In [17]: # Import required libraries
import pandas as pd
import numpy as np
In [18]: # Create a sample dataset (no file needed in Colab)
# Age: 20–60, Income: 20,000–2,00,000
[Link](0) # to get same random values every time
df = [Link]({
'Age': [Link](20, 60, 100),
'Income': [Link](20000, 200000, 100)
})
df # show the dataset
Out[18]: Age Income
0 20 44777
1 23 145311
2 23 33824
3 59 153490
4 29 163915
... ... ...
95 23 114275
96 54 63986
97 33 25103
98 59 142627
99 41 55050
100 rows × 2 columns
In [19]: # Show shape (rows, columns)
print("Shape:", [Link])
# Show first few rows
print([Link]())
Shape: (100, 2)
Age Income
0 20 44777
1 23 145311
2 23 33824
3 59 153490
4 29 163915
In [20]: # Mean of each numeric column
print("Mean Age:", df['Age'].mean())
print("Mean Income:", df['Income'].mean())
# Variance of each numeric column
print("Variance of Age:", df['Age'].var())
print("Variance of Income:", df['Income'].var())
Mean Age: 38.96
Mean Income: 107640.01
Variance of Age: 161.37212121212127
Variance of Income: 2532408683.161515
In [21]: # Standardization: (x - mean) / std
from [Link] import StandardScaler
scaler_std = StandardScaler()
data_std = scaler_std.fit_transform(df) # returns numpy array
df_std = [Link](data_std, columns=[Link]) # convert to dataframe
df_std
Out[21]: Age Income
0 -1.500053 -1.255483
1 -1.262702 0.752355
2 -1.262702 -1.474233
3 1.585499 0.915703
4 -0.788002 1.123908
... ... ...
95 -1.262702 0.132512
96 1.189915 -0.871846
97 -0.471536 -1.648406
98 1.585499 0.698750
99 0.161398 -1.050313
100 rows × 2 columns
In [22]: # Manual standardization using lambda
standardize = lambda x: (x - [Link]()) / [Link]()
df_std_manual = [Link](standardize)
df_std_manual
Out[22]: Age Income
0 -1.492533 -1.249189
1 -1.256373 0.748583
2 -1.256373 -1.466843
3 1.577551 0.911113
4 -0.784052 1.118275
... ... ...
95 -1.256373 0.131848
96 1.183951 -0.867476
97 -0.469172 -1.640143
98 1.577551 0.695248
99 0.160589 -1.045048
100 rows × 2 columns
In [23]: # Check if sklearn's standardization matches manual standardization
[Link](df_std.values, df_std_manual.values)
Out[23]: False
In [24]: # Normalization: scales data between 0 and 1
from [Link] import MinMaxScaler
scaler_minmax = MinMaxScaler()
data_norm = scaler_minmax.fit_transform(df)
df_norm = [Link](data_norm, columns=[Link])
df_norm
Out[24]: Age Income
0 0.000000 0.137905
1 0.076923 0.708259
2 0.076923 0.075766
3 1.000000 0.754661
4 0.230769 0.813804
... ... ...
95 0.076923 0.532184
96 0.871795 0.246883
97 0.333333 0.026290
98 1.000000 0.693032
99 0.538462 0.196186
100 rows × 2 columns
In [25]: # Manual normalization: (x - min) / (max - min)
normalize = lambda x: (x - [Link]()) / ([Link]() - [Link]())
df_norm_manual = [Link](normalize)
df_norm_manual
Out[25]: Age Income
0 0.000000 0.137905
1 0.076923 0.708259
2 0.076923 0.075766
3 1.000000 0.754661
4 0.230769 0.813804
... ... ...
95 0.076923 0.532184
96 0.871795 0.246883
97 0.333333 0.026290
98 1.000000 0.693032
99 0.538462 0.196186
100 rows × 2 columns
In [26]: # Log transformation (natural logarithm) on numeric columns
# Helps reduce skewness
df_log = [Link](lambda x: [Link](x))
df_log
Out[26]: Age Income
0 2.995732 10.709450
1 3.135494 11.886632
2 3.135494 10.428926
3 4.077537 11.941391
4 3.367296 12.007103
... ... ...
95 3.135494 11.646363
96 3.988984 11.066420
97 3.496508 10.130743
98 4.077537 11.867988
99 3.713572 10.915997
100 rows × 2 columns
In [27]: # Discretize 'Age' into 5 equal-width intervals
age_bins = [Link](df['Age'], 5)
age_bins.value_counts()
Out[27]: count
Age
(19.961, 27.8] 24
(51.2, 59.0] 24
(27.8, 35.6] 21
(35.6, 43.4] 18
(43.4, 51.2] 13
dtype: int64
In [28]: # Add labels to each age bin
labels = ['Very Young', 'Young', 'Middle', 'Older', 'Very Old']
df['Age_Category'] = [Link](df['Age'], 5, labels=labels)
df[['Age', 'Age_Category']].head(10)
Out[28]: Age Age_Category
0 20 Very Young
1 23 Very Young
2 23 Very Young
3 59 Very Old
4 29 Young
5 39 Middle
6 41 Middle
7 56 Very Old
8 43 Middle
9 26 Very Young
In [29]: # Binarization: Convert Income into 0/1 based on a threshold
# Here, 'High_Income' = 1 if Income >= 100000, else 0
df['High_Income'] = df['Income'].apply(lambda x: 1 if x >= 100000 else 0)
df[['Income', 'High_Income']].head(10)
Out[29]: Income High_Income
0 44777 0
1 145311 1
2 33824 0
3 153490 1
4 163915 1
5 98778 0
6 56223 0
7 81570 0
8 26521 0
9 94659 0
In [30]: # Aggregate: average income per age category
avg_income_by_age_cat = [Link]('Age_Category')['Income'].mean()
avg_income_by_age_cat
/tmp/[Link]: FutureWarning: The default of observed=False
is deprecated and will be changed to True in a future version of pandas. Pass o
bserved=False to retain current behavior or observed=True to adopt the future d
efault and silence this warning.
avg_income_by_age_cat = [Link]('Age_Category')['Income'].mean()
Out[30]: Income
Age_Category
Very Young 104923.416667
Young 106322.619048
Middle 106137.111111
Older 105895.307692
Very Old 113581.541667
dtype: float64
In [31]: # Average age for Low vs High income groups
avg_age_by_income_level = [Link]('High_Income')['Age'].mean()
avg_age_by_income_level
Out[31]: Age
High_Income
0 37.733333
1 39.963636
dtype: float64
In [32]: # Take a random sample of 10 rows (without replacement)
df_sample_n = [Link](n=10, replace=False, random_state=1)
df_sample_n
Out[32]: Age Income Age_Category High_Income
80 55 136551 Very Old 1
84 41 124489 Middle 1
33 39 135896 Middle 1
81 43 185560 Middle 1
93 54 35741 Very Old 0
17 44 51785 Older 0
36 59 99128 Very Old 0
82 35 189112 Young 1
69 40 55489 Middle 0
65 20 21913 Very Young 0
In [33]: # Take a random sample of 10% of the data
df_sample_frac = [Link](frac=0.10, random_state=1)
df_sample_frac
Out[33]: Age Income Age_Category High_Income
80 55 136551 Very Old 1
84 41 124489 Middle 1
33 39 135896 Middle 1
81 43 185560 Middle 1
93 54 35741 Very Old 0
17 44 51785 Older 0
36 59 99128 Very Old 0
82 35 189112 Young 1
69 40 55489 Middle 0
65 20 21913 Very Young 0