Steps in Data Preprocessing
In [1]: # Import the libraries
import numpy as np
import pandas as pd
import [Link] as plt
%matplotlib inline
import seaborn as sns
In [2]: #Import the data
df=pd.read_excel('[Link]')
In [3]: [Link]()
Out[3]: Reg StudyTim Attendanc Gender Age Marks
No. e e
0 22FMUCHH010490 12.5 85.0 Female NaN 9.3
1 22FMUCHH010491 12.0 72.0 Male 21.0 8.6
2 22FMUCHH010493 2.0 85.0 Female NaN 9.6
3 22FMUCHH010495 3.0 10.0 Male NaN 3.6
4 22FMUCHH010496 10.5 78.0 Female NaN 9.2
In [4]: df
Out[4]: Reg StudyTim Attendanc Gender Age Marks
No. e e
0 22FMUCHH010490 12.5 85.0 Female NaN 9.3
1 22FMUCHH010491 12.0 72.0 Male 21.0 8.6
2 22FMUCHH010493 2.0 85.0 Female NaN 9.6
3 22FMUCHH010495 3.0 10.0 Male NaN 3.6
4 22FMUCHH010496 10.5 78.0 Female NaN 9.2
5 22FMUCHH010497 1.0 68.0 Male NaN 4.0
6 22FMUCHH010499 9.5 74.0 Female NaN 8.4
7 22FMUCHH010500 9.0 77.0 Male NaN 7.7
8 22FMUCHH010503 3.0 NaN Female NaN 4.3
9 22FMUCHH010501 8.0 NaN Male 23.0 7.4
10 22FMUCHH010500 9.0 77.0 Male NaN 7.7
In [5]: [Link]()
<class '[Link]'>
RangeIndex: 11 entries, 0 to 10
Data columns (total 6 columns):
# Column Non-Null Count Dtype
0 Reg No. 11 non-null object
1 StudyTime 11 non-null float64
2 Attendance 9 non-null float64
3 Gender 11 non-null object
4 Age 2 non-null float64
5 Marks 11 non-null float64
dtypes: float64(4), object(2)
memory usage: 660.0+ bytes
In [6]: [Link]()
Out[6]: StudyTim Attendanc Age Marks
e e
count 11.000000 9.000000 2.000000 11.00000
0
mean 7.227273 69.555556 22.00000 7.254545
0
std 4.185473 23.006038 1.414214 2.229064
min 1.000000 10.000000 21.00000 3.600000
0
25% 3.000000 72.000000 21.50000 5.850000
0
50% 9.000000 77.000000 22.00000 7.700000
0
75% 10.000000 78.000000 22.50000 8.900000
0
max 12.500000 85.000000 23.00000 9.600000
0
In [7]: df['Gender'].value_counts()
Out[7] Gender
:
Male 6
Female 5
Name: count, dtype: int64
Handling Missing Data
#Identifying the missing values
In [Link]().sum()
[8]:
Reg No. 0
StudyTime 0
Out[8]
Attendance 2
:
Gender 0
Age 9
Marks 0
dtype: int64
#Replacing the missing values
df['Attendance'].fillna(df['Attendance'].mean(),inplace=True)
In
[9]:
C:\Users\KANDELA RAMESH\AppData\Local\Temp\ipykernel_26628\[Link]:
Futur eWarning: A value is trying to be set on a copy of a DataFrame or
Series through chained assignment using an inplace method.
The behavior will change in pandas 3.0. This inplace method will never work
becau se the intermediate object on which we are setting values always
behaves as a cop y.
For example, when doing 'df[col].method(value, inplace=True)', try using
'[Link] od({col: value}, inplace=True)' or df[col] = df[col].method(value)
instead, to pe rform the operation inplace on the original object.
df['Attendance'].fillna(df['Attendance'].mean(),inplace=True)
In [10]: [Link]().sum()
Out[10] Reg No. 0
:
StudyTime 0
Attendance 0
Gender 0
Age 9
Marks 0
dtype: int64
[Link]('Age',axis=1,inplace=True)
In
[11]:
[Link]().sum()
In
[12]: Reg No. 0
StudyTime 0
Out[12] Attendance 0
: Gender 0
Marks 0
dtype: int64
In [13]: df
Out[13]: Reg StudyTim Attendanc Gende Marks
No. e e r
0 22FMUCHH010490 12.5 85.000000 Female 9.3
1 22FMUCHH010491 12.0 72.000000 Male 8.6
2 22FMUCHH010493 2.0 85.000000 Female 9.6
3 22FMUCHH010495 3.0 10.000000 Male 3.6
4 22FMUCHH010496 10.5 78.000000 Female 9.2
5 22FMUCHH010497 1.0 68.000000 Male 4.0
6 22FMUCHH010499 9.5 74.000000 Female 8.4
7 22FMUCHH010500 9.0 77.000000 Male 7.7
8 22FMUCHH010503 3.0 69.555556 Female 4.3
9 22FMUCHH010501 8.0 69.555556 Male 7.4
10 22FMUCHH010500 9.0 77.000000 Male 7.7
Remove Duplicate Rows
In [14]: [Link]().sum()
Out[14] np.int64(1)
:
df = df.drop_duplicates()
In np.int64(0)
[15]:
[Link]().sum()
In
[16]:
Out[16]:
In [17]: df
Out[17]: Reg StudyTim Attendanc Gende Marks
No. e e r
0 22FMUCHH01049 12.5 85.000000 Female 9.3
0
1 22FMUCHH01049 12.0 72.000000 Male 8.6
1
2 22FMUCHH01049 2.0 85.000000 Female 9.6
3
3 22FMUCHH01049 3.0 10.000000 Male 3.6
5
4 22FMUCHH01049 10.5 78.000000 Female 9.2
6
5 22FMUCHH01049 1.0 68.000000 Male 4.0
7
6 22FMUCHH01049 9.5 74.000000 Female 8.4
9
7 22FMUCHH01050 9.0 77.000000 Male 7.7
0
8 22FMUCHH01050 3.0 69.555556 Female 4.3
3
9 22FMUCHH01050 8.0 69.555556 Male 7.4
1
Encoding categorical data
Encoding the Independent Variable
In [18]: gender=pd.get_dummies(df['Gender'], drop_first=True)
In [19]: gender
Out[19]: Mal
e
0 False
1 True
2 False
3 True
4 False
5 True
6 False
7 True
8 False
9 True
In [20]: # Convert boolean column to integers (1 and 0)
gender = [Link](int)
In [21]: [Link](['Gender'],axis=1,inplace=True)
C:\Users\KANDELA RAMESH\AppData\Local\Temp\ipykernel_26628\[Link]:
Setti ngWithCopyWarning:
A value is trying to be set on a copy of a slice from a DataFrame
See the caveats in the documentation: [Link]
docs/stabl e/user_guide/[Link]#returning-a-view-versus-a-copy
[Link](['Gender'],axis=1,inplace=True)
In [22]: [Link]()
Out[22]: Reg No. StudyTim Attendanc Marks
e e
0 12.5 85.0 9.3
22FMUCHH010490
1 12.0 72.0 8.6
22FMUCHH010491
2 2.0 85.0 9.6
22FMUCHH010493
3 3.0 10.0 3.6
22FMUCHH010495
4 10.5 78.0 9.2
22FMUCHH010496
In [23]: df=[Link]([df,gender],axis=1)
In [24]: [Link](2)
Out[24]: Reg StudyTim Attendanc Marks Male
No. e e
0 22FMUCHH010490 12.5 85.0 9.3 0
1 22FMUCHH010491 12.0 72.0 8.6 1
In [25]: [Link]('Reg No.', axis=1, inplace=True)
In [26]: df
Out[26]: StudyTim Attendanc Marks Male
e e
0 12.5 85.000000 9.3 0
1 12.0 72.000000 8.6 1
2 2.0 85.000000 9.6 0
3 3.0 10.000000 3.6 1
4 10.5 78.000000 9.2 0
5 1.0 68.000000 4.0 1
6 9.5 74.000000 8.4 0
7 9.0 77.000000 7.7 1
8 3.0 69.555556 4.3 0
9 8.0 69.555556 7.4 1
Outliers
In [27]: # Compute Quartile1
Q1 = df['Attendance'].quantile(0.25)
In [28]: # Compute Quartile2
Q3 = df['Attendance'].quantile(0.75)
In [29]: # Compute IQR
IQR = Q3 - Q1
In [30]: # Define Outlier Thresholds
lower_bound = Q1 - 1.5 * IQR
upper_bound = Q3 + 1.5 * IQR
In [31]: # Identify the Outliers
outliers = df[(df['Attendance'] < lower_bound) | (df['Attendance'] >
upper_bound
In [32]: outliers
Out[32]: StudyTim Attendanc Marks Male
e e
3 3.0 10.0 3.6 1
In [33]: #Remove the Outliers
[Link](index=3,axis=0, inplace=True)
In [34]: df
Out[34]: StudyTim Attendanc Marks Male
e e
0 12.5 85.000000 9.3 0
1 12.0 72.000000 8.6 1
2 2.0 85.000000 9.6 0
4 10.5 78.000000 9.2 0
5 1.0 68.000000 4.0 1
6 9.5 74.000000 8.4 0
7 9.0 77.000000 7.7 1
8 3.0 69.555556 4.3 0
9 8.0 69.555556 7.4 1
In [35]: [Link]()
<class '[Link]'>
Index: 9 entries, 0 to 9
Data columns (total 4 columns):
# Column Non-Null Count Dtype
0 StudyTime 9 non-null float64
1 Attendance 9 non-null float64
2 Marks 9 non-null float64
3 Male 9 non-null int64
dtypes: float64(3), int64(1)
memory usage: 360.0 bytes
In [36]: X = [Link]('Marks', axis=1)
y=df['Marks']
In [37]: X
Out[37]: StudyTim Attendanc Male
e e
0 12.5 85.000000 0
1 12.0 72.000000 1
2 2.0 85.000000 0
4 10.5 78.000000 0
5 1.0 68.000000 1
6 9.5 74.000000 0
7 9.0 77.000000 1
8 3.0 69.555556 0
9 8.0 69.555556 1
In [38]: y
Out[38]: 0 9.3
1 8.6
2 9.6
4 9.2
5 4.0
6 8.4
7 7.7
8 4.3
9 7.4
Name: Marks, dtype: float64
Feature Scaling
In [39]: from [Link] import StandardScaler
In [40]: scaler=StandardScaler()
In [41]: X
Out[41]: StudyTi Attendanc Male
me e
0 12.5 85.000000 0
1 12.0 72.000000 1
2 2.0 85.000000 0
4 10.5 78.000000 0
5 1.0 68.000000 1
6 9.5 74.000000 0
7 9.0 77.000000 1
8 3.0 69.555556 0
9 8.0 69.555556 1
In [42]: X[['StudyTime', 'Attendance']] = scaler.fit_transform(X[['StudyTime',
'Attendanc
In [43]: X
Out[43]: StudyTime Attendanc Male
e
0 1.593855 0
1.210701
1 -0.552346 1
1.089631
2 - 1.593855 0
1.331772
4 0.438208 0
0.726421
5 - -1.212716 1
1.573912
6 -0.222161 0
0.484281
7 0.273116 1
0.363210
8 - -0.955905 0
1.089631
9 -0.955905 1
0.121070