# Module 4: Data Cleaning and Post Visualization - Dataset-Driven Solution
# Replace DATA_PATH with the actual dataset file path (CSV)
# Outputs: ./figures/*.png and ./outputs/*.csv
import os
from pathlib import Path
import numpy as np
import pandas as pd
import [Link] as plt
import seaborn as sns
from [Link] import SimpleImputer, KNNImputer
from [Link] import OneHotEncoder, StandardScaler, MinMaxScaler,
RobustScaler, Normalizer
from [Link] import ColumnTransformer
from [Link] import Pipeline
# ------------------------- CONFIG -------------------------
DATA_PATH = "your_dataset.csv" # <-- set this to the actual dataset path
FIG_DIR = Path("figures"); FIG_DIR.mkdir(exist_ok=True)
OUT_DIR = Path("outputs"); OUT_DIR.mkdir(exist_ok=True)
[Link](style="whitegrid")
[Link](42)
# ------------------------- 4.1 Read Dataset (multiple methods illustrated)
-------------------------
# a) Standard comma-delimited CSV
df_a = pd.read_csv(DATA_PATH, delimiter=",")
print("[Read] via delimiter=',' :", df_a.shape)
# b) Directory + filename variables
data_dir, fname = [Link](DATA_PATH)
if data_dir == "": data_dir = "."
df_b = pd.read_csv([Link](data_dir, fname))
print("[Read] via dir+filename:", df_b.shape)
# c) Using Python open
with open(DATA_PATH, "r", encoding="utf-8") as f:
df_c = pd.read_csv(f)
print("[Read] via open():", df_c.shape)
# d) Using URL-style file URI (if accessible)
try:
file_uri = Path(DATA_PATH).resolve().as_uri()
df_d = pd.read_csv(file_uri)
print("[Read] via file:// URI:", df_d.shape)
except Exception as e:
print("[Read] file:// URI skipped:", e)
# Use df_a as the working DataFrame
df = df_a.copy()
# ------------------------- Column Type Inference -------------------------
# Identify numeric and categorical columns
numeric_cols = df.select_dtypes(include=["number"]).[Link]()
categorical_cols = df.select_dtypes(include=["object", "category",
"bool"]).[Link]()
print("Numeric columns:", numeric_cols)
print("Categorical columns:", categorical_cols)
# ------------------------- 4.1(ii) Create Dummy Variables
-------------------------
if categorical_cols:
df_dummies = pd.get_dummies(df, columns=categorical_cols, drop_first=False,
prefix=[f"{c}_d" for c in categorical_cols])
else:
df_dummies = [Link]()
df_dummies.to_csv(OUT_DIR / "with_dummies.csv", index=False)
print("[Dummy] Saved:", OUT_DIR / "with_dummies.csv")
# ------------------------- 4.1(iii) Detect Outliers (IQR + Z-score)
-------------------------
def iqr_bounds(s, k=1.5):
q1, q3 = [Link](0.25), [Link](0.75)
iqr = q3 - q1
return q1 - k*iqr, q3 + k*iqr
outlier_flags = [Link](index=[Link])
for col in numeric_cols:
s = pd.to_numeric(df[col], errors="coerce")
low, high = iqr_bounds([Link]())
outlier_flags[f"outlier_iqr_{col}"] = (s < low) | (s > high)
mu, sigma = [Link](skipna=True), [Link](skipna=True)
if sigma and sigma > 0:
z = (s - mu) / sigma
outlier_flags[f"outlier_z_{col}"] = [Link]() > 3
else:
outlier_flags[f"outlier_z_{col}"] = False
flags_out = [Link]([df[numeric_cols], outlier_flags], axis=1)
flags_out.to_csv(OUT_DIR / "outlier_flags.csv", index=False)
print("[Outliers] Flags saved:", OUT_DIR / "outlier_flags.csv")
# ------------------------- 4.1(iv) Box plots: with vs without missing
-------------------------
# Pick a numeric column to visualize distribution; prefer the one with most missing
na_counts = df[numeric_cols].isna().sum() if numeric_cols else [Link](dtype=int)
if len(na_counts) and (na_counts > 0).any():
var_box = na_counts.sort_values(ascending=False).index[0]
elif numeric_cols:
var_box = numeric_cols[0]
else:
var_box = None
if var_box is not None:
[Link](figsize=(8, 4))
[Link](1, 2, 1)
[Link](y=df[var_box])
[Link](f"{var_box} with NaNs")
[Link](1, 2, 2)
[Link](y=df[var_box].dropna())
[Link](f"{var_box} without NaNs (dropna)")
plt.tight_layout()
fname = FIG_DIR / f"box_{var_box}_with_without_na.png"
[Link](fname, dpi=140); [Link]()
print("[Box] Saved:", fname)
else:
print("[Box] Skipped (no numeric columns)")
# ------------------------- 4.2(i) Check Missing Values -------------------------
mv_report = [Link]({
"missing_count": [Link]().sum(),
"missing_pct": [Link]().mean().round(4)
}).sort_values("missing_count", ascending=False)
mv_report.to_csv(OUT_DIR / "missing_value_report.csv")
print("[Missing] Report saved:", OUT_DIR / "missing_value_report.csv")
# ------------------------- 4.2(ii) Deletion and Imputation
-------------------------
# Deletion example: drop rows where any of the most-missing columns are NaN
subset_for_drop = mv_report.index[mv_report["missing_count"] > 0].tolist()[:3] #
up to 3 worst columns
if subset_for_drop:
df_drop = [Link](subset=subset_for_drop)
df_drop.to_csv(OUT_DIR / "after_deletion_subset.csv", index=False)
print("[Deletion] Saved:", OUT_DIR / "after_deletion_subset.csv")
else:
print("[Deletion] Skipped (no missing values)")
# SimpleImputer: numeric=median, categorical=most_frequent
numeric_imputer = SimpleImputer(strategy="median")
categorical_imputer = SimpleImputer(strategy="most_frequent")
preprocess_impute = ColumnTransformer(
transformers=[
("num", numeric_imputer, numeric_cols),
("cat", categorical_imputer, categorical_cols)
],
remainder="drop"
)
imputed_arr = preprocess_impute.fit_transform(df)
imputed_cols = numeric_cols + categorical_cols
df_imputed = [Link](imputed_arr, columns=imputed_cols)
# Coerce types
for c in numeric_cols:
df_imputed[c] = pd.to_numeric(df_imputed[c], errors="coerce")
for c in categorical_cols:
df_imputed[c] = df_imputed[c].astype(str)
df_imputed.to_csv(OUT_DIR / "after_imputation_simple.csv", index=False)
print("[Impute] SimpleImputer saved:", OUT_DIR / "after_imputation_simple.csv")
# Optional: KNNImputer on numeric only
if numeric_cols:
knn_imp = KNNImputer(n_neighbors=5)
knn_numeric = [Link](knn_imp.fit_transform(df[numeric_cols]),
columns=numeric_cols)
knn_numeric.to_csv(OUT_DIR / "after_imputation_knn_numeric.csv", index=False)
print("[Impute] KNN numeric saved:", OUT_DIR /
"after_imputation_knn_numeric.csv")
# ------------------------- 4.2(iii) Histogram: with vs imputed
-------------------------
# Choose a numeric variable with missing if possible
if len(na_counts) and (na_counts > 0).any():
var_hist = na_counts.sort_values(ascending=False).index[0]
elif numeric_cols:
var_hist = numeric_cols[0]
else:
var_hist = None
if var_hist is not None and var_hist in df_imputed.columns:
[Link](figsize=(10, 4))
[Link](1, 2, 1)
[Link](pd.to_numeric(df[var_hist], errors="coerce"), bins=30,
color="steelblue", edgecolor="black")
[Link](f"{var_hist} (with NaNs)")
[Link](1, 2, 2)
[Link](pd.to_numeric(df_imputed[var_hist], errors="coerce"), bins=30,
color="seagreen", edgecolor="black")
[Link](f"{var_hist} (imputed)")
plt.tight_layout()
fname = FIG_DIR / f"hist_{var_hist}_with_vs_imputed.png"
[Link](fname, dpi=140); [Link]()
print("[Hist] Saved:", fname)
else:
print("[Hist] Skipped (no numeric column for histogram)")
# ------------------------- 4.2(iv) Relationship visualization
-------------------------
# Scatter for two numeric cols; color by first categorical if available
if len(numeric_cols) >= 2:
xcol, ycol = numeric_cols[:2]
[Link](figsize=(10, 4))
[Link](1, 2, 1)
if categorical_cols:
[Link](x=df[xcol], y=df[ycol], hue=df[categorical_cols[0]],
legend=False)
[Link](f"{xcol} vs {ycol} (original)")
else:
[Link](x=df[xcol], y=df[ycol])
[Link](f"{xcol} vs {ycol} (original)")
[Link](1, 2, 2)
if categorical_cols:
[Link](x=df_imputed[xcol], y=df_imputed[ycol],
hue=df_imputed[categorical_cols[0]], legend=False)
[Link](f"{xcol} vs {ycol} (imputed)")
else:
[Link](x=df_imputed[xcol], y=df_imputed[ycol])
[Link](f"{xcol} vs {ycol} (imputed)")
plt.tight_layout()
fname = FIG_DIR / f"scatter_{xcol}_{ycol}_orig_vs_imputed.png"
[Link](fname, dpi=140); [Link]()
print("[Scatter] Saved:", fname)
else:
print("[Scatter] Skipped (need >=2 numeric columns)")
# ------------------------- 4.3 Normalization: MinMax, Standard, Robust, Normalizer
-------------------------
# Prepare fully imputed + encoded frame
if categorical_cols:
cat_pipeline = Pipeline(steps=[
("imp", SimpleImputer(strategy="most_frequent")),
("ohe", OneHotEncoder(handle_unknown="ignore", sparse_output=False))
])
else:
cat_pipeline = "drop"
preprocess_full = ColumnTransformer(
transformers=[
("num", SimpleImputer(strategy="median"), numeric_cols),
("cat", cat_pipeline, categorical_cols)
],
remainder="drop"
)
X_full = preprocess_full.fit_transform(df)
# Build column names
ohe_cols = []
if categorical_cols:
ohe_cols =
preprocess_full.named_transformers_["cat"].named_steps["ohe"].get_feature_names_out
(categorical_cols).tolist()
full_cols = numeric_cols + ohe_cols
full_df = [Link](X_full, columns=full_cols)
# Define scalers
scalers = {
"standard": StandardScaler(),
"minmax": MinMaxScaler(),
"robust": RobustScaler(),
"normalize": Normalizer() # row-wise L2 normalization
}
for name, scaler in [Link]():
out = full_df.copy()
if numeric_cols:
num_scaled = scaler.fit_transform(out[numeric_cols].values)
out[numeric_cols] = num_scaled
out.to_csv(OUT_DIR / f"scaled_{name}.csv", index=False)
# Histograms before vs after for numeric columns
if numeric_cols:
[Link](figsize=(12, 4*len(numeric_cols)))
for i, col in enumerate(numeric_cols, 1):
[Link](len(numeric_cols), 2, 2*i - 1)
[Link](full_df[col], bins=30, color="steelblue", edgecolor="black")
[Link](f"{col} - Before")
[Link](len(numeric_cols), 2, 2*i)
[Link](out[col], bins=30, color="salmon", edgecolor="black")
[Link](f"{col} - After ({[Link]()})")
plt.tight_layout()
fname = FIG_DIR / f"hists_{name}.png"
[Link](fname, dpi=140); [Link]()
print(f"[Scale-Hist] Saved: {fname}")
# Boxplots
[Link](figsize=(10, 4))
data = []
labels = []
for col in numeric_cols:
[Link](full_df[col].values); [Link](f"{col}-B")
[Link](out[col].values); [Link](f"{col}-A")
[Link](data, labels=labels, showfliers=True)
[Link](f"Before vs After ({[Link]()})")
[Link](rotation=45)
plt.tight_layout()
fname = FIG_DIR / f"box_{name}.png"
[Link](fname, dpi=140); [Link]()
print(f"[Scale-Box] Saved: {fname}")
print("Done. Check ./figures and ./outputs for results.")