3/14/26, 11:19 PM final_code_classification

In [63]: import numpy as np

import pandas as pd

from sklearn.model_selection import train_test_split, RandomizedSearchCV

from skopt import BayesSearchCV
from [Link] import Pipeline
from [Link] import clone
from imblearn.over_sampling import SMOTE
from [Link] import StandardScaler, PowerTransformer, RobustScaler
from sklearn.feature_selection import SelectKBest, f_classif, RFE
from [Link] import PCA, FastICA, KernelPCA
from sklearn.cross_decomposition import PLSRegression, PLSCanonical
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA, Qua
from [Link] import TSNE
from [Link] import accuracy_score, average_precision_score, recall_scor
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold, cross_val_score

from sklearn.linear_model import LogisticRegression

from [Link] import RandomForestClassifier, ExtraTreesClassifier
from [Link] import SVC
from sklearn.neural_network import MLPClassifier
from xgboost import XGBClassifier
from catboost import CatBoostClassifier
from [Link] import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from [Link] import AdaBoostClassifier, HistGradientBoostingClassifier
from sklearn.linear_model import RidgeClassifier
from [Link] import DecisionTreeClassifier
import [Link] as plt
import seaborn as sns
sns.set_theme(style="whitegrid")

df = pd.read_csv(r"C:\Users\sahay\OneDrive\Desktop\Classification_Problem\Final_

if 'Unnamed: 0' in [Link]:

df = [Link]('Unnamed: 0', axis=1)

X = [Link]('Machine_failure', axis=1)
y = df['Machine_failure']

#Split
#1st split
X_train, X_temp, y_train, y_temp = train_test_split(
X,
y,
test_size=0.40,
random_state=42,
stratify=y
)
#2nd split
X_val, X_test, y_val, y_test = train_test_split(
X_temp,
y_temp,
test_size=0.50,
random_state=42,

[Link] 1/20
3/14/26, 11:19 PM final_code_classification

stratify=y_temp
)

EDA OF ALL COLUMNS

In [ ]: import pandas as pd
import numpy as np
import [Link] as plt
import seaborn as sns

target_col = 'Machine_failure'

numerical_features = df.select_dtypes(include=[Link]).[Link](target_col

sns.set_theme(style="whitegrid")

print(f"Generating Univariate and Bivariate plots for {len(numerical_features)}

for col in numerical_features:

fig, axes = [Link](1, 2, figsize=(14, 4))

[Link](data=df, x=col, kde=True, ax=axes[0], color='steelblue', bins=3

axes[0].set_title(f'Distribution of {col}', fontsize=12, fontweight='bold')
axes[0].set_ylabel('Frequency')

[Link](data=df, x=target_col, y=col, ax=axes[1], hue=target_col, palett

axes[1].set_title(f'{col} split by Machine State', fontsize=12, fontweight='
axes[1].set_xlabel('0 = Healthy, 1 = Broken')
axes[1].set_ylabel(col)

plt.tight_layout()
[Link]()

Generating Univariate and Bivariate plots for 13 features...

[Link] 2/20
3/14/26, 11:19 PM final_code_classification

[Link] 3/20
3/14/26, 11:19 PM final_code_classification

In [ ]: [Link](figsize=(12, 10))

corr_matrix = [Link]()

[Link](corr_matrix, annot=True, fmt='.2f', cmap='coolwarm', square=True)

[Link]('Multivariate: Sensor Correlation Heatmap')

[Link]()

[Link] 4/20
3/14/26, 11:19 PM final_code_classification

In [ ]: pipe = Pipeline([
('scaler', StandardScaler()),
('smote', SMOTE(random_state=29)),
('selector', None),
('extractor', None),
('reducer', None),
('clf',LogisticRegression())
])

# Dictionary
scalers = {
'standard': StandardScaler(),
'robust': RobustScaler(),
'power': PowerTransformer(),
'quantile': QuantileTransformer(),
'minmax': MinMaxScaler(),
}
selectors = {
'kbest': SelectKBest(f_classif, k=6),
'rfe': RFE(RandomForestClassifier(random_state=29), n_features_to_select=6),
None: None
}
extractors = {
'pls_regression': PLSRegression(n_components=3),
'pls_canonical': PLSCanonical(n_components=3),
'lda': LDA(n_components=1),
None: None
}

[Link] 5/20
3/14/26, 11:19 PM final_code_classification

reducers = {
'pca': PCA(n_components=0.95),
'kpca': KernelPCA(n_components=5, kernel='rbf'),
'ica': FastICA(n_components=5, random_state=29),
None: None
}
models = {
'logreg': LogisticRegression(max_iter=1000),
'rf': RandomForestClassifier(random_state=29),
'svc': SVC(random_state=29),
'mlp': MLPClassifier(max_iter=300, random_state=29),
'extratrees': ExtraTreesClassifier(random_state=29),
'xgb': XGBClassifier(eval_metric='logloss', random_state=29),
'catboost': CatBoostClassifier(verbose=0, random_state=29),
}

Finding best scaler (with SMOTE)

In [14]: pipe.set_params(
selector=None,
extractor=None,
reducer=None,
clf=models['rf']
)
print("The best scaler-")
scaler_results=[]
for name, scalers in [Link]():
pipe.set_params(scaler=scaler)
[Link](X_train, y_train)
y_pred = [Link](X_val)
f1 = f1_score(y_val, y_pred, pos_label=1)
scaler_results.append({'Scaler': name, 'F1': f1})
print(f"{name}: F1-Score = {f1:.4f}")

scaler_df = [Link](scaler_results).sort_values(by='F1', ascending=Fals

best_scaler_name = scaler_df.iloc[0]['Scaler']
print(f"\n🏆 Winner: {best_scaler_name}")

The best scaler-

---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
Cell In[14], line 9
7 print("The best scaler-")
8 scaler_results=[]
----> 9 for name, scalers in [Link]():
10 pipe.set_params(scaler=scaler)
11 [Link](X_train, y_train)

AttributeError: 'StandardScaler' object has no attribute 'items'

In [49]: pipe.set_params(
selector=None,
extractor=None,
reducer=None,
clf=models['rf']
)
print("The best scaler-")
scaler_results=[]
for scaler_name, scaler in [Link]():

[Link] 6/20
3/14/26, 11:19 PM final_code_classification

pipe_copy = clone(pipe)
pipe_copy.set_params(scaler=scaler)
pipe_copy.set_params(scaler=scaler)
pipe_copy.fit(X_train, y_train)
y_pred = pipe_copy.predict(X_val)
f1 = f1_score(y_val, y_pred, pos_label=1)
scaler_results.append({'Scaler': scaler_name, 'F1': f1})
print(f"{scaler_name}: F1-Score = {f1:.4f}")

scaler_df = [Link](scaler_results).sort_values(by='F1', ascending=False)

best_scaler_name = scaler_df.iloc[0]['Scaler']

The best scaler-

standard: F1-Score = 0.6541
robust: F1-Score = 0.7034
power: F1-Score = 0.6624
quantile: F1-Score = 0.6584
minmax: F1-Score = 0.6316

SO I REMOVE SMOTE BECAUSE IT IS CAUSING SKEWNESS

In [50]: pipe = Pipeline([

('scaler', StandardScaler()),
('selector', None),
('extractor', None),
('reducer', None),
('clf',LogisticRegression())
])

# Dictionary
scalers = {
'standard': StandardScaler(),
'robust': RobustScaler(),
'power': PowerTransformer(),
'quantile': QuantileTransformer(),
'minmax': MinMaxScaler(),
}
selectors = {
'kbest': SelectKBest(f_classif, k=6),
'rfe': RFE(RandomForestClassifier(random_state=29), n_features_to_select=6),
None: None
}
extractors = {
'pls_regression': PLSRegression(n_components=3),
'pls_canonical': PLSCanonical(n_components=3),
'lda': LDA(n_components=1),
None: None
}
reducers = {
'pca': PCA(n_components=0.95),
'kpca': KernelPCA(n_components=5, kernel='rbf'),
'ica': FastICA(n_components=5, random_state=29),
None: None
}
models = {
'logreg': LogisticRegression(max_iter=1000),
'rf': RandomForestClassifier(random_state=29),
'svc': SVC(random_state=29),
'mlp': MLPClassifier(max_iter=300, random_state=29),
'extratrees': ExtraTreesClassifier(random_state=29),

[Link] 7/20
3/14/26, 11:19 PM final_code_classification

'xgb': XGBClassifier(eval_metric='logloss', random_state=29),

'catboost': CatBoostClassifier(verbose=0, random_state=29)
}

In [51]: pipe.set_params(
selector=None,
extractor=None,
reducer=None,
clf=models['rf']
)
print("The best scaler-")
scaler_results=[]
for scaler_name, scaler in [Link]():
pipe_copy = clone(pipe)
pipe_copy.set_params(scaler=scaler)
pipe_copy.set_params(scaler=scaler)
pipe_copy.fit(X_train, y_train)
y_pred = pipe_copy.predict(X_val)
f1 = f1_score(y_val, y_pred, pos_label=1)
scaler_results.append({'Scaler': scaler_name, 'F1': f1})
print(f"{scaler_name}: F1-Score = {f1:.4f}")

scaler_df = [Link](scaler_results).sort_values(by='F1', ascending=False)

best_scaler_name = scaler_df.iloc[0]['Scaler']

The best scaler-

standard: F1-Score = 0.8333
robust: F1-Score = 0.8333
power: F1-Score = 0.8235
quantile: F1-Score = 0.8333
minmax: F1-Score = 0.8333

In [52]: pipe.set_params(
scaler=scalers['robust'],
extractor=None,
reducer=None,
clf=models['rf']
)
print("The best feature selector -")
selector_results = []
for name, selector in [Link]():
pipe_copy = clone(pipe)
pipe_copy.set_params(selector=selector)

pipe_copy.fit(X_train, y_train)
y_pred = pipe_copy.predict(X_val)

f1 = f1_score(y_val, y_pred, pos_label=1)

selector_results.append({'Selector': name, 'F1' : f1})
print(f"{name}: F1-Score = {f1:.4f}")

selector_df = [Link](selector_results).sort_values(by='F1', ascending=Fals

best_selector_name = selector_df.iloc[0]['Selector']

The best feature selector -

kbest: F1-Score = 0.7652
rfe: F1-Score = 0.8235
None: F1-Score = 0.8333

[Link] 8/20
3/14/26, 11:19 PM final_code_classification

In [53]: pipe.set_params(
scaler=scalers['robust'],
selector = None,
reducer=None,
clf=models['rf']
)
print("The best feature extractor -")
extractor_results = []
for name, extractor in [Link]():
pipe_copy = clone(pipe)
pipe_copy.set_params(extractor=extractor)

pipe_copy.fit(X_train, y_train)
y_pred = pipe_copy.predict(X_val)

f1 = f1_score(y_val, y_pred, pos_label=1)

extractor_results.append({'Extractor': name, 'F1' : f1})
print(f"{name}: F1-Score = {f1:.4f}")

extractor_df = [Link](extractor_results).sort_values(by='F1', ascending=Fa

best_extractor_name = extractor_df.iloc[0]['Extractor']

The best feature extractor -

[Link] 9/20
3/14/26, 11:19 PM final_code_classification

---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
Cell In[53], line 13
10 pipe_copy = clone(pipe)
11 pipe_copy.set_params(extractor=extractor)
---> 13 pipe_copy.fit(X_train, y_train)
14 y_pred = pipe_copy.predict(X_val)
16 f1 = f1_score(y_val, y_pred, pos_label=1)

File c:\Users\sahay\AppData\Local\Programs\Python\Python313\Lib\site-packages\skl
earn\[Link], in _fit_context.<locals>.decorator.<locals>.wrapper(estimator,
*args, **kwargs)
1329 estimator._validate_params()
1331 with config_context(
1332 skip_parameter_validation=(
1333 prefer_skip_nested_validation or global_skip_validation
1334 )
1335 ):
-> 1336 return fit_method(estimator, *args, **kwargs)

File c:\Users\sahay\AppData\Local\Programs\Python\Python313\Lib\site-packages\imb
learn\[Link], in [Link](self, X, y, **params)
516 if self._final_estimator != "passthrough":
517 last_step_params = self._get_metadata_for_step(
518 step_idx=len(self) - 1,
519 step_params=routed_params[[Link][-1][0]],
520 all_params=params,
521 )
--> 522 self._final_estimator.fit(Xt, yt, **last_step_params["fit"])
523 return self

File c:\Users\sahay\AppData\Local\Programs\Python\Python313\Lib\site-packages\skl
earn\[Link], in _fit_context.<locals>.decorator.<locals>.wrapper(estimator,
*args, **kwargs)
1329 estimator._validate_params()
1331 with config_context(
1332 skip_parameter_validation=(
1333 prefer_skip_nested_validation or global_skip_validation
1334 )
1335 ):
-> 1336 return fit_method(estimator, *args, **kwargs)

File c:\Users\sahay\AppData\Local\Programs\Python\Python313\Lib\site-packages\skl
earn\ensemble\_forest.py:359, in [Link](self, X, y, sample_weight)
356 if issparse(y):
357 raise ValueError("sparse multilabel-indicator for y is not supporte
d.")
--> 359 X, y = validate_data(
360 self,
361 X,
362 y,
363 multi_output=True,
364 accept_sparse="csc",
365 dtype=DTYPE,
366 ensure_all_finite=False,
367 )
368 # _compute_missing_values_in_feature_mask checks if X has missing values
and
369 # will raise an error if the underlying tree base estimator can't handle
missing

[Link] 10/20
3/14/26, 11:19 PM final_code_classification

370 # values. Only the criterion is required to determine if the tree support
s
371 # missing values.
372 estimator = type([Link])(criterion=[Link])

File c:\Users\sahay\AppData\Local\Programs\Python\Python313\Lib\site-packages\skl
earn\utils\[Link], in validate_data(_estimator, X, y, reset, validate
_separately, skip_check_array, **check_params)
2917 y = check_array(y, input_name="y", **check_y_params)
2918 else:
-> 2919 X, y = check_X_y(X, y, **check_params)
2920 out = X, y
2922 if not no_val_X and check_params.get("ensure_2d", True):

File c:\Users\sahay\AppData\Local\Programs\Python\Python313\Lib\site-packages\skl
earn\utils\[Link], in check_X_y(X, y, accept_sparse, accept_large_spa
rse, dtype, order, copy, force_writeable, ensure_all_finite, ensure_2d, allow_nd,
multi_output, ensure_min_samples, ensure_min_features, y_numeric, estimator)
1309 estimator_name = _check_estimator_name(estimator)
1310 raise ValueError(
1311 f"{estimator_name} requires y to be passed, but the target y is N
one"
1312 )
-> 1314 X = check_array(
1315 X,
1316 accept_sparse=accept_sparse,
1317 accept_large_sparse=accept_large_sparse,
1318 dtype=dtype,
1319 order=order,
1320 copy=copy,
1321 force_writeable=force_writeable,
1322 ensure_all_finite=ensure_all_finite,
1323 ensure_2d=ensure_2d,
1324 allow_nd=allow_nd,
1325 ensure_min_samples=ensure_min_samples,
1326 ensure_min_features=ensure_min_features,
1327 estimator=estimator,
1328 input_name="X",
1329 )
1331 y = _check_y(y, multi_output=multi_output, y_numeric=y_numeric, estimator
=estimator)
1333 check_consistent_length(X, y)

File c:\Users\sahay\AppData\Local\Programs\Python\Python313\Lib\site-packages\skl
earn\utils\[Link], in check_array(array, accept_sparse, accept_large_
sparse, dtype, order, copy, force_writeable, ensure_all_finite, ensure_non_negati
ve, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator, inpu
t_name)
1063 raise ValueError(
1064 "dtype='numeric' is not compatible with arrays of bytes/strings."
1065 "Convert your data to numeric values explicitly instead."
1066 )
1067 if not allow_nd and [Link] >= 3:
-> 1068 raise ValueError(
1069 f"Found array with dim {[Link]},"
1070 f" while dim <= 2 is required{context}."
1071 )
1073 if ensure_all_finite:
1074 _assert_all_finite(
1075 array,

[Link] 11/20
3/14/26, 11:19 PM final_code_classification

1076 input_name=input_name,
1077 estimator_name=estimator_name,
1078 allow_nan=ensure_all_finite == "allow-nan",
1079 )

ValueError: Found array with dim 3, while dim <= 2 is required by RandomForestCla
ssifier.

In [54]: pipe = Pipeline([

('scaler', StandardScaler()),
('selector', None),
('extractor', None),
('reducer', None),
('clf',LogisticRegression())
])

# Dictionary
scalers = {
'standard': StandardScaler(),
'robust': RobustScaler(),
'power': PowerTransformer(),
'quantile': QuantileTransformer(),
'minmax': MinMaxScaler(),
}
selectors = {
'kbest': SelectKBest(f_classif, k=6),
'rfe': RFE(RandomForestClassifier(random_state=29), n_features_to_select=6),
None: None
}
extractors = {
'lda': LDA(n_components=1),
None: None
}
reducers = {
'pca': PCA(n_components=0.95),
'kpca': KernelPCA(n_components=5, kernel='rbf'),
'ica': FastICA(n_components=5, random_state=29),
None: None
}
models = {
'logreg': LogisticRegression(max_iter=1000),
'rf': RandomForestClassifier(random_state=29),
'svc': SVC(random_state=29),
'mlp': MLPClassifier(max_iter=300, random_state=29),
'extratrees': ExtraTreesClassifier(random_state=29),
'xgb': XGBClassifier(eval_metric='logloss', random_state=29),
'catboost': CatBoostClassifier(verbose=0, random_state=29)
}

In [55]: pipe.set_params(
scaler=scalers['robust'],
selector = None,
reducer=None,
clf=models['rf']
)
print("The best feature extractor -")
extractor_results = []
for name, extractor in [Link]():
pipe_copy = clone(pipe)
pipe_copy.set_params(extractor=extractor)

[Link] 12/20
3/14/26, 11:19 PM final_code_classification

pipe_copy.fit(X_train, y_train)
y_pred = pipe_copy.predict(X_val)

f1 = f1_score(y_val, y_pred, pos_label=1)

extractor_results.append({'Extractor': name, 'F1' : f1})
print(f"{name}: F1-Score = {f1:.4f}")

extractor_df = [Link](extractor_results).sort_values(by='F1', ascending=Fa

best_extractor_name = extractor_df.iloc[0]['Extractor']

The best feature extractor -

lda: F1-Score = 0.2000
None: F1-Score = 0.8333

In [57]: pipe.set_params(
scaler=scalers['robust'],
selector = None,
extractor=None,
clf=models['rf']
)
print("The best feature reducer -")
reducer_results = []
for name, reducer in [Link]():
pipe_copy = clone(pipe)
pipe_copy.set_params(reducer=reducer if name != 'none' else None)

pipe_copy.fit(X_train, y_train)
y_pred = pipe_copy.predict(X_val)

f1 = f1_score(y_val, y_pred, pos_label=1)

reducer_results.append({'Reducer': name, 'F1' : f1})
print(f"{name}: F1-Score = {f1:.4f}")

reducer_df = [Link](reducer_results).sort_values(by='F1', ascending=False)

best_reducer_name = reducer_df.iloc[0]['Reducer']

The best feature reducer -

pca: F1-Score = 0.5000
kpca: F1-Score = 0.3297
ica: F1-Score = 0.5882
None: F1-Score = 0.8333

NOW RUNNING THE FINALIZED PIPELINE ON ALL THE IMPORTED CLASSIFIERS

In [61]: pipe = Pipeline([

('scaler', StandardScaler()),
('selector', None),
('extractor', None),
('reducer', None),
('clf',LogisticRegression())
])

# Dictionary
scalers = {
'standard': StandardScaler(),
'robust': RobustScaler(),
'power': PowerTransformer(),
'quantile': QuantileTransformer(),
'minmax': MinMaxScaler(),

[Link] 13/20
3/14/26, 11:19 PM final_code_classification

}
selectors = {
'kbest': SelectKBest(f_classif, k=6),
'rfe': RFE(RandomForestClassifier(random_state=29), n_features_to_select=6),
None: None
}
extractors = {
'lda': LDA(n_components=1),
None: None
}
reducers = {
'pca': PCA(n_components=0.95),
'kpca': KernelPCA(n_components=5, kernel='rbf'),
'ica': FastICA(n_components=5, random_state=29),
None: None
}
models = {
'logreg': LogisticRegression(max_iter=1000),
'rf': RandomForestClassifier(random_state=29),
'svc': SVC(random_state=29),
'mlp': MLPClassifier(max_iter=300, random_state=29),
'extratrees': ExtraTreesClassifier(random_state=29),
'xgb': XGBClassifier(eval_metric='logloss', random_state=29),
'catboost': CatBoostClassifier(verbose=0, random_state=29),
'ridge': RidgeClassifier(class_weight='balanced', random_state=29),
'nb': GaussianNB(),
'knn': KNeighborsClassifier(n_neighbors=5, weights='distance'),
'dt': DecisionTreeClassifier(class_weight='balanced', random_state=29),
'adaboost': AdaBoostClassifier(random_state=29),
'hist_gb': HistGradientBoostingClassifier(class_weight='balanced', random_st
}

Including Stratified k fold validation on the data

In [ ]: pipe = Pipeline([
('scaler', RobustScaler()),
('clf', LogisticRegression())
])

print("THE BEST MODEL IS -")

model_results = []

models = {
'logreg': LogisticRegression(max_iter=1000, class_weight='balanced', random_
'ridge': RidgeClassifier(class_weight='balanced', random_state=29),
'svc': SVC(class_weight='balanced', random_state=29),

'nb': GaussianNB(),
'knn': KNeighborsClassifier(n_neighbors=5, weights='distance'),

'mlp': MLPClassifier(max_iter=500, random_state=29),

'dt': DecisionTreeClassifier(class_weight='balanced', random_state=29),

'rf': RandomForestClassifier(class_weight='balanced', random_state=29),

[Link] 14/20
3/14/26, 11:19 PM final_code_classification

'extratrees': ExtraTreesClassifier(class_weight='balanced', random_state=29)

'adaboost': AdaBoostClassifier(random_state=29),
'hist_gb': HistGradientBoostingClassifier(class_weight='balanced', random_st
'xgb': XGBClassifier(eval_metric='logloss', scale_pos_weight=30, random_stat
'catboost': CatBoostClassifier(verbose=0, auto_class_weights='Balanced', ran
}

skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=29)

print("The best model for our data is = ")

model_results = []

for name, model in [Link]():

pipe_copy = clone(pipe)

pipe_copy.set_params(clf=model)
cv_scores = cross_val_score(
pipe_copy,
X_train,
y_train,
cv=skf,
scoring='f1',
n_jobs=-1
)

mean_f1 = cv_scores.mean()
std_f1 = cv_scores.std()

model_results.append({
'Model': name,
'Mean F1': mean_f1,
'Std Dev': std_f1
})

print(f"Finished {name: <12} -> Mean F1: {mean_f1:.4f} (± {std_f1:.4f})")

final_df = [Link](model_results).sort_values(by='Mean F1', ascending=False
print("\nFINAL MODEL LEADERBOARD")
print(final_df.to_string())

[Link] 15/20
3/14/26, 11:19 PM final_code_classification

THE BEST MODEL IS -

The best model for our data is =
Finished logreg -> Mean F1: 0.2826 (± 0.0207)
Finished ridge -> Mean F1: 0.2957 (± 0.0246)
Finished svc -> Mean F1: 0.4143 (± 0.0206)
Finished nb -> Mean F1: 0.2861 (± 0.0638)
Finished knn -> Mean F1: 0.5040 (± 0.0449)
Finished mlp -> Mean F1: 0.7063 (± 0.0599)
Finished dt -> Mean F1: 0.6993 (± 0.0753)
Finished rf -> Mean F1: 0.7625 (± 0.0578)
Finished extratrees -> Mean F1: 0.6356 (± 0.0676)
Finished adaboost -> Mean F1: 0.5106 (± 0.0723)
Finished hist_gb -> Mean F1: 0.7403 (± 0.0535)
Finished xgb -> Mean F1: 0.8085 (± 0.0602)
Finished catboost -> Mean F1: 0.7757 (± 0.0398)

FINAL MODEL LEADERBOARD

Model Mean F1 Std Dev
0 xgb 0.808541 0.060158
1 catboost 0.775738 0.039803
2 rf 0.762452 0.057844
3 hist_gb 0.740275 0.053518
4 mlp 0.706301 0.059890
5 dt 0.699312 0.075258
6 extratrees 0.635642 0.067598
7 adaboost 0.510644 0.072316
8 knn 0.503976 0.044890
9 svc 0.414304 0.020612
10 ridge 0.295694 0.024623
11 nb 0.286103 0.063815
12 logreg 0.282616 0.020697

HYPERPARAMETER TUNING

In [ ]: import time
from sklearn.model_selection import GridSearchCV

final_pipe = Pipeline([
('scaler', RobustScaler()),
('clf', XGBClassifier(eval_metric='logloss', scale_pos_weight=30, random_sta
])

skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

param_grid = {
'clf__n_estimators': [100, 300],
'clf__max_depth': [3, 5, 7],
'clf__learning_rate': [0.01, 0.1]
}

grid_search = GridSearchCV(
estimator=final_pipe,
param_grid=param_grid,
scoring='f1',
cv=skf,
n_jobs=-1
)

print("RUNNING GRID SEARCH")

start_time = [Link]()

[Link] 16/20
3/14/26, 11:19 PM final_code_classification

grid_search.fit(X_train, y_train)

elapsed_minutes = ([Link]() - start_time) / 60

print(f"Grid Search finished in {elapsed_minutes:.2f} minutes.")
print(f"Best F1-Score: {grid_search.best_score_:.4f}")
print("Best Parameters:")
for param, value in grid_search.best_params_.items():
print(f" - {[Link]('clf__', '')}: {value}")

RUNNING GRID SEARCH

Grid Search finished in 0.02 minutes.
Best F1-Score: 0.8014
Best Parameters:
- learning_rate: 0.1
- max_depth: 7
- n_estimators: 300

In [ ]: import time
from [Link] import uniform, randint
from sklearn.model_selection import RandomizedSearchCV

param_dist = {
'clf__n_estimators': randint(100, 400),
'clf__max_depth': randint(3, 8),
'clf__learning_rate': uniform(0.01, 0.15)
}

# 2. Setup Randomized Search

random_search = RandomizedSearchCV(
estimator=final_pipe,
param_distributions=param_dist,
n_iter=60,
scoring='f1',
cv=skf,
n_jobs=-1,
random_state=42
)

print("\nRUNNING RANDOMIZED SEARCH ")

start_time = [Link]()

random_search.fit(X_train, y_train)

elapsed_minutes = ([Link]() - start_time) / 60

print(f"Randomized Search finished in {elapsed_minutes:.2f} minutes.")
print(f"Best F1-Score: {random_search.best_score_:.4f}")
print("Best Parameters:")
for param, value in random_search.best_params_.items():
print(f" - {[Link]('clf__', '')}: {value}")

RUNNING RANDOMIZED SEARCH

Randomized Search finished in 0.10 minutes.
Best F1-Score: 0.8034
Best Parameters:
- learning_rate: 0.11670129291229749
- max_depth: 5
- n_estimators: 388

In [69]: import time

from skopt import BayesSearchCV

[Link] 17/20
3/14/26, 11:19 PM final_code_classification

from [Link] import Real, Integer

param_bayes = {
'clf__n_estimators': Integer(100, 400),
'clf__max_depth': Integer(3, 8),
'clf__learning_rate': Real(0.01, 0.15, prior='log-uniform')
}

bayes_search = BayesSearchCV(
estimator=final_pipe,
search_spaces=param_bayes,
n_iter=60,
scoring='f1',
cv=skf,
n_jobs=-1,
random_state=42
)

print("\nRUNNING BAYESIAN SEARCH")

start_time = [Link]()

bayes_search.fit(X_train, y_train)

elapsed_minutes = ([Link]() - start_time) / 60

print(f"Bayesian Search finished in {elapsed_minutes:.2f} minutes.")
print(f"Best F1-Score: {bayes_search.best_score_:.4f}")
print("Best Parameters:")
for param, value in bayes_search.best_params_.items():
print(f" - {[Link]('clf__', '')}: {value}")

RUNNING BAYESIAN SEARCH

Bayesian Search finished in 0.94 minutes.
Best F1-Score: 0.8170
Best Parameters:
- learning_rate: 0.11372963888953223
- max_depth: 6
- n_estimators: 305

FINAL TESTING OF MODEL ON TEST DATA

In [ ]: import [Link] as plt

import seaborn as sns
from [Link] import classification_report, f1_score, confusion_matrix
from [Link] import Pipeline
from [Link] import RobustScaler
from xgboost import XGBClassifier
import pandas as pd

final_optimized_pipe = Pipeline([
('scaler', RobustScaler()),
('clf', XGBClassifier(
n_estimators=305,
max_depth=6,
learning_rate=0.11372963888953223,
eval_metric='logloss',
scale_pos_weight=30,
random_state=29
))
])

[Link] 18/20
3/14/26, 11:19 PM final_code_classification

X_train_final = [Link]([X_train, X_val])

y_train_final = [Link]([y_train, y_val])

print("Training the final optimized model on combined Train+Val data...")

final_optimized_pipe.fit(X_train_final, y_train_final)

print("Predicting on the untouched Test set...\n")

y_test_pred = final_optimized_pipe.predict(X_test)

final_f1 = f1_score(y_test, y_test_pred, pos_label=1)

print(f"FINAL UNSEEN F1-SCORE: {final_f1:.4f}\n")

print("--- CLASSIFICATION REPORT ---")

print(classification_report(y_test, y_test_pred))

[Link](figsize=(6, 4))
cm = confusion_matrix(y_test, y_test_pred)
[Link](cm, annot=True, fmt='d', cmap='Blues', cbar=False)
[Link]('Final Confusion Matrix (Test Data)', fontweight='bold')
[Link]('Predicted: 0=Healthy, 1=Broken')
[Link]('Actual: 0=Healthy, 1=Broken')
[Link]()

Training the final optimized model on combined Train+Val data...

Predicting on the untouched Test set...

FINAL UNSEEN F1-SCORE: 0.7746

--- CLASSIFICATION REPORT ---

precision recall f1-score support

0 0.99 0.99 0.99 1932

1 0.74 0.81 0.77 68

accuracy 0.98 2000

macro avg 0.87 0.90 0.88 2000
weighted avg 0.98 0.98 0.98 2000

[Link] 19/20
3/14/26, 11:19 PM final_code_classification

[Link] 20/20