!pip install shap pyswarm alipy pdpbox==0.2.
0 lime
dataframe_image==0.1.7
import numpy as np
import [Link] as plt
import pandas as pd
from datetime import datetime
from sklearn import metrics
from [Link] import precision_score, recall_score, f1_score,
confusion_matrix, roc_curve, roc_auc_score
from sklearn.model_selection import cross_val_score, train_test_split
from [Link] import GradientBoostingClassifier as gbr
from [Link] import LabelEncoder, StandardScaler
from [Link] import DecisionTreeClassifier as dt
import seaborn as sns
import requests
import zipfile
from [Link] import ExperimentAnalyser
from pdpbox import pdp, info_plots
import io
from [Link] import drive
import lightgbm as lgb
import xgboost as xgb
import pyswarm as ps
from pyswarm import pso
import shap
import lime, lime.lime_tabular
import dataframe_image as dfi
[Link]('/content/drive')
df = pd.read_csv('/content/drive/MyDrive/[Link]', header=0)
# Display the DataFrame with column names (execute the code yourself to
see the output)
[Link]()
#Rename Dataset to Label to make it easy to understand
df = [Link](columns={'obj':'Label'})
########################
#Scatter plot
# Get the data for the scatter plot
x = [Link][:, 2] # Third column (index 2)
y = [Link][:, 6] # Seventh column (index 6)
colors = [Link][:, 1].map({'Y': 'green', 'N': 'red'}) # Second column
(index 1) for color
# Create the scatter plot
[Link](figsize=(8, 6)) # Adjust figure size as needed
# Create the scatter plot with color coding
[Link](x, y, c=colors, alpha=0.7)
[Link]([Link][2]) # Set x-axis label
[Link]([Link][6]) # Set y-axis label
[Link]('Scatter Plot with Color-Coded Clusters')
[Link]()
####### Replace categorical values with numbers########
df['Label'].value_counts()
#Define the dependent variable that needs to be predicted (labels)
y = df["Label"].values
# Encoding categorical data
# Encoding categorical data
labelencoder = LabelEncoder()
Y = labelencoder.fit_transform(y) # Y=1 and N=0
#Define x and normalize values
#Define the independent variables. Let's also drop Gender, so we can
normalize other data
X = [Link](labels = ["Label", "ID"], axis=1)
feature_names = [Link]([Link]) #Convert dtype string?
scaler = StandardScaler()
[Link](X)
X = [Link](X)
##Split data into train and test to verify accuracy after fitting the
model.
X_train, X_test, y_train, y_test = train_test_split(X, Y,
test_size=0.2, random_state=42)
#####################################################
#PSO Light GBM
d_train = [Link](X_train, label=y_train)
# [Link]
# Define the objective function to minimize
# Set LightGBM parameters
params_lgb = [0.1, 100, 10]
p=0.75
start=[Link]()
def objective_function(params_lgb):
# Set LightGBM parameters
lgbm_params = {'learning_rate': params_lgb[0], 'boosting_type':
'dart', # or 'gbdt'
'objective': 'binary',
'metric': ['auc', 'binary_logloss'],
'num_leaves': int(params_lgb[1]),
'max_depth': int(params_lgb[2])}
# Train LightGBM model with error handling
try:
clf = [Link](lgbm_params, d_train, 100)
scores = cross_val_score(clf, X, Y, cv=5, scoring='roc_auc')
return -[Link]() # Return the negative mean score for
minimization
except Exception as e:
print(f"Error during objective function evaluation: {e}")
return float('inf') # Return a large value to indicate failure
# Define the bounds for hyperparameters for pso optimization
lb_lgb = [0.01, 5, 3] # Lower bounds for learning_rate, num_leaves,
max_depth
ub_lgb = [0.3, 100, 10] # Upper bounds for learning_rate, num_leaves,
max_depth
# Perform PSO to find optimal hyperparameters
best_params_pso_lgbm, _ = pso(objective_function, lb_lgb, ub_lgb,
swarmsize=10, maxiter=50)
print("Best hyperparameters for PSO-LGBM:", best_params_pso_lgbm)
# Print the best hyperparameters
print("Best pso lgbm hyperparameters:", best_params_pso_lgbm)
# Train PSO LightGBM model with the best hyperparameters
best_lgbm_params = {
'learning_rate': best_params_pso_lgbm[0],
'boosting_type': 'dart', # or 'gbdt'
'objective': 'binary',
'metric': ['auc', 'binary_logloss'],
'num_leaves': int(best_params_pso_lgbm[1]), # num_leaves must be
an integer
'max_depth': int(best_params_pso_lgbm[2]), # max_depth must be an
integer
}
best_clf = [Link](best_lgbm_params, d_train, 100)
#print("PSO-LGBM execution time is: ", execution_time_lgbm)
#Prediction on test data
y_pred_pso_lgbm=best_clf.predict(X_test)
#convert into binary values 0/1 for classification
for i in range(0, X_test.shape[0]):
if y_pred_pso_lgbm[i]>=p: # setting threshold
y_pred_pso_lgbm[i]=1
else:
y_pred_pso_lgbm[i]=0
stop=[Link]()
execution_time_pso_lgbm = stop-start
#Print accuracy
#print ("Accuracy with PSO-LGBM = ",
metrics.accuracy_score(y_pred_lgbm,y_test))
#Confusion matrix
cm_pso_lgbm = confusion_matrix(y_test, y_pred_pso_lgbm)
#[Link](cm_pso_lgbm, annot=True)
#print("AUC score with PSO-LGBM is: ",
roc_auc_score(y_pred_pso_lgbm,y_test))
#######################################
# Determining Optimal variables with PSO_LGBM
####################
variable = [[Link][11, 2].min(),[[Link][11, 3].min(), [Link][11,
4].min(), [Link][11, 5].min(), [Link][11, 6].min(), [Link][11,
7].min(), [Link][11, 8].min(), [Link][11, 9].min()]]
# Optimal variables using PSO-LGB
def objective_function_opt(variable):
# Set LightGBM parameters
lgbm_params_opt = {'learning_rate': best_params_pso_lgbm[0],
'boosting_type': 'dart', # or 'gbdt'
'objective': 'binary',
'metric': ['auc', 'binary_logloss'],
'num_leaves': int(best_params_pso_lgbm[1]),
'max_depth': int(best_params_pso_lgbm[2]),
'params': variable}
# Train LightGBM model with error handling
try:
clf = [Link](lgbm_params_opt, d_train, 100)
scores = cross_val_score(clf, X, Y, cv=5, scoring='roc_auc')
return -[Link]() # Return the negative mean score for
minimization
except Exception as e:
print(f"Error during objective function evaluation: {e}")
return float('inf') # Return a large value to indicate failure
# Define the bounds for hyperparameters for pso optimization
lb_lgb_opt = [[Link][:, 2].min(),[Link][:, 3].min(), [Link][:,
4].min(), [Link][:, 5].min(), 5.4,[Link][:, 6].max(), [Link][:,
8].min(), [Link][:, 9].min()] # minimum UTS set at LN(400)
ub_lgb_opt = [[Link][:, 2].max(),[Link][:, 3].max(), [Link][:,
4].max(), 0.693, [Link][:, 6].max(), [Link][:, 7].max(), [Link][:,
8].max(), [Link][:, 9].max()] # maximum density set at LN(2)
# Perform PSO to find optimal variables
optimalvar, _ = pso( objective_function_opt, lb_lgb_opt, ub_lgb_opt,
swarmsize=10, maxiter=50)
# Column names
column_names = ["LNSiC_vol_perc", "LNSiC_size_um", "LNav_grain_um",
"LNDensity_gcc", "LNspec_YS", "LNspec_UTS", "LNtotal_perc_elong",
"LNspec_E"]
# Create a Pandas DataFrame
data_optimal = {"Column Name": column_names, "Optimal Value":
optimalvar}
df_optimal = [Link](data_optimal)
# Display the table
print(df_optimal)
exp=2.718281828459045
optimal_values=exp**optimalvar
# print(optimal_values)
column_names_values = ["SiC_vol_perc", "SiC_size_um", "av_grain_um",
"Density_gcc", "spec_YS", "spec_UTS", "total_perc_elong", "spec_E"]
# Create a Pandas DataFrame
data_optimal_values = {"Column Name": column_names_values, "Optimal
Value": optimal_values}
df_optimal_values = [Link](data_optimal_values)
# Display the table
print(df_optimal_values)
#
optimal_YS=optimal_values[3]*optimal_values[4]
optimal_UTS=optimal_values[3]*optimal_values[5]
optimal_E=optimal_values[3]*optimal_values[7]
optimal_values_opt=[optimal_YS, optimal_UTS, optimal_E]
# print(optimal_values_opt)
column_names_values_opt = ["Optimal YS", "Optimal UTS", "Optimal E"]
# Create a Pandas DataFrame
data_optimal_values_opt = {"Column Name": column_names_values_opt,
"Optimal Values": optimal_values_opt}
df_optimal_values_opt = [Link](data_optimal_values_opt)
# Display the table
print(df_optimal_values_opt)
###################################
# SHAP Analysis
X_train_shap = [Link](X_train, columns=[Link][2:11])
# Initialize the explainer with the LightGBM model
explainer = [Link](best_clf)
# Compute SHAP values for the test set
shap_values = explainer.shap_values(X_test)
# Plot the summary plot for overall feature importance
X_test_df = [Link](X_test, columns=X_train_shap.columns)
# Plot SHAP values for individual predictions
shap.summary_plot(shap_values, X_test_df,
feature_names=X_test_df.columns)
# ALE Analysis
# Define the features to be analyzed
features = ['LNSiC_vol_perc', 'LNSiC_size_um', 'LNav_grain_um']
# Create a figure and axes for the plots
fig, axes = [Link](1, len(features), figsize=(15, 5)) # 1 row, 3
columns
# Loop through each feature and create a PDP plot
for i, feature_name in enumerate(features):
pdp_goals = pdp.pdp_isolate(model=best_clf, dataset=X_test_df,
model_features=X_train_shap.columns, feature=feature_name)
pdp.pdp_plot(pdp_goals, feature_name, axes[i])
# Adjust layout and display the plot
plt.tight_layout()
[Link]()
# LIMEAnalysis
# Initialize LIME explainer
explainer = lime.lime_tabular.LimeTabularExplainer(
X_train,
feature_names=feature_names,
class_names=['N', 'Y'], # Replace with your class names if
different
mode='classification'
)
# Define a function to get predicted probabilities
def predict_proba_func(X):
# Get raw predictions
y_pred_raw = best_clf.predict(X)
# Apply sigmoid function if necessary (for binary classification)
y_pred_proba =[Link](y_pred_raw) / (1+[Link](y_pred_raw))
# Return probabilities for both classes (assuming binary
classification)
return np.column_stack([(1 - y_pred_proba), y_pred_proba])
# Explain an instance from the test set (e.g., the first instance)
i = 0 # Index of the instance to explain
exp = explainer.explain_instance(
X_test[i],
predict_proba_func, # Use the custom function
num_features=len(feature_names) # Number of features to include in
the explanation
)
# Display the explanation
exp.show_in_notebook(show_table=True, show_all=True)
# PSO-xgBoost
start = [Link]()
dtrain=[Link](X_train,label=y_train)
#setting parameters for xgboost
params_xgboost=[3,0.3]
# Define the objective function for PSO-XGBoost
def objective_function_xgb(params_xgboost):
# Create XGBoost parameters dictionary
xgb_params = {
'max_depth': int(params_xgboost[0]), # Convert to integer
'objective': 'binary:logistic',
'eval_metric': 'auc',
'learning_rate': params_xgboost[1]
}
# Evaluate using cross-validation with ROC AUC score
cv_results = [Link](
xgb_params,
dtrain,
num_boost_round=100, # Adjust as needed
nfold=5, # 5-fold cross-validation
metrics='auc',
early_stopping_rounds=10, # Early stopping for efficiency
seed=42, # Keep random_state for reproducibility
)
# Return the negative mean score (PSO minimizes)
return -cv_results['test-auc-mean'].iloc[-1] # Get the final AUC
score
# Define hyperparameter bounds for PSO
lb_xgb = [3, 0.01] # Lower bounds for max_depth, learning_rate
ub_xgb = [10, 0.3] # Upper bounds for max_depth, learning_rate
# Run PSO to find optimal hyperparameters
best_params_pso_xgb, _ = pso(objective_function_xgb, lb_xgb, ub_xgb,
swarmsize=10, maxiter=50)
# Print the best hyperparameters
print("Best hyperparameters for PSO-XGB:", best_params_pso_xgb)
# Create XGBoost model with the best hyperparameters
best_xgb_params = {
'max_depth': int(best_params_pso_xgb[0]),
'objective': 'binary:logistic',
'eval_metric': 'auc',
'learning_rate': best_params_pso_xgb[1]
}
# Train the best XGBoost model
best_pso_xgb = [Link](best_xgb_params, dtrain, 100)
#print("PSO-XGBoost execution time is: ", execution_time_xgb)
#now predicting the model on the test set
dtest=[Link](X_test)
y_pred_pso_xgb = best_pso_xgb.predict(dtest)
#print("XGBoost execution time is: ", execution_time_xgb)
#Converting probabilities into 1 or 0
for i in range(0, X_test.shape[0]):
if y_pred_pso_xgb[i]>=p: # setting threshold
y_pred_pso_xgb[i]=1
else:
y_pred_pso_xgb[i]=0
cm_pso_xgb = confusion_matrix(y_test, y_pred_pso_xgb)
#[Link](cm_xgb, annot=True)
stop = [Link]()
#Execution time of the model
execution_time_pso_xgb = stop-start
#print ("Accuracy with PSO-XGBoost= ",
metrics.accuracy_score(y_pred_pso_xgb, y_test))
#print("AUC score with PSO-XGBoost is: ", roc_auc_score(y_pred_pso_xgb,
y_test))
################
# Gradient Boosting Classifier (GBR)
start = [Link]()
#setting parameters for PSO-GBR
params_gbr=[50,0.3,3]
def objective_function_pso_gbr(params_gbr):
# Set GBR parameters
# Create GBR classifier with hyperparameters from params
n_estimators = int(params_gbr[0]) # Convert to integer
learning_rate = params_gbr[1]
max_depth = int(params_gbr[2]) # Convert to integer
gbr_classifier = gbr(
n_estimators=n_estimators,
learning_rate=learning_rate,
max_depth=max_depth,
random_state=42 # Keep random_state for reproducibility
)
# Evaluate using cross-validation with ROC AUC score, handling
potential errors
try:
scores = cross_val_score(gbr_classifier, X, Y, cv=5,
scoring='roc_auc')
return -[Link]() # Return the negative mean score (PSO
minimizes)
except Exception as e:
#print(f"Error during objective function evaluation: {e}")
return float('inf') # Return a large value to indicate failure
# Define hyperparameter bounds for PSO
lb_gbr = [50, 0.01, 3] # Lower bounds for n_estimators, learning_rate,
max_depth
ub_gbr = [200, 0.3, 10] # Upper bounds for n_estimators,
learning_rate, max_depth
# Run PSO to find optimal hyperparameters
best_params_pso_gbr, _ = pso(objective_function_pso_gbr, lb_gbr,
ub_gbr, swarmsize=10, maxiter=50)
# Print the best hyperparameters
print("Best hyperparameters for PSO-GBR:", best_params_pso_gbr)
# Create PSO-GBR classifier with the best hyperparameters
best_pso_gbr = gbr(
n_estimators=int(best_params_pso_gbr[0]),
learning_rate=best_params_pso_gbr[1],
max_depth=int(best_params_pso_gbr[2]),
random_state=42
)
# Fit the best GBR model to the data
best_pso_gbr.fit(X_train, y_train)
#print("GBR execution time is: ", execution_time_gbr)
#now predicting the model on the test set
y_pred_pso_gbr = best_pso_gbr.predict(X_test)
#Converting probabilities into 1 or 0
for i in range(0, X_test.shape[0]):
if y_pred_pso_gbr[i]>=p: # setting threshold
y_pred_pso_gbr[i]=1
else:
y_pred_pso_gbr[i]=0
cm_pso_gbr = confusion_matrix(y_test, y_pred_pso_gbr)
#[Link](cm_spo_gbr, annot=True)
stop = [Link]()
#Execution time of the model
execution_time_pso_gbr = stop-start
###########################################
# PSO-DT
# Define the objective function for PSO
start = [Link]()
params_pso_dt=[3]
def objective_function_pso_dt(params_pso_dt):
# Create Decision Tree classifier with hyperparameters from params
max_depth = int(params_pso_dt[0]) # Convert to integer
dt_classifier = dt(
max_depth=max_depth,
random_state=42 # Keep random_state for reproducibility
)
# Evaluate using cross-validation with ROC AUC score
scores = cross_val_score(dt_classifier, X, Y, cv=5,
scoring='roc_auc')
# Return the negative mean score (PSO minimizes)
return -[Link]()
# Define hyperparameter bounds for PSO
lb_dt = [2] # Lower bound for max_depth
ub_dt = [10] # Upper bound for max_depth
# Run PSO to find optimal hyperparameters
best_params_pso_dt, _ = pso(objective_function_pso_dt, lb_dt, ub_dt,
swarmsize=10, maxiter=50)
# Print the best hyperparameters
print("Best hyperparameters for PSO-DT:", best_params_pso_dt)
# Create Decision Tree classifier with the best hyperparameters
best_pso_dt = dt(
max_depth=int(best_params_pso_dt[0]),
random_state=42
)
# Fit the best Decision Tree model to the data
best_pso_dt.fit(X_train, y_train)
y_pred_pso_dt = best_pso_dt.predict(X_test)
#print(y_pred_pso_dt)
#Converting probabilities into 1 or 0
for i in range(0, X_test.shape[0]):
if y_pred_pso_dt[i]>=p: # setting threshold to .4
y_pred_pso_dt[i]=1
else:
y_pred_pso_dt[i]=0
cm_pso_dt = confusion_matrix(y_test, y_pred_pso_dt)
#[Link](cm_pso_dt, annot=True)
stop = [Link]()
#Execution time of the model
execution_time_pso_dt = stop-start
# Create a figure with two subplots
fig, axes = [Link](1, 2, figsize=(12, 5)) # 2 rows, 2 columns
# Plot the first confusion matrix (PSO-LGBM)
[Link](cm_pso_lgbm, annot=True, ax=axes[0])
axes[0].set_title("PSO-LGBM Confusion Matrix")
# Plot the second confusion matrix (PSO-XGBoost)
[Link](cm_pso_xgb, annot=True, ax=axes[1])
axes[1].set_title("PSO-XBoost Confusion Matrix")
# Create a figure with two subplots
fig, axes = [Link](1, 2, figsize=(12, 5)) # 1 row, 2 columns
# Plot the first confusion matrix (PSO-GBR)
[Link](cm_pso_gbr, annot=True, ax=axes[0])
axes[0].set_title("PSO-GBR Confusion Matrix")
# Plot the second confusion matrix (PSO-XGBoost - assuming you have
cm_xgb)
[Link](cm_pso_dt, annot=True, ax=axes[1])
axes[1].set_title("PSO-DT Confusion Matrix")
# Adjust layout and display the plot
plt.tight_layout()
[Link]()
#SUMMARY
print("################################################")
print("PSO-LGBM execution time is: ", execution_time_pso_lgbm)
print("PSO-XGBoost execution time is: ", execution_time_pso_xgb)
print("PSO-GBR execution time is: ", execution_time_pso_gbr)
print("PSO-DT execution time is: ", execution_time_pso_dt)
print("################################################")
print ("Accuracy with PSO-LGBM = ",
metrics.accuracy_score(y_pred_pso_lgbm,y_test))
print ("Accuracy with PSO-XGBoost= ",
metrics.accuracy_score(y_pred_pso_xgb, y_test))
print ("Accuracy with PSO-GBR= ",
metrics.accuracy_score(y_pred_pso_gbr, y_test))
print ("Accuracy with PSO-DT= ", metrics.accuracy_score(y_pred_pso_dt,
y_test))
print("################################################")
print ("Precision with PSO-LGBM = ",
metrics.precision_score(y_pred_pso_lgbm,y_test))
print ("Precision with PSO-XGBoost= ",
metrics.precision_score(y_pred_pso_xgb, y_test))
print ("Precision with PSO-GBR= ",
metrics.precision_score(y_pred_pso_gbr, y_test))
print ("Precision with PSO-DT= ",
metrics.precision_score(y_pred_pso_dt, y_test))
print("################################################")
print ("Recall score with PSO-LGBM = ",
metrics.recall_score(y_pred_pso_lgbm,y_test))
print ("Recall score with PSO-XGBoost= ",
metrics.recall_score(y_pred_pso_xgb, y_test))
print ("Recall score with PSO-GBR= ",
metrics.recall_score(y_pred_pso_gbr, y_test))
print ("Recall score with PSO-DT= ",
metrics.recall_score(y_pred_pso_dt, y_test))
print("################################################")
print ("F1-score with PSO-LGBM = ",
metrics.f1_score(y_pred_pso_lgbm,y_test))
print ("F1-score with PSO-XGBoost= ", metrics.f1_score(y_pred_pso_xgb,
y_test))
print ("F1-score with PSO-GBR= ", metrics.f1_score(y_pred_pso_gbr,
y_test))
print ("F1-score with PSO-DT= ", metrics.f1_score(y_pred_pso_dt,
y_test))
print("################################################")
print("AUC score with PSO-LGBM is: ",
roc_auc_score(y_pred_pso_lgbm,y_test))
print("AUC score with PSO-XGBoost is: ", roc_auc_score(y_pred_pso_xgb,
y_test))
print("AUC score with PSO-GBR is: ",
roc_auc_score(y_pred_pso_gbr,y_test))
print("AUC score with PSO-DT is: ", roc_auc_score(y_pred_pso_dt,
y_test))
# From these results, PSO-LGBM and PSO-DT show the best overall
performance in terms of accuracy, recall, and F1-score. However, PSO-
XGBoost has the shortest execution time, making it the fastest
algorithm. PSO-GBR has the longest execution time but performs well in
precision and F1-score.
# PSO-LGBM and PSO-XGBoost: Both algorithms have a recall of 1.0,
indicating they correctly identified all positive instances without
missing any. This is ideal for applications where it is crucial to
capture all positive cases, such as medical diagnoses or fraud
detection.
# PSO-GBR: With a recall of 0.83, PSO-GBR correctly identified 83% of
the positive instances. While this is still a good performance, it
means that 17% of the actual positives were missed.
# PSO-DT has a recall of 0.91, meaning it correctly identified 91% of
the positive instances. This is a strong performance, though not as
perfect as PSO-LGBM and PSO-XGBoost.
# PSO-DT and PSO-LGBM are the top performers in terms of F1-score,
indicating a strong balance between precision and recall.
# PSO-GBR also performs well, particularly in precision.
# PSO-XGBoost has the lowest precision and F1-score, suggesting it may
not be as reliable as the other algorithms in this context.
# Function to plot ROC curve and print AUC score
def plot_roc_curve(model_name, y_true, y_pred_proba):
fpr, tpr, _ = roc_curve(y_true, y_pred_proba)
auc_score = roc_auc_score(y_true, y_pred_proba)
[Link](fpr, tpr, label=f'{model_name} (AUC = {auc_score:.2f})')
[Link]([0, 1], [0, 1], 'k--') # Diagonal line for random
classifier
[Link]('False Positive Rate')
[Link]('True Positive Rate')
[Link]('Receiver Operating Characteristic (ROC) Curve')
[Link](loc='lower right')
# Plot ROC curves for all models
plot_roc_curve("PSO-LGBM", y_test, y_pred_pso_lgbm)
plot_roc_curve("PSO-XGBoost", y_test, y_pred_pso_xgb)
plot_roc_curve("PSO-GBR", y_test, y_pred_pso_gbr)
plot_roc_curve("PSO-DT", y_test, y_pred_pso_dt)
[Link]() # Display the the graphs