import numpy as np
import pandas as pd
import optuna
from sklearn.model_selection import train_test_split, KFold
from [Link] import mean_squared_error
from sklearn.linear_model import Ridge
import lightgbm as lgb
import xgboost as xgb
from catboost import CatBoostRegressor
# =====================
# Load Data
# =====================
train = pd.read_csv("/kaggle/input/[Link]")
test = pd.read_csv("/kaggle/input/[Link]")
X = [Link](columns=["id", "BeatsPerMinute"])
y = train["BeatsPerMinute"]
X_test = [Link](columns=["id"])
# =====================
# Optuna Tuning - LightGBM (already done, using best params)
# =====================
best_lgb_params = {
"objective": "regression",
"metric": "rmse",
"learning_rate": 0.028551392479384357,
"num_leaves": 42,
"max_depth": 8,
"feature_fraction": 0.9589137177373734,
"bagging_fraction": 0.9509935933870622,
"bagging_freq": 7,
"min_data_in_leaf": 42,
"lambda_l1": 0.007529602616697326,
"lambda_l2": 4.811359028566392e-06,
"seed": 42
}
# =====================
# Optuna Tuning - XGBoost
# =====================
def objective_xgb(trial):
params = {
"objective": "reg:squarederror",
"eval_metric": "rmse",
"tree_method": "hist",
"learning_rate": trial.suggest_float("learning_rate", 0.01, 0.1, log=True),
"max_depth": trial.suggest_int("max_depth", 4, 12),
"subsample": trial.suggest_float("subsample", 0.6, 1.0),
"colsample_bytree": trial.suggest_float("colsample_bytree", 0.6, 1.0),
"lambda": trial.suggest_float("lambda", 1e-8, 10.0, log=True),
"alpha": trial.suggest_float("alpha", 1e-8, 10.0, log=True),
"n_estimators": 5000,
"random_state": 42,
}
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2,
random_state=42)
model = [Link](**params)
[Link](
X_train, y_train,
eval_set=[(X_val, y_val)],
early_stopping_rounds=50,
verbose=False
)
preds = [Link](X_val)
return [Link](mean_squared_error(y_val, preds))
study_xgb = optuna.create_study(direction="minimize")
study_xgb.optimize(objective_xgb, n_trials=30, show_progress_bar=True)
best_xgb_params = study_xgb.best_params
print("Best XGB Params:", best_xgb_params)
# =====================
# Optuna Tuning - CatBoost
# =====================
def objective_cat(trial):
params = {
"iterations": 5000,
"learning_rate": trial.suggest_float("learning_rate", 0.01, 0.1, log=True),
"depth": trial.suggest_int("depth", 4, 12),
"l2_leaf_reg": trial.suggest_float("l2_leaf_reg", 1.0, 10.0),
"random_seed": 42,
"loss_function": "RMSE",
"od_type": "Iter",
"od_wait": 50,
"verbose": False
}
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2,
random_state=42)
model = CatBoostRegressor(**params)
[Link](X_train, y_train, eval_set=(X_val, y_val), verbose=False)
preds = [Link](X_val)
return [Link](mean_squared_error(y_val, preds))
study_cat = optuna.create_study(direction="minimize")
study_cat.optimize(objective_cat, n_trials=30, show_progress_bar=True)
best_cat_params = study_cat.best_params
print("Best Cat Params:", best_cat_params)
# =====================
# OOF Stacking
# =====================
NFOLDS = 5
kf = KFold(n_splits=NFOLDS, shuffle=True, random_state=42)
oof_lgb = [Link](len(X))
oof_xgb = [Link](len(X))
oof_cat = [Link](len(X))
test_preds_lgb = [Link]((NFOLDS, len(X_test)))
test_preds_xgb = [Link]((NFOLDS, len(X_test)))
test_preds_cat = [Link]((NFOLDS, len(X_test)))
for fold, (train_idx, val_idx) in enumerate([Link](X, y)):
print(f"FOLD {fold+1}")
X_train, X_val = [Link][train_idx], [Link][val_idx]
y_train, y_val = [Link][train_idx], [Link][val_idx]
# LightGBM
dtrain = [Link](X_train, label=y_train)
dval = [Link](X_val, label=y_val)
lgb_model = [Link](
best_lgb_params,
dtrain,
valid_sets=[dval],
num_boost_round=10000,
callbacks=[lgb.early_stopping(stopping_rounds=50, verbose=False)]
)
oof_lgb[val_idx] = lgb_model.predict(X_val)
test_preds_lgb[fold] = lgb_model.predict(X_test)
# XGBoost
xgb_model = [Link](
n_estimators=5000,
random_state=42,
tree_method="hist",
**best_xgb_params
)
xgb_model.fit(
X_train, y_train,
eval_set=[(X_val, y_val)],
early_stopping_rounds=50,
verbose=False
)
oof_xgb[val_idx] = xgb_model.predict(X_val)
test_preds_xgb[fold] = xgb_model.predict(X_test)
# CatBoost
cat_model = CatBoostRegressor(
iterations=5000,
random_seed=42,
loss_function="RMSE",
od_type="Iter",
od_wait=50,
verbose=False,
**best_cat_params
)
cat_model.fit(X_train, y_train, eval_set=(X_val, y_val), verbose=False)
oof_cat[val_idx] = cat_model.predict(X_val)
test_preds_cat[fold] = cat_model.predict(X_test)
# =====================
# Meta Model (Stacking)
# =====================
stack_train = [Link]([oof_lgb, oof_xgb, oof_cat]).T
stack_test = [Link]([
test_preds_lgb.mean(axis=0),
test_preds_xgb.mean(axis=0),
test_preds_cat.mean(axis=0)
]).T
meta_model = Ridge(alpha=1.0)
meta_model.fit(stack_train, y)
final_preds = meta_model.predict(stack_test)
# =====================
# Evaluate OOF
# =====================
rmse = [Link](mean_squared_error(y, meta_model.predict(stack_train)))
print("OOF Stacking RMSE:", rmse)
# =====================
# Submission
# =====================
submission = [Link]({
"id": test["id"],
"BeatsPerMinute": final_preds
})
submission.to_csv("[Link]", index=False)
print("Submission saved as [Link]")