Kaggle competition: Tabular Playground Series - Feb 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-feb-2021<load_from_csv>
import plotly.express as px
import plotly.graph_objects as go
import plotly.figure_factory as ff
from plotly.subplots import make_subplots
import matplotlib.pyplot as plt
from colorama import Fore
from pandas_profiling import ProfileReport
import seaborn as sns
from sklearn import metrics
from scipy import stats
import math
from tqdm.notebook import tqdm
from copy import deepcopy
from sklearn.ensemble import RandomForestRegressor
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
import lightgbm as lgb
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score
import optuna
from optuna import Trial, visualization
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score,confusion_matrix
from sklearn.metrics import accuracy_score, mean_squared_error
sample_submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv" )<drop_column>
train_df = pd.read_csv('/kaggle/input/tabular-playground-series-feb-2021/train.csv')
 test_df = pd.read_csv('/kaggle/input/tabular-playground-series-feb-2021/test.csv')
 sub_df = pd.read_csv('/kaggle/input/tabular-playground-series-feb-2021/sample_submission.csv')
 train_df.head()
train_ID = train['Id']
test_ID = test['Id']
train.drop("Id", axis = 1, inplace = True)
 test.drop("Id", axis = 1, inplace = True )<drop_column>
train_indexs = train_df.index
test_indexs = test_df.index
df = pd.concat(objs=[train_df, test_df], axis=0 ).reset_index(drop=True)
 df = df.drop('id', axis=1 )
train = train[train.GrLivArea < 4500]
train.reset_index(drop=True, inplace=True)
 outliers = [30, 88, 462, 631, 1322]
train = train.drop(train.index[outliers] )<concatenate>
def modify_df(df):
df['cat4'] = df['cat4'].apply(lambda x: x if x == 'B' else 'Z')
 df['cat5'] = df['cat5'].apply(lambda x: x if x in ['B', 'D'] else 'Z')
 df['cat6'] = df['cat6'].apply(lambda x: x if x == 'A' else 'Z')
 df['cat7'] = df['cat7'].apply(lambda x: x if x in ['E', 'D'] else 'Z')
 df['cat8'] = df['cat8'].apply(lambda x: x if x in ['E', 'C', 'G', 'A'] else 'Z')
 df['cont001'] = df['cont8'] * df['cont0']
df['cont002'] = df['cont9'] * df['cont0']
df['cont003'] = df['cont9'] * df['cont5']
df['cont004'] = df['cont8'] * df['cont5']
df['cont005'] = df['cont2'] * df['cont4']
df['cont006'] = df['cont1'] * df['cont3']
df['cont007'] = df['cont13'] * df['cont1']
return df
mod_train_df = modify_df(train_df.copy())
 mod_test_df = modify_df(test_df.copy() )
train_X = train.drop('SalePrice',axis=1)
 data_features = pd.concat(( train_X, test)).reset_index(drop=True)
 print(data_features.shape)
 data_features.columns 
<sort_values>
for feature in categorical_columns:
le = LabelEncoder()
le.fit(train_df[feature])
 train_df[feature] = le.transform(train_df[feature])
 test_df[feature] = le.transform(test_df[feature])
 for feature in categorical_columns:
le = LabelEncoder()
le.fit(mod_train_df[feature])
 mod_train_df[feature] = le.transform(mod_train_df[feature])
 mod_test_df[feature] = le.transform(mod_test_df[feature] )
data_features_na = data_features.isnull().sum().sort_values(ascending=False)
 data_features_na = data_features_na[data_features_na>0]
data_features_na<concatenate>
x = train_df[feature_cols]
y = train_df['target']
feature_cols_mod = mod_train_df.drop(['id', 'target'], axis=1 ).columns
xmod, ymod = mod_train_df[feature_cols_mod], mod_train_df['target']
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.3, random_state=42 )
percent =(data_features.isnull().sum() /data_features.isnull().count() ).sort_values(ascending=False)
 percent = percent[percent>0]
pd.concat([data_features_na, percent],axis=1,keys=['total', 'percent'] )<prepare_x_and_y>
clf = XGBRegressor(random_state=42, tree_method='gpu_hist')
 clf.fit(x_train, y_train )
x = data_features.loc[(data_features['LotArea'].notnull()), 'LotArea']
y = data_features.loc[(data_features['LotFrontage'].notnull()), 'LotFrontage']
t =(x<25000)&(y<150)
 coef = np.polyfit(x[t], y[t],1)
 formula = np.poly1d(coef)
 poly_y = formula(data_features['LotArea'])
 condition_frontage =(data_features['LotFrontage'].isnull())
 data_features.loc[condition_frontage,'LotFrontage'] = formula(data_features.loc[condition_frontage,'LotArea'])
 garage_var = ['GarageYrBlt','GarageCond','GarageFinish','GarageQual']
condition1 =(data_features['GarageYrBlt'].isnull() & data_features['GarageType'].notnull())
 for col in garage_var:
data_features.loc[condition1,col] = data_features[(data_features['GarageType'] == 'Detchd')][col].mode() [0]
condition2 =(data_features['BsmtFinType1'].notnull() & data_features['BsmtExposure'].isnull())
 data_features.loc[condition2,'BsmtExposure'] = 'No'
condition3 =(data_features['BsmtFinType1'].notnull() & data_features['BsmtCond'].isnull())
 data_features.loc[condition3,'BsmtCond'] = 'TA'
condition4 =(data_features['BsmtFinType1'].notnull() & data_features['BsmtQual'].isnull())
 data_features.loc[condition4,'BsmtQual'] = data_features.loc[(data_features['BsmtExposure'] == 'No'),'BsmtQual'].mode() [0]
condition5 =(data_features['BsmtFinType2'].isnull() & data_features['BsmtFinType1'].notnull())
 data_features.loc[condition5, 'BsmtFinType2'] = 'Unf'
bsmt_var = ['BsmtCond','BsmtExposure','BsmtQual','BsmtFinType1','BsmtFinType2']
garage_var = ['GarageType','GarageCond','GarageFinish','GarageQual']
NONE_var = ['PoolQC','MiscFeature','Alley','Fence','FireplaceQu']
for col in bsmt_var, garage_var,NONE_var:
data_features[col] = data_features[col].fillna('None')
 data_features['GarageYrBlt'] = data_features['GarageYrBlt'].fillna(0)
 condition6 =(data_features['MasVnrType'].isnull() & data_features['MasVnrArea'].notnull())
 data_features.loc[condition6,'MasVnrType'] = 'Stone'
data_features['MasVnrType'] = data_features['MasVnrType'].fillna('None')
 data_features['MasVnrArea'] = data_features['MasVnrArea'].fillna(0)
 data_features['MSZoning'] = data_features['MSZoning'].groupby(
 data_features['MSSubClass'] ).transform(lambda x:x.fillna(x.mode() [0]))
NA_for_0 = ['BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF','BsmtFullBath','BsmtHalfBath',
'GarageArea', 'GarageCars','MasVnrArea']
for col in NA_for_0:
data_features[col] = data_features[col].fillna(0)
 common_for_NA = ['Exterior1st','Exterior2nd','SaleType','Electrical','KitchenQual']
for col in common_for_NA:
data_features[col].fillna(data_features[col].mode() [0], inplace = True)
 data_features['Functional'] = data_features['Functional'].fillna('Typ')
 data_features['Utilities'] = data_features['Utilities'].fillna('None' )<sort_values>
predictions = clf.predict(x_test)
 score_rmse = math.sqrt(mean_squared_error(y_test, predictions))
print(Fore.GREEN + 'Base XGBoost RMSE: {}'.format(score_rmse))
missing_data = data_features.isnull().sum().sort_values(ascending = False)
 missing_data = missing_data[missing_data > 0]
missing_data<feature_engineering>
sub_xgb_base = clf.predict(test_df[feature_cols] )
str_var = ["MSSubClass",'MoSold','YrSold']
for var in str_var:
data_features[var] = data_features[var].apply(str )<categorify>
clf = LGBMRegressor(random_state=42, device='gpu')
 clf.fit(x_train, y_train )
data_features["MSSubClass"] = data_features.MSSubClass.map({'180':1,
'30':2, '45':2,
'190':3, '50':3, '90':3,
'85':4, '40':4, '160':4,
'70':5, '20':5, '75':5, '80':5, '150':5,
'120': 6, '60':6})
 data_features["MSZoning"] = data_features.MSZoning.map({'C(all)':1, 'RH':2, 'RM':2, 'RL':3, 'FV':4})
 data_features["Street"] = data_features.Street.map({'Grvl':1, 'Pave':2})
 data_features["Alley"] = data_features.Alley.map({'None':0, 'Grvl':1, 'Pave':2})
 data_features["LotShape"] = data_features.LotShape.map({'Reg':1, 'IR1':2, 'IR2':3, 'IR3':3})
 data_features["LandContour"] = data_features.LandContour.map({'Bnk':1, 'Lvl':2, 'Low':3, 'HLS':4})
 data_features["Utilities"] = data_features.Utilities.map({'NoSeWa':0, 'None':0,'AllPub':1})
 data_features["LotConfig"] = data_features.LotConfig.map({'Inside':1, 'Corner':1, 'CulDSac':2, 'FR2':1, 'FR3':2})
 data_features["LandSlope"] = data_features.LandSlope.map({'Gtl':1, 'Mod':2, 'Sev':2})
 data_features["Neighborhood"] = data_features.Neighborhood.map({'MeadowV':1,
'IDOTRR':2, 'BrDale':2,
'OldTown':3, 'Edwards':3, 'BrkSide':3,
'Sawyer':4, 'Blueste':4, 'SWISU':4, 'NAmes':4,
'NPkVill':5, 'Mitchel':5,
'SawyerW':6, 'Gilbert':6, 'NWAmes':6,
'Blmngtn':7, 'CollgCr':7, 'ClearCr':7, 'Crawfor':7,
'Veenker':8, 'Somerst':8, 'Timber':8,
'StoneBr':9,
'NoRidge':10, 'NridgHt':10})
 data_features["Condition1"] = data_features.Condition1.map({'Artery':1,
'Feedr':2, 'RRAe':2,
'Norm':3, 'RRAn':3,
'PosN':4, 'RRNe':4,
'PosA':5 ,'RRNn':5})
 data_features["BldgType"] = data_features.BldgType.map({'2fmCon':1, 'Duplex':1, 'Twnhs':1, '1Fam':2, 'TwnhsE':2})
 data_features["HouseStyle"] = data_features.HouseStyle.map({'1.5Unf':1,
'1.5Fin':2, '2.5Unf':2, 'SFoyer':2,
'1Story':3, 'SLvl':3,
'2Story':4, '2.5Fin':4})
 data_features['RoofStyle'] = data_features.RoofStyle.map({'Gambrel':1, 'Gable':2, 'Flat':3, 'Hip':3, 'Mansard':3, 'Shed':4})
 data_features["Exterior1st"] = data_features.Exterior1st.map({'BrkComm':1,
'AsphShn':2, 'CBlock':2, 'AsbShng':2,
'WdShing':3, 'Wd Sdng':3, 'MetalSd':3, 'Stucco':3, 'HdBoard':3,
'BrkFace':4, 'Plywood':4,
'VinylSd':5,
'CemntBd':6,
'Stone':7, 'ImStucc':7})
 data_features["MasVnrType"] = data_features.MasVnrType.map({'BrkCmn':1, 'None':1, 'BrkFace':2, 'Stone':3})
 data_features["ExterQual"] = data_features.ExterQual.map({'Fa':1, 'TA':2, 'Gd':3, 'Ex':4})
 data_features["Foundation"] = data_features.Foundation.map({'Slab':1,
'BrkTil':2, 'CBlock':2, 'Stone':2,
'Wood':3, 'PConc':4})
 data_features["BsmtQual"] = data_features.BsmtQual.map({'Fa':2, 'None':1, 'TA':3, 'Gd':4, 'Ex':5})
 data_features["BsmtCond"] = data_features.BsmtCond.map({'Fa':2, 'None':0, 'TA':3, 'Gd':4, 'Po':1})
 data_features["BsmtExposure"] = data_features.BsmtExposure.map({'None':1, 'No':2, 'Av':3, 'Mn':3, 'Gd':4})
 data_features["Heating"] = data_features.Heating.map({'Floor':1, 'Grav':1, 'Wall':2, 'OthW':3, 'GasW':4, 'GasA':5})
 data_features["HeatingQC"] = data_features.HeatingQC.map({'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5})
 data_features['CentralAir'] = data_features.CentralAir.map({'N':1, 'Y':2})
 data_features["Electrical"] = data_features.Electrical.map({'Mix':1, 'FuseP':2, 'FuseF':3, 'FuseA':4, 'SBrkr':5})
 data_features["KitchenQual"] = data_features.KitchenQual.map({'Fa':1, 'TA':2, 'Gd':3, 'Ex':4})
 data_features["Functional"] = data_features.Functional.map({'Maj2':1, 'Maj1':2, 'Min1':2, 'Min2':2, 'Mod':2, 'Sev':2, 'Typ':3})
 data_features["FireplaceQu"] = data_features.FireplaceQu.map({'None':1, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5})
 data_features["GarageType"] = data_features.GarageType.map({'CarPort':1, 'None':1,
'Detchd':2,
'2Types':3, 'Basment':3,
'Attchd':4, 'BuiltIn':5})
 data_features["GarageFinish"] = data_features.GarageFinish.map({'None':1, 'Unf':2, 'RFn':3, 'Fin':4})
 data_features["GarageQual"] = data_features.GarageQual.map({'None':0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':4})
 data_features["PavedDrive"] = data_features.PavedDrive.map({'N':1, 'P':2, 'Y':3})
 data_features["PoolQC"] = data_features.PoolQC.map({'None':0, 'Gd':1, 'Fa':1, 'Ex':2})
 data_features["SaleType"] = data_features.SaleType.map({'COD':1, 'ConLD':1, 'ConLI':1, 'ConLw':1, 'Oth':1, 'WD':1,
'CWD':2, 'Con':3, 'New':3})
 data_features["SaleCondition"] = data_features.SaleCondition.map({'AdjLand':1, 'Abnorml':2, 'Alloca':2, 'Family':2, 'Normal':3, 'Partial':4} )<categorify>
predictions = clf.predict(x_test)
 score_rmse = math.sqrt(mean_squared_error(y_test, predictions))
print(Fore.GREEN + 'Base LGBM RMSE: {}'.format(score_rmse))
le = preprocessing.LabelEncoder()
Year_var = ['YearBuilt','YearRemodAdd','GarageYrBlt']
for var in Year_var:
data_features[var] = le.fit_transform(data_features[var])
 data_features[Year_var]<feature_engineering>
train_oof = np.zeros(( 300000,))
test_preds = 0
train_oof.shape
data_features['MasVnrScore'] = data_features['MasVnrType'] * data_features['MasVnrArea']
data_features['BsmtScore2'] = data_features['BsmtQual'] * data_features['BsmtCond'] * data_features['BsmtExposure']
data_features['TotalSF'] = data_features['TotalBsmtSF'] + data_features['1stFlrSF'] + data_features['2ndFlrSF']
data_features["AllSF"] = data_features["GrLivArea"] + data_features["TotalBsmtSF"]
data_features['TotalBath'] = data_features['FullBath'] + 0.5*data_features['HalfBath'] + data_features['BsmtFullBath'] + 0.5*data_features['BsmtHalfBath']
data_features['Total_porch_sf'] = data_features['OpenPorchSF'] + data_features['3SsnPorch'] + data_features['EnclosedPorch'] + data_features['ScreenPorch'] + data_features['WoodDeckSF']<feature_engineering>
NUM_FOLDS = 5
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)
 for f,(train_ind, val_ind)in tqdm(enumerate(kf.split(x, y))):
tmp_train_df, tmp_val_df = x.iloc[train_ind][feature_cols], x.iloc[val_ind][feature_cols]
train_target, val_target = y[train_ind], y[val_ind]
model = LGBMRegressor(random_state=42, device='gpu')
 model.fit(tmp_train_df, train_target, eval_set=[(tmp_val_df, val_target)], verbose=False)
 temp_oof = model.predict(tmp_val_df)
 temp_test = model.predict(test_df[feature_cols])
 train_oof[val_ind] = temp_oof
test_preds += temp_test/NUM_FOLDS
print(mean_squared_error(temp_oof, val_target, squared=False))
train1 = data_features[:len(y_train)]
train1.loc[:,'SalePrice'] = y_train
corr1 = train1.corr() ['SalePrice'].sort_values(ascending = False)
 corr15 = corr1[:15]
corr15<feature_engineering>
sub_df['target'] = test_preds
sub_df.to_csv('submission_lgbm_cv.csv', index=False)
 sub_df.head()
sub_lgbm_cv = test_preds
corr10 = corr15.drop(['SalePrice','AllSF','1stFlrSF','GarageFinish','GarageArea'])
 corr10_var = corr10.index.tolist()
for col in corr10_var:
data_features[col + '-2'] = data_features[col] **2
data_features[col + '-3'] = data_features[col] **3
data_features[col + '-sqrt'] = np.sqrt(data_features[col])
<sort_values>
xgb_params = {
'booster':'gbtree',
'n_estimators':20000,
'max_depth':5,
'eta':0.008,
'gamma':3.5,
'objective':'reg:squarederror',
'verbosity':0,
'subsample':0.75,
'colsample_bytree':0.35,
'reg_lambda':0.23,
'reg_alpha':0.52,
'scale_pos_weight':1,
'objective':'reg:squarederror',
'eval_metric':'rmse',
'seed': 42,
'tree_method':'gpu_hist',
'gpu_id':0
}
skew_features = data_num.skew().sort_values(ascending = False)
 skew_features<import_modules>
train_oof = np.zeros(( 300000,))
test_preds = 0
train_oof.shape
NUM_FOLDS = 5
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=42)
 for f,(train_ind, val_ind)in tqdm(enumerate(kf.split(x, y))):
tmp_train_df, tmp_val_df = x.iloc[train_ind][feature_cols], x.iloc[val_ind][feature_cols]
train_target, val_target = y[train_ind], y[val_ind]
model = XGBRegressor(**xgb_params)
 model.fit(
 tmp_train_df,
train_target,
eval_set=[(tmp_val_df, val_target)],
early_stopping_rounds = 50,
verbose = False 
)
 temp_oof = model.predict(tmp_val_df)
 temp_test = model.predict(test_df[feature_cols])
 train_oof[val_ind] = temp_oof
test_preds += temp_test/NUM_FOLDS
print(mean_squared_error(temp_oof, val_target, squared=False))
from scipy.special import boxcox1p
from scipy.stats import boxcox_normmax
from sklearn.base import BaseEstimator, TransformerMixin<categorify>
sub_df['target'] = test_preds
sub_df.to_csv('submission_xgb_cv_optimized.csv', index=False)
 sub_df.head()
sub_xgb_cv_optimized = test_preds
class skewness__dummies(BaseEstimator, TransformerMixin):
def __init__(self, skew = 0.5):
pass
def fit(self, X, y = None):
return self
def transform(self, X, y = None):
num_fea = X.select_dtypes(exclude = ['object'])
 skew_fea = X[abs(num_fea.skew())> skew ].index
X[skew_fea] = boxcox1p(X.loc[:,skew_fea], boxcox_normmax(X.loc[:,skew_fea]))
X = pd.get_dummies(X)
 return X 
<feature_engineering>
xgb_params = {
'booster':'gbtree',
'n_estimators':10000,
'max_depth':7,
'eta':0.01,
'gamma':1.8,
'objective':'reg:squarederror',
'verbosity':0,
'subsample':0.85,
'colsample_bytree':0.4,
'lambda':2.7,
'alpha':6,
'scale_pos_weight':1,
'objective':'reg:squarederror',
'eval_metric':'rmse',
'seed': 42,
'tree_method':'gpu_hist',
'gpu_id':0
}
skew_features_index = skew_features.index
for features in skew_features_index:
data_num.loc[:,features] = boxcox1p(data_num.loc[:,features], boxcox_normmax(data_num.loc[:,features] + 1))
data_num.loc[:,features] = boxcox1p(data_num.loc[:,features], boxcox_normmax(data_num.loc[:,features] + 1))
print('After the transformation the mean skewness of the features is {}'.format(np.mean(data_num.skew())))
 data_num.skew().sort_values(ascending = False )<categorify>
train_oof = np.zeros(( 300000,))
test_preds = 0
train_oof.shape
NUM_FOLDS = 5
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=42)
 for f,(train_ind, val_ind)in tqdm(enumerate(kf.split(xmod, ymod))):
tmp_train_df, tmp_val_df = xmod.iloc[train_ind][feature_cols_mod], xmod.iloc[val_ind][feature_cols_mod]
train_target, val_target = ymod[train_ind], ymod[val_ind]
model = XGBRegressor(**xgb_params)
 model.fit(
 tmp_train_df,
train_target,
eval_set=[(tmp_val_df, val_target)],
early_stopping_rounds = 50,
verbose = False 
)
 temp_oof = model.predict(tmp_val_df)
 temp_test = model.predict(mod_test_df[feature_cols_mod])
 train_oof[val_ind] = temp_oof
test_preds += temp_test/NUM_FOLDS
print(mean_squared_error(temp_oof, val_target, squared=False))
final_features = pd.get_dummies(data_features)
 X_train = final_features.iloc[:len(y_train),:]
X_test = final_features.iloc[len(y_train):,:]
print('The shape of train set is{},y set is{},and the shape of test set is{}'.format(X_train.shape,y_train.shape,X_test.shape))<count_missing_values>
sub_df['target'] = test_preds
sub_df.to_csv('submission_xgb_mod_cv_optimized.csv', index=False)
 sub_df.head()
sub_xgb_mod_cv_optimized = test_preds
X.isnull().sum().sort_values(ascending = False )<import_modules>
def objective(trial,data=x,target=y):
train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.15,random_state=42)
 param = {
'device':'gpu',
'metric': 'rmse',
'random_state': 42,
'reg_lambda': trial.suggest_loguniform(
 'reg_lambda', 1e-3, 10.0 
),
'reg_alpha': trial.suggest_loguniform(
 'reg_alpha', 1e-3, 10.0 
),
'colsample_bytree': trial.suggest_categorical(
 'colsample_bytree', [0.3,0.5,0.6,0.7,0.8,0.9,1.0] 
),
'subsample': trial.suggest_categorical(
 'subsample', [0.6,0.7,0.8,1.0] 
),
'learning_rate': trial.suggest_categorical(
 'learning_rate', [0.008,0.009,0.01,0.012,0.014,0.016,0.018, 0.02] 
),
'n_estimators': trial.suggest_categorical(
 "n_estimators", [150, 200, 300, 3000] 
),
'max_depth': trial.suggest_categorical(
 'max_depth', [4,5,7,9,11,13,15,17,20] 
),
'min_child_samples': trial.suggest_int(
 'min_child_samples', 1, 300 
),
'num_leaves': trial.suggest_int(
 'num_leaves', 15, 120 
),
}
model = LGBMRegressor(**param)
 model.fit(train_x,train_y,eval_set=[(test_x,test_y)], early_stopping_rounds=300, verbose=False)
 preds = model.predict(test_x)
 rmse = mean_squared_error(test_y, preds,squared=False)
 return rmse
from datetime import datetime
from sklearn.preprocessing import RobustScaler
from sklearn.model_selection import KFold, cross_val_score
from sklearn.metrics import mean_squared_error , make_scorer
from sklearn.linear_model import ElasticNetCV, LassoCV, RidgeCV
from sklearn.pipeline import make_pipeline
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.svm import SVR
from mlxtend.regressor import StackingCVRegressor
from sklearn.linear_model import LinearRegression
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor<compute_train_metric>
study = optuna.create_study(direction='minimize')
 study.optimize(objective, n_trials=5)
 print('Number of finished trials:', len(study.trials))
print('Best trial:', study.best_trial.params )
kfolds = KFold(n_splits=18, shuffle=True, random_state=42)
 def cv_rmse(model, X, y):
rmse = np.sqrt(-cross_val_score(model, X, y,scoring="neg_mean_squared_error",cv=kfolds))
return(rmse)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))<choose_model_class>
study.best_params
models = {'XGBoost':XGBRegressor(n_estimators=1000,
max_depth=6,
objective ='reg:squarederror')}<train_on_grid>
best_params = {
'reg_lambda': 0.015979956459638782,
'reg_alpha': 9.103977313355028,
'colsample_bytree': 0.3,
'subsample': 1.0,
'learning_rate': 0.009,
'n_estimators': 3000,
'max_depth': 15,
'min_child_samples': 142,
'num_leaves': 84,
'random_state': 42,
'device': 'gpu',
}
clf = LGBMRegressor(**best_params)
 clf.fit(x_train, y_train)
 predictions = clf.predict(x_test)
 score_rmse = math.sqrt(mean_squared_error(y_test, predictions))
print(Fore.GREEN + 'Base LGBM RMSE: {}'.format(score_rmse))
for name in models.keys() :
model = models[name]
model.fit(X_train, y_train)
 scores = cv_rmse(model,X_train, y_train)
 print(scores.mean )<predict_on_test>
sub_preds = clf.predict(test_df[feature_cols])
 sub_df['target'] = sub_preds
sub_df.to_csv('submission_lgbm_optuna.csv', index=False)
 sub_df.head()
sub_lgbm_optuna = sub_preds
sample_submission.iloc[:,1] =(np.expm1(model.predict(X_test)))
 sample_submission<save_to_csv>
lgbm_params = {
"random_state": 2021,
"metric": "rmse",
"n_jobs": -1,
"cat_feature": [x for x in range(len(categorical_columns)) ],
"early_stopping_round": 150,
"reg_alpha": 6.147694913504962,
"reg_lambda": 0.002457826062076097,
"colsample_bytree": 0.3,
"learning_rate": 0.01,
"max_depth": 30,
"num_leaves": 100,
"min_child_samples": 275,
"n_estimators": 30000,
"cat_smooth": 40.0,
"max_bin": 512,
"min_data_per_group": 100,
"bagging_freq": 1,
"bagging_fraction": 0.7,
"cat_l2": 12.0,
}
sample_submission.to_csv("submission.csv", index=False )<import_modules>
train_oof = np.zeros(( 300000,))
test_preds = 0
train_oof.shape
NUM_FOLDS = 5
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=42)
 for f,(train_ind, val_ind)in tqdm(enumerate(kf.split(x, y))):
tmp_train_df, tmp_val_df = x.iloc[train_ind][feature_cols], x.iloc[val_ind][feature_cols]
train_target, val_target = y[train_ind], y[val_ind]
model = LGBMRegressor(**lgbm_params)
 model.fit(
 tmp_train_df,
train_target,
eval_set=[(tmp_val_df, val_target)],
early_stopping_rounds = 100,
verbose=False 
)
 temp_oof = model.predict(tmp_val_df)
 temp_test = model.predict(test_df[feature_cols])
 train_oof[val_ind] = temp_oof
test_preds += temp_test/NUM_FOLDS
print(mean_squared_error(temp_oof, val_target, squared=False))
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
import lightgbm as lgb
import xgboost as xgb
import gc
import matplotlib.pyplot as plt<define_variables>
sub_df['target'] = test_preds
sub_df.to_csv('submission_lgbm_cv_optimized.csv', index=False)
 sub_df.head()
sub_lgbm_cv_optimized = test_preds
path = '.. /input/'
path_train = path + 'train.csv'
path_test = path + 'test.csv'
unbalance_fact = 10
train_cols = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed']
test_cols = ['ip', 'app', 'device', 'os', 'channel', 'click_time' ]
dtypes = {
'ip' : 'uint64',
'app' : 'uint16',
'device' : 'uint16',
'os' : 'uint16',
'channel' : 'uint16',
'is_attributed' : 'uint8',
'click_id' : 'uint64'
}<load_from_csv>
train_oof = np.zeros(( 300000,))
test_preds = 0
train_oof.shape
NUM_FOLDS = 5
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=42)
 for f,(train_ind, val_ind)in tqdm(enumerate(kf.split(xmod, ymod))):
tmp_train_df, tmp_val_df = xmod.iloc[train_ind][feature_cols_mod], xmod.iloc[val_ind][feature_cols_mod]
train_target, val_target = ymod[train_ind], ymod[val_ind]
model = LGBMRegressor(**lgbm_params)
 model.fit(
 tmp_train_df,
train_target,
eval_set=[(tmp_val_df, val_target)],
early_stopping_rounds = 100,
verbose=False 
)
 temp_oof = model.predict(tmp_val_df)
 temp_test = model.predict(mod_test_df[feature_cols_mod])
 train_oof[val_ind] = temp_oof
test_preds += temp_test/NUM_FOLDS
print(mean_squared_error(temp_oof, val_target, squared=False))
print("Loading Data")
 train = pd.read_csv(path_train, usecols=train_cols, dtype=dtypes, parse_dates=["click_time"])
 print("Loading is done" )<randomize_order>
sub_df['target'] = test_preds
sub_df.to_csv('submission_lgbm_mod_cv_optimized.csv', index=False)
 sub_df.head()
sub_lgbm_mod_cv_optimized = test_preds
print(len(train))
train_pos = train[train["is_attributed"] == 1]
num_attr = len(train_pos)
 print("Number of attributed: {:d}".format(num_attr))
train_neg = train[train["is_attributed"] == 0].sample(num_attr)
 train = pd.concat([train_pos, train_neg])
 gc.collect()<data_type_conversions>
h2o.init()
def prepare_dataframe(df):
print("Display frames:")
 display(df.head())
 display(df.dtypes)
 display(df.shape)
 print("Creating new time features:")
 df['hour'] = df["click_time"].dt.hour.astype('uint8')
 df['day'] = df["click_time"].dt.day.astype('uint8')
 df["minute"] = df["click_time"].dt.minute.astype('uint8')
 print("
n_chans = df[['ip','day','hour','channel']].groupby(by=['ip','day',
'hour'])[['channel']].count().reset_index().rename(columns={'channel': 'ip_day_hour'})
 df = df.merge(n_chans, on=['ip','day','hour'], how='left')
 del n_chans
gc.collect()
n_chans = df[['ip','app', 'channel']].groupby(by=['ip',
'app'])[['channel']].count().reset_index().rename(columns={'channel': 'ip_app_count'})
 df = df.merge(n_chans, on=['ip','app'], how='left')
 del n_chans
gc.collect()
n_chans = df[['ip','app', 'os', 'channel']].groupby(by=['ip', 'app',
'os'])[['channel']].count().reset_index().rename(columns={'channel': 'ip_app_os_count'})
 df = df.merge(n_chans, on=['ip','app', 'os'], how='left')
 del n_chans
gc.collect()
n_chans = df[['ip','channel']].groupby(by=['ip'])[['channel']].count().reset_index().rename(columns={'channel': 'count_by_ip'})
 print('Merging the channels data with the main data set...')
 df = df.merge(n_chans, on=['ip'], how='left')
 n_chans = df[['ip','hour','channel','os']].groupby(by=['ip','hour','channel'
])[['os']].count().reset_index().rename(columns={'os': 'ip_hour_channel'})
 df = df.merge(n_chans, on=['ip','hour','channel'], how='left')
 del n_chans
gc.collect()
n_chans = df[['ip','hour','channel','os']].groupby(by=['ip','hour','os'
])[['channel']].count().reset_index().rename(columns={'channel': 'ip_hour_os'})
 df = df.merge(n_chans, on=['ip','hour','os'], how='left')
 del n_chans
gc.collect()
n_chans = df[['ip','hour','channel','app']].groupby(by=['ip','hour','app'
])[['channel']].count().reset_index().rename(columns={'channel': 'ip_hour_app'})
 df = df.merge(n_chans, on=['ip','hour','app'], how='left')
 del n_chans
gc.collect()
n_chans = df[['ip','hour','channel','device']].groupby(by=['ip','hour','device'
])[['channel']].count().reset_index().rename(columns={'channel': 'ip_hour_device'})
 df = df.merge(n_chans, on=['ip','hour','device'], how='left')
 del n_chans
gc.collect()
print("Adjusting the data types of the new count features...")
 df.info()
df['ip_day_hour'] = df['ip_day_hour'].astype('uint8')
 df['ip_app_count'] = df['ip_app_count'].astype('uint8')
 df['ip_app_os_count'] = df['ip_app_os_count'].astype('uint8')
 df['count_by_ip'] = df['count_by_ip'].astype('uint16')
 df['ip_hour_channel'] = df['ip_hour_channel'].astype('uint16')
 df['ip_hour_os'] = df['ip_hour_os'].astype('uint16')
 df['ip_hour_app'] = df['ip_hour_app'].astype('uint16')
 df['ip_hour_device'] = df['ip_hour_device'].astype('uint16')
 return df 
<create_dataframe>
train_hf = h2o.H2OFrame(train_df)
 test_hf = h2o.H2OFrame(test_df)
 predictors = list(feature_cols)
 response = 'target'
train, valid = train_hf.split_frame(ratios=[.8], seed=1234 )
train = prepare_dataframe(train )<split>
aml = H2OAutoML(
 max_models=20,
max_runtime_secs=200,
exclude_algos = ["DeepLearning", "DRF"],
seed=42,
)
train_df, test_df = train_test_split(train, test_size=0.2, stratify=train["is_attributed"])
 train_df, valid_df = train_test_split(train_df, test_size=0.2, stratify=train_df["is_attributed"] )<drop_column>
aml.train(x=predictors,
y=response,
training_frame=train,
validation_frame=valid
)
print(len(train_df))
print(len(valid_df))
print(len(test_df))
del train
gc.collect()<set_options>
print('The model performance in RMSE: {}'.format(aml.leader.rmse(valid=True)))
 print('The model performance in MAE: {}'.format(aml.leader.mae(valid=True)) )
target = 'is_attributed'
predictors = ['ip', 'device', 'app', 'os', 'channel', 'hour', "minute",
'ip_day_hour','count_by_ip','ip_app_count', 'ip_app_os_count',
"ip_hour_channel", "ip_hour_os", "ip_hour_app","ip_hour_device"]
gc.collect()<compute_train_metric>
preds = aml.predict(test_hf ).as_data_frame()
preds.head()
print("Testing against test split")
 dtest = xgb.DMatrix(test_df[predictors])
 test_res = model.predict(dtest, ntree_limit=model.best_ntree_limit)
 test_score = roc_auc_score(test_df[target].values, test_res)
 print("Test score = {:f}".format(test_score))<save_to_csv>
sub_df['target'] = preds['predict']
sub_df.to_csv('submission_h2o.csv', index=False)
 sub_df.head()
sub_automl = preds['predict']
print("Loading test data for generating submission.")
 test = pd.read_csv(path_test, dtype=dtypes, parse_dates=["click_time"])
 test = prepare_dataframe(test)
 print("Preparing data for submission...")
 dtest = xgb.DMatrix(test[predictors])
 test['is_attributed'] = model.predict(dtest, ntree_limit=model.best_ntree_limit)
 print("Writing the submission data into a csv file...")
 test[["click_id","is_attributed"]].to_csv("submission_xgb_v2.csv",index=False)
 print("All Done..." )<define_variables>
sub1 = 0.3*sub_xgb_cv_optimized + 0.3*sub_lgbm_cv_optimized + 0.4*sub_lgbm_optuna
sub2 = 0.4*sub_xgb_cv_optimized + 0.4*sub_lgbm_cv_optimized + 0.2*sub_lgbm_optuna
sub3 = 0.3*sub_xgb_cv_optimized + 0.4*sub_lgbm_cv_optimized + 0.3*sub_lgbm_optuna
sub4 = 0.3*sub_xgb_cv_optimized + 0.3*sub_lgbm_cv_optimized + 0.3*sub_lgbm_optuna + 0.1*sub_automl
sub5 = 0.3*sub_xgb_cv_optimized + 0.3*sub_lgbm_cv_optimized + 0.2*sub_lgbm_optuna + 0.2*sub_automl
sub6 = 0.5*sub_lgbm_mod_cv_optimized + 0.5*sub_xgb_mod_cv_optimized
sub7 = 0.7*sub_lgbm_mod_cv_optimized + 0.3*sub_xgb_mod_cv_optimized
sub8 = 0.2*sub_xgb_cv_optimized + 0.2*sub_lgbm_cv_optimized + 0.3*sub_lgbm_mod_cv_optimized + 0.3*sub_xgb_mod_cv_optimized
sub9 = 0.1*sub_xgb_cv_optimized + 0.1*sub_lgbm_cv_optimized + 0.4*sub_lgbm_mod_cv_optimized + 0.4*sub_xgb_mod_cv_optimized
sub10 = 0.1*sub_xgb_cv_optimized + 0.2*sub_lgbm_cv_optimized + 0.4*sub_lgbm_mod_cv_optimized + 0.3*sub_xgb_mod_cv_optimized
sub11 = 0.5*sub_xgb_cv_optimized + 0.5*sub_lgbm_cv_optimized
sub12 = 0.6*sub_xgb_cv_optimized + 0.4*sub_lgbm_cv_optimized
sub13 = 0.4*sub_xgb_cv_optimized + 0.6*sub_lgbm_cv_optimized
sub14 = 0.2*sub_xgb_cv_optimized + 0.7*sub_lgbm_cv_optimized + 0.1*sub_lgbm_mod_cv_optimized
sub14 = 0.2*sub_xgb_cv_optimized + 0.7*sub_lgbm_cv_optimized + 0.1*sub_xgb_mod_cv_optimized
debug = True
validaten_flg = False
EXP_ID = 'exp004'
CV_NUM = 'cv1'
def unpickle(filename):
with open(filename, mode='rb')as fo:
p = pickle.load(fo)
 return p
class CFG:
INPUT_DIR = '.. /input/riiid-test-answer-prediction'
OUT_DIR = '.. /input/riiid-trained-models-exp004'
DATA_DIR = '.. /input/riiid-cv-datasets'
TRAIN_PATH = os.path.join(INPUT_DIR, "train.csv")
 TEST_PATH = os.path.join(INPUT_DIR, "example_test.csv")
 LECTURE_PATH = os.path.join(INPUT_DIR, "lectures.csv")
 QUESTION_PATH = os.path.join(INPUT_DIR, "questions.csv")
 SUBMISSION_PATH = os.path.join(INPUT_DIR, "example_sample_submission.csv")
 SEED = 6718
train_pickle = '.. /input/riiid-cross-validation-files/cv1_train.pickle'
valid_pickle = '.. /input/riiid-cross-validation-files/cv1_valid.pickle'
question_file = '.. /input/riiid-test-answer-prediction/questions.csv'
debug = True
validaten_flg = False<load_from_csv>
sub_df['target'] = sub1
sub_df.to_csv('submission_01.csv', index=False)
 sub_df['target'] = sub2
sub_df.to_csv('submission_02.csv', index=False)
 sub_df['target'] = sub3
sub_df.to_csv('submission_03.csv', index=False)
 sub_df['target'] = sub4
sub_df.to_csv('submission_04.csv', index=False)
 sub_df['target'] = sub5
sub_df.to_csv('submission_05.csv', index=False)
 sub_df['target'] = sub6
sub_df.to_csv('submission_06.csv', index=False)
 sub_df['target'] = sub7
sub_df.to_csv('submission_07.csv', index=False)
 sub_df['target'] = sub8
sub_df.to_csv('submission_08.csv', index=False)
 sub_df['target'] = sub9
sub_df.to_csv('submission_09.csv', index=False)
 sub_df['target'] = sub10
sub_df.to_csv('submission_10.csv', index=False)
 sub_df['target'] = sub11
sub_df.to_csv('submission_11.csv', index=False)
 sub_df['target'] = sub12
sub_df.to_csv('submission_12.csv', index=False)
 sub_df['target'] = sub13
sub_df.to_csv('submission_13.csv', index=False)
 sub_df['target'] = sub14
sub_df.to_csv('submission_14.csv', index=False )