Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<define_variables>
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler, LabelEncoder
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.neighbors import KNeighborsClassifier
from xgboost import XGBClassifier
from sklearn.model_selection import StratifiedKFold, GridSearchCV, learning_curve, cross_val_score
MODEL_VERSION = 'v15_param_tuning' 
<define_variables>
train = pd.read_csv(".. /input/titanic/train.csv")
 test = pd.read_csv(".. /input/titanic/test.csv")
 testid = test['PassengerId']
train_len = len(train)
 y = train['Survived']
X = pd.concat([train, test] )
TEST_RUN = False
pd.set_option('display.max_columns', 50)
 KAGGLE_DATA_FOLDER = '/kaggle/input/m5-forecasting-accuracy'
BACKWARD_LAG = 60
END_DAY = 1913
BEGIN_DATE = '2013-08-01'
BEGIN_DAY = str(( datetime.strptime(BEGIN_DATE, '%Y-%m-%d')- datetime.strptime('2011-01-29', '%Y-%m-%d')).days)
 TRAIN_SPLIT = '2016-03-27'
EVAL_SPLIT = '2016-04-24'
print(datetime.strptime(EVAL_SPLIT, '%Y-%m-%d'))
TASK_TYPE='CPU'<define_variables>
X['Title'] = X['Name']
for name_string in X['Name']:
X['Title'] = X['Name'].str.extract('([A-Za-z]+)\.', expand=True)
 mapping = {'Mlle': 'Miss', 'Major': 'Mr', 'Col': 'Mr', 'Sir': 'Mr', 'Don': 'Mr', 'Mme': 'Miss',
'Jonkheer': 'Mr', 'Lady': 'Mrs', 'Capt': 'Mr', 'Countess': 'Mrs', 'Ms': 'Miss', 'Dona': 'Mrs'}
X.replace({'Title': mapping}, inplace=True)
 titles = ['Dr', 'Master', 'Miss', 'Mr', 'Mrs', 'Rev']
for title in titles:
age_to_impute = X.groupby('Title')['Age'].median() [titles.index(title)]
X.loc[(X['Age'].isnull())&(X['Title'] == title), 'Age'] = age_to_impute
gr = sns.countplot(x="Title",data=X)
 gr.set_xticklabels(gr.get_xticklabels() , rotation=75)
 plt.show(gr)
 sns.catplot(x="Title",y="Survived",data=X.iloc[:train_len],kind="bar")
 plt.show(sns)
 X.drop('Title', axis = 1, inplace = True )
CALENDAR_DTYPES = {
'date': 'str',
'wm_yr_wk': 'int16',
'weekday': 'category',
'wday': 'int16',
'month': 'int16',
'year': 'int16',
'd': 'object',
'event_name_1': 'category',
'event_type_1': 'category',
'event_name_2': 'category',
'event_type_2': 'category',
'snap_CA': 'int16',
'snap_TX': 'int16',
'snap_WI': 'int16'
}
PARSE_DATES = ['date']
SALES_PRICES_DTYPES = {
'store_id': 'category',
'item_id': 'category',
'wm_yr_wk': 'int16',
'sell_price': 'float32'
}<load_pretrained>
X['Family_Size'] = X['Parch'] + X['SibSp']
def load_data(train=True):

 print("Loading train and validation data")
 numcols = [f"d_{day}" for day in range(int(BEGIN_DAY),END_DAY+1)]
catcols = ['id', 'item_id', 'dept_id','store_id', 'cat_id', 'state_id']
dtype = {numcol:"float32" for numcol in numcols}
dtype.update({col: "category" for col in catcols if col != "id"})
 sales_train_validation = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sales_train_validation.csv'),
usecols=catcols+numcols, dtype=dtype)
 for col in catcols:
if col != "id":
sales_train_validation[col] = sales_train_validation[col].cat.codes.astype("int16")
 sales_train_validation[col] -= sales_train_validation[col].min()
if not train:
for day in range(END_DAY+1, END_DAY+28+1):
sales_train_validation[f"d_{day}"] = 0
value_vars = [column for column in sales_train_validation.columns
if(column.startswith('d_')and int(column.replace('d_', '')) >= END_DAY - BACKWARD_LAG)]
else:
value_vars = [col for col in sales_train_validation.columns
if(col.startswith('d_')and(int(col.replace('d_', '')) >= int(BEGIN_DAY)))]
print("Shape:", sales_train_validation.shape)
 print("Memory usage(Mb)before melting:", sales_train_validation.memory_usage().sum() / 1024**2)
 sales_train_validation = pd.melt(
 sales_train_validation,
id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
value_vars=value_vars,
var_name = 'day',
value_name = 'sales')
 print("Completed melting, new shape:", sales_train_validation.shape)
 print("Colums after melting:", sales_train_validation.columns)
 print("Memory usage(Mb)after melting:", sales_train_validation.memory_usage().sum() / 1024**2)
 columns = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']
print("Loading calendar")
 calendar = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'calendar.csv'), dtype=CALENDAR_DTYPES, parse_dates=['date'])
 print("Calendar columns: ", calendar.columns)
 print("Memory usage(Mb)calendar: ", calendar.memory_usage().sum() / 1024**2)
 calendar.rename(columns={'d':'day'}, inplace=True)
 for col, col_dtype in CALENDAR_DTYPES.items() :
if col_dtype == "category":
calendar[col] = calendar[col].cat.codes.astype("int16")
 calendar[col] -= calendar[col].min()
sales_train_validation = sales_train_validation.merge(calendar, on="day", copy=False)
 del calendar; gc.collect()
print("Merged calendar(in place)")
 print("Memory usage(Mb)after merging calendar:", sales_train_validation.memory_usage().sum() / 1024**2)
 print("Colums after merge:", sales_train_validation.columns)
 sell_prices = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sell_prices.csv'), dtype=SALES_PRICES_DTYPES)
 for col, col_dtype in SALES_PRICES_DTYPES.items() :
if col_dtype == "category":
sell_prices[col] = sell_prices[col].cat.codes.astype("int16")
 sell_prices[col] -= sell_prices[col].min()
print("Memory usage(Mb)sell prices:", sell_prices.memory_usage().sum() / 1024**2)
 columns = ['item_id', 'store_id', 'sell_price']
for feature in columns:
if feature == 'sell_price':
sell_prices[feature].fillna(0, inplace=True)
 sales_train_validation = sales_train_validation.merge(sell_prices, on=["store_id","item_id","wm_yr_wk"], copy=False)
 del sell_prices; gc.collect()
print("Merged sales prices(in place)")
 print("Memory usage(Mb)after merging sales:", sales_train_validation.memory_usage().sum() / 1024**2)
 print("Colums after merge:", sales_train_validation.columns)
 return sales_train_validation<categorify>
X['Last_Name'] = X['Name'].apply(lambda x: str.split(x, ",")[0])
 X['Fare'].fillna(X['Fare'].mean() , inplace=True)
 DEFAULT_SURVIVAL_VALUE = 0.5
X['Family_Survival'] = DEFAULT_SURVIVAL_VALUE
for grp, grp_df in X[['Survived','Name', 'Last_Name', 'Fare', 'Ticket', 'PassengerId',
'SibSp', 'Parch', 'Age', 'Cabin']].groupby(['Last_Name', 'Fare']):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
X.loc[X['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
X.loc[X['PassengerId'] == passID, 'Family_Survival'] = 0
for _, grp_df in X.groupby('Ticket'):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
if(row['Family_Survival'] == 0)|(row['Family_Survival']== 0.5):
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
X.loc[X['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
X.loc[X['PassengerId'] == passID, 'Family_Survival'] = 0
sns.catplot(x="Family_Size",y="Survived",data = X.iloc[:train_len],kind="bar" )
def make_lag_features(strain):

 print('in dataframe:', strain.shape)
 print("headers:", strain.columns)
 lags = [7, 28]
lag_cols = ['lag_{}'.format(lag)for lag in lags ]
for lag, lag_col in zip(lags, lag_cols):
strain[lag_col] = strain[['id', 'sales']].groupby('id')['sales'].shift(lag)
 print('lag sales done')
 windows= [7, 28]
for window in windows:
for lag, lag_col in zip(lags, lag_cols):
window_col = f'rmean_{lag_col}_{window}'
strain[window_col] = strain[['id', lag_col]].groupby('id')[lag_col].transform(
 lambda x: x.rolling(window ).mean() 
)
 print(f'Rolling mean sales for done for window {window}')
 def make_date_features(dt):
date_features = {
'week': 'weekofyear',
'quarter': 'quarter',
'mday': 'day',
"wday": "weekday"
}
for date_feat_name, date_feat_func in date_features.items() :
if not date_feat_name in dt.columns:
dt[date_feat_name] = getattr(dt['date'].dt, date_feat_func ).astype('int16')
 print('date features done')
 dt['day'] = dt['day'].apply(lambda x: int(x.replace('d_', '')))
 print('out dataframe:', dt.shape )<load_pretrained>
X['Fare'].fillna(X['Fare'].median() , inplace = True)
 X['FareBin'] = pd.qcut(X['Fare'], 5)
 label = LabelEncoder()
X['FareBin_Code'] = label.fit_transform(X['FareBin'])
 X.drop(['Fare'], 1, inplace=True )
%%time
sales_train_validation = load_data()<data_type_conversions>
X['AgeBin'] = pd.qcut(X['Age'], 4)
 label = LabelEncoder()
X['AgeBin_Code'] = label.fit_transform(X['AgeBin'])
 X.drop(['Age'], 1, inplace=True )
sales_train_validation["sale"] =(( sales_train_validation['sell_price'] * 100 % 10)< 6 ).astype('int8' )<feature_engineering>
X['Sex'].replace(['male','female'],[0,1],inplace=True)
 X.drop(['Name', 'PassengerId', 'SibSp', 'Parch', 'Ticket', 'Cabin',
'Embarked', 'Last_Name', 'FareBin', 'AgeBin', 'Survived'], axis = 1, inplace = True )
%%time
make_lag_features(sales_train_validation)
 make_date_features(sales_train_validation )<define_variables>
X_train = X[:train_len]
X_test = X[train_len:]
y_train = y
total = len(sales_train_validation['event_type_2'])
 zero = len(sales_train_validation.loc[sales_train_validation['event_type_2'] == 0])
 one = len(sales_train_validation.loc[sales_train_validation['event_type_2'] == 1])
 two = len(sales_train_validation.loc[sales_train_validation['event_type_2'] == 2])
 print('0: '+ str(zero))
print('1: ' + str(one))
print('2: ' + str(two))
print('-'*7 + '
Total: ' + str(total))
del total; del zero; del one; del two<count_missing_values>
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test )
sales_train_validation.isnull().sum()<categorify>
kfold = StratifiedKFold(n_splits=8 )
before = len(sales_train_validation)
 sales_train_validation.dropna(inplace = True)
 after = len(sales_train_validation)
 print(f"Reduced {(before-after)/before}%")
<split>
RFC = RandomForestClassifier()
rf_param_grid = {"max_depth": [None],
"max_features": [3,"sqrt", "log2"],
"min_samples_split": [2, 4],
"min_samples_leaf": [5, 7],
"bootstrap": [False, True],
"n_estimators" :[200, 500],
"criterion": ["gini", "entropy"]}
rf_param_grid_best = {"max_depth": [None],
"max_features": [3],
"min_samples_split": [4],
"min_samples_leaf": [5],
"bootstrap": [False],
"n_estimators" :[200],
"criterion": ["gini"]}
gs_rf = GridSearchCV(RFC, param_grid = rf_param_grid_best, cv=kfold, scoring="roc_auc", n_jobs= 4, verbose = 1)
 gs_rf.fit(X_train, y_train)
 RFC.fit(X_train, y_train)
 rf_best = gs_rf.best_estimator_
print(f'RandomForest GridSearch best params: {gs_rf.best_params_}')
 print(f'RandomForest GridSearch best score: {gs_rf.best_score_}' )
print("Splitting data into train, validation, evaluation set")
<features_selection>
KNN = KNeighborsClassifier()
knn_param_grid = {'algorithm': ['auto'],
'weights': ['uniform', 'distance'],
'leaf_size': [20, 25, 30],
'n_neighbors': [12, 14, 16]}
gs_knn = GridSearchCV(KNN, param_grid = knn_param_grid, cv=kfold, scoring = "roc_auc", n_jobs= 4, verbose = 1)
 gs_knn.fit(X_train, y_train)
 KNN.fit(X_train, y_train)
 knn_best = gs_knn.best_estimator_
print(f'KNN GridSearch best params: {gs_knn.best_params_}')
 print(f'KNN GridSearch best score: {gs_knn.best_score_}' )
%%time
np.random.seed(777)
 validation_idx = np.random.choice(sales_train_validation.index.values, 2_000_000, replace = False)
 train_idx = np.setdiff1d(sales_train_validation.index.values, validation_idx)
 train = sales_train_validation.loc[train_idx]
validation = sales_train_validation.loc[validation_idx]
evaluation = sales_train_validation[(sales_train_validation['date'] > EVAL_SPLIT)]<train_model>
GB = GradientBoostingClassifier()
gb_param_grid = {'loss' : ["deviance"],
'n_estimators' : [1000],
'learning_rate': [0.02, 0.05],
'min_samples_split': [15, 20, 25],
'max_depth': [4, 6],
'min_samples_leaf': [50, 60],
'max_features': ["sqrt"]
}
gb_param_grid_best = {'loss' : ["deviance"],
'n_estimators' : [1000],
'learning_rate': [0.02],
'min_samples_split': [25],
'max_depth': [4],
'min_samples_leaf': [60],
'max_features': ["sqrt"]
}
gs_gb = GridSearchCV(GB, param_grid = gb_param_grid_best, cv=kfold, scoring="roc_auc", n_jobs= 4, verbose = 1)
 gs_gb.fit(X_train,y_train)
 GB.fit(X_train, y_train)
 gb_best = gs_gb.best_estimator_
print(f'GradienBoost GridSearch best params: {gs_gb.best_params_}')
 print(f'GradienBoost GridSearch best score: {gs_gb.best_score_}' )
print("Train:", train.shape)
 print("Validation:", validation.shape)
 print("Evaluation:", evaluation.shape )<define_variables>
XGB = XGBClassifier()
xgb_param_grid = {'learning_rate':[0.05, 0.1],
'reg_lambda':[0.3, 0.5],
'gamma': [0.8, 1],
'subsample': [0.8, 1],
'max_depth': [2, 3],
'n_estimators': [200, 300]
}
xgb_param_grid_best = {'learning_rate':[0.1],
'reg_lambda':[0.3],
'gamma': [1],
'subsample': [0.8],
'max_depth': [2],
'n_estimators': [300]
}
gs_xgb = GridSearchCV(XGB, param_grid = xgb_param_grid_best, cv=kfold, scoring="roc_auc", n_jobs= 4, verbose = 1)
 gs_xgb.fit(X_train,y_train)
 XGB.fit(X_train, y_train)
 xgb_best = gs_xgb.best_estimator_
print(f'XGB GridSearch best params: {gs_xgb.best_params_}')
 print(f'XGB GridSearch best score: {gs_xgb.best_score_}' )
categorical_features = [
'item_id', 'dept_id', 'store_id', 'cat_id', 'state_id',
'event_name_1', 'event_type_1', 'event_name_2', 'event_type_2'
]<create_dataframe>
knn1 = KNeighborsClassifier(algorithm='auto', leaf_size=26, metric='minkowski',
metric_params=None, n_jobs=1, n_neighbors=6, p=2,
weights='uniform')
 knn1.fit(X_train, y_train )
train_pool = lgb.Dataset(
 data=train[train_columns],
label=train["sales"],
categorical_feature=categorical_features,
free_raw_data=False)
 del train; gc.collect()<create_dataframe>
y_pred = knn1.predict(X_test)
 test_Survived = pd.Series(y_pred, name="Survived")
 results = pd.concat([testid,test_Survived],axis=1)
 results.to_csv("submit.csv",index=False )