Kaggle competition: Titanic - Machine Learning from Disaster
user_prior_question_elapsed_time = train_df[['user_id','prior_question_elapsed_time']].groupby(['user_id'] ).tail(1)
 user_prior_question_elapsed_time.columns = ['user_id', 'prior_question_elapsed_time']<feature_engineering>
warnings.filterwarnings('ignore' )
train_df['delta_prior_question_elapsed_time'] = train_df.groupby('user_id')['prior_question_elapsed_time'].shift()
train_df['delta_prior_question_elapsed_time']=train_df['prior_question_elapsed_time']-train_df['delta_prior_question_elapsed_time']
train_df['delta_prior_question_elapsed_time'].fillna(0, inplace=True)
<data_type_conversions>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 print(train_df.shape)
 print(test_df.shape)
 train_label = train_df.Survived
test_Ids = test_df.PassengerId
train_df.delta_prior_question_elapsed_time=train_df.delta_prior_question_elapsed_time.astype('int32' )<data_type_conversions>
def checking_missing(data):
for col in data.columns:
print(f' Count of NAs in {col} : {np.sum(data[col].isnull())} ')
 print("NAs in train dataset: ")
 checking_missing(train_df)
 print("NAs in test dataset: ")
 checking_missing(test_df )
train_df['timestamp']=train_df['timestamp']/(1000*3600)
 train_df.timestamp=train_df.timestamp.astype('int16')
<groupby>
print(total_df.SibSp.value_counts())
 print('-'*10)
 print(total_df.Parch.value_counts() )
train_df['lag'] = train_df.groupby('user_id')[target].shift()<data_type_conversions>
total_df['FamSize'] = total_df.SibSp + total_df.Parch + 1
print(total_df.FamSize.value_counts())
 total_df.FamSize.replace(to_replace = [1], value = 'single', inplace = True)
 total_df.FamSize.replace(to_replace = [2,3,4,5], value = 'median', inplace = True)
 total_df.FamSize.replace(to_replace = [6,7,8,11], value = 'large', inplace = True)
 total_df = total_df.drop(['SibSp'], axis = 1 )
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df['user_correct_cumsum'] = cum['cumsum']
train_df['user_correct_cumcount'] = cum['cumcount']
train_df.drop(columns=['lag'], inplace=True)
 train_df['user_correct_cumsum'].fillna(0, inplace=True)
 train_df.user_correctness=train_df.user_correctness.astype('float16')
 train_df.user_correct_cumcount=train_df.user_correct_cumcount.astype('int16')
 train_df.user_correct_cumsum=train_df.user_correct_cumsum.astype('int16' )<data_type_conversions>
total_df.Fare = total_df.Fare.fillna(total_df.Fare.median())
 cuttingArr = np.array([0,50,100,total_df.Fare.max() ])
 total_df['Fare_Group'] = pd.cut(total_df.Fare,cuttingArr,include_lowest = True)
 print(total_df.Fare_Group.value_counts())
 Fare_intervals = total_df.Fare_Group.unique()
total_df.Fare_Group.replace(to_replace = Fare_intervals[0],value = 0, inplace = True)
 total_df.Fare_Group.replace(to_replace = Fare_intervals[1],value = 1, inplace = True)
 total_df.Fare_Group.replace(to_replace = Fare_intervals[2],value = 2, inplace = True)
 print(total_df.Fare_Group.value_counts() )
train_df.prior_question_had_explanation=train_df.prior_question_had_explanation.astype('int8')
 train_df['lag'] = train_df.groupby('user_id')['prior_question_had_explanation'].shift()<data_type_conversions>
total_df.Cabin = total_df.Cabin.fillna('X')
 total_df.Cabin = total_df.Cabin.apply(lambda x: x[0])
 total_df.Cabin = total_df.Cabin == 'X'
total_df.Cabin = total_df.Cabin.apply(lambda x: 0 if x == False else 1)
 print(total_df.Cabin.value_counts() )
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 train_df['explanation_mean'] = cum['cumsum'] / cum['cumcount']
train_df['explanation_cumsum'] = cum['cumsum']
train_df.drop(columns=['lag'], inplace=True)
 train_df['explanation_mean'].fillna(0, inplace=True)
 train_df['explanation_cumsum'].fillna(0, inplace=True)
 train_df.explanation_mean=train_df.explanation_mean.astype('float16')
 train_df.explanation_cumsum=train_df.explanation_cumsum.astype('int16' )<set_options>
total_df.Embarked = total_df.Embarked.fillna(total_df.Embarked.mode() [0])
 total_df.Embarked.value_counts()
del cum
gc.collect()<data_type_conversions>
total_df = total_df.drop(['Ticket'], axis = 1 )
train_df["attempt_no"] = 1
train_df.attempt_no=train_df.attempt_no.astype('int8')
 train_df["attempt_no"] = train_df[["user_id","content_id",'attempt_no']].groupby(["user_id","content_id"])["attempt_no"].cumsum()<data_type_conversions>
temp = copy.deepcopy(total_df[total_df.columns[total_df.columns != 'Age']])
 temp.Title.replace(to_replace = ['Mr'] , value = 0 , inplace = True)
 temp.Title.replace(to_replace = ['Mrs'] , value = 1 , inplace = True)
 temp.Title.replace(to_replace = ['Miss'] , value = 2, inplace = True)
 temp.Title.replace(to_replace = ['respect'] , value = 3 , inplace = True)
 temp = temp.agg(LabelEncoder().fit_transform)
 temp['Age'] = total_df['Age']
Corr = temp.corr() ["Age"].apply(lambda x : abs(x)).sort_values(ascending = False)
 print(Corr[:10])
 total_df['Missing_Age'] = total_df.Age.isnull()
explanation_agg = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['sum', 'count'])
 explanation_agg=explanation_agg.astype('int16')
<groupby>
total_df.Age = temp.groupby(['Pclass','Title'])['Age'].transform(lambda v: v.fillna(v.median()))
checking_missing(total_df )
user_agg = train_df.groupby('user_id')[target].agg(['sum', 'count'])
 content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count','var'])
 task_container_agg = train_df.groupby('task_container_id')[target].agg(['sum', 'count','var'])
<data_type_conversions>
LE_columns = ['FamSize','Fare_Group','Sex','Age_Group','Missing_Age']
OHE_columns = ['Embarked','Title']
Other_columns = total_df.columns[~total_df.columns.isin(LE_columns)& ~total_df.columns.isin(OHE_columns)]
LE_data = total_df[LE_columns]
OHE_data = total_df[OHE_columns]
Other_data = total_df[Other_columns]
for col in LE_data:
LE_data[col] = LabelEncoder().fit_transform(LE_data[col])
 OHE_data = pd.get_dummies(OHE_data)
 total_df = pd.concat([Other_data,LE_data,OHE_data], axis = 1)
 total_df.head(5 )
user_agg=user_agg.astype('int16')
 content_agg=content_agg.astype('float32')
 task_container_agg=task_container_agg.astype('float32' )<categorify>
train_df = train_df.drop(['PassengerId','Survived'],axis = 1)
 test_df = test_df.drop(['PassengerId','Survived'], axis = 1)
 ntrain = len(train_df)
 ntest = len(test_df )
attempt_no_agg=train_df.groupby(["user_id","content_id"])["attempt_no"].agg(['sum'])
 attempt_no_agg=attempt_no_agg.astype('int8')
<data_type_conversions>
xgbParams = {'max_depth':np.arange(3,11,2),'learning_rate':np.arange(0.01,0.2,0.05)
 ,'n_estimator':np.arange(100,1100,50),'gamma':np.arange(0.01,0.3,0.05)}
ABParams = {'n_estimators':np.arange(100,1100,50),'learning_rate':np.arange(0.01,0.2,0.05)}
RFParams = {'n_estimators':np.arange(100,1100,50),'max_depth':np.arange(3,11,2),
'min_samples_split':np.arange(2,20,2),'min_samples_leaf':np.arange(1,6,2)}
ETParams = {'n_estimators':np.arange(100,1100,50),'max_depth':np.arange(3,11,2),
'min_samples_leaf':np.arange(1,6,2),'verbose':[0]}
gbcParams = {'n_estimators' : np.arange(100,1100,50), 'learning_rate' : np.arange(0.01,0.2,0.05)
 , 'min_samples_split' : np.arange(2,20,2), 'min_samples_leaf' : np.arange(1,6,2),
'max_depth' : np.arange(3,11,2)}
LGBMCParams = {'n_estimators' : np.arange(100,1000,50), 'learning_rate' : np.arange(0.01,0.2,0.05),
'num_leaves' : np.arange(2,12,2)}
def tuneParams(classifier, params, train_x, train_y, cv = 5):
md = GridSearchCV(classifier, params, cv = cv, scoring = 'accuracy', n_jobs = -1)
 md.fit(train_x,train_y)
 return md.best_params_,np.round(md.best_score_*100,2)
 def tuneParamsRandom(classifier, params, train_x, train_y, cv=5):
rs = RandomizedSearchCV(classifier, params, n_iter = 30,scoring = 'accuracy', n_jobs = -1, verbose = 0)
 rs.fit(train_x,train_y)
 return rs.best_params_, abs(rs.best_score_ )
train_df['content_count'] = train_df['content_id'].map(content_agg['count'] ).astype('int32')
 train_df['content_sum'] = train_df['content_id'].map(content_agg['sum'] ).astype('int32')
 train_df['content_correctness'] = train_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 train_df.content_correctness=train_df.content_correctness.astype('float16')
 train_df['task_container_sum'] = train_df['task_container_id'].map(task_container_agg['sum'] ).astype('int32')
 train_df['task_container_std'] = train_df['task_container_id'].map(task_container_agg['var'] ).astype('float16')
 train_df['task_container_correctness'] = train_df['task_container_id'].map(task_container_agg['sum'] / task_container_agg['count'])
 train_df.task_container_correctness=train_df.task_container_correctness.astype('float16' )<load_from_csv>
xgb_best_Params,xgb_best_score = tuneParamsRandom(XGBClassifier() ,xgbParams,train_df,train_label)
 print("XGB:",xgb_best_Params,xgb_best_score)
 AB_best_Params,AB_best_score = tuneParamsRandom(AdaBoostClassifier() ,ABParams,train_df,train_label)
 print("AdaBoost:",AB_best_Params,AB_best_score)
 RF_best_Params,RF_best_score = tuneParamsRandom(RandomForestClassifier() ,RFParams,train_df,train_label)
 print("RandomForest:",RF_best_Params,RF_best_score)
 ET_best_Params,ET_best_score = tuneParamsRandom(ExtraTreesClassifier() ,ETParams,train_df,train_label)
 print("ExtraTrees:",ET_best_Params,ET_best_score)
 GBC_best_Params,GBC_best_score = tuneParamsRandom(GradientBoostingClassifier() ,gbcParams,train_df,train_label)
 print("GradientBoostingClassifier:",GBC_best_Params,GBC_best_score)
 LGBMC_best_Params, LGBMC_best_score = tuneParamsRandom(LGBMClassifier() ,LGBMCParams,train_df,train_label)
 print("LGBMClassifier:",LGBMC_best_Params,LGBMC_best_score )
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 1,3,4],
dtype={'question_id': 'int16','bundle_id': 'int16', 'part': 'int8','tags': 'str'} 
)
 questions_df['part_bundle_id']=questions_df['part']*100000+questions_df['bundle_id']
questions_df.part_bundle_id=questions_df.part_bundle_id.astype('int32')
 tag = questions_df["tags"].str.split(" ", n = 10, expand = True)
 tag.columns = ['tags1','tags2','tags3','tags4','tags5','tags6']
tag.fillna(0, inplace=True)
 tag = tag.astype('int16')
 questions_df = pd.concat([questions_df,tag],axis=1 ).drop(['tags'],axis=1 )<rename_columns>
kf = KFold(n_splits = 5, random_state = 2019)
 def fold_training(model, train_x, train_y, test_x):
oof_train = np.zeros(( ntrain))
oof_test = np.zeros(( ntest))
oof_kf_test = np.zeros(( 5,ntest))
for i,(train_idx,test_idx)in enumerate(kf.split(train_x)) :
kf_train_x = train_x.iloc[train_idx]
kf_train_y = train_y.iloc[train_idx]
kf_test_x = train_x.iloc[test_idx]
model.fit(kf_train_x,kf_train_y)
 oof_train[test_idx] = model.predict(kf_test_x)
 oof_kf_test[i,:] = model.predict(test_x)
 oof_test = oof_kf_test.mean(axis=0)
 return oof_train.reshape(-1,1), oof_test.reshape(-1,1 )
questions_df.rename(columns={'question_id':'content_id'}, inplace=True )<data_type_conversions>
xgb_oof_train,xgb_oof_test = fold_training(XGBClassifier(**xgb_best_Params),train_df,train_label,test_df)
 AB_oof_train,AB_oof_test = fold_training(AdaBoostClassifier(**AB_best_Params),train_df,train_label,test_df)
 RF_oof_train,RF_oof_test = fold_training(RandomForestClassifier(**RF_best_Params),train_df,train_label,test_df)
 ET_oof_train,ET_oof_test = fold_training(ExtraTreesClassifier(**ET_best_Params),train_df,train_label,test_df)
 gbc_oof_train,gbc_oof_test = fold_training(GradientBoostingClassifier(**GBC_best_Params),train_df,train_label,test_df)
 LGBMC_oof_train,LGBMC_oof_test = fold_training(LGBMClassifier(**LGBMC_best_Params),train_df,train_label,test_df )
questions_df['content_correctness'] = questions_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 questions_df.content_correctness=questions_df.content_correctness.astype('float16')
 questions_df['content_correctness_std'] = questions_df['content_id'].map(content_agg['var'])
 questions_df.content_correctness_std=questions_df.content_correctness_std.astype('float16' )<categorify>
final_train_df = pd.DataFrame(np.concatenate(( xgb_oof_train,AB_oof_train,
RF_oof_train,ET_oof_train,gbc_oof_train,LGBMC_oof_train),axis=1))
final_test_df = pd.DataFrame(np.concatenate(( xgb_oof_test,AB_oof_test,
RF_oof_test,ET_oof_test,gbc_oof_test,LGBMC_oof_test),axis=1))
LRParams = {'penalty':['l1','l2'],'C':[0.01,0.05,0.1,0.2]}
LR_best_Params, LR_best_score = tuneParams(LogisticRegression() ,LRParams,final_train_df, train_label)
 print("LR:",LR_best_Params,LR_best_score )
part_agg = questions_df.groupby('part')['content_correctness'].agg(['mean', 'var'])
 questions_df['part_correctness_mean'] = questions_df['part'].map(part_agg['mean'])
 questions_df['part_correctness_std'] = questions_df['part'].map(part_agg['var'])
 questions_df.part_correctness_mean=questions_df.part_correctness_mean.astype('float16')
 questions_df.part_correctness_std=questions_df.part_correctness_std.astype('float16' )<categorify>
LR = LogisticRegression(**LR_best_Params)
 LR.fit(final_train_df,train_label)
 prediction = LR.predict(final_test_df )
bundle_agg = questions_df.groupby('bundle_id')['content_correctness'].agg(['mean'])
 questions_df['bundle_correctness'] = questions_df['bundle_id'].map(bundle_agg['mean'])
 questions_df.bundle_correctness=questions_df.bundle_correctness.astype('float16' )<categorify>
submission = pd.DataFrame({'PassengerId':test_Ids,'Survived':prediction})
 submission.to_csv('submission_v9.csv',index = False )