Kaggle competition: Titanic - Machine Learning from Disaster
ts = time.time()
matrix = lag_feature(matrix, [1,2,3,6,12], 'item_cnt_month')
 time.time() - ts<import_modules>
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, AdaBoostClassifier, BaggingClassifier, ExtraTreesClassifier
from sklearn.metrics import mean_absolute_error, accuracy_score, f1_score
from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import LinearRegression, Lasso
from sklearn.model_selection import GridSearchCV, KFold, cross_validate
from sklearn.neighbors import KNeighborsClassifier
from sklearn.pipeline import make_pipeline, Pipeline
from sklearn.preprocessing import OneHotEncoder, MinMaxScaler, Normalizer, OrdinalEncoder, StandardScaler
from sklearn.impute import SimpleImputer
<prepare_x_and_y>
train = pd.read_csv('/kaggle/input/titanic/train.csv')
 test = pd.read_csv('/kaggle/input/titanic/test.csv')
 sub = pd.read_csv('/kaggle/input/titanic/gender_submission.csv' )
<load_pretrained>
y = train['Survived']
data = pd.read_pickle('/kaggle/input/all-datta5/data2.pkl')
 print(data.shape)
 data.head(3 )<count_unique_values>
traintest = pd.concat([train.drop('Survived', axis=1),test] ).reset_index()
a = []
for i in range(3,34):
b = data[data.date_block_num==i]
c = b.sum() ['item_cnt_month']
d = len(b.shop_id.unique())
 a.append(c/d)
 print(a )<create_dataframe>
traintest[traintest['Age'].isnull() ]
weekarr = []
t = 2
count = 0
for w in range(3):
for i in [31,28,31,30,31,30,31,31,30,31,30,31]:
a = [0,0,0,0,0,0,0,count]
count+=1
for j in range(i):
a[t]+=1
if t==6:
t=-1
t+=1
weekarr.append(a)
 weekarr = pd.DataFrame(np.vstack(weekarr), columns=['week0','week1','week2','week3','week4','week5','week6','date_block_num'])
 data = pd.merge(data, weekarr, on=['date_block_num'], how='left')
 del weekarr
gc.collect()<prepare_x_and_y>
traintest[traintest['Age']<14]
X_zong = data.drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
Y_valid = data[data.date_block_num == 33]['item_cnt_month']
del data
gc.collect()<categorify>
traintest['Title'] = traintest.Name.str.extract('([A-Za-z]+)\.' )
minMax = MinMaxScaler()
X_zong_std = minMax.fit_transform(X_zong.iloc[:,:-7])
 X_zong.iloc[:,:-7] = pd.DataFrame(np.vstack(X_zong_std),columns=X_zong.columns[:-7])
 X_zong.iloc[:,:-7] = downcast_dtypes(X_zong.iloc[:,:-7])
 X_zong['week0'] = X_zong['week0'].astype(np.int8)
 X_zong['week1'] = X_zong['week1'].astype(np.int8)
 X_zong['week2'] = X_zong['week2'].astype(np.int8)
 X_zong['week3'] = X_zong['week3'].astype(np.int8)
 X_zong['week4'] = X_zong['week4'].astype(np.int8)
 X_zong['week5'] = X_zong['week5'].astype(np.int8)
 X_zong['week6'] = X_zong['week6'].astype(np.int8)
 del X_zong_std
gc.collect()<filter>
traintest_drop3 = traintest.drop(['Name', 'Ticket', 'Cabin'], axis=1)
X_train = X_zong[X_zong.date_block_num < 0.96679688]
X_valid = X_zong[X_zong.date_block_num == 0.96679688]
X_test = X_zong[X_zong.date_block_num == 1]
del X_zong
gc.collect()<train_model>
traintest_drop3[traintest_drop3['Title']=='Mrs']['Age'].sort_values().value_counts(sort=False,dropna=False )
ts = time.time()
model = XGBRegressor(
 max_depth=10,
n_estimators=1000,
min_child_weight=0.5,
colsample_bytree=0.9,
subsample=0.8,
eta=0.1,
seed=1)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=True,
early_stopping_rounds = 10)
 time.time() - ts<save_to_csv>
traintest_drop3[traintest_drop3['Title']=='Master']['Age'].value_counts(dropna=False )
Y_test = model.predict(X_test ).clip(0, 20)
 test = pd.read_csv('/kaggle/input/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID')
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('xgb_submission1.csv', index=False )<set_options>
traintest_drop3[traintest_drop3['Title']=='Mr']['Age'].sort_values().value_counts(sort=False,dropna=False )
env = riiideducation.make_env()<load_from_csv>
traintest_drop3[traintest_drop3['Title']=='Miss']['Age'].sort_values().value_counts(sort=False,dropna=False )
train = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',
usecols=[1, 2, 3, 4, 5, 7, 8, 9],
dtype={'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id': 'int8',
'task_container_id': 'int16',
'answered_correctly':'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'boolean'} 
)
 train.info()<load_from_csv>
master_nan_median = np.median(traintest_drop3[traintest_drop3['Title']=='Master']['Age'].dropna().values)
 master_nan_median
questions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 3],
dtype={'question_id': 'int16',
'part': 'int8'} 
)
 questions_df<sort_values>
traintest_drop3[traintest_drop3['Title']=='Master']['Age']
train = train[train.content_type_id == False].sort_values('timestamp' ).reset_index(drop = True )<groupby>
traintest_drop3.loc[(traintest_drop3['Title']=='Master')&(traintest_drop3['Age'].isna()),'Age'] = master_nan_median
group1 = train.loc[(train.content_type_id == False), ['task_container_id', 'user_id']].groupby(['task_container_id'] ).agg(['count'])
 group1.columns = ['avg_questions']
group2 = train.loc[(train.content_type_id == False), ['task_container_id', 'user_id']].groupby(['task_container_id'] ).agg(['nunique'])
 group2.columns = ['avg_questions']
group3 = group1 / group2
group3['avg_questions_seen'] = group3.avg_questions.cumsum()
print('The amount of questions seen by the average user:')
 group3.iloc[0].avg_questions_seen<merge>
traintest_drop3.loc[traintest_drop3['Title']=='Master', 'Age']
train = pd.merge(train, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left' )<groupby>
mr_nan_median = np.median(traintest_drop3[traintest_drop3['Title']=='Mr']['Age'].dropna().values)
 mr_nan_median
results_q2_final = train.loc[train.content_type_id == False, ['question_id','part']].groupby(['question_id'] ).agg(['count'])
 results_q2_final.columns = ['count']<merge>
traintest_drop3.loc[(traintest_drop3['Title']=='Mr')&(traintest_drop3['Age'].isna()),'Age'] = mr_nan_median
question2 = pd.merge(questions_df, results_q_final, left_on = 'question_id', right_on = 'question_id', how = 'left')
 question2 = pd.merge(question2, results_q2_final, left_on = 'question_id', right_on = 'question_id', how = 'left')
 question2.quest_pct = round(question2.quest_pct,5)
 question2<groupby>
mrs_nan_median = np.median(traintest_drop3[traintest_drop3['Title']=='Mrs']['Age'].dropna().values)
 mrs_nan_median
prior_mean_user = results_u2_final.explanation_mean_user.mean()
prior_mean_user<drop_column>
traintest_drop3.loc[(traintest_drop3['Title']=='Mrs')&(traintest_drop3['Age'].isna()),'Age'] = mrs_nan_median
train.drop(['timestamp', 'content_type_id', 'question_id', 'part'], axis=1, inplace=True )<groupby>
miss_nan_median = np.median(traintest_drop3[traintest_drop3['Title']=='Miss']['Age'].dropna().values)
 miss_nan_median
print('The old length of the training set:')
 print(len(train))
validation = train.groupby('user_id' ).tail(10)
 train = train[~train.index.isin(validation.index)]
print('The length of the training set plus the length of the validation set:')
 print(len(train)+ len(validation))<groupby>
traintest_drop3.loc[(traintest_drop3['Title']=='Miss')&(traintest_drop3['Age'].isna()),'Age'] = miss_nan_median
results_u_val = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_val.columns = ['answered_correctly_user']
results_u2_val = train[['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_val.columns = ['explanation_mean_user']<groupby>
traintest_drop3['Age'].isna().sum()
X = train.groupby('user_id' ).tail(30)
 train = train[~train.index.isin(X.index)]
print('The length of the training set plus the length of the validation set plus the length of the set to be discarded:')
 print(len(X)+ len(validation)+ len(train))<groupby>
still_nan = np.where(traintest_drop3['Age'].isna())[0].tolist()
still_nan
results_u_X = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_X.columns = ['answered_correctly_user']
results_u2_X = train[['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_X.columns = ['explanation_mean_user']<drop_column>
traintest_drop3.iloc[still_nan]
del(train )<merge>
traintest_drop3.loc[(traintest_drop3['Title']=='Ms')&(traintest_drop3['Age'].isna()),'Age'] = mrs_nan_median
X = pd.merge(X, group3, left_on=['task_container_id'], right_index= True, how="left")
 X = pd.merge(X, results_u_X, on=['user_id'], how="left")
 X = pd.merge(X, results_u2_X, on=['user_id'], how="left" )<merge>
traintest_drop3.loc[(traintest_drop3['Title']=='Dr')&(traintest_drop3['Age'].isna()),'Age'] = mr_nan_median
validation = pd.merge(validation, group3, left_on=['task_container_id'], right_index= True, how="left")
 validation = pd.merge(validation, results_u_val, on=['user_id'], how="left")
 validation = pd.merge(validation, results_u2_val, on=['user_id'], how="left" )<categorify>
fare_nan = np.where(traintest_drop3['Fare'].isna())[0].tolist()
lb_make = LabelEncoder()
X.prior_question_had_explanation.fillna(False, inplace = True)
 validation.prior_question_had_explanation.fillna(False, inplace = True)
 validation["prior_question_had_explanation_enc"] = lb_make.fit_transform(validation["prior_question_had_explanation"])
 X["prior_question_had_explanation_enc"] = lb_make.fit_transform(X["prior_question_had_explanation"] )<categorify>
traintest_drop3.iloc[fare_nan]
question2.quest_pct = question2.quest_pct.mask(( question2['count'] < 3),.65)
 question2.quest_pct = question2.quest_pct.mask(( question2.quest_pct <.2)&(question2['count'] < 21),.2)
 question2.quest_pct = question2.quest_pct.mask(( question2.quest_pct >.95)&(question2['count'] < 21),.95 )<merge>
pclass3_median = np.median(traintest_drop3[traintest_drop3['Pclass']==3]['Fare'].dropna().values)
 pclass3_median
X = pd.merge(X, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 validation = pd.merge(validation, question2, left_on = 'content_id', right_on = 'question_id', how = 'left' )<prepare_x_and_y>
traintest_drop3.loc[(traintest_drop3['Fare'].isna()),'Fare'] = pclass3_median
y = X['answered_correctly']
X = X.drop(['answered_correctly'], axis=1)
 X.head()
y_val = validation['answered_correctly']
X_val = validation.drop(['answered_correctly'], axis=1 )<drop_column>
traintest_drop3['Fare'].isna().sum()
X = X[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part']]
X_val = X_val[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part']]<data_type_conversions>
emb_nan = np.where(traintest_drop3['Embarked'].isna())[0].tolist()
traintest_drop3.iloc[emb_nan]
X['answered_correctly_user'].fillna(0.65, inplace=True)
 X['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 X['quest_pct'].fillna(content_mean, inplace=True)
 X['part'].fillna(4, inplace = True)
 X['avg_questions_seen'].fillna(1, inplace = True)
 X['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 X['prior_question_had_explanation_enc'].fillna(0, inplace = True )<data_type_conversions>
traintest_drop3[traintest_drop3['Fare']==80.0]
X_val['answered_correctly_user'].fillna(0.65, inplace=True)
 X_val['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 X_val['quest_pct'].fillna(content_mean, inplace=True)
 X_val['part'].fillna(4, inplace = True)
 X['avg_questions_seen'].fillna(1, inplace = True)
 X_val['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 X_val['prior_question_had_explanation_enc'].fillna(0, inplace = True )<create_dataframe>
traintest_drop3[traintest_drop3['Pclass']==1]
lgb_train = lgb.Dataset(X, y, categorical_feature = ['part', 'prior_question_had_explanation_enc'],free_raw_data=False)
 lgb_eval = lgb.Dataset(X_val, y_val, categorical_feature = ['part', 'prior_question_had_explanation_enc'], reference=lgb_train, free_raw_data=False )<init_hyperparams>
traintest_drop3[traintest_drop3['PassengerId']==61]
y_val_array = np.array(y_val)
 def obj_and_bounds(trial):
params = {
'num_leaves': trial.suggest_int('num_leaves', 160, 162),
'boosting_type': 'gbdt',
'max_bin': trial.suggest_int('max_bin', 889, 891),
'objective': 'binary',
'metric': 'auc',
'max_depth': trial.suggest_int('max_depth', 10, 12),
'min_child_weight': trial.suggest_int('min_child_weight', 10, 12),
'feature_fraction': trial.suggest_uniform('feature_fraction', 0.69, 0.7),
'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.92, 0.93),
'bagging_freq': trial.suggest_int('bagging_freq', 6, 8),
'min_child_samples': trial.suggest_int('min_child_samples', 76, 78),
'lambda_l1': trial.suggest_loguniform('lambda_l1', 2e-2, 3e-2),
'lambda_l2': trial.suggest_loguniform('lambda_l2', 9e-8, 1e-7),
'early_stopping_rounds': 10
}
model = lgb.train(params, lgb_train, valid_sets=[lgb_train, lgb_eval], verbose_eval=1000, num_boost_round=300)
 val_pred = model.predict(X_val)
 score = roc_auc_score(y_val_array, val_pred)
 print("ROC/AUC = ")
 print(score)
 return score<find_best_params>
traintest_drop3[traintest_drop3['PassengerId']==63]
study = optuna.create_study(direction='maximize')
 study.optimize(obj_and_bounds, n_trials=2)
<find_best_params>
traintest_drop3[traintest_drop3['Pclass']==1]['Embarked'].value_counts()
print('Number of finished trials: {}'.format(len(study.trials)))
 trial = study.best_trial
print('ROC/AUC of the best trial: {}'.format(trial.value))
print('Parameters of the best trial: ')
 for key, value in trial.params.items() :
print(' {}: {}'.format(key, value))<train_model>
traintest_drop3.loc[(traintest_drop3['Embarked'].isna()),'Embarked'] = 'S'
model = lgb.train(
 trial.params,
lgb_train,
valid_sets=[lgb_train, lgb_eval],
verbose_eval=1000,
num_boost_round=2000
)<compute_train_metric>
oenc = OrdinalEncoder()
traintest_objenc = pd.DataFrame(oenc.fit_transform(traintest_drop3.select_dtypes('object')) ,columns = traintest_drop3.select_dtypes('object' ).columns )
y_pred = model.predict(X_val)
 roc_auc_score(y_val_array, y_pred )<merge>
traintest_num = traintest_drop3.select_dtypes(['int64','float64'] ).drop(['index','PassengerId'], axis=1 ).join(traintest_objenc )
iter_test = env.iter_test()
for(test_df, sample_prediction_df)in iter_test:
test_df['task_container_id'] = test_df.task_container_id.mask(test_df.task_container_id > 9999, 9999)
 test_df = pd.merge(test_df, group3, left_on=['task_container_id'], right_index= True, how="left")
 test_df = pd.merge(test_df, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df = pd.merge(test_df, results_u_final, on=['user_id'], how="left")
 test_df = pd.merge(test_df, results_u2_final, on=['user_id'], how="left")
 test_df['answered_correctly_user'].fillna(0.65, inplace=True)
 test_df['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 test_df['quest_pct'].fillna(content_mean, inplace=True)
 test_df['part'].fillna(4, inplace = True)
 test_df['avg_questions_seen'].fillna(1, inplace = True)
 test_df['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df["prior_question_had_explanation_enc"] = lb_make.fit_transform(test_df["prior_question_had_explanation"])
 test_df['answered_correctly'] = model.predict(test_df[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part']])
 env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<set_options>
traintest_norm = traintest_drop3[['index','PassengerId']].join(pd.DataFrame(StandardScaler().fit_transform(traintest_num), columns = traintest_num.columns))
env = riiideducation.make_env()<load_from_csv>
train_norm = traintest_norm[0:891].drop('index', axis=1 )
train = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',
usecols=[1, 2, 3, 4, 5, 7, 8, 9],
dtype={'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id': 'int8',
'task_container_id': 'int16',
'answered_correctly':'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'boolean'}
)<sort_values>
test_norm = traintest_norm[891::].drop('index', axis=1 ).reset_index().drop('index',axis=1 )
train = train[train.content_type_id == False].sort_values('timestamp' ).reset_index(drop = True )<count_unique_values>
X_train, X_test, y_train, y_test = train_test_split(train_norm, y )
train[(train.content_type_id == False)].task_container_id.nunique()<groupby>
modelgbr = GradientBoostingClassifier(random_state=42)
 modelgbr.fit(X_train, y_train)
 y_predgbr = modelgbr.predict(X_test)
 accuracy_score(y_test, y_predgbr)
group1 = train.loc[(train.content_type_id == False), ['task_container_id', 'user_id']].groupby(['task_container_id'] ).agg(['count'])
 group1.columns = ['avg_questions']
group2 = train.loc[(train.content_type_id == False), ['task_container_id', 'user_id']].groupby(['task_container_id'] ).agg(['nunique'])
 group2.columns = ['avg_questions']
group3 = group1 / group2<groupby>
pd.DataFrame(modelgbr.feature_importances_, X_train.columns ).sort_values(by=0,ascending=False)
group3['avg_questions_seen'] = group3.avg_questions.cumsum()<set_options>
def model_test(testmodel):
model = testmodel()
model.fit(X_train, y_train)
 y_pred = model.predict(X_test)
 return accuracy_score(y_test, y_pred )
group3.iloc[0].avg_questions_seen<groupby>
models = [RandomForestClassifier, GradientBoostingClassifier, AdaBoostClassifier,
BaggingClassifier, ExtraTreesClassifier, DecisionTreeClassifier, KNeighborsClassifier]
results_u_final = train.loc[train.content_type_id == False, ['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_final.columns = ['answered_correctly_user']
results_u2_final = train.loc[train.content_type_id == False, ['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_final.columns = ['explanation_mean_user']<drop_column>
for i in models:
print(i)
 print(model_test(i))
train.drop(['timestamp', 'content_type_id'], axis=1, inplace=True )<filter>
modelclf = GradientBoostingClassifier()
parameters = {'n_estimators':[50,200,300], 'learning_rate':[0.001,0.01,0.1],'max_depth':[1,3,6]}
clf = GridSearchCV(modelclf, parameters)
 clf.fit(X_train, y_train)
 y_predclf = clf.predict(X_test)
 accuracy_score(y_test, y_predclf)
 clf.cv_results_
clf.best_params_
validation = train.groupby('user_id' ).tail(5)
 train = train[~train.index.isin(validation.index)]
len(train)+ len(validation )<groupby>
modelsub = GradientBoostingClassifier(learning_rate = 0.01, max_depth = 3, n_estimators = 300)
 modelsub.fit(train_norm.drop('PassengerId',axis=1), y)
 preds_test = modelsub.predict(test_norm.drop('PassengerId',axis=1))
<groupby><EOS> 
submission = pd.DataFrame({
"PassengerId": test["PassengerId"],
"Survived": preds_test
})
 submission.to_csv('submission.csv', index=False)