Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<groupby>
%matplotlib inline
sns.set()
results_u_X = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_X.columns = ['answered_correctly_user']
results_u2_X = train[['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_X.columns = ['explanation_mean_user']<drop_column>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv' )
del(train )<merge>
train['Survived'].value_counts(normalize=True )
X = pd.merge(X, group3, left_on=['task_container_id'], right_index= True, how="left")
 X = pd.merge(X, results_u_X, on=['user_id'], how="left")
 X = pd.merge(X, results_u2_X, on=['user_id'], how="left" )<merge>
train['Survived'].groupby(train['Pclass'] ).mean()
validation = pd.merge(validation, group3, left_on=['task_container_id'], right_index= True, how="left")
 validation = pd.merge(validation, results_u_val, on=['user_id'], how="left")
 validation = pd.merge(validation, results_u2_val, on=['user_id'], how="left" )<categorify>
train.isnull().sum()
lb_make = LabelEncoder()
X.prior_question_had_explanation.fillna(False, inplace = True)
 validation.prior_question_had_explanation.fillna(False, inplace = True)
 validation["prior_question_had_explanation_enc"] = lb_make.fit_transform(validation["prior_question_had_explanation"])
 X["prior_question_had_explanation_enc"] = lb_make.fit_transform(X["prior_question_had_explanation"] )<load_from_csv>
train.isnull().sum()
question2 = pd.read_csv('/kaggle/input/question2/question2.csv' )<categorify>
test.isnull().sum()
question2.quest_pct = question2.quest_pct.mask(( question2['count'] < 3),.65)
 question2.quest_pct = question2.quest_pct.mask(( question2.quest_pct <.2)&(question2['count'] < 21),.2)
 question2.quest_pct = question2.quest_pct.mask(( question2.quest_pct >.95)&(question2['count'] < 21),.95 )<merge>
test.isnull().sum()
X = pd.merge(X, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 validation = pd.merge(validation, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 X.part = X.part - 1
validation.part = validation.part - 1<prepare_x_and_y>
train_test_data = [train, test]
for dataset in train_test_data:
dataset['Title'] = dataset['Name'].str.extract('([A-Za-z]+)\.', expand=False )
y = X['answered_correctly']
X = X.drop(['answered_correctly'], axis=1)
 X.head()
y_val = validation['answered_correctly']
X_val = validation.drop(['answered_correctly'], axis=1 )<drop_column>
train['Name_Title'] = train['Name'].apply(lambda x: x.split(',')[1] ).apply(lambda x: x.split() [0])
 train['Name_Title'].value_counts()
X = X[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part']]
X_val = X_val[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part']]<data_type_conversions>
train['Survived'].groupby(train['Name_Title'] ).mean()
X['answered_correctly_user'].fillna(0.65, inplace=True)
 X['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 X['quest_pct'].fillna(content_mean, inplace=True)
 X['part'].fillna(4, inplace = True)
 X['avg_questions_seen'].fillna(1, inplace = True)
 X['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 X['prior_question_had_explanation_enc'].fillna(0, inplace = True)
<data_type_conversions>
train['Name_Len'] = train['Name'].apply(lambda x: len(x))
train['Survived'].groupby(pd.qcut(train['Name_Len'],5)).mean()
X_val['answered_correctly_user'].fillna(0.65, inplace=True)
 X_val['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 X_val['quest_pct'].fillna(content_mean, inplace=True)
 X_val['part'].fillna(4, inplace = True)
 X['avg_questions_seen'].fillna(1, inplace = True)
 X_val['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 X_val['prior_question_had_explanation_enc'].fillna(0, inplace = True )<create_dataframe>
pd.qcut(train['Name_Len'],5 ).value_counts()
params = {
'objective': 'binary',
'boosting' : 'gbdt',
'max_bin': 800,
'learning_rate': 0.0175,
'num_leaves': 80
}
lgb_train = lgb.Dataset(X, y, categorical_feature = ['part', 'prior_question_had_explanation_enc'])
 lgb_eval = lgb.Dataset(X_val, y_val, categorical_feature = ['part', 'prior_question_had_explanation_enc'], reference=lgb_train )<train_model>
train['Sex'].value_counts(normalize=True )
model = lgb.train(
 params, lgb_train,
valid_sets=[lgb_train, lgb_eval],
verbose_eval=50,
num_boost_round=1300,
early_stopping_rounds=8
)<compute_train_metric>
train['Survived'].groupby(train['Sex'] ).mean()
y_pred = model.predict(X_val)
 y_true = np.array(y_val)
 roc_auc_score(y_true, y_pred )<import_modules>
train['Survived'].groupby(train['Age'].isnull() ).mean()
import matplotlib.pyplot as plt
import seaborn as sns<split>
train['Survived'].groupby(pd.qcut(train['Age'],5)).mean()
iter_test = env.iter_test()<merge>
pd.qcut(train['Age'],5 ).value_counts()
for(test_df, sample_prediction_df)in iter_test:
test_df['task_container_id'] = test_df.task_container_id.mask(test_df.task_container_id > 9999, 9999)
 test_df = pd.merge(test_df, group3, left_on=['task_container_id'], right_index= True, how="left")
 test_df = pd.merge(test_df, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df = pd.merge(test_df, results_u_final, on=['user_id'], how="left")
 test_df = pd.merge(test_df, results_u2_final, on=['user_id'], how="left")
 test_df['answered_correctly_user'].fillna(0.65, inplace=True)
 test_df['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 test_df['quest_pct'].fillna(content_mean, inplace=True)
 test_df['part'] = test_df.part - 1
test_df['part'].fillna(4, inplace = True)
 test_df['avg_questions_seen'].fillna(1, inplace = True)
 test_df['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df["prior_question_had_explanation_enc"] = lb_make.fit_transform(test_df["prior_question_had_explanation"])
 test_df['answered_correctly'] = model.predict(test_df[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part']])
 env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<import_modules>
train["Age"].fillna(train.groupby("Title")["Age"].transform("median"), inplace=True)
 test["Age"].fillna(test.groupby("Title")["Age"].transform("median"), inplace=True )
import optuna
import lightgbm as lgb
import pickle
import riiideducation
import dask.dataframe as dd
import pandas as pd
import numpy as np
from sklearn.metrics import roc_auc_score<load_from_csv>
train['Survived'].groupby(train['SibSp'] ).mean()
train = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',
usecols=[1, 2, 3,4,7,8,9], dtype={'timestamp': 'int64', 'user_id': 'int32' ,'content_id': 'int16','content_type_id': 'int8','answered_correctly':'int8','prior_question_elapsed_time': 'float32','prior_question_had_explanation': 'boolean'} 
)
 train = train[train.content_type_id == False]
train = train.sort_values(['timestamp'], ascending=True)
 train.drop(['timestamp','content_type_id'], axis=1, inplace=True)
 results_c = train[['content_id','answered_correctly']].groupby(['content_id'] ).agg(['mean'])
 results_c.columns = ["answered_correctly_content"]
results_u = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean', 'sum'])
 results_u.columns = ["answered_correctly_user", 'sum']<load_from_csv>
train['SibSp'].value_counts()
questions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',
usecols=[0,1, 3,4],
dtype={'question_id': 'int16',
'part': 'int8','bundle_id': 'int8','tags': 'str'} 
)
 tag = questions_df["tags"].str.split(" ", n = 10, expand = True)
 tag.columns = ['tags1','tags2','tags3','tags4','tags5','tags6']
questions_df = pd.concat([questions_df,tag],axis=1)
 questions_df['tags1'] = pd.to_numeric(questions_df['tags1'], errors='coerce')
 questions_df['tags2'] = pd.to_numeric(questions_df['tags2'], errors='coerce')
 questions_df['tags3'] = pd.to_numeric(questions_df['tags3'], errors='coerce')
 questions_df['tags4'] = pd.to_numeric(questions_df['tags4'], errors='coerce')
 questions_df['tags5'] = pd.to_numeric(questions_df['tags5'], errors='coerce')
 questions_df['tags6'] = pd.to_numeric(questions_df['tags6'], errors='coerce' )<categorify>
train['Survived'].groupby(train['Parch'] ).mean()
lb_make = LabelEncoder()
train['prior_question_had_explanation'].fillna(False, inplace=True)
 train["prior_question_had_explanation_enc"] = lb_make.fit_transform(train["prior_question_had_explanation"] )<load_pretrained>
train['Parch'].value_counts()
file = '.. /input/riiid-lgb-training-and-save-model/trained_model.pkl'
model = pickle.load(open(file, 'rb'))
print('Trained LGB model was loaded!' )<find_best_params>
train['Ticket_Len'] = train['Ticket'].apply(lambda x: len(x))
env = riiideducation.make_env()<merge>
train['Ticket_Len'].value_counts()
iter_test = env.iter_test()
for(test_df, sample_prediction_df)in iter_test:
test_df = test_df.sort_values(['user_id','timestamp'], ascending=False)
 test_df['answer_time'] = test_df.groupby(['user_id'])['prior_question_elapsed_time'].shift(1)
 test_df = pd.merge(test_df, results_u, on=['user_id'], how="left")
 test_df = pd.merge(test_df, results_c, on=['content_id'], how="left")
 test_df = pd.merge(test_df, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df['answered_correctly_user'].fillna(0.5, inplace=True)
 test_df['answered_correctly_content'].fillna(0.5, inplace=True)
 test_df['sum'].fillna(0, inplace=True)
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df["prior_question_had_explanation_enc"] = lb_make.transform(test_df["prior_question_had_explanation"])
 test_df['answered_correctly'] = model.predict(test_df[['answered_correctly_user', 'answered_correctly_content', 'sum','bundle_id','part','prior_question_elapsed_time','prior_question_had_explanation_enc',
'tags1','tags2','tags3']])
 env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<set_options>
train['Ticket_Lett'] = train['Ticket'].apply(lambda x: str(x)[0] )
env = riiideducation.make_env()<load_from_csv>
train['Ticket_Lett'].value_counts()
train = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',
usecols=[1, 2, 3, 4, 7, 8, 9],
dtype={'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id': 'int8',
'answered_correctly':'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'boolean'}
)<sort_values>
train.groupby(['Ticket_Lett'])['Survived'].mean()
train = train[train.content_type_id == False]
train = train.sort_values(['timestamp'], ascending=True ).reset_index(drop = True)
 train.head(10 )<groupby>
pd.qcut(train['Fare'], 3 ).value_counts()
results_c_final = train[['content_id','answered_correctly']].groupby(['content_id'] ).agg(['mean'])
 results_c_final.columns = ["answered_correctly_content"]
results_u_final = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean', 'sum', 'count'])
 results_u_final.columns = ['answered_correctly_user', 'sum', 'count']<filter>
train['Survived'].groupby(pd.qcut(train['Fare'], 3)).mean()
train.loc[(train.timestamp < 1000000)&(train.timestamp > 0)].answered_correctly.mean()<count_values>
train['Cabin_Letter'] = train['Cabin'].apply(lambda x: str(x)[0] )
train.prior_question_had_explanation.value_counts()<drop_column>
train['Cabin_Letter'].value_counts()
train.drop(['timestamp', 'content_type_id'], axis=1, inplace=True )<create_dataframe>
train['Survived'].groupby(train['Cabin_Letter'] ).mean()
validation = pd.DataFrame()<remove_duplicates>
train['Cabin_num'] = train['Cabin'].apply(lambda x: str(x ).split(' ')[-1][1:])
 train['Cabin_num'].replace('an', np.NaN, inplace = True)
 train['Cabin_num'] = train['Cabin_num'].apply(lambda x: int(x)if not pd.isnull(x)and x != '' else np.NaN )
for i in range(4):
last_records = train.drop_duplicates('user_id', keep = 'last')
 train = train[~train.index.isin(last_records.index)]
validation = validation.append(last_records )<create_dataframe>
pd.qcut(train['Cabin_num'],3 ).value_counts()
X = pd.DataFrame()<remove_duplicates>
train['Survived'].groupby(pd.qcut(train['Cabin_num'], 3)).mean()
for i in range(15):
last_records = train.drop_duplicates('user_id', keep = 'last')
 train = train[~train.index.isin(last_records.index)]
X = X.append(last_records )<groupby>
train['Embarked'].value_counts()
results_c = train[['content_id','answered_correctly']].groupby(['content_id'] ).agg(['mean'])
 results_c.columns = ["answered_correctly_content"]
results_u = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean', 'sum', 'count'])
 results_u.columns = ["answered_correctly_user", 'sum', 'count']<drop_column>
train['Embarked'].value_counts(normalize=True )
del(train)
<merge>
train['Survived'].groupby(train['Embarked'] ).mean()
X = pd.merge(X, results_u, on=['user_id'], how="left")
 X = pd.merge(X, results_c_final, on=['content_id'], how="left" )<merge>
def names(train, test):
for i in [train, test]:
i['Name_Len'] = i['Name'].apply(lambda x: len(x))
i['Name_Title'] = i['Name'].apply(lambda x: x.split(',')[1] ).apply(lambda x: x.split() [0])
 del i['Name']
return train, test
validation = pd.merge(validation, results_u, on=['user_id'], how="left")
 validation = pd.merge(validation, results_c_final, on=['content_id'], how="left" )<categorify>
def age_impute(train, test):
for i in [train, test]:
i['Age_Null_Flag'] = i['Age'].apply(lambda x: 1 if pd.isnull(x)else 0)
 data = train.groupby(['Name_Title', 'Pclass'])['Age']
i['Age'] = data.transform(lambda x: x.fillna(x.mean()))
return train, test
lb_make = LabelEncoder()
X.prior_question_had_explanation.fillna(False, inplace = True)
 validation.prior_question_had_explanation.fillna(False, inplace = True)
 validation["prior_question_had_explanation_enc"] = lb_make.fit_transform(validation["prior_question_had_explanation"])
 X["prior_question_had_explanation_enc"] = lb_make.fit_transform(X["prior_question_had_explanation"])
 X.head()<load_from_csv>
def fam_size(train, test):
for i in [train, test]:
i['Fam_Size'] = np.where(( i['SibSp']+i['Parch'])== 0 , 'Solo',
np.where(( i['SibSp']+i['Parch'])<= 3,'Nuclear', 'Big'))
del i['SibSp']
del i['Parch']
return train, test
questions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 3],
dtype={'question_id': 'int16',
'part': 'int8'}
)<merge>
def cabin(train, test):
for i in [train, test]:
i['Cabin_Letter'] = i['Cabin'].apply(lambda x: str(x)[0])
 del i['Cabin']
return train, test
X = pd.merge(X, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left')
 validation = pd.merge(validation, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left')
 X.part = X.part - 1
validation.part = validation.part - 1
X.part.value_counts()<prepare_x_and_y>
def cabin_num(train, test):
for i in [train, test]:
i['Cabin_num1'] = i['Cabin'].apply(lambda x: str(x ).split(' ')[-1][1:])
 i['Cabin_num1'].replace('an', np.NaN, inplace = True)
 i['Cabin_num1'] = i['Cabin_num1'].apply(lambda x: int(x)if not pd.isnull(x)and x != '' else np.NaN)
 i['Cabin_num'] = pd.qcut(train['Cabin_num1'],3)
 train = pd.concat(( train, pd.get_dummies(train['Cabin_num'], prefix = 'Cabin_num')) , axis = 1)
 test = pd.concat(( test, pd.get_dummies(test['Cabin_num'], prefix = 'Cabin_num')) , axis = 1)
 del train['Cabin_num']
del test['Cabin_num']
del train['Cabin_num1']
del test['Cabin_num1']
return train, test
y = X['answered_correctly']
X = X.drop(['answered_correctly'], axis=1)
 X.head()
y_val = validation['answered_correctly']
X_val = validation.drop(['answered_correctly'], axis=1 )<drop_column>
def embarked_impute(train, test):
for i in [train, test]:
i['Embarked'] = i['Embarked'].fillna('S')
 return train, test
X = X[['answered_correctly_user', 'answered_correctly_content', 'sum', 'count',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part']]
X_val = X_val[['answered_correctly_user', 'answered_correctly_content', 'sum', 'count',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part']]<data_type_conversions>
test['Fare'].fillna(train['Fare'].mean() , inplace = True )
X_val['answered_correctly_user'].fillna(0.5, inplace=True)
 X_val['answered_correctly_content'].fillna(0.5, inplace=True)
 X_val['part'].fillna(4, inplace = True)
 X_val['count'].fillna(0, inplace = True)
 X_val['sum'].fillna(0, inplace = True)
 X_val['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 X_val['prior_question_had_explanation_enc'].fillna(0, inplace = True )<create_dataframe>
def dummies(train, test, columns = ['Pclass', 'Sex', 'Embarked', 'Ticket_Lett', 'Cabin_Letter', 'Name_Title', 'Fam_Size']):
for column in columns:
train[column] = train[column].apply(lambda x: str(x))
test[column] = test[column].apply(lambda x: str(x))
good_cols = [column+'_'+i for i in train[column].unique() if i in test[column].unique() ]
train = pd.concat(( train, pd.get_dummies(train[column], prefix = column)[good_cols]), axis = 1)
 test = pd.concat(( test, pd.get_dummies(test[column], prefix = column)[good_cols]), axis = 1)
 del train[column]
del test[column]
return train, test
params = {
'objective': 'binary',
'max_bin': 700,
'learning_rate': 0.0175,
'num_leaves': 80
}
lgb_train = lgb.Dataset(X, y, categorical_feature = ['part', 'prior_question_had_explanation_enc'])
 lgb_eval = lgb.Dataset(X_val, y_val, categorical_feature = ['part', 'prior_question_had_explanation_enc'], reference=lgb_train )<train_model>
def drop(train, test, bye = ['PassengerId']):
for i in [train, test]:
for z in bye:
del i[z]
return train, test
model = lgb.train(
 params, lgb_train,
valid_sets=[lgb_train, lgb_eval],
verbose_eval=50,
num_boost_round=10000,
early_stopping_rounds=12
)<compute_train_metric>
train = train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv')
 train, test = names(train, test)
 train, test = age_impute(train, test)
 train, test = cabin_num(train, test)
 train, test = cabin(train, test)
 train, test = embarked_impute(train, test)
 train, test = fam_size(train, test)
 test['Fare'].fillna(train['Fare'].mean() , inplace = True)
 train, test = ticket_grouped(train, test)
 train, test = dummies(train, test, columns = ['Pclass', 'Sex', 'Embarked', 'Ticket_Lett',
'Cabin_Letter', 'Name_Title', 'Fam_Size'])
 train, test = drop(train, test )
y_pred = model.predict(X_val)
 y_true = np.array(y_val)
 roc_auc_score(y_true, y_pred )<import_modules>
rf = RandomForestClassifier(criterion='gini',
n_estimators=700,
min_samples_split=10,
min_samples_leaf=1,
max_features='auto',
oob_score=True,
random_state=1,
n_jobs=-1)
 rf.fit(train.iloc[:, 1:], train.iloc[:, 0])
 print("%.4f" % rf.oob_score_ )
import matplotlib.pyplot as plt
import seaborn as sns<split>
round(np.mean(rf.oob_score_)*100, 2 )
iter_test = env.iter_test()<merge>
pd.concat(( pd.DataFrame(train.iloc[:, 1:].columns, columns = ['variable']),
pd.DataFrame(rf.feature_importances_, columns = ['importance'])) ,
axis = 1 ).sort_values(by='importance', ascending = False)[:20]
for(test_df, sample_prediction_df)in iter_test:
test_df = pd.merge(test_df, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df = pd.merge(test_df, results_u_final, on=['user_id'], how="left")
 test_df = pd.merge(test_df, results_c_final, on=['content_id'], how="left")
 test_df['answered_correctly_user'].fillna(0.5, inplace=True)
 test_df['answered_correctly_content'].fillna(0.5, inplace=True)
 test_df['part'] = test_df.part - 1
test_df['part'].fillna(4, inplace = True)
 test_df['sum'].fillna(0, inplace=True)
 test_df['count'].fillna(0, inplace=True)
 test_df['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df["prior_question_had_explanation_enc"] = lb_make.fit_transform(test_df["prior_question_had_explanation"])
 test_df['answered_correctly'] = model.predict(test_df[['answered_correctly_user', 'answered_correctly_content', 'sum', 'count',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part']])
 env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<define_variables>
predictions = rf.predict(test)
 predictions = pd.DataFrame(predictions, columns=['Survived'])
 test = pd.read_csv(os.path.join('.. /input/titanic/test.csv'))
predictions = pd.concat(( test.iloc[:, 0], predictions), axis = 1)
 predictions.to_csv('vysledok.csv', sep=",", index = False )