Kaggle competition: Titanic - Machine Learning from Disaster
train_df['task_container_uncor_count'] = train_df['task_container_id'].map(task_container_agg['count']-task_container_agg['sum'] ).astype('int32')
 train_df['task_container_cor_count'] = train_df['task_container_id'].map(task_container_agg['sum'] ).astype('int32')
 train_df['task_container_std'] = train_df['task_container_id'].map(task_container_agg['var'] ).astype('float16')
 train_df['task_container_correctness'] = train_df['task_container_id'].map(task_container_agg['sum'] / task_container_agg['count'])
 train_df.task_container_correctness=train_df.task_container_correctness.astype('float16' )<groupby>
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 train_data.head(5)
content_elapsed_time_agg=train_df.groupby('content_id')['prior_question_elapsed_time'].agg(['mean'])
 content_had_explanation_agg=train_df.groupby('content_id')['prior_question_had_explanation'].agg(['mean'] )<train_model>
def outlier_detection(dataframe,features, critical=1.5):
ind = []
for column in features:
q1 = np.percentile(dataframe[column],25)
 q3 = np.percentile(dataframe[column],75)
 res = q3 - q1
new = critical * res
out = dataframe[(dataframe[column] < q1 - new)|(dataframe[column] > q3 + new)].index
ind.extend(out)
 ind = Counter(ind)
 val = list(s for s,i in ind.items())
 return val
ott = outlier_detection(train_data,["Fare"])
 train_data.iloc[ott]
print('start questions data...' )<load_from_csv>
women = train_data.loc[train_data.Sex == 'female']["Survived"]
rate_women = sum(women)/len(women)
 print("%.3f of women who survived" % rate_women)
 men = train_data.loc[train_data.Sex == 'male']["Survived"]
rate_men = sum(men)/len(men)
 print("%.3f of men who survived" % rate_men )
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 1,3,4],
dtype={'question_id': 'int16','bundle_id': 'int16', 'part': 'int8','tags': 'str'}
)<groupby>
features = ["PassengerId", "Name", "Pclass", "Sex", "Age", "SibSp", "Parch", "Ticket", "Fare", "Cabin", "Embarked"]
num_train, num_test = len(train_data), len(test_data)
 print(num_train, num_test)
 merged_data = pd.concat([train_data[features], test_data[features]], axis=0, ignore_index=True)
 merged_data['Age'] = merged_data['Age'].fillna(merged_data['Age'].mean())
 merged_data['Fare'] = merged_data['Fare'].fillna(merged_data['Fare'].mean())
 merged_data[['Cabin','Embarked']] = merged_data[['Cabin','Embarked']].fillna(value='Unknown')
 merged_data
bundle_agg = questions_df.groupby('bundle_id')['question_id'].agg(['count'] )<data_type_conversions>
y = train_data["Survived"]
winsorized_X = pd.get_dummies(merged_data[["Pclass", "Name", "Sex", "Age", "SibSp", "Parch", "Ticket", "Fare", "Embarked"]])
 winsorized_X.info()
questions_df['content_sub_bundle'] = questions_df['bundle_id'].map(bundle_agg['count'] ).astype('int8' )<set_options>
X_train = winsorized_X.iloc[0:num_train]
X_test = winsorized_X.iloc[num_train:]
ros = RandomOverSampler(random_state=1)
 X_ros, y_ros = ros.fit_resample(X_train, y)
 x_train, x_valid, y_train, y_valid = train_test_split(X_ros, y_ros, test_size=0.2, random_state=1)
 d_train = xgb.DMatrix(x_train, label=y_train)
 d_valid = xgb.DMatrix(x_valid, label=y_valid)
 d_test = xgb.DMatrix(X_test)
 params = {
'objective':'binary:hinge',
'max_depth':12,
'learning_rate':0.02,
'eval_metric':'error',
'min_child_weight':1,
'subsample':1,
'colsample_bytree':0.4,
'seed':29,
'reg_lambda':2.8,
'reg_alpha':0,
'gamma':0,
'nthread':-1
}
watchlist = [(d_train, 'train'),(d_valid, 'valid')]
nrounds=5000
model = xgb.train(params, d_train, nrounds, watchlist, early_stopping_rounds=600, maximize=False, verbose_eval=30)
questions_df['tags'].fillna('188', inplace=True )<string_transform>
y_pred = model.predict(d_valid)
 print('Accuracy :{0:0.5f}'.format(metrics.accuracy_score(y_valid, y_pred)))
 print('AUC : {0:0.5f}'.format(metrics.roc_auc_score(y_valid, y_pred)))
 print('Precision : {0:0.5f}'.format(metrics.precision_score(y_valid, y_pred)))
 print('Recall : {0:0.5f}'.format(metrics.recall_score(y_valid, y_pred)))
 print('F1 : {0:0.5f}'.format(metrics.f1_score(y_valid, y_pred)) )
<categorify><EOS> 
predictions = model.predict(d_test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions.astype('int32')})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )