Kaggle competition: Riiid Answer Correctness Prediction
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 df_sample= pd.read_csv('.. /input/gender_submission.csv' )<define_variables>
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1
def detect_outliers(df,n,features):

 outlier_indices = []
for col in features:
Q1 = np.percentile(df[col], 25)
 Q3 = np.percentile(df[col],75)
 IQR = Q3 - Q1
outlier_step = 1.5 * IQR
outlier_list_col = df[(df[col] < Q1 - outlier_step)|(df[col] > Q3 + outlier_step)].index
outlier_indices.extend(outlier_list_col)
 outlier_indices = Counter(outlier_indices)
 multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)
 return multiple_outliers<drop_column>
!pip install /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.1-py3-none-any.whl
Outliers_to_drop = detect_outliers(df_train,2,["Age","SibSp","Parch","Fare"])
 df_train = df_train.drop(Outliers_to_drop, axis = 0 ).reset_index(drop=True )<categorify>
from collections import defaultdict
import datatable as dt
import statsmodels.api as sm
from sklearn.metrics import roc_auc_score
from matplotlib import pyplot as plt
import riiideducation
from pathlib import Path
import seaborn as sns
from pytorch_tabnet.tab_model import TabNetClassifier
dataset_title = [i.split(",")[1].split(".")[0].strip() for i in df_train["Name"]]
df_train["Title"] = pd.Series(dataset_title)
 df_train["Title"] = df_train["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 df_train["Title"] = df_train["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 df_train["Title"] = df_train["Title"].astype(int)
 df_train.drop(labels = ["Name"], axis = 1, inplace = True )<categorify>
path = Path('/kaggle/input')
 assert path.exists()
def impute_age(cols):
Age = cols[0]
Pclass = cols[1]
if pd.isnull(Age):
if Pclass == 1:
return 37
elif Pclass == 2:
return 29
else:
return 24
else:
return Age<categorify>
%%time
data_types_dict = {
'user_id': 'int32',
'content_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'
train_df = dt.fread(path/"riidtrainjay/train.jay", columns=set(data_types_dict.keys())).to_pandas()
def impute_fare(cols):
Fare = cols[0]
Pclass = cols[1]
if pd.isnull(Fare):
if Pclass == 1:
return 84
elif Pclass == 2:
return 20
else:
return 13
else:
return Fare<feature_engineering>
%%time
train_df = train_df[train_df[target] != -1].reset_index(drop=True)
 train_df.drop(columns=['timestamp'], inplace=True )
df_train['Age'] = df_train[['Age','Pclass']].apply(impute_age,axis=1 )<categorify>
train_df['prior_question_had_explanation'].fillna(False, inplace=True)
 train_df['prior_question_had_explanation'] = train_df['prior_question_had_explanation'].astype('uint8' )
sex = pd.get_dummies(df_train['Sex'],drop_first=True)
 embark = pd.get_dummies(df_train['Embarked'],drop_first=True)
 df_train = pd.concat([df_train,sex,embark],axis=1 )<feature_engineering>
train_df['lag'] = train_df.groupby('user_id')[target].shift()
train_df['lag'].fillna(0, inplace=True )
df_train["Family"] = df_train["SibSp"] + df_train["Parch"] + 1
df_train['Single'] = df_train['Family'].map(lambda s: 1 if s == 1 else 0)
 df_train['SmallF'] = df_train['Family'].map(lambda s: 1 if s == 2 else 0)
 df_train['MedF'] = df_train['Family'].map(lambda s: 1 if 3 <= s <= 4 else 0)
 df_train['LargeF'] = df_train['Family'].map(lambda s: 1 if s >= 5 else 0)
 df_train['Senior'] = df_train['Age'].map(lambda s:1 if s>60 else 0 )<categorify>
%%time
cum = train_df.groupby(['user_id'])['lag'].agg(['cumsum', 'cumcount'])
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df.drop(columns=['lag'], inplace=True)
 del cum
dataset_title = [i.split(",")[1].split(".")[0].strip() for i in df_test["Name"]]
df_test["Title"] = pd.Series(dataset_title)
 df_test["Title"] = df_test["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 df_test["Title"] = df_test["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 df_test["Title"] = df_test["Title"].astype(int)
 df_test.drop(labels = ["Name"], axis = 1, inplace = True )<categorify>
user_agg = train_df.groupby('user_id')[target].agg(['sum', 'count'])
 content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count'] )
df_test['Age'] = df_test[['Age','Pclass']].apply(impute_age,axis=1)
 sex = pd.get_dummies(df_test['Sex'],drop_first=True)
 embark = pd.get_dummies(df_test['Embarked'],drop_first=True)
 df_test = pd.concat([df_test,sex,embark],axis=1)
 df_test['Fare'].fillna(value=df_test['Fare'].median() ,inplace=True )<feature_engineering>
for f in ['prior_question_elapsed_time']:
train_df[f] = pd.to_numeric(train_df[f], downcast='float' )
df_test['Fare'] = df_test[['Fare','Pclass']].apply(impute_fare,axis=1 )<feature_engineering>
%%time
train_df['residual'] = train_df[target] - train_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 residual_agg = train_df.groupby('user_id')['residual'].agg(['sum'] )
df_test["Fare"] = df_test["Fare"].map(lambda i: np.log(i)if i > 0 else 0 )<feature_engineering>
prior_question_elapsed_time_agg = train_df.groupby('user_id' ).agg({'prior_question_elapsed_time': ['sum', lambda x: len(x)]})
 prior_question_elapsed_time_agg.columns = ['sum', 'count']
prior_question_elapsed_time_agg['count'] = prior_question_elapsed_time_agg['count'].astype('int32')
 prior_question_elapsed_time_agg.info()
df_test["Family"] = df_test["SibSp"] + df_test["Parch"] + 1<feature_engineering>
train_df['user_prior_question_elapsed_time_diff'] =(train_df[target] - train_df['user_id'].map(user_agg['sum'] / user_agg['count'])) *(train_df['prior_question_elapsed_time'] - train_df['user_id'].map(prior_question_elapsed_time_agg['sum'] / prior_question_elapsed_time_agg['count']))
user_prior_question_elapsed_time_diff_agg = train_df.groupby('user_id')['user_prior_question_elapsed_time_diff'].agg(['sum'])
 train_df['user_prior_question_elapsed_time_diff_mean'] = train_df['user_id'].map(user_prior_question_elapsed_time_diff_agg['sum'] / user_agg['count'] )
df_test['Single'] = df_test['Family'].map(lambda s: 1 if s == 1 else 0)
 df_test['SmallF'] = df_test['Family'].map(lambda s: 1 if s == 2 else 0)
 df_test['MedF'] = df_test['Family'].map(lambda s: 1 if 3 <= s <= 4 else 0)
 df_test['LargeF'] = df_test['Family'].map(lambda s: 1 if s >= 5 else 0)
 df_test['Senior'] = df_test['Age'].map(lambda s:1 if s>60 else 0 )<categorify>
USER_TRIES = 70
VALID_TRIES = math.ceil(USER_TRIES / 10 )
df_train['Person'] = df_train[['Age','Sex']].apply(get_person,axis=1)
 df_test['Person'] = df_test[['Age','Sex']].apply(get_person,axis=1)
 person_dummies_train = pd.get_dummies(df_train['Person'])
 person_dummies_train.columns = ['Child','Female','Male']
person_dummies_train.drop(['Male'], axis=1, inplace=True)
 person_dummies_test = pd.get_dummies(df_test['Person'])
 person_dummies_test.columns = ['Child','Female','Male']
person_dummies_test.drop(['Male'], axis=1, inplace=True)
 df_train = df_train.join(person_dummies_train)
 df_test = df_test.join(person_dummies_test)
 df_train.drop(['Person'],axis=1,inplace=True)
 df_test.drop(['Person'],axis=1,inplace=True )<drop_column>
train_df = train_df.groupby('user_id' ).tail(USER_TRIES ).reset_index(drop=True )
df_train.drop('male',axis=1,inplace=True)
 df_test.drop('male',axis=1,inplace=True )<drop_column>
data_types_dict = {'question_id': 'int16', 'part': 'int8', 'bundle_id': 'int16', 'tags': 'string'}
questions_df = pd.read_csv(
 path/'riiid-test-answer-prediction/questions.csv',
usecols=data_types_dict.keys() ,
dtype=data_types_dict
)
df_train.drop(['Cabin','Ticket'],axis = 1, inplace= True)
 df_test.drop(['Ticket','Cabin'],axis = 1, inplace= True )<drop_column>
unique_tags_combos_keys = {v:i for i,v in enumerate(questions_df['tags'].unique())}
questions_df['tags_encoded'] = questions_df['tags'].apply(lambda x : unique_tags_combos_keys[x])
 questions_df['tags_encoded'] = pd.to_numeric(questions_df['tags_encoded'], downcast='integer')
 questions_df.info()
df_train.drop(['Sex','Embarked'],axis=1,inplace=True)
 df_test.drop(['Sex','Embarked'],axis=1,inplace=True )<split>
def extract_tag_factory(tag_pos):
def extract_tag(x):
if isinstance(x, str)and tag_pos < len(x.split()):
splits = x.split()
splits.sort()
return int(splits[tag_pos])
 else:
return 255
return extract_tag
for i in range(0, 3):
questions_df[f'tag_{i + 1}'] = questions_df['tags'].apply(extract_tag_factory(i))
questions_df[f'tag_{i + 1}'] = questions_df[f'tag_{i + 1}'].astype('uint8' )
X_train, X_test, y_train, y_test = train_test_split(df_train.drop('Survived',axis=1),
df_train['Survived'], test_size=0.15,
random_state=101 )<train_model>
train_df = pd.merge(train_df, questions_df, left_on='content_id', right_on='question_id', how='left')
 train_df.drop(columns=['question_id'], inplace=True )
dt = DecisionTreeClassifier()
dt.fit(X_train,y_train);
plt.figure(figsize=(18,18))
plot_tree(dt,filled=True);<train_model>
train_df['content_count'] = train_df['content_id'].map(content_agg['count'] ).astype('int32')
 train_df['content_id'] = train_df['content_id'].map(content_agg['sum'] / content_agg['count'] )
XGB = XGBClassifier(max_depth=4,learning_rate=0.005,n_estimators=500,n_jobs=-1,min_child_weight=2)
 XGB.fit(X_train,y_train )<save_to_csv>
train_df['prior_question_elapsed_time_mean'] = train_df['user_id'].map(prior_question_elapsed_time_agg['sum'] / prior_question_elapsed_time_agg['count'] )
y_pred = pd.DataFrame(XGB.predict(df_test))
y_pred['Survived'] = y_pred[0]
y_pred.drop(0,axis=1,inplace=True)
 y_pred['PassengerId'] = df_test['PassengerId']
y_pred_xgb = y_pred
y_pred.to_csv('titanic_pred_xgb.csv',index=False )<normalization>
train_df['residual_user_mean'] = train_df['user_id'].map(residual_agg['sum'] / user_agg['count'] )
Scaler1 = StandardScaler()
Scaler2 = StandardScaler()
X_train_scaled = Scaler1.fit_transform(X_train)
 df_test_scaled = Scaler2.fit_transform(df_test )<save_to_csv>
train_df['prior_question_elapsed_time'].fillna(train_df['prior_question_elapsed_time'].mean() , inplace=True)
 train_df['user_correctness'].fillna(train_df['user_correctness'].mean() , inplace=True )
logmodel = LogisticRegression(C=10 ).fit(X_train,y_train)
 y_pred = pd.DataFrame(logmodel.predict(df_test))
y_pred['Survived'] = y_pred[0]
y_pred.drop(0,axis=1,inplace=True)
 y_pred['PassengerId'] = df_test['PassengerId']
y_pred_lr = y_pred
y_pred.to_csv('titanic_pred_logistic.csv',index=False )<train_model>
for f in ['user_correctness', 'content_id']:
train_df[f] = pd.to_numeric(train_df[f], downcast='float' )
RFC = RandomForestClassifier(n_estimators=500,max_depth=9,min_samples_split=3)
 RFC.fit(X_train,y_train )<save_to_csv>
train_df['user_correctness'] = train_df['user_correctness'].replace(train_df['user_correctness'].mean() , 0.0)
 valid_df['user_correctness'] = valid_df['user_correctness'].replace(valid_df['user_correctness'].mean() , 0.0 )
y_pred = pd.DataFrame(RFC.predict(df_test))
y_pred['Survived'] = y_pred[0]
y_pred.drop(0,axis=1,inplace=True)
 y_pred['PassengerId'] = df_test['PassengerId']
y_pred_rf = y_pred
y_pred.to_csv('titanic_pred_rfc.csv',index=False )<train_model>
features = [
'prior_question_elapsed_time',
'prior_question_had_explanation',
'user_correctness',
'part',
'content_id',
'content_count',
'tags_encoded',
'tag_1',
'tag_2',
'prior_question_elapsed_time_mean',
'residual_user_mean'
]
lgb = LGBMClassifier(learning_rate=0.01,max_depth=5,n_estimators=500,num_leaves=3 ).fit(X_train,y_train )<save_to_csv>
%%time
model = sm.OLS(train_df[target], train_df[features])
 lin_model = model.fit()
roc_auc_score(valid_df[target], lin_model.predict(valid_df[features]))
y_pred = pd.DataFrame(lgb.predict(df_test))
y_pred['Survived'] = y_pred[0]
y_pred.drop(0,axis=1,inplace=True)
 y_pred['PassengerId'] = df_test['PassengerId']
y_pred_lgb = y_pred
y_pred.to_csv('titanic_pred_lgb.csv',index=False )<compute_test_metric>
lin_model.predict(valid_df[features].values[:10] )
print("XGB train score: ",round(XGB.score(X_train,y_train),2), " XGB test score: ",round(XGB.score(X_test,y_test),2))
print("Log-Reg.train score: ",round(logmodel.score(X_train,y_train),2)," Log-Reg.test score: ",round(logmodel.score(X_test,y_test),2))
print("Random Forest's train score: ",round(RFC.score(X_train,y_train),2), " Random Forest test score: ",round(RFC.score(X_test,y_test),2))
print("LGBM train score: ",round(lgb.score(X_train,y_train),2), " LGB Model test score: ",round(lgb.score(X_test,y_test),2))<predict_on_test>
EPOCHS=5
BATCH_SIZE=4096
y_valid_xgb = XGB.predict(X_test)
 y_valid_log = logmodel.predict(X_test)
 y_valid_rfc = RFC.predict(X_test)
 y_valid_lgb = lgb.predict(X_test )<compute_train_metric>
cat_features = ['tags_encoded', 'tag_1', 'tag_2']
fpr_xgb, tpr_xgb, thresholds_xgb = roc_curve(y_test, y_valid_xgb)
 roc_auc_xgb = auc(fpr_xgb, tpr_xgb)
 fpr_log, tpr_log, thresholds_log = roc_curve(y_test, y_valid_log)
 roc_auc_log = auc(fpr_log, tpr_log)
 fpr_rfc, tpr_rfc, thresholds_rfc = roc_curve(y_test, y_valid_rfc)
 roc_auc_rfc = auc(fpr_rfc, tpr_rfc)
 fpr_lgb, tpr_lgb, thresholds_lgb = roc_curve(y_test, y_valid_lgb)
 roc_auc_lgb = auc(fpr_lgb, tpr_lgb )<save_to_csv>
cont_features = [x for x in features if x not in cat_features]
y_pred_final = y_pred
y_pred_final['Survived'] = round(0.25 * y_pred_lgb['Survived']
+ 0.25 * y_pred_rf['Survived']
+ 0.25 * y_pred_xgb['Survived']
+ 0.25 * y_pred_lr['Survived'])
 y_pred_final['PassengerId'] = df_test['PassengerId']
y_pred_final['Survived'] = y_pred_final['Survived'].astype(int)
 y_pred_final.to_csv('titanic_pred_final.csv',index=False )<count_values>
fastai.__version__
y_pred_final['Survived'].value_counts()<import_modules>
train_df[target] = train_df[target].astype('float32' )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from collections import Counter<load_from_csv>
%%time
dls = TabularDataLoaders.from_df(train_df,
procs=[Categorify, FillMissing, Normalize],
cat_names=cat_features,
cont_names=cont_features,
y_names=target, valid_idx=valid_df.index, bs=BATCH_SIZE )
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 df_sample= pd.read_csv('.. /input/gender_submission.csv' )<define_variables>
def my_auc(inp, targ):
"Simple wrapper around scikit's roc_auc_score function for regression problems"
inp,targ = flatten_check(inp,targ)
 return roc_auc_score(targ.cpu().numpy() , inp.cpu().numpy() )
def detect_outliers(df,n,features):

 outlier_indices = []
for col in features:
Q1 = np.percentile(df[col], 25)
 Q3 = np.percentile(df[col],75)
 IQR = Q3 - Q1
outlier_step = 1.5 * IQR
outlier_list_col = df[(df[col] < Q1 - outlier_step)|(df[col] > Q3 + outlier_step)].index
outlier_indices.extend(outlier_list_col)
 outlier_indices = Counter(outlier_indices)
 multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)
 return multiple_outliers<drop_column>
def bce(inp,targ):
"Binary cross entropy"
inp,targ = flatten_check(inp,targ)
 loss = F.binary_cross_entropy(inp, targ)
 return loss
Outliers_to_drop = detect_outliers(df_train,2,["Age","SibSp","Parch","Fare"])
 df_train = df_train.drop(Outliers_to_drop, axis = 0 ).reset_index(drop=True )<categorify>
learn = tabular_learner(dls, layers=[200,100], metrics=my_auc )
dataset_title = [i.split(",")[1].split(".")[0].strip() for i in df_train["Name"]]
df_train["Title"] = pd.Series(dataset_title)
 df_train["Title"] = df_train["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 df_train["Title"] = df_train["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 df_train["Title"] = df_train["Title"].astype(int)
 df_train.drop(labels = ["Name"], axis = 1, inplace = True )<categorify>
learn.model
def impute_age(cols):
Age = cols[0]
Pclass = cols[1]
if pd.isnull(Age):
if Pclass == 1:
return 37
elif Pclass == 2:
return 29
else:
return 24
else:
return Age<categorify>
learn.model.layers.add_module('sigmoid', nn.Sigmoid())
 learn.loss_func = bce
def impute_fare(cols):
Fare = cols[0]
Pclass = cols[1]
if pd.isnull(Fare):
if Pclass == 1:
return 84
elif Pclass == 2:
return 20
else:
return 13
else:
return Fare<feature_engineering>
lr_find_res = learn.lr_find()
df_train['Age'] = df_train[['Age','Pclass']].apply(impute_age,axis=1 )<categorify>
%%time
learn.fit_one_cycle(3, lr=lr_find_res.lr_min )
sex = pd.get_dummies(df_train['Sex'],drop_first=True)
 embark = pd.get_dummies(df_train['Embarked'],drop_first=True)
 df_train = pd.concat([df_train,sex,embark],axis=1 )<feature_engineering>
def predict_batch(self, df):
dl = self.dls.test_dl(df)
 dl.dataset.conts = dl.dataset.conts.astype(np.float32)
 inp,preds,_,dec_preds = self.get_preds(dl=dl, with_input=True, with_decoded=True)
 return preds.numpy()
setattr(learn, 'predict_batch', predict_batch )
df_train["Family"] = df_train["SibSp"] + df_train["Parch"] + 1
df_train['Single'] = df_train['Family'].map(lambda s: 1 if s == 1 else 0)
 df_train['SmallF'] = df_train['Family'].map(lambda s: 1 if s == 2 else 0)
 df_train['MedF'] = df_train['Family'].map(lambda s: 1 if 3 <= s <= 4 else 0)
 df_train['LargeF'] = df_train['Family'].map(lambda s: 1 if s >= 5 else 0)
 df_train['Senior'] = df_train['Age'].map(lambda s:1 if s>60 else 0 )<categorify>
import lightgbm as lgb
dataset_title = [i.split(",")[1].split(".")[0].strip() for i in df_test["Name"]]
df_test["Title"] = pd.Series(dataset_title)
 df_test["Title"] = df_test["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 df_test["Title"] = df_test["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 df_test["Title"] = df_test["Title"].astype(int)
 df_test.drop(labels = ["Name"], axis = 1, inplace = True )<categorify>
cat_features = ['tags_encoded', 'tag_1', 'tag_2']
lgb_train = lgb.Dataset(train_df[features], train_df[target], categorical_feature = cat_features, free_raw_data=False)
 lgb_eval = lgb.Dataset(valid_df[features], valid_df[target], categorical_feature = cat_features, reference=lgb_train, free_raw_data=False )
df_test['Age'] = df_test[['Age','Pclass']].apply(impute_age,axis=1)
 sex = pd.get_dummies(df_test['Sex'],drop_first=True)
 embark = pd.get_dummies(df_test['Embarked'],drop_first=True)
 df_test = pd.concat([df_test,sex,embark],axis=1)
 df_test['Fare'].fillna(value=df_test['Fare'].median() ,inplace=True )<feature_engineering>
METRICS = ['auc']
params = {
'objective': 'binary',
'seed': 42,
'metric': METRICS,
'learning_rate': 0.05,
'max_bin': 800,
'num_leaves': 80
}
df_test['Fare'] = df_test[['Fare','Pclass']].apply(impute_fare,axis=1 )<feature_engineering>
%%time
NUM_BOOST_ROUNDS = 300
evals_result = {}
lgb_model = lgb.train(
 params,
lgb_train,
valid_sets=[lgb_train, lgb_eval],
verbose_eval=20,
num_boost_round=NUM_BOOST_ROUNDS,
early_stopping_rounds=20,
evals_result=evals_result
)
df_test["Fare"] = df_test["Fare"].map(lambda i: np.log(i)if i > 0 else 0 )<feature_engineering>
clf_tabnet = TabNetClassifier(cat_idxs=[list(train_df[features].columns ).index(x)for x in cat_features],
scheduler_params={"step_size": 50, "gamma": 0.9},
scheduler_fn=torch.optim.lr_scheduler.StepLR)
 clf_tabnet
df_test["Family"] = df_test["SibSp"] + df_test["Parch"] + 1<feature_engineering>
%%time
clf_tabnet.fit(
 X_train=train_df[features].values, y_train=train_df[target].values,
eval_set=[(valid_df[features].values, valid_df[target].values)],
max_epochs=2,
batch_size=8192 * 4
)
df_test['Single'] = df_test['Family'].map(lambda s: 1 if s == 1 else 0)
 df_test['SmallF'] = df_test['Family'].map(lambda s: 1 if s == 2 else 0)
 df_test['MedF'] = df_test['Family'].map(lambda s: 1 if 3 <= s <= 4 else 0)
 df_test['LargeF'] = df_test['Family'].map(lambda s: 1 if s >= 5 else 0)
 df_test['Senior'] = df_test['Age'].map(lambda s:1 if s>60 else 0 )<categorify>
preds = clf_tabnet.predict_proba(valid_df[features].values[:1000])
 preds[:,1].shape
df_train['Person'] = df_train[['Age','Sex']].apply(get_person,axis=1)
 df_test['Person'] = df_test[['Age','Sex']].apply(get_person,axis=1)
 person_dummies_train = pd.get_dummies(df_train['Person'])
 person_dummies_train.columns = ['Child','Female','Male']
person_dummies_train.drop(['Male'], axis=1, inplace=True)
 person_dummies_test = pd.get_dummies(df_test['Person'])
 person_dummies_test.columns = ['Child','Female','Male']
person_dummies_test.drop(['Male'], axis=1, inplace=True)
 df_train = df_train.join(person_dummies_train)
 df_test = df_test.join(person_dummies_test)
 df_train.drop(['Person'],axis=1,inplace=True)
 df_test.drop(['Person'],axis=1,inplace=True )<drop_column>
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int))
content_sum_dict = content_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
content_count_dict = content_agg['count'].astype('int32' ).to_dict(defaultdict(int))
residual_sum_dict = residual_agg['sum'].astype('float32' ).to_dict(defaultdict(int))
df_train.drop('male',axis=1,inplace=True)
 df_test.drop('male',axis=1,inplace=True )<drop_column>
prior_question_elapsed_time_sum_dict = prior_question_elapsed_time_agg['sum'].astype('int64' ).to_dict(defaultdict(int))
prior_question_elapsed_time_count_dict = prior_question_elapsed_time_agg['count'].astype('int32' ).to_dict(defaultdict(int))
df_train.drop(['Cabin','Ticket'],axis = 1, inplace= True)
 df_test.drop(['Ticket','Cabin'],axis = 1, inplace= True )<drop_column>
env = riiideducation.make_env()
iter_test = env.iter_test()
prior_test_df = None
df_train.drop(['Sex','Embarked'],axis=1,inplace=True)
 df_test.drop(['Sex','Embarked'],axis=1,inplace=True )<split>
def clip(count): return np.clip(count, 1e-8, np.inf )
X_train, X_test, y_train, y_test = train_test_split(df_train.drop('Survived',axis=1),
df_train['Survived'], test_size=0.15,
random_state=101 )<train_model>
for(test_df, sample_prediction_df)in iter_test:
if prior_test_df is not None:
prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)
 user_ids = prior_test_df['user_id'].values
content_ids = prior_test_df['content_id'].values
prior_question_elapsed_times = prior_test_df['prior_question_elapsed_time'].values
targets = prior_test_df[target].values
for user_id, content_id, prior_question_elapsed_time, answered_correctly in zip(user_ids, content_ids, prior_question_elapsed_times, targets):
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
content_sum_dict[content_id] += answered_correctly
content_count_dict[content_id] += 1
mean_accuracy = content_sum_dict[content_id] / clip(content_count_dict[content_id])
 residual_sum_dict[user_id] += answered_correctly - mean_accuracy
prior_question_elapsed_time_sum_dict[user_id] += 0 if np.isnan(prior_question_elapsed_time)else prior_question_elapsed_time
prior_question_elapsed_time_count_dict[user_id] += 0 if np.isnan(prior_question_elapsed_time)else 1
prior_test_df = test_df.copy()
test_df = pd.merge(test_df, questions_df, left_on='content_id', right_on='question_id', how='left')
 test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(False ).astype('uint8')
 user_sum = np.zeros(len(test_df), dtype=np.int16)
 user_count = np.zeros(len(test_df), dtype=np.int16)
 res_sum = np.zeros(len(test_df), dtype=np.float32)
 content_sum = np.zeros(len(test_df), dtype=np.int32)
 content_count = np.zeros(len(test_df), dtype=np.int32)
 prior_question_elapsed_time_sum = np.zeros(len(test_df), dtype=np.int32)
 prior_question_elapsed_time_count = np.zeros(len(test_df), dtype=np.int32)
 for i,(user_id, content_id)in enumerate(zip(test_df['user_id'].values, test_df['content_id'].values)) :
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
res_sum[i] = residual_sum_dict[user_id]
content_sum[i] = content_sum_dict[content_id]
content_count[i] = content_count_dict[content_id]
prior_question_elapsed_time_sum[i] = prior_question_elapsed_time_sum_dict[user_id]
prior_question_elapsed_time_count[i] = prior_question_elapsed_time_count_dict[user_id]
content_count = clip(content_count)
 user_count = clip(user_count)
 prior_question_elapsed_time_count = clip(prior_question_elapsed_time_count)
 test_df['user_correctness'] = user_sum / user_count
test_df['residual_user_mean'] = res_sum / user_count
test_df['content_count'] = content_count
test_df['content_id'] = content_sum / content_count
test_df['prior_question_elapsed_time_mean'] = prior_question_elapsed_time_sum / prior_question_elapsed_time_count
test_df['prior_question_elapsed_time'].fillna(train_df['prior_question_elapsed_time'].mean() , inplace=True)
 test_df[cat_features] = test_df[cat_features].apply(pd.to_numeric, downcast='integer')
 test_df.fillna(0, inplace=True)
 test_df[target] = np.average([
clf_tabnet.predict_proba(test_df[features].values)[:,1],
lin_model.predict(test_df[features]),
learn.predict_batch(learn, test_df[features])[:,0],
lgb_model.predict(test_df[features])
 ], weights=[0.25, 0.2, 0.25, 0.3], axis=0)
 env.predict(test_df[['row_id', target]] )
dt = DecisionTreeClassifier()
dt.fit(X_train,y_train);
plt.figure(figsize=(18,18))
plot_tree(dt,filled=True);<train_model>
test_df[target] = np.average([
clf_tabnet.predict_proba(test_df[features].values)[:,1],
lin_model.predict(test_df[features]),
learn.predict_batch(learn, test_df[features])[:,0],
lgb_model.predict(test_df[features])
 ], weights=[0.25, 0.2, 0.25, 0.3], axis=0 )
XGB = XGBClassifier(max_depth=4,learning_rate=0.005,n_estimators=500,n_jobs=-1,min_child_weight=2)
 XGB.fit(X_train,y_train )<save_to_csv>
lgb_model.predict(test_df[features] )