Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<merge>
warnings.filterwarnings('ignore')
 %matplotlib inline
df_merged = merge_train_test(train_df, test_df)
 df_merged.tail()<categorify>
training_set = pd.read_csv('.. /input/train.csv')
 testing_set = pd.read_csv('.. /input/test.csv')
 pID = testing_set['PassengerId']
%%time
count_enc = [None] * 200
df_real = df_merged[df_merged["target"]!=-2]
for v in range(200):
enc = CountEncoder()
enc.fit(df_real[f"var_{v}"])
 count_enc[v] = enc.transform(df_merged[f"var_{v}"])
 for v in range(200):
df_merged[f"cnt_{v}"] = count_enc[v]
del df_real<split>
print(training_set.isnull().sum() ,"
")
 print(testing_set.isnull().sum() )
train_df, test_df = split_train_test(df_merged)
 target = train_df['target']
gc.collect()
print(train_df.shape)
 test_df.head()<init_hyperparams>
for dataset in [training_set,testing_set]:
dataset['Age'].fillna(dataset['Age'].median() , inplace = True)
 dataset['Embarked'].fillna(dataset['Embarked'].mode() [0], inplace = True)
 dataset['Fare'].fillna(dataset['Fare'].median() , inplace = True)
 drop_column = ['PassengerId','Cabin', 'Ticket']
training_set.drop(drop_column, axis=1, inplace = True)
 testing_set.drop(drop_column, axis=1, inplace = True)
 print(training_set.isnull().sum())
 print("-"*10)
 print(testing_set.isnull().sum() )
param = {
"objective": "binary",
"boost": "gbdt",
"metric": "auc",
"boost_from_average": False,
"learning_rate": 0.01,
"num_leaves": 5,
"max_depth": -1,
"tree_learner": "serial",
"feature_fraction": 1.0,
"bagging_freq": 5,
"bagging_fraction": 0.4,
"min_data_in_leaf": 80,
"min_sum_hessian_in_leaf": 10.0,
"verbosity": 1,
"seed": 44000,
}<define_variables>
for dataset in [training_set,testing_set]:
dataset['FamilySize'] = dataset ['SibSp'] + dataset['Parch'] + 1
dataset['IsAlone'] = 1
dataset['IsAlone'].loc[dataset['FamilySize'] > 1] = 0
dataset['FareBin'] = pd.qcut(dataset['Fare'], 4)
 dataset['AgeBin'] = pd.cut(dataset['Age'].astype(int), 5)
 training_set.info()
testing_set.info()
training_set.sample(10 )
target = train_df['target']
df_merged_cut = [df_merged[[f"var_{v}",
f"cnt_{v}",
]] for v in range(200)]
gc.collect()<split>
training_set[training_set["Name"].str.contains("Master")]
%%time
skf = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED_SKF)
 oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(skf.split(train_df.values, target.values)) :
print("fold n°{}".format(fold_))
df_meta = df_merged[["ID_code", "target"]]
trn_X, trn_y = train_df.iloc[trn_idx], target.iloc[trn_idx]
val_X, val_y = train_df.iloc[val_idx], target.iloc[val_idx]
for v in range(200):
print(f"var {v}")
 features = [f"var_{v}",
f"cnt_{v}",
]
trn_data = lgb.Dataset(trn_X[features], label=trn_y)
 val_data = lgb.Dataset(val_X[features], label=val_y)
 num_round = 1000000
clf = lgb.train(param,
trn_data,
num_round,
valid_sets=[trn_data, val_data],
verbose_eval=1000,
early_stopping_rounds=100)
 df_meta[f"{v}_meta"] = clf.predict(df_merged_cut[v], num_iteration=clf.best_iteration ).astype(np.float32)
 df_meta.to_pickle(f"fold_{fold_}_meta.pickle" )<init_hyperparams>
label = LabelEncoder()
for dataset in [training_set,testing_set]:
dataset['Sex_Code'] = label.fit_transform(dataset['Sex'])
 dataset['Embarked_Code'] = label.fit_transform(dataset['Embarked'])
 dataset['AgeBin_Code'] = label.fit_transform(dataset['AgeBin'])
 dataset['FareBin_Code'] = label.fit_transform(dataset['FareBin'])
 Target = ['Survived']
training_set_x = ['Sex','Pclass', 'Embarked','SibSp', 'Parch', 'Age', 'Fare', 'FamilySize', 'IsAlone']
training_set_x_calc = ['Sex_Code','Pclass', 'Embarked_Code','SibSp', 'Parch', 'Age', 'Fare']
training_set_xy = Target + training_set_x
print('Original X Y: ', training_set_xy, '
')
 training_set_x_bin = ['Sex_Code','Pclass', 'Embarked_Code', 'FamilySize', 'AgeBin_Code', 'FareBin_Code']
training_set_xy_bin = Target + training_set_x_bin
print('Bin X Y: ', training_set_xy_bin, '
')
 training_set_dummy = pd.get_dummies(training_set[training_set_x],drop_first=True)
 training_set_x_dummy = training_set_dummy.columns.tolist()
training_set_xy_dummy = Target + training_set_x_dummy
print('Dummy X Y: ', training_set_xy_dummy, '
')
 training_set_dummy.head()
param = {
"objective": "binary",
"boost": "gbdt",
"metric": "auc",
"boost_from_average": "false",
"learning_rate": 0.01,
"num_leaves": 2,
"max_depth": -1,
"tree_learner": "serial",
"feature_fraction": 0.5,
"bagging_freq": 5,
"bagging_fraction": 0.4,
"min_data_in_leaf": 80,
"min_sum_hessian_in_leaf": 10.0,
"verbosity": 1,
"seed": 44000,
}<split>
y = training_set['Survived']
X = training_set_dummy
%%time
skf = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED_SKF)
 oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(skf.split(train_df.values, target.values)) :
print("fold n°{}".format(fold_))
df_meta = pd.read_pickle(f"fold_{fold_}_meta.pickle")
 train_df, test_df = split_train_test(df_meta)
 features = [f"{v}_meta" for v in range(200)]
trn_data = lgb.Dataset(train_df.iloc[trn_idx][features], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(train_df.iloc[val_idx][features], label=target.iloc[val_idx])
 num_round = 1000000
clf = lgb.train(param,
trn_data,
num_round,
valid_sets=[trn_data, val_data],
verbose_eval=1000,
early_stopping_rounds=2000)
 oof[val_idx] = clf.predict(train_df.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test_df[features], num_iteration=clf.best_iteration)/ N_SPLITS
print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)) )<save_to_csv>
testing_set_dummy = pd.get_dummies(testing_set[training_set_x],drop_first=True )
sub_df = pd.DataFrame({"ID_code":test_df["ID_code"].values})
 sub_df["target"] = predictions
sub_df.to_csv("submission.csv", index=False )<save_to_csv>
ss = MinMaxScaler()
training_set_dummy_ss= ss.fit_transform(training_set_dummy)
 testing_set_dummy_ss= ss.fit_transform(testing_set_dummy )
feature_importance_df.to_csv("feature_importance_df.csv", index=False )<set_options>
classifiers = {'Gradient Boosting Classifier':GradientBoostingClassifier() ,'Adaptive Boosting Classifier':AdaBoostClassifier() ,'RadiusNN':RadiusNeighborsClassifier(radius=40.0),
'Linear Discriminant Analysis':LinearDiscriminantAnalysis() , 'GaussianNB': GaussianNB() , 'BerNB': BernoulliNB() , 'KNN': KNeighborsClassifier() ,
'Random Forest Classifier': RandomForestClassifier(min_samples_leaf=10,min_samples_split=20,max_depth=4),'Decision Tree Classifier': DecisionTreeClassifier() ,'Logistic Regression':LogisticRegression() , "XGBoost": xgb.XGBClassifier() }
warnings.filterwarnings('ignore' )<set_options>
X_training, X_validating, y_training, y_validating = train_test_split(training_set_dummy, y, test_size=0.20, random_state=11 )
def get_logger() :
FORMAT = '[%(levelname)s]%(asctime)s:%(name)s:%(message)s'
logging.basicConfig(format=FORMAT)
 logger = logging.getLogger('main')
 logger.setLevel(logging.DEBUG)
 return logger
logger = get_logger()<load_from_csv>
base_accuracy = 0
for Name,classify in classifiers.items() :
classify.fit(X_training,y_training)
 y_predictng = classify.predict(X_validating)
 print('Accuracy Score of '+str(Name)+ " : " +str(met.accuracy_score(y_validating,y_predictng)))
 if met.accuracy_score(y_validating,y_predictng)> base_accuracy:
predictions_test = classify.predict(testing_set_dummy)
 base_accuracy = met.accuracy_score(y_validating,y_predictng)
 else:
continue
predicted_test_value = pd.DataFrame({ 'PassengerId': pID,
'Survived': predictions_test })
 predicted_test_value.to_csv("PredictedTestScore.csv", index=False )
def read_data(nrows=None):
logger.info('Input data')
 train_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/train.csv',nrows=nrows)
 test_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/test.csv')
 return train_df, test_df<load_pretrained>
cbr = xgb.XGBClassifier()
cbr.fit(X_training,y_training)
 predictions_train = cbr.predict(X_validating)
 print(met.accuracy_score(y_validating,predictions_train))
def process_data(train_df, test_df):
logger.info('Features engineering')
 synthetic = np.load('.. /input/publicprivate/synthetic_samples_indexes.npy')
 synthetic = synthetic-200000
synthetic = np.array(synthetic)
 test_df = test_df.iloc[~test_df.index.isin(synthetic)]
idx = [c for c in train_df.columns if c not in ['ID_code', 'target']]
traintest = pd.concat([train_df, test_df])
 traintest = traintest.reset_index(drop=True)
 for col in idx:
varname = col + '_IsUnique'
traintest[varname] = 0
_, index_, count_ = np.unique(traintest.loc[:,col].values, return_counts=True, return_index=True)
 traintest[varname][index_[count_ == 1]] += 1
traintest[varname] = traintest[varname] /(traintest[varname] == 1 ).sum()
for col in idx:
traintest[col+'_freq'] = traintest[col].map(traintest.groupby(col ).size())
 for col in idx:
varname = col + '_IsUnique'
tmp_col = traintest.loc[traintest[varname] > 0][col]
traintest[col + '_OnlyUnique'] = tmp_col
traintest[col + '_OnlyUnique'] = traintest[col + '_OnlyUnique'].fillna(0)
 traintest[col + '_NotUnique'] = traintest[col] - traintest[col + '_OnlyUnique']
traintest[col + '_NotUnique'] = traintest[col + '_NotUnique'].replace(0,np.nan)
 traintest[col + '_OnlyUnique'] = traintest[col + '_OnlyUnique'].replace(0,np.nan)
 traintest.pop(varname)
 train_df = traintest[:200000]
test_df = traintest[200000:]
print('Train and test shape:',train_df.shape, test_df.shape)
 return train_df, test_df 
<train_model>
clf1 = GradientBoostingClassifier()
clf3 = LinearDiscriminantAnalysis()
clf4 = LogisticRegression()
clf5 = xgb.XGBClassifier()
exTreeClf = VotingClassifier(estimators=[('svc', clf1),('gbc', clf3),('lr',clf4),('lda',clf5)])
 exTreeClf.fit(X_training,y_training)
def run_model(train_df, test_df):
logger.info('Prepare the model')
 features = [c for c in train_df.columns if c not in ['ID_code', 'target']]
target = train_df['target']
logger.info('Run model')
 param = {
'bagging_freq': 5,
'bagging_fraction': 0.38,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.045,
'learning_rate': 0.0095,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 20,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 3,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1
}
num_round = 1000000
folds = StratifiedKFold(n_splits=5, shuffle=False, random_state=42)
 oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_df.values, target.values)) :
print("Fold {}".format(fold_))
trn_data = lgb.Dataset(train_df.iloc[trn_idx][features], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(train_df.iloc[val_idx][features], label=target.iloc[val_idx])
 clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 3500)
 oof[val_idx] = clf.predict(train_df.iloc[val_idx][features], num_iteration=clf.best_iteration)
 predictions += clf.predict(test_df[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)))
 return predictions<save_to_csv>
def submit(test_df, predictions):
logger.info('Prepare submission')
 all_test_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/test.csv')
 sub = pd.DataFrame({"ID_code": all_test_df.ID_code.values})
 sub["target"] = 0
sub_real = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub_real["target"] = predictions
sub = sub.set_index('ID_code')
 sub_real = sub_real.set_index('ID_code')
 sub.update(sub_real)
 sub = sub.reset_index()
sub.to_csv("submission.csv", index=False )<split>
def main(nrows=None):
train_df, test_df = read_data(nrows)
 train_df, test_df = process_data(train_df, test_df)
 predictions = run_model(train_df, test_df)
 submit(test_df, predictions )<define_variables>
predicted_test = []
for x in exTreeClf.predict(testing_set_dummy):
predicted_test.append(x)
 predicted_test_value = pd.DataFrame({ 'PassengerId': pID,
'Survived': predicted_test })
 predicted_test_value.to_csv("PredictedTestScore.csv", index=False )
<load_from_csv><EOS>