Kaggle competition: Titanic - Machine Learning from Disaster
path_index = np.argsort(path[:-1])
 total_score = initial_score
print(f'Total score is {total_score:.2f}.')
 for _ in range(3):
for step,(id1, id2)in enumerate(tqdm(pairs), 1):
if step % 10**6 == 0:
new_total_score = score_path(path)
 print(f'Score: {new_total_score:.2f}; improvement over last 10^6 steps: {total_score - new_total_score:.2f}; total improvement: {initial_score - new_total_score:.2f}.')
 total_score = new_total_score
i, j = path_index[id1], path_index[id2]
i, j = min(i, j), max(i, j)
 chunk, reversed_chunk = path[i-1:j+2], np.concatenate([path[i-1:i], path[j:i-1:-1], path[j+1:j+2]])
 chunk_score, reversed_chunk_score = score_chunk(i-1, chunk), score_chunk(i-1, reversed_chunk)
 if j - i > 2:
chunk_abc = np.concatenate([path[i-1:i+1], path[j:i:-1], path[j+1:j+2]])
 chunk_acb = np.concatenate([path[i-1:i], path[j:j+1], path[i:j], path[j+1:j+2]])
 chunk_abcb = np.concatenate([path[i-1:i+1], path[j:j+1], path[i+1:j], path[j+1:j+2]])
 abc_score, acb_score, abcb_score = map(lambda chunk: score_chunk(i-1, chunk), [chunk_abc, chunk_acb, chunk_abcb])
 for chunk, score, name in zip(( chunk_abc, chunk_acb, chunk_abcb),(abc_score, acb_score, abcb_score),('abc', 'acb', 'abcb')) :
if score < chunk_score:
path[i-1:j+2] = chunk
path_index = np.argsort(path[:-1])
 chunk_score = score
if reversed_chunk_score < chunk_score:
path[i-1:j+2] = reversed_chunk
path_index = np.argsort(path[:-1] )<compute_test_metric>
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 train_data.head()
print(f'Total improvement is {initial_score - total_score:.2f}.' )<save_to_csv>
print("
print("
print()
print("Train Features = ", train_data.columns.values)
 print("Test Features = ", test_data.columns.values)
 print()
print("NaNs in each training Feature")
 print(train_data.isnull().sum())
 print()
print("NaNs in each testing Feature")
 print(test_data.isnull().sum() )
def make_submission(name, path):
pd.DataFrame({'Path': path} ).to_csv(f'{name}.csv', index=False )<compute_test_metric>
train_data.drop(columns='Cabin', inplace=True)
 train_data.drop(columns='Ticket', inplace=True )
make_submission(score_path(path), path )<load_from_csv>
def detect_outliers(df,n,features):
outlier_indices = []
for col in features:
Q1 = np.percentile(df[col], 25)
 Q3 = np.percentile(df[col],75)
 IQR = Q3 - Q1
outlier_step = 1.5 * IQR
outlier_list_col = df[(df[col] < Q1 - outlier_step)|(df[col] > Q3 + outlier_step)].index
outlier_indices.extend(outlier_list_col)
 outlier_indices = Counter(outlier_indices)
 multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)
 return multiple_outliers
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv(".. /input/test.csv")
 base_features = [x for x in train_df.columns.values.tolist() if x.startswith('var_')]<feature_engineering>
train_data['Embarked'].fillna('S', inplace=True )
train_df['real'] = 1
for col in base_features:
test_df[col] = test_df[col].map(test_df[col].value_counts())
 a = test_df[base_features].min(axis=1)
 test_df = pd.read_csv('.. /input/test.csv')
 test_df['real'] =(a == 1 ).astype('int')
 train = train_df.append(test_df ).reset_index(drop=True)
 del test_df, train_df; gc.collect()<count_values>
meanAges = train_data.groupby(['Pclass'])['Age'].mean()
print(meanAges )
for col in tqdm(base_features):
train[col + 'size'] = train[col].map(train.loc[train.real==1, col].value_counts())
 cnt_features = [col + 'size' for col in base_features]<feature_engineering>
train_data['newGender'] = train_data['Sex']
train_data.loc[train_data['Age']<16., 'newGender'] = 'child'
train_data.drop(columns='Sex', inplace=True)
 print(train_data.head(8))
for col in tqdm(base_features):
train.loc[train[col+'size']>1,col+'no_noise'] = train.loc[train[col+'size']>1,col]
noise1_features = [col + 'no_noise' for col in base_features]<data_type_conversions>
train_data['Title'] = train_data['Name']
for name_string in train_data['Name']:
train_data['Title'] = train_data['Name'].str.extract('([A-Za-z]+)\.', expand=True)
 mapping = {'Mlle': 'Miss', 'Major': 'Rare', 'Col': 'Rare', 'Sir': 'Rare', 'Don': 'Rare', 'Mme': 'Miss',
'Jonkheer': 'Rare', 'Lady': 'Rare', 'Capt': 'Rare', 'Countess': 'Rare', 'Ms': 'Miss', 'Dona': 'Mrs', 'Rev': 'Rare'}
train_data.replace({'Title': mapping}, inplace=True)
 train_data.drop(columns='Name', inplace=True )
train[noise1_features] = train[noise1_features].fillna(train[noise1_features].mean() )<feature_engineering>
train_data['familyNumber'] = train_data['SibSp'] + train_data['Parch']
print(train_data.head(8))
Outliers_to_drop = detect_outliers(train_data,1,["Age","familyNumber","Fare"])
 train_data = train_data.drop(Outliers_to_drop, axis = 0 ).reset_index(drop=True)
 train_data['familySize'] = np.nan
train_data.loc[train_data['familyNumber']==0, 'familySize'] = 'alone'
train_data.loc[train_data['familyNumber']==1, 'familySize'] = 'couple'
train_data.loc[train_data['familyNumber']==2, 'familySize'] = 'small'
train_data.loc[train_data['familyNumber']==3, 'familySize'] = 'medium'
train_data.loc[train_data['familyNumber']>=4, 'familySize'] = 'large'
train_data.drop(columns=['familyNumber', 'SibSp', 'Parch'], inplace=True )
for col in tqdm(base_features):
train.loc[train[col+'size']>2,col+'no_noise2'] = train.loc[train[col+'size']>2,col]
noise2_features = [col + 'no_noise2' for col in base_features]<data_type_conversions>
train_data['newPclass'] = np.nan
train_data.loc[train_data['Pclass']==1, 'newPclass'] = 'first'
train_data.loc[train_data['Pclass']==2, 'newPclass'] = 'second'
train_data.loc[train_data['Pclass']==3, 'newPclass'] = 'third'
train_data.drop(columns='Pclass', inplace=True)
 columns = ['newGender', 'Embarked', 'Title', 'familySize', 'newPclass']
one_hot = pd.get_dummies(train_data.loc[:, columns], drop_first=True)
 train_data.drop(columns=columns, inplace=True)
 train_data = train_data.join(one_hot)
 print(train_data.head(2))
print(train_data.columns.values )
train[noise2_features] = train[noise2_features].fillna(train[noise2_features].mean() )<split>
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.model_selection import GridSearchCV
from sklearn.preprocessing import StandardScaler
from sklearn.neighbors import KNeighborsClassifier
train_df = train[train['target'].notnull() ]
test_df = train[train['target'].isnull() ]
all_features = base_features + noise1_features + noise2_features<prepare_x_and_y>
trainScaler = StandardScaler()
trainScaler.fit(train_data.drop(['Survived', 'PassengerId'], axis = 1))
X_train_scaled = trainScaler.transform(train_data.drop(['Survived', 'PassengerId'], axis = 1))
X_train = train_data.drop(['Survived', 'PassengerId'], axis = 1)
 y_train = train_data['Survived']
scaler = preprocessing.StandardScaler().fit(train_df[all_features].values)
 df_trn = pd.DataFrame(scaler.transform(train_df[all_features].values), columns=all_features)
 df_tst = pd.DataFrame(scaler.transform(test_df[all_features].values), columns=all_features)
 y = train_df['target'].values<prepare_x_and_y>
grid_params = {
'n_neighbors': [3, 5, 7, 9, 11, 13],
'weights': ['uniform', 'distance'],
'metric':['euclidean', 'manhattan']
}
KNN_CV = GridSearchCV(estimator = KNeighborsClassifier() , param_grid=grid_params, cv = 3)
 KNN_CV.fit(X_train_scaled, y_train)
 print(KNN_CV.best_score_)
 weights = KNN_CV.best_params_.get('weights')
 n = KNN_CV.best_params_.get('n_neighbors')
 metric = KNN_CV.best_params_.get('metric')
 bestKNN = KNeighborsClassifier(n_neighbors=n, weights=weights, metric=metric)
 bestKNN.fit(X_train_scaled, y_train )
def get_keras_data(dataset, cols_info):
X = {}
base_feats, noise_feats, noise2_feats = cols_info
X['base'] = np.reshape(np.array(dataset[base_feats].values),(-1, len(base_feats), 1))
X['noise1'] = np.reshape(np.array(dataset[noise_feats].values),(-1, len(noise_feats), 1))
X['noise2'] = np.reshape(np.array(dataset[noise2_feats].values),(-1, len(noise2_feats), 1))
return X<prepare_x_and_y>
param_grid = {
'criterion' : ['gini'],
'n_estimators': [70, 80, 90, 100, 110, 120],
'max_features': ['auto', 'log2'],
'max_depth' : [5, 7, 9, 11, 13]
}
randomForest_CV = GridSearchCV(estimator = RandomForestClassifier() , param_grid = param_grid, cv = 3)
 randomForest_CV.fit(X_train, y_train)
 print(randomForest_CV.best_score_)
 criterion = randomForest_CV.best_params_.get('criterion')
 nRF = randomForest_CV.best_params_.get('n_estimators')
 max_features = randomForest_CV.best_params_.get('max_features')
 max_depth = randomForest_CV.best_params_.get('max_depth')
 bestRF = RandomForestClassifier(n_estimators=1000, max_depth=max_depth, max_features=max_features, criterion=criterion)
 bestRF.fit(X_train, y_train )
cols_info = [base_features, noise1_features, noise2_features]
X_test = get_keras_data(df_tst[all_features], cols_info )<choose_model_class>
for i in range(3):
test_data.loc[test_data['Pclass']==int(i+1), 'Age'] = test_data.loc[test_data['Pclass']==int(i+1), 'Age'].fillna(meanAges.values[i])
 meanFare = test_data['Fare'].median()
test_data['Fare'].fillna(meanFare, inplace=True )
def Convnet(cols_info, classes=1):
base_feats, noise1_feats, noise2_feats = cols_info
X_base_input = Input(shape=(len(base_feats), 1), name='base')
 X_base = Dense(16 )(X_base_input)
 X_base = Activation('relu' )(X_base)
 X_base = Flatten(name='base_last' )(X_base)
 X_noise1_input = Input(shape=(len(noise1_feats), 1), name='noise1')
 X_noise1 = Dense(16 )(X_noise1_input)
 X_noise1 = Activation('relu' )(X_noise1)
 X_noise1 = Flatten(name='nose1_last' )(X_noise1)
 X_noise2_input = Input(shape=(len(noise2_feats), 1), name='noise2')
 X_noise2 = Dense(16 )(X_noise2_input)
 X_noise2 = Activation('relu' )(X_noise2)
 X_noise2 = Flatten(name='nose2_last' )(X_noise2)
 X = concatenate([X_base, X_noise1, X_noise2])
 X = Dense(classes, activation='sigmoid' )(X)
 model = Model(inputs=[X_base_input, X_noise1_input, X_noise2_input],outputs=X)
 return model
model = Convnet(cols_info)
 model.summary()<drop_column>
test_data['newGender'] = test_data['Sex']
test_data.loc[test_data['Age']<18., 'newGender'] = 'child'
test_data.drop(columns='Sex', inplace=True)
 test_data['Title'] = test_data['Name']
for name_string in test_data['Name']:
test_data['Title'] = test_data['Name'].str.extract('([A-Za-z]+)\.', expand=True)
 mapping = {'Mlle': 'Miss', 'Major': 'Rare', 'Col': 'Rare', 'Sir': 'Rare', 'Don': 'Rare', 'Mme': 'Miss',
'Jonkheer': 'Rare', 'Lady': 'Rare', 'Capt': 'Rare', 'Countess': 'Rare', 'Ms': 'Miss', 'Dona': 'Mrs', 'Rev': 'Rare'}
test_data.replace({'Title': mapping}, inplace=True)
 test_data.drop(columns='Name', inplace=True)
 test_data['familyNumber'] = test_data['SibSp'] + test_data['Parch']
test_data['familySize'] = np.nan
test_data.loc[test_data['familyNumber']==0, 'familySize'] = 'alone'
test_data.loc[test_data['familyNumber']==1, 'familySize'] = 'couple'
test_data.loc[test_data['familyNumber']==2, 'familySize'] = 'small'
test_data.loc[test_data['familyNumber']==3, 'familySize'] = 'medium'
test_data.loc[test_data['familyNumber']>=4, 'familySize'] = 'large'
test_data.drop(columns=['familyNumber', 'SibSp', 'Parch'], inplace=True)
 test_data['newPclass'] = np.nan
test_data.loc[test_data['Pclass']==1, 'newPclass'] = 'first'
test_data.loc[test_data['Pclass']==2, 'newPclass'] = 'second'
test_data.loc[test_data['Pclass']==3, 'newPclass'] = 'third'
test_data.drop(columns='Pclass', inplace=True )
try:
del df_tst
except:
pass
gc.collect()<choose_model_class>
test_data.drop(columns=['Cabin', 'Ticket'], inplace=True)
 one_hot = pd.get_dummies(test_data.loc[:, columns], drop_first=True)
 test_data.drop(columns=columns, inplace=True)
 test_data = test_data.join(one_hot)
 print("NaNs in each testing Feature")
 print(test_data.isnull().sum() )
SEED = 2019
n_folds = 5
debug_flag = True
folds = 5
skf = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=SEED )<split>
X_test_scaled = trainScaler.transform(test_data.drop(['PassengerId'], axis = 1))
X_test = test_data.drop(columns='PassengerId')
 testPredictionsKNN = bestKNN.predict(X_test_scaled)
 testPredictionsRF = bestRF.predict(X_test)
 outputKNN = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': testPredictionsKNN})
 outputKNN.to_csv('my_submissionKNN.csv', index=False)
 outputRF = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': testPredictionsRF})
 outputRF.to_csv('my_submissionRF.csv', index=False )
i = 0
result = pd.DataFrame({"ID_code": test_df.ID_code.values})
 val_aucs = []
valid_X = train_df[['target']]
valid_X['predict'] = 0
for train_idx, val_idx in skf.split(df_trn, y):
if i == folds:
break
i += 1
X_train, y_train = df_trn.iloc[train_idx], y[train_idx]
X_valid, y_valid = df_trn.iloc[val_idx], y[val_idx]
X_train = get_keras_data(X_train, cols_info)
 X_valid = get_keras_data(X_valid, cols_info)
 model_name = 'NN_fold{}.h5'.format(str(i))
model = Convnet(cols_info)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy', 'binary_crossentropy', auc_2])
 checkpoint = ModelCheckpoint(model_name, monitor='val_auc_2', verbose=1,
save_best_only=True, mode='max', save_weights_only = True)
 reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=4,
verbose=1, mode='min', epsilon=0.0001)
 earlystop = EarlyStopping(monitor='val_auc_2', mode='max', patience=10, verbose=1)
 history = model.fit(X_train, y_train,
epochs=300,
batch_size=1024 * 2,
validation_data=(X_valid, y_valid),
callbacks=[checkpoint, reduceLROnPlat, earlystop])
 train_history = pd.DataFrame(history.history)
 train_history.to_csv('train_profile_fold{}.csv'.format(str(i)) , index=None)
 model.load_weights(model_name)
 y_pred_keras = model.predict(X_valid ).ravel()
valid_X['predict'].iloc[val_idx] = y_pred_keras
fpr_keras, tpr_keras, thresholds_keras = roc_curve(y_valid, y_pred_keras)
 auc_valid = roc_auc_score(y_valid, y_pred_keras)
 val_aucs.append(auc_valid)
 prediction = model.predict(X_test)
 result["fold{}".format(str(i)) ] = prediction<compute_test_metric>
param_grid = {
'learning_rate' : [0.1, 0.2],
'n_estimators': [90, 100, 110],
'loss': ['deviance', 'exponential']
}
gradBoost_CV = GridSearchCV(estimator = GradientBoostingClassifier() , param_grid = param_grid, cv = 3)
 gradBoost_CV.fit(X_train, y_train)
 print(gradBoost_CV.best_score_)
 learning = gradBoost_CV.best_params_.get('learning_rate')
 nGB = gradBoost_CV.best_params_.get('n_estimators')
 loss = gradBoost_CV.best_params_.get('loss')
 bestGB = GradientBoostingClassifier(learning_rate=learning, n_estimators=nGB, loss=loss)
 bestGB.fit(X_train, y_train)
 testPredictionsGB = bestGB.predict(X_test)
 outputGB = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': testPredictionsGB})
 outputGB.to_csv('my_submissionGB.csv', index=False )
for i in range(len(val_aucs)) :
print('Fold_%d AUC: %.6f' %(i+1, val_aucs[i]))<compute_test_metric>
param_grid = {
'learning_rate' : [0.1, 0.2],
'max_depth': [3, 5, 7],
}
xgBoost_CV = GridSearchCV(estimator = XGBClassifier() , param_grid = param_grid, cv = 3)
 xgBoost_CV.fit(X_train, y_train)
 print(xgBoost_CV.best_score_)
 learning = xgBoost_CV.best_params_.get('learning_rate')
 depth = xgBoost_CV.best_params_.get('max_depth')
 bestXGB = XGBClassifier(learning_rate=learning, max_depth=depth)
 print(bestXGB)
 bestXGB.fit(X_train, y_train)
 testPredictionsXGB = bestXGB.predict(X_test)
 outputGB = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': testPredictionsXGB})
 outputGB.to_csv('my_submissionXGB.csv', index=False )
auc_mean = np.mean(val_aucs)
 auc_std = np.std(val_aucs)
 auc_all = roc_auc_score(valid_X.target, valid_X.predict)
 print('%d-fold auc mean: %.9f, std: %.9f.All auc: %6f.' %(n_folds, auc_mean, auc_std, auc_all))<save_to_csv>
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 train_data.head()
y_all = result.values[:, 1:]
result['target'] = np.mean(y_all, axis = 1)
 to_submit = result[['ID_code', 'target']]
to_submit.to_csv('NN_submission.csv', index=None)
 result.to_csv('NN_all_prediction.csv', index=None)
 valid_X['ID_code'] = train_df['ID_code']
valid_X = valid_X[['ID_code', 'target', 'predict']].to_csv('NN_oof.csv', index=None )<load_from_csv>
print("
print("
print()
print("Train Features = ", train_data.columns.values)
 print("Test Features = ", test_data.columns.values)
 print()
print("NaNs in each training Feature")
 print(train_data.isnull().sum())
 print()
print("NaNs in each testing Feature")
 print(test_data.isnull().sum() )
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )<drop_column>
train_data.drop(columns='Cabin', inplace=True)
 train_data.drop(columns='Ticket', inplace=True )
train_features = train_df.drop(['target','ID_code'], axis = 1)
 test_features = test_df.drop(['ID_code'],axis = 1)
 train_target = train_df['target']<count_unique_values>
def detect_outliers(df,n,features):
outlier_indices = []
for col in features:
Q1 = np.percentile(df[col], 25)
 Q3 = np.percentile(df[col],75)
 IQR = Q3 - Q1
outlier_step = 1.5 * IQR
outlier_list_col = df[(df[col] < Q1 - outlier_step)|(df[col] > Q3 + outlier_step)].index
outlier_indices.extend(outlier_list_col)
 outlier_indices = Counter(outlier_indices)
 multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)
 return multiple_outliers
df_test = test_features.values
unique_samples = []
unique_count = np.zeros_like(df_test)
 basic_features = [c for c in train_df.columns if c not in ['ID_code', 'target']]
for feature in range(df_test.shape[1]):
_, index_, count_ = np.unique(df_test[:, feature], return_counts=True, return_index=True)
 unique_count[index_[count_ == 1], feature] += 1
real_samples_indexes = np.argwhere(np.sum(unique_count, axis=1)> 0)[:, 0]
synthetic_samples_indexes = np.argwhere(np.sum(unique_count, axis=1)== 0)[:, 0]<data_type_conversions>
train_data['Embarked'].fillna('S', inplace=True )
def process_data(train_df, test_df):
idx = [c for c in train_df.columns if c not in ['ID_code', 'target']]
df = pd.concat([train_df,test_df.ix[real_samples_indexes]])
 for feat in idx:
temp = df[feat].value_counts(dropna=True)
 train_df["count_"+feat] = train_df[feat].map(temp)
 test_df["count_"+feat] = test_df[feat].map(temp)
 train_df["sum_"+feat] =(( train_df[feat]-df[feat].mean())*train_df["count_"+feat].map(lambda x:int(x>1)) ).astype(np.float32)
 test_df["sum_"+feat] =(( test_df[feat]-df[feat].mean())*test_df["count_"+feat].map(lambda x:int(x>1)) ).astype(np.float32)
 train_df["copy_"+feat]=train_df[feat] *(train_df["count_"+feat] > 1 ).astype(int)
 test_df["copy_"+feat]=test_df[feat] *(test_df["count_"+feat] > 1 ).astype(int)
 train_df["copy_"+feat] = train_df["copy_"+feat].replace(0,df[feat].median())
 test_df["copy_"+feat] = test_df["copy_"+feat].replace(0,df[feat].median())
 return train_df, test_df<create_dataframe>
meanAges = train_data.groupby(['Pclass'])['Age'].mean()
print(meanAges )
train_features , test_features = process_data(train_features,test_features )<prepare_output>
train_data['newGender'] = train_data['Sex']
train_data.loc[train_data['Age']<16., 'newGender'] = 'child'
train_data.drop(columns='Sex', inplace=True)
 print(train_data.head(8))
train_features = train_features[feature_name]
test_features = test_features[feature_name]
gc.collect()<normalization>
train_data['Title'] = train_data['Name']
for name_string in train_data['Name']:
train_data['Title'] = train_data['Name'].str.extract('([A-Za-z]+)\.', expand=True)
 mapping = {'Mlle': 'Miss', 'Major': 'Rare', 'Col': 'Rare', 'Sir': 'Rare', 'Don': 'Rare', 'Mme': 'Miss',
'Jonkheer': 'Rare', 'Lady': 'Rare', 'Capt': 'Rare', 'Countess': 'Rare', 'Ms': 'Miss', 'Dona': 'Mrs', 'Rev': 'Rare'}
train_data.replace({'Title': mapping}, inplace=True)
 train_data.drop(columns='Name', inplace=True )
sc = StandardScaler()
train_features = sc.fit_transform(train_features)
 test_features = sc.transform(test_features )<split>
train_data['familyNumber'] = train_data['SibSp'] + train_data['Parch']
print(train_data.head(8))
Outliers_to_drop = detect_outliers(train_data,1,["Age","familyNumber","Fare"])
 train_data = train_data.drop(Outliers_to_drop, axis = 0 ).reset_index(drop=True)
 train_data['familySize'] = np.nan
train_data.loc[train_data['familyNumber']==0, 'familySize'] = 'alone'
train_data.loc[train_data['familyNumber']==1, 'familySize'] = 'couple'
train_data.loc[train_data['familyNumber']==2, 'familySize'] = 'small'
train_data.loc[train_data['familyNumber']==3, 'familySize'] = 'medium'
train_data.loc[train_data['familyNumber']>=4, 'familySize'] = 'large'
train_data.drop(columns=['familyNumber', 'SibSp', 'Parch'], inplace=True )
n_splits = 10
splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=4590 ).split(train_features, train_target))<choose_model_class>
train_data['newPclass'] = np.nan
train_data.loc[train_data['Pclass']==1, 'newPclass'] = 'first'
train_data.loc[train_data['Pclass']==2, 'newPclass'] = 'second'
train_data.loc[train_data['Pclass']==3, 'newPclass'] = 'third'
train_data.drop(columns='Pclass', inplace=True)
 columns = ['newGender', 'Embarked', 'Title', 'familySize', 'newPclass']
one_hot = pd.get_dummies(train_data.loc[:, columns], drop_first=True)
 train_data.drop(columns=columns, inplace=True)
 train_data = train_data.join(one_hot)
 print(train_data.head(2))
print(train_data.columns.values )
class CyclicLR(object):
def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,
step_size=2000, mode='triangular', gamma=1.,
scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):
if not isinstance(optimizer, Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if isinstance(base_lr, list)or isinstance(base_lr, tuple):
if len(base_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} base_lr, got {}".format(
 len(optimizer.param_groups), len(base_lr)))
 self.base_lrs = list(base_lr)
 else:
self.base_lrs = [base_lr] * len(optimizer.param_groups)
 if isinstance(max_lr, list)or isinstance(max_lr, tuple):
if len(max_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} max_lr, got {}".format(
 len(optimizer.param_groups), len(max_lr)))
 self.max_lrs = list(max_lr)
 else:
self.max_lrs = [max_lr] * len(optimizer.param_groups)
 self.step_size = step_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.batch_step(last_batch_iteration + 1)
 self.last_batch_iteration = last_batch_iteration
def batch_step(self, batch_iteration=None):
if batch_iteration is None:
batch_iteration = self.last_batch_iteration + 1
self.last_batch_iteration = batch_iteration
for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
param_group['lr'] = lr
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):
step_size = float(self.step_size)
 cycle = np.floor(1 + self.last_batch_iteration /(2 * step_size))
x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)
 lrs = []
param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)
 for param_group, base_lr, max_lr in param_lrs:
base_height =(max_lr - base_lr)* np.maximum(0,(1 - x))
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)
 lrs.append(lr)
 return lrs<choose_model_class>
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.model_selection import GridSearchCV
from sklearn.preprocessing import StandardScaler
from sklearn.neighbors import KNeighborsClassifier
class Simple_NN(nn.Module):
def __init__(self ,input_dim ,hidden_dim, dropout = 0.5):
super(Simple_NN, self ).__init__()
self.inpt_dim = input_dim
self.hidden_dim = hidden_dim
self.relu = nn.ReLU()
self.dropout = nn.Dropout(dropout)
 self.fc1 = nn.Linear(1, hidden_dim)
 self.fc2 = nn.Linear(int(hidden_dim*input_dim), 1)
 def forward(self, x):
b_size = x.size(0)
 x = x.view(-1, 1)
 y = self.fc1(x)
 y = self.relu(y)
 y = y.view(b_size, -1)
 out= self.fc2(y)
 return out<compute_test_metric>
trainScaler = StandardScaler()
trainScaler.fit(train_data.drop(['Survived', 'PassengerId'], axis = 1))
X_train_scaled = trainScaler.transform(train_data.drop(['Survived', 'PassengerId'], axis = 1))
X_train = train_data.drop(['Survived', 'PassengerId'], axis = 1)
 y_train = train_data['Survived']
def sigmoid(x):
return 1 /(1 + np.exp(-x))<train_model>
grid_params = {
'n_neighbors': [3, 5, 7, 9, 11, 13],
'weights': ['uniform', 'distance'],
'metric':['euclidean', 'manhattan']
}
KNN_CV = GridSearchCV(estimator = KNeighborsClassifier() , param_grid=grid_params, cv = 3)
 KNN_CV.fit(X_train_scaled, y_train)
 print(KNN_CV.best_score_)
 weights = KNN_CV.best_params_.get('weights')
 n = KNN_CV.best_params_.get('n_neighbors')
 metric = KNN_CV.best_params_.get('metric')
 bestKNN = KNeighborsClassifier(n_neighbors=n, weights=weights, metric=metric)
 bestKNN.fit(X_train_scaled, y_train )
class EarlyStopping:

 def __init__(self, patience=2, verbose=False):

 self.patience = patience
self.verbose = verbose
self.counter = 0
self.best_score = None
self.early_stop = False
self.val_auc_min = 0
def __call__(self, val_auc, model):
score = val_auc
if self.best_score is None:
self.best_score = score
self.save_checkpoint(val_auc, model)
 elif score < self.best_score:
self.counter += 1
print(f'EarlyStopping counter: {self.counter} out of {self.patience}')
 if self.counter >= self.patience:
self.early_stop = True
else:
self.best_score = score
self.save_checkpoint(val_auc, model)
 self.counter = 0
def save_checkpoint(self, val_auc, model):

 if self.verbose:
print(f'Validation loss decreased({self.val_auc_min:.6f} --> {val_auc:.6f} ).Saving model...')
 torch.save(model.state_dict() , 'checkpoint.pt')
 self.val_auc_min = val_auc<categorify>
param_grid = {
'criterion' : ['gini'],
'n_estimators': [70, 80, 90, 100, 110, 120],
'max_features': ['auto', 'log2'],
'max_depth' : [5, 7, 9, 11, 13]
}
randomForest_CV = GridSearchCV(estimator = RandomForestClassifier() , param_grid = param_grid, cv = 3)
 randomForest_CV.fit(X_train, y_train)
 print(randomForest_CV.best_score_)
 criterion = randomForest_CV.best_params_.get('criterion')
 nRF = randomForest_CV.best_params_.get('n_estimators')
 max_features = randomForest_CV.best_params_.get('max_features')
 max_depth = randomForest_CV.best_params_.get('max_depth')
 bestRF = RandomForestClassifier(n_estimators=1000, max_depth=max_depth, max_features=max_features, criterion=criterion)
 bestRF.fit(X_train, y_train )
def augment(x,y,t=2):
xs,xn = [],[]
for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xs.append(x1)
 for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xn.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x,xs,xn])
 y = np.concatenate([y,ys,yn])
 return x,y<define_variables>
for i in range(3):
test_data.loc[test_data['Pclass']==int(i+1), 'Age'] = test_data.loc[test_data['Pclass']==int(i+1), 'Age'].fillna(meanAges.values[i])
 meanFare = test_data['Fare'].median()
test_data['Fare'].fillna(meanFare, inplace=True )
n_epochs = 20000
batch_size = 256
train_preds = np.zeros(( len(train_features)))
 test_preds = np.zeros(( len(test_features)))
 x_test = np.array(test_features)
 x_test_cuda = torch.tensor(x_test, dtype=torch.float ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 avg_losses_f = []
avg_val_losses_f = []
val_auc=[]
for i,(train_idx, valid_idx)in enumerate(splits):
valid_preds = []
x_train = np.array(train_features)
 y_train = np.array(train_target)
 x_train_fold = torch.tensor(x_train[train_idx.astype(int)], dtype=torch.float ).cuda()
y_train_fold = torch.tensor(y_train[train_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(x_train[valid_idx.astype(int)], dtype=torch.float ).cuda()
y_val_fold = torch.tensor(y_train[valid_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
loss_fn = torch.nn.BCEWithLogitsLoss()
model = Simple_NN(train_features.shape[1],32)
 model.cuda()
optimizer = torch.optim.Adam(model.parameters() , lr = 0.001,weight_decay=1e-5)
 step_size = 2000
base_lr, max_lr = 0.0001, 0.002
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
lr=max_lr)
 scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,
step_size=step_size, mode='exp_range',
gamma=0.99994)
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 early_stopping = EarlyStopping(patience=8, verbose=True)
 for epoch in range(n_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for i,(x_batch, y_batch)in enumerate(train_loader):
y_pred = model(x_batch)
 if scheduler:
scheduler.batch_step()
loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() /len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(( len(test_features)))
 avg_val_loss = 0.
 avg_val_auc = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_auc += round(roc_auc_score(y_batch.cpu() ,sigmoid(y_pred.cpu().numpy())[:, 0]),4)/ len(valid_loader)
 avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
valid_preds.append(valid_preds_fold)
 elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))
early_stopping(avg_val_auc, model)
 if early_stopping.early_stop:
valid_preds_fold = valid_preds[-9]
print("Early stopping")
 break
model.load_state_dict(torch.load('checkpoint.pt'))
avg_losses_f.append(avg_loss)
 avg_val_losses_f.append(avg_val_loss)
 for i,(x_batch,)in enumerate(test_loader):
y_pred = model(x_batch ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits)
 val_auc.append(roc_auc_score(train_target[valid_idx],train_preds[valid_idx]))
print(val_auc)
 print(np.mean(val_auc))
auc = round(roc_auc_score(train_target,train_preds),4)
 print('All \t loss={:.4f} \t val_loss={:.4f} \t auc={:.4f}'.format(np.average(avg_losses_f),np.average(avg_val_losses_f),auc))<save_to_csv>
test_data['newGender'] = test_data['Sex']
test_data.loc[test_data['Age']<18., 'newGender'] = 'child'
test_data.drop(columns='Sex', inplace=True)
 test_data['Title'] = test_data['Name']
for name_string in test_data['Name']:
test_data['Title'] = test_data['Name'].str.extract('([A-Za-z]+)\.', expand=True)
 mapping = {'Mlle': 'Miss', 'Major': 'Rare', 'Col': 'Rare', 'Sir': 'Rare', 'Don': 'Rare', 'Mme': 'Miss',
'Jonkheer': 'Rare', 'Lady': 'Rare', 'Capt': 'Rare', 'Countess': 'Rare', 'Ms': 'Miss', 'Dona': 'Mrs', 'Rev': 'Rare'}
test_data.replace({'Title': mapping}, inplace=True)
 test_data.drop(columns='Name', inplace=True)
 test_data['familyNumber'] = test_data['SibSp'] + test_data['Parch']
test_data['familySize'] = np.nan
test_data.loc[test_data['familyNumber']==0, 'familySize'] = 'alone'
test_data.loc[test_data['familyNumber']==1, 'familySize'] = 'couple'
test_data.loc[test_data['familyNumber']==2, 'familySize'] = 'small'
test_data.loc[test_data['familyNumber']==3, 'familySize'] = 'medium'
test_data.loc[test_data['familyNumber']>=4, 'familySize'] = 'large'
test_data.drop(columns=['familyNumber', 'SibSp', 'Parch'], inplace=True)
 test_data['newPclass'] = np.nan
test_data.loc[test_data['Pclass']==1, 'newPclass'] = 'first'
test_data.loc[test_data['Pclass']==2, 'newPclass'] = 'second'
test_data.loc[test_data['Pclass']==3, 'newPclass'] = 'third'
test_data.drop(columns='Pclass', inplace=True )
test_ID = test_df['ID_code'].values
submission_nn = pd.DataFrame({ "ID_code": test_ID, "target": test_preds})
 submission_nn.to_csv('test_preds_submission_nn_10fold.csv', index=False )<save_to_csv>
test_data.drop(columns=['Cabin', 'Ticket'], inplace=True)
 one_hot = pd.get_dummies(test_data.loc[:, columns], drop_first=True)
 test_data.drop(columns=columns, inplace=True)
 test_data = test_data.join(one_hot)
 print("NaNs in each testing Feature")
 print(test_data.isnull().sum() )
train_ID = train_df['ID_code'].values
submission_nn = pd.DataFrame({ "ID_code": train_ID, "target": train_preds})
 submission_nn.to_csv('train_preds_submission_nn_10fold.csv', index=False )<set_options>
X_test_scaled = trainScaler.transform(test_data.drop(['PassengerId'], axis = 1))
X_test = test_data.drop(columns='PassengerId')
 testPredictionsKNN = bestKNN.predict(X_test_scaled)
 testPredictionsRF = bestRF.predict(X_test)
 outputKNN = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': testPredictionsKNN})
 outputKNN.to_csv('my_submissionKNN.csv', index=False)
 outputRF = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': testPredictionsRF})
 outputRF.to_csv('my_submissionRF.csv', index=False )
debug = False
warnings.simplefilter('ignore')
 plt.style.use('seaborn')
 random_state = 333<load_from_csv>
param_grid = {
'learning_rate' : [0.1, 0.2],
'n_estimators': [90, 100, 110],
'loss': ['deviance', 'exponential']
}
gradBoost_CV = GridSearchCV(estimator = GradientBoostingClassifier() , param_grid = param_grid, cv = 3)
 gradBoost_CV.fit(X_train, y_train)
 print(gradBoost_CV.best_score_)
 learning = gradBoost_CV.best_params_.get('learning_rate')
 nGB = gradBoost_CV.best_params_.get('n_estimators')
 loss = gradBoost_CV.best_params_.get('loss')
 bestGB = GradientBoostingClassifier(learning_rate=learning, n_estimators=nGB, loss=loss)
 bestGB.fit(X_train, y_train)
 testPredictionsGB = bestGB.predict(X_test)
 outputGB = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': testPredictionsGB})
 outputGB.to_csv('my_submissionGB.csv', index=False )
train_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/train.csv', index_col='ID_code')
 test_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/test.csv', index_col='ID_code')
 public = np.load('.. /input/helpingdata/public_LB.npy')
 private = np.load('.. /input/helpingdata/private_LB.npy')
 fake = np.load('.. /input/helpingdata/synthetic_samples_indexes.npy')
 target = train_df['target']<compute_test_metric>
param_grid = {
'learning_rate' : [0.1, 0.2],
'max_depth': [3, 5, 7],
}
xgBoost_CV = GridSearchCV(estimator = XGBClassifier() , param_grid = param_grid, cv = 3)
 xgBoost_CV.fit(X_train, y_train)
 print(xgBoost_CV.best_score_)
 learning = xgBoost_CV.best_params_.get('learning_rate')
 depth = xgBoost_CV.best_params_.get('max_depth')
 bestXGB = XGBClassifier(learning_rate=learning, max_depth=depth)
 print(bestXGB)
 bestXGB.fit(X_train, y_train)
 testPredictionsXGB = bestXGB.predict(X_test)
 outputGB = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': testPredictionsXGB})
 outputGB.to_csv('my_submissionXGB.csv', index=False )