Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<compute_test_metric>
!pip install seaborn==0.11.0
if debug:
print("debug:",roc_auc_score(y_test, y_preds))
print("debug:",roc_auc_score(y_test, y_preds2))
print("debug:",roc_auc_score(y_test,(y_preds + y_preds2)*0.5))<prepare_x_and_y>
pd.options.display.max_rows=200
pd.set_option('mode.chained_assignment', None)
 simplefilter("ignore", category=ConvergenceWarning)
 simplefilter("ignore", category=RuntimeWarning )
features = [x for x in X_train.columns]
cate = [x for x in X_train.columns if(x == 'ProductCD' or x in ['card1','card2'] or x.startswith("addr")or
x.endswith("domain")or x.startswith("Device")) and not x.endswith("count")and not x == "id_11" ]
print(cate)
 verbose_eval = 30
num_rounds = 800
folds = 3
kf = KFold(n_splits = folds, shuffle = True, random_state=seed+1)
 y_preds3 = np.zeros(X_test.shape[0])
 feature_importance_df = pd.DataFrame()
i = 0
for tr_idx, val_idx in kf.split(X_train, y_train):
X_tr = X_train[features].iloc[tr_idx, :].fillna(-1)
 y_tr = y_train.iloc[tr_idx]
model=cb.CatBoostClassifier(iterations=num_rounds,depth=14,learning_rate=0.04,loss_function='Logloss',eval_metric='Logloss'
,task_type = "GPU")
 if debug:
model.fit(X_tr,y_tr,cat_features=cate,verbose_eval = 30)
 else:
model.fit(X_tr,y_tr,cat_features=cate,verbose_eval = 30)
 del X_tr
y_preds3+= model.predict_proba(X_test[features].fillna(-1)) [:,1] / folds
if debug:
print("debug:",roc_auc_score(y_test, model.predict_proba(X_test[features].fillna(-1)) [:,1] / folds))
i+=1
if debug:
print("debug:",roc_auc_score(y_test, y_preds3))
print("debug:",roc_auc_score(y_test,(y_preds + y_preds3)*0.5))
print("debug:",roc_auc_score(y_test,(y_preds + y_preds2 + y_preds3)*0.33))<compute_test_metric>
train = pd.read_csv('/kaggle/input/titanic/train.csv', index_col='PassengerId')
 test = pd.read_csv('/kaggle/input/titanic/test.csv', index_col='PassengerId' )
if debug:
print("debug:",roc_auc_score(y_test, y_preds))
print("debug:",roc_auc_score(y_test, y_preds2))
print("debug:",roc_auc_score(y_test, y_preds3))
print("debug:",roc_auc_score(y_test,(y_preds + y_preds3)*0.5))
print("debug:",roc_auc_score(y_test,(y_preds + y_preds2 + y_preds3*0.5)*0.33))
print("debug:",roc_auc_score(y_test,(y_preds11*0.5 + y_preds*0.5 + y_preds2 + y_preds3*0.5)*0.33))<save_to_csv>
train.isna().sum()
if not debug:
sample_submission['isFraud'] =(y_preds11*0.5 + y_preds*0.5 + y_preds2 + y_preds3*0.5)*0.33
sample_submission.to_csv('simple_ensemble6.csv')
<set_options>
test.isna().sum()
print('loading libs...')
 warnings.filterwarnings("ignore")
 print('done' )<load_from_csv>
def imputer(df):
age_impute_series = df.groupby(['Pclass', 'Sex'] ).Age.transform('mean')
 df.Age.fillna(age_impute_series, inplace=True)
 df.Cabin = df.Cabin.str.extract(pat='([A-Z])')
 df.Cabin.fillna('M', inplace=True)
 df['Deck'] = df.Cabin.replace({'A':'ABC', 'B':'ABC', 'C':'ABC', 'D':'DE', 'E':'DE', 'F':'FG',
'G':'FG', 'T':'ABC'})
 df.drop('Cabin', axis=1, inplace=True)
 for feature in df.columns:
df[feature].fillna(df[feature].mode() [0], inplace=True)
 return df
%%time
print('loading data...')
 train = pd.read_pickle('.. /input/ieee-fe-with-some-eda/train_df.pkl')
 test = pd.read_pickle('.. /input/ieee-fe-with-some-eda/test_df.pkl')
 remove_features = pd.read_pickle('.. /input/ieee-fe-with-some-eda/remove_features.pkl')
 sample_submission = pd.read_csv('.. /input/ieee-fraud-detection/sample_submission.csv')
 print('done')
<prepare_x_and_y>
train_imputed = imputer(train.copy())
 test_imputed = imputer(test.copy() )
%%time
print('dropping target...')
 y_train = train['isFraud'].copy()
X_train = train.drop('isFraud', axis=1)
 X_test = test.copy()
train_cols = list(train.columns)
 del train, test
gc.collect()
print('selecting features...')
 remove_features = list(remove_features['features_to_remove'].values)
 features_columns = [col for col in train_cols if col not in remove_features]
X_train = X_train[features_columns]
X_test=X_test[features_columns]
print('Done' )<drop_column>
def ticket_extractor(ticket):
alpha = re.sub('\d', '', ticket)
 if alpha:
return alpha
else:
num = re.search('\d{1,9}', ticket)
 return ticket
temp = train_imputed.copy()
temp['Ticket_extracted'] = temp.Ticket.apply(ticket_extractor)
 for i in range(len(temp.Ticket)) :
try:
int(temp.Ticket_extracted.iloc[i])
 temp.Ticket_extracted.iloc[i] = f'Num_{len(temp.Ticket_extracted.iloc[i])}'
except:
continue
for label, pattern in [('_ca', 'C[.]?A[.]?'),('_PC', 'PC'),('_SOTON', 'SOTON'),('_STON', 'STON'), 
('_WC', 'W[.]?[/]?C'),('_SC', 'S[.]?C[.]?'),('_A', 'A[.]?'),('_SOC', 'S[.]?O[.]?[/]?C'), 
('_PP', 'PP'),('_FC', '(F.C.|F.C.C.) '),('_LS_number', 'Num_(6|7)'),('_SS_number', 'Num_(3|4|5)'), 
('rare', '^[^_]')]:
temp.Ticket_extracted[temp.Ticket_extracted.str.contains(pattern)] = label
temp['Ticket_extracted'].value_counts(dropna=False)
 fig, ax = plt.subplots(2, 1, figsize=(15, 15))
plt.style.use('fivethirtyeight')
 sns.countplot(data=temp, x='Ticket_extracted', hue='Survived', ax=ax[0], palette=palette_6)
 ax[0].set_title('Extracted Ticket - Survival "count" plot', size=20, loc='Left', y=1.03)
 sns.barplot(data=temp, x='Ticket_extracted', y='Survived', ax=ax[1], palette=palette_6)
 ax[1].set_title('Extracted Ticket - Survival "chance" plot', size=20, loc='Left', y=1.03)
 sns.despine()
plt.show()
X_train = reduce_mem_usage(X_train)
 X_test = reduce_mem_usage(X_test )<init_hyperparams>
temp = train_imputed.copy()
temp['Title'] = temp.Name.str.extract(pat='([a-zA-Z]+\.) ')
 temp.Title[~temp.Title.isin(['Mr.', 'Miss.', 'Mrs.', 'Master.'])] = 'rare'
params = {
'objective':'binary',
'boosting_type':'gbdt',
'metric':'auc',
'n_jobs':-1,
'max_depth':-1,
'tree_learner':'serial',
'min_data_in_leaf':30,
'n_estimators':1800,
'max_bin':255,
'verbose':-1,
'seed': 1229,
'learning_rate': 0.01,
'early_stopping_rounds':200,
'colsample_bytree': 0.5,
'num_leaves': 256,
'reg_alpha': 0.35,
}<split>
def feature_creator(df_train, df_test):
df_train['Fare_cat'] = pd.qcut(df_train['Fare'], 7)
 df_test['Fare_cat'] = pd.qcut(df_test['Fare'], 7)
 df_train['Fare_cat'] = LabelEncoder().fit_transform(df_train['Fare_cat'])
 df_test['Fare_cat'] = LabelEncoder().fit_transform(df_test['Fare_cat'])
 df_train['Age_cat'] = pd.cut(df_train.Age, bins=[0, 5, 24, 30, 36, np.inf])
 df_test['Age_cat'] = pd.cut(df_test.Age, bins=[0, 5, 24, 30, 36, np.inf])
 df_train['Age_cat'] = LabelEncoder().fit_transform(df_train['Age_cat'])
 df_test['Age_cat'] = LabelEncoder().fit_transform(df_test['Age_cat'])
 for df in [df_train, df_test]:
df['Title'] = df.Name.str.extract(pat='([a-zA-Z]+\.) ')
 df.Title[~df.Title.isin(['Mr.', 'Miss.', 'Mrs.', 'Master.'])] = 'rare'
df_full = pd.concat([df_train, df_test], axis=0)
 df_full['Ticket_frequency'] = df_full.Ticket.map(df_full.Ticket.value_counts())
 df_train = df_full.loc[df_train.index]
df_test = df_full.loc[df_test.index].loc[df_test.index].drop('Survived', axis=1)
 for df in [df_train, df_test]:
df['Ticket_extracted'] = df.Ticket.apply(ticket_extractor)
 for i in range(len(df.Ticket)) :
try:
int(df.Ticket_extracted.iloc[i])
 df.Ticket_extracted.iloc[i] = f'Num_{len(df.Ticket_extracted.iloc[i])}'
except:
continue
for label, pattern in [('_ca', 'C[.]?A[.]?'),('_PC', 'PC'),('_SOTON', 'SOTON'),('_STON', 'STON'), 
('_WC', 'W[.]?[/]?C'),('_SC', 'S[.]?C[.]?'),('_A', 'A[.]?'),('_SOC', 'S[.]?O[.]?[/]?C'), 
('_PP', 'PP'),('_FC', '(F.C.|F.C.C.) '),('_LS_number', 'Num_(6|7)'), 
('_SS_number', 'Num_(3|4|5)'),('rare', '^[^_]')]:
df.Ticket_extracted[df.Ticket_extracted.str.contains(pattern)] = label
df_train['Family_size'] = df_train['SibSp'] + df_train['Parch'] + 1
df_test['Family_size'] = df_test['SibSp'] + df_test['Parch'] + 1
df_train['Family_size_cat'] = df_train.Family_size.replace({1:'alone', 2:'small_family', 3:'small_family', 4:'small_family'
,5:'large_family', 6:'large_family', 7:'large_family'
,8:'large_family', 9:'large_family', 10:'large_family'
,11:'large_family'})
 df_test['Family_size_cat'] = df_test.Family_size.replace({1:'alone', 2:'small_family', 3:'small_family', 4:'small_family'
,5:'large_family', 6:'large_family', 7:'large_family'
,8:'large_family', 9:'large_family', 10:'large_family'
,11:'large_family'})
 df_train['Name_length'] = df_train.Name.str.replace(pat='[^a-zA-Z]', repl='' ).str.len()
df_test['Name_length'] = df_test.Name.str.replace(pat='[^a-zA-Z]', repl='' ).str.len()
df_full = pd.concat([df_train, df_test], axis=0 ).reset_index()
df_full['Family_name'] = df_full.Name.str.split(',', n=1, expand=True ).iloc[:, 0]
df_full['Family_Survival'] = 0.5
for grp, grp_df in df_full.groupby(['Family_name', 'Fare']):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
df_full.loc[df_full['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
df_full.loc[df_full['PassengerId'] == passID, 'Family_Survival'] = 0
for _, grp_df in df_full.groupby('Ticket'):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
if(row['Family_Survival'] == 0)|(row['Family_Survival']== 0.5):
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
df_full.loc[df_full['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
df_full.loc[df_full['PassengerId'] == passID, 'Family_Survival'] = 0
df_full.set_index('PassengerId', inplace=True)
 df_train = df_full[:891]
df_test = df_full[891:]
df_train.drop(['Family_name'], axis=1, inplace=True)
 df_test.drop(['Family_name', 'Survived'], axis=1, inplace=True)
 y_train =df_train.Survived
df_train.drop('Survived', axis=1, inplace=True)
 return df_train, y_train, df_test
%%time
NFOLDS = 6
folds = KFold(n_splits=NFOLDS)
 columns = X_train.columns
splits = folds.split(X_train, y_train)
 y_preds = np.zeros(X_test.shape[0])
 y_oof = np.zeros(X_train.shape[0])
 score = 0
for fold_n,(train_index, valid_index)in enumerate(splits):
X_tr, X_val = X_train[columns].iloc[train_index], X_train[columns].iloc[valid_index]
y_tr, y_val = y_train.iloc[train_index], y_train.iloc[valid_index]
dtrain = lgb.Dataset(X_tr, label=y_tr)
 dvalid = lgb.Dataset(X_val, label=y_val)
 clf = lgb.train(params, dtrain, valid_sets = [dtrain, dvalid], verbose_eval=500)
 y_pred_valid = clf.predict(X_val)
 y_oof[valid_index] = y_pred_valid
print(f"Fold {fold_n + 1} | AUC: {roc_auc_score(y_val, y_pred_valid)}")
 score += roc_auc_score(y_val, y_pred_valid)/ NFOLDS
del X_tr, X_val, y_tr, y_val
gc.collect()
y_preds += clf.predict(X_test)/ NFOLDS
print(f"
Mean AUC = {score}")
 print(f"Out of folds AUC = {roc_auc_score(y_train, y_oof)}")
 print('submission...')
 sample_submission['isFraud'] = y_preds
sample_submission.to_csv("submission_lgb.csv", index=False )<load_from_csv>
X_train, y_train, X_test = feature_creator(train_imputed.copy() , test_imputed.copy())
 y_train = y_train.astype(int )
train_identity = pd.read_csv('/kaggle/input/ieee-fraud-detection/train_identity.csv', index_col = 'TransactionID')
 print('Successfully loaded train_identity')
 train_transaction = pd.read_csv('/kaggle/input/ieee-fraud-detection/train_transaction.csv', index_col = 'TransactionID')
 print('Successfully loaded train_transaction')
 test_identity = pd.read_csv('/kaggle/input/ieee-fraud-detection/test_identity.csv', index_col = 'TransactionID')
 print('Successfully loaded test_identity')
 test_transaction = pd.read_csv('/kaggle/input/ieee-fraud-detection/test_transaction.csv', index_col = 'TransactionID')
 print('Successfully loaded test_transaction')
 sub = pd.read_csv('/kaggle/input/ieee-fraud-detection/sample_submission.csv')
 print('Successfully loaded sample_submisssion')
 print('Data was successfully loades!')
 print('Merging data.... ')
 train = train_transaction.merge(train_identity, how = 'left', left_index = True, right_index = True)
 test = test_transaction.merge(test_identity, how = 'left', left_index = True, right_index = True)
 print('Data was successfully merged!')
 del train_identity, train_transaction, test_identity, test_transaction
print('Train dataset has {} rows and {} columns'.format(train.shape[0], train.shape[1]))
print('Test dataset has {} rows and {} columns'.format(test.shape[0], test.shape[1]))<feature_engineering>
class FeatureEngineering(BaseEstimator, TransformerMixin):
def __init__(self, bin_fare=False, bin_age=True, family_size=True, bin_family_size=True,
drop_Name_length=False, drop_Ticket_frequency=False, drop_all=True, drop_Family_Survival=True,
drop_Ticket_extracted=False, scaling='StandardScaler', target_encode_title=True, test=False):
self.bin_fare = bin_fare
self.bin_age = bin_age
self.family_size = family_size
self.bin_family_size = bin_family_size
self.drop_Name_length = drop_Name_length
self.drop_Ticket_frequency = drop_Ticket_frequency
self.drop_all = drop_all
self.drop_Ticket_extracted = drop_Ticket_extracted
self.scaling = scaling
self.drop_Family_Survival = drop_Family_Survival
self.target_encode_title = target_encode_title
self.transformer = None
self.drop_list = []
self.test = test
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X['Pclass'] = X['Pclass'].astype(str)
 dummies = pd.get_dummies(X.loc[:, ['Pclass', 'Sex']], drop_first=True)
 X = pd.concat([X, dummies], axis=1)
 X.drop(['Pclass', 'Sex'], axis=1, inplace=True)
 X.drop(['Name', 'Ticket'], axis=1, inplace=True)
 if self.bin_fare:
self.drop_list.append('Fare')
 else:
self.drop_list.append('Fare_cat')
 if self.bin_age:
self.drop_list.append('Age')
 else:
self.drop_list.append('Age_cat')
 if self.family_size:
self.drop_list.extend(['SibSp', 'Parch'])
 if self.bin_family_size:
self.drop_list.append('Family_size')
 else:
self.drop_list.append('Family_size_cat')
 else:
self.drop_list.extend(['Family_size', 'Family_size_cat'])
 if self.drop_Name_length:
X.drop('Name_length', axis=1, inplace=True)
 if self.drop_Ticket_frequency:
X.drop('Ticket_frequency', axis=1, inplace=True)
 if self.drop_Family_Survival:
X.drop('Family_Survival', axis=1, inplace=True)
 if self.drop_all:
X.drop(self.drop_list, axis=1, inplace=True)
 self.cols = ['Title', 'Ticket_extracted'] if self.target_encode_title else ['Ticket_extracted']
if not self.test and len(X)> 400:
self.target_encoder = TargetEncoder(cols=self.cols, smoothing=5)
 X.loc[:, self.cols] = self.target_encoder.fit_transform(X.loc[:, self.cols], y_train.loc[X.index])
 else:
X.loc[:, self.cols] = self.target_encoder.transform(X.loc[:, self.cols])
 if self.drop_Ticket_extracted:
X.drop('Ticket_extracted', axis=1, inplace=True)
 X = pd.get_dummies(X, drop_first=True)
 features_to_scale = [feature for feature in X.columns if X[feature].nunique() != 2]
if not self.test and len(X)> 400:
if self.scaling == 'StandardScaler':
self.transformer = StandardScaler()
X.loc[:, features_to_scale] = self.transformer.fit_transform(X.loc[:, features_to_scale])
 elif self.scaling == 'RobustScaler':
self.transformer = RobustScaler()
X.loc[:, features_to_scale] = self.transformer.fit_transform(X.loc[:, features_to_scale])
 elif self.scaling == 'MinMaxScaler':
self.transformer = MinMaxScaler()
X.loc[:, features_to_scale] = self.transformer.fit_transform(X.loc[:, features_to_scale])
 else:
X.loc[:, features_to_scale] = self.transformer.transform(X.loc[:, features_to_scale])
 return X
def missing_values(df):
df1 = pd.DataFrame(df.isnull().sum() ).reset_index()
df1.columns = ['features', 'freq']
df1['percentage'] = df1['freq']/df.shape[0]
df1.sort_values('percentage', ascending = False, inplace = True)
 return df1
missing_train = missing_values(train)
 missing_train.columns = ['features', 'freq_tr', 'percentage_tr']
missing_train<rename_columns>
param_grid_pipeline = {'feature_engineering__bin_fare':[True, False],
'feature_engineering__bin_age':[True, False],
'feature_engineering__family_size':[True, False],
'feature_engineering__bin_family_size':[True, False],
'feature_engineering__drop_Name_length':[False],
'feature_engineering__drop_Ticket_frequency':[False],
'feature_engineering__drop_all':[True],
'feature_engineering__drop_Ticket_extracted':[True, False],
'feature_engineering__target_encode_title':[True, False],
'feature_engineering__scaling':['StandardScaler']}
pipeline_ = Pipeline([('feature_engineering', FeatureEngineering()), 
('model', LogisticRegression())])
 cv = StratifiedShuffleSplit(n_splits=20, test_size=0.15, random_state=101)
 grid = GridSearchCV(pipeline_, param_grid_pipeline,
cv=cv.split(X_train, X_train.Deck),
scoring='accuracy', verbose=2, n_jobs=-1)
 grid.fit(X_train.copy() , y_train )
missing_test = missing_values(test)
 missing_test.columns = ['features', 'freq_te', 'percentage_te']
missing_test<merge>
pd.DataFrame(grid.cv_results_)['mean_test_score'].isna().sum()
missing = missing_train.merge(missing_test, on = 'features')
 missing.head(10 )<count_values>
X_train_fe = fe.fit_transform(X_train.copy())
 X_train_fe
train['id_24'].value_counts(normalize = True, dropna = False )<count_values>
fe.test = True
test['id_24'].value_counts(normalize = True, dropna = False )<concatenate>
X_test_fe = fe.transform(X_test.copy())
 X_test_fe
drop_features = []
drop_features.append('id_24' )<drop_column>
def learning_curve_plotter(Model, X, y, params_1, params_2, step=50):
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
 plt.figure(figsize=(16, 7))
for i,(name, params)in enumerate([params_1, params_2]):
train_score = []
val_score = []
plt.subplot(1, 2, i+1)
 for j in range(100, len(X_train), step):
model = Model(**params ).fit(X_train[:j], y_train[:j])
 train_score.append(model.score(X_train[:j], y_train[:j]))
val_score.append(model.score(X_test, y_test))
plt.plot(train_score, 'r-', label='Training accuracy')
 plt.plot(val_score, 'b-', label='Validation accuracy')
 plt.title(f'{name}')
 plt.xlabel('Training set size')
 plt.ylabel('Accuracy')
 plt.legend()
plt.show()
for i in ['id_24', 'id_25', 'id_08', 'id_07', 'id_21', 'id_26', 'id_27', 'id_23', 'id_22']:
drop_features.append(i)
 drop_features<count_values>
param_grid_logreg = {'penalty':['elasticnet'],
'C':0.01 * np.arange(100),
'l1_ratio':0.1 * np.arange(10),
'solver':['saga']}
train['dist2'].value_counts(normalize = True, dropna = False ).head()<count_values>
grid_logreg = GridSearchCV(LogisticRegression() , param_grid_logreg,
cv=RepeatedStratifiedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='accuracy', verbose=2, n_jobs=-1 )
test['dist2'].value_counts(normalize = True, dropna = False ).head()<define_variables>
grid_logreg.fit(X_train_fe, y_train )
check_features = ['dist2']<filter>
params_logreg = {'C': 0.28, 'l1_ratio': 0.9, 'penalty': 'elasticnet', 'solver': 'saga'}
missing[missing['features']=='D7']<concatenate>
param_grid_knn = {'n_neighbors':np.arange(50),
'weights':['uniform'],
'algorithm':['ball_tree'],
'leaf_size':np.arange(1, 40, 2)}
check_features.append('D7' )<filter>
grid_knn = GridSearchCV(KNeighborsClassifier() , param_grid_knn,
cv=RepeatedStratifiedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='accuracy', verbose=2, n_jobs=-1 )
missing[missing['features']=='id_18']<concatenate>
grid_knn.fit(X_train_fe, y_train )
drop_features.append('id_18' )<filter>
params_knn = {'algorithm': 'ball_tree', 'leaf_size': 1, 'n_neighbors': 7, 'weights': 'uniform'}
missing[missing['features']=='D13']<filter>
param_grid_svc = {'C':[0.001, 0.01, 0.1, 1, 5],
'kernel':['rbf'],
'gamma':0.01 * np.arange(100),
'probability':[True]}
missing[missing['features']=='D14']<filter>
grid_svc = GridSearchCV(SVC() , param_grid_svc, cv=RepeatedStratifiedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='accuracy', verbose=2, n_jobs=-1 )
missing[missing['features']=='D12']<filter>
grid_svc.fit(X_train_fe, y_train )
missing[missing['features']=='id_04']<filter>
params_svc = {'C': 1, 'gamma': 0.09, 'kernel': 'rbf', 'probability': True}
missing[missing['features']=='id_03']<filter>
param_grid_random = {'n_estimators':[300, 500, 1000],
'max_depth':[5, 9],
'max_samples':[0.5, 0.7, 0.9],
'max_features':[0.5, 0.7, 0.9],
'min_samples_split':[2, 5, 8]
}
missing[missing['features']=='D6']<filter>
grid_random = GridSearchCV(RandomForestClassifier() , param_grid_random,
cv=RepeatedStratifiedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='accuracy', verbose=2, n_jobs=-1 )
missing[missing['features']=='id_33']<filter>
grid_random.fit(X_train_fe, y_train )
missing[missing['features']=='id_09']<filter>
params_random = {'max_depth': 5, 'max_features': 0.5, 'max_samples': 0.9,
'min_samples_split': 8, 'n_estimators': 300}
missing[missing['features']=='id_10']<concatenate>
param_grid_gradient = {'max_depth':[3, 4],
'n_estimators':[300, 400, 500],
'learning_rate':[0.01, 0.03, 0.05],
'subsample':[0.5, 0.7],
'max_features':[0.5, 0.7],
}
check_features.append('id_10' )<filter>
grid_gradient = GridSearchCV(GradientBoostingClassifier() , param_grid_gradient,
cv=RepeatedStratifiedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='accuracy', verbose=2, n_jobs=-1 )
missing[missing['features']=='D9']<filter>
grid_gradient.fit(X_train_fe, y_train )
missing[missing['features']=='D8']<filter>
params_gradient = {'learning_rate': 0.01, 'max_depth': 3, 'max_features': 0.5, 'n_estimators': 500, 'subsample': 0.5}
missing[missing['features']=='id_30']<filter>
param_grid_xgb = {'n_estimators':[400, 600],
'learning_rate':[0.01, 0.03, 0.05],
'max_depth':[3, 4],
'subsample':[0.5, 0.7],
'colsample_bylevel':[0.5, 0.7],
'reg_lambda':[15, None],
}
missing[missing['features']=='id_32']<filter>
grid_xgb = GridSearchCV(XGBClassifier() , param_grid_xgb,
cv=RepeatedStratifiedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='accuracy', verbose=2, n_jobs=-1 )
missing[missing['features']=='id_34']<concatenate>
grid_xgb.fit(X_train_fe, y_train )
drop_features.append('id_34' )<filter>
params_xgb = {'colsample_bylevel': 0.7, 'learning_rate': 0.03, 'max_depth': 3,
'n_estimators': 400, 'reg_lambda': 15, 'subsample': 0.5}
missing[missing['features']=='id_14']<filter>
logreg = LogisticRegression(**params_logreg)
 svc = SVC(**params_svc)
 knn = KNeighborsClassifier(**params_knn)
 rfc = RandomForestClassifier(**params_random)
 gradient = GradientBoostingClassifier(**params_gradient)
 xgb = XGBClassifier(**params_xgb)
 estimators = [('logreg', logreg),('knn', knn),('svc', svc),('rfc', rfc),('gradient', gradient), 
('xgb', xgb)]
stack = StackingClassifier(estimators=estimators,
cv=10, n_jobs=-1 )
missing[missing['features']=='V141']<concatenate>
model = XGBClassifier(**{'colsample_bylevel': 0.7, 'learning_rate': 0.03, 'max_depth': 3, 'n_estimators': 400,
'reg_lambda': 15, 'subsample': 0.5} ).fit(X_train_fe, y_train )
drop_features.append('V141' )<filter>
y_preds = model.predict(X_test_fe )
missing[missing['features']=='V157']<concatenate>
submission = pd.DataFrame({'PassengerId':test.index,
'Survived':y_preds} )
check_features.append('V157' )<filter>
submission.to_csv('submission.csv', index=False )
missing[missing['features']=='V162']<filter>
submission.to_csv('submission.csv', index=False )
<filter><EOS> 
pd.read_csv('submission.csv' )