Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<feature_engineering>
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
plt.style.use('ggplot')
 warnings.filterwarnings('ignore')
 plt.rc('font', size=18)
 plt.rc('axes', titlesize=22)
 plt.rc('axes', labelsize=18)
 plt.rc('xtick', labelsize=12)
 plt.rc('ytick', labelsize=12)
 plt.rc('legend', fontsize=12)
 plt.rcParams['font.sans-serif'] = ['Verdana']
def format_1000(value, tick_number):
return int(value / 1_000)
 pd.options.mode.chained_assignment = None
pd.options.display.max_seq_items = 500
pd.options.display.max_rows = 500
pd.set_option('display.float_format', lambda x: '%.5f' % x)
 local = False
debug = False
if(local):
BASE_PATH = "./"
else:
BASE_PATH = ".. /input/titanic/"
tokenizer = text.Tokenizer(num_words=max_features)
 all_text = np.hstack([train['comment_text'].str.lower() ])
 tokenizer.fit_on_texts(all_text)
 print("Fitting Done...Start text to sequence transform")
 train['seq_comment']= tokenizer.texts_to_sequences(train.comment_text.str.lower())
 print("Transform done for train ")
<split>
df = pd.read_csv(f"{BASE_PATH}train.csv",index_col='PassengerId')
 df_test = pd.read_csv(f"{BASE_PATH}test.csv",index_col='PassengerId' )
x_train, x_valid, y_train, y_valid = train_test_split(train, y_train, train_size=0.95, random_state=233)
 print(x_train.shape, x_valid.shape)
 print(y_train.shape, y_valid.shape )<normalization>
missing = [(c, df[c].isna().mean() *100)for c in df]
missing = pd.DataFrame(missing, columns=["column_name", "percentage"])
 missing = missing[missing.percentage > 0]
display(missing.sort_values("percentage", ascending=False))
def get_keras_data(dataset):
X = {
'comment_text' : pad_sequences(dataset.seq_comment, maxlen=maxlen),
'senti_score_scaled': np.array(dataset.senti_score)
 }
return X<prepare_x_and_y>
def cabin_location(df):
df['Cabin'].unique()
df['Cabin'].fillna('Unknown',inplace = True)
 df['Cabin_Location'] = df['Cabin'].str[0]
df['Cabin_Location'] = np.where(( df.Pclass==1)&(df['Cabin_Location']=='U'),'C',
np.where(( df.Pclass==2)&(df['Cabin_Location']=='U'),'D',
np.where(( df.Pclass==3)&(df['Cabin_Location']=='U'),'G',
np.where(df['Cabin_Location']=='T','C',df['Cabin_Location']))))
return df
def title_extract(df):
name_splice = df['Name'].str.split(pat=', ',n=-1,expand=True)
 df['Title'] = name_splice[1].str.split(pat='.',n=-1,expand=True)[0]
title_correction_map = {'Mme':'Mrs','Mlle':'Mss','Miss':'Mss', 'Don':'Mr','Donna':'Mrs',
'Lady':'Mrs','Sir':'Mr'}
title_list = {'Mr','Mrs','Mss','Dr','Master','Rev'}
df['Title'].replace(title_list,inplace=True)
 df['Title'] = df['Title'].apply(lambda x:'Other' if(not any(x in title for title in title_list)) else x)
 return df
def age_fill(df):
df_train = pd.read_csv(f"{BASE_PATH}train.csv",index_col='PassengerId')
 df_test = pd.read_csv(f"{BASE_PATH}test.csv",index_col='PassengerId')
 df_total = pd.concat([df_train, df_test] ).copy()
df_total = title_extract(df_total)
 df = title_extract(df)
 mean_age_by_title = np.round(df_total.groupby('Title' ).Age.mean() ).astype('int64')
 df['Age'] = df.groupby('Title')['Age'].apply(lambda x: x.fillna(mean_age_by_title[x.name]))
return df
def fare_fill(df):
df_train = pd.read_csv(f"{BASE_PATH}train.csv",index_col='PassengerId')
 df_test = pd.read_csv(f"{BASE_PATH}test.csv",index_col='PassengerId')
 df_total = pd.concat([df_train, df_test] ).copy()
mean_fare_by_class = df_total.groupby('Pclass' ).Fare.mean()
df['Fare'] = df.groupby('Pclass')['Fare'].apply(lambda x: x.fillna(mean_fare_by_class[x.name]))
df['is_free'] = np.where(df['Fare']== 0,1,0)
 df['Fare'] = np.where(( df.Pclass==1)&(df['Fare']== 0),mean_fare_by_class[1],
np.where(( df.Pclass==2)&(df['Fare']== 0),mean_fare_by_class[2],
np.where(( df.Pclass==3)&(df['Fare']== 0),mean_fare_by_class[3],
df['Fare'])))
 return df
def embarked_fill(df):
df_train = pd.read_csv(f"{BASE_PATH}train.csv",index_col='PassengerId')
 df_test = pd.read_csv(f"{BASE_PATH}test.csv",index_col='PassengerId')
 df_total = pd.concat([df_train, df_test] ).copy()
mode_embarked = df_total['Embarked'].mode()
df['Embarked'] = df['Embarked'].fillna(mode_embarked[0])
 return df
def totally_cleaned(df):
df_copy = df.copy()
df_copy = cabin_location(df)
 df_copy = title_extract(df)
 df_copy = age_fill(df)
 df_copy = fare_fill(df)
 df_copy = embarked_fill(df)
 return df_copy
X_train = get_keras_data(x_train)
 X_valid = get_keras_data(x_valid )<compute_train_metric>
def is_female(df):
df['is_female']=df['Sex'].apply(lambda x: 1 if x=='female' else 0)
 return df
def traveling_party(df):
df['traveling_party']=df['Parch'] + df['SibSp'] + 1
df['is_mother'] = np.where(((df['Sex']=='female')&(df['Parch'] > 0)&(df['Age'] > 30.)),1,0)
 df['is_wife'] = np.where(((df['Sex']=='female')&(df['Title'] == 'Mrs')&(df['Age'] > 30.)),1,0)
 df['is_alone'] = np.where(df['traveling_party'] == 1,1,0)
 return df
def bin_data(df,columns,nbins,log=False):
frames = []
for cut_col in columns:
if(log):
tmp = pd.DataFrame(pd.cut(np.log1p(feat[cut_col]), bins=nbins, labels=np.arange(0,nbins)))
 else:
tmp = pd.DataFrame(pd.cut(feat[cut_col], bins=nbins, labels=np.arange(0,nbins)))
 tmp.columns = [cut_col + "_binned"]
frames.append(tmp)
 binned = pd.concat(frames, axis=1 ).astype(int)
 df = pd.concat([df, binned], axis=1)
 return df
def ticket_pre(df):
ticket_prefix = []
for i in list(df.Ticket):
if not i.isdigit() :
ticket_prefix.append(( i.replace(".","" ).replace("/","" ).strip().split(' ')[0] ).upper())
 else:
ticket_prefix.append("X")
 df["Ticket_Pre"] = ticket_prefix
return df
def feature_engineering(df):
df = is_female(df)
 df = traveling_party(df)
 df = ticket_pre(df)
 return df
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
print(self.interval)
 self.X_val, self.y_val = validation_data
print(self.y_val)
 def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: %d - score: %.6f
" %(epoch+1, score))
def get_model1() :
inp = Input(shape = [X_train['comment_text'].shape[1]], name = 'comment_text')
 x = Embedding(max_features, embed_size )(inp)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(GRU(40, return_sequences=True, dropout = 0.15, recurrent_dropout = 0.15))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 outp = Dense(6, activation='sigmoid' )(conc)
 model = Model([inp], outp)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model
model1 = get_model1()
batch_size = 128
epochs = 1
RocAuc = RocAucEvaluation(validation_data=(X_valid, y_valid), interval=1)
 os.environ['OMP_NUM_THREADS'] = '4'
exp_decay = lambda init, fin, steps:(init/fin)**(1/(steps-1)) - 1
steps = int(len(X_train)/batch_size)* epochs
lr_init, lr_fin = 0.009, 0.0045
lr_decay = exp_decay(lr_init, lr_fin, steps)
 K.set_value(model1.optimizer.lr, lr_init)
 K.set_value(model1.optimizer.decay, lr_decay)
 for i in range(3):
hist = model1.fit(X_train, y_train, batch_size=batch_size+(batch_size*(2*i)) , epochs=epochs, validation_data=(X_valid,y_valid), callbacks=[RocAuc], verbose=1)
 print("Model 1 Done" )<load_pretrained>
def catboost_encode(df,columns):
df_train = pd.read_csv(f"{BASE_PATH}train.csv",index_col='PassengerId')
 df_test = pd.read_csv(f"{BASE_PATH}test.csv",index_col='PassengerId')
 df_total = pd.concat([df_train, df_test] ).copy()
df_total = totally_cleaned(df_total)
 df_total = feature_engineering(df_total)
 cb_enc = ce.CatBoostEncoder(cols=columns)
 count_encoded = cb_enc.fit(df_total[columns],df_total['Survived'])
 df = df.join(cb_enc.transform(df[columns] ).add_suffix('_cb'))
return df
def onehot_encode(df,columns):
df_obj = df[columns].copy()
cols = []
for c in df_obj:
dummies = pd.get_dummies(df_obj[c])
 dummies.columns = [c + "_" + str(x)for x in dummies.columns]
cols.append(dummies)
 df_obj = pd.concat(cols, axis=1)
 df = pd.concat([df, df_obj], axis=1)
 df.reset_index(inplace=True, drop=True)
 return df
def get_dummy(df,columns):
for col in columns:
df = pd.get_dummies(df, columns = [col], prefix = col)
 return df
def drop_catagotical(df):
catagorical_cols = [col for col in df.columns if df[col].dtype == "object"]
df = df.drop(catagorical_cols,axis=1)
 return df
w2v = gensim.models.KeyedVectors.load_word2vec_format('.. /input/googlenews-vectors-negative300/GoogleNews-vectors-negative300.bin', binary=True)
 print("Done loading model" )<categorify>
df = pd.read_csv(f"{BASE_PATH}train.csv",index_col='PassengerId')
 df_test = pd.read_csv(f"{BASE_PATH}test.csv",index_col='PassengerId')
 feat = pd.concat([df, df_test] ).copy()
vocab_size = len(tokenizer.word_index)+1
EMBEDDING_DIM = 300
embedding_matrix = np.zeros(( vocab_size, EMBEDDING_DIM))
print(embedding_matrix.shape)
 c = 0
c1 = 0
w_Y = []
w_No = []
for word, i in tokenizer.word_index.items() :
if word in w2v:
c +=1
embedding_vector = w2v[word]
w_Y.append(word)
 else:
embedding_vector = None
w_No.append(word)
 c1 +=1
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
print(c,c1, len(w_No), len(w_Y))
print(embedding_matrix.shape)
 class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
print(self.interval)
 self.X_val, self.y_val = validation_data
print(self.y_val)
 def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: %d - score: %.6f
" %(epoch+1, score))
def get_model2() :
inp = Input(shape = [X_train['comment_text'].shape[1]], name = 'comment_text')
 print("Here")
 x = Embedding(vocab_size, EMBEDDING_DIM, weights = [embedding_matrix], trainable = True )(inp)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(GRU(40, return_sequences=True, dropout = 0.15, recurrent_dropout = 0.15))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 outp = Dense(6, activation="sigmoid" )(conc)
 model = Model([inp], outp)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model
model2 = get_model2()
RocAuc = RocAucEvaluation(validation_data=(X_valid, y_valid), interval=1)
 os.environ['OMP_NUM_THREADS'] = '4'
exp_decay = lambda init, fin, steps:(init/fin)**(1/(steps-1)) - 1
steps = int(len(X_train)/batch_size)* epochs
lr_init, lr_fin = 0.009, 0.0045
lr_decay = exp_decay(lr_init, lr_fin, steps)
 K.set_value(model2.optimizer.lr, lr_init)
 K.set_value(model2.optimizer.decay, lr_decay)
 for i in range(3):
hist = model2.fit(X_train, y_train, batch_size=batch_size+(batch_size*(2*i)) , epochs=epochs, validation_data=(X_valid,y_valid), callbacks=[RocAuc], verbose=1)
 print("Model 2 Done")
<drop_column>
df = totally_cleaned(df)
 df = feature_engineering(df)
 df = catboost_encode(df,['Title','Cabin_Location'])
 df = bin_data(df,['Age'],5,False)
 df = bin_data(df,['Fare'],10,True)
 df = onehot_encode(df,['Embarked'])
 df = get_dummy(df,['Ticket_Pre'])
del X_train, X_valid, y_valid
gc.collect()<prepare_x_and_y>
df = pd.read_csv(f"{BASE_PATH}train.csv",index_col='PassengerId')
 df_test = pd.read_csv(f"{BASE_PATH}test.csv",index_col='PassengerId')
 feat = totally_cleaned(feat)
 feat = feature_engineering(feat)
 feat = catboost_encode(feat,['Title','Cabin_Location'])
 feat = bin_data(feat,['Age'],5,False)
 feat = bin_data(feat,['Fare'],10,True)
 feat = onehot_encode(feat,['Embarked'])
 feat = get_dummy(feat,['Ticket_Pre'])
 feat = drop_catagotical(feat)
 dtrain = feat[feat.Survived.notnull() ].copy()
dtest = feat[feat.Survived.isnull() ].copy()
dtest = dtest.drop(['Survived'], axis=1)
 dtest.index = df_test.index
print(f"Raw data shape : {df.shape} {df_test.shape}")
 print(f"Clean data shape : {dtrain.shape} {dtest.shape}" )
y_train = train[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]].values<load_from_csv>
X = dtrain[dtrain['Fare']<300].drop(["Survived"], axis=1)
 y = dtrain[dtrain['Fare']<300].Survived
metric = 'accuracy'
rf = RandomForestClassifier()
kfold = KFold(n_splits=10, shuffle=True, random_state=1)
 print(f"{cross_val_score(rf, X, y, cv=kfold, scoring=metric ).mean() *100:.4f} % Accuracy")
t1 = time.time()
def load_test() :
for df in pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv', chunksize= 150000):
yield df
print("Yield complete")
 test_ids = np.array([], dtype=np.int32)
 preds= np.zeros(( 0,6), dtype = np.int32)
 print("Start Batch Prediction")
 c = 0
for df in load_test() :
c +=1
print("Chunk number is "+str(c))
sentiment_score(df)
 k2 = df['senti_score'].reshape(-1,1)
 df['senti_score_scaled'] = scaler.transform(k2)
 df['seq_comment'] = tokenizer.texts_to_sequences(df.comment_text.str.lower())
 print("Transform done for test ")
 X_test = get_keras_data(df)
 y_pred1 = model1.predict(X_test, batch_size=1024, verbose =1)
 y_pred2 = model2.predict(X_test, batch_size=1024, verbose =1)
 test_id = df['id']
del df['seq_comment'], df['senti_score'], X_test
gc.collect()
k =(y_pred1+y_pred2)/2
print(k.shape)
 preds= np.append(preds,k, axis = 0)
 print(preds.shape)
 test_ids = np.append(test_ids, test_id)
 print("All chunks done")
 t2 = time.time()
print("Total time for Parallel Batch Prediction is "+str(t2-t1))<save_to_csv>
metric = 'accuracy'
kfold = KFold(n_splits=10, shuffle=True)
 random_state = 1
classifiers = [
GaussianProcessClassifier(random_state = random_state),
Perceptron(random_state = random_state),
RidgeClassifier(random_state = random_state),
SGDClassifier(random_state = random_state),
SVC(random_state = random_state),
RandomForestClassifier(random_state = random_state),
GradientBoostingClassifier(random_state = random_state),
lgb.LGBMClassifier(random_state = random_state),
xgb.XGBClassifier(objective="reg:squarederror",random_state = random_state),
xgb.XGBClassifier(objective="reg:squarederror",booster='dart',random_state = random_state),
AdaBoostClassifier(random_state = random_state),
BaggingClassifier(random_state = random_state),
KNeighborsClassifier() ,
LogisticRegression(random_state = random_state)
 ]
clf_names = [
"GaussianProcessClassifier",
"Perceptron",
"Ridge",
"SGDClassifier",
"SVC",
"rndmforest",
"gbmC",
"lgbm",
"xgboost",
"xgboost_dart",
"AdaBoost",
"Bagging",
"KNeighbors",
"Logistic"
]
cv_results = []
for clf in classifiers:
cv_results.append(cross_val_score(clf, X, y, cv=kfold, scoring=metric,n_jobs=4))
submission = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv')
 submission[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]] = preds
submission.to_csv('submission.csv', index=False)
 end_time = time.time()
print("Total time taken is "+str(end_time-start_time))<define_variables>
ridge = RidgeClassifier()
ridge_pg = {'alpha':[1,10,100],'tol':[0.001,0.0001,0.00001]}
gs_ridge = GridSearchCV(ridge,param_grid = ridge_pg, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gs_ridge.fit(X,y)
 ridge_best = gs_ridge.best_estimator_
DENSITY_COEFF = 0.1
assert DENSITY_COEFF >= 0.0 and DENSITY_COEFF <= 1.0
OVER_CORR_CUTOFF = 0.98
assert OVER_CORR_CUTOFF >= 0.0 and OVER_CORR_CUTOFF <= 1.0
INPUT_DIR = '.. /input/private-toxic-comment-sumbmissions/'
def load_submissions() :
files = os.listdir(INPUT_DIR)
 csv_files = []
for f in files:
if f.endswith(".csv"):
csv_files.append(f)
 frames = {f:pd.read_csv(INPUT_DIR+f ).sort_values('id')for f in csv_files}
return frames
def get_corr_mat(col,frames):
c = pd.DataFrame()
for name,df in frames.items() :
c[name] = df[col]
cor = c.corr()
for name in cor.columns:
cor.set_value(name,name,0.0)
 return cor
def highest_corr(mat):
n_cor = np.array(mat.values)
 corr = np.max(n_cor)
 idx = np.unravel_index(np.argmax(n_cor, axis=None), n_cor.shape)
 f1 = mat.columns[idx[0]]
f2 = mat.columns[idx[1]]
return corr,f1,f2
def get_merge_weights(m1,m2,densities):
d1 = densities[m1]
d2 = densities[m2]
d_tot = d1 + d2
weights1 = 0.5*DENSITY_COEFF +(d1/d_tot)*(1-DENSITY_COEFF)
 weights2 = 0.5*DENSITY_COEFF +(d2/d_tot)*(1-DENSITY_COEFF)
 return weights1, weights2
def ensemble_col(col,frames,densities):
if len(frames)== 1:
_, fr = frames.popitem()
return fr[col]
mat = get_corr_mat(col,frames)
 corr,merge1,merge2 = highest_corr(mat)
 new_col_name = merge1 + '_' + merge2
w1,w2 = get_merge_weights(merge1,merge2,densities)
 new_df = pd.DataFrame()
new_df[col] =(frames[merge1][col]*w1)+(frames[merge2][col]*w2)
 del frames[merge1]
del frames[merge2]
frames[new_col_name] = new_df
if corr >= OVER_CORR_CUTOFF:
print('\t',merge1,merge2,'(OVER CORR)')
 densities[new_col_name] = max(densities[merge1],densities[merge2])
 else:
print('\t',merge1,merge2)
 densities[new_col_name] = densities[merge1] + densities[merge2]
del densities[merge1]
del densities[merge2]
return ensemble_col(col,frames,densities)
 ens_submission = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv' ).sort_values('id')
 for col in ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]:
frames = load_submissions()
print('
',col)
 densities = {k:1.0 for k in frames.keys() }
ens_submission[col] = ensemble_col(col,frames,densities)
 print(ens_submission)
 ens_submission.to_csv('lazy_ensemble_submission.csv', index=False)
<import_modules>
lrc = LogisticRegression()
lrc_pg = {'penalty':['l1','l2','elasticnet'],'max_iter':[100,500,1000],'warm_start':[True],'tol':[0.001,0.0001,0.00001],'solver':['saga']
,'l1_ratio':[0.5]}
gs_lrc = GridSearchCV(lrc,param_grid = lrc_pg, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gs_lrc.fit(X,y)
 lrc_best = gs_lrc.best_estimator_
os.environ['OMP_NUM_THREADS'] = '4'
print(os.listdir(".. /input")) 
<string_transform>
RFC = RandomForestClassifier()
rf_pg = {"max_depth": [None],
"max_features": [1, 3, 10],
"min_samples_split": [2, 3, 10],
"min_samples_leaf": [1, 3, 10],
"bootstrap": [False],
"n_estimators" :[100,300],
"criterion": ["gini"]}
gsRFC = GridSearchCV(RFC,param_grid = rf_pg, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsRFC.fit(X,y)
 RFC_best = gsRFC.best_estimator_
stops = set(stopwords.words('english'))<feature_engineering>
XGBC =xgb.XGBClassifier(objective="reg:squarederror",random_state = random_state)
 if(debug):
XGBC_pg = {'max_depth':[2]}
else:
XGBC_pg = {'max_depth':[2,4,100],'learning_rate':[0.0001,0.001,0.005],'n_estimators':[100,250,500,1000],
'reg_alpha':[0.00001,0.00005,0.0001,0.0005],'colsample_bytree':[.5,.6,.7,.8]}
gsXGBC = GridSearchCV(XGBC,param_grid = XGBC_pg, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsXGBC.fit(X,y)
 XGBC_best = gsXGBC.best_estimator_
def standardize_text(df, text_field):
df[text_field] = df[text_field].str.replace(r"http\S+", "")
 df[text_field] = df[text_field].str.replace(r"http", "")
 df[text_field] = df[text_field].str.replace(r"@\S+", "")
 df[text_field] = df[text_field].str.replace(r"[^A-Za-z0-9() ,!?@'\`"\_
]", " ")
 df[text_field] = df[text_field].str.replace(r"@", "at")
 df[text_field] = df[text_field].str.lower()
return df<string_transform>
XGBDC =xgb.XGBClassifier(objective="reg:squarederror",booster = 'dart',random_state = random_state)
 if(debug):
XGBDC_pg = {'max_depth':[2]}
else:
XGBDC_pg = {'max_depth':[2,4],'learning_rate':[0.0001,0.001,0.005],'n_estimators':[100,250,500],
'reg_alpha':[0.00001,0.00005,0.0001,0.0005],'colsample_bytree':[.5,.7,.8]}
gsXGBDC = GridSearchCV(XGBDC,param_grid = XGBDC_pg, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsXGBDC.fit(X,y)
 XGBDC_best = gsXGBDC.best_estimator_
def removing_stopwords(df, text_field):
for i in range(len(df[text_field])) :
df[text_field][i] = ' '.join([word for word in df[text_field][i].split() if word not in stops])
 return df<define_variables>
BaggC = BaggingClassifier()
BaggC_pg = {'n_estimators':[5,10,25],'bootstrap':[False,True],'max_features':[.1,.4,.5,.7],
'max_samples':[.2,.5,.7,1]}
gsBaggC = GridSearchCV(BaggC,param_grid = BaggC_pg, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsBaggC.fit(X,y)
 BaggC_best = gsBaggC.best_estimator_
EMBEDDING_FILE = '.. /input/fast-text-vector/crawl-300d-2M.vec'<load_from_csv>
adac = AdaBoostClassifier()
if(debug):
adac_pg = {'algorithm':['SAMME']}
else:
adac_pg = {'algorithm':['SAMME','SAMME.R'],'learning_rate':[.001,.00001,.01,.5,1],'n_estimators':[25,50,100,150,200]}
gsADAC = GridSearchCV(adac,param_grid = adac_pg, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsADAC.fit(X,y)
 ADAC_best = gsADAC.best_estimator_
train = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/train.csv")
 test = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/test.csv" )<drop_column>
def important_features(names_classifiers):
ncols = 2
nrows = int(np.ceil(len(names_classifiers)/ncols))
nclassifier = 0
for row in range(nrows):
for col in range(ncols):
if(nclassifier == len(names_classifiers)) :
break
name = names_classifiers[nclassifier][0]
classifier = names_classifiers[nclassifier][1]
indices = np.argsort(classifier.feature_importances_)[::-1][:10]
g = sns.barplot(y=X.columns[indices][:10],x =(classifier.feature_importances_/(classifier.feature_importances_.sum())) [indices][:10] , orient='h',ax=axes[row][col])
 g.set_xlabel("Relative importance",fontsize=10)
 g.set_ylabel("Features",fontsize=10)
 g.tick_params(labelsize=8)
 g.set_title(name + " feature importance")
 nclassifier += 1
train["comment_text"].fillna("fillna")
 test["comment_text"].fillna("fillna")
 train = removing_stopwords(train,"comment_text")
 test = removing_stopwords(test,"comment_text" )<string_transform>
test_RFC = pd.Series(RFC_best.predict(dtest), name="RFC")
 test_lrc = pd.Series(lrc_best.predict(dtest), name="LRC")
 test_XGBC = pd.Series(XGBC_best.predict(dtest), name="XGBoost")
 test_XGBDC = pd.Series(XGBDC_best.predict(dtest), name="XGBoost_Dart")
 test_BaggC = pd.Series(BaggC_best.predict(dtest), name="Bagging")
 test_Ridge = pd.Series(ridge_best.predict(dtest), name="Ridge")
 test_ADAC = pd.Series(ADAC_best.predict(dtest), name="ADAC")
 ensemble_results = pd.concat([test_RFC,test_lrc,test_XGBC,test_XGBDC, test_BaggC,test_Ridge,test_ADAC],axis=1)
 train_RFC = pd.Series(RFC_best.predict(X), name="RFC")
 train_lrc = pd.Series(lrc_best.predict(X), name="LRC")
 train_XGBC = pd.Series(XGBC_best.predict(X), name="XGBoost")
 train_XGBDC = pd.Series(XGBDC_best.predict(X), name="XGBoost_Dart")
 train_BaggC = pd.Series(BaggC_best.predict(X), name="Bagging")
 train_Ridge = pd.Series(ridge_best.predict(X), name="Ridge")
 train_ADAC = pd.Series(ADAC_best.predict(X), name="ADAC")
 ensemble_results_train = pd.concat([train_RFC,train_lrc,train_XGBC,train_XGBDC, train_BaggC,train_Ridge,train_ADAC],axis=1)
 g= sns.heatmap(ensemble_results.corr() ,annot=True)
train = standardize_text(train,"comment_text")
 test = standardize_text(test,"comment_text" )<prepare_x_and_y>
lgbmc = lgb.LGBMClassifier()
if(debug):
lgbmc_pg = {'num_leaves':[30]}
else:
lgbmc_pg = {'num_leaves':[30,50,60],'max_depth':[2,3,7,-1],'learning_rate':[.001,.00001,.01,.5],'n_estimators':[100,150,200,500]}
gsLGBMC = GridSearchCV(lgbmc,param_grid = lgbmc_pg, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 ensemble_results.head(3)
 gsLGBMC.fit(ensemble_results_train,y)
 LGBMC_best = gsLGBMC.best_estimator_
g = plot_learning_curve(LGBMC_best,"Emsemble LGBM learning curves",ensemble_results_train,y,cv=kfold)
<define_variables><EOS> 
test_Survived = pd.Series(LGBMC_best.predict(ensemble_results), name="Survived")
 output = pd.DataFrame({'PassengerId': dtest.index,
'Survived': test_Survived.astype('int32')})
 output.to_csv("ensemble_python_voting.csv",index=False )