Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<choose_model_class>
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score
import gensim as gn
from tqdm import tqdm_notebook
from keras.layers import LSTM,Bidirectional,TimeDistributed, Embedding,Dense,Input,GlobalMaxPool1D,Flatten,Dropout
from keras.layers import CuDNNLSTM,CuDNNGRU,GlobalAveragePooling1D,GlobalMaxPooling1D,concatenate
from keras.models import Sequential,Model
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.callbacks import ModelCheckpoint
from keras import backend as K
from keras.optimizers import Adam
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints
import gc
lambda_elastic = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
elastic = make_pipeline(RobustScaler() ,
ElasticNetCV(alphas = lambda_elastic, cv= kf, max_iter=1e7))<choose_model_class>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
randfor = RandomForestRegressor(
 n_estimators=1200,
max_depth=15,
min_samples_split=5,
min_samples_leaf=5,
oob_score=True,
random_state=123 )<choose_model_class>
df_train = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv")
 df = pd.concat([df_train ,df_test])
 print("Unsincere avg: ", df_train.target.mean())
 print("total: ", df_train.target.count())
 print('Test samples: ', df_test.qid.count())
 df_train.head()
svmr = make_pipeline(RobustScaler() ,
SVR(
 C=21,
epsilon=0.01,
gamma=0.0003))<choose_model_class>
def add_lower(embedding, vocab):
count = 0
for word in vocab:
if word in embedding and word.lower() not in embedding:
embedding[word.lower() ] = embedding[word]
count += 1
print(f"Added {count} words to embedding")
 def build_vocab(texts):
sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
vocab = build_vocab(df['question_text'] )
xgb = XGBRegressor(
 learning_rate=0.01,
n_estimators=5000,
max_depth=4,
min_child_weight=0,
subsample=0.7,
colsample_bytree=0.7,
nthread=-1,
seed=123 )<choose_model_class>
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have",
"couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not",
"hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did",
"how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will",
"I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will",
"i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have",
"it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not",
"might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not",
"mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not",
"oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would",
"she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have",
"shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would",
"that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is",
"here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have",
"they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have",
"we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will",
"what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is",
"when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will",
"who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have",
"won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have",
"y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have",
"you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are",
"you've": "you have" }
lgbm = LGBMRegressor(
 n_estimators=5000,
learning_rate=0.01,
num_leaves=6,
max_bin=150,
colsample_bytree=0.3,
min_child_samples=8,
random_state=123,
verbose=-1 )<compute_train_metric>
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
score_ridge = -cross_val_score(ridge, X, y, scoring='neg_root_mean_squared_error', cv=kf ).mean()
print("Ridge Regression RMSE score: {}".format(score_ridge))
score_lasso = -cross_val_score(lasso, X, y, scoring='neg_root_mean_squared_error', cv=kf ).mean()
print("Lasso Regression RMSE score: {}".format(score_lasso))
score_elastic = -cross_val_score(elastic, X, y, scoring='neg_root_mean_squared_error', cv=kf ).mean()
print("Elasticnet Regression RMSE score: {}".format(score_elastic))
score_randfor = -cross_val_score(randfor, X, y, scoring='neg_root_mean_squared_error', cv=kf ).mean()
print("Random Forest Regressor RMSE score: {}".format(score_randfor))
score_svmr = -cross_val_score(svmr, X, y, scoring='neg_root_mean_squared_error', cv=kf ).mean()
print("Support Vector Regressor RMSE score: {}".format(score_svmr))
score_xgb = -cross_val_score(xgb, X, y, scoring='neg_root_mean_squared_error', cv=kf ).mean()
print("XGBoost Regressor RMSE score: {}".format(score_xgb))
score_lgbm = -cross_val_score(lgbm, X, y, scoring='neg_root_mean_squared_error', cv=kf ).mean()
print("Light GBM RMSE score: {}".format(score_lgbm))<create_dataframe>
punct = "/-'?!.,
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-",
"–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity',
'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3',
'π': 'pi', }
scores_df = pd.DataFrame(data=np.array([score_ridge, score_lasso, score_elastic, score_randfor,
score_svmr, score_xgb, score_lgbm]),
index=['Ridge','Lasso','ElasticNet', 'Random Forest', 'Support Vector', 'XGBoost', 'LGBM'],
columns=['Root_Mean_Sq_Err'])
 scores_df = scores_df.sort_values(by='Root_Mean_Sq_Err')
 scores_df<choose_model_class>
def clean_special_chars(text, punct, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text
stacked_model = StackingCVRegressor(
 regressors=(elastic, lasso, ridge, svmr, lgbm, xgb),
meta_regressor=elastic,
use_features_in_secondary=True,
n_jobs=-1 )<train_model>
mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling',
'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor',
'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ',
'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do',
'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do',
'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate',
"mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist',
'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend',
'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization',
'demonitization': 'demonetization', 'demonetisation': 'demonetization', 'pokémon': 'pokemon'}
stacked_fit = stacked_model.fit(np.array(X), np.array(y))
y_preds = stacked_fit.predict(np.array(X))
stacked_rmse = mse(y, y_preds, squared=False)
 print('RMSE score of Stacked Model(training data): {}'.format(stacked_rmse))<load_from_csv>
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission['SalePrice'] = np.floor(np.expm1(stacked_fit.predict(np.array(X_test))))<feature_engineering>
df_train['treated_question'] = df_train['question_text'].apply(lambda x: x.lower())
 df_test['treated_question'] = df_test['question_text'].apply(lambda x: x.lower())
 df_train['treated_question'] = df_train['treated_question'].apply(lambda x: clean_contractions(x, contraction_mapping))
df_test['treated_question'] = df_test['treated_question'].apply(lambda x: clean_contractions(x, contraction_mapping))
df_train['treated_question'] = df_train['treated_question'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))
df_test['treated_question'] = df_test['treated_question'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))
df_train['treated_question'] = df_train['treated_question'].apply(lambda x: correct_spelling(x, mispell_dict))
df_test['treated_question'] = df_test['treated_question'].apply(lambda x: correct_spelling(x, mispell_dict))
q1 = submission['SalePrice'].quantile(0.005)
 q2 = submission['SalePrice'].quantile(0.995)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x > q1 else x*0.77)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x < q2 else x*1.1 )<save_to_csv>
train, val = train_test_split(df_train, test_size=0.1, random_state=42)
 n_words = 50000
tokenizer = Tokenizer(num_words=n_words)
 tokenizer.fit_on_texts(list(train.treated_question))
q_train = tokenizer.texts_to_sequences(train.treated_question)
 q_val = tokenizer.texts_to_sequences(val.treated_question)
 q_test = tokenizer.texts_to_sequences(df_test.treated_question)
 max_len = 100
q_train = pad_sequences(q_train,maxlen=max_len)
 q_val = pad_sequences(q_val,maxlen=max_len)
 q_test = pad_sequences(q_test,maxlen=max_len)
 y_train = train.target
y_val = val.target
del train,val,df_train,df
gc.collect()
submission.to_csv("submission.csv", index=False )<save_to_csv>
def f1(y_true, y_pred):
def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
def optim_thres(y_val,y_pred):
score = 0
thresholds = np.arange(0.1,0.501,0.01)
 for thres in thresholds:
thres = np.round(thres,2)
 temp_pred =(y_pred > thres ).astype(int)
 temp_score = f1_score(y_val,temp_pred)
 print("Thres: {} --------- F1: {}".format(thres,temp_score))
if temp_score > score:
score = temp_score
final_thres = thres
return final_thres
submission.to_csv("submission.csv", index=False )<import_modules>
emb_file = ".. /input/embeddings/glove.840B.300d/glove.840B.300d.txt"
glove_dic = {}
for line in tqdm_notebook(open(emb_file)) :
temp = line.split(" ")
 glove_dic[temp[0]] = np.asarray(temp[1:],dtype='float32')
 add_lower(glove_dic, vocab)
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
word_index = tokenizer.word_index
emb_size = glove_dic['.'].shape[0]
emb_matrix = np.zeros(( n_words,emb_size))
for w,index in word_index.items() :
if index >= n_words:
continue
vec = glove_dic.get(w)
 if vec is not None:
emb_matrix[index,:] = vec
train=pd.read_csv('/kaggle/input/titanic/train.csv')
 test=pd.read_csv('/kaggle/input/titanic/test.csv')
 dataset = pd.concat(objs=[train, test], axis=0,sort=False ).reset_index(drop=True )<categorify>
inp = Input(shape=(max_len,))
x = Embedding(input_dim=n_words,output_dim=emb_size, weights=[emb_matrix],trainable=False )(inp)
 x1 = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x2 = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x1 = GlobalAveragePooling1D()(x1)
 x2 = GlobalAveragePooling1D()(x2)
 concat = concatenate([x1, x2])
 x = Dense(64, activation="relu" )(concat)
 drop = Dropout(0.1 )(x)
 x = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=inp,output=x)
 model.summary()
model_name = 'lstm_glove_emb'
checkpoint = ModelCheckpoint(filepath='./{}.hdf5'.format(model_name),
monitor='val_loss',mode='min',verbose=1,
save_best_only=True )
dataset['Ticket_Frequency'] = dataset.groupby('Ticket')['Ticket'].transform('count')
 dataset['Cabin_n'] = dataset['Cabin'].str[0]
dataset['Cabin_n'].fillna('M',inplace=True)
 dataset['Deck'] = dataset['Cabin'].apply(lambda s: s[0] if pd.notnull(s)else 'M')
 dataset['Deck'] = dataset['Deck'].replace(['A', 'B', 'C'], 'ABC')
 dataset['Deck'] = dataset['Deck'].replace(['D', 'E'], 'DE')
 dataset['Deck'] = dataset['Deck'].replace(['F', 'G'], 'FG')
 dataset['Deck'] = dataset['Deck'].replace(['T'], 'A')
 dataset.drop(labels = ["Cabin",'Ticket'], axis = 1, inplace = True)
 dataset["TFamily"] = dataset["Parch"] + dataset["SibSp"] + 1
dataset.loc[dataset['TFamily']==1 ,'IsAlone']=1
dataset.loc[dataset['TFamily']!=1 ,'IsAlone']=0
dataset['IsAlone']= dataset['IsAlone'].astype(int)
 dataset["Embarked"] = dataset["Embarked"].fillna("S")
 dataset['Age'] = dataset.groupby(["Sex", "Pclass"])["Age"].apply(lambda x: x.fillna(x.median()))
med_fare = dataset.groupby(['Pclass', 'Parch', 'SibSp'] ).Fare.median() [3][0][0]
dataset['Fare'] = dataset['Fare'].fillna(med_fare)
 dataset['Age_bin']=pd.cut(dataset['Age'], bins=10)
 dataset['TFamily_bin']=pd.cut(dataset['TFamily'], bins=2)
 dataset['Fare_bin']=pd.cut(dataset['Fare'], bins=13)
 dataset['Sex']=dataset['Sex'].map({'male':1,'female':0})
 dataset = pd.get_dummies(dataset, columns=['Embarked','Age_bin','Cabin_n','Fare_bin','TFamily_bin','Deck'])
 dataset.columns = dataset.columns.str.replace(',','')
 dataset.columns = dataset.columns.str.replace(']','')
 dataset_title = [i.split(",")[1].split(".")[0].strip() for i in dataset["Name"]]
Ticket = []
for i in list(train.Ticket):
if not i.isdigit() :
Ticket.append(i.replace(".","" ).replace("/","" ).strip().split(' ')[0])
 else:
Ticket.append("X")
 train["Ticket"] = Ticket
dataset.drop(labels = ['Name'], axis = 1, inplace = True)
 train_n=dataset[dataset['Survived'].notnull() ]
x_train=train_n.drop(labels=['Survived','PassengerId'],axis=1)
 y_train=train_n['Survived'].astype(int)
 X_train,X_test,Y_train,Y_test = train_test_split(x_train,y_train,test_size=0.25, random_state=0)
<choose_model_class>
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['acc',f1])
 history = model.fit(q_train,y_train,batch_size=1500,epochs=5,
validation_data=(q_val,y_val),verbose=1,callbacks=[checkpoint] )
kfold = StratifiedKFold(n_splits=10 )<choose_model_class>
del glove_dic
gc.collect()
model.load_weights('./{}.hdf5'.format(model_name))
y_pred_glove = model.predict(q_val,batch_size=1064,verbose=1)
 pred_glove = model.predict(q_test,batch_size=1064,verbose=1)
random_state = 2
classifiers = []
classifiers.append(SVC(random_state=random_state))
classifiers.append(DecisionTreeClassifier(random_state=random_state))
classifiers.append(AdaBoostClassifier(DecisionTreeClassifier(random_state=random_state),random_state=random_state,learning_rate=0.1))
classifiers.append(RandomForestClassifier(random_state=random_state))
classifiers.append(ExtraTreesClassifier(random_state=random_state))
classifiers.append(GradientBoostingClassifier(random_state=random_state))
classifiers.append(MLPClassifier(random_state=random_state))
classifiers.append(KNeighborsClassifier())
 classifiers.append(LogisticRegression(random_state = random_state))
classifiers.append(LinearDiscriminantAnalysis())
 classifiers.append(XGBClassifier())
 classifiers.append(LGBMClassifier())
 cv_results = []
for classifier in classifiers :
cv_results.append(cross_val_score(classifier, X_train, y = Y_train, scoring = "accuracy", cv = kfold, n_jobs=4))
cv_means = []
cv_std = []
for cv_result in cv_results:
cv_means.append(cv_result.mean())
 cv_std.append(cv_result.std())
 cv_res = pd.DataFrame({"CrossValMeans":cv_means,"CrossValerrors": cv_std,"Algorithm":["SVC","DecisionTree","AdaBoost",
"RandomForest","ExtraTrees","GradientBoosting","MultipleLayerPerceptron","KNeighboors","LogisticRegression","LinearDiscriminantAnalysis",'XGBClassifier','LGBMClassifier']})
 plt.figure(figsize=(20,10))
g = sns.barplot("CrossValMeans","Algorithm",data = cv_res, palette="Set3",orient = "h",**{'xerr':cv_std})
 plt.axvline(0.81)
 plt.axvline(0.84)
 g.set_xlabel("Mean Accuracy")
 g = g.set_title("Cross validation scores" )<train_model>
emb_file = ".. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt"
emb_dic = {}
for line in tqdm_notebook(open(emb_file,encoding='utf-8',errors='ignore')) :
temp = line.split(" ")
 emb_dic[temp[0]] = np.asarray(temp[1:],dtype='float32')
 emb_matrix = np.zeros(( n_words,emb_size))
for w,index in word_index.items() :
if index >= n_words:
continue
vec = emb_dic.get(w)
 if vec is not None:
emb_matrix[index,:] = vec
gbm = GradientBoostingClassifier(random_state=2)
 gbm.fit(X_train,Y_train)
 print('Score: ',gbm.score(X_test,Y_test))
feature_importances = pd.DataFrame(gbm.feature_importances_,index = X_test.columns,columns=['importance'] ).sort_values('importance', ascending=False)
 feature_importances.head(34 )<save_to_csv>
inp = Input(shape=(max_len,))
x = Embedding(input_dim=n_words,output_dim=emb_size, weights=[emb_matrix],trainable=False )(inp)
 x1 = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x2 = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x1 = GlobalAveragePooling1D()(x1)
 x2 = GlobalAveragePooling1D()(x2)
 concat = concatenate([x1, x2])
 x = Dense(64, activation="relu" )(concat)
 drop = Dropout(0.1 )(x)
 x = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=inp,output=x)
 model.summary()
model_name = 'lstm_paragram_emb'
checkpoint = ModelCheckpoint(filepath='./{}.hdf5'.format(model_name),
monitor='val_loss',mode='min',verbose=1,
save_best_only=True )
test_n=dataset[dataset['Survived'].isnull() ]
test_n.drop(labels = ["Survived"], axis = 1, inplace = True)
 submission=pd.DataFrame(columns=['PassengerId','Survived'])
 submission['PassengerId']=test_n['PassengerId']
test_n.drop(labels = ["PassengerId"], axis = 1, inplace = True)
 Y_pred=gbm.predict(test_n)
 submission['Survived']=Y_pred.astype(int)
 submission.to_csv('submission_gbm.csv',header=True,index=False )<set_options>
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['acc',f1])
 history = model.fit(q_train,y_train,batch_size=1500,epochs=5,
validation_data=(q_val,y_val),verbose=1,callbacks=[checkpoint] )
plt.rc("font", size=14)
 sns.set(style="dark")
 sns.set(style="darkgrid", color_codes=True)
 RED = "\033[1;31m"
BLUE = "\033[1;34m"
CYAN = "\033[1;36m"
GREEN = "\033[0;32m" 
<load_from_csv>
del emb_dic
gc.collect()
model.load_weights('./{}.hdf5'.format(model_name))
y_pred_paragram = model.predict(q_val,batch_size=1064,verbose=1)
 pred_paragram = model.predict(q_test,batch_size=1064,verbose=1)
titanic_df = pd.read_csv(".. /input/titanic/train.csv")
 test_df = pd.read_csv(".. /input/titanic/test.csv")
 titanic_df.head(5)
<drop_column>
emb_file = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
emb_dic = {}
for line in tqdm_notebook(open(emb_file,encoding='utf-8',errors='ignore')) :
temp = line.split(" ")
 emb_dic[temp[0]] = np.asarray(temp[1:],dtype='float32')
 emb_matrix = np.zeros(( n_words,emb_size))
for w,index in word_index.items() :
if index >= n_words:
continue
vec = emb_dic.get(w)
 if vec is not None:
emb_matrix[index,:] = vec
train_data = titanic_df
train_data["Age"].fillna(28, inplace=True)
 train_data["Embarked"].fillna("S", inplace=True)
 train_data.drop('Cabin', axis=1, inplace=True )<categorify>
inp = Input(shape=(max_len,))
x = Embedding(input_dim=n_words,output_dim=emb_size, weights=[emb_matrix] )(inp)
 x1 = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x2 = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x1 = GlobalAveragePooling1D()(x1)
 x2 = GlobalAveragePooling1D()(x2)
 concat = concatenate([x1, x2])
 x = Dense(64, activation="relu" )(concat)
 drop = Dropout(0.1 )(x)
 x = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=inp,output=x)
 model.summary()
model_name = 'lstm_wiki_emb'
checkpoint = ModelCheckpoint(filepath='./{}.hdf5'.format(model_name),
monitor='val_loss',mode='min',verbose=1,
save_best_only=True )
train_data['TravelBuds']=train_data["SibSp"]+train_data["Parch"]
train_data['TravelAlone']=np.where(train_data['TravelBuds']>0, 0, 1)
 train_data.drop('SibSp', axis=1, inplace=True)
 train_data.drop('Parch', axis=1, inplace=True)
 train_data.drop('TravelBuds', axis=1, inplace=True)
 train2 = pd.get_dummies(train_data, columns=["Pclass"])
 train3 = pd.get_dummies(train2, columns=["Embarked"])
 train4=pd.get_dummies(train3, columns=["Sex"])
 train4.drop('Sex_female', axis=1, inplace=True)
 train4.drop('PassengerId', axis=1, inplace=True)
 train4.drop('Name', axis=1, inplace=True)
 train4.drop('Ticket', axis=1, inplace=True)
 train4.head(5)
 df_final = train4<drop_column>
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['acc',f1])
 history = model.fit(q_train,y_train,batch_size=1500,epochs=5,
validation_data=(q_val,y_val),verbose=1,callbacks=[checkpoint] )
test_df["Age"].fillna(28, inplace=True)
 test_df["Fare"].fillna(14.45, inplace=True)
 test_df.drop('Cabin', axis=1, inplace=True )<categorify>
del emb_dic
gc.collect()
model.load_weights('./{}.hdf5'.format(model_name))
y_pred_wiki = model.predict(q_val,batch_size=1064,verbose=1)
 pred_wiki = model.predict(q_test,batch_size=1064,verbose=1 )
test_df['TravelBuds']=test_df["SibSp"]+test_df["Parch"]
test_df['TravelAlone']=np.where(test_df['TravelBuds']>0, 0, 1)
 test_df.drop('SibSp', axis=1, inplace=True)
 test_df.drop('Parch', axis=1, inplace=True)
 test_df.drop('TravelBuds', axis=1, inplace=True)
 test2 = pd.get_dummies(test_df, columns=["Pclass"])
 test3 = pd.get_dummies(test2, columns=["Embarked"])
 test4=pd.get_dummies(test3, columns=["Sex"])
 test4.drop('Sex_female', axis=1, inplace=True)
 test4.drop('PassengerId', axis=1, inplace=True)
 test4.drop('Name', axis=1, inplace=True)
 test4.drop('Ticket', axis=1, inplace=True)
 final_test = test4<feature_engineering>
y_pred_all = np.mean(np.array([y_pred_glove,y_pred_paragram,y_pred_wiki]),axis=0)
 print(y_pred_all.shape)
 final_thresh = optim_thres(y_val,y_pred_all )
df_final['IsMinor']=np.where(train_data['Age']<=16, 1, 0 )<feature_engineering>
final_pred = np.mean(np.array([pred_glove,pred_paragram,pred_wiki]),axis=0)
 sub_pred =(final_pred > final_thresh ).astype(int)
 sub = pd.DataFrame({"qid":df_test["qid"].values})
 sub['prediction'] = sub_pred
sub.to_csv("submission.csv", index=False )