Kaggle competition: Quora Insincere Questions Classification
train["Survived"] = train["Survived"].astype(int)
 X_train = train.drop(labels = ["Survived"],axis = 1)
 Y_train = train["Survived"]<prepare_x_and_y>
dim = 300
num_words = 75966
max_len = 100
print('Fiting tokenizer')
 tokenizer = Tokenizer(num_words=num_words)
 tokenizer.fit_on_texts(list(df['question_text'])+list(df_final['question_text']))
print('text to sequence')
 x_train = tokenizer.texts_to_sequences(df['question_text'])
 print('pad sequence')
 x_train = pad_sequences(x_train,maxlen=max_len)
 y_train = df['target'].values
print(x_train.shape)
 print(y_train.shape)
 x_test=tokenizer.texts_to_sequences(df_final['question_text'])
 x_test = pad_sequences(x_test,maxlen=max_len)
 print('Test data loaded:',x_test.shape )
X_test=test<choose_model_class>
print('Glove...')
 def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_glov = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_glov[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_glov.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()
kfold= StratifiedKFold(n_splits=10 )<choose_model_class>
print('Para...')
 EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_para = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_para[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_glov.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()
random_state=2
classifiers=[]
classifiers.append(SVC(random_state=random_state))
classifiers.append(DecisionTreeClassifier(random_state=random_state))
classifiers.append(RandomForestClassifier(random_state=random_state))
classifiers.append(KNeighborsClassifier())
 classifiers.append(LogisticRegression(random_state=random_state))
classifiers.append(LinearDiscriminantAnalysis())
 classifiers.append(AdaBoostClassifier(DecisionTreeClassifier(random_state=random_state),random_state=random_state))
classifiers.append(ExtraTreesClassifier(random_state=random_state))
classifiers.append(GradientBoostingClassifier(random_state=random_state))
classifiers.append(XGBClassifier(random_state=random_state))
cv_results=[]
for classifier in classifiers:
cv_results.append(cross_val_score(estimator=classifier,X=X_train,y=Y_train,cv=kfold,scoring='accuracy',n_jobs=-1))
cv_means=[]
cv_std=[]
for cv_result in cv_results:
cv_means.append(cv_result.mean())
 cv_std.append(cv_result.std())
 cv_res=pd.DataFrame({'CrossValMeans':cv_means , 'CrossValErrors':cv_std , 'Algorithm':['SVC','DTC','RFC','KNN','LR','LDA','ADA','XT','GBC','XGB']})
 sns.barplot('CrossValMeans','Algorithm',data=cv_res)
<choose_model_class>
matrixes = [embedding_matrix_glov,embedding_matrix_para]
matrix = np.mean(matrixes,axis=0)
 del embedding_matrix_glov,embedding_matrix_para
gc.collect()
svc_classifier=SVC(probability=True)
 svc_param_grid = [{'C': [1], 'kernel': ['rbf'],
'gamma': [0.1],
'cache_size':[100],
'coef0':[0.1],
'degree':[1],
'tol':[0.001]}]
gs_SVC = GridSearchCV(estimator = svc_classifier,
param_grid = svc_param_grid,
scoring = 'accuracy',
cv = kfold,
n_jobs = -1)
 gs_SVC = gs_SVC.fit(X_train, Y_train)
 svc_best_params = gs_SVC.best_params_
svc_best_score = gs_SVC.best_score_
svc_best=gs_SVC.best_estimator_
svc_best_score 
<choose_model_class>
def get_model(trainable=False):
inp1 = Input(shape=(max_len,))
emb = Embedding(num_words, dim, weights=[matrix],trainable = trainable, )(inp1)
 x,h_f,c_f,h_b,c_b = Bidirectional(CuDNNLSTM(128,return_sequences=True,return_state=True))(emb)
 x = concatenate([h_f,h_b])
 x = Dense(128, activation="relu" )(x)
 x = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=inp1, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
batch_size = 512
print('Batch size = ',batch_size)
 patience = 2
best_model=None
all_results = {}
train_meta = np.zeros(y_train.shape)
 test_meta = np.zeros(x_test.shape[0])
 splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=14 ).split(x_train, y_train))
for idx,(train_idx, valid_idx)in enumerate(splits):
print('----'+str(idx)+'-----')
 X_train1 = x_train[train_idx]
y_train1 = y_train[train_idx]
X_val = x_train[valid_idx]
y_val = y_train[valid_idx]
model_file = 'model_'+str(idx)+'.h5'
modelcheck = ModelCheckpoint(model_file,save_best_only=True)
 stop = EarlyStopping(patience=patience)
 model = get_model()
history = model.fit(X_train1,y_train1,
batch_size=batch_size,
validation_data=(X_val,y_val),
epochs=100,
callbacks=[modelcheck,stop],
verbose=2)
 print('training finished...')
 model.load_weights(model_file)
 y_pred = model.predict(X_val,batch_size=batch_size, verbose=1)
 train_meta[valid_idx] = y_pred.reshape(-1)
 search_result = threshold_search(y_val, y_pred)
 print(search_result)
 y_pred = y_pred>search_result['threshold']
y_pred = y_pred.astype(int)
 print('RESULTS ON VALIDATION SET:
',classification_report(y_val,y_pred))
all_results[model_file] = search_result['f1']
y_pred = model.predict(x_test,batch_size=batch_size, verbose=1)
 test_meta += y_pred.reshape(-1)/ len(splits)
 if best_model is None or best_model['f1'] < search_result['f1']:
best_model={'model':model_file,'f1':search_result['f1']}
print('-'*80)
 print(all_results)
 print('-'*80)
 print(best_model)
 print('-'*80 )
<choose_model_class><EOS> 
search_result = threshold_search(y_train, train_meta)
 print(search_result)
 df_subm = pd.DataFrame()
df_subm['qid'] = df_final.qid
df_subm['prediction'] = test_meta > search_result['threshold']
print(df_subm.head())
 df_subm.to_csv('submission.csv', index=False )