Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<groupby>
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers
open_years = []
for i in train_select['Open Date']:
open_years.append(count_years(i))
df_open_years = pd.DataFrame({ 'open_years' : open_years })
 group_years = df_open_years['open_years'].value_counts()
city_most = train_select['City'].value_counts() [train_select['City'].value_counts() > 2].index
city_transform = []
for i in train_select['City']:
if i in city_most:
city_transform.append(i)
 else:
city_transform.append('other')
 df_city_transform = pd.DataFrame({ 'city_transform' : city_transform })
 group_city = df_city_transform['city_transform'].value_counts()
train_final = pd.concat([train_select, df_open_years, df_city_transform], axis=1 ).drop(['Open Date', 'City'], axis=1 )<categorify>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )
X = train_final.drop(['revenue_log'], axis=1)
 y = train_final['revenue_log']
X = pd.get_dummies(X)
 X = np.array(X)
 y = np.array(y )<import_modules>
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)
 embed_size = 300
max_features = 50000
maxlen = 100
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
from sklearn.model_selection import GridSearchCV, cross_val_score, learning_curve
from sklearn.metrics import mean_absolute_error
from sklearn.linear_model import Lasso
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Ridge
from sklearn.linear_model import ElasticNet
from sklearn.neighbors import KNeighborsRegressor
from sklearn.svm import SVR
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.ensemble import AdaBoostRegressor
from sklearn.tree import DecisionTreeRegressor
from xgboost import XGBRegressor<choose_model_class>
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )
random_state = 2
classifiers = []
classifiers.append(Lasso(random_state=random_state))
classifiers.append(LinearRegression())
 classifiers.append(Ridge(random_state=random_state))
classifiers.append(ElasticNet(random_state=random_state))
classifiers.append(KNeighborsRegressor())
 classifiers.append(SVR())
 classifiers.append(RandomForestRegressor(random_state=random_state))
classifiers.append(GradientBoostingRegressor())
 classifiers.append(AdaBoostRegressor(random_state = random_state))
classifiers.append(DecisionTreeRegressor())
 classifiers.append(XGBRegressor())
 cv_results = []
for classifier in classifiers :
cv_results.append(cross_val_score(classifier, X, y, scoring='neg_mean_squared_error', cv =10, n_jobs=4))
cv_means = []
cv_std = []
for cv_result in cv_results:
cv_means.append(cv_result.mean())
 cv_std.append(cv_result.std())
 cv_res = pd.DataFrame({"CrossValMeans":cv_means,"CrossValerrors": cv_std,"Algorithm":["Lasso","LinearRegression","Ridge",
"ElasticNet","KNeighborsRegressor","SVR","RandomForestRegressor","GradientBoostingRegressor","AdaBoostRegressor","DecisionTreeRegressor", "XGBRegressor"]})
<sort_values>
model.fit(train_X, train_y, batch_size=512, epochs=4, validation_data=(val_X, val_y))
cv_res.sort_values(ascending=False, by='CrossValMeans' )<train_on_grid>
pred_noemb_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_noemb_val_y>thresh ).astype(int))))
model = SVR()
ex_param_grid = {
'kernel': ['rbf'],
'gamma': [1e-4, 1e-3, 0.01, 0.1],
'C': [1, 10, 100]
}
gsSVR = GridSearchCV(model,
param_grid = ex_param_grid,
cv=10,
scoring="neg_mean_squared_error")
 gsSVR.fit(X, y)
 SVR_best = gsSVR.best_estimator_
print('SVR')
 print('Best score : ', gsSVR.best_score_)
 print('Best params : ', gsSVR.best_params_ )<train_on_grid>
pred_noemb_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
model = AdaBoostRegressor(random_state = random_state)
 ex_param_grid = {
'n_estimators': [50, 100],
'learning_rate' : [0.01,0.05,0.1,0.3,1],
'loss' : ['linear', 'square', 'exponential']
}
gsABR = GridSearchCV(model,
param_grid = ex_param_grid,
cv=10,
scoring="neg_mean_squared_error")
 gsABR.fit(X, y)
 ABR_best = gsABR.best_estimator_
print('ABR')
 print('Best score : ', gsABR.best_score_)
 print('Best params : ', gsABR.best_params_ )<train_on_grid>
del model, inp, x
time.sleep(10 )
model = RandomForestRegressor(random_state = random_state)
 ex_param_grid = {
'n_estimators' : [10,20,30,40,50],
'max_features' : ["auto", "sqrt", "log2"],
'min_samples_split' : [2,4,8,10,12,14,16]
}
gsRFR = GridSearchCV(model,
param_grid = ex_param_grid,
cv=10,
scoring="neg_mean_squared_error")
 gsRFR.fit(X, y)
 RFR_best = gsRFR.best_estimator_
print('RFR')
 print('Best score : ', gsRFR.best_score_)
 print('Best params : ', gsRFR.best_params_ )<train_on_grid>
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))
model = KNeighborsRegressor()
ex_param_grid = {
'n_neighbors': [4,6,8,10],
'leaf_size': [30,40,50,60],
'weights': ['uniform','distance']
}
gsKNR = GridSearchCV(model,
param_grid = ex_param_grid,
cv=10,
scoring="neg_mean_squared_error")
 gsKNR.fit(X, y)
 KNR_best = gsKNR.best_estimator_
print('KNR')
 print('Best score : ', gsKNR.best_score_)
 print('Best params : ', gsKNR.best_params_ )<train_on_grid>
pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))))
model = GradientBoostingRegressor(random_state = random_state)
 ex_param_grid = {
'learning_rate': [0.25, 0.1, 0.05, 0.01],
'n_estimators': [50,100,200,300],
'max_depth': [3,5,7]
}
gsGBR = GridSearchCV(model,
param_grid = ex_param_grid,
cv=10,
scoring="neg_mean_squared_error")
 gsGBR.fit(X, y)
 GBR_best = gsGBR.best_estimator_
print('GBR')
 print('Best score : ', gsGBR.best_score_)
 print('Best params : ', gsGBR.best_params_ )<drop_column>
pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1)
 pred_glove_test_y =(pred_glove_test_y>0.35 ).astype(int )
attr_select = ['Open Date', 'City', 'City Group', 'Type', 'P2', 'P6', 'P13', 'P28', 'P29']
test_select = data_test[attr_select]<groupby>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
open_years = []
for i in test_select['Open Date']:
open_years.append(count_years(i))
df_open_years = pd.DataFrame({ 'open_years' : open_years })
 group_years = df_open_years['open_years'].value_counts()
city_most = train_select['City'].value_counts() [train_select['City'].value_counts() > 2].index
city_transform = []
for i in test_select['City']:
if i in city_most:
city_transform.append(i)
 else:
city_transform.append('other')
 df_city_transform = pd.DataFrame({ 'city_transform' : city_transform })
 group_city = df_city_transform['city_transform'].value_counts()
test_final = pd.concat([test_select, df_open_years, df_city_transform], axis=1 ).drop(['Open Date', 'City'], axis=1 )<categorify>
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )
X_test = pd.get_dummies(test_final ).drop(['Type_MB'], axis=1)
 test = pd.get_dummies(X_test )<train_model>
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))
votingR = VotingRegressor(estimators=[('svr', SVR_best),('abr', ABR_best), 
('gbr', GBR_best),('rfr', RFR_best),('knr', KNR_best)], n_jobs=4)
 votingR = votingR.fit(X, y )<save_to_csv>
pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_fasttext_val_y>thresh ).astype(int))))
predict_test = pd.Series(np.exp(votingR.predict(test)) , name="Prediction")
 results = pd.concat([IDtest, predict_test],axis=1)
 results.to_csv("my_prediction.csv",index=False )<import_modules>
pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1)
 pred_fasttext_test_y =(pred_fasttext_test_y>0.33 ).astype(int )
import pandas as pd
import numpy as np<import_modules>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
import pandas as pd
import numpy as np<load_from_csv>
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )
data = pd.read_csv('.. /input/train.csv')
 test_data = pd.read_csv('.. /input/test.csv')
 data.head()<feature_engineering>
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))
data['Type_IL'] = np.where(data['Type'] == 'IL', 1, 0)
 data['Type_FC'] = np.where(data['Type'] == 'FC', 1, 0)
 data['Type_DT'] = np.where(data['Type'] == 'DT', 1, 0)
 data['Big_Cities'] = np.where(data['City Group'] == 'Big Cities', 1, 0)
 data['Days_Open'] =(pd.to_datetime('2015-03-23')- pd.to_datetime(data['Open Date'])).dt.days
data = data.drop('Type', axis=1)
 data = data.drop('City Group', axis=1)
 data = data.drop('City', axis=1)
 data = data.drop('Open Date', axis=1)
 test_data['Type_IL'] = np.where(test_data['Type'] == 'IL', 1, 0)
 test_data['Type_FC'] = np.where(test_data['Type'] == 'FC', 1, 0)
 test_data['Type_DT'] = np.where(test_data['Type'] == 'DT', 1, 0)
 test_data['Big_Cities'] = np.where(test_data['City Group'] == 'Big Cities', 1, 0)
 test_data['Days_Open'] =(pd.to_datetime('2015-03-23')- pd.to_datetime(test_data['Open Date'])).dt.days
test_data = test_data.drop('Type', axis=1)
 test_data = test_data.drop('City Group', axis=1)
 test_data = test_data.drop('City', axis=1)
 test_data = test_data.drop('Open Date', axis=1 )<prepare_x_and_y>
pred_paragram_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_paragram_val_y>thresh ).astype(int))))
X = data.drop(['Id', 'revenue'], axis=1)
 Y = data.revenue<compute_train_metric>
pred_paragram_test_y = model.predict([test_X], batch_size=1024, verbose=1)
 pred_paragram_test_y =(pred_paragram_test_y>0.34 ).astype(int )
def check_rmse(X, Y, alpha):
RMSE_lasso = []
RMSE_ridge = []
for i in alpha:
lasso = Lasso(alpha=i)
 lasso.fit(X, Y)
 ridge = Ridge(alpha=i)
 ridge.fit(X, Y)
 RMSE_lasso.append(metrics.mean_squared_error(Y, lasso.predict(X)))
 RMSE_ridge.append(metrics.mean_squared_error(Y, ridge.predict(X)))
 return(RMSE_lasso, RMSE_ridge )<train_model>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
lasso = Lasso(alpha=5.5)
 lasso.fit(X, Y)
 metrics.mean_squared_error(Y, lasso.predict(X))<save_to_csv>
pred_val_y = 1.5*pred_glove_val_y + 1.3*pred_fasttext_val_y + 0.9*pred_paragram_val_y
pred_val_y =(pred_val_y>=2 ).astype(int)
 print(metrics.f1_score(val_y,(pred_val_y)))
<save_to_csv><EOS> 
pred_test_y = 1.5*pred_glove_test_y + 1.3*pred_fasttext_test_y + 0.9*pred_paragram_test_y
pred_test_y =(pred_test_y>=2 ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )