Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<correct_missing_values>
import math
import re
import numpy as np
import pandas as pd
from gensim.models import KeyedVectors
from keras.models import Sequential, Model
from keras.layers import Dense, CuDNNGRU, Dropout, Embedding, SpatialDropout1D, GlobalAveragePooling1D, Bidirectional
from keras.layers import GlobalMaxPooling1D, CuDNNLSTM, Input, concatenate, Lambda, CuDNNLSTM
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.callbacks import Callback
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score, precision_score, recall_score
print(len(train))
train = train.dropna(subset=['Embarked'])
 print(len(train))<categorify>
num_features = 300
num_steps = 120
num_steps_lstm = num_steps // 2
batch_size = 512
val_size = 0
train['Age'] = train.groupby("Pclass")['Age'].transform(lambda x: x.fillna(x.median()))
test['Age'] = test.groupby("Pclass")['Age'].transform(lambda x: x.fillna(x.median()))<count_missing_values>
def load_csv() :
train_df = pd.read_csv(".. /input/train.csv")
 train_df, val_df = train_test_split(train_df, test_size=val_size)
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train size: %d, Validation size: %d, Test size: %d" %(len(train_df), len(val_df), len(test_df)))
 return train_df, val_df, test_df
print(train["Age"].isnull().sum())
 print(test["Age"].isnull().sum() )<categorify>
def pre_process_data(data_set_name, sentences):
replaced_sentences = []
if len(sentences)> 0:
non_word_re = re.compile('([^a-zA-Z])')
 replaced_sentences = [non_word_re.sub(r' \1 ', s)for s in sentences]
print("%s processed(%d lines)" %(data_set_name, len(sentences)))
 return replaced_sentences
train = pd.get_dummies(data = train, columns = ["Sex", "Embarked", "Pclass"])
 test = pd.get_dummies(data = test, columns = ["Sex", "Embarked", "Pclass"] )<prepare_x_and_y>
def load_data() :
train_df, val_df, test_df = load_csv()
X_train = pre_process_data("train question_text", train_df.question_text)
 X_val = pre_process_data("validation question_text", val_df.question_text)
 X_test = pre_process_data("test question_text", test_df.question_text)
 tokenizer = Tokenizer(filters='', lower=False)
 tokenizer.fit_on_texts(X_train)
 tokenizer.fit_on_texts(X_val)
 tokenizer.fit_on_texts(X_test)
 X_train = tokenizer.texts_to_sequences(X_train)
 X_val = tokenizer.texts_to_sequences(X_val)
 X_test = tokenizer.texts_to_sequences(X_test)
 X_train = pad_sequences(X_train, maxlen=num_steps, padding='post')
 X_val = pad_sequences(X_val, maxlen=num_steps, padding='post')
 X_test = pad_sequences(X_test, maxlen=num_steps, padding='post')
 print("words to sequences mapped")
 return X_train, train_df.target.values, X_val, val_df.target.values, test_df.qid.values, X_test, tokenizer.word_index
X_train = train.drop("Survived", axis = 1)
 y_train = train["Survived"]<find_best_model_class>
X_train, y_train, X_val, y_val, test_qids, X_test, word_index = load_data()
def fit_model(algo, X_train, y_train, cv):
model = algo.fit(X_train, y_train)
 y_pred = algo.predict(X_train)
 accuracy = round(accuracy_score(y_train, y_pred)* 100 , 2)
 y_pred_cv = cross_val_predict(algo, X_train, y_train, cv = cv)
 accuracy_cv = round(accuracy_score(y_train, y_pred_cv)* 100 , 2)
 return y_pred_cv, accuracy, accuracy_cv<train_on_grid>
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32' )
y_pred_cv_lr, accuracy_lr, accuracy_cv_lr = fit_model(LogisticRegression(random_state = 3),
X_train, y_train, 10)
 print("Accuracy : ",accuracy_lr)
 print("Accuracy CV :",accuracy_cv_lr )<train_model>
def load_glove() :
path = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
glove = dict(get_coefs(*o.split(" ")) for o in open(path)if len(o)> 100)
 print("glove loaded")
 return glove
rf = RandomForestClassifier(n_estimators = 100, random_state = 3)
 rf.fit(X_train, y_train)
 y_train_pred = rf.predict(X_train)
 print('Confusion Matrix : ','
', confusion_matrix(y_train, y_train_pred))
print()
print("Accuracy : ", round(accuracy_score(y_train, y_train_pred)* 100, 2))<choose_model_class>
def get_word_vec(word, vocab):
if word in vocab:
return vocab[word]
wl = word.lower()
if wl in vocab:
return vocab[wl]
wu = word.upper()
if wu in vocab:
return vocab[wu]
vec = np.zeros(num_features)
 i = word_index[word]
j = 0
while i > 0:
vec[j] =(i % 1000)/ 1000
j = j + 1
i = i // 1000
return vec
rfc = RandomForestClassifier(random_state=3)
 params = {'n_estimators' : sp_randint(50,200),
'max_depth' : sp_randint(2,100),
'max_depth' : sp_randint(2,100),
'min_samples_split' : sp_randint(2,100),
'min_samples_leaf' : sp_randint(1,200),
'criterion' : ['gini', 'entropy']}
rsearch_rfc = RandomizedSearchCV(rfc, param_distributions = params, n_iter = 100, cv = 3, scoring = 'roc_auc', n_jobs = -1,\
return_train_score = True, random_state = 3)
 rsearch_rfc.fit(X_train,y_train )<find_best_params>
def build_embedding_matrix(load_vocab_func):
vocab = load_vocab_func()
unk_dict = dict()
embedding_matrix = np.zeros(( len(word_index)+ 1, num_features))
for word, i in word_index.items() :
embedding_matrix[i] = get_word_vec(word, vocab)
 print("embedding weight matrix built")
 return embedding_matrix
rsearch_rfc.best_params_<train_model>
embedding_matrix = build_embedding_matrix(load_glove )
rfc = RandomForestClassifier(**rsearch_rfc.best_params_, random_state = 3)
 rfc.fit(X_train, y_train)
 y_train_pred = rfc.predict(X_train)
 print('Confusion Matrix : ','
', confusion_matrix(y_train, y_train_pred))
print()
print("Accuracy : ", round(accuracy_score(y_train, y_train_pred)* 100, 2))<create_dataframe>
def calculate_f1(y_true, y_pred, threshold):
y_pred_int =(y_pred > threshold ).astype(int)
 s = np.sum(y_pred_int)
 rate = s / len(y_pred_int)
 if(s == 0 or s == len(y_pred_int)) :
return 0.0, y_pred_int, rate
f1 = f1_score(y_true, y_pred_int)
 return f1, y_pred_int, rate
imp = pd.DataFrame(rfc.feature_importances_, index = X_train.columns, columns = ['imp'])
 imp = imp.sort_values(by ='imp', ascending = False)
 imp<predict_on_test>
def f1_matrics(y_true, y_pred):
t_max, rate_max, f1_max, p, r = 0.0, 0.0, 0.0, 0.0, 0.0
for i in range(200, 500, 5):
t = i * 0.001
f1, y_pred_int, rate = calculate_f1(y_true, y_pred, t)
 if(f1 > f1_max):
f1_max, y_int_max, rate_max, t_max = f1, y_pred_int, rate, t
if f1_max > 0.0:
p = precision_score(y_true, y_int_max)
 r = recall_score(y_true, y_int_max)
 print("[Threshold: %.6f, Rate: %.6f, F1_Score: %.6f, Precision: %.6f, Recall: %.6f]
" %(t_max, rate_max, f1_max, p, r))
y_test_pred = rfc.predict(test )<prepare_output>
class F1Evaluation(Callback):
def __init__(self, X, y_true):
super(Callback, self ).__init__()
self.X = X
self.y_true = y_true
def on_epoch_end(self, epoch, logs={}):
if(len(self.X)> 0):
y_preds = self.model.predict(self.X)
 for y in y_preds:
f1_matrics(self.y_true, y )
submission = pd.DataFrame()
submission['PassengerId'] = df_test['PassengerId']
submission['Survived'] = y_test_pred
submission.head()<save_to_csv>
def build_model_branch(x, gru=True):
if gru:
rnn = Bidirectional(CuDNNGRU(200, return_sequences=True))(x)
 else:
rnn = Bidirectional(CuDNNLSTM(200, return_sequences=True))(x)
 part = 2
cells = []
for i in range(part):
rnn_p = Lambda(lambda x: x[:,i::part,:] )(rnn)
 avg = GlobalAveragePooling1D()(rnn_p)
 drp = Dropout(0.3 )(avg)
 den = Dense(16, activation='relu' )(drp)
 cells.append(den)
 x = concatenate(cells)
 x = Dropout(0.3 )(x)
 return Dense(1, activation='sigmoid' )(x )
submission.to_csv('rf_submission.csv', index=False)
 print('Submission CSV is ready!' )<load_from_csv>
def build_model() :
inp = Input(shape=(num_steps,))
emb = Embedding(len(embedding_matrix), num_features, weights=[embedding_matrix], trainable=False )(inp)
 emb = SpatialDropout1D(0.3 )(emb)
 y = build_model_branch(emb)
 x = Lambda(lambda x: x[:, :num_steps_lstm, :] )(emb)
 z = build_model_branch(x, gru=False)
 x = concatenate([y, z])
 x = Dense(8, activation='relu' )(x)
 x = Dense(1, activation='sigmoid' )(x)
 mdl = Model(inputs=inp, outputs=[x, y, z])
 mdl.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 mdl.summary()
return mdl
submissions_check = pd.read_csv("rf_submission.csv")
 submissions_check.head()<set_options>
model = build_model()
pd.plotting.register_matplotlib_converters()
%matplotlib inline 
<load_from_csv>
f1_evl = F1Evaluation(X_val, y_val)
 model.fit(X_train, [y_train] * 3, batch_size=batch_size, epochs=6, callbacks=[f1_evl] )
X_path = '.. /input/house-prices-advanced-regression-techniques/train.csv'
X = pd.read_csv(X_path, index_col='Id')
 test_path = ".. /input/house-prices-advanced-regression-techniques/test.csv"
X_test = pd.read_csv(test_path, index_col='Id')
 X.dropna(axis=0, subset=['SalePrice'], inplace=True)
 y = X.SalePrice
X.drop(['SalePrice'], axis=1, inplace=True)
 X.head()<define_variables>
def get_pred_int(y_pred):
y = y_pred.reshape(-1)
 n_pos = math.floor(len(y)* 0.07)
 ind_pred = np.argsort(y)[-n_pos:]
y_int = np.zeros(y_pred.shape, dtype=int)
 y_int.flat[ind_pred] = 1
return y_int, y[ind_pred[0]]
num_cols = [col for col in X.columns if X[col].dtypes in ['int64', 'float64']]
print(num_cols)
 print()
print(len(num_cols))<define_variables>
y_preds = model.predict(X_test, batch_size=1024, verbose=0 )
cat_cols = [col for col in X.columns if X[col].dtypes=='object']
print(cat_cols)
 print()
print(len(cat_cols))<count_missing_values>
sub_df = pd.DataFrame()
sub_df['qid'] = test_qids
sub_df['prediction'], _ = get_pred_int(y_preds[0])
 sub_df.to_csv("submission.csv", columns=['qid', 'prediction'], index_label=False, index=False )