Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<load_from_csv>
import numpy as np
import pandas as pd
import string
from tqdm import tqdm
import operator
import os
import gc
import time
import re
from random import shuffle
from gensim.models import KeyedVectors
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from sklearn.metrics import f1_score
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.models import Sequential, Input
from keras.layers import Dense, Dropout, LSTM, Embedding, GlobalMaxPool1D, Conv1D, MaxPooling1D
from keras.layers import CuDNNLSTM, Bidirectional, CuDNNGRU, GlobalAvgPool1D, concatenate
from keras.models import Model
import tensorflow as tf
from keras import backend as K
from keras.engine.topology import Layer
import Levenshtein
traindf = pd.read_csv('.. /input/titanic/train.csv' ).set_index('PassengerId')
 testdf = pd.read_csv('.. /input/titanic/test.csv' ).set_index('PassengerId' )<feature_engineering>
d = pd.read_csv('.. /input/train.csv')
 Y_train = d['target']
X_train = d['question_text']
d_test = pd.read_csv('.. /input/test.csv')
 X_test = d_test['question_text']
df = pd.concat([traindf, testdf], axis=0, sort=False)
 df['Title'] = df.Name.str.split(',' ).str[1].str.split('.' ).str[0].str.strip()
df['IsWomanOrBoy'] =(( df.Title == 'Master')|(df.Sex == 'female'))
df['LastName'] = df.Name.str.split(',' ).str[0]
family = df.groupby(df.LastName ).Survived
df['WomanOrBoyCount'] = family.transform(lambda s: s[df.IsWomanOrBoy].fillna(0 ).count())
 df['WomanOrBoyCount'] = df.mask(df.IsWomanOrBoy, df.WomanOrBoyCount - 1, axis=0)
 df['FamilySurvivedCount'] = family.transform(lambda s: s[df.IsWomanOrBoy].fillna(0 ).sum())
 df['FamilySurvivedCount'] = df.mask(df.IsWomanOrBoy, df.FamilySurvivedCount - \
df.Survived.fillna(0), axis=0)
 df['WomanOrBoySurvived'] = df.FamilySurvivedCount / df.WomanOrBoyCount.replace(0, np.nan)
 df.WomanOrBoyCount = df.WomanOrBoyCount.replace(np.nan, 0)
 df['Alone'] =(df.WomanOrBoyCount == 0)
 df['Title'] = df['Title'].replace('Ms','Miss')
 df['Title'] = df['Title'].replace('Mlle','Miss')
 df['Title'] = df['Title'].replace('Mme','Mrs')
 df['Embarked'] = df['Embarked'].fillna('S')
 med_fare = df.groupby(['Pclass', 'Parch', 'SibSp'] ).Fare.median() [3][0][0]
df['Fare'] = df['Fare'].fillna(med_fare)
 df['Age'] = df.groupby(['Sex', 'Pclass', 'Title'])['Age'].apply(lambda x: x.fillna(x.median()))
df['Family_Size'] = df['SibSp'] + df['Parch'] + 1
df['Deck'] = df['Cabin'].apply(lambda s: s[0] if pd.notnull(s)else 'M')
 df.loc[(df['Deck'] == 'T'), 'Deck'] = 'A'
df.WomanOrBoySurvived = df.WomanOrBoySurvived.fillna(0)
 df.WomanOrBoyCount = df.WomanOrBoyCount.fillna(0)
 df.FamilySurvivedCount = df.FamilySurvivedCount.fillna(0)
 df.Alone = df.Alone.fillna(0 )<drop_column>
def encode_digit(x):

 x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
def clean_digits(string_vec):

 cleaned_string = [encode_digit(s)for s in string_vec]
return pd.Series(cleaned_string)
 def clean_ws(string_vec):

 cleaned_string = [re.sub('\s+', ' ', s ).strip() for s in string_vec]
return pd.Series(cleaned_string)
 def clean_word(char, punct):

 for p in punct:
char = char.replace(p, f' {p} ')
 return(char)
 def clean_punct(string_vec, punct):

 cleaned_string = []
for char in tqdm(string_vec):
char = [clean_word(x, punct)for x in char.split() ]
cleaned_string.append(' '.join(char))
return pd.Series(cleaned_string)
 def tokenize_text(string_vec, tokenizer, max_len):

 token = tokenizer.texts_to_sequences(string_vec)
 token = pad_sequences(token, maxlen = max_len)
 return token
train_x, test_x = df.loc[traindf.index], df.loc[testdf.index]
test_x = test_x.drop('Survived', axis=1 )<save_to_csv>
def load_from_text(path):

 def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(path, errors='ignore'))
return embeddings_index
def read_embedding(path, reading_type, binary = False):

 if(reading_type == 'text'):
model_embed = load_from_text(path)
 if(reading_type == 'word2vec'):
model_embed = KeyedVectors.load_word2vec_format(path, binary = binary)
 return model_embed
def create_embedding_matrix(model_embed, tokenizer, max_features, embed_size):

 embedding_matrix = np.zeros(( max_features, embed_size))
for word, index in tokenizer.word_index.items() :
if index > max_features - 1:
break
else:
try:
embedding_matrix[index] = model_embed[word]
except:
continue
return embedding_matrix
%%time
profile = ProfileReport(train_x, title='Pandas Profiling Report for training dataset', minimal=True)
 profile.to_file(output_file="train_short_profile.html" )<create_dataframe>
def find_most_similar(words, model_embed, window = 1, return_top = 1):

 all_keys = list(model_embed.keys())
 key_dict = {}
for key in all_keys:
key_dict.update({key: len(key)})
 vocab_mapper = []
for word in tqdm(words):
w_l = len(word)
 sub_key_dict = [k for k, v in key_dict.items() if w_l - window <= v <= w_l + window]
dist_list = [{word : key, 'dist': Levenshtein.distance(word, key)} for key in sub_key_dict]
dist_list = sorted(dist_list, key = lambda k: k['dist'])[:return_top]
vocab_mapper.append(dist_list)
 vocab = pd.DataFrame(columns = ['orig_word', 'suggestion', 'dist'])
 for entry in vocab_mapper:
for j in range(return_top):
keys = [*entry[j].keys() ]
suggestion = entry[j][keys[0]]
dist = entry[j][keys[1]]
vocab = vocab.append({'orig_word' : keys[0], 'suggestion' : suggestion, 'dist' : dist}, ignore_index = True)
 vocab = vocab.sort_values('orig_word')
 return vocab
def build_vocab(sentences, verbose = True):

 vocab = {}
for sentence in tqdm(sentences, disable =(not verbose)) :
for word in sentence.split() :
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
def check_coverage(vocab, model_embed):

 a = {}
oov = {}
k = 0
i = 0
for word in tqdm(vocab):
try:
a[word] = model_embed[word]
k += vocab[word]
except:
oov[word] = vocab[word]
i += vocab[word]
pass
print('Found embeddings for {:.2%} of vocab'.format(len(a)/ len(vocab)))
 print('Found embeddings for {:.2%} of all text'.format(k /(k + i)))
 sorted_x = sorted(oov.items() , key=operator.itemgetter(1)) [::-1]
return sorted_x
def spell_checker(string_vec, vocab_mapper):

 cleaned_strings = []
for char in string_vec:
cleaned_words = []
for x in char.split() :
if vocab_mapper.get(x)is not None:
x = vocab_mapper.get(x)
 cleaned_words.append(x)
 cleaned_words = ' '.join(cleaned_words)
 cleaned_strings.append(cleaned_words)
 return pd.Series(cleaned_strings)
test_x = pd.concat([test_x.WomanOrBoySurvived.fillna(0), test_x.Alone, \
test_x.Sex.replace({'male': 0, 'female': 1})], axis=1)
 pd.DataFrame({'Survived':(((test_x.WomanOrBoySurvived <= 0.2381)&(test_x.Sex > 0.5)&(test_x.Alone > 0.5)) | \ 
(( test_x.WomanOrBoySurvived > 0.2381)& \
~(( test_x.WomanOrBoySurvived > 0.55)&(test_x.WomanOrBoySurvived <= 0.633)))).astype(int)}, index=testdf.index ).reset_index().to_csv('submission.csv', index=False )<prepare_x_and_y>
def to_binary(p_array, treshold):

 y_hat = []
for i in range(len(p_array)) :
if p_array[i] > treshold:
y_hat.append(1)
 else:
y_hat.append(0)
 return y_hat
def optimal_treshold(y, yhat):

 best_threshold = 0
best_score = 0
for threshold in [i/100 for i in range(10,90)]:
score = f1_score(y, yhat > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
rawXy_train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv', index_col='Id')
 y = np.log(rawXy_train.SalePrice)
 rawX_train = rawXy_train.drop(columns=['SalePrice'])
 numeric_columns = [col for col in rawX_train.columns if rawX_train[col].dtype != "object"]
def transform_X(X):
X = X.fillna(0)
 X = X.join(np.sqrt(X[numeric_columns]), rsuffix="_SQRT")
 X = pd.get_dummies(X)
 return X
X_train = transform_X(rawX_train)
 cv_errors = np.abs(cross_val_predict(BayesianRidge() , X_train, y, n_jobs=-1, cv=10)- y)
 outliers = list(cv_errors[cv_errors >(np.mean(cv_errors)+ 2*np.std(cv_errors)) ].index)
 X_train = X_train.drop(outliers)
 y = y.drop(outliers)
 final_model = BayesianRidge().fit(X_train, y)
 X_test = transform_X(pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv', index_col='Id'))
X_test = pd.DataFrame({col: X_test.get(col, 0)for col in X_train.columns})
 predictions = np.exp(final_model.predict(X_test))
pd.DataFrame({'Id': X_test.index, 'SalePrice': predictions} ).to_csv('baseline-submission.csv', index=False )<load_from_csv>
def share_unique(string_vec):

 share_list = []
for char in string_vec:
sh = len(set(char.split())) /len(char.split())
 share_list.append(sh)
 return share_list
def share_capslock(string_vec):

 share_list = []
for char in string_vec:
to_upper = char.upper().split()
sh = len(set(char.split() ).intersection(to_upper)) /len(char.split())
 share_list.append(sh)
 return share_list
def ends_with_symbol(string_vec, symbol):

 return [int(x.endswith(symbol)) for x in string_vec]
def count_words(string_vec):

 return [len(x.split())for x in string_vec]
def count_occurance(string_vec):

 return [x.count('!')+ x.count('?')for x in string_vec]
test=pd.read_csv(".. /input/test.csv")
 test.head()<load_from_csv>
embed_dict = {'google': {'path' : '.. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin',
'reading' : 'word2vec',
'binary': True},
'glove': {'path': '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt',
'reading' : 'text',
'binary': False},
'wiki': {'path' : '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec',
'reading' : 'text',
'binary': False},
'paragram': {'path': '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt',
'reading' : 'text',
'binary' : False}
}
gender_submission=pd.read_csv(".. /input/gender_submission.csv")
 gender_submission.head()<count_missing_values>
embeds_to_use = ['glove']
df.isnull().sum()<filter>
index = 0
for embed in embeds_to_use:
print('Reading: ' + embed)
 path = embed_dict.get(embed)['path']
type_of_file = embed_dict.get(embed)['reading']
bin = embed_dict.get(embed)['binary']
embedding = read_embedding(path, type_of_file, bin)
 if embed == embeds_to_use[0]:
model_embed = embedding
else:
all_keys = [k for k in embedding.keys() ]
index += 1
for key in tqdm(all_keys):
try:
vect = np.concatenate([model_embed[key], embedding[key]])
 except:
vect = np.concatenate([np.zeros(( 1, 300 * index), dtype = 'float32')[0], embedding[key]])
 model_embed.update({key : vect})
 del vect
del all_keys
del embedding
gc.collect()
time.sleep(5 )
df[(df['Fare'].isnull())|(df['Embarked'].isnull())]<groupby>
vocab_mapper = {'Quorans' : 'Qurans',
'Blockchain' : 'blockchain',
'blockchains' : 'blockchain',
'demonetisation' : 'demonetization',
'ethereum' : 'Ethereum',
'Qoura' : 'Quora',
'SJWs' : 'SJW',
'bhakts' : 'bhakti',
'Bhakts' : 'Bhakti',
'kotlin' : 'Kotlin',
'narcissit' : 'narcissist',
'Trumpism' : 'Trump',
'Tamilans' : 'Tamilians',
'acturial' : 'actuarial',
'demonitization' : 'demonetization',
'Demonetization' : 'demonetization',
'demonitisation' : 'demonetization',
'Demonetisation' : 'demonetization',
'Whyis' : 'Why is',
'AirPods' : 'AirPod',
'Drumpf': 'Trumpf',
'Zhihu' : 'Zhihua',
'Neuralink' : 'Neurolink',
'fullform' : 'full-form',
'biharis' : 'Biharis',
'madheshi' : 'Madheshi',
'Xiomi' : 'Xiaomi',
'rohingya' : 'Rohingya',
'Despacito' : 'Desposito',
'schizoids' : 'schizoid',
'MHTCET' : 'MHT-CET',
'fortnite' : 'Fortnite',
'Bittrex' : 'Bitrex',
'ReactJS' : 'JavaScript',
'hyperloop' : 'Hyperloop',
'adhaar' : 'Aadhaar',
'Adhaar' : 'Aadhaar',
'Baahubali' : 'Bahubali',
'Cryptocurrency' : 'cryptocurrency',
'cryptocurrencies' : 'cryptocurrency',
'cryptocoins' : 'cryptocurrency',
"\u200b":" ",
"\ufeff" : "",
"2k17" : '2017',
"2k18" : '2018',
"nofap": 'no fap',
'Brexiting' : 'Brexit',
'mastuburation' : 'masturbation',
'quara' : 'Quora',
'Quoras' : 'Quora',
"fiancé" : "fiance",
'π' : 'pi',
'Pokémon' : 'Pokemon',
'€' : 'euro'
}
word_expansion = {"aren't" : 'are not',
"I'm" : 'I am',
"What's" : 'What is',
"don’t" : "do not",
"isn't" : "is not",
"I’m" : "I am",
'aren’t' : 'are not',
"Can't" : "cannot",
"can't" : "cannot",
"don't" : 'do not',
"How's" : "how is",
"we're" : 'we are',
"won't" : 'will not',
"they're" : "they are",
"he's" : "he is",
"doesn’t" : "does not",
"shouldn't" : "should not",
"Shouldn't" : "should not",
"hasn't" : "has not",
"couldn't" : "could not",
"I’ve" : "I have",
"aren't" : "are not",
"weren't" : 'were not'}
cabin_df.groupby('PassengerCount' ).count()<feature_engineering>
punct = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√']
punct = ''.join(punct)
 def preproc_pipeline(string_df, model_embed, punct, word_expansion = None, vocab_mapper = None):

 if(word_expansion is not None):
string_df = spell_checker(string_df, word_expansion)
 string_df = clean_punct(string_df, punct)
 if(vocab_mapper is not None):
string_df = spell_checker(string_df, vocab_mapper)
 string_df = clean_digits(string_df)
 string_df = clean_ws(string_df)
 return string_df
X_tr = preproc_pipeline(X_train, model_embed, punct, vocab_mapper = vocab_mapper)
 X_te = preproc_pipeline(X_test, model_embed, punct, vocab_mapper = vocab_mapper )
cabin_df['CabinOccupancy']=np.where(cabin_df['PassengerCount']==1,1,'')
 cabin_df['CabinOccupancy']=np.where(cabin_df['PassengerCount']==2,2,cabin_df['CabinOccupancy'])
 cabin_df['CabinOccupancy']=np.where(cabin_df['PassengerCount']==3,3,cabin_df['CabinOccupancy'])
 cabin_df['CabinOccupancy']=np.where(cabin_df['PassengerCount']==4,4,cabin_df['CabinOccupancy'])
 cabin_df['CabinOccupancy']=np.where(cabin_df['PassengerCount']==5,5,cabin_df['CabinOccupancy'])
 cabin_df['CabinOccupancy']=np.where(cabin_df['PassengerCount']==6,6,cabin_df['CabinOccupancy'])
 cabin_df['CabinOccupancy']=cabin_df['CabinOccupancy'].astype(str)
 cabin_df.tail()<merge>
vocab = build_vocab(X_tr)
 oov = check_coverage(vocab, model_embed)
 oov = [x[0] for x in oov if x[1] > 60]
synonyms = find_most_similar(oov, model_embed, return_top = 3)
 print(synonyms )
df=pd.merge(df,cabin_df[['Cabin','CabinOccupancy']],how='left',on='Cabin')
 df.head()<feature_engineering>
maxlen = 70
max_features = 120000
batch_size = 512
numb_epoch = 2
embed_size = 300 * len(embeds_to_use )
block=df['Cabin'].str.split('([A-Za-z]+ )(\d+)', expand=True)
 block['block']=np.where(block[1].isnull() ,block[3],block[1])
 block['block']=np.where(block['block'].isnull() ,block[5],block['block'])
 block['block']=np.where(block['block'].isnull() ,block[7],block['block'])
 block['block']=np.where(block['block'].isnull() ,block[9],block['block'])
 block['block']=np.where(block['block'].isnull() ,block[11],block['block'])
 block.head()<merge>
def RNN_model(maxlen, embed_size, max_features, embedding_matrix,
loss_f = 'binary_crossentropy', opti = 'adam', metr = f1):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences = True))(x)
 x = Bidirectional(CuDNNLSTM(100, return_sequences = True))(x)
 x = Bidirectional(CuDNNLSTM(72, return_sequences = True))(x)
 atten = Attention(maxlen )(x)
 avg_pool = GlobalAvgPool1D()(x)
 max_pool = GlobalMaxPool1D()(x)
 conc = concatenate([atten, avg_pool, max_pool])
 conc = Dense(64, activation="relu" )(conc)
 outp = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss = loss_f, optimizer = opti , metrics=[metr])
 return model
<string_transform><EOS> 
tokenizer = Tokenizer(num_words = max_features)
 tokenizer.fit_on_texts(list(X_tr))
X_tr = tokenize_text(X_tr, tokenizer, maxlen)
 X_te = tokenize_text(X_te, tokenizer, maxlen)
 embedding_matrix = create_embedding_matrix(model_embed,
tokenizer,
max_features,
embed_size)
 model = RNN_model(X_tr.shape[1], embed_size, max_features, embedding_matrix)
 model_fited = model.fit(
 X_tr,
Y_train.values,
batch_size = batch_size,
nb_epoch = numb_epoch)
 y_hat_probs = model.predict(X_te)
 y_hat = to_binary(y_hat_probs, 0.38)
 print('The submission file has ' + str(np.sum(y_hat)* 100/len(y_hat)) + ' percent insincere')
 d_test = d_test.reset_index()
d_test['prediction'] = y_hat
to_upload = d_test[['qid', 'prediction']]
to_upload.to_csv('submission.csv', index = False )