Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<drop_column>
print(K.tensorflow_backend._get_available_gpus() )
X_nullageblk_test = nullageblk_test<drop_column>
_traindataset = '.. /input/train.csv'
_testdataset = '.. /input/test.csv'
_glove = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
_paragram = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
_wiki_news = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
_google_news = '.. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'
embeddings = [{'name': 'glove', 'path': _glove},
{'name': 'paragram', 'path': _paragram},
{'name': 'fasttext', 'path': _wiki_news}]
X_nonull_df_train=X_nonull_df_train.drop(columns=['Titles_ Capt','Titles_ Col','Titles_ Dona','Titles_ Lady','Titles_ Major',
'Titles_ Mlle','Titles_ Mme','Titles_ Sir','Titles_ the Countess','block_G',
'block_A','block_B','block_C','block_D','block_E','block_F','CabinOccupancy'])
 X_nullageblk_test=X_nullageblk_test.drop(columns=['CabinOccupancy','Titles_ Ms'] )<train_model>
t0 = time.time()
df_train = pd.read_csv(_traindataset)
 df_test = pd.read_csv(_testdataset)
 timeSince(t0 )
dtreeReg.fit(X_nonull_df_train,y_nonull_df_train )<predict_on_test>
t0 = time.time()
df_train['processed_questions'] = df_train['question_text'].fillna("_nan_" ).apply(lambda x: x.lower())
 df_test['processed_questions'] = df_test['question_text'].fillna("_nan_" ).apply(lambda x: x.lower())
 timeSince(t0 )
predictions_nullageblk = dtreeReg.predict(X_nullageblk_test)
 predictions_nullageblk<drop_column>
t0 = time.time()
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because",
"could've": "could have", "couldn't": "could not", "didn't": "did not",
"doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not",
"haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is",
"how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is",
"I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have",
"I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have",
"i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have",
"isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will",
"it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam",
"mayn't": "may not", "might've": "might have","mightn't": "might not",
"mightn't've": "might not have", "must've": "must have", "mustn't": "must not",
"mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have",
"o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have",
"shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have",
"she'd": "she would", "she'd've": "she would have", "she'll": "she will",
"she'll've": "she will have", "she's": "she is", "should've": "should have",
"shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have",
"so's": "so as", "this's": "this is","that'd": "that would",
"that'd've": "that would have", "that's": "that is", "there'd": "there would",
"there'd've": "there would have", "there's": "there is", "here's": "here is",
"they'd": "they would", "they'd've": "they would have", "they'll": "they will",
"they'll've": "they will have", "they're": "they are", "they've": "they have",
"to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have",
"we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have",
"weren't": "were not", "what'll": "what will", "what'll've": "what will have",
"what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is",
"when've": "when have", "where'd": "where did", "where's": "where is",
"where've": "where have", "who'll": "who will", "who'll've": "who will have",
"who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have",
"will've": "will have", "won't": "will not", "won't've": "will not have",
"would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have",
"y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have",
"y'all're": "you all are","y'all've": "you all have","you'd": "you would",
"you'd've": "you would have", "you'll": "you will", "you'll've": "you will have",
"you're": "you are", "you've": "you have"}
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
df_train['processed_questions'] = df_train['processed_questions'].apply(lambda x:
clean_contractions(x,
contraction_mapping))
df_test['processed_questions'] = df_test['processed_questions'].apply(lambda x:
clean_contractions(x,
contraction_mapping))
timeSince(t0 )
nullageblk.drop(columns=['Age'])
 nullageblk['Age']=predictions_nullageblk
nullageblk<categorify>
t0 = time.time()
punct = "/-'?!.,
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x",
"²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"',
'“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.',
'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi'}
def clean_special_chars(text, punct, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text
df_train['processed_questions'] = df_train['processed_questions'].apply(lambda x: clean_special_chars(x,
punct,
punct_mapping))
df_test['processed_questions'] = df_test['processed_questions'].apply(lambda x: clean_special_chars(x,
punct,
punct_mapping))
timeSince(t0 )
cat_feats=['Embarked','Sex','Titles','block','Pclass','CabinOccupancy']
nonull_df_train = pd.get_dummies(nonull_df,columns=cat_feats,drop_first=False)
 nonull_df_train=nonull_df_train.drop(columns=['Cabin','Name','PassengerId','Survived','Ticket'])
 nonull_df_train.head() 
<prepare_x_and_y>
t0 = time.time()
timeSince(t0 )
X_nonull_df_train = nonull_df_train.drop(columns=['CabinOccupancy_1','CabinOccupancy_2','CabinOccupancy_3','CabinOccupancy_4','CabinOccupancy_5','CabinOccupancy_6'],axis=1)
 y_nonull_df_train = nonull_df_train[['CabinOccupancy_1','CabinOccupancy_2','CabinOccupancy_3','CabinOccupancy_4','CabinOccupancy_5','CabinOccupancy_6']]<categorify>
t0 = time.time()
def build_vocab(texts):
sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
def vocab_to_integer(vocab):

 return {word: ii for ii, word in enumerate(vocab, 1)}
all_questions = pd.concat([df_train['processed_questions'], df_test['processed_questions']])
 final_vocab = build_vocab(all_questions)
 word_to_idx = vocab_to_integer(final_vocab)
 timeSince(t0 )
cat_feats=['Embarked','Sex','Titles','block','Pclass']
nullageblk_test = pd.get_dummies(nullageblk,columns=cat_feats,drop_first=False)
 nullageblk_test=nullageblk_test.drop(columns=['Name','Ticket','PassengerId','Cabin','Survived'])
 nullageblk_test.head() 
<drop_column>
t0 = time.time()
hparam = {}
hparam['VOCAB_SIZE'] = len(final_vocab)+ 1
hparam['PAD_LENGTH'] = 77
hparam['MINIBATCH_SIZE'] = 512
hparam['LEARNING_RATE'] = 1e-3
hparam['EPOCHS'] = 4
hparam['LSTM_HIDDEN_SIZE'] = 128
hparam['WORD_EMB_DIM'] = 0
hparam['KFOLDS'] = 10
timeSince(t0 )
X_nullageblk_test = nullageblk_test<drop_column>
t0 = time.time()
vocab_original = build_vocab(pd.concat([df_train['question_text'], df_test['question_text']]))
timeSince(t0 )
X_nonull_df_train=X_nonull_df_train.drop(columns=['Titles_ Capt','Titles_ Col','Titles_ Dona','Titles_ Lady','Titles_ Major',
'Titles_ Mlle','Titles_ Mme','Titles_ Sir','Titles_ the Countess','block_G',
'block_A','block_B','block_C','block_D','block_E','block_F'])
 X_nullageblk_test=X_nullageblk_test.drop(columns=['Titles_ Ms','CabinOccupancy'] )<train_model>
t0 = time.time()
def load_embed(file):

 def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 if file.split('/')[-1] == 'wiki-news-300d-1M.vec':
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file)if len(o)> 100)
 else:
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file, encoding='latin'))
return embeddings_index
def check_coverage(vocab, embeddings_index):

 known_words = {}
unknown_words = {}
nb_known_words = 0
nb_unknown_words = 0
for word in vocab.keys() :
try:
known_words[word] = embeddings_index[word]
nb_known_words += vocab[word]
except:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
pass
print('{} iv words, {} unique'.format(nb_known_words, len(known_words)))
 print('{} oov words, {} unique'.format(nb_unknown_words, len(unknown_words)))
 print('Found embeddings for {:.3%} of vocab'.format(len(known_words)/ len(vocab)))
 print('Found embeddings for {:.3%} of all text'.format(nb_known_words /(nb_known_words + nb_unknown_words)))
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words
def create_emb_matrix(nb_words, embed_size):

 return np.zeros(( nb_words, embed_size), dtype=np.float32)
 def fill_emb_matrix(word_idx, emb_matrix, emb_index):

 for word, i in word_idx:
emb_vector = emb_index.get(word)
 if emb_vector is not None:
emb_matrix[i] = emb_vector
return emb_matrix
def add_lower(embedding, vocab):
count = 0
for word in vocab:
if word in embedding and word.lower() not in embedding:
embedding[word.lower() ] = embedding[word]
count += 1
print(f"Added {count} words to embedding")
 timeSince(t0 )
dtreeClass.fit(X_nonull_df_train,y_nonull_df_train )<predict_on_test>
t0 = time.time()
conc_embedding = None
word_index = word_to_idx
nb_words = min(hparam['VOCAB_SIZE'], len(word_index)+ 1)
 hparam['VOCAB_SIZE'] = nb_words
print(hparam['VOCAB_SIZE'], len(word_index)+ 1)
 print(f"Got a vocab size of {nb_words} number of words")
 for embedding in embeddings:
emb_name = embedding['name']
emb_path = embedding['path']
print("Running procedure on {}".format(emb_name))
print("Loading {}".format(emb_name))
emb_index = load_embed(emb_path)
 print("Adding lowercase to {}".format(emb_name))
add_lower(emb_index, vocab_original)
 _ = check_coverage(final_vocab, emb_index)
 emb_size = 300
hparam['WORD_EMB_DIM'] += emb_size
emb_matrix = create_emb_matrix(nb_words, emb_size)
 print(emb_matrix.size)
 print(emb_matrix.shape)
 emb_matrix = fill_emb_matrix(word_index.items() , emb_matrix, emb_index)
 if conc_embedding is not None:
conc_embedding = np.concatenate(( conc_embedding, emb_matrix), axis=1)
 print("concatenated! now got shape: {}".format(conc_embedding.shape))
else:
conc_embedding = emb_matrix
del emb_matrix, emb_index, emb_name, emb_path, emb_size
timeSince(t0 )
predictions_nullageblk = dtreeClass.predict(X_nullageblk_test)
 predictions_nullageblk<prepare_output>
t0 = time.time()
def embed_word_to_int(X, vocab_to_int):
embedded_X = []
for q in X:
tmp_X = []
for w in q.split() :
tmp_X.append(vocab_to_int[w])
 embedded_X.append(tmp_X)
 return embedded_X
X_train = embed_word_to_int(df_train['processed_questions'].values, word_to_idx)
 X_test = embed_word_to_int(df_test['processed_questions'].values, word_to_idx)
 pad_length = hparam['PAD_LENGTH']
X_train_pad = pad_sequences(X_train, maxlen=pad_length, padding='pre', truncating='pre')
 X_test_pad = pad_sequences(X_test, maxlen=pad_length, padding='pre', truncating='pre')
 print(df_train['processed_questions'][3333])
 print(X_train[3333])
 print(X_train_pad[3333])
 timeSince(t0 )
predictions_nullageblk=pd.DataFrame(predictions_nullageblk, columns=['1','2','3','4','5','6'])
 predictions_nullageblk['CabinOccupancy']=np.where(predictions_nullageblk['1']==1,'1','')
 predictions_nullageblk['CabinOccupancy']=np.where(predictions_nullageblk['2']==1,'2',predictions_nullageblk['CabinOccupancy'])
 predictions_nullageblk['CabinOccupancy']=np.where(predictions_nullageblk['3']==1,'3',predictions_nullageblk['CabinOccupancy'])
 predictions_nullageblk['CabinOccupancy']=np.where(predictions_nullageblk['4']==1,'4',predictions_nullageblk['CabinOccupancy'])
 predictions_nullageblk['CabinOccupancy']=np.where(predictions_nullageblk['5']==1,'5',predictions_nullageblk['CabinOccupancy'])
 predictions_nullageblk['CabinOccupancy']=np.where(predictions_nullageblk['6']==1,'6',predictions_nullageblk['CabinOccupancy'])
 predictions_nullageblk.head()<prepare_output>
t0 = time.time()
kfold = StratifiedKFold(n_splits=hparam['KFOLDS'], shuffle=True, random_state=2019)
 timeSince(t0 )
predictions_nullageblk=predictions_nullageblk['CabinOccupancy'].values<drop_column>
t0 = time.time()
X = X_train_pad
y = df_train['target'].values
results = []
for fold,(train_index, val_index)in enumerate(kfold.split(X, y)) :
X_train, X_val = X[train_index], X[val_index]
y_train, y_val = y[train_index], y[val_index]
print(f"Training on {len(X_train)} and validating on {len(X_val)} number of words")
 dataset = {'X_train': X_train, 'y_train': y_train,
'X_val': X_val, 'y_val': y_val,
'X_test': X_test_pad}
test_preds, val_preds, thresh, f1 = train_val_pred(dataset, hparam, conc_embedding)
 print("len(test_preds)= {}, len(val_preds)= {}, thresh = {} at f1 = {}".format(len(test_preds),
len(val_preds),
thresh,
f1))
new_result = {'name': 'fold-' + str(fold),
'test_preds': test_preds,
'val_preds': val_preds,
'thresh': thresh,
'f1': f1}
results.append(new_result)
 timeSince(t0 )
nullageblk.drop(columns=['CabinOccupancy'])
 nullageblk['CabinOccupancy']=predictions_nullageblk
nullageblk<categorify>
t0 = time.time()
print("Got {} number of results!".format(len(results)))
 avg_thresh = 0
for result in results:
print("{} gave f1 score {} with thresh {}".format(result['name'], result['f1'], result['thresh']))
avg_thresh += result['thresh']
avg_thresh = avg_thresh / len(results)
 print("Got an average threshold at {}".format(avg_thresh))
timeSince(t0 )
cat_feats=['Embarked','Sex','Titles','block','Pclass']
nonull_df_train = pd.get_dummies(nonull_df,columns=cat_feats,drop_first=False)
 nonull_df_train=nonull_df_train.drop(columns=['Cabin','Name','PassengerId','Survived','Ticket'])
 nonull_df_train.head() 
<prepare_x_and_y>
t0 = time.time()
print("Using treshold {}".format(avg_thresh))
factor = 1.0 / len(results)
 pred_test_y = results[0]['test_preds'] * factor
print("Using factor: ", factor)
 for i in range(1, len(results)) :
pred_test_y += factor * results[i]['test_preds']
pred_test_y_res =(pred_test_y > avg_thresh ).astype(int)
 results_dict = {'qid':df_test['qid'].values, 'prediction':[]}
for prediction in pred_test_y_res:
results_dict['prediction'].append(prediction[0])
 print(results_dict['qid'][:15])
 print(results_dict['prediction'][:15])
 df = pd.DataFrame(data=results_dict)
 df.to_csv('submission.csv', index=False)
 print("Saved csv to disk!")
 timeSince(t0 )
X_nonull_df_train = nonull_df_train.drop(['block_A','block_B','block_C','block_D','block_E','block_F','block_G'],axis=1)
 y_nonull_df_train = nonull_df_train[['block_A','block_B','block_C','block_D','block_E','block_F','block_G']]<categorify>