Kaggle competition: Quora Insincere Questions Classification
print(f'Accuracy: {accuracy_score(y_test, y_pred)*100:.2f}%')
 print('-'*55)
 print(classification_report(y_test, y_pred))
print('-'*55)
 print(confusion_matrix(y_test, y_pred))<import_modules>
np.random.seed(0)
 tqdm.pandas()
np.random.seed(1 )
from sklearn.ensemble import RandomForestClassifier<compute_train_metric>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv" )
rf = RandomForestClassifier(random_state=121)
 rf.fit(X_train, y_train)
 y_pred = rf.predict(X_test)
 accuracy = accuracy_score(y_test, y_pred)
 print('Accuracy:', accuracy )<define_search_space>
def clean_text(x):
x = str(x)
 for punct in "/-'":
x = x.replace(punct, ' ')
 for punct in '&':
x = x.replace(punct, f' {punct} ')
 for punct in '?!.,"
x = x.replace(punct, '')
 return x
<train_model>
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: clean_text(x))
best_rf = RandomForestClassifier(random_state=121, criterion='entropy', max_depth=15, min_samples_leaf=5, min_samples_split=2, n_estimators=50)
 best_rf.fit(X_train, y_train)
 y_pred = best_rf.predict(X_test)
 accuracy = accuracy_score(y_test, y_pred )<compute_test_metric>
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018 )
print(f'Accuracy: {accuracy_score(y_test, y_pred)*100:.2f}%')
 print('-'*55)
 print(classification_report(y_test, y_pred))
print('-'*55)
 print(confusion_matrix(y_test, y_pred))<import_modules>
def get_synonyms(word, pos=None):
wordnet_pos = {
"NN": wordnet.NOUN,
"VB": wordnet.VERB,
"VBD": wordnet.VERB,
"VBG": wordnet.VERB,
"VBN": wordnet.VERB,
"VBP": wordnet.VERB,
"JJ": wordnet.ADJ,
"RB": wordnet.ADV,
"RBR": wordnet.ADV,
"RBS": wordnet.ADV,
}
if pos:
if pos in list(wordnet_pos.keys()):
synsets = wordnet.synsets(word, pos=wordnet_pos[pos])
 synonyms = []
for synset in synsets:
synonyms += [str(lemma.name())for lemma in synset.lemmas() ]
synonyms = [synonym.replace("_", " ")for synonym in synonyms]
synonyms = list(set(synonyms))
synonyms = [synonym for synonym in synonyms if synonym != word]
if synonyms:
return synonyms[0]
return ''
def get_syn_sentence(text):
words = text.split()
words_with_pos_tag = pos_tag(words)
 words_with_pos_tag
new_sentence_words = []
for word, pos in words_with_pos_tag:
synonym = get_synonyms(word, pos)
 if synonym:
new_sentence_words.append(synonym)
 else:
new_sentence_words.append(word)
 synonym_sentence = ' '.join(new_sentence_words)
 return synonym_sentence
from xgboost import XGBClassifier<compute_train_metric>
df_obscene = train_df[train_df['target'] == 1]
df_obscene = df_obscene.reset_index()
xgb = XGBClassifier(random_state=121)
 xgb.fit(X_train, y_train)
 y_pred = xgb.predict(X_test)
 accuracy = accuracy_score(y_test, y_pred)
 print('Accuracy:', accuracy )<predict_on_test>
sentence_with_synonyms = []
for idx, row in df_obscene.iterrows() :
sentence_with_synonyms.append(get_syn_sentence(row['question_text']))
df_obscene['question_text'] = sentence_with_synonyms
classifiers = [('Logistic Regression', best_lr), 
('KNN', knn), 
('Random Forest', best_rf), 
('Xgboost', xgb)]
for name_clf, clf in classifiers:
y_pred = clf.predict(X_test)
 acc = accuracy_score(y_test, y_pred)
 print(f'{name_clf} accuracy: {round(acc, 3)}%' )<import_modules>
train_df = train_df.append(df_obscene, ignore_index=True, sort = False )
from sklearn.ensemble import VotingClassifier<compute_train_metric>
embed_size = 300
max_features = 50000
maxlen = 100
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
vc = VotingClassifier(estimators=classifiers)
 vc.fit(X_train, y_train)
 y_pred = vc.predict(X_test)
 acc_vc = accuracy_score(y_test, y_pred)
 print(f'Ensembler Accuracy: {round(acc_vc, 3)}%' )<save_to_csv>
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
vc.fit(X, y)
 prediction = vc.predict(df_test)
 submission['Survived'] = prediction
submission.to_csv('Submission.csv', index=False )<import_modules>
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = LSTM(128, return_sequences = True )(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )
print("Tensorflow DL Version: " + tf.__version__)

print("Setup Completed" )<load_from_csv>
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))
train_file = ".. /input/digit-recognizer/train.csv"
predict_file = ".. /input/digit-recognizer/test.csv"
submission_file = ".. /input/digit-recognizer/sample_submission.csv"
train_data = pd.read_csv(train_file, sep=',')
 predict_data = pd.read_csv(predict_file, sep=',')
 print("Files Preparation Completed" )<count_missing_values>
pred_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_val_y>thresh ).astype(int))))
print('**************************Train File Preliminary Investigation**************************')
 print('1.Train File Shape:', train_data.shape)
 missing_val_count_by_column =(train_data.isnull().sum())
 print('2.Train File Missing Valu:', missing_val_count_by_column[missing_val_count_by_column > 0])
 print(" Missing Values in Train File: {}".format(train_data.isna().any().any()))
print('3.Train File Variables Information:')
 print(train_data.info())
 print('4.Train File Variables Unique Number:')
 print(train_data.nunique())
 print(' ')
 print('**************************Prediction File Preliminary Investigation**************************')
 print('1.Test File Shape:', predict_data.shape)
 missing_val_count_by_column =(predict_data.isnull().sum())
 print('2.Test File Missing Value:', missing_val_count_by_column[missing_val_count_by_column > 0])
 print(" Missing Values in Test File: {}".format(predict_data.isna().any().any()))
print('3.Test File Variables Information:')
 print(predict_data.info())
 print('4.Test File Variables Unique Number:')
 print(predict_data.nunique() )<categorify>
pred_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
<prepare_x_and_y><EOS> 
pred_test_y =(pred_test_y>0.35 ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )