Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<count_missing_values>
import os
import time
from sklearn import tree
from sklearn.metrics import accuracy_score
from sklearn.naive_bayes import GaussianNB
import numpy as np
import pandas as pd
from tqdm import tqdm
import string
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from nltk.corpus import stopwords
import seaborn as sns
import matplotlib.pylab as pylab
import matplotlib.pyplot as plt
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers
train.isnull().sum()<count_missing_values>
train.isnull().sum()
train.isnull().sum()<count_values>
train.isnull().sum()
print(train.Embarked [train.Embarked == 'S'].count())
 print(train.Embarked [train.Embarked == 'C'].count())
 print(train.Embarked [train.Embarked == 'Q'].count() )<categorify>
train, val = train_test_split(train, test_size=0.1, random_state=2018)
 embed_size = 300
max_features = 50000
maxlen = 100
train_X = train["question_text"].fillna("_na_" ).values
val_X = val["question_text"].fillna("_na_" ).values
test_X = test["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train['target'].values
val_y = val['target'].values
train = train.fillna({"Embarked": "S"})
 embarked_mapping = {"S": 1, "C": 2, "Q": 3}
train['Embarked'] = train['Embarked'].map(embarked_mapping)
 test['Embarked'] = test['Embarked'].map(embarked_mapping)
 train.head()<categorify>
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
train['Sex'] = train['Sex'].map({"male": 0, "female": 1})
 test['Sex'] = test['Sex'].map({"male": 0, "female": 1})
 train.head()<drop_column>
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )
train = train.drop(['Cabin'], axis = 1)
 test = test.drop(['Cabin'], axis = 1 )<drop_column>
tree = tree.DecisionTreeClassifier()
gauss = GaussianNB()
train = train.drop(['Ticket'], axis = 1)
 test = test.drop(['Ticket'], axis = 1 )<drop_column>
train = train.drop(['Name'], axis = 1)
 test = test.drop(['Name'], axis = 1 )<data_type_conversions>
tree.fit(train_X, train_y)
 gauss.fit(train_X,train_y)
 y_pred=tree.predict(val_X)
 y_pred_1=gauss.predict(val_X)
 print('decision tree accuracy is: ',accuracy_score(val_y, y_pred))
print('naive bayes accuracy is: ',accuracy_score(val_y, y_pred_1))
train.Age.fillna(value=train.Age.mean() , inplace=True)
 train.Fare.fillna(value=train.Fare.mean() , inplace=True)
 test.Age.fillna(value=test.Age.mean() , inplace=True)
 test.Fare.fillna(value=test.Fare.mean() , inplace=True )<categorify>
pred_test_y = model.predict([test_X], batch_size=1024, verbose=1)
 pred_test_y =(pred_test_y>0.35 ).astype(int)
 out_df = pd.DataFrame({"qid":test["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False)
 print(out_df.tail() )
train['CabinBool'] = train['CabinBool'].map({True: 0, False: 1})
 test['CabinBool'] = test['CabinBool'].map({True: 0, False: 1} )<categorify>
train["num_words"] = train["question_text"].apply(lambda x: len(str(x ).split()))
test["num_words"] = test["question_text"].apply(lambda x: len(str(x ).split()))
train["num_unique_words"] = train["question_text"].apply(lambda x: len(set(str(x ).split())))
 test["num_unique_words"] = test["question_text"].apply(lambda x: len(set(str(x ).split())))
 train["num_chars"] = train["question_text"].apply(lambda x: len(str(x)))
 test["num_chars"] = test["question_text"].apply(lambda x: len(str(x)))
 train["num_stopwords"] = train["question_text"].apply(lambda x: len([w for w in str(x ).lower().split() if w in eng_stopwords]))
test["num_stopwords"] = test["question_text"].apply(lambda x: len([w for w in str(x ).lower().split() if w in eng_stopwords]))
train["num_punctuations"] =train['question_text'].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
test["num_punctuations"] =test['question_text'].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
train["mean_word_len"] = train["question_text"].apply(lambda x: np.mean([len(w)for w in str(x ).split() ]))
test["mean_word_len"] = test["question_text"].apply(lambda x: np.mean([len(w)for w in str(x ).split() ]))
sns.violinplot(data=train,x="target", y="num_words" )