Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<filter>
tqdm.pandas()
first_class = X.loc[X.Pclass == 1]["Survived"]
first_rate = sum(first_class)/len(first_class)
 second_class = X.loc[X.Pclass == 2]["Survived"]
second_rate = sum(second_class)/len(second_class)
 third_class = X.loc[X.Pclass == 3]["Survived"]
third_rate = sum(third_class)/len(third_class)
 print("Rate of 1st class who survived:", "%.5f"% first_rate)
 print("Rate of 2nd class who survived:", "%.5f"% second_rate)
 print("Rate of 3rd class who survived:", "%.5f"% third_rate )<filter>
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not",
"didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not",
"he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is",
"I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would",
"i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would",
"it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not",
"might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have",
"needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not",
"sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have",
"she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as",
"this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have",
"there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have",
"they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will",
"we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have",
"what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is",
"where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have",
"will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have",
"y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would",
"you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have"}
contraction_patterns = [(r'won't', 'will not'),(r'can't', 'cannot'),(r'i'm', 'i am'),(r'ain't', 'is not'),(r'(\w+)'ll', '\g<1> will'),(r'(\w+)n't', '\g<1> not'), 
(r'(\w+)'ve', '\g<1> have'),(r'(\w+)'s', '\g<1> is'),(r'(\w+)'re', '\g<1> are'),(r'(\w+)'d', '\g<1> would'),(r'&', 'and'),(r'dammit', 'damn it'), 
(r'dont', 'do not'),(r'wont', 'will not')]
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }
punct = "/-'?!.,
mispell_dict = {'advanatges': 'advantages', 'irrationaol': 'irrational' , 'defferences': 'differences','lamboghini':'lamborghini','hypothical':'hypothetical', 'colour': 'color',
'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor',
'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'qoura' : 'quora', 'sallary': 'salary', 'Whta': 'What',
'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do',
'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating',
'pennis': 'penis', 'Etherium': 'Ethereum', 'etherium': 'ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota',
'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization',
'demonitization': 'demonetization', 'demonetisation': 'demonetization', 'pokémon': 'pokemon'}
sibs = X.loc[X.SibSp == 1]["Survived"]
sibs_rate = sum(sibs)/len(sibs)
 parents = X.loc[X.Parch == 1]["Survived"]
parents_rate = sum(parents)/len(parents)
 print("Rate of survivors with siblings:", "%.5f"% sibs_rate)
 print("Rate of survivors with parents:", "%.5f"% parents_rate )<create_dataframe>
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
def replaceContraction(text):
patterns = [(re.compile(regex), repl)for(regex, repl)in contraction_patterns]
for(pattern, repl)in patterns: 
(text, count)= re.subn(pattern, repl, text)
 return text
def clean_text(x):
x = str(x)
 for punct in "/-'":
x = x.replace(punct, ' ')
 for punct in '&':
x = x.replace(punct, f' {punct} ')
 for punct in '?!.,"
x = x.replace(punct, '')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', ' number ', x)
 x = re.sub('[0-9]{4}', ' number ', x)
 x = re.sub('[0-9]{3}', ' number ', x)
 x = re.sub('[0-9]{2}', ' number ', x)
 return x
def clean_special_chars(text, punct, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
df = pd.DataFrame({'title':name[0], 'name':name[2], 'last_name' : name[1],
'survived': X['Survived'], 'sibsp': X['SibSp'], 'parch': X['Parch'],
'age': X['Age'], 'sex': X['Sex']})
 df_test = pd.DataFrame({'title':name_test[0], 'name':name_test[2], 'last_name' : name_test[1],
'age': X_test_full['Age'], 'sex':X_test_full['Sex']})
 df1 = df.copy()
df2 = df.copy()
for row in range(891):
if df1.loc[row,'sibsp']==0 and df1.loc[row,'parch']==0:
df1.drop([row], inplace=True)
 if df2.loc[row,'sibsp']!=0 or df2.loc[row,'parch']!=0:
df2.drop([row], inplace=True )<count_values>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )
df1['last_name'].value_counts()<define_variables>
def statistical_features(df):
stat_features = pd.DataFrame()
stat_features['txt_len'] = df['question_text'].progress_apply(lambda x: len(str(x)))
 stat_features['word_count'] = df['question_text'].progress_apply(lambda x: len(str(x ).split(" ")))
 stat_features['!_count'] = df['question_text'].progress_apply(lambda x: x.count('!'))
stat_features['?_count'] = df['question_text'].progress_apply(lambda x: x.count('?'))
stat_features['upper_word_count'] = df['question_text'].progress_apply(lambda x: len([x for x in x.split() if x.isupper() ]))
stat_features['unique_word_count'] = df['question_text'].progress_apply(lambda x: len(set(x.split())))
 return stat_features
fam = df1["survived"]
fam_rate = sum(fam)/len(fam)
 lonely = df2["survived"]
lonely_rate = sum(lonely)/len(lonely)
 print("Rate of survivors with family onboard:", "%.5f"% fam_rate)
 print("Rate of survivors without family onboard:", "%.5f"% lonely_rate )<groupby>
def load_embed(file):
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 if file == '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec':
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file)if len(o)>100)
 else:
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file, encoding='latin'))
return embeddings_index
X.groupby('Pclass' ).Fare.mean()<filter>
glove = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
paragram = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
print("Extrayendo GloVe embedding...")
 embed_glove = load_embed(glove )
emb_s = X.loc[X.Embarked == 'S']["Survived"]
s_rate = sum(emb_s)/len(emb_s)
 emb_c = X.loc[X.Embarked == 'C']["Survived"]
c_rate = sum(emb_c)/len(emb_c)
 emb_q = X.loc[X.Embarked == 'Q']["Survived"]
q_rate = sum(emb_q)/len(emb_q)
 print("Rate of survivors that embarked from Southampton:", "%.5f"% s_rate)
 print("Rate of survivors that embarked from Cherbourg:","%.5f"% c_rate)
 print("Rate of survivors that embarked from Queenstown:","%.5f"% q_rate )<count_values>
X['Ticket'].value_counts()<groupby>
def build_vocab(texts):
sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
def check_coverage(vocab, embeddings_index):
known_words = {}
unknown_words = {}
nb_known_words = 0
nb_unknown_words = 0
for word in vocab.keys() :
try:
known_words[word] = embeddings_index[word]
nb_known_words += vocab[word]
except:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
pass
print('Found embeddings for {:.3%} of vocab'.format(len(known_words)/ len(vocab)))
 print('Found embeddings for {:.3%} of all text'.format(nb_known_words /(nb_known_words + nb_unknown_words)))
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words
df.groupby('title' ).age.mean()<categorify>
def add_lower(embedding, vocab):
count = 0
for word in vocab:
if word in embedding and word.lower() not in embedding:
embedding[word.lower() ] = embedding[word]
count += 1
print(f"Se agregaron {count} palabras al embedding" )
def replace_titles(x):
title = x['title']
if title in ['Don', 'Major', 'Capt', 'Jonkheer', 'Rev', 'Col', 'Sir']:
return 'Mr'
elif title in ['the Countess', 'Mme', 'Dona', 'Lady']:
return 'Mrs'
elif title in ['Mlle', 'Ms']:
return 'Miss'
elif title =='Dr':
if x['sex']=='male':
return 'Mr'
else:
return 'Mrs'
else:
return title
df['title']=df.apply(replace_titles, axis=1 )<feature_engineering>
df = pd.concat([train_df ,test_df])
 vocab = build_vocab(df['question_text'] )
X['Title'] = df['title']<groupby>
print("GloVe v2: ")
 add_lower(embed_glove, vocab)
X_test_full.groupby('Pclass' ).Fare.mean()<filter>
train_df['treated_question'] = train_df['question_text'].progress_apply(lambda x: x.lower())
 train_df['treated_question'] = train_df['treated_question'].progress_apply(lambda x: clean_contractions(x, contraction_mapping))
train_df['treated_question'] = train_df['treated_question'].progress_apply(lambda x: clean_special_chars(x, punct, punct_mapping))
train_df['treated_question'] = train_df['treated_question'].progress_apply(lambda x: correct_spelling(x, mispell_dict))
test_df['treated_question'] = test_df['question_text'].progress_apply(lambda x: x.lower())
 test_df['treated_question'] = test_df['treated_question'].progress_apply(lambda x: clean_contractions(x, contraction_mapping))
test_df['treated_question'] = test_df['treated_question'].progress_apply(lambda x: clean_special_chars(x, punct, punct_mapping))
test_df['treated_question'] = test_df['treated_question'].progress_apply(lambda x: correct_spelling(x, mispell_dict))
X_test_full.loc[X_test_full.Fare.isnull() ]<feature_engineering>
embed_size = 300
max_features = 95000
max_len = 100
X_test_full.Fare = X_test_full.Fare.fillna(12.46 )<count_values>
def make_data(X, X_test):
t = Tokenizer(num_words=max_features, filters='', oov_token='<OOV>')
 t.fit_on_texts(X)
 X = t.texts_to_sequences(X)
 X_test = t.texts_to_sequences(X_test)
 X = pad_sequences(X, maxlen=max_len)
 X_test = pad_sequences(X_test, maxlen=max_len)
 return X, X_test, t.word_index
df_test.groupby('title' ).title.count()<filter>
X, X_test, word_index = make_data(train_df['treated_question'], test_df['treated_question'] )
df_test.loc[df_test.title.isin(['Col', 'Dona', 'Don', 'Dr', 'Ms', 'Rev'])]<groupby>
y = train_df['target'].values
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.1, random_state=42 )
df_test.groupby('title' ).age.mean()<feature_engineering>
def make_embed_matrix(embeddings_index, word_index, len_voc):
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = word_index
nb_words = min(len_voc, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= len_voc:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix
df_test['title']=df_test.apply(replace_titles, axis=1)
 print(df_test.groupby('title' ).title.count())
 print(df_test.groupby('title' ).age.mean())
 print(df_test.groupby('title' ).age.median() )<feature_engineering>
embedding_matrix_glove = make_embed_matrix(embed_glove, word_index, max_features)
 del word_index
del embed_glove
gc.collect()
X_test_full['Title'] = df_test['title']<feature_engineering>
print("Extrayendo Paragram embedding...")
 embed_paragram = load_embed(paragram )
def f(row):
if row['Age'] <= 16: val = 1
else: val = 0
return val
X['Minor'] = X.apply(f, axis=1)
 X_test_full['Minor'] = X_test_full.apply(f, axis=1)
 def f(row):
if row['Parch'] > 0 and row['Title'] == 'Mrs' and row['Age'] > 18: val = 1
else: val = 0
return val
X['Mother'] = X.apply(f, axis=1)
 X_test_full['Mother'] = X_test_full.apply(f, axis=1 )<drop_column>
add_lower(embed_paragram, vocab)
X['Embarked'].fillna(X['Embarked'].value_counts().idxmax() , inplace=True)
 X['Deck'] = X['Cabin'].apply(lambda s: s[0] if pd.notnull(s)else 'U')
 X_test_full['Deck'] = X_test_full['Cabin'].apply(lambda s: s[0] if pd.notnull(s)else 'U')
 X['Deck'].unique()
X.drop('Cabin', axis=1, inplace=True)
 X_test_full.drop('Cabin', axis=1, inplace=True )<categorify>
_, _, word_index = make_data(train_df['treated_question'], test_df['treated_question'])
 embedding_matrix_paragram = make_embed_matrix(embed_paragram, word_index, max_features)
 del word_index
del embed_paragram
gc.collect()
plt.figure(figsize=(10,5))
plt.title('Training dataset heatmap')
 X['Title'] = LabelEncoder().fit_transform(X['Title'])
 X['Embarked'] = X['Embarked'].astype('|S')
 X['Embarked'] = LabelEncoder().fit_transform(X['Embarked'])
 X['Deck'] = X['Deck'].astype('|S')
 X['Deck'] = LabelEncoder().fit_transform(X['Deck'])
 sns.heatmap(X.corr() , annot=True, cmap='coolwarm')
 plt.show()<feature_engineering>
embedding_matrix_final = np.sum([embedding_matrix_glove*0.7,embedding_matrix_paragram*0.3], axis=0 )
df['age'] = np.where(( df.age.isnull())&(df.title=="Master"),5,
np.where(( df.age.isnull())&(df.title=="Miss"),21,
np.where(( df.age.isnull())&(df.title=="Mr"),33,
np.where(( df.age.isnull())&(df.title=="Mrs"),36,
df.age))))<count_missing_values>
train_stats = statistical_features(train_df)
 train_stats[train_df.target==1].head()
df.isnull().sum()<groupby>
test_stats = statistical_features(test_df)
 test_stats.head()
df[['title', 'survived']].groupby(['title'], as_index=False ).mean()<feature_engineering>
sc = StandardScaler()
train_stats = sc.fit_transform(train_stats)
 test_stats = sc.transform(test_stats )
X['Age'] = df['age']
X.isnull().sum()<count_missing_values>
y = train_df['target'].values
X_train_stats, X_val_stats, _, _ = train_test_split(train_stats, y, test_size=0.1, random_state=42 )
X.isnull().sum()<categorify>
def model() :
optim = Adam(lr=0.0010)
 main_inp = Input(shape=(max_len,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix_final], trainable=True )(main_inp)
 x = SpatialDropout1D(0.15 )(x)
 x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x = Conv1D(filters=64, kernel_size=1 )(x)
 x = GlobalMaxPool1D()(x)
 stat_inp = Input(shape=(X_train_stats.shape[1],))
x_2 = Dense(64, activation="relu" )(stat_inp)
 x_2 = Dropout(0.15 )(x_2)
 x_2 = Dense(32, activation="relu" )(x_2)
 x_f = concatenate([x, x_2])
 x_f = Dense(128, activation="relu" )(x_f)
 x_f = Dropout(0.15 )(x_f)
 x_f = BatchNormalization()(x_f)
 x_f = Dense(1, activation="sigmoid" )(x_f)
 model = Model(inputs=[main_inp, stat_inp], outputs = x_f)
 model.compile(loss='binary_crossentropy', optimizer=optim, metrics=['binary_accuracy'])
 return model
plt.figure(figsize=(10,5))
plt.title('Test dataset heatmap')
 X_test_full['Title'] = LabelEncoder().fit_transform(X_test_full['Title'])
 X_test_full['Embarked'] = X_test_full['Embarked'].astype('|S')
 X_test_full['Embarked'] = LabelEncoder().fit_transform(X_test_full['Embarked'])
 X_test_full['Deck'] = X_test_full['Deck'].astype('|S')
 X_test_full['Deck'] = LabelEncoder().fit_transform(X_test_full['Deck'])
 sns.heatmap(X_test_full.corr() , annot=True, cmap='coolwarm')
 plt.show()<feature_engineering>
BATCH_SIZE = 512
EPOCHS = 2
df_test['age'] = np.where(( df_test.age.isnull())&(df_test.title=="Master"),7,
np.where(( df_test.age.isnull())&(df_test.title=="Miss"),22,
np.where(( df_test.age.isnull())&(df_test.title=="Mr"),32,
np.where(( df_test.age.isnull())&(df_test.title=="Mrs"),39,
df_test.age))))<count_missing_values>
model1.fit([X_train,X_train_stats], y_train, batch_size=BATCH_SIZE, epochs=EPOCHS, validation_data=([X_val,X_val_stats],y_val))
df_test.isnull().sum()<feature_engineering>
def f1_score(pred_model):
max_t = 0
max_f1 = 0
for thresh in np.arange(0.1, 0.701, 0.01):
thresh = np.round(thresh, 2)
 f1 = metrics.f1_score(y_val,(pred_model>thresh ).astype(int))
if(f1>max_f1):
max_f1 = f1
max_t = thresh
print(max_t, max_f1)
 return max_t
X_test_full['Age'] = df_test['age'].copy()
X_test_full.isnull().sum()<categorify>
pred_model1 = model1.predict([X_val,X_val_stats], batch_size=1024, verbose=1)
 f1_score(pred_model1 )
for df in([X, X_test_full]):
df['FamilySize'] = df.SibSp + df.Parch + 1
df['Fare'] = LabelEncoder().fit_transform(df['Fare'])
 df['Age'] = LabelEncoder().fit_transform(df['Age'])
 df.drop(['SibSp'], axis=1, inplace=True)
 df.drop(['Parch'], axis=1, inplace=True)
 df.drop(['Ticket'], axis=1, inplace=True)
 df.drop(['Name'], axis=1, inplace=True )<feature_engineering>
model2 = model()
model2.fit([X_train,X_train_stats], y_train, batch_size=BATCH_SIZE, epochs=EPOCHS, validation_data=([X_val,X_val_stats],y_val))
X['Deck_class'] = X['Pclass']*X['Deck']
X_test_full['Deck_class'] = X_test_full['Pclass']*X_test_full['Deck']
X['Emb_class'] = X['Pclass']*X['Embarked']
X_test_full['Emb_class'] = X_test_full['Pclass']*X_test_full['Embarked']<categorify>
pred_model2 = model2.predict([X_val,X_val_stats], batch_size=1024, verbose=1)
 f1_score(pred_model2 )
train_dummies = pd.get_dummies(X, columns=['Sex'])
 test_dummies = pd.get_dummies(X_test_full, columns=['Sex'])
 X = train_dummies.copy()
X_test_full = test_dummies.copy()
for df in([X, X_test_full]):
for col in(['Fare', 'Age', 'FamilySize', 'Deck', 'Title', 'Pclass', 'Deck_class', 'Emb_class']):
df[col] =(df[col]-df[col].mean())/np.std(df[col])
 df.drop(['Sex_male'], axis=1, inplace=True )<split>
pred_model = pred_model1*0.5 + pred_model2*0.5
max_t = f1_score(pred_model )
<import_modules><EOS> 
pred_model1_test = model1.predict([X_test,test_stats], batch_size=1024, verbose=1)
 pred_model2_test = model2.predict([X_test,test_stats], batch_size=1024, verbose=1)
 pred_model_test = pred_model1_test*0.5 + pred_model2_test*0.5
pred_test =(pred_model_test>max_t ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test
out_df.to_csv("submission.csv", index=False )