Kaggle competition: Quora Insincere Questions Classification
train.drop(train[train.TotalBsmtSF>3000].index, inplace = True)
 train.reset_index(drop = True, inplace = True )<prepare_x_and_y>
df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
y_train = train.SalePrice
train.drop('SalePrice', axis = 1, inplace = True)
 train.shape
merged_df = pd.concat([train, test], axis = 0)
 merged_df.shape 
<data_type_conversions>
cList = {
"ain't": "am not",
"aren't": "are not",
"can't": "cannot",
"can't've": "cannot have",
"'cause": "because",
"could've": "could have",
"couldn't": "could not",
"couldn't've": "could not have",
"didn't": "did not",
"doesn't": "does not",
"don't": "do not",
"hadn't": "had not",
"hadn't've": "had not have",
"hasn't": "has not",
"haven't": "have not",
"he'd": "he would",
"he'd've": "he would have",
"he'll": "he will",
"he'll've": "he will have",
"he's": "he is",
"how'd": "how did",
"how'd'y": "how do you",
"how'll": "how will",
"how's": "how is",
"I'd": "I would",
"I'd've": "I would have",
"I'll": "I will",
"I'll've": "I will have",
"I'm": "I am",
"I've": "I have",
"isn't": "is not",
"it'd": "it had",
"it'd've": "it would have",
"it'll": "it will",
"it'll've": "it will have",
"it's": "it is",
"let's": "let us",
"ma'am": "madam",
"mayn't": "may not",
"might've": "might have",
"mightn't": "might not",
"mightn't've": "might not have",
"must've": "must have",
"mustn't": "must not",
"mustn't've": "must not have",
"needn't": "need not",
"needn't've": "need not have",
"o'clock": "of the clock",
"oughtn't": "ought not",
"oughtn't've": "ought not have",
"shan't": "shall not",
"sha'n't": "shall not",
"shan't've": "shall not have",
"she'd": "she would",
"she'd've": "she would have",
"she'll": "she will",
"she'll've": "she will have",
"she's": "she is",
"should've": "should have",
"shouldn't": "should not",
"shouldn't've": "should not have",
"so've": "so have",
"so's": "so is",
"that'd": "that would",
"that'd've": "that would have",
"that's": "that is",
"there'd": "there had",
"there'd've": "there would have",
"there's": "there is",
"they'd": "they would",
"they'd've": "they would have",
"they'll": "they will",
"they'll've": "they will have",
"they're": "they are",
"they've": "they have",
"to've": "to have",
"wasn't": "was not",
"we'd": "we had",
"we'd've": "we would have",
"we'll": "we will",
"we'll've": "we will have",
"we're": "we are",
"we've": "we have",
"weren't": "were not",
"what'll": "what will",
"what'll've": "what will have",
"what're": "what are",
"what's": "what is",
"what've": "what have",
"when's": "when is",
"when've": "when have",
"where'd": "where did",
"where's": "where is",
"where've": "where have",
"who'll": "who will",
"who'll've": "who will have",
"who's": "who is",
"who've": "who have",
"why's": "why is",
"why've": "why have",
"will've": "will have",
"won't": "will not",
"won't've": "will not have",
"would've": "would have",
"wouldn't": "would not",
"wouldn't've": "would not have",
"y'all": "you all",
"y'alls": "you alls",
"y'all'd": "you all would",
"y'all'd've": "you all would have",
"y'all're": "you all are",
"y'all've": "you all have",
"you'd": "you had",
"you'd've": "you would have",
"you'll": "you you will",
"you'll've": "you you will have",
"you're": "you are",
"you've": "you have"
}
c_re = re.compile('(%s)' % '|'.join(cList.keys()))
merged_df.loc[:, ['MSSubClass', 'OverallQual', 'OverallCond']] = merged_df.loc[:, ['MSSubClass', 'OverallQual', 'OverallCond']].astype('object')
 print('New Shape of merged data:
', merged_df.shape)
 print('Merged structure :
', merged_df.dtypes.value_counts() )<filter>
def expandContractions(text, c_re=c_re):
def replace(match):
return cList[match.group(0)]
return c_re.sub(replace, text)
 def clean_numbers(x):
x = re.sub('[0-9]{5,}', ' number ', x)
 x = re.sub('[0-9]{4}', ' number ', x)
 x = re.sub('[0-9]{3}', ' number ', x)
 x = re.sub('[0-9]{2}', ' number ', x)
 return x
def clean_special_chars(text, punct, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text
missing_col = len(merged_df)- merged_df.loc[:, np.sum(merged_df.isnull())>0].count()
missing_col<correct_missing_values>
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
replace_none = merged_df.loc[:, ['PoolQC','Fence','MiscFeature','GarageCond','GarageQual','GarageFinish','GarageType','Alley','FireplaceQu','BsmtFinType2','BsmtFinType1','BsmtExposure','BsmtCond','BsmtQual']]
for i in replace_none.columns:
merged_df[i].fillna('None', inplace = True)
 missing_col = len(merged_df)- merged_df.loc[:, np.sum(merged_df.isnull())>0].count()
missing_col<correct_missing_values>
df['question_text'] = df['question_text'].apply(lambda x: expandContractions(x))
df['question_text'] = df['question_text'].apply(lambda x: clean_numbers(x))
df['question_text'] = df['question_text'].apply(lambda x: x.lower())
 df['question_text'] = df['question_text'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))
df['question_text'] = df['question_text'].apply(lambda x: correct_spelling(x, mispell_dict))
df.head(100)
replace_mode = merged_df.loc[:, ['MSZoning', 'Electrical', 'Utilities', 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'KitchenQual', 'Functional', 'SaleType']]
for i in replace_mode.columns:
merged_df[i].fillna(merged_df[i].mode() [0], inplace = True)
 missing_col = len(merged_df)- merged_df.loc[:, np.sum(merged_df.isnull())>0].count()
missing_col<correct_missing_values>
test_df['question_text'] = test_df['question_text'].apply(lambda x: expandContractions(x))
test_df['question_text'] = test_df['question_text'].apply(lambda x: clean_numbers(x))
test_df['question_text'] = test_df['question_text'].apply(lambda x: x.lower())
 test_df['question_text'] = test_df['question_text'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))
test_df['question_text'] = test_df['question_text'].apply(lambda x: correct_spelling(x, mispell_dict))
test_df.head()
replace_mode = merged_df.loc[:, ['LotFrontage','MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath', 'GarageYrBlt', 'GarageCars', 'GarageArea']]
for i in replace_mode.columns:
merged_df[i].fillna(merged_df[i].median() , inplace = True)
 missing_col = len(merged_df)- merged_df.loc[:, np.sum(merged_df.isnull())>0].count()
missing_col<create_dataframe>
df_train, df_val = train_test_split(df, test_size=0.1, random_state=42 )
numeric_skew = pd.DataFrame(data = merged_df.select_dtypes(include = ['int64', 'float64'] ).skew() , columns = ['Skewness'])
 numeric_skew_sort = numeric_skew.sort_values(ascending = False, by = 'Skewness')
 numeric_skew_sort<normalization>
embed_size = 200
max_features = 50000
maxlen = 100
x_train = df_train["question_text"].fillna("_na_" ).values
x_val = df_val["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(x_train))
x_train = tokenizer.texts_to_sequences(x_train)
 x_val = tokenizer.texts_to_sequences(x_val)
 x_train = pad_sequences(x_train, maxlen=maxlen)
 x_val = pad_sequences(x_val, maxlen=maxlen)
 y_train = df_train['target'].values
y_val = df_val['target'].values
standard_scaler = StandardScaler()
merged_df_num_scaled = standard_scaler.fit_transform(merged_df_num)
 merged_df_num_scaled = pd.DataFrame(data = merged_df_num_scaled, columns = merged_df_num.columns, index = merged_df_num.index )<categorify>
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
merged_df_cat = merged_df.select_dtypes(include = ['object'] ).astype('category')
 merged_df_cat.shape
merged_df_cat_encoded = merged_df_cat.apply(LabelEncoder().fit_transform)
 merged_df_cat_encoded.shape
merged_df_one_hot = merged_df_cat.select_dtypes(include = ['category'])
 merged_df_one_hot = pd.get_dummies(merged_df_one_hot, drop_first = True)
 merged_df_one_hot.shape
merged_df_cat_encoded_final = pd.concat([merged_df_one_hot, merged_df_cat_encoded], axis = 1)
 merged_df_final_processed =pd.concat([merged_df_num_scaled, merged_df_cat_encoded_final], axis = 1)
 merged_df_final_processed.shape<prepare_x_and_y>
model = Sequential()
model.add(Embedding(max_features,
embed_size,
weights=[embedding_matrix]))
model.add(Bidirectional(CuDNNGRU(64, return_sequences=True)))
 model.add(Bidirectional(CuDNNGRU(32, return_sequences=True)))
 model.add(GlobalMaxPool1D())
 model.add(Dropout(0.1))
model.add(Dense(16, activation='relu'))
model.add(Dropout(0.1))
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['acc'] )
train_final = merged_df_final_processed.iloc[0:1438, :]
test_final = merged_df_final_processed.iloc[1438:, :]
y_train = y_train
X = train_final
y = y_train<compute_train_metric>
history = model.fit(x_train, y_train, batch_size=512, epochs=2, validation_data=(x_val, y_val))
state = 40
kfolds = KFold(n_splits=10, shuffle=True, random_state=state)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, y, scoring="neg_mean_squared_error", cv=kfolds))
return(rmse )<define_search_space>
x_test = test_df["question_text"].fillna("_na_" ).values
x_test = tokenizer.texts_to_sequences(x_test)
 x_test = pad_sequences(x_test, maxlen=maxlen )
<choose_model_class><EOS> 
y_test = model.predict([x_test], batch_size=1024, verbose=1)
 y_test =(y_test > 0.4 ).astype(int)
 test_df = pd.DataFrame({"qid": test_df["qid"].values})
 test_df['prediction'] = y_test
test_df.to_csv("submission.csv", index=False )