Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<categorify>
!unzip -n.. /input/quora-insincere-questions-classification/embeddings.zip
df_shops.loc[df_shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
df_shops['city'] = df_shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 df_shops.loc[df_shops.city == '!Якутск', 'city'] = 'Якутск'
df_shops['city_code'] = LabelEncoder().fit_transform(df_shops['city'])
 df_shops = df_shops[['shop_id','city_code']]
df_categories['split'] = df_categories['item_category_name'].str.split('-')
 df_categories['type'] = df_categories['split'].map(lambda x: x[0].strip())
 df_categories['type_code'] = LabelEncoder().fit_transform(df_categories['type'])
 df_categories['subtype'] = df_categories['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 df_categories['subtype_code'] = LabelEncoder().fit_transform(df_categories['subtype'])
 df_categories = df_categories[['item_category_id','type_code', 'subtype_code']]
df_items.drop(['item_name'], axis=1, inplace=True )<data_type_conversions>
tqdm.pandas()
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = df_train[df_train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True )<feature_engineering>
print(f"tf version: {tf.__version__}")
 print(f"keras version: {keras.__version__}" )
df_train['revenue'] = df_train['item_price'] * df_train['item_cnt_day']<merge>
max_length = 60
embedding_size = 600
learning_rate = 0.001
batch_size = 512
num_epoch = 4
group = df_train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
matrix.head()<set_options>
train = pd.read_csv(".. /input/quora-insincere-questions-classification/train.csv")
 test = pd.read_csv(".. /input/quora-insincere-questions-classification/test.csv" )
gc.collect()<data_type_conversions>
regex = re.compile("["+re.escape(string.punctuation)+ "0-9\\r\\t\
]")
 def clean_text(x):
return regex.sub("",x )
df_test['date_block_num'] = 34
df_test['date_block_num'] = df_test['date_block_num'].astype(np.int8)
 df_test['shop_id'] = df_test['shop_id'].astype(np.int8)
 df_test['item_id'] = df_test['item_id'].astype(np.int16 )<concatenate>
mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
def correct_spelling(x):
for word in mispell_dict.keys() :
x = x.replace(word, mispell_dict[word])
 return x
matrix = pd.concat([matrix, df_test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True )<data_type_conversions>
train["question_text"] = train["question_text"].progress_apply(lambda x: x.lower())
 test["question_text"] = test["question_text"].progress_apply(lambda x: x.lower())
 train["question_text"] = train["question_text"].progress_apply(clean_text)
 test["question_text"] = test["question_text"].progress_apply(clean_text)
 train["question_text"] = train["question_text"].progress_apply(correct_spelling)
 test["question_text"] = test["question_text"].progress_apply(correct_spelling )
matrix = pd.merge(matrix, df_shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, df_items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, df_categories, on=['item_category_id'], how='left')
 matrix['city_code'] = matrix['city_code'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
 matrix.head()<merge>
train_X = train["question_text"].values
test_X = test["question_text"].values
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df<concatenate>
tokenizer = Tokenizer()
tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=max_length)
 test_X = pad_sequences(test_X, maxlen=max_length)
 train_y = train['target'].values
matrix = lag_feature(matrix, [1,2,3,6,12], 'item_cnt_month')
 matrix.head()<merge>
embedding_files={"glove":r"glove.840B.300d/glove.840B.300d.txt","para":r"paragram_300_sl999/paragram_300_sl999.txt","wiki":r"wiki-news-300d-1M/wiki-news-300d-1M.vec"}
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 def load_data(embedding,word_index):
embedding_file=embedding_files.get(embedding)
 if embedding=="glove":
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(embedding_file))
elif embedding=="para":
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(embedding_file, encoding="utf8", errors='ignore')if len(o)>100)
 elif embedding=="wiki":
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(embedding_file)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = len(word_index)+1
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix
group = matrix.groupby(['date_block_num'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num'], how='left')
 matrix['date_avg_item_cnt'] = matrix['date_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_avg_item_cnt')
 matrix.drop(['date_avg_item_cnt'], axis=1, inplace=True)
 matrix.head()<merge>
def build_model(embedding_matrix, nb_words, embedding_size=300):
inp = Input(shape=(max_length,))
x = Embedding(nb_words, embedding_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.3 )(x)
 x1 = Bidirectional(LSTM(256, return_sequences=True))(x)
 x2 = Bidirectional(GRU(128, return_sequences=True))(x1)
 max_pool1 = GlobalMaxPooling1D()(x1)
 max_pool2 = GlobalMaxPooling1D()(x2)
 conc = Concatenate()([max_pool1, max_pool2])
 predictions = Dense(1, activation='sigmoid' )(conc)
 model = Model(inputs=inp, outputs=predictions)
 adam = optimizers.Adam(lr=learning_rate)
 model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])
 return model
group = matrix.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_cnt'] = matrix['date_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_item_avg_item_cnt')
 matrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)
 matrix.head()<merge>
train_pred = np.zeros(( len(train_X),), dtype=np.float32)
 pred_prob = np.zeros(( len(test_X),), dtype=np.float32)
 print("Loading embedding matrix...")
 embedding_matrix_wiki = load_data("wiki",tokenizer.word_index)
 embedding_matrix_glove = load_data("glove",tokenizer.word_index)
 embedding_matrix_para = load_data("para",tokenizer.word_index )
group = matrix.groupby(['date_block_num', 'shop_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_shop_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_avg_item_cnt'] = matrix['date_shop_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_shop_avg_item_cnt')
 matrix.drop(['date_shop_avg_item_cnt'], axis=1, inplace=True)
 matrix.head()<merge>
embedding_matrix = np.concatenate(( embedding_matrix_glove, embedding_matrix_wiki), axis=1)
 model = build_model(embedding_matrix, len(embedding_matrix), embedding_size)
 model.fit(train_X, train_y, batch_size=batch_size, epochs=num_epoch,validation_split=0.01 ,verbose=1)
 train_pred += 0.5*np.squeeze(model.predict(train_X, batch_size=batch_size, verbose=1))
pred_prob += 0.5*np.squeeze(model.predict(test_X, batch_size=batch_size, verbose=1))
group = matrix.groupby(['date_block_num', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_cat_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_category_id'], how='left')
 matrix['date_cat_avg_item_cnt'] = matrix['date_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_cat_avg_item_cnt')
 matrix.drop(['date_cat_avg_item_cnt'], axis=1, inplace=True)
 matrix.head()<merge>
embedding_matrix = np.concatenate(( embedding_matrix_glove, embedding_matrix_para), axis=1)
 model = build_model(embedding_matrix, len(embedding_matrix), embedding_size)
 model.fit(train_X, train_y, batch_size=batch_size, epochs=num_epoch,validation_split=0.01 ,verbose=1)
 train_pred += 0.5*np.squeeze(model.predict(train_X, batch_size=batch_size, verbose=1))
pred_prob += 0.5*np.squeeze(model.predict(test_X, batch_size=batch_size, verbose=1))
group = matrix.groupby(['date_block_num', 'shop_id', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_cat_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 matrix['date_shop_cat_avg_item_cnt'] = matrix['date_shop_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_cat_avg_item_cnt')
 matrix.drop(['date_shop_cat_avg_item_cnt'], axis=1, inplace=True)
 matrix.head()<merge>
i=0.01
maxi=0.99
maxf1=0
while i<0.99:
pred_train =(train_pred > i ).astype(np.int)
 f1=f1_score(train_y, pred_train)
 if f1>maxf1:
maxi=i
maxf1=f1
i+=0.01
print(f"threshold: {maxi}")
 print(f"max f1 score: {maxf1}" )
<merge><EOS> 
submission = pd.DataFrame.from_dict({'qid': test['qid']})
 submission['prediction'] =(pred_prob>maxi ).astype(int)
 submission.to_csv('submission.csv', index=False )