Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<merge>
tqdm().pandas()
pd_ctx = pd.option_context('display.max_colwidth', 100)
 pd.set_option('display.float_format', lambda x: '%.3f' % x )
group = matrix.groupby(['date_block_num', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'city_code'], how='left')
 matrix['date_city_avg_item_cnt'] = matrix['date_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_city_avg_item_cnt')
 matrix.drop(['date_city_avg_item_cnt'], axis=1, inplace=True)
 matrix.head()<merge>
TRAIN_FILE = '/kaggle/input/quora-insincere-questions-classification/train.csv'
TEST_FILE = '/kaggle/input/quora-insincere-questions-classification/test.csv'
df = pd.read_csv(TRAIN_FILE)
 df.info()
test_df = pd.read_csv(TEST_FILE)
 with pd_ctx:
print("Sincere question")
 display(df[df['target'] == 0].head())
 print("Insincere question")
 display(df[df['target'] == 1].head() )
group = matrix.groupby(['date_block_num', 'item_id', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'city_code'], how='left')
 matrix['date_item_city_avg_item_cnt'] = matrix['date_item_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_item_city_avg_item_cnt')
 matrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True)
 matrix.head()<merge>
def statistic(df):
stats = pd.DataFrame() ;
stats['question_text'] = df['question_text']
stats['sp_char_words'] = stats['question_text'].str.findall(r'[^a-zA-Z0-9 ]' ).str.len()
stats['num_capital'] = stats['question_text'].progress_map(lambda x: len([c for c in str(x)if c.isupper() ]))
stats['num_numerics'] = stats['question_text'].progress_map(lambda x: sum(c.isdigit() for c in x))
stats['num_stopwords'] = stats['question_text'].progress_map(lambda x: len([c for c in str(x ).lower().split() if c in STOPWORDS]))
return stats
group = matrix.groupby(['date_block_num', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_type_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'type_code'], how='left')
 matrix['date_type_avg_item_cnt'] = matrix['date_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_type_avg_item_cnt')
 matrix.drop(['date_type_avg_item_cnt'], axis=1, inplace=True)
 matrix.head()<merge>
contractions= {"i'm": 'i am',"i'm'a": 'i am about to',"i'm'o": 'i am going to',"i've": 'i have',"i'll": 'i will',"i'll've": 'i will have',"i'd": 'i would',"i'd've": 'i would have',"Whatcha": 'What are you',"amn't": 'am not',"ain't": 'are not',"aren't": 'are not',"'cause": 'because',"can't": 'can not',"can't've": 'can not have',"could've": 'could have',"couldn't": 'could not',"couldn't've": 'could not have',"daren't": 'dare not',"daresn't": 'dare not',"dasn't": 'dare not',"didn't": 'did not','didn’t': 'did not',"don't": 'do not','don’t': 'do not',"doesn't": 'does not',"e'er": 'ever',"everyone's": 'everyone is',"finna": 'fixing to',"gimme": 'give me',"gon't": 'go not',"gonna": 'going to',"gotta": 'got to',"hadn't": 'had not',"hadn't've": 'had not have',"hasn't": 'has not',"haven't": 'have not',"he've": 'he have',"he's": 'he is',"he'll": 'he will',"he'll've": 'he will have',"he'd": 'he would',"he'd've": 'he would have',"here's": 'here is',"how're": 'how are',"how'd": 'how did',"how'd'y": 'how do you',"how's": 'how is',"how'll": 'how will',"isn't": 'is not',"it's": 'it is',"'tis": 'it is',"'twas": 'it was',"it'll": 'it will',"it'll've": 'it will have',"it'd": 'it would',"it'd've": 'it would have',"kinda": 'kind of',"let's": 'let us',"luv": 'love',"ma'am": 'madam',"may've": 'may have',"mayn't": 'may not',"might've": 'might have',"mightn't": 'might not',"mightn't've": 'might not have',"must've": 'must have',"mustn't": 'must not',"mustn't've": 'must not have',"needn't": 'need not',"needn't've": 'need not have',"ne'er": 'never',"o'": 'of',"o'clock": 'of the clock',"ol'": 'old',"oughtn't": 'ought not',"oughtn't've": 'ought not have',"o'er": 'over',"shan't": 'shall not',"sha'n't": 'shall not',"shalln't": 'shall not',"shan't've": 'shall not have',"she's": 'she is',"she'll": 'she will',"she'd": 'she would',"she'd've": 'she would have',"should've": 'should have',"shouldn't": 'should not',"shouldn't've": 'should not have',"so've": 'so have',"so's": 'so is',"somebody's": 'somebody is',"someone's": 'someone is',"something's": 'something is',"sux": 'sucks',"that're": 'that are',"that's": 'that is',"that'll": 'that will',"that'd": 'that would',"that'd've": 'that would have',"em": 'them',"there're": 'there are',"there's": 'there is',"there'll": 'there will',"there'd": 'there would',"there'd've": 'there would have',"these're": 'these are',"they're": 'they are',"they've": 'they have',"they'll": 'they will',"they'll've": 'they will have',"they'd": 'they would',"they'd've": 'they would have',"this's": 'this is',"those're": 'those are',"to've": 'to have',"wanna": 'want to',"wasn't": 'was not',"we're": 'we are',"we've": 'we have',"we'll": 'we will',"we'll've": 'we will have',"we'd": 'we would',"we'd've": 'we would have',"weren't": 'were not',"what're": 'what are',"what'd": 'what did',"what've": 'what have',"what's": 'what is',"what'll": 'what will',"what'll've": 'what will have',"when've": 'when have',"when's": 'when is',"where're": 'where are',"where'd": 'where did',"where've": 'where have',"where's": 'where is',"which's": 'which is',"who're": 'who are',"who've": 'who have',"who's": 'who is',"who'll": 'who will',"who'll've": 'who will have',"who'd": 'who would',"who'd've": 'who would have',"why're": 'why are',"why'd": 'why did',"why've": 'why have',"why's": 'why is',"will've": 'will have',"won't": 'will not',"won't've": 'will not have',"would've": 'would have',"wouldn't": 'would not',"wouldn't've": 'would not have',"y'all": 'you all',"y'all're": 'you all are',"y'all've": 'you all have',"y'all'd": 'you all would',"y'all'd've": 'you all would have',"you're": 'you are',"you've": 'you have',"you'll've": 'you shall have',"you'll": 'you will',"you'd": 'you would',"you'd've": 'you would have','jan.': 'january','feb.': 'february','mar.': 'march','apr.': 'april','jun.': 'june','jul.': 'july','aug.': 'august','sep.': 'september','oct.': 'october','nov.': 'november','dec.': 'december','I’m': 'I am','I’m’a': 'I am about to','I’m’o': 'I am going to','I’ve': 'I have','I’ll': 'I will','I’ll’ve': 'I will have','I’d': 'I would','I’d’ve': 'I would have','amn’t': 'am not','ain’t': 'are not','aren’t': 'are not','’cause': 'because','can’t': 'can not','can’t’ve': 'can not have','could’ve': 'could have','couldn’t': 'could not','couldn’t’ve': 'could not have','daren’t': 'dare not','daresn’t': 'dare not','dasn’t': 'dare not','doesn’t': 'does not','e’er': 'ever','everyone’s': 'everyone is','gon’t': 'go not','hadn’t': 'had not','hadn’t’ve': 'had not have','hasn’t': 'has not','haven’t': 'have not','he’ve': 'he have','he’s': 'he is','he’ll': 'he will','he’ll’ve': 'he will have','he’d': 'he would','he’d’ve': 'he would have','here’s': 'here is','how’re': 'how are','how’d': 'how did','how’d’y': 'how do you','how’s': 'how is','how’ll': 'how will','isn’t': 'is not','it’s': 'it is','’tis': 'it is','’twas': 'it was','it’ll': 'it will','it’ll’ve': 'it will have','it’d': 'it would','it’d’ve': 'it would have','let’s': 'let us','ma’am': 'madam','may’ve': 'may have','mayn’t': 'may not','might’ve': 'might have','mightn’t': 'might not','mightn’t’ve': 'might not have','must’ve': 'must have','mustn’t': 'must not','mustn’t’ve': 'must not have','needn’t': 'need not','needn’t’ve': 'need not have','ne’er': 'never','o’': 'of','o’clock': 'of the clock','ol’': 'old','oughtn’t': 'ought not','oughtn’t’ve': 'ought not have','o’er': 'over','shan’t': 'shall not','sha’n’t': 'shall not','shalln’t': 'shall not','shan’t’ve': 'shall not have','she’s': 'she is','she’ll': 'she will','she’d': 'she would','she’d’ve': 'she would have','should’ve': 'should have','shouldn’t': 'should not','shouldn’t’ve': 'should not have','so’ve': 'so have','so’s': 'so is','somebody’s': 'somebody is','someone’s': 'someone is','something’s': 'something is','that’re': 'that are','that’s': 'that is','that’ll': 'that will','that’d': 'that would','that’d’ve': 'that would have','there’re': 'there are','there’s': 'there is','there’ll': 'there will','there’d': 'there would','there’d’ve': 'there would have','these’re': 'these are','they’re': 'they are','they’ve': 'they have','they’ll': 'they will','they’ll’ve': 'they will have','they’d': 'they would','they’d’ve': 'they would have','this’s': 'this is','those’re': 'those are','to’ve': 'to have','wasn’t': 'was not','we’re': 'we are','we’ve': 'we have','we’ll': 'we will','we’ll’ve': 'we will have','we’d': 'we would','we’d’ve': 'we would have','weren’t': 'were not','what’re': 'what are','what’d': 'what did','what’ve': 'what have','what’s': 'what is','what’ll': 'what will','what’ll’ve': 'what will have','when’ve': 'when have','when’s': 'when is','where’re': 'where are','where’d': 'where did','where’ve': 'where have','where’s': 'where is','which’s': 'which is','who’re': 'who are','who’ve': 'who have','who’s': 'who is','who’ll': 'who will','who’ll’ve': 'who will have','who’d': 'who would','who’d’ve': 'who would have','why’re': 'why are','why’d': 'why did','why’ve': 'why have','why’s': 'why is','will’ve': 'will have','won’t': 'will not','won’t’ve': 'will not have','would’ve': 'would have','wouldn’t': 'would not','wouldn’t’ve': 'would not have','y’all': 'you all','y’all’re': 'you all are','y’all’ve': 'you all have','y’all’d': 'you all would','y’all’d’ve': 'you all would have','you’re': 'you are','you’ve': 'you have','you’ll’ve': 'you shall have','you’ll': 'you will','you’d': 'you would','you’d’ve': 'you would have'}
missing_spell = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'bitcoin', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization','electroneum':'bitcoin','nanodegree':'degree','hotstar':'star','dream11':'dream','ftre':'fire','tensorflow':'framework','unocoin':'bitcoin','lnmiit':'limit','unacademy':'academy','altcoin':'bitcoin','altcoins':'bitcoin','litecoin':'bitcoin','coinbase':'bitcoin','cryptocurency':'cryptocurrency','simpliv':'simple','quoras':'quora','schizoids':'psychopath','remainers':'remainder','twinflame':'soulmate','quorans':'quora','brexit':'demonetized','iiest':'institute','dceu':'comics','pessat':'exam','uceed':'college','bhakts':'devotee','boruto':'anime','cryptocoin':'bitcoin','blockchains':'blockchain','fiancee':'fiance','redmi':'smartphone','oneplus':'smartphone','qoura':'quora','deepmind':'framework','ryzen':'cpu','whattsapp':'whatsapp','undertale':'adventure','zenfone':'smartphone','cryptocurencies':'cryptocurrencies','koinex':'bitcoin','zebpay':'bitcoin','binance':'bitcoin','whtsapp':'whatsapp','reactjs':'framework','bittrex':'bitcoin','bitconnect':'bitcoin','bitfinex':'bitcoin','yourquote':'your quote','whyis':'why is','jiophone':'smartphone','dogecoin':'bitcoin','onecoin':'bitcoin','poloniex':'bitcoin','7700k':'cpu','angular2':'framework','segwit2x':'bitcoin','hashflare':'bitcoin','940mx':'gpu','openai':'framework','hashflare':'bitcoin','1050ti':'gpu','nearbuy':'near buy','freebitco':'bitcoin','antminer':'bitcoin','filecoin':'bitcoin','whatapp':'whatsapp','empowr':'empower','1080ti':'gpu','crytocurrency':'cryptocurrency','8700k':'cpu','whatsaap':'whatsapp','g4560':'cpu','payymoney':'pay money','fuckboys':'fuck boys','intenship':'internship','zcash':'bitcoin','demonatisation':'demonetization','narcicist':'narcissist','mastuburation':'masturbation','trignometric':'trigonometric','cryptocurreny':'cryptocurrency','howdid':'how did','crytocurrencies':'cryptocurrencies','phycopath':'psychopath','bytecoin':'bitcoin','possesiveness':'possessiveness','scollege':'college','humanties':'humanities','altacoin':'bitcoin','demonitised':'demonetized','brasília':'brazilia','accolite':'accolyte','econimics':'economics','varrier':'warrier','quroa':'quora','statergy':'strategy','langague':'language','splatoon':'game','7600k':'cpu','gate2018':'gate 2018','in2018':'in 2018','narcassist':'narcissist','jiocoin':'bitcoin','hnlu':'hulu','7300hq':'cpu','weatern':'western','interledger':'blockchain','deplation':'deflation', 'cryptocurrencies':'cryptocurrency', 'bitcoin':'blockchain cryptocurrency'}
def clean_tag(x):
if '[math]' in x:
x = re.sub('\[math\].*?math\]', 'math equation', x)
 if 'http' in x or 'www' in x:
x = re.sub('(?:(?:https?|ftp):\/\/)?[\w/\-?=%.]+\.[\w/\-?=%.]+', 'url', x)
 return x
def contraction_fix(word):
try:
a=contractions[word]
except KeyError:
a=word
return a
def misspell_fix(word):
try:
a=missing_spell[word]
except KeyError:
a=word
return a
def clean_text(text):
text = clean_tag(text)
 text = " ".join([contraction_fix(w)for w in text.split() ])
 text = " ".join([misspell_fix(w)for w in text.split() ])
 text = re.sub(r'[^a-zA-Z0-9]', ' ', text)
 return text
def apply_clean_text(question_text):
tmp = pd.DataFrame()
tmp['question_text'] = question_text;
tmp['clean'] = tmp.question_text.progress_map(clean_text)
 with pd_ctx:
display(tmp)
 return tmp['clean']
sample['clean'] = apply_clean_text(sample.question_text )
group = matrix.groupby(['date_block_num', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_subtype_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'subtype_code'], how='left')
 matrix['date_subtype_avg_item_cnt'] = matrix['date_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_subtype_avg_item_cnt')
 matrix.drop(['date_subtype_avg_item_cnt'], axis=1, inplace=True)
 matrix.head()<drop_column>
OOV_TOKEN = '<OOV>'
def create_tokenizer(docs):
tokenizer = Tokenizer(oov_token=OOV_TOKEN)
 tokenizer.fit_on_texts(list(docs))
print("Size of vocabulary: ", len(tokenizer.word_index))
return tokenizer
tokenizer = create_tokenizer(sample['clean'])
 print("20 từ đầu tiên trong từ điển:")
 list(tokenizer.word_index.items())[:20]
del df_items
del df_shops
del df_categories<drop_column>
word_sequences = tokenizer.texts_to_sequences(sample['clean'])
 print("Length of 20 first word_sequences:")
 print(list(map(lambda x: len(x),word_sequences[:20])))
 print("
20 first word_sequences:")
 for sequence in word_sequences[:20]:
print(sequence )
matrix = reduce_mem_usage(matrix )<merge>
MAX_SENTENCE_LENGTH = 60
PADDING_TYPE = 'post'
TRUNCATE_TYPE = 'post'
def create_sequence(tokenizer, docs):
word_sequeces = tokenizer.texts_to_sequences(docs)
 padded_word_sequences = pad_sequences(word_sequeces, maxlen=MAX_SENTENCE_LENGTH, padding=PADDING_TYPE, truncating=TRUNCATE_TYPE)
 return padded_word_sequences
padded_sequences = create_sequence(tokenizer, sample['clean'])
 print("Kích thước mảng:",padded_sequences.shape)
 print("Length of 20 first word_sequences:")
 print(list(map(lambda x: len(x),padded_sequences[:20])))
 print("
10 first word_sequences:")
 for sequence in padded_sequences[:10]:
print(sequence)
group = df_train.groupby(['item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['item_id'], how='left')
 matrix['item_avg_item_price'] = matrix['item_avg_item_price'].astype(np.float16)
 group = df_train.groupby(['date_block_num','item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['date_item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_price'] = matrix['date_item_avg_item_price'].astype(np.float16)
 lags = [1,2,3,4,5,6]
matrix = lag_feature(matrix, lags, 'date_item_avg_item_price')
 for i in lags:
matrix['delta_price_lag_'+str(i)] = \ 
(matrix['date_item_avg_item_price_lag_'+str(i)] - matrix['item_avg_item_price'])/ matrix['item_avg_item_price']
def select_trend(row):
for i in lags:
if row['delta_price_lag_'+str(i)]:
return row['delta_price_lag_'+str(i)]
return 0
matrix['delta_price_lag'] = matrix.apply(select_trend, axis=1)
 matrix['delta_price_lag'] = matrix['delta_price_lag'].astype(np.float16)
 matrix['delta_price_lag'].fillna(0, inplace=True)
 fetures_to_drop = ['item_avg_item_price', 'date_item_avg_item_price']
for i in lags:
fetures_to_drop += ['date_item_avg_item_price_lag_'+str(i)]
fetures_to_drop += ['delta_price_lag_'+str(i)]
matrix.drop(fetures_to_drop, axis=1, inplace=True)
 gc.collect()
matrix.head()<merge>
trainY = train.target
print("Clean train question")
 trainX_text = apply_clean_text(train.question_text)
 print("Clean test question")
 testX_text = apply_clean_text(test_df.question_text )
group = df_train.groupby(['date_block_num','shop_id'] ).agg({'revenue': ['sum']})
 group.columns = ['date_shop_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(['shop_id'] ).agg({'date_shop_revenue': ['mean']})
 group.columns = ['shop_avg_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['shop_id'], how='left')
 matrix['shop_avg_revenue'] = matrix['shop_avg_revenue'].astype(np.float32)
 matrix['delta_revenue'] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix['delta_revenue'] = matrix['delta_revenue'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'delta_revenue')
 matrix.drop(['date_shop_revenue','shop_avg_revenue','delta_revenue'], axis=1, inplace=True )<feature_engineering>
X_train, X_val, y_train, y_val = train_test_split(trainX_text, trainY, test_size=0.2, random_state=123 )
matrix['month'] = matrix['date_block_num'] % 12<categorify>
EMBEDDING_DIM = 300
learning_rate = 0.001
def createModel_bidirectional_LSTM_GRU(features,embedding_matrix = None):
output_bias = Constant(np.log([len(data_pos)/len(data_neg)]))
x_input = Input(shape=(MAX_SENTENCE_LENGTH))
if not(embedding_matrix is None):
embedding = Embedding(features, EMBEDDING_DIM, input_length=MAX_SENTENCE_LENGTH, weights=[embedding_matrix], trainable=False )(x_input)
 else:
embedding = Embedding(features, EMBEDDING_DIM, input_length=MAX_SENTENCE_LENGTH )(x_input)
 x = SpatialDropout1D(0.2 )(embedding)
 lstm = Bidirectional(LSTM(256, return_sequences=True))(x)
 gru = Bidirectional(GRU(128, return_sequences=True))(lstm)
 x = Concatenate()([lstm, gru])
 x = GlobalAveragePooling1D()(x)
 x_output = Dense(1, activation='sigmoid', bias_initializer=output_bias )(x)
 model = Model(inputs=x_input, outputs=x_output)
 opt = Adam(lr=learning_rate)
 model.compile(loss='binary_crossentropy', optimizer= opt, metrics=[f1_m])
 return model
def f1_m(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 precision = true_positives /(predicted_positives + K.epsilon())
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 matrix['days'] = matrix['month'].map(days ).astype(np.int8 )<feature_engineering>
def best_threshold(y_train,train_preds):
tmp = [0,0,0]
delta = 0
for tmp[0] in tqdm(np.arange(0.1, 0.9, 0.01)) :
tmp[1] = metrics.f1_score(y_train, np.array(train_preds)>tmp[0])
 if tmp[1] > tmp[2]:
delta = tmp[0]
tmp[2] = tmp[1]
return delta, tmp[2]
cache = {}
matrix['item_shop_last_sale'] = -1
matrix['item_shop_last_sale'] = matrix['item_shop_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = str(row.item_id)+' '+str(row.shop_id)
 if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
matrix.at[idx, 'item_shop_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num 
<feature_engineering>
strategy = None
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect()
strategy = tf.distribute.TPUStrategy(tpu)
 print('Use TPU')
 except ValueError:
if len(tf.config.list_physical_devices('GPU')) > 0:
strategy = tf.distribute.MirroredStrategy()
print('Use GPU')
 else:
strategy = tf.distribute.get_strategy()
print('Use CPU' )
cache = {}
matrix['item_last_sale'] = -1
matrix['item_last_sale'] = matrix['item_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = row.item_id
if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
if row.date_block_num>last_date_block_num:
matrix.at[idx, 'item_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num<categorify>
class_weight = {
0: 1,
1: 3,
}
batch_size = 1024
n_epochs = 30
early_stopping=tf.keras.callbacks.EarlyStopping(
 monitor="val_loss",
patience=3,
mode="min",
restore_best_weights=True 
)
 reduce_lr=tf.keras.callbacks.ReduceLROnPlateau(
 monitor="val_loss",
factor=0.2,
patience=2,
verbose=1,
mode="auto" 
)
 my_callbacks=[early_stopping,reduce_lr]
def train_model_and_predict(X_train, y_train, X_val, y_val, X_test, embedding_vec=None):
print("Create vocab...")
 tokenizer = create_tokenizer(X_train)
 print("Create sequences...")
 X_train_seq = create_sequence(tokenizer, X_train)
 X_val_seq = create_sequence(tokenizer, X_val)
 X_test_seq = create_sequence(tokenizer, X_test)
 embedding_matrix = None
if not(embedding_vec is None):
embedding_matrix = load_embedding(embedding_vec, tokenizer.word_index)
 model = createModel_bidirectional_LSTM_GRU(len(tokenizer.word_index)+1, embedding_matrix)
 model.fit(X_train_seq, y_train, batch_size=batch_size, epochs=n_epochs, validation_data=(X_val_seq, y_val), class_weight=class_weight, callbacks=my_callbacks)
 val_pred = model.predict(X_val_seq, verbose=1, batch_size=256)
 threshold, f1_score = best_threshold(y_val, val_pred)
 test_pred = model.predict(X_test_seq, verbose=1, batch_size=256)
 return threshold, f1_score, val_pred, test_pred
matrix['item_shop_first_sale'] = matrix['date_block_num'] - matrix.groupby(['item_id','shop_id'])['date_block_num'].transform('min')
 matrix['item_first_sale'] = matrix['date_block_num'] - matrix.groupby('item_id')['date_block_num'].transform('min' )<filter>
with strategy.scope() :
threshold, f1_score, val_pred, test_pred = train_model_and_predict(X_train, y_train, X_val, y_val, testX_text)
 print(metrics.classification_report(y_val,(val_pred>threshold ).astype(int)) )
matrix = matrix[matrix.date_block_num > 11]<correct_missing_values>
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def get_lines_count(file_name):
return sum(1 for _ in open(file_name, encoding="utf8", errors='ignore'))
def load_vec(file_name):
return dict(
 get_coefs(*o.split(" "))
for o in tqdm(open(
 file_name, encoding="utf8", errors='ignore'),
total=get_lines_count(file_name)
)if len(o)> 100
)
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
if('item_cnt' in col):
df[col].fillna(0, inplace=True)
 return df
matrix = fill_na(matrix )<load_pretrained>
EMBEDDING_DIM = 300
ps = PorterStemmer()
lc = LancasterStemmer()
sb = SnowballStemmer('english')
 def load_embedding(word2vec, word2index):
oov_count = 0
vocab_count = 0
vocab_size = len(word2index)
 embedding_weights = np.zeros(( vocab_size+1, EMBEDDING_DIM))
unknown_vector = np.zeros(( EMBEDDING_DIM,), dtype=np.float32)- 1
unknown_words = []
for key, i in tqdm(word2index.items()):
word = key
if word in word2vec:
embedding_weights[i] = word2vec[word]
continue
word = key.capitalize()
if word in word2vec:
embedding_weights[i] = word2vec[word]
continue
word = key.upper()
if word in word2vec:
embedding_weights[i] = word2vec[word]
continue
word = key.lower()
if word in word2vec:
embedding_weights[i] = word2vec[word]
continue
word = ps.stem(key)
 if word in word2vec:
embedding_weights[i] = word2vec[word]
continue
word = lc.stem(key)
 if word in word2vec:
embedding_weights[i] = word2vec[word]
continue
word = sb.stem(key)
 if word in word2vec:
embedding_weights[i] = word2vec[word]
continue
unknown_words.append(key)
 embedding_weights[i] = unknown_vector
print('Top 10 Null word embeddings: ')
 print(unknown_words[:10])
 print('
Null word embeddings: %d' % len(unknown_words))
print("There are {:.2f}% word out of embedding file".format(100*len(unknown_words)/vocab_size))
return embedding_weights
matrix.to_pickle('data.pkl')
 del matrix
del cache
del group
del df_train
gc.collect() ;<load_pretrained>
GLOVE_FILE = 'glove.840B.300d/glove.840B.300d.txt'
!unzip -n /kaggle/input/quora-insincere-questions-classification/embeddings.zip {GLOVE_FILE} -d.
 print('loading glove_vec')
 glove_vec = load_vec(GLOVE_FILE )
data = pd.read_pickle('data.pkl' )<create_dataframe>
with strategy.scope() :
glove_threshold, glove_f1_score, glove_val_pred, glove_test_pred = train_model_and_predict(X_train, y_train, X_val, y_val, testX_text, glove_vec)
 print(metrics.classification_report(y_val,(glove_val_pred>glove_threshold ).astype(int)) )
data = data[['date_block_num', 'shop_id', 'item_id', 'item_cnt_month', 'city_code', 'item_category_id',
'type_code', 'subtype_code', 'item_cnt_month_lag_1', 'item_cnt_month_lag_2',
'item_cnt_month_lag_3', 'item_cnt_month_lag_6', 'item_cnt_month_lag_12',
'date_avg_item_cnt_lag_1', 'date_item_avg_item_cnt_lag_1', 'date_item_avg_item_cnt_lag_2',
'date_item_avg_item_cnt_lag_3', 'date_item_avg_item_cnt_lag_6', 'date_item_avg_item_cnt_lag_12',
'date_shop_avg_item_cnt_lag_1', 'date_shop_avg_item_cnt_lag_2', 'date_shop_avg_item_cnt_lag_3',
'date_shop_avg_item_cnt_lag_6', 'date_shop_avg_item_cnt_lag_12', 'date_cat_avg_item_cnt_lag_1',
'date_shop_cat_avg_item_cnt_lag_1', 'item_shop_first_sale', 'item_first_sale',
'date_city_avg_item_cnt_lag_1', 'date_item_city_avg_item_cnt_lag_1',
'delta_price_lag', 'month', 'days', 'item_shop_last_sale', 'item_last_sale']]<prepare_x_and_y>
del glove_vec
gc.collect()
X_train = data[data.date_block_num < 34].drop(['item_cnt_month'], axis=1)
 y_train = data[data.date_block_num < 34]['item_cnt_month'].clip(0.,20.)
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1)
 X_val = X_train[X_train.date_block_num > 30]
X_train = X_train[X_train.date_block_num <= 30]
y_val = y_train[~y_train.index.isin(X_train.index)]
y_train = y_train[y_train.index.isin(X_train.index)]
X_val_test = X_val[X_val.date_block_num > 32]
X_val = X_val[X_val.date_block_num <= 32]
y_val_test = y_val[~y_val.index.isin(X_val.index)]
y_val = y_val[y_val.index.isin(X_val.index)]
X_train.head()<set_options>
PARA_FILE = 'paragram_300_sl999/paragram_300_sl999.txt'
!unzip -n /kaggle/input/quora-insincere-questions-classification/embeddings.zip {PARA_FILE} -d.
 print('loading para_vec')
 para_vec = load_vec(PARA_FILE )
del data
gc.collect()<normalization>
with strategy.scope() :
para_threshold, para_f1_score, para_val_pred, para_test_pred = train_model_and_predict(X_train, y_train, X_val, y_val, testX_text, para_vec)
 print(metrics.classification_report(y_val,(para_val_pred>para_threshold ).astype(int)) )
thresh = 5 * 10**(-4)
 model = GradientBoostingRegressor()
model.fit(X_train, y_train)
 selection = SelectFromModel(model, threshold=thresh, prefit=True)
 X_train = selection.transform(X_train)
 X_val = selection.transform(X_val)
 X_val_test = selection.transform(X_val_test)
 X_test = selection.transform(X_test)
 del model, selection
gc.collect()<define_search_space>
del para_vec
gc.collect()
hyper_space = {'objective': 'regression',
'metric':'rmse',
'boosting':'gbdt',
'max_depth': hp.choice('max_depth', [3, 5, 8, 10, 12, 15]),
'num_leaves': hp.choice('num_leaves', [25, 50, 75, 100, 125, 150, 225, 250, 350, 400, 500]),
'subsample': hp.choice('subsample', [.3,.5,.7,.8,.9, 1]),
'colsample_bytree': hp.choice('colsample_bytree', [.5,.6,.7,.8,.9, 1]),
'learning_rate': hp.choice('learning_rate', [.01,.1,.05,.2]),
'reg_alpha': hp.choice('reg_alpha', [.1,.2,.3,.4,.5,.6,.7]),
'reg_lambda': hp.choice('reg_lambda', [.1,.2,.3,.4,.5,.6]),
'feature_fraction': hp.choice('feature_fraction', [.6,.7,.8,.9, 1]),
'bagging_frequency': hp.choice('bagging_frequency', [.3,.4,.5,.6,.7,.8,.9]),
'min_child_samples': hp.choice('min_child_samples', [10, 20, 30, 40])}<compute_test_metric>
WIKI_FILE = 'wiki-news-300d-1M/wiki-news-300d-1M.vec'
!unzip -n /kaggle/input/quora-insincere-questions-classification/embeddings.zip {WIKI_FILE} -d.
 print('loading wiki_vec')
 wiki_vec = load_vec(WIKI_FILE )
def rmse(y_pred, y):
return np.sqrt(np.mean(np.square(y - y_pred)) )<compute_train_metric>
with strategy.scope() :
wiki_threshold, wiki_f1_score, wiki_val_pred, wiki_test_pred = train_model_and_predict(X_train, y_train, X_val, y_val, testX_text, wiki_vec)
 print(metrics.classification_report(y_val,(wiki_val_pred>wiki_threshold ).astype(int)) )
lgtrain = lightgbm.Dataset(X_train, label=y_train)
 lgval = lightgbm.Dataset(X_val, label=y_val)
 def evaluate_metric(params):
model_lgb = lightgbm.train(params, lgtrain, 600,
valid_sets=[lgtrain, lgval], early_stopping_rounds=100,
verbose_eval=300)
 pred = model_lgb.predict(X_val_test, num_iteration=1000)
 score = rmse(pred, y_val_test)
 print(score, params)
 return {
'loss': score,
'status': STATUS_OK,
'stats_running': STATUS_RUNNING
}<find_best_params>
del wiki_vec
gc.collect()
trials = Trials()
algo = partial(tpe.suggest,
n_startup_jobs=-1)
 MAX_EVALS= 30
best_vals = fmin(evaluate_metric, space=hyper_space, verbose=1,
algo=algo, max_evals=MAX_EVALS, trials=trials)
 best_params = space_eval(hyper_space, best_vals )<train_model>
val_prod = np.zeros(( len(X_val),), dtype=np.float32)
 val_prod += 1/3 * np.squeeze(glove_val_pred)
 val_prod += 1/3 * np.squeeze(para_val_pred)
 val_prod += 1/3 *np.squeeze(wiki_val_pred)
 threshold_global, f1_global = best_threshold(y_val, val_prod)
 print(metrics.classification_report(y_val,(val_prod>threshold_global ).astype(int)) )
<train_model><EOS> 
pred_prob = np.zeros(( len(testX_text),), dtype=np.float32)
 pred_prob += 1/3 * np.squeeze(glove_test_pred)
 pred_prob += 1/3 * np.squeeze(para_test_pred)
 pred_prob += 1/3 * np.squeeze(wiki_test_pred)
 y_test_pre=(( pred_prob>threshold_global ).astype(int))
submit = pd.DataFrame()
submit["qid"]=test_df.qid
submit["prediction"]=y_test_pre
submit.to_csv("submission.csv",index=False )