Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<filter>
pd_ctx = pd.option_context('display.max_colwidth', 100)
train = train[train.item_price<100000]
train = train[train.item_cnt_day<1001]<feature_engineering>
df_train = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv')
 df_test = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')
 quora_data = df_train['question_text'].append(df_test['question_text'] )
median = train[(train.shop_id==32)&(train.item_id==2973)&(train.date_block_num==4)&(train.item_price>0)].item_price.median()
train.loc[train.item_price<0, 'item_price'] = median<feature_engineering>
df_train.target.value_counts()
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11<categorify>
def clean_tag(x):
if '[math]' in x:
x = re.sub('\[math\].*?math\]', 'MATH EQUATION', x)
 if 'http' in x or 'www' in x:
x = re.sub('(?:(?:https?|ftp):\/\/)?[\w/\-?=%.]+\.[\w/\-?=%.]+', 'URL', x)
 return x
shops.loc[shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 shops = shops[['shop_id','city_code']]
cats['split'] = cats['item_category_name'].str.split('-')
 cats['type'] = cats['split'].map(lambda x: x[0].strip())
 cats['type_code'] = LabelEncoder().fit_transform(cats['type'])
 cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id','type_code', 'subtype_code']]
items.drop(['item_name'], axis=1, inplace=True )<concatenate>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'•', '~', '@', '£', '·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′',
'█', '…', '“', '★', '”', '–', '●', '►', '−', '¢', '¬', '░', '¡', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓',
'—', '‹', '─', '▒', '：', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', '¯', '♦', '¤', '▲', '¸', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '・', '╦', '╣', '╔', '╗', '▬', '❤', '≤', '‡', '√', '◄', '━',
'⇒', '▶', '≥', '╝', '♡', '◊', '。', '✈', '≡', '☺', '✔', '↵', '≈', '✓', '♣', '☎', '℃', '◦', '└', '‟', '～', '！', '○',
'◆', '№', '♠', '▌', '✿', '▸', '⁄', '□', '❖', '✦', '．', '÷', '｜', '┃', '／', '￥', '╠', '↩', '✭', '▐', '☼', '☻', '┐',
'├', '«', '∼', '┌', '℉', '☮', '฿', '≦', '♬', '✧', '〉', '－', '⌂', '✖', '･', '◕', '※', '‖', '◀', '‰', '\x97', '↺',
'∆', '┘', '┬', '╬', '،', '⌘', '⊂', '＞', '〈', '⎙', '？', '☠', '⇐', '▫', '∗', '∈', '≠', '♀', '♔', '˚', '℗', '┗', '＊',
'┼', '❀', '＆', '∩', '♂', '‿', '∑', '‣', '➜', '┛', '⇓', '☯', '⊖', '☀', '┳', '；', '∇', '⇑', '✰', '◇', '♯', '☞', '´',
'↔', '┏', '｡', '◘', '∂', '✌', '♭', '┣', '┴', '┓', '✨', '\xa0', '˜', '❥', '┫', '℠', '✒', '［', '∫', '\x93', '≧', '］',
'\x94', '∀', '♛', '\x96', '∨', '◎', '↻', '⇩', '＜', '≫', '✩', '✪', '♕', '؟', '₤', '☛', '╮', '␊', '＋', '┈', '％',
'╋', '▽', '⇨', '┻', '⊗', '￡', '।', '▂', '✯', '▇', '＿', '➤', '✞', '＝', '▷', '△', '◙', '▅', '✝', '∧', '␉', '☭',
'┊', '╯', '☾', '➔', '∴', '\x92', '▃', '↳', '＾', '׳', '➢', '╭', '➡', '＠', '⊙', '☢', '˝', '∏', '„', '∥', '❝', '☐',
'▆', '╱', '⋙', '๏', '☁', '⇔', '▔', '\x91', '➚', '◡', '╰', '\x85', '♢', '˙', '۞', '✘', '✮', '☑', '⋆', 'ⓘ', '❒',
'☣', '✉', '⌊', '➠', '∣', '❑', '◢', 'ⓒ', '\x80', '〒', '∕', '▮', '⦿', '✫', '✚', '⋯', '♩', '☂', '❞', '‗', '܂', '☜',
'‾', '✜', '╲', '∘', '⟩', '＼', '⟨', '·', '✗', '♚', '∅', 'ⓔ', '◣', '͡', '‛', '❦', '◠', '✄', '❄', '∃', '␣', '≪', '｢',
'≅', '◯', '☽', '∎', '｣', '❧', '̅', 'ⓐ', '↘', '⚓', '▣', '˘', '∪', '⇢', '✍', '⊥', '＃', '⎯', '↠', '۩', '☰', '◥',
'⊆', '✽', '⚡', '↪', '❁', '☹', '◼', '☃', '◤', '❏', 'ⓢ', '⊱', '➝', '̣', '✡', '∠', '｀', '▴', '┤', '∝', '♏', 'ⓐ',
'✎', ';', '␤', '＇', '❣', '✂', '✤', 'ⓞ', '☪', '✴', '⌒', '˛', '♒', '＄', '✶', '▻', 'ⓔ', '◌', '◈', '❚', '❂', '￦',
'◉', '╜', '̃', '✱', '╖', '❉', 'ⓡ', '↗', 'ⓣ', '♻', '➽', '׀', '✲', '✬', '☉', '▉', '≒', '☥', '⌐', '♨', '✕', 'ⓝ',
'⊰', '❘', '＂', '⇧', '̵', '➪', '▁', '▏', '⊃', 'ⓛ', '‚', '♰', '́', '✏', '⏑', '̶', 'ⓢ', '⩾', '￠', '❍', '≃', '⋰', '♋',
'､', '̂', '❋', '✳', 'ⓤ', '╤', '▕', '⌣', '✸', '℮', '⁺', '▨', '╨', 'ⓥ', '♈', '❃', '☝', '✻', '⊇', '≻', '♘', '♞',
'◂', '✟', '⌠', '✠', '☚', '✥', '❊', 'ⓒ', '⌈', '❅', 'ⓡ', '♧', 'ⓞ', '▭', '❱', 'ⓣ', '∟', '☕', '♺', '∵', '⍝', 'ⓑ',
'✵', '✣', '٭', '♆', 'ⓘ', '∶', '⚜', '◞', '்', '✹', '➥', '↕', '̳', '∷', '✋', '➧', '∋', '̿', 'ͧ', '┅', '⥤', '⬆', '⋱',
'☄', '↖', '⋮', '۔', '♌', 'ⓛ', '╕', '♓', '❯', '♍', '▋', '✺', '⭐', '✾', '♊', '➣', '▿', 'ⓑ', '♉', '⏠', '◾', '▹',
'⩽', '↦', '╥', '⍵', '⌋', '։', '➨', '∮', '⇥', 'ⓗ', 'ⓓ', '⁻', '⎝', '⌥', '⌉', '◔', '◑', '✼', '♎', '♐', '╪', '⊚',
'☒', '⇤', 'ⓜ', '⎠', '◐', '⚠', '╞', '◗', '⎕', 'ⓨ', '☟', 'ⓟ', '♟', '❈', '↬', 'ⓓ', '◻', '♮', '❙', '♤', '∉', '؛',
'⁂', 'ⓝ', '־', '♑', '╫', '╓', '╳', '⬅', '☔', '☸', '┄', '╧', '׃', '⎢', '❆', '⋄', '⚫', '̏', '☏', '➞', '͂', '␙',
'ⓤ', '◟', '̊', '⚐', '✙', '↙', '̾', '℘', '✷', '⍺', '❌', '⊢', '▵', '✅', 'ⓖ', '☨', '▰', '╡', 'ⓜ', '☤', '∽', '╘',
'˹', '↨', '♙', '⬇', '♱', '⌡', '⠀', '╛', '❕', '┉', 'ⓟ', '̀', '♖', 'ⓚ', '┆', '⎜', '◜', '⚾', '⤴', '✇', '╟', '⎛',
'☩', '➲', '➟', 'ⓥ', 'ⓗ', '⏝', '◃', '╢', '↯', '✆', '˃', '⍴', '❇', '⚽', '╒', '̸', '♜', '☓', '➳', '⇄', '☬', '⚑',
'✐', '⌃', '◅', '▢', '❐', '∊', '☈', '॥', '⎮', '▩', 'ு', '⊹', '‵', '␔', '☊', '➸', '̌', '☿', '⇉', '⊳', '╙', 'ⓦ',
'⇣', '｛', '̄', '↝', '⎟', '▍', '❗', '״', '΄', '▞', '◁', '⛄', '⇝', '⎪', '♁', '⇠', '☇', '✊', 'ி', '｝', '⭕', '➘',
'⁀', '☙', '❛', '❓', '⟲', '⇀', '≲', 'ⓕ', '⎥', '\u06dd', 'ͤ', '₋', '̱', '̎', '♝', '≳', '▙', '➭', '܀', 'ⓖ', '⇛', '▊',
'⇗', '̷', '⇱', '℅', 'ⓧ', '⚛', '̐', '̕', '⇌', '␀', '≌', 'ⓦ', '⊤', '̓', '☦', 'ⓕ', '▜', '➙', 'ⓨ', '⌨', '◮', '☷',
'◍', 'ⓚ', '≔', '⏩', '⍳', '℞', '┋', '˻', '▚', '≺', 'ْ', '▟', '➻', '̪', '⏪', '̉', '⎞', '┇', '⍟', '⇪', '▎', '⇦', '␝',
'⤷', '≖', '⟶', '♗', '̴', '♄', 'ͨ', '̈', '❜', '̡', '▛', '✁', '➩', 'ா', '˂', '↥', '⏎', '⎷', '̲', '➖', '↲', '⩵', '̗', '❢',
'≎', '⚔', '⇇', '̑', '⊿', '̖', '☍', '➹', '⥊', '⁁', '✢']
def clean_punct(x):
x = str(x)
 for punct in puncts:
if punct in x:
x = x.replace(punct, ' ')
 return x
len(list(set(test.item_id)- set(test.item_id ).intersection(set(train.item_id)))) , len(list(set(test.item_id))), len(test )<data_type_conversions>
mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'bitcoin', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization',
'electroneum':'bitcoin','nanodegree':'degree','hotstar':'star','dream11':'dream','ftre':'fire','tensorflow':'framework','unocoin':'bitcoin',
'lnmiit':'limit','unacademy':'academy','altcoin':'bitcoin','altcoins':'bitcoin','litecoin':'bitcoin','coinbase':'bitcoin','cryptocurency':'cryptocurrency',
'simpliv':'simple','quoras':'quora','schizoids':'psychopath','remainers':'remainder','twinflame':'soulmate','quorans':'quora','brexit':'demonetized',
'iiest':'institute','dceu':'comics','pessat':'exam','uceed':'college','bhakts':'devotee','boruto':'anime',
'cryptocoin':'bitcoin','blockchains':'blockchain','fiancee':'fiance','redmi':'smartphone','oneplus':'smartphone','qoura':'quora','deepmind':'framework','ryzen':'cpu','whattsapp':'whatsapp',
'undertale':'adventure','zenfone':'smartphone','cryptocurencies':'cryptocurrencies','koinex':'bitcoin','zebpay':'bitcoin','binance':'bitcoin','whtsapp':'whatsapp',
'reactjs':'framework','bittrex':'bitcoin','bitconnect':'bitcoin','bitfinex':'bitcoin','yourquote':'your quote','whyis':'why is','jiophone':'smartphone',
'dogecoin':'bitcoin','onecoin':'bitcoin','poloniex':'bitcoin','7700k':'cpu','angular2':'framework','segwit2x':'bitcoin','hashflare':'bitcoin','940mx':'gpu',
'openai':'framework','hashflare':'bitcoin','1050ti':'gpu','nearbuy':'near buy','freebitco':'bitcoin','antminer':'bitcoin','filecoin':'bitcoin','whatapp':'whatsapp',
'empowr':'empower','1080ti':'gpu','crytocurrency':'cryptocurrency','8700k':'cpu','whatsaap':'whatsapp','g4560':'cpu','payymoney':'pay money',
'fuckboys':'fuck boys','intenship':'internship','zcash':'bitcoin','demonatisation':'demonetization','narcicist':'narcissist','mastuburation':'masturbation',
'trignometric':'trigonometric','cryptocurreny':'cryptocurrency','howdid':'how did','crytocurrencies':'cryptocurrencies','phycopath':'psychopath',
'bytecoin':'bitcoin','possesiveness':'possessiveness','scollege':'college','humanties':'humanities','altacoin':'bitcoin','demonitised':'demonetized',
'brasília':'brazilia','accolite':'accolyte','econimics':'economics','varrier':'warrier','quroa':'quora','statergy':'strategy','langague':'language',
'splatoon':'game','7600k':'cpu','gate2018':'gate 2018','in2018':'in 2018','narcassist':'narcissist','jiocoin':'bitcoin','hnlu':'hulu','7300hq':'cpu',
'weatern':'western','interledger':'blockchain','deplation':'deflation', 'cryptocurrencies':'cryptocurrency', 'bitcoin':'blockchain cryptocurrency',}
def correct_mispell(x):
words = x.split()
for i in range(0, len(words)) :
if mispell_dict.get(words[i])is not None:
words[i] = mispell_dict.get(words[i])
 elif mispell_dict.get(words[i].lower())is not None:
words[i] = mispell_dict.get(words[i].lower())
 words = " ".join(words)
 return words
ts = time.time()
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True)
 time.time() - ts<feature_engineering>
def remove_stopwords(x):
x = [word for word in x.split() if word not in STOPWORDS]
x = ' '.join(x)
 return x
train['revenue'] = train['item_price'] * train['item_cnt_day']<merge>
contraction_mapping = {
"I'm": 'I am',
"I'm'a": 'I am about to',
"I'm'o": 'I am going to',
"I've": 'I have',
"I'll": 'I will',
"I'll've": 'I will have',
"I'd": 'I would',
"I'd've": 'I would have',
"i'm": 'i am',
"i'm'a": 'i am about to',
"i'm'o": 'i am going to',
"i've": 'i have',
"i'll": 'i will',
"i'll've": 'i will have',
"i'd": 'i would',
"i'd've": 'i would have',
'Whatcha': 'What are you',
'whatcha': 'what are you',
"amn't": 'am not',
"ain't": 'are not',
"aren't": 'are not',
"'cause": 'because',
"can't": 'can not',
"can't've": 'can not have',
"could've": 'could have',
"couldn't": 'could not',
"couldn't've": 'could not have',
"daren't": 'dare not',
"daresn't": 'dare not',
"dasn't": 'dare not',
"didn't": 'did not',
'didn’t': 'did not',
"don't": 'do not',
'don’t': 'do not',
"doesn't": 'does not',
"e'er": 'ever',
"everyone's": 'everyone is',
'finna': 'fixing to',
'gimme': 'give me',
"gon't": 'go not',
'gonna': 'going to',
'gotta': 'got to',
"hadn't": 'had not',
"hadn't've": 'had not have',
"hasn't": 'has not',
"haven't": 'have not',
"he've": 'he have',
"he's": 'he is',
"he'll": 'he will',
"he'll've": 'he will have',
"he'd": 'he would',
"he'd've": 'he would have',
"here's": 'here is',
"how're": 'how are',
"how'd": 'how did',
"how'd'y": 'how do you',
"how's": 'how is',
"how'll": 'how will',
"isn't": 'is not',
"it's": 'it is',
"'tis": 'it is',
"'twas": 'it was',
"it'll": 'it will',
"it'll've": 'it will have',
"it'd": 'it would',
"it'd've": 'it would have',
'kinda': 'kind of',
"let's": 'let us',
'luv': 'love',
"ma'am": 'madam',
"may've": 'may have',
"mayn't": 'may not',
"might've": 'might have',
"mightn't": 'might not',
"mightn't've": 'might not have',
"must've": 'must have',
"mustn't": 'must not',
"mustn't've": 'must not have',
"needn't": 'need not',
"needn't've": 'need not have',
"ne'er": 'never',
"o'": 'of',
"o'clock": 'of the clock',
"ol'": 'old',
"oughtn't": 'ought not',
"oughtn't've": 'ought not have',
"o'er": 'over',
"shan't": 'shall not',
"sha'n't": 'shall not',
"shalln't": 'shall not',
"shan't've": 'shall not have',
"she's": 'she is',
"she'll": 'she will',
"she'd": 'she would',
"she'd've": 'she would have',
"should've": 'should have',
"shouldn't": 'should not',
"shouldn't've": 'should not have',
"so've": 'so have',
"so's": 'so is',
"somebody's": 'somebody is',
"someone's": 'someone is',
"something's": 'something is',
'sux': 'sucks',
"that're": 'that are',
"that's": 'that is',
"that'll": 'that will',
"that'd": 'that would',
"that'd've": 'that would have',
'em': 'them',
"there're": 'there are',
"there's": 'there is',
"there'll": 'there will',
"there'd": 'there would',
"there'd've": 'there would have',
"these're": 'these are',
"they're": 'they are',
"they've": 'they have',
"they'll": 'they will',
"they'll've": 'they will have',
"they'd": 'they would',
"they'd've": 'they would have',
"this's": 'this is',
"those're": 'those are',
"to've": 'to have',
'wanna': 'want to',
"wasn't": 'was not',
"we're": 'we are',
"we've": 'we have',
"we'll": 'we will',
"we'll've": 'we will have',
"we'd": 'we would',
"we'd've": 'we would have',
"weren't": 'were not',
"what're": 'what are',
"what'd": 'what did',
"what've": 'what have',
"what's": 'what is',
"what'll": 'what will',
"what'll've": 'what will have',
"when've": 'when have',
"when's": 'when is',
"where're": 'where are',
"where'd": 'where did',
"where've": 'where have',
"where's": 'where is',
"which's": 'which is',
"who're": 'who are',
"who've": 'who have',
"who's": 'who is',
"who'll": 'who will',
"who'll've": 'who will have',
"who'd": 'who would',
"who'd've": 'who would have',
"why're": 'why are',
"why'd": 'why did',
"why've": 'why have',
"why's": 'why is',
"will've": 'will have',
"won't": 'will not',
"won't've": 'will not have',
"would've": 'would have',
"wouldn't": 'would not',
"wouldn't've": 'would not have',
"y'all": 'you all',
"y'all're": 'you all are',
"y'all've": 'you all have',
"y'all'd": 'you all would',
"y'all'd've": 'you all would have',
"you're": 'you are',
"you've": 'you have',
"you'll've": 'you shall have',
"you'll": 'you will',
"you'd": 'you would',
"you'd've": 'you would have',
'jan.': 'january',
'feb.': 'february',
'mar.': 'march',
'apr.': 'april',
'jun.': 'june',
'jul.': 'july',
'aug.': 'august',
'sep.': 'september',
'oct.': 'october',
'nov.': 'november',
'dec.': 'december',
'I’m': 'I am',
'I’m’a': 'I am about to',
'I’m’o': 'I am going to',
'I’ve': 'I have',
'I’ll': 'I will',
'I’ll’ve': 'I will have',
'I’d': 'I would',
'I’d’ve': 'I would have',
'i’m': 'i am',
'i’m’a': 'i am about to',
'i’m’o': 'i am going to',
'i’ve': 'i have',
'i’ll': 'i will',
'i’ll’ve': 'i will have',
'i’d': 'i would',
'i’d’ve': 'i would have',
'amn’t': 'am not',
'ain’t': 'are not',
'aren’t': 'are not',
'’cause': 'because',
'can’t': 'can not',
'can’t’ve': 'can not have',
'could’ve': 'could have',
'couldn’t': 'could not',
'couldn’t’ve': 'could not have',
'daren’t': 'dare not',
'daresn’t': 'dare not',
'dasn’t': 'dare not',
'doesn’t': 'does not',
'e’er': 'ever',
'everyone’s': 'everyone is',
'gon’t': 'go not',
'hadn’t': 'had not',
'hadn’t’ve': 'had not have',
'hasn’t': 'has not',
'haven’t': 'have not',
'he’ve': 'he have',
'he’s': 'he is',
'he’ll': 'he will',
'he’ll’ve': 'he will have',
'he’d': 'he would',
'he’d’ve': 'he would have',
'here’s': 'here is',
'how’re': 'how are',
'how’d': 'how did',
'how’d’y': 'how do you',
'how’s': 'how is',
'how’ll': 'how will',
'isn’t': 'is not',
'it’s': 'it is',
'’tis': 'it is',
'’twas': 'it was',
'it’ll': 'it will',
'it’ll’ve': 'it will have',
'it’d': 'it would',
'it’d’ve': 'it would have',
'let’s': 'let us',
'ma’am': 'madam',
'may’ve': 'may have',
'mayn’t': 'may not',
'might’ve': 'might have',
'mightn’t': 'might not',
'mightn’t’ve': 'might not have',
'must’ve': 'must have',
'mustn’t': 'must not',
'mustn’t’ve': 'must not have',
'needn’t': 'need not',
'needn’t’ve': 'need not have',
'ne’er': 'never',
'o’': 'of',
'o’clock': 'of the clock',
'ol’': 'old',
'oughtn’t': 'ought not',
'oughtn’t’ve': 'ought not have',
'o’er': 'over',
'shan’t': 'shall not',
'sha’n’t': 'shall not',
'shalln’t': 'shall not',
'shan’t’ve': 'shall not have',
'she’s': 'she is',
'she’ll': 'she will',
'she’d': 'she would',
'she’d’ve': 'she would have',
'should’ve': 'should have',
'shouldn’t': 'should not',
'shouldn’t’ve': 'should not have',
'so’ve': 'so have',
'so’s': 'so is',
'somebody’s': 'somebody is',
'someone’s': 'someone is',
'something’s': 'something is',
'that’re': 'that are',
'that’s': 'that is',
'that’ll': 'that will',
'that’d': 'that would',
'that’d’ve': 'that would have',
'there’re': 'there are',
'there’s': 'there is',
'there’ll': 'there will',
'there’d': 'there would',
'there’d’ve': 'there would have',
'these’re': 'these are',
'they’re': 'they are',
'they’ve': 'they have',
'they’ll': 'they will',
'they’ll’ve': 'they will have',
'they’d': 'they would',
'they’d’ve': 'they would have',
'this’s': 'this is',
'those’re': 'those are',
'to’ve': 'to have',
'wasn’t': 'was not',
'we’re': 'we are',
'we’ve': 'we have',
'we’ll': 'we will',
'we’ll’ve': 'we will have',
'we’d': 'we would',
'we’d’ve': 'we would have',
'weren’t': 'were not',
'what’re': 'what are',
'what’d': 'what did',
'what’ve': 'what have',
'what’s': 'what is',
'what’ll': 'what will',
'what’ll’ve': 'what will have',
'when’ve': 'when have',
'when’s': 'when is',
'where’re': 'where are',
'where’d': 'where did',
'where’ve': 'where have',
'where’s': 'where is',
'which’s': 'which is',
'who’re': 'who are',
'who’ve': 'who have',
'who’s': 'who is',
'who’ll': 'who will',
'who’ll’ve': 'who will have',
'who’d': 'who would',
'who’d’ve': 'who would have',
'why’re': 'why are',
'why’d': 'why did',
'why’ve': 'why have',
'why’s': 'why is',
'will’ve': 'will have',
'won’t': 'will not',
'won’t’ve': 'will not have',
'would’ve': 'would have',
'wouldn’t': 'would not',
'wouldn’t’ve': 'would not have',
'y’all': 'you all',
'y’all’re': 'you all are',
'y’all’ve': 'you all have',
'y’all’d': 'you all would',
'y’all’d’ve': 'you all would have',
'you’re': 'you are',
'you’ve': 'you have',
'you’ll’ve': 'you shall have',
'you’ll': 'you will',
'you’d': 'you would',
'you’d’ve': 'you would have'
}
def clean_contractions(text):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([contraction_mapping[t] if t in contraction_mapping else t for t in text.split(" ")])
 return text
ts = time.time()
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
time.time() - ts<data_type_conversions>
def data_cleaning(x):
x = clean_tag(x)
 x = clean_punct(x)
 x = correct_mispell(x)
 x = remove_stopwords(x)
 x = clean_contractions(x)
 return x
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16 )<concatenate>
def Preprocess(doc):
corpus=[]
for text in tqdm(doc):
text=clean_contractions(text)
 text=correct_mispell(text)
 text=re.sub(r'[^a-z0-9A-Z]'," ",text)
 text=re.sub(r'[0-9]{1}',"
text=re.sub(r'[0-9]{2}','
text=re.sub(r'[0-9]{3}','
text=re.sub(r'[0-9]{4}','
text=re.sub(r'[0-9]{5,}','
corpus.append(text)
 return corpus
ts = time.time()
matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True)
 time.time() - ts<data_type_conversions>
%%time
!unzip.. /input/quora-insincere-questions-classification/embeddings.zip
ts = time.time()
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['city_code'] = matrix['city_code'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
 time.time() - ts<merge>
%%time
file_name="./GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin"
model_embed=KeyedVectors.load_word2vec_format(file_name,binary=True)
 def vocab_build(corpus):
vocab={}
for text in tqdm(corpus):
for word in text.split() :
try:
vocab[word]+=1
except KeyError:
vocab[word]=1
return vocab
def check_voc(vocab,model):
embed_words=[]
out_vocab={}
total_words=0
total_text=0
for i in tqdm(vocab):
try:
vec=model[i]
embed_words.append(vec)
 total_words+=vocab[i]
except KeyError:
out_vocab[i]=vocab[i]
total_text+=vocab[i]
print("The {:.2f}% of vocabularies have Covered of corpus".format(100*len(embed_words)/len(vocab)))
 print("The {:.2f}% of total text had coverded ".format(( 100*total_words/(total_words+total_text))))
return out_vocab
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df<merge>
total_text=pd.concat([df_train.question_text_cleaned,df_test.question_text_cleaned])
 vocabulary=vocab_build(total_text)
 oov=check_voc(vocabulary,model_embed )
ts = time.time()
group = matrix.groupby(['date_block_num'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num'], how='left')
 matrix['date_avg_item_cnt'] = matrix['date_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_avg_item_cnt')
 matrix.drop(['date_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
df_test['question_text_cleaned_2'] = Preprocess(df_test['question_text'])
 df_train['question_text_cleaned_2'] = Preprocess(df_train['question_text'])
 total_text_2=pd.concat([df_train.question_text_cleaned_2,df_test.question_text_cleaned_2])
 vocabulary2=vocab_build(total_text_2)
 oov2=check_voc(vocabulary2, model_embed )
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_cnt'] = matrix['date_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_item_avg_item_cnt')
 matrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
sort_oov=dict(sorted(oov2.items() , key=operator.itemgetter(1),reverse=True))
dict(list(sort_oov.items())[:50] )
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_shop_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_avg_item_cnt'] = matrix['date_shop_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_shop_avg_item_cnt')
 matrix.drop(['date_shop_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
del oov, oov2,sort_oov,total_text,total_text_2
gc.collect()
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_cat_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_category_id'], how='left')
 matrix['date_cat_avg_item_cnt'] = matrix['date_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_cat_avg_item_cnt')
 matrix.drop(['date_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
def get_word_index(vocab):
word_index=dict(( w,i+1)for i,w in enumerate(vocab.keys()))
return word_index
def fit_one_hot(word_index,corpus):
sent=[]
for text in tqdm(corpus):
li=[]
for word in text.split() :
try:
li.append(word_index[word])
 except KeyError:
li.append(0)
 sent.append(li)
 return sent
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_cat_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 matrix['date_shop_cat_avg_item_cnt'] = matrix['date_shop_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_cat_avg_item_cnt')
 matrix.drop(['date_shop_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
train,val=train_test_split(df_train,test_size=0.2,stratify=df_train.target,random_state=123)
 vocab_size=len(vocabulary2)+1
max_len=40
word_index=get_word_index(vocabulary2)
 train_text=train['question_text_cleaned_2']
val_text=val['question_text_cleaned_2']
test_text=df_test['question_text_cleaned_2']
encodes=fit_one_hot(word_index,train_text)
 train_padded=pad_sequences(encodes,maxlen=max_len,padding="post")
 encodes_=fit_one_hot(word_index,val_text)
 val_padded=pad_sequences(encodes_,maxlen=max_len,padding="post")
 encodes__=fit_one_hot(word_index,test_text)
 test_padded=pad_sequences(encodes__,maxlen=max_len,padding="post" )
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_type_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'type_code'], how='left')
 matrix['date_shop_type_avg_item_cnt'] = matrix['date_shop_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_type_avg_item_cnt')
 matrix.drop(['date_shop_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
count=0
embedding_mat=np.zeros(( vocab_size,300))
for word,i in tqdm(word_index.items()):
try:
vec=model_embed[word]
embedding_mat[i]=vec
except KeyError:
count+=1
continue
print("Number of Out of Vocabulary",count )
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_subtype_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'subtype_code'], how='left')
 matrix['date_shop_subtype_avg_item_cnt'] = matrix['date_shop_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_subtype_avg_item_cnt')
 matrix.drop(['date_shop_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
def get_model_origin(matrix):
inp = Input(shape=(max_len,))
x = Embedding(vocab_size,300,weights=[matrix],input_length=max_len,trainable=False )(inp)
 x = Bidirectional(LSTM(128, return_sequences=True))(x)
 x = Conv1D(64,3,activation="relu" )(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(128, activation="relu" )(x)
 x = Dropout(0.2 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 return model
ts = time.time()
group = matrix.groupby(['date_block_num', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'city_code'], how='left')
 matrix['date_city_avg_item_cnt'] = matrix['date_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_city_avg_item_cnt')
 matrix.drop(['date_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
opt=Adam(learning_rate=0.001)
 BATCH_SIZE = 1024
bin_loss=tf.keras.losses.BinaryCrossentropy(
 from_logits=False,
label_smoothing=0,
name='binary_crossentropy' 
)
 early_stopping=tf.keras.callbacks.EarlyStopping(
 monitor="val_loss",
patience=3,
mode="min",
restore_best_weights=True 
)
 reduce_lr=tf.keras.callbacks.ReduceLROnPlateau(
 monitor="val_loss",
factor=0.2,
patience=2,
verbose=1,
mode="auto" 
)
 my_callbacks=[early_stopping,reduce_lr]
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'city_code'], how='left')
 matrix['date_item_city_avg_item_cnt'] = matrix['date_item_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_item_city_avg_item_cnt')
 matrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
strategy = None
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect()
strategy = tf.distribute.TPUStrategy(tpu)
 print('Use TPU')
 except ValueError:
if len(tf.config.list_physical_devices('GPU')) > 0:
strategy = tf.distribute.MirroredStrategy()
print('Use GPU')
 else:
strategy = tf.distribute.get_strategy()
print('Use CPU' )
ts = time.time()
group = matrix.groupby(['date_block_num', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_type_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'type_code'], how='left')
 matrix['date_type_avg_item_cnt'] = matrix['date_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_type_avg_item_cnt')
 matrix.drop(['date_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
with strategy.scope() :
google_model = get_model_origin(embedding_mat)
 google_model.compile(loss=bin_loss, optimizer=opt, metrics=['accuracy'])
 history=google_model.fit(train_padded, train.target, batch_size=BATCH_SIZE, epochs=30, validation_data=(val_padded, val.target),callbacks=my_callbacks )
ts = time.time()
group = matrix.groupby(['date_block_num', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_subtype_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'subtype_code'], how='left')
 matrix['date_subtype_avg_item_cnt'] = matrix['date_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_subtype_avg_item_cnt')
 matrix.drop(['date_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<data_type_conversions>
google_y_pre=google_model.predict(val_padded, verbose=1)
 best_score = 0
best_thresh = 0
for thresh in np.arange(0.1,0.5,0.01):
if(best_score < metrics.f1_score(val.target,(google_y_pre>thresh ).astype(int))):
best_score = metrics.f1_score(val.target,(google_y_pre>thresh))
best_thresh = round(thresh, 2)
 print("threshold {0:2.2f} f1 score:{1:2.3f}".format(thresh,metrics.f1_score(val.target,(google_y_pre>thresh ).astype(int))))
print("\033[1mBest result {0:2.3f} in thresh_hold {1:2.2f}\033[0m".format(best_score, best_thresh))
ts = time.time()
group = train.groupby(['item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['item_id'], how='left')
 matrix['item_avg_item_price'] = matrix['item_avg_item_price'].astype(np.float16)
 group = train.groupby(['date_block_num','item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['date_item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_price'] = matrix['date_item_avg_item_price'].astype(np.float16)
 lags = [1,2,3,4,5,6]
matrix = lag_feature(matrix, lags, 'date_item_avg_item_price')
 for i in lags:
matrix['delta_price_lag_'+str(i)] = \ 
(matrix['date_item_avg_item_price_lag_'+str(i)] - matrix['item_avg_item_price'])/ matrix['item_avg_item_price']
def select_trend(row):
for i in lags:
if row['delta_price_lag_'+str(i)]:
return row['delta_price_lag_'+str(i)]
return 0
matrix['delta_price_lag'] = matrix.apply(select_trend, axis=1)
 matrix['delta_price_lag'] = matrix['delta_price_lag'].astype(np.float16)
 matrix['delta_price_lag'].fillna(0, inplace=True)
 fetures_to_drop = ['item_avg_item_price', 'date_item_avg_item_price']
for i in lags:
fetures_to_drop += ['date_item_avg_item_price_lag_'+str(i)]
fetures_to_drop += ['delta_price_lag_'+str(i)]
matrix.drop(fetures_to_drop, axis=1, inplace=True)
 time.time() - ts<merge>
del model_embed, history, best_score, best_thresh, google_model, google_y_pre
gc.collect()
ts = time.time()
group = train.groupby(['date_block_num','shop_id'] ).agg({'revenue': ['sum']})
 group.columns = ['date_shop_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(['shop_id'] ).agg({'date_shop_revenue': ['mean']})
 group.columns = ['shop_avg_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['shop_id'], how='left')
 matrix['shop_avg_revenue'] = matrix['shop_avg_revenue'].astype(np.float32)
 matrix['delta_revenue'] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix['delta_revenue'] = matrix['delta_revenue'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'delta_revenue')
 matrix.drop(['date_shop_revenue','shop_avg_revenue','delta_revenue'], axis=1, inplace=True)
 time.time() - ts<feature_engineering>
def get_model(matrix):
inp = Input(shape=(max_len,))
x = Embedding(vocab_size, 300, weights=[matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.3 )(x)
 x1 = Bidirectional(LSTM(256, return_sequences=True))(x)
 x2 = Bidirectional(GRU(128, return_sequences=True))(x1)
 max_pool1 = GlobalMaxPooling1D()(x1)
 max_pool2 = GlobalMaxPooling1D()(x2)
 conc = Concatenate()([max_pool1, max_pool2])
 predictions = Dense(1, activation='sigmoid' )(conc)
 model = Model(inputs=inp, outputs=predictions)
 adam = optimizers.Adam(lr=0.001)
 model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])
 return model
matrix['month'] = matrix['date_block_num'] % 12<categorify>
strategy = None
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect()
strategy = tf.distribute.TPUStrategy(tpu)
 print('Use TPU')
 except ValueError:
if len(tf.config.list_physical_devices('GPU')) > 0:
strategy = tf.distribute.MirroredStrategy()
print('Use GPU')
 else:
strategy = tf.distribute.get_strategy()
print('Use CPU' )
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 matrix['days'] = matrix['month'].map(days ).astype(np.int8 )<data_type_conversions>
with strategy.scope() :
google_model = get_model(embedding_mat)
 google_model.compile(loss=bin_loss, optimizer=Adam(lr=0.001), metrics=['accuracy'])
 history=google_model.fit(train_padded, train.target, batch_size=BATCH_SIZE, epochs=30, validation_data=(val_padded, val.target),callbacks=my_callbacks )
ts = time.time()
cache = {}
matrix['item_shop_last_sale'] = -1
matrix['item_shop_last_sale'] = matrix['item_shop_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = str(row.item_id)+' '+str(row.shop_id)
 if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
matrix.at[idx, 'item_shop_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts<data_type_conversions>
google_y_pre=google_model.predict(val_padded, verbose=1)
 best_score = 0
best_thresh = 0
for thresh in np.arange(0.1,0.5,0.01):
if(best_score < metrics.f1_score(val.target,(google_y_pre>thresh ).astype(int))):
best_score = metrics.f1_score(val.target,(google_y_pre>thresh))
best_thresh = round(thresh, 2)
 print("threshold {0:2.2f} f1 score:{1:2.3f}".format(thresh,metrics.f1_score(val.target,(google_y_pre>thresh ).astype(int))))
print("\033[1mBest result {0:2.3f} in thresh_hold {1:2.2f}\033[0m".format(best_score, best_thresh))
ts = time.time()
cache = {}
matrix['item_last_sale'] = -1
matrix['item_last_sale'] = matrix['item_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = row.item_id
if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
if row.date_block_num>last_date_block_num:
matrix.at[idx, 'item_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts<feature_engineering>
threshold=best_thresh
google_y_test_pre=google_model.predict(test_padded, batch_size=BATCH_SIZE, verbose=1)
 google_y_test_pre=(google_y_test_pre>thresh ).astype(int )
ts = time.time()
matrix['item_shop_first_sale'] = matrix['date_block_num'] - matrix.groupby(['item_id','shop_id'])['date_block_num'].transform('min')
 matrix['item_first_sale'] = matrix['date_block_num'] - matrix.groupby('item_id')['date_block_num'].transform('min')
 time.time() - ts<filter>
del embedding_mat, history, best_score, best_thresh
gc.collect()
ts = time.time()
matrix = matrix[matrix.date_block_num > 11]
time.time() - ts<correct_missing_values>
def load_embed(file):
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 if file == '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec':
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file)if len(o)>100)
 else:
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file, encoding='latin'))
return embeddings_index
ts = time.time()
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
if('item_cnt' in col):
df[col].fillna(0, inplace=True)
 return df
matrix = fill_na(matrix)
 time.time() - ts<load_pretrained>
glove_path = './glove.840B.300d/glove.840B.300d.txt'
paragram_path = './paragram_300_sl999/paragram_300_sl999.txt'
wiki_path = './wiki-news-300d-1M/wiki-news-300d-1M.vec'
matrix.to_pickle('data.pkl')
 del matrix
del cache
del group
del items
del shops
del cats
del train
gc.collect() ;<load_pretrained>
%%time
glove_embed = load_embed(glove_path)
 print("\033[1mGlove Coverage: \033[0m]")
 oov_glove = check_voc(vocabulary2, glove_embed )
data = pd.read_pickle('data.pkl' )<drop_column>
strategy = None
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect()
strategy = tf.distribute.TPUStrategy(tpu)
 print('Use TPU')
 except ValueError:
if len(tf.config.list_physical_devices('GPU')) > 0:
strategy = tf.distribute.MirroredStrategy()
print('Use GPU')
 else:
strategy = tf.distribute.get_strategy()
print('Use CPU' )
data = data[[
'date_block_num',
'shop_id',
'item_id',
'item_cnt_month',
'city_code',
'item_category_id',
'type_code',
'subtype_code',
'item_cnt_month_lag_1',
'item_cnt_month_lag_2',
'item_cnt_month_lag_3',
'item_cnt_month_lag_6',
'item_cnt_month_lag_12',
'date_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_2',
'date_item_avg_item_cnt_lag_3',
'date_item_avg_item_cnt_lag_6',
'date_item_avg_item_cnt_lag_12',
'date_shop_avg_item_cnt_lag_1',
'date_shop_avg_item_cnt_lag_2',
'date_shop_avg_item_cnt_lag_3',
'date_shop_avg_item_cnt_lag_6',
'date_shop_avg_item_cnt_lag_12',
'date_cat_avg_item_cnt_lag_1',
'date_shop_cat_avg_item_cnt_lag_1',
'date_city_avg_item_cnt_lag_1',
'date_item_city_avg_item_cnt_lag_1',
'delta_price_lag',
'month',
'days',
'item_shop_last_sale',
'item_last_sale',
'item_shop_first_sale',
'item_first_sale',
]]<prepare_x_and_y>
count=0
glove_embedding_mat=np.zeros(( vocab_size,300))
for word,i in tqdm(word_index.items()):
try:
vec=glove_embed[word]
glove_embedding_mat[i]=vec
except KeyError:
count+=1
continue
print("Number of Out of Vocabulary",count )
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )<set_options>
with strategy.scope() :
glove_model = get_model(glove_embedding_mat)
 glove_model.compile(loss=bin_loss, optimizer=Adam(lr=0.001), metrics=['accuracy'])
 glove_history=glove_model.fit(train_padded, train.target, batch_size=BATCH_SIZE, epochs=30, validation_data=(val_padded, val.target),callbacks=my_callbacks )
del data
gc.collect() ;<train_model>
glove_y_pre=glove_model.predict(val_padded, verbose=1)
 best_score = 0
best_thresh = 0
for thresh in np.arange(0.1,0.5,0.01):
if(best_score < metrics.f1_score(val.target,(glove_y_pre>thresh ).astype(int))):
best_score = metrics.f1_score(val.target,(glove_y_pre>thresh))
best_thresh = round(thresh, 2)
 print("threshold {0:2.2f} f1 score:{1:2.3f}".format(thresh,metrics.f1_score(val.target,(glove_y_pre>thresh ).astype(int))))
print("\033[1mBest result {0:2.3f} in thresh_hold {1:2.2f}\033[0m".format(best_score, best_thresh))
ts = time.time()
model = XGBRegressor(
 max_depth=8,
n_estimators=1000,
min_child_weight=300,
colsample_bytree=0.8,
subsample=0.8,
eta=0.3,
seed=42)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=True,
early_stopping_rounds = 10)
 time.time() - ts<save_to_csv>
threshold=best_thresh
glove_y_test_pre=glove_model.predict(test_padded, batch_size=BATCH_SIZE, verbose=1)
 glove_y_test_pre=(glove_y_test_pre>thresh ).astype(int )
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('xgb_submission.csv', index=False)
 pickle.dump(Y_pred, open('xgb_train.pickle', 'wb'))
pickle.dump(Y_test, open('xgb_test.pickle', 'wb'))<set_options>
del glove_embed, glove_embedding_mat, glove_history, best_score, best_thresh
gc.collect()
init_notebook_mode(connected=True)
 cufflinks.go_offline(connected=True)
 warnings.filterwarnings("ignore" )<load_from_csv>
strategy = None
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect()
strategy = tf.distribute.TPUStrategy(tpu)
 print('Use TPU')
 except ValueError:
if len(tf.config.list_physical_devices('GPU')) > 0:
strategy = tf.distribute.MirroredStrategy()
print('Use GPU')
 else:
strategy = tf.distribute.get_strategy()
print('Use CPU' )
df_train = pd.read_csv('.. /input/sales_train.csv')
 df_categories = pd.read_csv(".. /input/item_categories.csv")
 df_items = pd.read_csv(".. /input/items.csv")
 df_shops = pd.read_csv(".. /input/shops.csv")
 df_test = pd.read_csv(".. /input/test.csv" )<merge>
%%time
paragram_embed = load_embed(paragram_path)
 print("\033[1mParagram Coverage: \033[0m]")
 oov_paragram = check_voc(vocabulary2, paragram_embed )
df_train = pd.merge(df_train, df_items, on='item_id', how='inner')
 df_train = pd.merge(df_train, df_categories, on='item_category_id', how='inner')
 df_train = pd.merge(df_train, df_shops, on='shop_id', how='inner')
 df_test = pd.merge(df_test, df_items, on='item_id', how='inner')
 df_test = pd.merge(df_test, df_categories, on='item_category_id', how='inner')
 df_test = pd.merge(df_test, df_shops, on='shop_id', how='inner')
 gc.collect()<define_variables>
count=0
para_embedding_mat=np.zeros(( vocab_size,300))
for word,i in tqdm(word_index.items()):
try:
vec=paragram_embed[word.lower() ]
para_embedding_mat[i]=vec
except KeyError:
count+=1
continue
print("Number of Out of Vocabulary",count )
dict_categories = ['Cinema - DVD', 'PC Games - Standard Editions',
'Music - Local Production CD', 'Games - PS3', 'Cinema - Blu-Ray',
'Games - XBOX 360', 'PC Games - Additional Editions', 'Games - PS4',
'Gifts - Stuffed Toys', 'Gifts - Board Games(Compact)',
'Gifts - Figures', 'Cinema - Blu-Ray 3D',
'Programs - Home and Office', 'Gifts - Development',
'Gifts - Board Games', 'Gifts - Souvenirs(on the hinge)',
'Cinema - Collection', 'Music - MP3', 'Games - PSP',
'Gifts - Bags, Albums, Mouse Pads', 'Gifts - Souvenirs',
'Books - Audiobooks', 'Gifts - Gadgets, robots, sports',
'Accessories - PS4', 'Games - PSVita',
'Books - Methodical materials 1C', 'Payment cards - PSN',
'PC Games - Digit', 'Games - Game Accessories', 'Accessories - XBOX 360',
'Accessories - PS3', 'Games - XBOX ONE', 'Music - Vinyl',
'Programs - 1C: Enterprise 8', 'PC Games - Collectible Editions',
'Gifts - Attributes', 'Service Tools',
'Music - branded production CD', 'Payment cards - Live!',
'Game consoles - PS4', 'Accessories - PSVita', 'Batteries',
'Music - Music Video', 'Game Consoles - PS3',
'Books - Comics, Manga', 'Game Consoles - XBOX 360',
'Books - Audiobooks 1C', 'Books - Digit',
'Payment cards(Cinema, Music, Games)', 'Gifts - Cards, stickers',
'Accessories - XBOX ONE', 'Pure media(piece)',
'Programs - Home and Office(Digital)', 'Programs - Educational',
'Game consoles - PSVita', 'Books - Artbooks, encyclopedias',
'Programs - Educational(Digit)', 'Accessories - PSP',
'Gaming consoles - XBOX ONE', 'Delivery of goods',
'Payment Cards - Live!(Figure)',' Tickets(Figure)',
'Music - Gift Edition', 'Service Tools - Tickets',
'Net media(spire)', 'Cinema - Blu-Ray 4K', 'Game consoles - PSP',
'Game Consoles - Others', 'Books - Audiobooks(Figure)',
'Gifts - Certificates, Services', 'Android Games - Digit',
'Programs - MAC(Digit)', 'Payment Cards - Windows(Digit)',
'Books - Business Literature', 'Games - PS2', 'MAC Games - Digit',
'Books - Computer Literature', 'Books - Travel Guides',
'PC - Headsets / Headphones', 'Books - Fiction',
'Books - Cards', 'Accessories - PS2', 'Game consoles - PS2',
'Books - Cognitive literature']
dict_shops = ['Moscow Shopping Center "Semenovskiy"',
'Moscow TRK "Atrium"',
"Khimki Shopping Center",
'Moscow TC "MEGA Teply Stan" II',
'Yakutsk Ordzhonikidze, 56',
'St.Petersburg TC "Nevsky Center"',
'Moscow TC "MEGA Belaya Dacha II"',
'Voronezh(Plekhanovskaya, 13)',
'Yakutsk Shopping Center "Central"',
'Chekhov SEC "Carnival"',
'Sergiev Posad TC "7Ya"',
'Tyumen TC "Goodwin"',
'Kursk TC "Pushkinsky"',
'Kaluga SEC "XXI Century"',
'N.Novgorod Science and entertainment complex "Fantastic"',
'Moscow MTRC "Afi Mall"',
'Voronezh SEC "Maksimir"', 'Surgut SEC "City Mall"',
'Moscow Shopping Center "Areal"(Belyaevo)', 'Krasnoyarsk Shopping Center "June"',
'Moscow TK "Budenovsky"(pav.K7)', 'Ufa "Family" 2',
'Kolomna Shopping Center "Rio"', 'Moscow Shopping Center "Perlovsky"',
'Moscow Shopping Center "New Century"(Novokosino)', 'Omsk Shopping Center "Mega"',
'Moscow Shop C21', 'Tyumen Shopping Center "Green Coast"',
'Ufa TC "Central"', 'Yaroslavl shopping center "Altair"',
'RostovNaDonu "Mega" Shopping Center', '"Novosibirsk Mega "Shopping Center',
'Samara Shopping Center "Melody"', 'St.Petersburg TC "Sennaya"',
"Volzhsky Shopping Center 'Volga Mall' ",
'Vologda Mall "Marmelad"', 'Kazan TC "ParkHouse" II',
'Samara Shopping Center ParkHouse', '1C-Online Digital Warehouse',
'Online store of emergencies', 'Adygea Shopping Center "Mega"',
'Balashikha shopping center "October-Kinomir"' , 'Krasnoyarsk Shopping center "Vzletka Plaza" ',
'Tomsk SEC "Emerald City"', 'Zhukovsky st.Chkalov 39m? ',
'Kazan Shopping Center "Behetle"', 'Tyumen SEC "Crystal"',
'RostovNaDonu TRK "Megacenter Horizon"',
'! Yakutsk Ordzhonikidze, 56 fran', 'Moscow TC "Silver House"',
'Moscow TK "Budenovsky"(pav.A2)', "N.Novgorod SEC 'RIO' ",
'! Yakutsk TTS "Central" fran', 'Mytishchi TRK "XL-3"',
'RostovNaDonu TRK "Megatsentr Horizon" Ostrovnoy', 'Exit Trade',
'Voronezh SEC City-Park "Grad"', "Moscow 'Sale'",
'Zhukovsky st.Chkalov 39m² ',' Novosibirsk Shopping Mall "Gallery Novosibirsk"']<feature_engineering>
with strategy.scope() :
para_model = get_model(para_embedding_mat)
 para_model.compile(loss=bin_loss, optimizer=Adam(lr=0.001), metrics=['accuracy'])
 para_history=para_model.fit(train_padded, train.target, batch_size=BATCH_SIZE, epochs=30, validation_data=(val_padded, val.target),callbacks=my_callbacks )
def date_process(df):
df["date"] = pd.to_datetime(df["date"], format="%d.%m.%Y")
 df["_weekday"] = df['date'].dt.weekday
df["_day"] = df['date'].dt.day
df["_month"] = df['date'].dt.month
return df
def cross_heatmap(df, cols, normalize=False, values=None, aggfunc=None):
temp = cols
cm = sns.light_palette("green", as_cmap=True)
 return pd.crosstab(df[temp[0]], df[temp[1]],
normalize=normalize, values=values, aggfunc=aggfunc ).style.background_gradient(cmap = cm)
 def quantiles(df, columns):
for name in columns:
print(name + " quantiles")
 print(df[name].quantile([.01,.25,.5,.75,.99]))
print("")
 def chi2_test(col):
stat, p, dof, expected = stats.chi2_contingency(( pd.crosstab(df_train[col], df_train.item_cnt_day)))
 prob = 0.95
critical = stats.chi2.ppf(prob, dof)
 print(f"Testing the {col} by Total Items Sold")
 print('dof=%d' % dof)
 print(p)
 print("Critical Result: ")
 if abs(stat)>= critical:
print(f"Critical {round(critical,4)}")
 print('Dependent(reject H0)')
 else:
print('Independent(fail to reject H0)')
 print("")
 alpha = 1.0 - prob
print("P Value: ")
 if p <= alpha:
print(f"P-Value: {round(p,8)}")
 print('Dependent(reject H0)')
 else:
print('Independent(fail to reject H0)')
 def log_transforms(df, cols):
for col in cols:
df[col+'_log'] = np.log(df[col] + 1)
 return df
def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage().sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage().sum() / 1024**2
if verbose:
print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df
def knowningData(df, limit=5):
print(f"Dataset Shape: {df.shape}")
 print('Unique values per column: ')
 print(df.nunique())
 print("
print("")
 for column in df.columns:
print("Column Name: ", column)
 entropy = round(stats.entropy(df[column].value_counts() , base=2),2)
 print("entropy ", entropy,
" | Total nulls: ",(round(df[column].isnull().sum() / len(df[column])* 100,2)) ,
" | Total unique values: ", df.nunique() [column],
" | Missing: ", df[column].isna().sum())
 print("Top 5 most frequent values: ")
 print(round(df[column].value_counts() [:limit] / df[column].value_counts().sum() * 100,2))
print("")
 print("<count_values>
para_y_pre=para_model.predict(val_padded, verbose=1)
 best_score = 0
best_thresh = 0
for thresh in np.arange(0.1,0.5,0.01):
if(best_score < metrics.f1_score(val.target,(para_y_pre>thresh ).astype(int))):
best_score = metrics.f1_score(val.target,(para_y_pre>thresh))
best_thresh = round(thresh, 2)
 print("threshold {0:2.2f} f1 score:{1:2.3f}".format(thresh,metrics.f1_score(val.target,(para_y_pre>thresh ).astype(int))))
print("\033[1mBest result {0:2.3f} in thresh_hold {1:2.2f}\033[0m".format(best_score, best_thresh))
df_train.item_category_name = df_train.item_category_name.map(dict(zip(df_train.item_category_name.value_counts().index, dict_categories)))
 df_train.shop_name = df_train.shop_name.map(dict(zip(df_train.shop_name.value_counts().index, dict_shops)) )<feature_engineering>
threshold=best_thresh
para_y_test_pre=para_model.predict(test_padded, batch_size=BATCH_SIZE, verbose=1)
 para_y_test_pre=(para_y_test_pre>thresh ).astype(int )
df_train['total_amount'] = df_train['item_price'] * df_train['item_cnt_day']<compute_test_metric>
del paragram_embed, para_embedding_mat, para_history, best_score, best_thresh
gc.collect()
quantiles(df_train, ['item_cnt_day','item_price', 'total_amount'] )<feature_engineering>
strategy = None
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect()
strategy = tf.distribute.TPUStrategy(tpu)
 print('Use TPU')
 except ValueError:
if len(tf.config.list_physical_devices('GPU')) > 0:
strategy = tf.distribute.MirroredStrategy()
print('Use GPU')
 else:
strategy = tf.distribute.get_strategy()
print('Use CPU' )
df_train = log_transforms(df_train, ['item_price', 'item_cnt_day', 'total_amount'] )<sort_values>
%%time
wiki_embed = load_embed(wiki_path)
 print("\033[1mWiki Coverage: \033[0m]")
 oov_wiki = check_voc(vocabulary2, wiki_embed )
df_train[['item_cnt_day', 'item_price', 'item_name']].sort_values('item_cnt_day', ascending=False ).head(20 )<sort_values>
count=0
wiki_embedding_mat=np.zeros(( vocab_size,300))
for word,i in tqdm(word_index.items()):
try:
vec=wiki_embed[word]
wiki_embedding_mat[i]=vec
except KeyError:
count+=1
continue
print("Number of Out of Vocabulary",count )
sub_categorys_5000 = df_train.sort_values('total_amount',
ascending=False)[['item_category_name', 'item_name',
'shop_name',
'item_cnt_day','item_price',
'total_amount']].head(5000)
 sub_categorys_5000.head(10 )<count_values>
with strategy.scope() :
wiki_model = get_model(wiki_embedding_mat)
 wiki_model.compile(loss=bin_loss, optimizer=Adam(lr=0.001), metrics=['accuracy'])
 wiki_history=wiki_model.fit(train_padded, train.target, batch_size=BATCH_SIZE, epochs=30, validation_data=(val_padded, val.target),callbacks=my_callbacks )
print(f"Total of solds that have only one unit: {round(len(df_train[df_train.item_cnt_day == 1])/ len(df_train)* 100,2)}%" )<groupby>
wiki_y_pre=wiki_model.predict(val_padded, verbose=1)
 best_score = 0
best_thresh = 0
for thresh in np.arange(0.1,0.5,0.01):
if(best_score < metrics.f1_score(val.target,(wiki_y_pre>thresh ).astype(int))):
best_score = metrics.f1_score(val.target,(wiki_y_pre>thresh))
best_thresh = round(thresh, 2)
 print("threshold {0:2.2f} f1 score:{1:2.3f}".format(thresh,metrics.f1_score(val.target,(wiki_y_pre>thresh ).astype(int))))
print("\033[1mBest result {0:2.3f} in thresh_hold {1:2.2f}\033[0m".format(best_score, best_thresh))
grouped_blocks = df_train.groupby(["date_block_num",
"shop_name","item_category_name"])["item_name",
"item_price",
"item_cnt_day"].agg({"item_name":["nunique", 'count'],
"item_price":["min",'mean','max'],
"item_cnt_day":["min",'mean','max','sum']} )<load_from_csv>
threshold=best_thresh
wiki_y_test_pre=wiki_model.predict(test_padded, batch_size=BATCH_SIZE, verbose=1)
 wiki_y_test_pre=(wiki_y_test_pre>thresh ).astype(int )
del df_train
del df_test
gc.collect()
df_train = pd.read_csv('.. /input/sales_train.csv')
 df_test = pd.read_csv(".. /input/test.csv" )<filter>
del wiki_embed, wiki_embedding_mat, wiki_history, best_score, best_thresh
gc.collect()
df_train = df_train[df_train.item_price<100000]
df_train = df_train[df_train.item_cnt_day<1001]<feature_engineering>
y_pre=0.25*(google_y_pre + glove_y_pre + para_y_pre + wiki_y_pre)
 best_score = 0
best_thresh = 0
for thresh in np.arange(0.1,0.5,0.01):
if(best_score < metrics.f1_score(val.target,(y_pre>thresh ).astype(int))):
best_score = metrics.f1_score(val.target,(y_pre>thresh))
best_thresh = round(thresh, 2)
 print("threshold {0:2.2f} f1 score:{1:2.3f}".format(thresh,metrics.f1_score(val.target,(y_pre>thresh ).astype(int))))
print("\033[1mBest result {0:2.3f} in thresh_hold {1:2.2f}\033[0m".format(best_score, best_thresh))
<feature_engineering><EOS> 
y_test_pre = 0.2*google_y_test_pre + 0.3*glove_y_test_pre + 0.2*para_y_test_pre + 0.3*wiki_y_test_pre
y_test_pre=(y_test_pre>thresh ).astype(int)
 submit=pd.DataFrame()
submit["qid"]=df_test.qid
submit["prediction"]=y_test_pre
submit.to_csv("submission.csv",index=False )