Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<load_from_csv>
warnings.filterwarnings(action='ignore')
 warnings.filterwarnings(action='ignore', category=DataConversionWarning)
 warnings.filterwarnings(action='ignore', category=ConvergenceWarning)
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 sub = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv' )<import_modules>
%matplotlib inline
sns.set()
import operator
import re
import gensim<load_pretrained>
TRAIN_PATH = '.. /input/house-prices-advanced-regression-techniques'
TEST_PATH = '.. /input/house-prices-advanced-regression-techniques'
def load_houses_data(TRAIN_PATH=TRAIN_PATH, TEST_PATH=TEST_PATH):
train_csv = os.path.join(TRAIN_PATH, 'train.csv')
 test_csv = os.path.join(TEST_PATH, 'test.csv')
 return pd.read_csv(train_csv), pd.read_csv(test_csv )
df = pd.concat([train.iloc[:, [0,2]] ,test.iloc[:, :2]])
 glove = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
def load_embed(file):
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 if file == '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec':
embeddings_index = gensim.models.KeyedVectors.load_word2vec_format(crawl)
 else:
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file, encoding='latin'))
return embeddings_index<train_model>
X_train, X_test = load_houses_data()
y_train = X_train['SalePrice']
y_train_first = y_train
print("Extracting GloVe embedding")
<categorify>
X_train.groupby('BedroomAbvGr' ).count() ['Id']
def build_vocab(texts):
sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
def check_coverage(vocab, embeddings_index):
known_words = {}
unknown_words = {}
nb_known_words = 0
nb_unknown_words = 0
for word in vocab.keys() :
try:
known_words[word] = embeddings_index[word]
nb_known_words += vocab[word]
except:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
pass
print('Found embeddings for {:.2%} of vocab'.format(len(known_words)/ len(vocab)))
 print('Found embeddings for {:.2%} of all text'.format(nb_known_words /(nb_known_words + nb_unknown_words)))
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words<feature_engineering>
X_train.groupby('BsmtFullBath' ).count() ['Id']
df['lowered_comment'] = df['comment_text'].apply(lambda x: x.lower())
 print("Glove : ")
<categorify>
X_train.groupby('GarageFinish' ).count() ['Id']
def add_lower(embedding, vocab):
count = 0
for word in vocab:
if word in embedding and word.lower() not in embedding:
embedding[word.lower() ] = embedding[word]
count += 1
print(f"Added {count} words to embedding")
 print("Glove : ")
<define_variables>
X_train.corr() ['SalePrice'].sort_values()
contraction_mapping = {
"Trump's" : 'trump is',"'cause": 'because',',cause': 'because',';cause': 'because',"ain't": 'am not','ain,t': 'am not',
'ain;t': 'am not','ain´t': 'am not','ain’t': 'am not',"aren't": 'are not',
'aren,t': 'are not','aren;t': 'are not','aren´t': 'are not','aren’t': 'are not',"can't": 'cannot',"can't've": 'cannot have','can,t': 'cannot','can,t,ve': 'cannot have',
'can;t': 'cannot','can;t;ve': 'cannot have',
'can´t': 'cannot','can´t´ve': 'cannot have','can’t': 'cannot','can’t’ve': 'cannot have',
"could've": 'could have','could,ve': 'could have','could;ve': 'could have',"couldn't": 'could not',"couldn't've": 'could not have','couldn,t': 'could not','couldn,t,ve': 'could not have','couldn;t': 'could not',
'couldn;t;ve': 'could not have','couldn´t': 'could not',
'couldn´t´ve': 'could not have','couldn’t': 'could not','couldn’t’ve': 'could not have','could´ve': 'could have',
'could’ve': 'could have',"didn't": 'did not','didn,t': 'did not','didn;t': 'did not','didn´t': 'did not',
'didn’t': 'did not',"doesn't": 'does not','doesn,t': 'does not','doesn;t': 'does not','doesn´t': 'does not',
'doesn’t': 'does not',"don't": 'do not','don,t': 'do not','don;t': 'do not','don´t': 'do not','don’t': 'do not',
"hadn't": 'had not',"hadn't've": 'had not have','hadn,t': 'had not','hadn,t,ve': 'had not have','hadn;t': 'had not',
'hadn;t;ve': 'had not have','hadn´t': 'had not','hadn´t´ve': 'had not have','hadn’t': 'had not','hadn’t’ve': 'had not have',"hasn't": 'has not','hasn,t': 'has not','hasn;t': 'has not','hasn´t': 'has not','hasn’t': 'has not',
"haven't": 'have not','haven,t': 'have not','haven;t': 'have not','haven´t': 'have not','haven’t': 'have not',"he'd": 'he would',
"he'd've": 'he would have',"he'll": 'he will',
"he's": 'he is','he,d': 'he would','he,d,ve': 'he would have','he,ll': 'he will','he,s': 'he is','he;d': 'he would',
'he;d;ve': 'he would have','he;ll': 'he will','he;s': 'he is','he´d': 'he would','he´d´ve': 'he would have','he´ll': 'he will',
'he´s': 'he is','he’d': 'he would','he’d’ve': 'he would have','he’ll': 'he will','he’s': 'he is',"how'd": 'how did',"how'll": 'how will',
"how's": 'how is','how,d': 'how did','how,ll': 'how will','how,s': 'how is','how;d': 'how did','how;ll': 'how will',
'how;s': 'how is','how´d': 'how did','how´ll': 'how will','how´s': 'how is','how’d': 'how did','how’ll': 'how will',
'how’s': 'how is',"i'd": 'i would',"i'll": 'i will',"i'm": 'i am',"i've": 'i have','i,d': 'i would','i,ll': 'i will',
'i,m': 'i am','i,ve': 'i have','i;d': 'i would','i;ll': 'i will','i;m': 'i am','i;ve': 'i have',"isn't": 'is not',
'isn,t': 'is not','isn;t': 'is not','isn´t': 'is not','isn’t': 'is not',"it'd": 'it would',"it'll": 'it will',"It's":'it is',
"it's": 'it is','it,d': 'it would','it,ll': 'it will','it,s': 'it is','it;d': 'it would','it;ll': 'it will','it;s': 'it is','it´d': 'it would','it´ll': 'it will','it´s': 'it is',
'it’d': 'it would','it’ll': 'it will','it’s': 'it is',
'i´d': 'i would','i´ll': 'i will','i´m': 'i am','i´ve': 'i have','i’d': 'i would','i’ll': 'i will','i’m': 'i am',
'i’ve': 'i have',"let's": 'let us','let,s': 'let us','let;s': 'let us','let´s': 'let us',
'let’s': 'let us',"ma'am": 'madam','ma,am': 'madam','ma;am': 'madam',"mayn't": 'may not','mayn,t': 'may not','mayn;t': 'may not',
'mayn´t': 'may not','mayn’t': 'may not','ma´am': 'madam','ma’am': 'madam',"might've": 'might have','might,ve': 'might have','might;ve': 'might have',"mightn't": 'might not','mightn,t': 'might not','mightn;t': 'might not','mightn´t': 'might not',
'mightn’t': 'might not','might´ve': 'might have','might’ve': 'might have',"must've": 'must have','must,ve': 'must have','must;ve': 'must have',
"mustn't": 'must not','mustn,t': 'must not','mustn;t': 'must not','mustn´t': 'must not','mustn’t': 'must not','must´ve': 'must have',
'must’ve': 'must have',"needn't": 'need not','needn,t': 'need not','needn;t': 'need not','needn´t': 'need not','needn’t': 'need not',"oughtn't": 'ought not','oughtn,t': 'ought not','oughtn;t': 'ought not',
'oughtn´t': 'ought not','oughtn’t': 'ought not',"sha'n't": 'shall not','sha,n,t': 'shall not','sha;n;t': 'shall not',"shan't": 'shall not',
'shan,t': 'shall not','shan;t': 'shall not','shan´t': 'shall not','shan’t': 'shall not','sha´n´t': 'shall not','sha’n’t': 'shall not',
"she'd": 'she would',"she'll": 'she will',"she's": 'she is','she,d': 'she would','she,ll': 'she will',
'she,s': 'she is','she;d': 'she would','she;ll': 'she will','she;s': 'she is','she´d': 'she would','she´ll': 'she will',
'she´s': 'she is','she’d': 'she would','she’ll': 'she will','she’s': 'she is',"should've": 'should have','should,ve': 'should have','should;ve': 'should have',
"shouldn't": 'should not','shouldn,t': 'should not','shouldn;t': 'should not','shouldn´t': 'should not','shouldn’t': 'should not','should´ve': 'should have',
'should’ve': 'should have',"that'd": 'that would',"that's": 'that is','that,d': 'that would','that,s': 'that is','that;d': 'that would',
'that;s': 'that is','that´d': 'that would','that´s': 'that is','that’d': 'that would','that’s': 'that is',"there'd": 'there had',
"there's": 'there is','there,d': 'there had','there,s': 'there is','there;d': 'there had','there;s': 'there is',
'there´d': 'there had','there´s': 'there is','there’d': 'there had','there’s': 'there is',
"they'd": 'they would',"they'll": 'they will',"they're": 'they are',"they've": 'they have',
'they,d': 'they would','they,ll': 'they will','they,re': 'they are','they,ve': 'they have','they;d': 'they would','they;ll': 'they will','they;re': 'they are',
'they;ve': 'they have','they´d': 'they would','they´ll': 'they will','they´re': 'they are','they´ve': 'they have','they’d': 'they would','they’ll': 'they will',
'they’re': 'they are','they’ve': 'they have',"wasn't": 'was not','wasn,t': 'was not','wasn;t': 'was not','wasn´t': 'was not',
'wasn’t': 'was not',"we'd": 'we would',"we'll": 'we will',"we're": 'we are',"we've": 'we have','we,d': 'we would','we,ll': 'we will',
'we,re': 'we are','we,ve': 'we have','we;d': 'we would','we;ll': 'we will','we;re': 'we are','we;ve': 'we have',
"weren't": 'were not','weren,t': 'were not','weren;t': 'were not','weren´t': 'were not','weren’t': 'were not','we´d': 'we would','we´ll': 'we will',
'we´re': 'we are','we´ve': 'we have','we’d': 'we would','we’ll': 'we will','we’re': 'we are','we’ve': 'we have',"what'll": 'what will',"what're": 'what are',"what's": 'what is',
"what've": 'what have','what,ll': 'what will','what,re': 'what are','what,s': 'what is','what,ve': 'what have','what;ll': 'what will','what;re': 'what are',
'what;s': 'what is','what;ve': 'what have','what´ll': 'what will',
'what´re': 'what are','what´s': 'what is','what´ve': 'what have','what’ll': 'what will','what’re': 'what are','what’s': 'what is',
'what’ve': 'what have',"where'd": 'where did',"where's": 'where is','where,d': 'where did','where,s': 'where is','where;d': 'where did',
'where;s': 'where is','where´d': 'where did','where´s': 'where is','where’d': 'where did','where’s': 'where is',
"who'll": 'who will',"who's": 'who is','who,ll': 'who will','who,s': 'who is','who;ll': 'who will','who;s': 'who is',
'who´ll': 'who will','who´s': 'who is','who’ll': 'who will','who’s': 'who is',"won't": 'will not','won,t': 'will not','won;t': 'will not',
'won´t': 'will not','won’t': 'will not',"wouldn't": 'would not','wouldn,t': 'would not','wouldn;t': 'would not','wouldn´t': 'would not',
'wouldn’t': 'would not',"you'd": 'you would',"you'll": 'you will',"you're": 'you are','you,d': 'you would','you,ll': 'you will',
'you,re': 'you are','you;d': 'you would','you;ll': 'you will',
'you;re': 'you are','you´d': 'you would','you´ll': 'you will','you´re': 'you are','you’d': 'you would','you’ll': 'you will','you’re': 'you are',
'´cause': 'because','’cause': 'because',"you've": "you have","could'nt": 'could not',
"havn't": 'have not',"here’s": "here is",'i""m': 'i am',"i'am": 'i am',"i'l": "i will","i'v": 'i have',"wan't": 'want',"was'nt": "was not","who'd": "who would",
"who're": "who are","who've": "who have","why'd": "why would","would've": "would have","y'all": "you all","y'know": "you know","you.i": "you i",
"your'e": "you are","arn't": "are not","agains't": "against","c'mon": "common","doens't": "does not",'don""t': "do not","dosen't": "does not",
"dosn't": "does not","shoudn't": "should not","that'll": "that will","there'll": "there will","there're": "there are",
"this'll": "this all","u're": "you are", "ya'll": "you all","you'r": "you are","you’ve": "you have","d'int": "did not","did'nt": "did not","din't": "did not","dont't": "do not","gov't": "government",
"i'ma": "i am","is'nt": "is not","‘I":'I',
'ᴀɴᴅ':'and','ᴛʜᴇ':'the','ʜᴏᴍᴇ':'home','ᴜᴘ':'up','ʙʏ':'by','ᴀᴛ':'at','…and':'and','civilbeat':'civil beat',\
'TrumpCare':'Trump care','Trumpcare':'Trump care', 'OBAMAcare':'Obama care','ᴄʜᴇᴄᴋ':'check','ғᴏʀ':'for','ᴛʜɪs':'this','ᴄᴏᴍᴘᴜᴛᴇʀ':'computer',\
'ᴍᴏɴᴛʜ':'month','ᴡᴏʀᴋɪɴɢ':'working','ᴊᴏʙ':'job','ғʀᴏᴍ':'from','Sᴛᴀʀᴛ':'start','gubmit':'submit','CO₂':'carbon dioxide','ғɪʀsᴛ':'first',\
'ᴇɴᴅ':'end','ᴄᴀɴ':'can','ʜᴀᴠᴇ':'have','ᴛᴏ':'to','ʟɪɴᴋ':'link','ᴏғ':'of','ʜᴏᴜʀʟʏ':'hourly','ᴡᴇᴇᴋ':'week','ᴇɴᴅ':'end','ᴇxᴛʀᴀ':'extra',\
'Gʀᴇᴀᴛ':'great','sᴛᴜᴅᴇɴᴛs':'student','sᴛᴀʏ':'stay','ᴍᴏᴍs':'mother','ᴏʀ':'or','ᴀɴʏᴏɴᴇ':'anyone','ɴᴇᴇᴅɪɴɢ':'needing','ᴀɴ':'an','ɪɴᴄᴏᴍᴇ':'income',\
'ʀᴇʟɪᴀʙʟᴇ':'reliable','ғɪʀsᴛ':'first','ʏᴏᴜʀ':'your','sɪɢɴɪɴɢ':'signing','ʙᴏᴛᴛᴏᴍ':'bottom','ғᴏʟʟᴏᴡɪɴɢ':'following','Mᴀᴋᴇ':'make',\
'ᴄᴏɴɴᴇᴄᴛɪᴏɴ':'connection','ɪɴᴛᴇʀɴᴇᴛ':'internet','financialpost':'financial post', 'ʜaᴠᴇ':' have ', 'ᴄaɴ':' can ', 'Maᴋᴇ':' make ', 'ʀᴇʟɪaʙʟᴇ':' reliable ', 'ɴᴇᴇᴅ':' need ',
'ᴏɴʟʏ':' only ', 'ᴇxᴛʀa':' extra ', 'aɴ':' an ', 'aɴʏᴏɴᴇ':' anyone ', 'sᴛaʏ':' stay ', 'Sᴛaʀᴛ':' start', 'SHOPO':'shop',
}<feature_engineering>
X_train['YrSold'] = X_train['YrSold'].astype(str)
 X_train['MoSold'] = X_train['MoSold'].astype(str )
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
df['treated_comment'] = df['lowered_comment'].apply(lambda x: clean_contractions(x, contraction_mapping))
print("Glove : ")
<categorify>
class Normalize(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
dataset = y_train.copy()
dataset = np.log1p(dataset)
 return dataset
y_train = Normalize().transform(y_train)
 sns.distplot(y_train)
punct = "/-'?!.,
def unknown_punct(embed, punct):
unknown = ''
for p in punct:
if p not in embed:
unknown += p
unknown += ' '
return unknown
print("Glove :")
<categorify>
X_train_label = X_train[label_attrs]
X_train_num = X_train[num_attrs]
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }
def clean_special_chars(text, punct, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text
df['treated_comment'] = df['treated_comment'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))
print("Glove : ")
<define_variables>
X_train_num_std = pd.DataFrame(StandardScaler().fit_transform(X_train_num), columns=X_train_num.columns )
mispell_dict = {'SB91':'senate bill','tRump':'trump','utmterm':'utm term','FakeNews':'fake news','Gʀᴇat':'great','ʙᴏᴛtoᴍ':'bottom','washingtontimes':'washington times','garycrum':'gary crum','htmlutmterm':'html utm term','RangerMC':'car','TFWs':'tuition fee waiver','SJWs':'social justice warrior','Koncerned':'concerned','Vinis':'vinys','Yᴏᴜ':'you','Trumpsters':'trump','Trumpian':'trump','bigly':'big league','Trumpism':'trump','Yoyou':'you','Auwe':'wonder','Drumpf':'trump','utmterm':'utm term','Brexit':'british exit','utilitas':'utilities','ᴀ':'a', '😉':'wink','😂':'joy','😀':'stuck out tongue', 'theguardian':'the guardian','deplorables':'deplorable', 'theglobeandmail':'the globe and mail', 'justiciaries': 'justiciary','creditdation': 'Accreditation','doctrne':'doctrine','fentayal': 'fentanyl','designation-': 'designation','CONartist' : 'con-artist','Mutilitated' : 'Mutilated','Obumblers': 'bumblers','negotiatiations': 'negotiations','dood-': 'dood','irakis' : 'iraki','cooerate': 'cooperate','COx':'cox','racistcomments':'racist comments','envirnmetalists': 'environmentalists',}<feature_engineering>
X_train.corr() ['SalePrice'].sort_values()
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
df['treated_comment'] = df['treated_comment'].apply(lambda x: correct_spelling(x, mispell_dict))
print("Glove : ")
<feature_engineering>
num_colinear_drop_attrs = ['GarageArea', 'BedroomAbvGr', 'TotRmsAbvGrd', 'GarageYrBlt', '1stFlrSF']
train['comment_text'] = df['treated_comment'][:1804874]
test['comment_text'] = df['treated_comment'][1804874:]<normalization>
class MergeBath(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self;
def transform(self, X, y=None):
X = X.copy()
X['Bath'] = X['HalfBath'] * X['FullBath']
X['HalfBath2'] = X['HalfBath'] ** 2
X['FullBath2'] = X['FullBath'] ** 2
X['BsmtBath'] = X['BsmtHalfBath'] * X['BsmtFullBath']
X['BsmtHalfBath2'] = X['BsmtHalfBath'] ** 2
X['BsmtFullBath2'] = X['BsmtFullBath'] ** 2
return X
X_num_merged = MergeBath().transform(X_train_num )
print('-' * 80)
 print('train')
 train = reduce_mem_usage(train)
 print('-' * 80)
 print('test')
 test = reduce_mem_usage(test )<save_to_csv>
class MergeBsmntFs(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self;
def transform(self, X, y=None):
X = X.copy()
X['BsmtFinSF'] = X['BsmtFinSF1'] * X['BsmtFinSF2']
X['BsmtFinSF12'] = X['BsmtFinSF1'] ** 2
X['BsmtFinSF22'] = X['BsmtFinSF2'] ** 2
return X
X_num_bsmnt_proved = MergeBsmntFs().transform(X_num_merged )
train.to_csv('train_cleaned.csv', index=None)
 test.to_csv('test_cleaned.csv', index=None )<import_modules>
class MergePorches(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self;
def transform(self, X, y=None):
X = X.copy()
X['Porch'] = X['OpenPorchSF'] + X['EnclosedPorch'] + X['3SsnPorch'] + X['ScreenPorch']
X.drop(['OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch'], axis=1, inplace=True)
 return X
X_num_porch_merged = MergePorches().transform(X_num_bsmnt_proved )
import numpy as np
import pandas as pd
from keras.models import Model
from keras.layers import Input, Dense, Embedding, SpatialDropout1D, Dropout, add, concatenate
from keras.layers import CuDNNLSTM, Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.preprocessing import text, sequence
from keras.callbacks import LearningRateScheduler
from keras.losses import binary_crossentropy
from keras import backend as K 
<define_variables>
class FilterLotAreaAndRooms(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self;
def transform(self, X, y=None):
X = X.copy()
X['LotArea'] = X['LotArea'].apply(lambda l: 50001 if l > 50000 else l)
 X['BedroomAbvGr'] = X['BedroomAbvGr'].apply(lambda l: 5 if l > 5 else l)
 return X
X_num_lot_filtered = FilterLotAreaAndRooms().transform(X_num_porch_merged )
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt'
]
NUM_MODELS = 2
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f)
 def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix
def custom_loss(y_true, y_pred):
return binary_crossentropy(K.reshape(y_true[:,0],(-1,1)) , y_pred)* y_true[:,1]
def build_model(embedding_matrix, num_aux_targets, loss_weight):
words = Input(shape=(MAX_LEN,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.3 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss=[custom_loss,'binary_crossentropy'], loss_weights=[loss_weight, 1.0], optimizer='adam')
 return model
def preprocess(data):

 punct = "/-'?!.,
def clean_special_chars(text, punct):
for p in punct:
text = text.replace(p, ' ')
 return text
data = data.astype(str ).apply(lambda x: clean_special_chars(x, punct))
return data 
<load_from_csv>
num_scatter_drop = ['MSSubClass', 'LowQualFinSF']
x_train = preprocess(train['comment_text'])
 identity_columns = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
weights = np.ones(( len(x_train),)) / 4
weights +=(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values>=0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values<0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values<0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
loss_weight = 1.0 / weights.mean()
y_train = np.vstack([(train['target'].values>=0.5 ).astype(np.int),weights] ).T
y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']].values
x_test = preprocess(test['comment_text'])
<string_transform>
to_delete_outlires = ['GrLivArea', 'OverallCond', 'BsmtFinSF1', 'GarageCars',
'2ndFlrSF', 'YearBuilt', 'YearRemodAdd']
tokenizer = text.Tokenizer()
tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN)
 embedding_matrix = np.concatenate(
 [build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1)
 with open('temporary.pickle', mode='wb')as f:
pickle.dump(x_test, f)
 del identity_columns, weights, tokenizer, train, test, x_test
gc.collect()
checkpoint_predictions = []
weights = []
for model_idx in range(NUM_MODELS):
model = build_model(embedding_matrix, y_aux_train.shape[-1], loss_weight)
 for global_epoch in range(EPOCHS):
model.fit(
 x_train,
[y_train, y_aux_train],
batch_size=BATCH_SIZE,
epochs=1,
verbose=1,
callbacks=[
LearningRateScheduler(lambda epoch: 1e-3 *(0.6 ** global_epoch))
] 
)
 with open('temporary.pickle', mode='rb')as f:
x_test = pickle.load(f)
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten())
 del x_test
gc.collect()
weights.append(2 ** global_epoch)
 del model
gc.collect()
predictions = np.average(checkpoint_predictions, weights=weights, axis=0)
 df_submit = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv')
 df_submit.prediction = predictions
df_submit.to_csv('submission.csv', index=False )<set_options>
X_train_num_std_imputed = pd.DataFrame(SimpleImputer().fit_transform(X_train_num_std),
columns= X_train_num_std.columns )
%reload_ext autoreload
%autoreload 2
%matplotlib inline
sns.set_style('whitegrid' )<define_variables>
z = pd.DataFrame(np.abs(stats.zscore(X_train_num)) , columns=X_train_num.columns )
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()
CRAWL_EMBEDDING_PATH = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
GLOVE_EMBEDDING_PATH = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
NUM_MODELS = 2
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
MAX_LEN = 220
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
emb_arr = []
for line in tqdm(f):
try:
emb_arr.append(get_coefs(*line.strip().split(' ')))
 except Exception as e:
print(e)
 return dict(emb_arr)
 def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
unknown_words = []
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words
def sigmoid(x):
return 1 /(1 + np.exp(-x))
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
def preprocess(data):

 punct = "/-'?!.,
def clean_special_chars(text, punct):
for p in punct:
text = text.replace(p, ' ')
 return text
data = data.astype(str ).apply(lambda x: clean_special_chars(x, punct))
return data
def train_model(learn,test,output_dim,lr=0.001,
batch_size=512, n_epochs=4,
enable_checkpoint_ensemble=True):
all_test_preds = []
checkpoint_weights = [2 ** epoch for epoch in range(n_epochs)]
test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 n = len(learn.data.train_dl)
 phases = [(TrainingPhase(n ).schedule_hp('lr', lr *(0.6**(i)))) for i in range(n_epochs)]
sched = GeneralScheduler(learn, phases)
 learn.callbacks.append(sched)
 for epoch in range(n_epochs):
learn.fit(1)
 test_preds = np.zeros(( len(test), output_dim))
for i, x_batch in enumerate(test_loader):
X = x_batch[0].cuda()
y_pred = sigmoid(learn.model(X ).detach().cpu().numpy())
 test_preds[i * batch_size:(i+1)* batch_size, :] = y_pred
all_test_preds.append(test_preds)
 if enable_checkpoint_ensemble:
test_preds = np.average(all_test_preds, weights=checkpoint_weights, axis=0)
 else:
test_preds = all_test_preds[-1]
return test_preds
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 x_train = preprocess(train['comment_text'])
 y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]
x_test = preprocess(test['comment_text'])
 identity_columns = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
weights = np.ones(( len(x_train),)) / 4
weights +=(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values>=0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values<0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values<0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
loss_weight = 1.0 / weights.mean()
y_train = np.vstack([(train['target'].values>=0.5 ).astype(np.int),weights] ).T
max_features = None
tokenizer = text.Tokenizer()
tokenizer.fit_on_texts(list(x_train)+ list(x_test))
crawl_matrix, unknown_words_crawl = build_matrix(tokenizer.word_index, CRAWL_EMBEDDING_PATH)
 print('n unknown words(crawl): ', len(unknown_words_crawl))
glove_matrix, unknown_words_glove = build_matrix(tokenizer.word_index, GLOVE_EMBEDDING_PATH)
 print('n unknown words(glove): ', len(unknown_words_glove))
max_features = max_features or len(tokenizer.word_index)+ 1
max_features
embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis=-1)
 embedding_matrix.shape
del crawl_matrix
del glove_matrix
gc.collect()
y_train_torch = torch.tensor(np.hstack([y_train, y_aux_train]), dtype=torch.float32 )<string_transform>
X_train_without_outlier = X_train[(z[to_delete_outlires] < 3 ).all(axis=1)]
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test )<define_variables>
y_train_without_outlier = y_train[(z[to_delete_outlires] < 3 ).all(axis=1)]
lengths = np.array([len(x)for x in x_train] )<set_options>
class DataFrameDropper(BaseEstimator, TransformerMixin):
def __init__(self, drop_attrs=[]):
self.drop_attrs = drop_attrs
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X = X.copy()
X.drop(self.drop_attrs, axis=1, inplace=True, errors='ignore')
 return X
np.percentile(lengths, q=95 )<define_variables>
num_drop_attrs = num_scatter_drop + num_colinear_drop_attrs
X_num_dropped = DataFrameDropper(num_drop_attrs ).transform(X_num_lot_filtered )
batch_size = 512<categorify>
label_drop_attrs = ['Alley', 'FireplaceQu', 'PoolQC', 'Fence', 'MiscFeature']
X_label_dropped = DataFrameDropper(label_drop_attrs ).transform(X_train_label )
class TextDataset(data.Dataset):
def __init__(self, text, lens, y=None):
self.text = text
self.lens = lens
self.y = y
def __len__(self):
return len(self.lens)
 def __getitem__(self, idx):
if self.y is None:
return self.text[idx], self.lens[idx]
return self.text[idx], self.lens[idx], self.y[idx]
class Collator(object):
def __init__(self, test=False, percentile=100):
self.test = test
self.percentile = percentile
def __call__(self, batch):
global MAX_LEN
if self.test:
texts, lens = zip(*batch)
 else:
texts, lens, target = zip(*batch)
 lens = np.array(lens)
 max_len = min(int(np.percentile(lens, self.percentile)) , MAX_LEN)
 texts = torch.tensor(sequence.pad_sequences(texts, maxlen=max_len), dtype=torch.long)
 if self.test:
return texts
return texts, torch.tensor(target, dtype=torch.float32 )<create_dataframe>
X_label_imputed = pd.DataFrame(SimpleImputer(strategy="most_frequent" ).fit_transform(X_label_dropped.values),
columns=X_label_dropped.columns )
train_collate = Collator(percentile=100)
 train_dataset = TextDataset(x_train, lengths, y_train_torch.numpy())
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, collate_fn=train_collate )<define_variables>
class OneHotGoodEncoder(BaseEstimator, TransformerMixin):
def __init__(self):
self.encoder = OneHotEncoder()
def fit(self, X, y=None):
self.encoder.fit(X)
 def transform(self, X, y=None):
columns = X.columns
X_transformed = self.encoder.transform(X ).toarray()
cats = self.encoder.categories_
i = 0
labels = []
for cat in cats:
for c in cat:
labels.append(columns[i] + ' : ' + c)
 i = i+1
return pd.DataFrame(X_transformed, columns=labels)
n_repeats = 10
start_time = time.time()
for _ in range(n_repeats):
for batch in tqdm(train_loader):
pass
method1_time =(time.time() - start_time)/ n_repeats<categorify>
encoder = OneHotGoodEncoder()
encoder.fit(X_label_imputed)
 X_label_encoded = encoder.transform(X_label_imputed )
class SequenceDataset(torch.utils.data.Dataset):

 def __init__(self, sequences, choose_length, other_features=None, labels=None,
indices=None, shuffle=False, batch_size=512):
super(SequenceDataset, self ).__init__()
self.sequences = np.array(sequences)
 self.lengths = np.array([len(x)for x in sequences])
 self.n_samples = len(sequences)
 self.choose_length = choose_length
self.other_features = other_features
self.labels = labels
if indices is not None:
self.indices = indices
else:
self.indices = np.arange(len(sequences))
self.batch_size = batch_size
self.shuffle = shuffle
if self.shuffle:
self._shuffle()
def __len__(self):
return math.ceil(len(self.indices)/ self.batch_size)
 def _shuffle(self):
self.indices = np.random.permutation(self.indices)
 def __getitem__(self, i):
idx = self.indices[(self.batch_size * i):(self.batch_size *(i + 1)) ]
if self.shuffle and i == len(self)- 1:
self._shuffle()
pad_length = math.ceil(self.choose_length(self.lengths[idx]))
padded_sequences = sequence.pad_sequences(self.sequences[idx], maxlen=pad_length)
 x_batch = [torch.tensor(padded_sequences, dtype=torch.long)]
if self.other_features is not None:
x_batch += [x[idx] for x in self.other_features]
if self.labels is not None:
out = x_batch, self.labels[idx]
else:
out = x_batch
return out<create_dataframe>
F, p_value = f_regression(X_label_encoded, y_train)
 np.array(X_label_encoded.columns)+ " = " +(p_value < 0.05 ).astype(str )
train_dataset = SequenceDataset(x_train, lambda lengths: lengths.max() , other_features=[lengths], shuffle=False, batch_size=batch_size)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=1, shuffle=False )<define_variables>
encoder = OrdinalEncoder()
X_label_encoded = pd.DataFrame(OrdinalEncoder().fit_transform(X_label_imputed), columns=X_label_imputed.columns )
n_repeats = 10
start_time = time.time()
for _ in range(n_repeats):
for batch in tqdm(train_loader):
pass
method2_time =(time.time() - start_time)/ n_repeats<categorify>
X_label_analys = X_label_encoded.copy()
X_label_analys['PriceSale'] = y_train.values
lengths = torch.from_numpy(np.array([len(x)for x in x_train]))
maxlen = lengths.max()
x_train_padded = torch.from_numpy(sequence.pad_sequences(x_train, maxlen=maxlen))
x_train_padded.shape<categorify>
label_new_drop_attrs = ['Utilities', 'LandSlope', 'YrSold', 'MoSold']
X_label_new_analys = DataFrameDropper(label_new_drop_attrs ).transform(X_label_analys )
class SequenceBucketCollator() :
def __init__(self, choose_length, sequence_index, length_index, label_index=None):
self.choose_length = choose_length
self.sequence_index = sequence_index
self.length_index = length_index
self.label_index = label_index
def __call__(self, batch):
batch = [torch.stack(x)for x in list(zip(*batch)) ]
sequences = batch[self.sequence_index]
lengths = batch[self.length_index]
length = self.choose_length(lengths)
 mask = torch.arange(start=maxlen, end=0, step=-1)< length
padded_sequences = sequences[:, mask]
batch[self.sequence_index] = padded_sequences
if self.label_index is not None:
return [x for i, x in enumerate(batch)if i != self.label_index], batch[self.label_index]
return batch<create_dataframe>
class DataFrameSelector(BaseEstimator, TransformerMixin):
def __init__(self, attrs):
self.attrs = attrs
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
return X.loc[:, self.attrs]
class LabelBinarizerPipelineFriendly(OneHotEncoder):
def fit(self, X, y=None):

 super(LabelBinarizerPipelineFriendly,self ).fit(X)
 def transform(self, X, y=None):
return super(LabelBinarizerPipelineFriendly, self ).transform(X ).toarray()
def fit_transform(self, X, y=None):
return super(LabelBinarizerPipelineFriendly, self ).fit(X ).transform(X)
dataset = data.TensorDataset(x_train_padded, lengths, y_train_torch)
 train_loader = data.DataLoader(dataset, batch_size=batch_size, collate_fn=SequenceBucketCollator(lambda x: x.max() , 0, 1, 2))<define_variables>
num_pipeline = Pipeline([ 
('selection', DataFrameSelector(num_attrs)) , 
('merge_bath', MergeBath()), 
('merge_bsmnt', MergeBsmntFs()), 
('merge_porch', MergePorches()), 
('filter', FilterLotAreaAndRooms()), 
('drop', DataFrameDropper(num_drop_attrs)) , 
('impute', SimpleImputer()), 
('std_scale', StandardScaler()),
])
 label_pipeline = Pipeline([ 
('selection', DataFrameSelector(label_attrs)) , 
('drop', DataFrameDropper(label_new_drop_attrs)) , 
('impute', SimpleImputer(strategy="most_frequent")) , 
('encode', OneHotEncoder(sparse=False, handle_unknown='ignore')) , 
('std_scale', StandardScaler()),
])
 full_pipeline = FeatureUnion([ 
('num_pipline', num_pipeline), 
('label_pipeline', label_pipeline),
])
 X_train_cleaned = pd.DataFrame(full_pipeline.fit_transform(X_train_without_outlier))
n_repeats = 10
start_time = time.time()
for _ in range(n_repeats):
for batch in tqdm(train_loader):
pass
method3_time =(time.time() - start_time)/ n_repeats<choose_model_class>
y_train = y_train_without_outlier
class NeuralNet(nn.Module):
def __init__(self, embedding_matrix, num_aux_targets):
super(NeuralNet, self ).__init__()
embed_size = embedding_matrix.shape[1]
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.3)
 self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS, 1)
 self.linear_aux_out = nn.Linear(DENSE_HIDDEN_UNITS, num_aux_targets)
 def forward(self, x, lengths=None):
h_embedding = self.embedding(x.long())
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm1, _ = self.lstm1(h_embedding)
 h_lstm2, _ = self.lstm2(h_lstm1)
 avg_pool = torch.mean(h_lstm2, 1)
 max_pool, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( max_pool, avg_pool), 1)
 h_conc_linear1 = F.relu(self.linear1(h_conc))
h_conc_linear2 = F.relu(self.linear2(h_conc))
hidden = h_conc + h_conc_linear1 + h_conc_linear2
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out<init_hyperparams>
linear_model = LinearRegression()
linear_model.fit(X_train_cleaned, y_train )
batch_size = 512
model = NeuralNet(embedding_matrix, y_aux_train.shape[-1])
 model.cuda()
train_dataset = data.TensorDataset(x_train_padded, lengths, y_train_torch)
 collator = SequenceBucketCollator(lambda lengths: lengths.max() , 0, 1, 2)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size,
shuffle=False, collate_fn=collator)
 start_time = time.time()
for x_batch in tqdm(train_loader):
model(x_batch[0][0].cuda())
 max_length_seq_bucketing_time = time.time() - start_time
train_dataset = data.TensorDataset(x_train_padded, lengths, y_train_torch)
 collator = SequenceBucketCollator(lambda lengths: np.percentile(lengths.numpy() , q=95), 0, 1, 2)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size,
shuffle=False, collate_fn=collator)
 start_time = time.time()
for x_batch in tqdm(train_loader):
model(x_batch[0][0].cuda())
 percentile_seq_bucketing_time = time.time() - start_time
x_train_padded = torch.tensor(sequence.pad_sequences(x_train, maxlen=MAX_LEN))
train_dataset = data.TensorDataset(x_train_padded)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=False)
 start_time = time.time()
for x_batch in tqdm(train_loader):
model(x_batch[0].cuda())
 maxlen_static_time = time.time() - start_time
percentile_length = math.ceil(np.percentile(lengths, q=95))
x_train_padded = torch.tensor(sequence.pad_sequences(x_train, maxlen=percentile_length))
train_dataset = data.TensorDataset(x_train_padded)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=False)
 start_time = time.time()
for x_batch in tqdm(train_loader):
model(x_batch[0].cuda())
 percentile_static_time = time.time() - start_time<categorify>
def analys_model(model):
some_data = X_train.iloc[:5]
some_label = y_train.iloc[:5]
some_data_prepared = full_pipeline.transform(some_data)
 print(f"\x1b[31mPredictions are \033[92m{model.predict(some_data_prepared)}")
 print(f"\x1b[31mLables are \033[92m{list(some_label)}")
 housing_prediction = model.predict(X_train_cleaned)
 scores = cross_validate(model, X_train_cleaned, y_train, scoring="neg_mean_squared_error", cv=3)
 rmse_scores = np.sqrt(-scores['test_score'])
 print(f"\x1b[31mScores : \033[92m{rmse_scores}")
 print(f"\x1b[31mMean : \033[92m{rmse_scores.mean() }")
 print(f"\x1b[31mStandard Deviation : \033[92m{rmse_scores.std() }" )
lengths = torch.from_numpy(np.array([len(x)for x in x_train]))
test_lengths = torch.from_numpy(np.array([len(x)for x in x_test]))
maxlen = lengths.max()
x_train_padded = torch.from_numpy(sequence.pad_sequences(x_train, maxlen=maxlen))
x_test_padded = torch.from_numpy(sequence.pad_sequences(x_test, maxlen=maxlen))<create_dataframe>
sgd_grid = {
'n_iter_no_change': [10, 20, 30, 40, 50, 60, 80, 100, 130, 140],
'eta0': [0.4, 0.2, 0.1, 0.05, 0.03, 0.01, 0.009, 0.004],
}
sgd_model = SGDRegressor()
sgd_best = RandomizedSearchCV(sgd_model, sgd_grid, verbose=2, cv=3,n_jobs=-1,
scoring="neg_mean_squared_error" ).fit(X_train_cleaned, y_train ).best_estimator_
sgd_best
sgd_best.fit(X_train_cleaned, y_train)
 analys_model(sgd_best )
batch_size = 512
test_dataset = data.TensorDataset(x_test_padded, test_lengths)
 train_dataset = data.TensorDataset(x_train_padded, lengths, y_train_torch)
 valid_dataset = data.Subset(train_dataset, indices=[0, 1])
 train_collator = SequenceBucketCollator(lambda lenghts: lenghts.max() ,
sequence_index=0,
length_index=1,
label_index=2)
 test_collator = SequenceBucketCollator(lambda lenghts: lenghts.max() , sequence_index=0, length_index=1)
 train_loader = data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=train_collator)
 valid_loader = data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, collate_fn=train_collator)
 test_loader = data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, collate_fn=test_collator)
 databunch = DataBunch(train_dl=train_loader, valid_dl=valid_loader, collate_fn=train_collator )<compute_test_metric>
ridge_grid = {
'alpha': np.linspace(0, 1, num=500),
'solver' : ['cholesky'],
}
ridge_model = Ridge()
ridge_best = RandomizedSearchCV(ridge_model, ridge_grid, verbose=2, cv=3,n_jobs=-1,
scoring="neg_mean_squared_error" ).fit(X_train_cleaned, y_train ).best_estimator_
ridge_best.fit(X_train_cleaned, y_train)
 analys_model(ridge_best )
def custom_loss(data, targets):

 bce_loss_1 = nn.BCEWithLogitsLoss(weight=targets[:,1:2] )(data[:,:1],targets[:,:1])
 bce_loss_2 = nn.BCEWithLogitsLoss()(data[:,1:],targets[:,2:])
 return(bce_loss_1 * loss_weight)+ bce_loss_2<choose_model_class>
lasso_grid = {
'alpha': np.linspace(0, 1e-3, num=1000),
}
lasso_model = Lasso()
lasso_best = RandomizedSearchCV(lasso_model, lasso_grid, verbose=2, cv=3,n_jobs=-1,
scoring="neg_mean_squared_error" ).fit(X_train_cleaned, y_train ).best_estimator_
lasso_best.fit(X_train_cleaned, y_train)
 analys_model(lasso_best )
all_test_preds = []
for model_idx in range(NUM_MODELS):
print('Model ', model_idx)
 seed_everything(1234 + model_idx)
 model = NeuralNet(embedding_matrix, y_aux_train.shape[-1])
 learn = Learner(databunch, model, loss_func=custom_loss)
 test_preds = train_model(learn,test_dataset,output_dim=7)
 all_test_preds.append(test_preds )<save_to_csv>
lasso_lars_grid = {
'alpha': np.linspace(0, 1e-3, num=10000),
'max_iter' : [int(x)for x in np.linspace(1, 110, num = 100)]
}
lasso_lars_model = LassoLars()
lasso_lars_best = RandomizedSearchCV(lasso_lars_model, lasso_lars_grid, verbose=2, cv=3,n_jobs=-1,
scoring="neg_mean_squared_error" ).fit(X_train_cleaned, y_train ).best_estimator_
lasso_lars_best.fit(X_train_cleaned, y_train)
 analys_model(lasso_lars_best )
submission = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': np.mean(all_test_preds, axis=0)[:, 0]
})
 submission.to_csv('submission.csv', index=False )<import_modules>
class GradientBoostingOtherRegressor(TransformerMixin, BaseEstimator):
def __init__(self, estimator, n_estimates = 3):
self.estimator = estimator
self.estimators = []
self.n_estimates = n_estimates
def fit(self, X, y_train=None):
last_estimator = self.estimator
last_estimator.fit(X, y_train)
 y = y_train.values
self.estimators.append(last_estimator)
 for i in range(self.n_estimates):
y = y - last_estimator.predict(X)
 new_estimator = clone(self.estimator)
 new_estimator.fit(X, y)
 last_estimator = new_estimator
self.estimators.append(last_estimator)
 return self
def predict(self, X_test):
y_pred = sum(tree.predict(X_test)for tree in self.estimators)
 return y_pred
print("TF version: ", tf.__version__)
 print("Hub version: ", hub.__version__ )<set_options>
gbor = GradientBoostingOtherRegressor(ridge_best, n_estimates=4)
 gbor.fit(X_train_cleaned, y_train)
 analys_model(gbor )
pd.set_option('display.max_colwidth', None)
<load_from_csv>
elastic_grid = {
'alpha': np.linspace(0, 1e-2, num=10000),
'l1_ratio' : np.linspace(0, 1, num=10)
 }
elastic_model = ElasticNet()
elastic_best = RandomizedSearchCV(elastic_model, elastic_grid, verbose=2, cv=3,n_jobs=-1,
scoring="neg_mean_squared_error" ).fit(X_train_cleaned, y_train ).best_estimator_
elastic_best.fit(X_train_cleaned, y_train)
 analys_model(elastic_best )
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 ids = test.id
print('Total length of the dataset: ', len(train)+len(test))
print('shape of training set: ', train.shape)
 print('shape of testing set: ', test.shape )<concatenate>
from sklearn.svm import SVR
df_concat = pd.concat([train, test], axis = 0 ).reset_index(drop = True)
 nulls = pd.DataFrame(np.c_[df_concat.isnull().sum() ,(df_concat.isnull().sum() / len(df_concat)) *100],
columns = ['
index = df_concat.columns)
 nulls<categorify>
svm_linear_grid = {
'epsilon' : np.linspace(0, 0.5, num=200),
}
svm_linear_model = SVR(kernel='linear')
 svm_linear_best = RandomizedSearchCV(svm_linear_model, svm_linear_grid, verbose=2, cv=3, n_jobs=-1,
scoring='neg_mean_squared_error' ).fit(X_train_cleaned, y_train ).best_estimator_
svm_linear_best.fit(X_train_cleaned, y_train)
 analys_model(svm_linear_best)
for df in [train, test, df_concat]:
df.keyword.fillna('no_keyword', inplace = True)
 df.location.fillna('no_location', inplace = True )<sort_values>
svm_poly_grid = {
'epsilon' : np.linspace(0, 0.5, num=200),
}
svm_poly_model = SVR(kernel='poly')
 svm_poly_best = RandomizedSearchCV(svm_poly_model, svm_poly_grid, verbose=2, cv=3, n_jobs=-1,
scoring='neg_mean_squared_error' ).fit(X_train_cleaned, y_train ).best_estimator_
svm_poly_best.fit(X_train_cleaned, y_train)
 analys_model(svm_poly_best )
df_concat.groupby(['location'] ).count().text.sort_values(ascending = False )<drop_column>
dt_model = DecisionTreeRegressor()
dt_model.fit(X_train_cleaned, y_train)
 max_features = [int(x)for x in np.linspace(1, 270, num = 30)]
max_depth = [1, 2, 4, 5, 6, 9, 10, 12 , None]
min_samples_leaf = [int(x)for x in np.linspace(1, 10, num = 5)]
random_grid = {'max_features': max_features,
'max_depth': max_depth,
'min_samples_leaf': min_samples_leaf}
random_search = RandomizedSearchCV(estimator = dt_model, param_distributions = random_grid,
n_iter = 100, cv = 3, verbose=2, random_state=42, n_jobs = -1, scoring="neg_mean_squared_error")
 random_search.fit(X_train_cleaned, y_train)
 dt_best = random_search.best_estimator_
dt_best.fit(X_train_cleaned, y_train)
 analys_model(dt_best )
for df in [train, test, df_concat]:
df.drop(columns = ['location', 'keyword', 'id'], inplace = True )<string_transform>
dt_ada_model = AdaBoostRegressor(dt_best, n_estimators=200, learning_rate=0.5)
 dt_ada_model.fit(X_train_cleaned, y_train)
 analys_model(dt_ada_model )
nlp = spacy.load("en")
 sp = spacy.load('en_core_web_sm')
 nltk.download('stopwords')
 nltk.download('punkt')
 spacy_st = nlp.Defaults.stop_words
nltk_st = stopwords.words('english')
 def clean(tweet, http = True, punc = True, lem = True, stop_w = True):
if http is True:
tweet = re.sub("https?:\/\/t.co\/[A-Za-z0-9]*", '', tweet)
 if stop_w == 'nltk':
tweet = [word for word in word_tokenize(tweet)if not word.lower() in nltk_st]
tweet = ' '.join(tweet)
 elif stop_w == 'spacy':
tweet = [word for word in word_tokenize(tweet)if not word.lower() in spacy_st]
tweet = ' '.join(tweet)
 if lem == True:
lemmatized = [word.lemma_ for word in sp(tweet)]
tweet = ' '.join(lemmatized)
 if punc is True:
tweet = tweet.translate(str.maketrans('', '', string.punctuation))
tweet = re.sub("\s+", ' ', tweet)
 return tweet<feature_engineering>
n_estimators = [int(x)for x in np.linspace(start = 100, stop = 2000, num = 5)]
max_features = ['auto', 10, 20, 40, 90, 140, 200, 250]
max_depth = [int(x)for x in np.linspace(1, 1000, num = 20)]
max_depth.append(None)
 min_samples_leaf = [5, 10, 15]
bootstrap = [True, False]
random_grid = {'n_estimators': n_estimators,
'max_features': max_features,
'max_depth': max_depth,
'min_samples_leaf': min_samples_leaf,
'bootstrap': bootstrap}
rf = RandomForestRegressor()
df_concat['cleaned_text'] = df_concat.text.apply(lambda x: clean(x, lem = False, stop_w = 'nltk', http = True, punc = True))<define_variables>
random_search = RandomizedSearchCV(estimator = rf, param_distributions = random_grid,
n_iter = 100, cv = 3, verbose=2, random_state=42, n_jobs = -1, scoring="neg_mean_squared_error")
 random_search.fit(X_train_cleaned, y_train )
cleaned_train = df_concat[:train.shape[0]]
cleaned_test = df_concat[train.shape[0]:]<set_options>
rf_best = random_search.best_estimator_
rf_best.fit(X_train_cleaned, y_train)
 analys_model(rf_best )
%%HTML
<a id = "Word_Embeddings"></a>
<center>
<iframe width="700" height="315" src="https://www.youtube.com/embed/t5wdTK-QtLA" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" style="position: relative;top: 0;left: 0;" allowfullscreen ng-show="showvideo"></iframe>
</center><load_from_url>
dt_ada_model = AdaBoostRegressor(rf_best, n_estimators=10, learning_rate=0.5)
 dt_ada_model.fit(X_train_cleaned, y_train)
 analys_model(dt_ada_model )
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
FullTokenizer = tokenization.FullTokenizer 
<define_variables>
voting_model = VotingRegressor(
 estimators=[('ridge', ridge_best),('lasso', lasso_best),('elastic', elastic_best),('svm', svm_linear_best), 
('rf', gradient_best),('dt', dt_ada_model)],
n_jobs=-1 
)
 voting_model.fit(X_train_cleaned, y_train)
 analys_model(voting_model )
ans = input("Which Bert should I use?
a.Base uncased
b.Large uncased
c.Basic cased
d.Large cased
")
 if ans is 'a':
BERT_MODEL_HUB = 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/2'
disc = 'Base_uncased'
elif ans is 'b':
BERT_MODEL_HUB = 'https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/2'
disc = 'Large_uncased'
elif ans is 'c':
BERT_MODEL_HUB = 'https://tfhub.dev/tensorflow/bert_en_cased_L-12_H-768_A-12/2'
disc = 'Base_cased'
elif ans is 'd':
BERT_MODEL_HUB = 'https://tfhub.dev/tensorflow/bert_en_cased_L-24_H-1024_A-16/2'
disc = 'Large_cased'
bert_layer = hub.KerasLayer(BERT_MODEL_HUB, trainable=True)
 print('Bert layer is ready to use!')
 if ans =='a' or ans =='b':
to_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
vocabulary_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
tokenizer = FullTokenizer(vocabulary_file, to_lower_case)
 elif ans =='c' or ans =='d':
vocabulary_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
tokenizer = FullTokenizer(vocabulary_file, do_lower_case=False)
 print('Bert Tokenizer is ready!!!')
<string_transform>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y=None):
X = X.values
y = y.values
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=42)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def get_metafeatures(self, X):
return np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )
sentence = 'Terrorist will crush the Tower'
print('Tokenized version of {} is :
{} '.format(sentence, tokenizer.tokenize(sentence)) )<string_transform>
stacked_averaged_models = StackingAveragedModels(base_models = [gradient_best, dt_ada_model, elastic_best, lasso_lars_best, svm_linear_best],
meta_model = LinearRegression())
 stacked_averaged_models.fit(X_train_cleaned, y_train )
def tokenize_tweets(text_):
return tokenizer.convert_tokens_to_ids(['[CLS]'] + tokenizer.tokenize(text_)+ ['[SEP]'])
 df_concat['tokenized_tweets'] = df_concat.cleaned_text.apply(lambda x: tokenize_tweets(x))
cleaned_train.head(2 )<feature_engineering>
meta_features = pd.DataFrame(stacked_averaged_models.get_metafeatures(X_train_cleaned))
max_len = len(max(df_concat.tokenized_tweets, key = len))
print('The maximum length of each sequence besed on tokenized tweets is:', max_len)
 df_concat['padded_tweets'] = df_concat.tokenized_tweets.apply(lambda x: x + [0] *(max_len - len(x)))
 df_concat.head(2 )<choose_model_class>
X_test_clean = full_pipeline.transform(X_test)
 predictions = stacked_averaged_models.predict(X_test_clean)
 final_prediction = pd.DataFrame({'Id': X_test['Id'],
'SalePrice': np.expm1(predictions)} )
<choose_model_class><EOS> 
final_prediction.to_csv('prediction.csv', index=False )