Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<create_dataframe>
warnings.filterwarnings('ignore' )
test_preds = np.zeros(( len(X_test)))
 test = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test, batch_size=512, shuffle=False)
 tk0 = tqdm(test_loader)
 for i,(x_batch,)in enumerate(tk0):
pred = model(x_batch.to(device), attention_mask=(x_batch > 0 ).to(device), labels=None)
 test_preds[i * 512:(i + 1)* 512] = pred[:, 0].detach().cpu().squeeze().numpy()
test_pred = torch.sigmoid(torch.tensor(test_preds)).numpy().ravel()<create_dataframe>
df_train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 df_test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )
submission_bert = pd.DataFrame.from_dict({
'id': test_df['id'],
'prediction': test_pred
} )<load_from_csv>
df_train = df_train.drop(df_train[(df_train.SalePrice >= 700000)].index)
 df_train = df_train.drop(df_train[(df_train.LotFrontage >= 200)].index)
 df_train = df_train.drop(df_train[(df_train.LotArea >= 100000)].index)
 df_train = df_train.drop(df_train[(df_train.OverallQual == 4)&(df_train.SalePrice > 200000)].index)
 df_train = df_train.drop(df_train[(df_train.OverallQual == 8)&(df_train.SalePrice > 500000)].index)
 df_train = df_train.drop(df_train[(df_train.OverallQual == 10)&(df_train.SalePrice < 300000)].index)
 df_train = df_train.drop(df_train[(df_train.OverallCond == 2)&(df_train.SalePrice > 300000)].index)
 df_train = df_train.drop(df_train[(df_train.OverallCond == 5)&(df_train.SalePrice > 700000)].index)
 df_train = df_train.drop(df_train[(df_train.OverallCond == 6)&(df_train.SalePrice > 700000)].index)
 df_train = df_train.drop(df_train[(df_train.YearBuilt <= 1900)&(df_train.SalePrice > 200000)].index)
 df_train = df_train.drop(df_train[(df_train.MasVnrArea >= 1200)].index)
 df_train = df_train.drop(df_train[(df_train.BsmtFinSF1 >= 3000)].index)
 df_train = df_train.drop(df_train[(df_train.BsmtFinSF2 >= 1200)].index)
 df_train = df_train.drop(df_train[(df_train.TotalBsmtSF >= 4000)].index)
 df_train = df_train.drop(df_train[(df_train['1stFlrSF'] >= 4000)].index)
 df_train = df_train.drop(df_train[(df_train.LowQualFinSF > 500)&(df_train.SalePrice > 400000)].index)
 df_train = df_train.drop(df_train[(df_train.GrLivArea >= 4000)].index)
 df_train = df_train.drop(df_train[(df_train.BedroomAbvGr >= 7)].index)
 df_train = df_train.drop(df_train[(df_train.KitchenAbvGr < 1)].index)
 df_train = df_train.drop(df_train[(df_train.TotRmsAbvGrd > 12)].index)
 df_train = df_train.drop(df_train[(df_train.Fireplaces > 2)].index)
 df_train = df_train.drop(df_train[(df_train.GarageCars > 3)].index)
 df_train = df_train.drop(df_train[(df_train.GarageArea > 1200)&(df_train.SalePrice < 300000)].index)
 df_train = df_train.drop(df_train[(df_train.WoodDeckSF > 700)].index)
 df_train = df_train.drop(df_train[(df_train.OpenPorchSF > 450)].index)
 df_train = df_train.drop(df_train[(df_train.EnclosedPorch > 450)].index)
 df_train = df_train.drop(df_train[(df_train['3SsnPorch'] > 350)].index )
train_df = reduce_mem_usage(pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv'))<define_variables>
full = pd.concat([df_train, df_test], ignore_index = True)
 full.drop(['SalePrice'], axis=1, inplace=True)
 words_set = set()
for data in full['MiscFeature']:
if not pd.isnull(data):
for word in data.split(' '):
words_set.add(word)
 for feature in words_set:
count = 0
for data in full['MiscFeature']:
if not pd.isnull(data):
if feature in data:
count += 1
print("Category {}: {} samples".format(feature, count))
symbols_to_isolate = '.,?!-;*"…:—() %
symbols_to_delete = '
🍕\r🐵😑\xa0\ue014\t\uf818\uf04a\xad😢🐶️\uf0e0😜😎👊\u200b\u200e😁عدويهصقأناخلىبمغر😍💖💵Е👎😀😂\u202a\u202c🔥😄🏻💥ᴍʏʀᴇɴᴅᴏᴀᴋʜᴜʟᴛᴄᴘʙғᴊᴡɢ😋👏שלוםבי😱‼\x81エンジ故障\u2009🚌ᴵ͞🌟😊😳😧🙀😐😕\u200f👍😮😃😘אעכח💩💯⛽🚄🏼ஜ😖ᴠ🚲‐😟😈💪🙏🎯🌹😇💔😡\x7f👌ἐὶήιὲκἀίῃἴξ🙄Ｈ😠\ufeff\u2028😉😤⛺🙂\u3000تحكسة👮💙فزط😏🍾🎉😞\u2008🏾😅😭👻😥😔😓🏽🎆🍻🍽🎶🌺🤔😪\x08‑🐰🐇🐱🙆😨🙃💕𝘊𝘦𝘳𝘢𝘵𝘰𝘤𝘺𝘴𝘪𝘧𝘮𝘣💗💚地獄谷улкнПоАН🐾🐕😆ה🔗🚽歌舞伎🙈😴🏿🤗🇺🇸мυтѕ⤵🏆🎃😩\u200a🌠🐟💫💰💎эпрд\x95🖐🙅⛲🍰🤐👆🙌\u2002💛🙁👀🙊🙉\u2004ˢᵒʳʸᴼᴷᴺʷᵗʰᵉᵘ\x13🚬🤓\ue602😵άοόςέὸתמדףנרךצט😒͝🆕👅👥👄🔄🔤👉👤👶👲🔛🎓\uf0b7\uf04c\x9f\x10成都😣⏺😌🤑🌏😯ех😲Ἰᾶὁ💞🚓🔔📚🏀👐\u202d💤🍇\ue613小土豆🏡❔⁉\u202f👠》कर्मा🇹🇼🌸蔡英文🌞🎲レクサス😛外国人关系Сб💋💀🎄💜🤢َِьыгя不是\x9c\x9d🗑\u2005💃📣👿༼つ༽😰ḷЗз▱ц￼🤣卖温哥华议会下降你失去所有的钱加拿大坏税骗子🐝ツ🎅\x85🍺آإشء🎵🌎͟ἔ油别克🤡🤥😬🤧й\u2003🚀🤴ʲшчИОРФДЯМюж😝🖑ὐύύ特殊作戦群щ💨圆明园קℐ🏈😺🌍⏏ệ🍔🐮🍁🍆🍑🌮🌯🤦\u200d𝓒𝓲𝓿𝓵안영하세요ЖљКћ🍀😫🤤ῦ我出生在了可以说普通话汉语好极🎼🕺🍸🥂🗽🎇🎊🆘🤠👩🖒🚪天一家⚲\u2006⚭⚆⬭⬯⏖新✀╌🇫🇷🇩🇪🇮🇬🇧😷🇨🇦ХШ🌐\x1f杀鸡给猴看ʁ𝗪𝗵𝗲𝗻𝘆𝗼𝘂𝗿𝗮𝗹𝗶𝘇𝗯𝘁𝗰𝘀𝘅𝗽𝘄𝗱📺ϖ\u2000үսᴦᎥһͺ\u2007հ\u2001ɩｙｅ൦ｌƽｈ𝐓𝐡𝐞𝐫𝐮𝐝𝐚𝐃𝐜𝐩𝐭𝐢𝐨𝐧Ƅᴨןᑯ໐ΤᏧ௦Іᴑ܁𝐬𝐰𝐲𝐛𝐦𝐯𝐑𝐙𝐣𝐇𝐂𝐘𝟎ԜТᗞ౦〔Ꭻ𝐳𝐔𝐱𝟔𝟓𝐅🐋ﬃ💘💓ё𝘥𝘯𝘶💐🌋🌄🌅𝙬𝙖𝙨𝙤𝙣𝙡𝙮𝙘𝙠𝙚𝙙𝙜𝙧𝙥𝙩𝙪𝙗𝙞𝙝𝙛👺🐷ℋ𝐀𝐥𝐪🚶𝙢Ἱ🤘ͦ💸ج패티Ｗ𝙇ᵻ👂👃ɜ🎫\uf0a7БУі🚢🚂ગુજરાતીῆ🏃𝓬𝓻𝓴𝓮𝓽𝓼☘﴾̯﴿₽\ue807𝑻𝒆𝒍𝒕𝒉𝒓𝒖𝒂𝒏𝒅𝒔𝒎𝒗𝒊👽😙\u200cЛ‒🎾👹⎌🏒⛸公寓养宠物吗🏄🐀🚑🤷操美𝒑𝒚𝒐𝑴🤙🐒欢迎来到阿拉斯ספ𝙫🐈𝒌𝙊𝙭𝙆𝙋𝙍𝘼𝙅ﷻ🦄巨收赢得白鬼愤怒要买额ẽ🚗🐳𝟏𝐟𝟖𝟑𝟕𝒄𝟗𝐠𝙄𝙃👇锟斤拷𝗢𝟳𝟱𝟬⦁マルハニチロ株式社⛷한국어ㄸㅓ니͜ʖ𝘿𝙔₵𝒩ℯ𝒾𝓁𝒶𝓉𝓇𝓊𝓃𝓈𝓅ℴ𝒻𝒽𝓀𝓌𝒸𝓎𝙏ζ𝙟𝘃𝗺𝟮𝟭𝟯𝟲👋🦊多伦🐽🎻🎹⛓🏹🍷🦆为和中友谊祝贺与其想象对法如直接问用自己猜本传教士没积唯认识基督徒曾经让相信耶稣复活死怪他但当们聊些政治题时候战胜因圣把全堂结婚孩恐惧且栗谓这样还♾🎸🤕🤒⛑🎁批判检讨🏝🦁🙋😶쥐스탱트뤼도석유가격인상이경제황을렵게만들지않록잘관리해야합다캐나에서대마초와화약금의품런성분갈때는반드시허된사용🔫👁凸ὰ💲🗯𝙈Ἄ𝒇𝒈𝒘𝒃𝑬𝑶𝕾𝖙𝖗𝖆𝖎𝖌𝖍𝖕𝖊𝖔𝖑𝖉𝖓𝖐𝖜𝖞𝖚𝖇𝕿𝖘𝖄𝖛𝖒𝖋𝖂𝕴𝖟𝖈𝕸👑🚿💡知彼百\uf005𝙀𝒛𝑲𝑳𝑾𝒋𝟒😦𝙒𝘾𝘽🏐𝘩𝘨ὼṑ𝑱𝑹𝑫𝑵𝑪🇰🇵👾ᓇᒧᔭᐃᐧᐦᑳᐨᓃᓂᑲᐸᑭᑎᓀᐣ🐄🎈🔨🐎🤞🐸💟🎰🌝🛳点击查版🍭𝑥𝑦𝑧ＮＧ👣\uf020っ🏉ф💭🎥Ξ🐴👨🤳🦍\x0b🍩𝑯𝒒😗𝟐🏂👳🍗🕉🐲چی𝑮𝗕𝗴🍒ꜥⲣⲏ🐑⏰鉄リ事件ї💊「」\uf203\uf09a\uf222\ue608\uf202\uf099\uf469\ue607\uf410\ue600燻製シ虚偽屁理屈Г𝑩𝑰𝒀𝑺🌤𝗳𝗜𝗙𝗦𝗧🍊ὺἈἡχῖΛ⤏🇳𝒙ψՁմեռայինրւդձ冬至ὀ𝒁🔹🤚🍎𝑷🐂💅𝘬𝘱𝘸𝘷𝘐𝘭𝘓𝘖𝘹𝘲𝘫کΒώ💢ΜΟΝΑΕ🇱♲𝝈↴💒⊘Ȼ🚴🖕🖤🥘📍👈➕🚫🎨🌑🐻𝐎𝐍𝐊𝑭🤖🎎😼🕷ｇｒｎｔｉｄｕｆｂｋ𝟰🇴🇭🇻🇲𝗞𝗭𝗘𝗤👼📉🍟🍦🌈🔭《🐊🐍\uf10aლڡ🐦\U0001f92f\U0001f92a🐡💳ἱ🙇𝗸𝗟𝗠𝗷🥜さようなら🔼'<string_transform>
def add_columns(dataframe):
add_categorical = ['Othr', 'TenC', 'Gar2', 'Shed']
for column in add_categorical:
data = dataframe['MiscVal'][dataframe['MiscFeature'] == column]
dataframe[column] = pd.Series(data, index = dataframe.index ).fillna(value=0)
 dataframe.drop(columns = ['MiscFeature'], inplace = True)
 dataframe.drop(columns = ['MiscVal'], inplace = True)
 add_columns(full)
 print("Full data shape is {}".format(full.shape))
tokenizer = TreebankWordTokenizer()
isolate_dict = {ord(c):f' {c} ' for c in symbols_to_isolate}
remove_dict = {ord(c):f'' for c in symbols_to_delete}<prepare_x_and_y>
categorical_features = [
'Alley',
'BldgType',
'BsmtCond',
'BsmtExposure',
'BsmtFinType1',
'BsmtFinType2',
'BsmtQual',
'CentralAir',
'Condition1',
'Condition2',
'Electrical',
'ExterCond',
'Exterior1st',
'Exterior2nd',
'ExterQual',
'Fence',
'FireplaceQu',
'Foundation',
'Functional',
'GarageCond',
'GarageFinish',
'GarageQual',
'GarageType',
'Heating',
'HeatingQC',
'HouseStyle',
'KitchenQual',
'LandContour',
'LandSlope',
'LotConfig',
'LotShape',
'MasVnrType',
'MSZoning',
'Neighborhood',
'PavedDrive',
'PoolQC',
'RoofMatl',
'RoofStyle',
'SaleCondition',
'SaleType',
'Street',
'Utilities'
]
numerical_features = [
'1stFlrSF',
'2ndFlrSF',
'3SsnPorch',
'BsmtFinSF1',
'BsmtFinSF2',
'BsmtFullBath',
'BsmtHalfBath',
'BsmtUnfSF',
'EnclosedPorch',
'Fireplaces',
'FullBath',
'GarageArea',
'GarageCars',
'GarageYrBlt',
'GrLivArea',
'HalfBath',
'LotArea',
'LotFrontage',
'LowQualFinSF',
'MasVnrArea',
'MoSold',
'MSSubClass',
'OpenPorchSF',
'OverallCond',
'OverallQual',
'PoolArea',
'ScreenPorch',
'TotalBsmtSF',
'TotRmsAbvGrd',
'WoodDeckSF',
'YearBuilt',
'YearRemodAdd',
'YrSold',
'BedroomAbvGr',
'KitchenAbvGr',
'Othr',
'TenC',
'Gar2',
'Shed'
]
x_train = train_df['comment_text'].progress_apply(lambda x:preprocess(x))
y_aux_train = train_df[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]
x_test = test_df['comment_text'].progress_apply(lambda x:preprocess(x))
identity_columns = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
weights = np.ones(( len(x_train),)) / 4
weights +=(train_df[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train_df['target'].values>=0.5 ).astype(bool ).astype(np.int)+ 
(train_df[identity_columns].fillna(0 ).values<0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train_df['target'].values<0.5 ).astype(bool ).astype(np.int)+ 
(train_df[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
loss_weight = 1.0 / weights.mean()
y_train = np.vstack([(train_df['target'].values>=0.5 ).astype(np.int),weights] ).T
max_features = 410047<string_transform>
def createAgeFeatures(dataframe):
dataframe['GarageAge'] = dataframe['GarageYrBlt'].apply(lambda x: 2019 - x)
 dataframe['RemodelAge'] = dataframe['YearRemodAdd'].apply(lambda x: 2019 - x)
 dataframe['HouseAge'] = dataframe['YearBuilt'].apply(lambda x: 2019 - x)
 def createPoolFeature(dataframe):
dataframe['Pool'] = dataframe['PoolArea'].apply(lambda x: x != 0 ).map({True: 1, False: 0})
 def handleNumericalFeatures(dataframe):
createAgeFeatures(dataframe)
 createPoolFeature(dataframe)
 handleNumericalFeatures(full)
 print("Full data shape is {}".format(full.shape))
tokenizer = text.Tokenizer(num_words = max_features, filters='',lower=False )<feature_engineering>
def mapSimpleMapping(dataframe):
dataframe['ExterCond'] = dataframe['ExterCond'].map({
'Ex' : 2,
'Gd' : 1,
'TA' : 0,
'Fa' : -1,
'Po' : -2
})
 dataframe['ExterQual'] = dataframe['ExterQual'].map({
'Ex' : 2,
'Gd' : 1,
'TA' : 0,
'Fa' : -1,
'Po' : -2
})
 dataframe['HeatingQC'] = dataframe['HeatingQC'].map({
'Ex' : 2,
'Gd' : 1,
'TA' : 0,
'Fa' : -1,
'Po' : -2
})
 dataframe['KitchenQual'] = dataframe['KitchenQual'].map({
'Ex' : 2,
'Gd' : 1,
'TA' : 0,
'Fa' : -1,
'Po' : -2
})
 dataframe['PavedDrive'] = dataframe['PavedDrive'].map({
'Y' : 1,
'P' : 0.5,
'N' : 0,
})
 dataframe['Street'] = dataframe['Street'].map({
'Grvl' : -1,
'Pave' : 1,
})
 def mapSimpleWithNan(dataframe):
dataframe['BsmtCond'] = dataframe['BsmtCond'].map({
'Ex' : 4,
'Gd' : 3,
'TA' : 2,
'Fa' : 1,
'Po' : 0,
'NA' : -1
} ).fillna(-1)
 dataframe['BsmtExposure'] = dataframe['BsmtExposure'].map({
'Gd' : 3,
'Av' : 2,
'Mn' : 1,
'No' : 0,
'NA' : -1
} ).fillna(-1)
 dataframe['BsmtQual'] = dataframe['BsmtQual'].map({
'Ex' : 4,
'Gd' : 3,
'TA' : 2,
'Fa' : 1,
'Po' : 0,
'NA' : -1
} ).fillna(-1)
 dataframe['FireplaceQu'] = dataframe['FireplaceQu'].map({
'Ex' : 4,
'Gd' : 3,
'TA' : 2,
'Fa' : 1,
'Po' : 0,
'NA' : -1
} ).fillna(-1)
 dataframe['GarageCond'] = dataframe['GarageCond'].map({
'Ex' : 4,
'Gd' : 3,
'TA' : 2,
'Fa' : 1,
'Po' : 0,
'NA' : -1
} ).fillna(-1)
 dataframe['GarageFinish'] = dataframe['GarageFinish'].map({
'Fin' : 2,
'RFn' : 1,
'Unf' : 0,
'NA' : -1
} ).fillna(-1)
 dataframe['GarageQual'] = dataframe['GarageQual'].map({
'Ex' : 4,
'Gd' : 3,
'TA' : 2,
'Fa' : 1,
'Po' : 0,
'NA' : -1
} ).fillna(-1)
 dataframe['PoolQC'] = dataframe['PoolQC'].map({
'Ex' : 3,
'Gd' : 2,
'TA' : 1,
'Fa' : 0,
'NA' : -1
} ).fillna(-1)
 def mapBoolean(dataframe):
dataframe['CentralAir'] = dataframe['CentralAir'].map({
'N' : 0,
'Y' : 1,
})
 def handleMappedCategorical(dataframe):
mapSimpleMapping(dataframe)
 mapSimpleWithNan(dataframe)
 mapBoolean(dataframe)
 handleMappedCategorical(full )
tokenizer.fit_on_texts(list(x_train)+ list(x_test))
crawl_matrix, unknown_words_crawl = build_matrix(tokenizer.word_index, CRAWL_EMBEDDING_PATH)
 print('n unknown words(crawl): ', len(unknown_words_crawl))
glove_matrix, unknown_words_glove = build_matrix(tokenizer.word_index, GLOVE_EMBEDDING_PATH)
 print('n unknown words(glove): ', len(unknown_words_glove))
max_features = max_features or len(tokenizer.word_index)+ 1
max_features
embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis=-1)
 embedding_matrix.shape
del crawl_matrix
del glove_matrix
gc.collect()
y_train_torch = torch.tensor(np.hstack([y_train, y_aux_train]), dtype=torch.float32 )<string_transform>
check_plot_features = [
'Alley',
'BldgType',
'BsmtFinType1',
'BsmtFinType2',
'Condition1',
'Condition2',
'Electrical',
'Exterior1st',
'Exterior2nd',
'Fence',
'Foundation',
'Functional',
'GarageType',
'Heating',
'HouseStyle',
'LandContour',
'LandSlope',
'LotConfig',
'LotShape',
'MasVnrType',
'MSZoning',
'RoofMatl',
'RoofStyle',
'SaleCondition',
'SaleType'
]
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test )<categorify>
one_hots = ['BldgType', 'BsmtFinType1', 'BsmtFinType2', 'Condition1', 'Condition2', 'Exterior1st', 'Exterior2nd',
'Foundation', 'GarageType', 'Heating', 'HouseStyle', 'LandContour', 'LandSlope', 'LotConfig',
'MasVnrType', 'MSZoning', 'RoofMatl', 'RoofStyle', 'SaleCondition', 'SaleType']
def doOneHot(dataframe):
try:
for category in one_hots:
df = pd.Categorical(dataframe[category])
 dfDummies = pd.get_dummies(df, prefix = category)
 dataframe = pd.concat([dataframe, dfDummies], axis=1)
 if category not in ['Condition1', 'Condition2']:
dataframe.drop(columns = [category], inplace = True)
 return dataframe
except KeyError:
print("Oops! Category {} not found! Probably this has already been done...".format(category))
return dataframe
full = doOneHot(full)
 print("Full data shape is {}".format(full.shape))
lengths = torch.from_numpy(np.array([len(x)for x in x_train]))
maxlen = 300
x_train_padded = torch.from_numpy(sequence.pad_sequences(x_train, maxlen=maxlen))<categorify>
def handleAlley(dataframe):
dataframe['Alley'] = dataframe['Alley'].map({'Grvl' : 0, 'Pave' : 1})
 def handleConditions(dataframe):
handler = {
'Normal' : 0,
'RRNn' : 1,
'RRNe' : 1,
'PosN' : 2,
'Artery' : 3,
'Feedr' : 3,
'RRAn' : 3,
'PosA' : 3,
'RRAe' : 3
}
dataframe['Condition1'] = dataframe['Condition1'].map(handler)
 dataframe['Condition2'] = dataframe['Condition2'].map(handler)
 def handleElectrical(dataframe):
handler = {'Mix' : 0, 'FuseP' : 1, 'FuseF' : 2, 'FuseA' : 3, 'SBrkr' : 4}
dataframe['Electrical'] = dataframe['Electrical'].map(handler)
 def handleFunctional(dataframe):
handler = {'Typ' : 0, 'Min1' : -1, 'Min2' : -2, 'Mod' : -3, 'Maj1' : -4, 'Maj2' : -5, 'Sev' : -6, 'Sal' : -7}
dataframe['Functional'] = dataframe['Functional'].map(handler)
 def handleLotShape(dataframe):
handler = {'Reg' : 0, 'IR1' : -1, 'IR2' : -2, 'IR3' : -3}
dataframe['LotShape'] = dataframe['LotShape'].map(handler)
 def handlePostOneHots(dataframe):
handleAlley(dataframe)
 handleConditions(dataframe)
 handleElectrical(dataframe)
 handleFunctional(dataframe)
 handleLotShape(dataframe)
 handlePostOneHots(full)
 print("Full data shape is {}".format(full.shape))
test_lengths = torch.from_numpy(np.array([len(x)for x in x_test]))
x_test_padded = torch.from_numpy(sequence.pad_sequences(x_test, maxlen=maxlen))<create_dataframe>
def handleFence(dataframe):
dataframe['FencePrivacy'] = dataframe['Fence'].map({'MnPrv' : 0, 'GdPrv': 1} ).fillna(-1)
 dataframe['FenceWood'] = dataframe['Fence'].map({'MnWw' : 0, 'GdWo': 1} ).fillna(-1)
 dataframe.drop(columns = ['Fence'], inplace = True)
 return dataframe
full = handleFence(full)
 print("Full data shape is {}".format(full.shape))
batch_size = 512
test_dataset = data.TensorDataset(x_test_padded, test_lengths)
 train_dataset = data.TensorDataset(x_train_padded, lengths, y_train_torch)
 valid_dataset = data.Subset(train_dataset, indices=[0, 1])
 train_collator = SequenceBucketCollator(lambda lenghts: lenghts.max() ,
sequence_index=0,
length_index=1,
label_index=2)
 test_collator = SequenceBucketCollator(lambda lenghts: lenghts.max() , sequence_index=0, length_index=1)
 train_loader = data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=train_collator)
 valid_loader = data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, collate_fn=train_collator)
 test_loader = data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, collate_fn=test_collator)
 databunch = DataBunch(train_dl=train_loader, valid_dl=valid_loader, collate_fn=train_collator )<choose_model_class>
def handleUtilities(dataframe):
dataframe['Electricity'] = dataframe['Utilities'].map({'AllPub' : 1, 'NoSewr' : 1, 'NoSeWa' : 1, 'ELO' : 1} ).fillna(0)
 dataframe['Gas'] = dataframe['Utilities'].map({'AllPub' : 1, 'NoSewr' : 1, 'NoSeWa' : 1, 'ELO' : 0} ).fillna(0)
 dataframe['Water'] = dataframe['Utilities'].map({'AllPub' : 1, 'NoSewr' : 1, 'NoSeWa' : 0, 'ELO' : 0} ).fillna(0)
 dataframe['Septic Tank'] = dataframe['Utilities'].map({'AllPub' : 1, 'NoSewr' : 0, 'NoSeWa' : 0, 'ELO' : 0} ).fillna(0)
 dataframe.drop(columns = ['Utilities'], inplace = True)
 return dataframe
full = handleUtilities(full)
 print("Full data shape is {}".format(full.shape))
all_test_preds = []
for model_idx in range(NUM_MODELS):
print('Model ', model_idx)
 seed_everything(1 + model_idx)
 model = NeuralNet(embedding_matrix, y_aux_train.shape[-1])
 learn = Learner(databunch, model, loss_func=custom_loss)
 test_preds = train_model(learn,test_dataset,output_dim=7)
 all_test_preds.append(test_preds )<create_dataframe>
lat_dict = {}
lon_dict = {}
for i in range(len(geo_heatmap['Neighborhood'])) :
neighborhood = geo_heatmap['Neighborhood'][i]
lat = geo_heatmap['Latitude'][i]
lon = geo_heatmap['Longitude'][i]
lat_dict[neighborhood] = lat
lon_dict[neighborhood] = lon
def add_lat_lon_columns(df):
df ['Latitude'] = df['Neighborhood'].map(lat_dict)
 df ['Longitude'] = df['Neighborhood'].map(lon_dict)
 df.drop(columns=['Neighborhood'], inplace=True)
 add_lat_lon_columns(full)
 print("Full data shape is {}".format(full.shape))
submission_lstm = pd.DataFrame.from_dict({
'id': test_df['id'],
'prediction': np.mean(all_test_preds, axis=0)[:, 0]
} )<save_to_csv>
missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )
submission = pd.read_csv(
 ".. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv" 
)
 weights = [0.333, 0.667]
submission["prediction"] = ensemble_predictions(
 [submission_bert.prediction.values, submission_lstm.prediction.values],
weights,
type_="rank", 
)
 submission.to_csv("submission.csv", index=False)
<set_options>
full['KitchenQual'] = full['KitchenQual'].fillna(0)
 missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )
tqdm.pandas()<load_from_csv>
full[full['GarageArea'].isnull() ]
TEXT_COL = 'comment_text'
EMB_PATH = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv', index_col='id')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv', index_col='id')
<categorify>
new = full.iloc[2576].copy()
new['GarageArea'] = 0
new['GarageCars'] = 0
new['GarageCond'] = -1
new['GarageFinish'] = -1
new['GarageQual'] = -1
full.iloc[2576] = new.copy()
missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 def load_embeddings(embed_dir=EMB_PATH):
embedding_index = dict(get_coefs(*o.strip().split(" ")) for o in tqdm(open(embed_dir)))
 return embedding_index
def build_embedding_matrix(word_index, embeddings_index, max_features, lower = True, verbose = True):
embedding_matrix = np.zeros(( max_features, 300))
for word, i in tqdm(word_index.items() ,disable = not verbose):
if lower:
word = word.lower()
if i >= max_features: continue
try:
embedding_vector = embeddings_index[word]
except:
embedding_vector = embeddings_index["unknown"]
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix
def build_matrix(word_index, embeddings_index):
embedding_matrix = np.zeros(( len(word_index)+ 1,300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embeddings_index[word]
except:
embedding_matrix[i] = embeddings_index["unknown"]
return embedding_matrix<prepare_x_and_y>
full['Electrical'] = full['Electrical'].fillna(2)
 missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )
maxlen = 220
max_features = 100000
embed_size = 300
tokenizer = Tokenizer(num_words=max_features, lower=True)
 print('fitting tokenizer')
 tokenizer.fit_on_texts(list(train[TEXT_COL])+ list(test[TEXT_COL]))
word_index = tokenizer.word_index
X_train = tokenizer.texts_to_sequences(list(train[TEXT_COL]))
y_train = train['target'].values
X_test = tokenizer.texts_to_sequences(list(test[TEXT_COL]))
X_train = pad_sequences(X_train, maxlen=maxlen)
 X_test = pad_sequences(X_test, maxlen=maxlen)
 del tokenizer
gc.collect() 
<load_pretrained>
full[full['BsmtFinSF1'].isnull() ]
embeddings_index = load_embeddings() 
<prepare_x_and_y>
full['BsmtFinSF1'] = full['BsmtFinSF1'].fillna(0)
 full['BsmtFinSF2'] = full['BsmtFinSF2'].fillna(0)
 full['TotalBsmtSF'] = full['TotalBsmtSF'].fillna(0)
 full['BsmtUnfSF'] = full['BsmtUnfSF'].fillna(0)
 missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )
embedding_matrix = build_matrix(word_index, embeddings_index )<set_options>
full[full['BsmtFullBath'].isnull() ]
del embeddings_index
gc.collect()<import_modules>
full['BsmtFullBath'] = full['BsmtFullBath'].fillna(0)
 full['BsmtHalfBath'] = full['BsmtHalfBath'].fillna(0)
 missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
full[full['Functional'].isnull() ]
def build_model(verbose = False, compile = True):
sequence_input = L.Input(shape=(maxlen,), dtype='int32')
 embedding_layer = L.Embedding(len(word_index)+ 1,
300,
weights=[embedding_matrix],
input_length=maxlen,
trainable=False)
 x = embedding_layer(sequence_input)
 x = L.SpatialDropout1D(0.2 )(x)
 x = L.Bidirectional(L.CuDNNLSTM(64, return_sequences=True))(x)
 att = Attention(maxlen )(x)
 avg_pool1 = L.GlobalAveragePooling1D()(x)
 max_pool1 = L.GlobalMaxPooling1D()(x)
 x = L.concatenate([att,avg_pool1, max_pool1])
 preds = L.Dense(1, activation='sigmoid' )(x)
 model = Model(sequence_input, preds)
 if verbose:
model.summary()
if compile:
model.compile(loss='binary_crossentropy',optimizer=Adam(0.005),metrics=['acc'])
 return model 
<train_on_grid>
full[full['MasVnrArea'].isnull() ]
splits = list(KFold(n_splits=5 ).split(X_train,y_train))
BATCH_SIZE = 2048
NUM_EPOCHS = 100
oof_preds = np.zeros(( X_train.shape[0]))
test_preds = np.zeros(( X_test.shape[0]))
for fold in [0,1,2,3,4]:
K.clear_session()
tr_ind, val_ind = splits[fold]
ckpt = ModelCheckpoint(f'gru_{fold}.hdf5', save_best_only = True)
 es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=3)
 model = build_model()
model.fit(X_train[tr_ind],
y_train[tr_ind]>0.5,
batch_size=BATCH_SIZE,
epochs=NUM_EPOCHS,
validation_data=(X_train[val_ind], y_train[val_ind]>0.5),
callbacks = [es,ckpt])
 oof_preds[val_ind] += model.predict(X_train[val_ind])[:,0]
test_preds += model.predict(X_test)[:,0]
test_preds /= 5<compute_test_metric>
full['MasVnrArea'] = full['MasVnrArea'].fillna(0)
 missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )
roc_auc_score(y_train>0.5,oof_preds )<load_from_csv>
full[full['GarageYrBlt'].isnull() ]
submission = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv', index_col='id')
 submission['prediction'] = test_preds
submission.reset_index(drop=False, inplace=True)
 submission.head() 
<save_to_csv>
null_df = full[full['GarageYrBlt'].isnull() ]
null_df['GarageAge'] = null_df['YearBuilt'].apply(lambda x: 2019 - x)
 null_df['GarageYrBlt'] = null_df['YearBuilt']
full[full['GarageYrBlt'].isnull() ] = null_df
missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )
submission.to_csv('submission.csv', index=False )<save_to_csv>
full[full['LotFrontage'].isnull() ]
submission.to_csv('submission.csv', index=False )<define_variables>
full['Condition1'] = full['Condition1'].fillna(0)
 full['Condition2'] = full['Condition2'].fillna(0)
 missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )
package_dir = ".. /input/ppbert/pytorch-pretrained-bert/pytorch-pretrained-BERT"
sys.path.append(package_dir )<import_modules>
full['Alley'] = full['Alley'].fillna(-1)
 missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function<set_options>
full['LogLotArea'] = np.log(full['LotArea'])
 full['LogLotFrontage'] = np.log(full['LotFrontage'] )
%reload_ext autoreload
%autoreload 2
%matplotlib inline 
<string_transform>
missing_mask = full.isnull() ['LotFrontage']
filled_mask = full.notnull() ['LotFrontage']
predict = full[missing_mask]
train = full[filled_mask]
X_train = train['LogLotArea']
y_train = train['LogLotFrontage']
X_predict = predict['LogLotArea']
model = XGBRegressor(n_jobs = -1)
 model.fit(np.array(X_train ).reshape(-1,1), np.array(y_train))
y_predict = model.predict(np.array(X_predict ).reshape(-1,1))
y_predict.shape
full['LogLotFrontage'][missing_mask] = y_predict
def convert_lines(example, max_seq_length,tokenizer):
max_seq_length -=2
all_tokens = []
longer = 0
for text in tqdm(example):
tokens_a = tokenizer.tokenize(text)
 if len(tokens_a)>max_seq_length:
tokens_a = tokens_a[:max_seq_length]
longer += 1
one_token = tokenizer.convert_tokens_to_ids(["[CLS]"]+tokens_a+["[SEP]"])+[0] *(max_seq_length - len(tokens_a))
all_tokens.append(one_token)
 return np.array(all_tokens)
 def is_interactive() :
return 'SHLVL' not in os.environ
def seed_everything(seed=123):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
emb_arr = KeyedVectors.load(path)
 return emb_arr
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( max_features + 1, 300))
unknown_words = []
for word, i in word_index.items() :
if i <= max_features:
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
try:
embedding_matrix[i] = embedding_index[word.lower() ]
except KeyError:
try:
embedding_matrix[i] = embedding_index[word.title() ]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words
def sigmoid(x):
return 1 /(1 + np.exp(-x))
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
def train_model(learn,test,output_dim,lr=0.001,
batch_size=512, n_epochs=4,
enable_checkpoint_ensemble=True):
all_test_preds = []
checkpoint_weights = [2 ** epoch for epoch in range(n_epochs)]
test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 n = len(learn.data.train_dl)
 phases = [(TrainingPhase(n ).schedule_hp('lr', lr *(0.6**(i)))) for i in range(n_epochs)]
sched = GeneralScheduler(learn, phases)
 learn.callbacks.append(sched)
 for epoch in range(n_epochs):
learn.fit(1)
 test_preds = np.zeros(( len(test), output_dim))
for i, x_batch in enumerate(test_loader):
X = x_batch[0].cuda()
y_pred = sigmoid(learn.model(X ).detach().cpu().numpy())
 test_preds[i * batch_size:(i+1)* batch_size, :] = y_pred
all_test_preds.append(test_preds)
 if enable_checkpoint_ensemble:
test_preds = np.average(all_test_preds, weights=checkpoint_weights, axis=0)
 else:
test_preds = all_test_preds[-1]
return test_preds
def handle_punctuation(x):
x = x.translate(remove_dict)
 x = x.translate(isolate_dict)
 return x
def handle_contractions(x):
x = tokenizer.tokenize(x)
 return x
def fix_quote(x):
x = [x_[1:] if x_.startswith("'")else x_ for x_ in x]
x = ' '.join(x)
 return x
def preprocess(x):
x = handle_punctuation(x)
 x = handle_contractions(x)
 x = fix_quote(x)
 return x
class SequenceBucketCollator() :
def __init__(self, choose_length, sequence_index, length_index, label_index=None):
self.choose_length = choose_length
self.sequence_index = sequence_index
self.length_index = length_index
self.label_index = label_index
def __call__(self, batch):
batch = [torch.stack(x)for x in list(zip(*batch)) ]
sequences = batch[self.sequence_index]
lengths = batch[self.length_index]
length = self.choose_length(lengths)
 mask = torch.arange(start=maxlen, end=0, step=-1)< length
padded_sequences = sequences[:, mask]
batch[self.sequence_index] = padded_sequences
if self.label_index is not None:
return [x for i, x in enumerate(batch)if i != self.label_index], batch[self.label_index]
return batch
class NeuralNet(nn.Module):
def __init__(self, embedding_matrix, num_aux_targets):
super(NeuralNet, self ).__init__()
embed_size = embedding_matrix.shape[1]
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.3)
 self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS, 1)
 self.linear_aux_out = nn.Linear(DENSE_HIDDEN_UNITS, num_aux_targets)
 def forward(self, x, lengths=None):
h_embedding = self.embedding(x.long())
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm1, _ = self.lstm1(h_embedding)
 h_lstm2, _ = self.lstm2(h_lstm1)
 avg_pool = torch.mean(h_lstm2, 1)
 max_pool, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( max_pool, avg_pool), 1)
 h_conc_linear1 = F.relu(self.linear1(h_conc))
h_conc_linear2 = F.relu(self.linear2(h_conc))
hidden = h_conc + h_conc_linear1 + h_conc_linear2
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out
def custom_loss(data, targets):
bce_loss_1 = nn.BCEWithLogitsLoss(weight=targets[:,1:2] )(data[:,:1],targets[:,:1])
 bce_loss_2 = nn.BCEWithLogitsLoss()(data[:,1:],targets[:,2:])
 return(bce_loss_1 * loss_weight)+ bce_loss_2
def reduce_mem_usage(df):
start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df<set_options>
missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )
warnings.filterwarnings(action='once')
 device = torch.device('cuda')
 MAX_SEQUENCE_LENGTH = 300
SEED = 1234
BATCH_SIZE = 512
BERT_MODEL_PATH = '.. /input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'
np.random.seed(SEED)
 torch.manual_seed(SEED)
 torch.cuda.manual_seed(SEED)
 torch.backends.cudnn.deterministic = True
bert_config = BertConfig('.. /input/arti-bert-inference/bert/bert_config.json')
 tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH, cache_dir=None,do_lower_case=True)
 tqdm.pandas()
CRAWL_EMBEDDING_PATH = '.. /input/gensim-embeddings-dataset/crawl-300d-2M.gensim'
GLOVE_EMBEDDING_PATH = '.. /input/gensim-embeddings-dataset/glove.840B.300d.gensim'
NUM_MODELS = 2
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
MAX_LEN = 220
if not is_interactive() :
def nop(it, *a, **k):
return it
tqdm = nop
fastprogress.fastprogress.NO_BAR = True
master_bar, progress_bar = force_console_behavior()
fastai.basic_train.master_bar, fastai.basic_train.progress_bar = master_bar, progress_bar
seed_everything()<load_from_csv>
full['LotFrontage'][missing_mask] = np.exp(full['LogLotFrontage'][missing_mask])
 missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )
test_df = pd.read_csv(".. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv")
 test_df['comment_text'] = test_df['comment_text'].astype(str)
 X_test = convert_lines(test_df["comment_text"].fillna("DUMMY_VALUE"), MAX_SEQUENCE_LENGTH, tokenizer )<load_pretrained>
full['Functional'] = full['Functional'].fillna(0)
 missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )
model = BertForSequenceClassification(bert_config, num_labels=1)
 model.load_state_dict(torch.load(".. /input/arti-bert-inference/bert/bert_pytorch.bin"))
model.to(device)
 for param in model.parameters() :
param.requires_grad = False
model.eval()<categorify>
full[full['GarageCars'].isnull() ]['GarageCond']
test_preds = np.zeros(( len(X_test)))
 test = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test, batch_size=512, shuffle=False)
 tk0 = tqdm(test_loader)
 for i,(x_batch,)in enumerate(tk0):
pred = model(x_batch.to(device), attention_mask=(x_batch > 0 ).to(device), labels=None)
 test_preds[i * 512:(i + 1)* 512] = pred[:, 0].detach().cpu().squeeze().numpy()
test_pred = torch.sigmoid(torch.tensor(test_preds)).numpy().ravel()<create_dataframe>
full['GarageCars'] = full['GarageCars'].fillna(-1)
 missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )
submission_bert = pd.DataFrame.from_dict({
'id': test_df['id'],
'prediction': test_pred
} )<load_from_csv>
full[full['GarageArea'].isnull() ]['GarageCond']
train_df = reduce_mem_usage(pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv'))<define_variables>
full['GarageArea'] = full['GarageArea'].fillna(-1)
 missing_values = full.isnull().sum()
missing_values[missing_values>0].sort_values(ascending = False )
symbols_to_isolate = '.,?!-;*"…:—() %
symbols_to_delete = '
🍕\r🐵😑\xa0\ue014\t\uf818\uf04a\xad😢🐶️\uf0e0😜😎👊\u200b\u200e😁عدويهصقأناخلىبمغر😍💖💵Е👎😀😂\u202a\u202c🔥😄🏻💥ᴍʏʀᴇɴᴅᴏᴀᴋʜᴜʟᴛᴄᴘʙғᴊᴡɢ😋👏שלוםבי😱‼\x81エンジ故障\u2009🚌ᴵ͞🌟😊😳😧🙀😐😕\u200f👍😮😃😘אעכח💩💯⛽🚄🏼ஜ😖ᴠ🚲‐😟😈💪🙏🎯🌹😇💔😡\x7f👌ἐὶήιὲκἀίῃἴξ🙄Ｈ😠\ufeff\u2028😉😤⛺🙂\u3000تحكسة👮💙فزط😏🍾🎉😞\u2008🏾😅😭👻😥😔😓🏽🎆🍻🍽🎶🌺🤔😪\x08‑🐰🐇🐱🙆😨🙃💕𝘊𝘦𝘳𝘢𝘵𝘰𝘤𝘺𝘴𝘪𝘧𝘮𝘣💗💚地獄谷улкнПоАН🐾🐕😆ה🔗🚽歌舞伎🙈😴🏿🤗🇺🇸мυтѕ⤵🏆🎃😩\u200a🌠🐟💫💰💎эпрд\x95🖐🙅⛲🍰🤐👆🙌\u2002💛🙁👀🙊🙉\u2004ˢᵒʳʸᴼᴷᴺʷᵗʰᵉᵘ\x13🚬🤓\ue602😵άοόςέὸתמדףנרךצט😒͝🆕👅👥👄🔄🔤👉👤👶👲🔛🎓\uf0b7\uf04c\x9f\x10成都😣⏺😌🤑🌏😯ех😲Ἰᾶὁ💞🚓🔔📚🏀👐\u202d💤🍇\ue613小土豆🏡❔⁉\u202f👠》कर्मा🇹🇼🌸蔡英文🌞🎲レクサス😛外国人关系Сб💋💀🎄💜🤢َِьыгя不是\x9c\x9d🗑\u2005💃📣👿༼つ༽😰ḷЗз▱ц￼🤣卖温哥华议会下降你失去所有的钱加拿大坏税骗子🐝ツ🎅\x85🍺آإشء🎵🌎͟ἔ油别克🤡🤥😬🤧й\u2003🚀🤴ʲшчИОРФДЯМюж😝🖑ὐύύ特殊作戦群щ💨圆明园קℐ🏈😺🌍⏏ệ🍔🐮🍁🍆🍑🌮🌯🤦\u200d𝓒𝓲𝓿𝓵안영하세요ЖљКћ🍀😫🤤ῦ我出生在了可以说普通话汉语好极🎼🕺🍸🥂🗽🎇🎊🆘🤠👩🖒🚪天一家⚲\u2006⚭⚆⬭⬯⏖新✀╌🇫🇷🇩🇪🇮🇬🇧😷🇨🇦ХШ🌐\x1f杀鸡给猴看ʁ𝗪𝗵𝗲𝗻𝘆𝗼𝘂𝗿𝗮𝗹𝗶𝘇𝗯𝘁𝗰𝘀𝘅𝗽𝘄𝗱📺ϖ\u2000үսᴦᎥһͺ\u2007հ\u2001ɩｙｅ൦ｌƽｈ𝐓𝐡𝐞𝐫𝐮𝐝𝐚𝐃𝐜𝐩𝐭𝐢𝐨𝐧Ƅᴨןᑯ໐ΤᏧ௦Іᴑ܁𝐬𝐰𝐲𝐛𝐦𝐯𝐑𝐙𝐣𝐇𝐂𝐘𝟎ԜТᗞ౦〔Ꭻ𝐳𝐔𝐱𝟔𝟓𝐅🐋ﬃ💘💓ё𝘥𝘯𝘶💐🌋🌄🌅𝙬𝙖𝙨𝙤𝙣𝙡𝙮𝙘𝙠𝙚𝙙𝙜𝙧𝙥𝙩𝙪𝙗𝙞𝙝𝙛👺🐷ℋ𝐀𝐥𝐪🚶𝙢Ἱ🤘ͦ💸ج패티Ｗ𝙇ᵻ👂👃ɜ🎫\uf0a7БУі🚢🚂ગુજરાતીῆ🏃𝓬𝓻𝓴𝓮𝓽𝓼☘﴾̯﴿₽\ue807𝑻𝒆𝒍𝒕𝒉𝒓𝒖𝒂𝒏𝒅𝒔𝒎𝒗𝒊👽😙\u200cЛ‒🎾👹⎌🏒⛸公寓养宠物吗🏄🐀🚑🤷操美𝒑𝒚𝒐𝑴🤙🐒欢迎来到阿拉斯ספ𝙫🐈𝒌𝙊𝙭𝙆𝙋𝙍𝘼𝙅ﷻ🦄巨收赢得白鬼愤怒要买额ẽ🚗🐳𝟏𝐟𝟖𝟑𝟕𝒄𝟗𝐠𝙄𝙃👇锟斤拷𝗢𝟳𝟱𝟬⦁マルハニチロ株式社⛷한국어ㄸㅓ니͜ʖ𝘿𝙔₵𝒩ℯ𝒾𝓁𝒶𝓉𝓇𝓊𝓃𝓈𝓅ℴ𝒻𝒽𝓀𝓌𝒸𝓎𝙏ζ𝙟𝘃𝗺𝟮𝟭𝟯𝟲👋🦊多伦🐽🎻🎹⛓🏹🍷🦆为和中友谊祝贺与其想象对法如直接问用自己猜本传教士没积唯认识基督徒曾经让相信耶稣复活死怪他但当们聊些政治题时候战胜因圣把全堂结婚孩恐惧且栗谓这样还♾🎸🤕🤒⛑🎁批判检讨🏝🦁🙋😶쥐스탱트뤼도석유가격인상이경제황을렵게만들지않록잘관리해야합다캐나에서대마초와화약금의품런성분갈때는반드시허된사용🔫👁凸ὰ💲🗯𝙈Ἄ𝒇𝒈𝒘𝒃𝑬𝑶𝕾𝖙𝖗𝖆𝖎𝖌𝖍𝖕𝖊𝖔𝖑𝖉𝖓𝖐𝖜𝖞𝖚𝖇𝕿𝖘𝖄𝖛𝖒𝖋𝖂𝕴𝖟𝖈𝕸👑🚿💡知彼百\uf005𝙀𝒛𝑲𝑳𝑾𝒋𝟒😦𝙒𝘾𝘽🏐𝘩𝘨ὼṑ𝑱𝑹𝑫𝑵𝑪🇰🇵👾ᓇᒧᔭᐃᐧᐦᑳᐨᓃᓂᑲᐸᑭᑎᓀᐣ🐄🎈🔨🐎🤞🐸💟🎰🌝🛳点击查版🍭𝑥𝑦𝑧ＮＧ👣\uf020っ🏉ф💭🎥Ξ🐴👨🤳🦍\x0b🍩𝑯𝒒😗𝟐🏂👳🍗🕉🐲چی𝑮𝗕𝗴🍒ꜥⲣⲏ🐑⏰鉄リ事件ї💊「」\uf203\uf09a\uf222\ue608\uf202\uf099\uf469\ue607\uf410\ue600燻製シ虚偽屁理屈Г𝑩𝑰𝒀𝑺🌤𝗳𝗜𝗙𝗦𝗧🍊ὺἈἡχῖΛ⤏🇳𝒙ψՁմեռայինրւդձ冬至ὀ𝒁🔹🤚🍎𝑷🐂💅𝘬𝘱𝘸𝘷𝘐𝘭𝘓𝘖𝘹𝘲𝘫کΒώ💢ΜΟΝΑΕ🇱♲𝝈↴💒⊘Ȼ🚴🖕🖤🥘📍👈➕🚫🎨🌑🐻𝐎𝐍𝐊𝑭🤖🎎😼🕷ｇｒｎｔｉｄｕｆｂｋ𝟰🇴🇭🇻🇲𝗞𝗭𝗘𝗤👼📉🍟🍦🌈🔭《🐊🐍\uf10aლڡ🐦\U0001f92f\U0001f92a🐡💳ἱ🙇𝗸𝗟𝗠𝗷🥜さようなら🔼'<feature_engineering>
skewed_feats = full.apply(lambda x: st.skew(x.dropna())).sort_values(ascending=False)
 skewness = pd.DataFrame({'Skew' :skewed_feats})
 skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
walk_through_lambdas = True
def getBestLambda(data):
statistic = []
pvalue = []
bclambda_range = np.linspace(-3, 3, 1000)
 for bclambda in bclambda_range:
transform = boxcox1p(data, bclambda)
 stat, pval = normaltest(transform)
 statistic.append(stat)
 pvalue.append(pval)
 lowest_stat = 8000
closest = -10
best_stat = -1
best_pval = -1
for i in range(len(bclambda_range)) :
if statistic[i] < lowest_stat:
lowest_stat = statistic[i]
best_stat = i
if(abs(pvalue[i])- 1)> closest:
closest =(abs(pvalue[i])- 1)
 best_pval = i
return bclambda_range[best_pval], bclambda_range[best_stat]
i = 0
with progressbar.ProgressBar(max_value=len(skewed_features)) as bar:
for feat in skewed_features:
if walk_through_lambdas:
full["BC_{}".format(feat)] = boxcox1p(full[feat], getBestLambda(full[feat])[0])
 else:
full["BC_{}".format(feat)] = boxcox1p(full[feat],.15)
 i+=1
bar.update(i)
 missing_values = full.isnull().sum()
for column in missing_values[missing_values>0].keys() :
print("Dropping feature {}".format(column))
full.drop(column, axis = 1, inplace = True )
tokenizer = TreebankWordTokenizer()
isolate_dict = {ord(c):f' {c} ' for c in symbols_to_isolate}
remove_dict = {ord(c):f'' for c in symbols_to_delete}<prepare_x_and_y>
X_train = full[full['Id'].isin(df_train['Id'])]
X_train.drop('Id', axis = 1, inplace = True)
 X_test = full[full['Id'].isin(df_test['Id'])]
X_test.drop('Id', axis = 1, inplace = True)
 y_train = df_train['SalePrice']
print("X_train shape is {}, y_train shape is {} and X_test shape is {}".format(
 X_train.shape,
y_train.shape,
X_test.shape 
))
x_train = train_df['comment_text'].progress_apply(lambda x:preprocess(x))
y_aux_train = train_df[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]
x_test = test_df['comment_text'].progress_apply(lambda x:preprocess(x))
identity_columns = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
weights = np.ones(( len(x_train),)) / 4
weights +=(train_df[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train_df['target'].values>=0.5 ).astype(bool ).astype(np.int)+ 
(train_df[identity_columns].fillna(0 ).values<0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train_df['target'].values<0.5 ).astype(bool ).astype(np.int)+ 
(train_df[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
loss_weight = 1.0 / weights.mean()
y_train = np.vstack([(train_df['target'].values>=0.5 ).astype(np.int),weights] ).T
max_features = 410047<string_transform>
scaler = StandardScaler()
X_train_scaled = scaler.fit(X_train ).transform(X_train)
 X_test_scaled = scaler.transform(X_test)
 bclambda = getBestLambda(y_train)[0]
y_train_transformed = boxcox1p(y_train, bclambda )
tokenizer = text.Tokenizer(num_words = max_features, filters='',lower=False )<train_model>
n_folds = 5
def rmse_cv(model, X, y, cv=5):
rmse = np.sqrt(-cross_val_score(model, X, y, scoring = "neg_mean_squared_error", cv = cv))
return rmse
def rmse(y_actual, y_predicted):
return sqrt(mean_squared_error(y_actual, y_predicted))
tokenizer.fit_on_texts(list(x_train)+ list(x_test))
crawl_matrix, unknown_words_crawl = build_matrix(tokenizer.word_index, CRAWL_EMBEDDING_PATH)
 print('n unknown words(crawl): ', len(unknown_words_crawl))
glove_matrix, unknown_words_glove = build_matrix(tokenizer.word_index, GLOVE_EMBEDDING_PATH)
 print('n unknown words(glove): ', len(unknown_words_glove))
max_features = max_features or len(tokenizer.word_index)+ 1
max_features
embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis=-1)
 embedding_matrix.shape
del crawl_matrix
del glove_matrix
gc.collect()
y_train_torch = torch.tensor(np.hstack([y_train, y_aux_train]), dtype=torch.float32 )<string_transform>
X_train_xgb = X_train [xgb_features_list]
X_train_xgb_scaled = scaler.fit(X_train_xgb ).transform(X_train_xgb)
 X_test_xgb = X_test [xgb_features_list]
X_test_xgb_scaled = scaler.transform(X_test_xgb )
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test )<categorify>
rfecv_features_list = ['1stFlrSF', '2ndFlrSF', 'BedroomAbvGr', 'BsmtCond', 'BsmtExposure',
'BsmtFinSF1', 'BsmtFinSF2', 'BsmtFullBath', 'BsmtQual', 'BsmtUnfSF',
'CentralAir', 'Condition1', 'Electrical', 'EnclosedPorch', 'ExterCond',
'ExterQual', 'FireplaceQu', 'FullBath', 'Functional', 'GarageArea',
'GarageCars', 'GarageFinish', 'GarageQual', 'GarageYrBlt', 'GrLivArea',
'HalfBath', 'HeatingQC', 'KitchenAbvGr', 'KitchenQual', 'LotArea',
'LotFrontage', 'LotShape', 'LowQualFinSF', 'MSSubClass', 'MasVnrArea',
'MoSold', 'OpenPorchSF', 'OverallCond', 'OverallQual', 'PavedDrive',
'ScreenPorch', 'TotRmsAbvGrd', 'TotalBsmtSF', 'WoodDeckSF', 'YearBuilt',
'YearRemodAdd', 'YrSold', 'BsmtFinType1_ALQ', 'BsmtFinType1_GLQ',
'BsmtFinType2_BLQ', 'Condition1_Artery', 'Exterior1st_BrkFace',
'Exterior1st_HdBoard', 'Exterior1st_MetalSd', 'Exterior1st_Plywood',
'Exterior1st_VinylSd', 'Exterior1st_Wd Sdng', 'Foundation_CBlock',
'Foundation_PConc', 'GarageType_Attchd', 'GarageType_Detchd',
'HouseStyle_1Story', 'LandContour_Lvl', 'LotConfig_CulDSac',
'LotConfig_FR2', 'MasVnrType_BrkFace', 'MasVnrType_Stone', 'MSZoning_RL',
'MSZoning_RM', 'RoofStyle_Gable', 'SaleCondition_Abnorml',
'SaleCondition_Normal', 'SaleType_New', 'SaleType_WD', 'FenceWood',
'Latitude', 'Longitude', 'BC_LotArea']
lengths = torch.from_numpy(np.array([len(x)for x in x_train]))
maxlen = 300
x_train_padded = torch.from_numpy(sequence.pad_sequences(x_train, maxlen=maxlen))<categorify>
X_train_rfecv = X_train [rfecv_features_list]
X_train_rfecv_scaled = scaler.fit(X_train_rfecv ).transform(X_train_rfecv)
 X_test_rfecv = X_test [rfecv_features_list]
X_test_rfecv_scaled = scaler.transform(X_test_rfecv )
test_lengths = torch.from_numpy(np.array([len(x)for x in x_test]))
x_test_padded = torch.from_numpy(sequence.pad_sequences(x_test, maxlen=maxlen))<create_dataframe>
vanilla_input_size = 0
def VanillaRegressor(input_size = None):
model = Sequential()
model.add(Dense(5, input_dim=input_size, kernel_initializer='normal', activation='relu'))
model.add(Dense(1, kernel_initializer='normal'))
model.compile(loss='mean_squared_error', optimizer='adam')
 return model
def BetaRegressor(input_size = None):
model = Sequential()
model.add(Dense(128, kernel_initializer='normal',input_dim = input_size, activation='relu'))
model.add(Dense(256, kernel_initializer='normal',activation='relu'))
model.add(Dense(256, kernel_initializer='normal',activation='relu'))
model.add(Dense(256, kernel_initializer='normal',activation='relu'))
model.add(Dense(1, kernel_initializer='normal',activation='linear'))
model.compile(loss='mean_squared_error', optimizer='adam')
 return model
train_sets = [
X_train,
X_train_xgb,
X_train_rfecv,
X_train_corr
]
other_models = [
ElasticNet() ,
Lasso() ,
BayesianRidge() ,
LassoLarsIC() ,
RandomForestRegressor() ,
GradientBoostingRegressor() ,
KernelRidge() ,
xgb.XGBRegressor() ,
xgb.XGBRFRegressor() ,
lgb.LGBMRegressor() ,
SVR()
]
other_models_names = [
'ElasticNet',
'Lasso',
'BayesianRidge',
'LassoLarsIC',
'RandomForestRegressor',
'GradientBoostingRegressor',
'KernelRidge',
'XGBRegressor',
'XGBRFRegressor',
'LGBMRegressor',
'SVR'
]
batch_size = 512
test_dataset = data.TensorDataset(x_test_padded, test_lengths)
 train_dataset = data.TensorDataset(x_train_padded, lengths, y_train_torch)
 valid_dataset = data.Subset(train_dataset, indices=[0, 1])
 train_collator = SequenceBucketCollator(lambda lenghts: lenghts.max() ,
sequence_index=0,
length_index=1,
label_index=2)
 test_collator = SequenceBucketCollator(lambda lenghts: lenghts.max() , sequence_index=0, length_index=1)
 train_loader = data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=train_collator)
 valid_loader = data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, collate_fn=train_collator)
 test_loader = data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, collate_fn=test_collator)
 databunch = DataBunch(train_dl=train_loader, valid_dl=valid_loader, collate_fn=train_collator )<choose_model_class>
chosen_X_train = X_train_rfecv.copy()
chosen_X_test = X_test_rfecv.copy()
all_test_preds = []
for model_idx in range(NUM_MODELS):
print('Model ', model_idx)
 seed_everything(1 + model_idx)
 model = NeuralNet(embedding_matrix, y_aux_train.shape[-1])
 learn = Learner(databunch, model, loss_func=custom_loss)
 test_preds = train_model(learn,test_dataset,output_dim=7)
 all_test_preds.append(test_preds )<create_dataframe>
class grid() :
def __init__(self, model):
self.model = model
def grid_get(self, X, y, param_grid):
grid_search = GridSearchCV(self.model, param_grid, scoring='neg_mean_squared_error', verbose=99)
 grid_search.fit(X, y)
 print(grid_search.best_params_, np.sqrt(-grid_search.best_score_))
grid_search.cv_results_['mean_test_score'] = np.sqrt(-grid_search.cv_results_['mean_test_score'])
 print(pd.DataFrame(grid_search.cv_results_)[['params', 'mean_test_score', 'std_test_score']])
 make_search = False
max_iter = 15000
dataset = chosen_X_train
submission_lstm = pd.DataFrame.from_dict({
'id': test_df['id'],
'prediction': np.mean(all_test_preds, axis=0)[:, 0]
} )<save_to_csv>
if(make_search):
grid(lgb.LGBMRegressor() ).grid_get(dataset, y_train_transformed, {
'n_jobs' : [-1],
'boosting_type' : ['gbdt'],
'num_leaves' : np.linspace(1, 50, 11 ).astype(int),
'learning_rate' : np.linspace(0.01, 0.1, 11),
'n_estimators' : np.linspace(100, 700, 6 ).astype(int)
 })
submission = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv')
 submission['prediction'] =(( submission_bert.prediction + submission_lstm.prediction)/2 + submission_lstm.prediction)/2
submission.to_csv('submission.csv', index=False )<set_options>
train_sets = [
X_train_rfecv,
]
br = BayesianRidge()
llic = LassoLarsIC()
gb = GradientBoostingRegressor(learning_rate=0.064, loss='huber', n_estimators=620)
 kr = KernelRidge()
lgbm = lgb.LGBMRegressor(boosting_type='gbdt', learning_rate=0.028000000000000004, n_estimators=700, n_jobs=-1, num_leaves=5)
 xgbr = xgb.XGBRegressor()
other_models = [
]
for j in range(len(other_models)) :
for i in range(len(train_sets)) :
score = rmse_cv(other_models[j], train_sets[i], y_train_transformed, cv=2)
 print("{} scored {:.4f}(+-{:.4f})with the set
warnings.filterwarnings("ignore" )<load_from_csv>
def make_combinations(iterable):
my_combs = []
for item in iterable.copy() :
iterable.remove(item)
 for i in range(len(iterable)) :
for comb in combinations(iterable, i+1):
my_combs.append(( item, comb))
iterable.append(item)
 return my_combs
models = [
BayesianRidge() ,
make_pipeline(MaxAbsScaler() , PolynomialFeatures(2), LassoLarsIC()),
GradientBoostingRegressor(learning_rate=0.064, loss='huber', n_estimators=620),
KernelRidge() ,
xgb.XGBRegressor() ,
make_pipeline(MinMaxScaler() , PolynomialFeatures(2), lgb.LGBMRegressor(boosting_type='gbdt', learning_rate=0.028000000000000004, n_estimators=700, n_jobs=-1, num_leaves=5))
]
my_combs = make_combinations(models)
 print("I have {} combinations to test!".format(len(my_combs)) )
train_path = '.. /input/restaurant-revenue-prediction/train.csv.zip'
df_train = pd.read_csv(train_path)
 df_train.head()<load_from_csv>
class CustomEnsemble(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models, meta_model, scaler=MaxAbsScaler() , feature_generator=None):
self.models = models
if scaler:
if feature_generator:
self.meta_model = make_pipeline(scaler, feature_generator, meta_model)
 else:
self.meta_model = make_pipeline(scaler, meta_model)
 else:
if feature_generator:
self.meta_model = make_pipeline(feature_generator, meta_model)
 else:
self.meta_model = meta_model
def fit(self,X,y):
predictions = np.zeros(( X.shape[0], len(self.models)))
 for i, model in enumerate(self.models):
model.fit(X, y)
 predictions[:,i] = model.predict(X)
 self.meta_model.fit(predictions, y)
 def predict(self,X):
predictions = np.zeros(( X.shape[0], len(self.models)))
 for i, model in enumerate(self.models):
predictions[:,i] = model.predict(X)
 return self.meta_model.predict(predictions)
 def __str__(self):
return "<CustomEnsemble(meta={}, models={})>".format(self.meta_model, self.models)
 def __repr__(self):
return self.__str__()
models = [
BayesianRidge() ,
KernelRidge() ,
make_pipeline(MinMaxScaler() , PolynomialFeatures(2), lgb.LGBMRegressor(boosting_type='gbdt', learning_rate=0.028000000000000004, n_estimators=700, n_jobs=-1, num_leaves=5))
]
my_combs = make_combinations(models)
 print("I chose {} combinations to test!".format(2*len(my_combs)))
 best_model = CustomEnsemble(
 models = [
make_pipeline(MinMaxScaler() , PolynomialFeatures(2), lgb.LGBMRegressor(boosting_type='gbdt', learning_rate=0.028000000000000004, n_estimators=700, n_jobs=-1, num_leaves=5)) ,
BayesianRidge()
],
meta_model = KernelRidge() ,
scaler = MaxAbsScaler() 
)
 best = 0.09228172938603223
test_path = '.. /input/restaurant-revenue-prediction/test.csv.zip'
df_test = pd.read_csv(test_path, index_col='Id')
 df_test.head()<feature_engineering>
print("And the best model goes to...")
 print(best_model)
 print("Its score was {}".format(best))
<filter><EOS> 
best_model.fit(chosen_X_train, y_train_transformed)
 sub = pd.DataFrame()
sub['Id'] = df_test['Id']
sub['SalePrice'] = inv_boxcox1p(best_model.predict(chosen_X_test), bclambda)
 sub.to_csv('submission.csv',index=False )