Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<groupby>
from PIL import Image
import math
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader, Subset, TensorDataset
from sklearn.model_selection import train_test_split
from tqdm import notebook
import torchvision
from torchvision import transforms
from torchvision import models
from collections import Counter
from pathlib import Path
from sklearn.metrics import confusion_matrix, recall_score
import seaborn as sn
import pyarrow.parquet as pq
from skimage.filters import threshold_otsu
from skimage.transform import AffineTransform, SimilarityTransform, warp, resize
df_all1["LotFrontage"] = df_all1.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))<count_missing_values>
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu' )
df_all1.isna().sum() [df_all1.isna().sum() >0].sort_values(ascending=False )<create_dataframe>
df_test = pd.read_csv('/kaggle/input/bengaliai-cv19/test.csv' )
df_all2 = df_all1.copy()<categorify>
def make_tensordataset_from_dfs(parquet_locs, label_loc=None):
ids = []
X = []
for parquet_loc in parquet_locs:
df = pd.read_parquet(parquet_loc)
 ids.extend(df.image_id.tolist())
 x = df.iloc[:, 1:].to_numpy(dtype=np.uint8)
 del df
X.append(x)
 X = np.vstack(X)
 X = X.reshape(-1, 1, 137, 236)
 X = torch.from_numpy(X)
 ids = dict(( s,i)for(i,s)in enumerate(ids))
if label_loc is None:
return TensorDataset(X)
 else:
graphemes = torch.zeros(X.shape[0], dtype=torch.long)
 vowel_diacs = torch.zeros(X.shape[0], dtype=torch.long)
 consonant_diacs = torch.zeros(X.shape[0], dtype=torch.long)
 lbl_df = pd.read_csv(label_loc)
 for row in lbl_df.itertuples() :
if row.image_id not in ids:
continue
idx = ids[row.image_id]
graphemes[idx] = row.grapheme_root
vowel_diacs[idx] = row.vowel_diacritic
consonant_diacs[idx] = row.consonant_diacritic
return TensorDataset(X, graphemes, vowel_diacs, consonant_diacs )
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(df_all2[c].values))
df_all2[c] = lbl.transform(list(df_all2[c].values))
print('Shape : {}'.format(df_all2.shape))<feature_engineering>
class DatasetWithImageTransforms(Dataset):
def __init__(self, ds, transforms):
super(DatasetWithImageTransforms, self ).__init__()
self.ds = ds
self.tr = transforms
self.nt = len(self.ds[0])
 self.ln = len(self.ds)
 def __getitem__(self, index):
img, *rest = self.ds[index]
img = self.tr(img)
 return(img,)+ tuple(rest)
 def __len__(self):
return self.ln
skewness = skewness[abs(skewness)> 0.5]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.45
for feat in skewed_features:
df_all2[feat] = boxcox1p(df_all2[feat], lam )<drop_column>
def thresh(img):
thresh_val = int(threshold_otsu(img))
img =(img > thresh_val)
 return img
def bounding_box(img):
img = thresh(img ).astype(np.uint8)
 col_min_val = np.min(img, axis=0)
 row_min_val = np.min(img, axis=1)
 col = np.argwhere(1 - col_min_val ).flatten()
row = np.argwhere(1 - row_min_val ).flatten()
return row.min() , row.max() , col.min() , col.max()
def scale_to_bb(img):
height = img.shape[0]
width = img.shape[1]
t, b, l, r = bounding_box(img)
 box_width = r - l
box_height = b - t
t, l = max(0, t - 10), max(0, l - 10)
 b, r = min(height, b + 10), min(width, r + 10)
 img = resize(img[t:b, l:r], output_shape=(256, 256), preserve_range=True, order=3, cval=1.0)
 return img
def random_scale(img):
height = img.shape[0]
width = img.shape[1]
t, b, l, r = bounding_box(img)
 box_width = r - l
box_height = b - t
max_width_scale =(box_width + min(l, width - r)) / box_width
max_height_scale =(box_height + min(t, height - b)) / box_height
max_scale = min(max_width_scale, max_height_scale)
 min_scale = min(1.0, 0.25 * max(( height / box_height),(width / box_width)))
 scale = random.uniform(min_scale, max_scale)
 tfm = SimilarityTransform(
 scale=(scale, scale), 
)
 img = warp(img, tfm.inverse, cval=1.0, order=3)
 return img
def random_translate(img):
height = img.shape[0]
width = img.shape[1]
t, b, l, r = bounding_box(img)
 box_width = r - l
box_height = b - t
translate_height = random.uniform(-t, height - b)
 translate_width = random.uniform(-l, width - r)
 tfm = SimilarityTransform(
 translation=(translate_width, translate_height), 
)
 img = warp(img, tfm.inverse, cval=1.0, order=3)
 return img
def random_rotate_and_shear(img):
max_theta = math.pi / 16
theta = random.uniform(-max_theta, max_theta)
 max_shear_theta = math.pi / 8
shear_theta = random.uniform(-max_shear_theta, max_shear_theta)
 tfm = AffineTransform(rotation=theta, shear=shear_theta)
 img = warp(img, tfm.inverse, cval=1.0, order=3)
 return img
def invert_color(t):
t.mul_(-1)
 t.add_(255)
 return t
def affine_transforms(img):
img = img.reshape(137, 236 ).numpy()
img = thresh(img ).astype(np.float32)
 img = random_translate(random_scale(img))
img = img.reshape(1, 137, 236)
 return torch.from_numpy(img)
 def tfms(img):
img = img.reshape(137, 236 ).numpy()
img = thresh(img ).astype(np.float32)
 img = scale_to_bb(img)
 img = img.reshape(1, 256, 256)
 return torch.from_numpy(img)
 def va_tfms(img):
img = img.reshape(137, 236 ).numpy()
img = thresh(img ).astype(np.float32)
 img = img.reshape(1, 137, 236)
 return torch.from_numpy(img)
 affine_transforms = transforms.Lambda(affine_transforms)
 mult = transforms.Lambda(lambda img: img * 255)
 to_float = transforms.Lambda(lambda img: img.float())
 invert_color = transforms.Lambda(invert_color)
 tfms = transforms.Compose([
tfms,
to_float,
] )
df_all2 = df_all2.drop(['Street', 'Utilities', 'Condition2', 'RoofMatl', 'Heating', 'PoolQC'], axis = 1 )<categorify>
n_graphemes, n_vowel_diacs, n_consonant_diacs = 168, 11, 7
df_all2 = pd.get_dummies(df_all2 ).reset_index(drop=True)
 print(df_all2.shape )<feature_engineering>
def make_linear_block_old(in_size, out_size):
block = nn.Sequential(
 nn.Linear(in_size, out_size),
nn.ReLU() ,
nn.BatchNorm1d(num_features=out_size), 
)
 nn.init.xavier_normal_(block[0].weight.data)
 nn.init.zeros_(block[0].bias.data)
 return block
def make_ff_predictor_old(in_size, intermediate_size, out_size, layer_count):
layers = [make_linear_block_old(in_size, intermediate_size)]
for i in range(layer_count):
layers.append(make_linear_block_old(intermediate_size, intermediate_size))
layers.append(make_linear_block_old(intermediate_size, out_size))
layers = nn.Sequential(*layers)
 return layers
def make_squeeze_predictor(in_size, out_size):
return nn.Sequential(
 nn.Dropout(p=0.5),
nn.Conv2d(in_size, out_size, kernel_size=(1, 1)) ,
nn.ReLU() ,
nn.AdaptiveAvgPool2d(output_size=(1,1)) ,
nn.Flatten() 
)
 class BanglaHandwrittenGraphemeNN(nn.Module):
def __init__(self):
super(BanglaHandwrittenGraphemeNN, self ).__init__()
base = models.resnet18(pretrained=False)
 base.fc = nn.Identity()
self.base = base
feature_size = 512
self.grapheme_predictor = make_ff_predictor_old(feature_size, 512, n_graphemes, 2)
 self.vowel_diac_predictor = make_ff_predictor_old(feature_size, 512, n_vowel_diacs, 1)
 self.consonant_diacs = make_ff_predictor_old(feature_size, 512, n_consonant_diacs, 1)
 def convert_to_grayscale(self):
with torch.no_grad() :
conv1 = nn.Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
 conv1.weight.data = torch.sum(model.base.conv1.weight.data, dim=1, keepdim=True)
 self.base.conv1 = conv1
def freeze(self):
for p in self.base.parameters() :
p.requires_grad = False
def unfreeze(self):
for p in self.base.parameters() :
p.requires_grad = True
def forward(self, x):
with torch.no_grad() :
features = self.base(x)
 g_pred = self.grapheme_predictor(features)
 v_pred = self.vowel_diac_predictor(features)
 c_pred = self.consonant_diacs(features)
 return g_pred, v_pred, c_pred
df_all2['TotalSF']=df_all2['TotalBsmtSF'] + df_all2['1stFlrSF'] + df_all2['2ndFlrSF']
df_all2['Total_sqr_footage'] =(df_all2['BsmtFinSF1'] + df_all2['BsmtFinSF2'] +
df_all2['1stFlrSF'] + df_all2['2ndFlrSF'])
 df_all2['Total_Bathrooms'] =(df_all2['FullBath'] +(0.5 * df_all2['HalfBath'])+
df_all2['BsmtFullBath'] +(0.5 * df_all2['BsmtHalfBath']))
df_all2['Total_porch_sf'] =(df_all2['OpenPorchSF'] + df_all2['3SsnPorch'] +
df_all2['EnclosedPorch'] + df_all2['ScreenPorch'] +
df_all2['WoodDeckSF'])
<feature_engineering>
def make_linear_block(in_size, out_size, dropout, bn):
if dropout:
if bn:
block = nn.Sequential(
 nn.Linear(in_size, out_size),
nn.Dropout(0.5),
nn.ReLU() ,
nn.BatchNorm1d(num_features=out_size), 
)
 else:
block = nn.Sequential(
 nn.Linear(in_size, out_size),
nn.Dropout(0.5),
nn.ReLU() , 
)
 else:
if bn:
block = nn.Sequential(
 nn.Linear(in_size, out_size),
nn.ReLU() ,
nn.BatchNorm1d(num_features=out_size), 
)
 else:
block = nn.Sequential(
 nn.Linear(in_size, out_size),
nn.ReLU() , 
)
 nn.init.xavier_normal_(block[0].weight.data)
 nn.init.zeros_(block[0].bias.data)
 return block
def make_ff_predictor(in_size, intermediate_size, out_size, layer_count, dropout, bn):
layers = [make_linear_block(in_size, intermediate_size, dropout, bn)]
for i in range(layer_count):
layers.append(make_linear_block(intermediate_size, intermediate_size, dropout, bn))
layers.append(nn.Linear(intermediate_size, out_size))
layers = nn.Sequential(*layers)
 return layers
def make_squeeze_predictor(in_size, out_size):
return nn.Sequential(
 nn.Dropout(p=0.5),
nn.Conv2d(in_size, out_size, kernel_size=(1, 1)) ,
nn.ReLU() ,
nn.AdaptiveAvgPool2d(output_size=(1,1)) ,
nn.Flatten() , 
)
 class MultiTaskNNR18(nn.Module):
def __init__(self, n_classes_tasks, depth_tasks, dropout=False, bn=False):
super(MultiTaskNNR18, self ).__init__()
base, feature_size = models.resnet18(pretrained=False), 512
base.fc = nn.Identity()
self.base = base
self.task_predictors = nn.ModuleList([
make_ff_predictor(feature_size, 512, n_classes, depth, dropout, bn)
 for n_classes, depth in zip(n_classes_tasks, depth_tasks)
 ])
 self.n_classes_tasks = n_classes_tasks
self.depth_tasks = depth_tasks
def convert_to_grayscale(self):
with torch.no_grad() :
conv1 = nn.Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
 conv1.weight.data = torch.sum(self.base.conv1.weight.data, dim=1, keepdim=True)
 self.base.conv1 = conv1
def freeze(self):
for p in self.base.parameters() :
p.requires_grad = False
def unfreeze(self):
for p in self.base.parameters() :
p.requires_grad = True
def forward(self, x):
features = self.base(x)
 preds = [predictor(features)for predictor in self.task_predictors]
return preds
df_all2['haspool'] = df_all2['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 df_all2['has2ndfloor'] = df_all2['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 df_all2['hasgarage'] = df_all2['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 df_all2['hasbsmt'] = df_all2['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 df_all2['hasfireplace'] = df_all2['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )<prepare_x_and_y>
class MultiTaskNN(nn.Module):
def __init__(self, base, task_predictors):
super(MultiTaskNN, self ).__init__()
self.base = base
self.task_predictors = nn.ModuleList(task_predictors)
 def freeze(self):
for p in self.base.parameters() :
p.requires_grad = False
def unfreeze(self):
for p in self.base.parameters() :
p.requires_grad = True
def forward(self, x):
features = self.base(x)
 preds = [predictor(features)for predictor in self.task_predictors]
return preds
train = df_all2[:ntrain]
test = df_all2[ntrain:] 
<import_modules>
base = torch.jit.load('/kaggle/input/bhgd-r152-trace/resnet152-corrected-more-transforms-base.trace')
 num_tasks = 3
task_predictors = [torch.jit.load('/kaggle/input/bhgd-r152-trace/resnet152-corrected-more-transforms-task-predictor-{}.trace'.format(i)) for i in range(num_tasks)]
model1 = MultiTaskNN(base, task_predictors )
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor, AdaBoostRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline, Pipeline
from sklearn.preprocessing import RobustScaler, StandardScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import make_scorer, mean_squared_error
from sklearn.linear_model import LassoLarsCV, RidgeCV, ElasticNetCV
from sklearn.model_selection import train_test_split
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import MinMaxScaler
from sklearn.svm import LinearSVR
from sklearn.pipeline import make_pipeline, make_union
from tpot.builtins import StackingEstimator
from tpot.builtins import ZeroCount
from sklearn.decomposition import PCA
from imblearn.pipeline import make_pipeline<compute_test_metric>
base, feature_size = torchvision.models.resnet152(pretrained=False), 2048
base.fc = nn.Identity()
conv1 = nn.Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), bias=False)
 conv1.weight.data = torch.sum(base.conv1.weight.data, dim=1, keepdim=True)
 base.conv1 = conv1
n_classes_tasks = [n_graphemes, n_vowel_diacs, n_consonant_diacs]
task_predictors = [nn.Linear(feature_size, n_classes, bias=False)for n_classes in n_classes_tasks]
model2 = MultiTaskNN(base, task_predictors ).to(device)
 model2.load_state_dict(torch.load('/kaggle/input/bhgd-r152-itstrat-mixup-partial/resnet152-itstrat-mixup.pth', map_location=device))
def score(model, train_x, train_y):
score = np.sqrt(-cross_val_score(model, train_x, train_y, cv=5, scoring="neg_mean_squared_error"))
print(score.mean() , score.std() )<choose_model_class>
base, feature_size = torchvision.models.resnet152(pretrained=False), 2048
base.fc = nn.Identity()
conv1 = nn.Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), bias=False)
 conv1.weight.data = torch.sum(base.conv1.weight.data, dim=1, keepdim=True)
 base.conv1 = conv1
task_predictors = [nn.Linear(feature_size, n_classes, bias=False)for n_classes in n_classes_tasks]
model3 = MultiTaskNN(base, task_predictors ).to(device)
 model3.load_state_dict(torch.load('/kaggle/input/bhgd-r152-itstrat-mixup-partial/resnet152-itstrat-mixup-120-1e-3-0.2-16-8.pth', map_location=device))
tpot = TPOTRegressor(generations=1,verbosity=2,scoring='neg_mean_squared_error' )<train_model>
class MultiTaskEnsemble(nn.Module):
def __init__(self, models):
super(MultiTaskEnsemble, self ).__init__()
self.models = nn.ModuleList(models)
 self.n = len(models)
 def forward(self, x):
preds = [m(x)for m in self.models]
preds = [[pred[i] for pred in preds] for i in range(len(preds[0])) ]
preds = [torch.stack(pred ).mean(dim=0)for pred in preds]
return preds
tpot.fit(train.values, y_train )<compute_test_metric>
model = BanglaHandwrittenGraphemeNN().to(device)
 model.convert_to_grayscale()
model.load_state_dict(torch.load('/kaggle/input/bhgd-r18-best/model.pth', map_location=device))
model_r18_best = model
tpot.score(train.values, y_train )<create_dataframe>
model = MultiTaskNNR18([n_graphemes, n_vowel_diacs, n_consonant_diacs], [2, 1, 1], bn=True ).to(device)
 model.convert_to_grayscale()
model.load_state_dict(torch.load('/kaggle/input/bhgd-r18-best-clones/resnet18-original-lossweight-111.pth', map_location=device))
model_r18_111 = model
models_tested = pd.DataFrame(tpot.evaluated_individuals_ ).transpose()<sort_values>
model = MultiTaskNNR18([n_graphemes, n_vowel_diacs, n_consonant_diacs], [2, 1, 1], bn=True ).to(device)
 model.convert_to_grayscale()
model.load_state_dict(torch.load('/kaggle/input/bhgd-r18-best-clones/resnet18-original-lossweight-211.pth', map_location=device))
model_r18_211 = model
models_tested.sort_values(['internal_cv_score'], ascending=False ).head(10 )<train_on_grid>
model = MultiTaskNNR18([n_graphemes, n_vowel_diacs, n_consonant_diacs], [2, 1, 1], bn=True ).to(device)
 model.convert_to_grayscale()
model.load_state_dict(torch.load('/kaggle/input/bhgd-r18-best-clones/resnet18-original-80ep.pth', map_location=device))
model_r18_80ep = model
exported_pipeline1 = make_pipeline(
 StandardScaler() ,
ElasticNetCV(l1_ratio=0.7000000000000001, tol=0.001, cv=5)
)
 exported_pipeline1.fit(train.values, y_train )<predict_on_test>
ens1 = MultiTaskEnsemble([model_r18_best, model_r18_111, model_r18_211, model_r18_80ep] ).to(device )
score(exported_pipeline1, train.values, y_train)
 exported_pipeline_pred1 = exported_pipeline1.predict(train.values)
 score(exported_pipeline1, train.values, exported_pipeline_pred1 )<train_on_grid>
ens2 = MultiTaskEnsemble([model1, model2, model3] ).to(device )
exported_pipeline2 = make_pipeline(
 MinMaxScaler() ,
ElasticNetCV(l1_ratio=0.7000000000000001, tol=0.01, cv=5)
)
 exported_pipeline2.fit(train.values, y_train )<predict_on_test>
def load_model(name, num_tasks):
base = torch.jit.load(name + '-base.trace')
 task_predictors = [torch.jit.load(name + '-task-predictor-{}.trace'.format(i)) for i in range(num_tasks)]
return MultiTaskNN(base, task_predictors )
score(exported_pipeline2, train.values, y_train)
 exported_pipeline_pred2 = exported_pipeline2.predict(train)
 score(exported_pipeline2, train.values, exported_pipeline_pred2 )<train_on_grid>
model1 = load_model('/kaggle/input/bhgd-final-attempt/seresnext101-mixup-convtail-small', 3 )
exported_pipeline3 = make_pipeline(
 ZeroCount() ,
PCA(iterated_power=4, svd_solver='randomized'),
RidgeCV() 
)
 exported_pipeline3.fit(train.values, y_train )<predict_on_test>
model2 = load_model('/kaggle/input/bhgd-final-attempt/seresnext101-mixup-convtail', 3 )
score(exported_pipeline3, train.values, y_train)
 exported_pipeline_pred3 = exported_pipeline3.predict(train)
 score(exported_pipeline3, train.values, exported_pipeline_pred3 )<train_on_grid>
model3 = load_model('/kaggle/input/bhgd-final-attempt/seresnext101-mixup-40ep-rotate-shear', 3 )
exported_pipeline4 = make_pipeline(
 LassoLarsCV(normalize=True, max_iter=60, cv=5)
)
 exported_pipeline4.fit(train.values, y_train )<predict_on_test>
model4 = load_model('/kaggle/input/bhgd-final-attempt/seresnext101-mixup-convtail-small-40ep', 3 )
score(exported_pipeline4, train.values, y_train)
 exported_pipeline_pred4 = exported_pipeline4.predict(train)
 score(exported_pipeline4, train.values, exported_pipeline_pred4 )<train_on_grid>
ens3 = MultiTaskEnsemble([model1, model2, model3, model4] ).to(device )
exported_pipeline5 = make_pipeline(
 RidgeCV() 
)
 exported_pipeline5.fit(train.values, y_train )<predict_on_test>
models = [ens1, ens2, ens3]
score(exported_pipeline5, train.values, y_train)
 exported_pipeline_pred5 = exported_pipeline5.predict(train)
 score(exported_pipeline5, train.values, exported_pipeline_pred5 )<compute_test_metric>
results = []
np.sqrt(mean_squared_error(y_train,(exported_pipeline_pred1*0.2 + exported_pipeline_pred2*0.2 +
exported_pipeline_pred3*0.2 + exported_pipeline_pred4*0.2 + exported_pipeline_pred5*0.2)) )<compute_test_metric>
with torch.no_grad() :
for i in range(4):
te_ds = make_tensordataset_from_dfs(
 ['/kaggle/input/bengaliai-cv19' + '/test_image_data_{}.parquet'.format(i)] 
)
 te_ds = DatasetWithImageTransforms(te_ds, tfms)
 num_tta = 1
te_dl = DataLoader(te_ds, batch_size=256, num_workers=2, pin_memory=True)
 tta_results = [[] for i in range(num_tta)]
for i in range(num_tta):
for imgs, in te_dl:
imgs = imgs.to(device)
 g_pred, v_pred, c_pred, *rest = model(imgs)
 tta_results[i].append([g_pred, v_pred, c_pred])
 for i in range(len(te_dl)) :
g_preds = [results[i][0] for results in tta_results]
v_preds = [results[i][1] for results in tta_results]
c_preds = [results[i][2] for results in tta_results]
g_pred = torch.stack(g_preds ).mean(dim=0)
 v_pred = torch.stack(v_preds ).mean(dim=0)
 c_pred = torch.stack(c_preds ).mean(dim=0)
 g_pred = g_pred.argmax(1 ).tolist()
v_pred = v_pred.argmax(1 ).tolist()
c_pred = c_pred.argmax(1 ).tolist()
for g, v, c in zip(g_pred, v_pred, c_pred):
results.extend([c, g, v])
 del imgs
del te_ds
del te_dl
del tta_results
best_value = []
min_value=1
for i in range(10000):
random = np.random.dirichlet(np.ones(5),size=1)
 best_value.append(np.sqrt(mean_squared_error(y_train,(exported_pipeline_pred1*random[0][0] + exported_pipeline_pred2*random[0][1] +
exported_pipeline_pred3*random[0][2] + exported_pipeline_pred4*random[0][3] + exported_pipeline_pred5*random[0][4]))))
if(np.min(best_value)< min_value):
min_value = np.min(best_value)
 min_array = random<predict_on_test>
del models
def blend_models(sub):
return(exported_pipeline1.predict(sub)*min_array[0][0] +
exported_pipeline2.predict(sub)*min_array[0][1] +
exported_pipeline3.predict(sub)*min_array[0][2] +
exported_pipeline4.predict(sub)*min_array[0][3] +
exported_pipeline4.predict(sub)*min_array[0][4] )<create_dataframe>
del model
print('Predict submission')
 submission = pd.DataFrame()
submission['Id'] = test_ID
submission['SalePrice'] = np.floor(np.expm1(blend_models(test)) )<save_to_csv>
submission_df = pd.read_csv('/kaggle/input/bengaliai-cv19' + '/sample_submission.csv' )
submission.to_csv('submission.csv',index=False )<load_from_csv>
submission_df['target'] = results
<concatenate><EOS> 
submission_df.to_csv('./submission.csv', index=False )