Kaggle competition: House Prices - Advanced Regression Techniques
package_path = '.. /input/myscripts/'
sys.path.append(package_path )<import_modules>
train = pd.read_csv('.. /input/train.csv' , index_col= 'Id')
 test = pd.read_csv('.. /input/test.csv' , index_col= 'Id')
 train = train[train.GrLivArea < 4500]
train.reset_index(drop=True, inplace=True)
 label = train[['SalePrice']]
train.drop('SalePrice' , axis = 1 , inplace=True)
 train.head(3 )
from bengali_optimizers import *<choose_model_class>
numerical_col = []
cat_col = []
for x in train.columns:
if train[x].dtype == 'object':
cat_col.append(x)
 print(x+': ' + str(len(train[x].unique())))
 else:
numerical_col.append(x)
 print('CAT col
', cat_col)
 print('Numerical col
')
 print(numerical_col )
class CNN_Model(nn.Module):
def __init__(self,model_name='efficientnet-b5',global_dim=2048):
super(CNN_Model,self ).__init__()
self.cnn_backbone = EfficientNet.from_name(model_name)
 self.cnn_backbone.set_swish(memory_efficient=False)
 self.cnn_head_grapheme_root = CNN_Head(168,global_dim,1024,512)
 self.cnn_head_vowel_diacritic = CNN_Head(11,global_dim,1024,512)
 self.cnn_head_consonant_diacritic = CNN_Head(7,global_dim,1024,512)
 def forward(self, x):
global_feat = self.cnn_backbone.extract_features(x)
 global_feat =(F.adaptive_avg_pool2d(global_feat,1)+F.adaptive_max_pool2d(global_feat,1)) *0.5
global_feat = global_feat.view(global_feat.size(0), -1)
 grapheme_output = self.cnn_head_grapheme_root(global_feat)
 vowel_diacritic_output = self.cnn_head_vowel_diacritic(global_feat)
 consonant_diacritic_output = self.cnn_head_consonant_diacritic(global_feat)
 return grapheme_output,vowel_diacritic_output,consonant_diacritic_output
class CNN_Head(nn.Sequential):
def __init__(self, classes, input_dim, linear1_output_dim, linear2_output_dim,
use_batch_norm=True, use_activation_function=False,
global_dropout=0.0, lin1_dropout=0.0, lin2_dropout=0.00):
super(CNN_Head, self ).__init__()
if global_dropout > 0.0:
self.add_module('drop_1',nn.Dropout(p=global_dropout,inplace=False))
self.add_module('linear_1',nn.Linear(input_dim,linear1_output_dim))
if use_batch_norm:
self.add_module('norm_1', nn.BatchNorm1d(linear1_output_dim))
if use_activation_function:
self.add_module('activ_1',nn.ReLU(inplace=False))
if lin1_dropout > 0.0:
self.add_module('drop_2',nn.Dropout(p=lin1_dropout,inplace=False))
self.add_module('linear_2',nn.Linear(linear1_output_dim,linear2_output_dim))
if use_batch_norm:
self.add_module('norm_2', nn.BatchNorm1d(linear2_output_dim))
if use_activation_function:
self.add_module('activ_2',nn.ReLU(inplace=False))
if lin2_dropout > 0.0:
self.add_module('drop_3',nn.Dropout(p=lin2_dropout,inplace=False))
self.add_module('linear_final',nn.Linear(linear2_output_dim,classes))<train_on_grid>
numerical_col.remove('MSSubClass')
 cat_col.append('MSSubClass' )
model_1 = CNN_Model()
checkpoint_dict = torch.load('/kaggle/input/modelsubmission46/checkpoint_best_Average_recall.pth')
 model_1.load_state_dict(checkpoint_dict['Model_state_dict'])
 model_1 = model_1.cuda()
model_1.eval()
print('Validation Average Recall {} \
Validation Grapheme Root Recall {} \
Validation Vowel Diacritic Recall {} \
Validation Consonant Diacritic Recall {} \
Validation Loss {} '.format(checkpoint_dict['Current_average_recall'],checkpoint_dict['Current_grapheme_root_recall'],checkpoint_dict['Current_vowel_diacritic_recall'], \
checkpoint_dict['Current_consonant_diacritic_recall'],checkpoint_dict['Current_val_Loss']))<load_pretrained>
imputer = Imputer(missing_values='NaN' , strategy='median' , axis = 0)
 imputer = imputer.fit(train_num)
 train_num = imputer.transform(train_num )
model_2 = CNN_Model()
checkpoint_dict = torch.load('/kaggle/input/modelsubmission58/checkpoint_best_Average_recall.pth')
 model_2.load_state_dict(checkpoint_dict['Model_state_dict'])
 model_2 = model_2.cuda()
model_2.eval()
print('Validation Average Recall {} \
Validation Grapheme Root Recall {} \
Validation Vowel Diacritic Recall {} \
Validation Consonant Diacritic Recall {} \
Validation Loss {} '.format(checkpoint_dict['Current_average_recall'],checkpoint_dict['Current_grapheme_root_recall'],checkpoint_dict['Current_vowel_diacritic_recall'], \
checkpoint_dict['Current_consonant_diacritic_recall'],checkpoint_dict['Current_val_Loss']))<load_pretrained>
test_num = imputer.transform(test[numerical_col] )
model_3 = CNN_Model()
checkpoint_dict = torch.load('/kaggle/input/modelsubmission57/checkpoint_best_Average_recall.pth')
 model_3.load_state_dict(checkpoint_dict['Model_state_dict'])
 model_3 = model_3.cuda()
model_3.eval()
print('Validation Average Recall {} \
Validation Grapheme Root Recall {} \
Validation Vowel Diacritic Recall {} \
Validation Consonant Diacritic Recall {} \
Validation Loss {} '.format(checkpoint_dict['Current_average_recall'],checkpoint_dict['Current_grapheme_root_recall'],checkpoint_dict['Current_vowel_diacritic_recall'], \
checkpoint_dict['Current_consonant_diacritic_recall'],checkpoint_dict['Current_val_Loss']))<load_pretrained>
X_train , X_test , y_train , y_test= train_test_split(train_num , label , test_size= 0.2 , random_state=123 )
model_4 = CNN_Model()
checkpoint_dict = torch.load('/kaggle/input/modelsubmission73/checkpoint_best_Average_recall.pth')
 model_4.load_state_dict(checkpoint_dict['Model_state_dict'])
 model_4 = model_4.cuda()
model_4.eval()
print('Validation Average Recall {} \
Validation Grapheme Root Recall {} \
Validation Vowel Diacritic Recall {} \
Validation Consonant Diacritic Recall {} \
Validation Loss {} '.format(checkpoint_dict['Current_average_recall'],checkpoint_dict['Current_grapheme_root_recall'],checkpoint_dict['Current_vowel_diacritic_recall'], \
checkpoint_dict['Current_consonant_diacritic_recall'],checkpoint_dict['Current_val_Loss']))<train_on_grid>
clf = LinearRegression(normalize=True)
 scores = cross_val_score(clf, X_train, y_train, cv=5 ).mean()
scores
model_5 = CNN_Model()
checkpoint_dict = torch.load('/kaggle/input/modelsubmission74/checkpoint_best_Average_recall.pth')
 model_5.load_state_dict(checkpoint_dict['Model_state_dict'])
 model_5 = model_5.cuda()
model_5.eval()
print('Validation Average Recall {} \
Validation Grapheme Root Recall {} \
Validation Vowel Diacritic Recall {} \
Validation Consonant Diacritic Recall {} \
Validation Loss {} '.format(checkpoint_dict['Current_average_recall'],checkpoint_dict['Current_grapheme_root_recall'],checkpoint_dict['Current_vowel_diacritic_recall'], \
checkpoint_dict['Current_consonant_diacritic_recall'],checkpoint_dict['Current_val_Loss']))<categorify>
clf = Lasso(alpha=0.3, normalize=True)
 scores = cross_val_score(clf, X_train, y_train, cv=5 ).mean()
scores
class BengaliAiDataset(Dataset):
def __init__(self, all_imgs):
super(BengaliAiDataset, self ).__init__()
self.all_imgs = all_imgs
def __len__(self):
return self.all_imgs.shape[0]
def __getitem__(self, index):
img = self.all_imgs[index,:,:]
img = img[:,:,np.newaxis]
img = np.concatenate([img,img,img],axis=2)
 img =(img/255 ).astype(np.float32)
 img = np.transpose(img,(2, 0, 1))
img = torch.from_numpy(img ).float()
return img<define_variables>
clf = ElasticNet(alpha=1, l1_ratio=0.5, normalize=False)
 scores = cross_val_score(clf, X_train, y_train, cv=5 ).mean()
scores
HEIGHT = 137
WIDTH = 236
TEST = ['/kaggle/input/bengaliai-cv19/test_image_data_0.parquet',
'/kaggle/input/bengaliai-cv19/test_image_data_1.parquet',
'/kaggle/input/bengaliai-cv19/test_image_data_2.parquet',
'/kaggle/input/bengaliai-cv19/test_image_data_3.parquet']
all_grapheme_labels = []
all_vowel_diacritic_labels = []
all_consonant_diacritic_labels = []
torch.set_grad_enabled(False)
 with torch.no_grad() :
for fname in TEST:
df = pd.read_parquet(fname)
 data = df.iloc[:, 1:].values.reshape(-1, HEIGHT, WIDTH ).astype(np.uint8)
 all_imgs = []
for idx in tqdm(range(len(df))):
img = data[idx].astype(np.uint8)
 img = img[np.newaxis,...]
all_imgs.append(img)
 all_imgs = np.concatenate(all_imgs,axis=0)
 ds_test = BengaliAiDataset(all_imgs)
 dl_test = DataLoader(ds_test, shuffle=False, batch_size=100, num_workers=4, pin_memory=True)
 for i,(imgs)in enumerate(dl_test):
imgs = imgs.cuda()
grapheme_output_1,vowel_diacritic_output_1,consonant_diacritic_output_1 = model_1(imgs)
 grapheme_output_2,vowel_diacritic_output_2,consonant_diacritic_output_2 = model_2(imgs)
 grapheme_output_3,vowel_diacritic_output_3,consonant_diacritic_output_3 = model_3(imgs)
 grapheme_output_4,vowel_diacritic_output_4,consonant_diacritic_output_4 = model_4(imgs)
 grapheme_output_5,vowel_diacritic_output_5,consonant_diacritic_output_5 = model_5(imgs)
 grapheme_output =(grapheme_output_1+grapheme_output_2+grapheme_output_3+grapheme_output_4+grapheme_output_5)/5
vowel_diacritic_output =(vowel_diacritic_output_1+vowel_diacritic_output_2+vowel_diacritic_output_3+vowel_diacritic_output_4+vowel_diacritic_output_5)/5
consonant_diacritic_output =(consonant_diacritic_output_1+consonant_diacritic_output_2+consonant_diacritic_output_3+consonant_diacritic_output_4+consonant_diacritic_output_5)/5
submission_grapheme_root_labels = torch.argmax(grapheme_output.clone() ,dim=1)
 submission_vowel_diacritic_labels = torch.argmax(vowel_diacritic_output.clone() ,dim=1)
 submission_consonant_diacritic_labels = torch.argmax(consonant_diacritic_output.clone() ,dim=1)
 submission_grapheme_root_labels = submission_grapheme_root_labels.tolist()
submission_vowel_diacritic_labels = submission_vowel_diacritic_labels.tolist()
submission_consonant_diacritic_labels = submission_consonant_diacritic_labels.tolist()
all_grapheme_labels = all_grapheme_labels+submission_grapheme_root_labels
all_vowel_diacritic_labels = all_vowel_diacritic_labels+submission_vowel_diacritic_labels
all_consonant_diacritic_labels = all_consonant_diacritic_labels+submission_consonant_diacritic_labels<save_to_csv>
clf=xgboost.XGBRegressor(n_estimators=100, learning_rate=0.08, gamma=0, subsample=0.75,
colsample_bytree=1, max_depth=7)
 scores = cross_val_score(clf, X_train, y_train, cv=5 ).mean()
scores
row_id = []
target = []
for i in tqdm(range(len(all_vowel_diacritic_labels))):
row_id += [f'Test_{i}_grapheme_root', f'Test_{i}_vowel_diacritic',
f'Test_{i}_consonant_diacritic']
target += [all_grapheme_labels[i], all_vowel_diacritic_labels[i], all_consonant_diacritic_labels[i]]
submission_df = pd.DataFrame({'row_id': row_id, 'target': target})
 submission_df.to_csv('submission.csv', index=False )<import_modules>
dropp = ['MiscFeature' , 'PoolQC' , 'Fence' ,'Alley' ]
train_cat.drop(columns=dropp , axis=1, inplace=True )
IMG_SIZE=86
N_CHANNELS=1
def resize(df, size=86, need_progress_bar=True):
resized = {}
resize_size=86
if need_progress_bar:
for i in tqdm(range(df.shape[0])) :
image=df.loc[df.index[i]].values.reshape(137,236)
 _, thresh = cv2.threshold(image, 30, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
 contours, _ = cv2.findContours(thresh,cv2.RETR_LIST,cv2.CHAIN_APPROX_SIMPLE)[-2:]
idx = 0
ls_xmin = []
ls_ymin = []
ls_xmax = []
ls_ymax = []
for cnt in contours:
idx += 1
x,y,w,h = cv2.boundingRect(cnt)
 ls_xmin.append(x)
 ls_ymin.append(y)
 ls_xmax.append(x + w)
 ls_ymax.append(y + h)
 xmin = min(ls_xmin)
 ymin = min(ls_ymin)
 xmax = max(ls_xmax)
 ymax = max(ls_ymax)
 roi = image[ymin:ymax,xmin:xmax]
resized_roi = cv2.resize(roi,(resize_size, resize_size),interpolation=cv2.INTER_AREA)
 resized[df.index[i]] = resized_roi.reshape(-1)
 else:
for i in range(df.shape[0]):
image=df.loc[df.index[i]].values.reshape(137,236)
 _, thresh = cv2.threshold(image, 30, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
 contours, _ = cv2.findContours(thresh,cv2.RETR_LIST,cv2.CHAIN_APPROX_SIMPLE)[-2:]
idx = 0
ls_xmin = []
ls_ymin = []
ls_xmax = []
ls_ymax = []
for cnt in contours:
idx += 1
x,y,w,h = cv2.boundingRect(cnt)
 ls_xmin.append(x)
 ls_ymin.append(y)
 ls_xmax.append(x + w)
 ls_ymax.append(y + h)
 xmin = min(ls_xmin)
 ymin = min(ls_ymin)
 xmax = max(ls_xmax)
 ymax = max(ls_ymax)
 roi = image[ymin:ymax,xmin:xmax]
resized_roi = cv2.resize(roi,(resize_size, resize_size),interpolation=cv2.INTER_AREA)
 resized[df.index[i]] = resized_roi.reshape(-1)
 resized = pd.DataFrame(resized ).T
return resized 
<load_pretrained>
test_cat.drop(columns=dropp , axis=1, inplace=True)
 test_cat = test_cat.astype('category')
 test_cat.info()
model1 = load_model('.. /input/bengali-alternative/densenet_169_rooty.h5')
 model2 = load_model('.. /input/bengali-alternative/densenet_CNN(3 ).h5')
 model3 = load_model('.. /input/bengali-alternative/densenet_rooty_tooty.h5')
 preds_dict = {
'grapheme_root': [],
'vowel_diacritic': [],
'consonant_diacritic': []
}
components = ['consonant_diacritic', 'grapheme_root', 'vowel_diacritic']
target=[]
row_id=[]
for i in range(4):
df_test_img = pd.read_parquet('/kaggle/input/bengaliai-cv19/test_image_data_{}.parquet'.format(i))
df_test_img.set_index('image_id', inplace=True)
 X_test = resize(df_test_img, need_progress_bar=False)/255
X_test = X_test.values.reshape(-1, IMG_SIZE, IMG_SIZE, N_CHANNELS)
 preds1 = model1.predict(X_test)
 preds2 = model2.predict(X_test)
 preds3 = model3.predict(X_test)
 preds = [np.mean([preds1[0],preds2[0],preds3[0]],axis=0), np.mean([preds1[1],preds2[1],preds3[1]],axis=0), np.mean([preds1[2],preds2[2],preds3[2]],axis=0)]
for i, p in enumerate(preds_dict):
preds_dict[p] = np.argmax(preds[i], axis=1)
 for k,id in enumerate(df_test_img.index.values):
for i,comp in enumerate(components):
id_sample=id+'_'+comp
row_id.append(id_sample)
 target.append(preds_dict[comp][k])
 del df_test_img
del X_test
gc.collect()
df_sample = pd.DataFrame(
 {
'row_id': row_id,
'target':target
},
columns = ['row_id','target'] 
)
 df_sample.to_csv('submission.csv',index=False)
 df_sample.head()<install_modules>
most_freq = {}
for col in train_cat.columns:
p = train_cat[col].mode() [0]
train_cat[col].fillna(p, inplace=True)
 most_freq[col] = p
!pip install.. /input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null<import_modules>
for col in train_cat.columns:
test_cat[col].fillna(most_freq[col], inplace=True )
import numpy as np
import pandas as pd
import os
import torch
import torch.nn as nn
from torch.utils.data import Dataset,DataLoader
from torchvision import transforms,models
from tqdm import tqdm_notebook as tqdm
import math
import torch.nn.functional as F
from torch.nn import init
import gc
import cv2
import pretrainedmodels
import torchvision<define_variables>
train_num =pd.DataFrame(train_num)
 train_num.head(2 )
HEIGHT = 137
WIDTH = 236
TRAIN = False<load_pretrained>
test_num =pd.DataFrame(test_num)
 test_num.head(2 )
def prepare_image(dataType = 'train', indices = [0,1,2,3]):
assert dataType in ['train', 'test']
HEIGHT = 137
WIDTH = 236
images = []
for i in indices:
image_df = pd.read_parquet(f'.. /input/bengaliai-cv19/{dataType}_image_data_{i}.parquet')
 images.append(image_df.iloc[:,1:].values.reshape(-1,HEIGHT,WIDTH))
del image_df
gc.collect()
images = np.concatenate(images, axis = 0)
 return images<load_from_csv>
for col in train_cat:
train_cat[col] = train_cat[col].cat.codes
for col in test_cat:
test_cat[col] = test_cat[col].cat.codes
if TRAIN:
train = pd.read_csv('/kaggle/input/bengaliai-cv19/train.csv')
 train_labels = train[['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']].values
train_images = prepare_image()<prepare_x_and_y>
train_num.index = train_cat.index
class BengaliAIDataset(Dataset):
def __init__(self, images, labels=None, indices=None):
super(BengaliAIDataset, self ).__init__()
self.images = images
self.labels = labels
if indices is None:
indices = np.arange(len(images))
self.indices = indices
self.train = labels is not None
def __len__(self):

 return len(self.indices)
 def __getitem__(self, i):

 i = self.indices[i]
x = self.images[i]
x =(255 - x ).astype(np.float32)
 x =(x*(255.0/x.max())).astype(np.float32)
 x = crop_resize(x,224)
 x = np.stack(( x,)*3,axis=-1)
 x = x * [0.229, 0.224, 0.225]
x = np.rollaxis(x, 2, 0)
 x = x.astype(np.float32)/255.0
if self.train:
y = self.labels[i]
return x, y
else:
return x<choose_model_class>
test_num.index = test_cat.index
def MyModel() :
model = pretrainedmodels.__dict__['densenet161'](pretrained=None)
 model.last_linear = nn.Linear(model.last_linear.in_features, 186)
 return model<set_options>
train_cat = pd.get_dummies(train_cat)
 test_cat = pd.get_dummies(test_cat )
device = torch.device('cuda:0' )<categorify>
train_ = train_num.join(train_cat )
model = MyModel().to(device )<create_dataframe>
test_ = test_num.join(test_cat )
if TRAIN:
optimizer = torch.optim.Adam(model.parameters() , lr=4e-4)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', patience=2, factor=0.3)
 criterion = nn.CrossEntropyLoss()
batch_size=32
num_images = len(train_images)
 train_data_size = int(num_images*0.9)
 test_data_size = num_images - train_data_size
perm = np.random.RandomState(111 ).permutation(num_images)
 train_dataset = BengaliAIDataset(train_images, train_labels, indices=perm[:train_data_size])
 valid_dataset = BengaliAIDataset(train_images, train_labels, indices=perm[train_data_size:train_data_size+test_data_size])
 train_loader = torch.utils.data.DataLoader(train_dataset,batch_size=batch_size,shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid_dataset,batch_size=batch_size,shuffle=True)
 epochs = 10
losses=[]
accs=[]
train_root_accs = []
val_root_accs = []
train_vowel_accs = []
val_vowel_accs = []
train_con_accs = []
val_con_accs = []
train_accs = []
val_accs = []<load_pretrained>
scalar = MinMaxScaler()
train_ = scalar.fit_transform(train_)
 test_ = scalar.transform(test_ )
if not TRAIN:
load_model_path = '/kaggle/input/dense161/seresnet50(3 ).pth'
checkpoint = torch.load(load_model_path)
 model.load_state_dict(checkpoint['model_state_dict'] )<import_modules>
lightgbm = lgb.LGBMRegressor(objective='regression',
num_leaves=8,
learning_rate=0.03,
n_estimators=4000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
verbose=-1, 
)
 scores = cross_val_score(lightgbm, train_, label, cv=5 ).mean()
scores
<train_model><EOS> 
clf.fit(train_ , label)
 pre = clf.predict(test_)
 submit = pd.read_csv('.. /input/sample_submission.csv')
 submit.head()
submit.SalePrice = pre
submit.to_csv('submit.csv', index = False )