Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<train_model>
style.use('ggplot')
 sns.set_style('whitegrid')
 warnings.filterwarnings('ignore' )
data =(ImageList.from_folder(path)
.split_by_rand_pct(seed=seed)
.label_from_folder() 
.transform(tfms, size=224)
.databunch().normalize(imagenet_stats))
learn.data = data
learn.bs = 32
data.train_ds[0][0].shape<choose_model_class>
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 print('test shape: ',test.shape)
 print('train shape:',train.shape)
learn.loss_func = FocalLoss()<train_model>
train.drop("Id", axis=1,inplace=True)
 test.drop("Id", axis=1,inplace=True)
 print(train.corr() ['SalePrice'].sort_values(ascending=False))
learn.fit_one_cycle(24, slice(3e-3), wd=5e-6 )<save_model>
train = train.drop(train[(train['GrLivArea']>4000)&(train['SalePrice']<200000)].index )
learn.save('resnext50_32x4d_3');
learn.load('resnext50_32x4d_3');<find_best_params>
print(train.isnull().sum().sort_values(ascending = False)[train.isnull().sum().sort_values(ascending = False)!= 0])
 print(test.isnull().sum().sort_values(ascending = False)[test.isnull().sum().sort_values(ascending = False)!= 0] )
learn.unfreeze() ;
learn = learn.clip_grad() ;<train_model>
y = train['SalePrice']
train.drop('SalePrice', axis=1,inplace=True )
lr = [1e-3/200, 1e-3/20, 1e-3/10]
learn.fit_one_cycle(32, lr )<save_model>
all_data = pd.concat(( train,test)).reset_index(drop = True)
 all_data.isnull().sum().sort_values(ascending = False)[all_data.isnull().sum().sort_values(ascending = False)!= 0]
learn.save('resnext50_32x4d_4');
learn.load('resnext50_32x4d_4');<import_modules>
miss_val_1 = ["Alley",
"PoolQC",
"MiscFeature",
"Fence",
"FireplaceQu",
"GarageType",
"GarageFinish",
"GarageQual",
"GarageCond",
'BsmtQual',
'BsmtCond',
'BsmtExposure',
'BsmtFinType1',
'BsmtFinType2',
'MasVnrType']
miss_val_2=['BsmtFinSF1',
'BsmtFinSF2',
'BsmtUnfSF',
'TotalBsmtSF',
'BsmtFullBath',
'BsmtHalfBath',
'GarageYrBlt',
'GarageArea',
'GarageCars',
'MasVnrArea']
for i in miss_val_1:
all_data[i]=all_data[i].fillna('None')
 for i in miss_val_2:
all_data[i]=all_data[i].fillna(0 )
learn.export('/kaggle/working/fastai_resnet.pkl');<load_pretrained>
all_data.isnull().sum().sort_values(ascending = False)[all_data.isnull().sum().sort_values(ascending = False)!= 0]
learn = load_learner('/kaggle/working/','fastai_resnet.pkl', ImageList.from_folder(path_test))
preds,_ = learn.get_preds(ds_type=DatasetType.Test)
 cls_pred = F.softmax(preds,1 ).argmax(1 ).cpu().numpy()<prepare_output>
all_data['LotFrontage'] = all_data.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.mean()))
all_data['MSSubClass'] = all_data['MSSubClass'].astype(str)
 all_data['MSZoning'] = all_data.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str)
 all_data['Functional'] = all_data['Functional'].fillna('Typ')
 all_data['Utilities'] = all_data['Utilities'].fillna('AllPub')
 all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0])
 all_data['KitchenQual'] = all_data['KitchenQual'].fillna("TA")
 all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0])
 all_data['Electrical'] = all_data['Electrical'].fillna("SBrkr" )
paths = list(map(str,list(learn.data.test_ds.x.items)))
 all_test_paths = [p.split('/')[-1][:-4] for p in paths]
df_preds = pd.DataFrame()
df_preds['test_paths'] = all_test_paths
df_preds['class_pred'] = cls_pred
df_preds.set_index('test_paths',inplace=True )<filter>
all_data.isnull().sum().sort_values(ascending = False)[all_data.isnull().sum().sort_values(ascending = False)!= 0]
no_dis_idx = df_preds[df_preds.class_pred==1].index
print(len(no_dis_idx))<load_from_csv>
numerical = all_data.dtypes[all_data.dtypes != "object"].index
sub = pd.read_csv('/kaggle/input/unet-plus-plus-with-efficientnet-encoder/submission.csv'
,index_col='ImageId')
 sub.head()<feature_engineering>
skew = all_data[numerical].apply(lambda x: skew(x)).sort_values(ascending=False)
 skew_index = skew[abs(skew)>0.5].index
for feat in skew_index:
all_data[feat] = boxcox1p(all_data[feat], boxcox_normmax(all_data[feat] + 1))
sub.loc[no_dis_idx] = ' -1'<save_to_csv>
cat = all_data.dtypes[all_data.dtypes == "object"].index
for i in cat:
all_data[i].dtype
le = LabelEncoder()
all_data[i] = le.fit_transform(all_data[i])
 all_data[i].dtype
sub.to_csv('sub_classifier_correction.csv' )<set_options>
X = all_data.iloc[:len(y), :]
X_sub = all_data.iloc[len(y):, :]
!pip install albumentations > /dev/null
!git clone https://github.com/qubvel/efficientnet.git
plt.style.use('seaborn-white')
 sns.set_style("white")
 seed = 10
np.random.seed(seed)
 random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.set_random_seed(seed)
 %matplotlib inline<load_pretrained>
def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, y, scoring="neg_mean_squared_error", cv=kfolds))
return(rmse )
!mkdir masks
!unzip -q.. /input/data-repack-and-image-statistics/masks.zip -d masks
!mkdir train
!unzip -q.. /input/data-repack-and-image-statistics/train.zip -d train
!mkdir test
!unzip -q.. /input/data-repack-and-image-statistics/test.zip -d test<feature_engineering>
kfolds = KFold(n_splits=10, shuffle=True, random_state=42)
 alphas_alt = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas2 = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=alphas_alt, cv=kfolds))
lasso = make_pipeline(RobustScaler() , LassoCV(max_iter=1e7, alphas=alphas2, random_state=42, cv=kfolds))
elasticnet = make_pipeline(RobustScaler() , ElasticNetCV(max_iter=1e7, alphas=e_alphas, cv=kfolds, l1_ratio=e_l1ratio))
svr = make_pipeline(RobustScaler() , SVR(C= 20, epsilon= 0.008, gamma=0.0003,))
xgboost = XGBRegressor(learning_rate=0.01,n_estimators=3460,
max_depth=3, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
nthread=-1,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006)
 lightgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=5000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
verbose=-1, 
)
all_mask_fn = glob.glob('./masks/*')
 mask_df = pd.DataFrame()
mask_df['file_names'] = all_mask_fn
mask_df['mask_percentage'] = 0
mask_df.set_index('file_names',inplace=True)
 for fn in all_mask_fn:
mask_df.loc[fn,'mask_percentage'] = np.array(Image.open(fn)).sum() /(256*256*255)
 mask_df.reset_index(inplace=True)
 sns.distplot(mask_df.mask_percentage)
 mask_df['labels'] = 0
mask_df.loc[mask_df.mask_percentage>0,'labels'] = 1<split>
score = cv_rmse(ridge)
 print("Ridge: {:.4f}
".format(score.mean()))
score = cv_rmse(lasso)
 print("LASSO: {:.4f}
".format(score.mean()))
score = cv_rmse(elasticnet)
 print("elastic net: {:.4f}
".format(score.mean()))
score = cv_rmse(svr)
 print("SVR: {:.4f}
".format(score.mean()))
score = cv_rmse(lightgbm)
 print("lightgbm: {:.4f}
".format(score.mean()))
score = cv_rmse(xgboost)
 print("xgboost: {:.4f}
".format(score.mean()))
all_train_fn = glob.glob('./train/*')
 total_samples = len(all_train_fn)
 idx = np.arange(total_samples)
 train_fn,val_fn = train_test_split(all_train_fn,stratify=mask_df.labels,test_size=0.1,random_state=10)
 print('No.of train files:', len(train_fn))
print('No.of val files:', len(val_fn))
masks_train_fn = [fn.replace('./train','./masks')for fn in train_fn]
masks_val_fn = [fn.replace('./train','./masks')for fn in val_fn]<define_variables>
elastic = elasticnet.fit(X, y)
 lasso = lasso.fit(X, y)
 ridge = ridge.fit(X, y)
 svr = svr.fit(X, y)
 xgb = xgboost.fit(X, y)
 lgb = lightgbm.fit(X, y )
train_im_path,train_mask_path = './keras_im_train','./keras_mask_train'
h,w,batch_size = 256,256,16
val_im_path,val_mask_path = './keras_im_val','./keras_mask_val'
class DataGenerator(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, train_im_path=train_im_path,train_mask_path=train_mask_path,
augmentations=None, batch_size=batch_size,img_size=256, n_channels=3, shuffle=True):
'Initialization'
self.batch_size = batch_size
self.train_im_paths = glob.glob(train_im_path+'/*')
 self.train_im_path = train_im_path
self.train_mask_path = train_mask_path
self.img_size = img_size
self.n_channels = n_channels
self.shuffle = shuffle
self.augment = augmentations
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.ceil(len(self.train_im_paths)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[index*self.batch_size:min(( index+1)*self.batch_size,len(self.train_im_paths)) ]
list_IDs_im = [self.train_im_paths[k] for k in indexes]
X, y = self.data_generation(list_IDs_im)
 if self.augment is None:
return X,np.array(y)/255
else:
im,mask = [],[]
for x,y in zip(X,y):
augmented = self.augment(image=x, mask=y)
 im.append(augmented['image'])
 mask.append(augmented['mask'])
 return np.array(im),np.array(mask)/255
def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.train_im_paths))
if self.shuffle == True:
np.random.shuffle(self.indexes)
 def data_generation(self, list_IDs_im):
'Generates data containing batch_size samples'
X = np.empty(( len(list_IDs_im),self.img_size,self.img_size, self.n_channels))
y = np.empty(( len(list_IDs_im),self.img_size,self.img_size, 1))
for i, im_path in enumerate(list_IDs_im):
im = np.array(Image.open(im_path))
mask_path = im_path.replace(self.train_im_path,self.train_mask_path)
 mask = np.array(Image.open(mask_path))
if len(im.shape)==2:
im = np.repeat(im[...,None],3,2)
 X[i,] = cv2.resize(im,(self.img_size,self.img_size))
y[i,] = cv2.resize(mask,(self.img_size,self.img_size)) [..., np.newaxis]
y[y>0] = 255
return np.uint8(X),np.uint8(y )<normalization>
def models_predict(X):
return(( 0.15 * elastic.predict(X)) + \ 
(0.15 * lasso.predict(X)) + \ 
(0.15 * ridge.predict(X)) + \ 
(0.15 * svr.predict(X)) + \ 
(0.20 * xgb.predict(X)) + \ 
(0.20 * lgb.predict(X)) )
Compose, HorizontalFlip, CLAHE, HueSaturationValue,
RandomBrightness, RandomContrast, RandomGamma,OneOf,
ToFloat, ShiftScaleRotate,GridDistortion, ElasticTransform, JpegCompression, HueSaturationValue,
RGBShift, RandomBrightness, RandomContrast, Blur, MotionBlur, MedianBlur, GaussNoise,CenterCrop,
IAAAdditiveGaussianNoise,GaussNoise,OpticalDistortion,RandomSizedCrop 
)
 AUGMENTATIONS_TRAIN = Compose([
HorizontalFlip(p=0.5),
OneOf([
RandomContrast() ,
RandomGamma() ,
RandomBrightness() ,
], p=0.3),
OneOf([
ElasticTransform(alpha=120, sigma=120 * 0.05, alpha_affine=120 * 0.03),
GridDistortion() ,
OpticalDistortion(distort_limit=2, shift_limit=0.5),
], p=0.3),
RandomSizedCrop(min_max_height=(128, 256), height=h, width=w,p=0.5),
ToFloat(max_value=1)
 ],p=1)
 AUGMENTATIONS_TEST = Compose([
ToFloat(max_value=1)
 ],p=1 )<compute_train_metric>
submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.iloc[:,1] = np.expm1(models_predict(X_sub))
submission.to_csv("submission111.csv", index=False )
<compute_test_metric><EOS>