Kaggle competition: House Prices - Advanced Regression Techniques
weights = [0.3345993850605896, 0.33387592693072893, 0.3315246880086815]<prepare_output>
BATCH_SIZE = 128
EPOCHS = 15
weighted_y_pred = pd.DataFrame()
weighted_y_pred['sig_id'] = predictions[0]['sig_id']
for column in target_columns:
column_data = []
for i in range(len(predictions)) :
column_data.append(predictions[i][column] * weights[i])
 weighted_y_pred[column] = np.sum(column_data, axis=0)
 weighted_y_pred.shape<prepare_output>
train = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv" )
y_pred = weighted_y_pred<save_to_csv>
sns.set_theme(rc = {'grid.linewidth': 0.5,
'axes.linewidth': 0.75, 'axes.facecolor': '
'figure.facecolor': '
'xtick.labelcolor': '
submission = pd.DataFrame(index = public_ids + private_ids, columns=target_columns)
 submission.index.name = 'sig_id'
submission[:] = 0
submission.loc[y_pred.sig_id,:] = y_pred[target_columns].values
submission.loc[test[test.cp_type == 'ctl_vehicle'].sig_id] = 0
submission.to_csv('submission.csv', index=True )<import_modules>
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.SalePrice.values
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data.drop(['SalePrice', 'GarageArea', 'TotRmsAbvGrd'], axis=1, inplace=True)
 print("all_data size is : {}".format(all_data.shape))
import numpy as np
import pandas as pd
import os
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import Dense, Flatten, GlobalAveragePooling2D, Activation, Flatten, Dropout, BatchNormalization
from tensorflow.python.keras import optimizers, regularizers
from tensorflow.python.keras.applications import Xception
from keras.applications.xception import preprocess_input
from tensorflow.python.keras.models import Model
from tensorflow.python.keras.optimizers import sgd
from tensorflow.python.keras.preprocessing.image import ImageDataGenerator<load_from_csv>
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head(20 )
df=pd.read_csv('.. /input/labels.csv')
 test_df=pd.read_csv('.. /input//sample_submission.csv')
 image_size = 299<choose_model_class>
all_data["PoolQC"] = all_data["PoolQC"].fillna("None" )
model = Sequential()
model.add(Xception(include_top=False, pooling='avg', weights="imagenet"))
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.8))
model.add(Dense(120, activation='softmax'))
model.layers[0].trainable = False
model.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['accuracy'] )<create_dataframe>
all_data["MiscFeature"] = all_data["MiscFeature"].fillna("None" )
datagen = ImageDataGenerator(preprocessing_function=preprocess_input,
rescale=1./255.,
horizontal_flip=True,
width_shift_range = 0.2,
height_shift_range = 0.2,
validation_split=0.25)
 train_generator=datagen.flow_from_dataframe(dataframe=df,directory=".. /input/train/",
x_col="id",y_col="breed",has_ext=False,
subset="training",batch_size=32,
shuffle=True,class_mode="categorical",
target_size=(image_size, image_size))
valid_generator=datagen.flow_from_dataframe(
 dataframe=df,
directory=".. /input/train/",
x_col="id",
y_col="breed",has_ext=False,
subset="validation",batch_size=1,
class_mode="categorical",target_size=(image_size, image_size))
test_datagen=ImageDataGenerator(preprocessing_function=preprocess_input,rescale=1./255.)
test_generator=test_datagen.flow_from_dataframe(
 dataframe=test_df,
directory=".. /input/test/",x_col="id",
y_col=None,has_ext=False,batch_size=1,
seed=42,shuffle=False,
class_mode=None,target_size=(image_size, image_size)) 
<train_model>
all_data["Alley"] = all_data["Alley"].fillna("None" )
model.fit_generator(generator=train_generator,steps_per_epoch=train_generator.n,validation_data=valid_generator,validation_steps=train_generator.n,epochs=1 )<predict_on_test>
all_data["Fence"] = all_data["Fence"].fillna("None" )
pred=model.predict_generator(test_generator,verbose=1 )<save_to_csv>
all_data["FireplaceQu"] = all_data["FireplaceQu"].fillna("None" )
labels =(train_generator.class_indices)
 labels = list(labels.keys())
 df = pd.DataFrame(data=pred,
columns=labels)
 columns = list(df)
 columns.sort()
df = df.reindex(columns=columns)
 filenames = test_df["id"]
df["id"] = filenames
cols = df.columns.tolist()
cols = cols[-1:] + cols[:-1]
df = df[cols]
df.head(5)
 df.to_csv("submission_nas.csv",index=False )<set_options>
all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
%reload_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
all_data[col] = all_data[col].fillna('None' )
import numpy as np
import pandas as pd
import os
from fastai.imports import *
from fastai.torch_imports import *
from fastai.transforms import *
from fastai.conv_learner import *
from fastai.model import *
from fastai.dataset import *
from fastai.sgdr import *
from fastai.plots import *<define_variables>
for col in('GarageYrBlt', 'GarageCars'):
all_data[col] = all_data[col].fillna(0 )
PATH = '.. /input/'
sz = 224
bs = 58
arch = resnet50
print(os.listdir(PATH)) 
<define_variables>
for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
all_data[col] = all_data[col].fillna(0 )
labels_dir = f'{PATH}labels.csv'
n = len(list(open(labels_dir)))-1
val_idxs = get_cv_idxs(n)
 len(val_idxs)
 n<count_values>
for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
all_data[col] = all_data[col].fillna('None' )
labels = pd.read_csv(labels_dir)
 labels.head()
labels['breed'].value_counts()<load_from_csv>
all_data["MasVnrType"] = all_data["MasVnrType"].fillna("None")
 all_data["MasVnrArea"] = all_data["MasVnrArea"].fillna(0 )
tfms = tfms_from_model(arch, sz, aug_tfms = transforms_side_on, max_zoom = 1.1)
 data = ImageClassifierData.from_csv(PATH, 'train', csv_fname = labels_dir,test_name = 'test', suffix = '.jpg',val_idxs = val_idxs,tfms=tfms, bs =bs)
<define_variables>
all_data['MSZoning'] = all_data['MSZoning'].fillna(all_data['MSZoning'].mode() [0] )
file = PATH + data.trn_ds.fnames[1000]
file<load_pretrained>
all_data = all_data.drop(['Utilities'], axis=1 )
img = PIL.Image.open(file)
 img<load_from_csv>
all_data["Functional"] = all_data["Functional"].fillna("Typ" )
def get_data(sz, bs):
tfms = tfms_from_model(arch, sz, aug_tfms = transforms_side_on, max_zoom = 1.1)
 data = ImageClassifierData.from_csv(PATH, 'train', csv_fname = labels_dir,test_name = 'test', suffix = '.jpg',val_idxs = val_idxs,tfms=tfms, bs =bs)
 if sz > 300:
return data
else:
data.resize(340, '/tmp')
 return data<define_variables>
all_data['Electrical'] = all_data['Electrical'].fillna(all_data['Electrical'].mode() [0] )
TMP_PATH = "/tmp/tmp"
MODEL_PATH = "/tmp/model/"<load_pretrained>
all_data['KitchenQual'] = all_data['KitchenQual'].fillna(all_data['KitchenQual'].mode() [0] )
data = get_data(sz, bs)
 learn = ConvLearner.pretrained(arch, data,tmp_name = TMP_PATH, models_name = MODEL_PATH, precompute = True)
<train_model>
all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0] )
%time learn.fit(1e-1,5,cycle_len=1 )<train_model>
all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0] )
learn.precompute = False
%time learn.fit(1e-2,5,cycle_len=1 )<save_model>
all_data['MSSubClass'] = all_data['MSSubClass'].fillna("None" )
learn.save("224_pre")
 learn.load("224_pre" )<train_model>
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head()
sz = 299
learn.set_data(get_data(sz,bs))
learn.freeze()
learn.fit(1e-2,3,cycle_len=1 )<train_model>
all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 all_data['OverallCond'] = all_data['OverallCond'].astype(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str )
learn.fit(1e-2, 3, cycle_len = 1, cycle_mult = 2 )<compute_test_metric>
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[c].values))
all_data[c] = lbl.transform(list(all_data[c].values))
print('Shape all_data: {}'.format(all_data.shape))
log_preds, y = learn.TTA()
probs = np.mean(np.exp(log_preds), 0)
 accuracy_np(probs, y), metrics.log_loss(y, probs )<save_model>
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
learn.save("229_pre")
 learn.load("229_pre" )<feature_engineering>
y_train = np.log(y_train )
log_preds, y = learn.TTA(is_test= True)
 probs = np.mean(np.exp(log_preds), 0 )<create_dataframe>
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
all_data[feat] = boxcox1p(all_data[feat], lam)
df = pd.DataFrame(probs)
 df.columns = data.classes<prepare_output>
all_data = pd.get_dummies(all_data)
 print(all_data.shape )
df.insert(0, 'id', [o[5:-4] for o in data.test_ds.fnames] )<save_to_csv>
train = all_data[:ntrain]
test = all_data[ntrain:]
df.to_csv("Submit.csv",index = False)
<import_modules>
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split
import xgboost as xgb
import lightgbm as lgb
import numpy as np
import pandas as pd 
<load_from_csv>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )
df = pd.read_csv('.. /input/severstal-steel-defect-detection/sample_submission.csv')
 df['EncodedPixels']=['' for i in range(len(df)) ]
df_submit = pd.read_csv(".. /input/severstal-final-test-preds-0111-3/severstal_final_test_preds_0111_3.csv" ).fillna('')
 if df.columns[0]=='ImageId_ClassId':
df.set_index('ImageId_ClassId', inplace=True)
 df_submit.set_index('ImageId_ClassId', inplace=True)
 for name, row in df_submit.iterrows() :
df.loc[name] = row
df.reset_index(inplace=True)
 df.to_csv('submission.csv', index=False )<import_modules>
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
import pdb
import os
import cv2
import torch
import pandas as pd
import numpy as np
from tqdm import tqdm
import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader, Dataset
from albumentations import(Normalize, Compose)
 from albumentations.pytorch import ToTensor
import torch.utils.data as data
import torchvision.models as models
import torch.nn as nn
from torch.nn import functional as F<define_search_model>
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
class Resnet34_classification(nn.Module):
def __init__(self,num_class=4):
super(Resnet34_classification, self ).__init__()
e = ResNet34()
self.block = nn.ModuleList([
e.block0,
e.block1,
e.block2,
e.block3,
e.block4,
])
 e = None
self.feature = nn.Conv2d(512,32, kernel_size=1)
 self.logit = nn.Conv2d(32,num_class, kernel_size=1)
 def forward(self, x):
batch_size,C,H,W = x.shape
for i in range(len(self.block)) :
x = self.block[i](x)
 x = F.dropout(x,0.5,training=self.training)
 x = F.adaptive_avg_pool2d(x, 1)
 x = self.feature(x)
 logit = self.logit(x)
 return logit 
<load_pretrained>
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5 )
model_classification = Resnet34_classification()
model_classification.load_state_dict(torch.load('.. /input/hengs-steel-classifier/00007500_model.pth',
map_location=lambda storage, loc: storage), strict=True )<categorify>
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5 )
class TestDataset(Dataset):

 def __init__(self, root, df, mean, std):
self.root = root
df['ImageId'] = df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 self.fnames = df['ImageId'].unique().tolist()
self.num_samples = len(self.fnames)
 self.transform = Compose(
 [
Normalize(mean=mean, std=std, p=1),
ToTensor() ,
] 
)
 def __getitem__(self, idx):
fname = self.fnames[idx]
path = os.path.join(self.root, fname)
 image = cv2.imread(path)
 images = self.transform(image=image)["image"]
return fname, images
def __len__(self):
return self.num_samples<define_variables>
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1 )
sample_submission_path = '.. /input/severstal-steel-defect-detection/sample_submission.csv'
test_data_folder = ".. /input/severstal-steel-defect-detection/test_images"<define_search_space>
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )
batch_size = 32
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225 )<load_from_csv>
score = rmsle_cv(lasso)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
df = pd.read_csv(sample_submission_path)
 testset = DataLoader(
 TestDataset(test_data_folder, df, mean, std),
batch_size=batch_size,
shuffle=False,
num_workers=0,
pin_memory=True
)<define_variables>
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
threshold_label = [0.50,0.50,0.50,0.50,]
augment = ['null', 'flip_lr']<prepare_output>
score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
probability_label, image_id = get_classification_preds(model_classification, testset)
 predict_label = probability_label>np.array(threshold_label ).reshape(1,4,1,1)
 image_id_class_id = []
encoded_pixel = []
for b in range(len(image_id)) :
for c in range(4):
image_id_class_id.append(image_id[b]+'_%d'%(c+1))
if predict_label[b,c]==0:
rle=''
else:
rle ='1 1'
encoded_pixel.append(rle)
 df_classification = pd.DataFrame(zip(image_id_class_id, encoded_pixel), columns=['ImageId_ClassId', 'EncodedPixels'] )<count_values>
score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
df_classification['EncodedPixels'].value_counts()<set_options>
score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
warnings.filterwarnings('ignore')
<load_pretrained>
score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
unet_se_resnext50_32x4d = load('/kaggle/input/severstalmodels/unet_se_resnext50_32x4d.pth' ).cuda()
unet_mobilenet2 = load('/kaggle/input/severstalmodels/unet_mobilenet2.pth' ).cuda()
unet_resnet34 = load('/kaggle/input/severstalmodels/unet_resnet34.pth' ).cuda()
unet_eff3_p2 = load('/kaggle/input/steel-models/Unet_effi_stage2.pth' ).cuda()<normalization>
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )
class Model:
def __init__(self, models):
self.models = models
def __call__(self, x):
res = []
x = x.cuda()
with torch.no_grad() :
for m in self.models:
res.append(m(x))
res = torch.stack(res)
 return torch.mean(res, dim=0)
 model = Model([unet_se_resnext50_32x4d, unet_mobilenet2, unet_resnet34, unet_eff3_p2])
<categorify>
averaged_models = AveragingModels(models =(ENet, GBoost, lasso, model_xgb, model_lgb))
score = rmsle_cv(averaged_models)
 print("Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
def create_transforms(additional):
res = list(additional)
 res.extend([
A.Normalize(
 mean=(0.485, 0.456, 0.406), std=(0.230, 0.225, 0.223)
),
ChannelTranspose()
])
 res = A.Compose(res)
 return res
img_folder = '/kaggle/input/severstal-steel-defect-detection/test_images'
batch_size = 2
num_workers = 0
transforms = [
[],
[A.VerticalFlip(p=1)],
[A.HorizontalFlip(p=1)]
]
transforms = [create_transforms(t)for t in transforms]
datasets = [TtaWrap(ImageDataset(img_folder=img_folder, transforms=t), tfms=t)for t in transforms]
loaders = [DataLoader(d, num_workers=num_workers, batch_size=batch_size, shuffle=False)for d in datasets]<define_search_space>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
thresholds = [0.5, 0.5, 0.5, 0.5]
min_area = [600, 600, 1000, 2000]
res = []
total = len(datasets[0])//batch_size
for loaders_batch in tqdm_notebook(zip(*loaders), total=total):
preds = []
image_file = []
for i, batch in enumerate(loaders_batch):
features = batch['features'].cuda()
p = torch.sigmoid(model(features))
p = datasets[i].inverse(p)
 preds.append(p)
 image_file = batch['image_file']
preds = torch.stack(preds)
 preds = torch.mean(preds, dim=0)
 preds = preds.detach().cpu().numpy()
for p, file in zip(preds, image_file):
file = os.path.basename(file)
 for i in range(4):
p_channel = p[i]
imageid_classid = file+'_'+str(i+1)
 p_channel =(p_channel>thresholds[i] ).astype(np.uint8)
 if p_channel.sum() < min_area[i]:
p_channel = np.zeros(p_channel.shape, dtype=p_channel.dtype)
 res.append({
'ImageId_ClassId': imageid_classid,
'EncodedPixels': mask2rle(p_channel)
 })
 df_seg = pd.DataFrame(res)
 df_seg = df_seg.fillna('')
 df_seg.to_csv('submission.csv', index=False )<create_dataframe>
averaged_models.fit(train.values, y_train)
 av_train_pred = averaged_models.predict(train.values)
 print(rmsle(y_train, av_train_pred))
df = df_seg.copy()<data_type_conversions>
av_test_pred = np.expm1(averaged_models.predict(test.values))
if 1:
df['Class'] = df['ImageId_ClassId'].str[-1].astype(np.int32)
 df['Label'] =(df['EncodedPixels']!='' ).astype(np.int32)
 pos1 =(( df['Class']==1)&(df['Label']==1)).sum()
pos2 =(( df['Class']==2)&(df['Label']==1)).sum()
pos3 =(( df['Class']==3)&(df['Label']==1)).sum()
pos4 =(( df['Class']==4)&(df['Label']==1)).sum()
num_image = len(df)//4
num = len(df)
 pos =(df['Label']==1 ).sum()
neg = num-pos
print('')
 print('\t\tnum_image = %5d(1801)'%num_image)
 print('\t\tnum = %5d(7204)'%num)
 print('\t\tneg = %5d(6172)%0.3f'%(neg,neg/num))
print('\t\tpos = %5d(1032)%0.3f'%(pos,pos/num))
print('\t\tpos1 = %5d(128)%0.3f %0.3f'%(pos1,pos1/num_image,pos1/pos))
print('\t\tpos2 = %5d(43)%0.3f %0.3f'%(pos2,pos2/num_image,pos2/pos))
print('\t\tpos3 = %5d(741)%0.3f %0.3f'%(pos3,pos3/num_image,pos3/pos))
print('\t\tpos4 = %5d(120)%0.3f %0.3f'%(pos4,pos4/num_image,pos4/pos))<define_variables>
sample_submission = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv")
 sample_submission
<feature_engineering><EOS> 
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = av_test_pred
sub.to_csv('submission.csv',index=False )