Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<choose_model_class>
pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 500)
 pd.set_option('display.width', 1000)
 if not sys.warnoptions:
warnings.simplefilter("ignore" )
class SnapshotCallbackBuilder:
def __init__(self, nb_epochs, nb_snapshots, init_lr=0.1):
self.T = nb_epochs
self.M = nb_snapshots
self.alpha_zero = init_lr
def get_callbacks(self, model_prefix='Model'):
callback_list = [
callbacks.ModelCheckpoint("./keras.model",monitor='val_loss',
mode = 'min', save_best_only=True, verbose=1),
swa,
callbacks.LearningRateScheduler(schedule=self._cosine_anneal_schedule)
 ]
return callback_list
def _cosine_anneal_schedule(self, t):
cos_inner = np.pi *(t %(self.T // self.M))
cos_inner /= self.T // self.M
cos_out = np.cos(cos_inner)+ 1
return float(self.alpha_zero / 2 * cos_out )<define_search_model>
desc = {
'SalePrice': "The property's sale price in dollars.This is the target variable to be predicted",
'MSSubClass': "The building class",
'MSZoning': "The general zoning classification",
'LotFrontage': "Linear feet of street connected to property",
'LotArea': "Lot size in square feet",
'Street': "Type of road access",
'Alley': "Type of alley access",
'LotShape': "General shape of property",
'LandContour': "Flatness of the property",
'Utilities': "Type of utilities available",
'LotConfig': "Lot configuration",
'LandSlope': "Slope of property",
'Neighborhood': "Physical locations within Ames city limits",
'Condition1': "Proximity to main road or railroad",
'Condition2': "Proximity to main road or railroad(if a second is present)",
'BldgType': "Type of dwelling",
'HouseStyle': "Style of dwelling",
'OverallQual': "Overall material and finish quality",
'OverallCond': "Overall condition rating",
'YearBuilt': "Original construction date",
'YearRemodAdd': "Remodel date",
'RoofStyle': "Type of roof",
'RoofMatl': "Roof material",
'Exterior1st': "Exterior covering on house",
'Exterior2nd': "Exterior covering on house(if more than one material)",
'MasVnrType': "Masonry veneer type",
'MasVnrArea': "Masonry veneer area in square feet",
'ExterQual': "Exterior material quality",
'ExterCond': "Present condition of the material on the exterior",
'Foundation': "Type of foundation",
'BsmtQual': "Height of the basement",
'BsmtCond': "General condition of the basement",
'BsmtExposure': "Walkout or garden level basement walls",
'BsmtFinType1': "Quality of basement finished area",
'BsmtFinSF1': "Type 1 finished square feet",
'BsmtFinType2': "Quality of second finished area(if present)",
'BsmtFinSF2': "Type 2 finished square feet",
'BsmtUnfSF': "Unfinished square feet of basement area",
'TotalBsmtSF': "Total square feet of basement area",
'Heating': "Type of heating",
'HeatingQC': "Heating quality and condition",
'CentralAir': "Central air conditioning",
'Electrical':" Electrical system",
'1stFlrSF': "First Floor square feet",
'2ndFlrSF': "Second floor square feet",
'LowQualFinSF': "Low quality finished square feet(all floors)",
'GrLivArea': "Above grade(ground)living area square feet",
'BsmtFullBath': "Basement full bathrooms",
'BsmtHalfBath': "Basement half bathrooms",
'FullBath': "Full bathrooms above grade",
'HalfBath': "Half baths above grade",
'Bedroom': "Number of bedrooms above basement level",
'Kitchen': "Number of kitchens",
'KitchenQual': "Kitchen quality",
'TotRmsAbvGrd': "Total rooms above grade(does not include bathrooms)",
'Functional': "Home functionality rating",
'Fireplaces': "Number of fireplaces",
'FireplaceQu': "Fireplace quality",
'GarageType': "Garage location",
'GarageYrBlt': "Year garage was built",
'GarageFinish': "Interior finish of the garage",
'GarageCars': "Size of garage in car capacity",
'GarageArea': "Size of garage in square feet",
'GarageQual': "Garage quality",
'GarageCond': "Garage condition",
'PavedDrive': "Paved driveway",
'WoodDeckSF': "Wood deck area in square feet",
'OpenPorchSF': "Open porch area in square feet",
'EnclosedPorch': "Enclosed porch area in square feet",
'3SsnPorch': "Three season porch area in square feet",
'ScreenPorch': "Screen porch area in square feet",
'PoolArea': "Pool area in square feet",
'PoolQC': "Pool quality",
'Fence': "Fence quality",
'MiscFeature': "Miscellaneous feature not covered in other categories",
'MiscVal': "Value of miscellaneous feature",
'MoSold': "Month Sold",
'YrSold': "Year Sold",
'SaleType': "Type of sale",
'SaleCondition': "Condition of sale"
}
def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True):
x = Conv2D(filters, size, strides=strides, padding=padding )(x)
 x = BatchNormalization()(x)
 if activation == True:
x = LeakyReLU(alpha=0.1 )(x)
 return x
def residual_block(blockInput, num_filters=16):
x = LeakyReLU(alpha=0.1 )(blockInput)
 x = BatchNormalization()(x)
 blockInput = BatchNormalization()(blockInput)
 x = convolution_block(x, num_filters,(3,3))
x = convolution_block(x, num_filters,(3,3), activation=False)
 x = Add()([x, blockInput])
 return x<choose_model_class>
json.dump(desc, open("desc.json",'w'))
with open("desc.json", "r")as read_file:
d= json.load(read_file )
def UEfficientNet(input_shape=(None, None, 3),dropout_rate=0.1):
backbone = EfficientNetB4(weights='imagenet',
include_top=False,
input_shape=input_shape)
 input = backbone.input
start_neurons = 16
conv4 = backbone.layers[342].output
conv4 = LeakyReLU(alpha=0.1 )(conv4)
 pool4 = MaxPooling2D(( 2, 2))(conv4)
 pool4 = Dropout(dropout_rate )(pool4)
 convm = Conv2D(start_neurons * 32,(3, 3), activation=None, padding="same" )(pool4)
 convm = residual_block(convm,start_neurons * 32)
 convm = residual_block(convm,start_neurons * 32)
 convm = LeakyReLU(alpha=0.1 )(convm)
 deconv4 = Conv2DTranspose(start_neurons * 16,(3, 3), strides=(2, 2), padding="same" )(convm)
 uconv4 = concatenate([deconv4, conv4])
 uconv4 = Dropout(dropout_rate )(uconv4)
 uconv4 = Conv2D(start_neurons * 16,(3, 3), activation=None, padding="same" )(uconv4)
 uconv4 = residual_block(uconv4,start_neurons * 16)
 uconv4 = residual_block(uconv4,start_neurons * 16)
 uconv4 = LeakyReLU(alpha=0.1 )(uconv4)
 deconv3 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(uconv4)
 conv3 = backbone.layers[154].output
uconv3 = concatenate([deconv3, conv3])
 uconv3 = Dropout(dropout_rate )(uconv3)
 uconv3 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(uconv3)
 uconv3 = residual_block(uconv3,start_neurons * 8)
 uconv3 = residual_block(uconv3,start_neurons * 8)
 uconv3 = LeakyReLU(alpha=0.1 )(uconv3)
 deconv2 = Conv2DTranspose(start_neurons * 4,(3, 3), strides=(2, 2), padding="same" )(uconv3)
 conv2 = backbone.layers[92].output
uconv2 = concatenate([deconv2, conv2])
 uconv2 = Dropout(0.1 )(uconv2)
 uconv2 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(uconv2)
 uconv2 = residual_block(uconv2,start_neurons * 4)
 uconv2 = residual_block(uconv2,start_neurons * 4)
 uconv2 = LeakyReLU(alpha=0.1 )(uconv2)
 deconv1 = Conv2DTranspose(start_neurons * 2,(3, 3), strides=(2, 2), padding="same" )(uconv2)
 conv1 = backbone.layers[30].output
uconv1 = concatenate([deconv1, conv1])
 uconv1 = Dropout(0.1 )(uconv1)
 uconv1 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(uconv1)
 uconv1 = residual_block(uconv1,start_neurons * 2)
 uconv1 = residual_block(uconv1,start_neurons * 2)
 uconv1 = LeakyReLU(alpha=0.1 )(uconv1)
 uconv0 = Conv2DTranspose(start_neurons * 1,(3, 3), strides=(2, 2), padding="same" )(uconv1)
 uconv0 = Dropout(0.1 )(uconv0)
 uconv0 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(uconv0)
 uconv0 = residual_block(uconv0,start_neurons * 1)
 uconv0 = residual_block(uconv0,start_neurons * 1)
 uconv0 = LeakyReLU(alpha=0.1 )(uconv0)
 uconv0 = Dropout(dropout_rate/2 )(uconv0)
 output_layer = Conv2D(1,(1,1), padding="same", activation="sigmoid" )(uconv0)
 model = Model(input, output_layer)
 model.name = 'u-xception'
return model<choose_model_class>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
K.clear_session()
img_size = 256
model = UEfficientNet(input_shape=(img_size,img_size,3),dropout_rate=0.25 )<train_model>
train_ID = train['Id']
test_ID = test['Id']
train.drop(['Id'], axis=1, inplace=True)
 test.drop(['Id'], axis=1, inplace=True)
 numerical = [x for x in train.columns if train.dtypes[x] != 'object']
numerical.remove('SalePrice')
 categorical = [x for x in train.columns if train.dtypes[x] == 'object']
y = train.SalePrice.reset_index(drop=True)
 train_features = train.drop(['SalePrice'], axis=1)
 test_features = test
features = pd.concat([train_features, test_features] ).reset_index(drop=True)
 data_overview(features )
class SWA(keras.callbacks.Callback):
def __init__(self, filepath, swa_epoch):
super(SWA, self ).__init__()
self.filepath = filepath
self.swa_epoch = swa_epoch
def on_train_begin(self, logs=None):
self.nb_epoch = self.params['epochs']
print('Stochastic weight averaging selected for last {} epochs.' 
.format(self.nb_epoch - self.swa_epoch))
def on_epoch_end(self, epoch, logs=None):
if epoch == self.swa_epoch:
self.swa_weights = self.model.get_weights()
elif epoch > self.swa_epoch:
for i in range(len(self.swa_weights)) :
self.swa_weights[i] =(self.swa_weights[i] * 
(epoch - self.swa_epoch)+ self.model.get_weights() [i])/(( epoch - self.swa_epoch)+ 1)
 else:
pass
def on_train_end(self, logs=None):
self.model.set_weights(self.swa_weights)
 print('Final model parameters set to stochastic weight average.')
 self.model.save_weights(self.filepath)
 print('Final stochastic averaged weights saved to file.' )<choose_model_class>
train["SalePrice"] = np.log1p(train["SalePrice"])
 y = train.SalePrice.reset_index(drop=True )
model.compile(loss=bce_dice_loss, optimizer='adam', metrics=[my_iou_metric] )<choose_model_class>
corr_dict = train.corr() ['SalePrice'][(train.corr() ['SalePrice'] > 0.5)&(train.corr() ['SalePrice'] < 1.0)].sort_values(ascending=False ).to_dict()
print("Higher correlations between predictor variables and the response variable ")
 print("-------------------------------------------------------------------------")
 for x in corr_dict.keys() :
print("(" , round(corr_dict[x],3), ")",x, ":" , " -",desc[x] )
epochs = 70
snapshot = SnapshotCallbackBuilder(nb_epochs=epochs,nb_snapshots=1,init_lr=1e-3)
 batch_size = 16
swa = SWA('./keras_swa.model',67)
 valid_im_path,valid_mask_path = './keras_im_val','./keras_mask_val'
training_generator = DataGenerator(augmentations=AUGMENTATIONS_TRAIN,img_size=img_size)
 validation_generator = DataGenerator(train_im_path = valid_im_path ,
train_mask_path=valid_mask_path,augmentations=AUGMENTATIONS_TEST,
img_size=img_size)
 history = model.fit_generator(generator=training_generator,
validation_data=validation_generator,
use_multiprocessing=False,
epochs=epochs,verbose=2,
callbacks=snapshot.get_callbacks() )<load_pretrained>
features['MSSubClass'] = features['MSSubClass'].apply(str)
 features['YrSold'] = features['YrSold'].astype(str)
 features['MoSold'] = features['MoSold'].astype(str )
model.load_weights('./keras.model' )<predict_on_test>
for f in('MasVnrType','Alley','PoolQC', 'Fence', 'FireplaceQu', 'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2',):
features[f] = features[f].fillna('None' )
def predict_result(model,validation_generator,img_size):
preds_test1 = model.predict_generator(validation_generator ).reshape(-1, img_size, img_size)
 return preds_test1<predict_on_test>
features['MiscFeature'] = features['MiscFeature'].fillna('None' )
validation_generator = DataGenerator(train_im_path = valid_im_path ,
train_mask_path=valid_mask_path,augmentations=AUGMENTATIONS_TEST,
img_size=img_size,shuffle=False)
 preds_valid = predict_result(model,validation_generator,img_size )<train_model>
for f in('Exterior1st', 'Exterior2nd', 'SaleType','Utilities'):
features[f] = features[f].fillna(features[f].mode() [0])
 features['Functional'] = features['Functional'].fillna('Typ')
 features['Electrical'] = features['Electrical'].fillna("SBrkr")
 features['KitchenQual'] = features['KitchenQual'].fillna("TA")
 features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
objects = []
for i in features.columns:
if features[i].dtype == object:
objects.append(i)
 features.update(features[objects].fillna('None'))
valid_fn = glob.glob('./keras_mask_val/*')
 y_valid_ori = np.array([cv2.resize(np.array(Image.open(fn)) ,(img_size,img_size)) for fn in valid_fn])
 assert y_valid_ori.shape == preds_valid.shape<define_variables>
features[features['GarageYrBlt'] == 2207][['YearBuilt','YearRemodAdd','GarageYrBlt']]
threshold_best = 0.5
max_images = 64
grid_width = 16
grid_height = int(max_images / grid_width)
 fig, axs = plt.subplots(grid_height, grid_width, figsize=(grid_width, grid_height))
validation_generator = DataGenerator(train_im_path = valid_im_path ,
train_mask_path=valid_mask_path,augmentations=AUGMENTATIONS_TEST,
img_size=img_size,batch_size=64,shuffle=False)
 images,masks = validation_generator.__getitem__(0)
 for i,(im, mask)in enumerate(zip(images,masks)) :
pred = preds_valid[i]
ax = axs[int(i / grid_width), i % grid_width]
ax.imshow(im[...,0], cmap="bone")
 ax.imshow(mask.squeeze() , alpha=0.5, cmap="Reds")
 ax.imshow(np.array(np.round(pred > threshold_best), dtype=np.float32), alpha=0.5, cmap="Greens")
 ax.axis('off')
 plt.suptitle("Green:Prediction , Red: Pneumothorax." )<compute_train_metric>
features.GarageYrBlt.iloc[2592] = 2007
thresholds = np.linspace(0.2, 0.9, 31)
 ious = np.array([iou_metric_batch(y_valid_ori, np.int32(preds_valid > threshold)) for threshold in tqdm_notebook(thresholds)] )<define_variables>
for f in ['GarageYrBlt', 'GarageArea', 'GarageCars']:
features[f] = features[f].fillna(0 )
max_images = 64
grid_width = 16
grid_height = int(max_images / grid_width)
 fig, axs = plt.subplots(grid_height, grid_width, figsize=(grid_width, grid_height))
validation_generator = DataGenerator(train_im_path = valid_im_path ,
train_mask_path=valid_mask_path,augmentations=AUGMENTATIONS_TEST,
img_size=img_size,batch_size=64,shuffle=False)
 images,masks = validation_generator.__getitem__(0)
 for i,(im, mask)in enumerate(zip(images,masks)) :
pred = preds_valid[i]
ax = axs[int(i / grid_width), i % grid_width]
ax.imshow(im[...,0], cmap="bone")
 ax.imshow(mask.squeeze() , alpha=0.5, cmap="Reds")
 ax.imshow(np.array(np.round(pred > threshold_best), dtype=np.float32), alpha=0.5, cmap="Greens")
 ax.axis('off')
 plt.suptitle("Green:Prediction , Red: Pneumothorax." )<predict_on_test>
for f in ['BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath']:
features[f] = features[f].fillna(0 )
test_fn = glob.glob('./test/*')
 x_test = [cv2.resize(np.array(Image.open(fn)) ,(img_size,img_size)) for fn in test_fn]
x_test = np.array(x_test)
 x_test = np.array([np.repeat(im[...,None],3,2)for im in x_test])
 print(x_test.shape)
 preds_test = model.predict(x_test,batch_size=batch_size)
<categorify>
features['MasVnrArea'] = features['MasVnrArea'].fillna(0 )
sys.path.insert(0, '.. /input/siim-acr-pneumothorax-segmentation')
 rles = []
i,max_img = 1,10
plt.figure(figsize=(16,4))
for p in tqdm_notebook(preds_test):
p = p.squeeze()
im = cv2.resize(p,(1024,1024))
im = im > threshold_best
if im.sum() <1024*2:
im[:] = 0
im =(im.T*255 ).astype(np.uint8)
 rles.append(mask2rle(im, 1024, 1024))
i += 1
if i<max_img:
plt.subplot(1,max_img,i)
 plt.imshow(im)
 plt.axis('off' )<prepare_output>
features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].apply(lambda x: x.fillna(x.median()))
ids = [o.split('/')[-1][:-4] for o in test_fn]
sub_df = pd.DataFrame({'ImageId': ids, 'EncodedPixels': rles})
 sub_df.loc[sub_df.EncodedPixels=='', 'EncodedPixels'] = '-1'
sub_df.head()<save_to_csv>
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics2 = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numerics2.append(i)
 skew_features = features[numerics2].apply(lambda x: stats.skew(x)).sort_values(ascending=False)
 high_skew = skew_features[skew_features > 0.5]
skew_index = high_skew.index
for i in skew_index:
features[i] = boxcox1p(features[i], boxcox_normmax(features[i] + 1))
sub_df.to_csv('orig_submission.csv', index=False )<load_from_csv>
features['HasPool'] = features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 features['Has2ndfloor'] = features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 features['HasGarage'] = features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 features['HasBsmt'] = features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 features['HasFireplace'] = features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 features['YrBltAndRemod']=features['YearBuilt']+features['YearRemodAdd']
features['TotalSF']=features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']
features['Total_SQR_Footage'] =(features['BsmtFinSF1'] + features['BsmtFinSF2'] +
features['1stFlrSF'] + features['2ndFlrSF'])
 features['Total_Bathrooms'] =(features['FullBath'] +(0.5 * features['HalfBath'])+
features['BsmtFullBath'] +(0.5 * features['BsmtHalfBath']))
features['Total_Porch_SF'] =(features['OpenPorchSF'] + features['3SsnPorch'] +
features['EnclosedPorch'] + features['ScreenPorch'] +
features['WoodDeckSF'])
 features = features.drop(['Utilities', 'Street', 'PoolQC',], axis=1 )
sub_df = pd.read_csv('orig_submission.csv')
 leak_prob = pd.read_csv('.. /input/leak-probabilities-siim/leak_probabilities.csv')
 sub_df = sub_df.merge(leak_prob,on='ImageId')
 sub_df['pneumothorax'] = leak_prob.pred > 0.15
sub_df['EncodedPixels'] = np.where(sub_df.pneumothorax,'-1',sub_df.EncodedPixels )<save_to_csv>
final_features = pd.get_dummies(features ).reset_index(drop=True)
 print(final_features.shape )
sub_df.to_csv('leak_exploit.csv',columns=['ImageId','EncodedPixels'],index=False )<load_from_csv>
X = final_features.iloc[:len(y), :]
X_pred = final_features.iloc[len(X):, :]
X_train = X
X_test = X_pred
y_train = y
print('X', X.shape, 'y', y.shape, 'X_pred', X_pred.shape )
xf2 = pd.read_csv('.. /input/lungv1/leaky_unet_submission.csv')
 xf1 = pd.read_csv('.. /input/unet-with-efficientnet-encoder-in-keras/orig_submission.csv')
<merge>
def rmse_cv(model):
rmse = np.sqrt(-cross_val_score(model, X_train, y_train, scoring="neg_mean_squared_error", cv = 10))
return(rmse )
xf1.columns = ['ImageId', 'enc1']
xf2.columns = ['ImageId', 'enc2']
xf3 = pd.merge(left = xf1, right = xf2, on = 'ImageId', how = 'inner')
 print(xf1.shape, xf2.shape, xf3.shape)
 xf3[xf3['enc1'] != xf3['enc2']]
id1 = np.where(xf3['enc1'] == '-1')[0]
id2 = np.where(xf3['enc2'] != '-1')[0]
idx = np.intersect1d(id1,id2)
 xf3['EncodedPixels'] = xf3['enc1']
xf3['EncodedPixels'][idx] = xf3['enc2'][idx]<save_to_csv>
model_ridge = Ridge(alpha = 10 )
xf3[['ImageId','EncodedPixels']].to_csv('hybrid_1_2.csv', index = False )<set_options>
model_lasso = make_pipeline(RobustScaler() , Lasso(alpha = 0.0004))
%reload_ext autoreload
%autoreload 2
%matplotlib inline
sys.path.insert(0, '.. /input/siim-acr-pneumothorax-segmentation')
 fastai.__version__<define_search_space>
cv_ridge = rmse_cv(model_ridge ).mean()
cv_lasso = rmse_cv(model_lasso ).mean()
sz = 256
bs = 16
n_acc = 64//bs
nfolds = 4
SEED = 2019
noise_th = 75.0*(sz/128.0)**2
best_thr0 = 0.2
if sz == 256:
stats =([0.540,0.540,0.540],[0.264,0.264,0.264])
 TRAIN = '.. /input/siimacr-pneumothorax-segmentation-data-256/train'
TEST = '.. /input/siimacr-pneumothorax-segmentation-data-256/test'
MASKS = '.. /input/siimacr-pneumothorax-segmentation-data-256/masks'
elif sz == 128:
stats =([0.615,0.615,0.615],[0.291,0.291,0.291])
 TRAIN = '.. /input/siimacr-pneumothorax-segmentation-data-128/train'
TEST = '.. /input/siimacr-pneumothorax-segmentation-data-128/test'
MASKS = '.. /input/siimacr-pneumothorax-segmentation-data-128/masks'
Path('/tmp/.cache/torch/checkpoints/' ).mkdir(exist_ok=True, parents=True)
 !cp '.. /input/resnet34/resnet34.pth' '/tmp/.cache/torch/checkpoints/resnet34-333f7ec4.pth'
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(SEED )<choose_model_class>
results = pd.DataFrame({
'Model': ['Ridge','Lasso'],
'Score': [cv_ridge,cv_lasso]})
 result_df = results.sort_values(by='Score', ascending=True ).reset_index(drop=True)
 result_df.head()
class Hcolumns(nn.Module):
def __init__(self, hooks:Collection[Hook], nc:Collection[int]=None):
super(Hcolumns,self ).__init__()
self.hooks = hooks
self.n = len(self.hooks)
 self.factorization = None
if nc is not None:
self.factorization = nn.ModuleList()
for i in range(self.n):
self.factorization.append(nn.Sequential(
 conv2d(nc[i],nc[-1],3,padding=1,bias=True),
conv2d(nc[-1],nc[-1],3,padding=1,bias=True)))
 def forward(self, x:Tensor):
n = len(self.hooks)
 out = [F.interpolate(self.hooks[i].stored if self.factorization is None
else self.factorization[i](self.hooks[i].stored), scale_factor=2**(self.n-i),
mode='bilinear',align_corners=False)for i in range(self.n)] + [x]
return torch.cat(out, dim=1)
 class DynamicUnet_Hcolumns(SequentialEx):
"Create a U-Net from a given architecture."
def __init__(self, encoder:nn.Module, n_classes:int, blur:bool=False, blur_final=True,
self_attention:bool=False,
y_range:Optional[Tuple[float,float]]=None,
last_cross:bool=True, bottle:bool=False, **kwargs):
imsize =(256,256)
 sfs_szs = model_sizes(encoder, size=imsize)
 sfs_idxs = list(reversed(_get_sfs_idxs(sfs_szs)))
 self.sfs = hook_outputs([encoder[i] for i in sfs_idxs])
 x = dummy_eval(encoder, imsize ).detach()
ni = sfs_szs[-1][1]
middle_conv = nn.Sequential(conv_layer(ni, ni*2, **kwargs),
conv_layer(ni*2, ni, **kwargs)).eval()
x = middle_conv(x)
 layers = [encoder, batchnorm_2d(ni), nn.ReLU() , middle_conv]
self.hc_hooks = [Hook(layers[-1], _hook_inner, detach=False)]
hc_c = [x.shape[1]]
for i,idx in enumerate(sfs_idxs):
not_final = i!=len(sfs_idxs)-1
up_in_c, x_in_c = int(x.shape[1]), int(sfs_szs[idx][1])
 do_blur = blur and(not_final or blur_final)
 sa = self_attention and(i==len(sfs_idxs)-3)
 unet_block = UnetBlock(up_in_c, x_in_c, self.sfs[i], final_div=not_final,
blur=blur, self_attention=sa, **kwargs ).eval()
layers.append(unet_block)
 x = unet_block(x)
 self.hc_hooks.append(Hook(layers[-1], _hook_inner, detach=False))
hc_c.append(x.shape[1])
 ni = x.shape[1]
if imsize != sfs_szs[0][-2:]: layers.append(PixelShuffle_ICNR(ni, **kwargs))
if last_cross:
layers.append(MergeLayer(dense=True))
ni += in_channels(encoder)
 layers.append(res_block(ni, bottle=bottle, **kwargs))
hc_c.append(ni)
 layers.append(Hcolumns(self.hc_hooks, hc_c))
layers += [conv_layer(ni*len(hc_c), n_classes, ks=1, use_activ=False, **kwargs)]
if y_range is not None: layers.append(SigmoidRange(*y_range))
super().__init__(*layers)
 def __del__(self):
if hasattr(self, "sfs"): self.sfs.remove()
def unet_learner(data:DataBunch, arch:Callable, pretrained:bool=True, blur_final:bool=True,
norm_type:Optional[NormType]=NormType, split_on:Optional[SplitFuncOrIdxList]=None,
blur:bool=False, self_attention:bool=False, y_range:Optional[Tuple[float,float]]=None,
last_cross:bool=True, bottle:bool=False, cut:Union[int,Callable]=None,
hypercolumns=True, **learn_kwargs:Any)->Learner:
"Build Unet learner from `data` and `arch`."
meta = cnn_config(arch)
 body = create_body(arch, pretrained, cut)
 M = DynamicUnet_Hcolumns if hypercolumns else DynamicUnet
model = to_device(M(body, n_classes=data.c, blur=blur, blur_final=blur_final,
self_attention=self_attention, y_range=y_range, norm_type=norm_type,
last_cross=last_cross, bottle=bottle), data.device)
 learn = Learner(data, model, **learn_kwargs)
 learn.split(ifnone(split_on, meta['split']))
if pretrained: learn.freeze()
apply_init(model[2], nn.init.kaiming_normal_)
 return learn<train_model>
model_lasso.fit(X_train, y_train)
 lasso_pred = np.expm1(model_lasso.predict(X_test))
model_ridge.fit(X_train, y_train)
 ridge_pred = np.expm1(model_ridge.predict(X_test))
class AccumulateOptimWrapper(OptimWrapper):
def step(self): pass
def zero_grad(self): pass
def real_step(self): super().step()
def real_zero_grad(self): super().zero_grad()
def acc_create_opt(self, lr:Floats, wd:Floats=0.) :
"Create optimizer with `lr` learning rate and `wd` weight decay."
self.opt = AccumulateOptimWrapper.create(self.opt_func, lr, self.layer_groups,
wd=wd, true_wd=self.true_wd, bn_wd=self.bn_wd)
 Learner.create_opt = acc_create_opt
@dataclass
class AccumulateStep(LearnerCallback):

 def __init__(self, learn:Learner, n_step:int = 1):
super().__init__(learn)
 self.n_step = n_step
def on_epoch_begin(self, **kwargs):
"init samples and batches, change optimizer"
self.acc_batches = 0
def on_batch_begin(self, last_input, last_target, **kwargs):
"accumulate samples and batches"
self.acc_batches += 1
def on_backward_end(self, **kwargs):
"step if number of desired batches accumulated, reset samples"
if(self.acc_batches % self.n_step)== self.n_step - 1:
for p in(self.learn.model.parameters()):
if p.requires_grad: p.grad.div_(self.acc_batches)
 self.learn.opt.real_step()
self.learn.opt.real_zero_grad()
self.acc_batches = 0
def on_epoch_end(self, **kwargs):
"step the rest of the accumulated grads"
if self.acc_batches > 0:
for p in(self.learn.model.parameters()):
if p.requires_grad: p.grad.div_(self.acc_batches)
 self.learn.opt.real_step()
self.learn.opt.real_zero_grad()
self.acc_batches = 0<init_hyperparams>
Stack =(lasso_pred + ridge_pred)/ 2
<feature_engineering><EOS> 
submission = pd.DataFrame({'Id':test_ID, 'SalePrice':lasso_pred})
 submission.to_csv('Stack.csv', index=False )