Kaggle competition: House Prices - Advanced Regression Techniques
submission.loc[submission['Image_Label'].isin(image_labels_empty), 'EncodedPixels'] = np.nan
submission.to_csv('submission_segmentation_and_classifier.csv', index=None )<set_options>
warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 10000)
 pd.set_option('display.max_rows', 10000 )
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def rle2mask(mask_rle, shape=(2100, 1400)) :
img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 if mask_rle != None and type(mask_rle)is str:
s = mask_rle.split()
starts, lengths = [np.asarray(x, dtype=int)for x in(s[0:][::2], s[1:][::2])]
starts -= 1
ends = starts + lengths
for lo, hi in zip(starts, ends):
img[lo:hi] = 1
return img.reshape(shape ).T
def normalize(images):
return images/128-1
def denormalize(images):
return(( images+1)*128 ).astype('uint8')
 def load_image(Image):
path = TEST_PATH + Image
img = cv2.imread(path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 if(img.shape !=(height, width, 3)) :
img = cv2.resize(img,(width, height))
return img
def resizeMask(mask, w, h):
resmask = np.zeros(( h, w, mask.shape[2]))
for i in range(mask.shape[2]):
resmask[...,i] = cv2.resize(mask[...,i],(w,h))
return resmask 
<load_from_csv>
df_train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 df_test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )
TEST_PATH = '.. /input/understanding_cloud_organization/test_images/'
test_df = pd.read_csv('.. /input/understanding_cloud_organization/sample_submission.csv')
 test_df['Label'] = test_df['Image_Label'].str.split("_", n = 1, expand = True)[1]
test_df['Image'] = test_df['Image_Label'].str.split("_", n = 1, expand = True)[0]
types = ['Fish', 'Flower', 'Gravel', 'Sugar']
pixel_thresholds = [0.5, 0.5, 0.5, 0.5 ]
mask_sum_threshold = [10000, 10000, 10000, 9000]
mask_threshold=[1000, 1000, 1000, 1000]
def mask_reduce(mask):
reduced_mask = np.zeros(mask.shape,np.float32)
 for idx in range(mask.shape[2]):
label_num, labeled_mask = cv2.connectedComponents(mask[:,:, idx].astype(np.uint8))
for label in range(1, label_num):
single_label_mask =(labeled_mask == label)
 if single_label_mask.sum() > mask_threshold[idx]:
reduced_mask[single_label_mask, idx] = 1
return reduced_mask.astype('uint8')
 def mask_filter(mask):
lim = np.sum(mask, axis=(0,1)) < mask_sum_threshold
for i in range(len(lim)) :
if lim[i]: mask[..., i] = 0
return mask
def cleanup(pred):
return(pred>pixel_thresholds ).astype('uint8')
 test_df.head()<load_pretrained>
y = df_train[['Id','SalePrice']]
df_train = df_train.drop('SalePrice',axis=1 )
path = '.. /input/single-models/'
models = []
model1 = sm.FPN('resnet34', encoder_weights=None, classes=4, input_shape=(None, None, 3), activation='sigmoid')
 model1.load_weights(path+'Deotte-NormalBCEJaccard-FPN-Resnet34-val_loss-256.h5')
 models.append({"model": model1, 'weight': 1})
 model2 = sm.Unet('efficientnetb0', encoder_weights=None, classes=4, input_shape=(None, None, 3), activation='sigmoid')
 model2.load_weights(path+'Deotte-NormalJackardBCE-NoPseudo-K0-256-ThisisGood-0.6483.h5')
 models.append({"model": model2, 'weight': 1})
 model3 = sm.FPN('efficientnetb0', encoder_weights=None, classes=4, input_shape=(None, None, 3), activation='sigmoid')
 model3.load_weights(path+'Deotte-NormalBCEJaccard-FPN-val_loss-256.h5')
 models.append({"model": model3, 'weight': 1})
 model4 = sm.Unet('efficientnetb0', encoder_weights=None, classes=4, input_shape=(None, None, 3), activation='sigmoid')
 model4.load_weights(path+'Deotte-perImageBCEJackard_real-noPseudo--256-0.6410.h5')
 models.append({"model": model4, 'weight': 1})
 model5 = sm.FPN('efficientnetb0', encoder_weights=None, classes=4, input_shape=(None, None, 3), activation='sigmoid')
 model5.load_weights(path+'PerImageBCEJaccard-FPN-pseudo-256.h5')
 models.append({"model": model5, 'weight': 1})
 model6 = sm.Unet('efficientnetb0', encoder_weights=None, classes=4, input_shape=(None, None, 3), activation='sigmoid')
 model6.load_weights(path+'NormalBCEJackard-Unet-Effnet-pseudo-256.h5')
 models.append({"model": model6, 'weight': 1})
 model7 = sm.FPN('resnet34', encoder_weights=None, classes=4, input_shape=(None, None, 3), activation='sigmoid')
 model7.load_weights(path+'NormalBCEJaccard-FPN-Resnet34-pseudo-256.h5')
 models.append({"model": model7, 'weight': 1})
 model8 = sm.Unet('efficientnetb0', encoder_weights=None, classes=4, input_shape=(None, None, 3), activation='sigmoid')
 model8.load_weights(path+'NormalBCEJackard-Unet-Effnet-pseudo-K1-256.h5')
 models.append({"model": model8, 'weight': 1})
 model9 = sm.Unet('resnet34', encoder_weights=None, classes=4, input_shape=(None, None, 3), activation='sigmoid')
 model9.load_weights(path+'NormalBCEJackard-Unet-Resnet34-pseudo-K1-256.h5')
 models.append({"model": model9, 'weight': 1} )<choose_model_class>
concat_df = [df_train,df_test]
df_all = pd.concat(concat_df ).reset_index(drop=True )
def getClassifier(name):
base = efn.EfficientNetB3(weights=None, include_top=False, input_shape=(None, None, 3), pooling='avg')
 base.trainable=True
dropout_dense_layer = 0.3
classifier_model = Sequential()
classifier_model.add(base)
 classifier_model.add(Dropout(dropout_dense_layer))
classifier_model.add(Dense(4, activation='sigmoid'))
classifier_model.compile(
 loss='binary_crossentropy',
optimizer=Adam() ,
metrics=['accuracy'] 
)
 classifier_model.summary()
classifier_model.load_weights(path+name)
 return classifier_model
classifier_models = []
classifier_models.append(getClassifier('classifierB3-256.h5'))
classifier_models.append(getClassifier('classifierB3-blackout00-smooth0-256.h5'))
classifier_models.append(getClassifier('classifierB3-blackout04-256.h5')) 
<categorify>
df_dummy = pd.get_dummies(df_all['MSZoning'],prefix='MSZoning',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['MSZoning']
ids = test_df['Image'].unique()
test_df.EncodedPixels = ''
height = 256
width = int(height * 1.5)
 class_thresholds = [0.5, 0.5, 0.5, 0.5]
for picIdx in range(len(ids)) :
filename = ids[picIdx]
img = load_image(filename)
 if picIdx % 100 == 0: print(picIdx)
 batch = np.zeros(( 4, height, width, 3))
batch[0] = img
batch[1] = img[ :, ::-1, :]
batch[2] = img[ ::-1, :, :]
batch[3] = img[ ::-1, ::-1, :]
batch = normalize(batch)
 predTTA = np.zeros(( batch.shape[0], img.shape[0], img.shape[1], 4))
for j in range(len(models)) :
predTTA += models[j]['model'].predict(batch)
 predTTA /= len(models)
 pred =(predTTA[0, :, :, :]+predTTA[1, :, ::-1, :]+predTTA[2, ::-1, :, :]+predTTA[3, ::-1, ::-1, :])/4.0
if len(classifier_models)>0:
classpred = np.zeros(( batch.shape[0], 4))
for j in range(len(classifier_models)) :
classpred += classifier_models[j].predict(batch)
 classpred /= len(classifier_models)
 classpred = np.mean(classpred, axis=0)
 if np.sum(classpred>class_thresholds)== 0:
classpred[np.argmax(classpred)]=1
pred = pred *(classpred>class_thresholds)
 pred = cleanup(pred)
 pred = mask_reduce(pred)
 pred = mask_filter(pred)
 pred = resizeMask(pred, 525, 350)
 for myType in types:
name = filename+"_"+myType
line = test_df[test_df.Image_Label == name].index[0]
i=types.index(myType)
 maskrle = mask2rle(pred[..., i])
 test_df.loc[line, 'EncodedPixels'] = maskrle
sub = test_df[['Image_Label', 'EncodedPixels']]
sub.to_csv('submission.csv', index=False)
 sub.head(30 )<count_values>
df_all['LotFrontage'] = df_all['LotFrontage'].fillna(df_all['LotFrontage'].mean() )
sub['Label'] = sub['Image_Label'].str.split("_", n = 1, expand = True)[1]
sub['Image'] = sub['Image_Label'].str.split("_", n = 1, expand = True)[0]
print(sub[(sub.Label == 'Fish')&(sub.EncodedPixels != '')]['Image'].count())
 print(sub[(sub.Label == 'Sugar')&(sub.EncodedPixels != '')]['Image'].count())
 print(sub[(sub.Label == 'Gravel')&(sub.EncodedPixels != '')]['Image'].count())
 print(sub[(sub.Label == 'Flower')&(sub.EncodedPixels != '')]['Image'].count() )<define_variables>
def LotFrontage(n):
if(n > 21)&(n < 50):
return 1
elif(n > 50)&(n <= 70):
return 2
elif(n > 70)&(n <= 75):
return 3
elif(n > 75)&(n <= 80):
return 4
elif(n > 80)&(n <= 90):
return 5
elif(n > 100)&(n <= 120):
return 6
elif(n > 120)&(n <= 140):
return 7
elif(n > 140)&(n <= 160):
return 8
elif(n > 160)&(n <= 180):
return 9
elif(n > 180)&(n <= 200):
return 9
else:
return 10
path = '.. /input/understanding_cloud_organization' 
<train_model>
df_all['LotFrontage'] = df_all['LotFrontage'].apply(LotFrontage )
Train_Dir = '/kaggle/input/understanding_cloud_organization/train_images'
Test_Dir = 'kaggle/input/understanding_cloud_organization/test_images'
for img in tqdm(os.listdir(Train_Dir)) :
imgr = cv2.imread(os.path.join(Train_Dir,img)) 
<load_from_csv>
df_dummy = pd.get_dummies(df_all['Neighborhood'],prefix='Neighborhood',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['Neighborhood']
train_csv_folder = '/kaggle/input/understanding_cloud_organization/train.csv'
train_csv = pd.read_csv(train_csv_folder)
<feature_engineering>
df_all['Exterior2nd'] = df_all['Exterior2nd'].fillna('Other')
 df_dummy = pd.get_dummies(df_all['Exterior2nd'],prefix='Exterior2nd',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['Exterior2nd']
train_csv['ImageId'] = train_csv['Image_Label'].apply(lambda x: x.split('_')[0])
 train_csv['ClassId'] = train_csv['Image_Label'].apply(lambda x: x.split('_')[1])
 train_csv['hasmask'] = ~train_csv['EncodedPixels'].isna() 
<groupby>
df_dummy = pd.get_dummies(df_all['Utilities'],prefix='Utilities',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['Utilities']
mask_count_df = train_csv.groupby('ImageId' ).agg(np.sum ).reset_index()
mask_count_df.sort_values('hasmask', ascending=False, inplace=True)
 print(mask_count_df.shape)
 mask_count_df.head(10 )<categorify>
df_dummy = pd.get_dummies(df_all['LandContour'],prefix='LandContour',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['LandContour']
def rle_decode(mask_rle: str = '', shape: tuple =(1400, 2100)) :

 s = mask_rle.split()
starts, lengths = [np.asarray(x, dtype=int)for x in(s[0:][::2], s[1:][::2])]
starts -= 1
ends = starts + lengths
img = np.zeros(shape[0] * shape[1], dtype=np.uint8)
 for lo, hi in zip(starts, ends):
img[lo:hi] = 1
return img.reshape(shape, order='F' )<categorify>
df_dummy = pd.get_dummies(df_all['Condition1'],prefix='Condition1',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['Condition1']
def np_resize(img, input_shape):

 height, width = input_shape
return cv2.resize(img,(width, height))
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def rle2mask(rle, input_shape):
width, height = input_shape[:2]
mask= np.zeros(width*height ).astype(np.uint8)
 array = np.asarray([int(x)for x in rle.split() ])
 starts = array[0::2]
lengths = array[1::2]
current_position = 0
for index, start in enumerate(starts):
mask[int(start):int(start+lengths[index])] = 1
current_position += lengths[index]
return mask.reshape(height, width ).T
def build_masks(rles, input_shape, reshape=None):
depth = len(rles)
 if reshape is None:
masks = np.zeros(( *input_shape, depth))
else:
masks = np.zeros(( *reshape, depth))
for i, rle in enumerate(rles):
if type(rle)is str:
if reshape is None:
masks[:, :, i] = rle2mask(rle, input_shape)
 else:
mask = rle2mask(rle, input_shape)
 reshaped_mask = np_resize(mask, reshape)
 masks[:, :, i] = reshaped_mask
return masks
def build_rles(masks, reshape=None):
width, height, depth = masks.shape
rles = []
for i in range(depth):
mask = masks[:, :, i]
if reshape:
mask = mask.astype(np.float32)
 mask = np_resize(mask, reshape ).astype(np.int64)
 rle = mask2rle(mask)
 rles.append(rle)
 return rles<train_model>
def Condition2(c):
if c == 'Norm':
return 1
elif c == 'Feedr':
return 2
elif c == 'PosA':
return 3
elif c == 'PosN':
return 4
elif c == 'Artery':
return 5
else:
return 0
class DataGenerator(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, df, target_df=None, mode='fit',
base_path='/kaggle/input/understanding_cloud_organization/train_images',
batch_size=32, dim=(1400, 2100), n_channels=3, reshape=None,
augment=False, n_classes=4, random_state=2019, shuffle=True):
self.dim = dim
self.batch_size = batch_size
self.df = df
self.mode = mode
self.base_path = base_path
self.target_df = target_df
self.list_IDs = list_IDs
self.reshape = reshape
self.n_channels = n_channels
self.augment = augment
self.n_classes = n_classes
self.shuffle = shuffle
self.random_state = random_state
self.on_epoch_end()
np.random.seed(self.random_state)
 def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
list_IDs_batch = [self.list_IDs[k] for k in indexes]
X = self.__generate_X(list_IDs_batch)
 if self.mode == 'fit':
y = self.__generate_y(list_IDs_batch)
 if self.augment:
X, y = self.__augment_batch(X, y)
 print(X,y)
 return X, y
elif self.mode == 'predict':
return X
else:
raise AttributeError('The mode parameter should be set to "fit" or "predict".')
 def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.seed(self.random_state)
 np.random.shuffle(self.indexes)
 def __generate_X(self, list_IDs_batch):
'Generates data containing batch_size samples'
if self.reshape is None:
X = np.empty(( self.batch_size, *self.dim, self.n_channels))
else:
X = np.empty(( self.batch_size, *self.reshape, self.n_channels))
for i, ID in enumerate(list_IDs_batch):
im_name = self.df['ImageId'].iloc[ID]
img_path = f"{self.base_path}/{im_name}"
img = self.__load_rgb(img_path)
 if self.reshape is not None:
img = np_resize(img, self.reshape)
 X[i,] = img
return X
def __generate_y(self, list_IDs_batch):
if self.reshape is None:
y = np.empty(( self.batch_size, *self.dim, self.n_classes), dtype=int)
 else:
y = np.empty(( self.batch_size, *self.reshape, self.n_classes), dtype=int)
 for i, ID in enumerate(list_IDs_batch):
im_name = self.df['ImageId'].iloc[ID]
image_df = self.target_df[self.target_df['ImageId'] == im_name]
rles = image_df['EncodedPixels'].values
if self.reshape is not None:
masks = build_masks(rles, input_shape=self.dim, reshape=self.reshape)
 else:
masks = build_masks(rles, input_shape=self.dim)
 y[i, ] = masks
return y
def __load_grayscale(self, img_path):
img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)
 img = img.astype(np.float32)/ 255.
 img = np.expand_dims(img, axis=-1)
 return img
def __load_rgb(self, img_path):
img = cv2.imread(img_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img = img.astype(np.float32)/ 255.
 return img
def __random_transform(self, img, masks):
composition = albu.Compose([
albu.HorizontalFlip() ,
albu.VerticalFlip() ,
albu.ShiftScaleRotate(rotate_limit=45, shift_limit=0.15, scale_limit=0.15)
 ])
 composed = composition(image=img, mask=masks)
 aug_img = composed['image']
aug_masks = composed['mask']
return aug_img, aug_masks
def __augment_batch(self, img_batch, masks_batch):
for i in range(img_batch.shape[0]):
img_batch[i, ], masks_batch[i, ] = self.__random_transform(
 img_batch[i, ], masks_batch[i, ])
 return img_batch, masks_batch 
<split>
df_all['Condition2'] = df_all['Condition2'].apply(Condition2 ).astype(int )
BATCH_SIZE = 4
train_idx, val_idx = train_test_split(
 mask_count_df.index, random_state=2019, test_size=0.2 
)
 train_generator = DataGenerator(
 train_idx,
df=mask_count_df,
target_df=train_csv,
batch_size=BATCH_SIZE,
reshape=(320, 480),
augment=True,
n_channels=3,
n_classes=4 
)
 val_generator = DataGenerator(
 val_idx,
df=mask_count_df,
target_df=train_csv,
batch_size=BATCH_SIZE,
reshape=(320, 480),
augment=False,
n_channels=3,
n_classes=4
)<install_modules>
df_dummy = pd.get_dummies(df_all['BldgType'],prefix='BldgType',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['BldgType']
!pip install -U git+https://github.com/qubvel/efficientnet<install_modules>
df_dummy = pd.get_dummies(df_all['HouseStyle'],prefix='HouseStyle',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['HouseStyle']
!pip install segmentation-models --quiet<import_modules>
df_dummy = pd.get_dummies(df_all['RoofMatl'],prefix='RoofMatl',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['RoofMatl']
from keras.layers import Dense
from keras.models import Model
from keras.optimizers import Adam, Nadam
from keras.callbacks import Callback, ModelCheckpoint
from keras.losses import binary_crossentropy
import albumentations as albu
import segmentation_models as sm<compute_test_metric>
df_all['Exterior1st']= df_all['Exterior1st'].fillna('Other' )
def dice_coef(y_true, y_pred, smooth=1):
y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = K.sum(y_true_f * y_pred_f)
 return(2.* intersection + smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 def dice_loss(y_true, y_pred):
smooth = 1.
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = y_true_f * y_pred_f
score =(2.* K.sum(intersection)+ smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 return 1.- score
def bce_dice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)+ dice_loss(y_true, y_pred )<choose_model_class>
def Exterior1st(c):
if c == 'VinylSd':
return 1
elif c == 'Wd Sdng':
return 2
elif c == 'HdBoard':
return 3
elif c == 'Plywood':
return 4
elif c == 'MetalSd':
return 5
elif c == 'CemntBd':
return 6
elif c == 'WdShing':
return 7
elif c == 'BrkFace':
return 8
elif c == 'AsbShng':
return 9
else:
return 0
model = sm.Unet(
 'resnet18',
classes=4,
input_shape=(320, 480, 3),
activation='sigmoid' 
)
 model.compile(optimizer=Nadam(lr=0.0002), loss=bce_dice_loss, metrics=[dice_coef])
 model.summary()<load_from_csv>
df_all['Exterior1st'] = df_all['Exterior1st'].apply(Exterior1st ).astype(int )
sub_df = pd.read_csv('/kaggle/input/understanding_cloud_organization/sample_submission.csv')
 sub_df['ImageId'] = sub_df['Image_Label'].apply(lambda x: x.split('_')[0])
 test_imgs = pd.DataFrame(sub_df['ImageId'].unique() , columns=['ImageId'] )<define_variables>
def BsmtFinSF1(n):
if(n > -1)&(n <= 150):
return 1
elif(n > 150)&(n <= 300):
return 2
elif(n > 300)&(n <= 450):
return 3
elif(n > 450)&(n <= 600):
return 4
elif(n > 600)&(n <= 750):
return 5
else:
return 10
resnetpath = '/kaggle/input/resnet18/modelefficient.h5'
model.load_weights(resnetpath)
 test_df = []
for i in range(0, test_imgs.shape[0], 500):
batch_idx = list(
 range(i, min(test_imgs.shape[0], i + 500)) 
)
 test_generator = DataGenerator(
 batch_idx,
df=test_imgs,
shuffle=False,
mode='predict',
dim=(350, 525),
reshape=(320, 480),
n_channels=3,
base_path='/kaggle/input/understanding_cloud_organization/test_images',
target_df=sub_df,
batch_size=1,
n_classes=4 
)
 batch_pred_masks = model.predict_generator(
 test_generator,
workers=1,
verbose=1 
)
 for j, b in enumerate(batch_idx):
filename = test_imgs['ImageId'].iloc[b]
image_df = sub_df[sub_df['ImageId'] == filename].copy()
pred_masks = batch_pred_masks[j, ].round().astype(int)
 pred_rles = build_rles(pred_masks, reshape=(350, 525))
image_df['EncodedPixels'] = pred_rles
test_df.append(image_df )<load_from_csv>
df_all['BsmtFinSF1'] = df_all['BsmtFinSF1'].apply(BsmtFinSF1 ).astype(int )
os.listdir('/kaggle/input/csvfiledl/SubmissionDLearningFinal.csv' )<load_from_csv>
def MasVnrArea(n):
if(n > -1)&(n <= 150):
return 1
elif(n > 150)&(n <= 200):
return 2
elif(n > 200)&(n <= 300):
return 3
else:
return 0
df = pd.read_csv('/kaggle/input/csvfiledl/SubmissionDLearningFinal.csv' )<save_to_csv>
df_all['MasVnrArea'] = df_all['MasVnrArea'].apply(MasVnrArea ).astype(int )
df.to_csv('FinaldeepLearning.csv',index = False )<set_options>
df_dummy = pd.get_dummies(df_all['ExterQual'],prefix='ExterQual',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['ExterQual']
seed(10)
 set_random_seed(10)
 %matplotlib inline<install_modules>
df_dummy = pd.get_dummies(df_all['ExterCond'],prefix='ExterCond',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['ExterCond']
!pip install keras-rectified-adam<define_variables>
df_all['BsmtQual'] = df_all['BsmtQual'].fillna('TA')
 df_dummy = pd.get_dummies(df_all['BsmtQual'],prefix='BsmtQual',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['BsmtQual']
test_imgs_folder = '.. /input/understanding_cloud_organization/test_images/'
train_imgs_folder = '.. /input/understanding_cloud_organization/train_images/'
num_cores = multiprocessing.cpu_count()<load_from_csv>
df_all['BsmtCond'] = df_all['BsmtCond'].fillna('TA')
 df_dummy = pd.get_dummies(df_all['BsmtCond'],prefix='BsmtCond',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['BsmtCond']
train_df = pd.read_csv('.. /input/understanding_cloud_organization/train.csv')
 train_df.head()<feature_engineering>
df_all['BsmtExposure'] = df_all['BsmtExposure'].fillna('TA')
 df_dummy = pd.get_dummies(df_all['BsmtExposure'],prefix='BsmtExposure',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['BsmtExposure']
train_df = train_df[~train_df['EncodedPixels'].isnull() ]
train_df['Image'] = train_df['Image_Label'].map(lambda x: x.split('_')[0])
 train_df['Class'] = train_df['Image_Label'].map(lambda x: x.split('_')[1])
 classes = train_df['Class'].unique()
train_df = train_df.groupby('Image')['Class'].agg(set ).reset_index()
for class_name in classes:
train_df[class_name] = train_df['Class'].map(lambda x: 1 if class_name in x else 0)
 train_df.head()<define_variables>
def Functional(f):
if f == 'Typ':
return 1
elif f == 'Min1':
return 2
elif f == 'Min2':
return 3
elif f == 'Mod':
return 4
elif f == 'Maj1':
return 5
elif f == 'Maj2':
return 6
elif f == 'Sev':
return 7
elif f == 'Sal':
return 8
else:
return 0
img_2_ohe_vector = {img:vec for img, vec in zip(train_df['Image'], train_df.iloc[:, 2:].values)}<split>
df_all['Functional'] = df_all['Functional'].apply(Functional ).astype(int )
train_imgs, val_imgs = train_test_split(train_df['Image'].values,
test_size=0.17,
stratify=train_df['Class'].map(lambda x: str(sorted(list(x)))) ,
random_state=2019 )<define_variables>
df_dummy = pd.get_dummies(df_all['GarageType'],prefix='GarageType',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['GarageType']
class DataGenenerator(Sequence):
def __init__(self, images_list=None, folder_imgs=train_imgs_folder,
batch_size=16, shuffle=True, augmentation=None,
resized_height=256, resized_width=256, num_channels=3):
self.batch_size = batch_size
self.shuffle = shuffle
self.augmentation = augmentation
if images_list is None:
self.images_list = os.listdir(folder_imgs)
 else:
self.images_list = deepcopy(images_list)
 self.folder_imgs = folder_imgs
self.len = len(self.images_list)// self.batch_size
self.resized_height = resized_height
self.resized_width = resized_width
self.num_channels = num_channels
self.num_classes = 4
self.is_test = not 'train' in folder_imgs
if not shuffle and not self.is_test:
self.labels = [img_2_ohe_vector[img] for img in self.images_list[:self.len*self.batch_size]]
def __len__(self):
return self.len
def on_epoch_start(self):
if self.shuffle:
random.shuffle(self.images_list)
 def __getitem__(self, idx):
current_batch = self.images_list[idx * self.batch_size:(idx + 1)* self.batch_size]
X = np.empty(( self.batch_size, self.resized_height, self.resized_width, self.num_channels))
y = np.empty(( self.batch_size, self.num_classes))
for i, image_name in enumerate(current_batch):
path = os.path.join(self.folder_imgs, image_name)
 img = cv2.resize(cv2.imread(path),(self.resized_height, self.resized_width)).astype(np.float32)
 if not self.augmentation is None:
augmented = self.augmentation(image=img)
 img = augmented['image']
X[i, :, :, :] = img/255.0
if not self.is_test:
y[i, :] = img_2_ohe_vector[image_name]
return X, y
def get_labels(self):
if self.shuffle:
images_current = self.images_list[:self.len*self.batch_size]
labels = [img_2_ohe_vector[img] for img in images_current]
else:
labels = self.labels
return np.array(labels )<concatenate>
df_dummy = pd.get_dummies(df_all['SaleType'],prefix='SaleType',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['SaleType']
albumentations_train = Compose([
VerticalFlip() , HorizontalFlip() , Rotate(limit=10), GridDistortion()
], p=1 )<choose_model_class>
df_dummy = pd.get_dummies(df_all['SaleCondition'],prefix='SaleCondition',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['SaleCondition']
data_generator_train = DataGenenerator(train_imgs, augmentation=albumentations_train)
 data_generator_train_eval = DataGenenerator(train_imgs, shuffle=False)
 data_generator_val = DataGenenerator(val_imgs, shuffle=False )<load_pretrained>
df_dummy = pd.get_dummies(df_all['Heating'],prefix='Heating',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['Heating']
class PrAucCallback(Callback):
def __init__(self, data_generator, num_workers=num_cores,
early_stopping_patience=5,
plateau_patience=3, reduction_rate=0.5,
stage='train', checkpoints_path='checkpoints/'):
super(Callback, self ).__init__()
self.data_generator = data_generator
self.num_workers = num_workers
self.class_names = ['Fish', 'Flower', 'Sugar', 'Gravel']
self.history = [[] for _ in range(len(self.class_names)+ 1)]
self.early_stopping_patience = early_stopping_patience
self.plateau_patience = plateau_patience
self.reduction_rate = reduction_rate
self.stage = stage
self.best_pr_auc = -float('inf')
 if not os.path.exists(checkpoints_path):
os.makedirs(checkpoints_path)
 self.checkpoints_path = checkpoints_path
def compute_pr_auc(self, y_true, y_pred):
pr_auc_mean = 0
print(f"
{'
")
 for class_i in range(len(self.class_names)) :
precision, recall, _ = precision_recall_curve(y_true[:, class_i], y_pred[:, class_i])
 pr_auc = auc(recall, precision)
 pr_auc_mean += pr_auc/len(self.class_names)
 print(f"PR AUC {self.class_names[class_i]}, {self.stage}: {pr_auc:.3f}
")
 self.history[class_i].append(pr_auc)
 print(f"
{'
PR AUC mean, {self.stage}: {pr_auc_mean:.3f}
{'
")
 self.history[-1].append(pr_auc_mean)
 return pr_auc_mean
def is_patience_lost(self, patience):
if len(self.history[-1])> patience:
best_performance = max(self.history[-1][-(patience + 1):-1])
 return best_performance == self.history[-1][-(patience + 1)] and best_performance >= self.history[-1][-1]
def early_stopping_check(self, pr_auc_mean):
if self.is_patience_lost(self.early_stopping_patience):
self.model.stop_training = True
def model_checkpoint(self, pr_auc_mean, epoch):
if pr_auc_mean > self.best_pr_auc:
for checkpoint in glob.glob(os.path.join(self.checkpoints_path, 'classifier_densenet169_epoch_*')) :
os.remove(checkpoint)
 self.best_pr_auc = pr_auc_mean
self.model.save(os.path.join(self.checkpoints_path, f'classifier_densenet169_epoch_{epoch}_val_pr_auc_{pr_auc_mean}.h5'))
print(f"
{'
Saved new checkpoint
{'
")
 def reduce_lr_on_plateau(self):
if self.is_patience_lost(self.plateau_patience):
new_lr = float(keras.backend.get_value(self.model.optimizer.lr)) * self.reduction_rate
keras.backend.set_value(self.model.optimizer.lr, new_lr)
 print(f"
{'
Reduced learning rate to {new_lr}.
 {'
")
 def on_epoch_end(self, epoch, logs={}):
y_pred = self.model.predict_generator(self.data_generator, workers=self.num_workers)
 y_true = self.data_generator.get_labels()
pr_auc_mean = self.compute_pr_auc(y_true, y_pred)
 if self.stage == 'val':
self.early_stopping_check(pr_auc_mean)
 self.model_checkpoint(pr_auc_mean, epoch)
 self.reduce_lr_on_plateau()
def get_pr_auc_history(self):
return self.history<groupby>
df_dummy = pd.get_dummies(df_all['KitchenQual'],prefix='KitchenQual',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['KitchenQual']
train_metric_callback = PrAucCallback(data_generator_train_eval)
 val_callback = PrAucCallback(data_generator_val, stage='val' )<compute_test_metric>
df_all['GarageQual'] = df_all['GarageQual'].fillna('TA')
 df_dummy = pd.get_dummies(df_all['GarageQual'],prefix='GarageQual',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['GarageQual']
def dice_coef(y_true, y_pred, smooth=1):
y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = K.sum(y_true_f * y_pred_f)
 return(2.* intersection + smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 def dice_loss(y_true, y_pred):
smooth = 1.
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = y_true_f * y_pred_f
score =(2.* K.sum(intersection)+ smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 return 1.- score
def bce_dice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)+ dice_loss(y_true, y_pred )<install_modules>
df_dummy = pd.get_dummies(df_all['Foundation'],prefix='Foundation',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['Foundation']
!pip install -U git+https://github.com/qubvel/efficientnet<choose_model_class>
df_dummy = pd.get_dummies(df_all['GarageCond'],prefix='GarageCond',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['GarageCond']
def get_model() :
K.clear_session()
base_model = efn.EfficientNetB4(weights='imagenet', include_top=False, pooling='avg', input_shape=(256, 256, 3))
x = base_model.output
y_pred = Dense(4, activation='sigmoid' )(x)
 return Model(inputs=base_model.input, outputs=y_pred)
 model = get_model()<import_modules>
df_dummy = pd.get_dummies(df_all['PavedDrive'],prefix='PavedDrive',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['PavedDrive']
from keras_radam import RAdam<train_model>
df_all['GarageYrBlt']= df_all['GarageYrBlt'].fillna('2003' ).astype(int)
 df_all['TotalBsmtSF'] = df_all['TotalBsmtSF'].fillna('1305' ).astype(int)
 df_all['BsmtFullBath'] = df_all['BsmtFullBath'].fillna('0' ).astype(int)
 df_all['BsmtHalfBath'] = df_all['BsmtHalfBath'].fillna('0' ).astype(int)
 df_all['GarageCars'] = df_all['GarageCars'].fillna('2' ).astype(int)
 df_all['GarageArea'] = df_all['GarageArea'].fillna('420' ).astype(int)
 df_all['PoolArea']=df_all['PoolArea'].astype(int )
for base_layer in model.layers[:-5]:
base_layer.trainable = False
model.compile(optimizer=RAdam(warmup_proportion=0.1, min_lr=1e-5), loss=bce_dice_loss)
 history_0 = model.fit_generator(generator=data_generator_train,
validation_data=data_generator_val,
epochs=30,
callbacks=[train_metric_callback, val_callback],
workers=num_cores,
verbose=1
)<train_model>
df_all =df_all.drop(['YrSold','MoSold','MiscVal','PoolQC','ScreenPorch','3SsnPorch','EnclosedPorch',
'KitchenAbvGr','CentralAir','Alley','LotConfig','LotShape','LandSlope','RoofStyle',
'BsmtFinSF2','BsmtUnfSF','HeatingQC','Electrical','BsmtFinType2','BsmtFinType1',
'GarageFinish','MiscFeature','FireplaceQu','Fence','Alley','Street','MasVnrType'], axis = 1 )
for base_layer in model.layers[:-1]:
base_layer.trainable = True
model.compile(optimizer=RAdam(warmup_proportion=0.1, min_lr=1e-5), loss=bce_dice_loss)
 history_1 = model.fit_generator(generator=data_generator_train,
validation_data=data_generator_val,
epochs=20,
callbacks=[train_metric_callback, val_callback],
workers=num_cores,
verbose=1,
initial_epoch=1
)<find_best_params>
Scaler = StandardScaler()
all_scaled = pd.DataFrame(Scaler.fit_transform(df_all))
train_scaled = pd.DataFrame(all_scaled[:1460])
 test_scaled = pd.DataFrame(all_scaled[1460:2920] )
class_names = ['Fish', 'Flower', 'Sugar', 'Gravel']
def get_threshold_for_recall(y_true, y_pred, class_i, recall_threshold=0.94, precision_threshold=0.94, plot=False):
precision, recall, thresholds = precision_recall_curve(y_true[:, class_i], y_pred[:, class_i])
 i = len(thresholds)- 1
best_recall_threshold = None
while best_recall_threshold is None:
next_threshold = thresholds[i]
next_recall = recall[i]
if next_recall >= recall_threshold:
best_recall_threshold = next_threshold
i -= 1
best_precision_threshold = [thres for prec, thres in zip(precision, thresholds)if prec >= precision_threshold][0]
if plot:
plt.figure(figsize=(10, 7))
plt.step(recall, precision, color='r', alpha=0.3, where='post')
 plt.fill_between(recall, precision, alpha=0.3, color='r')
 plt.axhline(y=precision[i + 1])
 recall_for_prec_thres = [rec for rec, thres in zip(recall, thresholds)
 if thres == best_precision_threshold][0]
plt.axvline(x=recall_for_prec_thres, color='g')
 plt.xlabel('Recall')
 plt.ylabel('Precision')
 plt.ylim([0.0, 1.05])
 plt.xlim([0.0, 1.0])
 plt.legend(['PR curve',
f'Precision {precision[i + 1]:.2f} corresponding to selected recall threshold',
f'Recall {recall_for_prec_thres:.2f} corresponding to selected precision threshold'])
 plt.title(f'Precision-Recall curve for Class {class_names[class_i]}')
 return best_recall_threshold, best_precision_threshold
y_pred = model.predict_generator(data_generator_val, workers=num_cores)
 y_true = data_generator_val.get_labels()
recall_thresholds = dict()
precision_thresholds = dict()
for i, class_name in tqdm(enumerate(class_names)) :
recall_thresholds[class_name], precision_thresholds[class_name] = get_threshold_for_recall(y_true, y_pred, i, plot=True )<predict_on_test>
X_train = train_scaled
y_train = y['SalePrice']
X_train, X_test, y_train, y_test = train_test_split(
 X_train, y_train, test_size=0.15, random_state=0 )
data_generator_test = DataGenenerator(folder_imgs=test_imgs_folder, shuffle=False)
 y_pred_test = model.predict_generator(data_generator_test, workers=num_cores )<define_variables>
XGBoost
param_xgb = {"max_depth": [1,3,5,10,100],
"learning_rate" : [0.0001,0.001,0.01],
"min_child_weight" : [1, 3, 5, 10],
"n_estimators": [1, 10, 100, 1000],
"subsample": [0.5,0.75,0.9],
"gamma":[0,0.1,0.2],
"eta": [0.3,0.15,0.10]}
gs_xgb = GridSearchCV(XGBRegressor() ,
param_xgb,
cv=4,
verbose=True,
n_jobs=-1)
 gs_xgb.fit(X_train, y_train)
 print(gs_xgb.best_estimator_ )
image_labels_empty = set()
for i,(img, predictions)in enumerate(zip(os.listdir(test_imgs_folder), y_pred_test)) :
for class_i, class_name in enumerate(class_names):
if predictions[class_i] < recall_thresholds[class_name]:
image_labels_empty.add(f'{img}_{class_name}' )<load_from_csv>
LightGBM
param_lgb = {"max_depth": [1, 3, 5, 10, 25, 50, 75],
"learning_rate" : [0.001,0.01,0.05,0.1],
"num_leaves": [1, 10, 100,1000],
"n_estimators": [1, 3, 10, 100, 1000]}
gs_lgb = GridSearchCV(LGBMRegressor() ,
param_lgb,
cv=4,
verbose=True,
n_jobs=-1)
 gs_lgb.fit(X_train, y_train)
 print(gs_lgb.best_estimator_ )
submission = pd.read_csv('.. /input/efficient-net-b4-unet-clouds/submission.csv')
 submission.head()<filter>
XGB = XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1, eta=0.3, gamma=0,
gpu_id=-1, importance_type='gain', interaction_constraints='',
learning_rate=0.01, max_delta_step=0, max_depth=5,
min_child_weight=1,monotone_constraints='() ',n_estimators=1000,
n_jobs=0, num_parallel_tree=1, random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=0.5,
tree_method='exact', validate_parameters=1, verbosity=None)
 XGB.fit(X_train,y_train )
predictions_nonempty = set(submission.loc[~submission['EncodedPixels'].isnull() , 'Image_Label'].values )<concatenate>
model_lgb = LGBMRegressor(learning_rate=0.05, max_depth=3, n_estimators=1000, num_leaves=10)
 model_lgb.fit(X_train , y_train )
print(f'{len(image_labels_empty.intersection(predictions_nonempty)) } masks would be removed' )<save_to_csv>
votingC = VotingRegressor(estimators=[('xgb_r', XGB),('lgb_r', model_lgb)],n_jobs=4)
 votingC = votingC.fit(X_train, y_train )
submission.loc[submission['Image_Label'].isin(image_labels_empty), 'EncodedPixels'] = np.nan
submission.to_csv('submission_segmentation_and_classifier_efficientnetb4_16btch_size.csv', index=None )<install_modules>
print("Training score:",XGB.score(X_train,y_train),"Test Score:",XGB.score(X_test,y_test))
print("Training score:",model_lgb.score(X_train,y_train),"Test Score:",model_lgb.score(X_test,y_test))
print("Training score:",votingC.score(X_train,y_train),"Test Score:",votingC.score(X_test,y_test))
!pip install catalyst
!pip install pretrainedmodels
!pip install git+https://github.com/qubvel/segmentation_models.pytorch
!pip install pytorch_toolbelt
!pip install torchvision==0.4
!pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext".. /input/apex-325f5a0/apex-master/ 
<set_options>
y_pred_voting = pd.DataFrame(votingC.predict(test_scaled))
train_on_gpu = True
%matplotlib inline
device=torch.device('cuda' )<define_variables>
y_pred=pd.DataFrame()
y_pred['SalePrice'] = y_pred_voting[0]
y_pred['Id'] = df_test['Id']
path = '.. /input/understanding_cloud_organization'
img_paths = '.. /input/understanding-clouds-resized'
os.listdir(path )<load_from_csv>
y_pred.to_csv('submission.csv',index=False )
train = pd.read_csv(f'{path}/train.csv')
 sub = pd.read_csv(f'{path}/sample_submission.csv' )<define_variables>
warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 10000)
 pd.set_option('display.max_rows', 10000 )
n_train = len(os.listdir(f'{img_paths}/train_images_525/train_images_525'))
n_test = len(os.listdir(f'{img_paths}/test_images_525/test_images_525'))
print(f'There are {n_train} images in train dataset')
 print(f'There are {n_test} images in test dataset' )<count_values>
df_train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 df_test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )
train['Image_Label'].apply(lambda x: x.split('_')[1] ).value_counts()<count_values>
y = df_train[['Id','SalePrice']]
df_train = df_train.drop('SalePrice',axis=1 )
train.loc[train['EncodedPixels'].isnull() == False, 'Image_Label'].apply(lambda x: x.split('_')[1] ).value_counts()<count_values>
concat_df = [df_train,df_test]
df_all = pd.concat(concat_df ).reset_index(drop=True )
train.loc[train['EncodedPixels'].isnull() == False, 'Image_Label'].apply(lambda x: x.split('_')[0] ).value_counts().value_counts()<feature_engineering>
df_dummy = pd.get_dummies(df_all['MSZoning'],prefix='MSZoning',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['MSZoning']
train['label'] = train['Image_Label'].apply(lambda x: x.split('_')[1])
 train['im_id'] = train['Image_Label'].apply(lambda x: x.split('_')[0])
 sub['label'] = sub['Image_Label'].apply(lambda x: x.split('_')[1])
 sub['im_id'] = sub['Image_Label'].apply(lambda x: x.split('_')[0] )<count_unique_values>
df_all['LotFrontage'] = df_all['LotFrontage'].fillna(df_all['LotFrontage'].mean() )
id_mask_count = train.loc[train['EncodedPixels'].isnull() == False, 'Image_Label'].apply(lambda x: x.split('_')[0] ).value_counts().\
reset_index().rename(columns={'index': 'img_id', 'Image_Label': 'count'})
 train_ids, valid_ids = train_test_split(id_mask_count['img_id'].values, random_state=42, stratify=id_mask_count['count'], test_size=0.1)
 test_ids = sub['Image_Label'].apply(lambda x: x.split('_')[0] ).drop_duplicates().values<categorify>
def LotFrontage(n):
if(n > 21)&(n < 50):
return 1
elif(n > 50)&(n <= 70):
return 2
elif(n > 70)&(n <= 75):
return 3
elif(n > 75)&(n <= 80):
return 4
elif(n > 80)&(n <= 90):
return 5
elif(n > 100)&(n <= 120):
return 6
elif(n > 120)&(n <= 140):
return 7
elif(n > 140)&(n <= 160):
return 8
elif(n > 160)&(n <= 180):
return 9
elif(n > 180)&(n <= 200):
return 9
else:
return 10
image_name = '8242ba0.jpg'
image = get_img(image_name)
 mask = make_mask(train, image_name )<create_dataframe>
df_all['LotFrontage'] = df_all['LotFrontage'].apply(LotFrontage )
class CloudDataset(Dataset):
def __init__(self, df: pd.DataFrame = None, datatype: str = 'train', img_ids: np.array = None,
transforms = albu.Compose([albu.HorizontalFlip() ,AT.ToTensor() ]),
preprocessing=None):
self.df = df
if datatype != 'test':
self.data_folder = f"{img_paths}/train_images_525/train_images_525"
else:
self.data_folder = f"{img_paths}/test_images_525/test_images_525"
self.img_ids = img_ids
self.transforms = transforms
self.preprocessing = preprocessing
def __getitem__(self, idx):
image_name = self.img_ids[idx]
mask = make_mask(self.df, image_name)
 image_path = os.path.join(self.data_folder, image_name)
 img = cv2.imread(image_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 augmented = self.transforms(image=img, mask=mask)
 img = augmented['image']
mask = augmented['mask']
if self.preprocessing:
preprocessed = self.preprocessing(image=img, mask=mask)
 img = preprocessed['image']
mask = preprocessed['mask']
return img, mask
def __len__(self):
return len(self.img_ids )<categorify>
df_dummy = pd.get_dummies(df_all['Neighborhood'],prefix='Neighborhood',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['Neighborhood']
ENCODER = 'resnet18'
ENCODER_WEIGHTS = 'imagenet'
ACTIVATION = None
model = smp.Unet(
 encoder_name=ENCODER,
encoder_weights=ENCODER_WEIGHTS,
classes=4,
activation=ACTIVATION, 
)
 preprocessing_fn = smp.encoders.get_preprocessing_fn(ENCODER, ENCODER_WEIGHTS )<create_dataframe>
df_all['Exterior2nd'] = df_all['Exterior2nd'].fillna('Other')
 df_dummy = pd.get_dummies(df_all['Exterior2nd'],prefix='Exterior2nd',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['Exterior2nd']
num_workers = 4
bs = 32
train_dataset = CloudDataset(df=train, datatype='train', img_ids=train_ids, transforms = get_training_augmentation() , preprocessing=get_preprocessing(preprocessing_fn))
valid_dataset = CloudDataset(df=train, datatype='valid', img_ids=valid_ids, transforms = get_validation_augmentation() , preprocessing=get_preprocessing(preprocessing_fn))
train_loader = DataLoader(train_dataset, batch_size=bs, shuffle=True, num_workers=num_workers)
 valid_loader = DataLoader(valid_dataset, batch_size=bs, shuffle=False, num_workers=num_workers)
 loaders = {
"train": train_loader,
"valid": valid_loader
}<compute_test_metric>
df_dummy = pd.get_dummies(df_all['Utilities'],prefix='Utilities',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['Utilities']
def dice_loss(input, target):
input = torch.sigmoid(input)
 smooth = 1.0
iflat = input.view(-1)
 tflat = target.view(-1)
 intersection =(iflat * tflat ).sum()
return(( 2.0 * intersection + smooth)/(iflat.sum() + tflat.sum() + smooth))
class FocalLoss(nn.Module):
def __init__(self, gamma):
super().__init__()
self.gamma = gamma
def forward(self, input, target):
if not(target.size() == input.size()):
raise ValueError("Target size({})must be the same as input size({})" 
.format(target.size() , input.size()))
max_val =(-input ).clamp(min=0)
 loss = input - input * target + max_val + \ 
(( -max_val ).exp() +(-input - max_val ).exp() ).log()
invprobs = F.logsigmoid(-input *(target * 2.0 - 1.0))
loss =(invprobs * self.gamma ).exp() * loss
return loss.mean()
class MixedLoss(nn.Module):
def __init__(self, alpha, gamma):
super().__init__()
self.alpha = alpha
self.focal = FocalLoss(gamma)
 def forward(self, input, target):
loss = self.alpha*self.focal(input, target)- torch.log(dice_loss(input, target))
return loss.mean()<choose_model_class>
df_dummy = pd.get_dummies(df_all['LandContour'],prefix='LandContour',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['LandContour']
num_epochs = 25
logdir = "./logs/segmentation_unet"
optimizer = torch.optim.Adam([
{'params': model.decoder.parameters() , 'lr': 1e-2},
{'params': model.encoder.parameters() , 'lr': 1e-3},
])
 opt_level = 'O1'
model.cuda()
model, optimizer = amp.initialize(model, optimizer, opt_level=opt_level)
 scheduler = ReduceLROnPlateau(optimizer, factor=0.5, patience=2)
 criterion = smp.utils.losses.BCEDiceLoss(eps=1.)
runner = SupervisedRunner()<train_model>
df_dummy = pd.get_dummies(df_all['Condition1'],prefix='Condition1',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['Condition1']
runner.train(
 model=model,
criterion=criterion,
optimizer=optimizer,
scheduler=scheduler,
loaders=loaders,
callbacks=[DiceCallback() , EarlyStoppingCallback(patience=5, min_delta=0.001)],
logdir=logdir,
num_epochs=num_epochs,
verbose=True
)<define_search_space>
def Condition2(c):
if c == 'Norm':
return 1
elif c == 'Feedr':
return 2
elif c == 'PosA':
return 3
elif c == 'PosN':
return 4
elif c == 'Artery':
return 5
else:
return 0
encoded_pixels = []
loaders = {"infer": valid_loader}
runner.infer(
 model=model,
loaders=loaders,
callbacks=[
CheckpointCallback(
 resume=f"{logdir}/checkpoints/best.pth"),
InferCallback()
], 
)
 valid_masks = []
probabilities = np.zeros(( 2220, 350, 525), dtype = np.float32)
 for i,(batch, output)in enumerate(tqdm.tqdm(zip(
 valid_dataset, runner.callbacks[0].predictions["logits"]))):
image, mask = batch
for m in mask:
if m.shape !=(350, 525):
m = cv2.resize(m, dsize=(525, 350), interpolation=cv2.INTER_LINEAR)
 valid_masks.append(m)
 for j, probability in enumerate(output):
if probability.shape !=(350, 525):
probability = cv2.resize(probability, dsize=(525, 350), interpolation=cv2.INTER_LINEAR)
 probabilities[i * 4 + j, :, :] = probability<set_options>
df_all['Condition2'] = df_all['Condition2'].apply(Condition2 ).astype(int )
torch.cuda.empty_cache()
gc.collect()<define_search_space>
df_dummy = pd.get_dummies(df_all['BldgType'],prefix='BldgType',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['BldgType']
class_params = {}
for class_id in range(4):
print(class_id)
 attempts = []
for t in range(0, 100, 5):
t /= 100
for ms in [5000, 10000, 15000, 20000, 22500, 25000]:
masks = []
for i in range(class_id, len(probabilities), 4):
probability = probabilities[i]
predict, num_predict = post_process(sigmoid(probability), t, ms)
 masks.append(predict)
 d = []
for i, j in zip(masks, valid_masks[class_id::4]):
if(i.sum() == 0)&(j.sum() == 0):
d.append(1)
 else:
d.append(dice(i, j))
attempts.append(( t, ms, np.mean(d)))
 attempts_df = pd.DataFrame(attempts, columns=['threshold', 'size', 'dice'])
 attempts_df = attempts_df.sort_values('dice', ascending=False)
 print(attempts_df.head())
 best_threshold = attempts_df['threshold'].values[0]
best_size = attempts_df['size'].values[0]
class_params[class_id] =(best_threshold, best_size )<set_options>
df_dummy = pd.get_dummies(df_all['HouseStyle'],prefix='HouseStyle',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['HouseStyle']
del masks
del valid_masks
del probabilities
gc.collect()<find_best_params>
df_dummy = pd.get_dummies(df_all['RoofMatl'],prefix='RoofMatl',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['RoofMatl']
print(class_params )<create_dataframe>
df_all['Exterior1st']= df_all['Exterior1st'].fillna('Other' )
attempts_df = pd.DataFrame(attempts, columns=['threshold', 'size', 'dice'] )<sort_values>
def Exterior1st(c):
if c == 'VinylSd':
return 1
elif c == 'Wd Sdng':
return 2
elif c == 'HdBoard':
return 3
elif c == 'Plywood':
return 4
elif c == 'MetalSd':
return 5
elif c == 'CemntBd':
return 6
elif c == 'WdShing':
return 7
elif c == 'BrkFace':
return 8
elif c == 'AsbShng':
return 9
else:
return 0
attempts_df = attempts_df.sort_values('dice', ascending=False)
 attempts_df.head(10 )<find_best_params>
df_all['Exterior1st'] = df_all['Exterior1st'].apply(Exterior1st ).astype(int )
best_threshold = attempts_df['threshold'].values[0]
best_size = attempts_df['size'].values[0]<train_on_grid>
def BsmtFinSF1(n):
if(n > -1)&(n <= 150):
return 1
elif(n > 150)&(n <= 300):
return 2
elif(n > 300)&(n <= 450):
return 3
elif(n > 450)&(n <= 600):
return 4
elif(n > 600)&(n <= 750):
return 5
else:
return 10
for i,(input, output)in enumerate(zip(
 valid_dataset, runner.callbacks[0].predictions["logits"])) :
image, mask = input
image_vis = image.transpose(1, 2, 0)
 mask = mask.astype('uint8' ).transpose(1, 2, 0)
 pr_mask = np.zeros(( 350, 525, 4))
for j in range(4):
probability = cv2.resize(output[:, :, j], dsize=(525, 350), interpolation=cv2.INTER_LINEAR)
 pr_mask[:, :, j], _ = post_process(sigmoid(probability), class_params[j][0], class_params[j][1])
 visualize_with_raw(image=image_vis, mask=pr_mask, original_image=image_vis, original_mask=mask, raw_image=image_vis, raw_mask=output.transpose(1, 2, 0))
if i >= 2:
break<set_options>
df_all['BsmtFinSF1'] = df_all['BsmtFinSF1'].apply(BsmtFinSF1 ).astype(int )
torch.cuda.empty_cache()
gc.collect()<create_dataframe>
def MasVnrArea(n):
if(n > -1)&(n <= 150):
return 1
elif(n > 150)&(n <= 200):
return 2
elif(n > 200)&(n <= 300):
return 3
else:
return 0
test_dataset = CloudDataset(df=sub, datatype='test', img_ids=test_ids, transforms = get_validation_augmentation() , preprocessing=get_preprocessing(preprocessing_fn))
test_loader = DataLoader(test_dataset, batch_size=8, shuffle=False, num_workers=0)
 loaders = {"test": test_loader}<drop_column>
df_all['MasVnrArea'] = df_all['MasVnrArea'].apply(MasVnrArea ).astype(int )
del train_dataset, train_loader<set_options>
df_dummy = pd.get_dummies(df_all['ExterQual'],prefix='ExterQual',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['ExterQual']
del valid_dataset, valid_loader
gc.collect()<predict_on_test>
df_dummy = pd.get_dummies(df_all['ExterCond'],prefix='ExterCond',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['ExterCond']
encoded_pixels = []
image_id = 0
for i, test_batch in enumerate(tqdm.tqdm(loaders['test'])) :
runner_out = runner.predict_batch({"features": test_batch[0].cuda() })['logits']
for i, batch in enumerate(runner_out):
for probability in batch:
probability = probability.cpu().detach().numpy()
if probability.shape !=(350, 525):
probability = cv2.resize(probability, dsize=(525, 350), interpolation=cv2.INTER_LINEAR)
 predict, num_predict = post_process(sigmoid(probability), class_params[image_id % 4][0], class_params[image_id % 4][1])
 if num_predict == 0:
encoded_pixels.append('')
 else:
r = mask2rle(predict)
 encoded_pixels.append(r)
 image_id += 1<save_to_csv>
df_all['BsmtQual'] = df_all['BsmtQual'].fillna('TA')
 df_dummy = pd.get_dummies(df_all['BsmtQual'],prefix='BsmtQual',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['BsmtQual']
sub['EncodedPixels'] = encoded_pixels
sub.to_csv('submission.csv', columns=['Image_Label', 'EncodedPixels'], index=False )<install_modules>
df_all['BsmtCond'] = df_all['BsmtCond'].fillna('TA')
 df_dummy = pd.get_dummies(df_all['BsmtCond'],prefix='BsmtCond',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['BsmtCond']
!pip install typeguard
!pip install -q --no-deps tensorflow-addons~=0.7
print('Tensorflow version ' + tf.__version__)
<set_options>
df_all['BsmtExposure'] = df_all['BsmtExposure'].fillna('TA')
 df_dummy = pd.get_dummies(df_all['BsmtExposure'],prefix='BsmtExposure',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['BsmtExposure']
AUTO = tf.data.experimental.AUTOTUNE
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 IMAGE_SIZE = [512, 512]
EPOCHS = 50
BATCH_SIZE = 16 * strategy.num_replicas_in_sync<define_variables>
def Functional(f):
if f == 'Typ':
return 1
elif f == 'Min1':
return 2
elif f == 'Min2':
return 3
elif f == 'Mod':
return 4
elif f == 'Maj1':
return 5
elif f == 'Maj2':
return 6
elif f == 'Sev':
return 7
elif f == 'Sal':
return 8
else:
return 0
IMAGE_SIZE = [512, 512]<define_variables>
df_all['Functional'] = df_all['Functional'].apply(Functional ).astype(int )
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
GCS_PATH_SELECT = {
192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',
224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',
331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',
512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'
}
GCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')+ tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec' )<define_variables>
df_dummy = pd.get_dummies(df_all['GarageType'],prefix='GarageType',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['GarageType']
CLASSES = ['pink primrose', 'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea', 'wild geranium', 'tiger lily', 'moon orchid', 'bird of paradise', 'monkshood', 'globe thistle',
'snapdragon', "colt's foot", 'king protea', 'spear thistle', 'yellow iris', 'globe-flower', 'purple coneflower', 'peruvian lily', 'balloon flower', 'giant white arum lily',
'fire lily', 'pincushion flower', 'fritillary', 'red ginger', 'grape hyacinth', 'corn poppy', 'prince of wales feathers', 'stemless gentian', 'artichoke', 'sweet william',
'carnation', 'garden phlox', 'love in the mist', 'cosmos', 'alpine sea holly', 'ruby-lipped cattleya', 'cape flower', 'great masterwort', 'siam tulip', 'lenten rose',
'barberton daisy', 'daffodil', 'sword lily', 'poinsettia', 'bolero deep blue', 'wallflower', 'marigold', 'buttercup', 'daisy', 'common dandelion',
'petunia', 'wild pansy', 'primula', 'sunflower', 'lilac hibiscus', 'bishop of llandaff', 'gaura', 'geranium', 'orange dahlia', 'pink-yellow dahlia',
'cautleya spicata', 'japanese anemone', 'black-eyed susan', 'silverbush', 'californian poppy', 'osteospermum', 'spring crocus', 'iris', 'windflower', 'tree poppy',
'gazania', 'azalea', 'water lily', 'rose', 'thorn apple', 'morning glory', 'passion flower', 'lotus', 'toad lily', 'anthurium',
'frangipani', 'clematis', 'hibiscus', 'columbine', 'desert-rose', 'tree mallow', 'magnolia', 'cyclamen ', 'watercress', 'canna lily',
'hippeastrum ', 'bee balm', 'pink quill', 'foxglove', 'bougainvillea', 'camellia', 'mallow', 'mexican petunia', 'bromelia', 'blanket flower',
'trumpet creeper', 'blackberry lily', 'common tulip', 'wild rose']<choose_model_class>
df_dummy = pd.get_dummies(df_all['SaleType'],prefix='SaleType',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['SaleType']
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<categorify>
df_dummy = pd.get_dummies(df_all['SaleCondition'],prefix='SaleCondition',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['SaleCondition']
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled = True, ordered = False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO)
 return dataset
def data_augment(image, label):
image = tf.image.random_flip_left_right(image)
 return image, label
def get_training_dataset(dataset):
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(dataset):
dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)* 0.75
NUM_VALIDATION_IMAGES = count_data_items(TRAINING_FILENAMES)* 0.25
NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))<choose_model_class>
df_dummy = pd.get_dummies(df_all['Heating'],prefix='Heating',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['Heating']
def get_model() :
with strategy.scope() :
rnet = DenseNet201(
 input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights='imagenet',
include_top=False 
)
 rnet.trainable = True
model = tf.keras.Sequential([
rnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 return model
def train_cross_validate(folds = 4):
histories = []
models = []
early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 4)
 kfold = KFold(folds, shuffle = True, random_state = 42)
 for fold,(trn_ind, val_ind)in enumerate(kfold.split(TRAINING_FILENAMES)) :
train_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES} ).loc[trn_ind]['TRAINING_FILENAMES']), labeled = True)
 val_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES} ).loc[val_ind]['TRAINING_FILENAMES']), labeled = True, ordered = True)
 checkpoint_name = f'model_fold_{fold + 1}' + '.h5'
model_checkpoint = tf.keras.callbacks.ModelCheckpoint(checkpoint_name, save_best_only = True, save_weights_only = True)
 model = get_model()
history = model.fit(
 get_training_dataset(train_dataset),
steps_per_epoch = STEPS_PER_EPOCH,
epochs = EPOCHS,
callbacks = [lr_callback, early_stopping, model_checkpoint],
validation_data = get_validation_dataset(val_dataset)
)
 print('Load best weights for model prediction')
 model.load_weights(checkpoint_name)
 models.append(model)
 histories.append(history)
 return histories, models
def train_and_predict(folds = 4):
test_ds = get_test_dataset(ordered=True)
 test_images_ds = test_ds.map(lambda image, idnum: image)
 print('Start training {} models'.format(folds))
histories, models = train_cross_validate(folds = folds)
 print('Computing predictions...')
 probabilities = np.average([models[i].predict(test_images_ds)for i in range(folds)], axis = 0)
 predictions = np.argmax(probabilities, axis=-1)
 print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')
 return histories, models
histories, models = train_and_predict(folds = 4 )<import_modules>
df_dummy = pd.get_dummies(df_all['KitchenQual'],prefix='KitchenQual',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['KitchenQual']
import re
import numpy as np
from matplotlib import pyplot as plt
from sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix<import_modules>
df_all['GarageQual'] = df_all['GarageQual'].fillna('TA')
 df_dummy = pd.get_dummies(df_all['GarageQual'],prefix='GarageQual',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['GarageQual']
print("Tensorflow version " + tf.__version__ )<set_options>
df_dummy = pd.get_dummies(df_all['Foundation'],prefix='Foundation',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['Foundation']
AUTO = tf.data.experimental.AUTOTUNE
AUTO<choose_model_class>
df_dummy = pd.get_dummies(df_all['GarageCond'],prefix='GarageCond',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['GarageCond']
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu )<define_variables>
df_dummy = pd.get_dummies(df_all['PavedDrive'],prefix='PavedDrive',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['PavedDrive']
IMAGE_SIZE = [512, 512]
EPOCHS = 20
BATCH_SIZE = 16 * strategy.num_replicas_in_sync<choose_model_class>
df_all['GarageYrBlt']= df_all['GarageYrBlt'].fillna('2003' ).astype(int)
 df_all['TotalBsmtSF'] = df_all['TotalBsmtSF'].fillna('1305' ).astype(int)
 df_all['BsmtFullBath'] = df_all['BsmtFullBath'].fillna('0' ).astype(int)
 df_all['BsmtHalfBath'] = df_all['BsmtHalfBath'].fillna('0' ).astype(int)
 df_all['GarageCars'] = df_all['GarageCars'].fillna('2' ).astype(int)
 df_all['GarageArea'] = df_all['GarageArea'].fillna('420' ).astype(int)
 df_all['PoolArea']=df_all['PoolArea'].astype(int )
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<define_variables>
df_all =df_all.drop(['YrSold','MoSold','MiscVal','PoolQC','ScreenPorch','3SsnPorch','EnclosedPorch',
'KitchenAbvGr','CentralAir','Alley','LotConfig','LotShape','LandSlope','RoofStyle',
'BsmtFinSF2','BsmtUnfSF','HeatingQC','Electrical','BsmtFinType2','BsmtFinType1',
'GarageFinish','MiscFeature','FireplaceQu','Fence','Alley','Street','MasVnrType'], axis = 1 )
GCS_DS_PATH = KaggleDatasets().get_gcs_path()<define_variables>
Scaler = StandardScaler()
all_scaled = pd.DataFrame(Scaler.fit_transform(df_all))
train_scaled = pd.DataFrame(all_scaled[:1460])
 test_scaled = pd.DataFrame(all_scaled[1460:2920] )
GCS_PATH_SELECT = {
192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',
224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',
331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',
512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'
}
GCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')
 VALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec' )<define_variables>
X_train = train_scaled
y_train = y['SalePrice']
X_train, X_test, y_train, y_test = train_test_split(
 X_train, y_train, test_size=0.15, random_state=0 )
SKIP_VALIDATION = True
if SKIP_VALIDATION:
TRAINING_FILENAMES = TRAINING_FILENAMES + VALIDATION_FILENAMES<define_variables>
XGBoost
param_xgb = {"max_depth": [1,3,5,10,100],
"learning_rate" : [0.0001,0.001,0.01],
"min_child_weight" : [1, 3, 5, 10],
"n_estimators": [1, 10, 100, 1000],
"subsample": [0.5,0.75,0.9],
"gamma":[0,0.1,0.2],
"eta": [0.3,0.15,0.10]}
gs_xgb = GridSearchCV(XGBRegressor() ,
param_xgb,
cv=4,
verbose=True,
n_jobs=-1)
 gs_xgb.fit(X_train, y_train)
 print(gs_xgb.best_estimator_ )
CLASSES = ['pink primrose', 'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea', 'wild geranium', 'tiger lily', 'moon orchid', 'bird of paradise', 'monkshood', 'globe thistle',
'snapdragon', "colt's foot", 'king protea', 'spear thistle', 'yellow iris', 'globe-flower', 'purple coneflower', 'peruvian lily', 'balloon flower', 'giant white arum lily',
'fire lily', 'pincushion flower', 'fritillary', 'red ginger', 'grape hyacinth', 'corn poppy', 'prince of wales feathers', 'stemless gentian', 'artichoke', 'sweet william',
'carnation', 'garden phlox', 'love in the mist', 'cosmos', 'alpine sea holly', 'ruby-lipped cattleya', 'cape flower', 'great masterwort', 'siam tulip', 'lenten rose',
'barberton daisy', 'daffodil', 'sword lily', 'poinsettia', 'bolero deep blue', 'wallflower', 'marigold', 'buttercup', 'daisy', 'common dandelion',
'petunia', 'wild pansy', 'primula', 'sunflower', 'lilac hibiscus', 'bishop of llandaff', 'gaura', 'geranium', 'orange dahlia', 'pink-yellow dahlia',
'cautleya spicata', 'japanese anemone', 'black-eyed susan', 'silverbush', 'californian poppy', 'osteospermum', 'spring crocus', 'iris', 'windflower', 'tree poppy',
'gazania', 'azalea', 'water lily', 'rose', 'thorn apple', 'morning glory', 'passion flower', 'lotus', 'toad lily', 'anthurium',
'frangipani', 'clematis', 'hibiscus', 'columbine', 'desert-rose', 'tree mallow', 'magnolia', 'cyclamen ', 'watercress', 'canna lily',
'hippeastrum ', 'bee balm', 'pink quill', 'foxglove', 'bougainvillea', 'camellia', 'mallow', 'mexican petunia', 'bromelia', 'blanket flower',
'trumpet creeper', 'blackberry lily', 'common tulip', 'wild rose']<categorify>
LightGBM
param_lgb = {"max_depth": [1, 3, 5, 10, 25, 50, 75],
"learning_rate" : [0.001,0.01,0.05,0.1],
"num_leaves": [1, 10, 100,1000],
"n_estimators": [1, 3, 10, 100, 1000]}
gs_lgb = GridSearchCV(LGBMRegressor() ,
param_lgb,
cv=4,
verbose=True,
n_jobs=-1)
 gs_lgb.fit(X_train, y_train)
 print(gs_lgb.best_estimator_ )
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def data_augment(image, label):
image = tf.image.random_flip_left_right(image)
 return image, label
def get_training_dataset() :
dataset = load_dataset(TRAINING_FILENAMES, labeled=True)
 dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(ordered=False):
dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)
 NUM_VALIDATION_IMAGES = 0 if SKIP_VALIDATION else count_data_items(VALIDATION_FILENAMES)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))<define_variables>
XGB = XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1, eta=0.3, gamma=0,
gpu_id=-1, importance_type='gain', interaction_constraints='',
learning_rate=0.01, max_delta_step=0, max_depth=5,
min_child_weight=1,monotone_constraints='() ',n_estimators=1000,
n_jobs=0, num_parallel_tree=1, random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=0.5,
tree_method='exact', validate_parameters=1, verbosity=None)
 XGB.fit(X_train,y_train )
models = []
histories = []<install_modules>
model_lgb = LGBMRegressor(learning_rate=0.05, max_depth=3, n_estimators=1000, num_leaves=10)
 model_lgb.fit(X_train , y_train )
!pip install -q efficientnet 
<choose_model_class>
votingC = VotingRegressor(estimators=[('xgb_r', XGB),('lgb_r', model_lgb)],n_jobs=4)
 votingC = votingC.fit(X_train, y_train )
with strategy.scope() :
enet = efn.EfficientNetB7(
 input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights='imagenet',
include_top=False 
)
 enet.trainable = True
model = tf.keras.Sequential([
enet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model.compile(
 optimizer=tf.keras.optimizers.Adam(lr=0.0001),
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 model.summary()
models.append(model )<train_model>
print("Training score:",XGB.score(X_train,y_train),"Test Score:",XGB.score(X_test,y_test))
print("Training score:",model_lgb.score(X_train,y_train),"Test Score:",model_lgb.score(X_test,y_test))
print("Training score:",votingC.score(X_train,y_train),"Test Score:",votingC.score(X_test,y_test))
history = model.fit(
 get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[lr_callback],
validation_data=None if SKIP_VALIDATION else get_validation_dataset() 
)
 histories.append(history )<import_modules>
y_pred_voting = pd.DataFrame(votingC.predict(test_scaled))
from tensorflow.keras.applications import DenseNet201<choose_model_class>
y_pred=pd.DataFrame()
y_pred['SalePrice'] = y_pred_voting[0]
y_pred['Id'] = df_test['Id']
<train_model><EOS> 
y_pred.to_csv('submission.csv',index=False )