Kaggle competition: Tabular Playground Series - Jan 2021
class UneXt50(nn.Module):
def __init__(self, stride=1, **kwargs):
super().__init__()
m = ResNet(Bottleneck, [3, 4, 6, 3], groups=32, width_per_group=4)
 self.enc0 = nn.Sequential(m.conv1, m.bn1, nn.ReLU(inplace=True))
self.enc1 = nn.Sequential(nn.MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1),
m.layer1)
 self.enc2 = m.layer2
self.enc3 = m.layer3
self.enc4 = m.layer4
self.aspp = ASPP(2048,256,out_c=512,dilations=[stride*1,stride*2,stride*3,stride*4])
 self.drop_aspp = nn.Dropout2d(0.5)
 self.dec4 = UnetBlock(512,1024,256)
 self.dec3 = UnetBlock(256,512,128)
 self.dec2 = UnetBlock(128,256,64)
 self.dec1 = UnetBlock(64,64,32)
 self.fpn = FPN([512,256,128,64],[16]*4)
 self.drop = nn.Dropout2d(0.1)
 self.final_conv = ConvLayer(32+16*4, 1, ks=1, norm_type=None, act_cls=None)
 def forward(self, x):
enc0 = self.enc0(x)
 enc1 = self.enc1(enc0)
 enc2 = self.enc2(enc1)
 enc3 = self.enc3(enc2)
 enc4 = self.enc4(enc3)
 enc5 = self.aspp(enc4)
 dec3 = self.dec4(self.drop_aspp(enc5),enc3)
 dec2 = self.dec3(dec3,enc2)
 dec1 = self.dec2(dec2,enc1)
 dec0 = self.dec1(dec1,enc0)
 x = self.fpn([enc5, dec3, dec2, dec1], dec0)
 x = self.final_conv(self.drop(x))
x = F.interpolate(x,scale_factor=2,mode='bilinear')
 return x
class UneXt101(nn.Module):
def __init__(self, stride=1, **kwargs):
super().__init__()
m = ResNet(Bottleneck, [3, 4, 23, 3], groups=32, width_per_group=16)
 self.enc0 = nn.Sequential(m.conv1, m.bn1, nn.ReLU(inplace=True))
self.enc1 = nn.Sequential(nn.MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1),
m.layer1)
 self.enc2 = m.layer2
self.enc3 = m.layer3
self.enc4 = m.layer4
self.aspp = ASPP(2048,256,out_c=512,dilations=[stride*1,stride*2,stride*3,stride*4])
 self.drop_aspp = nn.Dropout2d(0.5)
 self.dec4 = UnetBlock(512,1024,256)
 self.dec3 = UnetBlock(256,512,128)
 self.dec2 = UnetBlock(128,256,64)
 self.dec1 = UnetBlock(64,64,32)
 self.fpn = FPN([512,256,128,64],[16]*4)
 self.drop = nn.Dropout2d(0.1)
 self.final_conv = ConvLayer(32+16*4, 1, ks=1, norm_type=None, act_cls=None)
 def forward(self, x):
enc0 = self.enc0(x)
 enc1 = self.enc1(enc0)
 enc2 = self.enc2(enc1)
 enc3 = self.enc3(enc2)
 enc4 = self.enc4(enc3)
 enc5 = self.aspp(enc4)
 dec3 = self.dec4(self.drop_aspp(enc5),enc3)
 dec2 = self.dec3(dec3,enc2)
 dec1 = self.dec2(dec2,enc1)
 dec0 = self.dec1(dec1,enc0)
 x = self.fpn([enc5, dec3, dec2, dec1], dec0)
 x = self.final_conv(self.drop(x))
x = F.interpolate(x,scale_factor=2,mode='bilinear')
 return x
class Unet50(nn.Module):
def __init__(self, stride=1, **kwargs):
super().__init__()
m = ResNet(Bottleneck, [3, 4, 6, 3])
 self.enc0 = nn.Sequential(m.conv1, m.bn1, nn.ReLU(inplace=True))
self.enc1 = nn.Sequential(nn.MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1),
m.layer1)
 self.enc2 = m.layer2
self.enc3 = m.layer3
self.enc4 = m.layer4
self.aspp = ASPP(2048,256,out_c=512,dilations=[stride*1,stride*2,stride*3,stride*4])
 self.drop_aspp = nn.Dropout2d(0.5)
 self.dec4 = UnetBlock(512,1024,256)
 self.dec3 = UnetBlock(256,512,128)
 self.dec2 = UnetBlock(128,256,64)
 self.dec1 = UnetBlock(64,64,32)
 self.fpn = FPN([512,256,128,64],[16]*4)
 self.drop = nn.Dropout2d(0.1)
 self.final_conv = ConvLayer(32+16*4, 1, ks=1, norm_type=None, act_cls=None)
 def forward(self, x):
enc0 = self.enc0(x)
 enc1 = self.enc1(enc0)
 enc2 = self.enc2(enc1)
 enc3 = self.enc3(enc2)
 enc4 = self.enc4(enc3)
 enc5 = self.aspp(enc4)
 dec3 = self.dec4(self.drop_aspp(enc5),enc3)
 dec2 = self.dec3(dec3,enc2)
 dec1 = self.dec2(dec2,enc1)
 dec0 = self.dec1(dec1,enc0)
 x = self.fpn([enc5, dec3, dec2, dec1], dec0)
 x = self.final_conv(self.drop(x))
x = F.interpolate(x,scale_factor=2,mode='bilinear')
 return x<load_pretrained>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import optuna
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import mean_squared_error
from sklearn.base import TransformerMixin
import xgboost as xgb
import lightgbm as lgb
MODELS = []
for models_list in MODELS_PATHS:
models_i = []
for ij,path in enumerate(models_list):
state_dict = torch.load(path,map_location=torch.device('cpu'))
if ij < 2:
model = UneXt101()
elif ij < 5:
model = smp.Unet(encoder_name='efficientnet-b7', classes=1, activation=None, encoder_weights=None)
 else:
model = smp.Unet(encoder_name='efficientnet-b5', classes=1, activation=None, encoder_weights=None)
 model.load_state_dict(state_dict)
 model.float()
model.eval()
model.to(device)
 models_i.append(model)
 del state_dict
MODELS.append(models_i )<prepare_x_and_y>
df = pd.read_csv('.. /input/tabular-playground-series-jan-2021/train.csv')
 df.head()
def _tile_resize_save(img, img_id, tile_sz, reduce=1):

 x = 0
while x < img.shape[0]:
y = 0
while y < img.shape[1]:
img_tile = img[x:x+tile_sz,y:y+tile_sz]
if reduce > 1:
new_dim =(img_tile.shape[1]//reduce,img_tile.shape[0]//reduce)
 img_tile = cv2.resize(img_tile, new_dim, interpolation = cv2.INTER_AREA)
 save_path = "%s_%d_%d.png" %(img_id, x//reduce, y//reduce)
 Image.fromarray(img_tile ).save(save_path)
 y += tile_sz
x += tile_sz
final_x =(( x-tile_sz)//tile_sz)*(tile_sz//reduce)+ img_tile.shape[0]
final_y =(( y-tile_sz)//tile_sz)*(tile_sz//reduce)+ img_tile.shape[1]
return(final_x, final_y, 3 )<train_model>
def objective_xgb(trial, data, target):
parameters = {
'tree_method': 'gpu_hist',
'lambda': trial.suggest_loguniform('lambda', 1e-3, 10.0),
'alpha': trial.suggest_loguniform('alpha', 1e-3, 10.0),
'colsample_bytree': trial.suggest_categorical('colsample_bytree', [0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9,1.0]),
'subsample': trial.suggest_categorical('subsample', [0.4, 0.5, 0.6, 0.7, 0.8, 1.0]),
'learning_rate': trial.suggest_categorical('learning_rate', [0.008, 0.009, 0.01, 0.012, 0.014, 0.016, 0.018, 0.02]),
'n_estimators': 1000,
'max_depth': trial.suggest_categorical('max_depth', [5, 7, 9, 11, 13, 15, 17, 20]),
'random_state': trial.suggest_categorical('random_state', [24, 48, 2020]),
'min_child_weight': trial.suggest_int('min_child_weight', 1, 300),
}
folds = KFold(n_splits=5, random_state=1337, shuffle=True)
 rmse = []
for train_idx, test_idx in folds.split(data, target):
X_train, X_test = X.iloc[train_idx], X.iloc[test_idx]
y_train, y_test = y.iloc[train_idx], y.iloc[test_idx]
model = xgb.XGBRegressor(**parameters)
 model.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=100, verbose=False)
 rmse.append(mean_squared_error(y_test, model.predict(X_test), squared=False))
print(f'Mean RMSE for all the folds: {np.mean(rmse)}')
 return np.mean(rmse )
def load_resize(idx, reduce):

 img = load_image(os.path.join(DATA_DIR,idx+'.tiff'))
init_shape = img.shape
shape = _tile_resize_save(img, idx,(MASK_SZ*REDUCTION), reduce=REDUCTION)
 img = _reconstruct_img(idx,(MASK_SZ*REDUCTION)//REDUCTION, shape)
 return img, init_shape<categorify>
xgb_parameters = {
'objective': 'reg:squarederror',
'tree_method': 'gpu_hist',
'n_estimators': 1000,
'lambda': 7.610705234008646,
'alpha': 0.0019377246932580476,
'colsample_bytree': 0.5,
'subsample': 0.7,
'learning_rate': 0.012,
'max_depth': 20,
'random_state': 24,
'min_child_weight': 229
}
def _get_nored_pads(initW, initH, upW, upH, xa, xb, ya, yb):

 px = xa/(xa+xb)
 py = ya/(ya+yb)
 padx = upW - initW
pady = upH - initH
assert padx > 0
assert pady > 0
xa = int(px*padx)
 xb = padx - xa
ya = int(py*pady)
 yb = pady - ya
return xa, xb, ya, yb
def _add_padding(img, init_sz, img_shape, p0, p1):

 start = time.time()
if VERBOSE:
print(" > Adding padding to make the image dividable into tiles...")
 if VERBOSE:
print(" > Before reduction:", img_shape)
 print(" > After reduction:", img.shape)
 pad0_ = TILE_SZ - img.shape[0]%TILE_SZ
x_pad = int(TILE_SZ*p0)
 xa =(pad0_//2 + x_pad)
 xb = pad0_+TILE_SZ-(pad0_//2 + x_pad)
 pad0_lr = [xa, xb]
pad1_ = TILE_SZ - img.shape[1]%TILE_SZ
y_pad = int(TILE_SZ*p1)
 ya =(pad1_//2 + y_pad)
 yb = pad1_+TILE_SZ-(pad1_//2 + y_pad)
 pad1_lr = [ya, yb]
img = np.pad(img,[pad0_lr, pad1_lr,[0,0]],constant_values=0)
 if VERBOSE:
print(" > After padding:", img.shape, "Time =", time.time() - start, "s")
 xa, xb, ya, yb = _get_nored_pads(img_shape[0], img_shape[1],
REDUCTION*img.shape[0], REDUCTION*img.shape[1],
xa, xb, ya, yb)
 return img, xa, xb, ya, yb, img.shape<train_model>
def objective_lgb(trial):
X, y = df.drop(columns=['target', 'id'] ).values, df['target'].values
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1337)
 ds_train = lgb.Dataset(X_train, label=y_train)
 ds_test = lgb.Dataset(X_test, label=y_test)
 parameters = {
'device_type': 'gpu',
'objective': 'regression',
'metric': 'rmse',
'verbosity': -1,
'boosting_type': 'gbdt',
'lambda_l1': trial.suggest_float('lambda_l1', 1e-8, 10.0, log=True),
'lambda_l2': trial.suggest_float('lambda_l2', 1e-8, 10.0, log=True),
'num_leaves': trial.suggest_int('num_leaves', 2, 256),
'feature_fraction': trial.suggest_float('feature_fraction', 0.4, 1.0),
'bagging_fraction': trial.suggest_float('bagging_fraction', 0.4, 1.0),
'bagging_freq': trial.suggest_int('bagging_freq', 1, 7),
'min_child_samples': trial.suggest_int('min_child_samples', 5, 100),
}
gbm = lgb.train(parameters, ds_train)
 prediction = gbm.predict(X_test)
 accuracy = mean_squared_error(y_test, prediction, squared=False)
 return accuracy
def _split_image(img):

 start = time.time()
if VERBOSE:
print(" > Splitting image into tiles...")
 assert not img.shape[0]%TILE_SZ
assert not img.shape[1]%TILE_SZ
img = img.reshape(img.shape[0]//TILE_SZ,
TILE_SZ,
img.shape[1]//TILE_SZ,
TILE_SZ,
3)
 img = img.transpose(0,2,1,3,4 ).reshape(-1,TILE_SZ,TILE_SZ,3)
 if VERBOSE:
print(" > Splitting done! Time =", time.time() - start)
 return img<data_type_conversions>
lgb_parameters = {
'objective': 'regression',
'metric': 'rmse',
'boosting': 'gbdt',
'lambda_l1': 3.2737454713243543e-07,
'lambda_l2': 3.685676983230042e-06,
'num_leaves': 190,
'feature_fraction': 0.47291296723211934,
'bagging_fraction': 0.8846579981793894,
'bagging_freq': 3,
'min_child_samples': 58,
'verbose': 0,
'device_type': 'gpu'
}
def img2tensor(img, dtype:np.dtype=np.float32):
if img.ndim==2: img = np.expand_dims(img,2)
 img = np.transpose(img,(2,0,1))
return torch.from_numpy(img.astype(dtype, copy=False))
class HuBMAPTestDataset(Dataset):
def __init__(self, idxs):
self.fnames = idxs
def __len__(self):
return len(self.fnames)
 def __getitem__(self, idx):
im = cv2.imread("tmp/%d.png" %(self.fnames[idx],))
return img2tensor(( im/255.0 - MEAN)/STD )<train_model>
class NonLinearTransformer(TransformerMixin):
def __init__(self):
pass
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X = X.drop(columns=['id'])
 for c in X.columns:
if c == 'target':
continue
X[f'{c}^2'] = X[c] ** 2
return X
def _make_tiles_dataloader(idxs):

 start = time.time()
ds = HuBMAPTestDataset(idxs)
 dl = DataLoader(ds, BATCH_SIZE,
num_workers=NUM_WORKERS,
shuffle=False,
pin_memory=True)
 if VERBOSE:
print(" > Tiles dataset created! Time =", time.time() - start)
 return dl<categorify>
pipe_xgb = Pipeline([ 
('custom', NonLinearTransformer()), 
('scaling', StandardScaler()), 
('regression', xgb.XGBRegressor(**xgb_parameters))
])
 pipe_lgb = Pipeline([ 
('custom', NonLinearTransformer()), 
('scaling', StandardScaler()), 
('regression', lgb.LGBMRegressor(**lgb_parameters))
] )
def _generate_masks(dl, idxs, n_tiles, init_sz, group):

 start = time.time()
if VERBOSE:
print(" > Generating masks...")
 red = CUSTOM_REDS[group]
mp = Model_pred(MODELS[group], dl, red)
 mask = torch.zeros(n_tiles,
init_sz,
init_sz,
dtype=torch.uint8)
 for i, p in zip(idxs,iter(mp)) : mask[i] = p.squeeze(-1)
 if VERBOSE:
print(" > Masks generated! Time =", time.time() - start)
 return mask<train_model>
df_train = pd.read_csv('.. /input/tabular-playground-series-jan-2021/train.csv')
 df_predict = pd.read_csv('.. /input/tabular-playground-series-jan-2021/test.csv' )
class Model_pred:
def __init__(self, models, dl, red, half:bool=False):
self.models = models
self.dl = dl
self.half = half
self.red = red
def __iter__(self):
with torch.no_grad() :
for x in iter(self.dl):
x = x.to(device)
 x = F.interpolate(x, scale_factor=1/self.red, mode='bilinear')
 if self.half: x = x.half()
py = 0.
 for rot_flip in ROT_TTA_FLIPS:
for model in self.models:
xr = torch.rot90(x, rot_flip, [-2, -1])
 p = model(xr)
 p = torch.rot90(p, -rot_flip, [-2, -1])
 p = torch.sigmoid(p ).detach()
py += p
for f in TTA_FLIPS:
xf = torch.rot90(x, rot_flip, [-2, -1])
 xf = torch.flip(xf,f)
 for model in self.models:
p = model(xf)
 p = torch.flip(p,f)
 p = torch.rot90(p, -rot_flip, [-2, -1])
 py += torch.sigmoid(p ).detach()
py /=(1+len(TTA_FLIPS)) *len(ROT_TTA_FLIPS)
 py /= len(self.models)
 py = F.upsample(py, scale_factor=REDUCTION*self.red, mode="bilinear")
 py = py.permute(0,2,3,1 ).float().cpu()
py =(N_BINS*py ).int()
batch_size = len(py)
 for i in range(batch_size):
yield py[i]
def __len__(self):
return len(self.dl.dataset )<categorify>
X, y = df_train.drop(columns=['target']), df_train['target']
def _reshape_depad_mask(mask, init_shape, init_sz, p0, p1, xa, xb, ya, yb):

 start = time.time()
if VERBOSE:
print(" > Merge tiled masks into one mask and crop padding...")
 mask = mask.view(init_shape[0]//TILE_SZ,
init_shape[1]//TILE_SZ,
init_sz,
init_sz ).\
permute(0,2,1,3 ).reshape(init_shape[0]*REDUCTION,
init_shape[1]*REDUCTION)
 mask = mask[xa:-xb,ya:-yb]
if VERBOSE:
print(" > Mask created! Shape =", mask.shape,"Time =", time.time() - start)
 return mask<train_model>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1337 )
def _save_mask_tiles(mask, idx, p0, p1):
start = time.time()
if VERBOSE:
print(" > Saving tiles in HDD memory...")
 x = 0
while x < mask.shape[0]:
y = 0
while y < mask.shape[1]:
mask_tile = mask[x:x+MASK_SZ,y:y+MASK_SZ].numpy()
save_path = "%s_%d_%d_%s_%s.png" %(idx, x, y, str(p0), str(p1))
Image.fromarray(mask_tile ).save(save_path)
 y += MASK_SZ
x += MASK_SZ
if VERBOSE:
print("Tiles saved! Time =", time.time() - start )<categorify>
pipe_xgb.fit(X_train, y_train)
 pipe_lgb.fit(X_train, y_train)
 print(f'XGB Score: {pipe_xgb.score(X_test, y_test)}, LGB Score: {pipe_lgb.score(X_test, y_test)}')
 print(f'XGB RMSE: {mean_squared_error(y_test, pipe_xgb.predict(X_test), squared=False)}, LGB RMSE: {mean_squared_error(y_test, pipe_lgb.predict(X_test), squared=False)}' )
def make_one_prediction(img, group, idx, img_shape, p0, p1):

 init_sz = TILE_SZ*REDUCTION
img, xa, xb, ya, yb, img_shape_p = _add_padding(img, init_sz, img_shape,
p0, p1)
 img = _split_image(img)
 n_tiles = img.shape[0]
idxs = _select_tiles(img)
 dl = _make_tiles_dataloader(idxs)
 mask = _generate_masks(dl, idxs, n_tiles, init_sz, group)
 mask = _reshape_depad_mask(mask, img_shape_p, init_sz,
p0, p1, xa, xb, ya, yb)
 gc.collect()
shutil.rmtree('tmp')
 _save_mask_tiles(mask, idx, p0, p1 )<normalization>
def ensemble_predict(X):
target_xgb = pipe_xgb.predict(X)
 target_lgb = pipe_lgb.predict(X)
 return [0.85 * x + 0.15 * l for(x, l)in zip(target_xgb, target_lgb)]
def get_mask_tiles(idx, p0_list, p1_list):

 group = _get_group(os.path.join(DATA_DIR,idx+'.tiff'))
TH = THS[group]
img, init_shape = load_resize(idx, REDUCTION)
 for p0 in p0_list:
for p1 in p1_list:
make_one_prediction(img, group, idx, init_shape, p0, p1)
 return init_shape, TH<predict_on_test>
print(f'Ensemble RMSE: {mean_squared_error(y_test, ensemble_predict(X_test), squared=False)}' )
def make_predictions(idx):

 init_shape, TH = get_mask_tiles(idx, X_OVERLAP, Y_OVERLAP)
 mask = torch.zeros(*init_shape[:2], dtype=torch.uint8)
 x = 0
while x < init_shape[0]:
y = 0
while y < init_shape[1]:
mask_tile = 0.
 for p0 in X_OVERLAP:
for p1 in Y_OVERLAP:
tile_path = "%s_%d_%d_%s_%s.png" %(idx, x, y, str(p0), str(p1))
mask_tile += torch.tensor(np.asarray(Image.open(tile_path), dtype=int))
os.remove(tile_path)
 NEW_TH = int(N_BINS*len(X_OVERLAP)*len(Y_OVERLAP)*TH)
 mask[x:x+MASK_SZ,y:y+MASK_SZ] = mask_tile>NEW_TH
y += MASK_SZ
x += MASK_SZ
if VERBOSE:
print(" > Converting to RLE...")
 rle = rle_encode_less_memory(mask.numpy())
 del mask
return rle<categorify>
pipe_xgb.fit(X, y)
 pipe_lgb.fit(X, y )
<load_from_csv><EOS> 
target = pd.DataFrame({
'id': df_predict['id'], 'target': ensemble_predict(df_predict)
 })
 target.to_csv('submission.csv', index=False )