Kaggle competition: Tabular Playground Series - Jan 2021
submission = pd.DataFrame.from_dict(subm, orient='index')
 submission.to_csv('submission.csv', index=False)
 submission.head()<install_modules>
import lightgbm as lgb
import optuna.integration.lightgbm as oplgb
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
from tqdm.notebook import tqdm
import matplotlib.pyplot as plt
import seaborn as sns
sys.path.append(".. /input/zarrkaggleinstall")
 sys.path.append(".. /input/segmentation-models-pytorch-install")
 !pip install -q --no-deps.. /input/deepflash2-lfs 
<categorify>
df_train = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/train.csv")
 df_test = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/test.csv")
 df_sample = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/sample_submission.csv" )
def rle_encode_less_memory(img):
pixels = img.T.flatten()
pixels[0] = 0
pixels[-1] = 0
runs = np.where(pixels[1:] != pixels[:-1])[0] + 2
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def load_model_weights(model, file, strict=True):
state = torch.load(file, map_location='cpu')
 stats = state['stats']
model_state = state['model']
model.load_state_dict(model_state, strict=strict)
 return model, stats<normalization>
train_id = df_train["id"]
test_id = df_test["id"]
df_train.drop("id", axis=1, inplace=True)
 df_test.drop("id", axis=1, inplace=True )
@patch
def read_img(self:BaseDataset, *args, **kwargs):
image = tifffile.imread(args[0])
 if len(image.shape)== 5:
image = image.squeeze().transpose(1, 2, 0)
 elif image.shape[0] == 3:
image = image.transpose(1, 2, 0)
 return image
@patch
def apply(self:DeformationField, data, offset=(0, 0), pad=(0, 0), order=1):
"Apply deformation field to image using interpolation"
outshape = tuple(int(s - p)for(s, p)in zip(self.shape, pad))
coords = [np.squeeze(d ).astype('float32' ).reshape(*outshape)for d in self.get(offset, pad)]
sl = []
for i in range(len(coords)) :
cmin, cmax = int(coords[i].min()), int(coords[i].max())
 dmax = data.shape[i]
if cmin<0:
cmax = max(-cmin, cmax)
 cmin = 0
elif cmax>dmax:
cmin = min(cmin, 2*dmax-cmax)
 cmax = dmax
coords[i] -= cmin
else: coords[i] -= cmin
sl.append(slice(cmin, cmax))
if len(data.shape)== len(self.shape)+ 1:
tile = np.empty(( *outshape, data.shape[-1]))
for c in range(data.shape[-1]):
tile[..., c] = cv2.remap(data[sl[0],sl[1], c]/255, coords[1],coords[0], interpolation=order, borderMode=cv2.BORDER_REFLECT)
 else:
tile = cv2.remap(data[sl[0], sl[1]], coords[1], coords[0], interpolation=order, borderMode=cv2.BORDER_REFLECT)
 return tile<define_variables>
feature_cols = [c for c in df_train.columns if c != "target"]
class CONFIG() :
data_path = Path('.. /input/hubmap-kidney-segmentation')
 models_path = Path('.. /input/hubmap-efficient-sampling-deepflash2-train')
 models_file = np.array([x for x in models_path.iterdir() if x.name.startswith('u')])
 scale = 3
tile_shape =(512, 512)
 padding =(100,100)
 encoder_name = "efficientnet-b4"
encoder_weights = None
in_channels = 3
classes = 2
batch_size = 16
threshold = 0.5
cfg = CONFIG()<load_from_csv>
train_x = df_train[feature_cols]
train_y = df_train.target
test_x = df_test
print(cfg.models_file)
 print(len(cfg.models_file))
df_sample = pd.read_csv(cfg.data_path/'sample_submission.csv', index_col='id')
 names,preds = [],[]
sub = None<categorify>
folds = KFold(n_splits=5, shuffle=True, random_state=2021 )
names,preds = [],[]
for idx, _ in df_sample.iterrows() :
print(f'
f = cfg.data_path/'test'/f'{idx}.tiff'
ds = TileDataset([f], scale=cfg.scale, tile_shape=cfg.tile_shape, padding=cfg.padding)
 shape = ds.data[f.name].shape
print('Shape:', shape)
 names.append(idx)
 msk = None
print('Prediction')
 for model_path in cfg.models_file:
model = smp.Unet(encoder_name=cfg.encoder_name,
encoder_weights=cfg.encoder_weights,
in_channels=cfg.in_channels,
classes=cfg.classes)
 model, stats = load_model_weights(model, model_path)
 batch_tfms = [Normalize.from_stats(*stats)]
dls = DataLoaders.from_dsets(ds, batch_size=cfg.batch_size, after_batch=batch_tfms, shuffle=False, drop_last=False)
 if torch.cuda.is_available() : dls.cuda() , model.cuda()
learn = Learner(dls, model, loss_func='')
 res = learn.predict_tiles(dl=dls.train, path='/kaggle/temp/', use_tta=False, uncertainty_estimates=False)
 if msk is None:
msk = res[0][f.name][..., 1]
else:
msk += res[0][f.name][..., 1]
del model, stats, learn
msk = msk/len(cfg.models_file)
 msk =(msk > cfg.threshold ).astype(np.uint8)
 print('Rezising')
 msk = cv2.resize(msk,(shape[1], shape[0]))
rle = rle_encode_less_memory(msk)
 preds.append(rle)
 print('Plotting')
 fig, ax = plt.subplots(figsize=(15,15))
ax.imshow(cv2.resize(msk,(1024, 1024)))
 plt.show()
_ = [shutil.rmtree(p, ignore_errors=True)for p in Path('/kaggle/temp/' ).iterdir() ]
_ = [shutil.rmtree(p, ignore_errors=True)for p in Path('/tmp/' ).iterdir() if p.name.startswith('zarr')]<save_to_csv>
class FoldsAverageLGBM:
def __init__(self, folds):
self.folds = folds
self.models = []
def fit(self, lgb_params, train_x, train_y):
oof_preds = np.zeros_like(train_y)
 self.train_x = train_x.values
self.train_y = train_y.values
for tr_idx, va_idx in tqdm(folds.split(train_x)) :
tr_x, va_x = self.train_x[tr_idx], self.train_x[va_idx]
tr_y, va_y = self.train_y[tr_idx], self.train_y[va_idx]
lgb_train_dataset = lgb.Dataset(tr_x, tr_y)
 lgb_valid_dataset = lgb.Dataset(va_x, va_y)
 model = lgb.train(lgb_params, lgb_train_dataset, valid_sets=[lgb_valid_dataset], verbose_eval=100)
 self.models.append(model)
 oof_pred = model.predict(va_x)
 oof_preds[va_idx] = oof_pred
self.oof_preds = oof_preds
def predict(self, test_x):
preds = []
for model in tqdm(self.models):
pred = model.predict(test_x)
 preds.append(pred)
 preds = np.mean(preds, axis=0)
 return preds
df = pd.DataFrame({'id':names,'predicted':preds} ).set_index('id')
 df_sample.loc[df.index.values] = df.values
df_sample.to_csv('submission.csv' )<set_options>
best_lgb_params = {
'seed': 2021,
'objective': 'regression',
'metric': 'rmse',
'verbosity': -1,
'feature_pre_filter': False,
'lambda_l1': 6.540486456085813,
'lambda_l2': 0.01548480538099245,
'num_leaves': 256,
'feature_fraction': 0.52,
'bagging_fraction': 0.6161835249194311,
'bagging_freq': 7,
'min_child_samples': 20
}
best_lgb_params["learning_rate"] = 0.001
best_lgb_params["early_stopping_round"] = 1000
best_lgb_params["num_iterations"] = 20000
warnings.filterwarnings("ignore" )<define_variables>
folds_average_lgbm = FoldsAverageLGBM(folds )
Threshold = 35<categorify>
folds_average_lgbm.fit(best_lgb_params, train_x, train_y )
def rle_encode_less_memory(img):
pixels = img.T.flatten()
pixels[0] = 0
pixels[-1] = 0
runs = np.where(pixels[1:] != pixels[:-1])[0] + 2
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<load_from_csv>
np.sqrt(mean_squared_error(df_train.target, folds_average_lgbm.oof_preds))
df_sample = pd.read_csv('.. /input/hubmap-kidney-segmentation/sample_submission.csv' )<define_variables>
y_pred = folds_average_lgbm.predict(test_x )
names,preds = [],[]<categorify>
sub = df_sample.copy()
sub["target"] = y_pred
sub.to_csv("submission_lgbm_1.csv", index=False)
 sub.head()