Kaggle competition: TGS Salt Identification Challenge
<SOS> metric: IntersectionOverUnionObjectSegmentation Kaggle data source: tgs-salt-identification-challenge<groupby>
!pip3 install pycocotools
print(f'Number of series: {MAX_EVAL}')
 agg_df = df.groupby('name' ).mean().drop(['id', 'fold_num'], axis=1)
 display(agg_df )<prepare_x_and_y>
%matplotlib inline
%reload_ext autoreload
%autoreload 2
print(torch.__version__)
 torch.cuda.is_available()
torch.backends.cudnn.benchmark=True
m_name = agg_df['smape'].idxmin()
print('Models name:', m_name)
 smape_cut = 60
fold_num = 0
ids = df[(df['smape'] > smape_cut)&(df['name'] == m_name)&(df['fold_num'] == fold_num)]['id'].unique()
i = ids[0]
for i in ids[:5]:
x = pred_df[(pred_df['id'] == i)&(pred_df['fold_num'] == fold_num)&(pred_df['name'] == m_name)]
smape_val = x['smape'].values[0]
y_hat = x['y_hat'].values[0]
y_val = x['y_val'].values[0]
x_train = x['x_train'].values[0]
plot_prediction(x_train, y_val, y_hat, plot_len=400, title=f"Id: {i}, SMAPE: {smape_val}")
<feature_engineering>
MASKS_FN = 'train.csv'
TRAIN_DN = Path('train/images/')
 MASKS_DN = Path('train/masks/')
 TEST = Path('test/images/')
 PATH = Path('/kaggle/input/tgs-salt-identification-challenge/')
 PATH128 = Path('/tmp/128/')
 TMP = Path('/tmp/')
 MODEL = Path('/tmp/model/')
 PRETRAINED = Path('/kaggle/input/is-there-salt-resnet34/model/resnet34_issalt.h5')
 seg = pd.read_csv(PATH/MASKS_FN ).set_index('id')
 seg.head()
sz = 128
bs = 64
nw = 4
subm = True
if subm:
print('Building the submission...')
 keys = pd.read_csv('/kaggle/input/web-traffic-time-series-forecasting/key_2.csv.zip')
 keys['date'] = keys['Page'].apply(lambda x: x.split("_")[-1])
 keys['name'] = keys['Page'].apply(lambda x: '_'.join(x.split("_")[:-1]))
keys['horisont_day'] = None
sdate = date(2017, 9, 13)
 edate = date(2017, 11, 13)
 delta = edate - sdate
dates_list = [str(sdate + timedelta(days=i)) for i in range(delta.days + 1)]
for n, d in enumerate(dates_list):
keys.loc[keys['date'] == d, 'horisont_day'] = n
m_list = ['weekly_median_w-49']
res = []
print('Predicting...')
 for k, i in enumerate(np.arange(train.shape[0])) :
if k % 10000 == 0:
print('Step:', k)
 x_df = train.iloc[i]
page = x_df['Page']
x_ts = pd.Series(x_df.values[1:].astype(float), index=x_df.index[1:])
 x_ts.index = pd.to_datetime(x_ts.index)
 if np.sum(x_ts[-50:].values)== 0:
y_hat = np.zeros(HORIZONT)
 else:
models = eval_models(x_ts, np.ones(HORIZONT), m_list)
 y_hat = models[0]['y_pred']
y_hat = y_hat[2:]
for m, y in enumerate(y_hat):
res.append({'name': page, 'horisont_day': m, 'Visits': y})
 res_df = pd.DataFrame(res)
 subm = res_df.merge(keys, how='left', on=['name', 'horisont_day'])
 subm['Visits'] = subm['Visits'].map(np.round ).astype(int)
 subm_df = subm[['Id', 'Visits']].sort_values(by='Id')
 subm_df.to_csv('subm.csv', encoding='utf-8', index=False)
<define_variables>
train_names_png = [TRAIN_DN/f for f in os.listdir(PATH/TRAIN_DN)]
train_names = list(seg.index.values)
 masks_names_png = [MASKS_DN/f for f in os.listdir(PATH/MASKS_DN)]
test_names_png = [TEST/f for f in os.listdir(PATH/TEST)]
col_list = ['minutes_past', 'radardist_km', 'Ref', 'Ref_5x5_10th',
'Ref_5x5_50th', 'Ref_5x5_90th', 'RefComposite', 'RefComposite_5x5_10th',
'RefComposite_5x5_50th', 'RefComposite_5x5_90th', 'RhoHV',
'RhoHV_5x5_10th', 'RhoHV_5x5_50th', 'RhoHV_5x5_90th', 'Zdr',
'Zdr_5x5_10th', 'Zdr_5x5_50th', 'Zdr_5x5_90th', 'Kdp', 'Kdp_5x5_10th',
'Kdp_5x5_50th', 'Kdp_5x5_90th', 'Expected']
d = {c : np.float32 for c in col_list}<load_from_csv>
with ThreadPoolExecutor(4)as e: e.map(resize_mask, train_names_png )
train = pd.read_csv(".. /input/how-much-did-it-rain-ii/train.zip", dtype=d)
 train<filter>
with ThreadPoolExecutor(4)as e: e.map(resize_mask, masks_names_png )
train.loc[train["Id"] == 862571]<filter>
with ThreadPoolExecutor(4)as e: e.map(resize_mask, test_names_png )
train.loc[train["Id"] == 5]<count_missing_values>
PATH = PATH128
train.isna().sum()<set_options>
from datasets import CustomDataset
pd.set_option('display.float_format', lambda x: '%.3f' % x )<compute_test_metric>
def dice(pred, targs):
pred =(pred>0 ).float()
return 2.*(pred*targs ).sum() /(pred+targs ).sum()
def IoU_np(pred, targs, thres=0):
pred =(pred>thres)
 intersection =(pred*targs ).sum()
return intersection /(( pred+targs ).sum() - intersection + 1.0)
 def IoU(pred, targs, thres=0):
pred =(pred>thres ).float()
intersection =(pred*targs ).sum()
return intersection /(( pred+targs ).sum() - intersection + 1.0 )
print(stats.percentileofscore(train["Expected"], 106))<drop_column>
def get_base() :
layers = cut_model(f(True), cut)
 return nn.Sequential(*layers)
 def load_pretrained(model, path):
weights = torch.load(PRETRAINED, map_location=lambda storage, loc: storage)
 model.load_state_dict(weights, strict=False)
 return model
train.drop(train[train["Expected"] >= 106].index, inplace=True)
 train<create_dataframe>
class SaveFeatures() :
features=None
def __init__(self, m): self.hook = m.register_forward_hook(self.hook_fn)
 def hook_fn(self, module, input, output): self.features = output
def remove(self): self.hook.remove()
train_grouped = train.groupby('Id')
 target = pd.DataFrame(train_grouped['Expected'].mean() )<prepare_output>
class UnetBlock(nn.Module):
def __init__(self, up_in, x_in, n_out):
super().__init__()
up_out = x_out = n_out//2
self.x_conv = nn.Conv2d(x_in, x_out, 1)
 self.tr_conv = nn.ConvTranspose2d(up_in, up_out, 2, stride=2)
 self.bn = nn.BatchNorm2d(n_out)
 def forward(self, up_p, x_p):
up_p = self.tr_conv(up_p)
 x_p = self.x_conv(x_p)
 cat_p = torch.cat([up_p,x_p], dim=1)
 return self.bn(F.relu(cat_p))
target.reset_index(inplace=True)
 target = target["Expected"]
target<categorify>
class UnetModel() :
def __init__(self,model,name='unet'):
self.model,self.name = model,name
def get_layer_groups(self, precompute):
lgs = list(split_by_idxs(children(self.model.rn), [lr_cut]))
return lgs + [children(self.model)[1:]]
def pad_series(X, target_len=19):
seq_len = X.shape[0]
pad_size = target_len-seq_len
if(pad_size > 0):
X = np.pad(X,(( 0,pad_size),(0,0)) , 'constant', constant_values=0.)
return X, seq_len<prepare_x_and_y>
x_names = [f'{x}.png' for x in train_names]
x_names_path = np.array([str(TRAIN_DN/x)for x in x_names])
 y_names = [x for x in x_names]
y_names_path = np.array([str(MASKS_DN/x)for x in x_names] )
INPUT_WIDTH = 19
data_size = len(train_grouped)
 X_train = np.empty(( data_size, INPUT_WIDTH, 22))
seq_lengths = np.zeros(data_size)
 y_train = np.zeros(data_size)
 i = 0
for _, group in train_grouped:
X = group.values
seq_len = X.shape[0]
X_train[i,:seq_len,:] = X[:,1:23]
y_train[i] = X[0,23]
i += 1
del X
del train_grouped<split>
aug_tfms = [RandomRotate(4, tfm_y=TfmType.CLASS),
RandomFlip(tfm_y=TfmType.CLASS),
RandomLighting(0.05, 0.05, tfm_y=TfmType.CLASS)]
x_train, x_valid, y_train, y_valid = train_test_split(X_train, target, random_state=42, shuffle=True )<prepare_x_and_y>
lr=3e-3
wd=1e-7
lrs = np.array([lr/100,lr/10,lr])
 n_folds = 10
out=np.zeros(( 18000,sz,sz))
alpha = 0
for i in range(n_folds):
val_size = 4000//n_folds
val_idxs=list(range(i*val_size,(i+1)*val_size)) 
(( val_x,trn_x),(val_y,trn_y)) = split_by_idx(val_idxs, x_names_path, y_names_path)
 test_x = np.array(test_names_png)
 tfms = tfms_from_model(resnet34, sz=sz, pad=0, crop_type=CropType.NO, tfm_y=TfmType.CLASS, aug_tfms=aug_tfms)
 datasets = ImageData.get_ds(CustomDataset,(trn_x,trn_y),(val_x,val_y), tfms,(test_x, test_x), path=PATH)
 md = ImageData(PATH, datasets, bs=64, num_workers=nw, classes=None)
 denorm = md.trn_ds.denorm
f = resnet34
cut,lr_cut = model_meta[f]
m_base = load_pretrained(get_base() ,PRETRAINED)
 m = to_gpu(Unet34(m_base))
models = UnetModel(m)
 learn = ConvLearner(md, models, tmp_name=TMP, models_name=MODEL)
 learn.opt_fn=optim.Adam
learn.crit = lovasz_hinge
learn.metrics=[accuracy_thresh(0.5),dice, IoU]
learn.freeze_to(2)
 learn.fit(lr,2,wds=wd,cycle_len=10,use_clr_beta=(10,10, 0.85, 0.9))
learn.unfreeze()
learn.fit(lrs, 3, wds=wd, cycle_len=10,use_clr_beta=(10,10, 0.85, 0.9))
print(f'computing test set: {i}')
 out+=learn.predict(is_test=True)
 print('Computing optimal threshold')
 preds, targs = learn.predict_with_targs()
IoUs=[]
for a in np.arange(0, 1, 0.1):
IoUs.append(IoU_np(preds, targs, a))
IoU_max = np.array(IoUs ).argmax()
print(f'optimal Threshold: {IoU_max/10.0}')
 alpha+=IoU_max/10.0
del X_train
del target
train_data = tf.data.Dataset.from_tensor_slices(( x_train, y_train))
valid_data = tf.data.Dataset.from_tensor_slices(( x_valid, y_valid))
train_data = train_data.batch(32)
 valid_data = valid_data.batch(32)
 del x_train
del x_valid
del y_train
del y_valid<choose_model_class>
out = out/n_folds
alpha = alpha/n_folds
def create_model(shape=(19, 22)) :
tfkl = tf.keras.layers
model = tf.keras.Sequential([
tfkl.Bidirectional(( tfkl.LSTM(128, return_sequences=True)) , input_shape=shape),
tfkl.Bidirectional(tfkl.LSTM(64)) ,
tfkl.Dense(64, activation="linear"),
tfkl.Dense(1, activation="linear")
 ])
 model.compile(loss='mean_absolute_error', optimizer="adam")
 return model
model = create_model()
model.summary()<train_model>
def rle_encode(im):

 pixels = im.flatten(order='F')
 pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
model.fit(train_data, epochs=100, validation_data=valid_data,
callbacks=[tf.keras.callbacks.ReduceLROnPlateau() ,
tf.keras.callbacks.EarlyStopping(patience = 10),
tf.keras.callbacks.ModelCheckpoint("model.h5", save_best_only=True)] )<load_from_csv>
sub = pd.DataFrame(list(zip(name_list, tmp_list)) , columns = ['id', 'rle_mask'] )
col_list.pop()
d = {c: np.float32 for c in col_list}
test = pd.read_csv(".. /input/how-much-did-it-rain-ii/test.zip", dtype=d)
 test[test.columns[1:]] = test[test.columns[1:]].astype(np.float32)
 test_ids = test['Id'].unique()
test = test.reset_index(drop=True)
 test.fillna(0.0, inplace=True)
 test_groups = test.groupby("Id")
 test_size = len(test_groups)
 X_test = np.zeros(( test_size, INPUT_WIDTH, 22), dtype=np.float32)
 i = 0
for _, group in test_groups:
X = group.values
seq_len = X.shape[0]
X_test[i,:seq_len,:] = X[:,1:23]
i += 1
del X
del test_groups
X_test.shape<load_from_csv>
sub.to_csv('submission.csv', index=False )
<save_to_csv><EOS> 
sub.to_csv('submission.csv', index=False )