Kaggle competition: Mechanisms of Action (MoA) Prediction
TOP_PREDICTIONS = 6
fig, m_axs = plt.subplots(TOP_PREDICTIONS, 3, figsize =(14, TOP_PREDICTIONS*4))
[c_ax.axis('off')for c_ax in m_axs.flatten() ]
def raw_prediction(img, path=test_image_dir):
c_img = imread(os.path.join(path, c_img_name))
c_img = np.expand_dims(c_img, 0)/255.0
cur_seg = fullres_model.predict(c_img)[0]
return cur_seg, c_img[0]
for(ax1, ax2, ax3), c_img_name in zip(m_axs, sub.ImageId.unique() [:TOP_PREDICTIONS]):
pred, c_img = raw_prediction(c_img_name)
 ax1.imshow(c_img)
 ax1.set_title('Image: ' + c_img_name)
 ax2.imshow(pred[...,0], cmap=get_cmap('jet'))
ax2.set_title('Prediction')
 ax3.imshow(masks_as_color(sub.query('ImageId=="{}"'.format(c_img_name)) ['EncodedPixels']))
ax3.set_title('Masks' )<save_to_csv>
warnings.filterwarnings('ignore' )
sub1 = pd.read_csv('.. /input/airbus-ship-detection/sample_submission_v2.csv')
 sub1 = pd.DataFrame(np.setdiff1d(sub1['ImageId'].unique() , sub['ImageId'].unique() , assume_unique=True), columns=['ImageId'])
 sub1['EncodedPixels'] = None
print(len(sub1), len(sub))
sub = pd.concat([sub, sub1])
 print(len(sub))
sub.to_csv('blended_submission.csv', index=False)
 sub.head()<define_search_space>
exp_name="original_torch_moa_5_folds_continued"
BATCH_SIZE = 48
EDGE_CROP = 16
GAUSSIAN_NOISE = 0.1
UPSAMPLE_MODE = 'SIMPLE'
NET_SCALING =(1, 1)
 IMG_SCALING =(3, 3)
 VALID_IMG_COUNT = 900
MAX_TRAIN_STEPS = 9
MAX_TRAIN_EPOCHS = 99
AUGMENT_BRIGHTNESS = False<categorify>
def reduce_mem_usage(props):
start_mem_usg = props.memory_usage().sum() / 1024**2
print("Memory usage of properties dataframe is :",start_mem_usg," MB")
 NAlist = []
for col in props.columns:
if props[col].dtype != object:
IsInt = False
mx = props[col].max()
mn = props[col].min()
if not np.isfinite(props[col] ).all() :
NAlist.append(col)
 props[col].fillna(mn-1,inplace=True)
 asint = props[col].fillna(0 ).astype(np.int64)
 result =(props[col] - asint)
 result = result.sum()
if result > -0.01 and result < 0.01:
IsInt = True
if IsInt:
if mn >= 0:
if mx < 255:
props[col] = props[col].astype(np.uint8)
 elif mx < 65535:
props[col] = props[col].astype(np.uint16)
 elif mx < 4294967295:
props[col] = props[col].astype(np.uint32)
 else:
props[col] = props[col].astype(np.uint64)
 else:
if mn > np.iinfo(np.int8 ).min and mx < np.iinfo(np.int8 ).max:
props[col] = props[col].astype(np.int8)
 elif mn > np.iinfo(np.int16 ).min and mx < np.iinfo(np.int16 ).max:
props[col] = props[col].astype(np.int16)
 elif mn > np.iinfo(np.int32 ).min and mx < np.iinfo(np.int32 ).max:
props[col] = props[col].astype(np.int32)
 elif mn > np.iinfo(np.int64 ).min and mx < np.iinfo(np.int64 ).max:
props[col] = props[col].astype(np.int64)
 else:
props[col] = props[col].astype(np.float32)
 print("___MEMORY USAGE AFTER COMPLETION:___")
 mem_usg = props.memory_usage().sum() / 1024**2
print("Memory usage is: ",mem_usg," MB")
 print("This is ",100*mem_usg/start_mem_usg,"% of the initial size")
 return props, NAlist
montage_rgb = lambda x: np.stack([montage(x[:, :, :, i])for i in range(x.shape[3])], -1)
 ship_dir = '.. /input'
train_image_dir = os.path.join(ship_dir, 'train')
 test_image_dir = os.path.join(ship_dir, 'test')
 def multi_rle_encode(img, **kwargs):

 labels = label(img)
 if img.ndim > 2:
return [rle_encode(np.sum(labels==k, axis=2), **kwargs)for k in np.unique(labels[labels>0])]
else:
return [rle_encode(labels==k, **kwargs)for k in np.unique(labels[labels>0])]
def rle_encode(img, min_max_threshold=1e-3, max_mean_threshold=None):

 if np.max(img)< min_max_threshold:
return ''
if max_mean_threshold and np.mean(img)> max_mean_threshold:
return ''
pixels = img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def rle_decode(mask_rle, shape=(768, 768)) :

 s = mask_rle.split()
starts, lengths = [np.asarray(x, dtype=int)for x in(s[0:][::2], s[1:][::2])]
starts -= 1
ends = starts + lengths
img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 for lo, hi in zip(starts, ends):
img[lo:hi] = 1
return img.reshape(shape ).T
def masks_as_image(in_mask_list):
all_masks = np.zeros(( 768, 768), dtype = np.uint8)
 for mask in in_mask_list:
if isinstance(mask, str):
all_masks |= rle_decode(mask)
 return all_masks
def masks_as_color(in_mask_list):
all_masks = np.zeros(( 768, 768), dtype = np.float)
 scale = lambda x:(len(in_mask_list)+x+1)/(len(in_mask_list)*2)
 for i,mask in enumerate(in_mask_list):
if isinstance(mask, str):
all_masks[:,:] += scale(i)* rle_decode(mask)
 return all_masks<load_from_csv>
def train_short_form_loader(feature_file,target_file,extra_target_file=None):

 train_features = pd.read_csv(feature_file)
 train_targets = pd.read_csv(target_file)
 train_features,_= reduce_mem_usage(train_features)
 train_targets,_ = reduce_mem_usage(train_targets)
 if extra_target_file is not None:
extra_targets = pd.read_csv(extra_target_file)
 extra_targets,_ = reduce_mem_usage(extra_targets)
 train_targets = pd.concat([train_targets,extra_targets])
 del extra_targets
targets = train_targets.columns[1:]
train_melt=train_targets.merge(train_features,how="left",on="sig_id")
 del train_features,train_targets
train_melt.set_index("sig_id",inplace=True)
 train_melt["cp_type"]= train_melt["cp_type"].astype('category')
 train_melt["cp_dose"]= train_melt["cp_dose"].astype('category')
 return train_melt , targets
def test_short_form_loader(feature_file):

 train_features = pd.read_csv(feature_file)
 train_features,_= reduce_mem_usage(train_features)
 train_melt = train_features.copy()
del train_features
train_melt.set_index("sig_id",inplace=True)
 train_melt["cp_type"]= train_melt["cp_type"].astype('category')
 train_melt["cp_dose"]= train_melt["cp_dose"].astype('category')
 return train_melt
masks = pd.read_csv(os.path.join('.. /input/', 'train_ship_segmentations.csv'))
not_empty = pd.notna(masks.EncodedPixels)
 print(not_empty.sum() , 'masks in', masks[not_empty].ImageId.nunique() , 'images')
 print(( ~not_empty ).sum() , 'empty images in', masks.ImageId.nunique() , 'total images')
 masks.head()<categorify>
train,target=train_short_form_loader('.. /input/lish-moa/train_features.csv','.. /input/lish-moa/train_targets_scored.csv' )
fig,(ax1, ax2, ax3, ax4)= plt.subplots(1, 4, figsize =(16, 5))
rle_0 = masks.query('ImageId=="00021ddc3.jpg"')['EncodedPixels']
img_0 = masks_as_image(rle_0)
 ax1.imshow(img_0)
 ax1.set_title('Mask as image')
 rle_1 = multi_rle_encode(img_0)
 img_1 = masks_as_image(rle_1)
 ax2.imshow(img_1)
 ax2.set_title('Re-encoded')
 img_c = masks_as_color(rle_0)
 ax3.imshow(img_c)
 ax3.set_title('Masks in colors')
 img_c = masks_as_color(rle_1)
 ax4.imshow(img_c)
 ax4.set_title('Re-encoded in colors')
 print('Check Decoding->Encoding',
'RLE_0:', len(rle_0), '->',
'RLE_1:', len(rle_1))
print(np.sum(img_0 - img_1), 'error' )<feature_engineering>
masks['ships'] = masks['EncodedPixels'].map(lambda c_row: 1 if isinstance(c_row, str)else 0)
 unique_img_ids = masks.groupby('ImageId' ).agg({'ships': 'sum'} ).reset_index()
unique_img_ids['has_ship'] = unique_img_ids['ships'].map(lambda x: 1.0 if x>0 else 0.0)
 unique_img_ids['has_ship_vec'] = unique_img_ids['has_ship'].map(lambda x: [x])
 unique_img_ids['file_size_kb'] = unique_img_ids['ImageId'].map(lambda c_img_id:
os.stat(os.path.join(train_image_dir,
c_img_id)).st_size/1024)
 unique_img_ids = unique_img_ids[unique_img_ids['file_size_kb'] > 50]
unique_img_ids['file_size_kb'].hist()
masks.drop(['ships'], axis=1, inplace=True)
 unique_img_ids.sample(7 )<split>
train_ids, valid_ids = train_test_split(balanced_train_df,
test_size = 0.2,
stratify = balanced_train_df['ships'])
 train_df = pd.merge(masks, train_ids)
 valid_df = pd.merge(masks, valid_ids)
 print(train_df.shape[0], 'training masks')
 print(valid_df.shape[0], 'validation masks' )<categorify>
def seed_everything(seed=42):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
def make_image_gen(in_df, batch_size = BATCH_SIZE):
all_batches = list(in_df.groupby('ImageId'))
out_rgb = []
out_mask = []
while True:
np.random.shuffle(all_batches)
 for c_img_id, c_masks in all_batches:
rgb_path = os.path.join(train_image_dir, c_img_id)
 c_img = imread(rgb_path)
 c_mask = np.expand_dims(masks_as_image(c_masks['EncodedPixels'].values), -1)
 if IMG_SCALING is not None:
c_img = c_img[::IMG_SCALING[0], ::IMG_SCALING[1]]
c_mask = c_mask[::IMG_SCALING[0], ::IMG_SCALING[1]]
out_rgb += [c_img]
out_mask += [c_mask]
if len(out_rgb)>=batch_size:
yield np.stack(out_rgb, 0)/255.0, np.stack(out_mask, 0)
 out_rgb, out_mask=[], []<train_model>
def supress_controls(df):
df = df[train['cp_type']!='ctl_vehicle']
df = df.drop('cp_type', axis=1)
 return df
train_gen = make_image_gen(train_df)
 train_x, train_y = next(train_gen)
 print('x', train_x.shape, train_x.min() , train_x.max())
 print('y', train_y.shape, train_y.min() , train_y.max() )<train_model>
def map_controls(df):
df['cp_type']=df['cp_type'].map({'ctl_vehicle': 0, 'trt_cp': 1})
 df['cp_type']=df['cp_type'].astype(int)
 return df
def map_dose(df):
df['cp_dose']=df['cp_dose'].map({'D1': 1, 'D2': 0})
 df['cp_dose']=df['cp_dose'].astype(int)
 return df
def map_time(df):
df['cp_time']=df['cp_time'].map({24: 0, 48: 1, 72: 2})
 df['cp_time']=df['cp_time'].astype(int)
 return df
%%time
valid_x, valid_y = next(make_image_gen(valid_df, VALID_IMG_COUNT))
print(valid_x.shape, valid_y.shape )<set_options>
def build_preprocess(preprocesses=[map_time,map_dose,map_controls]):
def preprocesser(df):
for proc in preprocesses:
df = proc(df)
 return df
return preprocesser
dg_args = dict(featurewise_center = False,
samplewise_center = False,
rotation_range = 45,
width_shift_range = 0.1,
height_shift_range = 0.1,
shear_range = 0.01,
zoom_range = [0.9, 1.25],
horizontal_flip = True,
vertical_flip = True,
fill_mode = 'reflect',
data_format = 'channels_last')
 if AUGMENT_BRIGHTNESS:
dg_args[' brightness_range'] = [0.5, 1.5]
image_gen = ImageDataGenerator(**dg_args)
 if AUGMENT_BRIGHTNESS:
dg_args.pop('brightness_range')
 label_gen = ImageDataGenerator(**dg_args)
 def create_aug_gen(in_gen, seed = None):
np.random.seed(seed if seed is not None else np.random.choice(range(9999)))
 for in_x, in_y in in_gen:
seed = np.random.choice(range(9999))
g_x = image_gen.flow(255*in_x,
batch_size = in_x.shape[0],
seed = seed,
shuffle=True)
 g_y = label_gen.flow(in_y,
batch_size = in_x.shape[0],
seed = seed,
shuffle=True)
 yield next(g_x)/255.0, next(g_y )<set_options>
preprocess_data=build_preprocess()
gc.collect()<define_search_model>
def upsample_conv(filters, kernel_size, strides, padding):
return layers.Conv2DTranspose(filters, kernel_size, strides=strides, padding=padding)
 def upsample_simple(filters, kernel_size, strides, padding):
return layers.UpSampling2D(strides)
 if UPSAMPLE_MODE=='DECONV':
upsample=upsample_conv
else:
upsample=upsample_simple
input_img = layers.Input(t_x.shape[1:], name = 'RGB_Input')
 pp_in_layer = input_img
if NET_SCALING is not None:
pp_in_layer = layers.AvgPool2D(NET_SCALING )(pp_in_layer)
 pp_in_layer = layers.GaussianNoise(GAUSSIAN_NOISE )(pp_in_layer)
 pp_in_layer = layers.BatchNormalization()(pp_in_layer)
 c1 = layers.Conv2D(8,(3, 3), activation='relu', padding='same' )(pp_in_layer)
 c1 = layers.Conv2D(8,(3, 3), activation='relu', padding='same' )(c1)
 p1 = layers.MaxPooling2D(( 2, 2))(c1)
 c2 = layers.Conv2D(16,(3, 3), activation='relu', padding='same' )(p1)
 c2 = layers.Conv2D(16,(3, 3), activation='relu', padding='same' )(c2)
 p2 = layers.MaxPooling2D(( 2, 2))(c2)
 c3 = layers.Conv2D(32,(3, 3), activation='relu', padding='same' )(p2)
 c3 = layers.Conv2D(32,(3, 3), activation='relu', padding='same' )(c3)
 p3 = layers.MaxPooling2D(( 2, 2))(c3)
 c4 = layers.Conv2D(64,(3, 3), activation='relu', padding='same' )(p3)
 c4 = layers.Conv2D(64,(3, 3), activation='relu', padding='same' )(c4)
 p4 = layers.MaxPooling2D(pool_size=(2, 2))(c4)
 c5 = layers.Conv2D(128,(3, 3), activation='relu', padding='same' )(p4)
 c5 = layers.Conv2D(128,(3, 3), activation='relu', padding='same' )(c5)
 u6 = upsample(64,(2, 2), strides=(2, 2), padding='same' )(c5)
 u6 = layers.concatenate([u6, c4])
 c6 = layers.Conv2D(64,(3, 3), activation='relu', padding='same' )(u6)
 c6 = layers.Conv2D(64,(3, 3), activation='relu', padding='same' )(c6)
 u7 = upsample(32,(2, 2), strides=(2, 2), padding='same' )(c6)
 u7 = layers.concatenate([u7, c3])
 c7 = layers.Conv2D(32,(3, 3), activation='relu', padding='same' )(u7)
 c7 = layers.Conv2D(32,(3, 3), activation='relu', padding='same' )(c7)
 u8 = upsample(16,(2, 2), strides=(2, 2), padding='same' )(c7)
 u8 = layers.concatenate([u8, c2])
 c8 = layers.Conv2D(16,(3, 3), activation='relu', padding='same' )(u8)
 c8 = layers.Conv2D(16,(3, 3), activation='relu', padding='same' )(c8)
 u9 = upsample(8,(2, 2), strides=(2, 2), padding='same' )(c8)
 u9 = layers.concatenate([u9, c1], axis=3)
 c9 = layers.Conv2D(8,(3, 3), activation='relu', padding='same' )(u9)
 c9 = layers.Conv2D(8,(3, 3), activation='relu', padding='same' )(c9)
 d = layers.Conv2D(1,(1, 1), activation='sigmoid' )(c9)
 if NET_SCALING is not None:
d = layers.UpSampling2D(NET_SCALING )(d)
 seg_model = models.Model(inputs=[input_img], outputs=[d])
 seg_model.summary()<compute_test_metric>
def multifold_indexer(train,target_columns,n_splits=10,random_state=12347,**kwargs):
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=n_splits,random_state=random_state,**kwargs)
 folds[ 'kfold']=0
for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=train[target_columns])) :
folds.iloc[v_idx,-1] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 return folds
def IoU(y_true, y_pred, eps=1e-6):
if np.max(y_true)== 0.0:
return IoU(1-y_true, 1-y_pred)
 intersection = K.sum(y_true * y_pred, axis=[1,2,3])
 union = K.sum(y_true, axis=[1,2,3])+ K.sum(y_pred, axis=[1,2,3])- intersection
return -K.mean(( intersection + eps)/(union + eps), axis=0 )<choose_model_class>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
weight_path="{}_weights.best.hdf5".format('seg_model')
 checkpoint = ModelCheckpoint(weight_path, monitor='val_loss', verbose=1, save_best_only=True, mode='min', save_weights_only=True)
 reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.33,
patience=1, verbose=1, mode='min',
min_delta=0.0001, cooldown=0, min_lr=1e-8)
 early = EarlyStopping(monitor="val_loss", mode="min", verbose=2,
patience=20)
 callbacks_list = [checkpoint, early, reduceLROnPlat]<train_model>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
if not scheduler.__class__ == torch.optim.lr_scheduler.ReduceLROnPlateau:
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, scheduler, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 if scheduler.__class__ == torch.optim.lr_scheduler.ReduceLROnPlateau:
scheduler.step(final_loss)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
def fit() :
seg_model.compile(optimizer=Adam(1e-3, decay=1e-6), loss=IoU, metrics=['binary_accuracy'])
 step_count = min(MAX_TRAIN_STEPS, train_df.shape[0]//BATCH_SIZE)
 aug_gen = create_aug_gen(make_image_gen(train_df))
loss_history = [seg_model.fit_generator(aug_gen,
steps_per_epoch=step_count,
epochs=MAX_TRAIN_EPOCHS,
validation_data=(valid_x, valid_y),
callbacks=callbacks_list,
workers=1 
)]
return loss_history
while True:
loss_history = fit()
if np.min([mh.history['val_loss'] for mh in loss_history])< -0.2:
break<load_pretrained>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
seg_model.load_weights(weight_path)
 seg_model.save('seg_model.h5' )<predict_on_test>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size,drop_rate1=0.2,drop_rate2=0.5,drop_rate3=0.8):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(drop_rate1)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(drop_rate2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(drop_rate3)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
pred_y = seg_model.predict(valid_x)
 print(pred_y.shape, pred_y.min(axis=0 ).max() , pred_y.max(axis=0 ).min() , pred_y.mean() )<choose_model_class>
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
if IMG_SCALING is not None:
fullres_model = models.Sequential()
fullres_model.add(layers.AvgPool2D(IMG_SCALING, input_shape =(None, None, 3)))
 fullres_model.add(seg_model)
 fullres_model.add(layers.UpSampling2D(IMG_SCALING))
else:
fullres_model = seg_model
fullres_model.save('fullres_model.h5' )<predict_on_test>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 100
BATCH_SIZE = 512
LEARNING_RATE = 2e-4
WEIGHT_DECAY = 2e-7
NFOLDS = 5
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
hidden_size=512
def raw_prediction(img, path=test_image_dir):
c_img = imread(os.path.join(path, c_img_name))
c_img = np.expand_dims(c_img, 0)/255.0
cur_seg = fullres_model.predict(c_img)[0]
return cur_seg, c_img[0]
def smooth(cur_seg):
return binary_opening(cur_seg>0.99, np.expand_dims(disk(2), -1))
def predict(img, path=test_image_dir):
cur_seg, c_img = raw_prediction(img, path=path)
 return smooth(cur_seg), c_img
samples = valid_df.groupby('ships' ).apply(lambda x: x.sample(1))
fig, m_axs = plt.subplots(samples.shape[0], 4, figsize =(15, samples.shape[0]*4))
[c_ax.axis('off')for c_ax in m_axs.flatten() ]
for(ax1, ax2, ax3, ax4), c_img_name in zip(m_axs, samples.ImageId.values):
first_seg, first_img = raw_prediction(c_img_name, train_image_dir)
 ax1.imshow(first_img)
 ax1.set_title('Image: ' + c_img_name)
 ax2.imshow(first_seg[:, :, 0], cmap=get_cmap('jet'))
ax2.set_title('Model Prediction')
 reencoded = masks_as_color(multi_rle_encode(smooth(first_seg)[:, :, 0]))
ax3.imshow(reencoded)
 ax3.set_title('Prediction Masks')
 ground_truth = masks_as_color(masks.query('ImageId=="{}"'.format(c_img_name)) ['EncodedPixels'])
 ax4.imshow(ground_truth)
 ax4.set_title('Ground Truth')
 fig.savefig('validation.png' )<define_variables>
def initialize_from_past_model(model,past_model_file):
pretrained_dict = torch.load(past_model_file)
 model_dict = model.state_dict()
pretrained_dict['dense3.bias']=pretrained_dict['dense3.bias'][:206]
pretrained_dict['dense3.weight_g']=pretrained_dict['dense3.weight_g'][:206]
pretrained_dict['dense3.weight_v']=pretrained_dict['dense3.weight_v'][:206]
pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict}
model_dict.update(pretrained_dict)
 model.load_state_dict(pretrained_dict)
test_paths = np.array(os.listdir(test_image_dir))
print(len(test_paths), 'test images found' )<predict_on_test>
def run_training(X_train,y_train,X_valid,y_valid,X_test,fold, seed,verbose=False,**kwargs):
seed_everything(seed)
 train_dataset = MoADataset(X_train, y_train)
 valid_dataset = MoADataset(X_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features= X_train.shape[1] ,
num_targets= y_train.shape[1],
hidden_size=hidden_size,**kwargs 
)
 model.to(DEVICE)
 initialize_from_past_model(model,f".. /input/pytorchauxtargets5f1/FOLD{fold}_original_torch_moa_5_folds_AUX.pth")
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,patience=3)
 loss_val = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 if verbose:
print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model,scheduler, loss_val, validloader, DEVICE)
 if verbose:
print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if verbose:
print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_{exp_name}.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
testdataset = TestDataset(X_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model.to(DEVICE)
 predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
def pred_encode(img, **kwargs):
cur_seg, _ = predict(img)
 cur_rles = multi_rle_encode(cur_seg, **kwargs)
 return [[img, rle] for rle in cur_rles if rle is not None]
out_pred_rows = []
for c_img_name in tqdm_notebook(test_paths[:30000]):
out_pred_rows += pred_encode(c_img_name, min_max_threshold=1.0 )<prepare_output>
def run_k_fold(folds,target_cols,test,NFOLDS, seed,verbose=False,**kwargs):
train = preprocess_data(folds)
 test_ = preprocess_data(test)
 oof = train[target_cols].copy()
predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
train_df = train[train['kfold'] != fold]
valid_df = train[train['kfold'] == fold]
feature_cols = [col for col in train_df.columns if not(col in target_cols.to_list() +['kfold'])]
X_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
X_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
X_test = test_[feature_cols].values
oof_, pred_ = run_training(X_train,y_train,X_valid,y_valid,X_test,fold, seed,verbose,**kwargs)
 oof[train['kfold'] == fold] = oof_
predictions += pred_ / NFOLDS
return oof, predictions
sub = pd.DataFrame(out_pred_rows)
 sub.columns = ['ImageId', 'EncodedPixels']
sub = sub[sub.EncodedPixels.notnull() ]
sub.head()<save_to_csv>
params ={'drop_rate1':0.5,'drop_rate2':0.2,'drop_rate3':0.2}
sub1 = pd.read_csv('.. /input/sample_submission.csv')
 sub1 = pd.DataFrame(np.setdiff1d(sub1['ImageId'].unique() , sub['ImageId'].unique() , assume_unique=True), columns=['ImageId'])
 sub1['EncodedPixels'] = None
print(len(sub1), len(sub))
sub = pd.concat([sub, sub1])
 print(len(sub))
sub.to_csv('submission.csv', index=False)
 sub.head()<load_from_csv>
SEED = [0]
train,target_cols = train_short_form_loader('.. /input/lish-moa/train_features.csv','.. /input/lish-moa/train_targets_scored.csv')
 test = test_short_form_loader(".. /input/lish-moa/test_features.csv")
 oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
folds = multifold_indexer(train,target_cols,n_splits=NFOLDS)
 oof_, predictions_ = run_k_fold(folds,target_cols,test,NFOLDS, seed,verbose=True,**params)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 test[target_cols] = predictions
train = pd.read_csv(".. /input/train/train.csv")
 test = pd.read_csv(".. /input/test/test.csv")
 breeds = pd.read_csv(".. /input/breed_labels.csv")
 colors = pd.read_csv(".. /input/color_labels.csv")
 states = pd.read_csv(".. /input/state_labels.csv" )<data_type_conversions>
y_true = train[target_cols].values
y_pred = oof
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred.iloc[:, i],labels=[0,1])
 score +=(score_ / len(target_cols))
print("CV log_loss: ", score)
<feature_engineering><EOS> 
sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission.set_index('sig_id',inplace=True)
 test_features.set_index('sig_id',inplace=True)
 test_features = test_features.loc[sample_submission.index]
sub = sample_submission.drop(columns=target_cols ).merge(test[target_cols], on='sig_id', how='left' ).fillna(0)
 sub.loc[test_features['cp_type']=='ctl_vehicle', target_cols] =0
sub.to_csv('./submission.csv', index=True )