Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<set_options>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
import tensorflow as tf
from kerastuner import HyperModel
from kerastuner.tuners import RandomSearch
from tensorflow.python.keras.callbacks import ModelCheckpoint
import seaborn as sns
from pylab import rcParams
from sklearn.metrics import confusion_matrix
def swish(x, beta=1.0):
return x * torch.sigmoid(beta*x)
 def get_all_cams(batch_cam_scaled, model, scales, ims_per_batch):
bs = ims_per_batch
with torch.no_grad() :
ori_w, ori_h = CFG.size, CFG.size
strided_up_size =(CFG.size, CFG.size)
 all_scale_cams = torch.from_numpy(np.zeros(( bs, len(scales), 19, CFG.size, CFG.size)) ).cuda()
all_scale_preds = torch.from_numpy(np.zeros(( bs, len(scales), 19)) ).cuda()
num_channels = 4
for i, images in enumerate(batch_cam_scaled):

 with torch.cuda.amp.autocast() :
logits, features = model(images, with_cam=True)
 features = swish(features)
 logits = logits.reshape(bs*4//bs//4, 4, bs, 19 ).mean(1 ).view(bs*4//4, 19)
 all_scale_preds[:, i, :] = logits
features = torch.cat([features[0:bs], features[bs:bs*2].flip(2), features[bs*2:bs*3].flip(3), torch.flip(features[bs*3:bs*4],(3,2)) ])
 size_feats = features.shape[-1]
features = features.reshape(bs*4//bs//4, 4, bs, 19, size_feats, size_feats ).sum(1 ).view(bs*4//4, 19, size_feats, size_feats)
 cams = F.interpolate(features,(CFG.size, CFG.size), mode='bicubic', align_corners=False)
 all_scale_cams[:, i, :, :, :] = cams
all_logits = np.sum(all_scale_preds.detach().cpu().numpy() , axis=1)
 all_cams = np.sum(all_scale_cams.detach().cpu().numpy() , axis=1)
 print("CAMS DONE")
 return {"hr_cams": all_cams, "logits" : all_logits} 
<define_variables>
rcParams['figure.figsize'] = 15, 15
NUM_CLASSES = 10
INPUT_SHAPE =(28, 28, 1 )
label_names = [
'0-Nucleoplasm',
'1-Nuclear membrane',
'2-Nucleoli',
'3-Nucleoli fibrillar center',
'4-Nuclear speckles',
'5-Nuclear bodies',
'6-Endoplasmic reticulum',
'7-Golgi apparatus',
'8-Intermediate filaments',
'9-Actin filaments',
'10-Microtubules',
'11-Mitotic spindle',
'12-Centrosome',
'13-Plasma membrane',
'14-Mitochondria',
'15-Aggresome',
'16-Cytosol',
'17-Vesicles + cytosolic patterns',
'18-Negative'
]<categorify>
train = pd.read_csv('.. /input/digit-recognizer/train.csv')
 test = pd.read_csv('.. /input/digit-recognizer/test.csv')
 X = train.iloc[:, 1:]
y = train.iloc[:, 0]
def encode_binary_mask(mask: np.ndarray)-> t.Text:

 if mask.dtype != np.bool:
raise ValueError(
 "encode_binary_mask expects a binary mask, received dtype == %s" %
mask.dtype)
 mask = np.squeeze(mask)
 if len(mask.shape)!= 2:
raise ValueError(
 "encode_binary_mask expects a 2d mask, received shape == %s" %
mask.shape)
 mask_to_encode = mask.reshape(mask.shape[0], mask.shape[1], 1)
 mask_to_encode = mask_to_encode.astype(np.uint8)
 mask_to_encode = np.asfortranarray(mask_to_encode)
 encoded_mask = coco_mask.encode(mask_to_encode)[0]["counts"]
binary_str = zlib.compress(encoded_mask, zlib.Z_BEST_COMPRESSION)
 base64_str = base64.b64encode(binary_str)
 return base64_str
def get_all_encoded_cells(mask):
print(mask.shape)
 cell_masks = []
for i in range(1, np.max(mask)+1):
enc_mask = encode_binary_mask(( mask == i))
cell_masks.append(enc_mask)
 return cell_masks<normalization>
X /= 256
test /= 256
X = X.values.reshape(( -1,)+ INPUT_SHAPE)
 test = test.values.reshape(( -1,)+ INPUT_SHAPE )
def resize_mask(mask):
resized_mask = resize_full_mask(mask, CFG.size)
 cell_masks = []
for i in range(1, np.max(mask)+1):
cell_masks.append(( resized_mask == i))
return cell_masks
def resize_full_mask(mask, size):
resized_mask = cv2.resize(mask,(size,size),interpolation=cv2.INTER_NEAREST_EXACT)
 return resized_mask 
<categorify>
class CNNHyperModel(HyperModel):
def __init__(self, input_shape, num_classes):
self.input_shape = input_shape
self.num_classes = num_classes
def build(self, hp):
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Conv2D(8, 3, padding='same', dilation_rate=(3, 3), \
activation='relu', input_shape=self.input_shape))
model.add(tf.keras.layers.Conv2D(16, 3, padding='same', \
activation='relu'))
max_or_avg =hp.Int(
 'max_or_avg',
min_value=1,
max_value=2,
step=1,
default=1)
 three = hp.Fixed('three', 3)
 model.add(tf.keras.layers.AvgPool2D(pool_size=max_or_avg, name=f'first_avg_pool_{max_or_avg}'))
model.add(tf.keras.layers.MaxPool2D(pool_size=three - max_or_avg))
model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.Conv2D(32, 3, padding='same', dilation_rate=(3, 3), \
activation='relu'))
model.add(tf.keras.layers.Conv2D(64, 3, padding='same', \
activation='relu'))
model.add(tf.keras.layers.AvgPool2D(pool_size=max_or_avg, name=f'second_avg_pool_{max_or_avg}'))
model.add(tf.keras.layers.MaxPool2D(pool_size=three - max_or_avg))
model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.Flatten())
 drop_rate = hp.Float('dropout',
min_value=0.3,
max_value=0.7,
default=0.3,
step=0.2, 
)
 model.add(tf.keras.layers.Dropout(rate=drop_rate,
name=f'drop{drop_rate}'))
units=hp.Int('units',
min_value=28,
max_value=4*28,
step=28,
default=28 
)
 model.add(tf.keras.layers.Dense(
 units=units,
name=f'Dense{units}',
activation='relu' 
))
model.add(tf.keras.layers.Dense(self.num_classes))
loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
 optimizer = tf.keras.optimizers.Adam()
model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy'])
 return model
def get_pred_string(mask_probas, cell_masks_fullsize_enc):
assert len(mask_probas)== len(cell_masks_fullsize_enc), "Probas have different length than masks"
string = ""
for enc_mask, mask_proba in zip(cell_masks_fullsize_enc, mask_probas):
for cls, proba in enumerate(mask_proba):
string += str(cls)+ " " + str(proba)+ " " + enc_mask.decode("utf-8")+ " "
return string<load_pretrained>
hypermodel = CNNHyperModel(input_shape=INPUT_SHAPE, num_classes=NUM_CLASSES)
 tuner = RandomSearch(
 hypermodel,
objective='val_accuracy',
max_trials=24,
executions_per_trial=1,
directory='./random_search',
project_name='MNIST' 
)
 tuner.search_space_summary()
PATH_SCALER_GRADBOOST = ".. /input/scaler-and-gradboost"
scaler_resnest0 = pickle.load(open(f"{PATH_SCALER_GRADBOOST}/scaler_resnest0.pkl", 'rb'))
scaler_resnest1 = pickle.load(open(f"{PATH_SCALER_GRADBOOST}/scaler_resnest1.pkl", 'rb'))
scaler_effnet0 = pickle.load(open(f"{PATH_SCALER_GRADBOOST}/scaler_effnet0.pkl", 'rb'))
scaler_effnet1 = pickle.load(open(f"{PATH_SCALER_GRADBOOST}/scaler_effnet1.pkl", 'rb'))
scaler_effnet2 = pickle.load(open(f"{PATH_SCALER_GRADBOOST}/scaler_effnet2.pkl", 'rb'))
model_gradboost_resnest0 = pickle.load(open(f"{PATH_SCALER_GRADBOOST}/GradientBoostingRegressor_resnest0.pkl", 'rb'))
model_gradboost_resnest1 = pickle.load(open(f"{PATH_SCALER_GRADBOOST}/GradientBoostingRegressor_resnest1.pkl", 'rb'))
model_gradboost_effnet0 = pickle.load(open(f"{PATH_SCALER_GRADBOOST}/GradientBoostingRegressor_effnet0.pkl", 'rb'))
model_gradboost_effnet1 = pickle.load(open(f"{PATH_SCALER_GRADBOOST}/GradientBoostingRegressor_effnet1.pkl", 'rb'))
model_gradboost_effnet2 = pickle.load(open(f"{PATH_SCALER_GRADBOOST}/GradientBoostingRegressor_effnet2.pkl", 'rb'))
def get_prob_from_cams_masks(cams, masks, labels, labels_from_labelmodel, verbose=True, typ=None):
print(f"GETTING MODEL {typ}")
 if typ=="resnest0":
scaler = scaler_resnest0
model_gradboost = model_gradboost_resnest0
if typ=="resnest1":
scaler = scaler_resnest1
model_gradboost = model_gradboost_resnest1
if typ=="resnest2":
scaler = scaler_resnest1
model_gradboost = model_gradboost_resnest1
if typ=="effnet0":
scaler = scaler_effnet0
model_gradboost = model_gradboost_effnet0
if typ=="effnet1":
scaler = scaler_effnet1
model_gradboost = model_gradboost_effnet1
if typ=="effnet2":
scaler = scaler_effnet2
model_gradboost = model_gradboost_effnet2
if typ=="effnet3":
scaler = scaler_effnet2
model_gradboost = model_gradboost_effnet2
masks_probas = np.zeros(( len(masks), 19))
for i, mask in enumerate(masks):
for label, cam in enumerate(cams):
cam_by_mask = np.multiply(mask, cam)
 cam_mask_product = np.multiply(cam_by_mask, labels[label])
 masks_probas[i, label] = np.sum(cam_mask_product)
 if verbose:
print(f"MASK: {i} PROB-RAW: {masks_probas[i, :]}")
 print("--------------------------------------")
 std_scaler = preprocessing.RobustScaler().fit(masks_probas.reshape(-1, 1))
for i, mask in enumerate(masks):
std_scaled = std_scaler.transform(masks_probas[i, :].reshape(-1, 1)) [:,0]
model_scaled = scaler.transform(masks_probas[i, :].reshape(-1, 1)) [:,0]
if verbose:
print(f"MASK: {i} STD-Scaled: {std_scaled}")
 print(f"MASK: {i} Model-Scaled: {model_scaled}")
 print("--------------------------------------")
 sigmoid_probas = sigmoid_factor(std_scaled, factor=CFG.sigmoid_factor, move=CFG.sigmoid_move)
 gradboost_probas = model_gradboost.predict(model_scaled.reshape(-1, 1))
if verbose:
print(f"MASK: {i} PROB-SIGMOID: {sigmoid_probas}")
 print(f"MASK: {i} PROB-GradBoost: {gradboost_probas}")
 print("--------------------------------------")
 if CFG.extra_model_for_labels:
masks_probas[i, :] = sigmoid_probas*CFG.split_sigmoid_graboost[0] + gradboost_probas*CFG.split_sigmoid_graboost[1]
masks_probas[i, :] = CFG.split[0] * masks_probas[i, :] + CFG.split[1] * labels_from_labelmodel + CFG.split[2] * labels
if verbose:
print(f"MASK: {i} PROB-WITH-LABELMODEL: {masks_probas[i, :]}")
 return masks_probas<import_modules>
X_train_val, X_test, y_train_val, y_test = train_test_split(X, y, test_size=1/7 )
class Classifier_EffNet(nn.Module, ABC_Model):
def __init__(self, backbone, num_classes=19):
super(Classifier_EffNet, self ).__init__()
self.enet = EfficientNet.from_name(backbone, num_classes=num_classes, in_channels=3, include_top=False)
 dict_sizes = {
'efficientnet-b0' : 1280,
'efficientnet-b1' : 1280,
'efficientnet-b2' : 1408,
'efficientnet-b3' : 1536,
'efficientnet-b4' : 1792,
'efficientnet-b5' : 2048
}
size_conv2d = dict_sizes[backbone]
self.classifier = nn.Conv2d(size_conv2d, num_classes, 1, bias=False)
 self.num_classes = num_classes
self.initialize([self.classifier])
 def forward(self, x, with_cam=False):
x = self.enet.extract_features(x)
 if with_cam:
features = self.classifier(x)
 logits = self.global_average_pooling_2d(features)
 return logits, features
else:
x = self.global_average_pooling_2d(x, keepdims=True)
 logits = self.classifier(x ).view(-1, self.num_classes)
 return logits
class Classifier_EffNet_GREEN(nn.Module, ABC_Model):
def __init__(self, backbone, num_classes=19):
super(Classifier_EffNet_GREEN, self ).__init__()
self.enet = EfficientNet.from_name(backbone, num_classes=num_classes, in_channels=3, include_top=False)
 dict_sizes = {
'efficientnet-b0' : 1280,
'efficientnet-b1' : 1280,
'efficientnet-b2' : 1408,
'efficientnet-b3' : 1536,
'efficientnet-b4' : 1792,
'efficientnet-b5' : 2048,
'efficientnet-b7' : 2560,
}
self.dense = nn.Linear(dict_sizes[backbone],19)
 self.initialize([self.dense])
 def forward(self, x, with_cam=False):
x = self.enet.extract_features(x)
 x = self.global_average_pooling_2d(x)
 logits = self.dense(x)
 return logits
class FixedBatchNorm(nn.BatchNorm2d):
def forward(self, x):
return F.batch_norm(x, self.running_mean, self.running_var, self.weight, self.bias, training=False, eps=self.eps)
 def group_norm(features):
return nn.GroupNorm(4, features)
 class Backbone(nn.Module, ABC_Model):
def __init__(self, model_name, num_classes=20, mode='fix', segmentation=False):
super().__init__()
self.mode = mode
if self.mode == 'fix':
self.norm_fn = FixedBatchNorm
else:
self.norm_fn = nn.BatchNorm2d
if 'resnet' in model_name:
self.model = resnet.ResNet(resnet.Bottleneck, resnet.layers_dic[model_name], strides=(2, 2, 2, 1), batch_norm_fn=self.norm_fn)
 state_dict = model_zoo.load_url(resnet.urls_dic[model_name])
 state_dict.pop('fc.weight')
 state_dict.pop('fc.bias')
 self.model.load_state_dict(state_dict)
 else:
if segmentation:
dilation, dilated = 4, True
else:
dilation, dilated = 2, False
self.model = eval("resnest." + model_name )(pretrained=False, dilated=dilated, dilation=dilation, norm_layer=self.norm_fn)
 del self.model.avgpool
del self.model.fc
self.stage1 = nn.Sequential(self.model.conv1,
self.model.bn1,
self.model.relu,
self.model.maxpool)
 self.stage2 = nn.Sequential(self.model.layer1)
 self.stage3 = nn.Sequential(self.model.layer2)
 self.stage4 = nn.Sequential(self.model.layer3)
 self.stage5 = nn.Sequential(self.model.layer4)
 class Classifier(Backbone):
def __init__(self, model_name, num_classes=20, mode='fix'):
super().__init__(model_name, num_classes, mode)
 self.classifier = nn.Conv2d(2048, num_classes, 1, bias=False)
 self.num_classes = num_classes
self.initialize([self.classifier])
 def forward(self, x, with_cam=False):
x = self.stage1(x)
 x = self.stage2(x)
 x = self.stage3(x)
 x = self.stage4(x)
 x = self.stage5(x)
 if with_cam:
features = self.classifier(x)
 logits = self.global_average_pooling_2d(features)
 return logits, features
else:
x = self.global_average_pooling_2d(x, keepdims=True)
 logits = self.classifier(x ).view(-1, self.num_classes)
 return logits<split>
search_results = tuner.search(X_train_val, y_train_val,
epochs=10, validation_split=1/6,
batch_size=100 )
def get_separate_labels(ims, model, model_for_labels_state, ims_per_batch, verbose=False):
bs = ims_per_batch
model.load_state_dict(model_for_labels_state["model_state_dict"])
 with torch.no_grad() :
image_batch = copy.deepcopy(ims)
 image_batch = image_batch.float()
image_batch_augs_fl2 = image_batch.flip(2)
 image_batch_augs_fl3 = image_batch.flip(3)
 image_batch_augs_fl32 = torch.flip(image_batch,(3,2))
images = torch.cat([image_batch, image_batch_augs_fl2, image_batch_augs_fl3, image_batch_augs_fl32], dim=0)
 images = images.cuda()
print(images.shape)
 with torch.cuda.amp.autocast() :
logits = model(images, with_cam=True)
 logits = logits.reshape(bs*4//bs//4, 4, bs, 19 ).mean(1 ).view(bs*4//4, 19)
 labels = torch.sigmoid(logits)
 if verbose:
print("LABELS FROM LABELS MODEL TORCH")
 print(labels)
 return labels.cpu().numpy()
def get_separate_labels_tf(ims, model, verbose=False, name=None):
bs = ims_per_batch
image_batch = copy.deepcopy(ims)
 image_batch_fl_lr = tf.image.flip_left_right(image_batch)
 image_batch_fl_up = tf.image.flip_up_down(image_batch)
 image_batch_fl_up_lr = tf.image.flip_up_down(image_batch_fl_lr)
 ims = tf.concat([image_batch, image_batch_fl_lr, image_batch_fl_up, image_batch_fl_up_lr], axis=0)
 labels = model.predict(ims, verbose=1)
 labels = labels.reshape(bs*4//bs//4, 4, bs, 19 ).mean(1)
 if verbose:
print(f"LABELS FROM LABELS MODEL {name}")
 print(labels)
 return labels[0] 
<set_options>
best_model = tuner.get_best_models(num_models=1)[0]
best_model.summary()
print(best_model.evaluate(X_test, y_test))
sys.path.append(".. /input/faustomorales-vitkeras/vit_keras/")
 CONFIG_B = {
"dropout": 0.1,
"mlp_dim": 3072,
"num_heads": 12,
"num_layers": 12,
"hidden_size": 768,
}
class TransformerBlock(tf.keras.layers.Layer):

 def __init__(self, *args, num_heads=12, mlp_dim=3072, dropout=0.1, **kwargs):
super().__init__(*args, **kwargs)
 self.num_heads = num_heads
self.mlp_dim = mlp_dim
self.dropout = dropout
def build(self, input_shape):
self.att = MultiHeadSelfAttention(
 num_heads=self.num_heads,
name="MultiHeadDotProductAttention_1", 
)
 self.mlpblock = tf.keras.Sequential(
 [
tf.keras.layers.Dense(
 self.mlp_dim,
activation="linear",
name=f"{self.name}/Dense_0", 
),
tf.keras.layers.Lambda(
 lambda x: tf.keras.activations.gelu(x, approximate=False)
)
 if hasattr(tf.keras.activations, "gelu")
 else tf.keras.layers.Lambda(
 lambda x: tfa.activations.gelu(x, approximate=False)
),
tf.keras.layers.Dropout(self.dropout),
tf.keras.layers.Dense(input_shape[-1], name=f"{self.name}/Dense_1"),
tf.keras.layers.Dropout(self.dropout),
],
name="MlpBlock_3", 
)
 self.layernorm1 = tf.keras.layers.LayerNormalization(
 epsilon=1e-6, name="LayerNorm_0" 
)
 self.layernorm2 = tf.keras.layers.LayerNormalization(
 epsilon=1e-6, name="LayerNorm_2" 
)
 self.dropout = tf.keras.layers.Dropout(self.dropout)
 def call(self, inputs, training):
x = self.layernorm1(inputs)
 x, weights = self.att(x)
 x = self.dropout(x, training=training)
 x = x + inputs
y = self.layernorm2(x)
 y = self.mlpblock(y)
 return x + y, weights
class MultiHeadSelfAttention(tf.keras.layers.Layer):
def __init__(self, *args, num_heads=12, **kwargs):
super().__init__(*args, **kwargs)
 self.num_heads = num_heads
def build(self, input_shape):
hidden_size = input_shape[-1]
num_heads = self.num_heads
if hidden_size % num_heads != 0:
raise ValueError(
 f"embedding dimension = {hidden_size} should be divisible by number of heads = {num_heads}" 
)
 self.hidden_size = hidden_size
self.projection_dim = hidden_size // num_heads
self.query_dense = tf.keras.layers.Dense(hidden_size, name="query")
 self.key_dense = tf.keras.layers.Dense(hidden_size, name="key")
 self.value_dense = tf.keras.layers.Dense(hidden_size, name="value")
 self.combine_heads = tf.keras.layers.Dense(hidden_size, name="out")
 def attention(self, query, key, value):
score = tf.matmul(query, key, transpose_b=True)
 dim_key = tf.cast(tf.shape(key)[-1], score.dtype)
 scaled_score = score / tf.math.sqrt(dim_key)
 weights = tf.nn.softmax(scaled_score, axis=-1)
 output = tf.matmul(weights, value)
 return output, weights
def separate_heads(self, x, batch_size):
x = tf.reshape(x,(batch_size, -1, self.num_heads, self.projection_dim))
return tf.transpose(x, perm=[0, 2, 1, 3])
 def call(self, inputs):
batch_size = tf.shape(inputs)[0]
query = self.query_dense(inputs)
 key = self.key_dense(inputs)
 value = self.value_dense(inputs)
 query = self.separate_heads(query, batch_size)
 key = self.separate_heads(key, batch_size)
 value = self.separate_heads(value, batch_size)
 attention, weights = self.attention(query, key, value)
 attention = tf.transpose(attention, perm=[0, 2, 1, 3])
 concat_attention = tf.reshape(attention,(batch_size, -1, self.hidden_size))
output = self.combine_heads(concat_attention)
 return output, weights 
<train_on_grid>
def schedule(epoch, lr):
return 10**(-(epoch//10)-3)
 lr_schedule = tf.keras.callbacks.LearningRateScheduler(schedule, verbose=1)
 cb_checkpointer_val = ModelCheckpoint(filepath = '.. /working/best_val.hdf5',
monitor = 'val_accuracy',
save_best_only = True,
mode = 'auto' )
if CFG.resnest:
model_resnest = Classifier(CFG.model_name_resnest, CFG.classes, mode="normal")
 if CFG.color_mode == "rgby":
weight = model_resnest.model.conv1[0].weight.clone()
model_resnest.model.conv1[0] = nn.Conv2d(4, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
 with torch.no_grad() :
model_resnest.model.conv1[0].weight[:, :3] = weight
model_resnest.model.conv1[0].weight[:, 3] = model_resnest.model.conv1[0].weight[:, 0]
model_resnest.to(device)
 model_resnest.eval()
model_states_resnet = [torch.load(MODEL_PATH + f"{model}")for model in MODELS_LIST_RESNEST]
if CFG.effnet:
model_effnet = Classifier_EffNet(CFG.model_name_effnet)
 if CFG.color_mode == "rgby":
model_effnet.enet._conv_stem.in_channels = 4
model_effnet.enet._conv_stem.weight = torch.nn.Parameter(torch.cat([model_effnet.enet._conv_stem.weight, model_effnet.enet._conv_stem.weight[:, 0:1, :, :]], axis=1))
model_effnet.to(device)
 model_effnet.eval()
model_states_effnet = [torch.load(MODEL_PATH + f"{model}")for model in MODELS_LIST_EFFNET]
if CFG.extra_model_for_labels:
if CFG.extra_model_is_tf:
model_green_eff = tf.keras.models.load_model(MODEL_LABELS_EFF)
 model_green_rn = tf.keras.models.load_model(MODEL_LABELS_RN)
 model_green_vit = tf.keras.models.load_model(MODEL_LABELS_VIT, custom_objects={
'ClassToken': layers.ClassToken,
'AddPositionEmbs' : layers.AddPositionEmbs,
'TransformerBlock' : TransformerBlock,
'MultiHeadSelfAttention' : MultiHeadSelfAttention})
 else:
model_for_labels_state = torch.load(MODEL_LABELS)
 model_green = Classifier_EffNet_GREEN("efficientnet-b7")
 model_green.to(device)
 model_green.eval()
dl_test = DataLoader(test_data, batch_size=batch_size_, shuffle=False, num_workers=0)
<define_variables>
X_train, X_val, y_train, y_val = train_test_split(X_train_val,
y_train_val,
test_size=1/6 )
def inference_one_batch(batch_cam, batch_ids_cam, batch_seg, ims_per_batch, batch_eff_tf=None, batch_rn_tf_600=None, batch_vit_tf_384=None, show_image=True, show_seg=True, verbose=True):
batch_ids_seg = tuple(batch_seg["ID"])
 print(batch_ids_cam)
 print(batch_ids_seg)
 assert batch_ids_cam == batch_ids_seg, "IDS OF SEGMENTATION AND CAMS DONT MATCH"
print(f"GETTING {ims_per_batch} CELL MASKS")
 cell_mask_list = batch_seg["mask"]
cell_mask_sizes = batch_seg["ori_size"]
cell_masks_full_size = []
for e,(mask, size)in enumerate(zip(cell_mask_list, cell_mask_sizes)) :
cell_masks_full_size.append(resize_full_mask(mask,size))
res_cell_masks = [resize_mask(cell_mask)for cell_mask in cell_mask_list]
if show_seg:
show_seg_cells(batch_cam, res_cell_masks, batch_ids_seg, batch_cam)
 del cell_mask_list
cell_masks_fullsize_enc_list = []
for cell_mask in cell_masks_full_size:
cell_masks_fullsize_enc = get_all_encoded_cells(cell_mask)
 cell_masks_fullsize_enc_list.append(cell_masks_fullsize_enc)
 del cell_masks_full_size
print(f"ENCODED {len(cell_masks_fullsize_enc_list)} CELL MASKS")
 labels_model_eff = get_separate_labels_tf(batch_eff_tf, model_green_eff, verbose=verbose, name=MODEL_LABELS_EFF)
 labels_model_rn = get_separate_labels_tf(batch_rn_tf_600, model_green_rn, verbose=verbose, name=MODEL_LABELS_RN)
 labels_model_vit = get_separate_labels_tf(batch_vit_tf_384, model_green_vit, verbose=verbose, name=MODEL_LABELS_VIT)
 labels_model = labels_model_eff*CFG.split_image_level[0] + labels_model_rn*CFG.split_image_level[1] + labels_model_vit*CFG.split_image_level[2]
print("LABELS FROM LABEL MODEL")
 print(labels_model)
 print("RESIZE IMAGES")
 batch_cam_scaled = []
scales = [1.0, 1.3, 1.6]
st = time.time()
for i, scale in enumerate(scales):
image_batch_pil = torch.from_numpy(np.zeros(( ims_per_batch, 4, round(CFG.size*scale), round(CFG.size*scale))))
image_batch = copy.deepcopy(batch_cam)
 for j, im in enumerate(image_batch):
im = ToPILImage()(im)
 im = im.resize(( round(CFG.size*scale), round(CFG.size*scale)) , resample=PIL.Image.BICUBIC)
 im = torchvision.transforms.functional.to_tensor(im)*255
image_batch_pil[j] = im
image_batch_resized = image_batch_pil.float()
image_batch_augs_fl2 = image_batch_resized.flip(2)
 image_batch_augs_fl3 = image_batch_resized.flip(3)
 image_batch_augs_fl32 = torch.flip(image_batch_resized,(3,2))
images = torch.cat([image_batch_resized, image_batch_augs_fl2, image_batch_augs_fl3, image_batch_augs_fl32], dim=0)
 images = images.cuda()
batch_cam_scaled.append(images)
 print(f"TIME FOR RESIZING WITH PIL {time.time() - st}")
 if CFG.resnest:
print("GETTING CAMS AND PREDS RESNEST")
 mask_probas_resnest_folds = []
folds_resnest = len(MODELS_LIST_RESNEST)
 all_ims_resnest = []
time_spent_mask_probas = []
for f, model_state_rn in enumerate(model_states_resnet):
all_hr_cams, sig_labels = get_hrcams_vis(batch_cam_scaled, show_image, model_resnest, model_state_rn, scales, ims_per_batch)
 mask_probas_resnest_batches = []
print(f"GETTING MASKS PROBAS FOLD {f}")
 for b,(cams, mask, sig_label, label_model)in enumerate(zip(all_hr_cams, res_cell_masks, sig_labels, labels_model)) :
print(f"GETTING MASKS PROBAS BATCH {b}")
 mask_probas_resnest = get_prob_from_cams_masks(cams, mask, sig_label, label_model, verbose, typ=f"resnest{f}")
 if verbose:
print(mask_probas_resnest)
 mask_probas_resnest_batches.append(mask_probas_resnest)
 mask_probas_resnest_folds.append(mask_probas_resnest_batches)
 for b in range(ims_per_batch):
all_ims_resnest.append(np.mean(np.array([mask_probas_resnest_folds[i][b] for i in range(folds_resnest)]), axis=0))
if CFG.effnet:
print("GETTING CAMS AND PREDS EFFICIENTNET")
 mask_probas_effnet_folds = []
folds_effnet = len(MODELS_LIST_EFFNET)
 all_ims_effnet = []
time_spent_mask_probas = []
for f, model_state_eff in enumerate(model_states_effnet):
all_hr_cams, sig_labels = get_hrcams_vis(batch_cam_scaled, show_image, model_effnet, model_state_eff, scales, ims_per_batch)
 mask_probas_effnet_batches = []
print(f"GETTING MASKS PROBAS FOLD {f}")
 for b,(cams, mask, sig_label, label_model)in enumerate(zip(all_hr_cams, res_cell_masks, sig_labels, labels_model)) :
print(f"GETTING MASKS PROBAS BATCH {b}")
 s_t = time.time()
mask_probas_effnet = get_prob_from_cams_masks(cams, mask, sig_label, label_model, verbose, typ=f"effnet{f}")
 if verbose:
print(mask_probas_effnet)
 mask_probas_effnet_batches.append(mask_probas_effnet)
 mask_probas_effnet_folds.append(mask_probas_effnet_batches)
 for b in range(ims_per_batch):
all_ims_effnet.append(np.mean(np.array([mask_probas_effnet_folds[i][b] for i in range(folds_effnet)]), axis=0))
if CFG.resnest and CFG.effnet:
mask_probas = CFG.split_cam_level[0]*np.array(all_ims_effnet)+ CFG.split_cam_level[1]*np.array(all_ims_resnest)
 elif CFG.resnest and not CFG.effnet:
mask_probas = all_ims_resnest
elif CFG.effnet and not CFG.resnest:
mask_probas = all_ims_effnet
if verbose:
print(mask_probas)
 return batch_ids_cam, sizes, cell_masks_fullsize_enc_list, mask_probas<set_options>
fit_history = best_model.fit(X_train, y_train, epochs=40, batch_size=100,
validation_data=(X_val, y_val),
callbacks = [lr_schedule, cb_checkpointer_val] )
libc = ctypes.CDLL("libc.so.6")
 libc.malloc_trim(0)
 gc.collect()
gc.collect()
gc.collect()
torch.cuda.empty_cache()
torch.cuda.empty_cache()
torch.cuda.empty_cache()
torch.cuda.empty_cache()<define_variables>
best_model.load_weights('./best_val.hdf5')
 best_model.evaluate(X_test, y_test )
if CFG.is_demo:
index = 0
batch0, ids0 = next(itertools.islice(dl_test, index, None))
if CFG.extra_model_is_tf:
batch_tf_green = next(itertools.islice(dtest_tf_green_600, index, None))
batch_tf_rgb_600 = next(itertools.islice(dtest_tf_rgb_600, index, None))
batch_tf_rgb_384 = next(itertools.islice(dtest_tf_ggg_384, index, None))
print(batch_tf_green.shape)
 print(batch_tf_rgb_600.shape)
 print(batch_tf_rgb_384.shape)
<split>
cb_checkpointer = ModelCheckpoint(filepath = '.. /working/best.hdf5',
monitor = 'accuracy',
save_best_only = True,
mode = 'auto' )
df = pd.DataFrame(columns=["image_id", "pred"])
 i = 0
start_time = time.time()
ims_done = 0
for i,(( batch_cam, batch_ids_cam),(batch_tf_green_600),(batch_tf_rgb_600),(batch_tf_ggg_384)) in enumerate(zip(dl_test, dtest_tf_green_600, dtest_tf_rgb_600, dtest_tf_ggg_384)) :
ims_per_batch = len(batch_ids_cam)
 batch_seg = cell_masks_df[i*batch_size_ : i*batch_size_ + batch_size_]
ids, sizes, cell_masks_fullsize_enc_list, probas = inference_one_batch(batch_cam, batch_ids_cam, batch_seg, ims_per_batch, batch_eff_tf=batch_tf_green_600, batch_rn_tf_600=batch_tf_rgb_600, batch_vit_tf_384=batch_tf_ggg_384, show_image=False, show_seg=False, verbose=False)
 for id, proba, cell_mask_enc in zip(ids, probas, cell_masks_fullsize_enc_list):
pred_string = get_pred_string(proba, cell_mask_enc)
 d = {"image_id":id, "pred": pred_string}
df = df.append(d, ignore_index=True)
 ims_done += ims_per_batch
print(f"---{ims_done} IMAGES DONE---")
 torch.cuda.empty_cache()
gc.collect()
print("--- %s seconds ---" %(time.time() - start_time)) 
<merge>
final_fit_history = best_model.fit(X, y, epochs=40, batch_size=100,
callbacks = [lr_schedule, cb_checkpointer] )
sub = pd.merge(
 data_df,
df,
how="left",
left_on='ID',
right_on='image_id', 
)
 def isNaN(num):
return num != num
for i, row in sub.iterrows() :
if isNaN(row['pred']): continue
sub.PredictionString.loc[i] = row['pred'] 
<save_to_csv>
best_model.load_weights('./best.hdf5' )
if all(df_from_files == data_df_sample_submission):
sub.to_csv("submission.csv",index=False)
<install_modules>
best_model.evaluate(X, y )
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index -q
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index -q<feature_engineering>
pred = pd.DataFrame({'ImageId' : np.arange(test.shape[0])+ 1,
'Label': best_model.predict(test ).argmax(axis=-1)})
 pred.to_csv('pred.csv', index=False)
 pred