Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<categorify>
train_set = pd.read_csv('.. /input/digit-recognizer/train.csv')
 test_set = pd.read_csv('.. /input/digit-recognizer/test.csv' )
def get_transforms() :
return Compose([
Resize(IMAGE_SIZE, IMAGE_SIZE),
Normalize() ,
ToTensorV2() ,
] )<choose_model_class>
X = train_set.drop('label', axis=1)
 labels = train_set['label']
class ResNet200D(nn.Module):
def __init__(self, model_name='resnet200d_320'):
super().__init__()
self.model = timm.create_model(model_name, pretrained=False)
 n_features = self.model.fc.in_features
self.model.global_pool = nn.Identity()
self.model.fc = nn.Identity()
self.pooling = nn.AdaptiveAvgPool2d(1)
 self.fc = nn.Linear(n_features, 11)
 def forward(self, x):
bs = x.size(0)
 features = self.model(x)
 pooled_features = self.pooling(features ).view(bs, -1)
 output = self.fc(pooled_features)
 return output
class SeResNet152D(nn.Module):
def __init__(self, model_name='seresnet152d_320'):
super().__init__()
self.model = timm.create_model(model_name, pretrained=False)
 n_features = self.model.fc.in_features
self.model.global_pool = nn.Identity()
self.model.fc = nn.Identity()
self.pooling = nn.AdaptiveAvgPool2d(1)
 self.fc = nn.Linear(n_features, 11)
 def forward(self, x):
bs = x.size(0)
 features = self.model(x)
 pooled_features = self.pooling(features ).view(bs, -1)
 output = self.fc(pooled_features)
 return output
class EfficientNetB5(nn.Module):
def __init__(self, model_name='tf_efficientnet_b5_ns'):
super().__init__()
self.model = timm.create_model(model_name, pretrained=False)
 n_features = self.model.classifier.in_features
self.model.global_pool = nn.Identity()
self.model.classifier = nn.Identity()
self.pooling = nn.AdaptiveAvgPool2d(1)
 self.classifier = nn.Linear(n_features, 11)
 def forward(self, x):
bs = x.size(0)
 features = self.model(x)
 pooled_features = self.pooling(features ).view(bs, -1)
 output = self.classifier(pooled_features)
 return output<categorify>
y = pd.get_dummies(labels )
def inference(models, test_loader, device):
tk0 = tqdm(enumerate(test_loader), total=len(test_loader))
probs = []
for i,(images)in tk0:
images = images.to(device)
 avg_preds = []
for model in models:
with torch.no_grad() :
y_preds1 = model(images)
 y_preds2 = model(images.flip(-1))
y_preds =(y_preds1.sigmoid().to('cpu' ).numpy() + y_preds2.sigmoid().to('cpu' ).numpy())/ 2
avg_preds.append(y_preds)
 avg_preds = np.mean(avg_preds, axis=0)
 probs.append(avg_preds)
 probs = np.concatenate(probs)
 return probs<choose_model_class>
X = X / 255.0
test_set = test_set / 255.0
models200D = []
model = ResNet200D()
model.load_state_dict(torch.load(MODEL_PATH_resnet200d)['model'])
 model.eval()
model.to(device)
 models200D.append(model)
 models152D = []
model = SeResNet152D()
model.load_state_dict(torch.load(MODEL_PATH_seresnet152d)['model'])
 model.eval()
model.to(device)
 models152D.append(model)
 modelsB5 = []
model = EfficientNetB5()
model.load_state_dict(torch.load(MODEL_PATH_efficientnet_b5)['model'])
 model.eval()
model.to(device)
 modelsB5.append(model )<create_dataframe>
X_train,X_val, y_train, y_val = train_test_split(X,y,test_size=0.2, random_state=42 )
test_dataset = TestDataset(test, transform=get_transforms())
 test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False,
num_workers=4 , pin_memory=True)
 predictions200d = inference(models200D, test_loader, device)
 del models200D
gc.collect()
predictions152d = inference(models152D, test_loader, device)
 del models152D
gc.collect()
predictionsB5 = inference(modelsB5, test_loader, device)
<drop_column>
batch_size = 32
epochs=50
variable_list = %who_ls
for _ in variable_list:
if _ not in("predictions200d", "predictions152d", "predictionsB5"):
del globals() [_]
%who_ls<define_variables>
simple_NN = keras.Sequential([
layers.Dense(100, activation='relu', input_shape=(28,28,1)) ,
layers.Dropout(0.3),
layers.Flatten() ,
layers.Dense(units=100, activation='relu'),
layers.Dense(units=10, activation='softmax')
 ])
ROOT = Path.cwd().parent
INPUT = ROOT / "input"
OUTPUT = ROOT / "output"
DATA = INPUT / "ranzcr-clip-catheter-line-classification"
TRAIN = DATA / "train"
TEST = DATA / "test"
TRAINED_MODEL = INPUT / "ranzcr-clip-weights-for-multi-head-model-v2"
TMP = ROOT / "tmp"
TMP.mkdir(exist_ok=True)
 RANDAM_SEED = 1086
N_CLASSES = 11
FOLDS = [0, 1, 2, 3, 4]
N_FOLD = len(FOLDS)
 IMAGE_SIZE =(512, 512)
 FAST_COMMIT = False
CLASSES = [
'ETT - Abnormal',
'ETT - Borderline',
'ETT - Normal',
'NGT - Abnormal',
'NGT - Borderline',
'NGT - Incompletely Imaged',
'NGT - Normal',
'CVC - Abnormal',
'CVC - Borderline',
'CVC - Normal',
'Swan Ganz Catheter Present'
]<load_from_csv>
simple_NN.compile(
 optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'] 
)
 simple_NN.optimizer.lr=0.001
for p in DATA.iterdir() :
print(p.name)
 train = pd.read_csv(DATA / "train.csv")
 smpl_sub = pd.read_csv(DATA / "sample_submission.csv")
 smpl_sub.shape<split>
early_stopping = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)
if FAST_COMMIT and len(smpl_sub)== 3582:
smpl_sub = smpl_sub.iloc[:64 * 3].reset_index(drop=True )<categorify>
lrr = ReduceLROnPlateau(monitor='val_loss',patience=3,verbose=1,factor=0.5, min_lr=0.00001 )
def multi_label_stratified_group_k_fold(label_arr: np.array, gid_arr: np.array, n_fold: int, seed: int=42):

 np.random.seed(seed)
 random.seed(seed)
 start_time = time.time()
n_train, n_class = label_arr.shape
gid_unique = sorted(set(gid_arr))
n_group = len(gid_unique)
 gid2aid = dict(zip(gid_unique, range(n_group)))
 aid_arr = np.vectorize(lambda x: gid2aid[x] )(gid_arr)
 cnts_by_class = label_arr.sum(axis=0)
 col, row = np.array(sorted(enumerate(aid_arr), key=lambda x: x[1])).T
cnts_by_group = coo_matrix(
(np.ones(len(label_arr)) ,(row, col))
).dot(coo_matrix(label_arr)).toarray().astype(int)
 del col
del row
cnts_by_fold = np.zeros(( n_fold, n_class), int)
 groups_by_fold = [[] for fid in range(n_fold)]
group_and_cnts = list(enumerate(cnts_by_group))
np.random.shuffle(group_and_cnts)
 print("finished preparation", time.time() - start_time)
 for aid, cnt_by_g in sorted(group_and_cnts, key=lambda x: -np.std(x[1])) :
best_fold = None
min_eval = None
for fid in range(n_fold):
cnts_by_fold[fid] += cnt_by_g
fold_eval =(cnts_by_fold / cnts_by_class ).std(axis=0 ).mean()
cnts_by_fold[fid] -= cnt_by_g
if min_eval is None or fold_eval < min_eval:
min_eval = fold_eval
best_fold = fid
cnts_by_fold[best_fold] += cnt_by_g
groups_by_fold[best_fold].append(aid)
 print("finished assignment.", time.time() - start_time)
 gc.collect()
idx_arr = np.arange(n_train)
 for fid in range(n_fold):
val_groups = groups_by_fold[fid]
val_indexs_bool = np.isin(aid_arr, val_groups)
 train_indexs = idx_arr[~val_indexs_bool]
val_indexs = idx_arr[val_indexs_bool]
print("[fold {}]".format(fid), end=" ")
 print("n_group:(train, val)=({}, {})".format(n_group - len(val_groups), len(val_groups)) , end=" ")
 print("n_sample:(train, val)=({}, {})".format(len(train_indexs), len(val_indexs)))
 yield train_indexs, val_indexs<categorify>
history_simple_NN = simple_NN.fit(
 x=X_train,
y=y_train,
validation_data=(X_val, y_val),
batch_size=batch_size,
epochs=epochs,
shuffle=True,
verbose=2,
callbacks=[early_stopping, lrr] 
)
label_arr = train[CLASSES].values
group_id = train.PatientID.values
train_val_indexs = list(
 multi_label_stratified_group_k_fold(
 label_arr, group_id, N_FOLD, RANDAM_SEED 
)
 )<feature_engineering>
CNN = keras.Sequential([
layers.Conv2D(32, kernel_size=(3,3), activation='relu', padding='same', input_shape=(28,28,1)) ,
layers.Conv2D(64, kernel_size=(3,3), activation='relu', padding='same'),
layers.Conv2D(64, kernel_size=(3,3), activation='relu', padding='same'),
layers.Flatten() ,
layers.Dense(10, activation='softmax')
 ])
train["fold"] = -1
for fold_id,(trn_idx, val_idx)in enumerate(train_val_indexs):
train.loc[val_idx, "fold"] = fold_id
train.groupby("fold")[CLASSES].sum()<train_model>
CNN.compile(
 optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'] 
)
 CNN.optimizer.lr=0.001
def resize_images(img_id, input_dir, output_dir, resize_to=(512, 512), ext="png"):
img_path = input_dir / f"{img_id}.jpg"
save_path = output_dir / f"{img_id}.{ext}"
img = cv2.imread(str(img_path), cv2.IMREAD_GRAYSCALE)
 img = cv2.resize(img, resize_to)
 cv2.imwrite(str(save_path), img,)
 TEST_RESIZED = TMP / "test_{0}x{1}".format(*IMAGE_SIZE)
 TEST_RESIZED.mkdir(exist_ok=True)
 TEST_RESIZED
_ = Parallel(n_jobs=2, verbose=5 )([
delayed(resize_images )(img_id, TEST, TEST_RESIZED, IMAGE_SIZE, "png")
 for img_id in smpl_sub.StudyInstanceUID.values
] )<choose_model_class>
history_CNN = CNN.fit(
 x=X_train,
y=y_train,
validation_data=(X_val, y_val),
batch_size=batch_size,
epochs=epochs,
shuffle= True,
verbose=2,
callbacks=[early_stopping, lrr]
)
def get_activation(activ_name: str="relu"):
act_dict = {
"relu": nn.ReLU(inplace=True),
"tanh": nn.Tanh() ,
"sigmoid": nn.Sigmoid() ,
"identity": nn.Identity() }
if activ_name in act_dict:
return act_dict[activ_name]
else:
raise NotImplementedError
class Conv2dBNActiv(nn.Module):

 def __init__(
 self, in_channels: int, out_channels: int,
kernel_size: int, stride: int=1, padding: int=0,
bias: bool=False, use_bn: bool=True, activ: str="relu" 
):
super(Conv2dBNActiv, self ).__init__()
layers = []
layers.append(nn.Conv2d(
 in_channels, out_channels,
kernel_size, stride, padding, bias=bias))
if use_bn:
layers.append(nn.BatchNorm2d(out_channels))
layers.append(get_activation(activ))
self.layers = nn.Sequential(*layers)
 def forward(self, x):

 return self.layers(x)
 class SpatialAttentionBlock(nn.Module):

 def __init__(
 self, in_channels: int,
out_channels_list: tp.List[int], 
):

 super(SpatialAttentionBlock, self ).__init__()
self.n_layers = len(out_channels_list)
 channels_list = [in_channels] + out_channels_list
assert self.n_layers > 0
assert channels_list[-1] == 1
for i in range(self.n_layers - 1):
in_chs, out_chs = channels_list[i: i + 2]
layer = Conv2dBNActiv(in_chs, out_chs, 3, 1, 1, activ="relu")
 setattr(self, f"conv{i + 1}", layer)
 in_chs, out_chs = channels_list[-2:]
layer = Conv2dBNActiv(in_chs, out_chs, 3, 1, 1, activ="sigmoid")
 setattr(self, f"conv{self.n_layers}", layer)
 def forward(self, x):

 h = x
for i in range(self.n_layers):
h = getattr(self, f"conv{i + 1}" )(h)
 h = h * x
return h<choose_model_class>
CNN_2 = keras.Sequential([
layers.Conv2D(32, kernel_size=(3,3), activation='relu', padding='same', input_shape=(28,28,1)) ,
layers.MaxPooling2D(pool_size=(2, 2)) ,
layers.Conv2D(64, kernel_size=(3,3), activation='relu', padding='same'),
layers.MaxPooling2D(pool_size=(2, 2)) ,
layers.Conv2D(64, kernel_size=(3,3), activation='relu', padding='same'),
layers.MaxPooling2D(pool_size=(2, 2)) ,
layers.Flatten() ,
layers.Dropout(0.4),
layers.Dense(10, activation='softmax')
 ] )
class MultiHeadResNet200D(nn.Module):
def __init__(
 self, out_dims_head: tp.List[int]=[3, 4, 3, 1], pretrained=False 
):
self.base_name = "resnet200d_320"
self.n_heads = len(out_dims_head)
 super(MultiHeadResNet200D, self ).__init__()
base_model = timm.create_model(
 self.base_name, num_classes=sum(out_dims_head), pretrained=False)
 in_features = base_model.num_features
if pretrained:
pretrained_model_path = '.. /input/startingpointschestx/resnet200d_320_chestx.pth'
state_dict = dict()
for k, v in torch.load(pretrained_model_path, map_location='cpu')["model"].items() :
if k[:6] == "model.":
k = k.replace("model.", "")
 state_dict[k] = v
base_model.load_state_dict(state_dict)
 base_model.reset_classifier(0, '')
 self.backbone = base_model
for i, out_dim in enumerate(out_dims_head):
layer_name = f"head_{i}"
layer = nn.Sequential(
 SpatialAttentionBlock(in_features, [64, 32, 16, 1]),
nn.AdaptiveAvgPool2d(output_size=1),
nn.Flatten(start_dim=1),
nn.Linear(in_features, in_features),
nn.ReLU(inplace=True),
nn.Dropout(0.5),
nn.Linear(in_features, out_dim))
setattr(self, layer_name, layer)
 def forward(self, x):
h = self.backbone(x)
 hs = [
getattr(self, f"head_{i}" )(h)for i in range(self.n_heads)]
y = torch.cat(hs, axis=1)
 return y
m = MultiHeadResNet200D([3, 4, 3, 1], False)
 m = m.eval()
x = torch.rand(1, 3, 256, 256)
 with torch.no_grad() :
y = m(x)
 print("[forward test]")
 print("input:\t{}
output:\t{}".format(x.shape, y.shape))
del m; del x; del y
gc.collect()<categorify>
CNN_2.compile(
 optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'] 
)
 CNN.optimizer.lr=0.001
class LabeledImageDataset(data.Dataset):

 def __init__(
 self,
file_list: tp.List[
tp.Tuple[tp.Union[str, Path], tp.Union[int, float, np.ndarray]]
],
transform_list: tp.List[tp.Dict], 
):

 self.file_list = file_list
self.transform = ImageTransformForCls(transform_list)
 def __len__(self):

 return len(self.file_list)
 def __getitem__(self, index):

 img_path, label = self.file_list[index]
img = self._read_image_as_array(img_path)
 img, label = self.transform(( img, label))
return img, label
def _read_image_as_array(self, path: str):

 img_arr = cv2.imread(str(path))
img_arr = cv2.cvtColor(img_arr, cv2.COLOR_BGR2RGB)
 return img_arr<create_dataframe>
history_CNN_2 = CNN_2.fit(
 x=X_train,
y=y_train,
validation_data=(X_val, y_val),
batch_size=batch_size,
epochs=epochs,
shuffle= True,
verbose=2,
callbacks=[early_stopping, lrr]
)
def get_dataloaders_for_inference(
 file_list: tp.List[tp.List], batch_size=64 
):

 dataset = LabeledImageDataset(
 file_list,
transform_list=[
[
"Normalize",
{
"always_apply": True,
"max_pixel_value": 255.0,
"mean": ["0.4887381077884414"],
"std": ["0.23064819430546407"]
}
],
["ToTensorV2", {"always_apply": True}],
] 
)
 loader = data.DataLoader(
 dataset,
batch_size=batch_size,
shuffle=False,
num_workers=2,
pin_memory=True,
drop_last=False 
)
 return loader<categorify>
datagen = ImageDataGenerator(
 featurewise_center=False,
featurewise_std_normalization=False,
rotation_range=10,
zoom_range=0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False,
validation_split=0.2 
)
 datagen.fit(X_train)
class ImageTransformBase:

 def __init__(self, data_augmentations: tp.List[tp.Tuple[str, tp.Dict]]):

 augmentations_list = [
self._get_augmentation(aug_name )(**params)
 for aug_name, params in data_augmentations]
self.data_aug = albumentations.Compose(augmentations_list)
 def __call__(self, pair: tp.Tuple[np.ndarray])-> tp.Tuple[np.ndarray]:

 raise NotImplementedError
def _get_augmentation(self, aug_name: str)-> tp.Tuple[ImageOnlyTransform, DualTransform]:

 if hasattr(albumentations, aug_name):
return getattr(albumentations, aug_name)
 else:
return eval(aug_name)
 class ImageTransformForCls(ImageTransformBase):

 def __init__(self, data_augmentations: tp.List[tp.Tuple[str, tp.Dict]]):

 super(ImageTransformForCls, self ).__init__(data_augmentations)
 def __call__(self, in_arrs: tp.Tuple[np.ndarray])-> tp.Tuple[np.ndarray]:

 img, label = in_arrs
augmented = self.data_aug(image=img)
 img = augmented["image"]
return img, label<init_hyperparams>
data_size = len(X_train)
 steps_per_epoch = int(data_size / batch_size)
 print(steps_per_epoch )
def load_setting_file(path: str):

 with open(path)as f:
settings = yaml.safe_load(f)
 return settings
def set_random_seed(seed: int = 42, deterministic: bool = False):

 random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = deterministic
def run_inference_loop(stgs, model, loader, device):
model.to(device)
 model.eval()
pred_list = []
with torch.no_grad() :
for x, t in tqdm(loader):
y_1 = model(x.to(device))
y_2 = model(x.to(device ).flip(-1))
y_preds =(
 y_1.sigmoid().detach().cpu().numpy() +
y_2.sigmoid().detach().cpu().numpy() 
)/ 2
pred_list.append(y_preds)
 pred_arr = np.concatenate(pred_list)
 del pred_list
return pred_arr<set_options>
history_CNN_2_datagen = CNN_2.fit(
 datagen.flow(X_train,y_train,batch_size=batch_size),
epochs=epochs,
shuffle=True,
validation_data=(X_val,y_val),
verbose=2,
callbacks=[early_stopping, lrr],
steps_per_epoch=steps_per_epoch
)
if not torch.cuda.is_available() :
device = torch.device("cpu")
 else:
device = torch.device("cuda")
 print(device )<load_pretrained>
val_predictions = CNN_2.predict(X_val)
 y_pred = val_predictions.argmax(axis=-1 )
model_dir = TRAINED_MODEL
test_dir = TEST_RESIZED
test_file_list = [ 
(test_dir / f"{img_id}.png", [-1] * 11)
 for img_id in smpl_sub["StudyInstanceUID"].values
]
test_loader = get_dataloaders_for_inference(test_file_list, batch_size=64)
 test_preds_arr = np.zeros(( N_FOLD, len(smpl_sub), N_CLASSES))
for fold_id in FOLDS:
print(f"[fold {fold_id}]")
 stgs = load_setting_file(model_dir / f"fold{fold_id}" / "settings.yml")
 stgs["model"]["params"]["pretrained"] = False
model = MultiHeadResNet200D(**stgs["model"]["params"])
 model_path = model_dir / f"best_model_fold{fold_id}.pth"
model.load_state_dict(torch.load(model_path, map_location=device))
test_pred = run_inference_loop(stgs, model, test_loader, device)
 test_preds_arr[fold_id] = test_pred
del model
torch.cuda.empty_cache()
gc.collect()<feature_engineering>
predictions = CNN_2.predict(test_set)
 results = predictions.argmax(axis=-1 )
sub = smpl_sub.copy()
sub[CLASSES] =(
 0.50 *(test_preds_arr.argsort(axis=1 ).argsort(axis=1 ).mean(axis=0))
+ 0.30 *(predictions200d.argsort(axis=0 ).argsort(axis=0))
+ 0.08 *(predictions152d.argsort(axis=0 ).argsort(axis=0))
+ 0.12 *(predictionsB5.argsort(axis=0 ).argsort(axis=0)) 
)
 sub.to_csv("submission.csv", index=False )<install_modules>
result = pd.DataFrame()
result['ImageId'] = list(range(1,28001))
result['Label'] = results
result.to_csv("output.csv", index = False)