Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<save_to_csv>
%matplotlib inline
predictions_list = []
model_dir_list = []
for config, model_dir in zip(normal_configs, normal_model_dirs):
predictions_list.append(main(config, model_dir))
model_dir_list.append(model_dir)
 for config, model_dir in zip(tta_configs, tta_model_dirs):
predictions_list.append(main_tta(config, model_dir))
model_dir_list.append(model_dir)
<save_to_csv>
print(tf.version.VERSION )
predictions = np.zeros(predictions_list[0].shape, dtype=predictions_list[0].dtype)
 for i, key in zip(range(len(predictions_list)) , model_dir_list):
predictions += predictions_list[i] * weights_dict[key]
test['label'] = predictions.argmax(1)
 test[['image_id', 'label']].to_csv(OUTPUT_DIR+'submission.csv', index=False)
 test.head()<install_modules>
train = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv" )
!mkdir -p /tmp/pip/cache/
!cp.. /input/omegaconf/PyYAML-5.4b2-cp38-cp38-manylinux1_x86_64.whl /tmp/pip/cache/
!cp.. /input/omegaconf/omegaconf-2.0.5-py3-none-any.whl /tmp/pip/cache/
!cp.. /input/omegaconf/typing_extensions-3.7.4.3-py3-none-any.whl /tmp/pip/cache/
!pip install --no-index --find-links /tmp/pip/cache/ omegaconf > /dev/null<set_options>
train_X = train.loc[:, "pixel0":"pixel783"]
train_y = train.loc[:, "label"]
sys.path.append('.. /input/timm-pytorch-image-models/pytorch-image-models-master')
 sys.path.append(".. /input/cleanlab/")
 warnings.filterwarnings('ignore')
<set_options>
treshhold=0.1
train_X[train_X<treshhold]=0
test[test<treshhold]=0
mean, std =(0.485, 0.456, 0.406),(0.229, 0.224, 0.225)
 def get_transforms(img_size=(512, 512)) :
transformations = Compose([
PadIfNeeded(min_height=img_size[0], min_width=img_size[1]),
CenterCrop(img_size[0], img_size[1]),
Normalize(mean=mean, std=std, max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.0)
 return transformations<choose_model_class>
train_X = train_X / 255.0
test_X = test / 255.0
train_X = train_X.values.reshape(-1,28,28,1)
 test_X = test_X.values.reshape(-1,28,28,1)
 train_y = to_categorical(train_y, num_classes = 10 )
def create_model(model_name: str,
pretrained: bool,
num_classes: int,
in_chans: int):
model = timm.create_model(model_name=model_name,
pretrained=pretrained,
num_classes=num_classes,
in_chans=in_chans)
 return model<load_pretrained>
def build_model(input_shape=(28, 28, 1)) :
model = Sequential()
model.add(Conv2D(32, kernel_size = 3, activation='swish', input_shape =(28, 28, 1)))
 model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size = 3, activation='swish'))
model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='swish'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(64, kernel_size = 3, activation='swish'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 3, activation='swish'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='swish'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(128, kernel_size = 4, activation='swish'))
model.add(BatchNormalization())
 model.add(Flatten())
 model.add(Dropout(0.4))
model.add(Dense(10, activation='softmax'))
return model
def get_state_dict_from_checkpoint(log_dir, fold_num):
ckpt_path = glob(os.path.join(log_dir, f'checkpoints/*fold{fold_num}*.ckpt')) [0]
state_dict = pl_load(ckpt_path, map_location='cpu')
 if 'state_dict' in state_dict:
state_dict = state_dict['state_dict']
did_distillation = False
state_dict = OrderedDict(( k.replace('model.', '')
 if 'model.' in k else k, v)for k, v in
state_dict.items())
 return state_dict<find_best_params>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False )
class LitTester(pl.LightningModule):
def __init__(self, network_cfg, state_dict):
super(LitTester, self ).__init__()
self.model = create_model(**network_cfg)
 self.model.load_state_dict(state_dict)
 self.model.eval()
def forward(self, x):
x = self.model(x)
 return x
def test_step(self, batch, batch_idx):
score = torch.nn.functional.softmax(self(batch), dim=1)
 score2 = torch.nn.functional.softmax(self(torch.flip(batch, [-1])) , dim=1)
 score3 = torch.nn.functional.softmax(self(torch.flip(batch, [-2])) , dim=1)
 out =(score + score2 + score3)/ 3.0
return {"pred": out}
def test_epoch_end(self, output_results):
all_outputs = torch.cat([out["pred"] for out in output_results], dim=0)
 all_outputs = all_outputs.cpu().numpy()
return {'prob': all_outputs}<define_search_space>
learning_rate_reduction = ReduceLROnPlateau(monitor='accuracy',
patience=3,
verbose=1,
factor=0.8,
min_lr=0.001)
 annealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x)
 early_stop=EarlyStopping(monitor='val_loss', min_delta=0, patience=10, verbose=1, mode='auto',
baseline=None, restore_best_weights=True )
eff_b0_cfg_s =
eff_b0_cfg = OmegaConf.create(eff_b0_cfg_s )<define_variables>
%%time
nets=10
model = [0] *nets
history = [0] * nets
skf = StratifiedKFold(n_splits=nets, shuffle = True, random_state=1)
 skf.get_n_splits(train_X, train['label'])
 print(skf)
 number=0
for train_index, test_index in skf.split(train_X, train['label']):
print("SPLIT ",number," TRAIN index:", train_index, "TEST index:", test_index)
 X_train, X_val = train_X[train_index], train_X[test_index]
y_train, y_val = train_y[train_index], train_y[test_index]
model[number]=build_model()
model[number].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
 history[number] =model[number].fit(datagen.flow(X_train,y_train), epochs=100 ,validation_data =(X_val,y_val),
batch_size=100, verbose = 0,callbacks = [annealer,early_stop])
 metrics=pd.DataFrame(history[number].history)
 display(metrics)
 number+=1
name = '14-10-36'
cfg = eff_b0_cfg
do_predict = True
do_submit = False
img_dir = '.. /input/cassava-leaf-disease-merged/train/'
label_path = '.. /input/cassava-leaf-disease-merged/merged.csv'
log_dir = os.path.join('.. /input/cassava-public-ckpt', name)
 n_folds = len(glob(os.path.join(log_dir, 'checkpoints/*.ckpt')))
 num2class = ["Cassava Bacterial Blight(CBB)", "Cassava Brown Streak Disease(CBSD)", "Cassava Green Mottle(CGM)", "Cassava Mosaic Disease(CMD)", "Healthy"]<feature_engineering>
for number in range(0,nets):
model[number].save("StratifiedKFold_10_batch100_double_val_loss_"+str(number)+".h5" )
seed_everything(42)
 label_df = pd.read_csv(label_path)
 if 'fold' not in label_df.columns:
skf = StratifiedKFold(n_splits=5, shuffle=True)
 label_df.loc[:, 'fold'] = 0
for fold_num,(train_index, val_index)in enumerate(skf.split(X=label_df.index, y=label_df.label.values)) :
label_df.loc[label_df.iloc[val_index].index, 'fold'] = fold_num
if do_predict:
infer = pl.Trainer(gpus=1)
 oof_dict = {'image_id': [], 'label': [], 'fold': []}
for fold_num in range(n_folds):
val_df = label_df[label_df.fold == fold_num]
test_dataset = TestDataset(img_dir, val_df, img_size=cfg.img_size)
 test_dataloader = DataLoader(test_dataset,
batch_size=cfg.batch_size,
num_workers=4,
shuffle=False)
 state_dict = get_state_dict_from_checkpoint(log_dir, fold_num)
 model = LitTester(cfg.network, state_dict)
 pred = infer.test(model, test_dataloaders=test_dataloader, verbose=False)[0]
oof_dict['image_id'].extend(val_df.image_id.values)
 oof_dict['label'].extend(pred['prob'].tolist())
 oof_dict['fold'].extend([fold_num] * len(pred['prob']))
pred_df = pd.DataFrame(oof_dict)
 pred_df.to_csv('oof.csv', index=False)
 else:
pred_df = pd.read_csv(os.path.join(log_dir, 'oof.csv'))<load_from_csv>
results = np.zeros(( test_X.shape[0],10))
for j in range(nets):
results = results + model[j].predict(test_X)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("StratifiedKFold_10_batch100_double_val_loss.csv",index=False )
if do_submit:
sub = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 infer = pl.Trainer(gpus=1)
 test_dataset = TestDataset('.. /input/cassava-leaf-disease-classification/test_images',
sub,
img_size=cfg.img_size)
 test_dataloader = DataLoader(test_dataset,
batch_size=cfg.batch_size,
num_workers=4,
shuffle=False)
 preds = []
for fold_num in range(n_folds):
state_dict = get_state_dict_from_checkpoint(log_dir, fold_num)
 model = LitTester(cfg.network, state_dict)
 pred = infer.test(model, test_dataloaders=test_dataloader, verbose=False)[0]
preds.append(pred['prob'])
 sub['label'] = np.argmax(np.mean(preds, axis=0), axis=1)
 sub.to_csv(os.path.join(os.getcwd() , 'submission.csv'), index=False )<sort_values>
show_test_digits(range(500,530))