Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<predict_on_test>
batch_size=128
keras.__version__
def inference(model, states, test_loader, device):
model.to(device)
 tk0 = tqdm(enumerate(test_loader), total=len(test_loader))
probs = []
for i,(features)in tk0:
features = features.to(device)
 avg_preds = []
for state in states:
model.load_state_dict(state['model'])
 model.eval()
with torch.no_grad() :
y_preds = model(features)
 avg_preds.append(y_preds.softmax(1 ).to('cpu' ).numpy())
 avg_preds = np.mean(avg_preds, axis=0)
 probs.append(avg_preds)
 probs = np.concatenate(probs)
 return probs<save_to_csv>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 pred_df = pd.read_csv('.. /input/sample_submission.csv')
model = CNNStacking(CFG['target_size'])
 states = [torch.load(STAGE2_DIR+f'/fold{fold}_best.pth')for fold in CFG['trn_fold']]
test_dataset = StackingDataset(stage1_predictions)
 test_loader = DataLoader(test_dataset, batch_size=CFG['batch_size'], shuffle=False,
num_workers=CFG['num_workers'], pin_memory=True)
 predictions = inference(model, states, test_loader, device)
 test['label'] = predictions.argmax(1)
 test[['image_id', 'label']].to_csv(OUTPUT_DIR+'submission.csv', index=False )<import_modules>
log_dir = '.. /work'
from fastai.vision.all import *
import albumentations<define_variables>
train_df_n = train_df.astype('float32')/ 255
test_df_n = test_df.astype('float32')/ 255
train_np_n = train_df_n.values
test_np_n = test_df_n.values
print(train_np_n.shape)
 print(test_np_n.shape )
set_seed(42 )<categorify>
X_train, X_val, y_train, y_val = train_test_split(train_np_n, train_label_np, test_size=0.25, random_state=42)
 print(X_train.shape)
 print(y_train.shape)
 print(X_val.shape)
 print(y_val.shape )
class AlbumentationsTransform(RandTransform):
"A transform handler for multiple `Albumentation` transforms"
split_idx,order=None,2
def __init__(self, train_aug, valid_aug): store_attr()
def before_call(self, b, split_idx):
self.idx = split_idx
def encodes(self, img: PILImage):
if self.idx == 0:
aug_img = self.train_aug(image=np.array(img)) ['image']
else:
aug_img = self.valid_aug(image=np.array(img)) ['image']
return PILImage.create(aug_img )<prepare_x_and_y>
print(y_train.shape)
 print(y_val.shape)
 Y_train = to_categorical(y_train)
 Y_val = to_categorical(y_val)
 print(Y_train.shape)
 print(Y_val.shape )
def get_x(row): return data_path/row['image_id']
def get_y(row): return row['label']<choose_model_class>
def step_decay_for_conv2(epoch):
x = 0.0005
if epoch >= 20: x = 0.0001
if epoch >= 40: x = 0.00005
return x
lr_decay = LearningRateScheduler(step_decay_for_conv2,verbose=0 )
class CassavaModel(Module):
def __init__(self, num_classes):
self.effnet = EfficientNet.from_pretrained("efficientnet-b3")
 self.dropout = nn.Dropout(0.1)
 self.out = nn.Linear(1536, num_classes)
 def forward(self, image):
batch_size, _, _, _ = image.shape
x = self.effnet.extract_features(image)
 x = F.adaptive_avg_pool2d(x, 1 ).reshape(batch_size, -1)
 outputs = self.out(self.dropout(x))
return outputs<define_variables>
def create_model2() :
inputs_mnist = Input(shape=(28,28,1))
inputs = Conv2D(filters=64, kernel_size=(3,3), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs_mnist)
 inputs = Conv2D(filters=128, kernel_size=(3,3), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs)
 inputs = BatchNormalization()(inputs)
 inputs = MaxPooling2D(pool_size=(2,2))(inputs)
 inputs = Conv2D(filters=128, kernel_size=(3,3), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs)
 inputs = Conv2D(filters=128, kernel_size=(3,3), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs)
 inputs = BatchNormalization()(inputs)
 inputs = MaxPooling2D(pool_size=(2,2))(inputs)
 inputs = Conv2D(filters=128, kernel_size=(3,3), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs)
 inputs = Conv2D(filters=128, kernel_size=(3,3), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs)
 inputs = BatchNormalization()(inputs)
 inputs = MaxPooling2D(pool_size=(2,2))(inputs)
 inputs = Conv2D(filters=128, kernel_size=(3,3), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs)
 inputs = Conv2D(filters=128, kernel_size=(3,3), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs)
 inputs = BatchNormalization()(inputs)
 inputs_last = MaxPooling2D(pool_size=(2,2))(inputs)
 inputs2 = Conv2D(filters=64, kernel_size=(5,5), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs_mnist)
 inputs2 = Conv2D(filters=128, kernel_size=(5,5), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs2)
 inputs2 = BatchNormalization()(inputs2)
 inputs2 = MaxPooling2D(pool_size=(2,2))(inputs2)
 inputs2 = Conv2D(filters=128, kernel_size=(5,5), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs2)
 inputs2 = Conv2D(filters=128, kernel_size=(5,5), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs2)
 inputs2 = BatchNormalization()(inputs2)
 inputs2 = MaxPooling2D(pool_size=(2,2))(inputs2)
 inputs2 = Conv2D(filters=128, kernel_size=(5,5), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs2)
 inputs2 = Conv2D(filters=128, kernel_size=(5,5), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs2)
 inputs2 = BatchNormalization()(inputs2)
 inputs2 = MaxPooling2D(pool_size=(2,2))(inputs2)
 inputs2 = Conv2D(filters=128, kernel_size=(5,5), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs2)
 inputs2 = Conv2D(filters=128, kernel_size=(5,5), padding='same', bias_regularizer=regularizers.l2(0.005))(inputs2)
 inputs2 = BatchNormalization()(inputs2)
 inputs2_last = MaxPooling2D(pool_size=(2,2))(inputs2)
 inputs_ad_ls = Flatten()(inputs_last)
 inputs_ad_ls = Dense(units=4096, activation='relu' )(inputs_ad_ls)
 inputs_ad_ls = Dropout(rate=0.5 )(inputs_ad_ls)
 inputs_ad_ls = Dense(units=4096, activation='relu' )(inputs_ad_ls)
 inputs_ad_ls = Dropout(rate=0.5 )(inputs_ad_ls)
 outputs_ad_ls = Dense(units=10, activation='softmax', name='1st_fc' )(inputs_ad_ls)
 inputs2_ad_ls = Flatten()(inputs2_last)
 inputs2_ad_ls = Dense(units=4096, activation='relu' )(inputs2_ad_ls)
 inputs2_ad_ls = Dropout(rate=0.5 )(inputs2_ad_ls)
 inputs2_ad_ls = Dense(units=4096, activation='relu' )(inputs2_ad_ls)
 inputs2_ad_ls = Dropout(rate=0.5 )(inputs2_ad_ls)
 outputs2_ad_ls = Dense(units=10, activation='softmax', name='2nd_fc' )(inputs2_ad_ls)
 inputs3 = keras.layers.concatenate([inputs_last, inputs2_last])
 inputs_2_fc = Flatten()(inputs3)
 inputs_2_fc = Dense(units=8192, activation='relu' )(inputs_2_fc)
 inputs_2_fc = Dropout(rate=0.5 )(inputs_2_fc)
 inputs_2_fc = Dense(units=4096, activation='relu' )(inputs_2_fc)
 inputs_2_fc = Dropout(rate=0.5 )(inputs_2_fc)
 inputs_2_fc = Dense(units=4096, activation='relu' )(inputs_2_fc)
 inputs_2_fc = Dropout(rate=0.5 )(inputs_2_fc)
 outputs = Dense(units=10, activation='softmax', name='last_fc' )(inputs_2_fc)
 model = Model(inputs=[inputs_mnist], outputs=[outputs, outputs_ad_ls, outputs2_ad_ls])
 model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'], loss_weights=[1.0, 0.2, 0.2])
 model.summary()
return model
Path('/kaggle/input' ).ls()<load_pretrained>
_model = create_model2()
learn = load_learner(Path('/kaggle/input/effnet-inference/inference(1)'), cpu=False )<define_variables>
def fit_the_model(_model, _epochs):
original_hist = _model.fit(np.array(X_train), [np.array(Y_train),np.array(Y_train),np.array(Y_train)], epochs=_epochs, batch_size=batch_size,
verbose=1,
callbacks=[lr_decay],
validation_data=(np.array(X_val), [np.array(Y_val),np.array(Y_val),np.array(Y_val)]))
return original_hist
path = Path(".. /input")
 data_path = path/'cassava-leaf-disease-classification'<load_from_csv>
def fit_the_model_with_data(_model, _epochs, X_train, Y_train, X_val, Y_val, _cp):
original_hist = _model.fit(np.array(X_train), [np.array(Y_train),np.array(Y_train),np.array(Y_train)], epochs=_epochs, batch_size=batch_size,
verbose=0,
callbacks=[lr_decay, _cp],
validation_data=(np.array(X_val), [np.array(Y_val),np.array(Y_val),np.array(Y_val)]))
return original_hist
test_df = pd.read_csv(data_path/'sample_submission.csv')
 test_df.head()<prepare_output>
from sklearn.model_selection import KFold
import numpy as np
test_copy = test_df.copy()
test_copy['image_id'] = test_copy['image_id'].apply(lambda x: f'test_images/{x}' )<train_model>
X = np.copy(train_np_n)
 y = np.copy(train_label_np)
 kf = KFold(n_splits=4,shuffle=True)
 kf.get_n_splits(X)
 print(kf )
test_dl = learn.dls.test_dl(test_copy )<predict_on_test>
_i = 0
for train_index, val_index in kf.split(X):
filepath=".. /work/kfold_cp"+str(_i)+".hdf5"
_cp = ModelCheckpoint(filepath, monitor='val_last_fc_acc', verbose=1, save_best_only=True, save_weights_only=False, mode='max', period=1)
 _m = create_model2() ;
X_t, X_v = X[train_index], X[val_index]
y_t, y_v = y[train_index], y[val_index]
X_t = X_t.reshape(X_t.shape[0],28,28,1)
 X_v = X_v.reshape(X_v.shape[0],28,28,1)
 Y_t = to_categorical(y_t)
 Y_v = to_categorical(y_v)
 print(X_t.shape, y_t.shape, Y_t.shape, X_v.shape, y_v.shape, Y_v.shape)
 epochs=100
history = fit_the_model_with_data(_m, epochs, X_t, Y_t, X_v, Y_v, _cp)
 check_result(history.history);
_m.save(".. /work/"+"kfold"+str(_i)+'.h5')
 _i += 1
preds, _ = learn.get_preds(dl=test_dl )<feature_engineering>
model_cp0 = load_model(".. /work/kfold_cp0.hdf5")
 model_cp1 = load_model(".. /work/kfold_cp1.hdf5")
 model_cp2 = load_model(".. /work/kfold_cp2.hdf5")
 model_cp3 = load_model(".. /work/kfold_cp3.hdf5" )
test_df['label'] = preds.argmax(dim=-1 ).numpy()<save_to_csv>
def pred_argmax(_model, _data, _label=np.array([0])) :
c_prob, c_prob1, c_prob2 = _model.predict(_data)
 pred= np.argmax(c_prob, axis=1)
 if len(_label)!= 1:
correct = np.argmax(_label,axis=1)
 else:
correct = 0
return c_prob, pred, correct
def pred_argmax2(_model,_gen, _data, _label=np.array([0])) :
_batch_size=128
_s = len(_data)/ _batch_size
c_prob = _model.predict_generator(_gen.flow(_data,batch_size=_batch_size,shuffle=None), steps=_s)
 pred= np.argmax(c_prob, axis=1)
 if len(_label)!= 1:
correct = np.argmax(_label,axis=1)
 else:
correct = 0
return c_prob, pred, correct
def disp_image(_data, _pred, _correct, _list, _isdisplay="off"):
pair_list = []
wrong_list = []
for _i in _list:
if _isdisplay == "on":
plt.imshow(_data[_i].reshape(28, 28), cmap=plt.get_cmap('gray'))
plt.show()
pair_list.append(str(_correct[_i])+" as "+str(_pred[_i]))
wrong_list.append(_pred[_i])
 return pair_list, wrong_list
test_df.to_csv('submission.csv', index=False )<define_variables>
X_all = train_np_n.reshape(train_np_n.shape[0], 28, 28, 1)
 Y_all = to_categorical(train_label_np)
 print(X_all.shape)
 print(Y_all.shape)
package_paths = [
'.. /input/pytorch-image-models/pytorch-image-models-master'
]
for pth in package_paths:
sys.path.append(pth )<import_modules>
X0_cp_all_prob, X0_cp_all_pred, X0_cp_all_correct = pred_argmax(model_cp0, X_all,Y_all)
 X1_cp_all_prob, X1_cp_all_pred, X1_cp_all_correct = pred_argmax(model_cp1, X_all,Y_all)
 X2_cp_all_prob, X2_cp_all_pred, X2_cp_all_correct = pred_argmax(model_cp2, X_all,Y_all)
 X3_cp_all_prob, X3_cp_all_pred, X3_cp_all_correct = pred_argmax(model_cp3, X_all,Y_all)
 _d0_cp = np.not_equal(X0_cp_all_pred, X0_cp_all_correct)
 _d1_cp = np.not_equal(X1_cp_all_pred, X1_cp_all_correct)
 _d2_cp = np.not_equal(X2_cp_all_pred, X2_cp_all_correct)
 _d3_cp = np.not_equal(X3_cp_all_pred, X3_cp_all_correct)
 print(sum(_d0_cp))
print(sum(_d1_cp))
print(sum(_d2_cp))
print(sum(_d3_cp))
DF_cp = np.argmin([sum(_d0_cp),sum(_d1_cp),sum(_d2_cp),sum(_d3_cp)], axis=0)
 print(DF_cp )
<load_from_csv>
X0_test_prob, X0_test_pred, X0_test_correct = pred_argmax(model_cp0, X_test)
 X1_test_prob, X1_test_pred, X1_test_correct = pred_argmax(model_cp1, X_test)
 X2_test_prob, X2_test_pred, X2_test_correct = pred_argmax(model_cp2, X_test)
 X3_test_prob, X3_test_pred, X3_test_correct = pred_argmax(model_cp3, X_test)
submission = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 submission.head()<set_options>
pred_info_test = np.concatenate([X0_test_pred.reshape(X0_test_pred.shape[0],1), X1_test_pred.reshape(X1_test_pred.shape[0],1), X2_test_pred.reshape(X2_test_pred.shape[0],1), X3_test_pred.reshape(X3_test_pred.shape[0],1)], axis=1)
 print(pred_info_test.shape)
 print(pred_info_test )
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
def get_img(path):
im_bgr = cv2.imread(path)
 im_rgb = im_bgr[:, :, ::-1]
return im_rgb
ShiftScaleRotate, Normalize, Compose, CenterCrop, Resize, HorizontalFlip,
VerticalFlip, Transpose, RandomResizedCrop, HueSaturationValue, RandomBrightnessContrast,
CoarseDropout, Cutout 
)
 def get_infer_transforms() :
return Compose([
CenterCrop(config['img_size'], config['img_size'], p=1.0),
Resize(config['img_size'], config['img_size']),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<categorify>
def return_result(preds):
_c = Counter(preds)
 _v = _c.most_common(1)[0][0]
_n = _c.most_common(1)[0][1]
if _n == 4 or _n ==3:
return _v
else:
return preds[DF_cp]
class LeafDataset(Dataset):
def __init__(self, df, img_dir, transforms=None, include_labels=True):
super().__init__()
self.df = df
self.img_dir = img_dir
self.transforms = transforms
self.include_labels = include_labels
if include_labels:
self.labels = self.df['label'].values
def __len__(self):
return len(self.df)
 def __getitem__(self, index: int):
img = get_img("{}/{}".format(self.img_dir, self.df.loc[index]['image_id']))
if self.transforms:
img = self.transforms(image=img)['image']
if self.include_labels:
label = self.labels[index]
return img, label
else:
return img;<choose_model_class>
rdf_test = pd.DataFrame(pred_info_test ).apply(return_result, axis=1)
 rdf_test
class LeafDiseaseClassifier(nn.Module):
def __init__(self, model_arch, num_classes, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, num_classes)
 def forward(self, x):
x = self.model(x)
 return x
def freeze_batch_norm(self):
layers = [mod for mod in self.model.children() ]
for layer in layers:
if isinstance(layer, nn.BatchNorm2d):
for param in layer.parameters() :
param.requires_grad = False
elif isinstance(layer, nn.Sequential):
for seq_layers in layer.children() :
if isinstance(layer, nn.BatchNorm2d):
param.requires_grad = False<create_dataframe>
pred_df["label"]=rdf_test
pred_df
if __name__ == '__main__':
seed_everything(config['seed'])
 test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds = LeafDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_infer_transforms() , include_labels=False)
 test_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=config['test_bs'],
num_workers=config['num_workers'],
shuffle=False, 
)
 device = torch.device(config['device'])
 model = LeafDiseaseClassifier(config['model_arch'], config['num_classes'] ).to(device)
 model.load_state_dict(torch.load('.. /input/effnet-b4/tf_efficientnet_b4_ns_Fold4_Epoch5_Acc_0.8952.pth'))
model.eval()
preds = []
pbar = tqdm(enumerate(test_loader), total=len(test_loader))
for step,(test_batch)in pbar:
test_batch = test_batch.to(device ).float()
test_preds = model(test_batch)
 preds += [torch.softmax(test_preds, 1 ).detach().cpu().numpy() ]
preds = np.concatenate(preds, axis=0)
<save_to_csv>
pred_df.to_csv('submission.csv', index=False )