Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<count_unique_values>
os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'
BATCH_SIZE = 64
ROW = 300
COL = 300
train_csv_loc = '.. /input/cassava-leaf-disease-classification/train.csv'
train_location = '.. /input/cassava-leaf-disease-classification/train_images/'
test_location = '.. /input/cassava-leaf-disease-classification/test_images'
for col in cat_trans_cols:
print(col, transaction_data[col].nunique() , "
" )<define_variables>
def augmentations(file):
file = read_file(file)
 file = image.decode_jpeg(file, channels=3)
 transform = A.Compose([
A.RandomRotate90() ,
A.Flip() ,
A.Transpose() ,
A.OneOf([
A.IAAAdditiveGaussianNoise() ,
A.GaussNoise() ,
], p=0.2),
A.OneOf([
A.MotionBlur(p=.2),
A.MedianBlur(blur_limit=3, p=0.1),
A.Blur(blur_limit=3, p=0.1),
], p=0.2),
A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=45, p=0.2),
A.OneOf([
A.OpticalDistortion(p=0.3),
A.GridDistortion(p=.1),
A.IAAPiecewiseAffine(p=0.3),
], p=0.2),
A.OneOf([
A.CLAHE(clip_limit=2),
A.IAASharpen() ,
A.IAAEmboss() ,
A.RandomBrightnessContrast() ,
A.HueSaturationValue(hue_shift_limit=30),
A.RandomFog(p=.3)
 ], p=0.7),
A.HueSaturationValue(p=0.3),
A.OneOf([
A.CoarseDropout(max_holes=16, max_height=16, max_width=16),
A.Cutout(num_holes=16, max_h_size=16, max_w_size=16),
A.ElasticTransform(alpha=1, sigma=25)
 ], p=0.4),
])
 file = transform(image=file.numpy())['image']
file = preprocess_input(file)
 file = image.resize(file, [ROW, COL])
 return file
low_card_trans_cols = ["ProductCD", "card4", "card6", "M1", "M2", "M3", "M4", "M6", "M7", "M8", "M9"]
high_card_trans_cols = ["P_emaildomain"]
print("lists successfully created!" )<categorify>
def fetch_image_without_aug(filename , label):
image_file = read_file(filename)
 image_file = image.decode_jpeg(image_file, channels=3)
 image_file = preprocess_input(image_file)
 image_file = image.resize(image_file, [ROW, COL])
 return image_file, label
for i in cat_trans_cols:
most_frequent_value = transaction_data[i].mode() [0]
print("For column: ", i, "the most frequent value is: ", most_frequent_value, "
")
 transaction_data[i].fillna(most_frequent_value, inplace = True )<categorify>
def fetch_image_with_aug(filename , label):
aug_img = numpy_function(func=augmentations, inp=[filename], Tout=float32)
 return aug_img, label
label_encoder = LabelEncoder()
print("transaction_data.shape before label-encoding: ", transaction_data.shape, "
")
 transaction_data[high_card_trans_cols] = label_encoder.fit_transform(transaction_data[high_card_trans_cols])
 print("transaction_data.shape after label-encoding: ", transaction_data.shape, "
")
 print("transaction_data[high_card_trans_cols] after label_encoding: ",transaction_data[high_card_trans_cols], "
" )<count_unique_values>
def getDatasetFromDataframe(train_files, train_labels, val_files, val_labels):
train_ds = Dataset.from_tensor_slices(( train_files, train_labels))
train_ds = train_ds.shuffle(len(train_files))
train_ds = train_ds.map(fetch_image_with_aug , num_parallel_calls=16)
 train_ds = train_ds.batch(BATCH_SIZE)
 train_ds = train_ds.prefetch(AUTOTUNE)
 val_ds = Dataset.from_tensor_slices(( val_files, val_labels))
val_ds = val_ds.shuffle(len(val_files))
val_ds = val_ds.map(fetch_image_without_aug , num_parallel_calls=16)
 val_ds = val_ds.batch(BATCH_SIZE)
 val_ds = val_ds.prefetch(AUTOTUNE)
 return train_ds , val_ds
for col in cat_id_cols:
print(col, identity_data[col].nunique() , "
" )<define_variables>
def create_model(training = True, weights = "imagenet"):
base_model = EfficientNetB3(weights=weights , include_top=False , input_shape=(ROW,COL,3))
x = Input(shape =(ROW,COL,3))
out_1 = base_model(x,training = training)
 out_1 = GlobalAveragePooling2D(name = 'encoding' )(out_1)
 out_1 = Dropout(0.5 )(out_1)
 output = Dense(5 , activation="softmax" )(out_1)
 final_model = Model(inputs = x , outputs = output)
 final_model.summary()
return final_model
low_card_id_cols = ["id_12", "id_15", "id_16", "id_28", "id_29", "id_34", "id_35", "id_36", "id_37", "id_38", "DeviceType"]
high_card_id_cols = ["id_30", "id_31", "id_33", "DeviceInfo"]
print("lists successfully created!" )<categorify>
train_csv = pd.read_csv(train_csv_loc)
 train_csv['image_id'] = train_csv['image_id'].map(lambda x: train_location+x)
 Folder_name = "Exp_"
create_k_folds = StratifiedKFold(n_splits=5)
 fold_number = 1
for train_index, test_index in create_k_folds.split(train_csv['image_id'] , train_csv['label']):
print("Fold Number {} is starting it's training".format(fold_number))
print("Creating Callbacks...")
 Folder_name = Folder_name + str(fold_number)
 callbacks, histories = create_callbacks(Folder_name)
 print("Importing Datasets...")
 X_train, X_test = train_csv['image_id'].loc[train_index], train_csv['image_id'].loc[test_index]
y_train, y_test = train_csv['label'].loc[train_index], train_csv['label'].loc[test_index]
train_ds, val_ds = getDatasetFromDataframe(X_train, y_train, X_test, y_test)
 final_model = create_model()
final_model.compile(optimizer=RMSprop(learning_rate=1e-4),
loss=SparseCategoricalCrossentropy() , metrics=['accuracy'])
 try:
print("Starting training...")
 hist = final_model.fit(train_ds, epochs=15,
validation_data=val_ds, callbacks=callbacks, workers=16)
 except:
print("Error...")
 fold_number += 1
for i in cat_id_cols:
most_frequent_value = identity_data[i].mode() [0]
print("For column: ", i, "the most frequent value is: ", most_frequent_value, "
")
 identity_data[i].fillna(most_frequent_value, inplace = True )<categorify>
checkpoints = glob.glob(os.path.join('.. /input/efficient-net-weights/Weights2' , 'Exp2_*.h5'))
Models = []
for checkpoint in checkpoints:
new_model = create_model(training = False, weights = None)
 new_model.load_weights(checkpoint)
 Models.append(new_model)
 print('Models loaded...')
 images = os.listdir(test_location)
 results = []
for single in images:
image = load_img(test_location+'/'+single , target_size=(ROW,COL))
image = img_to_array(image)
 image = preprocess_input(image)
 batch = np.array([image])
 res = []
for model in Models:
res.append(model.predict(batch))
res = np.mean(res , 0)
 results.append({"image_id" : single , "label" : np.argmax(res)} )
<categorify><EOS> 
submission = pd.DataFrame(results)
 submission.to_csv('submission.csv' , index = False )