Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<train_model>
import numpy as np
import pandas as pd
import os
gbm = lgb.train(params,
lgb_train,
num_boost_round=10000,
valid_sets=lgb_eval,
early_stopping_rounds=500 )<predict_on_test>
BASE_DIR = "/kaggle/input/cassava-leaf-disease-classification/"
TRAIN_DIR = "/kaggle/input/cassava-leaf-disease-classification/train_images/"
TEST_DIR = "/kaggle/input/cassava-leaf-disease-classification/test_images/"
lr.fit(new_train, train['target'])
 LR_TEST=lr.predict_proba(new_test)[:, 1]
new_test=new_test.astype(float)
 LGBM_TEST= gbm.predict(new_test, num_iteration=gbm.best_iteration)
<save_to_csv>
import numpy as np
import pandas as pd
import os
import matplotlib.pyplot as plt
import json
import cv2
from PIL import Image
prediction=pd.DataFrame({'id':test.id,'LR_TEST':LR_TEST,'LGBM_TEST':LGBM_TEST})
 submit=pd.DataFrame({'id':test.id,'target':LR_TEST})
 prediction.to_csv('prediction.csv',index=False)
 submit.to_csv('submission.csv',index=False )<load_from_csv>
with open(os.path.join(BASE_DIR, "label_num_to_disease_map.json")) as file:
map_classes = json.loads(file.read())
 print(json.dumps(map_classes, indent=2))
df_train = pd.read_csv('.. /input/cat-in-the-dat/train.csv')
 df_test = pd.read_csv('.. /input/cat-in-the-dat/test.csv')
 print(df_train.shape)
 print(df_test.shape)
 df_train.info()<count_unique_values>
input_files = os.listdir(os.path.join(BASE_DIR, "train_images"))
print(f"Number of train images: {len(input_files)}" )
for i in range(10):
col = 'nom_{}'.format(i)
 print(col, df_train[col].nunique())
 for i in range(6):
col = 'ord_{}'.format(i)
 print(col, df_train[col].nunique() )<feature_engineering>
IMG_HEIGHT = 400
IMG_WIDTH = 400
batch_size = 32
PRE_TRAINED_MODEL = '.. /input/resnet50v02/Cassava_Best_ResNet50_Model_V02.hdf5'
for df in [df_train, df_test]:
for col in cyclic_cols:
df[col+'_sin'] = np.sin(( 2 * np.pi * df[col])/ max(df[col]))
df[col+'_cos'] = np.cos(( 2 * np.pi * df[col])/ max(df[col]))<categorify>
from keras.preprocessing.image import load_img
from keras.preprocessing.image import img_to_array
from keras.preprocessing.image import ImageDataGenerator
target = df_train['target']
train_id = df_train['id']
test_id = df_test['id']
train0 = df_train.copy().drop(['target', 'id'], axis=1)
 test0 = df_test.copy().drop('id', axis=1)
 print(train0.shape)
 print(test0.shape)
 for df in [train0, test0]:
df['bin_3'] = df['bin_3'].map({'T': 1, 'F': 0})
 df['bin_4'] = df['bin_4'].map({'Y': 1, 'N': 0})
 df[cyclic_cols] /= df[cyclic_cols].max(axis=0)
 df.drop(['ord_{}'.format(i)for i in range(6)], axis=1, inplace=True)
 df[['ord_{}_sort'.format(i)for i in range(6)]] /= df[['ord_{}_sort'.format(i)for i in range(6)]].max(axis=0)
 train0.head()<categorify>
test_filenames = os.listdir(TEST_DIR)
 test_df = pd.DataFrame({
'image_id': test_filenames
})
 test_samples = test_df.shape[0]
test_samples
%%time
col_names = ['nom_{}'.format(i)for i in range(5)] + ['ord_{}'.format(i)for i in range(5)] + ['day', 'month']
train_ohe = df_train[col_names]
test_ohe = df_test[col_names]
traintest = pd.concat([train_ohe, test_ohe])
 dummies = pd.get_dummies(traintest, columns=traintest.columns, drop_first=True, sparse=True)
 train_ohe1 = dummies.iloc[:df_train.shape[0], :]
test_ohe1 = dummies.iloc[df_train.shape[0]:, :]
print('train_ohe1.shape', train_ohe1.shape)
 col_names = ['nom_{}'.format(i)for i in range(5, 10)] + ['ord_{}'.format(i)for i in range(5, 6)]
train_ohe = df_train[col_names]
test_ohe = df_test[col_names]
traintest = pd.concat([train_ohe, test_ohe])
 dummies = pd.get_dummies(traintest, columns=traintest.columns, drop_first=True, sparse=True)
 train_ohe2 = dummies.iloc[:df_train.shape[0], :].sparse.to_coo().tocsr()
test_ohe2 = dummies.iloc[df_train.shape[0]:, :].sparse.to_coo().tocsr()
print('train_ohe2.shape', train_ohe2.shape)
 n_components = 200
svd = TruncatedSVD(n_components=n_components)
 train_svd = svd.fit_transform(train_ohe2)
 test_svd = svd.transform(test_ohe2)
 print('train_svd.shape', train_svd.shape)
 train = train0.drop(['nom_{}'.format(i)for i in range(10)], axis=1)
 test = test0.drop(['nom_{}'.format(i)for i in range(10)], axis=1)
 col_names = ['pca_{}'.format(i)for i in range(n_components)]
train = pd.concat([train, train_ohe1, pd.DataFrame(train_svd, columns=col_names)], axis=1)
 test = pd.concat([test, test_ohe1, pd.DataFrame(test_svd, columns=col_names)], axis=1)
 print('train.shape', train.shape )<categorify>
keras.backend.clear_session()
np.random.seed(42)
 tf.random.set_seed(42 )
col_names = ['bin_{}'.format(i)for i in range(5)] \
+ ['nom_{}'.format(i)for i in range(10)] \
+ ['ord_{}'.format(i)for i in range(6)] \
+ ['day', 'month']
train_ohe = df_train[col_names]
test_ohe = df_test[col_names]
traintest = pd.concat([train_ohe, test_ohe])
 dummies = pd.get_dummies(traintest, columns=traintest.columns, drop_first=True, sparse=True)
 train_ohe = dummies.iloc[:df_train.shape[0], :].sparse.to_coo().tocsr()
test_ohe = dummies.iloc[df_train.shape[0]:, :].sparse.to_coo().tocsr()
print('train_ohe.shape', train_ohe.shape )<compute_train_metric>
from keras.models import load_model
lr1 = LogisticRegression(solver='lbfgs', C=0.1)
 lr2 = LogisticRegression(solver='lbfgs', C=0.1)
 with ignore_warnings(category=FutureWarning):
%time print('Dense CV:', cross_val_score(lr1, train, target, cv=2, scoring='roc_auc', n_jobs=-1 ).mean())
 %time print('OHE CV:', cross_val_score(lr2, train_ohe, target, cv=2, scoring='roc_auc', n_jobs=-1 ).mean() )<train_model>
def get_augmented_images(image_id):
image_path = os.path.join(TEST_DIR, image_id)
 image_data = Image.open(image_path)
 image_data = image_data.resize(( IMG_HEIGHT, IMG_WIDTH), Image.ANTIALIAS)
 image_data = np.expand_dims(image_data, axis = 0)
 test_datagen = ImageDataGenerator(validation_split = 0.2,
rotation_range = 45,
zoom_range = 0.4,
horizontal_flip = True,
vertical_flip = True,
fill_mode = 'nearest',
shear_range = 0.1,
height_shift_range = 0.1,
width_shift_range = 0.1)
 test_datagen = ImageDataGenerator(rescale=1.0/255, validation_split = 0.2,
rotation_range = 45,
zoom_range = 0.4,
horizontal_flip = True,
vertical_flip = True,
fill_mode = 'nearest',
shear_range = 0.1,
height_shift_range = 0.1,
width_shift_range = 0.1,
featurewise_center = True,
featurewise_std_normalization = True)
 test_datagen.fit(image_data)
 it = test_datagen.flow(image_data, batch_size=1)
 image_list = []
image_list.append(image_data)
 for i in range(5):
batch = it.next()
image_list.append(batch)
 return image_list
<save_to_csv><EOS> 
test_results_list = []
for image_id in test_df["image_id"]:
image_list = get_augmented_images(image_id)
 predict_list = []
for image_data in image_list:
predict_class_1 = model.predict(image_data)
 print("Probabilities :: {}, :: Predicted Class : {}".format(predict_class_1[0],np.argmax(predict_class_1[0])))
 predict_list.append(predict_class_1[0])
 predict_array = np.array(predict_list)
 predict_class = np.mean(predict_array, axis=0)
 print("Augmented Probabilities :: {}, :: Augmented Predicted Class : {}".format(predict_class,np.argmax(predict_class)))
 test_results_dict = {}
test_results_dict["image_id"] = image_id
test_results_dict["label"] = np.argmax(predict_class)
 test_results_list.append(test_results_dict)
 test_results_df = pd.DataFrame(test_results_list)
 test_results_df.to_csv("submission.csv" ,index = False)