Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<concatenate>
!pip install --quiet /kaggle/input/kerasapplications
!pip install --quiet /kaggle/input/efficientnet-git
new_data=pd.concat([train,test],axis=0)
 new_data.head()<count_missing_values>
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
seed = 21
seed_everything(seed)
 warnings.filterwarnings('ignore' )
new_data.isnull().sum()<filter>
BATCH_SIZE = 32 * REPLICAS
HEIGHT = 512
WIDTH = 512
CHANNELS = 3
N_CLASSES = 5
TTA_STEPS = 8
new_data[new_data['release_date'].isnull() ]<feature_engineering>
def data_augment(image, label):
p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if p_spatial >.75:
image = tf.image.transpose(image)
 if p_rotate >.75:
image = tf.image.rot90(image, k=3)
 elif p_rotate >.5:
image = tf.image.rot90(image, k=2)
 elif p_rotate >.25:
image = tf.image.rot90(image, k=1)
 if p_pixel_1 >=.4:
image = tf.image.random_saturation(image, lower=.7, upper=1.3)
 if p_pixel_2 >=.4:
image = tf.image.random_contrast(image, lower=.8, upper=1.2)
 if p_pixel_3 >=.4:
image = tf.image.random_brightness(image, max_delta=.1)
 if p_crop >.7:
if p_crop >.9:
image = tf.image.central_crop(image, central_fraction=.7)
 elif p_crop >.8:
image = tf.image.central_crop(image, central_fraction=.8)
 else:
image = tf.image.central_crop(image, central_fraction=.9)
 elif p_crop >.4:
crop_size = tf.random.uniform([], int(HEIGHT*.8), HEIGHT, dtype=tf.int32)
 image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])
 return image, label
new_data['release_date']=new_data['release_date'].fillna('3/20/01' )<data_type_conversions>
def get_name(file_path):
parts = tf.strings.split(file_path, os.path.sep)
 name = parts[-1]
return name
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
return image
def center_crop(image):
image = tf.reshape(image, [600, 800, CHANNELS])
 h, w = image.shape[0], image.shape[1]
if h > w:
image = tf.image.crop_to_bounding_box(image,(h - w)// 2, 0, w, w)
 else:
image = tf.image.crop_to_bounding_box(image, 0,(w - h)// 2, h, h)
 image = tf.image.resize(image, [HEIGHT, WIDTH])
 return image
def resize_image(image, label):
image = tf.image.resize(image, [HEIGHT, WIDTH])
 image = tf.reshape(image, [HEIGHT, WIDTH, CHANNELS])
 return image, label
def process_path(file_path):
name = get_name(file_path)
 img = tf.io.read_file(file_path)
 img = decode_image(img)
 return img, name
def get_dataset(files_path, shuffled=False, tta=False, extension='jpg'):
dataset = tf.data.Dataset.list_files(f'{files_path}*{extension}', shuffle=shuffled)
 dataset = dataset.map(process_path, num_parallel_calls=AUTO)
 if tta:
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.map(resize_image, num_parallel_calls=AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n )
new_data['release_year']=pd.to_datetime(new_data['release_date'] ).dt.year
new_data['release_month']=pd.to_datetime(new_data['release_date'] ).dt.month
new_data['release_day']=pd.to_datetime(new_data['release_date'] ).dt.day<feature_engineering>
model_path_list = glob.glob('/kaggle/input/cassava-leaf-disease-training-with-tpu-v2-pods/*.h5')
 model_path_list.sort()
print('Models to predict:')
 print(*model_path_list, sep='
' )
new_data['release_year'].loc[new_data['release_year']>=2018]-=100<drop_column>
def model_fn(input_shape, N_CLASSES):
inputs = L.Input(shape=input_shape, name='input_image')
 base_model = efn.EfficientNetB4(input_tensor=inputs,
include_top=False,
weights=None,
pooling='avg')
 x = L.Dropout (.5 )(base_model.output)
 output = L.Dense(N_CLASSES, activation='softmax', name='output' )(x)
 model = Model(inputs=inputs, outputs=output)
 return model
with strategy.scope() :
model = model_fn(( None, None, CHANNELS), N_CLASSES)
 model.summary()
new_data=new_data.drop('release_date',axis=1 )<define_variables>
files_path = f'{database_base_path}test_images/'
test_size = len(os.listdir(files_path))
test_preds = np.zeros(( test_size, N_CLASSES))
for model_path in model_path_list:
print(model_path)
 K.clear_session()
model.load_weights(model_path)
 if TTA_STEPS > 0:
test_ds = get_dataset(files_path, tta=True ).repeat()
ct_steps = TTA_STEPS *(( test_size/BATCH_SIZE)+ 1)
 preds = model.predict(test_ds, steps=ct_steps, verbose=1)[:(test_size * TTA_STEPS)]
preds = np.mean(preds.reshape(test_size, TTA_STEPS, N_CLASSES, order='F'), axis=1)
 test_preds += preds / len(model_path_list)
 else:
test_ds = get_dataset(files_path, tta=False)
 x_test = test_ds.map(lambda image, image_name: image)
 test_preds += model.predict(x_test)/ len(model_path_list)
 test_preds = np.argmax(test_preds, axis=-1)
 test_names_ds = get_dataset(files_path)
 image_names = [img_name.numpy().decode('utf-8')for img, img_name in iter(test_names_ds.unbatch())]
weiji=[1929,1930,1930,1931,1932,1933,1934,1935,1936,1937,1938,1939,1940,1941,1942,1943,1944,1945,1950,1951,1952,1953,1961,1962,1963,1964,1965,1966,1967,1968,1969,1971,1973,1974,1975,1980,1981,1982,1983,1984,1985,1986,1987,1988,1989,
1990,1991,1992,1993,1994,1995,1996,1997,1998,1999,2007,2008,2009,2010,2011]<feature_engineering>
submission = pd.DataFrame({'image_id': image_names, 'label': test_preds})
 submission.to_csv('submission.csv', index=False)
 display(submission.head() )