Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<drop_column>
!pip install --quiet /kaggle/input/kerasapplications
!pip install --quiet /kaggle/input/efficientnet-git
train = train.drop(['homepage'], axis=1)
 test = test.drop(['homepage'], axis=1 )<drop_column>
import math, re, os
import tensorflow as tf
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from kaggle_datasets import KaggleDatasets
from tensorflow import keras
from functools import partial
from sklearn.model_selection import train_test_split
from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau
from efficientnet.tfkeras import EfficientNetB3, EfficientNetB4, EfficientNetB5
import tensorflow.keras.backend as K
train = train.drop(['poster_path'], axis=1)
 test = test.drop(['poster_path'], axis=1 )<drop_column>
AUTOTUNE = tf.data.experimental.AUTOTUNE
WORK_PATH = '.. /input/cassava-leaf-disease-classification'
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
IMAGE_SIZE = [512, 512]
CHANNELS = 3
CLASSES = ['0', '1', '2', '3', '4']
EPOCHS = 30
train = train.drop(['status'], axis=1)
 test = test.drop(['status'], axis=1 )<feature_engineering>
def decode_image(image):
image = tf.image.decode_jpeg(image, channels=CHANNELS)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
for col in ['title', 'tagline', 'overview', 'original_title']:
train['len_' + col] = train[col].fillna('' ).apply(lambda x: len(str(x)))
 train['words_' + col] = train[col].fillna('' ).apply(lambda x: len(str(x.split(' '))))
test['len_' + col] = test[col].fillna('' ).apply(lambda x: len(str(x)))
 test['words_' + col] = test[col].fillna('' ).apply(lambda x: len(str(x.split(' '))))<drop_column>
def read_tfrecord(example, labeled):
tfrecord_format = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64)
 } if labeled else {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string)
 }
example = tf.io.parse_single_example(example, tfrecord_format)
 image = decode_image(example['image'])
 if labeled:
label = tf.cast(example['target'], tf.int32)
 return image, label
idnum = example['image_name']
return image, idnum
train = train.drop(["imdb_id", "original_title", "overview", "tagline", "title"], axis=1)
 test = test.drop(["imdb_id", "original_title", "overview", "tagline", "title"], axis=1 )<prepare_x_and_y>
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE)
 return dataset
X = train.drop(['id', 'revenue'], axis=1)
 Y = np.log1p(train['revenue'])
 X_test = test.drop(['id'], axis=1 )<split>
TEST_FILENAMES = tf.io.gfile.glob(WORK_PATH + '/test_tfrecords/ld_test*.tfrec' )
X_train, X_valid, Y_train, Y_valid = train_test_split(X, Y, test_size=0.1 )<choose_model_class>
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTOTUNE)
 return dataset
params = {'num_leaves': 30,
'min_data_in_leaf': 20,
'objective': 'regression',
'max_depth': 5,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
model = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 model.fit(X_train, Y_train,
eval_set=[(X_train, Y_train),(X_valid, Y_valid)], eval_metric='rmse',
verbose=1000, early_stopping_rounds=200 )<predict_on_test>
with strategy.scope() :
base_model = EfficientNetB5(weights=None, include_top=False, input_shape =(None, None, 3), pooling='avg')
 model = tf.keras.Sequential([
base_model,
tf.keras.layers.Dropout (.3),
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model.compile(
 optimizer=tf.keras.optimizers.Adam(lr = 0.001),
loss='sparse_categorical_crossentropy',
metrics=['acc'] )
y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration_ )<save_to_csv>
model.load_weights('.. /input/cassava-leaf-disease-tpu-efficientnetb4/EffNetB5_best_weights.h5' )
sample_submission['revenue'] = np.expm1(y_pred)
 sample_submission.to_csv("submission.csv", index=False )<set_options>
def to_float32(image, label):
return tf.cast(image, tf.float32), label
pd.set_option('max_columns', None)
 %matplotlib inline
plt.style.use('ggplot')
 stop = set(stopwords.words('english'))
py.init_notebook_mode(connected=True)
 print(os.listdir(".. /input")) 
<load_from_csv>
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES )
<load_from_csv><EOS> 
test_ds = get_test_dataset(ordered=True ).map(to_float32)
 print('Computing predictions...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilities = model.predict(test_images_ds)
 test_preds = np.argmax(probabilities, axis=-1)
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
image_names = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 submission = pd.DataFrame({'image_id': image_names, 'label': test_preds})
 submission.to_csv('submission.csv', index=False)
 display(submission.head() )