Kaggle competition: Cassava Leaf Disease Classification
count=new_train['production_countries'].value_counts()
count=count[count>5]<groupby>
print("Tensorflow version " + tf.__version__)
bud=new_train.groupby('production_countries' ).budget.mean()<sort_values>
AUTOTUNE = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
FILENAMES = tf.io.gfile.glob(".. /input/cassava-leaf-disease-classification" + '/test_tfrecords/ld_test*.tfrec')
 BATCH_SIZE = 128 * strategy.num_replicas_in_sync
IMAGE_SIZE = [512, 512]
classes = ['0', '1', '2', '3', '4']
os.environ['PYTHONHASHSEED']=str(34)
 os.environ['TF_CUDNN_DETERMINISTIC'] = '1'
random.seed(34)
 np.random.seed(34)
 tf.random.set_seed(34 )
bud=bud.sort_values(ascending=False)[:10]<groupby>
def decode_image(image):
image = tf.image.decode_jpeg(image, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['target'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example, return_image_name):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum if return_image_name else 0
def get_val_dataset(files, one_hot = False,
shuffle = False, repeat = False,
labeled = True, return_image_names = True,
batch_size = BATCH_SIZE, dim = IMAGE_SIZE[0]):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTOTUNE)
 if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(2048)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTOTUNE)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTOTUNE)
 if one_hot:
ds = ds.map(onehot, num_parallel_calls = AUTOTUNE)
 ds = ds.batch(batch_size)
 ds = ds.prefetch(AUTOTUNE)
 return ds
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 def onehot(image,label):
CLASSES = len(classes)
 return image,tf.one_hot(label,CLASSES )
rev=new_train.groupby('production_countries' ).revenue.mean()<sort_values>
VALIDATE = False
rev=rev.sort_values(ascending=False)[:10]<filter>
FOLDS=5
SEED=34
if VALIDATE:
GCS_PATH = KaggleDatasets().get_gcs_path('cassava-leaf-disease-tfrecords-512x512')
 TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/*.tfrec')
 AUG_TYPE = 'CUTMIXUP'
new_train['production_countries'].loc[new_train['production_countries']=='ET']='Ethiopia'
along_co=new_train[new_train['production_countries'].apply(lambda x:1 if len(x)==2 else 0)==1]
along_co.head()<define_variables>
if VALIDATE:
histories = []
oof_pred = []; oof_labels = []
kfold = KFold(FOLDS, shuffle = True, random_state = SEED)
 for f,(train_index, val_index)in enumerate(kfold.split(TRAINING_FILENAMES)) :
print('
print('Getting datasets...'); print('')
 val_ds = get_val_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES} ).loc[val_index]['TRAINING_FILENAMES']),
one_hot=True,labeled=True, return_image_names=False, repeat=False, shuffle=False)
 print('Getting model...'); print(''); print('Training model...'); print('')
 model = tf.keras.models.load_model(f'.. /input/cassava-tensorflow-starter-training/EFFNET_{f}_34_CUTMIXUP_512_full.h5')
 oof_labels.append([target.numpy() for img, target in iter(val_ds.unbatch())])
 x_oof = val_ds.map(lambda image, image_name: image)
 oof_pred.append(np.argmax(model.predict(x_oof), axis=-1))
del model; z = gc.collect()
count_rev=along_co[['production_countries','revenue']]<prepare_output>
if VALIDATE:
y_true = np.concatenate(oof_labels)
 y_preds = np.concatenate(oof_pred)
 print(classification_report(np.argmax(y_true, axis=1)if AUG_TYPE is 'CUTMIXUP' else y_true, y_preds))
print(f"OOF accuracy score: {accuracy_score(np.argmax(y_true, axis=1)if AUG_TYPE is 'CUTMIXUP' else y_true, y_preds)}" )
dd=count_rev['production_countries'].unique()
mn=pd.DataFrame(dd,columns=['address'])
 mn.head()<load_pretrained>
JPEG_PATH = ".. /input/cassava-leaf-disease-classification/test_images"
JPEG_PATH_TR = ".. /input/cassava-leaf-disease-classification/train_images"
def load_image(jpeg_path, image_id):
img = cv2.imread(os.path.join(jpeg_path, image_id)) /255.0
img = cv2.resize(img,(512, 512)) [:, :, ::-1]
return img
def generator(filepath, paths, batch_size=32):
i=0
print(len(paths))
while i <= len(paths):
batch = []
for cpt in range(batch_size):
if i + cpt >= len(paths):
i += batch_size
break
batch.append(load_image(filepath, paths[i+cpt]))
i += batch_size
yield np.stack(batch )
import geopandas as ge 
<load_from_csv>
submission = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 tr = pd.read_csv('.. /input/cassava-leaf-disease-classification/train.csv' )
world = ge.read_file(ge.datasets.get_path('naturalearth_lowres'))<import_modules>
preds_all = []
preds_model = []
for fold in range(FOLDS):
print(f"
ds_test = generator(JPEG_PATH,submission.image_id.values)
 K.clear_session()
print('Loading and inferring...')
 model = tf.keras.models.load_model(f'.. /input/cassava-tensorflow-starter-training/EFFNET_{fold}_34_CUTMIXUP_512_full.h5')
 preds = model.predict(ds_test, verbose=True)
 preds_model.append(preds)
 preds_model = np.stack(preds_model ).mean(0)
 preds_all.append(preds_model)
 preds_all = np.stack(preds_all )
from mpl_toolkits.axes_grid1 import make_axes_locatable 
<count_values>
submission["label"] = preds_all.mean(0 ).argmax(1)
 submission.to_csv("submission.csv", index=False )