Kaggle competition: Plant Pathology 2020 - FGVC7
%%time
X_train, X_val, y_train, y_val, X_test = Create_train_test_dataset(df=master, random_state=10 )<set_options>
!pip install -q efficientnet
del master
gc.collect()<create_dataframe>
import math, re, os
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from kaggle_datasets import KaggleDatasets
import tensorflow as tf
import tensorflow.keras.layers as L
import efficientnet.tfkeras as efn
from sklearn import metrics
from sklearn.model_selection import train_test_split
def lgbm_val(X_train, X_val, y_train, y_val):
train = lgb.Dataset(X_train, label = y_train)
 valid = lgb.Dataset(X_val, label = y_val)
 params = {
'metric': 'rmse',
'objective': 'poisson',
'seed': 200,
'force_row_wise' : True,
'learning_rate' : 0.07,
'lambda': 0.1,
'num_leaves': 60,
'sub_row' : 0.7,
'bagging_freq' : 1,
'colsample_bytree': 0.85
}
lgbm = lgb.train(params,
train,
num_boost_round = 800,
valid_sets = [valid],
early_stopping_rounds = 50,
verbose_eval = 5)
 lgb.plot_importance(lgbm, importance_type="gain", precision=0, figsize=(6, 13)) ;
pred_train = lgbm.predict(X_train)
 pred_val = lgbm.predict(X_val)
 return pred_train, pred_val, lgbm<compute_test_metric>
AUTO = tf.data.experimental.AUTOTUNE
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 40
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
%%time
pred_train, pred_val, lgbm = lgbm_val(X_train, X_val, y_train, y_val )<compute_test_metric>
def format_path(st):
return GCS_DS_PATH + '/images/' + st + '.jpg'
print("MSE train %.3f" % mean_squared_error(y_true=y_train, y_pred=pred_train))
print("MSE validation %.3f" % mean_squared_error(y_true=y_val, y_pred=pred_val))
print("R2 score train %.3f" % r2_score(y_true=y_train, y_pred=pred_train))
print("R2 score validation %.3f" % r2_score(y_true=y_val, y_pred=pred_val))<drop_column>
train = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/train.csv')
 test = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/test.csv')
 sub = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/sample_submission.csv')
 train_paths = train.image_id.apply(format_path ).values
test_paths = test.image_id.apply(format_path ).values
train_labels = train.loc[:, 'healthy':].values
train_paths, valid_paths, train_labels, valid_labels = train_test_split(
 train_paths, train_labels, test_size=0.15, random_state=2020 )
del X_train, X_val, y_train, y_val, pred_train, pred_val
gc.collect()<prepare_output>
np.set_printoptions(threshold=15, linewidth=80)
 def batch_to_numpy_images_and_labels(data):
images, labels = data
numpy_images = images.numpy()
numpy_labels = labels.numpy()
if numpy_labels.dtype == object:
numpy_labels = [None for _ in enumerate(numpy_images)]
return numpy_images, numpy_labels
def batchtest_to_numpy_images_and_labels(data):
images= data
numpy_images = images.numpy()
return numpy_images,None
def title_from_label_and_target(label, correct_label):
if correct_label is None:
return CLASSES[label], True
correct =(label == correct_label)
 return "{} [{}{}{}]".format(CLASSES[label], 'OK' if correct else 'NO', u"\u2192" if not correct else '',
CLASSES[correct_label] if not correct else ''), correct
def display_one_flower(image, title, subplot, red=False, titlesize=16):
plt.subplot(*subplot)
 plt.axis('off')
 plt.imshow(image)
 if len(title)> 0:
plt.title(title, fontsize=int(titlesize)if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))
return(subplot[0], subplot[1], subplot[2]+1)
 def display_batch_of_images(databatch, predictions=None):

 images, labels = batch_to_numpy_images_and_labels(databatch)
 if labels is None:
labels = [None for _ in enumerate(images)]
rows = int(math.sqrt(len(images)))
 cols = len(images)//rows
FIGSIZE = 13.0
SPACING = 0.1
subplot=(rows,cols,1)
 if rows < cols:
plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))
else:
plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))
tempo=""
for i,(image, label)in enumerate(zip(images[:rows*cols], labels[:rows*cols])) :
if(label[0]==1):
tempo="healthy"
elif(label[1]==1):
tempo="multiple_diseases"
elif(label[2]==1):
tempo="rust"
else:
tempo="scab"
title = '' if label is None else tempo
correct = True
if predictions is not None:
title, correct = title_from_label_and_target(predictions[i], label)
 dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3
subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)
 plt.tight_layout()
if label is None and predictions is None:
plt.subplots_adjust(wspace=0, hspace=0)
 else:
plt.subplots_adjust(wspace=SPACING, hspace=SPACING)
 plt.show()
def display_confusion_matrix(cmat, score, precision, recall):
plt.figure(figsize=(15,15))
ax = plt.gca()
ax.matshow(cmat, cmap='Reds')
 ax.set_xticks(range(len(CLASSES)))
 ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})
 plt.setp(ax.get_xticklabels() , rotation=45, ha="left", rotation_mode="anchor")
 ax.set_yticks(range(len(CLASSES)))
 ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})
 plt.setp(ax.get_yticklabels() , rotation=45, ha="right", rotation_mode="anchor")
 titlestring = ""
if score is not None:
titlestring += 'f1 = {:.3f} '.format(score)
 if precision is not None:
titlestring += '
precision = {:.3f} '.format(precision)
 if recall is not None:
titlestring += '
recall = {:.3f} '.format(recall)
 if len(titlestring)> 0:
ax.text(101, 1, titlestring, fontdict={'fontsize': 18, 'horizontalalignment':'right', 'verticalalignment':'top', 'color':'
plt.show()
def display_training_curves(training, validation, title, subplot):
if subplot%10==1:
plt.subplots(figsize=(10,10), facecolor='
plt.tight_layout()
ax = plt.subplot(subplot)
 ax.set_facecolor('
ax.plot(training)
 ax.plot(validation)
 ax.set_title('model '+ title)
 ax.set_ylabel(title)
 ax.set_xlabel('epoch')
 ax.legend(['train', 'valid.'])
 def displaytest_batch_of_images(databatch, predictions=None):

 images,labels = batchtest_to_numpy_images_and_labels(databatch)
 if labels is None:
labels = [None for _ in enumerate(images)]
rows = int(math.sqrt(len(images)))
 cols = len(images)//rows
FIGSIZE = 13.0
SPACING = 0.1
subplot=(rows,cols,1)
 if rows < cols:
plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))
else:
plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))
tempo=""
for i,(image, label)in enumerate(zip(images[:rows*cols], labels[:rows*cols])) :
title = ''
correct = True
if predictions is not None:
title, correct = title_from_label_and_target(predictions[i], label)
 dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3
subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)
 plt.tight_layout()
if label is None and predictions is None:
plt.subplots_adjust(wspace=0, hspace=0)
 else:
plt.subplots_adjust(wspace=SPACING, hspace=SPACING)
 plt.show()
%%time
pred_test = lgbm.predict(X_test)
 pred = np.array(pred_test, dtype='float16')
 pred = np.reshape(pred,(30490, 28), order='F')
 pred_df = pd.DataFrame(pred)
 pred_df.head()<save_to_csv>
def decode_image(filename, label=None, image_size=(512, 512)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if label is None:
return image
else:
return image, label
col = sample.loc[:,'F1':].columns
sample[col] = pd.concat([pred_df, pred_df] ).values
sample.to_csv("submission.csv", index=False )<set_options>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.cache() 
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( valid_paths, valid_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )
pd.set_option('max_columns', 50)
 plt.style.use('bmh')
 color_cycle = cycle(plt.rcParams['axes.prop_cycle'].by_key() ['color'] )<load_from_csv>
validating_dataset = valid_dataset.unbatch().batch(20)
 valid_batch = iter(validating_dataset )
osj = os.path.join
INPUT_DIR = '.. /input/m5-forecasting-accuracy/'
train = pd.read_csv(osj(INPUT_DIR, 'sales_train_validation.csv'))
price = pd.read_csv(osj(INPUT_DIR, 'sell_prices.csv'))
calender = pd.read_csv(osj(INPUT_DIR, 'calendar.csv'))
sample_submit = pd.read_csv(osj(INPUT_DIR, 'sample_submission.csv'))<data_type_conversions>
display_batch_of_images(next(valid_batch))
calender['date'] = pd.to_datetime(calender['date'] )<define_variables>
training_dataset = train_dataset.unbatch().batch(20)
 train_batch = iter(training_dataset )
d_cols = [c for c in train.columns if 'd_' in c]<merge>
display_batch_of_images(next(train_batch))
past_sales = train.set_index('id')[d_cols].T.merge(calender.set_index('d')['date'],
left_index=True,
right_index=True,
validate='1:1' ).set_index('date' )<create_dataframe>
testing_dataset = test_dataset.unbatch().batch(20)
 test_batch = iter(testing_dataset )
def make_total_sales_lowess(p):
total_sales = pd.DataFrame(p.sum(axis=1), columns=['total sales'])
 total_sales_noXmas = total_sales.drop(index=[dt.datetime(2011,12,25),
dt.datetime(2012,12,25),
dt.datetime(2013,12,25),
dt.datetime(2014,12,25),
dt.datetime(2015,12,25)])
 df = total_sales_noXmas
lowess = sm.nonparametric.lowess(df['total sales'], df.index, frac=.3)
 lowess_x = list(zip(*lowess)) [0]
lowess_y = list(zip(*lowess)) [1]
f = interp1d(lowess_x, lowess_y, bounds_error=False)
 new_lowess_x = df.index
new_lowess_y = f(new_lowess_x)
 total_sales_lowess = total_sales_noXmas
total_sales_lowess['lowess'] = new_lowess_y
total_sales_lowess['total sales-lowess'] = total_sales_lowess['total sales'] - total_sales_lowess['lowess']
return total_sales_lowess<groupby>
displaytest_batch_of_images(next(test_batch))
total_sales_lowess = make_total_sales_lowess(past_sales )<load_from_csv>
def build_lrfn(lr_start=0.00001, lr_max=0.0008,
lr_min=0.00001, lr_rampup_epochs=20,
lr_sustain_epochs=0, lr_exp_decay=.8):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn
submission = pd.read_csv('.. /input/m5-three-shades-of-dark-darker-magic/submission_v1.csv' )<feature_engineering>
with strategy.scope() :
model = tf.keras.Sequential([
efn.EfficientNetB7(
 input_shape=(512, 512, 3),
weights='imagenet',
include_top=False 
),
L.GlobalAveragePooling2D() ,
L.Dense(train_labels.shape[1], activation='softmax')
 ])
 model.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 model.summary()
teams_now = 3215
teams_before = 3186
submission.loc[submission['id'].str.contains('CA_1'), 'F1'] *= 1.009
submission.loc[submission['id'].str.contains('CA_2'), 'F1'] *= 1.02
submission.loc[submission['id'].str.contains('CA_3'), 'F1'] *= 1.009
submission.loc[submission['id'].str.contains('CA_4'), 'F1'] *= 1.011
submission.loc[submission['id'].str.contains('TX_1'), 'F1'] *= 1.01
submission.loc[submission['id'].str.contains('TX_2'), 'F1'] *= 1.011
submission.loc[submission['id'].str.contains('TX_3'), 'F1'] *= 1.011
submission.loc[submission['id'].str.contains('WI_1'), 'F1'] *= 1.015
submission.loc[submission['id'].str.contains('WI_2'), 'F1'] *= 1.02
submission.loc[submission['id'].str.contains('WI_3'), 'F1'] *= 1.015
submission['F1'].sum()<feature_engineering>
lrfn = build_lrfn()
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)
 STEPS_PER_EPOCH = train_labels.shape[0] // BATCH_SIZE
history = model.fit(
 train_dataset,
epochs=EPOCHS,
callbacks=[lr_schedule],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset
)
magic = submission['F2'].sum()
japan_emergency = 119
submission.loc[submission['id'].str.contains('CA_1'), 'F2'] *= 0.994
submission.loc[submission['id'].str.contains('CA_2'), 'F2'] *= 0.998
submission.loc[submission['id'].str.contains('CA_3'), 'F2'] *= 0.994
submission.loc[submission['id'].str.contains('CA_4'), 'F2'] *= 0.996
submission.loc[submission['id'].str.contains('TX_1'), 'F2'] *= 0.995
submission.loc[submission['id'].str.contains('TX_2'), 'F2'] *= 0.996
submission.loc[submission['id'].str.contains('TX_3'), 'F2'] *= 0.996
submission.loc[submission['id'].str.contains('WI_1'), 'F2'] *= 0.998
submission.loc[submission['id'].str.contains('WI_2'), 'F2'] *= 1
submission.loc[submission['id'].str.contains('WI_3'), 'F2'] *= 0.998
submission['F2'].sum()<feature_engineering>
probs = model.predict(test_dataset, verbose=1)
 sub.loc[:, 'healthy':] = probs
sub.to_csv('submission.csv', index=False)
 sub.head()
submission.loc[submission['id'].str.contains('CA_1'), 'F3'] *= 0.994
submission.loc[submission['id'].str.contains('CA_2'), 'F3'] *= 0.998
submission.loc[submission['id'].str.contains('CA_3'), 'F3'] *= 0.994
submission.loc[submission['id'].str.contains('CA_4'), 'F3'] *= 0.996
submission.loc[submission['id'].str.contains('TX_1'), 'F3'] *= 0.995
submission.loc[submission['id'].str.contains('TX_2'), 'F3'] *= 0.996
submission.loc[submission['id'].str.contains('TX_3'), 'F3'] *= 0.996
submission.loc[submission['id'].str.contains('WI_1'), 'F3'] *= 1
submission.loc[submission['id'].str.contains('WI_2'), 'F3'] *= 1.005
submission.loc[submission['id'].str.contains('WI_3'), 'F3'] *= 1
submission['F3'].sum()<feature_engineering>
with strategy.scope() :
model1 = tf.keras.Sequential([
efn.EfficientNetB7(
 input_shape=(512, 512, 3),
weights='imagenet',
include_top=False 
),
L.GlobalAveragePooling2D() ,
L.Dense(train_labels.shape[1], activation='softmax')
 ])
 model1.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 model1.summary()
sex_constant = 1.0073
submission.loc[submission['id'].str.contains('CA_1'), 'F4'] *= 0.98
submission.loc[submission['id'].str.contains('CA_2'), 'F4'] *= 1.02
submission.loc[submission['id'].str.contains('CA_3'), 'F4'] *= 0.98
submission.loc[submission['id'].str.contains('CA_4'), 'F4'] *= 1
submission.loc[submission['id'].str.contains('TX_1'), 'F4'] *= 0.99
submission.loc[submission['id'].str.contains('TX_2'), 'F4'] *= 1
submission.loc[submission['id'].str.contains('TX_3'), 'F4'] *= 1
submission.loc[submission['id'].str.contains('WI_1'), 'F4'] *= 1.00
submission.loc[submission['id'].str.contains('WI_2'), 'F4'] *= 1.02
submission.loc[submission['id'].str.contains('WI_3'), 'F4'] *= 1.00
submission['F4'].sum()<feature_engineering>
lrfn = build_lrfn()
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)
 STEPS_PER_EPOCH = train_labels.shape[0] // BATCH_SIZE
history = model1.fit(
 train_dataset,
epochs=50,
callbacks=[lr_schedule],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset
)
color_of_the_night = 995874
color_of_the_night /= 1000000
submission.loc[submission['id'].str.contains('CA_1'), 'F5'] *= 0.995
submission.loc[submission['id'].str.contains('CA_2'), 'F5'] *= 1
submission.loc[submission['id'].str.contains('CA_3'), 'F5'] *= 0.995
submission.loc[submission['id'].str.contains('CA_4'), 'F5'] *= 0.998
submission.loc[submission['id'].str.contains('TX_1'), 'F5'] *= 0.996
submission.loc[submission['id'].str.contains('TX_2'), 'F5'] *= 0.998
submission.loc[submission['id'].str.contains('TX_3'), 'F5'] *= 0.998
submission.loc[submission['id'].str.contains('WI_1'), 'F5'] *= 1
submission.loc[submission['id'].str.contains('WI_2'), 'F5'] *= 1.01
submission.loc[submission['id'].str.contains('WI_3'), 'F5'] *= 1
submission['F5'].sum()<feature_engineering>
probs1 = model1.predict(test_dataset, verbose=1)
 sub.loc[:, 'healthy':] = probs1
sub.to_csv('submission1.csv', index=False)
 sub.head()
injury = 1.000376
submission.loc[submission['id'].str.contains('CA_1'), 'F6'] *= 1.0002
submission.loc[submission['id'].str.contains('CA_2'), 'F6'] *= 1.001
submission.loc[submission['id'].str.contains('CA_3'), 'F6'] *= 1.0002
submission.loc[submission['id'].str.contains('CA_4'), 'F6'] *= 1.001
submission.loc[submission['id'].str.contains('TX_1'), 'F6'] *= 1.000
submission.loc[submission['id'].str.contains('TX_2'), 'F6'] *= 1.001
submission.loc[submission['id'].str.contains('TX_3'), 'F6'] *= 1.001
submission.loc[submission['id'].str.contains('WI_1'), 'F6'] *= 1.001
submission.loc[submission['id'].str.contains('WI_2'), 'F6'] *= 1.01
submission.loc[submission['id'].str.contains('WI_3'), 'F6'] *= 1.001
submission['F6'].sum()<feature_engineering>
probs2=(probs+probs1)/2
sub.loc[:,'healthy':]=probs2
<feature_engineering><EOS> 
sub.to_csv("submission2.csv",index=False )