Kaggle competition: Plant Pathology 2020 - FGVC7
<SOS> metric: MCAUC Kaggle data source: plant-pathology-2020-fgvc7<save_to_csv>
import numpy as np
import pandas as pd
import os
submission.to_csv('submission.csv', index=False)
 print('Submission shape', submission.shape )<define_variables>
!pip install -q efficientnet
MODEL_VERSION = 'v14' 
<define_variables>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
TEST_RUN = False
pd.set_option('display.max_columns', 50)
 KAGGLE_DATA_FOLDER = '/kaggle/input/m5-forecasting-accuracy'
BACKWARD_LAG = 60
END_DAY = 1913
BEGIN_DATE = '2013-08-01'
BEGIN_DAY = str(( datetime.strptime(BEGIN_DATE, '%Y-%m-%d')- datetime.strptime('2011-01-29', '%Y-%m-%d')).days)
 TRAIN_SPLIT = '2016-03-27'
EVAL_SPLIT = '2016-04-24'
print(datetime.strptime(EVAL_SPLIT, '%Y-%m-%d'))
TASK_TYPE='CPU'<define_variables>
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
CALENDAR_DTYPES = {
'date': 'str',
'wm_yr_wk': 'int16',
'weekday': 'category',
'wday': 'int16',
'month': 'int16',
'year': 'int16',
'd': 'object',
'event_name_1': 'category',
'event_type_1': 'category',
'event_name_2': 'category',
'event_type_2': 'category',
'snap_CA': 'int16',
'snap_TX': 'int16',
'snap_WI': 'int16'
}
PARSE_DATES = ['date']
SALES_PRICES_DTYPES = {
'store_id': 'category',
'item_id': 'category',
'wm_yr_wk': 'int16',
'sell_price': 'float32'
}<load_pretrained>
def format_path(fn):
return GCS_DS_PATH + '/images/' + fn + '.jpg'
train = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/train.csv')
 test = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/test.csv')
 sub = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/sample_submission.csv')
 train_paths = train['image_id'].apply(format_path ).values
test_paths = test['image_id'].apply(format_path ).values
train_labels = train.loc[:, 'healthy':].values
list_train_ds = tf.data.Dataset.from_tensor_slices(( train_paths, train_labels))
list_test_ds = tf.data.Dataset.from_tensor_slices(test_paths )
def load_data(train=True):

 print("Loading train and validation data")
 numcols = [f"d_{day}" for day in range(int(BEGIN_DAY),END_DAY+1)]
catcols = ['id', 'item_id', 'dept_id','store_id', 'cat_id', 'state_id']
dtype = {numcol:"float32" for numcol in numcols}
dtype.update({col: "category" for col in catcols if col != "id"})
 sales_train_validation = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sales_train_validation.csv'),
usecols=catcols+numcols, dtype=dtype)
 for col in catcols:
if col != "id":
sales_train_validation[col] = sales_train_validation[col].cat.codes.astype("int16")
 sales_train_validation[col] -= sales_train_validation[col].min()
if not train:
for day in range(END_DAY+1, END_DAY+28+1):
sales_train_validation[f"d_{day}"] = 0
value_vars = [column for column in sales_train_validation.columns
if(column.startswith('d_')and int(column.replace('d_', '')) >= END_DAY - BACKWARD_LAG)]
else:
value_vars = [col for col in sales_train_validation.columns
if(col.startswith('d_')and(int(col.replace('d_', '')) >= int(BEGIN_DAY)))]
print("Shape:", sales_train_validation.shape)
 print("Memory usage(Mb)before melting:", sales_train_validation.memory_usage().sum() / 1024**2)
 sales_train_validation = pd.melt(
 sales_train_validation,
id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
value_vars=value_vars,
var_name = 'day',
value_name = 'sales')
 print("Completed melting, new shape:", sales_train_validation.shape)
 print("Colums after melting:", sales_train_validation.columns)
 print("Memory usage(Mb)after melting:", sales_train_validation.memory_usage().sum() / 1024**2)
 columns = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']
print("Loading calendar")
 calendar = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'calendar.csv'), dtype=CALENDAR_DTYPES, parse_dates=['date'])
 print("Calendar columns: ", calendar.columns)
 print("Memory usage(Mb)calendar: ", calendar.memory_usage().sum() / 1024**2)
 calendar.rename(columns={'d':'day'}, inplace=True)
 for col, col_dtype in CALENDAR_DTYPES.items() :
if col_dtype == "category":
calendar[col] = calendar[col].cat.codes.astype("int16")
 calendar[col] -= calendar[col].min()
sales_train_validation = sales_train_validation.merge(calendar, on="day", copy=False)
 del calendar; gc.collect()
print("Merged calendar(in place)")
 print("Memory usage(Mb)after merging calendar:", sales_train_validation.memory_usage().sum() / 1024**2)
 print("Colums after merge:", sales_train_validation.columns)
 sell_prices = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sell_prices.csv'), dtype=SALES_PRICES_DTYPES)
 for col, col_dtype in SALES_PRICES_DTYPES.items() :
if col_dtype == "category":
sell_prices[col] = sell_prices[col].cat.codes.astype("int16")
 sell_prices[col] -= sell_prices[col].min()
print("Memory usage(Mb)sell prices:", sell_prices.memory_usage().sum() / 1024**2)
 columns = ['item_id', 'store_id', 'sell_price']
for feature in columns:
if feature == 'sell_price':
sell_prices[feature].fillna(0, inplace=True)
 sales_train_validation = sales_train_validation.merge(sell_prices, on=["store_id","item_id","wm_yr_wk"], copy=False)
 del sell_prices; gc.collect()
print("Merged sales prices(in place)")
 print("Memory usage(Mb)after merging sales:", sales_train_validation.memory_usage().sum() / 1024**2)
 print("Colums after merge:", sales_train_validation.columns)
 return sales_train_validation<categorify>
AUTOTUNE = tf.data.experimental.AUTOTUNE
EPOCHS = 40
BATCH_SIZE_1 = 16 * strategy.num_replicas_in_sync
BATCH_SIZE_2 = 64
image_size = 800
def make_lag_features(strain):

 print('in dataframe:', strain.shape)
 print("headers:", strain.columns)
 lags = [7, 28]
lag_cols = ['lag_{}'.format(lag)for lag in lags ]
for lag, lag_col in zip(lags, lag_cols):
strain[lag_col] = strain[['id', 'sales']].groupby('id')['sales'].shift(lag)
 print('lag sales done')
 windows= [7, 28]
for window in windows:
for lag, lag_col in zip(lags, lag_cols):
window_col = f'rmean_{lag_col}_{window}'
strain[window_col] = strain[['id', lag_col]].groupby('id')[lag_col].transform(
 lambda x: x.rolling(window ).mean() 
)
 print(f'Rolling mean sales for done for window {window}')
 def make_date_features(dt):
date_features = {
'week': 'weekofyear',
'quarter': 'quarter',
'mday': 'day',
"wday": "weekday"
}
for date_feat_name, date_feat_func in date_features.items() :
if not date_feat_name in dt.columns:
dt[date_feat_name] = getattr(dt['date'].dt, date_feat_func ).astype('int16')
 print('date features done')
 dt['day'] = dt['day'].apply(lambda x: int(x.replace('d_', '')))
 print('out dataframe:', dt.shape )<load_pretrained>
def decode_image(filename, label=None, image_size=(image_size, image_size)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if label is None:
return image
else:
return image, label
def prepare_for_training(ds, batch_size, cache=True, shuffle_buffer_size=512):
if cache:
if isinstance(cache, str):
ds = ds.cache(cache)
 else:
ds = ds.cache()
ds = ds.map(data_augment, num_parallel_calls=AUTOTUNE)
 ds = ds.shuffle(buffer_size=shuffle_buffer_size)
 ds = ds.repeat()
ds = ds.batch(batch_size)
 ds = ds.prefetch(buffer_size=AUTOTUNE)
 return ds
%%time
sales_train_validation = load_data()<data_type_conversions>
train_ds1 = list_train_ds.map(decode_image, num_parallel_calls=AUTOTUNE)
 train_ds1 = prepare_for_training(train_ds1, BATCH_SIZE_1)
 train_ds2 = list_train_ds.map(decode_image, num_parallel_calls=AUTOTUNE)
 train_ds2 = prepare_for_training(train_ds2, BATCH_SIZE_2)
 test_ds = list_test_ds.map(decode_image, num_parallel_calls=AUTOTUNE)
 test_ds = test_ds.batch(BATCH_SIZE_1 )
sales_train_validation["sale"] =(( sales_train_validation['sell_price'] * 100 % 10)< 6 ).astype('int8' )<feature_engineering>
with strategy.scope() :
model1 = tf.keras.Sequential([
InceptionResNetV2(
 input_shape=(image_size, image_size, 3),
weights='imagenet',
include_top=False 
),
L.GlobalAveragePooling2D() ,
L.Dense(train_labels.shape[1], activation='softmax')
 ])
 model1.compile(
 optimizer = 'adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 model1.summary()
%%time
make_lag_features(sales_train_validation)
 make_date_features(sales_train_validation )<define_variables>
with strategy.scope() :
model2 = tf.keras.Sequential([
efn.EfficientNetB7(
 input_shape=(image_size, image_size, 3),
weights='imagenet',
include_top=False 
),
L.GlobalAveragePooling2D() ,
L.Dense(train_labels.shape[1], activation='softmax'),
])
 model2.compile(
 optimizer = 'adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 model2.summary()
total = len(sales_train_validation['event_type_2'])
 zero = len(sales_train_validation.loc[sales_train_validation['event_type_2'] == 0])
 one = len(sales_train_validation.loc[sales_train_validation['event_type_2'] == 1])
 two = len(sales_train_validation.loc[sales_train_validation['event_type_2'] == 2])
 print('0: '+ str(zero))
print('1: ' + str(one))
print('2: ' + str(two))
print('-'*7 + '
Total: ' + str(total))
del total; del zero; del one; del two<count_missing_values>
LR_START = 0.0001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.0001
LR_RAMPUP_EPOCHS = 4
LR_SUSTAIN_EPOCHS = 6
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr = np.random.random_sample() * LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True )
sales_train_validation.isnull().sum()<categorify>
STEPS_PER_EPOCH = train_labels.shape[0] // BATCH_SIZE_1
history1 = model1.fit(
 train_ds1,
epochs=EPOCHS,
callbacks=[lr_callback],
steps_per_epoch=STEPS_PER_EPOCH,
)
before = len(sales_train_validation)
 sales_train_validation.dropna(inplace = True)
 after = len(sales_train_validation)
 print(f"Reduced {(before-after)/before}%")
<split>
STEPS_PER_EPOCH = train_labels.shape[0] // BATCH_SIZE_2
history2 = model2.fit(
 train_ds2,
epochs=EPOCHS,
callbacks=[lr_callback],
steps_per_epoch=STEPS_PER_EPOCH,
)
<count_values><EOS> 
probs1 = model1.predict(test_ds, verbose=1)
 probs2 = model2.predict(test_ds, verbose=1)
 probs_avg =(probs1 + probs2)/ 2
sub.loc[:, 'healthy':] = probs_avg
sub.to_csv('submission.csv', index=False)
 sub.head()