Kaggle competition: Plant Pathology 2020 - FGVC7
<SOS> metric: MCAUC Kaggle data source: plant-pathology-2020-fgvc7<find_best_model_class>
import math, re, os
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from kaggle_datasets import KaggleDatasets
import tensorflow as tf
import tensorflow.keras.layers as L
from sklearn import metrics
from sklearn.model_selection import train_test_split
from keras.callbacks import ModelCheckpoint
def learn_from_batch_ae(model, data, device):
seq = data["sequence"].clone()
seq[:, :, :14] = F.dropout2d(seq[:, :, :14], p=0.3)
 target = data["sequence"][:, :, :14]
out = model(seq.to(device), data["bpp"].to(device))
loss = F.binary_cross_entropy(out, target.to(device))
return loss
def train_ae(model, train_data, optimizer, lr_scheduler, epochs=10, device="cpu",
start_epoch: int = 0, start_it: int = 0, log_path: str = "./logs"):
print(f"device: {device}")
 losses = []
it = start_it
model_save_path = Path(MODEL_SAVE_PATH)
 start_epoch = start_epoch
end_epoch = start_epoch + epochs
min_loss = 10.0
min_loss_epoch = 0
if not model_save_path.exists() :
model_save_path.mkdir(parents=True)
 for epoch in progress_bar(range(start_epoch, end_epoch)) :
print(f"epoch: {epoch}")
 model.train()
for i, data in enumerate(train_data):
optimizer.zero_grad()
loss = learn_from_batch_ae(model, data, device)
 loss.backward()
nn.utils.clip_grad_norm_(model.parameters() , 0.5)
 optimizer.step()
if lr_scheduler:
lr_scheduler.step()
loss_v = loss.item()
losses.append(loss_v)
 it += 1
loss_m = np.mean(losses)
 if loss_m < min_loss:
min_loss_epoch = epoch
min_loss = loss_m
print(f'epoch: {epoch} loss: {loss_m}')
 losses = []
torch.save(optimizer.state_dict() , str(model_save_path / "optimizer.pt"))
torch.save(model.state_dict() , str(model_save_path / f"model-{epoch}.pt"))
return dict(end_epoch=end_epoch, it=it, min_loss_epoch=min_loss_epoch)
<choose_model_class>
AUTO = tf.data.experimental.AUTOTUNE
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
set_seed(123)
 shutil.rmtree("./model", True)
 shutil.rmtree("./logs", True)
 save_path = Path("./model_prediction")
 if not save_path.exists() :
save_path.mkdir(parents=True)
 lr_scheduler = None
device = "cuda" if torch.cuda.is_available() else "cpu"
model = AEModel()
model = model.to(device)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3)
 res = dict(end_epoch=0, it=0, min_loss_epoch=0)
 epochs = [5, 5, 5, 5]
for e in epochs:
res = train_ae(model, loader0, optimizer, lr_scheduler, e, device=device,
start_epoch=res["end_epoch"], start_it=res["it"])
 res = train_ae(model, loader1, optimizer, lr_scheduler, e, device=device,
start_epoch=res["end_epoch"], start_it=res["it"])
 res = train_ae(model, loader2, optimizer, lr_scheduler, e, device=device,
start_epoch=res["end_epoch"], start_it=res["it"])
 epoch = res["min_loss_epoch"]
shutil.copyfile(str(Path(MODEL_SAVE_PATH)/ f"model-{epoch}.pt"), "ae-model.pt" )<compute_test_metric>
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 40
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
IM_Z = 768
def MCRMSE(y_true, y_pred):
colwise_mse = torch.mean(torch.square(y_true - y_pred), dim=1)
 return torch.mean(torch.sqrt(colwise_mse), dim=1)
 def sn_mcrmse_loss(predict, target, signal_to_noise):
loss = MCRMSE(target, predict)
 weight = 0.5 * torch.log(signal_to_noise + 1.01)
 loss =(loss * weight ).mean()
return loss
def learn_from_batch(model, data, optimizer, lr_scheduler, device):
optimizer.zero_grad()
out = model(data["sequence"].to(device), data["bpp"].to(device))
signal_to_noise = data["signal_to_noise"] * data["score"]
loss = sn_mcrmse_loss(out, data["label"].to(device), signal_to_noise.to(device))
loss.backward()
nn.utils.clip_grad_norm_(model.parameters() , 0.5)
 optimizer.step()
if lr_scheduler:
lr_scheduler.step()
return out, loss
def evaluate(model, valid_data, device):
model.eval()
loss_list = []
mcrmse = []
for i, data in enumerate(valid_data):
with torch.no_grad() :
y = model(data["sequence"].to(device), data["bpp"].to(device))
mcrmse_ = MCRMSE(data["label"].to(device), y)[data["signal_to_noise"] > 1]
mcrmse.append(mcrmse_.mean().item())
 loss = sn_mcrmse_loss(y, data["label"].to(device), data["signal_to_noise"].to(device))
loss_list.append(loss.item())
 model.train()
return dict(loss=np.mean(loss_list), mcmse=np.mean(mcrmse))
def train(model, train_data, valid_data, optimizer, lr_scheduler, epochs=10, device="cpu",
start_epoch: int = 0, log_path: str = "./logs"):
print(f"device: {device}")
 losses = []
writer = SummaryWriter(log_path)
 it = 0
model_save_path = Path(MODEL_SAVE_PATH)
 start_epoch = start_epoch
end_epoch = start_epoch + epochs
if not model_save_path.exists() :
model_save_path.mkdir(parents=True)
 min_eval_loss = 10.0
min_eval_epoch = None
for epoch in progress_bar(range(start_epoch, end_epoch)) :
print(f"epoch: {epoch}")
 model.train()
for i, data in enumerate(train_data):
_, loss = learn_from_batch(model, data, optimizer, lr_scheduler, device)
 loss_v = loss.item()
writer.add_scalar('loss', loss_v, it)
 losses.append(loss_v)
 it += 1
print(f'epoch: {epoch} loss: {np.mean(losses)}')
 losses = []
eval_result = evaluate(model, valid_data, device)
 eval_loss = eval_result["loss"]
if eval_loss <= min_eval_loss:
min_eval_epoch = epoch
min_eval_loss = eval_loss
print(f"eval loss: {eval_loss} {eval_result['mcmse']}")
 writer.add_scalar(f"evaluate/loss", eval_loss, epoch)
 writer.add_scalar(f"evaluate/mcmse", eval_result["mcmse"], epoch)
 model.train()
torch.save(optimizer.state_dict() , str(model_save_path / "optimizer.pt"))
torch.save(model.state_dict() , str(model_save_path / f"model-{epoch}.pt"))
print(f'min eval loss: {min_eval_loss} epoch {min_eval_epoch}')
 return min_eval_epoch 
<load_pretrained>
def format_path(st):
return GCS_DS_PATH + '/images/' + st + '.jpg'
device = torch.device('cuda')
 BATCH_SIZE = 64
base_train_data = pd.read_json(str(Path(BASE_PATH)/ 'train.json'), lines=True)
 samples = base_train_data
save_path = Path("./model_prediction")
 if not save_path.exists() :
save_path.mkdir(parents=True)
 shutil.rmtree("./model", True)
 shutil.rmtree("./logs", True)
 split = ShuffleSplit(n_splits=5, test_size=.1)
 ids = samples.reset_index() ["id"]
set_seed(124)
 for fold,(train_index, test_index)in enumerate(split.split(samples)) :
print(f"fold: {fold}")
 train_df = samples.loc[train_index].reset_index()
val_df = samples.loc[test_index].reset_index()
train_loader = create_loader(train_df, BATCH_SIZE)
 valid_loader = create_loader(val_df, BATCH_SIZE)
 print(train_df.shape, val_df.shape)
 ae_model = AEModel()
state_dict = torch.load("./ae-model.pt")
 ae_model.load_state_dict(state_dict)
 del state_dict
model = FromAeModel(ae_model.seq)
 model = model.to(device)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3)
 lr_scheduler = None
epoch = train(model, train_loader, valid_loader, optimizer, lr_scheduler, 200, device=device,
log_path=f"logs/{fold}")
 shutil.copyfile(str(Path(MODEL_SAVE_PATH)/ f"./model-{epoch}.pt"), f"model_prediction/model-{fold}.pt")
 del model<predict_on_test>
train = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/train.csv')
 test = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/test.csv')
 sub = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/sample_submission.csv')
 train.head()
def predict_batch(model, data, device):
with torch.no_grad() :
pred = model(data["sequence"].to(device), data["bpp"].to(device))
pred = pred.detach().cpu().numpy()
return_values = []
ids = data["ids"]
for idx, p in enumerate(pred):
id_ = ids[idx]
assert p.shape ==(model.pred_len, len(target_cols))
for seqpos, val in enumerate(p):
assert len(val)== len(target_cols)
 dic = {key: val for key, val in zip(target_cols, val)}
dic["id_seqpos"] = f"{id_}_{seqpos}"
return_values.append(dic)
 return return_values
def predict_data(model, loader, device, batch_size):
data_list = []
for i, data in enumerate(progress_bar(loader)) :
data_list += predict_batch(model, data, device)
 expected_length = model.pred_len * len(loader)* batch_size
assert len(data_list)== expected_length, f"len = {len(data_list)} expected = {expected_length}"
return data_list 
<load_pretrained>
train_paths = train.image_id.apply(format_path ).values
test_paths = test.image_id.apply(format_path ).values
train_labels = train.loc[:, 'healthy':].values
train_paths, valid_paths, train_labels, valid_labels = train_test_split(
 train_paths, train_labels, test_size=0.1, random_state=2020 )
device = torch.device('cuda')if torch.cuda.is_available() else "cpu"
BATCH_SIZE = 1
base_test_data = pd.read_json(str(Path(BASE_PATH)/ 'test.json'), lines=True)
 public_df = base_test_data.query("seq_length == 107" ).copy()
private_df = base_test_data.query("seq_length == 130" ).copy()
print(f"public_df: {public_df.shape}")
 print(f"private_df: {private_df.shape}")
 public_df = public_df.reset_index()
private_df = private_df.reset_index()
pub_loader = create_loader(public_df, BATCH_SIZE, is_test=True)
 pri_loader = create_loader(private_df, BATCH_SIZE, is_test=True)
 pred_df_list = []
c = 0
for fold in range(5):
model_load_path = f"./model_prediction/model-{fold}.pt"
ae_model0 = AEModel()
ae_model1 = AEModel()
model_pub = FromAeModel(pred_len=107, seq=ae_model0.seq)
 model_pub = model_pub.to(device)
 model_pri = FromAeModel(pred_len=130, seq=ae_model1.seq)
 model_pri = model_pri.to(device)
 state_dict = torch.load(model_load_path, map_location=device)
 model_pub.load_state_dict(state_dict)
 model_pri.load_state_dict(state_dict)
 del state_dict
data_list = []
data_list += predict_data(model_pub, pub_loader, device, BATCH_SIZE)
 data_list += predict_data(model_pri, pri_loader, device, BATCH_SIZE)
 pred_df = pd.DataFrame(data_list, columns=["id_seqpos"] + target_cols)
 print(pred_df.head())
 print(pred_df.tail())
 pred_df_list.append(pred_df)
 c += 1
data_dic = dict(id_seqpos=pred_df_list[0]["id_seqpos"])
 for col in target_cols:
vals = np.zeros(pred_df_list[0][col].shape[0])
 for df in pred_df_list:
vals += df[col].values
data_dic[col] = vals / float(c)
 pred_df_avg = pd.DataFrame(data_dic, columns=["id_seqpos"] + target_cols)
 print(pred_df_avg.head())
 pred_df_avg.to_csv("./submission.csv", index=False)
<load_from_csv>
def decode_image(filename, label=None, image_size=(IM_Z, IM_Z)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if label is None:
return image
else:
return image, label
train = pd.read_csv("/kaggle/input/m5-forecasting-accuracy/sales_train_validation.csv")
 calendar = pd.read_csv("/kaggle/input/m5-forecasting-accuracy/calendar.csv")
 price = pd.read_csv("/kaggle/input/m5-forecasting-accuracy/sell_prices.csv")
 sample = pd.read_csv("/kaggle/input/m5-forecasting-accuracy/sample_submission.csv" )<create_dataframe>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.cache() 
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( valid_paths, valid_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )
valid_df = pd.DataFrame(np.array(train.iloc[:,-28:])).astype("int16" )<data_type_conversions>
def build_lrfn(lr_start=0.00001, lr_max=0.000075,
lr_min=0.000001, lr_rampup_epochs=20,
lr_sustain_epochs=0, lr_exp_decay=.8):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn
ch_p = ModelCheckpoint(filepath="model_ir.h5", monitor='val_loss', save_weights_only=True,
verbose=1 )
def calendar_preprocessing(df=calendar):
int16_col = ["wm_yr_wk", "wday", "month", "year", "snap_CA", "snap_TX", "snap_WI"]
df[int16_col] = df[int16_col].astype("int16")
 df["date"] = pd.to_datetime(df["date"])
 df["mday"] = df["date"].dt.day.astype("int8")
 list_col = ["event_name_1", "event_type_1", "event_name_2", "event_type_2"]
df[list_col] = df[list_col].fillna("na")
 for i in list_col:
ce_oe = ce.OrdinalEncoder(cols=i, handle_unknown='impute')
 df = ce_oe.fit_transform(df)
 df[i] = df[i].astype("int16")
 col = ['wm_yr_wk', 'wday', 'month', 'year', 'event_name_1', 'event_type_1',
'event_name_2', 'event_type_2', 'snap_CA', 'snap_TX', 'snap_WI','mday']
for c in col:
df['shift28_{}'.format(c)] = df[c].shift(-28 ).fillna(1000 ).astype("int16")
 df.drop(col, axis=1, inplace=True)
 df.drop("weekday", axis=1, inplace=True)
 df.drop(["shift28_event_type_1", "shift28_event_name_2", "shift28_event_type_2"], axis=1, inplace=True)
 return df
def price_preprocessing(df=price):
int16_col = ["wm_yr_wk"]
df[int16_col] = df[int16_col].astype("int16")
 float16_col = ["sell_price"]
df[float16_col] = df[float16_col].astype("float16")
 col = ['wm_yr_wk', 'sell_price']
df['shift28_{}'.format(col[0])] = df[col[0]].shift(-28 ).fillna(1000 ).astype("int16")
 df['shift28_{}'.format(col[1])] = df[col[1]].shift(-28 ).fillna(1000 ).astype("float16")
 df.drop(col, axis=1, inplace=True)
 return df
def train_preprocessing(df=train):
int16_col = df.loc[:,"d_1":].columns
df[int16_col] = df[int16_col].astype("int16")
 drop_col = df.loc[:,"d_1":"d_817"]
df.drop(drop_col, axis=1, inplace=True)
 id_col = ["id", "item_id", "dept_id", "cat_id", "store_id", "state_id"]
df = df.melt(id_vars=id_col, var_name="d", value_name="volume")
 return df<categorify>
with strategy.scope() :
model = tf.keras.Sequential([
tf.keras.applications.InceptionResNetV2(
 input_shape=(IM_Z, IM_Z, 3),
weights='imagenet',
include_top=False 
),
L.GlobalAveragePooling2D() ,
L.Dense(train_labels.shape[1], activation='softmax')
 ])
 model.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
train = train_preprocessing(df=train)
 train.dtypes<merge>
lrfn = build_lrfn()
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)
 STEPS_PER_EPOCH = train_labels.shape[0] // BATCH_SIZE
master = pd.merge(train, calendar, left_on="d", right_on="d", how="left")
 master = pd.merge(master, price, left_on=["store_id", "item_id", "shift28_wm_yr_wk"],
right_on=["store_id", "item_id", "shift28_wm_yr_wk"], how='left' )<set_options>
history = model.fit(
 train_dataset,
epochs=EPOCHS,
callbacks=[lr_schedule, ch_p],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset
)
del train, calendar, price
gc.collect()<categorify>
probs = model.predict(test_dataset, verbose=1 )
def Create_train_test_dataset(df=master, random_state=10):
df.dropna(inplace=True)
 id_col = ["id", "item_id", "dept_id", "cat_id", "store_id", "state_id"]
for i in id_col:
ce_oe = ce.OrdinalEncoder(cols=i, handle_unknown='impute')
 df = ce_oe.fit_transform(df)
 df[i] = df[i].astype("int16")
 df.drop("d", axis=1, inplace=True)
 lags = [7, 14, 28]
for l in lags:
df["lag_{}".format(l)] = df.groupby("id")["volume"].transform(lambda x: x.shift(l)).fillna(1000 ).astype("int16")
 df["lag_d_{}".format(l)] = df.groupby("id")["volume"].transform(lambda x: x.shift(l)- x.shift(0)).fillna(1000 ).astype("int16")
 window = [7,28]
for w in window:
for l in lags:
df["rolling_{}_lag_{}".format(w,l)] = df.groupby("id")["volume"].transform(lambda x: x.rolling(w ).mean().shift(l)).fillna(1000 ).astype("float16")
 df["rolling_{}_lag_d_{}".format(w,l)] = df.groupby("id")["volume"].transform(lambda x: x.rolling(w ).mean().shift(l)- x.rolling(w ).mean().shift(0)).fillna(1000 ).astype("float16")
 list_vol_flag = []
for v in df["volume"]:
if v == 0:
flg = 0
list_vol_flag.append(flg)
 elif v == 1:
flg = 1
list_vol_flag.append(flg)
 elif v == 2:
flg = 2
list_vol_flag.append(flg)
 elif v > 2 and v <= 5:
flg = 3
list_vol_flag.append(flg)
 elif v > 5 and v <=20:
flg = 4
list_vol_flag.append(flg)
 else :
flg=5
list_vol_flag.append(flg)
 df["vol_range"] = list_vol_flag
df["vol_range"] = df["vol_range"].astype("int16")
 s_lag = [1,7]
for sl in s_lag:
df["lag_price_{}".format(sl)] = df.groupby("id")["shift28_sell_price"].transform(lambda x: x.shift(sl)- x.shift(0)).fillna(1000 ).astype("float16")
 df["target"] = df.groupby("id")["volume"].transform(lambda x:x.shift(-28)).fillna(0 ).astype("int16")
 Xmas_date = [pd.datetime(2013,12,25), pd.datetime(2014,12,25), pd.datetime(2015,12,25)]
X = df[(df["date"] <= df["date"].max() - datetime.timedelta(days=28))
&(df["date"] > df["date"].min() + datetime.timedelta(days=56)) ].set_index("date" ).drop(Xmas_date ).drop(["target"], axis=1)
 y = df[(df["date"] <= df["date"].max() - datetime.timedelta(days=28))
&(df["date"] > df["date"].min() + datetime.timedelta(days=56)) ].set_index("date" ).drop(Xmas_date)["target"]
X_test = df[df["date"] > df["date"].max() - datetime.timedelta(days=28)].set_index("date" ).drop(["target"], axis=1)
 X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.15, random_state=random_state)
 return X_train, X_val, y_train, y_val, X_test<split>
sub.loc[:, 'healthy':] = probs
sub.to_csv('submission.csv', index=False)
 sub.head()