Kaggle competition: M5 Forecasting - Accuracy
user_secrets = UserSecretsClient()
token = user_secrets.get_secret("token")
 chat_id = user_secrets.get_secret("chat_id")
 warnings.filterwarnings('ignore' )<define_search_space>
warnings.filterwarnings(action='once' )
SEED = 42
BATCH_SIZE = 8
SIZE = [520,520]
LR = 0.001
WEIGHT_DECAY = 0
EPOCHS = 35
WARMUP = 15
STEP_SIZE = 5
TTA = 4<set_options>
calendar = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/calendar.csv', parse_dates=['date'], index_col='date')
 calendar.head(5 )
def seed_everything(SEED):
np.random.seed(SEED)
 torch.manual_seed(SEED)
 torch.cuda.manual_seed(SEED)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(SEED)
 device = torch.device("cuda:0" )<load_from_csv>
calendar.fillna('0', inplace=True)
 label_encoder = LabelEncoder()
label_cols = ['event_name_1', 'event_type_1', 'event_name_2', 'event_type_2']
for col in label_cols:
calendar[col] = label_encoder.fit_transform(calendar[col])
 calendar.head(5 )
DIR_INPUT = '/kaggle/input/plant-pathology-2020-fgvc7'
train_df = pd.read_csv(DIR_INPUT + '/train.csv')
 test_df = pd.read_csv(DIR_INPUT + '/test.csv')
 cols = list(train_df.columns[1:] )<categorify>
calendar['is_weekend'] = calendar['wday'].apply(lambda x: 1 if x == 1 or x == 2 else 0)
 seasons = {1: 1, 2: 1, 12: 1, 3: 2, 4: 2, 5: 2, 6: 3, 7: 3, 8: 3, 9: 4, 10: 4, 11: 4 }
calendar['season'] = calendar['month'].apply(lambda x: seasons[x])
 calendar.head()
class PLANT(Dataset):
def __init__(self,df,transform=None,train=True):
self.df = df
self.transform = transform
self.train = train
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
path = self.df.iloc[idx]['image_id']
image = cv2.imread(DIR_INPUT+f"/images/{path}.jpg")
 image = cv2.cvtColor(image,cv2.COLOR_BGR2RGB)
 if self.transform is not None:
image = self.transform(image=image)['image']
if self.train==True:
label = np.argmax(self.df[cols].iloc[idx].values ).reshape(1,1)
 return {'image': image,'label': label }
if self.train==False:
return {'image':image }<split>
calendar['wm_yr_wk'] = calendar['wm_yr_wk'].astype(np.int16)
 calendar['wday'] = calendar['wday'].astype(np.int8)
 calendar['month'] = calendar['month'].astype(np.int8)
 calendar['year'] = calendar['year'].astype(np.int16)
 calendar['snap_CA'] = calendar['snap_CA'].astype(np.int8)
 calendar['snap_TX'] = calendar['snap_TX'].astype(np.int8)
 calendar['snap_WI'] = calendar['snap_WI'].astype(np.int8)
 calendar['is_weekend'] = calendar['is_weekend'].astype(np.int8)
 calendar['season'] = calendar['season'].astype(np.int8)
 calendar['event_name_1'] = calendar['event_name_1'].astype(np.int16)
 calendar['event_type_1'] = calendar['event_type_1'].astype(np.int8)
 calendar['event_name_2'] = calendar['event_name_2'].astype(np.int8)
 calendar['event_type_2'] = calendar['event_type_2'].astype(np.int8 )
train,valid = train_test_split(train_df,test_size = 0.2,random_state = SEED)
 dataset_train = PLANT(df=train, transform=transform['train'])
 dataloader_train = DataLoader(dataset_train, batch_size=BATCH_SIZE, num_workers=4, shuffle=True, drop_last=True)
 dataset_valid = PLANT(df=valid, transform=transform['valid'])
 dataloader_valid = DataLoader(dataset_valid, batch_size=BATCH_SIZE, num_workers=4, shuffle=False, drop_last=True)
 dataset_test = PLANT(test_df,transform=transform['valid'],train=False)
 dataloader_test = DataLoader(dataset_test, batch_size=BATCH_SIZE, num_workers=4, shuffle=False)
 dataset_test_tta = PLANT(test_df,transform=transform['test_tta'],train=False)
 dataloader_test_tta = DataLoader(dataset_test_tta, batch_size=BATCH_SIZE, num_workers=4, shuffle=False )<choose_model_class>
train_data = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sales_train_evaluation.csv')
 train_data.head()
def getmodel() :
model = EfficientNet.from_pretrained('efficientnet-b5')
 model._fc = nn.Sequential(
 nn.Linear(in_features=2048, out_features=4, bias=True))
model = model.to(device)
 return model<choose_model_class>
def melt_item_group(group):
numcols = [f"d_{day}" for day in range(0,1941 + 1)]
catcols = ['id', 'item_id', 'dept_id','store_id', 'cat_id', 'state_id']
for day in range(1942, 1942 + 28):
group[f"d_{day}"] = np.nan
dt = pd.melt(
 group,
id_vars = catcols,
value_vars = [col for col in group.columns if col.startswith("d_")],
var_name = "d",
value_name = "sales" 
)
 return dt
new_model = getmodel()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(new_model.parameters() , lr=LR,weight_decay=WEIGHT_DECAY)
 scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=WARMUP, num_training_steps=EPOCHS )<train_model>
prices = pd.read_csv('.. /input/m5-forecasting-accuracy/sell_prices.csv')
 prices['wm_yr_wk'] = prices['wm_yr_wk'].astype(np.int16)
 prices['sell_price'] = prices['sell_price'].astype(np.float32)
 prices['wm_yr_wk'] = prices['wm_yr_wk'].astype(np.int16)
 prices.set_index(['store_id', 'item_id', 'wm_yr_wk'], inplace=True)
 prices = prices.sort_index()
results = pd.DataFrame(columns=['training_loss','training_accuracy','validation_loss','validation_accuracy','precision','recall','roc_auc_score'])
 @telegram_sender(token=token, chat_id=int(chat_id))
def train(model, criterion, optimizer,scheduler, dataloader_train, dataloader_valid):
global results
for epoch in range(EPOCHS):
print('Epoch {}/{}'.format(epoch,EPOCHS-1))
since = time.time()
model.train()
training_accuracy = []
training_loss = []
for bi, d in enumerate(tqdm(dataloader_train, total=int(len(dataloader_train)))) :
inputs = d["image"]
labels = d["label"]
inputs = inputs.to(device, dtype=torch.float)
 labels = labels.to(device, dtype=torch.long)
 optimizer.zero_grad()
with torch.set_grad_enabled(True):
outputs = model(inputs)
 labels = labels.squeeze()
loss = criterion(outputs, labels)
 loss.backward()
optimizer.step()
outputs = torch.max(outputs,1)[1]
outputs = outputs.cpu().detach().numpy()
labels = labels.cpu().numpy()
training_accuracy.append(accuracy_score(outputs,labels))
training_loss.append(loss.item())
 print('Training accuracy: {:.4f} and Training Loss: {:.4f}'.format(np.mean(training_accuracy),np.mean(training_loss)))
 model.eval()
validation_loss = []
validation_labels = []
validation_outputs = []
with torch.no_grad() :
for bi,d in enumerate(tqdm(dataloader_valid,total=int(len(dataloader_valid)))) :
inputs = d["image"]
labels = d["label"]
inputs = inputs.to(device, dtype=torch.float)
 labels = labels.to(device, dtype=torch.long)
 outputs = model(inputs)
 labels = labels.squeeze()
loss = criterion(outputs,labels)
 outputs_softmax = F.softmax(outputs ).cpu().detach().numpy()
labels_onehot = torch.eye(4)[labels].cpu().numpy()
validation_labels.extend(labels_onehot)
 validation_outputs.extend(outputs_softmax)
 validation_loss.append(loss.item())
 precision = precision_score(np.argmax(validation_labels,axis=1),np.argmax(validation_outputs,axis=1),average='macro')
 recall = recall_score(np.argmax(validation_labels,axis=1),np.argmax(validation_outputs,axis=1),average='macro')
 accuracy = accuracy_score(np.argmax(validation_labels,axis=1),np.argmax(validation_outputs,axis=1))
roc = roc_auc_score(validation_labels,validation_outputs,average='macro')
 print('Validation accuracy: {:.4f} and Validation Loss: {:.4f} and roc_auc_score: {:.4f}'.format(accuracy,\
np.mean(validation_loss),roc))
res = pd.DataFrame([[np.mean(training_loss),np.mean(training_accuracy),np.mean(validation_loss),\
accuracy,precision,recall,roc]],columns=results.columns)
 results = pd.concat([results,res])
 scheduler.step()
return results.iloc[-1]<train_model>
def convert_categorical_columns(df, columns):
for column in columns:
df[column] = df[column].astype('category')
 df[column] = df[column].cat.codes
df[column] = df[column].astype('category' )
train(new_model, criterion, optimizer,scheduler, dataloader_train, dataloader_valid )<find_best_params>
def get_dataset_simple(df, label_column):
test_df = df.loc['d_1914':,:].copy()
valid_df = df.loc['d_1914': 'd_1941',:].copy()
train_df = df.loc[df.index[0]: 'd_1913',:].copy()
train_df.loc[:,label_column] = train_df.loc[:,label_column]
return train_df, valid_df, test_df
display_training_curves(1, results['precision'], 'precision', 211)
 display_training_curves(1, results['recall'], 'recall', 212 )<predict_on_test>
def get_dataset(df, label_column):
test_df = df.loc['d_1914':,:].copy()
df_shape = df.loc[:'d_1941',:].shape
samples_idx = random.sample(range(df_shape[0]), int(0.2 * df_shape[0]))
valid_idxs = samples_idx
train_idxs = list(set(list(range(0, df_shape[0])))- set(samples_idx))
return train_idxs, valid_idxs, test_df
new_model.eval()
test_pred = np.zeros(( len(test_df),4))
with torch.no_grad() :
for i, data in enumerate(tqdm(dataloader_test,total=int(len(dataloader_test)))) :
inputs = data['image']
inputs = inputs.to(device, dtype=torch.float)
 predict = new_model(inputs)
 test_pred[i*len(predict):(i+1)*len(predict)] = predict.detach().cpu().squeeze().numpy()<save_to_csv>
evaluation_data = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sales_train_evaluation.csv')
 evaluation_data.loc[:,'id'] = evaluation_data['id'].apply(lambda x: x[:-10] + 'validation')
 evaluation_data.set_index('id', inplace=True)
 evaluation_data = evaluation_data.loc[:,'d_1914': 'd_1941'].copy()
def get_rmse(submission_validation):
sub = submission_validation.set_index('id')
 error = mean_squared_error(sub.values, evaluation_data.loc[sub.index,:].values)
 return error
submission_df = pd.read_csv(DIR_INPUT + '/sample_submission.csv')
 submission_df[['healthy', 'multiple_diseases', 'rust', 'scab']] = F.softmax(torch.from_numpy(test_pred),dim=1)
 submission_df.to_csv('submission.csv', index=False)
 pd.Series(np.argmax(submission_df[cols].values,axis=1)).value_counts()<predict_on_test>
def split_prediction(prediction):
columns = ['id'] + ['F%s' % i for i in range(1, 29)]
prediction = prediction.reset_index().set_index('id')
 prediction_evaluation = prediction.loc[:,'d_1914':'d_1941'].reset_index().copy()
prediction_evaluation.columns = columns
prediction_validation = prediction.loc[:,'d_1942':].reset_index().copy()
prediction_validation.columns = columns
prediction_validation.loc[:, 'id'] = prediction_validation['id'].str[:-10] + 'validation'
return prediction_validation, prediction_evaluation
new_model.eval()
test_pred = np.zeros(( len(test_df),4))
for i in range(TTA):
with torch.no_grad() :
for i, data in enumerate(tqdm(dataloader_test_tta,total=int(len(dataloader_test_tta)))) :
inputs = data['image']
inputs = inputs.to(device, dtype=torch.float)
 predict = new_model(inputs)
 test_pred[i*len(predict):(i+1)*len(predict)] += predict.detach().cpu().squeeze().numpy()<save_to_csv>
def create_price_features(group_df, horizon=28):
lag = horizon // 7
group_df['sell_price_diff_shift_1'] = group_df.groupby('id')['sell_price'].transform(lambda x: x - x.shift(1)).astype(np.float32)
 group_df[f'sell_price_diff_shift_{horizon}'] = group_df.groupby('id')['sell_price'].transform(lambda x: x - x.shift(horizon)).astype(np.float32)
 group_df['sell_price_diff_rolling_7'] = group_df.groupby('id')['sell_price'].transform(lambda x: x - x.rolling(7 ).mean() ).astype(np.float32)
 group_df[f'sell_price_diff_shift_{horizon}_shift_1'] = group_df.groupby('id')['sell_price_diff_shift_1'].transform(lambda x: x.shift(horizon)).astype(np.float32)
 group_df[f'sell_price_diff_shift_{horizon}_shift_{horizon}'] = group_df.groupby('id')[f'sell_price_diff_shift_{horizon}'].transform(lambda x: x.shift(horizon)).astype(np.float32)
 group_df[f'sell_price_diff_shift_{horizon}_rolling_7'] = group_df.groupby('id')['sell_price_diff_rolling_7'].transform(lambda x: x.shift(horizon)).astype(np.float32)
 group_df[f'sell_price_diff_rolling_7_diff_rolling_7_shift{horizon}'] = group_df.groupby('id')['sell_price_diff_rolling_7'].transform(lambda x: x - x.shift(horizon)).astype(np.float32 )
submission_df = pd.read_csv(DIR_INPUT + '/sample_submission.csv')
 submission_df[['healthy', 'multiple_diseases', 'rust', 'scab']] = F.softmax(torch.from_numpy(test_pred/TTA),dim=1)
 submission_df.to_csv('submission_tta.csv', index=False)
 pd.Series(np.argmax(submission_df[cols].values,axis=1)).value_counts()<import_modules>
def get_created_features(horizon=28):
feature_columns = [
'sell_price_diff_shift_1',
'sell_price_diff_rolling_7',
f'sell_price_diff_shift_{horizon}',
f'sell_price_diff_shift_{horizon}_shift_1',
f'sell_price_diff_shift_{horizon}_shift_{horizon}',
f'sell_price_diff_shift_{horizon}_rolling_7',
f'sell_price_diff_rolling_7_diff_rolling_7_shift{horizon}',
]
return feature_columns
import numpy as np
import pandas as pd
import os
import random, re, math
import tensorflow as tf, tensorflow.keras.backend as K
from tensorflow.keras.layers import Dense
from tensorflow.keras.models import Model
from tensorflow.keras import optimizers
from kaggle_datasets import KaggleDatasets
from tensorflow.keras.models import Sequential
import tensorflow.keras.layers as L
from tensorflow.keras.applications import ResNet152V2, InceptionResNetV2, InceptionV3, Xception, VGG19
from tensorflow.keras.layers import Dense, GlobalAveragePooling2D
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt<install_modules>
%%time
idx_feature = ['id']
categorical_feature = [
'dept_id' ,
'state_id',
'event_name_1',
'event_type_1',
'snap_CA',
'snap_TX',
'snap_WI',
'event_name_2',
'event_type_2',
'is_weekend',
'season',
]
feature_columns = categorical_feature + [
'sales_shift_28',
'sales_mean_rolling_4_wday_shift_4',
'sales_mean_rolling_4_wday_shift_8',
'sell_price',
]
label_column = 'sales'
dummy_subm = pd.DataFrame(columns=['id'] + ['F%s' % i for i in range(1, 29)])
 submission_validation = dummy_subm.copy()
submission_evaluation = dummy_subm.copy()
submission_validation_lgb = dummy_subm.copy()
submission_evaluation_lgb = dummy_subm.copy()
groups = train_data.groupby('store_id')
 len_group = len(groups)
 for i,(store_id, group)in enumerate(groups):
print(store_id)
 group = melt_item_group(group)
 group_df = group.join(calendar.set_index('d'), on='d')
 del group
group_df.sales.fillna(0, inplace=True)
 group_df = group_df.set_index(['d'])
 group_df['sales_shift_28'] = group_df.groupby(['id'])['sales'].transform(lambda x: x.shift(28)).astype(np.float32)
 group_df['sales_mean_rolling_4_wday'] = group_df.groupby(['id', 'wday'])['sales'].transform(lambda x: x.rolling(4 ).mean() ).astype(np.float32)
 group_df['sales_mean_rolling_4_wday_shift_4'] = group_df.groupby(['id', 'wday'])['sales_mean_rolling_4_wday'].transform(lambda x: x.shift(4)).astype(np.float32)
 group_df['sales_mean_rolling_4_wday_shift_8'] = group_df.groupby(['id', 'wday'])['sales_mean_rolling_4_wday'].transform(lambda x: x.shift(8)).astype(np.float32)
 print('add prices')
 group_df = group_df.join(
 prices.loc[store_id], on=['item_id', 'wm_yr_wk'] 
)
 label_cols_prices = ['state_id', 'dept_id']
convert_categorical_columns(group_df, label_cols_prices)
 create_price_features(group_df)
 feature_columns += get_created_features()
group_df.dropna(inplace=True)
 print('drop rows with na')
 train_idxs, valid_idxs, test_df = get_dataset(
 group_df[feature_columns + [label_column] + idx_feature],
label_column=label_column 
)
 print('get_dataset_simple')
 prediction = test_df.reset_index().set_index(['d', 'id',])['sales_mean_rolling_4_wday_shift_4']
prediction = prediction.unstack(level=0)
 prediction_validation, prediction_evaluation = split_prediction(prediction)
 try:
dtrain = lgb.Dataset(group_df.iloc[train_idxs][feature_columns], label=group_df.iloc[train_idxs][label_column], categorical_feature=categorical_feature)
 dvalid = lgb.Dataset(group_df.iloc[valid_idxs][feature_columns], label=group_df.iloc[valid_idxs][label_column], categorical_feature=categorical_feature)
 param = {
'boosting_type': 'gbdt',
'objective': 'tweedie',
'metric': 'rmse',
'subsample': 0.5,
'subsample_freq': 1,
'learning_rate': 0.03,
'num_leaves': 1024,
'min_data_in_leaf': 1024,
'feature_fraction': 0.2,
'max_bin': 10,
'boost_from_average': False,
'verbose': -1,
}
bst = lgb.train(param, dtrain, valid_sets=[dvalid], num_boost_round = 2000, early_stopping_rounds=30, verbose_eval=True, categorical_feature=categorical_feature)
 prediction_lgb = bst.predict(test_df.reset_index().set_index(['d', 'id',])[feature_columns])
 prediction_lgb_df = test_df.reset_index() [['d', 'id']].copy()
prediction_lgb_df['prediction'] = prediction_lgb
prediction_lgb_df = prediction_lgb_df.set_index(['d', 'id',] ).unstack(level=0 ).reset_index()
prediction_lgb_df.columns = ['id'] + ['d_{}'.format(i)for i in range(1914, 1970)]
prediction_validation_lgb, prediction_evaluation_lgb = split_prediction(prediction_lgb_df)
 except Exception as e:
print(e)
 prediction_validation_lgb = prediction_validation
prediction_evaluation_lgb = prediction_evaluation
finally:
submission_validation_lgb = submission_validation_lgb.append(prediction_validation_lgb)
 submission_evaluation_lgb = submission_evaluation_lgb.append(prediction_evaluation_lgb)
 submission_validation = submission_validation.append(prediction_validation)
 submission_evaluation = submission_evaluation.append(prediction_evaluation )
!pip install efficientnet 
<set_options>
submission = submission_validation.append(submission_evaluation)
 submission.to_csv('/kaggle/working/submission_mean.csv', index=False)
 submission_lgb = submission_validation_lgb.append(submission_evaluation_lgb)
 submission_lgb.to_csv('/kaggle/working/submission_lgb.csv', index=False )