Kaggle competition: Plant Pathology 2020 - FGVC7
<SOS> metric: MCAUC Kaggle data source: plant-pathology-2020-fgvc7<define_variables>
!pip install efficientnet
def get_dataset_simple(df, label_column):
test_df = df.loc['d_1914':,:].copy()
valid_models_df = df.loc['d_1914': 'd_1941',:].copy()
_df = df.loc[df.index[0]: 'd_1941',:].copy()
samples_idx = random.sample(range(_df.shape[0]), int(0.2 * _df.shape[0]))
valid_df = _df.iloc[samples_idx]
idx_train = list(set(list(range(0, item_df.shape[0])))- set(samples_idx))
train_df = df.iloc[idx_train].copy()
return train_df, valid_df, test_df, valid_models_df<load_from_csv>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
evaluation_data = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sales_train_evaluation.csv')
 evaluation_data.loc[:,'id'] = evaluation_data['id'].apply(lambda x: x[:-10] + 'validation')
 evaluation_data.set_index('id', inplace=True)
 evaluation_data = evaluation_data.loc[:,'d_1914': 'd_1941'].copy()
def get_rmse(submission_validation_boost):
sub = submission_validation_boost.set_index('id')
 error = mean_squared_error(sub.values, evaluation_data.loc[sub.index,:].values)
 return error<categorify>
seed = 69
img_dir = ".. /input/plant-pathology-2020-fgvc7//images/"
path = ".. /input/plant-pathology-2020-fgvc7/"
input_shape =(512,512,3)
def apply_target_encoder(item_df, cat_features_to_encoding):
target_enc = ce.TargetEncoder(cols=cat_features_to_encoding)
 target_enc.fit(item_df.loc[:'d_1913'][cat_features_to_encoding], item_df.loc[:'d_1913']['sales'])
 return item_df.join(target_enc.transform(item_df[cat_features_to_encoding] ).add_suffix('_target'))<feature_engineering>
train = pd.read_csv(path + 'train.csv')
 test = pd.read_csv(path + 'test.csv')
 sample_submission = pd.read_csv(path + 'sample_submission.csv')
def create_sales_features(item_df, horizon=7):
lag = horizon // 7
item_df[f'sales_shift_{horizon}'] = item_df['sales'].shift(horizon)
 item_df[f'sales_shift_{horizon}_shift_{horizon}'] = item_df['sales'].shift(horizon * 2)
 item_df['sales_mean_rolling_4_wday'] = item_df.groupby(['wday'])['sales'].transform(lambda x: x.rolling(4 ).mean())
 item_df[f'sales_mean_rolling_4_wday_shift_{lag}'] = item_df.groupby(['wday'])['sales_mean_rolling_4_wday'].transform(lambda x: x.shift(lag))
item_df[f'sales_mean_rolling_shift_{lag}_4_wday_shift_{lag}'] = item_df.groupby(['wday'])['sales_mean_rolling_4_wday'].transform(lambda x: x.shift(lag * 2))
item_df[f'sales_diff_rolling_shift_{lag}_4_wday_shift_{lag}'] = item_df[f'sales_mean_rolling_4_wday_shift_{lag}'] - item_df[f'sales_mean_rolling_shift_{lag}_4_wday_shift_{lag}']<feature_engineering>
gm_exp = tf.Variable(3.0, dtype = tf.float32)
 def generalized_mean_pool_2d(X):
pool =(tf.reduce_mean(tf.abs(X**(gm_exp)) ,
axis = [1, 2],
keepdims = False)+ 1.e-7)**(1./gm_exp)
 return pool
def create_model(input_shape):
input = Input(shape = input_shape)
 x_model = ResNet50(weights = 'imagenet', include_top = False, input_tensor = input, pooling = None,
classes = None)
 for layer in x_model.layers:
layer.trainable = True
lambda_layer = Lambda(generalized_mean_pool_2d)
 lambda_layer.trainable_weights.extend([gm_exp])
 x = lambda_layer(x_model.output)
 healthy = Dense(4, activation = 'softmax', name = 'plan_diseases' )(x)
 model = Model(inputs = x_model.input, outputs = healthy)
 return model
def create_price_features(item_df, horizon=7):
lag = horizon // 7
item_df['sell_price_diff_shift_1'] = item_df['sell_price'] - item_df['sell_price'].shift(1)
 item_df[f'sell_price_diff_shift_{horizon}'] = item_df['sell_price'] - item_df['sell_price'].shift(horizon)
 item_df['sell_price_diff_rolling_7'] = item_df['sell_price'] - item_df['sell_price'].rolling(7 ).mean()
item_df[f'sell_price_diff_shift_{horizon}_shift_1'] = item_df['sell_price'].shift(horizon)- item_df['sell_price'].shift(horizon + 1)
 item_df[f'sell_price_diff_shift_{horizon}_shift_{horizon}'] = item_df['sell_price'].shift(horizon)- item_df['sell_price'].shift(horizon * 2)
 item_df[f'sell_price_diff_shift_{horizon}_rolling_7'] = item_df['sell_price'].shift(28)- item_df['sell_price'].shift(horizon ).rolling(7 ).mean()
item_df[f'sell_price_diff_rolling_7_diff_rolling_7_shift{horizon}'] = item_df['sell_price_diff_rolling_7'] - item_df['sell_price_diff_rolling_7'].shift(horizon )<feature_engineering>
def _read(path):
img = cv2.imread(path)
 return img
class TrainDataGenerator(tf.keras.utils.Sequence):
def __init__(self , X_set, Y_set, img_dir, ids, batch_size = 2, img_size =(512,512,3), augmentation = False, GridMask = False):
self.X = X_set
self.Y = Y_set
self.batch_size = batch_size
self.ids = ids
self.img_size = img_size
self.img_dir = img_dir
self.augmentation = augmentation
self.GridMask = GridMask
self.on_epoch_end()
self.x_indexed = X_set[self.ids]
def __len__(self):
return int(ceil(len(self.ids)/self.batch_size))
def __getitem__(self, index):
indices = self.ids[index*self.batch_size:(index+1)*self.batch_size]
X, Y = self.__generator__(indices)
 return X, Y
def on_epoch_end(self):
self.indices = np.arange(len(self.ids))
def __generator__(self, indices):
X = np.empty(( self.batch_size, *self.img_size))
Y = np.empty(( self.batch_size, 4))
for i, index in enumerate(indices):
ID = self.X[index]
image = _read(self.img_dir+ID+".jpg")
 image = cv2.resize(image,(512,512))
if self.augmentation == True:
aug = strong_aug(p=1.0)
 image = augment(aug, image)
 elif self.GridMask == True:
image = augment(transforms_train, image)
 X[i,] = image/255.
 Y[i,] = self.Y.loc[index].values
return X, Y
class TestDataGenerator(tf.keras.utils.Sequence):
def __init__(self, Test_set, img_dir, ids, batch_size = 3, img_size =(512,512,3), augmentation = None):
self.X = Test_set
self.img_dir = img_dir
self.ids = ids
self.batch_size = batch_size
self.img_size = img_size
self.augmentation = augmentation
def __len__(self):
return int(ceil(len(self.ids)/self.batch_size))
def __getitem__(self, index):
indices = self.ids[index*self.batch_size:(index+1)*self.batch_size]
X = self.__generator__(indices)
 return X
def __generator__(self, indices):
X = np.empty(( self.batch_size, *self.img_size))
for i, index in enumerate(indices):
ID = self.X[index]
image = _read(self.img_dir+ID+".jpg")
 image = cv2.resize(image,(512,512))
if self.augmentation is not None:
if self.augmentation == "HorizontalFlip":
image = augment(HorizontalFlip(p=1), image)
 elif self.augmentation == "VerticalFlip":
image = augment(VerticalFlip(p=1), image)
 elif self.augmentation == 'VH':
image = VH_augment(image)
 X[i,] = image/255.
 return X
def create_categorical_features(item_df, horizon=7):
item_df[f'month_target_diff_{horizon}'] = item_df['month_target'] - item_df['month_target'].shift(horizon)
 item_df[f'season_target_diff_{horizon}'] = item_df['season_target'] - item_df['season_target'].shift(horizon)
 item_df[f'event_name_1_target_diff_{horizon}'] = item_df['event_name_1_target'] - item_df['event_name_1_target'].shift(horizon)
 item_df[f'event_type_1_target_diff_{horizon}'] = item_df['event_type_1_target'] - item_df['event_type_1_target'].shift(horizon)
 item_df[f'event_name_2_target_diff_{horizon}'] = item_df['event_name_2_target'] - item_df['event_name_2_target'].shift(horizon)
 item_df[f'event_type_2_target_diff_{horizon}'] = item_df['event_type_2_target'] - item_df['event_type_2_target'].shift(horizon)
 item_df[f'snap_CA_target_diff_{horizon}'] = item_df['snap_CA_target'] - item_df['snap_CA_target'].shift(horizon)
 item_df[f'snap_TX_target_diff_{horizon}'] = item_df['snap_TX_target'] - item_df['snap_TX_target'].shift(horizon)
 item_df[f'snap_WI_target_diff_{horizon}'] = item_df['snap_WI_target'] - item_df['snap_WI_target'].shift(horizon)
 item_df[f'season_target_diff_{horizon}_shift{horizon}'] = item_df['season_target'].shift(horizon)- item_df['season_target'].shift(horizon)
 item_df[f'event_name_1_target_diff_{horizon}_shift{horizon}'] = item_df['event_name_1_target'].shift(horizon)- item_df['event_name_1_target'].shift(horizon * 2)
 item_df[f'event_type_1_target_diff_{horizon}_shift{horizon}'] = item_df['event_type_1_target'].shift(horizon)- item_df['event_type_1_target'].shift(horizon * 2)
 item_df[f'event_name_2_target_diff_{horizon}_shift{horizon}'] = item_df['event_name_2_target'].shift(horizon)- item_df['event_name_2_target'].shift(horizon * 2)
 item_df[f'event_type_2_target_diff_{horizon}_shift{horizon}'] = item_df['event_type_2_target'].shift(horizon)- item_df['event_type_2_target'].shift(horizon * 2)
 item_df[f'snap_CA_target_diff_{horizon}_shift{horizon}'] = item_df['snap_CA_target'].shift(horizon)- item_df['snap_CA_target'].shift(horizon * 2)
 item_df[f'snap_TX_target_diff_{horizon}_shift{horizon}'] = item_df['snap_TX_target'].shift(horizon)- item_df['snap_TX_target'].shift(horizon * 2)
 item_df[f'snap_WI_target_diff_{horizon}_shift{horizon}'] = item_df['snap_WI_target'].shift(horizon)- item_df['snap_WI_target'].shift(horizon * 2 )<define_variables>
tgt_cols = ['healthy', 'multiple_diseases', 'rust' , 'scab']
train_df = train[tgt_cols]
Y = pd.get_dummies(train_df)
 X = train['image_id']
X_train, X_test, Y_train, Y_test = train_test_split(X,Y, test_size = 0.15, random_state = seed)
 ids_train = np.array(X_train.index)
 ids_test = X_test.index
del train
del X
del Y
def get_created_features(horizon=7):
lag = horizon // 7
linear_feature_columns = [
f'sales_mean_rolling_4_wday_shift_{lag}',
f'sell_price_diff_rolling_7_diff_rolling_7_shift{horizon}',
f'month_target_diff_{horizon}',
f'season_target_diff_{horizon}',
f'event_name_1_target_diff_{horizon}',
f'event_type_1_target_diff_{horizon}',
f'event_name_2_target_diff_{horizon}',
f'event_type_2_target_diff_{horizon}',
f'snap_CA_target_diff_{horizon}',
f'snap_TX_target_diff_{horizon}',
f'snap_WI_target_diff_{horizon}',
]
feature_columns = [
f'sales_mean_rolling_4_wday_shift_{lag}',
f'sales_mean_rolling_shift_{lag}_4_wday_shift_{lag}',
f'sales_diff_rolling_shift_{lag}_4_wday_shift_{lag}',
'sell_price_diff_shift_1',
'sell_price_diff_rolling_7',
f'sell_price_diff_shift_{horizon}',
f'sell_price_diff_shift_{horizon}_shift_1',
f'sell_price_diff_shift_{horizon}_shift_{horizon}',
f'sell_price_diff_shift_{horizon}_rolling_7',
f'sell_price_diff_rolling_7_diff_rolling_7_shift{horizon}',
f'month_target_diff_{horizon}',
f'season_target_diff_{horizon}',
f'event_name_1_target_diff_{horizon}',
f'event_type_1_target_diff_{horizon}',
f'event_name_2_target_diff_{horizon}',
f'event_type_2_target_diff_{horizon}',
f'snap_CA_target_diff_{horizon}',
f'snap_TX_target_diff_{horizon}',
f'snap_WI_target_diff_{horizon}',
f'season_target_diff_{horizon}_shift{horizon}',
f'event_name_1_target_diff_{horizon}_shift{horizon}',
f'event_type_1_target_diff_{horizon}_shift{horizon}',
f'event_name_2_target_diff_{horizon}_shift{horizon}',
f'event_type_2_target_diff_{horizon}_shift{horizon}',
f'snap_CA_target_diff_{horizon}_shift{horizon}',
f'snap_TX_target_diff_{horizon}_shift{horizon}',
f'snap_WI_target_diff_{horizon}_shift{horizon}',
]
return feature_columns, linear_feature_columns<define_variables>
model = create_model(input_shape)
 model.compile(optimizer = Adam(lr = 0.00016),
loss = 'categorical_crossentropy',
metrics = ['categorical_accuracy'] )
%%time
idx_feature = ['id']
categorical_feature = [
'wday',
'month',
'year',
'event_name_1',
'event_type_1',
'snap_CA',
'snap_TX',
'snap_WI',
'event_name_2',
'event_type_2',
'is_weekend',
'season',
]
cat_features_to_encoding = [
'wday',
'month',
'year',
'event_name_1',
'event_type_1',
'snap_CA',
'snap_TX',
'snap_WI',
'event_name_2',
'event_type_2',
'is_weekend',
'season',
]
encoded_cat_features = [i + '_target' for i in cat_features_to_encoding]
label_column = 'sales'
sub_columns = ['id'] + ['F%s' % i for i in range(1, 29)]
submission_validation = pd.DataFrame(columns=sub_columns)
 submission_evaluation = pd.DataFrame(columns=sub_columns)
 submission_validation_lgb = pd.DataFrame(columns=sub_columns)
 submission_evaluation_lgb = pd.DataFrame(columns=sub_columns)
 submission_validation_sgd = pd.DataFrame(columns=sub_columns)
 submission_evaluation_sgd = pd.DataFrame(columns=sub_columns)
 submission_validation_linear = pd.DataFrame(columns=sub_columns)
 submission_evaluation_linear = pd.DataFrame(columns=sub_columns)
 submission_validation_mean = pd.DataFrame(columns=sub_columns)
 submission_evaluation_mean = pd.DataFrame(columns=sub_columns)
 random.seed(2)
 samples_idx = random.sample(range(train_data.shape[0]), 300)
 models = {'linear': 0, 'lgb': 0, 'mean': 0, 'linearSGD': 0}
feature_importance_init =False
feature_importance = None
feature_importance_n = 0
for iteration, i in enumerate(range(train_data.shape[0])) :
if iteration % 1000 == 1:
print(iteration, train_data.shape[0])
 print('mean', get_rmse(submission_validation_mean))
print('linear', get_rmse(submission_validation_linear))
print('sgd', get_rmse(submission_validation_sgd))
print('lgb', get_rmse(submission_validation_lgb))
print('pred', get_rmse(submission_validation))
row = train_data.loc[i]
all_id = row.id
item_id = row.item_id
dept_id = row.dept_id
cat_id = row.cat_id
store_id = row.store_id
state_id = row.state_id
sales = row['d_1':]
item_df = calendar.join(sales.to_frame('sales'), on='d')
 item_df = item_df.join(prices.loc[store_id, item_id], on=['wm_yr_wk'])
 item_df.sales.fillna(0, inplace=True)
 item_df.loc[:,'sales'] = item_df.sales.astype('int64')
 item_df = item_df.set_index('d')
 first_sale = item_df[item_df.sales!=0].index[0]
first_sale_int = int(first_sale[2:])
 if first_sale_int > 1914 - 90:
first_sale = 'd_{}'.format(1914 - 90)
 item_df = item_df.loc[first_sale:,:]
item_df = apply_target_encoder(item_df, cat_features_to_encoding)
 feature_columns = []
linear_feature_columns = []
for horizon in [28,]:
lag = horizon // 7
create_price_features(item_df, horizon=horizon)
 create_categorical_features(item_df, horizon=horizon)
 create_sales_features(item_df, horizon=horizon)
 features, linear_features = get_created_features(horizon=horizon)
 feature_columns += features
linear_feature_columns = linear_features
feature_columns = feature_columns + encoded_cat_features + categorical_feature
item_df.dropna(inplace=True)
 predictions_list = []
try:
train_df, valid_df, test_df, valid_models_df = get_dataset_simple(item_df[feature_columns + [label_column]], label_column=label_column)
 sc_X = StandardScaler()
sc_y = StandardScaler()
X_train = sc_X.fit_transform(train_df[linear_feature_columns])
 y_train = sc_y.fit_transform(train_df[label_column].values.reshape(-1,1))
X_test = sc_X.fit_transform(test_df[linear_feature_columns])
 X_valid = sc_X.fit_transform(valid_df[linear_feature_columns])
 y_valid = sc_y.fit_transform(valid_df[label_column].values.reshape(-1,1))
model_sgd = SGDRegressor(max_iter=3000)
 model_sgd.fit(X_train, y_train.reshape(-1))
prediction_sgd = sc_y.inverse_transform(model_sgd.predict(X_test ).reshape(-1,1)).reshape(-1 ).tolist()
predictions_list.append(['linearSGD', prediction_sgd, mean_squared_error(valid_models_df.sales.values, prediction_sgd[0:28])])
 model = LinearRegression()
model.fit(X_train, y_train.reshape(-1))
prediction_linear = sc_y.inverse_transform(model.predict(X_test ).reshape(-1,1)).reshape(-1 ).tolist()
predictions_list.append(['linear', prediction_linear, mean_squared_error(valid_models_df.sales.values, prediction_linear[0:28])])
 dtrain = lgb.Dataset(train_df[feature_columns], label=train_df[label_column], categorical_feature=categorical_feature)
 dvalid = lgb.Dataset(valid_df[feature_columns], label=valid_df[label_column], categorical_feature=categorical_feature)
 param = {
'boosting_type': 'gbdt',
'objective': 'tweedie',
'metric': 'rmse',
'subsample': 0.5,
'subsample_freq': 1,
'learning_rate': 0.03,
'num_leaves': 64,
'min_data_in_leaf': min(1024, test_df.shape[0] // 4),
'feature_fraction': 0.1,
'boost_from_average': False,
'verbose': -1,
'lambda_l1': 0.8,
}
bst = lgb.train(param, dtrain, valid_sets=[dvalid], num_boost_round = 2400, early_stopping_rounds=500, verbose_eval=False, categorical_feature=categorical_feature)
 if not feature_importance_init:
feature_importance_init = True
feature_importance = bst.feature_importance()
else:
feature_importance += bst.feature_importance()
feature_importance_n += 1
prediction_lgb = bst.predict(test_df[feature_columns] ).tolist()
predictions_list.append(['lgb', prediction_lgb, mean_squared_error(valid_models_df.sales.values, prediction_lgb[0:28])])
 except Exception as e:
print(i, all_id, item_df.shape)
 print(e)
 prediction_lgb = test_df[f'sales_mean_rolling_4_wday_shift_{lag}'].values.tolist()
prediction_mean =(test_df[f'sales_mean_rolling_4_wday_shift_{lag}'] ).values.tolist()
predictions_list.append(['mean', prediction_mean, mean_squared_error(valid_models_df.sales.values, prediction_mean[0:28])])
 best_predictions = sorted(predictions_list, key=lambda x: x[2])[0]
models[best_predictions[0]] += 1
key = all_id[:-10] + 'validation'
submission_validation.loc[len(submission_validation)] = [key] + best_predictions[1][0:28]
submission_evaluation.loc[len(submission_evaluation)] = [all_id] + best_predictions[1][28:56]
submission_validation_lgb.loc[len(submission_validation_lgb)] = [key] + prediction_lgb[0:28]
submission_evaluation_lgb.loc[len(submission_evaluation_lgb)] = [all_id] + prediction_lgb[28:56]
submission_validation_sgd.loc[len(submission_validation_sgd)] = [key] + prediction_sgd[0:28]
submission_evaluation_sgd.loc[len(submission_evaluation_sgd)] = [all_id] + prediction_sgd[28:56]
submission_validation_linear.loc[len(submission_validation_linear)] = [key] + prediction_linear[0:28]
submission_evaluation_linear.loc[len(submission_evaluation_linear)] = [all_id] + prediction_linear[28:56]
submission_validation_mean.loc[len(submission_validation_mean)] = [key] + prediction_mean[0:28]
submission_evaluation_mean.loc[len(submission_evaluation_mean)] = [all_id] + prediction_mean[28:56]
print('Final')
 print('mean', get_rmse(submission_validation_mean))
print('linear', get_rmse(submission_validation_linear))
print('sgd', get_rmse(submission_validation_sgd))
print('lgb', get_rmse(submission_validation_lgb))
print('pred', get_rmse(submission_validation))
print(models )<save_to_csv>
best_w = ModelCheckpoint('plant_best.h5',
monitor='val_loss',
verbose=0,
save_best_only=True,
save_weights_only=True,
mode='auto',
period=1)
 last_w = ModelCheckpoint('plant_last.h5',
monitor='val_loss',
verbose=0,
save_best_only=False,
mode='auto',
period=1)
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
patience=2, min_lr=0.000001, verbose=1,cooldown=1)
 callbacks = [best_w, last_w, reduce_lr]
submission = submission_validation.append(submission_evaluation)
 submission.to_csv('/kaggle/working/my_submission.csv', index=False )<save_to_csv>
batch_size = 5
model.fit_generator(TrainDataGenerator(X_train,Y_train , img_dir, ids_train,
batch_size, augmentation = True, GridMask = True),
epochs=15,
verbose=1,
callbacks=callbacks,
validation_data=TrainDataGenerator(X_test, Y_test , img_dir, ids_test,
batch_size, augmentation = False, GridMask = False),
workers=1,
use_multiprocessing=False,
shuffle=True )
submission_mean = submission_validation_mean.append(submission_evaluation_mean)
 submission_mean.to_csv('/kaggle/working/my_submission_mean.csv', index=False )<save_to_csv>
gc.collect()
submission_lgb = submission_validation_lgb.append(submission_evaluation_lgb)
 submission_lgb.to_csv('/kaggle/working/my_submission_lgb.csv', index=False )<save_to_csv>
model.load_weights('plant_best.h5' )
submission_linear = submission_validation_linear.append(submission_evaluation_linear)
 submission_linear.to_csv('/kaggle/working/my_submission_linear.csv', index=False )<save_to_csv>
ids = test.index
Test_set = test['image_id']
data_generator_test = TestDataGenerator(Test_set, img_dir, ids, batch_size, input_shape, augmentation = None)
 data_generator_test_Horizontal = TestDataGenerator(Test_set, img_dir, ids, batch_size, input_shape, augmentation = "HorizontalFlip")
 data_generator_test_Vertical = TestDataGenerator(Test_set, img_dir, ids, batch_size, input_shape, augmentation = "VerticalFlip")
 data_generator_test_VH = TestDataGenerator(Test_set, img_dir, ids, batch_size, input_shape, augmentation = "VH")
 preds_1 = model.predict_generator(data_generator_test, verbose = 1)
 preds_2 = model.predict_generator(data_generator_test_Horizontal, verbose = 1)
 preds_3 = model.predict_generator(data_generator_test_Vertical, verbose = 1)
 preds_4 = model.predict_generator(data_generator_test_VH, verbose = 1)
 for index, pred in enumerate(zip(preds_1,preds_2, preds_3, preds_4)) :
sample_submission["healthy"][index] =(pred[0][0] + pred[1][0] + pred[2][0] + pred[3][0])/4
sample_submission["multiple_diseases"][index] =(pred[0][1] + pred[1][1] + pred[2][1] + pred[3][1])/4
sample_submission["rust"][index] =(pred[0][2] + pred[1][2] + pred[2][2] + pred[3][2])/4
sample_submission["scab"][index] =(pred[0][3] + pred[1][3] + pred[2][3] + pred[3][3])/4
submission_sgd = submission_validation_sgd.append(submission_evaluation_sgd)
 submission_sgd.to_csv('/kaggle/working/my_submission_sgd.csv', index=False )<import_modules>
sample_submission.to_csv("submission.csv", index=False )
from datetime import datetime, timedelta
import gc
import numpy as np, pandas as pd
import lightgbm as lgb<define_variables>
!pip install efficientnet
CAL_DTYPES={"event_name_1": "category", "event_name_2": "category", "event_type_1": "category",
"event_type_2": "category", "weekday": "category", 'wm_yr_wk': 'int16', "wday": "int16",
"month": "int16", "year": "int16", "snap_CA": "float32", 'snap_TX': 'float32', 'snap_WI': 'float32' }
PRICE_DTYPES = {"store_id": "category", "item_id": "category", "wm_yr_wk": "int16","sell_price":"float32" }<set_options>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
pd.options.display.max_columns = 50<define_variables>
seed = 69
img_dir = ".. /input/plant-pathology-2020-fgvc7//images/"
path = ".. /input/plant-pathology-2020-fgvc7/"
input_shape =(512,512,3)
h = 28
max_lags = 57
tr_last = 1913 + 28
fday = datetime(2016,4, 25)+ timedelta(days= 28)
 fday,tr_last,max_lags<load_from_csv>
train = pd.read_csv(path + 'train.csv')
 test = pd.read_csv(path + 'test.csv')
 sample_submission = pd.read_csv(path + 'sample_submission.csv')
def create_dt(is_train = True, nrows = None, first_day = 1200):
prices = pd.read_csv(".. /input/m5-forecasting-accuracy/sell_prices.csv", dtype = PRICE_DTYPES)
 for col, col_dtype in PRICE_DTYPES.items() :
if col_dtype == "category":
prices[col] = prices[col].cat.codes.astype("int16")
 prices[col] -= prices[col].min()
cal = pd.read_csv(".. /input/m5-forecasting-accuracy/calendar.csv", dtype = CAL_DTYPES)
 cal["date"] = pd.to_datetime(cal["date"])
 for col, col_dtype in CAL_DTYPES.items() :
if col_dtype == "category":
cal[col] = cal[col].cat.codes.astype("int16")
 cal[col] -= cal[col].min()
start_day = max(1 if is_train else tr_last-max_lags, first_day)
 numcols = [f"d_{day}" for day in range(start_day,tr_last+1)]
catcols = ['id', 'item_id', 'dept_id','store_id', 'cat_id', 'state_id']
dtype = {numcol:"float32" for numcol in numcols}
dtype.update({col: "category" for col in catcols if col != "id"})
 dt = pd.read_csv(".. /input/m5-forecasting-accuracy/sales_train_evaluation.csv",
nrows = nrows, usecols = catcols + numcols, dtype = dtype)
 for col in catcols:
if col != "id":
dt[col] = dt[col].cat.codes.astype("int16")
 dt[col] -= dt[col].min()
if not is_train:
for day in range(tr_last+1, tr_last+ 28 +1):
dt[f"d_{day}"] = np.nan
dt = pd.melt(dt,
id_vars = catcols,
value_vars = [col for col in dt.columns if col.startswith("d_")],
var_name = "d",
value_name = "sales")
 dt = dt.merge(cal, on= "d", copy = False)
 dt = dt.merge(prices, on = ["store_id", "item_id", "wm_yr_wk"], copy = False)
 return dt<categorify>
gm_exp = tf.Variable(3.0, dtype = tf.float32)
 def generalized_mean_pool_2d(X):
pool =(tf.reduce_mean(tf.abs(X**(gm_exp)) ,
axis = [1, 2],
keepdims = False)+ 1.e-7)**(1./gm_exp)
 return pool
def create_model(input_shape):
input = Input(shape = input_shape)
 x_model = ResNet50(weights = 'imagenet', include_top = False, input_tensor = input, pooling = None,
classes = None)
 for layer in x_model.layers:
layer.trainable = True
lambda_layer = Lambda(generalized_mean_pool_2d)
 lambda_layer.trainable_weights.extend([gm_exp])
 x = lambda_layer(x_model.output)
 healthy = Dense(4, activation = 'softmax', name = 'plan_diseases' )(x)
 model = Model(inputs = x_model.input, outputs = healthy)
 return model
def create_fea(dt):
lags = [7, 28]
lag_cols = [f"lag_{lag}" for lag in lags ]
for lag, lag_col in zip(lags, lag_cols):
dt[lag_col] = dt[["id","sales"]].groupby("id")["sales"].shift(lag)
 wins = [7, 28]
for win in wins :
for lag,lag_col in zip(lags, lag_cols):
dt[f"rmean_{lag}_{win}"] = dt[["id", lag_col]].groupby("id")[lag_col].transform(lambda x : x.rolling(win ).mean())
 date_features = {
"wday": "weekday",
"week": "weekofyear",
"month": "month",
"quarter": "quarter",
"year": "year",
"mday": "day",
}
for date_feat_name, date_feat_func in date_features.items() :
if date_feat_name in dt.columns:
dt[date_feat_name] = dt[date_feat_name].astype("int16")
 else:
dt[date_feat_name] = getattr(dt["date"].dt, date_feat_func ).astype("int16" )<define_variables>
def _read(path):
img = cv2.imread(path)
 return img
class TrainDataGenerator(tf.keras.utils.Sequence):
def __init__(self , X_set, Y_set, img_dir, ids, batch_size = 2, img_size =(512,512,3), augmentation = False, GridMask = False):
self.X = X_set
self.Y = Y_set
self.batch_size = batch_size
self.ids = ids
self.img_size = img_size
self.img_dir = img_dir
self.augmentation = augmentation
self.GridMask = GridMask
self.on_epoch_end()
self.x_indexed = X_set[self.ids]
def __len__(self):
return int(ceil(len(self.ids)/self.batch_size))
def __getitem__(self, index):
indices = self.ids[index*self.batch_size:(index+1)*self.batch_size]
X, Y = self.__generator__(indices)
 return X, Y
def on_epoch_end(self):
self.indices = np.arange(len(self.ids))
def __generator__(self, indices):
X = np.empty(( self.batch_size, *self.img_size))
Y = np.empty(( self.batch_size, 4))
for i, index in enumerate(indices):
ID = self.X[index]
image = _read(self.img_dir+ID+".jpg")
 image = cv2.resize(image,(512,512))
if self.augmentation == True:
aug = strong_aug(p=1.0)
 image = augment(aug, image)
 elif self.GridMask == True:
image = augment(transforms_train, image)
 X[i,] = image/255.
 Y[i,] = self.Y.loc[index].values
return X, Y
class TestDataGenerator(tf.keras.utils.Sequence):
def __init__(self, Test_set, img_dir, ids, batch_size = 3, img_size =(512,512,3), augmentation = None):
self.X = Test_set
self.img_dir = img_dir
self.ids = ids
self.batch_size = batch_size
self.img_size = img_size
self.augmentation = augmentation
def __len__(self):
return int(ceil(len(self.ids)/self.batch_size))
def __getitem__(self, index):
indices = self.ids[index*self.batch_size:(index+1)*self.batch_size]
X = self.__generator__(indices)
 return X
def __generator__(self, indices):
X = np.empty(( self.batch_size, *self.img_size))
for i, index in enumerate(indices):
ID = self.X[index]
image = _read(self.img_dir+ID+".jpg")
 image = cv2.resize(image,(512,512))
if self.augmentation is not None:
if self.augmentation == "HorizontalFlip":
image = augment(HorizontalFlip(p=1), image)
 elif self.augmentation == "VerticalFlip":
image = augment(VerticalFlip(p=1), image)
 elif self.augmentation == 'VH':
image = VH_augment(image)
 X[i,] = image/255.
 return X
FIRST_DAY = 350<correct_missing_values>
tgt_cols = ['healthy', 'multiple_diseases', 'rust' , 'scab']
train_df = train[tgt_cols]
Y = pd.get_dummies(train_df)
 X = train['image_id']
X_train, X_test, Y_train, Y_test = train_test_split(X,Y, test_size = 0.15, random_state = seed)
 ids_train = np.array(X_train.index)
 ids_test = X_test.index
del train
del X
del Y
df.dropna(inplace = True)
 df.shape<prepare_x_and_y>
model = create_model(input_shape)
 model.compile(optimizer = Adam(lr = 0.00016),
loss = 'categorical_crossentropy',
metrics = ['categorical_accuracy'] )
cat_feats = ['item_id', 'dept_id','store_id', 'cat_id', 'state_id'] + ["event_name_1", "event_name_2", "event_type_1", "event_type_2"]
useless_cols = ["id", "date", "sales","d", "wm_yr_wk", "weekday"]
train_cols = df.columns[~df.columns.isin(useless_cols)]
X_train = df[train_cols]
y_train = df["sales"]<create_dataframe>
best_w = ModelCheckpoint('plant_best.h5',
monitor='val_loss',
verbose=0,
save_best_only=True,
save_weights_only=True,
mode='auto',
period=1)
 last_w = ModelCheckpoint('plant_last.h5',
monitor='val_loss',
verbose=0,
save_best_only=False,
mode='auto',
period=1)
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
patience=2, min_lr=0.000001, verbose=1,cooldown=1)
 callbacks = [best_w, last_w, reduce_lr]
<create_dataframe>
batch_size = 5
model.fit_generator(TrainDataGenerator(X_train,Y_train , img_dir, ids_train,
batch_size, augmentation = True, GridMask = True),
epochs=15,
verbose=1,
callbacks=callbacks,
validation_data=TrainDataGenerator(X_test, Y_test , img_dir, ids_test,
batch_size, augmentation = False, GridMask = False),
workers=1,
use_multiprocessing=False,
shuffle=True )
%%time
np.random.seed(777)
 fake_valid_inds = np.random.choice(X_train.index.values, 2_000_000, replace = False)
 train_inds = np.setdiff1d(X_train.index.values, fake_valid_inds)
 train_data = lgb.Dataset(X_train.loc[train_inds] , label = y_train.loc[train_inds],
categorical_feature=cat_feats, free_raw_data=False)
 fake_valid_data = lgb.Dataset(X_train.loc[fake_valid_inds], label = y_train.loc[fake_valid_inds],
categorical_feature=cat_feats,
free_raw_data=False )<prepare_x_and_y>
gc.collect()
del df, X_train, y_train, fake_valid_inds,train_inds ; gc.collect()<init_hyperparams>
model.load_weights('plant_best.h5' )
params = {
"objective" : "poisson",
"metric" :"rmse",
"force_row_wise" : True,
"learning_rate" : 0.075,
"sub_row" : 0.75,
"bagging_freq" : 1,
"lambda_l2" : 0.1,
"metric": ["rmse"],
'verbosity': 1,
'num_iterations' : 1200,
'num_leaves': 128,
"min_data_in_leaf": 100,
}<train_model>
ids = test.index
Test_set = test['image_id']
data_generator_test = TestDataGenerator(Test_set, img_dir, ids, batch_size, input_shape, augmentation = None)
 data_generator_test_Horizontal = TestDataGenerator(Test_set, img_dir, ids, batch_size, input_shape, augmentation = "HorizontalFlip")
 data_generator_test_Vertical = TestDataGenerator(Test_set, img_dir, ids, batch_size, input_shape, augmentation = "VerticalFlip")
 data_generator_test_VH = TestDataGenerator(Test_set, img_dir, ids, batch_size, input_shape, augmentation = "VH")
 preds_1 = model.predict_generator(data_generator_test, verbose = 1)
 preds_2 = model.predict_generator(data_generator_test_Horizontal, verbose = 1)
 preds_3 = model.predict_generator(data_generator_test_Vertical, verbose = 1)
 preds_4 = model.predict_generator(data_generator_test_VH, verbose = 1)
 for index, pred in enumerate(zip(preds_1,preds_2, preds_3, preds_4)) :
sample_submission["healthy"][index] =(pred[0][0] + pred[1][0] + pred[2][0] + pred[3][0])/4
sample_submission["multiple_diseases"][index] =(pred[0][1] + pred[1][1] + pred[2][1] + pred[3][1])/4
sample_submission["rust"][index] =(pred[0][2] + pred[1][2] + pred[2][2] + pred[3][2])/4
sample_submission["scab"][index] =(pred[0][3] + pred[1][3] + pred[2][3] + pred[3][3])/4
%%time
m_lgb = lgb.train(params, train_data, valid_sets = [fake_valid_data], verbose_eval=20 )<save_model>
sample_submission.to_csv("submission.csv", index=False )
m_lgb.save_model("model.lgb" )<count_unique_values>
!pip install efficientnet
sub.id.nunique() , sub["id"].str.contains("evaluation$" ).sum()<import_modules>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
import os
import gc
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from datetime import datetime, timedelta, date
from tqdm.notebook import tqdm
import lightgbm as lgb<define_variables>
seed = 69
img_dir = ".. /input/plant-pathology-2020-fgvc7//images/"
path = ".. /input/plant-pathology-2020-fgvc7/"
input_shape =(512,512,3)
TEST_RUN = False
pd.set_option('display.max_columns', 50)
 KAGGLE_DATA_FOLDER = '/kaggle/input/m5-forecasting-accuracy'
SUBMISSION_PATH = "/kaggle/input/lgbmensemblebestvalidationsubmission/submission.csv"
PATH_TO_ALL_CATS = "/kaggle/input/single-model-evaluation-submission/model_final.lgb"
PATH_TO_CAT0 = "/kaggle/input/category-models-evaluation-submission/model_cat0_final.lgb"
PATH_TO_CAT1 = "/kaggle/input/category-models-evaluation-submission/model_cat1_final.lgb"
PATH_TO_CAT2 = "/kaggle/input/category-models-evaluation-submission/model_cat2_final.lgb"
PATH_MODELS = [PATH_TO_ALL_CATS, PATH_TO_CAT0, PATH_TO_CAT1, PATH_TO_CAT2]
KAGGLE_DATA_FOLDER = '/kaggle/input/m5-forecasting-accuracy'
BACKWARD_LAG = 28+28
END_DAY = 1913+28
BEGIN_DATE = '2013-08-01'
BEGIN_DAY = str(( datetime.strptime(BEGIN_DATE, '%Y-%m-%d')- datetime.strptime('2011-01-29', '%Y-%m-%d')).days)
 EVAL_SPLIT = '2016-05-22'
print(datetime.strptime(EVAL_SPLIT, '%Y-%m-%d'))<define_variables>
train = pd.read_csv(path + 'train.csv')
 test = pd.read_csv(path + 'test.csv')
 sample_submission = pd.read_csv(path + 'sample_submission.csv')
CALENDAR_DTYPES = {
'date': 'str',
'wm_yr_wk': 'int16',
'weekday': 'category',
'wday': 'int16',
'month': 'int16',
'year': 'int16',
'd': 'object',
'event_name_1': 'category',
'event_type_1': 'category',
'event_name_2': 'category',
'event_type_2': 'category',
'snap_CA': 'int16',
'snap_TX': 'int16',
'snap_WI': 'int16'
}
PARSE_DATES = ['date']
SALES_PRICES_DTYPES = {
'store_id': 'category',
'item_id': 'category',
'wm_yr_wk': 'int16',
'sell_price': 'float32',
'sales': 'float32'
}<load_pretrained>
gm_exp = tf.Variable(3.0, dtype = tf.float32)
 def generalized_mean_pool_2d(X):
pool =(tf.reduce_mean(tf.abs(X**(gm_exp)) ,
axis = [1, 2],
keepdims = False)+ 1.e-7)**(1./gm_exp)
 return pool
def create_model(input_shape):
input = Input(shape = input_shape)
 x_model = ResNet50(weights = 'imagenet', include_top = False, input_tensor = input, pooling = None,
classes = None)
 for layer in x_model.layers:
layer.trainable = True
lambda_layer = Lambda(generalized_mean_pool_2d)
 lambda_layer.trainable_weights.extend([gm_exp])
 x = lambda_layer(x_model.output)
 healthy = Dense(4, activation = 'softmax', name = 'plan_diseases' )(x)
 model = Model(inputs = x_model.input, outputs = healthy)
 return model
def load_data(train=True):

 print("Loading train and validation data")
 numcols = [f"d_{day}" for day in range(int(BEGIN_DAY),END_DAY+1)]
catcols = ['id', 'item_id', 'dept_id','store_id', 'cat_id', 'state_id']
dtype = {numcol:"float32" for numcol in numcols}
dtype.update({col: "category" for col in catcols if col != "id"})
 sales_train_validation = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sales_train_evaluation.csv'),
usecols=catcols+numcols, dtype=dtype)
 for col in catcols:
if col != "id":
sales_train_validation[col] = sales_train_validation[col].cat.codes.astype("int16")
 sales_train_validation[col] -= sales_train_validation[col].min()
if not train:
for day in range(END_DAY+1, END_DAY+28+1):
sales_train_validation[f"d_{day}"] = 0
value_vars = [column for column in sales_train_validation.columns
if(column.startswith('d_')and int(column.replace('d_', '')) >= END_DAY - BACKWARD_LAG)]
else:
value_vars = [col for col in sales_train_validation.columns
if(col.startswith('d_')and(int(col.replace('d_', '')) >= int(BEGIN_DAY)))]
print("Shape:", sales_train_validation.shape)
 print("Memory usage(Mb)before melting:", sales_train_validation.memory_usage().sum() / 1024**2)
 sales_train_validation = pd.melt(
 sales_train_validation,
id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
value_vars=value_vars,
var_name = 'day',
value_name = 'sales')
 print("Completed melting, new shape:", sales_train_validation.shape)
 print("Colums after melting:", sales_train_validation.columns)
 print("Memory usage(Mb)after melting:", sales_train_validation.memory_usage().sum() / 1024**2)
 columns = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']
print("Loading calendar")
 calendar = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'calendar.csv'), dtype=CALENDAR_DTYPES, parse_dates=['date'])
 print("Calendar columns: ", calendar.columns)
 print("Memory usage(Mb)calendar: ", calendar.memory_usage().sum() / 1024**2)
 calendar.rename(columns={'d':'day'}, inplace=True)
 for col, col_dtype in CALENDAR_DTYPES.items() :
if col_dtype == "category":
calendar[col] = calendar[col].cat.codes.astype("int16")
 calendar[col] -= calendar[col].min()
sales_train_validation = sales_train_validation.merge(calendar, on="day", copy=False)
 del calendar; gc.collect()
print("Merged calendar(in place)")
 print("Memory usage(Mb)after merging calendar:", sales_train_validation.memory_usage().sum() / 1024**2)
 print("Colums after merge:", sales_train_validation.columns)
 sell_prices = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sell_prices.csv'), dtype=SALES_PRICES_DTYPES)
 for col, col_dtype in SALES_PRICES_DTYPES.items() :
if col_dtype == "category":
sell_prices[col] = sell_prices[col].cat.codes.astype("int16")
 sell_prices[col] -= sell_prices[col].min()
print("Memory usage(Mb)sell prices:", sell_prices.memory_usage().sum() / 1024**2)
 columns = ['item_id', 'store_id', 'sell_price']
for feature in columns:
if feature == 'sell_price':
sell_prices[feature].fillna(0, inplace=True)
 sales_train_validation = sales_train_validation.merge(sell_prices, on=["store_id","item_id","wm_yr_wk"], copy=False)
 del sell_prices; gc.collect()
print("Merged sales prices(in place)")
 print("Memory usage(Mb)after merging sales:", sales_train_validation.memory_usage().sum() / 1024**2)
 print("Colums after merge:", sales_train_validation.columns)
 return sales_train_validation<categorify>
def _read(path):
img = cv2.imread(path)
 return img
class TrainDataGenerator(tf.keras.utils.Sequence):
def __init__(self , X_set, Y_set, img_dir, ids, batch_size = 2, img_size =(512,512,3), augmentation = False, GridMask = False):
self.X = X_set
self.Y = Y_set
self.batch_size = batch_size
self.ids = ids
self.img_size = img_size
self.img_dir = img_dir
self.augmentation = augmentation
self.GridMask = GridMask
self.on_epoch_end()
self.x_indexed = X_set[self.ids]
def __len__(self):
return int(ceil(len(self.ids)/self.batch_size))
def __getitem__(self, index):
indices = self.ids[index*self.batch_size:(index+1)*self.batch_size]
X, Y = self.__generator__(indices)
 return X, Y
def on_epoch_end(self):
self.indices = np.arange(len(self.ids))
def __generator__(self, indices):
X = np.empty(( self.batch_size, *self.img_size))
Y = np.empty(( self.batch_size, 4))
for i, index in enumerate(indices):
ID = self.X[index]
image = _read(self.img_dir+ID+".jpg")
 image = cv2.resize(image,(512,512))
if self.augmentation == True:
aug = strong_aug(p=1.0)
 image = augment(aug, image)
 elif self.GridMask == True:
image = augment(transforms_train, image)
 X[i,] = image/255.
 Y[i,] = self.Y.loc[index].values
return X, Y
class TestDataGenerator(tf.keras.utils.Sequence):
def __init__(self, Test_set, img_dir, ids, batch_size = 3, img_size =(512,512,3), augmentation = None):
self.X = Test_set
self.img_dir = img_dir
self.ids = ids
self.batch_size = batch_size
self.img_size = img_size
self.augmentation = augmentation
def __len__(self):
return int(ceil(len(self.ids)/self.batch_size))
def __getitem__(self, index):
indices = self.ids[index*self.batch_size:(index+1)*self.batch_size]
X = self.__generator__(indices)
 return X
def __generator__(self, indices):
X = np.empty(( self.batch_size, *self.img_size))
for i, index in enumerate(indices):
ID = self.X[index]
image = _read(self.img_dir+ID+".jpg")
 image = cv2.resize(image,(512,512))
if self.augmentation is not None:
if self.augmentation == "HorizontalFlip":
image = augment(HorizontalFlip(p=1), image)
 elif self.augmentation == "VerticalFlip":
image = augment(VerticalFlip(p=1), image)
 elif self.augmentation == 'VH':
image = VH_augment(image)
 X[i,] = image/255.
 return X
def make_lag_features(strain):

 print('in dataframe:', strain.shape)
 print("headers:", strain.columns)
 lags = [7, 28]
lag_cols = ['lag_{}'.format(lag)for lag in lags ]
for lag, lag_col in zip(lags, lag_cols):
strain[lag_col] = strain[['item_id', 'sales']].groupby('item_id')['sales'].shift(lag)
 print('lag sales done')
 windows= [7, 28]
for window in windows:
for lag, lag_col in zip(lags, lag_cols):
window_col = f'item_id_rmean_{lag_col}_{window}'
strain[window_col] = strain[['item_id', lag_col]].groupby('item_id')[lag_col].transform(
 lambda x: x.rolling(window ).mean() 
)
 print(f'Rolling mean sales done for window {window} per item_id')
 def make_date_features(dt):
date_features = {
'week': 'weekofyear',
'quarter': 'quarter',
'mday': 'day',
"wday": "weekday",
"month": "month",
"year": "year"
}
for date_feat_name, date_feat_func in date_features.items() :
if date_feat_name in dt.columns:
dt[date_feat_name] = dt[date_feat_name].astype("int16")
 else:
dt[date_feat_name] = getattr(dt['date'].dt, date_feat_func ).astype('int16')
 print('date features done')
 dt['day'] = dt['day'].apply(lambda x: int(x.replace('d_', '')))
 print('out dataframe:', dt.shape )<categorify>
tgt_cols = ['healthy', 'multiple_diseases', 'rust' , 'scab']
train_df = train[tgt_cols]
Y = pd.get_dummies(train_df)
 X = train['image_id']
X_train, X_test, Y_train, Y_test = train_test_split(X,Y, test_size = 0.15, random_state = seed)
 ids_train = np.array(X_train.index)
 ids_test = X_test.index
del train
del X
del Y
model_all = lgb.Booster(model_file = PATH_MODELS[0])
 model_cat0 = lgb.Booster(model_file = PATH_MODELS[1])
 model_cat1 = lgb.Booster(model_file = PATH_MODELS[2])
 model_cat2 = lgb.Booster(model_file = PATH_MODELS[3])
 model_per = [lgb.Booster(model_file = PATH_MODELS[category])for category in range(1, 3+1)]<data_type_conversions>
model = create_model(input_shape)
 model.compile(optimizer = Adam(lr = 0.00016),
loss = 'categorical_crossentropy',
metrics = ['categorical_accuracy'] )
df = load_data(train=False)
 df["sale"] =(( df['sell_price'] * 100 % 10)< 6 ).astype('int8')
 make_lag_features(df)
 make_date_features(df )<feature_engineering>
best_w = ModelCheckpoint('plant_best.h5',
monitor='val_loss',
verbose=0,
save_best_only=True,
save_weights_only=True,
mode='auto',
period=1)
 last_w = ModelCheckpoint('plant_last.h5',
monitor='val_loss',
verbose=0,
save_best_only=False,
mode='auto',
period=1)
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
patience=2, min_lr=0.000001, verbose=1,cooldown=1)
 callbacks = [best_w, last_w, reduce_lr]
def lag_features_for_day(dt, day):
print(type(dt))
lags = [7, 28]
lag_cols = [f"lag_{lag}" for lag in lags]
for lag, lag_col in zip(lags, lag_cols):
dt.loc[dt['date'] == str(day), lag_col] = \
dt.loc[dt['date'] == str(day-timedelta(days=lag)) , 'sales'].values
windows = [7, 28]
for window in windows:
for lag, lag_col in zip(lags, lag_cols):
df_window = dt[(dt['date'] <= str(day-timedelta(days=lag)))&(dt['date'] > str(day-timedelta(days=lag+window)))]
df_window_grouped = df_window.groupby("item_id" ).agg({'sales':'mean'} ).reindex(dt.loc[dt['date']==str(day),'item_id'])
 dt.loc[dt['date'] == str(day),f'item_id_rmean_{lag_col}_{window}'] = df_window_grouped.sales.values
lag = 28
lag_col = 'lag_28'
window = 28
df_window = dt[(dt['date'] <= str(day-timedelta(days=lag)))&(dt['date'] > str(day-timedelta(days=lag+window)))]
df_window_grouped = df_window.groupby("store_id" ).agg({'sales':'mean'} ).reindex(dt.loc[dt['date']==str(day),'store_id'])
 dt.loc[dt['date'] == str(day),f'store_id_rmean_{lag_col}_{window}'] = df_window_grouped.sales.values
print(f"Features done for day {str(day)}")
<define_variables>
batch_size = 5
model.fit_generator(TrainDataGenerator(X_train,Y_train , img_dir, ids_train,
batch_size, augmentation = True, GridMask = True),
epochs=15,
verbose=1,
callbacks=callbacks,
validation_data=TrainDataGenerator(X_test, Y_test , img_dir, ids_test,
batch_size, augmentation = False, GridMask = False),
workers=1,
use_multiprocessing=False,
shuffle=True )
%%time
END_DATE = EVAL_SPLIT
PREDICT_DAYS = 28
for f_day in tqdm(range(1,PREDICT_DAYS+1)) :
pred_date =(datetime.strptime(END_DATE, '%Y-%m-%d')+ timedelta(days=f_day)).date()
print(f"Forecasting day {END_DAY+f_day}, date: {str(pred_date)}")
 pred_begin_date = pred_date - timedelta(days=BACKWARD_LAG+1)
 print(pred_begin_date)
 prediction_data = df[(df['date'] >= str(pred_begin_date)) &(df['date'] <= str(pred_date)) ].copy()
lag_features_for_day(prediction_data, pred_date)
 prediction_data = prediction_data.loc[prediction_data['date'] == str(pred_date), train_columns]
prediction_all_cats = model_all.predict(prediction_data)
 prediction_per_cat = pd.DataFrame(data=None, index=prediction_data.index, columns=["sales"])
 for category in range(3):
prediction_data_cat = prediction_data.loc[prediction_data["cat_id"] == category].drop("cat_id", axis = 1)
 prediction = model_per[category].predict(prediction_data_cat)
 prediction_per_cat.loc[prediction_per_cat.index.isin(prediction_data_cat.index), "sales"] = prediction
prediction_per_cat = prediction_per_cat['sales'].to_numpy(dtype='float64')
 prediction = np.average([prediction_all_cats, prediction_per_cat], axis = 0, weights=[1./2, 1./2])
 print("Prediction", prediction.size, prediction)
 df.loc[df['date'] == str(pred_date), 'sales'] = prediction<set_options>
gc.collect()
del prediction_data
gc.collect()<load_from_csv>
model.load_weights('plant_best.h5' )
sales_ = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sales_train_evaluation.csv'))
submission = pd.read_csv(os.path.join(SUBMISSION_PATH))<feature_engineering>
ids = test.index
Test_set = test['image_id']
data_generator_test = TestDataGenerator(Test_set, img_dir, ids, batch_size, input_shape, augmentation = None)
 data_generator_test_Horizontal = TestDataGenerator(Test_set, img_dir, ids, batch_size, input_shape, augmentation = "HorizontalFlip")
 data_generator_test_Vertical = TestDataGenerator(Test_set, img_dir, ids, batch_size, input_shape, augmentation = "VerticalFlip")
 data_generator_test_VH = TestDataGenerator(Test_set, img_dir, ids, batch_size, input_shape, augmentation = "VH")
 preds_1 = model.predict_generator(data_generator_test, verbose = 1)
 preds_2 = model.predict_generator(data_generator_test_Horizontal, verbose = 1)
 preds_3 = model.predict_generator(data_generator_test_Vertical, verbose = 1)
 preds_4 = model.predict_generator(data_generator_test_VH, verbose = 1)
 for index, pred in enumerate(zip(preds_1,preds_2, preds_3, preds_4)) :
sample_submission["healthy"][index] =(pred[0][0] + pred[1][0] + pred[2][0] + pred[3][0])/4
sample_submission["multiple_diseases"][index] =(pred[0][1] + pred[1][1] + pred[2][1] + pred[3][1])/4
sample_submission["rust"][index] =(pred[0][2] + pred[1][2] + pred[2][2] + pred[3][2])/4
sample_submission["scab"][index] =(pred[0][3] + pred[1][3] + pred[2][3] + pred[3][3])/4
<data_type_conversions><EOS> 
sample_submission.to_csv("submission.csv", index=False )