Kaggle competition: Plant Pathology 2020 - FGVC7
from typing import Union
import numpy as np
import pandas as pd
from tqdm.notebook import tqdm_notebook as tqdm 
<feature_engineering>
sub1 = pd.read_csv('.. /input/average-efficientnet/submission.csv')
 sub1.head()
class WRMSSEEvaluator(object):
def __init__(self, train_df: pd.DataFrame, valid_df: pd.DataFrame, calendar: pd.DataFrame, prices: pd.DataFrame):
train_y = train_df.loc[:, train_df.columns.str.startswith('d_')]
train_target_columns = train_y.columns.tolist()
weight_columns = train_y.iloc[:, -28:].columns.tolist()
train_df['all_id'] = 0
id_columns = train_df.loc[:, ~train_df.columns.str.startswith('d_')].columns.tolist()
valid_target_columns = valid_df.loc[:, valid_df.columns.str.startswith('d_')].columns.tolist()
if not all([c in valid_df.columns for c in id_columns]):
valid_df = pd.concat([train_df[id_columns], valid_df], axis=1, sort=False)
 self.train_df = train_df
self.valid_df = valid_df
self.calendar = calendar
self.prices = prices
self.weight_columns = weight_columns
self.id_columns = id_columns
self.valid_target_columns = valid_target_columns
weight_df = self.get_weight_df()
self.group_ids =(
 'all_id',
'state_id',
'store_id',
'cat_id',
'dept_id',
['state_id', 'cat_id'],
['state_id', 'dept_id'],
['store_id', 'cat_id'],
['store_id', 'dept_id'],
'item_id',
['item_id', 'state_id'],
['item_id', 'store_id'] 
)
 for i, group_id in enumerate(tqdm(self.group_ids)) :
train_y = train_df.groupby(group_id)[train_target_columns].sum()
scale = []
for _, row in train_y.iterrows() :
series = row.values[np.argmax(row.values != 0):]
scale.append(((series[1:] - series[:-1])** 2 ).mean())
 setattr(self, f'lv{i + 1}_scale', np.array(scale))
setattr(self, f'lv{i + 1}_train_df', train_y)
 setattr(self, f'lv{i + 1}_valid_df', valid_df.groupby(group_id)[valid_target_columns].sum())
 lv_weight = weight_df.groupby(group_id)[weight_columns].sum().sum(axis=1)
 setattr(self, f'lv{i + 1}_weight', lv_weight / lv_weight.sum())
 def get_weight_df(self)-> pd.DataFrame:
day_to_week = self.calendar.set_index('d')['wm_yr_wk'].to_dict()
weight_df = self.train_df[['item_id', 'store_id'] + self.weight_columns].set_index(['item_id', 'store_id'])
 weight_df = weight_df.stack().reset_index().rename(columns={'level_2': 'd', 0: 'value'})
 weight_df['wm_yr_wk'] = weight_df['d'].map(day_to_week)
 weight_df = weight_df.merge(self.prices, how='left', on=['item_id', 'store_id', 'wm_yr_wk'])
 weight_df['value'] = weight_df['value'] * weight_df['sell_price']
weight_df = weight_df.set_index(['item_id', 'store_id', 'd'] ).unstack(level=2)['value']
weight_df = weight_df.loc[zip(self.train_df.item_id, self.train_df.store_id), :].reset_index(drop=True)
 weight_df = pd.concat([self.train_df[self.id_columns], weight_df], axis=1, sort=False)
 return weight_df
def rmsse(self, valid_preds: pd.DataFrame, lv: int)-> pd.Series:
valid_y = getattr(self, f'lv{lv}_valid_df')
 score =(( valid_y - valid_preds)** 2 ).mean(axis=1)
 scale = getattr(self, f'lv{lv}_scale')
 return(score / scale ).map(np.sqrt)
 def score(self, valid_preds: Union[pd.DataFrame, np.ndarray])-> float:
assert self.valid_df[self.valid_target_columns].shape == valid_preds.shape
if isinstance(valid_preds, np.ndarray):
valid_preds = pd.DataFrame(valid_preds, columns=self.valid_target_columns)
 valid_preds = pd.concat([self.valid_df[self.id_columns], valid_preds], axis=1, sort=False)
 all_scores = []
for i, group_id in enumerate(self.group_ids):
lv_scores = self.rmsse(valid_preds.groupby(group_id)[self.valid_target_columns].sum() , i + 1)
 weight = getattr(self, f'lv{i + 1}_weight')
 lv_scores = pd.concat([weight, lv_scores], axis=1, sort=False ).prod(axis=1)
 all_scores.append(lv_scores.sum())
 return np.mean(all_scores)
 df_train_full = pd.read_csv(".. /input/m5-forecasting-accuracy/sales_train_evaluation.csv")
 df_calendar = pd.read_csv(".. /input/m5-forecasting-accuracy/calendar.csv")
 df_prices = pd.read_csv(".. /input/m5-forecasting-accuracy/sell_prices.csv")
 df_train = df_train_full.iloc[:, :-28]
df_valid = df_train_full.iloc[:, -28:]
evaluator = WRMSSEEvaluator(df_train, df_valid, df_calendar, df_prices)
 def WRMSSEE(sub):
val_preds = sample_sub[['id']].merge(sub, on = 'id')
 val_preds.columns = ['id'] + list(df_valid.columns)
 valid_preds = val_preds.iloc[:30490, -28:]
return evaluator.score(valid_preds )<categorify>
sub2 = pd.read_csv('.. /input/classification-densenet201-efficientnetb7/submission.csv')
 sub2.head()
def function(solution):
solution = np.abs(solution)
 submission = OperateBaseModels(submission0,submission1, a=solution[0], b=solution[1])
 submission = OperateBaseModels(submission,submission2, a=1, b=solution[2])
 submission = OperateBaseModels(submission,submission3, a=1, b=solution[3])
 submission = OperateBaseModels(submission,submission4, a=1, b=solution[4])
 error = WRMSSEE(submission)
 return error<merge>
sub3 = pd.read_csv('.. /input/tf-zoo-models-on-tpu/submission.csv')
 sub3.head()
diference = validation.merge(submission,how='left')
 error = np.mean(np.mean(np.abs(perfect_sub._get_numeric_data() -submission._get_numeric_data())) )<save_to_csv>
sub4 = pd.read_csv('.. /input/fork-of-plant-2020-tpu-915e9c/submission.csv')
 sub4.head()
submission.to_csv("submission.csv", index=False)
 submission<set_options>
sub = pd.read_csv('.. /input/plant-pathology-2020-fgvc7/sample_submission.csv')
 sub.head()
warnings.filterwarnings(action='once' )<load_from_csv>
sub.healthy =(sub1.healthy + sub2.healthy + sub3.healthy + sub4.healthy)/4
calendar = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/calendar.csv', parse_dates=['date'], index_col='date')
 calendar.head(5 )<categorify>
sub.multiple_diseases =(sub1.multiple_diseases + sub2.multiple_diseases + sub3.multiple_diseases + sub4.multiple_diseases)/4
calendar.fillna('Regular', inplace=True)
 label_encoder = LabelEncoder()
label_cols = ['event_name_1', 'event_type_1', 'event_name_2', 'event_type_2']
for col in label_cols:
calendar[col] = label_encoder.fit_transform(calendar[col] )<feature_engineering>
sub.rust =(sub1.rust + sub2.rust + sub3.rust + sub4.rust)/4
calendar['is_weekend'] = calendar['wday'].apply(lambda x: 1 if x == 1 or x == 2 else 0)
 seasons = {1: 1, 2: 1, 12: 1, 3: 2, 4: 2, 5: 2, 6: 3, 7: 3, 8: 3, 9: 4, 10: 4, 11: 4 }
calendar['season'] = calendar['month'].apply(lambda x: seasons[x])
 calendar.head()<load_from_csv>
sub.to_csv('submission.csv', index=False )
<data_type_conversions><EOS> 
sub.to_csv('submission.csv', index=False )