Kaggle competition: M5 Forecasting - Accuracy
<SOS> metric: M5_WRMSSE Kaggle data source: m5-forecasting-accuracy<load_from_csv>
warnings.filterwarnings('ignore' )
train = pd.read_csv(TRAIN_RESAMPLED_AUDIO_DIRS[0] / "train_mod.csv")
 if not TEST_AUDIO_DIR.exists() :
TEST_AUDIO_DIR = INPUT_ROOT / "birdcall-check" / "test_audio"
test = pd.read_csv(INPUT_ROOT / "birdcall-check" / "test.csv")
 else:
test = pd.read_csv(RAW_DATA / "test.csv" )<categorify>
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 def df_parallelize_run(func, t_split):
num_cores = np.min([N_CORES,len(t_split)])
 pool = Pool(num_cores)
 df = pd.concat(pool.map(func, t_split), axis=1)
 pool.close()
pool.join()
return df
class DropStripes(nn.Module):
def __init__(self, dim, drop_width, stripes_num):

 super(DropStripes, self ).__init__()
assert dim in [2, 3]
self.dim = dim
self.drop_width = drop_width
self.stripes_num = stripes_num
def forward(self, input):

 assert input.ndimension() == 4
if self.training is False:
return input
else:
batch_size = input.shape[0]
total_width = input.shape[self.dim]
for n in range(batch_size):
self.transform_slice(input[n], total_width)
 return input
def transform_slice(self, e, total_width):

 for _ in range(self.stripes_num):
distance = torch.randint(low=0, high=self.drop_width, size=(1,)) [0]
bgn = torch.randint(low=0, high=total_width - distance, size=(1,)) [0]
if self.dim == 2:
e[:, bgn : bgn + distance, :] = 0
elif self.dim == 3:
e[:, :, bgn : bgn + distance] = 0
class SpecAugmentation(nn.Module):
def __init__(self, time_drop_width, time_stripes_num, freq_drop_width,
freq_stripes_num):

 super(SpecAugmentation, self ).__init__()
self.time_dropper = DropStripes(dim=2, drop_width=time_drop_width,
stripes_num=time_stripes_num)
 self.freq_dropper = DropStripes(dim=3, drop_width=freq_drop_width,
stripes_num=freq_stripes_num)
 def forward(self, input):
x = self.time_dropper(input)
 x = self.freq_dropper(x)
 return x<train_model>
def get_data_by_store(store):
df = pd.concat([pd.read_pickle(BASE),
pd.read_pickle(PRICE ).iloc[:,2:],
pd.read_pickle(CALENDAR ).iloc[:,2:]],
axis=1)
 df = df[df['store_id']==store]
df2 = pd.read_pickle(MEAN_ENC)[mean_features]
df2 = df2[df2.index.isin(df.index)]
df3 = pd.read_pickle(LAGS ).iloc[:,3:]
df3 = df3[df3.index.isin(df.index)]
df = pd.concat([df, df2], axis=1)
 del df2
df = pd.concat([df, df3], axis=1)
 del df3
features = [col for col in list(df)if col not in remove_features]
df = df[['id','d',TARGET]+features]
df = df[df['d']>=START_TRAIN].reset_index(drop=True)
 return df, features
def get_base_test() :
base_test = pd.DataFrame()
for store_id in STORES_IDS:
temp_df = pd.read_pickle(AUX_MODELS+'test_'+store_id+'.pkl')
 temp_df['store_id'] = store_id
base_test = pd.concat([base_test, temp_df] ).reset_index(drop=True)
 return base_test
def make_lag(LAG_DAY):
lag_df = base_test[['id','d',TARGET]]
col_name = 'sales_lag_'+str(LAG_DAY)
 lag_df[col_name] = lag_df.groupby(['id'])[TARGET].transform(lambda x: x.shift(LAG_DAY)).astype(np.float16)
 return lag_df[[col_name]]
def make_lag_roll(LAG_DAY):
shift_day = LAG_DAY[0]
roll_wind = LAG_DAY[1]
lag_df = base_test[['id','d',TARGET]]
col_name = 'rolling_mean_tmp_'+str(shift_day)+'_'+str(roll_wind)
 lag_df[col_name] = lag_df.groupby(['id'])[TARGET].transform(lambda x: x.shift(shift_day ).rolling(roll_wind ).mean())
 return lag_df[[col_name]]
def init_layer(layer):
nn.init.xavier_uniform_(layer.weight)
 if hasattr(layer, "bias"):
if layer.bias is not None:
layer.bias.data.fill_(0.)
def init_bn(bn):
bn.bias.data.fill_(0.)
bn.weight.data.fill_(1.0)
 def interpolate(x: torch.Tensor, ratio: int):

(batch_size, time_steps, classes_num)= x.shape
upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)
 upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)
 return upsampled
def pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):

 pad = framewise_output[:, -1:, :].repeat(
 1, frames_num - framewise_output.shape[1], 1)

output = torch.cat(( framewise_output, pad), dim=1)

return output
class ConvBlock(nn.Module):
def __init__(self, in_channels: int, out_channels: int):
super().__init__()
self.conv1 = nn.Conv2d(
 in_channels=in_channels,
out_channels=out_channels,
kernel_size=(3, 3),
stride=(1, 1),
padding=(1, 1),
bias=False)
 self.conv2 = nn.Conv2d(
 in_channels=out_channels,
out_channels=out_channels,
kernel_size=(3, 3),
stride=(1, 1),
padding=(1, 1),
bias=False)
 self.bn1 = nn.BatchNorm2d(out_channels)
 self.bn2 = nn.BatchNorm2d(out_channels)
 self.init_weight()
def init_weight(self):
init_layer(self.conv1)
 init_layer(self.conv2)
 init_bn(self.bn1)
 init_bn(self.bn2)
 def forward(self, input, pool_size=(2, 2), pool_type='avg'):
x = input
x = F.relu_(self.bn1(self.conv1(x)))
 x = F.relu_(self.bn2(self.conv2(x)))
 if pool_type == 'max':
x = F.max_pool2d(x, kernel_size=pool_size)
 elif pool_type == 'avg':
x = F.avg_pool2d(x, kernel_size=pool_size)
 elif pool_type == 'avg+max':
x1 = F.avg_pool2d(x, kernel_size=pool_size)
 x2 = F.max_pool2d(x, kernel_size=pool_size)
 x = x1 + x2
else:
raise Exception('Incorrect argument!')
 return x
class AttBlock(nn.Module):
def __init__(self,
in_features: int,
out_features: int,
activation="linear",
temperature=1.0):
super().__init__()
self.activation = activation
self.temperature = temperature
self.att = nn.Conv1d(
 in_channels=in_features,
out_channels=out_features,
kernel_size=1,
stride=1,
padding=0,
bias=True)
 self.cla = nn.Conv1d(
 in_channels=in_features,
out_channels=out_features,
kernel_size=1,
stride=1,
padding=0,
bias=True)
 self.bn_att = nn.BatchNorm1d(out_features)
 self.init_weights()
def init_weights(self):
init_layer(self.att)
 init_layer(self.cla)
 init_bn(self.bn_att)
 def forward(self, x):
norm_att = torch.softmax(torch.clamp(self.att(x), -10, 10), dim=-1)
 cla = self.nonlinear_transform(self.cla(x))
x = torch.sum(norm_att * cla, dim=2)
 return x, norm_att, cla
def nonlinear_transform(self, x):
if self.activation == 'linear':
return x
elif self.activation == 'sigmoid':
return torch.sigmoid(x )<set_options>
lgb_params = {
'boosting_type': 'gbdt',
'objective': 'tweedie',
'tweedie_variance_power': 1.1,
'metric': 'rmse',
'subsample': 0.5,
'subsample_freq': 1,
'learning_rate': 0.03,
'num_leaves': 2**11-1,
'min_data_in_leaf': 2**12-1,
'feature_fraction': 0.5,
'max_bin': 100,
'n_estimators': 1400,
'boost_from_average': False,
'verbose': 1,
}
class PANNsCNN14Att(nn.Module):
def __init__(self, sample_rate: int, window_size: int, hop_size: int,
mel_bins: int, fmin: int, fmax: int, classes_num: int):
super().__init__()
window = 'hann'
center = True
pad_mode = 'reflect'
ref = 1.0
amin = 1e-10
top_db = None
self.interpolate_ratio = 32
self.spectrogram_extractor = Spectrogram(
 n_fft=window_size,
hop_length=hop_size,
win_length=window_size,
window=window,
center=center,
pad_mode=pad_mode,
freeze_parameters=True)
 self.logmel_extractor = LogmelFilterBank(
 sr=sample_rate,
n_fft=window_size,
n_mels=mel_bins,
fmin=fmin,
fmax=fmax,
ref=ref,
amin=amin,
top_db=top_db,
freeze_parameters=True)
 self.spec_augmenter = SpecAugmentation(
 time_drop_width=64,
time_stripes_num=2,
freq_drop_width=8,
freq_stripes_num=2)
 self.bn0 = nn.BatchNorm2d(mel_bins)
 self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)
 self.conv_block2 = ConvBlock(in_channels=64, out_channels=128)
 self.conv_block3 = ConvBlock(in_channels=128, out_channels=256)
 self.conv_block4 = ConvBlock(in_channels=256, out_channels=512)
 self.conv_block5 = ConvBlock(in_channels=512, out_channels=1024)
 self.conv_block6 = ConvBlock(in_channels=1024, out_channels=2048)
 self.fc1 = nn.Linear(2048, 2048, bias=True)
 self.att_block = AttBlock(2048, classes_num, activation='sigmoid')
 self.init_weight()
def init_weight(self):
init_bn(self.bn0)
 init_layer(self.fc1)
 def cnn_feature_extractor(self, x):
x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training)
 x = self.conv_block2(x, pool_size=(2, 2), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training)
 x = self.conv_block3(x, pool_size=(2, 2), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training)
 x = self.conv_block4(x, pool_size=(2, 2), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training)
 x = self.conv_block5(x, pool_size=(2, 2), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training)
 x = self.conv_block6(x, pool_size=(1, 1), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training)
 return x
def preprocess(self, input, mixup_lambda=None):
x = self.spectrogram_extractor(input)
 x = self.logmel_extractor(x)
 frames_num = x.shape[2]
x = x.transpose(1, 3)
 x = self.bn0(x)
 x = x.transpose(1, 3)
 if self.training:
x = self.spec_augmenter(x)
 if self.training and mixup_lambda is not None:
x = do_mixup(x, mixup_lambda)
 return x, frames_num
def forward(self, input, mixup_lambda=None):

 x, frames_num = self.preprocess(input, mixup_lambda=mixup_lambda)
 x = self.cnn_feature_extractor(x)
 x = torch.mean(x, dim=3)
 x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)
 x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)
 x = x1 + x2
x = F.dropout(x, p=0.5, training=self.training)
 x = x.transpose(1, 2)
 x = F.relu_(self.fc1(x))
x = x.transpose(1, 2)
 x = F.dropout(x, p=0.5, training=self.training)
(clipwise_output, norm_att, segmentwise_output)= self.att_block(x)
 segmentwise_output = segmentwise_output.transpose(1, 2)
 framewise_output = interpolate(segmentwise_output,
self.interpolate_ratio)
 framewise_output = pad_framewise_output(framewise_output, frames_num)
 output_dict = {
'framewise_output': framewise_output,
'clipwise_output': clipwise_output
}
return output_dict<load_pretrained>
SEED = 42
seed_everything(SEED)
 lgb_params['seed'] = SEED
N_CORES = psutil.cpu_count()
TARGET = 'sales'
START_TRAIN = 0
END_TRAIN = 1913
P_HORIZON = 28
USE_AUX = True
remove_features = ['id','state_id','store_id',
'date','wm_yr_wk','d',TARGET]
mean_features = ['enc_cat_id_mean','enc_cat_id_std',
'enc_dept_id_mean','enc_dept_id_std',
'enc_item_id_mean','enc_item_id_std']
ORIGINAL = '.. /input/m5-forecasting-accuracy/'
BASE = '.. /input/m5-simple-fe/grid_part_1.pkl'
PRICE = '.. /input/m5-simple-fe/grid_part_2.pkl'
CALENDAR = '.. /input/m5-simple-fe/grid_part_3.pkl'
LAGS = '.. /input/m5-lags-features/lags_df_28.pkl'
MEAN_ENC = '.. /input/m5-custom-features/mean_encoding_df.pkl'
AUX_MODELS = '.. /input/m5groupkfold/'
STORES_IDS = ['CA_1']
CV_FOLDS = [0,1,2]
SHIFT_DAY = 28
N_LAGS = 15
LAGS_SPLIT = [col for col in range(SHIFT_DAY,SHIFT_DAY+N_LAGS)]
ROLS_SPLIT = []
for i in [1,7,14]:
for j in [7,14,30,60]:
ROLS_SPLIT.append([i,j] )
SR = 32000
y, _ = librosa.load(TRAIN_RESAMPLED_AUDIO_DIRS[0] / "aldfly" / "XC134874.wav",
sr=SR,
res_type="kaiser_fast",
mono=True)
 Audio(y, rate=SR )<choose_model_class>
for store_id in STORES_IDS:
print('Train', store_id)
 grid_df, features_columns = get_data_by_store(store_id)
 train_mask = grid_df['d']<=END_TRAIN
preds_mask = grid_df['d']>(END_TRAIN-100)
 folds = GroupKFold(n_splits=3)
 grid_df['groups'] = grid_df['tm_w'].astype(str)+ '_' + grid_df['tm_y'].astype(str)
 split_groups = grid_df[train_mask]['groups']
X,y = grid_df[train_mask][features_columns], grid_df[train_mask][TARGET]
grid_df = grid_df[preds_mask].reset_index(drop=True)
 keep_cols = [col for col in list(grid_df)if '_tmp_' not in col]
grid_df = grid_df[keep_cols]
grid_df.to_pickle('test_'+store_id+'.pkl')
 del grid_df
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X, y, groups=split_groups)) :
print('Fold:',fold_)
 print(len(trn_idx),len(val_idx))
tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]
v_X, v_y = X.iloc[val_idx,:], y[val_idx]
train_data = lgb.Dataset(tr_x, label=tr_y)
 valid_data = lgb.Dataset(v_X, label=v_y)
 seed_everything(SEED)
 estimator = lgb.train(
 lgb_params,
train_data,
valid_sets = [train_data, valid_data],
verbose_eval = 100 
)
 model_name = 'lgb_model_'+STORES_IDS[0]+'_'+str(fold_)+'.bin'
pickle.dump(estimator, open(model_name, 'wb'))
del train_data, valid_data, estimator
gc.collect()
MODEL_FEATURES = features_columns
model_config = {
"sample_rate": 32000,
"window_size": 1024,
"hop_size": 320,
"mel_bins": 64,
"fmin": 50,
"fmax": 14000,
"classes_num": 264
}
model = PANNsCNN14Att(**model_config )<predict_on_test>
for fold_ in CV_FOLDS:
print("FOLD:", fold_)
 all_preds = pd.DataFrame()
base_test = get_base_test()
main_time = time.time()
for PREDICT_DAY in range(1,29):
print('Predict | Day:', PREDICT_DAY)
 start_time = time.time()
grid_df = base_test.copy()
grid_df = pd.concat([grid_df, df_parallelize_run(make_lag_roll, ROLS_SPLIT)], axis=1)
 for store_id in STORES_IDS:
model_path = 'lgb_model_'+store_id+'_'+str(fold_)+'.bin'
if USE_AUX:
model_path = AUX_MODELS + model_path
estimator = pickle.load(open(model_path, 'rb'))
day_mask = base_test['d']==(END_TRAIN+PREDICT_DAY)
 store_mask = base_test['store_id']==store_id
mask =(day_mask)&(store_mask)
 base_test[TARGET][mask] = estimator.predict(grid_df[mask][MODEL_FEATURES])
 temp_df = base_test[day_mask][['id',TARGET]]
temp_df.columns = ['id','F'+str(PREDICT_DAY)]
if 'id' in list(all_preds):
all_preds = all_preds.merge(temp_df, on=['id'], how='left')
 else:
all_preds = temp_df.copy()
all_preds = all_preds.reset_index(drop=True)
 print('
' %0.2f min total |' %(( time.time() - main_time)/ 60),
' %0.2f day sales |' %(temp_df['F'+str(PREDICT_DAY)].sum()))
all_preds.to_csv('all_preds_'+str(fold_)+'.csv',index=False)
 del temp_df, all_preds
melspec_numpy = melspec.detach().numpy() [0, 0].transpose(1, 0)
 display.specshow(melspec_numpy, sr=SR, y_axis="mel");<define_variables>
all_preds_0=pd.read_csv(AUX_MODELS+'all_preds_0'+'.csv')
 all_preds_1=pd.read_csv(AUX_MODELS+'all_preds_1'+'.csv')
 all_preds_2=pd.read_csv(AUX_MODELS+'all_preds_2'+'.csv' )
BIRD_CODE = {
'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,
'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,
'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,
'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,
'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,
'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,
'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,
'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,
'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,
'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,
'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,
'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,
'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,
'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,
'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,
'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,
'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,
'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,
'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,
'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,
'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,
'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,
'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,
'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,
'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,
'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,
'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,
'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,
'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,
'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,
'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,
'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,
'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,
'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,
'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,
'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,
'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,
'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,
'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,
'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,
'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,
'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,
'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,
'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,
'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,
'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,
'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,
'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,
'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,
'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,
'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,
'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,
'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263
}
INV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items() }<prepare_x_and_y>
final_all_preds = pd.DataFrame()
final_all_preds['id'] = all_preds_1['id']
for item in all_preds_1:
if item!='id':
final_all_preds[item]=(all_preds_0[item]*(1/3)) +(all_preds_1[item]*(1/3)) +(all_preds_2[item]*(1/3))
final_all_preds
<normalization><EOS> 
submission = pd.read_csv(ORIGINAL+'sample_submission.csv')[['id']]
submission = submission.merge(final_all_preds, on=['id'], how='left' ).fillna(0)
 submission.to_csv('submission'+'.csv', index=False )