Kaggle competition: M5 Forecasting - Accuracy
<SOS> metric: M5_WRMSSE Kaggle data source: m5-forecasting-accuracy<load_pretrained>
from datetime import datetime, timedelta
import gc
import numpy as np, pandas as pd
import lightgbm as lgb
class F1Callback(Callback):
def __init__(self,
input_key: str = "targets",
output_key: str = "logits",
model_output_key: str = "clipwise_output",
prefix: str = "f1"):
super().__init__(CallbackOrder.Metric)
 self.input_key = input_key
self.output_key = output_key
self.model_output_key = model_output_key
self.prefix = prefix
def on_loader_start(self, state: State):
self.prediction: List[np.ndarray] = []
self.target: List[np.ndarray] = []
def on_batch_end(self, state: State):
targ = state.input[self.input_key].detach().cpu().numpy()
out = state.output[self.output_key]
clipwise_output = out[self.model_output_key].detach().cpu().numpy()
self.prediction.append(clipwise_output)
 self.target.append(targ)
 y_pred = clipwise_output.argmax(axis=1)
 y_true = targ.argmax(axis=1)
 score = f1_score(y_true, y_pred, average="macro")
 state.batch_metrics[self.prefix] = score
def on_loader_end(self, state: State):
y_pred = np.concatenate(self.prediction, axis=0 ).argmax(axis=1)
 y_true = np.concatenate(self.target, axis=0 ).argmax(axis=1)
 score = f1_score(y_true, y_pred, average="macro")
 state.loader_metrics[self.prefix] = score
if state.is_valid_loader:
state.epoch_metrics[state.valid_loader + "_epoch_" +
self.prefix] = score
else:
state.epoch_metrics["train_epoch_" + self.prefix] = score
class mAPCallback(Callback):
def __init__(self,
input_key: str = "targets",
output_key: str = "logits",
model_output_key: str = "clipwise_output",
prefix: str = "mAP"):
super().__init__(CallbackOrder.Metric)
 self.input_key = input_key
self.output_key = output_key
self.model_output_key = model_output_key
self.prefix = prefix
def on_loader_start(self, state: State):
self.prediction: List[np.ndarray] = []
self.target: List[np.ndarray] = []
def on_batch_end(self, state: State):
targ = state.input[self.input_key].detach().cpu().numpy()
out = state.output[self.output_key]
clipwise_output = out[self.model_output_key].detach().cpu().numpy()
self.prediction.append(clipwise_output)
 self.target.append(targ)
 score = average_precision_score(targ, clipwise_output, average=None)
 score = np.nan_to_num(score ).mean()
state.batch_metrics[self.prefix] = score
def on_loader_end(self, state: State):
y_pred = np.concatenate(self.prediction, axis=0)
 y_true = np.concatenate(self.target, axis=0)
 score = average_precision_score(y_true, y_pred, average=None)
 score = np.nan_to_num(score ).mean()
state.loader_metrics[self.prefix] = score
if state.is_valid_loader:
state.epoch_metrics[state.valid_loader + "_epoch_" +
self.prefix] = score
else:
state.epoch_metrics["train_epoch_" + self.prefix] = score<merge>
CAL_DTYPES={"event_name_1": "category", "event_name_2": "category", "event_type_1": "category",
"event_type_2": "category", "weekday": "category", 'wm_yr_wk': 'int16', "wday": "int16",
"month": "int16", "year": "int16", "snap_CA": "float32", 'snap_TX': 'float32', 'snap_WI': 'float32' }
PRICE_DTYPES = {"store_id": "category", "item_id": "category", "wm_yr_wk": "int16","sell_price":"float32" }
tmp_list = []
for audio_d in TRAIN_RESAMPLED_AUDIO_DIRS:
if not audio_d.exists() :
continue
for ebird_d in audio_d.iterdir() :
if ebird_d.is_file() :
continue
for wav_f in ebird_d.iterdir() :
tmp_list.append([ebird_d.name, wav_f.name, wav_f.as_posix() ])
 train_wav_path_exist = pd.DataFrame(
 tmp_list, columns=["ebird_code", "resampled_filename", "file_path"])
 del tmp_list
train_all = pd.merge(
 train, train_wav_path_exist, on=["ebird_code", "resampled_filename"], how="inner")
 print(train.shape)
 print(train_wav_path_exist.shape)
 print(train_all.shape )<feature_engineering>
pd.options.display.max_columns = 50
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 train_all["fold"] = -1
for fold_id,(train_index, val_index)in enumerate(skf.split(train_all, train_all["ebird_code"])) :
train_all.iloc[val_index, -1] = fold_id
fold_proportion = pd.pivot_table(train_all, index="ebird_code", columns="fold", values="xc_id", aggfunc=len)
 print(fold_proportion.shape )<data_type_conversions>
h = 28
max_lags = 70
tr_last = 1913
fday = datetime(2016,4, 25)
 fday
use_fold = 0
train_file_list = train_all.query("fold != @use_fold")[["file_path", "ebird_code"]].values.tolist()
val_file_list = train_all.query("fold == @use_fold")[["file_path", "ebird_code"]].values.tolist()
print("[fold {}] train: {}, val: {}".format(use_fold, len(train_file_list), len(val_file_list)) )<create_dataframe>
def create_dt(is_train = True, nrows = None, first_day = 1200):
prices = pd.read_csv(".. /input/m5-forecasting-accuracy/sell_prices.csv", dtype = PRICE_DTYPES)
 for col, col_dtype in PRICE_DTYPES.items() :
if col_dtype == "category":
prices[col] = prices[col].cat.codes.astype("int16")
 prices[col] -= prices[col].min()
cal = pd.read_csv(".. /input/m5-forecasting-accuracy/calendar.csv", dtype = CAL_DTYPES)
 cal["date"] = pd.to_datetime(cal["date"])
 for col, col_dtype in CAL_DTYPES.items() :
if col_dtype == "category":
cal[col] = cal[col].cat.codes.astype("int16")
 cal[col] -= cal[col].min()
start_day = max(1 if is_train else tr_last-max_lags, first_day)
 numcols = [f"d_{day}" for day in range(start_day,tr_last+1)]
catcols = ['id', 'item_id', 'dept_id','store_id', 'cat_id', 'state_id']
dtype = {numcol:"float32" for numcol in numcols}
dtype.update({col: "category" for col in catcols if col != "id"})
 dt = pd.read_csv(".. /input/m5-forecasting-accuracy/sales_train_validation.csv",
nrows = nrows, usecols = catcols + numcols, dtype = dtype)
 for col in catcols:
if col != "id":
dt[col] = dt[col].cat.codes.astype("int16")
 dt[col] -= dt[col].min()
if not is_train:
for day in range(tr_last+1, tr_last+ 28 +1):
dt[f"d_{day}"] = np.nan
dt = pd.melt(dt,
id_vars = catcols,
value_vars = [col for col in dt.columns if col.startswith("d_")],
var_name = "d",
value_name = "sales")
 dt = dt.merge(cal, on= "d", copy = False)
 dt = dt.merge(prices, on = ["store_id", "item_id", "wm_yr_wk"], copy = False)
 return dt
device = torch.device("cuda:0")
 loaders = {
"train": data.DataLoader(PANNsDataset(train_file_list, None),
batch_size=64,
shuffle=True,
num_workers=2,
pin_memory=True,
drop_last=True),
"valid": data.DataLoader(PANNsDataset(val_file_list, None),
batch_size=64,
shuffle=False,
num_workers=2,
pin_memory=True,
drop_last=False)
 }
model_config["classes_num"] = 527
model = PANNsCNN14Att(**model_config)
 weights = torch.load(".. /input/pannscnn14-decisionlevelatt-weight/Cnn14_DecisionLevelAtt_mAP0.425.pth")
 model.load_state_dict(weights["model"])
 model.att_block = AttBlock(2048, 264, activation='sigmoid')
 model.att_block.init_weights()
model.to(device)
 optimizer = optim.Adam(model.parameters() , lr=0.001)
 scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10)
 criterion = PANNsLoss().to(device)
 callbacks = [
F1Callback(input_key="targets", output_key="logits", prefix="f1"),
mAPCallback(input_key="targets", output_key="logits", prefix="mAP"),
CheckpointCallback(save_n_best=0)
 ]<train_model>
def create_fea(dt):
lags = [7, 28]
lag_cols = [f"lag_{lag}" for lag in lags ]
for lag, lag_col in zip(lags, lag_cols):
dt[lag_col] = dt[["id","sales"]].groupby("id")["sales"].shift(lag)
 wins = [7, 28]
for win in wins :
for lag,lag_col in zip(lags, lag_cols):
dt[f"rmean_{lag}_{win}"] = dt[["id", lag_col]].groupby("id")[lag_col].transform(lambda x : x.rolling(win ).mean())
 date_features = {
"wday": "weekday",
"week": "weekofyear",
"month": "month",
"quarter": "quarter",
"year": "year",
"mday": "day",
}
for date_feat_name, date_feat_func in date_features.items() :
if date_feat_name in dt.columns:
dt[date_feat_name] = dt[date_feat_name].astype("int16")
 else:
dt[date_feat_name] = getattr(dt["date"].dt, date_feat_func ).astype("int16" )
warnings.simplefilter("ignore")
 runner = SupervisedRunner(
 device=device,
input_key="waveform",
input_target_key="targets")
 runner.train(
 model=model,
criterion=criterion,
loaders=loaders,
optimizer=optimizer,
scheduler=scheduler,
num_epochs=10,
verbose=True,
logdir=f"fold0",
callbacks=callbacks,
main_metric="epoch_f1",
minimize_metric=False )<define_variables>
FIRST_DAY = 800
model_config = {
"sample_rate": 32000,
"window_size": 1024,
"hop_size": 320,
"mel_bins": 64,
"fmin": 50,
"fmax": 14000,
"classes_num": 264
}
weights_path = ".. /input/birdcall-pannsatt-aux-weak/best.pth"<load_pretrained>
df.dropna(inplace = True)
 df.shape
def get_model(config: dict, weights_path: str):
model = PANNsCNN14Att(**config)
 checkpoint = torch.load(weights_path)
 model.load_state_dict(checkpoint["model_state_dict"])
 device = torch.device("cuda")
 model.to(device)
 model.eval()
return model<categorify>
cat_feats = ['item_id', 'dept_id','store_id', 'cat_id', 'state_id'] + ["event_name_1", "event_name_2", "event_type_1", "event_type_2"]
useless_cols = ["id", "date", "sales","d", "wm_yr_wk", "weekday"]
train_cols = df.columns[~df.columns.isin(useless_cols)]
X_train = df[train_cols]
y_train = df["sales"]
def prediction_for_clip(test_df: pd.DataFrame,
clip: np.ndarray,
model: PANNsCNN14Att,
threshold=0.5):
PERIOD = 30
audios = []
y = clip.astype(np.float32)
 len_y = len(y)
 start = 0
end = PERIOD * SR
while True:
y_batch = y[start:end].astype(np.float32)
 if len(y_batch)!= PERIOD * SR:
y_pad = np.zeros(PERIOD * SR, dtype=np.float32)
 y_pad[:len(y_batch)] = y_batch
audios.append(y_pad)
 break
start = end
end += PERIOD * SR
audios.append(y_batch)
 array = np.asarray(audios)
 tensors = torch.from_numpy(array)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model.eval()
estimated_event_list = []
global_time = 0.0
site = test_df["site"].values[0]
audio_id = test_df["audio_id"].values[0]
for image in progress_bar(tensors):
image = image.view(1, image.size(0))
image = image.to(device)
 with torch.no_grad() :
prediction = model(image)
 framewise_outputs = prediction["framewise_output"].detach(
 ).cpu().numpy() [0]
thresholded = framewise_outputs >= threshold
for target_idx in range(thresholded.shape[1]):
if thresholded[:, target_idx].mean() == 0:
pass
else:
detected = np.argwhere(thresholded[:, target_idx] ).reshape(-1)
 head_idx = 0
tail_idx = 0
while True:
if(tail_idx + 1 == len(detected)) or(
 detected[tail_idx + 1] -
detected[tail_idx] != 1):
onset = 0.01 * detected[
head_idx] + global_time
offset = 0.01 * detected[
tail_idx] + global_time
onset_idx = detected[head_idx]
offset_idx = detected[tail_idx]
max_confidence = framewise_outputs[
onset_idx:offset_idx, target_idx].max()
mean_confidence = framewise_outputs[
onset_idx:offset_idx, target_idx].mean()
estimated_event = {
"site": site,
"audio_id": audio_id,
"ebird_code": INV_BIRD_CODE[target_idx],
"onset": onset,
"offset": offset,
"max_confidence": max_confidence,
"mean_confidence": mean_confidence
}
estimated_event_list.append(estimated_event)
 head_idx = tail_idx + 1
tail_idx = tail_idx + 1
if head_idx >= len(detected):
break
else:
tail_idx += 1
global_time += PERIOD
prediction_df = pd.DataFrame(estimated_event_list)
 return prediction_df<load_pretrained>
train_data = lgb.Dataset(X_train, label = y_train, categorical_feature=cat_feats, free_raw_data=False)
 fake_valid_inds = np.random.choice(len(X_train), 1000000)
 fake_valid_data = lgb.Dataset(X_train.iloc[fake_valid_inds], label = y_train.iloc[fake_valid_inds],categorical_feature=cat_feats,
free_raw_data=False )
def prediction(test_df: pd.DataFrame,
test_audio: Path,
model_config: dict,
weights_path: str,
threshold=0.5):
model = get_model(model_config, weights_path)
 unique_audio_id = test_df.audio_id.unique()
warnings.filterwarnings("ignore")
 prediction_dfs = []
for audio_id in unique_audio_id:
with timer(f"Loading {audio_id}"):
clip, _ = librosa.load(test_audio /(audio_id + ".mp3"),
sr=SR,
mono=True,
res_type="kaiser_fast")
 test_df_for_audio_id = test_df.query(
 f"audio_id == '{audio_id}'" ).reset_index(drop=True)
 with timer(f"Prediction on {audio_id}"):
prediction_df = prediction_for_clip(test_df_for_audio_id,
clip=clip,
model=model,
threshold=threshold)
 prediction_dfs.append(prediction_df)
 prediction_df = pd.concat(prediction_dfs, axis=0, sort=False ).reset_index(drop=True)
 return prediction_df<predict_on_test>
params = {
"objective" : "poisson",
"metric" :"rmse",
"force_row_wise" : True,
"learning_rate" : 0.075,
"sub_row" : 0.85,
"bagging_freq" : 1,
"lambda_l2" : 0.1,
"metric": ["rmse"],
'verbosity': 1,
'num_iterations' : 2500,
}
prediction_df = prediction(test_df=test,
test_audio=TEST_AUDIO_DIR,
model_config=model_config,
weights_path=weights_path,
threshold=0.5 )<groupby>
%%time
m_lgb = lgb.train(params, train_data, valid_sets = [fake_valid_data],early_stopping_rounds=200, verbose_eval=100 )
labels = {}
for audio_id, sub_df in prediction_df.groupby("audio_id"):
events = sub_df[["ebird_code", "onset", "offset", "max_confidence", "site"]].values
n_events = len(events)
 removed_event = []
for i in range(n_events):
for j in range(n_events):
if i == j:
continue
if i in removed_event:
continue
if j in removed_event:
continue
event_i = events[i]
event_j = events[j]
if(event_i[1] - event_j[2] >= 0)or(event_j[1] - event_i[2] >= 0):
pass
else:
later_onset = max(event_i[1], event_j[1])
 sooner_onset = min(event_i[1], event_j[1])
 sooner_offset = min(event_i[2], event_j[2])
 later_offset = max(event_i[2], event_j[2])
 intersection = sooner_offset - later_onset
union = later_offset - sooner_onset
iou = intersection / union
if iou > 0.4:
if event_i[3] > event_j[3]:
removed_event.append(j)
 else:
removed_event.append(i)
 site = events[0][4]
for i in range(n_events):
if i in removed_event:
continue
event = events[i][0]
onset = events[i][1]
offset = events[i][2]
if site in {"site_1", "site_2"}:
start_section = int(( onset // 5)* 5)+ 5
end_section = int(( offset // 5)* 5)+ 5
cur_section = start_section
row_id = f"{site}_{audio_id}_{start_section}"
if labels.get(row_id)is not None:
labels[row_id].add(event)
 else:
labels[row_id] = set()
labels[row_id].add(event)
 while cur_section != end_section:
cur_section += 5
row_id = f"{site}_{audio_id}_{cur_section}"
if labels.get(row_id)is not None:
labels[row_id].add(event)
 else:
labels[row_id] = set()
labels[row_id].add(event)
 else:
row_id = f"{site}_{audio_id}"
if labels.get(row_id)is not None:
labels[row_id].add(event)
 else:
labels[row_id] = set()
labels[row_id].add(event )<prepare_output>
m_lgb.save_model("model.lgb" )
for key in labels:
labels[key] = " ".join(sorted(list(labels[key])))
 row_ids = list(labels.keys())
 birds = list(labels.values())
 post_processed = pd.DataFrame({
"row_id": row_ids,
"birds": birds
})
 post_processed.head()<save_to_csv>
sub.id.nunique() , sub["id"].str.contains("validation$" ).sum()