Kaggle competition: M5 Forecasting - Accuracy
all_row_id = test[["row_id"]]
submission = all_row_id.merge(post_processed, on="row_id", how="left")
 submission = submission.fillna("nocall")
 submission.to_csv("submission.csv", index=False)
 submission.head(20 )<set_options>
def read_data() :
print('Reading files...')
 calendar = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/calendar.csv')
 calendar = reduce_mem_usage(calendar)
 print('Calendar has {} rows and {} columns'.format(calendar.shape[0], calendar.shape[1]))
sell_prices = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sell_prices.csv')
 sell_prices = reduce_mem_usage(sell_prices)
 print('Sell prices has {} rows and {} columns'.format(sell_prices.shape[0], sell_prices.shape[1]))
sales_train_val = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sales_train_validation.csv')
 print('Sales train validation has {} rows and {} columns'.format(sales_train_val.shape[0], sales_train_val.shape[1]))
submission = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sample_submission.csv')
 return calendar, sell_prices, sales_train_val, submission
pd.options.display.max_rows = 500
pd.options.display.max_columns = 500<feature_engineering>
calendar, sell_prices, sales_train_val, submission = read_data()
def set_seed(seed: int = 42):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 @contextmanager
def timer(name: str)-> None:

 t0 = time.time()
print("[{}] start".format(name))
yield
print("[{}] done in {:.0f} s".format(name, time.time() - t0))<define_variables>
NUM_ITEMS = sales_train_val.shape[0]
DAYS_PRED = submission.shape[1] - 1
set_seed(1213)
 ROOT = Path.cwd().parent
INPUT_ROOT = ROOT / "input"
RAW_DATA = INPUT_ROOT / "birdsong-recognition"
TRAIN_AUDIO_DIR = RAW_DATA / "train_audio"
TEST_AUDIO_DIR = RAW_DATA / "test_audio"<load_from_csv>
def encode_categorical(df, cols):
for col in cols:
le = LabelEncoder()
df[col] = df[col].fillna('nan')
 df[col] = pd.Series(le.fit_transform(df[col]), index=df.index)
 return df
calendar = encode_categorical(
 calendar, ["event_name_1", "event_type_1", "event_name_2", "event_type_2"]
).pipe(reduce_mem_usage)
 sales_train_val = encode_categorical(
 sales_train_val, ["item_id", "dept_id", "cat_id", "store_id", "state_id"],
).pipe(reduce_mem_usage)
 sell_prices = encode_categorical(sell_prices, ["item_id", "store_id"] ).pipe(
 reduce_mem_usage 
)
train = pd.read_csv(RAW_DATA / "train.csv" )<load_from_csv>
product = sales_train_val[['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']].drop_duplicates()
if not TEST_AUDIO_DIR.exists() :
TEST_AUDIO_DIR = INPUT_ROOT / "birdcall-check" / "test_audio"
test = pd.read_csv(INPUT_ROOT / "birdcall-check" / "test.csv")
 else:
test = pd.read_csv(RAW_DATA / "test.csv" )<save_to_csv>
nrows = 365 * 2 * NUM_ITEMS
sub = pd.read_csv(".. /input/birdsong-recognition/sample_submission.csv")
 sub.to_csv("submission.csv", index=False )<init_hyperparams>
d_name = ['d_' + str(i+1)for i in range(1913)]
sales_train_val_values = sales_train_val[d_name].values
tmp = np.tile(np.arange(1,1914),(sales_train_val_values.shape[0],1))
df_tmp =(( sales_train_val_values>0)* tmp)
 start_no = np.min(np.where(df_tmp==0,9999,df_tmp),axis=1)-1
flag = np.dot(np.diag(1/(start_no+1)) , tmp)<1
sales_train_val_values = np.where(flag,np.nan,sales_train_val_values)
 sales_train_val[d_name] = sales_train_val_values
del tmp,sales_train_val_values
gc.collect()
TARGET_SR = 32000
model_config = {
"base_model_name": "resnest50_fast_1s1x64d",
"pretrained": False,
"num_classes": 264,
"trained_weights": ".. /input/training-birdsong-baseline-resnest50-fast/best_model.pth"
}
melspectrogram_parameters = {
"n_mels": 128,
"fmin": 20,
"fmax": 16000
}<define_variables>
1913-np.max(start_no )
BIRD_CODE = {
'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,
'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,
'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,
'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,
'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,
'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,
'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,
'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,
'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,
'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,
'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,
'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,
'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,
'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,
'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,
'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,
'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,
'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,
'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,
'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,
'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,
'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,
'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,
'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,
'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,
'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,
'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,
'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,
'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,
'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,
'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,
'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,
'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,
'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,
'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,
'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,
'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,
'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,
'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,
'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,
'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,
'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,
'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,
'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,
'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,
'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,
'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,
'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,
'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,
'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,
'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,
'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,
'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263
}
INV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items() }<define_search_model>
sales_train_val = pd.melt(sales_train_val,
id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
var_name = 'day', value_name = 'demand' )
class SplAtConv2d(Module):

 def __init__(self, in_channels, channels, kernel_size, stride=(1, 1), padding=(0, 0),
dilation=(1, 1), groups=1, bias=True,
radix=2, reduction_factor=4,
rectify=False, rectify_avg=False, norm_layer=None,
dropblock_prob=0.0, **kwargs):
super(SplAtConv2d, self ).__init__()
padding = _pair(padding)
 self.rectify = rectify and(padding[0] > 0 or padding[1] > 0)
 self.rectify_avg = rectify_avg
inter_channels = max(in_channels*radix//reduction_factor, 32)
 self.radix = radix
self.cardinality = groups
self.channels = channels
self.dropblock_prob = dropblock_prob
if self.rectify:
self.conv = RFConv2d(in_channels, channels*radix, kernel_size, stride, padding, dilation,
groups=groups*radix, bias=bias, average_mode=rectify_avg, **kwargs)
 else:
self.conv = Conv2d(in_channels, channels*radix, kernel_size, stride, padding, dilation,
groups=groups*radix, bias=bias, **kwargs)
 self.use_bn = norm_layer is not None
if self.use_bn:
self.bn0 = norm_layer(channels*radix)
 self.relu = ReLU(inplace=True)
 self.fc1 = Conv2d(channels, inter_channels, 1, groups=self.cardinality)
 if self.use_bn:
self.bn1 = norm_layer(inter_channels)
 self.fc2 = Conv2d(inter_channels, channels*radix, 1, groups=self.cardinality)
 if dropblock_prob > 0.0:
self.dropblock = DropBlock2D(dropblock_prob, 3)
 self.rsoftmax = rSoftMax(radix, groups)
 def forward(self, x):
x = self.conv(x)
 if self.use_bn:
x = self.bn0(x)
 if self.dropblock_prob > 0.0:
x = self.dropblock(x)
 x = self.relu(x)
 batch, rchannel = x.shape[:2]
if self.radix > 1:
if torch.__version__ < '1.5':
splited = torch.split(x, int(rchannel//self.radix), dim=1)
 else:
splited = torch.split(x, rchannel//self.radix, dim=1)
 gap = sum(splited)
 else:
gap = x
gap = F.adaptive_avg_pool2d(gap, 1)
 gap = self.fc1(gap)
 if self.use_bn:
gap = self.bn1(gap)
 gap = self.relu(gap)
 atten = self.fc2(gap)
 atten = self.rsoftmax(atten ).view(batch, -1, 1, 1)
 if self.radix > 1:
if torch.__version__ < '1.5':
attens = torch.split(atten, int(rchannel//self.radix), dim=1)
 else:
attens = torch.split(atten, rchannel//self.radix, dim=1)
 out = sum([att*split for(att, split)in zip(attens, splited)])
 else:
out = atten * x
return out.contiguous()
class rSoftMax(nn.Module):
def __init__(self, radix, cardinality):
super().__init__()
self.radix = radix
self.cardinality = cardinality
def forward(self, x):
batch = x.size(0)
 if self.radix > 1:
x = x.view(batch, self.cardinality, self.radix, -1 ).transpose(1, 2)
 x = F.softmax(x, dim=1)
 x = x.reshape(batch, -1)
 else:
x = torch.sigmoid(x)
 return x<import_modules>
sales_train_val = sales_train_val.iloc[-nrows:,:]
sales_train_val = sales_train_val[~sales_train_val.demand.isnull() ]
class DropBlock2D(object):
def __init__(self, *args, **kwargs):
raise NotImplementedError
class GlobalAvgPool2d(nn.Module):
def __init__(self):

 super(GlobalAvgPool2d, self ).__init__()
def forward(self, inputs):
return nn.functional.adaptive_avg_pool2d(inputs, 1 ).view(inputs.size(0), -1)
 class Bottleneck(nn.Module):

 expansion = 4
def __init__(self, inplanes, planes, stride=1, downsample=None,
radix=1, cardinality=1, bottleneck_width=64,
avd=False, avd_first=False, dilation=1, is_first=False,
rectified_conv=False, rectify_avg=False,
norm_layer=None, dropblock_prob=0.0, last_gamma=False):
super(Bottleneck, self ).__init__()
group_width = int(planes *(bottleneck_width / 64.))* cardinality
self.conv1 = nn.Conv2d(inplanes, group_width, kernel_size=1, bias=False)
 self.bn1 = norm_layer(group_width)
 self.dropblock_prob = dropblock_prob
self.radix = radix
self.avd = avd and(stride > 1 or is_first)
 self.avd_first = avd_first
if self.avd:
self.avd_layer = nn.AvgPool2d(3, stride, padding=1)
 stride = 1
if dropblock_prob > 0.0:
self.dropblock1 = DropBlock2D(dropblock_prob, 3)
 if radix == 1:
self.dropblock2 = DropBlock2D(dropblock_prob, 3)
 self.dropblock3 = DropBlock2D(dropblock_prob, 3)
 if radix >= 1:
self.conv2 = SplAtConv2d(
 group_width, group_width, kernel_size=3,
stride=stride, padding=dilation,
dilation=dilation, groups=cardinality, bias=False,
radix=radix, rectify=rectified_conv,
rectify_avg=rectify_avg,
norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 elif rectified_conv:
self.conv2 = RFConv2d(
 group_width, group_width, kernel_size=3, stride=stride,
padding=dilation, dilation=dilation,
groups=cardinality, bias=False,
average_mode=rectify_avg)
 self.bn2 = norm_layer(group_width)
 else:
self.conv2 = nn.Conv2d(
 group_width, group_width, kernel_size=3, stride=stride,
padding=dilation, dilation=dilation,
groups=cardinality, bias=False)
 self.bn2 = norm_layer(group_width)
 self.conv3 = nn.Conv2d(
 group_width, planes * 4, kernel_size=1, bias=False)
 self.bn3 = norm_layer(planes*4)
 if last_gamma:
zeros_(self.bn3.weight)
 self.relu = nn.ReLU(inplace=True)
 self.downsample = downsample
self.dilation = dilation
self.stride = stride
def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 if self.dropblock_prob > 0.0:
out = self.dropblock1(out)
 out = self.relu(out)
 if self.avd and self.avd_first:
out = self.avd_layer(out)
 out = self.conv2(out)
 if self.radix == 0:
out = self.bn2(out)
 if self.dropblock_prob > 0.0:
out = self.dropblock2(out)
 out = self.relu(out)
 if self.avd and not self.avd_first:
out = self.avd_layer(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.dropblock_prob > 0.0:
out = self.dropblock3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out += residual
out = self.relu(out)
 return out
class ResNet(nn.Module):

 def __init__(self, block, layers, radix=1, groups=1, bottleneck_width=64,
num_classes=1000, dilated=False, dilation=1,
deep_stem=False, stem_width=64, avg_down=False,
rectified_conv=False, rectify_avg=False,
avd=False, avd_first=False,
final_drop=0.0, dropblock_prob=0,
last_gamma=False, norm_layer=nn.BatchNorm2d):
self.cardinality = groups
self.bottleneck_width = bottleneck_width
self.inplanes = stem_width*2 if deep_stem else 64
self.avg_down = avg_down
self.last_gamma = last_gamma
self.radix = radix
self.avd = avd
self.avd_first = avd_first
super(ResNet, self ).__init__()
self.rectified_conv = rectified_conv
self.rectify_avg = rectify_avg
if rectified_conv:
conv_layer = RFConv2d
else:
conv_layer = nn.Conv2d
conv_kwargs = {'average_mode': rectify_avg} if rectified_conv else {}
if deep_stem:
self.conv1 = nn.Sequential(
 conv_layer(3, stem_width, kernel_size=3, stride=2, padding=1, bias=False, **conv_kwargs),
norm_layer(stem_width),
nn.ReLU(inplace=True),
conv_layer(stem_width, stem_width, kernel_size=3, stride=1, padding=1, bias=False, **conv_kwargs),
norm_layer(stem_width),
nn.ReLU(inplace=True),
conv_layer(stem_width, stem_width*2, kernel_size=3, stride=1, padding=1, bias=False, **conv_kwargs), 
)
 else:
self.conv1 = conv_layer(3, 64, kernel_size=7, stride=2, padding=3,
bias=False, **conv_kwargs)
 self.bn1 = norm_layer(self.inplanes)
 self.relu = nn.ReLU(inplace=True)
 self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
 self.layer1 = self._make_layer(block, 64, layers[0], norm_layer=norm_layer, is_first=False)
 self.layer2 = self._make_layer(block, 128, layers[1], stride=2, norm_layer=norm_layer)
 if dilated or dilation == 4:
self.layer3 = self._make_layer(block, 256, layers[2], stride=1,
dilation=2, norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 self.layer4 = self._make_layer(block, 512, layers[3], stride=1,
dilation=4, norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 elif dilation==2:
self.layer3 = self._make_layer(block, 256, layers[2], stride=2,
dilation=1, norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 self.layer4 = self._make_layer(block, 512, layers[3], stride=1,
dilation=2, norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 else:
self.layer3 = self._make_layer(block, 256, layers[2], stride=2,
norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 self.layer4 = self._make_layer(block, 512, layers[3], stride=2,
norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 self.avgpool = GlobalAvgPool2d()
self.drop = nn.Dropout(final_drop)if final_drop > 0.0 else None
self.fc = nn.Linear(512 * block.expansion, num_classes)
 for m in self.modules() :
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2./ n))
elif isinstance(m, norm_layer):
m.weight.data.fill_(1)
 m.bias.data.zero_()
def _make_layer(self, block, planes, blocks, stride=1, dilation=1, norm_layer=None,
dropblock_prob=0.0, is_first=True):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
down_layers = []
if self.avg_down:
if dilation == 1:
down_layers.append(nn.AvgPool2d(kernel_size=stride, stride=stride,
ceil_mode=True, count_include_pad=False))
else:
down_layers.append(nn.AvgPool2d(kernel_size=1, stride=1,
ceil_mode=True, count_include_pad=False))
down_layers.append(nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=1, stride=1, bias=False))
else:
down_layers.append(nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=1, stride=stride, bias=False))
down_layers.append(norm_layer(planes * block.expansion))
downsample = nn.Sequential(*down_layers)
 layers = []
if dilation == 1 or dilation == 2:
layers.append(block(self.inplanes, planes, stride, downsample=downsample,
radix=self.radix, cardinality=self.cardinality,
bottleneck_width=self.bottleneck_width,
avd=self.avd, avd_first=self.avd_first,
dilation=1, is_first=is_first, rectified_conv=self.rectified_conv,
rectify_avg=self.rectify_avg,
norm_layer=norm_layer, dropblock_prob=dropblock_prob,
last_gamma=self.last_gamma))
elif dilation == 4:
layers.append(block(self.inplanes, planes, stride, downsample=downsample,
radix=self.radix, cardinality=self.cardinality,
bottleneck_width=self.bottleneck_width,
avd=self.avd, avd_first=self.avd_first,
dilation=2, is_first=is_first, rectified_conv=self.rectified_conv,
rectify_avg=self.rectify_avg,
norm_layer=norm_layer, dropblock_prob=dropblock_prob,
last_gamma=self.last_gamma))
else:
raise RuntimeError("=> unknown dilation size: {}".format(dilation))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes,
radix=self.radix, cardinality=self.cardinality,
bottleneck_width=self.bottleneck_width,
avd=self.avd, avd_first=self.avd_first,
dilation=dilation, rectified_conv=self.rectified_conv,
rectify_avg=self.rectify_avg,
norm_layer=norm_layer, dropblock_prob=dropblock_prob,
last_gamma=self.last_gamma))
return nn.Sequential(*layers)
 def forward(self, x):
x = self.conv1(x)
 x = self.bn1(x)
 x = self.relu(x)
 x = self.maxpool(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 x = self.avgpool(x)
 x = torch.flatten(x, 1)
 if self.drop:
x = self.drop(x)
 x = self.fc(x)
 return x<choose_model_class>
test1_rows = [row for row in submission['id'] if 'validation' in row]
test2_rows = [row for row in submission['id'] if 'evaluation' in row]
test1 = submission[submission['id'].isin(test1_rows)]
test2 = submission[submission['id'].isin(test2_rows)]
test1.columns = ["id"] + [f"d_{d}" for d in range(1914, 1914 + DAYS_PRED)]
test2.columns = ["id"] + [f"d_{d}" for d in range(1942, 1942 + DAYS_PRED)]
test2['id'] = test2['id'].str.replace('_evaluation','_validation')
 test1 = test1.merge(product, how = 'left', on = 'id')
 test2 = test2.merge(product, how = 'left', on = 'id')
 test1 = pd.melt(test1, id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
var_name = 'day', value_name = 'demand')
 test2 = pd.melt(test2, id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
var_name = 'day', value_name = 'demand')
 sales_train_val['part'] = 'train'
test1['part'] = 'test1'
test2['part'] = 'test2'
data = pd.concat([sales_train_val, test1, test2], axis = 0)
 del sales_train_val, test1, test2
data = data[data['part'] != 'test2']
gc.collect()
def get_model(args: tp.Dict):
model = ResNet(
 Bottleneck, [3, 4, 6, 3],
radix=1, groups=1, bottleneck_width=64,
deep_stem=True, stem_width=32, avg_down=True,
avd=True, avd_first=True)
 del model.fc
model.fc = nn.Sequential(
 nn.Linear(2048, 1024), nn.ReLU() , nn.Dropout(p=0.2),
nn.Linear(1024, 1024), nn.ReLU() , nn.Dropout(p=0.2),
nn.Linear(1024, args["num_classes"]))
state_dict = torch.load(args["trained_weights"])
 model.load_state_dict(state_dict)
 device = torch.device("cuda")
 model.to(device)
 model.eval()
return model<create_dataframe>
calendar.drop(['weekday', 'wday', 'month', 'year'],
inplace = True, axis = 1)
 data = pd.merge(data, calendar, how = 'left', left_on = ['day'], right_on = ['d'])
 data.drop(['d', 'day'], inplace = True, axis = 1)
 del calendar
gc.collect()
data = data.merge(sell_prices, on = ['store_id', 'item_id', 'wm_yr_wk'], how = 'left')
 print('Our final dataset to train has {} rows and {} columns'.format(data.shape[0], data.shape[1]))
del sell_prices
gc.collect()
def prediction_for_clip(test_df: pd.DataFrame,
clip: np.ndarray,
model: ResNet,
mel_params: dict,
threshold=0.5):
dataset = TestDataset(df=test_df,
clip=clip,
img_size=224,
melspectrogram_parameters=mel_params)
 loader = data.DataLoader(dataset, batch_size=1, shuffle=False)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model.eval()
prediction_dict = {}
for image, row_id, site in progress_bar(loader):
site = site[0]
row_id = row_id[0]
if site in {"site_1", "site_2"}:
image = image.to(device)
 with torch.no_grad() :
prediction = F.sigmoid(model(image))
proba = prediction.detach().cpu().numpy().reshape(-1)
 events = proba >= threshold
labels = np.argwhere(events ).reshape(-1 ).tolist()
else:
image = image.squeeze(0)
 batch_size = 16
whole_size = image.size(0)
 if whole_size % batch_size == 0:
n_iter = whole_size // batch_size
else:
n_iter = whole_size // batch_size + 1
all_events = set()
for batch_i in range(n_iter):
batch = image[batch_i * batch_size:(batch_i + 1)* batch_size]
if batch.ndim == 3:
batch = batch.unsqueeze(0)
 batch = batch.to(device)
 with torch.no_grad() :
prediction = F.sigmoid(model(batch))
proba = prediction.detach().cpu().numpy()
events = proba >= threshold
for i in range(len(events)) :
event = events[i, :]
labels = np.argwhere(event ).reshape(-1 ).tolist()
for label in labels:
all_events.add(label)
 labels = list(all_events)
 if len(labels)== 0:
prediction_dict[row_id] = "nocall"
else:
labels_str_list = list(map(lambda x: INV_BIRD_CODE[x], labels))
label_string = " ".join(labels_str_list)
 prediction_dict[row_id] = label_string
return prediction_dict<categorify>
def simple_fe(data):
for diff in [0, 1, 2]:
shift = DAYS_PRED + diff
data[f"shift_t{shift}"] = data.groupby(["id"])["demand"].transform(
 lambda x: x.shift(shift)
)

for size in [7, 30, 60, 90, 180]:
data[f"rolling_mean_t{size}"] = data.groupby(["id"])["demand"].transform(
 lambda x: x.shift(DAYS_PRED ).rolling(size ).mean() 
)

data["shift_price_t1"] = data.groupby(["id"])["sell_price"].transform(
 lambda x: x.shift(1)
)
 data["price_change_t1"] =(data["shift_price_t1"] - data["sell_price"])/(
 data["shift_price_t1"] 
)
 data["rolling_price_max_t365"] = data.groupby(["id"])["sell_price"].transform(
 lambda x: x.shift(1 ).rolling(365 ).max() 
)
 data["price_change_t365"] =(data["rolling_price_max_t365"] - data["sell_price"])/(
 data["rolling_price_max_t365"] 
)
 data["rolling_price_std_t7"] = data.groupby(["id"])["sell_price"].transform(
 lambda x: x.rolling(7 ).std() 
)
 data["rolling_price_std_t30"] = data.groupby(["id"])["sell_price"].transform(
 lambda x: x.rolling(30 ).std() 
)
 dt_col = "date"
data[dt_col] = pd.to_datetime(data[dt_col])
 attrs = [
"year",
"quarter",
"month",
"week",
"day",
"dayofweek",
"is_year_end",
"is_year_start",
"is_quarter_end",
"is_quarter_start",
"is_month_end",
"is_month_start",
]
for attr in attrs:
dtype = np.int16 if attr == "year" else np.int8
data[attr] = getattr(data[dt_col].dt, attr ).astype(dtype)
 data["is_weekend"] = data["dayofweek"].isin([5, 6] ).astype(np.int8)
 return data
data = simple_fe(data)
 data = reduce_mem_usage(data )
def prediction(test_df: pd.DataFrame,
test_audio: Path,
model_config: dict,
mel_params: dict,
target_sr: int,
threshold=0.5):
model = get_model(model_config)
 unique_audio_id = test_df.audio_id.unique()
warnings.filterwarnings("ignore")
 prediction_dfs = []
for audio_id in unique_audio_id:
with timer(f"Loading {audio_id}"):
clip, _ = librosa.load(test_audio /(audio_id + ".mp3"),
sr=target_sr,
mono=True,
res_type="kaiser_fast")
 test_df_for_audio_id = test_df.query(
 f"audio_id == '{audio_id}'" ).reset_index(drop=True)
 with timer(f"Prediction on {audio_id}"):
prediction_dict = prediction_for_clip(test_df_for_audio_id,
clip=clip,
model=model,
mel_params=mel_params,
threshold=threshold)
 row_id = list(prediction_dict.keys())
 birds = list(prediction_dict.values())
 prediction_df = pd.DataFrame({
"row_id": row_id,
"birds": birds
})
 prediction_dfs.append(prediction_df)
 prediction_df = pd.concat(prediction_dfs, axis=0, sort=False ).reset_index(drop=True)
 return prediction_df<save_to_csv>
x_train = data[data['date'] <= '2016-03-27']
y_train = x_train['demand']
x_val = data[(data['date'] > '2016-03-27')&(data['date'] <= '2016-04-24')]
y_val = x_val['demand']
test = data[(data['date'] > '2016-04-24')]
submission = prediction(test_df=test,
test_audio=TEST_AUDIO_DIR,
model_config=model_config,
mel_params=melspectrogram_parameters,
target_sr=TARGET_SR,
threshold=0.8)
 submission.to_csv("submission.csv", index=False )<define_variables>
features = [
"item_id",
"dept_id",
"cat_id",
"store_id",
"state_id",
"event_name_1",
"event_type_1",
"event_name_2",
"event_type_2",
"snap_CA",
"snap_TX",
"snap_WI",
"sell_price",
"shift_t28",
"shift_t29",
"shift_t30",
"rolling_mean_t7",
"rolling_mean_t30",
"rolling_mean_t60",
"rolling_mean_t90",
"rolling_mean_t180",
"price_change_t1",
"price_change_t365",
"rolling_price_std_t7",
"rolling_price_std_t30",
"year",
"month",
"week",
"day",
"dayofweek",
"is_year_end",
"is_year_start",
"is_quarter_end",
"is_quarter_start",
"is_month_end",
"is_month_start",
"is_weekend",
]
params = {
'boosting_type': 'gbdt',
'metric': 'rmse',
'objective': 'regression',
'n_jobs': -1,
'seed': 236,
'learning_rate': 0.1,
'bagging_fraction': 0.75,
'bagging_freq': 10,
'colsample_bytree': 0.75}
train_set = lgb.Dataset(x_train[features], y_train)
 val_set = lgb.Dataset(x_val[features], y_val)
 del x_train, y_train
RANDOM_SEED = 42<load_from_csv>
train_df = pd.read_csv('/kaggle/input/nyc-taxi-trip-duration/train.zip', compression='zip')
 test_df = pd.read_csv('/kaggle/input/nyc-taxi-trip-duration/test.zip', compression='zip')
 train_fastroute_df = pd.concat(( pd.read_csv('/kaggle/input/osrm-oscarleo/fastest_routes_train_part_1.csv'), pd.read_csv('/kaggle/input/osrm-oscarleo/fastest_routes_train_part_2.csv')) , axis = 0 ).reset_index(drop = True)
 test_fastroute_df= pd.read_csv('/kaggle/input/osrm-oscarleo/fastest_routes_test.csv')
 ny_shape_df = gpd.read_file('/kaggle/input/neighborhoods-in-new-york/ZillowNeighborhoods-NY.shp')
 remain_train_pickup_df = pd.concat(( pd.read_csv('/kaggle/input/nyc-tax-train-test-remain-2/train_city_pickup.csv'), pd.read_csv('/kaggle/input/nyc-tax-train-test-remain/train_city_pickup.csv')) , axis = 0)
 remain_train_dropoff_df = pd.concat(( pd.read_csv('/kaggle/input/nyc-tax-train-test-remain-2/train_city_dropoff.csv'), pd.read_csv('/kaggle/input/nyc-tax-train-test-remain/train_city_dropoff.csv')) , axis = 0)
 remain_test_pickup_df = pd.concat(( pd.read_csv('/kaggle/input/nyc-tax-train-test-remain-2/test_city_pickup.csv'), pd.read_csv('/kaggle/input/nyc-tax-train-test-remain/test_city_pickup.csv')) , axis = 0)
 remain_test_dropoff_df = pd.concat(( pd.read_csv('/kaggle/input/nyc-tax-train-test-remain-2/test_city_dropoff.csv'), pd.read_csv('/kaggle/input/nyc-tax-train-test-remain/test_city_dropoff.csv')) , axis = 0)
 temperature_df = pd.read_csv('/kaggle/input/historical-hourly-weather-data/temperature.csv')
 humidity_df = pd.read_csv('/kaggle/input/historical-hourly-weather-data/humidity.csv')
 wind_speed_df = pd.read_csv('/kaggle/input/historical-hourly-weather-data/wind_speed.csv')
 wind_direction_df = pd.read_csv('/kaggle/input/historical-hourly-weather-data/wind_direction.csv')
 gc.collect()<data_type_conversions>
weight_mat = np.c_[np.ones([NUM_ITEMS,1] ).astype(np.int8),
pd.get_dummies(product.state_id.astype(str),drop_first=False ).astype('int8' ).values,
pd.get_dummies(product.store_id.astype(str),drop_first=False ).astype('int8' ).values,
pd.get_dummies(product.cat_id.astype(str),drop_first=False ).astype('int8' ).values,
pd.get_dummies(product.dept_id.astype(str),drop_first=False ).astype('int8' ).values,
pd.get_dummies(product.state_id.astype(str)+ product.cat_id.astype(str),drop_first=False ).astype('int8' ).values,
pd.get_dummies(product.state_id.astype(str)+ product.dept_id.astype(str),drop_first=False ).astype('int8' ).values,
pd.get_dummies(product.store_id.astype(str)+ product.cat_id.astype(str),drop_first=False ).astype('int8' ).values,
pd.get_dummies(product.store_id.astype(str)+ product.dept_id.astype(str),drop_first=False ).astype('int8' ).values,
pd.get_dummies(product.item_id.astype(str),drop_first=False ).astype('int8' ).values,
pd.get_dummies(product.state_id.astype(str)+ product.item_id.astype(str),drop_first=False ).astype('int8' ).values,
np.identity(NUM_ITEMS ).astype(np.int8)
 ].T
weight_mat_csr = csr_matrix(weight_mat)
 del weight_mat; gc.collect()
def weight_calc(data,product):
sales_train_val = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sales_train_validation.csv')
 d_name = ['d_' + str(i+1)for i in range(1913)]
sales_train_val = weight_mat_csr * sales_train_val[d_name].values
df_tmp =(( sales_train_val>0)* np.tile(np.arange(1,1914),(weight_mat_csr.shape[0],1)))
 start_no = np.min(np.where(df_tmp==0,9999,df_tmp),axis=1)-1
flag = np.dot(np.diag(1/(start_no+1)) , np.tile(np.arange(1,1914),(weight_mat_csr.shape[0],1)))<1
sales_train_val = np.where(flag,np.nan,sales_train_val)
 weight1 = np.nansum(np.diff(sales_train_val,axis=1)**2,axis=1)/(1913-start_no)
 df_tmp = data[(data['date'] > '2016-03-27')&(data['date'] <= '2016-04-24')]
df_tmp['amount'] = df_tmp['demand'] * df_tmp['sell_price']
df_tmp =df_tmp.groupby(['id'])['amount'].apply(np.sum)
 df_tmp = df_tmp[product.id].values
weight2 = weight_mat_csr * df_tmp
weight2 = weight2/np.sum(weight2)
 del sales_train_val
gc.collect()
return weight1, weight2
weight1, weight2 = weight_calc(data,product)
 def wrmsse(preds, data):
y_true = data.get_label()
y_true = y_true[-(NUM_ITEMS * DAYS_PRED):]
preds = preds[-(NUM_ITEMS * DAYS_PRED):]
num_col = DAYS_PRED
reshaped_preds = preds.reshape(num_col, NUM_ITEMS ).T
reshaped_true = y_true.reshape(num_col, NUM_ITEMS ).T
train = weight_mat_csr*np.c_[reshaped_preds, reshaped_true]
score = np.sum(
 np.sqrt(
 np.mean(
 np.square(
 train[:,:num_col] - train[:,num_col:])
 ,axis=1)/ weight1)* weight2)
 return 'wrmsse', score, False
def wrmsse_simple(preds, data):
y_true = data.get_label()
y_true = y_true[-(NUM_ITEMS * DAYS_PRED):]
preds = preds[-(NUM_ITEMS * DAYS_PRED):]
num_col = DAYS_PRED
reshaped_preds = preds.reshape(num_col, NUM_ITEMS ).T
reshaped_true = y_true.reshape(num_col, NUM_ITEMS ).T
train = np.c_[reshaped_preds, reshaped_true]
weight2_2 = weight2[:NUM_ITEMS]
weight2_2 = weight2_2/np.sum(weight2_2)
 score = np.sum(
 np.sqrt(
 np.mean(
 np.square(
 train[:,:num_col] - train[:,num_col:])
 ,axis=1)/ weight1[:NUM_ITEMS])*weight2_2)
 return 'wrmsse', score, False
def add_df_date_hour(df, string):

 df['pickup_date'] = pd.to_datetime(df['datetime'] ).dt.date
df['pickup_hour'] = pd.to_datetime(df['datetime'] ).dt.hour
return df[['pickup_date', 'pickup_hour', 'New York']].rename(columns = {'New York' : string})
 def get_haversine_distance(lat_1, long_1, lat_2, long_2):

 AVG_EARTH_RADIUS = 6371
lat_1, long1, lat_2, long_2 = map(np.radians,(lat_1, long_1, lat_2, long_2))
lat = lat_2 - lat_1 ; long = long_2 - long_1
d = np.sin(lat * 0.5)** 2 + np.cos(lat_1)* np.cos(lat_2)* np.sin(long * 0.5)** 2
h = 2 * AVG_EARTH_RADIUS * np.arcsin(np.sqrt(d))
return h
def get_direction(lat_1, long_1, lat_2, long_2):

 AVG_EARTH_RADIUS = 6371
long_delta_rad = np.radians(long_2 - long_1)
 lat_1, long_1, lat_2, long_2 = map(np.radians,(lat_1, long_1, lat_2, long_2))
y = np.sin(long_delta_rad)* np.cos(lat_2)
 x = np.cos(lat_1)* np.sin(lat_2)- np.sin(lat_1)* np.cos(lat_2)* np.cos(long_delta_rad)
 return np.degrees(np.arctan2(y, x))
def map_dic_to_dic(dic_1, dic_2):

 return {k : dic_2[v] for k, v in dic_1.items() if v in dic_2.keys() }
def merge_location_df(df, fastroute_df, pickup_df, dropoff_df):

 df = df.merge(fastroute_df, on = 'id')
 df = df.merge(pickup_df[['id', 'pickup_city']], on = 'id', how = 'left')
 df = df.merge(dropoff_df[['id', 'dropoff_city']], on = 'id', how = 'left')
 return df
def merge_weather_df(df, temp_df, hum_df, ws_df, wd_df):

 df = df.merge(temp_df, on = ['pickup_date', 'pickup_hour'], how = 'left')
 df = df.merge(hum_df, on = ['pickup_date', 'pickup_hour'], how = 'left')
 df = df.merge(ws_df, on = ['pickup_date', 'pickup_hour'], how = 'left')
 df = df.merge(wd_df, on = ['pickup_date', 'pickup_hour'], how = 'left')
 return df
def map_df_to_dic(df, dic, cols):

 for k, v in cols.items() :
df[k] = df[v].map(dic)
 return df
def data_preprocessing(df):

 df['pickup_dropoff_same_county'] = 0
df.loc[df['pickup_county'] == df['dropoff_county'], 'pickup_dropoff_same_county'] = 1
df['pickup_dropoff_same_city'] = 0
df.loc[df['pickup_city'] == df['dropoff_city'], 'pickup_dropoff_same_city'] = 1
df['f_road_count'] = df['street_for_each_step'].str.count('Road')
 df['f_street_count'] = df['street_for_each_step'].str.count('Street')
 df['f_drive_count'] = df['street_for_each_step'].str.count('Drive')
 df['f_avenue_count'] = df['street_for_each_step'].str.count('Avenue')
 df['f_plaza_count'] = df['street_for_each_step'].str.count('Plaza')
 df['f_square_count'] = df['street_for_each_step'].str.count('Square')
 df['f_parkway_count'] = df['street_for_each_step'].str.count('Parkway')
 df['f_boulevard_count'] = df['street_for_each_step'].str.count('Boulevard')
 df['f_expressway_count'] = df['street_for_each_step'].str.count('Expressway')
 df['f_left_count'] = df['step_direction'].str.count('left')
 congested_expressways = ['Brooklyn Queens Expressway', 'Cross Bronx Expressway', 'Van Wyck Expressway', 'Long Island Expressway']
df['in_congested_expressways'] = df['street_for_each_step'].apply(lambda x: 1 if any(way in x for way in congested_expressways)else 0)
 tourist_attractions = ['Financial District', 'Little Italy', 'Chinatown', 'Tribeca', 'SoHo', 'Lower East Side', 'Greenwich Village', 'Flatiron District',
'Chelsea', 'The Garment District', 'Hell Kitchen', 'Broadway', 'Times Square', 'Fifth Avenue', 'Central Park', 'Upper East',
'West Sides', 'Harlem',
'Gramercy', 'Union Square', 'Union Square Park', 'Lower East Side', 'East Village', 'Inwood', 'Upper West Side', 'Upper East Side', 'Midtown',
'West Village', 'Central Park', 'East Harlem', 'DUMBO']
df['pickup_hour'] = pd.to_datetime(df['pickup_datetime'] ).dt.hour
df['pickup_min'] = pd.to_datetime(df['pickup_datetime'] ).dt.minute
df['pickup_day'] = pd.to_datetime(df['pickup_datetime'] ).dt.day
df['pickup_date'] = pd.to_datetime(df['pickup_datetime'] ).dt.date
df['pickup_time'] = df['pickup_hour'] +(df['pickup_min'] / 60)
 df['pickup_dayofweek'] = pd.to_datetime(df['pickup_datetime'] ).dt.dayofweek
df['pickup_weekday'] = 0
df.loc[df['pickup_dayofweek'].isin([0, 1, 2, 3, 4]), 'pickup_weekday'] = 1
df['pickup_tourist_area'] = 0
df.loc[df['pickup_city'].isin(tourist_attractions), 'pickup_tourist_area'] = 1
df['dropoff_tourist_area'] = 0
df.loc[df['dropoff_city'].isin(tourist_attractions), 'dropoff_tourist_area'] = 1
df['second_last_street'] = df['street_for_each_step'].str.split('|' ).str[-1]
df['second_first_street'] = df['street_for_each_step'].str.split('|' ).str[1]
df['displacement'] = get_haversine_distance(df['pickup_latitude'], df['pickup_longitude'], df['dropoff_latitude'], df['dropoff_longitude'])
 df['direction'] = get_direction(df['pickup_latitude'], df['pickup_longitude'], df['dropoff_latitude'], df['dropoff_longitude'])
 return df
def agggregate_processing(train, test):

 coords = np.vstack(( train[['pickup_latitude', 'pickup_longitude']].values,
train[['dropoff_latitude', 'dropoff_longitude']].values,
test[['pickup_latitude', 'pickup_longitude']].values,
test[['dropoff_latitude', 'dropoff_longitude']].values))
pca = PCA().fit(coords)
 train['pickup_pca0'] = pca.transform(train[['pickup_latitude', 'pickup_longitude']])[:, 0]
train['pickup_pca1'] = pca.transform(train[['pickup_latitude', 'pickup_longitude']])[:, 1]
train['dropoff_pca0'] = pca.transform(train[['dropoff_latitude', 'dropoff_longitude']])[:, 0]
train['dropoff_pca1'] = pca.transform(train[['dropoff_latitude', 'dropoff_longitude']])[:, 1]
test['pickup_pca0'] = pca.transform(test[['pickup_latitude', 'pickup_longitude']])[:, 0]
test['pickup_pca1'] = pca.transform(test[['pickup_latitude', 'pickup_longitude']])[:, 1]
test['dropoff_pca0'] = pca.transform(test[['dropoff_latitude', 'dropoff_longitude']])[:, 0]
test['dropoff_pca1'] = pca.transform(test[['dropoff_latitude', 'dropoff_longitude']])[:, 1]
label_encoder = LabelEncoder()
label_features = ['dropoff_city', 'dropoff_county', 'end_street', 'second_last_street', 'pickup_city', 'pickup_county', 'starting_street',
'second_first_street', 'store_and_fwd_flag']
big_df = pd.concat(( train, test), axis = 0, sort = True)
 train_len = train.shape[0]
for label_feature in label_features:
big_df[label_feature].fillna('-99', inplace = True)
 big_df[label_feature] = label_encoder.fit_transform(big_df[label_feature].values)
 count_dropoff_city_df = pd.DataFrame(big_df.groupby(['pickup_date', 'pickup_hour', 'dropoff_city'])['passenger_count'].count() ).reset_index().rename(columns = {'passenger_count' : 'count_dropoff_city'})
 count_pickup_city_df = pd.DataFrame(big_df.groupby(['pickup_date', 'pickup_hour', 'pickup_city'])['passenger_count'].count() ).reset_index().rename(columns = {'passenger_count' : 'count_pickup_city'})
 count_dropoff_county_df = pd.DataFrame(big_df.groupby(['pickup_date', 'pickup_hour', 'dropoff_county'])['passenger_count'].count() ).reset_index().rename(columns = {'passenger_count' : 'count_dropoff_county'})
 count_pickup_county_df = pd.DataFrame(big_df.groupby(['pickup_date', 'pickup_hour', 'pickup_county'])['passenger_count'].count() ).reset_index().rename(columns = {'passenger_count' : 'count_pickup_county'})
 count_pickup_start_street_df = pd.DataFrame(big_df.groupby(['pickup_date', 'pickup_hour', 'starting_street'])['passenger_count'].count() ).reset_index().rename(columns = {'passenger_count' : 'count_pickup_street'})
 count_pickup_start_sec_street_df = pd.DataFrame(big_df.groupby(['pickup_date', 'pickup_hour', 'second_first_street'])['passenger_count'].count() ).reset_index().rename(columns = {'passenger_count' : 'count_pickup_sec_street'})
 count_pickup_end_street_df = pd.DataFrame(big_df.groupby(['pickup_date', 'pickup_hour', 'end_street'])['passenger_count'].count() ).reset_index().rename(columns = {'passenger_count' : 'count_dropoff_street'})
 count_pickup_end_sec_street_df = pd.DataFrame(big_df.groupby(['pickup_date', 'pickup_hour', 'second_last_street'])['passenger_count'].count() ).reset_index().rename(columns = {'passenger_count' : 'count_dropoff_sec_street'})
 big_df = big_df.merge(count_dropoff_county_df, on = ['pickup_date', 'pickup_hour', 'dropoff_county'], how = 'left')
 big_df = big_df.merge(count_pickup_county_df, on = ['pickup_date', 'pickup_hour', 'pickup_county'], how = 'left')
 big_df = big_df.merge(count_dropoff_city_df, on = ['pickup_date', 'pickup_hour', 'dropoff_city'], how = 'left')
 big_df = big_df.merge(count_pickup_city_df, on = ['pickup_date', 'pickup_hour', 'pickup_city'], how = 'left')
 big_df = big_df.merge(count_pickup_start_street_df, on = ['pickup_date', 'pickup_hour', 'starting_street'], how = 'left')
 big_df = big_df.merge(count_pickup_start_sec_street_df, on = ['pickup_date', 'pickup_hour', 'second_first_street'], how = 'left')
 big_df = big_df.merge(count_pickup_end_street_df, on = ['pickup_date', 'pickup_hour', 'end_street'], how = 'left')
 big_df = big_df.merge(count_pickup_end_sec_street_df, on = ['pickup_date', 'pickup_hour', 'second_last_street'], how = 'left')
 train = big_df.iloc[:train.shape[0]]
test = big_df.iloc[train.shape[0]:]
return train, test<categorify>
params = {
'boosting_type': 'gbdt',
'metric': 'custom',
'objective': 'poisson',
'n_jobs': -1,
'seed': 236,
'learning_rate': 0.1,
'bagging_fraction': 0.75,
'bagging_freq': 10,
'colsample_bytree': 0.75}
model = lgb.train(params, train_set, num_boost_round = 2500, early_stopping_rounds = 50,
valid_sets = [train_set, val_set], verbose_eval = 100, feval= wrmsse)
 val_pred = model.predict(x_val[features])
 val_score = np.sqrt(metrics.mean_squared_error(val_pred, y_val))
print(f'Our val wrmsse score is {val_score}')
 y_pred = model.predict(test[features])
 test['demand'] = y_pred
<feature_engineering><EOS> 
predictions = test[['id', 'date', 'demand']]
predictions = pd.pivot(predictions, index = 'id', columns = 'date', values = 'demand' ).reset_index()
predictions.columns = ['id'] + ['F' + str(i + 1)for i in range(28)]
evaluation_rows = [row for row in submission['id'] if 'evaluation' in row]
evaluation = submission[submission['id'].isin(evaluation_rows)]
validation = submission[['id']].merge(predictions, on = 'id')
 final = pd.concat([validation, evaluation])
 final.to_csv('submission2.csv', index = False )