Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<groupby>
df = dtable.fread('/kaggle/input/jane-street-market-prediction/train.csv' ).to_pandas()
df = df.query('date > 85' ).reset_index(drop = True)
 df = df[df.weight > 0]
df.reset_index(inplace=True, drop=True)
 df = df.astype({c: np.float32 for c in df.select_dtypes(include='float64' ).columns})
 df_labels = df[['date', 'weight', 'resp_1', 'resp_2', 'resp_3', 'resp_4', 'resp']]
features = [c for c in df.columns if 'feature' in c]
df = df.drop(df_labels.columns, axis=1)
 df.fillna(df.mean() , inplace=True)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp_4', 'resp']
X = df[features]
y = np.stack([(df_labels[c] > 0 ).astype('int')for c in resp_cols] ).T
f_mean = np.mean(df[features[1:]].values, axis=0)
 device = 'cuda' if t.cuda.is_available() else 'cpu'
sel_cols = ['Country_Region','Province_State','ConfirmedCasesPos','FatalitiesPos']
df_tr_cnt = df_train[sel_cols].groupby(by=['Country_Region','Province_State'] ).sum()
df_te_cnt = df_test[sel_cols].groupby(by=['Country_Region','Province_State'] ).sum()<filter>
class Model(nn.Module):
def __init__(self, input_size):
super(Model, self ).__init__()
hs = 256
self.batch_norm0 = nn.BatchNorm1d(input_size)
 self.layer1 = LinBnDrop(input_size, hs, bn=True, p=0, act=Mish() , lin_first=False)
 self.layer2 = LinBnDrop(hs + input_size, hs, bn=True, p=0.2289, act=Mish() , lin_first=False)
 self.layer3 = LinBnDrop(hs + hs, hs, bn=True, p=0.2289, act=Mish() , lin_first=False)
 self.layer4 = LinBnDrop(hs + hs, hs, bn=True, p=0.2, act=Mish() , lin_first=False)
 self.dense5 = nn.Linear(hs + hs, 5)
 def forward(self, x):
x = self.batch_norm0(x)
 x1 = self.layer1(x)
 x = torch.cat([x, x1], 1)
 x2 = self.layer2(x)
 x = torch.cat([x1, x2], 1)
 x3 = self.layer3(x)
 x = torch.cat([x2, x3], 1)
 x4 = self.layer4(x)
 x = torch.cat([x3, x4], 1)
 x = self.dense5(x)
 return x
sel_cols = ['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']
df_tr_last = df_train[df_train['Date']==max(df_train['Date'])][sel_cols]
df_te_last = df_test[df_test['Date']==max(df_test['Date'])][sel_cols]<merge>
model_nn = Model(len(features))
model_nn = model_nn.to(device)
 learn = Learner(None, model_nn, loss_func=1)
 learn.load('/kaggle/input/roclossjs3/dense_model')
df_tr_last = df_tr_last.merge(df_tr_cnt,on=['Country_Region','Province_State'])
 df_te_last = df_te_last.merge(df_te_cnt,on=['Country_Region','Province_State'] )<merge>
@njit
def fillna_npwhere_njit(array, values):
if np.isnan(array.sum()):
array = np.where(np.isnan(array), values, array)
 return array
def for_loop(method, matrix, values):
for i in range(matrix.shape[0]):
matrix[i] = method(matrix[i], values)
 return matrix
df_last = df_tr_last.merge(df_te_last,on=['Country_Region','Province_State'])
 df_last['DeltaConfirmedCases']=100*(df_last['ConfirmedCases_y']-df_last['ConfirmedCases_x'])/df_last['ConfirmedCases_x']
df_last['DeltaFatalities']=100*(df_last['Fatalities_y']-df_last['Fatalities_x'])/df_last['Fatalities_x']
df_last['DeltaConfirmedCases']=df_last['DeltaConfirmedCases'].apply(np.abs)
 df_last['DeltaFatalities']=df_last['DeltaFatalities'].apply(np.abs)
 df_last.fillna(0,inplace=True)
 df_last = df_last.replace([np.inf, -np.inf], 0 )<sort_values>
%%time
%%capture
env = janestreet.make_env()
learn.model.eval()
preds = []
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
x_tt[:, 1:] = for_loop(fillna_npwhere_njit, x_tt[:, 1:], f_mean)
 pred = 0.
 pred = learn.model(t.from_numpy(x_tt ).to(device, t.float)).sigmoid() [0][-1].item()
preds.append(pred)
 action = 1 if pred >=.5 else 0
pred_df.action = action
else:
pred_df.action = 0
env.predict(pred_df )
n_head = 5
d_lim1 = 120
d_lim2 = 200
sel_var ='ConfirmedCases'
sel_cols = ['Country_Region','Province_State',sel_var+'_x','Delta'+sel_var,sel_var+'Pos_x']
condition =(df_last['Delta'+sel_var]>d_lim1)&(df_last['Delta'+sel_var]<d_lim2)
 df_last[condition][sel_cols].sort_values(by=['Delta'+sel_var], ascending=True ).head(n_head )<sort_values>
preds = np.array(preds)
 preds.mean() , preds.std() , sum(preds >=.5), sum(preds < 5 )