Kaggle competition: Jane Street Market Prediction
import numpy as np
import pandas as pd
import datetime as datetime
from sklearn import preprocessing<set_options>
pd.set_option('display.max_columns', 100)
 pd.set_option('display.max_rows', 100)
 seed = 42
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
if not os.path.exists("results"):
os.mkdir("results")
 TRAINING = True
read_path = '/kaggle/input/jane-street-market-prediction/train.csv'
save_path = os.path.join("results", str(seed))
device = torch.device("cuda:0")
 if not os.path.exists(save_path):
os.mkdir(save_path)
register_matplotlib_converters() 
<import_modules>
TRAINING = True
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestRegressor
from xgboost import XGBRegressor<load_from_csv>
train = pd.read_csv(read_path)
 features = [c for c in train.columns if 'feature' in c]
f_mean = train[features].mean()
train = train.loc[train.weight > 0].reset_index(drop = True)
 train[features] = train[features].fillna(f_mean)
 train = train.astype("float32")
 train['action'] =(train['resp'] > 0 ).astype('int')
 train['action1'] =(train['resp_1'] > 0 ).astype('int')
 train['action2'] =(train['resp_2'] > 0 ).astype('int')
 train['action3'] =(train['resp_3'] > 0 ).astype('int')
 train['action4'] =(train['resp_4'] > 0 ).astype('int')
 targets = ['resp']
targets = ['action', 'action1', 'action2', 'action3', 'action4']
def add_features(df, features):
new_features = copy.deepcopy(features)
 df["cross_1_2"] = df["feature_1"] /(df["feature_2"] + 1e-5)
 df["cross_41_42_43"] = df["feature_41"] + df["feature_42"] + df["feature_43"]
new_features.extend(["cross_1_2", "cross_41_42_43"])
 return df, new_features
train, train_features = add_features(train, features)
training_set = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 training_set.shape<load_from_csv>
test = train.loc[(train.date >= 450)&(train.date < 500)].reset_index(drop=True)
test_set = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 test_set.shape<filter>
class MyDataset:
def __init__(self, df, features, targets):
self.features = df[features].values
self.labels = df[targets].values
self.weights = df['weight'].values
def __len__(self):
return len(self.labels)
 def __getitem__(self, idx):
feat_ = torch.tensor(self.features[idx], dtype=torch.float)
 label_ = torch.tensor(self.labels[idx], dtype=torch.float)
 weight_ = torch.tensor(self.weights[idx], dtype=torch.float)
 return feat_, label_, weight_
N_train=len(training_set[training_set['Country_Region']==my_country])
 N_train<filter>
class Model(nn.Module):
def __init__(self, all_feat_cols, target_cols):
super(Model, self ).__init__()
self.batch_norm0 = nn.BatchNorm1d(len(all_feat_cols))
self.dropout0 = nn.Dropout(0.1)
 dropout_rate = 0.1
hidden_size = 256
self.dense1 = nn.Linear(len(all_feat_cols), hidden_size)
 self.batch_norm1 = nn.BatchNorm1d(hidden_size)
 self.dropout1 = nn.Dropout(dropout_rate)
 self.dense2 = nn.Linear(hidden_size+len(all_feat_cols), hidden_size)
 self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(dropout_rate)
 self.dense3 = nn.Linear(hidden_size+hidden_size, hidden_size)
 self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(dropout_rate)
 self.dense4 = nn.Linear(hidden_size+hidden_size, hidden_size)
 self.batch_norm4 = nn.BatchNorm1d(hidden_size)
 self.dropout4 = nn.Dropout(dropout_rate)
 self.dense5 = nn.Linear(2 * hidden_size, 128)
 self.batch_norm5 = nn.BatchNorm1d(128)
 self.dropout5 = nn.Dropout(dropout_rate)
 self.dense6 = nn.Linear(128, 128)
 self.batch_norm6 = nn.BatchNorm1d(128)
 self.dropout6 = nn.Dropout(dropout_rate)
 self.dense7 = nn.Linear(128, len(target_cols))
self.Relu = nn.ReLU(inplace=True)
 self.PReLU = nn.PReLU()
self.LeakyReLU = nn.LeakyReLU(negative_slope=0.01, inplace=True)
 self.RReLU = nn.RReLU()
def forward(self, x):
x = self.batch_norm0(x)
 x = self.dropout0(x)
 x1 = self.dense1(x)
 x1 = self.batch_norm1(x1)
 x1 = self.RReLU(x1)
 x1 = self.dropout1(x1)
 x = torch.cat([x, x1], 1)
 x2 = self.dense2(x)
 x2 = self.batch_norm2(x2)
 x2 = self.RReLU(x2)
 x2 = self.dropout2(x2)
 x = torch.cat([x1, x2], 1)
 x3 = self.dense3(x)
 x3 = self.batch_norm3(x3)
 x3 = self.RReLU(x3)
 x3 = self.dropout3(x3)
 x = torch.cat([x2, x3], 1)
 x4 = self.dense4(x)
 x4 = self.batch_norm4(x4)
 x4 = self.RReLU(x4)
 x4 = self.dropout4(x4)
 x = torch.cat([x3, x4], 1)
 x = self.dense5(x)
 x = self.batch_norm5(x)
 x = self.RReLU(x)
 x = self.dropout5(x)
 x = self.dense6(x)
 x = self.batch_norm6(x)
 x = self.RReLU(x)
 x = self.dropout6(x)
 x = self.dense7(x)
 return x
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets, weights=None):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets, weights)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
class EarlyStopping:
def __init__(self, patience=7, mode="max", delta=0.) :
self.patience = patience
self.counter = 0
self.mode = mode
self.best_score = None
self.early_stop = False
self.delta = delta
if self.mode == "min":
self.val_score = np.Inf
else:
self.val_score = -np.Inf
def __call__(self, epoch_score, model, model_path):
if self.mode == "min":
score = -1.0 * epoch_score
else:
score = np.copy(epoch_score)
 if self.best_score is None:
self.best_score = score
self.save_checkpoint(epoch_score, model, model_path)
 elif score < self.best_score:
self.counter += 1
print('EarlyStopping counter: {} out of {}'.format(self.counter, self.patience))
if self.counter >= self.patience:
self.early_stop = True
else:
self.best_score = score
self.save_checkpoint(epoch_score, model, model_path)
 self.counter = 0
def save_checkpoint(self, epoch_score, model, model_path):
if epoch_score not in [-np.inf, np.inf, -np.nan, np.nan]:
print('Validation score improved.Saving model!')
 torch.save(model.state_dict() , model_path)
 self.val_score = epoch_score
N_test=len(test_set[test_set['Country_Region']==my_country])
 N_test<prepare_output>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for feature, label, weight in dataloader:
feature = feature.to(device)
 label = label.to(device)
 weight = weight.to(device)
 optimizer.zero_grad()
outputs = model(feature)
 loss = loss_fn(outputs.reshape(-1, 1), label.reshape(-1, 1))
loss.backward()
optimizer.step()
if scheduler:
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def inference_fn(model, dataloader, device):
model.eval()
preds = []
labels = []
final_loss = 0
for feature, label, weight in dataloader:
feature = feature.to(device)
 label = label.to(device)
 with torch.no_grad() :
outputs = model(feature)
 loss = loss_fn(outputs.reshape(-1, 1), label.reshape(-1, 1))
preds.append(outputs.sigmoid().cpu().numpy())
 labels.append(label.cpu().numpy())
 final_loss += loss.item()
final_loss/= len(dataloader)
 preds = np.concatenate(preds, axis=0)
 labels = np.concatenate(labels, axis=0)
 return preds, labels, final_loss
def utility_score(date, weight, resp, action):
values = weight * resp * action
to_bincount = {}
for d, v in zip(date, values):
to_bincount.setdefault(d, [] ).append(v)
 Pi = []
for val in to_bincount.values() :
Pi.append(np.sum(val))
Pi = np.array(Pi)
 count_i = len(np.unique(date))
t = np.sum(Pi)/ np.sqrt(np.sum(Pi ** 2)) * np.sqrt(250 / count_i)
 u = np.clip(t, 0, 6)* np.sum(Pi)
 return u
def loss_mse(preds, targets):
return(( preds-targets)**2 ).mean()
def loss_ce(preds, targets, weight=None):
return F.binary_cross_entropy_with_logits(preds, targets, weight )
df_test = test_set.copy()
df_test['DsType']='test'
df_test['ConfirmedCases']=0
df_test['Fatalities']=0
df_test.info()<concatenate>
f_handler=open('out.txt', 'w')
 sys.stdout=f_handler
if TRAINING:
batch_size = 1024
label_smoothing = 1e-2
learning_rate = 1e-3
start_time = time.time()
oof = np.zeros(len(train['action']))
gkf = GroupKFold(n_splits = 5)
 models = []
for fold,(tr, te)in enumerate(gkf.split(train['action'].values, train['action'].values, train['date'].values)) :
train_set = MyDataset(train.loc[tr], train_features, targets)
 train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=1)
 valid_set = MyDataset(train.loc[te], train_features, targets)
 valid_loader = DataLoader(valid_set, batch_size=batch_size, shuffle=False, num_workers=1)
 torch.cuda.empty_cache()
device = torch.device("cuda:0")
 model = Model(train_features, targets)
 model.to(device)
 optimizer = torch.optim.Adam(model.parameters() , lr=learning_rate)
 loss_fn = SmoothBCEwLogits(smoothing=label_smoothing)
 ckp_path = f'JSModel_{fold}.pth'
es = EarlyStopping(patience=100, mode="min")
 for epoch in range(100):
if epoch > 0 &(epoch % 5 == 0):
learning_rate *= 0.9
train_loss = train_fn(model, optimizer, None, loss_fn, train_loader, device)
 valid_pred, true_valid_labels, valid_loss = inference_fn(model, valid_loader, device)
 auc_score = roc_auc_score(true_valid_labels.reshape(-1, 1), valid_pred.reshape(-1, 1))
valid_pred = np.median(valid_pred, axis=1)
 valid_pred = np.where(valid_pred >= 0.5, 1, 0 ).astype(int)
 u_score = utility_score(date=train.loc[te].date.values.reshape(-1),
weight=train.loc[te].weight.values.reshape(-1),
resp=train.loc[te].resp.values.reshape(-1),
action=valid_pred.reshape(-1))
u_score_max = utility_score(date=train.loc[te].date.values.reshape(-1),
weight=train.loc[te].weight.values.reshape(-1),
resp=train.loc[te].resp.values.reshape(-1),
action=train.loc[te].action.values.reshape(-1))
print(f"FOLD{fold} EPOCH:{epoch:3}, train_loss:{train_loss:.5f}, valid_loss:{valid_loss:.5f}, auc:{auc_score:.5f},u_score:{u_score:.5f},max_u_score:{u_score_max:.5f}"
f"time: {(time.time() - start_time)/ 60:.2f}min")
 es(valid_loss, model, model_path=ckp_path)
 if es.early_stop:
print("Early stopping")
 break
models.append(model)
 test_set = MyDataset(test, train_features, targets)
 test_loader = DataLoader(test_set, batch_size=batch_size, shuffle=False, num_workers=1)
 test_preds, true_test_labels, test_loss = inference_fn(model, test_loader, device)
 test_preds = np.median(test_preds, axis=1)
 test_preds = np.where(test_preds >= 0.5, 1, 0 ).astype(int)
 auc_score = roc_auc_score(test['action'].values.reshape(-1, 1), test_preds)
 print("auc on test set:", auc_score)
 u_score = utility_score(date=test.date.values.reshape(-1),
weight=test.weight.values.reshape(-1),
resp=test.resp.values.reshape(-1),
action=test_preds.reshape(-1))
print("utility score on test set:", u_score)
 u_score2 = utility_score(date=test.date.values.reshape(-1),
weight=test.weight.values.reshape(-1),
resp=test.resp.values.reshape(-1),
action=test.action.values.reshape(-1))
print("max utility score on test set(if using true action):", u_score2)
 break
df_union=pd.concat([df_train,df_test],sort=False ).copy()
df_union.fillna('ND', inplace = True )<feature_engineering>
if not TRAINING:
models = []
for i in [0, 1, 2, 3, 4]:
torch.cuda.empty_cache()
device = torch.device("cuda:0")
 model = Model(train_features, targets)
 model.to(device)
 model.eval()
ckp_path = f'/kaggle/input/skeleton-with-pytorch/JSModel_{i}.pth'
model.load_state_dict(torch.load(ckp_path))
models.append(model )
df_union['Month']=df_union['Date'].apply(lambda s : int(s.replace('-','')[4:6]))
df_union['Day']=df_union['Date'].apply(lambda s : int(s.replace('-','')[6:9]))
df_union['Date']=df_union['Date'].apply(lambda s : datetime.datetime.strptime(s, '%Y-%m-%d'))
df_union['DateOrd']=df_union['Date'].apply(lambda s : s.toordinal())
 df_union['Province_Norm']=df_union['Country_Region']+'-'+df_union['Province_State']
df_union.tail()<categorify>
if TRAINING:
models = []
for i in [1, 3]:
torch.cuda.empty_cache()
device = torch.device("cuda:0")
 model = Model(train_features, targets)
 model.to(device)
 model.eval()
ckp_path = f'./JSModel_{i}.pth'
model.load_state_dict(torch.load(ckp_path))
models.append(model )
le1 = preprocessing.LabelEncoder()
le1.fit(df_union['Country_Region'])
 df_union['Country'] = le1.transform(df_union['Country_Region'] )<categorify>
models = [models[1], models[3], models[2]]
batch_size = 4096
label_smoothing = 1e-2
loss_fn = SmoothBCEwLogits(smoothing=label_smoothing)
 test_pred = np.zeros(( len(test), len(targets)))
 test_set = MyDataset(test, train_features, targets)
 test_loader = DataLoader(test_set, batch_size=4096, shuffle=False, num_workers=4)
 for id_, model in enumerate(models):
test_pred_, _, __ = inference_fn(model, test_loader, device)
 test_pred += test_pred_ / len(models)
 test_pred_ = np.median(test_pred_, axis=1)
 test_pred_ = np.where(test_pred_ >= 0.50, 1, 0 ).astype(int)
 auc_score = roc_auc_score(test['action'].values.reshape(-1, 1), test_pred_)
 u_score = utility_score(date=test.date.values.reshape(-1),
weight=test.weight.values.reshape(-1),
resp=test.resp.values.reshape(-1),
action=test_pred_.reshape(-1))
print("model", id_, "auc:", auc_score, "u_score:", u_score)
 test_preds = np.median(test_pred, axis=1)
 test_preds = np.where(test_preds >= 0.50, 1, 0 ).astype(int)
 auc_score = roc_auc_score(test['action'].values.reshape(-1, 1), test_preds)
 u_score = utility_score(date=test.date.values.reshape(-1),
weight=test.weight.values.reshape(-1),
resp=test.resp.values.reshape(-1),
action=test_preds.reshape(-1))
print(auc_score, u_score )
le2 = preprocessing.LabelEncoder()
le2.fit(df_union['Province_Norm'])
 df_union['Province'] = le2.transform(df_union['Province_Norm'])
 df_union.drop('Province_Norm',axis=1,inplace=True )<define_variables>
env = janestreet.make_env()
env_iter = env.iter_test()
th = 0.5
for(test_df, pred_df)in tqdm(env_iter):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt.sum()):
x_tt = np.nan_to_num(x_tt)+ np.isnan(x_tt)* f_mean.values.reshape(1, -1)
 feature_inp = pd.DataFrame(x_tt)
 feature_inp.columns = features
feature_inp, _ = add_features(feature_inp,features)
 feature_inp = torch.tensor(feature_inp.values, dtype=torch.float ).to(device)
 pred = np.zeros(( 1, len(targets)))
 for model in models:
pred += model(feature_inp ).detach().cpu().numpy()
pred /= len(models)
 pred = pred.mean(axis=1 ).item()
pred_df.action = int(pred >= 0)
 else:
pred_df.action = 0
env.predict(pred_df )