Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<save_to_csv>
df = dtable.fread('/kaggle/input/jane-street-market-prediction/train.csv' ).to_pandas()
df = df.query('date > 85' ).reset_index(drop = True)
 df = df[df.weight > 0]
df.reset_index(inplace=True, drop=True)
 df = df.astype({c: np.float32 for c in df.select_dtypes(include='float64' ).columns})
 df_labels = df[['date', 'weight', 'resp_1', 'resp_2', 'resp_3', 'resp_4', 'resp']]
features = [c for c in df.columns if 'feature' in c]
df = df.drop(df_labels.columns, axis=1)
 df.fillna(df.mean() , inplace=True)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp_4', 'resp']
X = df[features]
y = np.stack([(df_labels[c] > 0 ).astype('int')for c in resp_cols] ).T
f_mean = np.mean(df[features[1:]].values, axis=0)
 device = 'cuda' if t.cuda.is_available() else 'cpu'
submit = pd.read_csv('/kaggle/input/nlpstarted/submit2(1 ).csv')
 submit.to_csv('submission.csv', index=False )<install_modules>
class Model(nn.Module):
def __init__(self, input_size):
super(Model, self ).__init__()
self.batch_norm0 = nn.BatchNorm1d(input_size)
 self.layer1 = LinBnDrop(input_size, 256, bn=True, p=0, act=Mish() , lin_first=False)
 a = LinBnDrop(256, 256, bn=True, p=.12, act=Mish() , lin_first=False)
 b = LinBnDrop(256, 256, bn=True, p=.12, act=Mish() , lin_first=False)
 self.res1 = SequentialEx(a, b, MergeLayer())
 self.layer2 = LinBnDrop(256, 512, bn=True, p=0.12, act=Mish() , lin_first=False)
 a = LinBnDrop(512, 512, bn=True, p=.12, act=Mish() , lin_first=False)
 b = LinBnDrop(512, 512, bn=True, p=.12, act=Mish() , lin_first=False)
 self.res2 = SequentialEx(a, b, MergeLayer())
 self.layer3 = LinBnDrop(512, 640, bn=True, p=0.12, act=Mish() , lin_first=False)
 a = LinBnDrop(640, 640, bn=True, p=.12, act=Mish() , lin_first=False)
 b = LinBnDrop(640, 640, bn=True, p=.12, act=Mish() , lin_first=False)
 self.res3 = SequentialEx(a, b, MergeLayer())
 self.layer4 = LinBnDrop(640, 256, bn=True, p=0.12, act=Mish() , lin_first=False)
 a = LinBnDrop(256, 256, bn=True, p=.12, act=Mish() , lin_first=False)
 b = LinBnDrop(256, 256, bn=True, p=.12, act=Mish() , lin_first=False)
 self.res4 = SequentialEx(a, b, MergeLayer())
 self.layer5 = LinBnDrop(256, 128, bn=True, p=0.12, act=Mish() , lin_first=False)
 a = LinBnDrop(128, 128, bn=True, p=.12, act=Mish() , lin_first=False)
 b = LinBnDrop(128, 128, bn=True, p=.12, act=Mish() , lin_first=False)
 self.res5 = SequentialEx(a, b, MergeLayer())
 self.layer6 = nn.Linear(128, 5)
 def forward(self, x):
x = self.batch_norm0(x)
 x = self.layer1(x)
 x = self.res1(x)
 x = self.layer2(x)
 x = self.res2(x)
 x = self.layer3(x)
 x = self.res3(x)
 x = self.layer4(x)
 x = self.res4(x)
 x = self.layer5(x)
 x = self.res5(x)
 x = self.layer6(x)
 return x
VERSION = "nightly"
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py
!python pytorch-xla-env-setup.py --version $VERSION<install_modules>
@njit
def fillna_npwhere_njit(array, values):
if np.isnan(array.sum()):
array = np.where(np.isnan(array), values, array)
 return array
def for_loop(method, matrix, values):
for i in range(matrix.shape[0]):
matrix[i] = method(matrix[i], values)
 return matrix
!pip install transformers==3.0.0<save_to_csv>
model_nn = Model(len(features))
model_nn = model_nn.to(device)
 learn = Learner(None, model_nn, loss_func=1)
 learn.load('/kaggle/input/roclossjs5/deep_resnet_model' )
train = pd.read_csv('data/train.csv')
 train = train.fillna('')
 train['text'] = train['keyword'] + ' ' + train['location'] + ' '+ train['text']
train = train[['text','target']]
df_train, df_valid = train_test_split(train, test_size=0.2, random_state=42)
 df_train.to_csv('train.csv')
 df_valid.to_csv('valid.csv' )<load_pretrained>
%%time
%%capture
env = janestreet.make_env()
learn.model.eval()
preds = []
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
x_tt[:, 1:] = for_loop(fillna_npwhere_njit, x_tt[:, 1:], f_mean)
 pred = 0.
 pred = learn.model(t.from_numpy(x_tt ).to(device, t.float)).sigmoid() [0][-1].item()
preds.append(pred)
 action = 1 if pred >=.5 else 0
pred_df.action = action
else:
pred_df.action = 0
env.predict(pred_df )
class BertStyleModel(torch.nn.Module):
def __init__(self, model_type):
super().__init__()
self.model_type = model_type
if(model_type == 'roberta'):
config_path = 'roberta-base'
model_path = 'roberta-base'
config = RobertaConfig.from_pretrained(config_path)
 config.output_hidden_states = True
self.bert = RobertaModel.from_pretrained(model_path, config=config)
 elif(model_type == 'distilbert'):
config_path = 'distilbert-base-uncased'
config = DistilBertConfig.from_json_file(config_path)
 model_path = 'distilbert-base-uncased'
config.output_hidden_states = True
self.bert = DistilBertModel.from_pretrained(model_path, config=config)
 elif(model_type == 'bert'):
config_path = 'bert-base-uncased'
config = BertConfig.from_pretrained(config_path)
 config.output_hidden_states = True
model_path = 'bert-base-uncased'
self.bert = BertModel.from_pretrained(model_path, config=config)
 self.dropout = nn.Dropout(p=0.2)
 self.high_dropout = nn.Dropout(p=0.5)
 self.cls_token_head = nn.Sequential(
 nn.Dropout(0.2),
nn.Linear(768 * 4, 768),
nn.ReLU(inplace=True), 
)
 self.classifier = nn.Linear(768, 1)
 def forward(self, input_ids=None, attention_mask=None, token_type_ids=None):
if(self.model_type == 'roberta'):
outputs = self.bert(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
 hidden_layers = outputs[2]
elif(self.model_type == 'distilbert'):
outputs = self.bert(input_ids, attention_mask=attention_mask)
 hidden_layers = outputs[1]
elif(self.model_type == 'bert'):
outputs = self.bert(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
 hidden_layers = outputs[2]
hidden_states_cls_embeddings = [x[:, 0] for x in hidden_layers[-4:]]
x = torch.cat(hidden_states_cls_embeddings, dim=-1)
 cls_output = self.cls_token_head(x)
 logits = torch.mean(torch.stack([
self.classifier(self.high_dropout(cls_output))
for _ in range(5)
 ], dim=0), dim=0)
 outputs = logits
return outputs<set_options>
preds = np.array(preds)
 preds.mean() , preds.std() , sum(preds >=.5), sum(preds < 5 )