Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<choose_model_class>
pio.templates.default = "ggplot2"
env = janestreet.make_env()
iter_test = env.iter_test()
%%time
clf = MultinomialNB(fit_prior=False)
 pipe = Pipeline([('preprop', PrePropTextTransformer(lemmatize=False,
lowercase=False,
remove_stopwords=False,
remove_accents=True,
normalize_contractions=False,
normalize_URL=True,
normalize_emoji=True,
normalize_html=True,
normalize_punctuation=False 
)) ,('vectorizer', TfidfVectorizer()),('predictor', clf)])
 pipe.fit(train_df["text"], train_df["target"])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions)) 
<choose_model_class>
print("Reading dataset using CUDA dataframes...", end='')
 train_cudf = cudf.read_csv('/kaggle/input/jane-street-market-prediction/train.csv')
 train_data = train_cudf.to_pandas()
del train_cudf
meta_cudf = cudf.read_csv('/kaggle/input/jane-street-market-prediction/features.csv')
 meta_data = meta_cudf.to_pandas()
del meta_cudf
sample_prediction_df = pd.read_csv('.. /input/jane-street-market-prediction/example_sample_submission.csv')
 print('Finished.
 ')
 print(f'Train shape: {format(train_data.shape)}')
 print(f'Features meta shape: {format(meta_data.shape)}' )
%%time
clf = MultinomialNB(fit_prior=False)
 pipe = Pipeline([('preprop', PrePropTextTransformer(lemmatize=False,
lowercase=False,
remove_stopwords=False,
remove_accents=True,
normalize_contractions=False,
normalize_URL=True,
normalize_emoji=True,
normalize_html=True,
normalize_punctuation=False 
)) ,('vectorizer', TfidfVectorizer(tokenizer=TweetTokenizer().tokenize)) ,('predictor', clf)])
 pipe.fit(train_df["text"], train_df["target"])
 predictions = pipe.predict(dev_df["text"]);
print(classification_report(dev_df['target'], predictions)) 
<categorify>
print('Preprocessing data...', end='')
 features = [c for c in train_data.columns if 'feature' in c]
train_data = train_data[train_data['weight'] > 0].reset_index(drop = True)
 train_data[features] = train_data[features].fillna(method = 'ffill' ).fillna(0)
 train_data['action'] =(train_data['resp'].values > 0 ).astype(int)
 print('Finished.' )
!pip install transformers
def mean_across_all_tokens(hidden_states):
return torch.mean(hidden_states[-1], dim=1)
 def sum_all_tokens(hidden_states):
return torch.sum(hidden_states[-1], dim=1)
 def concat_all_tokens(hidden_states):
batch_size, max_tokens, emb_dim = hidden_states[-1].shape
return torch.reshape(hidden_states[-1],(batch_size, max_tokens * emb_dim))
def CLS_token_embedding(hidden_states):
return hidden_states[-1][:, 0, :]
class BertTransformer(BaseEstimator, TransformerMixin):
def __init__(
 self,
max_length: int = 60,
tokenizer = BertTokenizer.from_pretrained("bert-base-uncased"),
model = BertModel.from_pretrained("bert-base-uncased", output_hidden_states=True),
embedding_func = mean_across_all_tokens,
combine_sentence_tokens=True 
):
self.tokenizer = tokenizer;
self.combine_sentence_tokens = combine_sentence_tokens;
self.embedding_func = embedding_func;
self.model = model
self.model.eval()
self.max_length = max_length
def _tokenize(self, text_list: List[str])-> Tuple[torch.tensor, torch.tensor]:
input_ids = self.tokenizer.batch_encode_plus(text_list,
add_special_tokens=True,
max_length=self.max_length,
pad_to_max_length=True 
)["input_ids"]
return torch.LongTensor(input_ids)
 def _tokenize_and_predict(self, text_list: List[str])-> torch.tensor:
input_ids_tensor = self._tokenize(text_list)
 out = self.model(input_ids=input_ids_tensor)
 hidden_states = out[2]
if(self.combine_sentence_tokens):
return self.embedding_func(hidden_states)
 else:
return hidden_states[-1]
def transform(self, text_list: List[str], batch_size=32):
if isinstance(text_list, pd.Series):
text_list = text_list.tolist()
batches = minibatch(text_list, size=batch_size)
 predictions = []
for batch in batches:
with torch.no_grad() :
batch_predictions = self._tokenize_and_predict(batch)
 predictions.append(batch_predictions)
 return torch.cat(predictions, dim=0)
 def fit(self, X, y=None):

 return self<categorify>
print('Creating XGBclassifier...
 ', end='')
 parameters = {'max_depth': 8,
'learning_rate': 0.015,
'random_state': 42,
'tree_method': 'gpu_hist',
'min_child_weight': 0.30,
'subsample': 0.46,
'colsample_bytree': 0.99,
'eval_metric': 'auc',
'gamma': 9.8,
'objective': 'binary:logistic'}
X_train = train_data.loc[train_data['date'] > 80, features].values
y_train = train_data.loc[train_data['date'] > 80, 'action'].values
d_train = xgb.DMatrix(X_train, y_train)
 %time clf = xgb.train(parameters, d_train, 1175)
 print('Finished training the classifier.' )
bertTransformer = BertTransformer(combine_sentence_tokens=False)
 bertTransformer.transform(["pablo", "I love Pablo"] ).shape<find_best_model_class>
@njit
def fast_fillna(array, values):
if np.isnan(array.sum()):
array = np.where(np.isnan(array), values, array)
 return array
train_data.loc[0, features[1:]] = fast_fillna(train_data.loc[0, features[1:]].values, 0 )
<choose_model_class><EOS> 
tmp = np.zeros(len(features))
for(test_df, sample_prediction_df)in tqdm(iter_test):
if test_df['weight'].item() > 0:
X_test = test_df.loc[:, features].values
X_test[0, :] = fast_fillna(X_test[0, :], tmp)
 tmp = X_test[0, :]
d_test = xgb.DMatrix(X_test)
 y_preds = clf.predict(d_test)
 sample_prediction_df.action = np.where(y_preds >= 0.5, 1, 0 ).astype(int)
 else:
sample_prediction_df.action = 0
env.predict(sample_prediction_df )