Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<feature_engineering>
df = dtable.fread('/kaggle/input/jane-street-market-prediction/train.csv' ).to_pandas()
df = df.query('date > 85' ).reset_index(drop = True)
 df = df[df.weight > 0]
df.reset_index(inplace=True, drop=True)
 df = df.astype({c: np.float32 for c in df.select_dtypes(include='float64' ).columns})
 df_labels = df[['date', 'weight', 'resp_1', 'resp_2', 'resp_3', 'resp_4', 'resp']]
features = [c for c in df.columns if 'feature' in c]
df = df.drop(df_labels.columns, axis=1)
 df.fillna(df.mean() , inplace=True)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp_4', 'resp']
X = df[features]
y = np.stack([(df_labels[c] > 0 ).astype('int')for c in resp_cols] ).T
f_mean = np.mean(df[features[1:]].values, axis=0)
 device = 'cuda' if t.cuda.is_available() else 'cpu'
def subtext_repeation_in_df(df, col, subtext, num):
text = "".join(df[col])
 result = pd.DataFrame(columns = ['subtext', 'count'])
 i = 0
if(len(df)> 0)and(len(subtext)> 0):
for c in subtext:
for j in range(num):
cs = c*(j+1)
 result.loc[i,'count'] = text.count(cs)
 if c == ' ':
cs = cs.replace(' ','<space>')
 result.loc[i,'subtext'] = cs
i += 1
print('Number of all data is', len(df))
result = result[result['count'] > 0].reset_index(drop=True)
 display(result.sort_values(by='subtext'))
print('Text examples')
 problem_examples = pd.DataFrame(columns = ['problem_examples'])
 problem_examples['problem_examples'] = ''
for i in range(len(result)) :
problem_examples.loc[i,'problem_examples'] = df[df[col].str.find(result.loc[i,'subtext'])>-1].reset_index(drop=True ).loc[0, col]
problem_examples = problem_examples.drop_duplicates()
display(problem_examples )<statistical_test>
class Model(nn.Module):
def __init__(self):
super(Model, self ).__init__()
self.layers = nn.Sequential(LinBnDrop(130, 400, bn=True, p=0, act=Mish() , lin_first=False),
LinBnDrop(400, 800, bn=True, p=0.2289, act=Mish() , lin_first=False),
LinBnDrop(800, 400, bn=True, p=0.2289, act=Mish() , lin_first=False),
LinBnDrop(400, 5, bn=False, act=None, lin_first=False))
def forward(self, x):
x = self.layers(x)
 return x
print('Statistics for punctuation marks repetition in training data')
 subtext_repeation_in_df(train_data, 'text', list(string.punctuation), 10 )<remove_duplicates>
model_nn = Model()
model_nn = model_nn.to(device)
 learn = Learner(None, model_nn, loss_func=1)
 learn.load('/kaggle/input/roc-jstreet1/roc_model' )
print('Statistics for punctuation marks repetition in test data')
 subtext_repeation_in_df(test_data, 'text', list(string.punctuation), 10 )<predict_on_test>
@njit
def fillna_npwhere_njit(array, values):
if np.isnan(array.sum()):
array = np.where(np.isnan(array), values, array)
 return array
def for_loop(method, matrix, values):
for i in range(matrix.shape[0]):
matrix[i] = method(matrix[i], values)
 return matrix
if not with_kfold:
model = ClassificationModel(model_type, model_name, args=model_args, weight=weight)
 model.train_model(train_data)
 result, model_outputs, wrong_predictions = model.eval_model(train_data, acc=sklearn.metrics.accuracy_score)
 y_preds, _, = model.predict(test_data['text'])
 pred_train, _ = model.predict(train_data['text'] )<train_model>
%%time
%%capture
env = janestreet.make_env()
learn.model.eval()
preds = []
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
x_tt[:, 1:] = for_loop(fillna_npwhere_njit, x_tt[:, 1:], f_mean)
 pred = 0.
 pred = learn.model(t.from_numpy(x_tt ).to(device, t.float)).sigmoid() [0][-1].item()
preds.append(pred)
 action = 1 if pred >=.5 else 0
pred_df.action = action
else:
pred_df.action = 0
env.predict(pred_df )
<compute_test_metric><EOS> 
preds = np.array(preds)
 preds.mean() , preds.std() , sum(preds >=.5), sum(preds < 5 )