Kaggle competition: Jane Street Market Prediction
import numpy as np
import pandas as pd 
<load_from_csv>
warnings.filterwarnings("ignore" )
df1 = pd.read_csv('.. /input/combining-your-model-with-a-model-without-o-0d991a/combining_submission.csv')
 df2 = pd.read_csv('.. /input/simple-lightgbm-without-blending/submission.csv' )<feature_engineering>
DEVICE = torch.device("cuda:0")if torch.cuda.is_available() else torch.device("cpu" )
df1['target'] =.5*df1['target'] +.5*df2['target']<save_to_csv>
def load_preprocess_data(filename='.. /input/jane-street-market-prediction/train.csv', isTrainData=True):
dtype = None
if isTrainData:
dtype = {
'date' : 'int64',
'weight' : 'float64',
'resp' : 'float64',
'ts_id' : 'int64',
'feature_0' : 'float64'
}
else:
dtype = {
'date' : 'int64',
'weight' : 'float64',
'feature_0' : 'float64'
}
for i in range(1, 130):
k = 'feature_' + str(i)
 dtype[k] = 'float32'
X = pd.read_csv(filename, dtype=dtype)
 resp_cols = ['resp_1', 'resp_2', 'resp_3','resp_4', 'resp']
X = X.query('date > 85')
 X = X[X['weight'] != 0].reset_index(drop = True)
 y = np.stack([(X[c] > 0 ).astype('int')for c in resp_cols] ).T
f_columns = [c for c in X.columns if "feature" in c]
if isTrainData:
X.drop(columns=['weight', 'date', 'resp', 'resp_1', 'resp_2', 'resp_3', 'resp_4', 'ts_id'], inplace=True)
 else:
X.drop(columns=['weight', 'date'], inplace=True)
 preprocess_pipe = Pipeline([ 
("imputer", SimpleImputer(missing_values=np.nan, strategy='mean')) ,
])
 X = preprocess_pipe.fit_transform(X)
 gc.collect()
return X, y, preprocess_pipe
df1.to_csv('ens.csv',index=False )<import_modules>
X, y, preprocess_pipe = load_preprocess_data()
X.shape
import numpy as np
import pandas as pd
import time
import lightgbm as lgb
from sklearn.model_selection import StratifiedKFold, KFold
from sklearn.metrics import mean_squared_error
from sklearn.metrics import log_loss<load_from_csv>
dtype = {
'feature' : 'str',
'tag_0' : 'int8'
}
for i in range(1, 29):
k = 'tag_' + str(i)
 dtype[k] = 'int8'
features_df = pd.read_csv('.. /input/jane-street-market-prediction/features.csv', usecols=range(1,30), dtype=dtype)
 N_FEATURES = features_df.shape[0]
N_FEAT_TAGS = features_df.shape[1]
features_df.head()
%%time
df_train = pd.read_csv('.. /input/predicting-outliers-to-improve-your-score/train_clean.csv')
 df_test = pd.read_csv('.. /input/predicting-outliers-to-improve-your-score/test_clean.csv' )<drop_column>
del features_df
gc.collect()
N_FEATURES, N_FEAT_TAGS
df_train = df_train[df_train['outliers'] == 0]
target = df_train['target']
del df_train['target']
features = [c for c in df_train.columns if c not in ['card_id', 'first_active_month','outliers']]
categorical_feats = [c for c in features if 'feature_' in c]<init_hyperparams>
def format_time(elapsed):

 elapsed_rounded = int(round(( elapsed)))
 return str(datetime.timedelta(seconds=elapsed_rounded))
def compute_metrics(labels, pred_logits):
preds = pred_logits.argmax(-1)
 precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='macro')
 acc = accuracy_score(labels, preds)
 mcc = matthews_corrcoef(labels, preds)
 softmax = nn.Softmax(dim=1)
 pred_pr = softmax(torch.tensor(pred_logits))
auc = roc_auc_score(labels, pred_pr[:, 1])
 metrics = {
'mcc' : mcc,
'accuracy' : acc,
'f1' : f1,
'precision': precision,
'recall' : recall,
'auc' : auc
}
return metrics
def compute_multiclass_metrics(labels, pred_logits):

 metrics = []
n_class = labels.shape[1]
for i in range(n_class):
label = labels[:, i]
prob1 = pred_logits[:, i]
prob0 = 1 - prob1
pred = np.hstack(( prob0.reshape(( -1, 1)) , prob1.reshape(( -1, 1))))
pred = compute_metrics(label, pred)
 metrics.append(pred)
 metrics_df = pd.DataFrame(metrics)
 metrics_df = metrics_df.mean()
return metrics_df.to_dict()
param = {'objective':'regression',
'num_leaves': 31,
'min_data_in_leaf': 25,
'max_depth': 7,
'learning_rate': 0.01,
'lambda_l1':0.13,
"boosting": "gbdt",
"feature_fraction":0.85,
'bagging_freq':8,
"bagging_fraction": 0.9 ,
"metric": 'rmse',
"verbosity": -1,
"random_state": 2333}<split>
class MyTrainer:
def __init__(self, model, args, train_dataset, eval_dataset, compute_metrics=compute_multiclass_metrics):
self.model = model
self.args = args
self.train_dataset = train_dataset
self.eval_dataset = eval_dataset
self.compute_metrics = compute_metrics
self.isTrained = False
self.device = self.get_device_type()
self.optimizer = AdamW(model.parameters() ,
lr = args.learning_rate,
eps = args.adam_epsilon 
)
 self.epochs = self.args.num_train_epochs
self.train_dataloader, self.validation_dataloader, self.lr_scheduler, self.num_training_steps = self.get_dataLoaders()
return
def get_device_type(self):
if torch.cuda.is_available() :
device = torch.device("cuda")
 print('There are %d GPU(s)available.' % torch.cuda.device_count())
 print('We will use the GPU:', torch.cuda.get_device_name(0))
else:
print('No GPU available, using the CPU instead.')
 device = torch.device("cpu")
 return device
def get_dataLoaders(self):
if isinstance(self.train_dataset, torch.utils.data.IterableDataset):
train_sampler = None
else:
train_sampler = RandomSampler(self.train_dataset)
 train_dataloader = DataLoader(
 self.train_dataset,
sampler = train_sampler,
batch_size = self.args.per_device_train_batch_size,
num_workers = 8 
)
 validation_dataloader = None
if self.eval_dataset:
validation_dataloader = DataLoader(
 self.eval_dataset,
sampler = SequentialSampler(self.eval_dataset),
batch_size = self.args.per_device_eval_batch_size,
num_workers = 8 
)
 num_training_steps = len(train_dataloader)* self.epochs
lr_scheduler = get_linear_schedule_with_warmup(self.optimizer,
num_warmup_steps = self.args.warmup_steps,
num_training_steps = num_training_steps)
 return train_dataloader, validation_dataloader, lr_scheduler, num_training_steps
def test_iterate_dataloader(self):
for step, batch in enumerate(self.train_dataloader):
print(step)
 print(batch)
 break
return
def train(self):
seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 training_stats = []
total_t0 = time.time()
min_val_loss = 9999
min_train_loss = 9999
max_val_auc = -1
step = 0
for epoch_i in range(0, self.epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, self.epochs))
print('Training...')
 t0 = time.time()
total_train_loss = 0
self.model.train()
for stp, batch in enumerate(self.train_dataloader):
step += 1
if step % 50 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(self.train_dataloader), elapsed))
if(self.args.max_steps > 0 and self.args.max_steps < step)or \ 
(self.args.eval_steps> 0 and step % self.args.eval_steps==0 and step>0):
avg_train_loss = total_train_loss / step
training_time = format_time(time.time() - t0)
 if self.validation_dataloader:
print("Running Validation...")
 avg_val_loss, avg_val_f1, avg_val_mcc, avg_val_auc, avg_val_precision, avg_val_recall, avg_val_accuracy, validation_time = self.evaluate()
training_stats.append({
'epoch' : epoch_i + 1,
'training_loss' : avg_train_loss,
'eval_loss' : avg_val_loss,
'eval_f1' : avg_val_f1,
'eval_mcc' : avg_val_mcc,
'eval_precision': avg_val_precision,
'eval_recall' : avg_val_recall,
'eval_auc' : avg_val_auc,
'eval_accuracy' : avg_val_accuracy,
'training_time' : training_time,
'eval_time' : validation_time
})
 if avg_val_loss < min_val_loss:
min_val_loss = avg_val_loss
torch.save(self.model.state_dict() , "jane_emb_nn_clf_rishi.pt")
 if self.args.max_steps > 0 and self.args.max_steps < step :
print("")
 print("Training complete!")
 print("Total training took {:}(h:mm:ss)".format(format_time(time.time() -total_t0)))
 self.isTrained = True
self.plot_train_stats_regression(training_stats)
 return training_stats
self.model.zero_grad()
', batch)
 batch['features'] = batch['features'].to(self.device ).float()
if 'labels' in batch:
batch['labels'] = batch['labels'].to(self.device ).long()
output = self.model(**batch)
 loss = output.loss
logits = output.logits
total_train_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(self.model.parameters() , 1.0)
 self.optimizer.step()
self.lr_scheduler.step()
avg_train_loss = total_train_loss / len(self.train_dataloader)
 training_time = format_time(time.time() - t0)
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epcoh took: {:}".format(training_time))
if self.validation_dataloader:
print("
Running Validation...")
 avg_val_loss, avg_val_f1, avg_val_mcc, avg_val_auc, avg_val_precision, avg_val_recall, avg_val_accuracy, validation_time = self.evaluate()
training_stats.append({
'epoch' : epoch_i + 1,
'training_loss' : avg_train_loss,
'eval_loss' : avg_val_loss,
'eval_f1' : avg_val_f1,
'eval_mcc' : avg_val_mcc,
'eval_precision': avg_val_precision,
'eval_recall' : avg_val_recall,
'eval_auc' : avg_val_auc,
'eval_accuracy' : avg_val_accuracy,
'training_time' : training_time,
'eval_time' : validation_time
})
 if avg_val_loss < min_val_loss:
min_val_loss = avg_val_loss
torch.save(model.state_dict() , "jane_emb_nn_clf_loss.pt")
 if avg_val_auc > max_val_auc:
max_val_auc = avg_val_auc
torch.save(self.model.state_dict() , "jane_emb_nn_clf_auc.pt")
 else:
training_stats.append({
'epoch' : epoch_i + 1,
'training_loss' : avg_train_loss,
'training_time' : training_time,
})
 if avg_train_loss < min_train_loss:
min_train_loss = avg_train_loss
torch.save(self.model.state_dict() , "jane_emb_nn_clf_rishi.pt")
 print("")
 print("Training complete!")
 print("Total training took {:}(h:mm:ss)".format(format_time(time.time() -total_t0)))
 self.isTrained = True
self.model.eval()
try:
self.plot_train_stats(training_stats)
 except:
pass
torch.save(self.model.state_dict() , "jane_emb_nn_clf_end.pt")
 return training_stats
def evaluate(self):
t0 = time.time()
self.model.eval()
total_eval_mcc = 0
total_eval_f1 = 0
total_eval_precision = 0
total_eval_recall = 0
total_eval_auc = 0
total_eval_accuracy = 0
total_eval_loss = 0
nb_eval_steps = 0
for batch in self.validation_dataloader:
with torch.no_grad() :
batch['features'] = batch['features'].to(self.device ).float()
batch['labels'] = batch['labels'].to(self.device ).long()
output = self.model(**batch)
 loss = output.loss
logits = output.logits
total_eval_loss += loss.item()
logits = logits.detach().cpu().numpy()
label_ids = batch['labels'].detach().cpu().numpy()
metrics = self.compute_metrics(label_ids, logits)
 total_eval_mcc += metrics['mcc']
total_eval_f1 += metrics['f1']
total_eval_precision += metrics['precision']
total_eval_recall += metrics['recall']
total_eval_auc += metrics['auc']
total_eval_accuracy += metrics['accuracy']
avg_val_f1 = total_eval_f1 / len(self.validation_dataloader)
 print(" F1: {0:.3f}".format(avg_val_f1))
avg_val_mcc = total_eval_mcc / len(self.validation_dataloader)
 print(" MCC: {0:.3f}".format(avg_val_mcc))
avg_val_precision = total_eval_precision / len(self.validation_dataloader)
 print(" Precision: {0:.3f}".format(avg_val_precision))
avg_val_recall = total_eval_recall / len(self.validation_dataloader)
 print(" Recall: {0:.3f}".format(avg_val_recall))
avg_val_auc = total_eval_auc / len(self.validation_dataloader)
 print(" AUC: {0:.3f}".format(avg_val_auc))
avg_val_accuracy = total_eval_accuracy / len(self.validation_dataloader)
 print(" Accuracy: {0:.3f}".format(avg_val_accuracy))
avg_val_loss = total_eval_loss / len(self.validation_dataloader)
 validation_time = format_time(time.time() - t0)
 print(" Validation Loss: {0:.2f}".format(avg_val_loss))
print(" Validation took: {:}".format(validation_time))
return avg_val_loss, avg_val_f1, avg_val_mcc, avg_val_auc, avg_val_precision, avg_val_recall, avg_val_accuracy, validation_time
def plot_train_stats(self, training_stats):

 mccs = accuracies = f1_scores = precisions = recalls = auc = losses = epochs = -1
epochs = len(training_stats)
 if 'eval_mcc' in training_stats[0]:
mccs = [e['eval_mcc'] for e in training_stats]
sns.lineplot(x=np.arange(1, epochs + 1), y=mccs, label='val_mcc')
 if 'eval_accuracy' in training_stats[0]:
accuracies = [e['eval_accuracy'] for e in training_stats]
sns.lineplot(x=np.arange(1, epochs + 1), y=accuracies, label='val_accuracy')
 if 'eval_f1' in training_stats[0]:
f1_scores = [e['eval_f1'] for e in training_stats]
sns.lineplot(x=np.arange(1, epochs + 1), y=f1_scores, label='val_f1')
 if 'eval_precision' in training_stats[0]:
precisions = [e['eval_precision'] for e in training_stats]
sns.lineplot(x=np.arange(1, epochs + 1), y=precisions, label='val_precision')
 if 'eval_recall' in training_stats[0]:
recalls = [e['eval_recall'] for e in training_stats]
sns.lineplot(x=np.arange(1, epochs + 1), y=recalls, label='val_recall')
 if 'eval_auc' in training_stats[0]:
auc = [e['eval_auc'] for e in training_stats]
sns.lineplot(x=np.arange(1, epochs + 1), y=mccs, label='val_auc')
 if 'eval_loss' in training_stats[0]:
losses = [e['eval_loss'] for e in training_stats]
if 'training_loss' in training_stats[0]:
tr_losses = [e['training_loss'] for e in training_stats]
sns.lineplot(x=np.arange(1, epochs + 1), y=tr_losses, label='tr_losses')
 plt.show()
print('mccs :', mccs)
 print('accuracies :', accuracies)
 print('precisions :', precisions)
 print('recalls :', recalls)
 print('f1_scores :', f1_scores)
 print('auc :', auc)
 print('losses :', losses)
 print('tr_losses :', tr_losses)
 plt.plot(recalls, precisions, marker='.', label='Prcision-Recall Curve')
 plt.xlabel('Recall')
 plt.ylabel('Precision')
 plt.legend()
plt.show()
return
def getTrainedModel(self):
if self.isTrained:
return self.model
return None
def predict(self, prediction_dataset, isRemoveLabels=True):

 prediction_sampler = SequentialSampler(prediction_dataset)
 prediction_dataloader = DataLoader(prediction_dataset, sampler=prediction_sampler, batch_size=self.args.per_device_eval_batch_size)
 print('Predicting labels for {:,} test sentences...'.format(len(prediction_dataset)))
 self.model.eval()
predictions = []
for batch in prediction_dataloader:
if 'labels' in batch and isRemoveLabels:
batch.pop('labels')
 batch['features'] = batch['features'].to(self.device ).float()
if 'labels' in batch:
batch['labels'] = batch['labels'].to(self.device ).long()
with torch.no_grad() :
outputs = model(**batch)
 logits = outputs.logits
logits = logits.detach().cpu().numpy()
predictions.append(logits)
 print('Done predictions for ', len(predictions), '/', len(prediction_dataloader), 'batches')
 print('Done prediction')
 pred_logits = np.concatenate(predictions, axis=0)
 return pred_logits, None, None
%%time
folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=2333)
 oof = np.zeros(len(df_train))
predictions = np.zeros(len(df_test))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(df_train,df_train['outliers'].values)) :
print("fold {}".format(fold_))
trn_data = lgb.Dataset(df_train.iloc[trn_idx][features], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(df_train.iloc[val_idx][features], label=target.iloc[val_idx])
 num_round = 10000
clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval= 100, early_stopping_rounds = 200)
 oof[val_idx] = clf.predict(df_train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(df_test[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(mean_squared_error(oof, target)**0.5))<create_dataframe>
class JaneDataset(torch.utils.data.Dataset):
def __init__(self, df, labels=None):
if isinstance(df, pd.DataFrame):
df.reset_index(drop=True, inplace=True)
 if isinstance(labels, pd.Series):
labels.reset_index(drop=True, inplace=True)
 self.df = df
self.labels = labels
return
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
if torch.is_tensor(idx):
idx = idx.tolist()
sample = None
if isinstance(self.df, pd.DataFrame):
features = np.array(self.df.loc[idx, f_columns])
 else:
features = self.df[idx]
if self.labels is not None:
sample = {'features': features, 'labels': self.labels[idx]}
else:
sample = {'features': features}
return sample
model_without_outliers = pd.DataFrame({"card_id":df_test["card_id"].values})
 model_without_outliers["target"] = predictions<load_from_csv>
train_dataset = JaneDataset(X, y)
 eval_dataset = None
train_dataset[1]
%%time
df_train = pd.read_csv('.. /input/predicting-outliers-to-improve-your-score/train_clean.csv')
 df_test = pd.read_csv('.. /input/predicting-outliers-to-improve-your-score/test_clean.csv' )<drop_column>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
target = df_train['outliers']
del df_train['outliers']
del df_train['target']<define_variables>
class FFN(nn.Module):
def __init__(self, inputCount=160, outputCount=0, hiddenLayerCounts=[80, 40, 20], drop_prob=0.2, nonlin=nn.SiLU()):
super(FFN, self ).__init__()
self.nonlin = nonlin
self.dropout = nn.Dropout(drop_prob)
 self.dense1 = nn.Linear(inputCount, hiddenLayerCounts[0])
 self.batchnorm1 = nn.BatchNorm1d(hiddenLayerCounts[0])
 self.dense2 = nn.Linear(hiddenLayerCounts[0], hiddenLayerCounts[1])
 self.batchnorm2 = nn.BatchNorm1d(hiddenLayerCounts[1])
 self.dense3 = nn.Linear(hiddenLayerCounts[1], hiddenLayerCounts[2])
 self.batchnorm3 = nn.BatchNorm1d(hiddenLayerCounts[2])
 self.outDense = None
if outputCount > 0:
self.outDense = nn.Linear(hiddenLayerCounts[-1], outputCount)
 self.outActivtn = None
if outputCount == 1 or outputCount == 2:
self.outActivtn = nn.Sigmoid()
elif outputCount > 0:
self.outActivtn = nn.Softmax(dim=-1)
 return
def forward(self, X, **kwargs):
X = self.dropout(self.nonlin(self.batchnorm1(self.dense1(X))))
X = self.dropout(self.nonlin(self.batchnorm2(self.dense2(X))))
X = self.dropout(self.nonlin(self.batchnorm3(self.dense3(X))))
if self.outActivtn:
X = self.outActivtn(self.outDense(X))
return X
features = [c for c in df_train.columns if c not in ['card_id', 'first_active_month']]
categorical_feats = [c for c in features if 'feature_' in c]<init_hyperparams>
class Clf_Model(nn.Module):
def __init__(self, embed_dim=N_FEAT_TAGS, csv_file='.. /input/jane-street-market-prediction/features.csv'):
super(Clf_Model, self ).__init__()
global N_FEAT_TAGS
N_FEAT_TAGS = 29
dtype = {'tag_0' : 'int8'}
for i in range(1, 29):
k = 'tag_' + str(i)
 dtype[k] = 'int8'
t_df = pd.read_csv(csv_file, usecols=range(1,N_FEAT_TAGS+1), dtype=dtype)
 t_df['tag_29'] = np.array([1] +([0] *(t_df.shape[0]-1)) ).astype('int8')
 self.features_tag_matrix = torch.tensor(t_df.to_numpy())
 N_FEAT_TAGS += 1
self.embed_dim = embed_dim
self.tag_embedding = nn.Embedding(N_FEAT_TAGS+1, embed_dim)
 self.tag_weights = nn.Linear(N_FEAT_TAGS, 1)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(inputCount=(130+embed_dim), outputCount=0, hiddenLayerCounts=[(160+embed_dim),(100+embed_dim),(50+embed_dim)], drop_prob=0.25)
 self.outDense = nn.Linear(50+embed_dim, 5)
 self.criterion = SmoothBCEwLogits(smoothing=0.005)
 return
def features2emb(self):

 all_tag_idxs = torch.LongTensor(np.arange(N_FEAT_TAGS)).to(DEVICE)
 tag_bools = self.features_tag_matrix
f_emb = self.tag_embedding(all_tag_idxs ).repeat(130, 1, 1)
 f_emb = f_emb * tag_bools[:, :, None]
s = torch.sum(tag_bools, dim=1)
 f_emb = torch.sum(f_emb, dim=-2)/ s[:, None]
return f_emb
def forward(self, features, labels=None):

 features = features.view(-1, N_FEATURES)
 if labels is not None:
labels = labels.view(-1, 5)
 f_emb = self.features2emb()
features_2 = torch.matmul(features, f_emb)
 features = torch.hstack(( features, features_2))
x = self.ffn(features)
 out_logits = self.outDense(x)
 if labels is None:
out_probs = F.sigmoid(out_logits)
 Probs = namedtuple('Probs',['logits'])
 out_probs = Probs(out_probs)
 return out_probs
batchLoss = self.criterion(out_logits, labels.float())
 out_probs = F.sigmoid(out_logits)
 Loss_Logits = namedtuple('Loss_Logits',['loss','logits'])
 loss_logits = Loss_Logits(batchLoss, out_probs)
 return loss_logits
param = {'num_leaves': 31,
'min_data_in_leaf': 30,
'objective':'binary',
'max_depth': 6,
'learning_rate': 0.01,
"boosting": "rf",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9 ,
"bagging_seed": 11,
"metric": 'binary_logloss',
"lambda_l1": 0.1,
"verbosity": -1,
"random_state": 2333}<split>
model = Clf_Model()
try:
model.load_state_dict(torch.load(".. /input/jane-smooth-embnn5-160-100-50/jane_smooth_embnn5_160_100_50.pt"))
except:
model.load_state_dict(torch.load(".. /input/jane-smooth-embnn5-160-100-50/jane_smooth_embnn5_160_100_50.pt", map_location='cpu'))
model = model.float().to(DEVICE )
%%time
folds = KFold(n_splits=5, shuffle=True, random_state=15)
 oof = np.zeros(len(df_train))
predictions = np.zeros(len(df_test))
feature_importance_df = pd.DataFrame()
start = time.time()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(df_train.values, target.values)) :
print("fold n°{}".format(fold_))
trn_data = lgb.Dataset(df_train.iloc[trn_idx][features], label=target.iloc[trn_idx], categorical_feature=categorical_feats)
 val_data = lgb.Dataset(df_train.iloc[val_idx][features], label=target.iloc[val_idx], categorical_feature=categorical_feats)
 num_round = 10000
clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=100, early_stopping_rounds = 200)
 oof[val_idx] = clf.predict(df_train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(df_test[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(log_loss(target, oof)) )<prepare_output>
training_args = TrainingArguments(
 output_dir = './results',
num_train_epochs= 250,
warmup_steps = 300,
eval_steps = 0,
max_steps = 0,
learning_rate = 5e-3,
per_device_train_batch_size= 80000,
per_device_eval_batch_size = 80000, 
)
 trainer = MyTrainer(
 model = model,
args = training_args,
train_dataset = train_dataset,
eval_dataset = eval_dataset,
)
df_outlier_prob = pd.DataFrame({"card_id":df_test["card_id"].values})
 df_outlier_prob["target"] = predictions
df_outlier_prob.head()<prepare_output>
outlier_id = pd.DataFrame(df_outlier_prob.sort_values(by='target',ascending = False ).head(25000)['card_id'] )<load_from_csv>
model.eval()
for batch in trainer.train_dataloader:
with torch.no_grad() :
batch['features'] = batch['features'].to(trainer.device ).float()
batch['labels'] = batch['labels'].to(trainer.device ).long()
output = model(**batch)
 break
best_submission = pd.read_csv('.. /input/predicting-outliers-to-improve-your-score/3.695.csv' )<merge>
def predict_torch(test_df):
test_df.drop(columns=['weight', 'date'], inplace=True)
 test_df.reset_index(drop=True, inplace=True)
 test_df = preprocess_pipe.transform(test_df)
 test_df = torch.tensor(test_df ).float().view(-1, 130 ).to(DEVICE)
 predictions = []
for i in range(test_df.shape[0]):
pred_p = model(test_df ).logits.detach().cpu().numpy().reshape(( -1, 5))
predictions.append(pred_p)
 predictions = np.vstack(predictions)
 predictions = np.median(predictions, axis=1)
 return(predictions >= 0.5 ).astype(int )
most_likely_liers = best_submission.merge(outlier_id,how='right')
 most_likely_liers.head()<filter>
env = janestreet.make_env()
env_iter = env.iter_test()
%%time
for card_id in most_likely_liers['card_id']:
model_without_outliers.loc[model_without_outliers['card_id']==card_id,'target']\
= most_likely_liers.loc[most_likely_liers['card_id']==card_id,'target'].values<save_to_csv>
for test_df, pred_df in env_iter:
if test_df["weight"].item() > 0:
pred_df.action = predict_torch(test_df)
 else:
pred_df.action = 0
env.predict(pred_df )
<import_modules><EOS> 
print('Done !' )