Kaggle competition: Otto Group Product Classification Challenge
BASE_ROBERTA = ".. /input/roberta-base"<feature_engineering>
class PreProcess:
def __init__(self):
self.df_otto_train = pd.read_csv('/kaggle/input/otto-group-product-classification-challenge/train.csv')
 self.df_otto_test = pd.read_csv('/kaggle/input/otto-group-product-classification-challenge/test.csv')
 def __call__(self):
conv = lambda x: int(x.replace('Class_', '')) - 1
self.df_otto_train['target2'] = self.df_otto_train.target.map(conv)
 x = self.df_otto_train.drop(['id', 'target', 'target2'], axis=1 ).values
y = self.df_otto_train.target2.values
x_test_id = self.df_otto_test.id.values
x_test = self.df_otto_test.drop(['id'], axis=1 ).values
x_train, x_valid, t_train, t_valid = train_test_split(x, y, test_size=0.2, shuffle=True)
 _, _, _, counts = np.unique(t_train, return_index=True, return_inverse=True, return_counts=True)
 loss_weight =(1 / torch.Tensor(counts)) *(x_train.shape[0]/len(np.unique(t_train)))
 loss_weight = loss_weight.to(device)
 return x_train, x_valid, t_train, t_valid, x_test, x_test_id, loss_weight
class TweetDataset(torch.utils.data.Dataset):
def __init__(self, df, max_len=96):
self.df = df
self.max_len = max_len
self.labeled = 'selected_text' in df
self.tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=f'{BASE_ROBERTA}/vocab.json',
merges_file=f'{BASE_ROBERTA}/merges.txt',
lowercase=True,
add_prefix_space=True)
 def __getitem__(self, index):
data = {}
row = self.df.iloc[index]
ids, masks, tweet, offsets = self.get_input_data(row)
 data['ids'] = ids
data['masks'] = masks
data['tweet'] = tweet
data['offsets'] = offsets
if self.labeled:
start_idx, end_idx = self.get_target_idx(row, tweet, offsets)
 data['start_idx'] = start_idx
data['end_idx'] = end_idx
return data
def __len__(self):
return len(self.df)
 def get_input_data(self, row):
tweet = " " + " ".join(row.text.lower().split())
 encoding = self.tokenizer.encode(tweet)
 sentiment_id = self.tokenizer.encode(row.sentiment ).ids
ids = [0] + sentiment_id + [2, 2] + encoding.ids + [2]
offsets = [(0, 0)] * 4 + encoding.offsets + [(0, 0)]
pad_len = self.max_len - len(ids)
 if pad_len > 0:
ids += [1] * pad_len
offsets += [(0, 0)] * pad_len
ids = torch.tensor(ids)
 masks = torch.where(ids != 1, torch.tensor(1), torch.tensor(0))
offsets = torch.tensor(offsets)
 return ids, masks, tweet, offsets
def get_target_idx(self, row, tweet, offsets):
selected_text = " " + " ".join(row.selected_text.lower().split())
 len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
target_idx = []
for j,(offset1, offset2)in enumerate(offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 start_idx = target_idx[0]
end_idx = target_idx[-1]
return start_idx, end_idx
def get_train_val_loaders(df, train_idx, val_idx, batch_size=8):
train_df = df.iloc[train_idx]
val_df = df.iloc[val_idx]
train_loader = torch.utils.data.DataLoader(
 TweetDataset(train_df),
batch_size=batch_size,
shuffle=True,
num_workers=2,
drop_last=True)
 val_loader = torch.utils.data.DataLoader(
 TweetDataset(val_df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 dataloaders_dict = {"train": train_loader, "val": val_loader}
return dataloaders_dict
def get_test_loader(df, batch_size=32):
loader = torch.utils.data.DataLoader(
 TweetDataset(df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 return loader<load_pretrained>
class MLP(nn.Module):
def __init__(self, in_dim, out_dim):
super(MLP, self ).__init__()
n_layers = 5
layers = []
features = [522, 1006, 1304, 706, 851]
p = [0.1287716033668473, 0.08497812991732616, 0.19230277862176845, 0.09567140996248122, 0.15776969001259095]
in_features = in_dim
for i in range(n_layers):
out_features = features[i]
layers.append(nn.Linear(in_features, out_features))
layers.append(nn.BatchNorm1d(out_features))
layers.append(nn.ReLU())
 layers.append(nn.Dropout(p[i]))
in_features = out_features
layers.append(nn.Linear(in_features, out_dim))
layers.append(nn.LogSoftmax(dim=-1))
self.network = nn.Sequential(*layers)
 def forward(self, x):
return self.network(x)
 class MLP_tune(nn.Module):
def __init__(self, trial, in_dim, out_dim):
super(MLP_tune, self ).__init__()
n_layers = trial.suggest_int("n_layers", 5, 10)
 layers = []
in_features = in_dim
for i in range(n_layers):
out_features = trial.suggest_int("n_units_l{}".format(i), in_dim, in_dim*20)
 layers.append(nn.Linear(in_features, out_features))
layers.append(nn.BatchNorm1d(out_features))
layers.append(nn.ReLU())
 p = trial.suggest_float("dropout_l{}".format(i), 0.0, 0.5)
 layers.append(nn.Dropout(p))
in_features = out_features
layers.append(nn.Linear(in_features, out_dim))
layers.append(nn.LogSoftmax(dim=-1))
self.network = nn.Sequential(*layers)
 def forward(self, x):
return self.network(x )
class TweetModel(nn.Module):
def __init__(self):
super(TweetModel, self ).__init__()
config = RobertaConfig.from_pretrained(
 f'{BASE_ROBERTA}/config.json', output_hidden_states=True)
 self.config = config
self.roberta = RobertaModel.from_pretrained(
 f'{BASE_ROBERTA}/pytorch_model.bin', config=self.config)
 self.dropout = nn.Dropout(p=0.2)
 self.high_dropout = nn.Dropout(p=0.5)
 n_weights = config.num_hidden_layers + 1
weights_init = torch.zeros(n_weights ).float()
weights_init.data[:-1] = -3
self.layer_weights = nn.Parameter(weights_init)
 self.intermediate_classifier = nn.Linear(config.hidden_size, config.hidden_size // 2)
 self.classifier = nn.Linear(config.hidden_size, self.config.num_labels)
 nn.init.normal_(self.classifier.weight, std=0.02)
 nn.init.normal_(self.classifier.bias, 0)
 def forward(self, input_ids, attention_mask):
_, _, hidden_layers = self.roberta(input_ids, attention_mask)

x = torch.stack([self.dropout(layer)for layer in hidden_layers],
dim=3)
 x =(torch.softmax(self.layer_weights, dim=0)* x ).sum(-1)
 x = torch.mean(torch.stack([
self.classifier(self.high_dropout(x))
for _ in range(5)
 ], dim=0), dim=0)
 start_logits, end_logits = x.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits<choose_model_class>
class Train:
def __init__(self):
self.x_train, self.x_valid, self.t_train, self.t_valid, self.x_test, self.x_test_id, self.loss_weight = PreProcess()()
def train(self, n_epochs, lr, weight_decay):
in_dim = self.x_train.shape[1]
out_dim = len(np.unique(self.t_train))
mlp = MLP(in_dim, out_dim ).to(device)
 optimizer = optim.RMSprop(mlp.parameters() , lr=lr, weight_decay=weight_decay)
 scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min')
 criterion = nn.NLLLoss(weight=self.loss_weight)
 ds_train = TensorDataset(torch.tensor(self.x_train ).float() , torch.tensor(self.t_train))
ds_valid = TensorDataset(torch.tensor(self.x_valid ).float() , torch.tensor(self.t_valid))
self.ds_test = TensorDataset(torch.tensor(self.x_test ).float())
 loader_train = DataLoader(ds_train, batch_size=256, shuffle=True)
 loader_valid = DataLoader(ds_valid, batch_size=256, shuffle=True)
 start_time = time.time()
for epoch in range(n_epochs):
losses_train = []
losses_valid = []
preds_train = []
preds_valid = []
trues_train = []
trues_valid = []
mlp.train()
for x, t in loader_train:
true = t.tolist()
trues_train.extend(true)
 mlp.zero_grad()
x = x.to(device)
 t = t.to(device)
 y = mlp.forward(x)
 loss = criterion(y, t.long())
 loss.backward()
optimizer.step()
pred = y.argmax(1 ).tolist()
preds_train.extend(pred)
 losses_train.append(loss.tolist())
 mlp.eval()
for x, t in loader_valid:
true = t.tolist()
trues_valid.extend(true)
 x = x.to(device)
 t = t.to(device)
 y = mlp.forward(x)
 loss = criterion(y, t.long())
 pred = y.argmax(1 ).tolist()
preds_valid.extend(pred)
 losses_valid.append(loss.tolist())
 print('EPOCH: {}, Train [Loss: {:.3f}, F1: {:.3f}, R: {:.3f}, P: {:.3f}], Valid [Loss: {:.3f}, F1: {:.3f}, R: {:.3f}, P: {:.3f}]'.format(
 epoch,
np.mean(losses_train),
f1_score(trues_train, preds_train, average='weighted'),
recall_score(trues_train, preds_train, average='weighted'),
precision_score(trues_train, preds_train, average='weighted'),
np.mean(losses_valid),
f1_score(trues_valid, preds_valid, average='weighted'),
recall_score(trues_valid, preds_valid, average='weighted'),
precision_score(trues_valid, preds_valid, average='weighted')
))
scheduler.step(np.mean(losses_valid))
elapsed_time = time.time() - start_time
print("elapsed_time:{:.1f}".format(elapsed_time / 60)+ "[min]")
 return mlp
def test(self, mlp):
loader_test = DataLoader(self.ds_test, batch_size=256, shuffle=False)
 preds_test = []
mlp.eval()
for x in loader_test:
x = x[0].to(device)
 y = mlp.forward(x)
 y = y.to('cpu' ).detach().numpy()
y = np.exp(y)
 preds_test.extend(y)
 return preds_test
def submit(self, preds_test):
df_sample_submission = pd.read_csv('/kaggle/input/otto-group-product-classification-challenge/sampleSubmission.csv')
 df_submission = pd.DataFrame(columns = df_sample_submission.columns)
 df_submission.id = self.x_test_id
df_submission[df_submission.columns[1:]] = preds_test
df_submission.to_csv('submission.csv', index=False)
 return
def get_optimizer(self, trial, model):
optimizer_names = ['Adam', 'MomentumSGD', 'rmsprop']
optimizer_name = trial.suggest_categorical('optimizer', optimizer_names)
 weight_decay = trial.suggest_loguniform('weight_decay', 1e-10, 1e-3)
 if optimizer_name == optimizer_names[0]:
adam_lr = trial.suggest_loguniform('adam_lr', 1e-5, 1e-1)
 optimizer = optim.Adam(model.parameters() , lr=adam_lr, weight_decay=weight_decay)
 elif optimizer_name == optimizer_names[1]:
momentum_sgd_lr = trial.suggest_loguniform('momentum_sgd_lr', 1e-5, 1e-1)
 optimizer = optim.SGD(model.parameters() , lr=momentum_sgd_lr, momentum=0.9, weight_decay=weight_decay)
 else:
rmsprop_lr = trial.suggest_loguniform('rmsprop_lr', 1e-5, 1e-1)
 optimizer = optim.RMSprop(model.parameters() , lr=rmsprop_lr, weight_decay=weight_decay)
 return optimizer
def objective(self, trial):
self.x_train, self.x_valid, self.t_train, self.t_valid, self.x_test, self.x_test_id, self.loss_weight = PreProcess()()
in_dim = self.x_train.shape[1]
out_dim = len(np.unique(self.t_train))
n_epochs = 100
mlp = MLP_tune(trial, in_dim, out_dim ).to(device)
 optimizer = self.get_optimizer(trial, mlp)
 scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min')
 criterion = nn.NLLLoss(weight=self.loss_weight)
 ds_train = TensorDataset(torch.tensor(self.x_train ).float() , torch.tensor(self.t_train))
ds_valid = TensorDataset(torch.tensor(self.x_valid ).float() , torch.tensor(self.t_valid))
loader_train = DataLoader(ds_train, batch_size=256, shuffle=True)
 loader_valid = DataLoader(ds_valid, batch_size=256, shuffle=True)
 start_time = time.time()
for epoch in range(n_epochs):
losses_train = []
losses_valid = []
preds_train = []
preds_valid = []
trues_train = []
trues_valid = []
mlp.train()
for x, t in loader_train:
true = t.tolist()
trues_train.extend(true)
 mlp.zero_grad()
x = x.to(device)
 t = t.to(device)
 y = mlp.forward(x)
 loss = criterion(y, t.long())
 loss.backward()
optimizer.step()
pred = y.argmax(1 ).tolist()
preds_train.extend(pred)
 losses_train.append(loss.tolist())
 mlp.eval()
for x, t in loader_valid:
true = t.tolist()
trues_valid.extend(true)
 x = x.to(device)
 t = t.to(device)
 y = mlp.forward(x)
 loss = criterion(y, t.long())
 pred = y.argmax(1 ).tolist()
preds_valid.extend(pred)
 losses_valid.append(loss.tolist())
 print('EPOCH: {}, Train [Loss: {:.3f}, F1: {:.3f}, R: {:.3f}, P: {:.3f}], Valid [Loss: {:.3f}, F1: {:.3f}, R: {:.3f}, P: {:.3f}]'.format(
 epoch,
np.mean(losses_train),
f1_score(trues_train, preds_train, average='weighted'),
recall_score(trues_train, preds_train, average='weighted'),
precision_score(trues_train, preds_train, average='weighted'),
np.mean(losses_valid),
f1_score(trues_valid, preds_valid, average='weighted'),
recall_score(trues_valid, preds_valid, average='weighted'),
precision_score(trues_valid, preds_valid, average='weighted')
))
scheduler.step(np.mean(losses_valid))
metric = np.mean(losses_valid)
 trial.report(metric, epoch)
 if trial.should_prune() :
raise optuna.exceptions.TrialPruned()
elapsed_time = time.time() - start_time
print("elapsed_time:{:.1f}".format(elapsed_time / 60)+ "[min]")
 return metric
model = TweetModel()<define_search_space>
def run_mlp(n_cross_valid):
n_cross_valid = n_cross_valid
preds_test_all = []
for n in range(n_cross_valid):
n_epochs = 50
lr = 7.560243948481283e-05
weight_decay = 0.0005896154168582327
train = Train()
mlp = train.train(n_epochs, lr, weight_decay)
 preds_test = train.test(mlp)
 preds_test_all.append(preds_test)
 preds_test = np.mean(preds_test_all, axis=0)
 return preds_test
output = model(torch.tensor([0, 1313, 2, 2, 42, 16, 41, 6344, 3545, 328, 2], dtype=torch.long ).unsqueeze(0),
torch.tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], dtype=torch.long ).unsqueeze(0), 
)
 start_pos_logit = output[0]
end_pos_logit = output[1]
print(start_pos_logit)
 print(end_pos_logit )<compute_test_metric>
def tune() :
train = Train()
study = optuna.create_study(direction="minimize")
 study.optimize(train.objective, n_trials=1000)
 pruned_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.PRUNED]
complete_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.COMPLETE]
print("Study statistics: ")
 print(" Number of finished trials: ", len(study.trials))
print(" Number of pruned trials: ", len(pruned_trials))
print(" Number of complete trials: ", len(complete_trials))
print("Best trial:")
 trial = study.best_trial
print(" Value: ", trial.value)
 print(" Params: ")
 for key, value in trial.params.items() :
print(" {}: {}".format(key, value))
def loss_fn(start_logits, end_logits, start_positions, end_positions):
ce_loss = nn.CrossEntropyLoss()
start_loss = ce_loss(start_logits, start_positions)
 end_loss = ce_loss(end_logits, end_positions)
 total_loss = start_loss + end_loss
return total_loss<statistical_test>
def run_lgb(n_cross_valid):
n_cross_valid = n_cross_valid
preds_test_all = []
for n in range(n_cross_valid):
x_train, x_valid, t_train, t_valid, x_test, x_test_id, _ = PreProcess()()
train_data = lgb.Dataset(x_train, label=t_train)
 eval_data = lgb.Dataset(x_valid, label=t_valid, reference=train_data)
 params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'multiclass',
"lambda_l1": 3.234625291884511e-05,
"lambda_l2": 0.0001436287400101119,
'num_leaves': 180,
"feature_fraction": 0.8978571608212189,
"bagging_fraction": 0.48465224832769926,
"bagging_freq": 1,
"min_child_samples": 15,
'learning_rate': 0.024535585298549177,
'num_class': 9,
'verbose': -1,
'device_type': 'gpu'
}
gbm = lgb.train(
 params,
train_data,
valid_sets=eval_data,
num_boost_round = 2000,
verbose_eval = 5,
early_stopping_rounds = 5 
)
 preds_test = gbm.predict(x_test)
 preds_test_all.append(preds_test)
 preds_test = np.mean(preds_test_all, axis=0)
 return preds_test
def get_selected_text(text, start_idx, end_idx, offsets):
selected_text = ""
for ix in range(start_idx, end_idx + 1):
selected_text += text[offsets[ix][0]: offsets[ix][1]]
if(ix + 1)< len(offsets)and offsets[ix][1] < offsets[ix + 1][0]:
selected_text += " "
return selected_text
def jaccard(str1, str2, smooth=0):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return(float(len(c)) + smooth)/(len(a)+ len(b)- len(c)+ smooth)
 def compute_jaccard_score(text, start_idx, end_idx, start_logits, end_logits, offsets):
start_pred = np.argmax(start_logits)
 end_pred = np.argmax(end_logits)
 if start_pred > end_pred:
pred = text
else:
pred = get_selected_text(text, start_pred, end_pred, offsets)
 true = get_selected_text(text, start_idx, end_idx, offsets)
 return jaccard(true, pred )<define_variables>
def lgb_objective(trial):
x_train, x_valid, t_train, t_valid, _, _, _ = PreProcess()()
train_data = lgb.Dataset(x_train, label=t_train)
 eval_data = lgb.Dataset(x_valid, label=t_valid, reference=train_data)
 params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'multiclass',
"lambda_l1": trial.suggest_float("lambda_l1", 1e-8, 10.0, log=True),
"lambda_l2": trial.suggest_float("lambda_l2", 1e-8, 10.0, log=True),
'num_leaves': trial.suggest_int("num_leaves", 2, 256),
"feature_fraction": trial.suggest_float("feature_fraction", 0.4, 1.0),
"bagging_fraction": trial.suggest_float("bagging_fraction", 0.4, 1.0),
"bagging_freq": trial.suggest_int("bagging_freq", 1, 7),
"min_child_samples": trial.suggest_int("min_child_samples", 5, 100),
'learning_rate': trial.suggest_float("learning_rate", 0.001, 1.0, log=True),
'num_class': 9,
'verbose': -1,
}
pruning_callback = optuna.integration.LightGBMPruningCallback(trial, "multi_logloss")
 gbm = lgb.train(
 params,
train_data,
valid_sets = eval_data,
num_boost_round = 2000,
verbose_eval = 5,
early_stopping_rounds = 5,
callbacks=[pruning_callback] 
)
 return gbm.best_score['valid_0']['multi_logloss']
def tune_lgb() :
pruner = optuna.pruners.MedianPruner(n_startup_trials=20, n_warmup_steps=20)
 study = optuna.create_study(direction="minimize", pruner = pruner)
 study.optimize(lgb_objective, n_trials=300)
 pruned_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.PRUNED]
complete_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.COMPLETE]
print("Study statistics: ")
 print(" Number of finished trials: ", len(study.trials))
print(" Number of pruned trials: ", len(pruned_trials))
print(" Number of complete trials: ", len(complete_trials))
print("Best trial:")
 trial = study.best_trial
print(" Value: {}".format(trial.value))
print(" Params: ")
 for key, value in trial.params.items() :
print(" {}: {}".format(key, value))
MODEL_FOLDS_BASE_FOLDER = ".. /input/tweeter-stratification-no-fine-tuning-seed-4444/"<load_from_csv>
def run_xgb(n_cross_valid):
n_cross_valid = n_cross_valid
preds_test_all = []
for n in range(n_cross_valid):
x_train, x_valid, t_train, t_valid, x_test, x_test_id, _ = PreProcess()()
train_data = xgb.DMatrix(x_train, label=t_train)
 eval_data = xgb.DMatrix(x_valid, label=t_valid)
 test_data = xgb.DMatrix(x_test)
 params = {
'objective': 'multi:softprob',
'eval_metric': 'mlogloss',
'num_class': 9,
"booster": "gbtree",
"lambda": 0.00047596412767141483,
"alpha": 0.14634359955484605,
"max_depth": 8,
"eta": 0.04169411268469549,
"gamma": 0.00048700929022167433,
"grow_policy": "depthwise",
'tree_method': 'gpu_hist'
}
evals = [(eval_data, 'eval')]
bst = xgb.train(
 params,
train_data,
num_boost_round = 2000,
evals = evals,
early_stopping_rounds = 5 
)
 print('Best Score:{0:.4f}, Iteratin:{1:d}, Ntree_Limit:{2:d}'.format(
 bst.best_score, bst.best_iteration, bst.best_ntree_limit))
preds_test = bst.predict(test_data, ntree_limit=bst.best_ntree_limit)
 preds_test_all.append(preds_test)
 preds_test = np.mean(preds_test_all, axis=0)
 return preds_test
%%time
test_df = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test_df['text'] = test_df['text'].astype(str)
 test_loader = get_test_loader(test_df)
 predictions = []
models = []
for fold in range(5):
model = TweetModel()
model.cuda()
model.load_state_dict(torch.load(f'{MODEL_FOLDS_BASE_FOLDER}/roberta_fold{fold+1}_{seed}.pth'))
model.eval()
models.append(model)
 count = 0
for data in test_loader:
ids = data['ids'].cuda()
masks = data['masks'].cuda()
tweet = data['tweet']
offsets = data['offsets'].numpy()
start_logits = []
end_logits = []
for model in models:
with torch.no_grad() :
output = model(ids, masks)
 start_logits.append(output[0].cpu().detach().numpy())
 end_logits.append(output[1].cpu().detach().numpy())
 start_logits = np.array(start_logits)
 end_logits = np.array(end_logits)
 start_logits = np.mean(start_logits, axis=0)
 end_logits = np.mean(end_logits, axis=0)
 for i in range(len(ids)) :
start_pred = np.argmax(start_logits[i])
 end_pred = np.argmax(end_logits[i])
 if start_pred > end_pred:
count += 1
pred = tweet[i]
else:
pred = get_selected_text(tweet[i], start_pred, end_pred, offsets[i])
 predictions.append(pred )<define_variables>
def xgb_objective(trial):
x_train, x_valid, t_train, t_valid, _, _, _ = PreProcess()()
train_data = xgb.DMatrix(x_train, label=t_train)
 eval_data = xgb.DMatrix(x_valid, label=t_valid)
 params = {
'objective': 'multi:softprob',
'eval_metric': 'mlogloss',
'num_class': 9,
"booster": trial.suggest_categorical("booster", ["gbtree", "gblinear", "dart"]),
"lambda": trial.suggest_float("lambda", 1e-8, 1.0, log=True),
"alpha": trial.suggest_float("alpha", 1e-8, 1.0, log=True),
'tree_method': 'gpu_hist'
}
if params["booster"] == "gbtree" or params["booster"] == "dart":
params["max_depth"] = trial.suggest_int("max_depth", 1, 9)
 params["eta"] = trial.suggest_float("eta", 1e-8, 1.0, log=True)
 params["gamma"] = trial.suggest_float("gamma", 1e-8, 1.0, log=True)
 params["grow_policy"] = trial.suggest_categorical("grow_policy", ["depthwise", "lossguide"])
 if params["booster"] == "dart":
params["sample_type"] = trial.suggest_categorical("sample_type", ["uniform", "weighted"])
 params["normalize_type"] = trial.suggest_categorical("normalize_type", ["tree", "forest"])
 params["rate_drop"] = trial.suggest_float("rate_drop", 1e-8, 1.0, log=True)
 params["skip_drop"] = trial.suggest_float("skip_drop", 1e-8, 1.0, log=True)
 pruning_callback = optuna.integration.XGBoostPruningCallback(trial, f"eval-{params['eval_metric']}")
 evals = [(eval_data, 'eval')]
bst = xgb.train(
 params,
train_data,
num_boost_round = 500,
evals = evals,
early_stopping_rounds = 5,
callbacks=[pruning_callback] 
)
 return bst.best_score
def tune_xgb() :
pruner = optuna.pruners.MedianPruner(n_startup_trials=20, n_warmup_steps=20)
 study = optuna.create_study(direction="minimize", pruner = pruner)
 study.optimize(xgb_objective, n_trials=500)
 pruned_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.PRUNED]
complete_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.COMPLETE]
print("Study statistics: ")
 print(" Number of finished trials: ", len(study.trials))
print(" Number of pruned trials: ", len(pruned_trials))
print(" Number of complete trials: ", len(complete_trials))
print("Best trial:")
 trial = study.best_trial
print(" Value: {}".format(trial.value))
print(" Params: ")
 for key, value in trial.params.items() :
print(" {}: {}".format(key, value))
pred_length = [len(p)for p in predictions]<save_to_csv>
def submit(preds_test, x_test_id, alg):
df_sample_submission = pd.read_csv('/kaggle/input/otto-group-product-classification-challenge/sampleSubmission.csv')
 df_submission = pd.DataFrame(columns = df_sample_submission.columns)
 df_submission.id = x_test_id
df_submission[df_submission.columns[1:]] = preds_test
df_submission.to_csv('submission_{}.csv'.format(alg), index=False)
 def run() :
_, _, _, _, _, x_test_id, _ = PreProcess()()
n_cross_valid = 40
preds_test_all = []
preds_test_mlp = run_mlp(n_cross_valid)
 submit(preds_test_mlp, x_test_id, 'mlp')
 preds_test_all.append(preds_test_mlp)
 preds_test_lgb = run_lgb(n_cross_valid)
 submit(preds_test_lgb, x_test_id, 'lgb')
 preds_test_all.append(preds_test_lgb)
 preds_test_xgb = run_xgb(n_cross_valid)
 submit(preds_test_xgb, x_test_id, 'xgb')
 preds_test_all.append(preds_test_xgb)
 preds_test = np.mean(preds_test_all, axis=0)
 submit(preds_test, x_test_id, 'mlp_lgb_xgb')
 run()