Kaggle competition: Otto Group Product Classification Challenge
<SOS> metric: MulticlassLoss Kaggle data source: otto-group-product-classification-challenge<load_from_csv>
!pip install optuna
%%time
test_df = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test_df['text'] = test_df['text'].astype(str)
 test_loader = get_test_loader(test_df)
 predictions = []
models = []
for fold in range(skf.n_splits):
model = TweetModel()
model.cuda()
model.load_state_dict(torch.load(f'roberta_fold{fold+1}.pth'))
model.eval()
models.append(model)
 for data in test_loader:
ids = data['ids'].cuda()
masks = data['masks'].cuda()
tweet = data['tweet']
offsets = data['offsets'].numpy()
start_logits = []
end_logits = []
for model in models:
with torch.no_grad() :
output = model(ids, masks)
 start_logits.append(torch.softmax(output[0], dim=1 ).cpu().detach().numpy())
 end_logits.append(torch.softmax(output[1], dim=1 ).cpu().detach().numpy())
 start_logits = np.mean(start_logits, axis=0)
 end_logits = np.mean(end_logits, axis=0)
 for i in range(len(ids)) :
start_pred = np.argmax(start_logits[i])
 end_pred = np.argmax(end_logits[i])
 if start_pred > end_pred:
pred = tweet[i]
else:
pred = get_selected_text(tweet[i], start_pred, end_pred, offsets[i])
 predictions.append(pred )<save_to_csv>
class PreProcess:
def __init__(self):
self.df_otto_train = pd.read_csv('/kaggle/input/otto-group-product-classification-challenge/train.csv')
 self.df_otto_test = pd.read_csv('/kaggle/input/otto-group-product-classification-challenge/test.csv')
 def __call__(self):
conv = lambda x: int(x.replace('Class_', '')) - 1
self.df_otto_train['target2'] = self.df_otto_train.target.map(conv)
 x = self.df_otto_train.drop(['id', 'target', 'target2'], axis=1 ).values
y = self.df_otto_train.target2.values
x_test_id = self.df_otto_test.id.values
x_test = self.df_otto_test.drop(['id'], axis=1 ).values
x_train, x_valid, t_train, t_valid = train_test_split(x, y, test_size=0.2, shuffle=True)
 _, _, _, counts = np.unique(t_train, return_index=True, return_inverse=True, return_counts=True)
 loss_weight =(1 / torch.Tensor(counts)) *(x_train.shape[0]/len(np.unique(t_train)))
 loss_weight = loss_weight.to(device)
 return x_train, x_valid, t_train, t_valid, x_test, x_test_id, loss_weight
sub_df = pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 sub_df['selected_text'] = predictions
sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('!!!!', '!')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('.. ', '.')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('...', '.')if len(x.split())==1 else x)
 sub_df.to_csv('submission.csv', index=False)
 sub_df.head()<define_variables>
class MLP(nn.Module):
def __init__(self, in_dim, out_dim):
super(MLP, self ).__init__()
n_layers = 5
layers = []
features = [522, 1006, 1304, 706, 851]
p = [0.1287716033668473, 0.08497812991732616, 0.19230277862176845, 0.09567140996248122, 0.15776969001259095]
in_features = in_dim
for i in range(n_layers):
out_features = features[i]
layers.append(nn.Linear(in_features, out_features))
layers.append(nn.BatchNorm1d(out_features))
layers.append(nn.ReLU())
 layers.append(nn.Dropout(p[i]))
in_features = out_features
layers.append(nn.Linear(in_features, out_dim))
layers.append(nn.LogSoftmax(dim=-1))
self.network = nn.Sequential(*layers)
 def forward(self, x):
return self.network(x)
 class MLP_tune(nn.Module):
def __init__(self, trial, in_dim, out_dim):
super(MLP_tune, self ).__init__()
n_layers = trial.suggest_int("n_layers", 5, 10)
 layers = []
in_features = in_dim
for i in range(n_layers):
out_features = trial.suggest_int("n_units_l{}".format(i), in_dim, in_dim*20)
 layers.append(nn.Linear(in_features, out_features))
layers.append(nn.BatchNorm1d(out_features))
layers.append(nn.ReLU())
 p = trial.suggest_float("dropout_l{}".format(i), 0.0, 0.5)
 layers.append(nn.Dropout(p))
in_features = out_features
layers.append(nn.Linear(in_features, out_dim))
layers.append(nn.LogSoftmax(dim=-1))
self.network = nn.Sequential(*layers)
 def forward(self, x):
return self.network(x )
MY_CONFIG_PATH = '.. /input/base-pre-cv072469/base_pre_cv072469/base_pre_cv072469_config_20200612_150232.pkl'
MODEL1_FOLDER = ".. /input/large-cv0726033/RoBERTa-large-0726033 /"
MODEL1_CONFIG = "config_20200611_000411.pkl"
MODEL2_FOLDER = ".. /input/base-cv0722436/RoBERTa-base-0722436/"
MODEL2_CONFIG = "config_20200610_230921.pkl"
MODEL3_FOLDER = ".. /input/large-pre-cv072592/large_pre_cv072592/"
MODEL3_CONFIG = "large_pre_cv072592_config_20200612_112512.pkl"
MODEL4_FOLDER = ".. /input/base-pre-cv072469/base_pre_cv072469/"
MODEL4_CONFIG = "base_pre_cv072469_config_20200612_150232.pkl"
MODEL5_FOLDER = ".. /input/ooka-large-seed101/"
MODEL5_CONFIG = "config_20200614_233735.pkl"
MODEL6_FOLDER = ".. /input/ooka-base-seed101/ooka_base_seed101/"
MODEL6_CONFIG = "config_20200612_183046.pkl"
MODEL7_FOLDER = ".. /input/hiromu-large-seed101/"
MODEL7_CONFIG = "config_20200614_134559.pkl"
MODEL8_FOLDER = ".. /input/hiromu-base-seed101/hiromu_base_seed101/"
MODEL8_CONFIG = "config_20200614_124718.pkl"
ROBERTA_PATH = ".. /input/roberta-base"
REMODEL0 = ".. /input/hiromu-reranking-cv07346/hiromu_reranking_cv0.7346/reranking_model_fold0.bin"
REMODEL1 = ".. /input/hiromu-reranking-cv07346/hiromu_reranking_cv0.7346/reranking_model_fold1.bin"
REMODEL2 = ".. /input/hiromu-reranking-cv07346/hiromu_reranking_cv0.7346/reranking_model_fold2.bin"
REMODEL3 = ".. /input/hiromu-reranking-cv07346/hiromu_reranking_cv0.7346/reranking_model_fold3.bin"
REMODEL4 = ".. /input/hiromu-reranking-cv07346/hiromu_reranking_cv0.7346/reranking_model_fold4.bin"
REMODEL5 = ".. /input/oof-bucketing-fold0-3/reranking_model_fold0.bin"
REMODEL6 = ".. /input/oof-bucketing-fold0-3/reranking_model_fold1.bin"
REMODEL7 = ".. /input/oof-bucketing-fold0-3/reranking_model_fold2.bin"
REMODEL8 = ".. /input/oof-bucketing-fold0-3/reranking_model_fold3.bin"
REMODEL9 = ".. /input/oof-bucketing-fold4/reranking_model_fold4.bin"<install_modules>
class Train:
def __init__(self):
self.x_train, self.x_valid, self.t_train, self.t_valid, self.x_test, self.x_test_id, self.loss_weight = PreProcess()()
def train(self, n_epochs, lr, weight_decay):
in_dim = self.x_train.shape[1]
out_dim = len(np.unique(self.t_train))
mlp = MLP(in_dim, out_dim ).to(device)
 optimizer = optim.RMSprop(mlp.parameters() , lr=lr, weight_decay=weight_decay)
 scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min')
 criterion = nn.NLLLoss(weight=self.loss_weight)
 ds_train = TensorDataset(torch.tensor(self.x_train ).float() , torch.tensor(self.t_train))
ds_valid = TensorDataset(torch.tensor(self.x_valid ).float() , torch.tensor(self.t_valid))
self.ds_test = TensorDataset(torch.tensor(self.x_test ).float())
 loader_train = DataLoader(ds_train, batch_size=256, shuffle=True)
 loader_valid = DataLoader(ds_valid, batch_size=256, shuffle=True)
 start_time = time.time()
for epoch in range(n_epochs):
losses_train = []
losses_valid = []
preds_train = []
preds_valid = []
trues_train = []
trues_valid = []
mlp.train()
for x, t in loader_train:
true = t.tolist()
trues_train.extend(true)
 mlp.zero_grad()
x = x.to(device)
 t = t.to(device)
 y = mlp.forward(x)
 loss = criterion(y, t.long())
 loss.backward()
optimizer.step()
pred = y.argmax(1 ).tolist()
preds_train.extend(pred)
 losses_train.append(loss.tolist())
 mlp.eval()
for x, t in loader_valid:
true = t.tolist()
trues_valid.extend(true)
 x = x.to(device)
 t = t.to(device)
 y = mlp.forward(x)
 loss = criterion(y, t.long())
 pred = y.argmax(1 ).tolist()
preds_valid.extend(pred)
 losses_valid.append(loss.tolist())
 print('EPOCH: {}, Train [Loss: {:.3f}, F1: {:.3f}, R: {:.3f}, P: {:.3f}], Valid [Loss: {:.3f}, F1: {:.3f}, R: {:.3f}, P: {:.3f}]'.format(
 epoch,
np.mean(losses_train),
f1_score(trues_train, preds_train, average='weighted'),
recall_score(trues_train, preds_train, average='weighted'),
precision_score(trues_train, preds_train, average='weighted'),
np.mean(losses_valid),
f1_score(trues_valid, preds_valid, average='weighted'),
recall_score(trues_valid, preds_valid, average='weighted'),
precision_score(trues_valid, preds_valid, average='weighted')
))
scheduler.step(np.mean(losses_valid))
elapsed_time = time.time() - start_time
print("elapsed_time:{:.1f}".format(elapsed_time / 60)+ "[min]")
 return mlp
def test(self, mlp):
loader_test = DataLoader(self.ds_test, batch_size=256, shuffle=False)
 preds_test = []
mlp.eval()
for x in loader_test:
x = x[0].to(device)
 y = mlp.forward(x)
 y = y.to('cpu' ).detach().numpy()
y = np.exp(y)
 preds_test.extend(y)
 return preds_test
def submit(self, preds_test):
df_sample_submission = pd.read_csv('/kaggle/input/otto-group-product-classification-challenge/sampleSubmission.csv')
 df_submission = pd.DataFrame(columns = df_sample_submission.columns)
 df_submission.id = self.x_test_id
df_submission[df_submission.columns[1:]] = preds_test
df_submission.to_csv('submission.csv', index=False)
 return
def get_optimizer(self, trial, model):
optimizer_names = ['Adam', 'MomentumSGD', 'rmsprop']
optimizer_name = trial.suggest_categorical('optimizer', optimizer_names)
 weight_decay = trial.suggest_loguniform('weight_decay', 1e-10, 1e-3)
 if optimizer_name == optimizer_names[0]:
adam_lr = trial.suggest_loguniform('adam_lr', 1e-5, 1e-1)
 optimizer = optim.Adam(model.parameters() , lr=adam_lr, weight_decay=weight_decay)
 elif optimizer_name == optimizer_names[1]:
momentum_sgd_lr = trial.suggest_loguniform('momentum_sgd_lr', 1e-5, 1e-1)
 optimizer = optim.SGD(model.parameters() , lr=momentum_sgd_lr, momentum=0.9, weight_decay=weight_decay)
 else:
rmsprop_lr = trial.suggest_loguniform('rmsprop_lr', 1e-5, 1e-1)
 optimizer = optim.RMSprop(model.parameters() , lr=rmsprop_lr, weight_decay=weight_decay)
 return optimizer
def objective(self, trial):
self.x_train, self.x_valid, self.t_train, self.t_valid, self.x_test, self.x_test_id, self.loss_weight = PreProcess()()
in_dim = self.x_train.shape[1]
out_dim = len(np.unique(self.t_train))
n_epochs = 100
mlp = MLP_tune(trial, in_dim, out_dim ).to(device)
 optimizer = self.get_optimizer(trial, mlp)
 scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min')
 criterion = nn.NLLLoss(weight=self.loss_weight)
 ds_train = TensorDataset(torch.tensor(self.x_train ).float() , torch.tensor(self.t_train))
ds_valid = TensorDataset(torch.tensor(self.x_valid ).float() , torch.tensor(self.t_valid))
loader_train = DataLoader(ds_train, batch_size=256, shuffle=True)
 loader_valid = DataLoader(ds_valid, batch_size=256, shuffle=True)
 start_time = time.time()
for epoch in range(n_epochs):
losses_train = []
losses_valid = []
preds_train = []
preds_valid = []
trues_train = []
trues_valid = []
mlp.train()
for x, t in loader_train:
true = t.tolist()
trues_train.extend(true)
 mlp.zero_grad()
x = x.to(device)
 t = t.to(device)
 y = mlp.forward(x)
 loss = criterion(y, t.long())
 loss.backward()
optimizer.step()
pred = y.argmax(1 ).tolist()
preds_train.extend(pred)
 losses_train.append(loss.tolist())
 mlp.eval()
for x, t in loader_valid:
true = t.tolist()
trues_valid.extend(true)
 x = x.to(device)
 t = t.to(device)
 y = mlp.forward(x)
 loss = criterion(y, t.long())
 pred = y.argmax(1 ).tolist()
preds_valid.extend(pred)
 losses_valid.append(loss.tolist())
 print('EPOCH: {}, Train [Loss: {:.3f}, F1: {:.3f}, R: {:.3f}, P: {:.3f}], Valid [Loss: {:.3f}, F1: {:.3f}, R: {:.3f}, P: {:.3f}]'.format(
 epoch,
np.mean(losses_train),
f1_score(trues_train, preds_train, average='weighted'),
recall_score(trues_train, preds_train, average='weighted'),
precision_score(trues_train, preds_train, average='weighted'),
np.mean(losses_valid),
f1_score(trues_valid, preds_valid, average='weighted'),
recall_score(trues_valid, preds_valid, average='weighted'),
precision_score(trues_valid, preds_valid, average='weighted')
))
scheduler.step(np.mean(losses_valid))
metric = np.mean(losses_valid)
 trial.report(metric, epoch)
 if trial.should_prune() :
raise optuna.exceptions.TrialPruned()
elapsed_time = time.time() - start_time
print("elapsed_time:{:.1f}".format(elapsed_time / 60)+ "[min]")
 return metric
!pip uninstall transformers -y<install_modules>
def run() :
n_cross_valid = 30
preds_test_all = []
for n in range(n_cross_valid):
n_epochs = 100
lr = 7.560243948481283e-05
weight_decay = 0.0005896154168582327
train = Train()
mlp = train.train(n_epochs, lr, weight_decay)
 preds_test = train.test(mlp)
 preds_test_all.append(preds_test)
 preds_test = np.mean(preds_test_all, axis=0)
 train.submit(list(preds_test))
run()
<install_modules><EOS> 
def tune() :
train = Train()
study = optuna.create_study(direction="minimize")
 study.optimize(train.objective, n_trials=1000)
 pruned_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.PRUNED]
complete_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.COMPLETE]
print("Study statistics: ")
 print(" Number of finished trials: ", len(study.trials))
print(" Number of pruned trials: ", len(pruned_trials))
print(" Number of complete trials: ", len(complete_trials))
print("Best trial:")
 trial = study.best_trial
print(" Value: ", trial.value)
 print(" Params: ")
 for key, value in trial.params.items() :
print(" {}: {}".format(key, value))