Kaggle competition: Otto Group Product Classification Challenge
<SOS> metric: MulticlassLoss Kaggle data source: otto-group-product-classification-challenge<compute_train_metric>
%matplotlib inline
jem_all = []
if IS_EXPECTATION_MAXIMISE:
def jem_expectation_maximisation(confidences, strings, check_string):
res = 0
for i in range(len(confidences)) :
res += confidences[i] * jaccard(strings[i], check_string)
 return res
for k in range(input_ids_t.shape[0]):
jem_strings = [jem_preds.loc[k,'string' + str(x)] for x in range(N_SPLIT)]
jem_confidences = [0.5 *(jem_preds.loc[k, 'start' + str(x)] + jem_preds.loc[k, 'end' + str(x)])for x in range(N_SPLIT)]
jem_best =(0, test.loc[k, 'text'])
 for fold in range(N_SPLIT):
jem_fold_conf = jem_expectation_maximisation(jem_confidences, jem_strings, jem_preds.loc[k, 'string' + str(fold)])
 if jem_fold_conf > jem_best[0]:
jem_best =(jem_fold_conf, jem_strings[fold])
 jem_all.append(jem_best[1] )<split>
train = pd.read_csv('.. /input/otto-group-product-classification-challenge/train.csv')
 test = pd.read_csv('.. /input/otto-group-product-classification-challenge/test.csv')
 sample_submit = pd.read_csv('.. /input/otto-group-product-classification-challenge/sampleSubmission.csv' )
if IS_GETTING_BEST_LOGITS:
def get_best_start_end_idxs(_start_logits, _end_logits):
best_logit = -1000
best_idxs = None
for start_idx, start_logit in enumerate(_start_logits):
for end_idx, end_logit in enumerate(_end_logits[start_idx:]):
logit_sum =(start_logit + end_logit ).item()
if logit_sum > best_logit:
best_logit = logit_sum
best_idxs =(start_idx, start_idx+end_idx)
 return best_idxs<categorify>
train['target'] = train['target'].str.replace('Class_', '')
 train['target'] = train['target'].astype(int)- 1
all = []
for k in range(input_ids_t.shape[0]):
if IS_GETTING_BEST_LOGITS:
a, b = get_best_start_end_idxs(preds_start[k,], preds_end[k,])
 else:
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 if IS_SUBSTITUTING_URLS:
throwaway, st = deprocess_urls(test_web_replacements, '', st, k)
 if IS_POSTPROCESSING:
st = post_processing(st)
 all.append(st )<save_to_csv>
data = pd.concat([train, test])
 cols = [c for c in data.columns if c not in ['id', 'target']]
for col in cols:
dictionary=data[col].value_counts().to_dict()
data['count_'+col]=data[col].map(dictionary)
 data['max_val'] = data[cols].max(axis=1)
 data['sum_val'] = data[cols].sum(axis=1)
 data['non_zero'] =(data[cols] > 0 ).sum(axis=1)
 data['count_one'] =(data[cols] == 1 ).sum(axis=1)
 data['count_two'] =(data[cols] == 2 ).sum(axis=1)
 data['count_three'] =(data[cols] == 3 ).sum(axis=1)
 train = data[~data['target'].isnull() ].reset_index(drop=True)
 test = data[data['target'].isnull() ].reset_index(drop=True )
if IS_EXPECTATION_MAXIMISE:
test['selected_text'] = jem_all
else:
test['selected_text'] = all
if IS_PSEUDO_LABELLING:
test.to_csv('extra_data.csv', index=False)
 else:
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 test.sample(10 )<define_variables>
NFOLDS = 5
RANDOM_STATE = 871972
excluded_column = ['target', 'id']
cols = [c for c in train.columns if c not in excluded_column]
folds = StratifiedKFold(n_splits=NFOLDS, shuffle=True,
random_state=RANDOM_STATE)
 params = {
'metric':'multi_logloss','objective': 'multiclass', 'num_class': 9, 'verbosity': 1,
'feature_fraction': 0.4, 'num_leaves': 139, 'bagging_fraction': 0.8254401463359962, 'bagging_freq': 3,
'lambda_l1': 0.02563829140437355, 'lambda_l2': 9.594334397031103, 'min_child_samples': 100
}
if IS_PREDICTING_TRAIN:
all = []
start = []
end = []
start_pred = []
end_pred = []
for k in range(input_ids.shape[0]):
if IS_GETTING_BEST_LOGITS:
a, b = get_best_start_end_idxs(preds_start_train[k,], preds_end_train[k,])
 else:
a = np.argmax(preds_start_train[k,])
 b = np.argmax(preds_end_train[k,])
 start.append(np.argmax(start_tokens[k]))
end.append(np.argmax(end_tokens[k]))
if a>b:
st = train.loc[k,'text']
start_pred.append(0)
 end_pred.append(len(st))
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 start_pred.append(a)
 end_pred.append(b)
 if IS_SUBSTITUTING_URLS:
throwaway, st = deprocess_urls(test_web_replacements, '', st, k)
 if IS_POSTPROCESSING:
st = post_processing(st)
 all.append(st)
 train['start'] = start
train['end'] = end
train['start_pred'] = start_pred
train['end_pred'] = end_pred
train['selected_text_pred'] = all
train.to_csv('outliers.csv', index=False)
 train.sample(10 )<set_options>
y_pred_test = np.zeros(( len(test), 9))
oof = np.zeros(( len(train), 9))
score = 0
feature_importance_df = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(train, y = train['target'])) :
print('Fold', fold_n)
 X_train, X_valid = train.iloc[train_index], train.iloc[valid_index]
y_train, y_valid = X_train['target'].astype(int), X_valid['target'].astype(int)
 train_data = lgb.Dataset(X_train[cols], label=y_train)
 valid_data = lgb.Dataset(X_valid[cols], label=y_valid)
 lgb_model = lgb.train(params,train_data,num_boost_round=30000,
valid_sets = [train_data, valid_data],verbose_eval=300,early_stopping_rounds = 300)
 y_pred_valid = lgb_model.predict(X_valid[cols], num_iteration=lgb_model.best_iteration)
 oof[valid_index] = y_pred_valid
score += log_loss(y_valid, y_pred_valid)
 y_pred_test += lgb_model.predict(test[cols], num_iteration=lgb_model.best_iteration)/NFOLDS
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = cols
fold_importance_df["importance"] = lgb_model.feature_importance(importance_type='gain')
 fold_importance_df["fold"] = fold_n + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 print('valid logloss average:', score/NFOLDS, log_loss(train['target'], oof))
warnings.filterwarnings('ignore' )<set_options>
submit = pd.concat([sample_submit[['id']], pd.DataFrame(y_pred_test)], axis = 1)
 submit.columns = sample_submit.columns
submit.to_csv('submit.csv', index=False )
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed = 4444
seed_everything(seed )<define_variables>
column_name = ['lgb_' + str(i)for i in range(9)]
pd.DataFrame(oof, columns = column_name ).to_csv('oof_lgb.csv', index=False)
 pd.DataFrame(y_pred_test, columns = column_name ).to_csv('submit_lgb.csv', index=False )