Kaggle competition: Otto Group Product Classification Challenge
<SOS> metric: MulticlassLoss Kaggle data source: otto-group-product-classification-challenge<load_pretrained>
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfTransformer
import lightgbm as lgb
MY_CONFIG = read_pickle(MY_CONFIG_PATH )<feature_engineering>
pd.set_option('display.max_columns', 100)
 pd.set_option('display.max_rows', 100)
 dtypes = {f"feat_{i}": "int32" for i in range(1, 94)}
dtypes["id"] = "int32"
dtypes["target"] = "string"
CONFIG1 = read_pickle(MODEL1_FOLDER+MODEL1_CONFIG)
 tmp = pd.DataFrame([[_p] for _p in glob.glob(MODEL1_FOLDER+'*')if _p[-4:]=='.pth'], columns=['filepath'])
 tmp['model'] = tmp['filepath'].apply(lambda x: x.split('/')[-1])
 tmp['fold'] = tmp['model'].apply(lambda x: int(x.split('_')[4]))
tmp['sentiment'] = tmp['model'].apply(lambda x: x.split('_')[3])
 tmp.sort_values('fold', inplace=True)
 tmp.reset_index(drop=True, inplace=True)
 filepath1 = tmp.loc[tmp['sentiment']=='pnn', 'filepath'].tolist()
--- [START {STR_NOW}] {"-"*64}
')
 print(CONFIG1 )<feature_engineering>
df_train = pd.read_csv(
 "/kaggle/input/otto-group-product-classification-challenge/train.csv",
dtype=dtypes
).set_index("id" )
CONFIG2 = read_pickle(MODEL2_FOLDER+MODEL2_CONFIG)
 tmp = pd.DataFrame([[_p] for _p in glob.glob(MODEL2_FOLDER+'*')if _p[-4:]=='.pth'], columns=['filepath'])
 tmp['model'] = tmp['filepath'].apply(lambda x: x.split('/')[-1])
 tmp['fold'] = tmp['model'].apply(lambda x: int(x.split('_')[4]))
tmp['sentiment'] = tmp['model'].apply(lambda x: x.split('_')[3])
 tmp.sort_values('fold', inplace=True)
 tmp.reset_index(drop=True, inplace=True)
 filepath2 = tmp.loc[tmp['sentiment']=='pnn', 'filepath'].tolist()
--- [START {STR_NOW}] {"-"*64}
')
 print(CONFIG2 )<feature_engineering>
class_to_order = dict()
order_to_class = dict()
for idx, col in enumerate(df_train.target.unique()):
order_to_class[idx] = col
class_to_order[col] = idx
df_train["target_ord"] = df_train["target"].map(class_to_order ).astype("int16")
 feature_columns = [
col for col in df_train.columns if col.startswith("feat_")
 ]
target_column = ["target_ord"]
CONFIG3 = read_pickle(MODEL3_FOLDER+MODEL3_CONFIG)
 tmp = pd.DataFrame([[_p] for _p in glob.glob(MODEL3_FOLDER+'*')if _p[-4:]=='.pth'], columns=['filepath'])
 tmp['model'] = tmp['filepath'].apply(lambda x: x.split('/')[-1])
 tmp['fold'] = tmp['model'].apply(lambda x: int(x.split('_')[7]))
tmp['sentiment'] = tmp['model'].apply(lambda x: x.split('_')[6])
 tmp.sort_values('fold', inplace=True)
 tmp.reset_index(drop=True, inplace=True)
 filepath3 = tmp.loc[tmp['sentiment']=='pnn', 'filepath'].tolist()
--- [START {STR_NOW}] {"-"*64}
')
 print(CONFIG3 )<feature_engineering>
X_train, X_valid, y_train, y_valid = train_test_split(
 df_train[feature_columns], df_train[target_column],
test_size=0.3, random_state=42,
stratify=df_train[target_column]
)
CONFIG4 = read_pickle(MODEL4_FOLDER+MODEL4_CONFIG)
 tmp = pd.DataFrame([[_p] for _p in glob.glob(MODEL4_FOLDER+'*')if _p[-4:]=='.pth'], columns=['filepath'])
 tmp['model'] = tmp['filepath'].apply(lambda x: x.split('/')[-1])
 tmp['fold'] = tmp['model'].apply(lambda x: int(x.split('_')[7]))
tmp['sentiment'] = tmp['model'].apply(lambda x: x.split('_')[6])
 tmp.sort_values('fold', inplace=True)
 tmp.reset_index(drop=True, inplace=True)
 filepath4 = tmp.loc[tmp['sentiment']=='pnn', 'filepath'].tolist()
--- [START {STR_NOW}] {"-"*64}
')
 print(CONFIG4 )<feature_engineering>
tfidf = TfidfTransformer()
tfidf_feature_train = tfidf.fit_transform(X_train ).toarray().astype("float32")
 tfidf_feature_valid = tfidf.transform(X_valid ).toarray().astype("float32" )
CONFIG5 = read_pickle(MODEL5_FOLDER+MODEL5_CONFIG)
 tmp = pd.DataFrame([[_p] for _p in glob.glob(MODEL5_FOLDER+'*')if _p[-4:]=='.pth'], columns=['filepath'])
 tmp['model'] = tmp['filepath'].apply(lambda x: x.split('/')[-1])
 tmp['fold'] = tmp['model'].apply(lambda x: int(x.split('_')[4]))
tmp['sentiment'] = tmp['model'].apply(lambda x: x.split('_')[3])
 tmp.sort_values('fold', inplace=True)
 tmp.reset_index(drop=True, inplace=True)
 filepath5 = tmp.loc[tmp['sentiment']=='pnn', 'filepath'].tolist()
--- [START {STR_NOW}] {"-"*64}
')
 print(CONFIG5 )<feature_engineering>
X_train_tfidf = np.hstack(( X_train.values, tfidf_feature_train))
X_valid_tfidf = np.hstack(( X_valid.values, tfidf_feature_valid))
CONFIG6 = read_pickle(MODEL6_FOLDER+MODEL6_CONFIG)
 tmp = pd.DataFrame([[_p] for _p in glob.glob(MODEL6_FOLDER+'*')if _p[-4:]=='.pth'], columns=['filepath'])
 tmp['model'] = tmp['filepath'].apply(lambda x: x.split('/')[-1])
 tmp['fold'] = tmp['model'].apply(lambda x: int(x.split('_')[4]))
tmp['sentiment'] = tmp['model'].apply(lambda x: x.split('_')[3])
 tmp.sort_values('fold', inplace=True)
 tmp.reset_index(drop=True, inplace=True)
 filepath6 = tmp.loc[tmp['sentiment']=='pnn', 'filepath'].tolist()
--- [START {STR_NOW}] {"-"*64}
')
 print(CONFIG6 )<feature_engineering>
params = {
'objective': "multiclass",
'metric': {"multi_logloss"},
'num_class': 9,
'seed': 42,
'lambda_l1': 0.0036682603550733813,
'lambda_l2': 8.924549306063208,
'num_leaves': 113,
'feature_fraction': 0.48000000000000004,
'bagging_fraction': 1.0,
'bagging_freq': 0,
'min_child_samples': 20
}
CONFIG7 = read_pickle(MODEL7_FOLDER+MODEL7_CONFIG)
 tmp = pd.DataFrame([[_p] for _p in glob.glob(MODEL7_FOLDER+'*')if _p[-4:]=='.pth'], columns=['filepath'])
 tmp['model'] = tmp['filepath'].apply(lambda x: x.split('/')[-1])
 tmp['fold'] = tmp['model'].apply(lambda x: int(x.split('_')[4]))
tmp['sentiment'] = tmp['model'].apply(lambda x: x.split('_')[3])
 tmp.sort_values('fold', inplace=True)
 tmp.reset_index(drop=True, inplace=True)
 filepath7 = tmp.loc[tmp['sentiment']=='pnn', 'filepath'].tolist()
--- [START {STR_NOW}] {"-"*64}
')
 print(CONFIG7 )<feature_engineering>
dataset_train = lgb.Dataset(X_train_tfidf, y_train)
 dataset_valid = lgb.Dataset(X_valid_tfidf, y_valid )
CONFIG8 = read_pickle(MODEL8_FOLDER+MODEL8_CONFIG)
 tmp = pd.DataFrame([[_p] for _p in glob.glob(MODEL8_FOLDER+'*')if _p[-4:]=='.pth'], columns=['filepath'])
 tmp['model'] = tmp['filepath'].apply(lambda x: x.split('/')[-1])
 tmp['fold'] = tmp['model'].apply(lambda x: int(x.split('_')[4]))
tmp['sentiment'] = tmp['model'].apply(lambda x: x.split('_')[3])
 tmp.sort_values('fold', inplace=True)
 tmp.reset_index(drop=True, inplace=True)
 filepath8 = tmp.loc[tmp['sentiment']=='pnn', 'filepath'].tolist()
--- [START {STR_NOW}] {"-"*64}
')
 print(CONFIG8 )<feature_engineering>
booster = lgb.train(
 params,
dataset_train,
feature_name=(
 [f"feat_{i}" for i in range(1, 94)]
+ [f"tfidf_{i}" for i in range(1, 94)] 
),
num_boost_round=500,
valid_sets=dataset_valid,
early_stopping_rounds=20,
)
for item in set(MY_CONFIG.keys())- set(CONFIG1.keys()):
CONFIG1[item] = MY_CONFIG[item]
CONFIG2[item] = MY_CONFIG[item]<categorify>
best_iteration = booster.best_iteration
print(best_iteration )
JAC_TH = 0.46
_, test_df = preprocess_df(ORI_DATA_DIR, use_sentiment=SENTIMENT_COMB[CONFIG1['USE_SENTIMENT']], show_head=False)
 pred_ensemble = create_test_candidates_ensemble(test_df, filepath1, filepath2, filepath3, filepath4, CONFIG1, CONFIG2, CONFIG3, CONFIG4, JAC_TH=JAC_TH, ratio=[0.4, 0.1, 0.3, 0.2, 0.4, 0.1, 0.3, 0.2], en_ratio=[0.5, 0.5], num=2 )<define_variables>
df_test = pd.read_csv(
 "/kaggle/input/otto-group-product-classification-challenge/test.csv",
dtype=dtypes
).set_index("id" )
seed_everything(100)
 MAX_LEN = 256
TOKENIZER = tokenizers.ByteLevelBPETokenizer(
 vocab_file=f"{ROBERTA_PATH}/vocab.json",
merges_file=f"{ROBERTA_PATH}/merges.txt",
lowercase=True,
add_prefix_space=True 
)
 cand_nums = 5
val_nums = 5<import_modules>
tfidf = TfidfTransformer()
tfidf_feature_train_all = tfidf.fit_transform(df_train[feature_columns] ).toarray().astype("float32")
 X_train_all_tfidf = np.hstack(( df_train[feature_columns].values, tfidf_feature_train_all))
dataset_train_all = lgb.Dataset(X_train_all_tfidf, df_train[target_column] )
class RerankingDataset:
def __init__(self, tweet, sentiment, selected_text, jaccard, score):
self.tweet = tweet
self.sentiment = sentiment
self.selected_text = selected_text
self.tokenizer = TOKENIZER
self.max_len = MAX_LEN
self.jaccard = jaccard
self.score = score
def __len__(self):
return len(self.tweet)
 def __getitem__(self, item):
tweet = self.tweet[item]
selected_text = self.selected_text[item]
sentiment = self.sentiment[item]
jaccard = self.jaccard[item]
score = self.score[item]
new_tweet = str(tweet)+ " 
 
 " + str(selected_text)
 tok_tweet = TOKENIZER.encode(new_tweet)
 input_ids_orig = tok_tweet.ids
sentiment_id = {
'positive': 1313,
'negative': 2430,
'neutral': 7974
}
input_ids = [0] + [sentiment_id[sentiment]] + [2] + [2] + input_ids_orig + [2]
token_type_ids = [0, 0, 0, 0] + [0] *(len(input_ids_orig)+ 1)
 mask = [1] * len(token_type_ids)
 return {
'orig_tweet' : tweet,
'sentiment' : sentiment,
'orig_selected' : selected_text,
'jaccard' : jaccard,
'score' : score,
'ids' : input_ids,
'mask' : mask,
'token_type_ids' : token_type_ids
}
class RerankingCollate:
def __init__(self):
self.CONFIG = {}
self.CONFIG['BUCKET'] = True
self.CONFIG['MAX_LEN'] = MAX_LEN
def __call__(self, batch):
out = {
'orig_tweet' : [],
'sentiment' : [],
'orig_selected' : [],
'jaccard' : [],
'score' : [],
'ids' : [],
'mask' : [],
'token_type_ids' : [],
}
for i in range(len(batch)) :
for k, v in batch[i].items() :
out[k].append(v)
 if self.CONFIG['BUCKET']:
max_pad = 0
for p in out['ids']:
if len(p)>max_pad:
max_pad = len(p)
 else:
max_pad = self.CONFIG['MAX_LEN']
for i in range(len(batch)) :
tokenized_text = out['ids'][i]
token_type_ids = out['token_type_ids'][i]
mask = out['mask'][i]
text_len = len(tokenized_text)
 out['ids'][i] =(tokenized_text + [1] *(max_pad - text_len)) [:max_pad]
out['token_type_ids'][i] =(token_type_ids + [0] *(max_pad - text_len)) [:max_pad]
out['mask'][i] =(mask + [0] *(max_pad - text_len)) [:max_pad]
out['jaccard'] = torch.tensor(out['jaccard'], dtype=torch.float)
 out['score'] = torch.tensor(out['score'], dtype=torch.float)
 out['ids'] = torch.tensor(out['ids'], dtype=torch.long)
 out['mask'] = torch.tensor(out['mask'], dtype=torch.long)
 out['token_type_ids'] = torch.tensor(out['token_type_ids'], dtype=torch.long)
 return out<normalization>
booster = lgb.train(
 params,
dataset_train_all,
feature_name=(
 [f"feat_{i}" for i in range(1, 94)]
+ [f"tfidf_{i}" for i in range(1, 94)] 
),
num_boost_round=best_iteration,
)
class myModel(transformers.BertPreTrainedModel):
def __init__(self, conf):
super(myModel, self ).__init__(conf)
 self.roberta = transformers.RobertaModel.from_pretrained(ROBERTA_PATH, config=conf)
 self.dropout = nn.Dropout(0.1)
 self.out_proj = nn.Linear(768*2, 1)
 def forward(self, ids, mask, token_type_ids):
o1, o2, _ = self.roberta(
 ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 apool = torch.mean(o1, 1)
 mpool, _ = torch.max(o1, 1)
 cat = torch.cat(( apool, mpool), 1)
 bo = self.dropout(cat)
 p2 = self.out_proj(bo)
 return p2<load_pretrained>
tfidf_feature_test = tfidf.transform(df_test ).toarray()
X_test_tfidf = np.hstack(( df_test[feature_columns].values, tfidf_feature_test))
device = torch.device("cuda")
 model_config = transformers.RobertaConfig.from_pretrained(ROBERTA_PATH)
 model_config.output_hidden_states = True
model0 = myModel(conf=model_config)
 model0.to(device)
 model0.load_state_dict(torch.load(REMODEL0))
model1 = myModel(conf=model_config)
 model1.to(device)
 model1.load_state_dict(torch.load(REMODEL1))
model2 = myModel(conf=model_config)
 model2.to(device)
 model2.load_state_dict(torch.load(REMODEL2))
model3 = myModel(conf=model_config)
 model3.to(device)
 model3.load_state_dict(torch.load(REMODEL3))
model4 = myModel(conf=model_config)
 model4.to(device)
 model4.load_state_dict(torch.load(REMODEL4))
model5 = myModel(conf=model_config)
 model5.to(device)
 model5.load_state_dict(torch.load(REMODEL5))
model6 = myModel(conf=model_config)
 model6.to(device)
 model6.load_state_dict(torch.load(REMODEL6))
model7 = myModel(conf=model_config)
 model7.to(device)
 model7.load_state_dict(torch.load(REMODEL7))
model8 = myModel(conf=model_config)
 model8.to(device)
 model8.load_state_dict(torch.load(REMODEL8))
model9 = myModel(conf=model_config)
 model9.to(device)
 model9.load_state_dict(torch.load(REMODEL9))
print('model load' )<load_from_csv>
pred = booster.predict(X_test_tfidf )
dev = pd.read_csv('/kaggle/working/test_candidate.csv')
 dev<create_dataframe>
for idx, col in order_to_class.items() :
df_test[col] = pred[:,idx]
valid_dataset = RerankingDataset(
 tweet=dev.tweet.values,
sentiment=dev.sentiment.values,
selected_text=dev.candidate.values,
jaccard=dev.jaccard.values,
score=dev.score.values 
)
 valid_data_loader = torch.utils.data.DataLoader(
 valid_dataset,
batch_size=val_nums,
collate_fn=RerankingCollate() ,
num_workers=0,
)<predict_on_test>
df_test[[f"Class_{i}" for i in range(1, 10)]].to_csv('submission.csv', index=True )