Kaggle competition: Otto Group Product Classification Challenge
<SOS> metric: MulticlassLoss Kaggle data source: otto-group-product-classification-challenge<load_from_url>
print(os.listdir(".. /input"))
<define_variables>
data = pd.read_csv('.. /input/train.csv')
 train_X = data.drop(["target","id"], axis=1 )
path_spiece_model = '/kaggle/input/albert-model/albert-base-v2-spiece.model'
path_albert_config = '/kaggle/input/albert-model/albert-base-v2-config.json'
path_albert_model = '/kaggle/input/albert-model/albert-base-v2-pytorch_model.bin'<load_pretrained>
le = LabelEncoder()
le.fit(data["target"])
 train_y = le.transform(data["target"] )
sp = spm.SentencePieceProcessor()
sp.load(path_spiece_model )<load_pretrained>
sss = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=0)
 for train_index, test_index in sss.split(train_X.values, train_y):
X_train = train_X.values[train_index]
X_val = train_X.values[test_index]
y_train = train_y[train_index]
y_val = train_y[test_index]
tokenizer = transformers.AlbertTokenizer.from_pretrained(path_spiece_model, do_lower_case=True)
 tokenizer.tokenize("Test tokenizer" )<categorify>
missing_val_count_by_column =(data.isnull().sum())
 print(missing_val_count_by_column.sum() )
class OffsetTokenizer() :
def __init__(self, path_model=path_spiece_model):
self.spt = sentencepiece_pb2.SentencePieceText()
self.sp = spm.SentencePieceProcessor()
self.sp.load(path_model)
 def encode(self, text, return_tokens=False, lower=True):
if lower:
text = text.lower()
offset = []
ids = []
self.spt.ParseFromString(self.sp.encode_as_serialized_proto(text))
for piece in self.spt.pieces:
offset.append(( piece.begin, piece.end))
ids.append(piece.id)
 if return_tokens:
return sp.encode_as_pieces(text), ids, offset
else:
return ids, offset 
<choose_model_class>
data["target"].value_counts()
o_tokenizer = OffsetTokenizer()
o_tokenizer.encode("Test tokenizer", return_tokens=False )<load_pretrained>
ros = RandomOverSampler()
X_ros, y_ros = ros.fit_sample(X_train, y_train)
 unique, counts = np.unique(y_ros, return_counts=True)
 print(np.asarray(( unique, counts)).T )
albert_config = transformers.AlbertConfig.from_pretrained(path_albert_config)
 albert_config.output_hidden_states=True<set_options>
scaler = StandardScaler()
scaler.fit(X_train)
 X_train_scaled = scaler.transform(X_train)
 X_val_scaled = scaler.transform(X_val )
device = 'cuda' if torch.cuda.is_available() else 'cpu'<concatenate>
test_data = pd.read_csv('.. /input/test.csv')
 test_X = test_data.drop(["id"], axis=1)
 scaler_all = StandardScaler()
train_X_scaled = scaler_all.fit_transform(train_X)
 test_X_scaled = scaler.transform(test_X )
def pad_or_truncate(list_num, max_len, SEP_ID, is_input_ids=False):
if len(list_num)< max_len:
list_num = list_num + [0] *(max_len - len(list_num))
else:
if is_input_ids:
list_num = list_num[:max_len-1] + SEP_ID
else:
return list_num[:max_len]
return list_num<concatenate>
pca = decomposition.PCA(n_components=20)
 pca.fit(X_train_scaled)
 X_train_pca = pca.transform(X_train_scaled)
 print(pca.explained_variance_ratio_)
 print(pca.explained_variance_ )
def issubset_sequence(s_text_ids, text_ids):
s_text_ids = [str(i)for i in s_text_ids]
ptn_s_text_ids = r'\b' + r'\b \b'.join(s_text_ids)+ r'\b'
text_ids = [str(i)for i in text_ids]
text_ids = ' '.join(text_ids)
 if re.search(ptn_s_text_ids, text_ids):
return True
else:
return False<categorify>
xgb = XGBClassifier()
xgb.fit(X_train_scaled, y_train)
 preds = xgb.predict_proba(X_val_scaled)
 score = log_loss(y_val, preds)
 print("test data log loss eval : {}".format(log_loss(y_val,preds)) )
def data_extractor(df, data_test=False, max_len=128, random=True):

 count = 0
sentiment_id = {'positive' : 2221,
'neutral' : 8387,
'negative' : 3682}
CLS_ID = tokenizer.encode('[CLS]', add_special_tokens=False)
 SEP_ID = tokenizer.encode('[SEP]', add_special_tokens=False)
 all_data = []
if random:
df = df.sample(frac=1, random_state=42)
 texts = df.text.tolist()
sentiments = df.sentiment.tolist()
if not data_test:
sel_texts = df.selected_text.tolist()
for idx, row in df.iterrows() :
start_target, end_target = 0, 0
text = ' '+ ' '.join(row.text.lower().split())
 s_text = ' '+ ' '.join(row.selected_text.lower().split())
 sentiment = row.sentiment.lower().strip()
text_ids, offsets = o_tokenizer.encode(text)
 s_text_ids, _ = o_tokenizer.encode(s_text)
 for i in range(len(s_text_ids)) :
if issubset_sequence(s_text_ids, text_ids):
break
else:
s_text_ids = s_text_ids[1:]
if len(s_text_ids)== 0:
s_text_ids = tokenizer.encode(s_text, add_special_tokens=False)
 for i in range(len(s_text_ids)) :
if issubset_sequence(s_text_ids, text_ids):
break
else:
s_text_ids = s_text_ids[:-1]
for i in range(len(text_ids)) :
if text_ids[i:i+len(s_text_ids)] == s_text_ids:
start_target, end_target = i+3, i+len(s_text_ids)+3-1
if(start_target - end_target)== 1:
end_target += 1
elif end_target-3 > len(text_ids):
print(text, '__|__', s_text)
 print(text_ids)
 print(start_target, end_target)
 print(s_text_ids)
 raise ValueError('tes, end_target > length text ids')
 break
input_ids = CLS_ID + [sentiment_id[sentiment]] + SEP_ID + text_ids + SEP_ID
token_type_ids = [0] * 3 + [1] *(len(text_ids)+1)
 attention_mask = [1] * len(input_ids)
 input_ids = pad_or_truncate(input_ids, max_len, SEP_ID, is_input_ids=True)
 token_type_ids = pad_or_truncate(token_type_ids, max_len, SEP_ID)
 attention_mask = pad_or_truncate(attention_mask, max_len, SEP_ID)
 if len(offsets)< max_len:
offsets = offsets +([(0,0)] *(max_len - len(offsets)))
 else:
offsets = offsets[:max_len]
if start_target == 0 or end_target == 0:
print(text, '_|_', s_text)
 print(tokenizer.tokenize(text), '_|_', tokenizer.tokenize(s_text))
print(text_ids)
 print(s_text_ids)
 raise ValueError('tes')
 albert_input = { 'input_ids':torch.tensor(input_ids ).to(device),
'token_type_ids':torch.tensor(token_type_ids ).to(device),
'attention_mask':torch.tensor(attention_mask ).to(device),
'start_targets':torch.tensor(start_target ).to(device),
'end_targets':torch.tensor(end_target ).to(device),
'offsets':torch.tensor(offsets ).to(device),
'original_texts':text,
'sentiments':sentiment,
}
all_data.append(albert_input)
 else:
for idx, row in df.iterrows() :
text = ' '+ ' '.join(row.text.lower().split())
 sentiment = row.sentiment.strip()
text_ids, offsets = o_tokenizer.encode(text)
 input_ids = CLS_ID + [sentiment_id[sentiment]] + SEP_ID + text_ids + SEP_ID
token_type_ids = [0] * 3 + [1] *(len(text_ids)+1)
 attention_mask = [1] * len(input_ids)
 input_ids = pad_or_truncate(input_ids, max_len, SEP_ID, is_input_ids=True)
 token_type_ids = pad_or_truncate(token_type_ids, max_len, SEP_ID)
 attention_mask = pad_or_truncate(attention_mask, max_len, SEP_ID)
 if len(offsets)< max_len:
offsets = offsets +([(0,0)] *(max_len - len(offsets)))
 else:
offsets = offsets[:max_len]
albert_input = { 'input_ids':torch.tensor(input_ids ).to(device),
'token_type_ids':torch.tensor(token_type_ids ).to(device),
'attention_mask':torch.tensor(attention_mask ).to(device),
'offsets':torch.tensor(offsets ).to(device),
'original_texts':text,
'sentiments':sentiment,
}
all_data.append(albert_input)
 return all_data 
<load_pretrained>
xgb.get_params
def generate_dataloader(albert_inputs, split=True, val_size=0.2, batch_size=32):
if split:
val_size = math.floor(len(albert_inputs)*val_size)
 train_data = albert_inputs[val_size:]
val_data = albert_inputs[:val_size]
train_dataloader = DataLoader(train_data, batch_size=batch_size)
 val_dataloader = DataLoader(val_data, batch_size=batch_size)
 return train_dataloader, val_dataloader
else:
return DataLoader(albert_inputs, batch_size=batch_size )<feature_engineering>
albert_inputs = data_extractor(train)
 train_dataloader, val_dataloader = generate_dataloader(albert_inputs, split=True )<define_variables>
scores = []
n_estimators = [100,200,400,450,500,525,550,600,700]
for nes in n_estimators:
xgb = XGBClassifier(learning_rate =0.1, n_estimators=nes, max_depth=7, min_child_weight=3, subsample=0.8,
colsample_bytree=0.8, nthread=4, seed=42, objective='multi:softprob')
 xgb.fit(X_train_scaled, y_train)
 preds = xgb.predict_proba(X_val_scaled)
 score = log_loss(y_val, preds)
 scores.append(score)
 print("test data log loss eval : {}".format(log_loss(y_val,preds)) )
len(( albert_inputs[0]['offsets'].sum(1)> 0 ).nonzero() )<choose_model_class>
scores_md = []
max_depths = [1,3,5,6,7,8,10]
for md in max_depths:
xgb = XGBClassifier(learning_rate =0.1, n_estimators=n_estimators[np.argmin(scores)],
max_depth=md, min_child_weight=3, subsample=0.8,
colsample_bytree=0.8, nthread=4, seed=42, objective='multi:softprob')
 xgb.fit(X_train_scaled, y_train)
 preds = xgb.predict_proba(X_val_scaled)
 score = log_loss(y_val, preds)
 scores_md.append(score)
 print("test data log loss eval : {}".format(log_loss(y_val,preds)) )
class TweetSelectionModel(transformers.AlbertPreTrainedModel):
def __init__(self, conf, hidden_dim=768):
super(TweetSelectionModel, self ).__init__(conf, hidden_dim)
 self.albert = transformers.AlbertModel.from_pretrained(path_albert_model,
config=conf, from_tf=False)
 self.drop_out = nn.Dropout(0.3)
 self.linear = nn.Linear(hidden_dim*2, 2)
 torch.nn.init.normal_(self.linear.weight, std=0.02)
 def forward(self, ids, token_type_ids, mask):
_, _, out = self.albert(
 ids,
token_type_ids=token_type_ids,
attention_mask=mask, 
)
 out = torch.cat(( out[-1], out[-2]), dim=-1)
 out = self.drop_out(out)
 logits = self.linear(out)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits 
<compute_test_metric>
scores_mcw = []
min_child_weights = [1,2,3,4,5]
for mcw in min_child_weights:
xgb = XGBClassifier(learning_rate =0.1, n_estimators=n_estimators[np.argmin(scores)],
max_depth=max_depths[np.argmin(scores_md)],
min_child_weight=mcw, subsample=0.8,
colsample_bytree=0.8, nthread=4, seed=42, objective='multi:softprob')
 xgb.fit(X_train_scaled, y_train)
 preds = xgb.predict_proba(X_val_scaled)
 score = log_loss(y_val, preds)
 scores_mcw.append(score)
 print("test data log loss eval : {}".format(log_loss(y_val,preds)) )
def loss_fn(start_logits, end_logits, start_positions, end_positions):

 loss_fct = nn.CrossEntropyLoss()
start_loss = loss_fct(start_logits, start_positions)
 end_loss = loss_fct(end_logits, end_positions)
 total_loss =(start_loss + end_loss)
 return total_loss 
<define_variables>
scores_ss = []
subsamples = [0.5,0.6,0.7,0.8,0.9,1]
for ss in subsamples:
xgb = XGBClassifier(learning_rate =0.1, n_estimators=n_estimators[np.argmin(scores)],
max_depth=max_depths[np.argmin(scores_md)],
min_child_weight=min_child_weights[np.argmin(scores_mcw)], subsample=ss,
colsample_bytree=0.8, nthread=4, seed=42, objective='multi:softprob')
 xgb.fit(X_train_scaled, y_train)
 preds = xgb.predict_proba(X_val_scaled)
 score = log_loss(y_val, preds)
 scores_ss.append(score)
 print("test data log loss eval : {}".format(log_loss(y_val,preds)) )
def eval(data, start_outputs, end_outputs):

 scores = []
input_ids = data['input_ids']
start_targets = data['start_targets']
end_targets = data['end_targets']
offsets = data['offsets']
texts = data['original_texts']
sentiments = data['sentiments']
c_wrong_end = 0
start_outputs = torch.argmax(start_outputs, dim=1)
 end_outputs = torch.argmax(end_outputs, dim=1)
 for text, sentiment, offset, s_target, e_target, s_output, e_output in zip(texts, sentiments, offsets, start_targets,
end_targets, start_outputs, end_outputs):
s_target, e_target, s_output, e_output = s_target.item() -3, e_target.item() -3, max(s_output.item() -3, 0), max(e_output.item() -3, 0)
 len_offset = len(( offset.sum(1)> 0 ).nonzero())
 offset = offset.tolist()
if s_output > e_output:
if sentiment == 'neutral':
e_output = len_offset
else:
s_output = e_output
c_wrong_end += 1
elif e_output > len_offset:
e_output = len_offset
s_char, e_char, s_char_output, e_char_output = offset[s_target][0], offset[e_target][1], offset[s_output][0], offset[e_output][1]
s_text = text[s_char:e_char]
pred_s_text = text[s_char_output:e_char_output]
score_jaccard = compute_jaccard(s_text, pred_s_text)
 scores.append(score_jaccard)
 return c_wrong_end, round(sum(scores)/ len(scores), 2)
<find_best_params>
scores_cb = []
colsample_bytrees = [0.5,0.6,0.7,0.8,0.9,1]
for cb in colsample_bytrees:
xgb = XGBClassifier(learning_rate =0.1, n_estimators=n_estimators[np.argmin(scores)],
max_depth=max_depths[np.argmin(scores_md)],
min_child_weight=min_child_weights[np.argmin(scores_mcw)],
subsample=subsamples[np.argmin(scores_ss)],
colsample_bytree=cb, nthread=4, seed=42, objective='multi:softprob')
 xgb.fit(X_train_scaled, y_train)
 preds = xgb.predict_proba(X_val_scaled)
 score = log_loss(y_val, preds)
 scores_cb.append(score)
 print("test data log loss eval : {}".format(log_loss(y_val,preds)) )
def eval_dataloader(model, dataloader):
model.eval()
total_c_wrong_end, jaccard_mean_score, loss_score = 0, 0, 0
for data in dataloader:
input_ids = data['input_ids'].to(device)
 token_type_ids = data['token_type_ids'].to(device)
 attention_mask = data['attention_mask'].to(device)
 start_targets = data['start_targets'].to(device)
 end_targets = data['end_targets'].to(device)
 with torch.no_grad() :
start_outputs, end_outputs = model(input_ids, token_type_ids, attention_mask)
 loss = loss_fn(start_outputs, end_outputs, start_targets, end_targets)
 c_wrong_end, jaccard_score = eval(data, start_outputs, end_outputs)
 total_c_wrong_end += c_wrong_end
if loss_score == 0:
loss_score = loss.item()
else:
loss_score = round(( loss_score + loss.item())/ 2, 3)
 if jaccard_mean_score == 0:
jaccard_mean_score = jaccard_score
else:
jaccard_mean_score = round(( jaccard_score + jaccard_mean_score)/ 2, 3)
 return total_c_wrong_end, loss_score, jaccard_mean_score<train_model>
scores_eta = []
etas = [0.001,0.01,0.1,0.2,0.3,0.5,1]
for eta in etas:
xgb = XGBClassifier(learning_rate =eta, n_estimators=n_estimators[np.argmin(scores)],
max_depth=max_depths[np.argmin(scores_md)],
min_child_weight=min_child_weights[np.argmin(scores_mcw)],
subsample=subsamples[np.argmin(scores_ss)],
colsample_bytree=colsample_bytrees[np.argmin(scores_cb)],
nthread=4, seed=42, objective='multi:softprob')
 xgb.fit(X_train_scaled, y_train)
 preds = xgb.predict_proba(X_val_scaled)
 score = log_loss(y_val, preds)
 scores_eta.append(score)
 print("test data log loss eval : {}".format(log_loss(y_val,preds)) )
def train_model(train_dataloader, val_dataloader, model, epochs=3):
model = model.to(device)
 optimizer = transformers.AdamW(model.parameters() , weight_decay=0.001, lr=1e-5)
 for i in range(epochs):
model.train()
jaccard_mean_score, loss_score = 0, 0
total_c_wrong_end = 0
for data in train_dataloader:
input_ids = data['input_ids'].to(device)
 token_type_ids = data['token_type_ids'].to(device)
 attention_mask = data['attention_mask'].to(device)
 start_targets = data['start_targets'].to(device)
 end_targets = data['end_targets'].to(device)
 model.zero_grad()
start_outputs, end_outputs = model(input_ids, token_type_ids, attention_mask)
 loss = loss_fn(start_outputs, end_outputs, start_targets, end_targets)
 loss.backward()
optimizer.step()
c_wrong_end, jaccard_score = eval(data, start_outputs, end_outputs)
 total_c_wrong_end += c_wrong_end
if loss_score == 0:
loss_score = loss.item()
else:
loss_score = round(( loss_score + loss.item())/ 2, 3)
 if jaccard_mean_score == 0:
jaccard_mean_score = jaccard_score
else:
jaccard_mean_score = round(( jaccard_score + jaccard_mean_score)/ 2, 3)
 val_total_c_wrong_end, val_loss, val_jaccard_mean_score = eval_dataloader(model, val_dataloader)
 print('epoch:', i, end=' ')
 print('train end < start:', str(total_c_wrong_end)+ '/' + str(len(train_dataloader.dataset)) , end=' ')
 print('val end < start:', str(val_total_c_wrong_end)+ '/' + str(len(val_dataloader.dataset)) , end=' ')
 print('train loss:', loss_score, end=' ')
 print('val loss:', val_loss, end=' ')
 print('train jaccard:', jaccard_score, end=' ')
 print('val jaccard:', val_jaccard_mean_score, end=' ')
 print()
return model 
<choose_model_class>
xgb = XGBClassifier(learning_rate =eta, n_estimators=n_estimators[np.argmin(scores)],
max_depth=max_depths[np.argmin(scores_md)],
min_child_weight=min_child_weights[np.argmin(scores_mcw)],
subsample=subsamples[np.argmin(scores_ss)],
colsample_bytree=colsample_bytrees[np.argmin(scores_cb)],
nthread=4, seed=42, objective='multi:softprob')
 calibrated_xgb = CalibratedClassifierCV(xgb, cv=5, method='isotonic')
 calibrated_xgb.fit(X_train_scaled, y_train)
 preds = calibrated_xgb.predict_proba(X_val_scaled)
 score = log_loss(y_val, preds)
 scores_eta.append(score)
 print("test data log loss eval : {}".format(log_loss(y_val,preds)) )
model = TweetSelectionModel(albert_config )<train_model>
xgb = XGBClassifier(learning_rate =0.1, n_estimators=525, max_depth=8, min_child_weight=3, subsample=0.7,
colsample_bytree=0.7, nthread=4, seed=42, objective='multi:softprob')
 my_model = CalibratedClassifierCV(xgb, cv=5, method='isotonic')
 my_model.fit(train_X_scaled,train_y)
 test_preds = my_model.predict_proba(test_X_scaled)
 output = pd.DataFrame(test_preds,columns=["Class_"+str(i)for i in range(1,10)])
 output.insert(loc=0, column='id', value=test_data.id)
 output.to_csv('submission.csv', index=False )