Kaggle competition: Leaf Classification
<SOS> metric: MulticlassLoss Kaggle data source: leaf-classification<split>
warnings.filterwarnings("ignore")
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
kf = KFold(n_splits=3, shuffle=True, random_state=1029)
 train_scores = []
valid_scores = []
train_optimized_scores = defaultdict(list)
 valid_optimized_scores = defaultdict(list)
 thresholds = defaultdict(list)
 for train_idx, valid_idx in kf.split(train_y):
train_fold_preds, train_fold_y = ema_train_preds[train_idx], train_y[train_idx]
valid_fold_preds, valid_fold_y = ema_train_preds[valid_idx], train_y[valid_idx]
train_scores.append(get_scores(train_fold_y, train_fold_preds))
valid_scores.append(get_scores(valid_fold_y, valid_fold_preds))
for i, target_name in enumerate(target_names):
optimizer = OptimizedRounder()
optimizer.fit(train_y[train_idx, i], ema_train_preds[train_idx, i])
 train_score = optimizer.score(train_y[train_idx, i], ema_train_preds[train_idx, i])
 valid_score = optimizer.score(train_y[valid_idx, i], ema_train_preds[valid_idx, i])
 train_optimized_scores[target_name].append(train_score)
 valid_optimized_scores[target_name].append(valid_score)
 thresholds[target_name].append(optimizer.threshold )<compute_test_metric>
test_ids = test.id
train.drop(['id'], axis=1, inplace=True)
 test.drop(['id'], axis=1, inplace=True )
train_score = pd.DataFrame(train_scores ).values.mean()
valid_score = pd.DataFrame(valid_scores ).values.mean()
train_optimized_score = np.mean(list(train_optimized_scores.values()))
valid_optimized_score = np.mean(list(valid_optimized_scores.values()))
print(f'train score: {train_score:.4f} -> {train_optimized_score:.4f}')
 print(f'valid score: {valid_score:.4f} -> {valid_optimized_score:.4f}' )<feature_engineering>
margin_cols = [col for col in train.columns if 'margin' in col]
shape_cols = [col for col in train.columns if 'shape' in col]
texture_cols = [col for col in train.columns if 'texture' in col]
for i, target_name in enumerate(target_names):
optimizer = OptimizedRounder()
optimizer.threshold = np.mean(thresholds[target_name], axis=0)
 ema_test_preds[:, i] = optimizer.transform(ema_test_preds[:, i] )<save_to_csv>
X = train.drop('species', axis=1)
 y = train[["species"]]
le=LabelEncoder().fit(y)
 y=le.transform(y )
submission = pd.read_csv(SAMPLE_SUBMISSION_PATH)
 submission[target_names] = ema_test_preds
submission.to_csv('submission.csv', index=False)
 print(f'all processes done in {(time.time() - start_time)/ 60:.2f} min.' )<define_variables>
sss = StratifiedShuffleSplit(n_splits = 10, test_size=0.2, random_state=123)
 scores = []
k = 0
for train_index, test_index in sss.split(X, y):
X_train, X_test = X.iloc[train_index], X.iloc[test_index]
y_train, y_test = y[train_index], y[test_index]
BS = 8
SEED = 42<install_modules>
classifiers = [
KNeighborsClassifier(3),
SVC(kernel="rbf", C=0.025, probability=True),
NuSVC(probability=True),
DecisionTreeClassifier() ,
RandomForestClassifier(n_estimators=1000),
AdaBoostClassifier() ,
GradientBoostingClassifier() ,
GaussianNB() ,
LinearDiscriminantAnalysis() ,
QuadraticDiscriminantAnalysis() ,
LogisticRegression()
]
log_cols=["Classifier", "Accuracy", "Log Loss"]
log = pd.DataFrame(columns=log_cols)
 for clf in classifiers:
clf.fit(X_train, y_train)
 name = clf.__class__.__name__
print("+"*30)
 print(name)
 print('****Results****')
 train_predictions = clf.predict(X_test)
 acc = accuracy_score(y_test, train_predictions)
 print("Accuracy: {:.4%}".format(acc))
train_predictions = clf.predict_proba(X_test)
 ll = log_loss(y_test, train_predictions)
 print("Log Loss: {}".format(ll))
log_entry = pd.DataFrame([[name, acc*100, ll]], columns=log_cols)
 log = log.append(log_entry)
 print("+"*30 )
!pip install.. /input/sacremoses/sacremoses-master/ > /dev/null
!pip install.. /input/transformers/transformers-master/ > /dev/null<set_options>
scaler = StandardScaler().fit(X_train)
 X_train_scale = scaler.transform(X_train )
tqdm.pandas()
device = torch.device('cuda' )<load_from_csv>
params = {'C':[1000, 2000], 'tol': [0.001, 0.0001]}
log_reg = LogisticRegression(solver='newton-cg', multi_class='multinomial',class_weight='balanced',max_iter=400)
 grid_search_lgr = GridSearchCV(log_reg, params, scoring='neg_log_loss', refit='True', n_jobs=-1, cv=5)
 grid_search_lgr.fit(X_train_scale, y_train)
 print('Best score: {}'.format(grid_search_lgr.best_score_))
print('Best parameters: {}'.format(grid_search_lgr.best_params_))
train = pd.read_csv('/kaggle/input/google-quest-challenge/train.csv' ).fillna(' ')
 test = pd.read_csv('/kaggle/input/google-quest-challenge/test.csv' ).fillna(' ')
 sub = pd.read_csv('.. /input/google-quest-challenge/sample_submission.csv' ).fillna(' ' )<string_transform>
X_test_scale = scaler.transform(test)
 y_pred_prob = grid_search_lgr.predict_proba(X_test_scale)
 print(y_pred_prob.shape)
def _get_segments_xlnet(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "<sep>":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return [0] *(max_seq_length - len(tokens)) + segments
def _get_segments_bert(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids_xlnet(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = [5] *(max_seq_length-len(token_ids)) + token_ids
return input_ids
def _get_ids_bert(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length=512-1,
t_max_len=70-1, q_max_len=219, a_max_len=219):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = norm_token_length(q, q_new_len)
 a = norm_token_length(a, a_new_len)
 return t, q, a
def norm_token_length(tokens, l):
if len(tokens)> l:
head = l//2
tail = l - head
return tokens[:head] + tokens[-tail:]
else:
return tokens[:l]
def _convert_to_bert_inputs(title, question, answer, cate, pretrained_weights, max_sequence_length=512):

 if "bert-base" in pretrained_weights:
stoken = ["[CLS]"] + [cate] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids_bert(stoken, tokenizer, max_sequence_length)
 input_segments = _get_segments_bert(stoken, max_sequence_length)
 elif pretrained_weights == "xlnet-base-cased":
stoken = [cate] + title + ["<sep>"] + question + ["<sep>"] + answer + ["<sep>", "<cls>"]
input_ids = _get_ids_xlnet(stoken, tokenizer, max_sequence_length)
 input_segments = _get_segments_xlnet(stoken, max_sequence_length)
 try:
cls_index = input_segments.index(5)- 1
except ValueError:
cls_index = -1
input_segments[cls_index] = 2
return [input_ids, input_segments]
def convert_row(row, pretrained_weights):
if pretrained_weights == "bert-base-uncased":
c = f"[{row['category'].lower() }]"
elif pretrained_weights == "bert-base-cased":
c = f"[{row['category']}]"
elif pretrained_weights == "xlnet-base-cased":
c = f"[{row['category']}]"
t, q, a = title = row["question_title"], row["question_body"], row["answer"]
t, q, a = _trim_input(t, q, a)
 ids, segments = _convert_to_bert_inputs(t, q, a, c, pretrained_weights)
 return np.array([[ids, segments]] )<choose_model_class>
submission = pd.DataFrame(y_pred_prob, columns=list(le.classes_))
submission.insert(0, 'id', test_ids)
 submission.reset_index()
submission.head()
class CustomBert(BertPreTrainedModel):
def __init__(self, config):
super(CustomBert, self ).__init__(config)
 config.num_labels = 30
config.output_hidden_states = True
self.n_use_layer = 4
self.bert = BertModel(config)
 self.dropout = nn.Dropout(config.hidden_dropout_prob)
 self.dense1 = nn.Linear(768*self.n_use_layer, 768*self.n_use_layer)
 self.dense2 = nn.Linear(768*self.n_use_layer, 768*self.n_use_layer)
 self.classifier = nn.Linear(768*self.n_use_layer, config.num_labels)
 def forward(self, input_ids=None, attention_mask=None, token_type_ids=None,
position_ids=None, head_mask=None, inputs_embeds=None, labels=None):
outputs = self.bert(input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask)
 pooled_output = torch.cat([outputs[2][-1*i][:,0] for i in range(1, self.n_use_layer+1)], dim=1)
 pooled_output = self.dense1(pooled_output)
 pooled_output = self.dense2(pooled_output)
 pooled_output = self.dropout(pooled_output)
 logits = self.classifier(pooled_output)
 outputs =(logits,)+ outputs[2:]
return outputs<load_pretrained>
submission.to_csv('submission.csv', index=False )
<prepare_x_and_y><EOS> 
submission.to_csv('submission.csv', index=False )