Kaggle competition: Leaf Classification
<load_pretrained><EOS> 
import pandas as pd
import numpy as np
<SOS> metric: MulticlassLoss Kaggle data source: leaf-classification<categorify>
import pandas as pd
import numpy as np
model_dir_target = ".. /input/quest-bertuncased-10fold/"
bert_pred_lst = []
for fold in range(10):
bert_path = f"{model_dir_target}/bert-base-uncased_f{fold}_best"
model.load_state_dict(torch.load(bert_path))
lst = []
for i,(x_batch,)in enumerate(test_loader):
input_ids = x_batch[:, :512]
token_ids = x_batch[:, 512:]
pred = model(input_ids.to(device), attention_mask=(input_ids > 0 ).to(device), token_type_ids=token_ids.to(device))
lst.append(pred[0].detach().cpu().squeeze().numpy())
 test_pred = np.vstack(lst)
 bert_pred_lst.append(test_pred )<load_pretrained>
df = pd.read_csv('.. /input/leaf-classification/train.csv.zip' )
tokenizer = BertTokenizer.from_pretrained('.. /input/quest-bertcased-tokenizer/')
 tokenizer.added_tokens_encoder<prepare_x_and_y>
set(( df.dtypes ).to_list() )
pretrained_weights = 'bert-base-cased'
X_test = test[["question_title", "question_body", "answer", "category"]].progress_apply(lambda x: convert_row(x, pretrained_weights), axis=1 ).values
X_test = np.vstack(X_test ).reshape(( len(X_test), 1024))
test_dataset = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=BS, shuffle=False )<load_pretrained>
%matplotlib inline
model = CustomBert.from_pretrained(".. /input/quest-bert-swa/")
 model = model.to(device)
 for param in model.parameters() :
param.requires_grad=False
model.resize_token_embeddings(len(tokenizer))
model.eval()<define_variables>
model_dir_target = ".. /input/quest-bertcased-10fold/"
cased_pred_lst = []
for fold in range(10):
bert_path = f"{model_dir_target}/bert-base-cased_f{fold}_best"
model.load_state_dict(torch.load(bert_path))
lst = []
for i,(x_batch,)in enumerate(test_loader):
input_ids = x_batch[:, :512]
token_ids = x_batch[:, 512:]
pred = model(input_ids.to(device), attention_mask=(input_ids > 0 ).to(device), token_type_ids=token_ids.to(device))
lst.append(pred[0].detach().cpu().squeeze().numpy())
 test_pred = np.vstack(lst)
 cased_pred_lst.append(test_pred )<choose_model_class>
image_dir = './images'
class CustomXLNet(XLNetForSequenceClassification):
def __init__(self, config):
super(CustomXLNet, self ).__init__(config)
 config.num_labels = 30
config.output_hidden_states = True
self.n_use_layer = 4
self.n_labels = config.num_labels
self.transformer = XLNetModel(config)
 self.dense1 = nn.Linear(768*self.n_use_layer, 768*self.n_use_layer)
 self.dense2 = nn.Linear(768*self.n_use_layer, 768*self.n_use_layer)
 self.logits_proj = nn.Linear(768*self.n_use_layer, config.num_labels)
 def forward(
 self,
input_ids=None,
attention_mask=None,
mems=None,
perm_mask=None,
target_mapping=None,
token_type_ids=None,
input_mask=None,
head_mask=None,
inputs_embeds=None,
labels=None, 
):
transformer_outputs = self.transformer(
 input_ids,
attention_mask=attention_mask,
mems=mems,
perm_mask=perm_mask,
target_mapping=target_mapping,
token_type_ids=token_type_ids,
input_mask=input_mask,
head_mask=head_mask,
inputs_embeds=inputs_embeds, 
)
 output = torch.cat([transformer_outputs[1][-1*(i+1)][:,-1] for i in range(self.n_use_layer)], dim=1)
 output = self.dense1(output)
 output = self.dense2(output)
 logits = self.logits_proj(output)
 outputs =(logits,)+ transformer_outputs[1:]
return outputs<load_pretrained>
X = df.drop(['id', 'species'], axis=1)
 y = df['species']
print(X.shape, y.shape )
tokenizer = XLNetTokenizer.from_pretrained('.. /input/quest-xlnet-tokenizer/')
 tokenizer.added_tokens_encoder<create_dataframe>
from sklearn.preprocessing import LabelEncoder
pretrained_weights = "xlnet-base-cased"
X_test = test[["question_title", "question_body", "answer", "category"]].progress_apply(lambda x: convert_row(x, pretrained_weights), axis=1 ).values
X_test = np.vstack(X_test ).reshape(( len(X_test), 1024))
test_dataset = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=BS, shuffle=False )<load_pretrained>
df['species'].unique()
model = CustomXLNet.from_pretrained(".. /input/quest-xlnet-tokenizer/")
 model = model.to(device)
 for param in model.parameters() :
param.requires_grad=False
model.resize_token_embeddings(len(tokenizer))
model.eval()<categorify>
classEncoder = LabelEncoder()
trainLabels = classEncoder.fit_transform(df.loc[:,'species'] )
model_dir_target = ".. /input/quest-xlnetcased-10fold/"
xlnet_pred_lst = []
for fold in range(10):
bert_path = f"{model_dir_target}/xlnet-base-cased_f{fold}_best"
model.load_state_dict(torch.load(bert_path))
lst = []
for i,(x_batch,)in enumerate(test_loader):
input_ids = x_batch[:, :512]
token_ids = x_batch[:, 512:]
pred = model(input_ids.to(device), attention_mask=(input_ids != 5 ).int().to(device), token_type_ids=token_ids.to(device))
lst.append(pred[0].detach().cpu().squeeze().numpy())
 test_pred = np.vstack(lst)
 xlnet_pred_lst.append(test_pred )<compute_test_metric>
classEncoder.classes_[trainLabels[:5]]
def sigmoid(x):
return 1.0 /(1.0 + np.exp(-x))
bert_pred = np.array(bert_pred_lst ).mean(0)
 cased_pred = np.array(cased_pred_lst ).mean(0)
 xlnet_pred = np.array(xlnet_pred_lst ).mean(0)
 bert_pred = sigmoid(bert_pred)
 cased_pred = sigmoid(cased_pred)
 xlnet_pred = sigmoid(xlnet_pred)
 test_pred = bert_pred*0.4 + cased_pred*0.2 + xlnet_pred*0.4<load_pretrained>
y = classEncoder.fit_transform(y)
 y[:5]
lgbm_models = pickle.load(open(".. /input/quest-lgbm/lgbm_question_type_spelling.pkl", 'rb'))
count_vectorizers = pickle.load(open(".. /input/quest-lgbm/tfidf_vectorizers.pkl", 'rb'))<concatenate>
from sklearn.preprocessing import StandardScaler
dfs = []
for idx, col_name in enumerate(["question_title", "question_body", "answer"]):
X = count_vectorizers[idx].transform(test[col_name])
 feat = [f"{col_name}_{c}".encode("utf-8")for c in count_vectorizers[idx].get_feature_names() ]
df = pd.DataFrame(X.toarray() , columns=feat)
 dfs.append(df)
 test_x = pd.concat(dfs, axis=1)
 test_x.shape<predict_on_test>
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X )
lgbm_pred = np.zeros(len(test_x))
for fold in range(4):
lgbm_pred += lgbm_models[fold].predict(test_x)/4
lgbm_pred.shape<compute_test_metric>
from sklearn.model_selection import train_test_split
def spearman_corr(y_true, y_pred):
if np.ndim(y_pred)== 2:
corr = np.nan_to_num([stats.spearmanr(y_true[:, i], y_pred[:, i])[0] for i in range(y_true.shape[1])] ).mean()
else:
corr = stats.spearmanr(y_true, y_pred)[0]
return corr
class OptimizedRounder(object):
def __init__(self,
n_overall: int = 5,
n_classwise: int = 5,
n_classes: int = 7,
metric: str = "qwk"):
self.n_overall = n_overall
self.n_classwise = n_classwise
self.n_classes = n_classes
self.coef = [1.0 / n_classes * i for i in range(1, n_classes)]
self.metric_str = metric
self.metric = spearman_corr
def _loss(self, X: np.ndarray, y: np.ndarray)-> float:
X_p = np.digitize(X, self.coef)
 ll = -self.metric(y, X_p)
 return ll
def fit(self, X: np.ndarray, y: np.ndarray):
golden1 = 0.618
golden2 = 1 - golden1
ab_start = [ 
(0.01, 1.0 / self.n_classes + 0.05),
]
for i in range(1, self.n_classes):
ab_start.append(( i * 1.0 / self.n_classes + 0.05, 
(i + 1)* 1.0 / self.n_classes + 0.05))
for _ in range(self.n_overall):
for idx in range(self.n_classes - 1):
a, b = ab_start[idx]
self.coef[idx] = a
la = self._loss(X, y)
 self.coef[idx] = b
lb = self._loss(X, y)
 for it in range(self.n_classwise):
if la > lb:
a = b -(b - a)* golden1
self.coef[idx] = a
la = self._loss(X, y)
 else:
b = b -(b - a)* golden2
self.coef[idx] = b
lb = self._loss(X, y)
 def predict(self, X: np.ndarray)-> np.ndarray:
X_p = np.digitize(X, self.coef)
 return X_p<predict_on_test>
X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size = 0.3, random_state = 99)
 X_train.shape, X_test.shape, y_train.shape, y_test.shape
optR_lst = pickle.load(open(".. /input/quest-optimizedrounder/optR_lst_10fold_ensemble_v3.pkl", 'rb'))
lst = []
for idx, optR in enumerate(optR_lst):
coeff = optR.predict(test_pred[:, idx])
 lst.append(coeff/norm_dict[sub.columns[1:][idx]])
 opt_preds = np.array(lst ).T
test_pred = opt_preds
test_pred[:,19] = lgbm_pred<feature_engineering>
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV
test_pred[test[test["category"] != "CULTURE"].index, 19] = 0.0
test["host_info"] = test["question_user_page"].map(lambda x: x.split("/")[2].replace(".stackexchange.com", ""))
test_pred[test[test["host_info"].map(lambda x: x not in ["english", "ell"])].index, 19] = 0.0<save_to_csv>
params = {'C':[1, 10, 50, 100, 500, 1000, 2000],
'tol': [0.001, 0.0001, 0.005]
}
sub[sub.columns[1:]] = test_pred
sub.to_csv("submission.csv", index=False)
 sub.head()<import_modules>
lr = LogisticRegression(solver='newton-cg', multi_class='multinomial')
pyLDAvis.enable_notebook()
np.random.seed(2018)
 warnings.filterwarnings('ignore' )<load_from_csv>
LogLoss = make_scorer(log_loss, greater_is_better=False, needs_proba=True )
sample = pd.read_csv('/kaggle/input/google-quest-challenge/sample_submission.csv')
 sample.head(3 )<load_from_csv>
clf = GridSearchCV(lr, params, scoring = LogLoss, refit = 'True', n_jobs = 1, cv = 5)
train = pd.read_csv('/kaggle/input/google-quest-challenge/train.csv')
 train.head(3 )<load_from_csv>
lr.fit(X_train, y_train )
test = pd.read_csv('/kaggle/input/google-quest-challenge/test.csv')
 test.head(3 )<define_variables>
clf_fit = clf.fit(X_scaled, y )
targets = [
'question_asker_intent_understanding',
'question_body_critical',
'question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others',
'question_interestingness_self',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'question_well_written',
'answer_helpful',
'answer_level_of_information',
'answer_plausible',
'answer_relevance',
'answer_satisfaction',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation',
'answer_well_written'
] 
<feature_engineering>
print('Logistic Regression Score {}'.format(lr.score(X_test, y_test)) )
lengths = train['question_title'].apply(len)
 train['lengths'] = lengths
lengths = train.loc[train['lengths']<4000]['lengths']
sns.distplot(lengths, color='b')
 plt.show()<feature_engineering>
print('Grid Search Score {}'.format(clf_fit.score(X_scaled, y)) )
stopwords=stopwords.words('english')
 train['que_stopwords']=train['question_body'].apply(lambda x : [x for x in x.split() if x in stopwords])
 train['ans_stopwords']=train['answer'].apply(lambda x: [x for x in x.split() if x in stopwords] )<count_unique_values>
print("best params: " + str(clf.best_params_))
print("best estimator: " + str(clf.best_estimator_))
clf.cv_results_['mean_test_score']
def common_ngrams(col,common=10):
corpus=[]
for question in train[col].values:
words=[str(x[0]+' '+x[1])for x in ngrams(question.split() ,2)]
corpus.append(words)
 flatten=[x for one in corpus for x in one]
counter=Counter(flatten)
 most_common=counter.most_common(common)
 string,value=zip(*(most_common))
return string,value 
<set_options>
test = pd.read_csv('.. /input/leaf-classification/test.csv.zip')
 test_ids = test.pop('id')
 x_test = test.values
scaler = StandardScaler()
x_test = scaler.fit_transform(x_test)
 y_pred = clf.predict_proba(x_test )
np.set_printoptions(suppress=True )<load_from_csv>
sample_sub = pd.read_csv('.. /input/leaf-classification/sample_submission.csv.zip', nrows = 5)
 sample_sub
<string_transform><EOS> 
subm = pd.DataFrame(y_pred, index=test_ids, columns = classEncoder.classes_)
 subm.to_csv('Submission_LogisticReg_v1.csv' )