Kaggle competition: Leaf Classification
<SOS> metric: MulticlassLoss Kaggle data source: leaf-classification<load_from_csv>
%pylab inline
train = pd.read_csv('/kaggle/input/google-quest-challenge/train.csv' ).fillna(' ')
 test = pd.read_csv('/kaggle/input/google-quest-challenge/test.csv' ).fillna(' ')
 sub = pd.read_csv('.. /input/google-quest-challenge/sample_submission.csv' ).fillna(' ' )<string_transform>
from sklearn.preprocessing import StandardScaler
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import LabelEncoder
def _get_segments_xlnet(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "<sep>":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return [0] *(max_seq_length - len(tokens)) + segments
def _get_segments_bert(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids_xlnet(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = [5] *(max_seq_length-len(token_ids)) + token_ids
return input_ids
def _get_ids_bert(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length=512-1,
t_max_len=70-1, q_max_len=219, a_max_len=219):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = norm_token_length(q, q_new_len)
 a = norm_token_length(a, a_new_len)
 return t, q, a
def norm_token_length(tokens, l):
if len(tokens)> l:
head = l//2
tail = l - head
return tokens[:head] + tokens[-tail:]
else:
return tokens[:l]
def _convert_to_bert_inputs(title, question, answer, cate, pretrained_weights, max_sequence_length=512):

 if "bert-base" in pretrained_weights:
stoken = ["[CLS]"] + [cate] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids_bert(stoken, tokenizer, max_sequence_length)
 input_segments = _get_segments_bert(stoken, max_sequence_length)
 elif pretrained_weights == "xlnet-base-cased":
stoken = [cate] + title + ["<sep>"] + question + ["<sep>"] + answer + ["<sep>", "<cls>"]
input_ids = _get_ids_xlnet(stoken, tokenizer, max_sequence_length)
 input_segments = _get_segments_xlnet(stoken, max_sequence_length)
 try:
cls_index = input_segments.index(5)- 1
except ValueError:
cls_index = -1
input_segments[cls_index] = 2
return [input_ids, input_segments]
def convert_row(row, pretrained_weights):
if pretrained_weights == "bert-base-uncased":
c = f"[{row['category'].lower() }]"
elif pretrained_weights == "bert-base-cased":
c = f"[{row['category']}]"
elif pretrained_weights == "xlnet-base-cased":
c = f"[{row['category']}]"
t, q, a = title = row["question_title"], row["question_body"], row["answer"]
t, q, a = _trim_input(t, q, a)
 ids, segments = _convert_to_bert_inputs(t, q, a, c, pretrained_weights)
 return np.array([[ids, segments]] )<choose_model_class>
from keras.models import Sequential
from keras.layers import Dense,Dropout,Activation
from keras.utils.np_utils import to_categorical
class CustomBert(BertPreTrainedModel):
def __init__(self, config):
super(CustomBert, self ).__init__(config)
 config.num_labels = 30
config.output_hidden_states = True
self.n_use_layer = 4
self.bert = BertModel(config)
 self.dropout = nn.Dropout(config.hidden_dropout_prob)
 self.dense1 = nn.Linear(768*self.n_use_layer, 768*self.n_use_layer)
 self.dense2 = nn.Linear(768*self.n_use_layer, 768*self.n_use_layer)
 self.classifier = nn.Linear(768*self.n_use_layer, config.num_labels)
 def forward(self, input_ids=None, attention_mask=None, token_type_ids=None,
position_ids=None, head_mask=None, inputs_embeds=None, labels=None):
outputs = self.bert(input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask)
 pooled_output = torch.cat([outputs[2][-1*i][:,0] for i in range(1, self.n_use_layer+1)], dim=1)
 pooled_output = self.dense1(pooled_output)
 pooled_output = self.dense2(pooled_output)
 pooled_output = self.dropout(pooled_output)
 logits = self.classifier(pooled_output)
 outputs =(logits,)+ outputs[2:]
return outputs<load_pretrained>
data = pd.read_csv('.. /input/train.csv')
 parent_data = data.copy()
ID = data.pop('id' )
tokenizer = BertTokenizer.from_pretrained('.. /input/quest-bert-swa/')
 tokenizer.added_tokens_encoder<prepare_x_and_y>
y = data.pop('species')
 y = LabelEncoder().fit(y ).transform(y)
 print(y.shape )
pretrained_weights = 'bert-base-uncased'
X_test = test[["question_title", "question_body", "answer", "category"]].progress_apply(lambda x: convert_row(x, pretrained_weights), axis=1 ).values
X_test = np.vstack(X_test ).reshape(( len(X_test), 1024))
test_dataset = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=BS, shuffle=False )<load_pretrained>
X = StandardScaler().fit(data ).transform(data)
 print(X.shape )
model = CustomBert.from_pretrained(".. /input/quest-bert-swa/")
 model = model.to(device)
 for param in model.parameters() :
param.requires_grad=False
model.resize_token_embeddings(len(tokenizer))
model.eval()<categorify>
y_cat = to_categorical(y)
 print(y_cat.shape )
model_dir_target = ".. /input/quest-bertuncased-10fold/"
bert_pred_lst = []
for fold in range(10):
bert_path = f"{model_dir_target}/bert-base-uncased_f{fold}_best"
model.load_state_dict(torch.load(bert_path))
lst = []
for i,(x_batch,)in enumerate(test_loader):
input_ids = x_batch[:, :512]
token_ids = x_batch[:, 512:]
pred = model(input_ids.to(device), attention_mask=(input_ids > 0 ).to(device), token_type_ids=token_ids.to(device))
lst.append(pred[0].detach().cpu().squeeze().numpy())
 test_pred = np.vstack(lst)
 bert_pred_lst.append(test_pred )<load_pretrained>
model = Sequential()
model.add(Dense(1024,input_dim=192))
model.add(Dropout(0.2))
model.add(Activation('sigmoid'))
model.add(Dense(512))
model.add(Dropout(0.3))
model.add(Activation('sigmoid'))
model.add(Dense(99))
model.add(Activation('softmax'))
y_cat = to_categorical(y)
 print(y_cat.shape )
tokenizer = BertTokenizer.from_pretrained('.. /input/quest-bertcased-tokenizer/')
 tokenizer.added_tokens_encoder<prepare_x_and_y>
model.compile(loss='categorical_crossentropy',optimizer='rmsprop' )
pretrained_weights = 'bert-base-cased'
X_test = test[["question_title", "question_body", "answer", "category"]].progress_apply(lambda x: convert_row(x, pretrained_weights), axis=1 ).values
X_test = np.vstack(X_test ).reshape(( len(X_test), 1024))
test_dataset = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=BS, shuffle=False )<load_pretrained>
history = model.fit(X,y_cat,batch_size=128,nb_epoch=100,verbose=0 )
model = CustomBert.from_pretrained(".. /input/quest-bert-swa/")
 model = model.to(device)
 for param in model.parameters() :
param.requires_grad=False
model.resize_token_embeddings(len(tokenizer))
model.eval()<define_variables>
test = pd.read_csv('.. /input/test.csv' )
model_dir_target = ".. /input/quest-bertcased-10fold/"
cased_pred_lst = []
for fold in range(10):
bert_path = f"{model_dir_target}/bert-base-cased_f{fold}_best"
model.load_state_dict(torch.load(bert_path))
lst = []
for i,(x_batch,)in enumerate(test_loader):
input_ids = x_batch[:, :512]
token_ids = x_batch[:, 512:]
pred = model(input_ids.to(device), attention_mask=(input_ids > 0 ).to(device), token_type_ids=token_ids.to(device))
lst.append(pred[0].detach().cpu().squeeze().numpy())
 test_pred = np.vstack(lst)
 cased_pred_lst.append(test_pred )<choose_model_class>
index = test.pop('id' )
class CustomXLNet(XLNetForSequenceClassification):
def __init__(self, config):
super(CustomXLNet, self ).__init__(config)
 config.num_labels = 30
config.output_hidden_states = True
self.n_use_layer = 4
self.n_labels = config.num_labels
self.transformer = XLNetModel(config)
 self.dense1 = nn.Linear(768*self.n_use_layer, 768*self.n_use_layer)
 self.dense2 = nn.Linear(768*self.n_use_layer, 768*self.n_use_layer)
 self.logits_proj = nn.Linear(768*self.n_use_layer, config.num_labels)
 def forward(
 self,
input_ids=None,
attention_mask=None,
mems=None,
perm_mask=None,
target_mapping=None,
token_type_ids=None,
input_mask=None,
head_mask=None,
inputs_embeds=None,
labels=None, 
):
transformer_outputs = self.transformer(
 input_ids,
attention_mask=attention_mask,
mems=mems,
perm_mask=perm_mask,
target_mapping=target_mapping,
token_type_ids=token_type_ids,
input_mask=input_mask,
head_mask=head_mask,
inputs_embeds=inputs_embeds, 
)
 output = torch.cat([transformer_outputs[1][-1*(i+1)][:,-1] for i in range(self.n_use_layer)], dim=1)
 output = self.dense1(output)
 output = self.dense2(output)
 logits = self.logits_proj(output)
 outputs =(logits,)+ transformer_outputs[1:]
return outputs<load_pretrained>
test = StandardScaler().fit(test ).transform(test )
tokenizer = XLNetTokenizer.from_pretrained('.. /input/quest-xlnet-tokenizer/')
 tokenizer.added_tokens_encoder<create_dataframe>
yPred = model.predict_proba(test )
pretrained_weights = "xlnet-base-cased"
X_test = test[["question_title", "question_body", "answer", "category"]].progress_apply(lambda x: convert_row(x, pretrained_weights), axis=1 ).values
X_test = np.vstack(X_test ).reshape(( len(X_test), 1024))
test_dataset = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=BS, shuffle=False )<load_pretrained>
yPred = pd.DataFrame(yPred,index=index,columns=sort(parent_data.species.unique()))
<categorify><EOS> 
fp = open('submission_nn_kernel.csv','w')
 fp.write(yPred.to_csv() )