Kaggle competition: Leaf Classification
<SOS> metric: MulticlassLoss Kaggle data source: leaf-classification<predict_on_test>
%pylab inline
xlnet_inputs =(
 data['xlnet_train_a'][tags[0]], data['xlnet_train_t_q'][tags[0]], data['xlnet_train_a'][tags[1]], data['xlnet_train_t_q'][tags[1]],
data['xlnet_train_a'][tags[2]], data['xlnet_train_t_q'][tags[2]] 
)
 predictions = xlnet_model.predict(xlnet_inputs)
 train_y = final_outputs[train_idx]
valid_y = final_outputs[valid_idx]
train_preds = predictions[train_idx]
valid_preds = predictions[valid_idx]<predict_on_test>
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
estimators = []
post_valid_preds = np.zeros_like(valid_preds)
 for i in range(train_preds.shape[1]):
lgb_train = lgb.Dataset(train_preds[:, i:i+1], label=train_y[:, i])
 params = {'objective': 'rmse', 'num_leaves': 3, 'learning_rate': 0.01, 'min_data_in_leaf': 20}
est = lgb.train(params, lgb_train, num_boost_round=50)
 estimators.append(est)
 post_valid_preds[:, i] = est.predict(valid_preds[:, i:i+1])
 if(post_valid_preds[:, i].max() - post_valid_preds[:, i].min())< 0.000001:
max_idx = np.argmax(post_valid_preds[:, i])
 post_valid_preds[:, i][max_idx] = min(0.9999999, post_valid_preds[:, i][max_idx] + 0.001)
 min_idx = np.argmin(post_valid_preds[:, i])
 post_valid_preds[:, i][min_idx] = max(0.0000001, post_valid_preds[:, i][min_idx] - 0.001)
 print(f"Validation Score(Before){SpearmanCorrCoeff(valid_y, valid_preds)}")
 print(f"Validation Score(After){SpearmanCorrCoeff(valid_y, post_valid_preds)}" )<predict_on_test>
from keras.models import Sequential
from keras.layers import Dense,Dropout,Activation
from keras.utils.np_utils import to_categorical
xlnet_test_inputs = inputs =(data['xlnet_test_a'][tags[0]], data['xlnet_test_t_q'][tags[0]], data['xlnet_test_a'][tags[1]],
data['xlnet_test_t_q'][tags[1]],data['xlnet_test_a'][tags[2]], data['xlnet_test_t_q'][tags[2]])
 test_preds = xlnet_model.predict(xlnet_test_inputs)
 post_test_preds = np.zeros_like(test_preds)
 for i in range(train_preds.shape[1]):
post_test_preds[:, i] = estimators[i].predict(test_preds[:, i:i+1])
 if(post_valid_preds[:, i].max() - post_valid_preds[:, i].min())< 0.000001:
max_idx = np.argmax(post_valid_preds[:, i])
 post_valid_preds[:, i][max_idx] = min(0.9999999, post_valid_preds[:, i][max_idx] + 0.00001)
 min_idx = np.argmin(post_valid_preds[:, i])
 post_valid_preds[:, i][min_idx] = max(0.0000001, post_valid_preds[:, i][min_idx] - 0.00001)
 submission = pd.read_csv(DIR+'/sample_submission.csv')
 submission.iloc[:,1:] = post_test_preds
submission.to_csv("submission.csv", index = False)
 submission.head()<import_modules>
rcParams['figure.figsize'] = 10,10
import pandas as pd
import numpy as np
import gc
import os
from math import floor, ceil
from tqdm.notebook import tqdm
from sklearn.model_selection import GroupKFold
from scipy.stats import spearmanr
import tensorflow as tf
import tensorflow_hub as hub
import tensorflow.keras.backend as K
from transformers import BertTokenizer
from transformers import BertConfig
from transformers import TFBertModel<load_from_csv>
data = pd.read_csv('.. /input/train.csv')
 parent_data = data.copy()
ID = data.pop('id' )
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-base-uncased-huggingface-transformer/'
tokenizer = BertTokenizer.from_pretrained(BERT_PATH+'bert-base-uncased-vocab.txt')
 MAX_SEQUENCE_LENGTH = 384
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )<categorify>
y = data.pop('species')
 y = LabelEncoder().fit(y ).transform(y)
 print(y.shape )
def _convert_to_transformer_inputs(title, question, answer, tokenizer, max_sequence_length):

 def return_id(str1, str2, truncation_strategy, length):
inputs = tokenizer.encode_plus(str1, str2,
add_special_tokens=True,
max_length=length,
truncation_strategy=truncation_strategy)
 input_ids = inputs["input_ids"]
input_masks = [1] * len(input_ids)
 input_segments = inputs["token_type_ids"]
padding_length = length - len(input_ids)
 padding_id = tokenizer.pad_token_id
input_ids = input_ids +([padding_id] * padding_length)
 input_masks = input_masks +([0] * padding_length)
 input_segments = input_segments +([0] * padding_length)
 return [input_ids, input_masks, input_segments]
input_ids_q, input_masks_q, input_segments_q = return_id(
 title + ' ' + question, None, 'longest_first', max_sequence_length)
 input_ids_a, input_masks_a, input_segments_a = return_id(
 answer, None, 'longest_first', max_sequence_length)
 return [input_ids_q, input_masks_q, input_segments_q,
input_ids_a, input_masks_a, input_segments_a]
def compute_input_arrays(df, columns, tokenizer, max_sequence_length):
input_ids_q, input_masks_q, input_segments_q = [], [], []
input_ids_a, input_masks_a, input_segments_a = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
ids_q, masks_q, segments_q, ids_a, masks_a, segments_a = \
_convert_to_transformer_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids_q.append(ids_q)
 input_masks_q.append(masks_q)
 input_segments_q.append(segments_q)
 input_ids_a.append(ids_a)
 input_masks_a.append(masks_a)
 input_segments_a.append(segments_a)
 return [np.asarray(input_ids_q, dtype=np.int32),
np.asarray(input_masks_q, dtype=np.int32),
np.asarray(input_segments_q, dtype=np.int32),
np.asarray(input_ids_a, dtype=np.int32),
np.asarray(input_masks_a, dtype=np.int32),
np.asarray(input_segments_a, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )<feature_engineering>
X = StandardScaler().fit(data ).transform(data)
 print(X.shape )
def compute_spearmanr_ignore_nan(trues, preds):
rhos = []
for tcol, pcol in zip(np.transpose(trues), np.transpose(preds)) :
rhos.append(spearmanr(tcol, pcol ).correlation)
 return np.nanmean(rhos)
 q_id = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_id = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 q_mask = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_mask = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 q_atn = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_atn = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 def create_model() :
config = BertConfig()
config.output_hidden_states = False
bert_model = TFBertModel.from_pretrained(
 BERT_PATH+'bert-base-uncased-tf_model.h5', config=config)
 q_embedding = bert_model(q_id, attention_mask=q_mask, token_type_ids=q_atn)[0]
a_embedding = bert_model(a_id, attention_mask=a_mask, token_type_ids=a_atn)[0]
q = tf.keras.layers.GlobalAveragePooling1D()(q_embedding)
 a = tf.keras.layers.GlobalAveragePooling1D()(a_embedding)
 x = tf.keras.layers.Concatenate()([q, a])
 x = tf.keras.layers.Dropout(0.2 )(x)
 x = tf.keras.layers.Dense(30, activation='sigmoid' )(x)
 model = tf.keras.models.Model(inputs=[q_id, q_mask, q_atn, a_id, a_mask, a_atn,], outputs=x)
 return model<categorify>
y_cat = to_categorical(y)
 print(y_cat.shape )
outputs = compute_output_arrays(df_train, output_categories)
 inputs = compute_input_arrays(df_train, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputs = compute_input_arrays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
<split>
model = Sequential()
model.add(Dense(256, input_dim=108, init='uniform', activation='relu'))
model.add(Dense(128, init='normal', activation='sigmoid'))
model.add(Dense(99, activation='softmax'))
gkf = GroupKFold(n_splits=5 ).split(X=df_train.question_body, groups=df_train.question_body)
 valid_preds = []
test_preds = []
for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold in [0, 2]:
train_inputs = [inputs[i][train_idx] for i in range(len(inputs)) ]
train_outputs = outputs[train_idx]
valid_inputs = [inputs[i][valid_idx] for i in range(len(inputs)) ]
valid_outputs = outputs[valid_idx]
K.clear_session()
model = create_model()
optimizer = tf.keras.optimizers.Adam(learning_rate=2e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 model.fit(train_inputs, train_outputs, epochs=3, batch_size=6)
 valid_preds.append(model.predict(valid_inputs))
test_preds.append(model.predict(test_inputs))
rho_val = compute_spearmanr_ignore_nan(valid_outputs, valid_preds[-1])
 print('validation score = ', rho_val )<save_to_csv>
model = Sequential()
model.add(Dense(512,input_dim=192, init='uniform', activation='relu'))
model.add(Dropout(0.3))
model.add(Dense(256, activation='sigmoid'))
model.add(Dropout(0.3))
model.add(Dense(99, activation='softmax'))
df_sub.iloc[:, 1:] = np.average(test_preds, axis=0)
 df_sub.to_csv('submission.csv', index=False )<import_modules>
model.compile(loss='categorical_crossentropy',optimizer='rmsprop', metrics = ["accuracy"] )
import pandas as pd
import numpy as np<import_modules>
history = model.fit(X,y_cat,batch_size=192,
nb_epoch=130,verbose=0, validation_split=0.1 )
import pandas as pd
import numpy as np<define_variables>
test = pd.read_csv('.. /input/test.csv' )
PATH = '.. /input/google-quest-challenge/'<load_from_csv>
index = test.pop('id' )
train = pd.read_csv(PATH+'train.csv')
 test = pd.read_csv(PATH+'test.csv' )<prepare_x_and_y>
test = StandardScaler().fit(test ).transform(test )
train_x = train.loc[:, 'qa_id':'host']<prepare_x_and_y>
yPred = model.predict_proba(test )
train_y = train.loc[:, 'question_asker_intent_understanding':'answer_well_written']<drop_column>
yPred = pd.DataFrame(yPred,index=index,columns=sort(parent_data.species.unique()))
<import_modules><EOS> 
fp = open('submission_nn_kernel.csv','w')
 fp.write(yPred.to_csv() )