Kaggle competition: Leaf Classification
<SOS> metric: MulticlassLoss Kaggle data source: leaf-classification<feature_engineering>
start = time.time()
for col_num, col_name in enumerate(target_columns[-30:]):
for fold_num in range(FOLD_NUM):
for model_num in range(len(model_names)) :
coef = all_coeffs[col_num][fold_num][model_num]
submission[col_num] += coef / FOLD_NUM * inf_sub[fold_num][model_num][col_name].values
sub_df.iloc[:, -30:] = submission.T
sub_df<feature_engineering>
%pylab inline
def norm_sub(df):
for col_name in df.columns[-30:]:
tmp_df = df[col_name].values
v_max = np.max(tmp_df)+ 0.01
v_min = np.min(tmp_df)- 0.01
df[col_name] = df[col_name].apply(lambda x:(x - v_min)/(v_max - v_min))
df[col_name] = df[col_name].values + np.random.normal(0, 1e-7, len(df))
return df
sub_df = norm_sub(sub_df )<load_from_csv>
from sklearn.preprocessing import StandardScaler
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import StratifiedShuffleSplit
<statistical_test>
from keras.models import Sequential
from keras.layers import merge
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers.advanced_activations import PReLU
from keras.layers import Convolution2D, MaxPooling2D
from keras.utils.np_utils import to_categorical
from keras.callbacks import EarlyStopping
class color:
PURPLE = '\033[95m'
CYAN = '\033[96m'
DARKCYAN = '\033[36m'
BLUE = '\033[94m'
GREEN = '\033[92m'
YELLOW = '\033[93m'
RED = '\033[91m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
END = '\033[0m'
def compute_spearmanr(trues, preds, columns=None):
rhos = []
for i,(col_true, col_pred)in enumerate(zip(trues.T, preds.T)) :
if i == 0:
continue
r = spearmanr(col_true, col_pred ).correlation
if columns:
print("{:.3f}\t{}".format(r, columns[i-1]))
rhos.append(r)
 return np.mean(rhos)
 def compare_spearmanr(trues, preds, pp_preds, columns=None):
rhos = []
pp_rhos = []
for i,(col_true, col_pred, col_pp_pred)in enumerate(zip(trues.T, preds.T, pp_preds.T)) :
if i == 0:
continue
r = spearmanr(col_true, col_pred ).correlation
pp_r = spearmanr(col_true, col_pp_pred ).correlation
if columns:
if r < pp_r:
print("{:.3f} ->".format(r)+ color.BOLD + " {:.3f}".format(pp_r)+ color.END + " \t{}".format(columns[i-1]))
elif(r - pp_r)< 10**(-6):
print("{:.3f} ->".format(r)+ " {:.3f}".format(pp_r)+ " \t{}".format(columns[i-1]))
else:
print("{:.3f} ->".format(r)+ color.RED + " {:.3f}".format(pp_r)+ color.END + " \t{}".format(columns[i-1]))
rhos.append(r)
 pp_rhos.append(pp_r)
 whole_base = np.mean(rhos)
 whole_pp = np.mean(pp_rhos)
 print("Whole Score")
 if whole_base < whole_pp:
print("{:.3f} ->".format(whole_base)+ color.BOLD + " {:.3f}".format(whole_pp)+ color.END + " +{}".format(abs(whole_pp - whole_base)))
 elif(whole_base - whole_pp)< 10**(-6):
print("{:.3f} ->".format(whole_base)+ " {:.3f}".format(whole_pp))
else:
print("{:.3f} ->".format(whole_base)+ color.RED + " {:.3f}".format(whole_pp)+ color.END + " -{}".format(abs(whole_pp - whole_base)))
 return whole_base, whole_pp
def inf_numbert_of_annotators(v):
s = set(v)
 d = 10
for v in s:
for u in s:
if v != u:
d = min(abs(v - u), d)
 for i in range(100):
if d * i >= 1 - 10**(-3):
return i
def zero_one_scale(v):
max_x = np.max(v)
 min_x = np.min(v)
 return(v - min_x)/(max_x - min_x)
 def post_process(train_df, train, pred_df):
print(len(train), len(pred_df))
df = pred_df.copy()
df.loc[(train["host"] != "english.stackexchange.com")&(train["host"] != "ell.stackexchange.com"), "question_type_spelling"] = 0
for col in ["question_type_spelling", "question_type_entity", "question_type_definition",
"question_type_compare", "question_conversational", "question_interestingness_others",
"question_type_instructions", "question_type_consequence", "question_interestingness_self",
"question_has_commonly_accepted_answer", "answer_relevance", "question_not_really_a_question"]:
n = inf_numbert_of_annotators(train_df[col].values)
 df[col] = zero_one_scale(df[col].values)
 for i in range(n+1):
base = i / n
low = base - 1 /(2 * n)
 up = base + 1 /(2 * n)
 df.loc[(df[col] >= low)&(df[col] <= up), col] = base
if len(set(df[col].values)) == 1:
df[col] = pred_df[col].values
return df<load_from_csv>
data = pd.read_csv('.. /input/train.csv')
 parent_data = data.copy()
ID = data.pop('id' )
train_df = pd.read_csv('.. /input/google-quest-challenge/train.csv')
 test_df = pd.read_csv('.. /input/google-quest-challenge/test.csv')
<save_to_csv>
y = data.pop('species')
 y = LabelEncoder().fit(y ).transform(y)
 print(y.shape )
sub_df.to_csv("submission.csv", index=False )<import_modules>
X = preprocessing.MinMaxScaler().fit(data ).transform(data)
 X = StandardScaler().fit(data ).transform(data)
 print(X.shape)
 X
import pandas as pd
import numpy as np
import transformers
import tensorflow as tf
from scipy.stats import spearmanr
import tensorflow_hub as hub
import re
from sklearn.preprocessing import MinMaxScaler
import gc
from sklearn.model_selection import GroupKFold,KFold
from scipy.stats import spearmanr, rankdata
from sklearn.linear_model import MultiTaskElasticNet
import string
from collections import Counter
from gensim.models import Word2Vec
from transformers import XLNetConfig, TFXLNetModel, XLNetTokenizer, TFXLNetMainLayer, BertConfig, TFBertMainLayer, BertTokenizer, TFBertModel
import lightgbm as lgb<define_variables>
y_cat = to_categorical(y)
 print(y_cat.shape )
DIR = '/kaggle/input/google-quest-challenge'<load_pretrained>
sss = StratifiedShuffleSplit(n_splits=10, test_size=0.2,random_state=12345)
 train_index, val_index = next(iter(sss.split(X, y)))
 x_train, x_val = X[train_index], X[val_index]
y_train, y_val = y_cat[train_index], y_cat[val_index]
print("x_train dim: ",x_train.shape)
 print("x_val dim: ",x_val.shape )
xlnet_tokenizer = XLNetTokenizer.from_pretrained('/kaggle/input/xlnetbasecased/tokenizer')
 bert_tokenizer = BertTokenizer.from_pretrained('/kaggle/input/bertbaseuncasedcomplete/bert-model' )<load_from_csv>
model = Sequential()
model.add(Dense(768,input_dim=192, kernel_initializer='glorot_normal', activation='tanh'))
model.add(Dropout(0.4))
model.add(Dense(768, activation='tanh'))
model.add(Dropout(0.4))
model.add(Dense(99, activation='softmax'))
train_df = pd.read_csv(DIR+'/train.csv')
 test_df = pd.read_csv(DIR+'/test.csv')
 cols = train_df.columns[11:]<categorify>
model.compile(loss='categorical_crossentropy',optimizer='rmsprop', metrics = ["accuracy"] )
def tokenize_input(tokenizer, s1, s2, tags, data_name, max_length, tokenizer_name = 'bert'):
if s2 is not None:
x = tokenizer.encode_plus(s1, s2, pad_to_max_length=False)
 if len(x['input_ids'])> max_length:
segment_1 = int(0.25*max_length)
 x['input_ids'] = x['input_ids'][:segment_1] + x['input_ids'][-(max_length-segment_1):]
x['attention_mask'] = x['attention_mask'][:segment_1] + x['attention_mask'][-(max_length-segment_1):]
x['token_type_ids'] = x['token_type_ids'][:segment_1] + x['token_type_ids'][-(max_length-segment_1):]
else:
diff = max_length - len(x['input_ids'])
 if tokenizer_name == 'xlnet':
x['input_ids'] = [tokenizer.pad_token_id]*diff + x['input_ids']
x['attention_mask'] = [0]*diff + x['attention_mask']
x['token_type_ids'] = [tokenizer.pad_token_type_id]*diff + x['token_type_ids']
else:
x['input_ids'] = x['input_ids'] + [tokenizer.pad_token_id]*diff
x['attention_mask'] = x['attention_mask'] + [0]*diff
x['token_type_ids'] = x['token_type_ids'] + [0]*diff
else:
x = tokenizer.encode_plus(s1)
 if len(x['input_ids'])> max_length:
segment_1 = int(0.25*max_length)
 x['input_ids'] = x['input_ids'][:segment_1] + x['input_ids'][-(max_length-segment_1):]
x['attention_mask'] = x['attention_mask'][:segment_1] + x['attention_mask'][-(max_length-segment_1):]
x['token_type_ids'] = x['token_type_ids'][:segment_1] + x['token_type_ids'][-(max_length-segment_1):]
else:
diff = max_length - len(x['input_ids'])
 if tokenizer_name == 'xlnet':
x['input_ids'] = [tokenizer.pad_token_id]*diff + x['input_ids']
x['attention_mask'] = [0]*diff + x['attention_mask']
x['token_type_ids'] = [tokenizer.pad_token_type_id]*diff + x['token_type_ids']
else:
x['input_ids'] = x['input_ids'] + [tokenizer.pad_token_id]*diff
x['attention_mask'] = x['attention_mask'] + [0]*diff
x['token_type_ids'] = x['token_type_ids'] + [tokenizer.pad_token_type_id]*diff
data[data_name][tags[0]].append(x['input_ids'])
 data[data_name][tags[1]].append(x['token_type_ids'])
 data[data_name][tags[2]].append(x['attention_mask'])
 data = {}
data['xlnet_train_t_a'] = {}
data['xlnet_train_q_a'] = {}
data['xlnet_train_t_q'] = {}
data['xlnet_train_q'] = {}
data['xlnet_train_a'] = {}
data['xlnet_test_t_a'] = {}
data['xlnet_test_q_a'] = {}
data['xlnet_test_t_q'] = {}
data['xlnet_test_q'] = {}
data['xlnet_test_a'] = {}
tags = ['input_ids', 'token_type_ids', 'attention_masks']
data['xlnet_train_t_a'][tags[0]], data['xlnet_train_t_a'][tags[1]], data['xlnet_train_t_a'][tags[2]] = [], [], []
data['xlnet_train_q_a'][tags[0]], data['xlnet_train_q_a'][tags[1]], data['xlnet_train_q_a'][tags[2]] = [], [], []
data['xlnet_train_t_q'][tags[0]], data['xlnet_train_t_q'][tags[1]], data['xlnet_train_t_q'][tags[2]] = [], [], []
data['xlnet_train_q'][tags[0]], data['xlnet_train_q'][tags[1]], data['xlnet_train_q'][tags[2]] = [], [], []
data['xlnet_train_a'][tags[0]], data['xlnet_train_a'][tags[1]], data['xlnet_train_a'][tags[2]] = [], [], []
data['xlnet_test_t_a'][tags[0]], data['xlnet_test_t_a'][tags[1]], data['xlnet_test_t_a'][tags[2]] = [], [], []
data['xlnet_test_q_a'][tags[0]], data['xlnet_test_q_a'][tags[1]], data['xlnet_test_q_a'][tags[2]] = [], [], []
data['xlnet_test_t_q'][tags[0]], data['xlnet_test_t_q'][tags[1]], data['xlnet_test_t_q'][tags[2]] = [], [], []
data['xlnet_test_q'][tags[0]], data['xlnet_test_q'][tags[1]], data['xlnet_test_q'][tags[2]] = [], [], []
data['xlnet_test_a'][tags[0]], data['xlnet_test_a'][tags[1]], data['xlnet_test_a'][tags[2]] = [], [], []
for i in range(train_df.shape[0]):
tokenize_input(xlnet_tokenizer, train_df.loc[i, 'question_title'], train_df.loc[i, 'answer'], tags, 'xlnet_train_t_a', 512, 'xlnet')
 tokenize_input(xlnet_tokenizer, train_df.loc[i, 'question_body'], train_df.loc[i, 'answer'], tags, 'xlnet_train_q_a', 512, 'xlnet')
 tokenize_input(xlnet_tokenizer, train_df.loc[i, 'question_title'], train_df.loc[i, 'question_body'], tags, 'xlnet_train_t_q', 512, 'xlnet')
 tokenize_input(xlnet_tokenizer, train_df.loc[i, 'question_body'], None, tags, 'xlnet_train_q', 512, 'xlnet')
 tokenize_input(xlnet_tokenizer, train_df.loc[i, 'answer'], None, tags, 'xlnet_train_a', 512, 'xlnet')
 for i in range(test_df.shape[0]):
tokenize_input(xlnet_tokenizer, test_df.loc[i, 'question_title'], test_df.loc[i, 'answer'], tags, 'xlnet_test_t_a', 512, 'xlnet')
 tokenize_input(xlnet_tokenizer, test_df.loc[i, 'question_body'], test_df.loc[i, 'answer'], tags, 'xlnet_test_q_a', 512, 'xlnet')
 tokenize_input(xlnet_tokenizer, test_df.loc[i, 'question_title'], test_df.loc[i, 'question_body'], tags, 'xlnet_test_t_q', 512, 'xlnet')
 tokenize_input(xlnet_tokenizer, test_df.loc[i, 'question_body'], None, tags, 'xlnet_test_q', 512, 'xlnet')
 tokenize_input(xlnet_tokenizer, test_df.loc[i, 'answer'], None, tags, 'xlnet_test_a', 512, 'xlnet')
 data['bert_train_t_a'] = {}
data['bert_train_q_a'] = {}
data['bert_train_t_q'] = {}
data['bert_train_q'] = {}
data['bert_train_a'] = {}
data['bert_test_t_a'] = {}
data['bert_test_q_a'] = {}
data['bert_test_t_q'] = {}
data['bert_test_q'] = {}
data['bert_test_a'] = {}
data['bert_train_t_a'][tags[0]], data['bert_train_t_a'][tags[1]], data['bert_train_t_a'][tags[2]] = [], [], []
data['bert_train_q_a'][tags[0]], data['bert_train_q_a'][tags[1]], data['bert_train_q_a'][tags[2]] = [], [], []
data['bert_train_t_q'][tags[0]], data['bert_train_t_q'][tags[1]], data['bert_train_t_q'][tags[2]] = [], [], []
data['bert_train_q'][tags[0]], data['bert_train_q'][tags[1]], data['bert_train_q'][tags[2]] = [], [], []
data['bert_train_a'][tags[0]], data['bert_train_a'][tags[1]], data['bert_train_a'][tags[2]] = [], [], []
data['bert_test_t_a'][tags[0]], data['bert_test_t_a'][tags[1]], data['bert_test_t_a'][tags[2]] = [], [], []
data['bert_test_q_a'][tags[0]], data['bert_test_q_a'][tags[1]], data['bert_test_q_a'][tags[2]] = [], [], []
data['bert_test_t_q'][tags[0]], data['bert_test_t_q'][tags[1]], data['bert_test_t_q'][tags[2]] = [], [], []
data['bert_test_q'][tags[0]], data['bert_test_q'][tags[1]], data['bert_test_q'][tags[2]] = [], [], []
data['bert_test_a'][tags[0]], data['bert_test_a'][tags[1]], data['bert_test_a'][tags[2]] = [], [], []
for i in range(train_df.shape[0]):
tokenize_input(bert_tokenizer, train_df.loc[i, 'question_title'], train_df.loc[i, 'answer'], tags, 'bert_train_t_a', 512, 'bert')
 tokenize_input(bert_tokenizer, train_df.loc[i, 'question_body'], train_df.loc[i, 'answer'], tags, 'bert_train_q_a', 512, 'bert')
 tokenize_input(bert_tokenizer, train_df.loc[i, 'question_title'], train_df.loc[i, 'question_body'], tags, 'bert_train_t_q', 512, 'bert')
 tokenize_input(bert_tokenizer, train_df.loc[i, 'question_body'], None, tags, 'bert_train_q', 512, 'bert')
 tokenize_input(bert_tokenizer, train_df.loc[i, 'answer'], None, tags, 'bert_train_a', 512, 'bert')
 for i in range(test_df.shape[0]):
tokenize_input(bert_tokenizer, test_df.loc[i, 'question_title'], test_df.loc[i, 'answer'], tags, 'bert_test_t_a', 512, 'bert')
 tokenize_input(bert_tokenizer, test_df.loc[i, 'question_body'], test_df.loc[i, 'answer'], tags, 'bert_test_q_a', 512, 'bert')
 tokenize_input(bert_tokenizer, test_df.loc[i, 'question_title'], test_df.loc[i, 'question_body'], tags, 'bert_test_t_q', 512, 'bert')
 tokenize_input(bert_tokenizer, test_df.loc[i, 'question_body'], None, tags, 'bert_test_q', 512, 'bert')
 tokenize_input(bert_tokenizer, test_df.loc[i, 'answer'], None, tags, 'bert_test_a', 512, 'bert')
 for key, _ in data.items() :
for k, _ in data[key].items() :
data[key][k] = np.array(data[key][k] )<compute_train_metric>
early_stopping = EarlyStopping(monitor='val_loss', patience=300)
 history = model.fit(x_train, y_train,batch_size=192,epochs=2500 ,verbose=0,
validation_data=(x_val, y_val),callbacks=[early_stopping] )
def SpearmanCorrCoeff(A, B):
overall_score = 0
for index in range(A.shape[1]):
overall_score += spearmanr(A[:, index], B[:, index] ).correlation
return overall_score/30
class PredictCallback(tf.keras.callbacks.Callback):
def __init__(self, data, labels):
self.data = data
self.labels = labels
def on_epoch_end(self, epoch, logs = {}):
predictions = self.model.predict(self.data)
 print('
\t Validation Score - ' + str(SpearmanCorrCoeff(self.labels, predictions)) )<define_search_model>
print('val_acc: ',max(history.history['val_acc']))
print('val_loss: ',min(history.history['val_loss']))
print('train_acc: ',max(history.history['acc']))
print('train_loss: ',min(history.history['loss']))
print()
print("train/val loss ratio: ", min(history.history['loss'])/min(history.history['val_loss']))
class BERT(TFBertModel):
def __init__(self, config, *inputs, **kwrgs):
super(BERT, self ).__init__(config, *inputs, **kwrgs)
 self.bert = TFBertMainLayer(config, name = 'bert')
 for i in range(1, 45):
self.bert.submodules[-i].trainable = False
def call(self, inputs, **kwrgs):
outputs = self.bert(inputs)
 hidden_states = outputs[2]
h12 = hidden_states[-1][:, 0, :]
h11 = hidden_states[-2][:, 0, :]
h10 = hidden_states[-3][:, 0, :]
h9 = hidden_states[-4][:, 0, :]
concat = tf.keras.layers.Concatenate(axis = -1 )([h9, h10, h11, h12])
 return concat
class XLNet(TFXLNetModel):
def __init__(self, config, *inputs, **kwrgs):
super(XLNet, self ).__init__(config, *inputs, **kwrgs)
 self.transformer = TFXLNetMainLayer(config, name = 'transformer')
 for i in range(1, 3):
self.transformer.layer[-i].trainable = False
def call(self, inputs, **kwrgs):
outputs = self.transformer(inputs)
 hidden_states = outputs[1]
h12 = hidden_states[-1][:, 0, :]
h11 = hidden_states[-2][:, 0, :]
h10 = hidden_states[-3][:, 0, :]
h9 = hidden_states[-4][:, 0, :]
concat = tf.keras.layers.Concatenate(axis = -1 )([h9, h10, h11, h12])
 return concat<load_pretrained>
test = pd.read_csv('.. /input/test.csv')
 index = test.pop('id')
 test = preprocessing.MinMaxScaler().fit(test ).transform(test)
 test = StandardScaler().fit(test ).transform(test)
 yPred = model.predict_proba(test )
def create_model(name):
id_1 = tf.keras.Input(shape =(512), dtype = tf.int32)
 id_2 = tf.keras.Input(shape =(512), dtype = tf.int32)
 type_id_1 = tf.keras.Input(shape =(512), dtype = tf.int32)
 type_id_2 = tf.keras.Input(shape =(512), dtype = tf.int32)
 a1 = tf.keras.Input(shape =(512), dtype = tf.int32)
 a2 = tf.keras.Input(shape =(512), dtype = tf.int32)
 if name == 'xlnet':
config = XLNetConfig.from_pretrained('/kaggle/input/xlnetbasecased/config-xlnet-base-cased', output_hidden_states = True)
 transformer = XLNet.from_pretrained('/kaggle/input/xlnetbasecased/model-xlnet-base-cased', config = config)
 else:
config = BertConfig.from_pretrained('/kaggle/input/bertbaseuncasedcomplete/bert-model', output_hidden_states = True)
 transformer = BERT.from_pretrained('/kaggle/input/bertbaseuncasedcomplete/bert-model', config = config)
 out_1 = transformer({'input_ids':id_1, 'attention_mask':a1, 'token_type_ids':type_id_1})
 out_2 = transformer({'input_ids':id_2, 'attention_mask':a2, 'token_type_ids':type_id_2})
 concat = tf.keras.layers.Concatenate(axis = -1 )([out_1, out_2])
 drop = tf.keras.layers.Dropout(rate = 0.1 )(concat)
 dense = tf.keras.layers.Dense(30, activation = 'sigmoid' )(drop)
 return tf.keras.Model(inputs = [id_1, id_2, type_id_1, type_id_2, a1, a2], outputs = [dense] )<define_variables>
yPred = pd.DataFrame(yPred,index=index,columns=sort(parent_data.species.unique()))
<split><EOS> 
fp = open('submission_nn_kernel.csv','w')
 fp.write(yPred.to_csv())
 end = time.time()
print(round(( end-start),2), "seconds" )