Kaggle competition: Leaf Classification
<SOS> metric: MulticlassLoss Kaggle data source: leaf-classification<load_pretrained>
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import make_moons, make_circles, make_classification
from sklearn.neural_network import MLPClassifier
from sklearn.svm import SVC
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.feature_selection import f_classif
from sklearn.feature_selection import SelectKBest
import matplotlib.pyplot as plt
import seaborn as sns
def load_embedding(embedding_path: str, word_index: Dict[str, int])-> np.ndarray:
embeddings_index = joblib.load(embedding_path)
 nb_words = min(max_features + 2, len(word_index))
embedding_matrix = np.zeros(( nb_words, embed_size))
for key, i in word_index.items() :
word = key
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
continue
word = key.lower()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
continue
return embedding_matrix
def w2v_fine_tune(all_texts: List[str], vocab: Dict, embedding_matrix: np.ndarray)-> np.ndarray:
model = Word2Vec(min_count=1, workers=1, iter=3, size=300)
 model.build_vocab_from_freq(vocab['word_freq'])
 idxmap = np.array(
 [vocab['token2id'][w] for w in model.wv.index2entity])
 model.wv.vectors[:] = embedding_matrix[idxmap]
model.trainables.syn1neg[:] = embedding_matrix[idxmap]
model.train(all_texts, total_examples=len(all_texts), epochs=model.epochs)
 embedding_matrix = np.vstack([np.zeros(( 1, 300)) , model.wv.vectors, np.zeros(( 1, 300)) ])
 return embedding_matrix<prepare_x_and_y>
data_train = pd.read_csv('.. /input/train.csv')
 data_train.head()
vocab = build_vocab(itertools.chain(*train_texts, *test_texts), max_features)
 embedding_matrix = load_embedding(EMBEDDING_PATH, vocab['token2id'])
 embedding_matrix = w2v_fine_tune(all_texts, vocab, embedding_matrix)
 train_q = tokenize(train_texts[:, 0], vocab['token2id'], max_q_len)
 train_a = tokenize(train_texts[:, 1], vocab['token2id'], max_a_len)
 train_x = np.array([train_q, train_a] ).T
train_y = train_df[target_names].values
train_group = train_df['question_body'].values
test_q = tokenize(test_texts[:, 0], vocab['token2id'], max_q_len)
 test_a = tokenize(test_texts[:, 1], vocab['token2id'], max_a_len)
 test_x = np.array([test_q, test_a] ).T
t_max = train_y.max(axis=0)[np.newaxis, :]
t_min = train_y.min(axis=0)[np.newaxis, :]
train_y =(train_y - t_min)/(t_max - t_min )<categorify>
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column] > 0.995)]
feature_selected = data_train.drop(['id', 'species']+to_drop, axis=1 )
class UniversalSentenceEncoder(object):
def __init__(self, model_dir: str, batch_size: int = 128):
self.module = hub.load(str(model_dir))
self.batch_size = batch_size
def __call__(self, texts: List[str], mode: str)-> torch.FloatTensor:
assert mode in ['question', 'answer']
embeddings = []
for i in range(0, len(texts), self.batch_size):
text = texts[i:(i + self.batch_size)]
if mode == 'question':
h_embedding = self.module.signatures['question_encoder'](
 tf.constant(text)) ['outputs']
else:
h_embedding = self.module.signatures['response_encoder'](
 input=tf.constant(text),
context=tf.constant(text)) ['outputs']
h_embedding = torch.FloatTensor(h_embedding.numpy())
 embeddings.append(h_embedding)
 return torch.cat(embeddings, 0)
<categorify>
featureScaler = StandardScaler()
featureScaler.fit(feature_selected)
 feature_scaled = featureScaler.transform(feature_selected )
use = UniversalSentenceEncoder(USE_DIR, batch_size=32)
 q_texts = list(itertools.chain(train_df['question'].values, test_df['question'].values))
a_texts = list(itertools.chain(train_df['answer'].values, test_df['answer'].values))
q_emb = use(q_texts, mode='question')
 a_emb = use(a_texts, mode='answer')
 train_q_emb, test_q_emb = q_emb[:len(train_df)], q_emb[len(train_df):]
train_a_emb, test_a_emb = a_emb[:len(train_df)], a_emb[len(train_df):]
train_text_emb = torch.cat(( train_q_emb, train_a_emb), 1)
 test_text_emb = torch.cat(( test_q_emb, test_a_emb), 1)
 del use
cuda.select_device(0)
 cuda.close()<categorify>
classifiers = [
MLPClassifier(hidden_layer_sizes=(512,256,128), max_iter=500, tol=0.0000005),
LinearDiscriminantAnalysis()
]
for clf in classifiers:
print(type(clf))
kfold = KFold(5)
 for train_indices, test_indices in kfold.split(data_train):
clf.fit(feature_scaled[train_indices], data_train['species'].iloc[train_indices])
 print(clf.score(feature_scaled[test_indices], data_train['species'].iloc[test_indices]))
class TextDataset(Dataset):
def __init__(self, seqs, targets=None):
self.seqs = seqs
self.targets = targets
def __len__(self):
return len(self.seqs)
 def get_keys(self):
return np.vectorize(len )(self.seqs ).sum(axis=1)
 def __getitem__(self, index):
if self.targets is None:
return index, self.seqs[index]
return index, self.seqs[index], self.targets[index]
def collate_fn(data):
def _pad_sequences(seqs):
lens = [len(seq)for seq in seqs]
max_len = max(lens)
 padded_seqs = torch.zeros(len(seqs), max_len ).long()
for i, seq in enumerate(seqs):
start = max_len - lens[i]
padded_seqs[i, start:] = torch.LongTensor(seq)
 return padded_seqs
transposed = list(zip(*data))
index = transposed[0]
q_seqs, a_seqs = zip(*transposed[1])
 q_seqs = _pad_sequences(q_seqs)
 a_seqs = _pad_sequences(a_seqs)
 seqs = [q_seqs, a_seqs]
if len(transposed)== 2:
return index, seqs
return index, seqs, torch.FloatTensor(transposed[2])
 class BucketSampler(Sampler):
def __init__(self, data_source, sort_keys, bucket_size=None, batch_size=1048, shuffle_data=True):
super().__init__(data_source)
 self.shuffle = shuffle_data
self.batch_size = batch_size
self.sort_keys = sort_keys
self.bucket_size = bucket_size if bucket_size is not None else len(sort_keys)
 self.weights = None
if not shuffle_data:
self.index = self.prepare_buckets()
else:
self.index = None
def set_weights(self, weights):
assert weights >= 0
total = np.sum(weights)
 if total != 1:
weights = weights / total
self.weights = weights
def __iter__(self):
indices = None
if self.weights is not None:
total = len(self.sort_keys)
 indices = np.random.choice(total,(total,), p=self.weights)
 if self.shuffle:
self.index = self.prepare_buckets(indices)
 return iter(self.index)
 def get_reverse_indexes(self):
indexes = np.zeros(( len(self.index),), dtype=np.int32)
 for i, j in enumerate(self.index):
indexes[j] = i
return indexes
def __len__(self):
return len(self.sort_keys)
 def prepare_buckets(self, indices=None):
lens = - self.sort_keys
assert self.bucket_size % self.batch_size == 0 or self.bucket_size == len(lens)
 if indices is None:
if self.shuffle:
indices = shuffle(np.arange(len(lens), dtype=np.int32))
lens = lens[indices]
else:
indices = np.arange(len(lens), dtype=np.int32)
 def divide_chunks(l, n):
if n == len(l):
yield np.arange(len(l), dtype=np.int32), l
else:
for i in range(0, len(l), n):
data = l[i:i + n]
yield np.arange(i, i + len(data), dtype=np.int32), data
new_indices = []
extra_batch = None
for chunk_index, chunk in divide_chunks(lens, self.bucket_size):
indices_sorted = chunk_index[np.argsort(chunk, axis=-1)]
batches = []
for _, batch in divide_chunks(indices_sorted, self.batch_size):
if len(batch)== self.batch_size:
batches.append(batch.tolist())
 else:
assert extra_batch is None
assert batch is not None
extra_batch = batch
if self.shuffle:
batches = shuffle(batches)
 for batch in batches:
new_indices.extend(batch)
 if extra_batch is not None:
new_indices.extend(extra_batch)
 return indices[new_indices]<create_dataframe>
final_clf = MLPClassifier(hidden_layer_sizes=(512,256,128), max_iter=500, tol=0.0000005)
 final_clf.fit(feature_scaled, data_train['species'] )
<choose_model_class><EOS> 
data_test = pd.read_csv('.. /input/test.csv')
 feature_test = featureScaler.transform(data_test.drop(['id']+to_drop, axis=1))
pd.concat([data_test[['id']], pd.DataFrame(final_clf.predict_proba(feature_test), columns=final_clf.classes_)], axis=1 ).to_csv('submission.csv', index=False )