Kaggle competition: Leaf Classification
<SOS> metric: MulticlassLoss Kaggle data source: leaf-classification<import_modules>
root = '.. /input'
np.random.seed(2016)
 split_random_state = 7
split =.9
def load_numeric_training(standardize=True):

 data = pd.read_csv(os.path.join(root, 'train.csv'))
ID = data.pop('id')
 y = data.pop('species')
 y = LabelEncoder().fit(y ).transform(y)
 X = StandardScaler().fit(data ).transform(data)if standardize else data.values
return ID, X, y
def load_numeric_test(standardize=True):

 test = pd.read_csv(os.path.join(root, 'test.csv'))
ID = test.pop('id')
 test = StandardScaler().fit(test ).transform(test)if standardize else test.values
return ID, test
def resize_img(img, max_dim=96):

 max_ax = max(( 0, 1), key=lambda i: img.size[i])
 scale = max_dim / float(img.size[max_ax])
 return img.resize(( int(img.size[0] * scale), int(img.size[1] * scale)))
 def load_image_data(ids, max_dim=96, center=True):

 X = np.empty(( len(ids), max_dim, max_dim, 1))
for i, idee in enumerate(ids):
x = resize_img(load_img(os.path.join(root, 'images', str(idee)+ '.jpg'), grayscale=True), max_dim=max_dim)
 x = img_to_array(x)
 length = x.shape[0]
width = x.shape[1]
if center:
h1 = int(( max_dim - length)/ 2)
 h2 = h1 + length
w1 = int(( max_dim - width)/ 2)
 w2 = w1 + width
else:
h1, w1 = 0, 0
h2, w2 =(length, width)
 X[i, h1:h2, w1:w2, 0:1] = x
return np.around(X / 255.0)
 def load_train_data(split=split, random_state=None):

 ID, X_num_tr, y = load_numeric_training()
X_img_tr = load_image_data(ID)
 sss = StratifiedShuffleSplit(n_splits=1, train_size=split, random_state=random_state)
 train_ind, test_ind = next(sss.split(X_num_tr, y))
X_num_val, X_img_val, y_val = X_num_tr[test_ind], X_img_tr[test_ind], y[test_ind]
X_num_tr, X_img_tr, y_tr = X_num_tr[train_ind], X_img_tr[train_ind], y[train_ind]
return(X_num_tr, X_img_tr, y_tr),(X_num_val, X_img_val, y_val)
 def load_test_data() :

 ID, X_num_te = load_numeric_test()
X_img_te = load_image_data(ID)
 return ID, X_num_te, X_img_te
print('Loading the training data...')
(X_num_tr, X_img_tr, y_tr),(X_num_val, X_img_val, y_val)= load_train_data(random_state=split_random_state)
 y_tr_cat = to_categorical(y_tr)
 y_val_cat = to_categorical(y_val)
 print('Training data loaded!' )
from collections import Counter, defaultdict
import copy
from functools import partial
import itertools
import os
from pathlib import Path
import random
import re
import string
import time
from typing import Dict, List, Tuple
import warnings
from gensim.models import Word2Vec
import joblib
from numba import cuda
import numpy as np
import pandas as pd
import plotly_express as px
import scipy
from nltk.stem import PorterStemmer, SnowballStemmer
from nltk.stem.lancaster import LancasterStemmer
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.model_selection import GroupKFold, KFold
from sklearn.utils import shuffle
import tensorflow as tf
import tensorflow_hub as hub
import torch
from torch import nn
from torch.optim.optimizer import Optimizer
from torch.utils.data import Dataset, Sampler, DataLoader<load_from_csv>
class ImageDataGenerator2(ImageDataGenerator):
def flow(self, X, y=None, batch_size=32, shuffle=True, seed=None,
save_to_dir=None, save_prefix='', save_format='jpeg'):
return NumpyArrayIterator2(
 X, y, self,
batch_size=batch_size, shuffle=shuffle, seed=seed,
dim_ordering=self.dim_ordering,
save_to_dir=save_to_dir, save_prefix=save_prefix, save_format=save_format)
 class NumpyArrayIterator2(NumpyArrayIterator):
def next(self):
with self.lock:
self.index_array, current_index, current_batch_size = next(self.index_generator)
 batch_x = np.zeros(tuple([current_batch_size] + list(self.X.shape)[1:]))
for i, j in enumerate(self.index_array):
x = self.X[j]
x = self.image_data_generator.random_transform(x.astype('float32'))
x = self.image_data_generator.standardize(x)
 batch_x[i] = x
if self.save_to_dir:
for i in range(current_batch_size):
img = array_to_img(batch_x[i], self.dim_ordering, scale=True)
 fname = '{prefix}_{index}_{hash}.{format}'.format(prefix=self.save_prefix,
index=current_index + i,
hash=np.random.randint(1e4),
format=self.save_format)
 img.save(os.path.join(self.save_to_dir, fname))
if self.y is None:
return batch_x
batch_y = self.y[self.index_array]
return batch_x, batch_y
print('Creating Data Augmenter...')
 imgen = ImageDataGenerator2(
 rotation_range=20,
zoom_range=0.2,
horizontal_flip=True,
vertical_flip=True,
fill_mode='nearest')
 imgen_train = imgen.flow(X_img_tr, y_tr_cat, seed=np.random.randint(1, 10000))
print('Finished making data augmenter...' )
start_time = time.time()
INPUT_DIR = Path('.. /input/')
 DATA_DIR = INPUT_DIR / 'google-quest-challenge'
TRAIN_PATH = DATA_DIR / 'train.csv'
TEST_PATH = DATA_DIR / 'test.csv'
SAMPLE_SUBMISSION_PATH = DATA_DIR / 'sample_submission.csv'
USE_DIR = INPUT_DIR / 'universal-sentence-encoder/universal-sentence-encoder-qa/universal-sentence-encoder-qa/'
EMBEDDING_PATH = INPUT_DIR / 'fasttext-vector/fasttext.pkl'
train_df = pd.read_csv(TRAIN_PATH)
 test_df = pd.read_csv(TEST_PATH)
 num_targets = 30
target_names = [
'question_asker_intent_understanding',
'question_body_critical',
'question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others',
'question_interestingness_self',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'question_well_written',
'answer_helpful',
'answer_level_of_information',
'answer_plausible',
'answer_relevance',
'answer_satisfaction',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation',
'answer_well_written']
train_df['question'] = train_df['question_title'] + ' ' + train_df['question_body']
test_df['question'] = test_df['question_title'] + ' ' + test_df['question_body']
n_splits = 5
n_epochs = 9
batch_size = 32
max_q_len = 512
max_a_len = 512
updates_per_epoch = 100
mu = 0.9
embed_size = 300
max_features = 60000
seed = 1029
device = torch.device('cuda')
 ps = PorterStemmer()
lc = LancasterStemmer()
sb = SnowballStemmer('english')
 warnings.filterwarnings('ignore')
 random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True<define_variables>
def combined_model() :
image = Input(shape=(96, 96, 1), name='image')
 x = Convolution2D(8, 5, 5, input_shape=(96, 96, 1), border_mode='same' )(image)
 x =(Activation('relu'))(x)
 x =(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)) )(x)
 x =(Convolution2D(32, 5, 5, border_mode='same'))(x)
 x =(Activation('relu'))(x)
 x =(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)) )(x)
 x = Flatten()(x)
 numerical = Input(shape=(192,), name='numerical')
 concatenated = merge([x, numerical], mode='concat')
 x = Dense(100, activation='relu' )(concatenated)
 x = Dropout (.5 )(x)
 out = Dense(99, activation='softmax' )(x)
 model = Model(input=[image, numerical], output=out)
 model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])
 return model
print('Creating the model...')
 model = combined_model()
print('Model created!' )
misspell_dict = {"aren't": "are not", "can't": "cannot", "couldn't": "could not",
"didn't": "did not", "doesn't": "does not", "don't": "do not",
"hadn't": "had not", "hasn't": "has not", "haven't": "have not",
"he'd": "he would", "he'll": "he will", "he's": "he is",
"i'd": "I had", "i'll": "I will", "i'm": "I am", "isn't": "is not",
"it's": "it is", "it'll": "it will", "i've": "I have", "let's": "let us",
"mightn't": "might not", "mustn't": "must not", "shan't": "shall not",
"she'd": "she would", "she'll": "she will", "she's": "she is",
"shouldn't": "should not", "that's": "that is", "there's": "there is",
"they'd": "they would", "they'll": "they will", "they're": "they are",
"they've": "they have", "we'd": "we would", "we're": "we are",
"weren't": "were not", "we've": "we have", "what'll": "what will",
"what're": "what are", "what's": "what is", "what've": "what have",
"where's": "where is", "who'd": "who would", "who'll": "who will",
"who're": "who are", "who's": "who is", "who've": "who have",
"won't": "will not", "wouldn't": "would not", "you'd": "you would",
"you'll": "you will", "you're": "you are", "you've": "you have",
"'re": " are", "wasn't": "was not", "we'll": " will", "tryin'": "trying"}
def replace_typical_misspell(text: str)-> str:
misspell_re = re.compile('(%s)' % '|'.join(misspell_dict.keys()))
def replace(match):
return misspell_dict[match.group(0)]
return misspell_re.sub(replace, text)
 puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']',
'>', '%', '=', '
'®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█',
'½', 'à', '…', '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶',
'↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '¼',
'⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲',
'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '∙', '）', '↓', '、', '│', '（', '»', '，', '♪',
'╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√']
def clean_text(text: str)-> str:
text = str(text)
 for punct in puncts + list(string.punctuation):
if punct in text:
text = text.replace(punct, f' {punct} ')
 return text
def clean_numbers(text: str)-> str:
return re.sub(r'\d+', ' ', text)
 def preprocess_text(text: str)-> str:
text = text.lower()
text = replace_typical_misspell(text)
 text = clean_text(text)
 text = clean_numbers(text)
 text = text.strip()
return text<categorify>
def combined_generator(imgen, X):

 while True:
for i in range(X.shape[0]):
batch_img, batch_y = next(imgen)
 x = X[imgen.index_array]
yield [batch_img, x], batch_y
best_model_file = "leafnet.h5"
best_model = ModelCheckpoint(best_model_file, monitor='val_loss', verbose=1, save_best_only=True)
 print('Training model...')
 history = model.fit_generator(combined_generator(imgen_train, X_num_tr),
samples_per_epoch=X_num_tr.shape[0],
nb_epoch=89,
validation_data=([X_img_val, X_num_val], y_val_cat),
nb_val_samples=X_num_val.shape[0],
verbose=0,
callbacks=[best_model])
 print('Loading the best model...')
 model = load_model(best_model_file)
 print('Best Model loaded!' )
<feature_engineering><EOS> 
LABELS = sorted(pd.read_csv(os.path.join(root, 'train.csv')).species.unique())
 index, test, X_img_te = load_test_data()
yPred_proba = model.predict([X_img_te, test])
 yPred = pd.DataFrame(yPred_proba,index=index,columns=LABELS)
 print('Creating and writing submission...')
 fp = open('submit.csv', 'w')
 fp.write(yPred.to_csv())
 print('Finished writing submission')
 yPred.tail()