Kaggle competition: Traffic signs classification
<SOS> metric: categorizationaccuracy Kaggle data source: traffic-signs-classification<filter>
import numpy as np
import pandas as pd
import skimage.io
from skimage.transform import resize
import matplotlib.pyplot as plt
from tqdm import tqdm
import os
ts = time.time()
matrix = matrix[matrix.date_block_num > 11]
time.time() - ts<correct_missing_values>
path_to_train = '.. /input/tl-signs-hse-itmo-2020-winter/train/train/'
data = pd.read_csv('.. /input/tl-signs-hse-itmo-2020-winter/train.csv')
 train_dataset_info = []
for name, label in zip(data['filename'], data['class_number']):
train_dataset_info.append({
'path':os.path.join(path_to_train, name),
'label':int(label)- 1})
 train_dataset_info = np.array(train_dataset_info )
ts = time.time()
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
if('item_cnt' in col):
df[col].fillna(0, inplace=True)
 return df
matrix = fill_na(matrix)
 time.time() - ts<load_pretrained>
class data_generator:

 def create_train(dataset_info, batch_size, shape):
while True:
random_indexes = np.random.choice(len(dataset_info), batch_size)
 batch_images = np.empty(( batch_size, shape[0], shape[1], shape[2]))
batch_labels = np.zeros(( batch_size, 66))
for i, idx in enumerate(random_indexes):
image = data_generator.load_image(
 dataset_info[idx]['path'], shape)
 batch_images[i] = image
batch_labels[i][dataset_info[idx]['label']] = 1
yield batch_images, batch_labels
def load_image(path, shape):
image = skimage.io.imread(path)
 image = resize(image,(shape[0], shape[1]), mode='reflect')
 return image
matrix.to_pickle('data.pkl')
 del matrix
del cache
del group
del items
del shops
del cats
del train
gc.collect() ;<load_pretrained>
train_datagen = data_generator.create_train(
 train_dataset_info, 5,(48, 48, 3))
data = pd.read_pickle('data.pkl' )<drop_column>
def create_model(input_shape, n_out):
model = Sequential([
Conv2D(32,(3,3), padding='same', activation=tf.nn.relu,
input_shape=(48, 48, 3)) ,
MaxPooling2D(( 2, 2), strides=2),
Conv2D(64,(3,3), padding='same', activation=tf.nn.relu),
MaxPooling2D(( 2, 2), strides=2),
Flatten() ,
Dense(128, activation=tf.nn.relu),
Dense(66, activation=tf.nn.softmax)
 ])
 return model
data = data[[
'date_block_num',
'shop_id',
'item_id',
'item_cnt_month',
'city_code',
'item_category_id',
'type_code',
'subtype_code',
'item_cnt_month_lag_1',
'item_cnt_month_lag_2',
'item_cnt_month_lag_3',
'item_cnt_month_lag_6',
'item_cnt_month_lag_12',
'date_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_2',
'date_item_avg_item_cnt_lag_3',
'date_item_avg_item_cnt_lag_6',
'date_item_avg_item_cnt_lag_12',
'date_shop_avg_item_cnt_lag_1',
'date_shop_avg_item_cnt_lag_2',
'date_shop_avg_item_cnt_lag_3',
'date_shop_avg_item_cnt_lag_6',
'date_shop_avg_item_cnt_lag_12',
'date_cat_avg_item_cnt_lag_1',
'date_shop_cat_avg_item_cnt_lag_1',
'date_city_avg_item_cnt_lag_1',
'date_item_city_avg_item_cnt_lag_1',
'delta_price_lag',
'month',
'days',
'item_shop_last_sale',
'item_last_sale',
'item_shop_first_sale',
'item_first_sale',
]]<prepare_x_and_y>
keras.backend.clear_session()
model = create_model(
 input_shape=(48, 48, 3),
n_out=66)
 model.compile(
 loss='categorical_crossentropy',
optimizer=Adam(1e-04),
metrics=['acc'])
 model.summary()
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )<set_options>
epochs = 100; batch_size = 16
checkpointer = ModelCheckpoint(
 '.. /working/InceptionResNetV2.model',
verbose=2,
save_best_only=True)
 np.random.seed(2018)
 indexes = np.arange(train_dataset_info.shape[0])
 np.random.shuffle(indexes)
 train_indexes = indexes[:20000]
valid_indexes = indexes[20000:]
train_generator = data_generator.create_train(
 train_dataset_info[train_indexes], batch_size,(48, 48, 3))
validation_generator = data_generator.create_train(
 train_dataset_info[valid_indexes], 100,(48, 48, 3))
history = model.fit_generator(
 train_generator,
steps_per_epoch=100,
validation_data=next(validation_generator),
epochs=epochs,
verbose=1,
callbacks=[checkpointer] )
del data
gc.collect() ;<train_model>
submit = pd.read_csv('.. /input/tl-signs-hse-itmo-2020-winter/sample_submission.csv')
 predicted = []
for name in tqdm(submit['filename']):
path = os.path.join('.. /input/tl-signs-hse-itmo-2020-winter/test/test', name)
 image = data_generator.load_image(path,(48, 48, 3))
score_predict = model.predict(image[np.newaxis])[0]
label_predict = np.argmax(score_predict)
 str_predict_label = str(label_predict + 1)
 predicted.append(str_predict_label )
<save_to_csv><EOS> 
submit['class_number'] = predicted
submit.to_csv('submission_neural.csv', index=False )