Kaggle competition: Quick, Draw! Doodle Recognition Challenge
<SOS> metric: MAP@{K} Kaggle data source: quick,-draw!-doodle-recognition-challenge<save_to_csv>
import json, gc, os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tqdm import tqdm
import cv2
from tensorflow import keras
from tensorflow.keras.applications import MobileNet
from tensorflow.keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D
from tensorflow.keras.models import load_model, Sequential
from sklearn.model_selection import train_test_split
from tensorflow.keras.metrics import top_k_categorical_accuracy
sub['revenue'] = np.expm1(prediction_xgb)
 sub.to_csv("XGB_model.csv", index=False )<save_to_csv>
np.random.seed(seed=58 )
sub['revenue'] = np.expm1(prediction_cat)
 sub.to_csv("cat_model.csv", index=False )<save_to_csv>
df_train = pd.read_csv('.. /input/quick-draw-doodle-recognition-challenge-shufflecsv/train_k0.csv.gz', nrows=100)
 df_test = pd.read_csv('.. /input/quickdraw-doodle-recognition/test_simplified.csv', nrows=2 )
sub['revenue'] = np.expm1(( prediction_lgb + prediction_xgb + prediction_cat)/ 3)
 sub.to_csv("combined.csv", index=False )<import_modules>
names = ['countrycode', 'drawing', 'key_id', 'recognized', 'timestamp', 'word']
files = sorted(os.listdir('.. /input/quickdraw-doodle-recognition/train_simplified/'), reverse=False)
 class_dict = {file[:-4].replace(" ", "_"): i for i, file in enumerate(files)}
classreverse_dict = {v: k for k, v in class_dict.items() }
df_train = df_train.drop(['shuffle'], axis=1)
 df_train['datatype'], df_test['datatype'] = 'training', 'testing'
df_train = df_train[['key_id', 'countrycode', 'drawing', 'datatype', 'word', 'recognized']]
df_test['word'], df_test['recognized'] = '', True
df_data = pd.concat([df_train, df_test], ignore_index=True )
from keras.models import Sequential, Model
from keras.optimizers import Adam, SGD
from keras.layers import Dense, Activation, Dropout, Input, concatenate,BatchNormalization
from keras.wrappers.scikit_learn import KerasRegressor
from keras.regularizers import l2
from keras.callbacks import EarlyStopping<choose_model_class>
chunksize = 680
img_size = 64
num_channels = 1
num_classes = 340
num_shuffles = 50
img_size_flat = img_size * img_size * num_channels
def build_model(input_shape,n_hidden=10, n_neurons=30, optimizer = SGD(3e-3,momentum=0.9)) :
model = Sequential()
options = {"input_shape": input_shape}
for layer in range(n_hidden):
BatchNormalization()
model.add(Dense(n_neurons, activation="relu",
kernel_regularizer=l2(0.01), **options))
options = {}
BatchNormalization()
model.add(Dense(1, **options))
model.compile(loss="mean_squared_error", optimizer=optimizer)
 return model<feature_engineering>
def draw2pixel(draw: list)-> np.ndarray:

 image, xmin, xmax, ymin, ymax = np.zeros(( 256, 256)) , 255, 0, 255, 0
for k, stroke in enumerate(draw):
for i in range(len(stroke[0])-1):
xmin, xmax = min(xmin, stroke[0][i], stroke[0][i + 1]), max(xmax, stroke[0][i], stroke[0][i + 1])
 ymin, ymax = min(ymin, stroke[1][i], stroke[1][i + 1]), max(ymax, stroke[1][i], stroke[1][i + 1])
 color =(255.0 - min(k, 10)* 13)/ 255.0
_ = cv2.line(image,(stroke[0][i], stroke[1][i]),(stroke[0][i + 1], stroke[1][i + 1]), color=color, thickness=5)
 if xmin == xmax: xmin, xmax = 0, 255
if ymin == ymax: ymin, ymax = 0, 255
return cv2.resize(image[ymin:ymax, xmin:xmax],(img_size, img_size))
train_dummies_prepared = num_pipeline.fit_transform(train_dummies )<choose_model_class>
def feature_extraction(df_data: pd.DataFrame)-> pd.DataFrame:

 df_data['drawing'] = df_data['drawing'].apply(lambda x: json.loads(x))
df_data['word'] = df_data['word'].apply(lambda x: -1 if x == '' else class_dict[x.replace(' ', '_')])
 df_data['pixel'] = df_data['drawing'].apply(lambda x: draw2pixel(x))
return df_data
model_NN = build_model(input_shape= train_dummies_prepared.shape[1:], n_hidden=6,optimizer=Adam(3e-5))<train_model>
def feature_extraction2(df_data: pd.DataFrame)-> pd.DataFrame:

 df_data = df_data.drop(['countrycode', 'drawing', 'datatype'], axis=1)
 return df_data
model_history = model_NN.fit(train_dummies_prepared, train_labels, epochs=2500, batch_size=5, validation_split=0.1,callbacks=[EarlyStopping(patience=100)] )<set_options>
df_data = feature_extraction(df_data )
plt.style.use('ggplot')
 %matplotlib inline 
<load_from_csv>
df_data = feature_extraction2(df_data )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.shape, test.shape<feature_engineering>
del df_data, df_train, df_test
gc.collect()
def refmt_str2dict(df, cols):
for col in cols:
df[col] = df[col].apply(lambda x: {} if pd.isna(x)else ast.literal_eval(x))
return df
dict_columns = ['belongs_to_collection', 'genres', 'production_companies', 'production_countries',
'spoken_languages', 'Keywords', 'cast', 'crew']
train = refmt_str2dict(train, dict_columns )<define_variables>
def train_generator() -> tuple:

 while True:
for k in np.random.permutation(range(num_shuffles - 1)) :
for df_data in pd.read_csv('.. /input/quick-draw-doodle-recognition-challenge-shufflecsv/train_k%d.csv.gz' %k, chunksize=chunksize):
df_data['drawing'] = df_data['drawing'].apply(lambda x: json.loads(x))
df_data['word'] = df_data['word'].apply(lambda x: -1 if x == '' else class_dict[x.replace(' ', '_')])
 x = np.zeros(( df_data.shape[0], img_size, img_size, 1))
for i, drawing in enumerate(df_data['drawing'].values): x[i, :, :, 0] = draw2pixel(drawing)
 y = keras.utils.to_categorical(df_data['word'], num_classes=num_classes)
 yield x, y
gen_train = train_generator()
TMDB_path = 'https://image.tmdb.org/t/p/w600_and_h900_bestv2/'<feature_engineering>
df_data = pd.read_csv('.. /input/quick-draw-doodle-recognition-challenge-shufflecsv/train_k%d.csv.gz' %(num_shuffles - 1), nrows=34000)
 df_data['drawing'] = df_data['drawing'].apply(lambda x: json.loads(x))
df_data['word'] = df_data['word'].apply(lambda x: -1 if x == '' else class_dict[x.replace(' ', '_')])
 x_validate = np.zeros(( df_data.shape[0], img_size, img_size, 1))
for i, drawing in enumerate(df_data['drawing'].values): x_validate[i, :, :, 0] = draw2pixel(drawing)
 y_validate = keras.utils.to_categorical(df_data['word'], num_classes=num_classes )
def fix_date(x):
yr = x.split('/')[2]
if int(yr)<= 19:
return x[:-2] + '20' + yr
else:
return x[:-2] + '19' + yr
train['release_date'] = train['release_date'].apply(lambda x: fix_date(x))
train['release_date'] = pd.to_datetime(train['release_date'] )<data_type_conversions>
del df_data
gc.collect()
def process_date(df):
date_attrs = ['year', 'month', 'day', 'weekday', 'weekofyear', 'quarter']
for attr in date_attrs:
new_col = 'release_date_' + attr
df[new_col] = getattr(df['release_date'].dt, attr ).astype(int)
 return df
train = process_date(train )<string_transform>
def top_3_categorical_accuracy(y_true: np.ndarray, y_pred: np.ndarray)-> float:

 return top_k_categorical_accuracy(y_true, y_pred, k=3 )
def WordCloud_fromDict(dict_name, key='name'):
list_dict = list(train[dict_name].apply(lambda x: [i[key] for i in x] if x != {} else [] ).values)
 list2txt = ' '.join(['_'.join(i.split(' ')) for j in list_dict for i in j])
 wordcloud = WordCloud(max_font_size = None, background_color = 'black', collocations = False,
width = 1200, height = 1000 ).generate(list2txt)
 return wordcloud<define_variables>
model_mobilenet = MobileNet(input_shape=(img_size, img_size, 1), alpha=1.0, dropout=1e-3, weights=None, classes=num_classes)
 model_mobilenet.summary()
list_genres = list(train['genres'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 list_keywords = list(train['Keywords'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 genres = set([y for x in list_genres for y in x])
 keywords = set([y for x in list_keywords for y in x])
 len(genres), len(keywords )<define_variables>
model_mobilenet.compile(optimizer='adam', loss='categorical_crossentropy', metrics=[top_3_categorical_accuracy])
 hist = model_mobilenet.fit_generator(gen_train, steps_per_epoch=800, epochs=96, verbose=2, validation_data=(x_validate, y_validate))
model_hist = pd.DataFrame(hist.history)
 model_mobilenet_score = model_mobilenet.evaluate(x_validate, y_validate, verbose=1)
 print('mobilenet
top 3 categorical accuracy score: %0.4f' %model_mobilenet_score[1] )
list_crew = list(train['crew'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 cast = set([y for x in list_cast_wURL for y in x])
 crew = set([y for x in list_crew for y in x])
 len(cast), len(crew )<define_variables>
model_mobilenet.save('model_mobilenet.h5' )
list_prodCompanies = list(train['production_companies'].apply(
 lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 list_prodCountries = list(train['production_countries'].apply(
 lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 prodCompanies = set([y for x in list_prodCompanies for y in x])
 prodCountries = set([y for x in list_prodCountries for y in x])
 len(prodCompanies), len(prodCountries )<string_transform>
del x_validate, y_validate
gc.collect()
OneMovie = train[train['title'] == 'The Terminator']
keywords = list(OneMovie['Keywords'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)[0]
keywords_str = ' '.join(keywords)
 text_merged = list(OneMovie['overview'])[0] + ' | ' + list(OneMovie['tagline'])[0] + ' | ' + keywords_str
text_merged<string_transform>
df_test = pd.read_csv('.. /input/quickdraw-doodle-recognition/test_simplified.csv')
 df_test['drawing'] = df_test['drawing'].apply(lambda x: json.loads(x))
text_merged2train = []
for id in train['id']:
mov = train[train['id'] == id]
keywords = list(mov['Keywords'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)[0]
keywords_str = ' '.join(keywords)
 if pd.isnull(list(mov['overview'])[0]):
overview = ''
else:
overview = list(mov['overview'])[0]
if pd.isnull(list(mov['tagline'])[0]):
tagline = ''
else:
tagline = list(mov['tagline'])[0]
text_merged = keywords_str
text_merged2train.append(text_merged)
 train['text_merged'] = text_merged2train<count_values>
x_test = np.zeros(( df_test.shape[0], img_size, img_size, 1))
for i, drawing in enumerate(df_test['drawing'].values): x_test[i, :, :, 0] = draw2pixel(drawing)
 y_test = np.argsort(-model_mobilenet.predict(x_test, verbose=1)) [:, 0:3]
df_word = pd.DataFrame({'top 1': y_test[:, 0], 'top 2': y_test[:, 1], 'top 3': y_test[:, 2]})
 df_word = df_word.replace(classreverse_dict)
 df_word['submission'] = df_word['top 1'] + ' ' + df_word['top 2'] + ' ' + df_word['top 3']
train['belongs_to_collection'].apply(lambda x: 1 if x != {} else 0 ).value_counts()<feature_engineering>
out = pd.DataFrame({'key_id': df_test['key_id'], 'word': df_word['submission']})
 out.to_csv('submission.csv', index=False )