Kaggle competition: Quick, Draw! Doodle Recognition Challenge
<SOS> metric: MAP@{K} Kaggle data source: quick,-draw!-doodle-recognition-challenge<feature_engineering>
%matplotlib inline
InteractiveShell.ast_node_interactivity = "all"
plt.rcParams['figure.figsize'] = [16, 10]
plt.rcParams['font.size'] = 14
start = dt.datetime.now()
data['num_languages'] = counting_number_of_values_dict(data,'spoken_languages')
 data['all_languages'] = data['spoken_languages'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_languages = [m[0] for m in Counter([i for j in languages for i in j] ).most_common(30)]
for g in top_languages:
data['language_' + g] = data['all_languages'].apply(lambda x: 1 if g in x else 0 )<drop_column>
DP_DIR = '.. /input/shuffle-csvs/'
INPUT_DIR = '.. /input/quickdraw-doodle-recognition/'
BASE_SIZE = 256
NCSVS = 100
NCATS = 340
np.random.seed(seed=1987)
 tf.set_random_seed(seed=1987)
 def f2cat(filename: str)-> str:
return filename.split('.')[0]
def list_all_categories() :
files = os.listdir(os.path.join(INPUT_DIR, 'train_simplified'))
return sorted([f2cat(f)for f in files], key=str.lower )
data.drop(['spoken_languages', 'all_languages'], axis=1, inplace=True )<count_values>
def apk(actual, predicted, k=3):

 if len(predicted)> k:
predicted = predicted[:k]
score = 0.0
num_hits = 0.0
for i, p in enumerate(predicted):
if p in actual and p not in predicted[:i]:
num_hits += 1.0
score += num_hits /(i + 1.0)
 if not actual:
return 0.0
return score / min(len(actual), k)
 def mapk(actual, predicted, k=3):

 return np.mean([apk(a, p, k)for a, p in zip(actual, predicted)])
 def preds2catids(predictions):
return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])
 def top_3_accuracy(y_true, y_pred):
return top_k_categorical_accuracy(y_true, y_pred, k=3 )
counting_number_of_values_dict(data,'production_countries' ).value_counts()<categorify>
STEPS = 1000
EPOCHS = 15
size = 80
batchsize = 530
data['num_countries'] = counting_number_of_values_dict(data, 'production_countries')
 dict_one_hot_code(data,name_column= 'production_countries', name_new_column='production_country')
<drop_column>
model = MobileNet(input_shape=(size, size, 1), alpha=1., weights=None, classes=NCATS)
 model.compile(optimizer=Adam(lr=0.002), loss='categorical_crossentropy',
metrics=[categorical_crossentropy, categorical_accuracy, top_3_accuracy])
 print(model.summary() )
data.drop('production_countries', axis=1, inplace=True )<count_values>
valid_df = pd.read_csv(os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(NCSVS - 1)) , nrows=60000)
 x_valid = df_to_image_array_xd(valid_df, size)
 y_valid = keras.utils.to_categorical(valid_df.y, num_classes=NCATS)
 print(x_valid.shape, y_valid.shape)
 print('Validation array memory {:.2f} GB'.format(x_valid.nbytes / 1024.**3))
counting_number_of_values_dict(data,'production_companies' ).value_counts()<feature_engineering>
train_datagen = image_generator_xd(size=size, batchsize=batchsize, ks=range(NCSVS - 1))
data['prod_companies_total_number'] = counting_number_of_values_dict(data,'production_companies' )<categorify>
callbacks = [
ReduceLROnPlateau(monitor='val_categorical_accuracy', factor=0.5, patience=5,
min_delta=0.005, mode='max', cooldown=3, verbose=1)
 ]
hists = []
hist = model.fit_generator(
 train_datagen, steps_per_epoch=STEPS, epochs=EPOCHS, verbose=1,
validation_data=(x_valid, y_valid),
callbacks = callbacks 
)
 hists.append(hist )
dict_one_hot_code(data,'production_companies','all_production_companies' )<drop_column>
hist = model.fit_generator(
 train_datagen, steps_per_epoch=STEPS, epochs=EPOCHS, verbose=1,
validation_data=(x_valid, y_valid),
callbacks = callbacks 
)
 hists.append(hist )
data.drop('production_companies', axis=1, inplace=True )<count_missing_values>
hist = model.fit_generator(
 train_datagen, steps_per_epoch=STEPS, epochs=EPOCHS, verbose=1,
validation_data=(x_valid, y_valid),
callbacks = callbacks 
)
 hists.append(hist )
data['homepage'].isnull().value_counts()<feature_engineering>
valid_predictions = model.predict(x_valid, batch_size=128, verbose=1)
 map3 = mapk(valid_df[['y']].values, preds2catids(valid_predictions ).values)
 print('Map3: {:.3f}'.format(map3))
data['has_homepage'] = 0
data.loc[data['homepage'].isnull() == False, 'has_homepage'] = 1
data.drop('homepage', axis=1, inplace=True )<prepare_output>
test = pd.read_csv(os.path.join(INPUT_DIR, 'test_simplified.csv'))
test.head()
x_test = df_to_image_array_xd(test, size)
 print(test.shape, x_test.shape)
 print('Test array memory {:.2f} GB'.format(x_test.nbytes / 1024.**3))
data_language = pd.DataFrame(data.groupby('original_language')["revenue"].sum().sort_values(ascending=False ).head(10))<feature_engineering>
test_predictions = model.predict(x_test, batch_size=128, verbose=1)
 top3 = preds2catids(test_predictions)
 top3.head()
top3.shape
cats = list_all_categories()
id2cat = {k: cat.replace(' ', '_')for k, cat in enumerate(cats)}
top3cats = top3.replace(id2cat)
 top3cats.head()
top3cats.shape
data['is_english_original_language'] = 0
data.loc[data['original_language'] == 'en', 'is_english_original_language'] = 1<drop_column>
test['word'] = top3cats['a'] + ' ' + top3cats['b'] + ' ' + top3cats['c']
submission = test[['key_id', 'word']]
submission.to_csv('gs_mn_submission_{}.csv'.format(int(map3 * 10**4)) , index=False)
 submission.head()
submission.shape
<data_type_conversions><EOS> 
end = dt.datetime.now()
print('Latest run {}.
 Total time {}s'.format(end,(end - start ).seconds))