Kaggle competition: Quick, Draw! Doodle Recognition Challenge
<SOS> metric: MAP@{K} Kaggle data source: quick,-draw!-doodle-recognition-challenge<filter>
%matplotlib inline
InteractiveShell.ast_node_interactivity = "all"
plt.rcParams['figure.figsize'] = [16, 10]
plt.rcParams['font.size'] = 14
start = dt.datetime.now()
test[test['release_date_year'] == 1927] 
<drop_column>
from tensorflow.keras.models import load_model
import h5py
cols2drop = ['id','belongs_to_collection', 'homepage', 'imdb_id', 'original_title', 'overview',
'poster_path', 'release_date', 'spoken_languages', 'status','tagline', 'Keywords']
train = train.drop(cols2drop, axis=1)
 test = test.drop(cols2drop, axis=1)
 train.columns, test.columns<drop_column>
model = load_model('.. /input/model-pretrained/model.h5' )
df4corr = train[['budget', 'popularity', 'runtime', 'release_date_year', 'release_date_month', 'release_date_day', \
'release_date_weekday', 'release_date_weekofyear', 'release_date_quarter', 'revenue']]<feature_engineering>
DP_DIR = '.. /input/shuffle-csvs/'
INPUT_DIR = '.. /input/quickdraw-doodle-recognition/'
BASE_SIZE = 256
NCSVS = 100
NCATS = 340
np.random.seed(seed=1987)
 tf.set_random_seed(seed=1987)
 def f2cat(filename: str)-> str:
return filename.split('.')[0]
def list_all_categories() :
files = os.listdir(os.path.join(INPUT_DIR, 'train_simplified'))
return sorted([f2cat(f)for f in files], key=str.lower )
train['budget_yn'] = train['budget'].apply(lambda x: 0 if x == 0 else 1)
 train['budget_per_year'] = train['budget']/train['release_date_year'] 
<feature_engineering>
def apk(actual, predicted, k=3):

 if len(predicted)> k:
predicted = predicted[:k]
score = 0.0
num_hits = 0.0
for i, p in enumerate(predicted):
if p in actual and p not in predicted[:i]:
num_hits += 1.0
score += num_hits /(i + 1.0)
 if not actual:
return 0.0
return score / min(len(actual), k)
 def mapk(actual, predicted, k=3):

 return np.mean([apk(a, p, k)for a, p in zip(actual, predicted)])
 def preds2catids(predictions):
return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])
 def top_3_accuracy(y_true, y_pred):
return top_k_categorical_accuracy(y_true, y_pred, k=3 )
train['n_genres'] = train['genres'].apply(lambda x: len(x)if x != {} else 0)
 train['n_production_companies'] = train['production_companies'].apply(lambda x: len(x)if x != {} else 0)
 train['n_production_countries'] = train['production_countries'].apply(lambda x: len(x)if x != {} else 0)
 train['n_cast'] = train['cast'].apply(lambda x: len(x)if x != {} else 0)
 train['n_crew'] = train['crew'].apply(lambda x: len(x)if x != {} else 0 )<feature_engineering>
STEPS = 800
EPOCHS = 16
size = 64
batchsize = 680
train['genres_collapsed'] = train['genres'].apply(lambda x: ' '.\
join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in genres:
train['genre_' + g] = train['genres_collapsed'].apply(lambda x: 1 if g in x else 0)
 keys_genre = ['genre_' + g for g in genres]<count_unique_values>
model = MobileNet(input_shape=(size, size, 1), alpha=1., weights=None, classes=NCATS)
 model.compile(optimizer=Adam(lr=0.002), loss='categorical_crossentropy',
metrics=[categorical_crossentropy, categorical_accuracy, top_3_accuracy])
 print(model.summary() )
n_prodCompanies = 30
Counter([i for j in list_prodCompanies for i in j] ).most_common(n_prodCompanies )<feature_engineering>
model.load_weights('.. /input/model-pretrained/model.h5' )
train['production_companies_collapsed'] = train['production_companies'].apply(lambda x: ' '.\
join(sorted([i['name'] for i in x])) if x != {} else '')
 top_prodCompanies = [m[0] for m in Counter([i for j in list_prodCompanies for i in j] ).most_common(n_prodCompanies)]
for comp in top_prodCompanies:
train['production_company_' + comp] = train['production_companies_collapsed'].\
apply(lambda x: 1 if comp in x else 0)
 keys_production_company = ['production_company_' + comp for comp in top_prodCompanies]<count_unique_values>
valid_df = pd.read_csv(os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(NCSVS - 1)) , nrows=34000)
 x_valid = df_to_image_array_xd(valid_df, size)
 y_valid = keras.utils.to_categorical(valid_df.y, num_classes=NCATS)
 print(x_valid.shape, y_valid.shape)
 print('Validation array memory {:.2f} GB'.format(x_valid.nbytes / 1024.**3))
n_prodCountries = 30
Counter([i for j in list_prodCountries for i in j] ).most_common(n_prodCountries )<feature_engineering>
train_datagen = image_generator_xd(size=size, batchsize=batchsize, ks=range(NCSVS - 1))
train['production_countries_collapsed'] = train['production_countries'].apply(lambda x: ' '.\
join(sorted([i['name'] for i in x])) if x != {} else '')
 top_prodCountries = [m[0] for m in Counter([i for j in list_prodCountries for i in j] ).most_common(n_prodCountries)]
for comp in top_prodCountries:
train['production_country_' + comp] = train['production_countries_collapsed'].\
apply(lambda x: 1 if comp in x else 0)
 keys_production_country = ['production_country_' + comp for comp in top_prodCountries]<feature_engineering>
%%timeit
x, y = next(train_datagen )
top_languages = [m[0] for m in Counter(train['original_language'] ).most_common(5)]
for l in top_languages:
train['language_' + l] = train['original_language'].\
apply(lambda x: 1 if l in x else 0)
 keys_language = ['language_' + l for l in top_languages]<define_variables>
valid_predictions = model.predict(x_valid, batch_size=128, verbose=1)
 map3 = mapk(valid_df[['y']].values, preds2catids(valid_predictions ).values)
 print('Map3: {:.3f}'.format(map3))
n_lead = 50
list_lead = list(train['cast'].apply(lambda x: [i['name'] for i in x if i['cast_id'] == 1] if x != {} else [] ).values)
 Counter([i for j in list_lead for i in j] ).most_common(n_lead )<concatenate>
test = pd.read_csv(os.path.join(INPUT_DIR, 'test_simplified.csv'))
test.head()
x_test = df_to_image_array_xd(test, size)
 print(test.shape, x_test.shape)
 print('Test array memory {:.2f} GB'.format(x_test.nbytes / 1024.**3))
fullset = pd.concat([train, test])
 list_lead = list(fullset['cast'].apply(lambda x: [i['name'] for i in x if i['cast_id'] == 1] if x != {} else [] ).values)
 Counter([i for j in list_lead for i in j] ).most_common(n_lead )<feature_engineering>
test_predictions = model.predict(x_test, batch_size=128, verbose=1)
 top3 = preds2catids(test_predictions)
 top3.head()
top3.shape
cats = list_all_categories()
id2cat = {k: cat.replace(' ', '_')for k, cat in enumerate(cats)}
top3cats = top3.replace(id2cat)
 top3cats.head()
top3cats.shape
train['cast_collapsed'] = train['cast'].apply(lambda x: ' '.\
join(sorted([i['name'] for i in x])) if x != {} else '')
 top_lead = [m[0] for m in Counter([i for j in list_lead for i in j] ).most_common(n_lead)]
for lead in top_lead:
train['cast_' + lead] = train['cast_collapsed'].apply(lambda x: 1 if lead in x else 0)
 keys_cast = ['cast_' + lead for lead in top_lead]<feature_engineering>
test['word'] = top3cats['a'] + ' ' + top3cats['b'] + ' ' + top3cats['c']
submission = test[['key_id', 'word']]
submission.to_csv('gs_mn_submission_{}.csv'.format(int(map3 * 10**4)) , index=False)
 submission.head()
submission.shape
train['directors_collapsed'] = train['crew'].apply(lambda x: ' '.\
join(sorted([i['name'] for i in x if i['job'] == 'Director'])) if x != {} else '')
 top_directors = [m[0] for m in Counter([i for j in list_directors for i in j] ).most_common(n_directors)]
for d in top_directors:
train['director_' + d] = train['directors_collapsed'].apply(lambda x: 1 if d in x else 0)
 keys_director = ['director_' + d for d in top_directors]<define_variables>
end = dt.datetime.now()
print('Latest run {}.
 Total time {}s'.format(end,(end - start ).seconds))