Kaggle competition: Quick, Draw! Doodle Recognition Challenge
<SOS> metric: MAP@{K} Kaggle data source: quick,-draw!-doodle-recognition-challenge<predict_on_test>
%matplotlib inline
!pip install pycountry-convert
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.log_device_placement = True
sess = tf.Session(config=config)
 set_session(sess )
model_gboost.fit(X, np.log1p(y))
model_gboost_pred_train = model_gboost.predict(X)
 RMSE(np.log1p(y), model_gboost_pred_train )<compute_test_metric>
batch_size = 340*3
valid_percent = 1
data_files = 100
size = 64
num_classes = 340
class_paths = os.listdir(".. /input/quickdraw-doodle-recognition/train_simplified/")
 cat_names = [item[:-4] for item in class_paths]
cat_names.sort(key=lambda x: str.lower(x))
UNKNOWN_COUNTRY = 'YY'
class ContextMode(Enum):
no_context = 1
area_context = 2
model_ensemble_pred_train =(model_CatBoost_pred_train+model_gboost_pred_train)/2
RMSE(np.log1p(y), model_ensemble_pred_train )<define_variables>
valid_countries_dict = pycountry_convert.map_countries(cn_name_format="default")
 valid_country_codes = list(set([v['alpha_2'] for k, v in valid_countries_dict.items() ]))
all_country_codes= set(valid_country_codes)| set([UNKNOWN_COUNTRY] )
model_baseline_pred_test = model_baseline_coeffs[0] + model_baseline_coeffs[1]*X<save_to_csv>
def ValidateCountry(country_code):
return country_code if country_code in all_country_codes else UNKNOWN_COUNTRY
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = model_baseline_pred_test
submission.to_csv('submission_baseline.csv', index = False )<drop_column>
our_mapping = pd.read_csv('.. /input/area-context-country-clustering/area_mapping.csv')
 print(our_mapping.head())
 our_dict = pd.Series(our_mapping.groups.values, index = our_mapping.alpha3 ).to_dict()
def get_area(iso_code):
try:
return our_dict[iso_code]
except KeyError:
return 0
test_features = test[['budget', 'popularity', 'runtime',
'release_date_year', 'release_date_weekday', 'release_date_month',
'budget_yn', 'belongs2coll_yn', 'homepage_yn',
'n_genres', 'n_production_companies', 'n_production_countries']+
keys_genre+keys_cast+keys_director+keys_production_company+keys_production_country+
keys_language]
X = test_features
test_features.columns[test_features.isna().any() ].tolist()<filter>
num_groups = max(our_mapping.groups)+1
print(num_groups )
test_features['runtime'][test_features['runtime'].isna() == True]<filter>
def image_generator(batch_size, isTraining):
while True:
if isTraining:
index_table = np.random.permutation(data_files-valid_percent)
 else:
index_table = np.random.permutation(valid_percent)+(data_files-valid_percent)
 _path = '.. /input/shuffle-and-filter/'
for k in index_table:
filename = os.path.join(_path,'train_k{}.csv.gz'.format(k))
for chunk in pd.read_csv(filename, chunksize=batch_size,
usecols=['countrycode','y','drawing','word'],
keep_default_na=False):
ch_size = len(chunk)
 if(ch_size<batch_size):
continue
else:
yield chunk
test_features['runtime'][test_features['runtime'].isna() == True] = test_features['runtime'].median()<predict_on_test>
def test_gen() :
_path = '.. /input/quickdraw-doodle-recognition/test_simplified.csv'
while True:
for _chunk in pd.read_csv(_path, chunksize=batch_size,
usecols=['drawing','countrycode'],
keep_default_na=False):
if(len(_chunk)<batch_size):
_chunk = pd.concat([_chunk, _chunk[0:batch_size-len(_chunk)]])
 yield _chunk
gen_TEST = test_gen()
model_RF_pred_test = model_RF.predict(X )<save_to_csv>
areas_oneh = OneHotEncoder().fit(np.arange(num_groups ).reshape(-1,1))
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = model_RF_pred_test
submission.to_csv('submission_RF.csv', index = False )<predict_on_test>
class GeneratorMode(Enum):
training = 1
validation = 2
testing = 3
def main_generator(whatGeneratorMode, whatContextMode=ContextMode.no_context):
while True:
if whatGeneratorMode==GeneratorMode.training:
results = next(train_generator)
 elif whatGeneratorMode==GeneratorMode.validation:
results = next(valid_generator)
 elif whatGeneratorMode==GeneratorMode.testing:
results = next(gen_TEST)
 results['drawing'] = results['drawing'].apply(json.loads)
 x = np.zeros(( batch_size, size, size))
for i, raw_strokes in enumerate(results.drawing.values):
x[i] = draw_cv2(raw_strokes, size=size)
 x = x / 255.
 x = x.reshape(( batch_size, size, size, 1)).astype(np.float32)
 if whatGeneratorMode!=GeneratorMode.testing:
y = keras.utils.to_categorical(results.y, num_classes=num_classes)
 if whatContextMode==ContextMode.no_context:
if whatGeneratorMode==GeneratorMode.testing:
yield x
else:
yield x, y
elif whatContextMode==ContextMode.area_context:
_countries = results['countrycode'].apply(ValidateCountry ).values
_areas = np.array(list(map(get_area, _countries)))
 xac = areas_oneh.transform(_areas.reshape(-1,1)).toarray()
if whatGeneratorMode==GeneratorMode.testing:
yield [x, xac]
else:
yield [x, xac], y
model_CatBoost_pred_test = model_CatBoost.predict(X)
 model_CatBoost_pred_test = np.expm1(model_CatBoost_pred_test )<save_to_csv>
def in_top_3(y_true, y_pred):
return top_k_categorical_accuracy(y_true, y_pred, k=3 )
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = model_CatBoost_pred_test
submission.to_csv('submission_CatBoost.csv', index = False )<predict_on_test>
def create_model(whatContextMode=ContextMode.no_context, alpha=1.0, show=False):
K.clear_session()
if whatContextMode.value==ContextMode.no_context.value:
my_model = MobileNet(input_shape=(size, size, 1), weights=None, alpha=alpha, classes=num_classes)
 my_model.compile(optimizer=Adam() , loss='categorical_crossentropy',
metrics=['accuracy',in_top_3])
 elif whatContextMode.value==ContextMode.area_context.value:
dropout=0.001
shape0 =(1, 1, int(1024*alpha))
shape1 =(1, 1, int(1024*alpha+num_groups))
Input_area = Input(shape=(num_groups,))
my_model = MobileNet(input_shape=(size, size, 1), weights=None, alpha=alpha, include_top=False)
 x = my_model.get_layer('conv_pw_13_relu' ).output
x = GlobalAveragePooling2D()(x)
 x = Reshape(shape0, name='reshape_0' )(x)
 x = Flatten()(x)
 x = concatenate([x, Input_area])
 x = Reshape(shape1, name='reshape_1' )(x)
 x = Dropout(dropout, name='dropout' )(x)
 x = Conv2D(num_classes,(1, 1), padding='same', name='conv_preds' )(x)
 x = Activation('softmax', name='act_softmax' )(x)
 preds = Reshape(( num_classes,), name='reshape_2' )(x)
 my_model = Model(inputs=[my_model.input, Input_area], outputs=preds)
 my_model.compile(optimizer=Adam() , loss='categorical_crossentropy',
metrics=['accuracy',in_top_3])
 else:
print('Context not recognized')
 return
if show:
my_model.summary()
return my_model
model_gboost_pred_test = model_gboost.predict(X)
 model_gboost_pred_test = np.expm1(model_gboost_pred_test )<save_to_csv>
ourContextMode=ContextMode.area_context
create_model(whatContextMode=ourContextMode, alpha=0.5, show=True )
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = model_gboost_pred_test
submission.to_csv('submission_gboost.csv', index = False )<define_variables>
nseed(2018)
 set_random_seed(2018 )
model_ensemble_pred_test =(model_CatBoost_pred_test + model_gboost_pred_test)/2<save_to_csv>
ourContextMode = ContextMode.area_context
train_generator = image_generator(batch_size=batch_size, isTraining=True)
 valid_generator = image_generator(batch_size=batch_size, isTraining=False)
 train_gen=main_generator(whatGeneratorMode=GeneratorMode.training,
whatContextMode=ourContextMode)
 valid_gen=main_generator(whatGeneratorMode=GeneratorMode.validation,
whatContextMode=ourContextMode)
 model = create_model(whatContextMode=ourContextMode )
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = model_ensemble_pred_test
submission.to_csv('submission_ensemble.csv', index = False )<install_modules>
train_epoch=44
train_steps=1000
validation_steps=34
cb = [ReduceLROnPlateau(monitor='val_acc', factor=0.5,
patience=5, mode='max', cooldown=3, verbose=0)]
history = model.fit_generator(
 generator=train_gen, steps_per_epoch=train_steps,
validation_data=valid_gen, validation_steps=validation_steps,
callbacks=cb,
epochs=train_epoch, verbose=1 )
!pip install --quiet /kaggle/input/kerasapplications
!pip install --quiet /kaggle/input/efficientnet-git<set_options>
_test_gen = main_generator(whatGeneratorMode=GeneratorMode.testing,
whatContextMode=ourContextMode)
 predictions = model.predict_generator(_test_gen, steps=np.ceil(112199/batch_size), verbose=1 )
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
seed = 0
seed_everything(seed)
 warnings.filterwarnings('ignore' )<set_options>
def top3cats(predictions):
return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3],
columns=['word','word2','word3'])
 test_simp = pd.read_csv('.. /input/quickdraw-doodle-recognition/test_simplified.csv',
nrows=np.shape(predictions)[0])
 categories_dict = {_id: cat_name.replace(' ', '_')for _id, cat_name in enumerate(cat_names)}
output = top3cats(predictions ).replace(categories_dict)
 output =(output.word + ' ' ).str.cat([output.word2 + ' ', output.word3])
 output.head()
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print(f'Running on TPU {tpu.master() }')
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<define_variables>
output = pd.concat([test_simp['key_id'], output.loc[0:112198]], axis=1)
 output.tail()
<load_from_csv><EOS> 
output.to_csv(ourContextMode.name + '_myResults.csv',index=False )