Kaggle competition: Quick, Draw! Doodle Recognition Challenge
<SOS> metric: MAP@{K} Kaggle data source: quick,-draw!-doodle-recognition-challenge<split>
%matplotlib inline
InteractiveShell.ast_node_interactivity = "all"
plt.rcParams['figure.figsize'] = [16, 10]
plt.rcParams['font.size'] = 14
start = dt.datetime.now()
( x_train, y_train),(x_test, y_test)= mnist.load_data()<categorify>
DP_DIR = '.. /input/shuffle-csvs/'
INPUT_DIR = '.. /input/quickdraw-doodle-recognition/'
BASE_SIZE = 256
NCSVS = 100
NCATS = 340
np.random.seed(seed=1987)
 tf.set_random_seed(seed=1987)
 def f2cat(filename: str)-> str:
return filename.split('.')[0]
def list_all_categories() :
files = os.listdir(os.path.join(INPUT_DIR, 'train_simplified'))
return sorted([f2cat(f)for f in files], key=str.lower )
pred_train = x_train.reshape(x_train.shape[0], 28, 28, 1)
 pred_test = x_test.reshape(x_test.shape[0], 28, 28, 1)
 pred_train = pred_train.astype('float32')
 pred_test = pred_test.astype('float32')
 pred_train /=255
pred_test /=255
class_train = np_utils.to_categorical(y_train, 10)
 class_test = np_utils.to_categorical(y_test, 10 )<choose_model_class>
def apk(actual, predicted, k=3):
if len(predicted)> k:
predicted = predicted[:k]
score = 0.0
num_hits = 0.0
for i, p in enumerate(predicted):
if p in actual and p not in predicted[:i]:
num_hits += 1.0
score += num_hits /(i + 1.0)
 if not actual:
return 0.0
return score / min(len(actual), k)
 def mapk(actual, predicted, k=3):
return np.mean([apk(a, p, k)for a, p in zip(actual, predicted)])
 def preds2catids(predictions):
return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])
 def top_3_accuracy(y_true, y_pred):
return top_k_categorical_accuracy(y_true, y_pred, k=3 )
classifier = Sequential()
classifier.add(Conv2D(32,(3,3), input_shape =(28,28,1), activation = 'relu'))
classifier.add(BatchNormalization())
 classifier.add(MaxPooling2D(pool_size =(2,2)))
 classifier.add(Conv2D(32,(3,3), activation = 'relu'))
classifier.add(BatchNormalization())
 classifier.add(MaxPooling2D(pool_size =(2,2)))
 classifier.add(Flatten())
 classifier.add(Dense(units = 128, activation = 'relu'))
classifier.add(Dropout(0.2))
classifier.add(Dense(units = 128, activation = 'relu'))
classifier.add(Dropout(0.2))
classifier.add(Dense(units = 10, activation = 'softmax'))
classifier.compile(loss = 'categorical_crossentropy', optimizer = 'adam', metrics=['accuracy'] )<train_model>
STEPS = 800
EPOCHS = 16
size = 64
batchsize = 680
classifier.fit(pred_train,
class_train,
batch_size = 128,
epochs = 30,
validation_data =(pred_test, class_test))<predict_on_test>
model = MobileNet(input_shape=(size, size, 1), alpha=1., weights=None, classes=NCATS)
 model.compile(optimizer=Adam(lr=0.002), loss='categorical_crossentropy',
metrics=[categorical_crossentropy, categorical_accuracy, top_3_accuracy])
 print(model.summary() )
X_test = test/255
X_test = X_test.values
X_test = X_test.reshape(X_test.shape[0], 28, 28, 1 ).astype('float32')
 pred = classifier.predict(X_test)
 pred.shape<load_from_csv>
valid_df = pd.read_csv(os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(NCSVS - 1)) , nrows=34000)
 x_valid = df_to_image_array_xd(valid_df, size)
 y_valid = keras.utils.to_categorical(valid_df.y, num_classes=NCATS)
 print(x_valid.shape, y_valid.shape)
 print('Validation array memory {:.2f} GB'.format(x_valid.nbytes / 1024.**3))
sample_submit = pd.read_csv("/kaggle/input/digit-recognizer/sample_submission.csv")
 sample_submit.head()<save_to_csv>
train_datagen = image_generator_xd(size=size, batchsize=batchsize, ks=range(NCSVS - 1))
y_pred = pred.argmax(axis=1)
 ImageID = np.arange(len(y_pred)) +1
submit = pd.DataFrame([ImageID,y_pred] ).T
submit.rename(columns = {0:'ImageId', 1:'Label'})
 submit.to_csv('submission.csv', header = ['ImageId', 'Label' ], index = None )<set_options>
callbacks = [
ReduceLROnPlateau(monitor='val_top_3_accuracy', factor=0.75, patience=3, min_delta=0.001,
mode='max', min_lr=1e-5, verbose=1),
ModelCheckpoint('model.h5', monitor='val_top_3_accuracy', mode='max', save_best_only=True,
save_weights_only=True),
]
hists = []
hist = model.fit_generator(
 train_datagen, steps_per_epoch=STEPS, epochs=70, verbose=1,
validation_data=(x_valid, y_valid),
callbacks = callbacks 
)
 hists.append(hist )
plt.style.use('seaborn-white')
 sns.set_style("white")
 %matplotlib inline 
<define_variables>
valid_predictions = model.predict(x_valid, batch_size=128, verbose=1)
 map3 = mapk(valid_df[['y']].values, preds2catids(valid_predictions ).values)
 print('Map3: {:.3f}'.format(map3))
img_size_ori = 101
img_size_target = 101
im_width = 101
im_height = 101
im_chan = 1
basicpath = '.. /input/'
path_train = basicpath + 'train/'
path_test = basicpath + 'test/'
path_train_images = path_train + 'images/'
path_train_masks = path_train + 'masks/'
path_test_images = path_test + 'images/' 
<load_from_csv>
test = pd.read_csv(os.path.join(INPUT_DIR, 'test_simplified.csv'))
test.head()
x_test = df_to_image_array_xd(test, size)
 print(test.shape, x_test.shape)
 print('Test array memory {:.2f} GB'.format(x_test.nbytes / 1024.**3))
train_df = pd.read_csv(".. /input/train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(".. /input/depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]
len(train_df )<feature_engineering>
test_predictions = model.predict(x_test, batch_size=128, verbose=1)
 top3 = preds2catids(test_predictions)
 top3.head()
top3.shape
cats = list_all_categories()
id2cat = {k: cat.replace(' ', '_')for k, cat in enumerate(cats)}
top3cats = top3.replace(id2cat)
 top3cats.head()
top3cats.shape
train_df["images"] = [np.array(load_img(".. /input/train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]<feature_engineering>
test['word'] = top3cats['a'] + ' ' + top3cats['b'] + ' ' + top3cats['c']
submission = test[['key_id', 'word']]
submission.to_csv('submission.csv', index=False)
 submission.head()
submission.shape
train_df["masks"] = [np.array(load_img(".. /input/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]<categorify>
end = dt.datetime.now()
print('Latest run {}.
 Total time {}s'.format(end,(end - start ).seconds))