Kaggle competition: Quick, Draw! Doodle Recognition Challenge
train['belongs2coll_yn'] = train['belongs_to_collection'].apply(lambda x: len(x)if x != {} else 0)
 train['homepage'].head(5 )<count_values>
def f2cat(filename: str)-> str:
return filename.split('.')[0]
class Simplified() :
def __init__(self, input_path='./input'):
self.input_path = input_path
def list_all_categories(self):
files = os.listdir(os.path.join(self.input_path, 'train_simplified'))
return sorted([f2cat(f)for f in files], key=str.lower)
 def read_training_csv(self, category, nrows=None, usecols=None, drawing_transform=False):
df = pd.read_csv(os.path.join(self.input_path, 'train_simplified', category + '.csv'),
nrows=nrows, parse_dates=['timestamp'], usecols=usecols)
 if drawing_transform:
df['drawing'] = df['drawing'].apply(json.loads)
 return df
train['homepage'].apply(lambda x: 1 if pd.isnull(x)== False else 0 ).value_counts()<feature_engineering>
Total time {}s'.format(end,(end - start ).seconds))
train['homepage_yn'] = train['homepage'].apply(lambda x: 1 if pd.isnull(x)== False else 0 )<string_transform>
INPUT_DIR = '.. /input/quickdraw-doodle-recognition/'
BASE_SIZE = 256
def split_train_val() :
ALL_FILES = glob.glob('.. /input/shuffle-csvs/*.csv.gz')
 VALIDATION_FILE = '.. /input/shuffle-csvs/train_k'+str(int(random.random() *93)) +'.csv.gz'
ALL_FILES.remove(VALIDATION_FILE)
 np.random.seed(seed=1987)
 return ALL_FILES, VALIDATION_FILE
def apk(actual, predicted, k=3):

 if len(predicted)> k:
predicted = predicted[:k]
score = 0.0
num_hits = 0.0
for i, p in enumerate(predicted):
if p in actual and p not in predicted[:i]:
num_hits += 1.0
score += num_hits /(i + 1.0)
 if not actual:
return 0.0
return score / min(len(actual), k)
 def mapk(actual, predicted, k=3):

 return np.mean([apk(a, p, k)for a, p in zip(actual, predicted)])
 def preds2catids(predictions):
return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])
 def f2cat(filename: str)-> str:
return filename.split('.')[0]
def list_all_categories() :
files = os.listdir(os.path.join(INPUT_DIR, 'train_simplified'))
return sorted([f2cat(f)for f in files], key=str.lower)
 def plot_batch(x):
cols = 4
rows = 6
fig, axs = plt.subplots(nrows=rows, ncols=cols, sharex=True, sharey=True, figsize=(18, 18))
for i in range(rows):
for k in range(0,3):
ax = axs[i, k]
ax.imshow(x[i, :, :, k], cmap=plt.cm.gray)
 ax.axis('off')
 ax = axs[i, 3]
ax.imshow(x[i, :, :],)
 ax.axis('off')
 fig.tight_layout()
plt.show() ;
' '.join(set(train['original_language'])) , len(set(train['original_language']))<count_values>
AUGMENTATION = True
STEPS = 200
BATCH_SIZE = 400
EPOCHS = 10
NCATS = 340
LEARNING_RATE = 0.002
IMG_SHAPE =(128,128,3)
 IMG_SIZE = IMG_SHAPE[0]
train['status'].apply(lambda x: 1 if x == 'Released' else 0 ).value_counts() , set(train['status'] )<compute_test_metric>
def image_generator(size, batchsize, lw=6, augmentation = False):
while True:
for filename in ALL_FILES:
for df in pd.read_csv(filename, chunksize=batchsize):
df['drawing'] = df['drawing'].apply(eval)
 x = np.zeros(( len(df), size, size,3))
for i, raw_strokes in enumerate(df.drawing.values):
x[i] = draw_cv2(raw_strokes, size=size, lw=lw, augmentation = augmentation)
 x = x / 255.
 x = x.reshape(( len(df), size, size, 3)).astype(np.float32)
 y = keras.utils.to_categorical(df.y, num_classes=NCATS)
 yield x, y
def valid_generator(valid_df, size, batchsize, lw=6):
while(True):
for i in range(0,len(valid_df),batchsize):
chunk = valid_df[i:i+batchsize]
x = np.zeros(( len(chunk), size, size,3))
for i, raw_strokes in enumerate(chunk.drawing.values):
x[i] = draw_cv2(raw_strokes, size=size, lw=lw)
 x = x / 255.
 x = x.reshape(( len(chunk), size, size,3)).astype(np.float32)
 y = keras.utils.to_categorical(chunk.y, num_classes=NCATS)
 yield x,y
def test_generator(test_df, size, batchsize, lw=6):
for i in range(0,len(test_df),batchsize):
chunk = test_df[i:i+batchsize]
x = np.zeros(( len(chunk), size, size,3))
for i, raw_strokes in enumerate(chunk.drawing.values):
x[i] = draw_cv2(raw_strokes, size=size, lw=lw)
 x = x / 255.
 x = x.reshape(( len(chunk), size, size, 3)).astype(np.float32)
 yield x
ALL_FILES, VALIDATION_FILE = split_train_val()
train_datagen = image_generator(size=IMG_SIZE, batchsize=BATCH_SIZE, augmentation = AUGMENTATION)
 valid_df = pd.read_csv(VALIDATION_FILE)
 valid_df['drawing'] = valid_df['drawing'].apply(eval)
 validation_steps = len(valid_df)//BATCH_SIZE
valid_datagen = valid_generator(valid_df, size=IMG_SIZE, batchsize=BATCH_SIZE )
test = refmt_str2dict(test, dict_columns )<feature_engineering>
def top_3_accuracy(y_true, y_pred):
return keras.metrics.top_k_categorical_accuracy(y_true, y_pred, k=3)
 reducer = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5)
 checkpointer = ModelCheckpoint(filepath='mobileNet_ckpt.hdf5', verbose=2, save_best_only=True)
 model = load_model('.. /input/mobilenetfile/mobileNet.hdf5', custom_objects = {'top_3_accuracy':top_3_accuracy})
 opt = Adam(lr = LEARNING_RATE)
 model.compile(optimizer = opt, loss = 'categorical_crossentropy', metrics = ['accuracy', top_3_accuracy])
 model.summary()
test.at[pd.isnull(test['release_date']), 'release_date'] = '1/1/11'
test['release_date'] = test['release_date'].apply(lambda x: fix_date(x))
test['release_date'] = pd.to_datetime(test['release_date'])
 test = process_date(test )<define_variables>
history = model.fit_generator(train_datagen,
steps_per_epoch=STEPS,
epochs=EPOCHS,
verbose=2,
validation_data=valid_datagen,
validation_steps=validation_steps,
callbacks=[checkpointer,reducer])
 model.save('mobileNet.hdf5' )
<feature_engineering><EOS> 
submission_df = pd.read_csv(os.path.join(INPUT_DIR, 'test_simplified.csv'))
submission_df['drawing'] = submission_df['drawing'].apply(eval)
 submission_datagen = test_generator(submission_df, size=IMG_SIZE, batchsize=BATCH_SIZE)
 submission_predictions = model.predict_generator(submission_datagen, math.ceil(len(submission_df)/BATCH_SIZE))
cats = list_all_categories()
id2cat = {k: cat.replace(' ', '_')for k, cat in enumerate(cats)}
top3 = preds2catids(submission_predictions)
 top3cats = top3.replace(id2cat)
 submission_df['word'] = top3cats['a'] + ' ' + top3cats['b'] + ' ' + top3cats['c']
submission = submission_df[['key_id', 'word']]
submission.to_csv('submission.csv', index=False )