Kaggle competition: Quick, Draw! Doodle Recognition Challenge
<SOS> metric: MAP@{K} Kaggle data source: quick,-draw!-doodle-recognition-challenge<feature_engineering>
%matplotlib inline
InteractiveShell.ast_node_interactivity = "all"
plt.rcParams['figure.figsize'] = [16, 10]
plt.rcParams['font.size'] = 14
start = dt.datetime.now()
data['title_length'] = data['original_title'].apply(lambda x: len(x)if x != '' else 0 )<sort_values>
DP_DIR = '.. /input/shuffle-csvs/'
INPUT_DIR = '.. /input/quickdraw-doodle-recognition/'
BASE_SIZE = 256
NCSVS = 100
NCATS = 340
np.random.seed(seed=1987)
 tf.set_random_seed(seed=1987)
 def f2cat(filename: str)-> str:
return filename.split('.')[0]
def list_all_categories() :
files = os.listdir(os.path.join(INPUT_DIR, 'train_simplified'))
return sorted([f2cat(f)for f in files], key=str.lower )
corr = data.corr()
corr['revenue'].sort_values(ascending=False ).head(20 )<count_missing_values>
def apk(actual, predicted, k=3):

 if len(predicted)> k:
predicted = predicted[:k]
score = 0.0
num_hits = 0.0
for i, p in enumerate(predicted):
if p in actual and p not in predicted[:i]:
num_hits += 1.0
score += num_hits /(i + 1.0)
 if not actual:
return 0.0
return score / min(len(actual), k)
 def mapk(actual, predicted, k=3):

 return np.mean([apk(a, p, k)for a, p in zip(actual, predicted)])
 def preds2catids(predictions):
return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])
 def top_3_accuracy(y_true, y_pred):
return top_k_categorical_accuracy(y_true, y_pred, k=3 )
data.isnull().sum().sort_values(ascending=False )<drop_column>
STEPS = 800
EPOCHS = 16
size = 64
batchsize = 680
data.drop(['imdb_id', 'poster_path', 'release_date', 'status','budget_log','cast_character_','language_'], axis=1, inplace=True )<drop_column>
model = MobileNet(input_shape=(size, size, 1), alpha=1., weights=None, classes=NCATS)
 model.compile(optimizer=Adam(lr=0.002), loss='categorical_crossentropy',
metrics=[categorical_crossentropy, categorical_accuracy, top_3_accuracy])
 print(model.summary() )
data_dropping_names = data.drop(['original_title','overview','tagline','title'], axis=1)
 train = data_dropping_names[data_dropping_names['source'] == 'train'].copy()
test = data_dropping_names[data_dropping_names['source'] == 'test'].copy()
train_labels = train['revenue_log']
train.drop(['id', 'revenue', 'source', 'revenue_log'], axis=1, inplace=True)
 test_final = test.drop(['id', 'source','revenue','revenue_log'], axis=1 )<count_missing_values>
valid_df = pd.read_csv(os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(NCSVS - 1)) , nrows=34000)
 x_valid = df_to_image_array_xd(valid_df, size)
 y_valid = keras.utils.to_categorical(valid_df.y, num_classes=NCATS)
 print(x_valid.shape, y_valid.shape)
 print('Validation array memory {:.2f} GB'.format(x_valid.nbytes / 1024.**3))
train.isnull().sum()<count_missing_values>
train_datagen = image_generator_xd(size=size, batchsize=batchsize, ks=range(NCSVS - 1))
train.isnull().sum()<choose_model_class>
%%timeit
x, y = next(train_datagen )
num_pipeline = Pipeline([ 
('imputer', SimpleImputer(strategy="median")) , 
('robust_scaler', RobustScaler()),
] )<train_model>
callbacks = [
ReduceLROnPlateau(monitor='val_top_3_accuracy', factor=0.75, patience=3, min_delta=0.001,
mode='max', min_lr=1e-5, verbose=1),
ModelCheckpoint('model.h5', monitor='val_top_3_accuracy', mode='max', save_best_only=True,
save_weights_only=True),
]
hists = []
hist = model.fit_generator(
 train_datagen, steps_per_epoch=STEPS, epochs=70, verbose=1,
validation_data=(x_valid, y_valid),
callbacks = callbacks 
)
 hists.append(hist )
n_fold = 10
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42)
 def train_model(X, X_test, y, params=None, folds=folds, model_type='lgb', plot_feature_importance=False, model=None):
prediction = np.zeros(X_test.shape[0])
 scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X)) :
print('Fold', fold_n, 'started at', time.ctime())
 if model_type == 'sklearn':
X_train, X_valid = X[train_index], X[valid_index]
else:
X_train, X_valid = X.values[train_index], X.values[valid_index]
y_train, y_valid = y[train_index], y[valid_index]
if model_type == 'lgb':
model = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='rmse',
verbose=1000, early_stopping_rounds=200)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration_)
 if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test.values), ntree_limit=model.best_ntree_limit)
 if model_type == 'sklearn':
model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid ).reshape(-1,)
 score = mean_squared_error(y_valid, y_pred_valid)
 y_pred = model.predict(X_test)
 if model_type == 'cat':
model = CatBoostRegressor(iterations=20000, eval_metric='RMSE', **params)
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 scores.append(mean_squared_error(y_valid, y_pred_valid)** 0.5)
 prediction += y_pred
if model_type == 'lgb':
fold_importance = pd.DataFrame()
fold_importance["feature"] = X.columns
fold_importance["importance"] = model.feature_importances_
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_fold
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 gc.enable()
del model, y_pred_valid, X_test,X_train,X_valid, y_pred, y_train
gc.collect()
if model_type == 'lgb':
feature_importance["importance"] /= n_fold
if plot_feature_importance:
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
return scores, prediction, feature_importance
return scores, prediction
else:
return scores, prediction 
<categorify>
valid_predictions = model.predict(x_valid, batch_size=128, verbose=1)
 map3 = mapk(valid_df[['y']].values, preds2catids(valid_predictions ).values)
 print('Map3: {:.3f}'.format(map3))
train_dummies = pd.get_dummies(train)
 test_dummies = pd.get_dummies(test)
 train_dummies, test_dummies = train_dummies.align(test_dummies, axis=1, join='inner' )<train_model>
test = pd.read_csv(os.path.join(INPUT_DIR, 'test_simplified.csv'))
test.head()
x_test = df_to_image_array_xd(test, size)
 print(test.shape, x_test.shape)
 print('Test array memory {:.2f} GB'.format(x_test.nbytes / 1024.**3))
params = {
'num_leaves': 30,
'min_data_in_leaf': 20,
'objective': 'regression',
'max_depth': 6,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
}
score_lgb, prediction_lgb, _ = train_model(train_dummies, test_dummies, train_labels, params=params, model_type='lgb', plot_feature_importance=True )<sort_values>
test_predictions = model.predict(x_test, batch_size=128, verbose=1)
 top3 = preds2catids(test_predictions)
 top3.head()
top3.shape
cats = list_all_categories()
id2cat = {k: cat.replace(' ', '_')for k, cat in enumerate(cats)}
top3cats = top3.replace(id2cat)
 top3cats.head()
top3cats.shape
score_lgb.sort(reverse=True)
 dictvalues = {'RMSE_lgb': score_lgb}<save_to_csv>
test['word'] = top3cats['a'] + ' ' + top3cats['b'] + ' ' + top3cats['c']
submission = test[['key_id', 'word']]
submission.to_csv('gs_mn_submission_{}.csv'.format(int(map3 * 10**4)) , index=False)
 submission.head()
submission.shape
<sort_values><EOS> 
end = dt.datetime.now()
print('Latest run {}.
 Total time {}s'.format(end,(end - start ).seconds))