Kaggle competition: Elo Merchant Category Recommendation
<SOS> metric: RMSE Kaggle data source: elo-merchant-category-recommendation<drop_column>
warnings.simplefilter(action='ignore', category=FutureWarning)
cols_to_drop = lag_variables[:-1] + ['item_price']
lag_train.drop(columns=cols_to_drop, inplace=True)
 lag_train = reduce_mem_usage(lag_train )<concatenate>
new_transactions = pd.read_csv('.. /input/elo-merchant-category-recommendation/new_merchant_transactions.csv', parse_dates=['purchase_date'])
 historical_transactions = pd.read_csv('.. /input/elo-merchant-category-recommendation/historical_transactions.csv', parse_dates=['purchase_date'])
 def binarize(df):
for col in ['authorized_flag', 'category_1']:
df[col] = df[col].map({'Y':1, 'N':0})
 return df
historical_transactions = binarize(historical_transactions)
 new_transactions = binarize(new_transactions )
test['date_block_num'] = 34
test = lag_features(test, sales_m)
 fillNaN(test)
 test = date_attr(test )<save_to_csv>
%%time
def read_data(input_file):
df = pd.read_csv(input_file)
 df['first_active_month'] = pd.to_datetime(df['first_active_month'])
 df['elapsed_time'] =(datetime.date(2018, 2, 1)- df['first_active_month'].dt.date ).dt.days
return df
train = read_data('.. /input/elo-merchant-category-recommendation/train.csv')
 test = read_data('.. /input/elo-merchant-category-recommendation/test.csv')
 target = train['target']
del train['target']
gc.collect()
regressor = RandomForestRegressor(n_estimators=10, n_jobs = -1)
 regressor.fit(lag_train.drop(columns=['item_cnt_day']),
lag_train['item_cnt_day'])
 feat_importances = pd.Series(regressor.feature_importances_, index=lag_train.drop(columns=['item_cnt_day'] ).columns)
 feat_importances.nlargest(10 ).plot(kind='barh')
 pred = regressor.predict(test)
 pred = pred.clip(0, 20)
 submission['item_cnt_month'] = pred
submission.to_csv('submission_rf.csv', index=False )<init_hyperparams>
%%time
historical_transactions = pd.get_dummies(historical_transactions, columns=['category_2', 'category_3'])
 new_transactions = pd.get_dummies(new_transactions, columns=['category_2', 'category_3'])
 historical_transactions = reduce_mem_usage(historical_transactions)
 new_transactions = reduce_mem_usage(new_transactions)
 agg_fun = {'authorized_flag': ['sum', 'mean']}
auth_mean = historical_transactions.groupby(['card_id'] ).agg(agg_fun)
 auth_mean.columns = ['_'.join(col ).strip() for col in auth_mean.columns.values]
auth_mean.reset_index(inplace=True)
 authorized_transactions = historical_transactions[historical_transactions['authorized_flag'] == 1]
historical_transactions = historical_transactions[historical_transactions['authorized_flag'] == 0]
gc.collect()
param = {'max_depth': 10,
'subsample': 1,
'min_child_weight': 1,
'eta': 0.3,
'num_round': 1000,
'eval_metric': 'rmse',
'verbosity': 0}<save_to_csv>
%%time
historical_transactions['purchase_month'] = historical_transactions['purchase_date'].dt.month
authorized_transactions['purchase_month'] = authorized_transactions['purchase_date'].dt.month
new_transactions['purchase_month'] = new_transactions['purchase_date'].dt.month
gc.collect()
xgbtrain = xgb.DMatrix(lag_train.drop(columns=['item_cnt_day']),
lag_train['item_cnt_day'])
 bst = xgb.train(param, xgbtrain)
 x = xgb.plot_importance(bst)
 x.figure.set_size_inches(10, 30)
 xgbpredict = xgb.DMatrix(test)
 pred = bst.predict(xgbpredict)
 pred = pred.clip(0, 20)
 submission['item_cnt_month'] = pred
submission.to_csv('submission_xgboost.csv', index=False )<install_modules>
%%time
def aggregate_transactions(history):
history.loc[:, 'purchase_date'] = pd.DatetimeIndex(history['purchase_date'] ).\
astype(np.int64)* 1e-9
agg_func = {
'category_1': ['sum', 'mean'],
'category_2_1.0': ['mean'],
'category_2_2.0': ['mean'],
'category_2_3.0': ['mean'],
'category_2_4.0': ['mean'],
'category_2_5.0': ['mean'],
'category_3_A': ['mean'],
'category_3_B': ['mean'],
'category_3_C': ['mean'],
'merchant_id': ['nunique'],
'merchant_category_id': ['nunique'],
'state_id': ['nunique'],
'city_id': ['nunique'],
'subsector_id': ['nunique'],
'purchase_amount': ['sum', 'mean', 'max', 'min', 'std'],
'installments': ['sum', 'mean', 'max', 'min', 'std'],
'purchase_month': ['mean', 'max', 'min', 'std'],
'purchase_date': [np.ptp, 'min', 'max'],
'month_lag': ['min', 'max']
}
agg_history = history.groupby(['card_id'] ).agg(agg_func)
 agg_history.columns = ['_'.join(col ).strip() for col in agg_history.columns.values]
agg_history.reset_index(inplace=True)
 df =(history.groupby('card_id')
.size() 
.reset_index(name='transactions_count'))
agg_history = pd.merge(df, agg_history, on='card_id', how='left')
 return agg_history
gc.collect()
!pip install -q efficientnet<import_modules>
%%time
def aggregate_per_month(history):
grouped = history.groupby(['card_id', 'month_lag'])
 agg_func = {
'purchase_amount': ['count', 'sum', 'mean', 'min', 'max', 'std'],
'installments': ['count', 'sum', 'mean', 'min', 'max', 'std'],
}
intermediate_group = grouped.agg(agg_func)
 intermediate_group.columns = ['_'.join(col ).strip() for col in intermediate_group.columns.values]
intermediate_group.reset_index(inplace=True)
 final_group = intermediate_group.groupby('card_id' ).agg(['mean', 'std'])
 final_group.columns = ['_'.join(col ).strip() for col in final_group.columns.values]
final_group.reset_index(inplace=True)
 return final_group
final_group = aggregate_per_month(historical_transactions)
 final_group[:10]
gc.collect()
import numpy as np
import pandas as pd
import os
import re
import cv2
import math
import time
from tqdm import tqdm_notebook as tqdm
from sklearn.model_selection import train_test_split
from kaggle_datasets import KaggleDatasets
import tensorflow as tf
import tensorflow.keras.backend as K
from tensorflow.keras.layers import Dense
from tensorflow.keras.models import Model,Sequential
from tensorflow.keras import optimizers
import efficientnet.tfkeras as efn
import seaborn as sns
import matplotlib.pyplot as plt
from plotly.subplots import make_subplots
import plotly.express as px<train_on_grid>
%%time
train = pd.merge(train, history, on='card_id', how='left')
 test = pd.merge(test, history, on='card_id', how='left')
 train = pd.merge(train, authorized, on='card_id', how='left')
 test = pd.merge(test, authorized, on='card_id', how='left')
 train = pd.merge(train, new, on='card_id', how='left')
 test = pd.merge(test, new, on='card_id', how='left')
 train = pd.merge(train, final_group, on='card_id', how='left')
 test = pd.merge(test, final_group, on='card_id', how='left')
 train = pd.merge(train, auth_mean, on='card_id', how='left')
 test = pd.merge(test, auth_mean, on='card_id', how='left')
 print("Train Shape:", train.shape)
 print("Test Shape:", test.shape)
 gc.collect()
print("Tensorflow version " + tf.__version__)
 AUTO = tf.data.experimental.AUTOTUNE
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
 DATASET = '512x512-melanoma-tfrecords-70k-images'
GCS_PATH = KaggleDatasets().get_gcs_path(DATASET )<load_from_csv>
features = [c for c in train.columns if c not in ['card_id', 'first_active_month']]
categorical_feats = [c for c in features if 'feature_' in c]
train_data = pd.read_csv('.. /input/siim-isic-melanoma-classification/train.csv')
 train_data.head()<count_missing_values>
param = {'num_leaves': 31,
'min_data_in_leaf': 32,
'objective':'regression',
'max_depth': -1,
'learning_rate': 0.005,
"min_child_samples": 20,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9 ,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.1,
"nthread": 4,
"verbosity": -1}
train_data.isna().sum()<define_variables>
folds = KFold(n_splits=5, shuffle=True, random_state=15)
 oof = np.zeros(len(train))
predictions = np.zeros(len(test))
start = time.time()
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train.values, target.values)) :
print("fold n°{}".format(fold_))
trn_data = lgb.Dataset(train.iloc[trn_idx][features], label=target.iloc[trn_idx], categorical_feature=categorical_feats)
 val_data = lgb.Dataset(train.iloc[val_idx][features], label=target.iloc[val_idx], categorical_feature=categorical_feats)
 num_round = 10000
clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=100, early_stopping_rounds = 200)
 oof[val_idx] = clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(mean_squared_error(oof, target)**0.5))
SEED = 42
BATCH_SIZE = 32 * strategy.num_replicas_in_sync
SIZE = [512,512]
LR = 0.00004
EPOCHS = 12
WARMUP = 5
WEIGHT_DECAY = 0
LABEL_SMOOTHING = 0.05
TTA = 4<define_variables>
lgbparam = {'num_leaves': 31,
'boosting_type': 'rf',
'min_data_in_leaf': 30,
'objective':'regression',
'max_depth': -1,
'learning_rate': 0.01,
"min_child_samples": 20,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9 ,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.1,
"verbosity": -1,
"nthread": 4,
"random_state": 4590}
def seed_everything(SEED):
np.random.seed(SEED)
 tf.random.set_seed(SEED)
 seed_everything(SEED)
 train_filenames = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')
 test_filenames = tf.io.gfile.glob(GCS_PATH + '/test*.tfrec' )<split>
folds = RepeatedKFold(n_splits=5, n_repeats=2, random_state=4520)
 oof_lgb = np.zeros(len(train))
predictions_lgb = np.zeros(len(test))
start = time.time()
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train.values, target.values)) :
print("fold n°{}".format(fold_))
trn_data = lgb.Dataset(train.iloc[trn_idx][features], label=target.iloc[trn_idx], categorical_feature=categorical_feats)
 val_data = lgb.Dataset(train.iloc[val_idx][features], label=target.iloc[val_idx], categorical_feature=categorical_feats)
 num_round = 11000
clf = lgb.train(lgbparam, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=100, early_stopping_rounds = 100)
 oof_lgb[val_idx] = clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions_lgb += clf.predict(test[features], num_iteration=clf.best_iteration)/(5 * 2)
 print("CV score: {:<8.5f}".format(mean_squared_error(oof_lgb, target)**0.5))
train_filenames,valid_filenames = train_test_split(train_filenames,test_size = 0.2,random_state = SEED )<normalization>
sub_df = pd.DataFrame({"card_id":test["card_id"].values})
 sub_df["target"] = predictions
sub_df.to_csv("submit_lgb.csv", index=False)
 sub_df1 = pd.DataFrame({"card_id":test["card_id"].values})
 sub_df1["target"] = predictions_lgb
sub_df1.to_csv("submit_lgb1.csv", index=False )
def decode_image(image):
image = tf.image.decode_jpeg(image, channels=3)
 image = tf.cast(image, tf.float32)/255.0
image = tf.reshape(image, [*SIZE, 3])
 return image
def data_augment(image, label=None, seed=SEED):
image = tf.image.rot90(image,k=np.random.randint(4))
image = tf.image.random_flip_left_right(image, seed=seed)
 image = tf.image.random_flip_up_down(image, seed=seed)
 if label is None:
return image
else:
return image, label
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64), }
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['target'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string), }
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 image_name = example['image_name']
return image, image_name
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset =(tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
.with_options(ignore_order)
.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO))
return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 def plot_transform(num_images):
plt.figure(figsize=(30,10))
x = load_dataset(train_filenames, labeled=False)
 image,_ = iter(x ).next()
for i in range(1,num_images+1):
plt.subplot(1,num_images+1,i)
 plt.axis('off')
 image = data_augment(image=image)
 plt.imshow(image )<normalization>
train_stack = np.vstack([oof,oof_lgb] ).transpose()
test_stack = np.vstack([predictions,predictions_lgb] ).transpose()
folds = RepeatedKFold(n_splits=5,n_repeats=1,random_state=4520)
 oof_stack = np.zeros(train_stack.shape[0])
 predictions_stack = np.zeros(test_stack.shape[0])
 for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_stack, target)) :
print("fold n°{}".format(fold_))
trn_data, trn_y = train_stack[trn_idx], target.iloc[trn_idx].values
val_data, val_y = train_stack[val_idx], target.iloc[val_idx].values
print("-" * 10 + "Stacking " + str(fold_)+ "-" * 10)
 clf = BayesianRidge()
clf.fit(trn_data, trn_y)
 oof_stack[val_idx] = clf.predict(val_data)
 predictions_stack += clf.predict(test_stack)/ 5
np.sqrt(mean_squared_error(target.values, oof_stack))
plot_transform(7 )<define_variables>
sample_submission = pd.read_csv('.. /input/elo-merchant-category-recommendation/sample_submission.csv')
 sample_submission['target'] = predictions_stack
sample_submission.to_csv('Bayesian_Ridge_Stacking.csv', index=False )
<choose_model_class><EOS> 
sample_submission = pd.read_csv('.. /input/elo-merchant-category-recommendation/sample_submission.csv')
 sample1 = pd.read_csv(".. /input/elo-blending/3.695.csv")
 sample2 = pd.read_csv(".. /input/elo-blending/combining_submission(1 ).csv")
 sample_submission['target'] = predictions * 0.5 + predictions_lgb * 0.5
sample_submission.to_csv("Blend1.csv", index = False)
 sample_submission['target'] = sample_submission['target'] * 0.2 + sample1['target'] * 0.2 + sample2['target'] * 0.6
sample_submission.to_csv('Blend2.csv', index=False )