Kaggle competition: Restaurant Revenue Prediction
<choose_model_class><EOS> 
import pandas as pd
import numpy as np
<SOS> metric: RMSE Kaggle data source: restaurant-revenue-prediction<choose_model_class>
import pandas as pd
import numpy as np
def create_rtn(num_columns, num_labels, rethink_iter, num_layers, hidden_units, dropout_rates, lr):
inp = tf.keras.layers.Input(shape =(num_columns,), name = 'inp')
 x = tf.keras.layers.BatchNormalization(name = 'bn0' )(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0], name = 'dp0' )(x)
 x = tf.keras.layers.RepeatVector(rethink_iter, name = 'rv' )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(hidden_units[0], name = 'd0'), name = 'wn0' )(x)
 x = tf.keras.layers.Activation('swish', name = 'a0' )(x)
 for i in range(num_layers):
if i != num_layers - 1:
x = tf.keras.layers.LSTM(hidden_units[1], return_sequences = True, dropout = dropout_rates[1], name = f'lstm{i}' )(x)
 else:
x = tf.keras.layers.LSTM(hidden_units[1], return_sequences = False, dropout = dropout_rates[1], name = f'lstm{i}' )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(num_labels, name = f'd{num_labels}'), name = f'wn{num_labels}' )(x)
 out = tf.keras.layers.Activation('sigmoid', name = f'out{num_labels}' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 model.compile(optimizer = tfa.optimizers.AdamW(weight_decay = 1e-5, learning_rate = lr),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.0008),
metrics = tf.keras.losses.BinaryCrossentropy(name = 'mean_loss'), 
)
 return model<load_from_csv>
data = pd.read_csv('.. /input/train.csv')
 test_data = pd.read_csv('.. /input/test.csv')
 data.head()
def inference_model(X_train, Y_train_2, Y_nonscored, features, model_name, model_name_2, save_path, num_seeds, num_splits,
model_params, X_test = None, sample_sub_path = None, verbose = 0):
start_time_all = time()
oof = Y_train_2.copy()
oof.loc[:, Y_train_2.columns] = 0
overall_score = []
if X_test is not None:
sub = pd.read_csv(sample_sub_path)
 sub.loc[:, Y_train_2.columns] = 0
else:
sub = None
if 'RTN' in model_name_2:
model = create_rtn(len(features), 206, **model_params)
 elif 'RESNET' in model_name_2:
model = create_resnet(len(features), **model_params)
 elif 'ELU' in model_name_2:
model = create_mlp_elu(len(features), **model_params)
 elif 'MLP' in model_name_2:
model = create_mlp(len(features), 206, **model_params)
 for nums, seed in enumerate(range(num_seeds)) :
start_time_seed = time()
tf.random.set_seed(seed)
 np.random.seed(seed)
 random.seed(seed)
 mean_score = 0
skf = MultilabelStratifiedKFold(n_splits = num_splits, random_state = seed, shuffle = True)
 for n,(tr, te)in enumerate(skf.split(Y_train_2, Y_train_2)) :
print(f'Model:{model_name}, Seed:{seed}, Fold:{n}', end = '\r')
 start_time_fold = time()
if CALCULATE_OOF:
x_tr, x_val = X_train.values[tr][:, features], X_train.values[te][:, features]
y_tr, y_val = Y_train_2.values[tr], Y_train_2.values[te]
if 'ELU' in model_name_2:
ckp_path = save_path + f'{model_name}_{seed}_{n}.hdf5'
else:
ckp_path = save_path + f'{model_name}_Seed_{seed}_Fold_{n}.hdf5'
model.load_weights(ckp_path)
 if X_test is not None:
x_tt = X_test.values[:, features]
test_predict = model.predict(x_tt, batch_size = 1024)
 sub.loc[:, Y_train_2.columns] += test_predict /(num_splits * num_seeds)
 if CALCULATE_OOF:
val_predict = model.predict(x_val, batch_size = 1024)
 fold_score = log_loss_metric(y_val, val_predict)
 mean_score += fold_score / num_splits
oof.loc[te, Y_train_2.columns] += val_predict / num_seeds
if CALCULATE_OOF:
pass
if X_test is not None:
sub.loc[X_test['cp_type'] == 1, Y_train_2.columns] = 0
if CALCULATE_OOF:
oof.loc[X_train['cp_type'] == 1, Y_train_2.columns] = 0
overall_score = log_loss_metric(Y_train_2.values, oof[Y_train_2.columns].values)
 print(f'[{str(datetime.timedelta(seconds = time() - start_time_all)) [0:7]}] {model_name} OOF Score:', overall_score)
 return overall_score, oof, sub<load_from_csv>
data['Type_IL'] = np.where(data['Type'] == 'IL', 1, 0)
 data['Type_FC'] = np.where(data['Type'] == 'FC', 1, 0)
 data['Type_DT'] = np.where(data['Type'] == 'DT', 1, 0)
 data['Big_Cities'] = np.where(data['City Group'] == 'Big Cities', 1, 0)
 data['Days_Open'] =(pd.to_datetime('2015-03-23')- pd.to_datetime(data['Open Date'])).dt.days
data = data.drop('Type', axis=1)
 data = data.drop('City Group', axis=1)
 data = data.drop('City', axis=1)
 data = data.drop('Open Date', axis=1)
 test_data['Type_IL'] = np.where(test_data['Type'] == 'IL', 1, 0)
 test_data['Type_FC'] = np.where(test_data['Type'] == 'FC', 1, 0)
 test_data['Type_DT'] = np.where(test_data['Type'] == 'DT', 1, 0)
 test_data['Big_Cities'] = np.where(test_data['City Group'] == 'Big Cities', 1, 0)
 test_data['Days_Open'] =(pd.to_datetime('2015-03-23')- pd.to_datetime(test_data['Open Date'])).dt.days
test_data = test_data.drop('Type', axis=1)
 test_data = test_data.drop('City Group', axis=1)
 test_data = test_data.drop('City', axis=1)
 test_data = test_data.drop('Open Date', axis=1 )
def inference_ensemble(X_train, Y_train_2, model_name, model_name_2, save_path, num_seeds, num_splits,
model_params, X_test = None, sample_sub_path = None, verbose = 0):
start_time_all = time()
oof = Y_train_2.copy()
oof.loc[:, Y_train_2.columns] = 0
overall_score = []
if X_test is not None:
sub = pd.read_csv(sample_sub_path)
 sub.loc[:, Y_train_2.columns] = 0
else:
sub = None
if 'RTN' in model_name_2:
model = create_rtn(X_train.shape[1], 206, **model_params)
 elif 'RESNET' in model_name_2:
model = create_resnet(X_train.shape[1], **model_params)
 elif 'ELU' in model_name_2:
model = create_mlp_elu(X_train.shape[1], **model_params)
 elif 'MLP' in model_name_2:
model = create_mlp(X_train.shape[1], 206, **model_params)
 for seed in range(num_seeds):
start_time_seed = time()
tf.random.set_seed(seed)
 np.random.seed(seed)
 random.seed(seed)
 mean_score = 0
skf = MultilabelStratifiedKFold(n_splits = num_splits, random_state = seed, shuffle = True)
 for n,(tr, te)in enumerate(skf.split(Y_train_2, Y_train_2)) :
print(f'Model:{model_name}, Seed:{seed}, Fold:{n}', end = '\r')
 start_time_fold = time()
if CALCULATE_OOF:
x_tr, x_val = X_train[tr], X_train[te]
y_tr, y_val = Y_train_2.values[tr], Y_train_2.values[te]
if 'ELU' in model_name_2:
ckp_path = save_path + f'{model_name}_{seed}_{n}.hdf5'
else:
ckp_path = save_path + f'{model_name}_Seed_{seed}_Fold_{n}.hdf5'
model.load_weights(ckp_path)
 if X_test is not None:
x_tt = X_test
test_predict = model.predict(x_tt, batch_size = 1024)
 sub.loc[:, Y_train_2.columns] += test_predict /(num_splits * num_seeds)
 if CALCULATE_OOF:
val_predict = model.predict(x_val, batch_size = 1024)
 fold_score = log_loss_metric(y_val, val_predict)
 mean_score += fold_score / num_splits
oof.loc[te, Y_train_2.columns] += val_predict / num_seeds
if CALCULATE_OOF:
pass
if X_test is not None:
sub.loc[test['cp_type'] == 1, Y_train_2.columns] = 0
if CALCULATE_OOF:
oof.loc[train['cp_type'] == 1, Y_train_2.columns] = 0
overall_score = log_loss_metric(Y_train_2.values, oof[Y_train_2.columns].values)
 print(f'[{str(datetime.timedelta(seconds = time() - start_time_all)) [0:7]}] {model_name} OOF Score:', overall_score)
 return overall_score, oof, sub<define_search_space>
X = data.drop(['Id', 'revenue'], axis=1)
 Y = data.revenue
model_names = ['ELU_0', 'ELU_1', 'ELU_2', 'RESNET_0', 'RESNET_1', 'RESNET_2', 'MLP_0', 'MLP_1', 'MLP_2', 'RTN']
model_params = [{'hidden_units': [512, 512, 4096],
'dropout_rates': 0.45,
'lr': 1e-4,
},
{'hidden_units': [512, 1024],
'dropout_rates': 0.463,
'lr': 1e-4,
},
{'hidden_units': [1024, 1024],
'dropout_rates': 0.5,
'lr': 1e-4,
},
{'hidden_units': [128, 896, 256, 1024],
'dropout_rates': [0.5615750059111406, 0.381766362379825],
'lr': 1e-4,
} ,
{'hidden_units': [256, 1024, 512, 896],
'dropout_rates': [0.5493076960151594, 0.5121645764863383],
'lr': 1e-4,
},
{'hidden_units': [384, 1024, 256, 896],
'dropout_rates': [0.5923827297670073, 0.37098621422772815],
'lr': 1e-4,
},
{'hidden_units': [128, 1024],
'dropout_rates': [0.41458519175008285, 0.38992411412605404, 0.10265155152086326],
'lr': 1e-4,
},
{'hidden_units': [896, 128, 1664],
'dropout_rates': [0.532412647744322, 0.2498368055693044, 0.36619131749273925, 0.1386672227832089],
'lr': 1e-4,
},
{'hidden_units': [128, 896, 1024, 1792],
'dropout_rates': [0.4131287697962003, 0.33921841003415876, 0.13058255266781393, 0.20075775903486198, 0.3354496944535896],
'lr': 1e-4,
},
{'rethink_iter': 3,
'num_layers': 1,
'hidden_units': [128, 512],
'dropout_rates': [0.3, 0.4],
'lr': 1e-4,
},]<define_variables>
def check_rmse(X, Y, alpha):
RMSE_lasso = []
RMSE_ridge = []
for i in alpha:
lasso = Lasso(alpha=i)
 lasso.fit(X, Y)
 ridge = Ridge(alpha=i)
 ridge.fit(X, Y)
 RMSE_lasso.append(metrics.mean_squared_error(Y, lasso.predict(X)))
 RMSE_ridge.append(metrics.mean_squared_error(Y, ridge.predict(X)))
 return(RMSE_lasso, RMSE_ridge )
VERBOSE = 0
sample_sub_path = '.. /input/lish-moa/sample_submission.csv'
overall_oof_scores = []
oof_elu = []
oof_resnet = []
oof_mlp = []
submission_elu = []
submission_resnet = []
submission_mlp = []
for m in range(len(model_params)) :
print(model_names[m], model_params[m])
 if 'ELU' in model_names[m]:
model_idx = model_names[m].split('_')[1]
save_path = '.. /input/multilabel-v2/'
model_name = f'Model_{model_idx}'
elif 'RESNET' in model_names[m]:
model_idx = model_names[m].split('_')[1]
save_path = '.. /input/multilabel-v4/'
model_name = f'Model{model_idx}'
elif 'MLP' in model_names[m]:
model_idx = model_names[m].split('_')[1]
save_path = '.. /input/miltilabel-mlpstack/'
model_name = f'Model{model_idx}'
else:
save_path = '.. /input/multilabel-rtn-single/'
model_name = f'Model0'
oof_score, res, ss = inference_model(train, train_targets, train_targets_nonscored, top_feats, model_name, model_names[m],
save_path, N_STARTS, N_SPLITS, model_params[m], test, sample_sub_path, VERBOSE)
 if 'ELU' in model_names[m]:
oof_elu.append(res)
 submission_elu.append(ss)
 elif 'RESNET' in model_names[m]:
oof_resnet.append(res)
 submission_resnet.append(ss)
 elif 'MLP' in model_names[m]:
oof_mlp.append(res)
 submission_mlp.append(ss)
 else:
oof_rtn = res
submission_rtn = ss
if CALCULATE_OOF:
tr_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' ).drop('sig_id', axis = 1)
 res_all = np.zeros(tr_targets[cols].shape)
 res_all[train_features['cp_type'] == 0] = res[cols].values
overall_oof_score = log_loss_metric(tr_targets[cols].values, res_all)
 overall_oof_scores.append(overall_oof_score)
 print(f'{model_names[m]} Overall OOF CV Score:', overall_oof_score )<compute_test_metric>
lasso = Lasso(alpha=5.5)
 lasso.fit(X, Y)
 metrics.mean_squared_error(Y, lasso.predict(X))
if CALCULATE_OOF:
for n, name in enumerate(model_names):
print(f'{name} OOF:\t', overall_oof_scores[n] )<concatenate>
model = Lasso(alpha=5.5)
 model.fit(X, Y)
 test_predicted = pd.DataFrame()
test_predicted['Id'] = test_data.Id
test_predicted['Prediction'] = model.predict(test_data.drop('Id', axis=1))
test_predicted.to_csv('submission-lasso-5.5.csv', index=False)
 test_predicted.describe()
if CALCULATE_OOF:
train_new_elu = np.concatenate([oof0[cols].values for oof0 in oof_elu], axis = 1)
 train_new_resnet = np.concatenate([oof0[cols].values for oof0 in oof_resnet], axis = 1)
 train_new_mlp = np.concatenate([oof0[cols].values for oof0 in oof_mlp], axis = 1)
 else:
train_new_elu = np.zeros(( train.shape[0], len(cols)* 3))
train_new_resnet = np.zeros(( train.shape[0], len(cols)* 3))
train_new_mlp = np.zeros(( train.shape[0], len(cols)* 3))
test_new_elu = np.concatenate([sub[cols].values for sub in submission_elu], axis = 1)
 test_new_resnet = np.concatenate([sub[cols].values for sub in submission_resnet], axis = 1)
 test_new_mlp = np.concatenate([sub[cols].values for sub in submission_mlp], axis = 1 )<init_hyperparams>
model = RandomForestRegressor(n_estimators=150)
 model.fit(X, Y)
 test_predicted = pd.DataFrame()
test_predicted['Id'] = test_data.Id
test_predicted['Prediction'] = model.predict(test_data.drop('Id', axis=1))
test_predicted.to_csv('submission-random-forest.csv', index=False)
 test_predicted.describe()
model_names = ['ELU_stack', 'RESNET_stack', 'MLP_stack']
model_params = [{'hidden_units': [1024, 1024],
'dropout_rates': 0.336,
'lr': 1e-4,
},
{'hidden_units': [1024, 896, 896, 256],
'dropout_rates': [0.5398428318872255, 0.5734093398641228],
'lr': 1e-4,
},
{'hidden_units': [1920, 768],
'dropout_rates': [0.36130273975713795, 0.38130486900003896, 0.44485672673556004],
'lr': 1e-4,
},]<choose_model_class>
model = Ridge(alpha=330)
 model.fit(X, Y)
 test_predicted = pd.DataFrame()
test_predicted['Id'] = test_data.Id
test_predicted['Prediction'] = model.predict(test_data.drop('Id', axis=1))
test_predicted.to_csv('submission-ridge-330.csv', index=False)
 test_predicted.describe()
overall_oof_stack_scores = []
for m in range(len(model_params)) :
print(model_names[m], model_params[m])
 if 'ELU' in model_names[m]:
save_path = '.. /input/multilabel-v2/'
model_name = f'EModel'
oof_score, res, ss = inference_ensemble(train_new_elu, train_targets, model_name, model_names[m], save_path,
N_STARTS, N_SPLITS, model_params[m], test_new_elu, sample_sub_path, VERBOSE)
 oof_elu_stack = res.copy()
submission_elu_stack = ss.copy()
elif 'RESNET' in model_names[m]:
save_path = '.. /input/multilabel-v4/'
model_name = f'EModel_Stack'
oof_score, res, ss = inference_ensemble(train_new_resnet, train_targets, model_name, model_names[m], save_path,
N_STARTS, N_SPLITS, model_params[m], test_new_resnet, sample_sub_path, VERBOSE)
 oof_resnet_stack = res.copy()
submission_resnet_stack = ss.copy()
elif 'MLP' in model_names[m]:
save_path = '.. /input/miltilabel-mlpstack/'
model_name = f'EModel_Stack'
oof_score, res, ss = inference_ensemble(train_new_mlp, train_targets, model_name, model_names[m], save_path,
N_STARTS, N_SPLITS, model_params[m], test_new_mlp, sample_sub_path, VERBOSE)
 oof_mlp_stack = res.copy()
submission_mlp_stack = ss.copy()
if CALCULATE_OOF:
tr_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' ).drop('sig_id', axis = 1)
 res_all = np.zeros(tr_targets[cols].shape)
 res_all[train_features['cp_type'] == 0] = res[cols].values
overall_oof_score = log_loss_metric(tr_targets[cols].values, res_all)
 overall_oof_stack_scores.append(overall_oof_score)
 print(f'{model_names[m]} Overall OOF CV Score:', overall_oof_score )<compute_test_metric>
model = Lasso(alpha=200000)
 model.fit(X, Y)
 test_predicted = pd.DataFrame()
test_predicted['Id'] = test_data.Id
test_predicted['Prediction'] = model.predict(test_data.drop('Id', axis=1))
test_predicted.to_csv('submission-lasso-high-alpha.csv', index=False)
 test_predicted.describe()
if CALCULATE_OOF:
for n, name in enumerate(model_names):
print(f'{name} OOF:\t', overall_oof_stack_scores[n] )<categorify>
for c in X.columns:
print(c, len(X[c].unique()))
@njit
def post_process(pred, low, high):
pred_copy = pred.copy()
num_idx = 0
for i in range(pred_copy.shape[0]):
flag = np.zeros(pred_copy.shape[1])
 array = pred_copy[i].copy()
for j in range(pred_copy.shape[1]):
if(pred_copy[i, j] <= low)or(pred_copy[i, j] >= high):
flag[j] = 1
array[j] = round(array[j])
 if flag.all() and pred_copy[i].any() :
pred_copy[i] = array
num_idx += 1
return pred_copy, num_idx<load_from_csv>
data['Time_Open'] = round(data['Days_Open'] / 700, 0)
 data = data.drop('Days_Open', axis=1)
 test_data['Time_Open'] = round(test_data['Days_Open'] / 700, 0)
 test_data = test_data.drop('Days_Open', axis=1 )
ss = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 ss[cols] = 0.0877289129911273 * submission_elu_stack[cols].values + \
0.026620985745766827 * submission_resnet_stack[cols].values + \
0.04467573105638814 * submission_mlp_stack[cols].values + \
0.3382592931952609 * submission_rtn[cols].values + \
0.5027150770114568 * submission_tabnet[cols].values
ss.head(10 )<sort_values>
X = data.drop(['Id', 'revenue'], axis=1)
 Y = data.revenue
if POST_PROCESS:
low = 0.012
high = 0.98
ss[cols], num_idx = post_process(ss[cols].values, low, high)
 print(num_idx )<drop_column>
model = Ridge(alpha=330)
 model.fit(X, Y)
 test_predicted = pd.DataFrame()
test_predicted['Id'] = test_data.Id
test_predicted['Prediction'] = model.predict(test_data.drop('Id', axis=1))
test_predicted.to_csv('submission-ridge-330-div-700.csv', index=False)
 test_predicted.describe()
<define_variables><EOS> 
model = Lasso(alpha=200000)
 model.fit(X, Y)
 test_predicted = pd.DataFrame()
test_predicted['Id'] = test_data.Id
test_predicted['Prediction'] = model.predict(test_data.drop('Id', axis=1))
test_predicted.to_csv('submission-lasso-high-alpha-div-700.csv', index=False)
 test_predicted.describe()