Kaggle competition: Restaurant Revenue Prediction
<SOS> metric: RMSE Kaggle data source: restaurant-revenue-prediction<choose_model_class>
trainData = pd.read_csv('.. /input/train.csv')
 testData = pd.read_csv('.. /input/test.csv')
 trainData = trainData.drop('Id', axis=1)
 testData = testData.drop('Id', axis=1 )
if FINETUNE:
if cfg.strategy == "KFOLD":
oof_preds_all = []
oof_targets_all = []
scores_all = []
scores_auc_all= []
preds_test = []
res = np.zeros(targets_tr.shape)
 for nums, seed in enumerate(range(cfg.num_ensembling)) :
print("
mskf = MultilabelStratifiedKFold(n_splits=cfg.SPLITS, random_state=cfg.seed+seed, shuffle=True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
p = []
for j,(train_idx, val_idx)in enumerate(mskf.split(np.zeros(len(cat_tr)) , targets_tr)) :
print("FOLDS : ", j)
 x_tr, y_tr = np.concatenate([cat_tr[train_idx], numerical_tr[train_idx]], axis=1), targets_tr[train_idx]
x_tr, y_tr = np.concatenate([x_tr, X_test[test['cp_type'] == 0, :]]), np.concatenate([y_tr, pseudo_targets])
 X_train, y_train = torch.as_tensor(x_tr), torch.as_tensor(y_tr)
 X_val, y_val = torch.as_tensor(np.concatenate([cat_tr[val_idx], numerical_tr[val_idx]], axis=1)) , torch.as_tensor(targets_tr[val_idx])
 model = TabNetRegressor(n_d=24, n_a=24, n_steps=1, gamma=1.3, lambda_sparse=0, cat_dims=cfg.cat_dims, cat_emb_dim=cfg.cat_emb_dim, cat_idxs=cfg.cats_idx, optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2, weight_decay=1e-5), mask_type='entmax', device_name=cfg.device, scheduler_params=dict(milestones=[100, 150], gamma=0.9),
scheduler_fn=torch.optim.lr_scheduler.MultiStepLR)
 name = cfg.save_name + f"_fold{j}_{seed}"
model.load_model(name)
 model.fit(X_train=X_train, y_train=y_train, X_valid=X_val, y_valid=y_val, max_epochs=200, patience=5, batch_size=1024, virtual_batch_size=128,
num_workers=0, drop_last=False, loss_fn=sbcewlogits, pretrain=True, optimizer_params=dict(lr=1e-4, weight_decay=1e-5))
model.load_best_model()
save_name = f"PL_tabnet_raw_step1_fold{j}_{seed}"
model.save_model(save_name)
 temp = model.predict(X_test)
 p.append(torch.sigmoid(torch.as_tensor(temp)).detach().cpu().numpy())
 if CALCULATE_OOF_PL:
preds = model.predict(X_val)
 preds = torch.sigmoid(torch.as_tensor(preds)).detach().cpu().numpy()
score = log_loss_multi(y_val, preds)
 res[val_idx] += preds / cfg.num_ensembling
oof_preds.append(preds)
 oof_targets.append(y_val)
 scores.append(score)
 scores_auc.append(auc_multi(y_val,preds))
print(f"validation fold {j} : {score}")
 p = np.stack(p)
 preds_test.append(p)
 if CALCULATE_OOF_PL:
oof_preds_all.append(np.concatenate(oof_preds))
oof_targets_all.append(np.concatenate(oof_targets))
scores_all.append(np.array(scores))
scores_auc_all.append(np.array(scores_auc))
preds_test = np.stack(preds_test )<load_from_csv>
trainData['Open Date'] = pd.to_datetime(trainData['Open Date'], format='%m/%d/%Y')
 testData['Open Date'] = pd.to_datetime(testData['Open Date'], format='%m/%d/%Y')
 trainData['OpenDays']=""
testData['OpenDays']=""
dateLastTrain = pd.DataFrame({'Date':np.repeat(['01/01/2015'],[len(trainData)])})
 dateLastTrain['Date'] = pd.to_datetime(dateLastTrain['Date'], format='%m/%d/%Y')
 dateLastTest = pd.DataFrame({'Date':np.repeat(['01/01/2015'],[len(testData)])})
 dateLastTest['Date'] = pd.to_datetime(dateLastTest['Date'], format='%m/%d/%Y')
 trainData['OpenDays'] = dateLastTrain['Date'] - trainData['Open Date']
testData['OpenDays'] = dateLastTest['Date'] - testData['Open Date']
trainData['OpenDays'] = trainData['OpenDays'].astype('timedelta64[D]' ).astype(int)
 testData['OpenDays'] = testData['OpenDays'].astype('timedelta64[D]' ).astype(int)
 trainData = trainData.drop('Open Date', axis=1)
 testData = testData.drop('Open Date', axis=1 )
if FINETUNE:
if CALCULATE_OOF_PL:
if cfg.strategy == "KFOLD":
for i in range(cfg.num_ensembling):
print("CV score fold : ", log_loss_multi(oof_targets_all[i], oof_preds_all[i]))
print("auc mean : ", sum(scores_auc_all[i])/len(scores_auc_all[i]))
tr_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' ).drop('sig_id', axis = 1)
 res_all = np.zeros(tr_targets[cols].shape)
 res_all[train_features['cp_type'] == 0] = res
overall_oof_score = log_loss_metric(tr_targets[cols].values, res_all)
 print(f'TabNet Overall OOF CV Score:', overall_oof_score)
 oof_tabnet2 = res_all<load_from_csv>
cityPerc = trainData[["City Group", "revenue"]].groupby(['City Group'],as_index=False ).mean()
citygroupDummy = pd.get_dummies(trainData['City Group'])
 trainData = trainData.join(citygroupDummy)
 citygroupDummyTest = pd.get_dummies(testData['City Group'])
 testData = testData.join(citygroupDummyTest)
 trainData = trainData.drop('City Group', axis=1)
 testData = testData.drop('City Group', axis=1 )
if FINETUNE:
submission_tabnet2 = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 submission_tabnet2[cols] = preds_test.mean(1 ).mean(0)
 submission_tabnet2.loc[test['cp_type'] == 1, cols] = 0<load_from_csv>
sns.set_context("notebook", font_scale=1.1)
 sns.set_style("ticks")
 xTrain = pd.DataFrame({'OpenDays':trainData['OpenDays'].apply(numpy.log),
'Big Cities':trainData['Big Cities'], 'Other':trainData['Other'],
'P2':trainData['P2'], 'P8':trainData['P8'], 'P22':trainData['P22'],
'P24':trainData['P24'], 'P28':trainData['P28'], 'P26':trainData['P26']})
 yTrain = trainData['revenue'].apply(numpy.log)
 xTest = pd.DataFrame({'OpenDays':testData['OpenDays'].apply(numpy.log),
'Big Cities':testData['Big Cities'], 'Other':testData['Other'],
'P2':testData['P2'], 'P8':testData['P8'], 'P22':testData['P22'],
'P24':testData['P24'], 'P28':testData['P28'], 'P26':testData['P26']})
 cls = RandomForestRegressor(n_estimators=150)
 cls.fit(xTrain, yTrain)
 pred = cls.predict(xTest)
 pred = numpy.exp(pred)
 cls.score(xTrain, yTrain )
def train_model(X_train, Y_train_2, Y_nonscored, features, model_name, model_name_2, save_path, num_seeds, num_splits,
model_params, X_test = None, sample_sub_path = None, pseudo_labeling = True, verbose = 0):
start_time_all = time()
oof = Y_train_2.copy()
oof.loc[:, Y_train_2.columns] = 0
overall_score = []
if X_test is not None:
sub = pd.read_csv(sample_sub_path)
 sub.loc[:, Y_train_2.columns] = 0
else:
sub = None
if 'RTN' in model_name_2:
model = create_rtn(len(features), 206, **model_params)
 elif 'RESNET' in model_name_2:
model = create_resnet(len(features), **model_params)
 elif 'ELU' in model_name_2:
model = create_mlp_elu(len(features), **model_params)
 elif 'MLP' in model_name_2:
model = create_mlp(len(features), 206, **model_params)
 for nums, seed in enumerate(range(num_seeds)) :
start_time_seed = time()
tf.random.set_seed(seed)
 np.random.seed(seed)
 random.seed(seed)
 mean_score = 0
skf = MultilabelStratifiedKFold(n_splits = num_splits, random_state = seed, shuffle = True)
 for n,(tr, te)in enumerate(skf.split(Y_train_2, Y_train_2)) :
print(f'Model:{model_name}, Seed:{seed}, Fold:{n}', end = '\r')
 start_time_fold = time()
x_tr, x_val = X_train.values[tr][:, features], X_train.values[te][:, features]
y_tr, y_val = Y_train_2.values[tr], Y_train_2.values[te]
if pseudo_labeling:
x_tr = np.concatenate([x_tr, pseudo_train[:, features]])
 y_tr = np.concatenate([y_tr, pseudo_targets])
 if X_test is not None:
x_tt = X_test.values[:, features]
if 'ELU' in model_name_2:
ckp_path = save_path + f'{model_name}_{seed}_{n}.hdf5'
else:
ckp_path = save_path + f'{model_name}_Seed_{seed}_Fold_{n}.hdf5'
model.load_weights(ckp_path)
 rlr = ReduceLROnPlateau(monitor = 'val_mean_loss', factor = 0.1, patience = 3,
verbose = verbose, min_delta = 1e-4, mode = 'min')
 ckp = ModelCheckpoint(f'{model_name}_Seed_{seed}_Fold_{n}.hdf5', monitor = 'val_mean_loss', verbose = 0,
save_best_only = True, save_weights_only = True, mode = 'min')
 es = EarlyStopping(monitor = 'val_mean_loss', min_delta = 1e-4, patience = 5, mode = 'min',
baseline = None, restore_best_weights = True, verbose = verbose)
 history = model.fit(x_tr, y_tr, validation_data =(x_val, y_val), epochs = 1000,
batch_size = 128, callbacks = [rlr, ckp, es], verbose = verbose)
 hist = pd.DataFrame(history.history)
 model.load_weights(f'{model_name}_Seed_{seed}_Fold_{n}.hdf5')
 if X_test is not None:
test_predict = model.predict(x_tt, batch_size = 1024)
 sub.loc[:, Y_train_2.columns] += test_predict /(num_splits * num_seeds)
 if CALCULATE_OOF_PL:
val_predict = model.predict(x_val, batch_size = 1024)
 fold_score = hist['val_mean_loss'].min()
mean_score += fold_score / num_splits
oof.loc[te, Y_train_2.columns] += val_predict / num_seeds
print(f'[{str(datetime.timedelta(seconds = time() - start_time_fold)) [0:7]}] {model_name} Seed {seed}, Fold {n}:', fold_score)
 if CALCULATE_OOF_PL:
print(f'[{str(datetime.timedelta(seconds = time() - start_time_seed)) [0:7]}] {model_name} Seed {seed} Mean Score:', mean_score)
 if X_test is not None:
sub.loc[X_test['cp_type'] == 1, Y_train_2.columns] = 0
if CALCULATE_OOF_PL:
oof.loc[X_train['cp_type'] == 1, Y_train_2.columns] = 0
overall_score = log_loss_metric(Y_train_2.values, oof[Y_train_2.columns].values)
 print(f'[{str(datetime.timedelta(seconds = time() - start_time_all)) [0:7]}] {model_name} OOF Score:', overall_score)
 return overall_score, oof, sub<define_search_space>
pred = cls.predict(xTest)
 pred = numpy.exp(pred )
model_names = ['ELU_0', 'ELU_1', 'ELU_2', 'RTN']
model_params = [{'hidden_units': [512, 512, 4096],
'dropout_rates': 0.45,
'lr': 1e-4,
},
{'hidden_units': [512, 1024],
'dropout_rates': 0.463,
'lr': 1e-4,
},
{'hidden_units': [1024, 1024],
'dropout_rates': 0.5,
'lr': 1e-4,
},
{'rethink_iter': 3,
'num_layers': 1,
'hidden_units': [128, 512],
'dropout_rates': [0.3, 0.4],
'lr': 1e-4,
},]<define_variables>
pred2 = []
for i in range(len(pred)) :
if pred[i] != float('Inf'):
pred2.append(pred[i])
 m = sum(pred2)/ float(len(pred2))
for i in range(len(pred)) :
if pred[i] == float('Inf'):
print("haha")
 pred[i] = m
if FINETUNE:
Pseudo_Labeling = True
VERBOSE = 0
sample_sub_path = '.. /input/lish-moa/sample_submission.csv'
overall_oof_scores = []
oof_elu2 = []
submission_elu2 = []
for m in range(len(model_params)) :
print(model_names[m], model_params[m])
 if 'ELU' in model_names[m]:
model_idx = model_names[m].split('_')[1]
save_path = '.. /input/multilabel-v2/'
model_name = f'Model_{model_idx}'
elif 'RTN' in model_names[m]:
save_path = '.. /input/multilabel-rtn-single/'
model_name = f'Model0'
oof_score, res, ss = train_model(train, train_targets, train_targets_nonscored, top_feats, model_name, model_names[m], save_path,
N_STARTS, N_SPLITS, model_params[m], test, sample_sub_path, Pseudo_Labeling, VERBOSE)
 if 'ELU' in model_names[m]:
oof_elu2.append(res)
 submission_elu2.append(ss)
 elif 'RTN' in model_names[m]:
oof_rtn2 = res
submission_rtn2 = ss
if CALCULATE_OOF_PL:
tr_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' ).drop('sig_id', axis = 1)
 res_all = np.zeros(tr_targets[cols].shape)
 res_all[train_features['cp_type'] == 0] = res[cols].values
overall_oof_score = log_loss_metric(tr_targets[cols].values, res_all)
 overall_oof_scores.append(overall_oof_score)
 print(f'{model_name} Overall OOF CV Score:', overall_oof_score )<compute_test_metric>
testData = pd.read_csv(".. /input/test.csv")
 submission = pd.DataFrame({
"Id": testData["Id"],
"Prediction": pred
})
 submission.to_csv('RandomForestSimple.csv',header=True, index=False )