Kaggle competition: Restaurant Revenue Prediction
<SOS> metric: RMSE Kaggle data source: restaurant-revenue-prediction<merge>
%matplotlib inline
valid_results = train_targets_scored.drop(columns=tar_col ).merge(train_target_with_sig_id, on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(tar_col)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )<prepare_output>
train_df = pd.read_csv('.. /input/restaurant-revenue-prediction/train.csv.zip',parse_dates=['Open Date'])
 test_df = pd.read_csv('.. /input/restaurant-revenue-prediction/test.csv.zip',parse_dates=['Open Date'] )
oof3 = valid_results.values[:,1:]
sub3 = sub.copy()<install_modules>
train_df.isnull().sum()
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/<set_options>
test_df.isnull().sum()
warnings.filterwarnings("ignore")
 sys.path.append(".. /input/rank-gauss")
 os.environ["CUDA_LAUNCH_BLOCKING"] = '1'
plt.style.use("fivethirtyeight")
 sys.path.append(".. /input/iterative-stratification")
 c_ = Fore.CYAN
m_ = Fore.MAGENTA
r_ = Fore.RED
b_ = Fore.BLUE
y_ = Fore.YELLOW
g_ = Fore.GREEN<set_options>
upper_limit = train_df.revenue.quantile(0.75)+(1.5* iqr(train_df.revenue))
lower_limit = train_df.revenue.quantile(0.25)-(1.5* iqr(train_df.revenue))
condition =(train_df.revenue > upper_limit)|(train_df.revenue<lower_limit)
 train_df[condition]
seed = 42
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(seed )<define_variables>
len(train_df.Id.unique())== train_df.shape[0]
data_path = ".. /input/lish-moa/"
no_ctl = True
scale = "rankgauss"
variance_threshould = 0.7
decompo = "PCA"
ncompo_genes = 80
ncompo_cells = 10
encoding = "dummy"<load_from_csv>
len(test_df.Id.unique())== test_df.shape[0]
train = pd.read_csv(data_path + "train_features.csv")
 targets = pd.read_csv(data_path + "train_targets_scored.csv")
 test = pd.read_csv(data_path + "test_features.csv")
 submission = pd.read_csv(data_path + "sample_submission.csv" )<filter>
train_df.drop('Id',axis=1,inplace=True)
 test_df.drop('Id', axis=1, inplace=True )
if no_ctl:
print(b_, "not_ctl")
 train = train[train["cp_type"] != "ctl_vehicle"]
test = test[test["cp_type"] != "ctl_vehicle"]
targets = targets.iloc[train.index]
train.reset_index(drop = True, inplace = True)
 test.reset_index(drop = True, inplace = True)
 targets.reset_index(drop = True, inplace = True )<define_variables>
train_df['Open Date'].value_counts()
GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]<concatenate>
train_df['open_year'] = train_df['Open Date'].dt.year
test_df['open_year'] = test_df['Open Date'].dt.year
data_all = pd.concat([train, test], ignore_index = True)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
data_all = pd.concat([data_all[["sig_id", "cp_type", "cp_time", "cp_dose"]], data_all[cols_numeric]], axis = 1)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]<normalization>
train_df['City Group'].value_counts()
def scale_minmax(col):
return(col - col.min())/(col.max() - col.min())
 def scale_norm(col):
return(col - col.mean())/ col.std()
if scale == "boxcox":
print(b_, "boxcox")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis = 0)
 trans = []
for feat in cols_numeric:
trans_var, lambda_var = stats.boxcox(data_all[feat].dropna() + 1)
 trans.append(scale_minmax(trans_var))
data_all[cols_numeric] = np.asarray(trans ).T
elif scale == "norm":
print(b_, "norm")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_norm, axis = 0)
 elif scale == "minmax":
print(b_, "minmax")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis = 0)
 elif scale == "rankgauss":
print(b_, "Rank Gauss")
 scaler = GaussRankScaler()
data_all[cols_numeric] = scaler.fit_transform(data_all[cols_numeric])
 else:
pass<train_model>
len(train_df.City.unique()),len(test_df.City.unique() )
if decompo == "PCA":
print(b_, "PCA")
 GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
pca_genes1 = PCA(n_components = ncompo_genes,
random_state = seed ).fit(data_all[GENES][:train.shape[0]])
 pca_genes = pca_genes1.transform(data_all[GENES])
 pca_cells1 = PCA(n_components = ncompo_cells,
random_state = seed ).fit(data_all[CELLS][:train.shape[0]])
 pca_cells = pca_cells1.transform(data_all[CELLS])
 pca_genes = pd.DataFrame(pca_genes, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells = pd.DataFrame(pca_cells, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 data_all = pd.concat([data_all, pca_genes, pca_cells], axis = 1)
 else:
pass<categorify>
train_df.drop('City',axis=1,inplace=True)
 test_df.drop('City',axis=1,inplace=True )
if encoding == "lb":
print(b_, "Label Encoding")
 for feat in ["cp_time", "cp_dose"]:
data_all[feat] = LabelEncoder().fit_transform(data_all[feat])
 elif encoding == "dummy":
print(b_, "One-Hot")
 data_all = pd.get_dummies(data_all, columns = ["cp_time", "cp_dose"] )<feature_engineering>
train_df.Type.value_counts()
GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
for stats in tqdm.tqdm(["sum", "mean", "std", "kurt", "skew"]):
data_all["g_" + stats] = getattr(data_all[GENES], stats )(axis = 1)
 data_all["c_" + stats] = getattr(data_all[CELLS], stats )(axis = 1)
 data_all["gc_" + stats] = getattr(data_all[GENES + CELLS], stats )(axis = 1 )<load_pretrained>
train_df.loc[124,'Type'] = 'IL'
with open("data_all.pickle", "wb")as f:
pickle.dump(data_all, f )<load_pretrained>
test_df.Type.value_counts()
with open("data_all.pickle", "rb")as f:
data_all = pickle.load(f )<drop_column>
train_df.P1.value_counts()
features_to_drop = ["sig_id", "cp_type"]
data_all.drop(features_to_drop, axis = 1, inplace = True)
 try:
targets.drop("sig_id", axis = 1, inplace = True)
 except:
pass
train_df = data_all[: train.shape[0]]
train_df.reset_index(drop = True, inplace = True)
 test_df = data_all[train_df.shape[0]: ]
test_df.reset_index(drop = True, inplace = True )<choose_model_class>
comp_df = pd.concat([train_df, test_df])
 comp_df.reset_index(drop=True, inplace=True )
MAX_EPOCH = 200
tabnet_params = dict(
 n_d = 32,
n_a = 32,
n_steps = 1,
gamma = 1.3,
lambda_sparse = 0,
optimizer_fn = optim.Adam,
optimizer_params = dict(lr = 2e-2, weight_decay = 1e-5),
mask_type = "entmax",
scheduler_params = dict(
 mode = "min", patience = 5, min_lr = 1e-5, factor = 0.9),
scheduler_fn = ReduceLROnPlateau,
seed = seed,
verbose = 10
)<normalization>
comp_df.Type.value_counts() ,comp_df['City Group'].value_counts()
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<import_modules>
comp_df.Type = comp_df.Type.map({'MB':0,'DT':1, 'IL':2,'FC':3})
 comp_df['City Group'] = comp_df['City Group'].map({'Big Cities':1, 'Other':0} )
class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux =(1 - y_true)* np.log(1 - logits + 1e-15)+ y_true * np.log(logits + 1e-15)
 return np.mean(-aux )<split>
p_name = ['P'+str(i)for i in range(1,38)]
comp_df[p_name] = MinMaxScaler().fit_transform(comp_df[p_name] )
scores_auc_all = []
test_cv_preds = []
NB_SPLITS = 8
oof_preds = []
oof_targets = []
scores = []
scores_auc = []
SEED =(42,43,44)
 for s in SEED:
mskf = MultilabelStratifiedKFold(n_splits = NB_SPLITS, random_state = s, shuffle = True)
 tabnet_params['seed'] = s
for fold_nb,(train_idx, val_idx)in enumerate(mskf.split(train_df, targets)) :
print(b_,"FOLDS: ", r_, fold_nb + 1, y_, 'seed:', tabnet_params['seed'])
 print(g_, '*' * 60, c_)
 X_train, y_train = train_df.values[train_idx, :], targets.values[train_idx, :]
X_val, y_val = train_df.values[val_idx, :], targets.values[val_idx, :]
model = TabNetRegressor(**tabnet_params)
 name = ".. /input/moa-predictions-overfitting-with-tabnet-f6b350/tabnet_model" + f"seed{s}_fold{fold_nb}.zip"
model.load_model(name)
 preds_val = model.predict(X_val)
 preds = 1 /(1 + np.exp(-preds_val))
logits = 1 /(1 + np.exp(-preds_val))
aux =(1 - y_val)* np.log(1 - logits + 1e-15)+ y_val * np.log(logits + 1e-15)
 score = np.mean(-aux)
 oof_preds.append(preds_val)
 oof_targets.append(y_val)
 scores.append(score)
 preds_test = model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 del model
gc.collect()
oof_preds_all = np.concatenate(oof_preds)
 oof_targets_all = np.concatenate(oof_targets)
 test_preds_all = np.stack(test_cv_preds )<compute_test_metric>
pca_list = ['pca'+str(i)for i in range(1,30,1)]
comp_df[pca_list] = PCA(n_components=29 ).fit_transform(comp_df[p_name])
 comp_df.drop(p_name,axis=1,inplace=True )
aucs = []
for task_id in range(oof_preds_all.shape[1]):
aucs.append(roc_auc_score(y_true = oof_targets_all[:, task_id],
y_score = oof_preds_all[:, task_id] 
))
print(f"{b_}Overall AUC: {r_}{np.mean(aucs)}")
 print(f"{b_}Average CV: {r_}{np.mean(scores)}" )<feature_engineering>
comp_df['launch_days'] =(datetime.datetime.now() - comp_df[['Open Date']])
 comp_df['launch_days'] = comp_df['launch_days'].dt.days
all_feat = [col for col in submission.columns if col not in ["sig_id"]]
test = pd.read_csv(data_path + "test_features.csv")
 sig_id = test[test["cp_type"] != "ctl_vehicle"].sig_id.reset_index(drop = True)
 tmp = pd.DataFrame(test_preds_all.mean(axis = 0), columns = all_feat)
 tmp["sig_id"] = sig_id
submission = pd.merge(test[["sig_id"]], tmp, on = "sig_id", how = "left")
 submission.fillna(0, inplace = True)
<create_dataframe>
comp_df.drop('Open Date',axis=1,inplace=True )
sub5 = submission.copy()<import_modules>
comp_df['launch_days'] = MinMaxScaler().fit_transform(comp_df[['launch_days']])
 comp_df['open_year'] = MinMaxScaler().fit_transform(comp_df[['open_year']] )
import datetime
import pandas as pd
from time import time
import tensorflow as tf
from autograd import grad
import autograd.numpy as np
from scipy.optimize import minimize, fsolve<compute_test_metric>
test_df = comp_df[comp_df['revenue'].isnull() ]
train_df = comp_df[comp_df['revenue'].notnull() ]
test_df.drop('revenue',axis=1, inplace=True )
def log_loss_numpy(y_pred):
loss = 0
y_pred_clip = np.clip(y_pred, 1e-16, 1 - 1e-16)
 for i in range(y_pred.shape[1]):
loss += - np.mean(y_true[:, i] * np.log(y_pred_clip[:, i])+(1 - y_true[:, i])* np.log(1 - y_pred_clip[:, i]))
return loss / y_pred.shape[1]
def func_numpy_metric(weights):
coef = 1e-6
oof_blend = weights[0] * oof1 + weights[1] * oof2 + weights[2] * oof3
score = log_loss_numpy(oof_blend)
 penalty = coef *(np.sum(weights)- 1)** 2
return score + penalty<load_from_csv>
x_train = train_df.drop('revenue',axis=1)
 y_train = train_df['revenue']
y_true = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv', index_col = 'sig_id' ).values 
<categorify>
from lightgbm import LGBMRegressor
from sklearn.model_selection import GridSearchCV, cross_validate, RepeatedKFold
oof1 = np.array(oof1,dtype='float')
 oof2 = np.array(oof2,dtype='float')
 oof3 = np.array(oof3,dtype='float')
<load_from_csv>
cv = RepeatedKFold(n_splits=10, n_repeats=3)
 scores = cross_validate(LGBMRegressor() , x_train,y_train, scoring=['r2','neg_root_mean_squared_error'],cv=cv )
y_true = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv', index_col = 'sig_id' ).values
print('Model 1 OOF(Numpy Log Loss):', log_loss_numpy(oof1))
print('Model 2 OOF(Numpy Log Loss):', log_loss_numpy(oof2))
print('Model 3 OOF(Numpy Log Loss):', log_loss_numpy(oof3))
print('-' * 50 )<compute_test_metric>
r2 = scores['test_r2']
rmse = scores['test_neg_root_mean_squared_error']
print(np.mean(r2),np.mean(rmse))
def Lagrange_func(params):
w1, w2, w3, _lambda = params
oof_blend = w1 * oof1 + w2 * oof2 + w3 * oof3
return log_loss_numpy(oof_blend)- _lambda *(w1 + w2 + w3 - 1 )<find_best_params>
final_model = LGBMRegressor(boosting_type='dart',max_depth=3,n_estimators=20,random_state=42, subsample=0.3 ).fit(x_train,y_train )
grad_L = grad(Lagrange_func )<find_best_params>
test_file = pd.read_csv('.. /input/restaurant-revenue-prediction/test.csv.zip')
 answer = pd.DataFrame(final_model.predict(test_df))
answer.columns = ['Prediction']
answer['Id'] = test_file.index.tolist()
answer.set_index('Id',inplace=True )
<find_best_params><EOS> 
answer.to_csv('result.csv' )