Kaggle competition: Restaurant Revenue Prediction
def run_train_one_fold(
 settings, model, train_all_dataset, train_val_index, device, output_path):

 train_dataset = data.Subset(train_all_dataset, train_val_index[0])
 val_dataset = data.Subset(train_all_dataset, train_val_index[1])
 train_loader = data.DataLoader(train_dataset, **settings["loader"]["train"])
 val_loader = data.DataLoader(val_dataset, **settings["loader"]["val"])
 print("train: {}, val: {}".format(len(train_dataset), len(val_dataset)))
 model.to(device)
 optimizer = get_optimizer(settings, model)
 scheduler = get_scheduler(settings, train_loader, optimizer)
 loss_func = get_loss_function(settings)
 loss_func.to(device)
 eval_mgr = EvalFuncManager(len(val_loader), {
"loss": loss_func,
"metric": MyLogLoss() ,
})
 manager = get_manager(
 settings, model, device, train_loader, val_loader,
optimizer, eval_mgr, output_path)
 run_train_loop(
 manager, settings, model, device, train_loader, optimizer, scheduler, loss_func )<groupby>
train = pd.read_csv('/kaggle/input/restaurant-revenue-prediction/train.csv.zip')
 test = pd.read_csv('/kaggle/input/restaurant-revenue-prediction/test.csv.zip')
 train_Id = train.Id
test_Id = test.Id
train.drop('Id', axis=1, inplace=True)
 test.drop('Id', axis=1, inplace=True )
def get_drug_multilabel_index(target_in_drug_df, target_cols_list, vc_num, n_folds, seed):

 vc = target_in_drug_df.drug_id.value_counts()
vc1 = vc.loc[vc <= vc_num].index
vc2 = vc.loc[vc > vc_num].index
dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=n_folds, random_state=seed, shuffle=True)
 tmp = target_in_drug_df.groupby('drug_id')[target_cols_list].mean().loc[vc1]
if tmp.shape[0] != 0:
for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[target_cols_list])) :
dd = {k:fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=n_folds, random_state=seed, shuffle=True)
 tmp = target_in_drug_df.loc[target_in_drug_df.drug_id.isin(vc2)].reset_index(drop = True)
 if tmp.shape[0] != 0:
for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[target_cols_list])) :
dd = {k:fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 folds = []
target_in_drug_df['fold'] = target_in_drug_df.drug_id.map(dct1)
 target_in_drug_df.loc[target_in_drug_df.fold.isna() ,'fold'] = target_in_drug_df.loc[target_in_drug_df.fold.isna() ,'sig_id'].map(dct2)
 target_in_drug_df.fold = target_in_drug_df.fold.astype('int8')
 folds.append(target_in_drug_df.fold.values)
 return np.stack(folds)[0]
def drug_multilabel_stratified_kfold(k, target_in_drug_df, target_cols_list, n_folds, seed):

 folds_array = get_drug_multilabel_index(target_in_drug_df, target_cols_list, 18, n_folds, seed)
 train_idx = [i for i, x in enumerate(folds_array)if x != k]
validation_idx = [i for i, x in enumerate(folds_array)if x == k]
return train_idx, validation_idx<load_from_csv>
pd.options.display.max_columns = None
pd.options.display.max_rows = 80
pd.options.display.float_format = '{:.2f}'.format
%matplotlib inline
warnings.filterwarnings('ignore')
 %matplotlib inline
target_in_drug = pd.merge(
 pd.read_csv(".. /input/lish-moa/train_targets_scored.csv"),
pd.read_csv(".. /input/lish-moa/train_drug.csv"),
how='left', on='sig_id')
 print(target_in_drug.shape)
 target_in_drug_trt = del_control(target_in_drug)
 print(target_in_drug_trt.shape )<concatenate>
train['pd_date'] = pd.to_datetime(train['Open Date'], format='%m/%d/%Y')
 train['Open_Year'] = train['pd_date'].dt.strftime('%Y')
 train['Open_Month'] = train['pd_date'].dt.strftime('%m')
 train = train.drop('pd_date',axis=1)
 train = train.drop('Open Date',axis=1 )
X = np.concatenate([df.values for df in oof_list], axis=1)
 print(X.shape)
 y = y_true.values
print(y.shape)
 X_2d = np.stack([df.values for df in oof_list], axis=2)
 print(X_2d.shape)
 X_3d = X_2d[:, None,...]
print(X_3d.shape)
 X_node = np.stack([df.values for df in oof_list], axis=2 )<create_dataframe>
test['pd_date'] = pd.to_datetime(test['Open Date'], format='%m/%d/%Y')
 test['Open_Year'] = test['pd_date'].dt.strftime('%Y')
 test['Open_Month'] = test['pd_date'].dt.strftime('%m')
 test = test.drop('pd_date',axis=1)
 test = test.drop('Open Date',axis=1 )
train_all_dataset = MoAStackingDatasetForCNN(X_3d, y)
<define_variables>
train.dtypes.value_counts()
TRAIN_OUTPUT = Path.cwd().parent / "tmp"
TRAIN_OUTPUT.mkdir(exist_ok=True)
 WORK_DIR = Path.cwd()<init_hyperparams>
train.nunique(axis=0 )
settings_str =
settings = yaml.safe_load(settings_str )<set_options>
print(f'categorical variables: {cats}')
 print(f'numerical variables: {nums}' )
gc.collect()<feature_engineering>
nominal_list =cats
num_list = nums
device = torch.device(settings["globals"]["device"])
 cv_score_df_list = []
if settings["globals"]["fast_commit"]:
settings["globals"]["max_epoch"] = 1
stgs_list = []
for tmp_seed in settings["globals"]["seeds_for_avg"]:
stgs_list.append(deepcopy(settings))
stgs_list[-1]["globals"]["seed"] = tmp_seed<categorify>
train = train.drop('Open_Month',axis=1)
 test= test.drop('Open_Month',axis=1)
 nominal_list.remove('Open_Month' )
for m_id, tmp_stgs in enumerate(stgs_list):
n_folds = settings["globals"]["n_folds_split"]
tmp_seed = tmp_stgs["globals"]["seed"]
train_val_indexs = [
drug_multilabel_stratified_kfold(
 fold_id, target_in_drug_trt, TARGET_COL, n_folds, tmp_seed)
 for fold_id in range(n_folds)]
train_all_dataset.reset_model_order()
train_all_dataset.shuffle_model_order(tmp_seed)
 print("[model {}'s order]: {}".format(m_id, train_all_dataset.model_order))
for fold_id in range(n_folds):
print("[fold: {} - model: {}]".format(fold_id, m_id))
set_random_seed(tmp_seed , True)
 model = eval(tmp_stgs["model"]["name"] )(**tmp_stgs["model"]["params"])
 run_train_one_fold(
 tmp_stgs, model, train_all_dataset, train_val_indexs[fold_id], device,
TRAIN_OUTPUT / "fold{}_model{}".format(fold_id, m_id))
del model
gc.collect()
best_eval_list = []
for fold_id in range(n_folds):
output_dir = TRAIN_OUTPUT / "fold{}_model{}".format(fold_id, m_id)
 log_df = pd.read_json(output_dir / "log")
 best_log = log_df.iloc[log_df["val/metric"].idxmin() ][["epoch", "train/loss", "val/loss", "val/metric"]]
best_eval_list.append(best_log)
 best_epoch = int(best_log["epoch"])
 model_path = output_dir / "snapshot_epoch_{}.pth".format(best_epoch)
 cp_path = WORK_DIR / "best_model_fold{}_model{}.pth".format(fold_id, m_id)
 shutil.copy(model_path, cp_path)
 for model_path in output_dir.glob("*.pth"):
model_path.unlink()
cv_score_df = pd.DataFrame(best_eval_list)
 cv_score_df.insert(0, "fold", range(settings["globals"]["n_folds_split"]))
cv_score_df.insert(0, "model", m_id)
 del best_eval_list
cv_score_df_list.append(cv_score_df )<sort_values>
mean_revenue_per_city = train[['City', 'revenue']].groupby('City', as_index=False ).mean()
mean_revenue_per_city.head()
mean_revenue_per_city['revenue'] = mean_revenue_per_city['revenue'].apply(lambda x: int(x/1e6))
mean_revenue_per_city
mean_dict = dict(zip(mean_revenue_per_city.City, mean_revenue_per_city.revenue))
mean_dict
def order_sub(sub):
return sub.sort_values('sig_id' ).reset_index(drop=True )<load_from_csv>
if DO_SUBMISSION or DO_VIRTUAL_SUBMISSION:
sub_list = [
pd.read_csv('./submission-sinchir0-nn.csv'),
pd.read_csv('./submission_sinchir0_tabnet.csv'),
pd.read_csv("./submission_takapy_tf-resnet.csv"),
pd.read_csv('./submission_tawara_thrnn_seed_cv.csv'),
pd.read_csv('./submission_tawara_thrnn_drug_seed_cv.csv'),
]
else:
sub_list = [
pd.read_csv('.. /input/nn-use-train-public/submission.csv'),
pd.read_csv('.. /input/tabnet-train-public-add-n-shared-1/submission.csv'),
pd.read_csv(".. /input/moa-takapy-tf-resnet-transfer/submission.csv"),
pd.read_csv('.. /input/moa-weight-thrnn-seed-cv/submission.csv'),
pd.read_csv('.. /input/moa-weight-thrnn-drug-seed-cv/submission.csv'),
]
for i, name in enumerate(["NN(drugCV)", "TabNet", "ResNet", "ThrNN", "ThrNN(drugCV)"]):
print(f"[{name}]:", sub_list[i].shape )<sort_values>
sub_list = [order_sub(sub_df)for sub_df in sub_list]<predict_on_test>
l1_l2_and = set(city_train_list)& set(city_test_list)
 print(l1_l2_and)
 print(len(l1_l2_and))
def inference_function(settings, model, loader, device):
model.to(device)
 model.eval()
pred_list = []
with torch.no_grad() :
for x, t in loader:
y = model(x.to(device))
pred_list.append(y.sigmoid().detach().cpu().numpy())
 pred_arr = np.concatenate(pred_list)
 del pred_list
return pred_arr<define_variables>
len(set(city_test_list ).difference(city_train_list))
16 * 206<concatenate>
len(set(city_train_list ).difference(city_test_list))
X_test = np.concatenate([
sub_df.iloc[:, 1:].values for sub_df in sub_list], axis=1)
 print(X_test.shape)
 X_test_2d = np.stack([
sub_df.iloc[:, 1:].values for sub_df in sub_list], axis=2)
 print(X_test_2d.shape)
 X_test_3d = X_test_2d[:, None,...]
print(X_test_3d.shape)
 X_test_node = np.stack([
sub_df.iloc[:, 1:].values for sub_df in sub_list], axis=2)
 test_dataset = MoAStackingDatasetForCNN(X_test_3d, None)
<define_variables>
def adjust_cities(full_full_data, train, k):
relevant_pvars = ["P1", "P2", "P11", "P19", "P20", "P23","P30"]
train = train.loc[:, relevant_pvars]
kmeans = cluster.KMeans(n_clusters=k)
 kmeans.fit(train)
 full_data['City_Cluster'] = kmeans.predict(full_data.loc[:, relevant_pvars])
 return full_data
oof_pred_arr_avg = np.zeros(( len(X), 206))
test_pred_arr_avg = np.zeros(( len(X_test), 206))
for m_id, tmp_stgs in enumerate(stgs_list):
n_folds = tmp_stgs["globals"]["n_folds_split"]
tmp_seed = tmp_stgs["globals"]["seed"]
train_val_indexs = [
drug_multilabel_stratified_kfold(
 fold_id, target_in_drug_trt, TARGET_COL, n_folds, tmp_seed)
 for fold_id in range(n_folds)]
oof_pred_arr = np.zeros(( len(X), 206))
test_preds_arr = np.zeros(( n_folds, len(X_test), 206))
train_all_dataset.reset_model_order()
train_all_dataset.shuffle_model_order(tmp_seed)
 test_dataset.model_order = train_all_dataset.model_order
print("[model {}'s order]: {}".format(m_id, train_all_dataset.model_order))
for fold_id in range(n_folds):
print("[fold: {} - model: {}]".format(fold_id, m_id))
model_path = WORK_DIR / "best_model_fold{}_model{}.pth".format(fold_id, m_id)
 model = eval(tmp_stgs["model"]["name"] )(**tmp_stgs["model"]["params"])
 model.load_state_dict(torch.load(model_path))
val_index = train_val_indexs[fold_id][1]
val_dataset = data.Subset(train_all_dataset, val_index)
 val_loader = data.DataLoader(val_dataset, **tmp_stgs["loader"]["val"])
 val_pred = inference_function(tmp_stgs, model, val_loader, device)
 oof_pred_arr[val_index] = val_pred
del val_dataset; del val_loader; del val_pred
test_loader = data.DataLoader(test_dataset, **tmp_stgs["loader"]["val"])
 test_pred = inference_function(tmp_stgs, model, test_loader, device)
 test_preds_arr[fold_id] = test_pred
del test_loader; del test_pred;
gc.collect()
oof_pred_arr_avg += oof_pred_arr
test_pred_arr_avg += test_preds_arr.mean(axis=0)
 oof_score = log_loss(y.ravel() , oof_pred_arr.ravel())
 cv_score_df_list[m_id] = cv_score_df_list[m_id].append(
 {"model": m_id, "fold": "oof", "val/metric": oof_score,},
ignore_index=True)
 oof_pred_arr_avg /= len(stgs_list)
 test_pred_arr_avg /= len(stgs_list )<concatenate>
num_train = train.shape[0]
num_test = test.shape[0]
print(num_train, num_test)
 full_data = pd.concat([train, test], ignore_index=True )
cv_score_all = pd.concat(cv_score_df_list, axis=0)
 cv_score_all<compute_test_metric>
full_data = adjust_cities(full_data, train, 20)
 full_data
full_data = full_data.drop(['City'], axis=1 )
print(log_loss(y.ravel() , oof_pred_arr_avg.ravel()))<load_from_csv>
train = full_data[:num_train]
test = full_data[num_train:]
print("Train :",train.shape)
 print("Test:",test.shape)
 test
if DO_SUBMISSION or DO_TRAIN_FOR_ENSEMBLE:
df_test = pd.read_csv(".. /input/lish-moa/test_features.csv")
 submission = pd.read_csv(".. /input/lish-moa/sample_submission.csv")
 elif DO_VIRTUAL_SUBMISSION:
df_test = pd.read_csv(".. /input/lish-moa/test_features.csv")
 submission = pd.read_csv(".. /input/lish-moa/sample_submission.csv")
 print(df_test.shape, submission.shape)
 df_test, submission = generate_virtual_private(df_test, submission)
 print(df_test.shape, submission.shape)
 else:
raise ValueError<load_from_csv>
mean_revenue_per_city = train[['City_Cluster', 'revenue']].groupby('City_Cluster', as_index=False ).mean()
mean_revenue_per_city.head()
mean_revenue_per_city['revenue'] = mean_revenue_per_city['revenue'].apply(lambda x: int(x/1e6))
mean_revenue_per_city
mean_dict = dict(zip(mean_revenue_per_city.City_Cluster, mean_revenue_per_city.revenue))
mean_dict
df = submission.copy()
public_id = list(df['sig_id'].values)
 test_id = list(df_test['sig_id'].values)
 private_id = list(set(test_id)-set(public_id))
df_submit = pd.DataFrame(index = public_id+private_id, columns=TARGET_COL)
 df_submit.index.name = 'sig_id'
df_submit[:] = 0
df_predict = BLEND.copy()
df_submit.loc[df_predict.sig_id,:] = df_predict[TARGET_COL].values
df_submit.loc[df_test[df_test.cp_type =='ctl_vehicle'].sig_id] = 0
df_submit.to_csv('submission.csv',index=True )<import_modules>
city_rev = []
for i in full_data['City_Cluster']:
for key, value in mean_dict.items() :
if i == key:
city_rev.append(value)
 df_city_rev = pd.DataFrame({'city_rev':city_rev})
 full_data = pd.concat([full_data,df_city_rev],axis=1)
 full_data.head
nominal_list.extend(['City_Cluster'])
 nominal_list.remove('City')
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import GroupKFold, StratifiedKFold, KFold, train_test_split, cross_val_score
from sklearn.metrics import log_loss, make_scorer
from sklearn.decomposition import PCA
from sklearn.preprocessing import QuantileTransformer
import torch
import matplotlib.pyplot as plt
import random<set_options>
le = LabelEncoder()
le_count = 0
for i in range(len(nominal_list)) :
if len(list(full_data[nominal_list[i]].unique())) <= 2:
le.fit(full_data[nominal_list[i]])
 full_data[nominal_list[i]] = le.transform(full_data[nominal_list[i]])
 le_count += 1
print('%d columns were label encoded.' % le_count )
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42)
<load_from_csv>
full_data = pd.get_dummies(full_data)
 print('full_dataing Features shape: ', full_data.shape )
test_features = pd.read_csv("/kaggle/input/lish-moa/test_features.csv")
 train_drug = pd.read_csv("/kaggle/input/lish-moa/train_drug.csv")
 train_features = pd.read_csv("/kaggle/input/lish-moa/train_features.csv")
 train_targets_scored = pd.read_csv("/kaggle/input/lish-moa/train_targets_scored.csv")
 train_targets_nonscored = pd.read_csv("/kaggle/input/lish-moa/train_targets_nonscored.csv" )<define_variables>
def tukey_outliers(x):
q1 = np.percentile(x,25)
 q3 = np.percentile(x,75)
 iqr = q3-q1
min_range = q1 - iqr*1.5
max_range = q3 + iqr*1.5
outliers = x[(x<min_range)|(x>max_range)]
return outliers
common_names = [name for name in train_targets_nonscored.columns
if name in train_targets_scored.columns]
common_names<filter>
cp_indexes_train = train_features[train_features["cp_type"] == "ctl_vehicle"].index
cp_indexes_test = test_features[test_features["cp_type"] == "ctl_vehicle"].index
print(len(cp_indexes_train), len(cp_indexes_test))<drop_column>
skewed_data = train[num_list].apply(lambda x: skew(x)).sort_values(ascending=False)
 skewed_data[:10]
<define_variables>
train = full_data[:num_train]
test = full_data[num_train:]
print("Train :",train.shape)
 print("Test:",test.shape )
GENES = [colname for colname in train_features.columns
if colname.startswith("g-")]
CELLS = [colname for colname in train_features.columns
if colname.startswith("c-")]<categorify>
( X_train, X_test, y_train, y_test)= train_test_split(train_X, train_y , test_size = 0.3 , random_state = 0)
 print("X_train: "+str(X_train.shape))
print("X_test: "+str(X_test.shape))
print("y_train: "+str(y_train.shape))
print("y_test: "+str(y_test.shape))
def map_dataset(df):
df['cp_type'] = df['cp_type'].map({'ctl_vehicle': 0, 'trt_cp':1})
 df['cp_dose'] = df['cp_dose'].map({"D1":0, "D2":1})
 return df<create_dataframe>
from sklearn.model_selection import GridSearchCV, cross_val_score, learning_curve
from sklearn.metrics import mean_absolute_error
from sklearn.linear_model import Lasso
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Ridge
from sklearn.linear_model import ElasticNet
from sklearn.neighbors import KNeighborsRegressor
from sklearn.svm import SVR
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.ensemble import AdaBoostRegressor
from sklearn.tree import DecisionTreeRegressor
from xgboost import XGBRegressor
train_features = map_dataset(train_features)
 test_features = map_dataset(test_features )<categorify>
random_state = 2
classifiers = []
classifiers.append(Lasso(random_state=random_state))
classifiers.append(LinearRegression())
 classifiers.append(Ridge(random_state=random_state))
classifiers.append(ElasticNet(random_state=random_state))
classifiers.append(KNeighborsRegressor())
 classifiers.append(SVR())
 classifiers.append(RandomForestRegressor(random_state=random_state))
classifiers.append(GradientBoostingRegressor())
 classifiers.append(AdaBoostRegressor(random_state = random_state))
classifiers.append(DecisionTreeRegressor())
 classifiers.append(XGBRegressor() )
def add_PCA(train, test):
n_gs = 200
n_cs = 20
pca_cs = PCA(n_components = n_cs)
 pca_gs = PCA(n_components = n_gs)
 united = pd.concat([train[GENES + CELLS], test[GENES+CELLS]], axis=0)
 united_pca_gs = pca_gs.fit_transform(united[GENES])
 united_pca_cs = pca_cs.fit_transform(united[CELLS])
 united_c_mean = np.expand_dims(united[CELLS].mean(axis=1), axis=1)
 united_g_mean = np.expand_dims(united[GENES].mean(axis=1), axis=1)
 generated_features = pd.DataFrame(np.concatenate([united_pca_gs, united_pca_cs,
united_g_mean, united_c_mean], axis=1))
train_generated = generated_features.iloc[:train.shape[0]].reset_index(drop=True)
 test_generated = generated_features.iloc[train.shape[0]:].reset_index(drop=True)
 train = pd.concat([train, train_generated], axis=1)
 print("before:", test.shape)
 print(test_generated.shape, test.shape)
 test = pd.concat([test, test_generated], axis=1)
 print("After:", test.shape)
 return train, test 
<categorify>
cv_res.sort_values(ascending=False, by='CrossValMeans' )
def QuantileTransform(train, test, transform_type):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution=transform_type)
 united = pd.concat([train[GENES+CELLS], test[GENES+CELLS]], axis=0)
 united = transformer.fit_transform(united)
 train[GENES+CELLS] = united[:train.shape[0]]
test[GENES+CELLS] = united[train.shape[0]:]
return train, test<feature_engineering>
def objective(trial):
params = {
'alpha': trial.suggest_loguniform("alpha", 0.1, 5),
'fit_intercept': trial.suggest_categorical('fit_intercept', [True, False]),
'normalize': trial.suggest_categorical('normalize', [True, False]),
}
reg = Ridge(**params)
 reg.fit(X_train, y_train)
 y_pred = reg.predict(X_test)
 mae = mean_absolute_error(y_test, y_pred)
 return mae
train_features, test_features = QuantileTransform(train_features, test_features, 'normal' )<categorify>
study = optuna.create_study()
study.optimize(objective, n_trials=100)
 print(f'best score: {study.best_value:.4f}, best params: {study.best_params}' )
train_features = train_features.merge(train_drug, on='sig_id')
 train_features['drug_id'] = LabelEncoder().fit_transform(train_features['drug_id'] )<import_modules>
params = {'alpha': 1.9510706324753746, 'fit_intercept': True, 'normalize': True}
reg = Ridge(**params)
 reg.fit(X_train, y_train)
 prediction_log = reg.predict(test_X)
 prediction =np.exp(prediction_log)
 print(prediction )
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.optim import Adam
from torch import optim
from torch.utils.data import Dataset, DataLoader<import_modules>
submission = pd.DataFrame({"Id":test_Id, "Prediction":prediction})
 submission.to_csv("submission.csv", index=False )
from pytorch_tabnet.tab_model import TabNetRegressor<data_type_conversions>
<define_search_model><EOS>