Kaggle competition: Restaurant Revenue Prediction
if FINETUNE:
if CALCULATE_OOF_PL:
print(f'{model_names} OOF:\t', overall_oof_scores )<train_on_grid>
warnings.filterwarnings('ignore' )
def train_ensemble(X_train, Y_train_2, model_name, model_name_2, save_path, num_seeds, num_splits,
model_params, X_test = None, sample_sub_path = None, verbose = 0):
start_time_all = time()
oof = Y_train_2.copy()
oof.loc[:, Y_train_2.columns] = 0
overall_score = []
if X_test is not None:
sub = pd.read_csv(sample_sub_path)
 sub.loc[:, Y_train_2.columns] = 0
else:
sub = None
if 'RTN' in model_name_2:
model = create_rtn(X_train.shape[1], 206, **model_params)
 elif 'RESNET' in model_name_2:
model = create_resnet(X_train.shape[1], **model_params)
 elif 'ELU' in model_name_2:
model = create_mlp_elu(X_train.shape[1], **model_params)
 elif 'MLP' in model_name_2:
model = create_mlp(X_train.shape[1], 206, **model_params)
 for seed in range(num_seeds):
start_time_seed = time()
tf.random.set_seed(seed)
 np.random.seed(seed)
 random.seed(seed)
 mean_score = 0
skf = MultilabelStratifiedKFold(n_splits = num_splits, random_state = seed, shuffle = True)
 for n,(tr, te)in enumerate(skf.split(Y_train_2, Y_train_2)) :
print(f'Model:{model_name}, Seed:{seed}, Fold:{n}', end = '\r')
 start_time_fold = time()
x_tr, x_val = X_train[tr], X_train[te]
y_tr, y_val = Y_train_2.values[tr], Y_train_2.values[te]
if 'ELU' in model_name_2:
ckp_path = save_path + f'{model_name}_{seed}_{n}.hdf5'
else:
ckp_path = save_path + f'{model_name}_Seed_{seed}_Fold_{n}.hdf5'
model.load_weights(ckp_path)
 rlr = ReduceLROnPlateau(monitor = 'val_mean_loss', factor = 0.1, patience = 3,
verbose = verbose, min_delta = 1e-4, mode = 'min')
 ckp = ModelCheckpoint(f'{model_name}_Seed_{seed}_Fold_{n}.hdf5', monitor = 'val_mean_loss', verbose = 0,
save_best_only = True, save_weights_only = True, mode = 'min')
 es = EarlyStopping(monitor = 'val_mean_loss', min_delta = 1e-4, patience = 5, mode = 'min',
baseline = None, restore_best_weights = True, verbose = verbose)
 history = model.fit(x_tr, y_tr, validation_data =(x_val, y_val), epochs = 1000,
batch_size = 128, callbacks = [rlr, ckp, es], verbose = verbose)
 hist = pd.DataFrame(history.history)
 model.load_weights(f'{model_name}_Seed_{seed}_Fold_{n}.hdf5')
 if X_test is not None:
x_tt = X_test
test_predict = model.predict(x_tt, batch_size = 1024)
 sub.loc[:, Y_train_2.columns] += test_predict /(num_splits * num_seeds)
 if CALCULATE_OOF_PL:
val_predict = model.predict(x_val, batch_size = 1024)
 fold_score = hist['val_mean_loss'].min()
mean_score += fold_score / num_splits
oof.loc[te, Y_train_2.columns] += val_predict / num_seeds
print(f'[{str(datetime.timedelta(seconds = time() - start_time_fold)) [0:7]}] {model_name} Seed {seed}, Fold {n}:', fold_score)
 if CALCULATE_OOF_PL:
print(f'[{str(datetime.timedelta(seconds = time() - start_time_seed)) [0:7]}] {model_name} Seed {seed} Mean Score:', mean_score)
 if X_test is not None:
sub.loc[test['cp_type'] == 1, Y_train_2.columns] = 0
if CALCULATE_OOF_PL:
oof.loc[train['cp_type'] == 1, Y_train_2.columns] = 0
overall_score = log_loss_metric(Y_train_2.values, oof[Y_train_2.columns].values)
 print(f'[{str(datetime.timedelta(seconds = time() - start_time_all)) [0:7]}] {model_name} OOF Score:', overall_score)
 return overall_score, oof, sub<concatenate>
df_train = pd.read_csv("/kaggle/input/restaurant-revenue-prediction/train.csv.zip")
 df_test = pd.read_csv("/kaggle/input/restaurant-revenue-prediction/test.csv.zip" )
if FINETUNE:
if CALCULATE_OOF_PL:
train_new_elu2 = np.concatenate([oof0[cols].values for oof0 in oof_elu2], axis = 1)
 else:
train_new_elu2 = np.zeros(( train.shape[0], len(cols)* 3))
test_new_elu2 = np.concatenate([sub[cols].values for sub in submission_elu2], axis = 1 )<define_search_space>
df_train = df_train.drop("Id",axis=1 )
model_names = ['ELU_stack']
model_params = [{'hidden_units': [1024, 1024],
'dropout_rates': 0.336,
'lr': 1e-4,
},]<load_pretrained>
quantitative_feats = [i for i in df_train if df_train[i].dtype != np.object]
qualitative_feats = [i for i in df_train if df_train[i].dtype == np.object]
print("Quantitative_features: {}".format(quantitative_feats))
print("Qualtative_features: {}".format(qualitative_feats))
if FINETUNE:
overall_oof_stack_scores = []
for m in range(len(model_params)) :
print(model_names[m], model_params[m])
 if 'ELU' in model_names[m]:
save_path = '.. /input/multilabel-v2/'
model_name = f'EModel'
oof_score, res, ss = train_ensemble(train_new_elu2, train_targets, model_name, model_names[m], save_path,
N_STARTS, N_SPLITS, model_params[m], test_new_elu2, sample_sub_path, VERBOSE)
 oof_elu_stack2 = res.copy()
submission_elu_stack2 = ss.copy()
if CALCULATE_OOF_PL:
tr_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' ).drop('sig_id', axis = 1)
 res_all = np.zeros(tr_targets[cols].shape)
 res_all[train_features['cp_type'] == 0] = res[cols].values
overall_oof_score = log_loss_metric(tr_targets[cols].values, res_all)
 overall_oof_stack_scores.append(overall_oof_score)
 print(f'{model_names[m]} Overall OOF CV Score:', overall_oof_score )<compute_test_metric>
quantitative_feats.remove("revenue")
 qualitative_feats.remove("Open Date" )
if CALCULATE_OOF_PL:
for n, name in enumerate(model_names):
print(f'{name} OOF:\t', overall_oof_stack_scores[n] )<load_from_csv>
revenue = df_train["revenue"]
if FINETUNE:
ss = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 ss[cols] = 0.0877289129911273 * submission_elu_stack2[cols].values + \
0.026620985745766827 * submission_resnet_stack[cols].values + \
0.04467573105638814 * submission_mlp_stack[cols].values + \
0.3382592931952609 * submission_rtn2[cols].values + \
0.5027150770114568 * submission_tabnet2[cols].values<save_to_csv>
discrete_feats = [feat for feat in quantitative_feats if df_train[feat].nunique() <= 25]
discrete_feats
ss.to_csv('submission.csv', index = False)
 ss.head(10 )<define_variables>
continuous_feats = [feat for feat in quantitative_feats if df_train[feat].nunique() > 25]
continuous_feats
DO_SUBMISSION = True
DO_TRAIN_FOR_ENSEMBLE = False
DO_VIRTUAL_SUBMISSION = False
assert(sum([DO_SUBMISSION, DO_TRAIN_FOR_ENSEMBLE, DO_VIRTUAL_SUBMISSION])== 1), "select `ONE` mode"<install_modules>
print(df_train.isnull().any() )
%%time
!pip install --no-index --find-links.. /input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet<install_modules>
def calc_vif(X):
vif = pd.DataFrame()
vif["variables"] = X.columns
vif["VIF"] = [variance_inflation_factor(X.values, i)for i in range(X.shape[1])]
return(vif)
 vif = calc_vif(df_train[quantitative_feats])
 high_vif = vif[vif["VIF"] > 10].sort_values("VIF",ascending=False)
 high_vif
%%time
!pip install.. /input/iterative-stratification/iterative-stratification-master/<set_options>
y = df_train["revenue"]
df_train = df_train.drop("revenue",axis=1)
 df_all = pd.concat([df_train,df_test],axis=0)
 df_all = df_all.drop("Id",axis=1)
 df_all.shape
warnings.resetwarnings()
warnings.simplefilter('ignore', FutureWarning)
 warnings.simplefilter('ignore', DeprecationWarning)
 os.environ["CUDA_VISIBLE_DEVICES"]="0"<define_variables>
def missing_value(df):

 number = df.isnull().sum().sort_values(ascending=False)
 number = number[number > 0]
percentage = df.isnull().sum() *100 /df.shape[0]
percentage = percentage[percentage > 0].sort_values(ascending=False)
 return pd.concat([number,percentage],axis=1,keys=["Total","Percentage"])
 missing_value(df_all )
def generate_virtual_private(test_feat, smpl_sub, rate_to_pub=4):
assert(test_feat.sig_id == smpl_sub.sig_id ).all()
test_feat_list = [test_feat]
smpl_sub_list = [smpl_sub]
for i in range(1, rate_to_pub):
tmp_test_feat = test_feat.copy()
tmp_smpl_sub = smpl_sub.copy()
tmp_test_feat.sig_id = tmp_test_feat.sig_id + f"_{i}"
tmp_smpl_sub.sig_id = tmp_smpl_sub.sig_id + f"_{i}"
assert(tmp_test_feat.sig_id == tmp_smpl_sub.sig_id ).all()
tmp_test_feat.iloc[:, 4:] += i * 10
assert(tmp_test_feat.iloc[:, 4:] != test_feat.iloc[:, 4:] ).all().all()
test_feat_list.append(tmp_test_feat)
 smpl_sub_list.append(tmp_smpl_sub)
 test_feat_concat = pd.concat(test_feat_list, axis=0, ignore_index=True)
 smpl_sub_concat = pd.concat(smpl_sub_list, axis=0, ignore_index=True)
 return test_feat_concat, smpl_sub_concat<define_variables>
df_all["OpenMonth"] = df_all["Open Date"].transform(lambda x: int(x.split("/")[0]))
df_all["OpenYear"] = df_all["Open Date"].transform(lambda x: int(x.split("/")[-1]))
df_all["Open Day"] = df_all["Open Date"].transform(lambda x: int(x.split("/")[1]))
df_all = df_all.drop(["Open Date"],axis=1 )
MODEL_NAMES = ["NN(drugCV)", "TabNet", "ResNet", "ThrNN", "ThrNN(drugCV)"]<load_pretrained>
bias_feat = []
for feat in df_all.columns:
counts = df_all[feat].value_counts().iloc[0]
if counts*100 / len(df_all)>99.94:
bias_feat.append(feat)
 bias_feat
%%time
if DO_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/nn-inference-0.01833.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/virtual-nn-use-train-public-inference.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError<load_pretrained>
df_all = pd.get_dummies(df_all ).reset_index(drop=True )
%%time
if DO_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/tabnet-inference-0.01840.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/virtual-tabnet-inference-add-param-n-shared.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError<install_modules>
X_train, X_test, y_train, y_test = train_test_split(df_train,y,test_size=0.33,random_state=42)
 print("Shapes: ", X_train.shape, X_test.shape, y_train.shape, y_test.shape )
%%time
if DO_SUBMISSION:
!python.. /input/moa-takapy-script/tf-rn-transfer-1layerother-selcol100.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-takapy-script/virtual-tf-rn-transfer-1layerother-selcol100.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError<install_modules>
best_estimators = []
%%time
if DO_SUBMISSION:
!python.. /input/moa-tawara-scripts-for-final-submission/moa-for-final-thrnn-seed-cv-0.01836.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-tawara-scripts-for-final-submission/virtual-moa-for-final-thrnn-seed-cv-0.01836.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError<define_variables>
from sklearn.model_selection import GridSearchCV
from sklearn.linear_model import Lasso, Ridge, ElasticNet, HuberRegressor, BayesianRidge
from sklearn.ensemble import RandomForestRegressor, BaggingRegressor, VotingRegressor, AdaBoostRegressor, GradientBoostingRegressor, ExtraTreesRegressor
from sklearn.tree import DecisionTreeRegressor
from xgboost import XGBRegressor
from sklearn.neighbors import KNeighborsRegressor
import lightgbm as lgbm
TARGET_COL = ['5-alpha_reductase_inhibitor', '11-beta-hsd1_inhibitor', 'acat_inhibitor', 'acetylcholine_receptor_agonist', 'acetylcholine_receptor_antagonist', 'acetylcholinesterase_inhibitor', 'adenosine_receptor_agonist', 'adenosine_receptor_antagonist', 'adenylyl_cyclase_activator', 'adrenergic_receptor_agonist', 'adrenergic_receptor_antagonist', 'akt_inhibitor', 'aldehyde_dehydrogenase_inhibitor', 'alk_inhibitor', 'ampk_activator', 'analgesic', 'androgen_receptor_agonist', 'androgen_receptor_antagonist', 'anesthetic_-_local', 'angiogenesis_inhibitor', 'angiotensin_receptor_antagonist', 'anti-inflammatory', 'antiarrhythmic', 'antibiotic', 'anticonvulsant', 'antifungal', 'antihistamine', 'antimalarial', 'antioxidant', 'antiprotozoal', 'antiviral', 'apoptosis_stimulant', 'aromatase_inhibitor', 'atm_kinase_inhibitor', 'atp-sensitive_potassium_channel_antagonist', 'atp_synthase_inhibitor', 'atpase_inhibitor', 'atr_kinase_inhibitor', 'aurora_kinase_inhibitor', 'autotaxin_inhibitor', 'bacterial_30s_ribosomal_subunit_inhibitor', 'bacterial_50s_ribosomal_subunit_inhibitor', 'bacterial_antifolate', 'bacterial_cell_wall_synthesis_inhibitor', 'bacterial_dna_gyrase_inhibitor', 'bacterial_dna_inhibitor', 'bacterial_membrane_integrity_inhibitor', 'bcl_inhibitor', 'bcr-abl_inhibitor', 'benzodiazepine_receptor_agonist', 'beta_amyloid_inhibitor', 'bromodomain_inhibitor', 'btk_inhibitor', 'calcineurin_inhibitor', 'calcium_channel_blocker', 'cannabinoid_receptor_agonist', 'cannabinoid_receptor_antagonist', 'carbonic_anhydrase_inhibitor', 'casein_kinase_inhibitor', 'caspase_activator', 'catechol_o_methyltransferase_inhibitor', 'cc_chemokine_receptor_antagonist', 'cck_receptor_antagonist', 'cdk_inhibitor', 'chelating_agent', 'chk_inhibitor', 'chloride_channel_blocker', 'cholesterol_inhibitor', 'cholinergic_receptor_antagonist', 'coagulation_factor_inhibitor', 'corticosteroid_agonist', 'cyclooxygenase_inhibitor', 'cytochrome_p450_inhibitor', 'dihydrofolate_reductase_inhibitor', 'dipeptidyl_peptidase_inhibitor', 'diuretic', 'dna_alkylating_agent', 'dna_inhibitor', 'dopamine_receptor_agonist', 'dopamine_receptor_antagonist', 'egfr_inhibitor', 'elastase_inhibitor', 'erbb2_inhibitor', 'estrogen_receptor_agonist', 'estrogen_receptor_antagonist', 'faah_inhibitor', 'farnesyltransferase_inhibitor', 'fatty_acid_receptor_agonist', 'fgfr_inhibitor', 'flt3_inhibitor', 'focal_adhesion_kinase_inhibitor', 'free_radical_scavenger', 'fungal_squalene_epoxidase_inhibitor', 'gaba_receptor_agonist', 'gaba_receptor_antagonist', 'gamma_secretase_inhibitor', 'glucocorticoid_receptor_agonist', 'glutamate_inhibitor', 'glutamate_receptor_agonist', 'glutamate_receptor_antagonist', 'gonadotropin_receptor_agonist', 'gsk_inhibitor', 'hcv_inhibitor', 'hdac_inhibitor', 'histamine_receptor_agonist', 'histamine_receptor_antagonist', 'histone_lysine_demethylase_inhibitor', 'histone_lysine_methyltransferase_inhibitor', 'hiv_inhibitor', 'hmgcr_inhibitor', 'hsp_inhibitor', 'igf-1_inhibitor', 'ikk_inhibitor', 'imidazoline_receptor_agonist', 'immunosuppressant', 'insulin_secretagogue', 'insulin_sensitizer', 'integrin_inhibitor', 'jak_inhibitor', 'kit_inhibitor', 'laxative', 'leukotriene_inhibitor', 'leukotriene_receptor_antagonist', 'lipase_inhibitor', 'lipoxygenase_inhibitor', 'lxr_agonist', 'mdm_inhibitor', 'mek_inhibitor', 'membrane_integrity_inhibitor', 'mineralocorticoid_receptor_antagonist', 'monoacylglycerol_lipase_inhibitor', 'monoamine_oxidase_inhibitor', 'monopolar_spindle_1_kinase_inhibitor', 'mtor_inhibitor', 'mucolytic_agent', 'neuropeptide_receptor_antagonist', 'nfkb_inhibitor', 'nicotinic_receptor_agonist', 'nitric_oxide_donor', 'nitric_oxide_production_inhibitor', 'nitric_oxide_synthase_inhibitor', 'norepinephrine_reuptake_inhibitor', 'nrf2_activator', 'opioid_receptor_agonist', 'opioid_receptor_antagonist', 'orexin_receptor_antagonist', 'p38_mapk_inhibitor', 'p-glycoprotein_inhibitor', 'parp_inhibitor', 'pdgfr_inhibitor', 'pdk_inhibitor', 'phosphodiesterase_inhibitor', 'phospholipase_inhibitor', 'pi3k_inhibitor', 'pkc_inhibitor', 'potassium_channel_activator', 'potassium_channel_antagonist', 'ppar_receptor_agonist', 'ppar_receptor_antagonist', 'progesterone_receptor_agonist', 'progesterone_receptor_antagonist', 'prostaglandin_inhibitor', 'prostanoid_receptor_antagonist', 'proteasome_inhibitor', 'protein_kinase_inhibitor', 'protein_phosphatase_inhibitor', 'protein_synthesis_inhibitor', 'protein_tyrosine_kinase_inhibitor', 'radiopaque_medium', 'raf_inhibitor', 'ras_gtpase_inhibitor', 'retinoid_receptor_agonist', 'retinoid_receptor_antagonist', 'rho_associated_kinase_inhibitor', 'ribonucleoside_reductase_inhibitor', 'rna_polymerase_inhibitor', 'serotonin_receptor_agonist', 'serotonin_receptor_antagonist', 'serotonin_reuptake_inhibitor', 'sigma_receptor_agonist', 'sigma_receptor_antagonist', 'smoothened_receptor_antagonist', 'sodium_channel_inhibitor', 'sphingosine_receptor_agonist', 'src_inhibitor', 'steroid', 'syk_inhibitor', 'tachykinin_antagonist', 'tgf-beta_receptor_inhibitor', 'thrombin_inhibitor', 'thymidylate_synthase_inhibitor', 'tlr_agonist', 'tlr_antagonist', 'tnf_inhibitor', 'topoisomerase_inhibitor', 'transient_receptor_potential_channel_antagonist', 'tropomyosin_receptor_kinase_inhibitor', 'trpv_agonist', 'trpv_antagonist', 'tubulin_inhibitor', 'tyrosine_kinase_inhibitor', 'ubiquitin_specific_protease_inhibitor', 'vegfr_inhibitor', 'vitamin_b', 'vitamin_d_receptor_agonist', 'wnt_inhibitor']<load_from_csv>
params = {
"alpha" : [.01,.1,.5,.7,.9,.95,.99, 1, 5, 10, 20],
"fit_intercept" : [True, False],
"normalize" : [True,False],
"solver" : ['svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga'],
"tol" : [0.0001, 0.001, 0.01, 0.1],
"random_state" : [42]
}
ridge = Ridge()
ridge_grid = GridSearchCV(ridge, params, scoring='r2', cv=7, n_jobs=-1)
 ridge_grid.fit(X_train, y_train)
 print("Best parameters: {}:".format(ridge_grid.best_params_))
print("Best score: {}".format(ridge_grid.best_score_))
best_estimators.append(["Ridge",ridge_grid.best_estimator_] )
def del_control(df: pd.DataFrame())-> pd.DataFrame() :
train = pd.read_csv('.. /input/lish-moa/train_features.csv')
 df['cp_type'] = train['cp_type']
df = df.loc[df['cp_type']=='trt_cp'].reset_index(drop=True)
 df = df.drop('cp_type', axis=1)
 return df
def drop_sig_id(df: pd.DataFrame())-> pd.DataFrame() :
return df.drop('sig_id', axis=1)
 def fix_oof(df: pd.DataFrame())-> pd.DataFrame() :
if df.shape[0] != 21948:
df = del_control(df)
 if 'sig_id' in df.columns.tolist() :
df = drop_sig_id(df)
 return df<load_from_csv>
params = {
'alpha' : [.01,.1,.5,.7,.9,.95,.99, 1, 5, 10, 20],
'fit_intercept' : [True, False],
'normalize' : [True,False],
'tol' : [0.0001, 0.001, 0.01, 0.1],
"random_state" : [42]
}
lasso = Lasso()
lasso_grid = GridSearchCV(lasso, params, scoring='r2', cv=7, n_jobs=-1)
 lasso_grid.fit(X_train, y_train)
 print("Best parameters: {}:".format(lasso_grid.best_params_))
print("Best score: {}".format(lasso_grid.best_score_))
best_estimators.append(["Lasso",lasso_grid.best_estimator_] )
oof_list = []
oof_list.append(fix_oof(
 pd.read_pickle('.. /input/nn-use-train-public/oof.pkl')))
 oof_list.append(fix_oof(
 pd.DataFrame(
 np.load('.. /input/tabnet-train-public-add-n-shared-1/oof_tabnet.npy'),
columns=TARGET_COL)) 
)
 oof_list.append(fix_oof(
 pd.read_csv(".. /input/moa-takapy-tf-resnet-transfer/oof_prediction_takapy_resnet.csv")))
 oof_list.append(fix_oof(
 pd.read_csv(".. /input/moa-weight-thrnn-seed-cv/oof_prediction.csv")))
 oof_list.append(fix_oof(
 pd.read_csv(".. /input/moa-weight-thrnn-drug-seed-cv/oof_prediction.csv")) )<load_from_csv>
params = {
"alpha" : [.01,.1,.5,.7,.9,.95,.99, 1, 5, 10, 20],
"fit_intercept" : [True, False],
"normalize" : [True,False],
"tol" : [0.0001, 0.001, 0.01, 0.1],
"random_state" : [42]
}
EL = ElasticNet()
EL_grid = GridSearchCV(EL, params, scoring='r2', cv=7, n_jobs=-1)
 EL_grid.fit(X_train, y_train)
 print("Best parameters: {}:".format(EL_grid.best_params_))
print("Best score: {}".format(EL_grid.best_score_))
best_estimators.append(["ElasticNet",EL_grid.best_estimator_] )
y_true = fix_oof(pd.read_csv('.. /input/lish-moa/train_targets_scored.csv'))
for i, name in enumerate(MODEL_NAMES):
print(f"[{name}]")
 print("shape:", oof_list[i].shape)
 print("local logloss:", log_loss(y_true.values.ravel() , oof_list[i].values.ravel()))<categorify>
params = {
"alpha" : [.01,.1,.5,.7,.9,.95,.99, 1, 5, 10, 20],
"fit_intercept" : [True, False],
"tol" : [0.0001, 0.001, 0.01, 0.1],
"max_iter": [100, 300 , 500]
}
Huber_R = HuberRegressor()
Huber_R_grid = GridSearchCV(Huber_R, params, scoring='r2', cv=7, n_jobs=-1)
 Huber_R_grid.fit(X_train, y_train)
 print("Best parameters: {}:".format(Huber_R_grid.best_params_))
print("Best score: {}".format(Huber_R_grid.best_score_))
best_estimators.append(["HuberRegressor",Huber_R_grid.best_estimator_] )
class MoAStackingDataset(data.Dataset):
def __init__(self, feat: np.ndarray, label: np.ndarray = None):

 self.feat = feat
if label is None:
self.label = np.full(( len(feat), 1), -1)
 else:
self.label = label
self.model_order = None
def __len__(self):

 return len(self.feat)
 def __getitem__(self, index: int):

 return [
torch.from_numpy(self.feat[index] ).float() ,
torch.from_numpy(self.label[index] ).float()
]
def reset_model_order(self):

 pass
def shuffle_model_order(self, seed):

 pass
class MoAStackingDatasetForCNN(data.Dataset):
def __init__(self, feat: np.ndarray, label: np.ndarray = None):

 self.feat = feat
if label is None:
self.label = np.full(( len(feat), 1), -1)
 else:
self.label = label
self.reset_model_order()
def reset_model_order(self):
self.model_order = np.arange(self.feat.shape[-1])
 def shuffle_model_order(self, seed):
np.random.seed(seed)
 self.model_order = np.random.permutation(self.model_order)
 def __len__(self):

 return len(self.feat)
 def __getitem__(self, index: int):

 return [
torch.from_numpy(self.feat[index][..., self.model_order] ).float() ,
torch.from_numpy(self.label[index] ).float()
]
class MoAStackingDatasetForGCN(data.Dataset):
def __init__(self, feat: np.ndarray, label: np.ndarray = None):

 self.feat = feat
if label is None:
self.label = np.full(( len(feat), 1), -1)
 else:
self.label = label
self.model_order = None
def reset_model_order(self):

 pass
def shuffle_model_order(self, seed):

 pass
def __len__(self):

 return len(self.feat)
 def __getitem__(self, index: int):

 return [
torch.from_numpy(self.feat[index] ).float() ,
torch.from_numpy(self.label[index] ).float()
]<define_search_model>
params = {
"max_depth": ["None",10, 30, 50, 75, 100],
"max_features": ["auto",0.3, 0.6],
"min_samples_leaf": [1,3,5,7],
"min_samples_split": [2, 4, 8, 12],
"n_estimators": [30, 50, 100, 200],
"random_state" : [42]
}
RFR = RandomForestRegressor()
RFR_grid = GridSearchCV(RFR, params, scoring='r2', cv=7, n_jobs=-1)
 RFR_grid.fit(X_train, y_train)
 print("Best parameters: {}:".format(RFR_grid.best_params_))
print("Best score: {}".format(RFR_grid.best_score_))
best_estimators.append(["RandomForestR",RFR_grid.best_estimator_] )
def get_activation(activ_name: str="relu"):

 act_dict = {
"relu": nn.ReLU() ,
"tanh": nn.Tanh() ,
"sigmoid": nn.Sigmoid() ,
"identity": nn.Identity() }
if activ_name in act_dict:
return act_dict[activ_name]
elif re.match(r"^htanh\_\d{4}$", activ_name):
bound = int(activ_name[-4:])/ 1000
return nn.Hardtanh(-bound, bound)
 else:
raise NotImplementedError
class LBAD(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(LBAD, self ).__init__()
layers = [nn.Linear(in_features, out_features)]
if use_wn:
layers[0] = nn.utils.weight_norm(layers[0])
 if use_bn:
layers.append(nn.BatchNorm1d(out_features))
layers.append(get_activation(activ))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class BDLA(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(BDLA, self ).__init__()
layers = []
if use_bn:
layers.append(nn.BatchNorm1d(in_features))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
layers.append(nn.Linear(in_features, out_features))
if use_wn:
layers[-1] = nn.utils.weight_norm(layers[-1])
 layers.append(get_activation(activ))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class LABD(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(LABD, self ).__init__()
layers = [nn.Linear(in_features, out_features), get_activation(activ)]
if use_wn:
layers[0] = nn.utils.weight_norm(layers[0])
 if use_bn:
layers.append(nn.BatchNorm1d(out_features))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x )<prepare_x_and_y>
params = {
"max_depth": ["None",10, 30, 50, 75, 100],
"max_features": ["auto",.3,.4,.5,.6],
"min_samples_leaf": [1,3,5,7],
"min_samples_split": [2, 4, 8, 12],
"n_estimators": [30, 50, 100, 200],
"random_state" : [42]
}
ETR = ExtraTreesRegressor()
ETR_grid = GridSearchCV(ETR, params, scoring='r2', cv=7, n_jobs=-1)
 ETR_grid.fit(X_train, y_train)
 print("Best parameters: {}:".format(ETR_grid.best_params_))
print("Best score: {}".format(ETR_grid.best_score_))
best_estimators.append(["ExtraTreesR",ETR_grid.best_estimator_] )
def vector_wise_matmul(X: torch.Tensor, W: torch.Tensor)-> torch.Tensor:

 X = torch.transpose(X, 0, 1)
 W = torch.transpose(W, 1, 2)
 H = torch.matmul(X, W)
 H = torch.transpose(H, 0, 1)
 return H
def vector_wise_shared_matmul(X: torch.Tensor, W: torch.Tensor)-> torch.Tensor:

 H = nn.functional.linear(X, W)
 return H<normalization>
params = {
"max_features": [0.2, 0.4, 0.6,1.0],
"n_estimators": [5, 10, 15, 20],
"random_state": [42]
}
BR = BaggingRegressor()
BR_grid = GridSearchCV(BR, params, scoring='r2', cv=7, n_jobs=-1)
 BR_grid.fit(X_train, y_train)
 print("Best parameters: {}:".format(BR_grid.best_params_))
print("Best score: {}".format(BR_grid.best_score_))
best_estimators.append(["BaggingRegressorR",BR_grid.best_estimator_] )
def _calculate_fan_in_and_fan_out_for_vwl(tensor)-> tp.Tuple[int]:

 dimensions = tensor.dim()
if dimensions < 2:
raise ValueError("Fan in and fan out can not be computed for tensor with fewer than 2 dimensions")
 fan_in = tensor.size(-1)
 fan_out = tensor.size(-2)
 return fan_in, fan_out
def _calculate_correct_fan_for_vwl(tensor, mode)-> int:

 mode = mode.lower()
valid_modes = ['fan_in', 'fan_out']
if mode not in valid_modes:
raise ValueError("Mode {} not supported, please use one of {}".format(mode, valid_modes))
fan_in, fan_out = _calculate_fan_in_and_fan_out_for_vwl(tensor)
 return fan_in if mode == 'fan_in' else fan_out
def kaiming_uniform_for_vwl(tensor, a=0, mode='fan_in', nonlinearity='leaky_relu'):

 fan = _calculate_correct_fan_for_vwl(tensor, mode)
 gain = nn.init.calculate_gain(nonlinearity, a)
 std = gain / np.sqrt(fan)
 bound = np.sqrt(3.0)* std
with torch.no_grad() :
return tensor.uniform_(-bound, bound )<prepare_x_and_y>
params = {
"learning_rate": [.1,.5,.7,.9,.95,.99, 1],
"colsample_bytree": [.3,.4,.5,.6],
"max_depth": [2, 4],
"alpha": [1, 3, 5],
"subsample": [.5],
"n_estimators": [30, 70, 100, 200],
"random_state" : [42]
}
XGBR = XGBRegressor()
XGBR_grid = GridSearchCV(XGBR, params, scoring='r2', cv=7, n_jobs=-1)
 XGBR_grid.fit(X_train, y_train)
 print("Best parameters: {}:".format(XGBR_grid.best_params_))
print("Best score: {}".format(XGBR_grid.best_score_))
best_estimators.append(["XGBoostR",XGBR_grid.best_estimator_] )
class VectorWiseLinear(nn.Module):

 def __init__(
 self,
in_features: int, out_features: int, n_vectors: int,
bias: bool=True, weight_shared: bool=True 
)-> None:

 super(VectorWiseLinear, self ).__init__()
self.in_features = in_features
self.out_features = out_features
self.n_vectors = n_vectors
self.weight_shared = weight_shared
if self.weight_shared:
self.weight = nn.Parameter(
 torch.Tensor(self.out_features, self.in_features))
self.matmul_func = vector_wise_shared_matmul
else:
self.weight = nn.Parameter(
 torch.Tensor(self.n_vectors, self.out_features, self.in_features))
self.matmul_func = vector_wise_matmul
if bias:
self.bias = nn.Parameter(torch.Tensor(out_features))
else:
self.register_parameter('bias', None)
 self.reset_parameters()
def reset_parameters(self)-> None:

 kaiming_uniform_for_vwl(self.weight, a=np.sqrt(5))
if self.bias is not None:
fan_in, _ = _calculate_fan_in_and_fan_out_for_vwl(self.weight)
 bound = 1 / np.sqrt(fan_in)
 nn.init.uniform_(self.bias, -bound, bound)
 def forward(self, X: torch.Tensor)-> torch.Tensor:

 H = self.matmul_func(X, self.weight)
 if self.bias is not None:
H = H + self.bias
return H<import_modules>
params = {
"max_depth": ["None",10, 40, 80],
"max_features": ["auto","sqrt","log2"],
"min_samples_leaf": [1,3,5,7],
"min_samples_split": [2, 6, 12],
"random_state" : [42],
"splitter" : ["best","random"]
}
DTR = DecisionTreeRegressor()
DTR_grid = GridSearchCV(DTR, params, scoring='r2', cv=7, n_jobs=-1)
 DTR_grid.fit(X_train, y_train)
 print("Best parameters: {}:".format(DTR_grid.best_params_))
print("Best score: {}".format(DTR_grid.best_score_))
best_estimators.append(["DecisionTreeR",DTR_grid.best_estimator_] )
class GraphConv(nn.Module):

 def __init__(
 self,
in_channels: int, out_channels: int, n_nodes: int, shrare_msg: bool=True,
model_self: bool=True, share_model_self: bool=True,
bias: bool=True, share_bias: bool=True 
)-> None:

 self.in_channels = in_channels
self.out_channels = out_channels
self.n_nodes = n_nodes
self.model_self = model_self
super(GraphConv, self ).__init__()
self.msg = VectorWiseLinear(
 in_channels, out_channels, n_nodes, False, shrare_msg)
 if model_self:
self.model_self = VectorWiseLinear(
 in_channels, out_channels, n_nodes, False, share_model_self)
 if bias:
if share_bias:
self.bias = nn.Parameter(torch.Tensor(out_channels))
else:
self.bias = nn.Parameter(torch.Tensor(n_nodes, out_channels))
bound = 1 / np.sqrt(out_channels)
 nn.init.uniform_(self.bias, -bound, bound)
 else:
self.register_parameter('bias', None)
 def forward(self, X: torch.Tensor, A: torch.Tensor, W: torch.Tensor=None)-> torch.Tensor:

 if W is not None:
A = A * W
M = X
M = self.msg(M)
 M = torch.matmul(A, M)
 H = M
if self.model_self:
H = H + self.model_self(X)
 if self.bias is not None:
H = H + self.bias
return H<choose_model_class>
params = {
"n_estimators": [10, 30, 50, 100],
"learning_rate": [.01, 0.1, 0.5, 0.9, 0.95, 1],
"random_state" : [42]
}
AdaBoostR = AdaBoostRegressor()
AdaBoostR_grid = GridSearchCV(AdaBoostR, params, scoring='r2', cv=7, n_jobs=-1)
 AdaBoostR_grid.fit(X_train, y_train)
 print("Best parameters: {}:".format(AdaBoostR_grid.best_params_))
print("Best score: {}".format(AdaBoostR_grid.best_score_))
best_estimators.append(["AdaBoostR",AdaBoostR_grid.best_estimator_] )
class MLP(nn.Module):

 def __init__(
 self, n_features_list: tp.List[int], use_tail_as_out: bool=False,
drop_rate: float=0.0, use_bn: bool=False, use_wn: bool=False,
activ:str="relu", block_name: str="LBAD" 
):

 super(MLP, self ).__init__()
n_layers = len(n_features_list)- 1
block_class = {
"LBAD": LBAD, "BDLA": BDLA, "LABD": LABD}[block_name]
layers = []
for i in range(n_layers):
in_feats, out_feats = n_features_list[i: i + 2]
if i == n_layers - 1 and use_tail_as_out:
if block_name in ["BDLA"]:
layer = block_class(in_feats, out_feats, drop_rate, use_bn, use_wn, "identity")
 else:
layer = nn.Linear(in_feats, out_feats)
 if use_wn:
layer = nn.utils.weight_norm(layer)
 else:
layer = block_class(in_feats, out_feats, drop_rate, use_bn, use_wn, activ)
 layers.append(layer)
 self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class CNNStacking1d(nn.Module):

 def __init__(
 self, n_models: int,
n_channels_list: tp.List[int], use_bias: bool=False,
kwargs_head: tp.Dict={}, 
):

 super(CNNStacking1d, self ).__init__()
self.n_conv_layers = len(n_channels_list)- 1
for i in range(self.n_conv_layers):
in_ch = n_channels_list[i]
out_ch = n_channels_list[i + 1]
layer = nn.Sequential(
 nn.Conv1d(
 in_ch, out_ch, kernel_size=3, stride=1, padding=0, bias=use_bias),
nn.ReLU(inplace=True))
setattr(self, "conv{}".format(i + 1), layer)
 kwargs_head["n_features_list"][0] =(n_models - 2 * self.n_conv_layers)* n_channels_list[-1]
self.head = MLP(**kwargs_head)
 def forward(self, x: torch.FloatTensor)-> torch.Tensor:

 bs = x.shape[0]
h = x
for i in range(self.n_conv_layers):
h = getattr(self, "conv{}".format(i + 1))(h)
 h = torch.reshape(h,(bs, -1))
h = self.head(h)
 return h
class CNNStacking2d(nn.Module):

 def __init__(
 self, n_models: int, n_classes: int,
n_channels_list: tp.List[int], use_bias: bool=False,
kwargs_head: tp.Dict={}, 
):

 super(CNNStacking2d, self ).__init__()
self.n_conv_layers = len(n_channels_list)- 1
for i in range(self.n_conv_layers):
in_ch = n_channels_list[i]
out_ch = n_channels_list[i + 1]
layer = nn.Sequential(
 nn.Conv2d(
 in_ch, out_ch, kernel_size=(1, 3), stride=1, padding=0, bias=use_bias),
nn.ReLU(inplace=True))
setattr(self, "conv{}".format(i + 1), layer)
 kwargs_head["n_features_list"][0] =(n_models - 2 * self.n_conv_layers)* n_classes * n_channels_list[-1]
self.head = MLP(**kwargs_head)
 def forward(self, x: torch.FloatTensor)-> torch.Tensor:

 bs = x.shape[0]
h = x
for i in range(self.n_conv_layers):
h = getattr(self, "conv{}".format(i + 1))(h)
 h = torch.reshape(h,(bs, -1))
h = self.head(h)
 return h
class GCNStacking(nn.Module):

 def __init__(
 self, n_classes: int,
n_channels_list: tp.List[int],
add_self_loop: bool=False,
kwargs_head: tp.Dict={}, 
):

 super(GCNStacking, self ).__init__()
self.n_conv_layers = len(n_channels_list)- 1
for i in range(self.n_conv_layers):
in_ch = n_channels_list[i]
out_ch = n_channels_list[i + 1]
layer = GraphConv(
 in_ch, out_ch, n_classes,
shrare_msg=False, share_model_self=False, share_bias=False)
 setattr(self, "conv{}".format(i + 1), layer)
 self.relu = nn.ReLU(inplace=True)
 if add_self_loop:
adj_mat = torch.ones(n_classes, n_classes)/ n_classes
else:
adj_mat =(1 - torch.eye(n_classes, n_classes)) /(n_classes - 1)
 self.register_buffer("A", adj_mat.float())
 kwargs_head["n_features_list"][0] = n_classes * n_channels_list[-1]
self.head = MLP(**kwargs_head)
 def forward(self, X: torch.FloatTensor)-> torch.Tensor:

 bs, n_classes = X.shape[:2]
H = X
for i in range(self.n_conv_layers):
H = getattr(self, "conv{}".format(i + 1))(H, self.A[None,...])
 H = self.relu(H)
 h = torch.reshape(H,(bs, -1))
h = self.head(h)
 return h<init_hyperparams>
params = {
"max_depth": [2, 3, 6, 10],
"max_features": ["auto",0.3, 0.6],
"min_samples_leaf": [1,3],
"min_samples_split": [2, 5],
"n_estimators": [30, 50, 100, 200],
"random_state" : [42],
"tol" : [0.0001, 0.001, 0.01, 0.1]
}
GBR = GradientBoostingRegressor()
GBR_grid = GridSearchCV(GBR, params, scoring='r2', cv=7, n_jobs=-1)
 GBR_grid.fit(X_train, y_train)
 print("Best parameters: {}:".format(GBR_grid.best_params_))
print("Best score: {}".format(GBR_grid.best_score_))
best_estimators.append(["GradientBoostR",GBR_grid.best_estimator_] )
class EvalFuncManager(nn.Module):

 def __init__(
 self, iters_per_epoch: int,
evalfunc_dict: tp.Dict[str, nn.Module], prefix: str = "val" 
)-> None:

 super(EvalFuncManager, self ).__init__()
self.tmp_iter = 0
self.iters_per_epoch = iters_per_epoch
self.prefix = prefix
self.metric_names = []
for k, v in evalfunc_dict.items() :
setattr(self, k, v)
 self.metric_names.append(k)
 self.reset()
def reset(self)-> None:

 self.tmp_iter = 0
for name in self.metric_names:
getattr(self, name ).reset()
def __call__(self, y: torch.Tensor, t: torch.Tensor)-> None:

 for name in self.metric_names:
getattr(self, name ).update(y, t)
 self.tmp_iter += 1
if self.tmp_iter == self.iters_per_epoch:
ppe.reporting.report({
"{}/{}".format(self.prefix, name): getattr(self, name ).compute()
for name in self.metric_names})
 self.reset()
class MeanLoss(nn.Module):
def __init__(self):
super(MeanLoss, self ).__init__()
self.loss_sum = 0
self.n_examples = 0
def forward(self, y: torch.Tensor, t: torch.Tensor):

 return self.loss_func(y, t)
 def reset(self):

 self.loss_sum = 0
self.n_examples = 0
def update(self, y: torch.Tensor, t: torch.Tensor):

 self.loss_sum += self(y, t ).item() * y.shape[0]
self.n_examples += y.shape[0]
def compute(self):

 return self.loss_sum / self.n_examples
class MyLogLoss(MeanLoss):
def __init__(self, **params):
super(MyLogLoss, self ).__init__()
self.loss_func = nn.BCEWithLogitsLoss(**params)
 class LSBCEWithLogitsLoss(nn.Module):

 def __init__(self, k: int, alpha: float=0.01):

 super(LSBCEWithLogitsLoss, self ).__init__()
self.k = k
self.alpha = alpha
self.loss_func = nn.BCEWithLogitsLoss()
def forward(self, y, t):

 t_s = t *(1 - self.alpha)+ self.alpha / self.k
loss = self.loss_func(y, t_s)
 return loss
class MyLSLogLoss(MeanLoss):
def __init__(self, **params):
super(MyLSLogLoss, self ).__init__()
self.loss_func = LSBCEWithLogitsLoss(**params )<train_model>
evaluate = pd.DataFrame(columns=["model","cv","std"])
 for name,model in pipelines:
kfold = KFold(n_splits=10,random_state=42)
 cv = cross_val_score(model, X_train, y_train, cv=kfold, n_jobs=-1, scoring="neg_root_mean_squared_error")
 row = evaluate.shape[0]
evaluate.loc[row,"model"] = name
evaluate.loc[row,"cv"] = round(cv.mean() ,3)
 evaluate.loc[row,"std"] = "+/- {}".format(round(cv.std() ,4))
evaluate = evaluate.sort_values("cv",ascending=False)
 evaluate
def run_train_loop(
 manager, args, model, device,
train_loader, optimizer, scheduler, loss_func 
):

 while not manager.stop_trigger:
model.train()
for batch in train_loader:
x, t = batch
with manager.run_iteration() :
optimizer.zero_grad()
y = model(x.to(device))
loss = loss_func(y, t.to(device))
ppe.reporting.report({'train/loss': loss.item() })
 loss.backward()
optimizer.step()
if scheduler is not None:
scheduler.step()
def run_eval(args, model, device, batch, eval_manager):

 model.eval()
x, t = batch
y = model(x.to(device))
eval_manager(y, t.to(device))<choose_model_class>
votings = []
votings.append(["Scaled_RFR",RFR_grid.best_estimator_])
 votings.append(( "Scaled_all_models",Pipeline([("Scaler",StandardScaler()), 
("Votings",VotingRegressor([("RFR",RFR_grid.best_estimator_), 
("AdaBoostR", AdaBoostR_grid.best_estimator_), 
("BaggingR",BR_grid.best_estimator_), 
("GBR",GBR_grid.best_estimator_)
 ])
)])))
 votings.append(( "Scaled_RFR_BR_GBR",Pipeline([("Scaler",StandardScaler()), 
("Votings",VotingRegressor([("RFR",RFR_grid.best_estimator_), 
("BaggingR",BR_grid.best_estimator_), 
("GBR",GBR_grid.best_estimator_)
 ])
)])))
 votings.append(( "Scaled_RFR_BR_AdaBR",Pipeline([("Scaler",StandardScaler()), 
("Votings",VotingRegressor([("RFR",RFR_grid.best_estimator_), 
("BaggingR",BR_grid.best_estimator_), 
("AdaBoostR", AdaBoostR_grid.best_estimator_)
 ])
)])))
 votings.append(( "Scaled_AdaBR_GBR",Pipeline([("Scaler",StandardScaler()), 
("Votings",VotingRegressor([("RFR",RFR_grid.best_estimator_), 
("AdaBoostR", AdaBoostR_grid.best_estimator_), 
("GBR",GBR_grid.best_estimator_)
 ])
)])))
 votings.append(( "Scaled_BR_GBR_AdaBR",Pipeline([("Scaler",StandardScaler()), 
("Votings",VotingRegressor([("BaggingR",BR_grid.best_estimator_), 
("GBR",GBR_grid.best_estimator_), 
("AdaBoostR", AdaBoostR_grid.best_estimator_)
 ])
)])))
 votings.append(( "Scaled_BR_GBR",Pipeline([("Scaler",StandardScaler()), 
("Votings",VotingRegressor([("BaggingR",BR_grid.best_estimator_), 
("GBR",GBR_grid.best_estimator_)
 ])
)])))
 votings.append(( "Scaled_BR_AdaBR",Pipeline([("Scaler",StandardScaler()), 
("Votings",VotingRegressor([("BaggingR",BR_grid.best_estimator_), 
("AdaBoostR", AdaBoostR_grid.best_estimator_)
 ])
)])))
 votings.append(( "Scaled_BR_RFR",Pipeline([("Scaler",StandardScaler()), 
("Votings",VotingRegressor([("BaggingR",BR_grid.best_estimator_), 
("RFR",RFR_grid.best_estimator_)
 ])
)])))
 votings.append(( "Scaled_GBR_AdaBR",Pipeline([("Scaler",StandardScaler()), 
("Votings",VotingRegressor([("GBR",GBR_grid.best_estimator_), 
("AdaBoostR", AdaBoostR_grid.best_estimator_)
 ])
)])))
 votings.append(( "Scaled_GBR_RFR",Pipeline([("Scaler",StandardScaler()), 
("Votings",VotingRegressor([("GBR",GBR_grid.best_estimator_), 
("RFR",RFR_grid.best_estimator_)
 ])
)])))
 votings.append(( "Scaled_AdaBR_RFR",Pipeline([("Scaler",StandardScaler()), 
("Votings",VotingRegressor([("AdaBoostR", AdaBoostR_grid.best_estimator_), 
("RFR",RFR_grid.best_estimator_)
 ])
)])) )
def get_optimizer(settings, model):
optimizer = getattr(torch.optim, settings["optimizer"]["name"] )(
 model.parameters() , **settings["optimizer"]["params"])
 return optimizer
def get_scheduler(settings, train_loader, optimizer):
if settings["scheduler"]["name"] is None:
scheduler = None
else:
if settings["scheduler"]["name"] == "OneCycleLR":
settings["scheduler"]["params"]["epochs"] = settings["globals"]["max_epoch"]
settings["scheduler"]["params"]["steps_per_epoch"] = len(train_loader)
 scheduler = getattr(torch.optim.lr_scheduler, settings["scheduler"]["name"] )(
 optimizer, **settings["scheduler"]["params"])
 return scheduler
def get_loss_function(settings):
if hasattr(nn, settings["loss"]["name"]):
loss_func = getattr(nn, settings["loss"]["name"] )(**settings["loss"]["params"])
 else:
loss_func = eval(settings["loss"]["name"] )(**settings["loss"]["params"])
 return loss_func
def get_manager(
 settings, model, device, train_loader, val_loader, optimizer, eval_manager, output_path 
):
trigger = ppe.training.triggers.EarlyStoppingTrigger(
 check_trigger=(1, 'epoch'), monitor='val/metric', mode="min",
patience=settings["globals"]["patience"], verbose=True,
max_trigger=(settings["globals"]["max_epoch"], 'epoch'))
manager = ppe.training.ExtensionsManager(
 model, optimizer, settings["globals"]["max_epoch"],
iters_per_epoch=len(train_loader), stop_trigger=trigger, out_dir=output_path)
 log_extentions = [
ppe_extensions.observe_lr(optimizer=optimizer),
ppe_extensions.LogReport() ,
ppe_extensions.PlotReport(["train/loss", "val/loss"], 'epoch', filename='loss.png'),
ppe_extensions.PlotReport(["lr"], 'epoch', filename='lr.png'),
ppe_extensions.PrintReport([
"epoch", "iteration", "lr", "train/loss", "val/loss", "val/metric", "elapsed_time"])
 ]
for ext in log_extentions:
manager.extend(ext)
 manager.extend(
 ppe_extensions.Evaluator(
 val_loader, model,
eval_func=lambda *batch: run_eval(settings, model, device, batch, eval_manager)) ,
trigger=(1, "epoch"))
manager.extend(
 ppe_extensions.snapshot(target=model, filename="snapshot_epoch_{.epoch}.pth"),
trigger=ppe.training.triggers.MinValueTrigger(key="val/metric", trigger=(1, 'epoch')))
 return manager<set_options>
evaluate_vote = pd.DataFrame(columns=["model","cv","std"])
 for name, model in votings:
kfold = KFold(n_splits=10,random_state=42)
 cv = cross_val_score(model,X_train,y_train, cv=kfold, scoring="neg_root_mean_squared_error",n_jobs=-1)
 row = evaluate_vote.shape[0]
evaluate_vote.loc[row,"model"] = name
evaluate_vote.loc[row,"cv"] = round(cv.mean() ,4)
 evaluate_vote.loc[row,"std"] = "+- {}".format(round(cv.std() ,5))
evaluate_vote = evaluate_vote.sort_values("cv",ascending=False)
 evaluate_vote
def set_random_seed(seed: int = 42, deterministic: bool = False):

 random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 if deterministic:
torch.backends.cudnn.deterministic = True<train_model>
best_model = Pipeline([("Scaler",StandardScaler()), 
("RFR",RFR_grid.best_estimator_)
 ])
 best_model = best_model.fit(df_train,y)
 submission = pd.read_csv("/kaggle/input/restaurant-revenue-prediction/sampleSubmission.csv")
 submission.iloc[:,1] = np.floor(np.expm1(best_model.predict(df_test)))
 submission.to_csv('submission', index=False)
 submission