Kaggle competition: Restaurant Revenue Prediction
c_model_filename = 'tabnet_classifier'
corr_train_ids, corr_val_ids = train_test_split(id_range, test_size=0.4, random_state=seed + 1)
 c_model = train_classifier(train_set.values, c_model_filename, 80, lr = 0.03, dropout = 0.5 )<predict_on_test>
warnings.filterwarnings("ignore" )
def make_prediction(test_set):
results_c = []
results_g = []
for k in range(num_folds):
c_model.load_weights(c_model_filename + '_checkpoint{:02d}.h5'.format(k))
results_c.append(c_model.predict(test_set)[0])
 result = np.mean(results_c, axis = 0)
 return result<load_from_csv>
train_path = '.. /input/restaurant-revenue-prediction/train.csv.zip'
df_train = pd.read_csv(train_path)
 df_train.head()
sample_submission = pd.read_csv(data_folder + '/sample_submission.csv')
 sample_submission.shape<predict_on_test>
test_path = '.. /input/restaurant-revenue-prediction/test.csv.zip'
df_test = pd.read_csv(test_path, index_col='Id')
 df_test.head()
submission_no_veh = ~vehicles_mask[n_train:]
sample_submission.loc[:, scored_columns]=0
results = make_prediction(test_set)
 sample_submission.loc[submission_no_veh, scored_columns]=results
sample_submission.head()<save_to_csv>
def calculate_age(born):
born = datetime.strptime(born, "%m/%d/%Y" ).date()
today = date.today()
return today.year - born.year -(( today.month, today.day)<(born.month, born.day))
df_train['Age'] = df_train['Open Date'].apply(calculate_age)
 df_test['Age'] = df_test['Open Date'].apply(calculate_age)
 df_train = df_train.drop('Open Date', axis=1)
 df_test = df_test.drop('Open Date', axis=1)
 df_train = df_train.drop('Id', axis=1)
 df_train.head()
sample_submission.to_csv('submission.csv', index=False )<set_options>
df_train[df_train['revenue'] > 10000000 ]
warnings.filterwarnings('ignore' )<install_modules>
print("--- Train set ---")
 print(df_train['City Group'].value_counts())
 print("---- Test set ----")
 print(df_test['City Group'].value_counts() )
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/ 
<load_from_csv>
print("--- Train set ---")
 print(df_train['Type'].value_counts())
 print("---- Test set ----")
 print(df_test['Type'].value_counts())
train_features = pd.read_csv(data_dir + 'train_features.csv')
 train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(data_dir + 'train_targets_nonscored.csv')
 train_drug = pd.read_csv(data_dir + 'train_drug.csv')
 test_features = pd.read_csv(data_dir + 'test_features.csv')
 sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 print('train_features: {}'.format(train_features.shape))
print('train_targets_scored: {}'.format(train_targets_scored.shape))
print('train_targets_nonscored: {}'.format(train_targets_nonscored.shape))
print('train_drug: {}'.format(train_drug.shape))
print('test_features: {}'.format(test_features.shape))
print('sample_submission: {}'.format(sample_submission.shape))<create_dataframe>
y = df_train['revenue']
df_train=df_train.drop('revenue', axis=1 )
train_features2=train_features.copy()
test_features2=test_features.copy()<define_variables>
print('There are {} cities which restaurant location have been collected.'.format(len(df_full['City'].unique())) )
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<create_dataframe>
df_full = df_full.drop('City', axis=1)
 df_full.shape
qt = QuantileTransformer(n_quantiles=100,random_state=42,output_distribution='normal')
 data = pd.concat([pd.DataFrame(train_features[GENES+CELLS]), pd.DataFrame(test_features[GENES+CELLS])])
 data2 = qt.fit_transform(data[GENES+CELLS])
 train_features[GENES+CELLS] = pd.DataFrame(data2[:train_features.shape[0]])
 test_features[GENES+CELLS] = pd.DataFrame(data2[-test_features.shape[0]:] )<set_options>
p_name = ['P'+str(i)for i in range(1,38)]
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42 )<create_dataframe>
pca_list = ['pca'+str(i)for i in range(1,30,1)]
df_full[pca_list] = PCA(n_components=29 ).fit_transform(df_full[p_name])
 df_full.drop(p_name,axis=1,inplace=True )
n_comp = 600
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]];
test2 = data2[-test_features.shape[0]:]
train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gpca), axis=1)
 test_features = pd.concat(( test_features, test_gpca), axis=1 )<create_dataframe>
df=pd.get_dummies(df_full, dtype=float )
n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_cpca), axis=1)
 test_features = pd.concat(( test_features, test_cpca), axis=1 )<create_dataframe>
numTrain=df_train.shape[0]
train = df[:numTrain]
test = df[numTrain:]
var_thresh = VarianceThreshold(0.85)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
<concatenate>
X_train, X_test, y_train, y_test = train_test_split(train,y,test_size=0.3,random_state=42)
 print("Shapes: ", X_train.shape, X_test.shape, y_train.shape, y_test.shape )
def fe_cluster(train, test, n_clusters_g = 22, n_clusters_c = 4, SEED = 42):
features_g = GENES
features_c = CELLS
def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_{kind}'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_{kind}'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features2 ,test_features2=fe_cluster(train_features2,test_features2 )<concatenate>
from sklearn.model_selection import GridSearchCV
from sklearn.linear_model import Lasso, Ridge, ElasticNet
from sklearn.ensemble import AdaBoostRegressor
from xgboost import XGBRegressor
train_pca=pd.concat(( train_gpca,train_cpca),axis=1)
 test_pca=pd.concat(( test_gpca,test_cpca),axis=1 )<categorify>
best_estimators=[]
def fe_cluster_pca(train, test,n_clusters=5,SEED = 42):
data=pd.concat([train,test],axis=0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_pca'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_pca'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_pca'])
 test = pd.get_dummies(test, columns = [f'clusters_pca'])
 return train, test
train_cluster_pca ,test_cluster_pca = fe_cluster_pca(train_pca,test_pca )<split>
params = {
"alpha" : [.01,.1,.5,.7,.9,.95,.99, 1, 5, 10, 20],
"fit_intercept" : [True, False],
"normalize" : [True,False],
"solver" : ['svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga'],
"tol" : [0.0001, 0.001, 0.01, 0.1],
"random_state" : [42]
}
ridge = Ridge()
ridge_grid = GridSearchCV(ridge, params, scoring='r2', cv=5, n_jobs=-1)
 ridge_grid.fit(X_train, y_train)
 print("Best parameters: {}:".format(ridge_grid.best_params_))
print("Best score: {}".format(ridge_grid.best_score_))
best_estimators.append(["Ridge",ridge_grid.best_estimator_] )
train_cluster_pca = train_cluster_pca.iloc[:,650:]
test_cluster_pca = test_cluster_pca.iloc[:,650:]<split>
params = {
'alpha' : [.01,.1,.5,.7,.9,.95,.99, 1, 5, 10, 20],
'fit_intercept' : [True, False],
'normalize' : [True,False],
'tol' : [0.0001, 0.001, 0.01, 0.1],
"random_state" : [42]
}
lasso = Lasso()
lasso_grid = GridSearchCV(lasso, params, scoring='r2', cv=5, n_jobs=-1)
 lasso_grid.fit(X_train, y_train)
 print("Best parameters: {}:".format(lasso_grid.best_params_))
print("Best score: {}".format(lasso_grid.best_score_))
best_estimators.append(["Lasso",lasso_grid.best_estimator_] )
train_features_cluster=train_features2.iloc[:,876:]
test_features_cluster=test_features2.iloc[:,876:]<define_variables>
params = {
"alpha" : [.01,.1,.5,.7,.9,.95,.99, 1, 5, 10, 20],
"fit_intercept" : [True, False],
"normalize" : [True,False],
"tol" : [0.0001, 0.001, 0.01, 0.1],
"random_state" : [42]
}
EL = ElasticNet()
EL_grid = GridSearchCV(EL, params, scoring='r2', cv=5, n_jobs=-1)
 EL_grid.fit(X_train, y_train)
 print("Best parameters: {}:".format(EL_grid.best_params_))
print("Best score: {}".format(EL_grid.best_score_))
best_estimators.append(["ElasticNet",EL_grid.best_estimator_] )
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']<feature_engineering>
params = {
"learning_rate": [.1,.5,.7,.9,.95,.99, 1],
"colsample_bytree": [.3,.4,.5,.6],
"max_depth": [2, 4],
"alpha": [1, 3, 5],
"subsample": [.5],
"n_estimators": [30, 70, 100, 200],
"random_state" : [42]
}
XGBR = XGBRegressor()
XGBR_grid = GridSearchCV(XGBR, params, scoring='r2', cv=5, n_jobs=-1)
 XGBR_grid.fit(X_train, y_train)
 print("Best parameters: {}:".format(XGBR_grid.best_params_))
print("Best score: {}".format(XGBR_grid.best_score_))
best_estimators.append(["XGBoostR",XGBR_grid.best_estimator_] )
def fe_stats(train, test):
features_g = GENES
features_c = CELLS
for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
train_features2,test_features2=fe_stats(train_features2,test_features2 )<split>
params = {
"n_estimators": [10, 30, 50, 100],
"learning_rate": [.01, 0.1, 0.5, 0.9, 0.95, 1],
"random_state" : [42]
}
AdaBoostR = AdaBoostRegressor()
AdaBoostR_grid = GridSearchCV(AdaBoostR, params, scoring='r2', cv=5, n_jobs=-1)
 AdaBoostR_grid.fit(X_train, y_train)
 print("Best parameters: {}:".format(AdaBoostR_grid.best_params_))
print("Best score: {}".format(AdaBoostR_grid.best_score_))
best_estimators.append(["AdaBoostR",AdaBoostR_grid.best_estimator_] )
train_features_stats=train_features2.iloc[:,902:]
test_features_stats=test_features2.iloc[:,902:] 
<concatenate>
pipelines = []
for name,model in best_estimators:
pipeline = Pipeline([("Scaler",StandardScaler()), 
(name,model)
 ])
 pipelines.append(["Scaled_"+name,pipeline] )
train_features = pd.concat(( train_features, train_features_cluster,train_cluster_pca,train_features_stats), axis=1)
 test_features = pd.concat(( test_features, test_features_cluster,test_cluster_pca,test_features_stats), axis=1 )<merge>
evaluate = pd.DataFrame(columns=["model","cv","std"])
 for name,model in pipelines:
kfold = KFold(n_splits=10,random_state=42)
 cv = cross_val_score(model, X_train, y_train, cv=kfold, n_jobs=-1, scoring="neg_root_mean_squared_error")
 row = evaluate.shape[0]
evaluate.loc[row,"model"] = name
evaluate.loc[row,"cv"] = round(cv.mean() ,3)
 evaluate.loc[row,"std"] = "+/- {}".format(round(cv.std() ,4))
evaluate = evaluate.sort_values("cv",ascending=False)
 evaluate
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train.merge(train_targets_nonscored, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 train = train[train['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True )<drop_column>
from sklearn.ensemble import VotingRegressor
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )<define_variables>
votings = []
votings.append(( "Scaled_all_models",Pipeline([("Scaler",StandardScaler()), 
("Votings",VotingRegressor([("XGBoostR",XGBR_grid.best_estimator_), 
("AdaBoostR", AdaBoostR_grid.best_estimator_), 
("Ridge",ridge_grid.best_estimator_)
 ])
)])))
 votings.append(( "Scaled_XGBR_AB",Pipeline([("Scaler",StandardScaler()), 
("Votings",VotingRegressor([("XGBoostR",XGBR_grid.best_estimator_), 
("AdaBoostR", AdaBoostR_grid.best_estimator_)
 ])) ])
))
votings.append(( "Scaled_XGBR_R",Pipeline([("Scaler",StandardScaler()), 
("Votings",VotingRegressor([("XGBoostR",XGBR_grid.best_estimator_), 
("Ridge",ridge_grid.best_estimator_)
 ])) ])))
 votings.append(( "Scaled_AB_R",Pipeline([("Scaler",StandardScaler()), 
("Votings",VotingRegressor([("AdaBoostR", AdaBoostR_grid.best_estimator_), 
("Ridge",ridge_grid.best_estimator_)
 ])) ])) )
target_cols = [x for x in train_targets_scored.columns if x != 'sig_id']
aux_target_cols = [x for x in train_targets_nonscored.columns if x != 'sig_id']
all_target_cols = target_cols + aux_target_cols
num_targets = len(target_cols)
 num_aux_targets = len(aux_target_cols)
 num_all_targets = len(all_target_cols)
 print('num_targets: {}'.format(num_targets))
print('num_aux_targets: {}'.format(num_aux_targets))
print('num_all_targets: {}'.format(num_all_targets))<categorify>
evaluate_vote = pd.DataFrame(columns=["model","cv","std"])
 for name, model in votings:
kfold = KFold(n_splits=10,random_state=42)
 cv = cross_val_score(model,X_train,y_train, cv=kfold, scoring="neg_root_mean_squared_error",n_jobs=-1)
 row = evaluate_vote.shape[0]
evaluate_vote.loc[row,"model"] = name
evaluate_vote.loc[row,"cv"] = round(cv.mean() ,4)
 evaluate_vote.loc[row,"std"] = "+- {}".format(round(cv.std() ,5))
evaluate_vote = evaluate_vote.sort_values("cv",ascending=False)
 evaluate_vote
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct<train_model>
best_model = Pipeline([("Scaler",StandardScaler()), 
("Votings",VotingRegressor([("XGBoostR",XGBR_grid.best_estimator_), 
("AdaBoostR", AdaBoostR_grid.best_estimator_)
 ])) ])
 best_model = best_model.fit(train,y)
 submission = pd.read_csv("/kaggle/input/restaurant-revenue-prediction/sampleSubmission.csv")
 submission.iloc[:,1] = best_model.predict(test)
 submission.to_csv('submission', index=False)
 submission