Kaggle competition: Restaurant Revenue Prediction
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )<drop_column>
df_train = pd.read_csv(f'{PATH}train.csv', parse_dates=['Open Date'])
 df_test = pd.read_csv(f'{PATH}test.csv', parse_dates=['Open Date'])
 df_joined = pd.concat([df_train.drop('revenue', axis=1), df_test], axis=0 )
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist() 
<load_from_csv>
def score(model, X_train, y_train, X_valid = [], y_valid = []):
rms = sqrt(mean_squared_error(np.square(np.exp(y_train)) , np.square(np.exp(model.predict(X_train)))))
 score = [rms, model.score(X_train, y_train)]
if len(X_valid)!= 0 and len(y_valid)!= 0:
score.append(sqrt(mean_squared_error(np.square(np.exp(y_valid)) , np.square(np.exp(model.predict(X_valid))))))
if model.oob_score:
score.append(model.oob_score_)
 return score
n_train = df_train.shape[0]
def prcs(df, fe=[]):
add_datepart(df, 'Open Date')
 drop_cols = ['Open Year', 'Open Month', 'Open Week', 'Open Day', 'Open Dayofweek',
'Open Dayofyear', 'Open Is_month_end', 'Open Is_month_start',
'Open Is_quarter_end', 'Open Is_quarter_start', 'Open Is_year_end',
'Open Is_year_start']
df = df.drop(drop_cols, axis=1)
 if 'city' in fe:
df = df.drop('City', axis=1)
 if 'MB' in fe:
df['Type'] = df['Type'].replace('MB', 'DT')
 if 'city_group' in fe:
df = df.drop('City Group', axis=1)
 if 'id' in fe:
df = df.drop('Id', axis=1)
 if 'dummies' in fe:
p_cols = [ f'P{n}' for n in range(1,38)]
df = pd.get_dummies(df, columns=p_cols)
 if 'city_group' not in fe:
df = pd.get_dummies(df, columns=['City Group'], drop_first=True)
 df = pd.get_dummies(df, columns=['Type'])
 if 'outlier' in fe:
df = df.drop(index=16, axis=0)
 train_cats(df)
 X, _, _ = proc_df(df, None)
 if 'scale_open' in fe:
X['Open Elapsed'] =(X['Open Elapsed']/1000 ).apply(np.log)
 X_train = X[:n_train]
X_test = X[n_train:]
return X_train, X_test
train_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 ss_svc = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 ss_lr = ss_svc.copy()
ss_rf = ss_svc.copy()
ss = ss_svc.copy()
cols = [c for c in ss_svc.columns.values if c != 'sig_id']
def preprocess(df):
df.loc[:, 'cp_type'] = df.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 del df['sig_id']
return df
def log_loss_metric(y_true, y_pred):
metrics = []
for _target in train_targets.columns:
metrics.append(log_loss(y_true.loc[:, _target], y_pred.loc[:, _target].astype(float), labels = [0,1]))
return np.mean(metrics)
 train = preprocess(train_features)
 test = preprocess(test_features)
 del train_targets['sig_id']<compute_test_metric>
X_train, X_test = prcs(df_joined.copy())
 y_train = df_train['revenue'].copy().apply(np.log )
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))<choose_model_class>
m = RandomForestRegressor(n_jobs=-1, n_estimators=400, oob_score=True, max_features=0.5)
 m.fit(X_train, y_train)
 score(m,X_train, y_train )
def create_model1(num_columns,target_columns):
model = tf.keras.Sequential([
tf.keras.layers.Input(num_columns),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1024, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(target_columns, activation="sigmoid"))
])
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam() , sync_period=10),
loss=losses.BinaryCrossentropy(label_smoothing=0.001),metrics=logloss 
)
 return model<choose_model_class>
df_preds = pd.DataFrame(columns=['Prediction'],index=X_test.index, data=np.square(np.exp(m.predict(X_test))))
df_preds.to_csv('submission0.csv', index=True, index_label='Id')
 df_preds.head()
def create_model2(num_columns,target_columns):
model = tf.keras.Sequential([
tf.keras.layers.Input(num_columns),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1048, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(500, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(target_columns, activation="sigmoid"))
])
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam() , sync_period=10),
loss=losses.BinaryCrossentropy(label_smoothing=0.001),metrics=logloss 
)
 return model<define_variables>
m = RandomForestRegressor(n_jobs=-1, n_estimators=400, oob_score=True, max_features=0.5)
 m.fit(X_train, y_train)
 score(m,X_train, y_train )
top_feats = range(1056)
 print(len(top_feats))<normalization>
df_preds = pd.DataFrame(columns=['Prediction'],index=X_test.index, data=np.square(np.exp(m.predict(X_test))))
df_preds.to_csv('submission1', index=True, index_label='Id')
 df_preds.head()
scaler = StandardScaler()
X = scaler.fit_transform(train.values)
 x_tt = scaler.transform(test_features.values )<data_type_conversions>
X_train, X_test = prcs(df_joined.copy() , fe=['id'])
 y_train = df_train['revenue'].copy().apply(np.sqrt ).apply(np.log )
train = train.astype('float')
 test = test.astype('float' )<define_variables>
m = RandomForestRegressor(n_jobs=-1, n_estimators=400, oob_score=True, max_features=0.5)
 m.fit(X_train, y_train)
 score(m,X_train, y_train )
p_min = 0.001
p_max = 0.999<init_hyperparams>
df_preds = pd.DataFrame(columns=['Prediction'],index=X_test.index, data=np.square(np.exp(m.predict(X_test))))
df_preds.to_csv('submission2.csv', index=True, index_label='Id')
 df_preds.head()
N_STARTS = 2
tf.random.set_seed(42)
 ss_NN = ss_svc.copy()
res_NN = train_targets.copy()
ss_NN.loc[:, train_targets.columns] = 0
res_NN.loc[:, train_targets.columns] = 0
for seed in range(N_STARTS):
for n,(tr, te)in enumerate(MultilabelStratifiedKFold(n_splits=10, random_state=seed, shuffle=True ).split(train_targets, train_targets)) :
if seed%2 == 0:
model = create_model1(len(top_feats),206)
 else:
model = create_model2(len(top_feats),206)
 model_weights_path = f'.. /input/keras-nn-pca-label-smoothing-transfer-learning/repeat{seed}_Fold{n}.hdf5'
model.load_weights(model_weights_path)
 test_predict = model.predict(test.values[:, top_feats])
 val_predict = model.predict(train.values[te][:, top_feats])
 ss_NN.loc[:, train_targets.columns] += test_predict
res_NN.loc[te, train_targets.columns] += val_predict
del model,model_weights_path
tf.keras.backend.clear_session()
gc.collect()
del test_predict,val_predict
gc.collect()
ss_NN.loc[:, train_targets.columns] /=(( n+1)* N_STARTS)
 res_NN.loc[:, train_targets.columns] /= N_STARTS<compute_test_metric>
X_train, X_test = prcs(df_joined.copy() , fe=['id', 'dummies'])
 y_train = df_train['revenue'].copy().apply(np.sqrt ).apply(np.log )
print(f'NN OOF Metric: {log_loss_metric(train_targets, res_NN)}')
 res_NN.loc[train['cp_type'] == 1, train_targets.columns] = 0
ss_NN.loc[test['cp_type'] == 1, train_targets.columns] = 0
print(f'NN OOF Metric with postprocessing: {log_loss_metric(train_targets, res_NN)}' )<prepare_output>
m = RandomForestRegressor(n_jobs=-1, n_estimators=400, oob_score=True, max_features=0.5)
 m.fit(X_train, y_train)
 score(m,X_train, y_train )
oof2 = res_NN.values
sub2 = ss_NN.copy()<drop_column>
df_preds = pd.DataFrame(columns=['Prediction'],index=X_test.index, data=np.square(np.exp(m.predict(X_test))))
df_preds.to_csv('submission3.csv', index=True, index_label='Id')
 df_preds.head()
del train,test,res_NN,ss_NN
gc.collect()<load_from_csv>
X_train, X_test = prcs(df_joined.copy() , fe=['id', 'dummies', 'city'])
 y_train = df_train['revenue'].copy().apply(np.sqrt ).apply(np.log )
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_features = train_features.drop(['sig_id','cp_type','cp_dose','cp_time'],axis=1)
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 train_targets_scored = train_targets_scored.drop('sig_id',axis=1)
 train_target_with_sig_id = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 labels_train = train_targets_scored.values
train_features = train_features.iloc[non_ctl_idx]
train_target_with_sig_id = train_target_with_sig_id.iloc[non_ctl_idx]
labels_train = labels_train[non_ctl_idx]
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_features = test_features.drop(['sig_id','cp_dose','cp_time'],axis=1)
 json_file_path = '.. /input/t-test-pca-rfe-logistic-regression/main_predictors.json'
with open(json_file_path, 'r')as j:
predictors = json.loads(j.read())
 predictors = predictors['start_predictors']<categorify>
m = RandomForestRegressor(n_jobs=-1, n_estimators=400, oob_score=True, max_features=0.5)
 m.fit(X_train, y_train)
 score(m,X_train, y_train )
cs = train_features.columns.str.contains('c-')
 gs = train_features.columns.str.contains('g-')
 def preprocessor(train,test):
n_gs = 2
n_cs = 100
pca_cs = PCA(n_components = n_cs)
 pca_gs = PCA(n_components = n_gs)
 train_pca_gs = pca_gs.fit_transform(train[:,gs])
 train_pca_cs = pca_cs.fit_transform(train[:,cs])
 test_pca_gs = pca_gs.transform(test[:,gs])
 test_pca_cs = pca_cs.transform(test[:,cs])
 train_c_mean = train[:,cs].mean(axis=1)
 test_c_mean = test[:,cs].mean(axis=1)
 train_g_mean = train[:,gs].mean(axis=1)
 test_g_mean = test[:,gs].mean(axis=1)
 train = np.concatenate(( train,train_pca_gs,train_pca_cs,train_c_mean[:,np.newaxis]
,train_g_mean[:,np.newaxis]),axis=1)
 test = np.concatenate(( test,test_pca_gs,test_pca_cs,test_c_mean[:,np.newaxis],
test_g_mean[:,np.newaxis]),axis=1)
 scaler = preprocessing.StandardScaler()
train = scaler.fit_transform(train)
 test = scaler.transform(test)
 return train, test<compute_test_metric>
df_preds = pd.DataFrame(columns=['Prediction'],index=X_test.index, data=np.square(np.exp(m.predict(X_test))))
df_preds.to_csv('submission4.csv', index=True, index_label='Id')
 df_preds.head()
n_labels = train_targets_scored.shape[1]
n_train = train_features.shape[0]
n_test = test_features.shape[0]
p_min = 0.0005
p_max = 0.9995
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))<choose_model_class>
X_train, X_test = prcs(df_joined.copy() , fe=['id', 'dummies', 'city', 'city_group'])
 y_train = df_train['revenue'].copy().apply(np.sqrt ).apply(np.log )
def build_model(n_features, n_features_2, n_labels, label_smoothing = 0.0005):
input_1 = layers.Input(shape =(n_features,), name = 'Input1')
 input_2 = layers.Input(shape =(n_features_2,), name = 'Input2')
 head_1 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.2),
layers.Dense(512, activation="elu"),
layers.BatchNormalization() ,
layers.Dense(256, activation = "elu")
 ],name='Head1')
 input_3 = head_1(input_1)
 input_3_concat = layers.Concatenate()([input_2, input_3])
 head_2 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.3),
layers.Dense(512, "relu"),
layers.BatchNormalization() ,
layers.Dense(512, "elu"),
layers.BatchNormalization() ,
layers.Dense(256, "relu"),
layers.BatchNormalization() ,
layers.Dense(256, "elu")
 ],name='Head2')
 input_4 = head_2(input_3_concat)
 input_4_avg = layers.Average()([input_3, input_4])
 head_3 = Sequential([
layers.BatchNormalization() ,
layers.Dense(256, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, activation="sigmoid")
 ],name='Head3')
 output = head_3(input_4_avg)
 model = Model(inputs = [input_1, input_2], outputs = output)
 model.compile(optimizer='adam', loss=losses.BinaryCrossentropy(label_smoothing=label_smoothing), metrics=logloss)
 return model<prepare_x_and_y>
m = RandomForestRegressor(n_jobs=-1, n_estimators=400, oob_score=True, max_features=0.5)
 m.fit(X_train, y_train)
 score(m,X_train, y_train )
y_val = np.zeros(( n_train,206))
y_pred = np.zeros(( n_test,206)) 
<prepare_x_and_y>
df_preds = pd.DataFrame(columns=['Prediction'],index=X_test.index, data=np.square(np.exp(m.predict(X_test))))
df_preds.to_csv('submission5.csv', index=True, index_label='Id')
 df_preds.head()
n_seeds = 5
np.random.seed(1)
 seeds = np.random.randint(0,100,size=n_seeds)
 n_folds = 10
y_pred = np.zeros(( n_test,n_labels))
oof = tf.constant(0.0)
 for seed in seeds:
fold = 0
kf = KFold(n_splits=n_folds,shuffle=True,random_state=seed)
 for train, test in kf.split(train_features):
_, X_test = preprocessor(train_features.iloc[train].values,train_features.iloc[test].values)
 _,data_test = preprocessor(train_features.iloc[train].values,test_features.drop('cp_type',axis=1 ).values)
 X_test_2 = train_features.iloc[test][predictors].values
data_test_2 = test_features[predictors].values
X_test_2 = train_features.iloc[test][predictors].values
data_test_2 = test_features[predictors].values
y_train = labels_train[train]
y_test = labels_train[test]
model_path = '.. /input/resnet-with-transfer/TwoHeads'+'_seed_'+str(seed)+'_fold_'+str(fold)
 model = tf.keras.models.load_model(model_path,custom_objects={'logloss':logloss})
 y_val[test] += model.predict([X_test,X_test_2])/n_seeds
y_pred += model.predict([data_test,data_test_2])/(n_folds*n_seeds)
 fold += 1
del model, X_test,X_test_2,data_test,data_test_2
tf.keras.backend.clear_session()
gc.collect()<load_from_csv>
X_train, X_test = prcs(df_joined.copy() , fe=['id', 'dummies', 'city', 'city_group'])
 y_train = df_train['revenue'].copy().apply(np.sqrt ).apply(np.log )
p_min = 0.001
p_max = 0.999
sub = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub.iloc[:,1:] = y_pred
sub.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0 
<feature_engineering>
m = RandomForestRegressor(n_jobs=-1, n_estimators=4000, oob_score=True, max_features=0.5)
 m.fit(X_train, y_train)
 score(m,X_train, y_train )
<load_from_csv><EOS> 
df_preds = pd.DataFrame(columns=['Prediction'],index=X_test.index, data=np.square(np.exp(m.predict(X_test))))
df_preds.to_csv('submission6.csv', index=True, index_label='Id')
 df_preds.head()