Kaggle competition: Restaurant Revenue Prediction
<SOS> metric: RMSE Kaggle data source: restaurant-revenue-prediction<define_variables>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
<create_dataframe>
FAKE_DATA_RATIO = 311.5
SEED = 0
train = pd.read_csv('/kaggle/input/restaurant-revenue-prediction/train.csv.zip', index_col = 0, parse_dates=[1])
 test = pd.read_csv('/kaggle/input/restaurant-revenue-prediction/test.csv.zip', index_col = 0, parse_dates=[1])
 print("Train Dimensions:")
 print(train.shape)
 print("Test Dimensions:")
 print(test.shape)
 df = pd.concat(( test, train), ignore_index=True)
 df.describe()
new_total = pd.DataFrame(total_proc, columns = total2.columns)
 added_features = pca_features
added_columns = pca_columns
new_total = pd.concat([new_total, pd.DataFrame(added_features, columns = added_columns)], axis = 1)
 train_targets=targets2[scored_columns]
train_set = new_total.loc[:trains_without_veh-1]
test_set = new_total.loc[trains_without_veh:]
num_columns = train_set.shape[1]
num_targets = train_targets.shape[1]
print('Train data', train_set.shape)
 print('Train targets', train_targets.shape)
 print('Test set', test_set.shape )<choose_model_class>
tofit = df.loc[(( df.T_FC==1)|(df.T_IL==1)) &(pd.notnull(df.revenue)) ]
tofill = df.loc[(( df.T_FC==0)&(df.T_IL==0)) ]
print('type training set shape:'), tofit.shape
print('data to impute:'), tofill.shape
y = tofit.T_FC
X = tofit.drop(["T_FC", "T_IL", "revenue"], axis=1 )
def glu(act, n_units):

 return act[:, :n_units] * tf.nn.sigmoid(act[:, n_units:])
 def dense_bn_glu(input_size, num_features, dropout = 0.5):
in_layer = L.Input(input_size)
 base_layer = L.Dropout(dropout )(in_layer)
 layer = L.Dense(2 * num_features )(base_layer)
 layer = L.BatchNormalization()(layer)
 layer = glu(layer, num_features)
 return Model(in_layer, layer)
 def combine_models(model1, model2):
model2_output = model2(model1.output)
 return Model(model1.input, model2_output )<choose_model_class>
model_grid = {'max_depth': [None, 8], 'min_samples_split': [4,9,16], 'min_samples_leaf':[1,4], 'max_features':['sqrt', 0.5, None]}
type_model = ExtraTreesClassifier(n_estimators=25, random_state=SEED)
 grid = RandomizedSearchCV(type_model, model_grid, n_iter=10, cv=5, scoring="roc_auc")
 grid.fit(X, y)
 print("Best parameters for Type Model:")
 print(grid.best_params_)
 type_model.set_params(**grid.best_params_)
 type_model.fit(X, y)
 imputations = type_model.predict(tofill.drop(["T_FC", "T_IL", "revenue"], axis=1))
df.loc[(df.T_FC==0)&(df.T_IL==0), "T_FC"] = imputations
df = df.drop(["T_IL"], axis=1)
 print("% labeled FC in the training set:"), df.T_FC.mean()
print("% of imputed values labeled FC:"), np.mean(imputations )
def create_TabNet2_model(num_columns, n_steps = 4, dropout = 0.5):
num_shared = 1
num_unique = 2
num_features = 2*num_scored
num_output_features = num_scored
num_steps = n_steps
relaxation = 1.5
input_layer = L.Input(num_columns)
 input_base = L.BatchNormalization()(input_layer)
 masked_input = input_base
output_agg = []
agg_compl_mask = 1
mask_agg = []
epsilon = 1e-6
output_mask = np.random.randint(2, size=[1, num_scored, num_steps])
 output_mask_compensation = num_steps /(np.sum(output_mask, axis=2, keepdims = True)+ 1e-6)
 output_mask = output_mask * output_mask_compensation
out_agg_compl_mask = 1
out_mask_agg = []
shared_model = dense_bn_glu(num_columns, num_features, dropout)
 for i in range(num_shared-1):
shared_model = combine_models(shared_model,
dense_bn_glu(num_features, num_features, dropout))
def feature_transformer(num_layers, num_features, num_shared_features = None):
cur_model = shared_model
if(num_shared_features is None):
num_shared_features = num_features
for i in range(num_layers):
cur_model = combine_models(cur_model,
dense_bn_glu(num_shared_features, num_features, dropout))
return cur_model
def attentive_transformer(num_features, input_data):
hidden = L.Dense(num_features )(input_data)
 hidden = L.BatchNormalization()(hidden)
 return hidden
cur = feature_transformer(num_unique, num_features )(masked_input)
 coef_features = cur[:,num_output_features:]
for i in range(num_steps):
mask = attentive_transformer(num_columns, coef_features)
 mask *= agg_compl_mask
mask = L.Activation('sigmoid' )(mask)
 mask_agg.append(tf.expand_dims(mask, axis = 2))
agg_compl_mask *=(relaxation - mask)
 masked_input = mask * input_base
cur = feature_transformer(num_unique, num_features )(masked_input)
 coef_features = cur[:,num_output_features:]
output_features = tf.expand_dims(L.Activation('sigmoid' )(cur[:,:num_output_features]), axis = 2)
 out_mask = attentive_transformer(num_scored, coef_features)
 out_mask *= out_agg_compl_mask
out_mask = L.Activation('sigmoid' )(out_mask)
 out_agg_compl_mask *=(relaxation - out_mask)
 out_mask = tf.expand_dims(out_mask, axis = 2)
 out_mask_agg.append(out_mask)
 output_agg.append(( 1.0/num_steps)*output_features)
 output = tf.concat(output_agg, axis=2)
 out_mask = tf.concat(out_mask_agg, axis = 2)
 mask_loss = L.Lambda(lambda x: x, name='mask' )(out_mask)
 out_mask_compensation = num_steps /(tf.reduce_sum(out_mask, axis=2, keepdims = True)+ epsilon)
 out_mask = out_mask_compensation*out_mask
output = out_mask * output
mask_total = tf.concat(mask_agg, axis = 2)
 sparse_val = -mask_total*tf.math.log(mask_total+epsilon)
 sparse_val = tf.reduce_sum(sparse_val, axis = 1)
 sparse_loss = L.Lambda(lambda x:tf.reduce_mean(x), name='sparse' )(sparse_val)
 output = L.Lambda(lambda x:tf.reduce_sum(x, axis = 2), name='output' )(output)
 return Model(input_layer, [output, mask_loss, sparse_loss] )<compute_train_metric>
print("Pre-binarizing columns:"), len(df.columns)
 for col in df.columns:
if col[0] == 'P':
print(col), len(df[col].unique()), "unique values"
df = df.join(pd.get_dummies(df[col], prefix=col))
df = df.drop([col, df.columns[-1]], axis=1)
 print("Post-binarizing columns:"), len(df.columns )
def dummy_loss(y_true, y_pred):
return y_pred
base_lr = 0.01
def compile_model(model, lr = base_lr):
focal_loss = tfa.losses.SigmoidFocalCrossEntropy()
binary_loss = tf.keras.losses.BinaryCrossentropy()
model.compile(optimizer=tf.keras.optimizers.Adam(lr),
loss=[binary_loss, binary_loss, dummy_loss],
metrics={'output':focal_loss},
loss_weights=[1.0, 0.0003, 0.0000001]
)<categorify>
train = df.loc[pd.notnull(df.revenue)]
test = df.loc[pd.isnull(df.revenue)].drop(['revenue'], axis=1)
 y = train.revenue.apply(np.sqrt)
 X = train.drop(["revenue"], axis=1 )
def smooth_labels(labels, factor=0.1):
labels *=(1.0 - factor)
 labels +=(factor / labels.shape[1])
 return labels
def get_mask_loss_data(train_shape, val_shape, f=np.zeros):
return f(train_shape), f(val_shape)
 def get_sparse_loss_data(train_shape, val_shape):
return get_mask_loss_data(train_shape, val_shape, np.zeros)
 class Cycle_LR() :
def __init__(self, base_lr, cycle_start, period):
self.base_lr = base_lr
self.cycle_start = cycle_start
self.period = period
self.alpha1 = 0.3 * base_lr
self.alpha2 = 0.03 * self.alpha1
def get_lr(self, epoch, cur):
if epoch == 35:
print('Cur lr', cur)
 return self.base_lr * 0.33
if epoch > self.cycle_start:
t =(( epoch - self.cycle_start)% self.period)/ self.period
cur = self.alpha2*t + self.alpha1 *(1-t)
 print('New lr:', cur)
 return cur<choose_model_class>
model_grid = [{'normalize': [True, False], 'alpha': np.logspace(0,10)}]
model = Ridge()
grid = GridSearchCV(model, model_grid, cv=LeaveOneOut() , scoring='neg_mean_squared_error')
 grid.fit(X, y)
 print("Best parameters set found on development set:")
 print(grid.best_params_)
 model.set_params(**grid.best_params_)
 model.fit(X, y )
num_folds = 1
num_steps = 8
id_range = np.arange(train_targets.shape[0])
 target_vals = train_targets.values[:,:num_scored].astype(float)
 def train_classifier(train_set, model_filename, epochs = 150, lr = base_lr, dropout = 0.5):
for k,(train_ids, test_ids)in enumerate([(id_range,id_range)]):
classifier = create_TabNet2_model(train_set.shape[1], num_steps, dropout)
 compile_model(classifier, lr)
 checkpoint_filepath = model_filename + '_checkpoint{:02d}.h5'.format(k)
 model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
 filepath=checkpoint_filepath,
metric = 'val_output_loss',
save_weights_only=True,
save_best_only=True)
 cycle_lr = Cycle_LR(lr, 70, 10)
 lr_schedule = tf.keras.callbacks.LearningRateScheduler(cycle_lr.get_lr, verbose=0)
 mask_train, mask_val = get_mask_loss_data([*(target_vals[train_ids].shape), num_steps],
[*(target_vals[test_ids].shape), num_steps], np.zeros)
 sparse_train, sparse_val = get_sparse_loss_data([target_vals[train_ids].shape[0], 1], [target_vals[test_ids].shape[0], 1])
 train_true_vals = [smooth_labels(target_vals[train_ids])]
val_true_vals = [target_vals[test_ids]]
train_true_vals.append(mask_train)
 val_true_vals.append(mask_val)
 train_true_vals.append(sparse_train)
 val_true_vals.append(sparse_val)
 sample_weights = train_nums2[train_ids]
classifier.fit(train_set[train_ids],
train_true_vals,
sample_weight = 8.0 / sample_weights,
validation_data=(train_set[test_ids], val_true_vals),
callbacks=[model_checkpoint_callback, lr_schedule],
epochs=epochs, batch_size=256)
 return classifier<split>
submission = pd.DataFrame(columns=['Prediction'],index=test.index, data=model.predict(test))
submission.Prediction = submission.Prediction.apply(np.square)
 submission.index.name='Id'
submission.to_csv("TFI_Ridge.csv")
 submission.describe().astype(int )