Kaggle competition: Avito Demand Prediction Challenge
tr_adj = train[train['Weeks']>=0].groupby(['Patient'],as_index=False)['Weeks'].min()
tr_adj = pd.merge(tr_adj,train,how='left',on=['Patient','Weeks'])
 tr_adj = tr_adj.groupby(['Patient','Weeks'],as_index=False)[['Percent','FVC']].mean()
tr_adj['intercept_percent'] = tr_adj['Percent'] -(tr_adj['Weeks'] * mean_slope_percent)
 tr_adj.rename(columns={'Weeks': 'Weeks_0', 'Percent': 'Percent_0','FVC':'FVC_0'}, inplace=True)
 tr_adj.head()<merge>
h2o.init(max_mem_size = "10G" )
train_variant = train.copy()
train_variant = pd.merge(train_variant,tr_adj,how='left',on=['Patient'])
 train_variant['Percent_mean_slope'] = train_variant['intercept_percent'] +(train_variant['Weeks'] * mean_slope_percent)
 if percent_model == "weeks_0":
train_variant['percent'] =(train_variant['Percent_0'] - data_percent_min)/(data_percent_max - data_percent_min)
 if percent_model == "avg_slope":
train_variant['percent'] =(train_variant['Percent_mean_slope'] - data_percent_min)/(data_percent_max - data_percent_min)
 train_variant.drop(columns=['Percent_mean_slope','intercept_percent'],inplace=True)
 print(train_variant.shape)
 train_variant.head()<compute_test_metric>
print("
Data Load Stage")
 training = pd.read_csv('.. /input/train.csv', index_col = "item_id", parse_dates = ["activation_date"])
 traindex = training.index
testing = pd.read_csv('.. /input/test.csv', index_col = "item_id", parse_dates = ["activation_date"])
 testdex = testing.index
y = training.deal_probability.copy()
print('Train shape: {} Rows, {} Columns'.format(*training.shape))
print('Test shape: {} Rows, {} Columns'.format(*testing.shape))
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model<data_type_conversions>
df = pd.concat([training,testing],axis=0)
 del training, testing
gc.collect()
print('
All Data shape: {} Rows, {} Columns'.format(*df.shape))
print("Feature Engineering")
 df["price"] = np.log(df["price"]+0.001)
 df["price"].fillna(-999,inplace=True)
 df["image_top_1"].fillna(-999,inplace=True)
 print("
Create Time Variables")
 df["Weekday"] = df['activation_date'].dt.weekday
df["Weekd of Year"] = df['activation_date'].dt.week
df["Day of Month"] = df['activation_date'].dt.day
df.drop(["activation_date","image"],axis=1,inplace=True)
 print("
Encode Variables")
 categorical = ["user_id","region","city","parent_category_name","category_name","item_seq_number","user_type","image_top_1"]
messy_categorical = ["param_1","param_2","param_3","title","description"]
print("Encoding :",categorical + messy_categorical)
 lbl = preprocessing.LabelEncoder()
for col in categorical + messy_categorical:
df[col] = lbl.fit_transform(df[col].astype(str))
X = df.loc[traindex,:].copy()
print("Training Set shape",X.shape)
 test = df.loc[testdex,:].copy()
print("Submission Set Shape: {} Rows, {} Columns".format(*test.shape))
del df
gc.collect()
y_train = train['FVC'].values.astype(np.float32)
 x_train = train[FE].values.astype(np.float32)
 x_train_variant = train_variant[FE].values.astype(np.float32)
 x_test = test[FE].values.astype(np.float32)
 x_submission = submission[FE].values.astype(np.float32)
 pred_test = np.zeros(( x_test.shape[0], 3))
pred_train = np.zeros(( x_train.shape[0], 3))
pred_submission = np.zeros(( x_submission.shape[0], 3))<find_best_params>
test.drop("deal_probability",axis=1, inplace=True )
num_inputs = x_train.shape[1]
net = make_model(num_inputs)
 print(net.summary())
 print(net.count_params() )<define_variables>
NFOLD = 5
repeats = 10<define_variables>
htrain = h2o.H2OFrame(X)
 htest = h2o.H2OFrame(test )
%%time
preds_df = pd.DataFrame()
conf_df = pd.DataFrame()
preds_test_df = pd.DataFrame()
conf_test_df = pd.DataFrame()
preds_submission_df = pd.DataFrame()
conf_submission_df = pd.DataFrame()
for random_state in range(repeats):
print('repeat:',random_state)
 pred_test = np.zeros(( x_test.shape[0], 3))
pred_train = np.zeros(( x_train.shape[0], 3))
pred_submission = np.zeros(( x_submission.shape[0], 3))
kf = KFold(n_splits=NFOLD,shuffle=True,random_state=random_state)
 cnt = 0
EPOCHS = 800
for tr_idx, val_idx in kf.split(x_train):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(num_inputs)
 net.fit(x_train[tr_idx], y_train[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(x_train[val_idx], y_train[val_idx]), verbose=0)
 print("train", net.evaluate(x_train[tr_idx], y_train[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(x_train[val_idx], y_train[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict train-val...")
 pred_train[val_idx] = net.predict(x_train_variant[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pred_test += net.predict(x_test, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
print("predict submission...")
 pred_submission += net.predict(x_submission, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
sigma_opt = mean_absolute_error(y_train, pred_train[:, 1])
 unc = pred_train[:,2] - pred_train[:, 0]
sigma_mean = np.mean(unc)
 print('----------------------------')
 print(sigma_opt, sigma_mean)
 preds_df[random_state] = pred_train[:, 1]
conf_df[random_state] = unc
preds_test_df[random_state] = pred_test[:, 1]
conf_test_df[random_state] = pred_test[:,2] - pred_test[:, 0]
preds_submission_df[random_state] = pred_submission[:, 1]
conf_submission_df[random_state] = pred_submission[:,2] - pred_submission[:, 0]<feature_engineering>
x =htrain.columns
y ='deal_probability'
x.remove(y)
 aml = H2OAutoML(max_runtime_secs = 18000)
 aml.train(x=x, y =y, training_frame=htrain)
 print('Generate predictions...')
 htrain.drop(['deal_probability'])
preds_df['mean'] = preds_df.iloc[:,0:repeats].mean(axis=1)
 conf_df['mean'] = conf_df.iloc[:,0:repeats].mean(axis=1)
 preds_df['median'] = preds_df.iloc[:,0:repeats].median(axis=1)
 conf_df['median'] = conf_df.iloc[:,0:repeats].median(axis=1)
 preds_test_df['mean'] = preds_test_df.iloc[:,0:repeats].mean(axis=1)
 conf_test_df['mean'] = conf_test_df.iloc[:,0:repeats].mean(axis=1)
 preds_test_df['median'] = preds_test_df.iloc[:,0:repeats].median(axis=1)
 conf_test_df['median'] = conf_test_df.iloc[:,0:repeats].median(axis=1)
 preds_submission_df['mean'] = preds_submission_df.iloc[:,0:repeats].mean(axis=1)
 conf_submission_df['mean'] = conf_submission_df.iloc[:,0:repeats].mean(axis=1)
 preds_submission_df['median'] = preds_submission_df.iloc[:,0:repeats].median(axis=1)
 conf_submission_df['median'] = conf_submission_df.iloc[:,0:repeats].median(axis=1 )<compute_test_metric>
scores = []
for i in range(repeats):
score = laplace_log_likelihood(y_train, preds_df[i], conf_df[i])
 print('solution',i,':',score)
 scores.append(score)
 print('----------------')
 print('mean score:',np.mean(scores))<compute_train_metric>
aml.leader
print('mean OOF lap_log:',laplace_log_likelihood(y_train, preds_df['mean'], conf_df['mean']))
print('median OOF lap_log:',laplace_log_likelihood(y_train, preds_df['median'], conf_df['median']))<compute_train_metric>
preds = aml.leader.predict(htest)
 preds = preds.as_data_frame()
<prepare_output><EOS> 
lgsub = pd.DataFrame(preds.predict.values,columns=["deal_probability"],index=testdex)
 lgsub['deal_probability'].clip(0.0, 1.0, inplace=True)
 lgsub.to_csv("lgsub.csv",index=True,header=True)
 print("Notebook Runtime: %0.2f Minutes"%(( time.time() - notebookstart)/60))