Kaggle competition: Instacart Market Basket Analysis
<SOS> metric: MeanFScore Kaggle data source: instacart-market-basket-analysis<categorify>
gc.enable()
gr = GroupMeanTransformer(['Sex', 'SmokingStatus'], ['FVC_base'])
 gr.fit(df_train)
 df_train = gr.transform(df_train)
 df_test = gr.transform(df_test)
 FEATURES += [f for f in gr.generated_features if re.match("^delta_", f)]<prepare_x_and_y>
orders = pd.read_csv('.. /input/orders.csv')
 order_products_train = pd.read_csv('.. /input/order_products__train.csv')
 order_products_prior = pd.read_csv('.. /input/order_products__prior.csv')
 products = pd.read_csv('.. /input/products.csv')
 aisles = pd.read_csv('.. /input/aisles.csv')
 departments = pd.read_csv('.. /input/departments.csv' )
MODE = ["PERC_FVC", "LOG"]
SIGMA_MODE = ["PERC"]
X = df_train[FEATURES].copy().values
y = df_train['FVC_target'].copy().values
group = df_train['Patient'].values
X_test = df_test[FEATURES].copy().values<filter>
aisles['aisle'] = aisles['aisle'].astype('category')
 departments['department'] = departments['department'].astype('category')
 orders['eval_set'] = orders['eval_set'].astype('category')
 products['product_name'] = products['product_name'].astype('category' )
df_train[mean_target == mean_target.max() ]<compute_train_metric>
op = orders.merge(order_products_prior, on='order_id', how='inner')
 op.head()
pred_oof = np.nan * np.zeros(( N_FOLDS, X.shape[0]))
pred_test = np.nan * np.zeros(( N_FOLDS, X_test.shape[0]))
for i,(idxT, idxV)in enumerate(tqdm(folds)) :
model = linear_model.LassoCV()
model.fit(Z[idxT], mean_target[idxT])
 pred_oof[i] = model.predict(Z)
 pred_oof[i, idxT] = np.nan
pred_test[i] = model.predict(Z_test)
 pred_mean = inverse_transform(np.nanmean(pred_oof, 0), MODE, df_train)
 test_mean = inverse_transform(np.nanmean(pred_test, 0), MODE, df_test )<choose_model_class>
user = op.groupby('user_id')['order_number'].max().to_frame('u_total_orders')
 user = user.reset_index()
user.head()
pred_oof = np.nan * np.zeros(( N_FOLDS, X.shape[0]))
pred_test = np.nan * np.zeros(( N_FOLDS, X_test.shape[0]))
for i,(idxT, idxV)in enumerate(tqdm(folds)) :
model = linear_model.LinearRegression()
model.fit(Z[idxT], sigma_target[idxT])
 pred_oof[i] = model.predict(Z)
 pred_oof[i, idxT] = np.nan
pred_test[i] = model.predict(Z_test)
 pred_sigma = inverse_transform(np.nanmean(pred_oof, 0), SIGMA_MODE, df_train)
 test_sigma = inverse_transform(np.nanmean(pred_test, 0), SIGMA_MODE, df_test )<compute_test_metric>
user = user.merge(u_reorder, on='user_id', how='left')
 del u_reorder
gc.collect()
user.head()
lll_overall = laplace_likelihood(y[valid_mask], [pred_mean[valid_mask], pred_sigma[valid_mask]])
 lllb_overall = laplace_likelihood_bound(y[valid_mask], pred_mean[valid_mask])
 y_rmse_overall = metrics.mean_squared_error(y[valid_mask], pred_mean[valid_mask])** 0.5
y_mae_overall = metrics.mean_absolute_error(y[valid_mask], pred_mean[valid_mask])
 s_rmse_overall = metrics.mean_squared_error(opt_sigma[valid_mask], pred_sigma[valid_mask])** 0.5
s_mae_overall = metrics.mean_absolute_error(opt_sigma[valid_mask], pred_sigma[valid_mask])
 _df_base = pd.DataFrame({'y': y, 'so': opt_sigma, 'p': pred_mean, 's': pred_sigma} ).merge(valid_folds, left_index=True, right_index=True)
 lllb = _df_base.groupby('fold' ).apply(lambda x: laplace_likelihood_bound(x['y'], x['p']))
lll = _df_base.groupby('fold' ).apply(lambda x: laplace_likelihood(x['y'], [x['p'], x['s']]))
y_rmse = _df_base.groupby('fold' ).apply(lambda x: metrics.mean_squared_error(x['y'], x['p'])** 0.5)
 y_mae = _df_base.groupby('fold' ).apply(lambda x: metrics.mean_absolute_error(x['y'], x['p']))
s_rmse = _df_base.groupby('fold' ).apply(lambda x: metrics.mean_squared_error(x['so'], x['s'])** 0.5)
 s_mae = _df_base.groupby('fold' ).apply(lambda x: metrics.mean_absolute_error(x['so'], x['s']))
print(" OVERALL || FOLD-WISE")
 print("Bound: %7.4f || %7.4f(+- %7.4f)" %(lllb_overall, lllb.mean() , 2 * lllb.std()))
print("Score: %7.4f || %7.4f(+- %7.4f)" %(lll_overall, lll.mean() , 2 * lll.std()))
print("yRMSE: %7.2f || %7.2f(+- %7.2f)" %(y_rmse_overall, y_rmse.mean() , 2 * y_rmse.std()))
print("y-MAE: %7.2f || %7.2f(+- %7.2f)" %(y_mae_overall, y_mae.mean() , 2 * y_mae.std()))
print("sRMSE: %7.2f || %7.2f(+- %7.2f)" %(s_rmse_overall, s_rmse.mean() , 2 * s_rmse.std()))
print("s-MAE: %7.2f || %7.2f(+- %7.2f)" %(s_mae_overall, s_mae.mean() , 2 * s_mae.std()))<create_dataframe>
prd = op.groupby('product_id')['order_id'].count().to_frame('p_total_purchases')
 prd = prd.reset_index()
prd.head()
pd.DataFrame({'LLL': lll, 'Bound': lllb} ).boxplot(figsize=(12, 4)) ;<prepare_output>
p_reorder = op.groupby('product_id' ).filter(lambda x: x.shape[0] >40)
 p_reorder.head()
submission = df_test.copy() [['Patient', 'Weeks_target']]
submission['Patient_Week'] = submission['Patient'] + "_" + submission['Weeks_target'].astype('str')
 submission['FVC'] = test_mean
submission['Confidence'] = test_sigma
submission = submission.sort_values(['Weeks_target', 'Patient'])[['Patient_Week', 'FVC', 'Confidence']]
submission.head()<save_to_csv>
p_reorder = p_reorder.groupby('product_id')['reordered'].mean().to_frame('p_reorder_ratio')
 p_reorder = p_reorder.reset_index()
p_reorder.head()
submission.to_csv("submission.csv", index=False )<save_to_csv>
prd = prd.merge(p_reorder, on='product_id', how='left')
 del p_reorder
gc.collect()
prd.head()
submission.to_csv("submission.csv", index=False )<import_modules>
prd['p_reorder_ratio'] = prd['p_reorder_ratio'].fillna(value=0)
 prd.head()
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold<import_modules>
uxp = op.groupby(['user_id', 'product_id'])['order_id'].count().to_frame('uxp_total_bought')
 uxp = uxp.reset_index()
uxp.head()
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M<define_variables>
op['order_number_back'] = op.groupby('user_id')['order_number'].transform(max)- op.order_number +1
op5 = op[op.order_number_back <= 5]
last_five = op5.groupby(['user_id','product_id'])[['order_id']].count()
last_five.columns = ['times_last5']
last_five['times_last5_ratio'] = last_five.times_last5 / 5
uxp = uxp.merge(last_five , on=['user_id', 'product_id'], how='left')
 del [last_five, op5]
gc.collect()
uxp.head()
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )<define_variables>
uxp['times_last5'] = uxp['times_last5'].fillna(value=0)
 uxp['times_last5_ratio'] = uxp['times_last5_ratio'].fillna(value=0)
 uxp.head()
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128<load_from_csv>
times = op.groupby(['user_id', 'product_id'])[['order_id']].count()
times.columns = ['Times_Bought_N']
times.head()
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )<concatenate>
total_orders = op.groupby('user_id')['order_number'].max().to_frame('total_orders')
 total_orders.head()
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )<count_unique_values>
span = pd.merge(total_orders, first_order_no, on='user_id', how='right')
 span.head()
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique())
<feature_engineering>
span['Order_Range_D'] = span.total_orders - span.first_order_number + 1
span.head()
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )<categorify>
uxp_ratio = pd.merge(times, span, on=['user_id', 'product_id'], how='left')
 uxp_ratio.head()
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )<merge>
uxp_ratio['uxp_order_ratio'] = uxp_ratio.Times_Bought_N / uxp_ratio.Order_Range_D
uxp_ratio.head()
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base<data_type_conversions>
uxp_ratio = uxp_ratio.drop(['Times_Bought_N', 'total_orders', 'first_order_number', 'Order_Range_D'], axis=1)
 uxp_ratio.head()
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
<feature_engineering>
del [times, first_order_no, span]
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']<filter>
uxp = uxp.merge(uxp_ratio, on=['user_id', 'product_id'], how='left')
 del uxp_ratio
uxp.head()
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data<compute_test_metric>
data = uxp.merge(user, on='user_id', how='left')
 data.head()
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model<prepare_x_and_y>
data = data.merge(prd, on='product_id', how='left')
 data.head()
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))<choose_model_class>
del op, user, prd, uxp
gc.collect()
NFOLD = 5
kf = KFold(n_splits=NFOLD )<predict_on_test>
orders_last = orders[(orders.eval_set=='train')|(orders.eval_set=='test')]
%%time
cnt = 0
EPOCHS = 800
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD 
<compute_test_metric>
data = data.merge(orders_last, on='user_id', how='left')
 data.head()
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<feature_engineering>
data_train = data[data.eval_set=='train']
data_train = data_train.merge(order_products_train, on=['product_id', 'order_id'], how='left')
 data_train = data_train.drop(['order_id','eval_set', 'add_to_cart_order'], axis=1)
 data_train = data_train.fillna(0)
 data_train.head()
sub['FVC1'] = 0.996*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]<prepare_output>
data_test = data[data.eval_set=='test']
data_test = data_test.drop(['eval_set', 'order_id'], axis=1)
 data_test = data_test.fillna(0)
 data_test.head()
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()<feature_engineering>
del data
del orders_last
gc.collect()
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<load_from_csv>
data_train = data_train.set_index(['user_id', 'product_id'])
 data_test = data_test.set_index(['user_id', 'product_id'] )
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1<save_to_csv>
data_train.loc[:, 'reordered'] = data_train.reordered.fillna(0)
 X_train, X_val, y_train, y_val = train_test_split(data_train.drop('reordered', axis=1), data_train.reordered,
test_size=0.2, random_state=42)
 del data_train
gc.collect()
d_train = xgboost.DMatrix(X_train, y_train)
 xgb_params = {
"objective" : "reg:logistic"
,"eval_metric" : "logloss"
,"eta" : 0.1
,"max_depth" : 6
,"min_child_weight" :10
,"gamma" :0.70
,"subsample" :0.76
,"colsample_bytree" :0.95
,"alpha" :2e-05
,"lambda" :10
}
watchlist= [(d_train, "train")]
bst = xgboost.train(params=xgb_params, dtrain=d_train, num_boost_round=80, evals=watchlist, verbose_eval=10)
 xgboost.plot_importance(bst )
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )<import_modules>
del [X_train, X_val, y_train, y_val]
gc.collect()
import numpy as np
import pandas as pd
import seaborn as sns
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold, GroupKFold<import_modules>
d_test = xgboost.DMatrix(data_test)
 data_test = data_test.reset_index()
data_test = data_test[['product_id', 'user_id']]
data_test["reordered"] =(bst.predict(d_test)> 0.21 ).astype(int)
 del bst
import tensorflow as tf
import tensorflow.keras.backend as backend
import tensorflow.keras.layers as layers
import tensorflow.keras.models as models<define_variables>
orders_test = orders[orders.eval_set=='test']
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 return seed<load_from_csv>
data_test = data_test.merge(orders_test[["user_id", "order_id"]], on='user_id', how='left')
 data_test.head()
path = ".. /input/osic-pulmonary-fibrosis-progression"
train = pd.read_csv(f"{path}/train.csv")
 test = pd.read_csv(f"{path}/test.csv")
 print('****training data head 1 value****
')
 print(train.head(1))
print('
****test data head 1 value****
')
 print(test.head(1))<count_duplicates>
del orders
del orders_test
gc.collect()
print('train_data_shape', train.shape)
 print('test_data_shape', test.shape)
 print('duplicates',train.duplicated().sum())
 print('duplicates',train.duplicated(subset=['Patient','Weeks'] ).sum())
 train.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'] )<feature_engineering>
data_test['product_id'] = data_test.product_id.astype(int)
 data_test = data_test.drop('user_id', axis=1)
 gc.collect()
sub = pd.read_csv(f"{path}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]<merge>
d = dict()
for row in data_test.itertuples() :
if row.reordered == 1:
try:
d[row.order_id] += ' ' + str(row.product_id)
 except:
d[row.order_id] = str(row.product_id)
 for order in data_test.order_id:
if order not in d:
d[order] = 'None'
gc.collect()
submission = sub.merge(test.drop('Weeks', axis=1), on="Patient")
 print('****submission_data head 1 value*****
')
 print(sub.head(1))
print('******test data head 1 value********')
 print(test.head(1))
submission.head(1 )<count_unique_values>
sub = pd.DataFrame.from_dict(d, orient='index')
 sub.reset_index(inplace=True)
 sub.columns = ['order_id', 'products']
sub.to_csv('sub.csv', index=False )
<feature_engineering><EOS> 
submission = pd.read_csv(".. /working/sub.csv")
 submission.head()