Kaggle competition: Instacart Market Basket Analysis
<SOS> metric: MeanFScore Kaggle data source: instacart-market-basket-analysis<train_model>
import numpy as np
import pandas as pd
import xgboost as xgb
from sklearn import metrics, model_selection
print('we are using fix seed value always to avoid RANDOMIZATION(NEED TO GET SAME RESULT)')
 print('Seed value =',SEED)
 print('Number of folds =',BATCH_SIZE)
 print('Number of epochs =',EPOCHS)
 print('
mean_absolute_error =',mean_absolute_error)
 print('unc_mean =',unc.mean())
 print('unc_min =',unc.min())
 print('unc_max =',unc.max())
 print('unc_mean =',(unc>=0 ).mean() )<feature_engineering>
data_path = ".. /input/"
orders_df = pd.read_csv(data_path + "orders.csv", usecols=["order_id","user_id","order_number"] )
submission['FVC1'] = pe[:, 1]
submission['Confidence1'] = pe[:, 2] - pe[:, 0]<create_dataframe>
aisles=pd.read_csv('.. /input/aisles.csv')
 departments=pd.read_csv('.. /input/departments.csv')
 orders=pd.read_csv('.. /input/orders.csv')
 orderp=pd.read_csv('.. /input/order_products__prior.csv')
 ordert=pd.read_csv('.. /input/order_products__train.csv')
 products=pd.read_csv('.. /input/products.csv' )
subm = submission[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()<feature_engineering>
goods = pd.merge(left=pd.merge(left=products, right=departments, how='left'), right=aisles, how='left')
 goods.product_name = goods.product_name.str.replace(' ', '_' ).str.lower()
goods.head()
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if unc_mean<70:
subm['Confidence'] = mean_absolute_error
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<load_from_csv>
prior_df = pd.read_csv(data_path + "order_products__prior.csv")
 prior_df = pd.merge(prior_df, orders_df, how="inner", on="order_id")
 prior_grouped_df = prior_df.groupby("user_id")["order_number"].aggregate("max" ).reset_index()
prior_df_latest = pd.merge(prior_df, prior_grouped_df, how="inner", on=["user_id", "order_number"])
 prior_df_latest = prior_df_latest[["user_id", "product_id", "reordered"]]
prior_df_latest.columns = ["user_id", "product_id", "reordered_latest"]
prior_df = prior_df.groupby(["user_id","product_id"])["reordered"].aggregate(["count", "sum"] ).reset_index()
prior_df.columns = ["user_id", "product_id", "reordered_count", "reordered_sum"]
prior_df = pd.merge(prior_df, prior_df_latest, how="left", on=["user_id","product_id"])
 prior_df.head()
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1<save_to_csv>
orders_df.drop(["order_number"],axis=1,inplace=True)
 train_df = pd.read_csv(data_path + "order_products__train.csv", usecols=["order_id"])
 train_df = train_df.groupby("order_id" ).aggregate("count" ).reset_index()
test_df = pd.read_csv(data_path + "sample_submission.csv", usecols=["order_id"])
 train_df = pd.merge(train_df, orders_df, how="inner", on="order_id")
 test_df = pd.merge(test_df, orders_df, how="inner", on="order_id")
 print(train_df.shape, test_df.shape )
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )<install_modules>
train_df = pd.merge(train_df, prior_df, how="inner", on="user_id")
 test_df = pd.merge(test_df, prior_df, how="inner", on="user_id")
 del prior_df, prior_grouped_df, prior_df_latest
print(train_df.shape, test_df.shape )
<import_modules>
products_df = pd.read_csv(data_path + "products.csv", usecols=["product_id", "aisle_id", "department_id"])
 train_df = pd.merge(train_df, products_df, how="inner", on="product_id")
 test_df = pd.merge(test_df, products_df, how="inner", on="product_id")
 del products_df
print(train_df.shape, test_df.shape )
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold<import_modules>
train_y_df = pd.read_csv(data_path + "order_products__train.csv", usecols=["order_id", "product_id", "reordered"])
 train_y_df = pd.merge(train_y_df, orders_df, how="inner", on="order_id")
 train_y_df = train_y_df[["user_id", "product_id", "reordered"]]
train_df = pd.merge(train_df, train_y_df, how="left", on=["user_id", "product_id"])
 train_df["reordered"].fillna(0, inplace=True)
 print(train_df.shape)
 del train_y_df
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M<set_options>
train_y = train_df.reordered.values
out_df = test_df[["order_id", "product_id"]]
train_df = np.array(train_df.drop(["order_id", "user_id", "reordered"], axis=1))
test_df = np.array(test_df.drop(["order_id", "user_id"], axis=1))
print(train_df.shape, test_df.shape )
warnings.filterwarnings("ignore")
<define_variables>
def runXGB(train_X, train_y, test_X, test_y=None, feature_names=None, seed_val=0):
params = {}
params["objective"] = "binary:logistic"
params['eval_metric'] = 'logloss'
params["eta"] = 0.05
params["subsample"] = 0.7
params["min_child_weight"] = 10
params["colsample_bytree"] = 0.7
params["max_depth"] = 8
params["silent"] = 1
params["seed"] = seed_val
num_rounds = 100
plst = list(params.items())
 xgtrain = xgb.DMatrix(train_X, label=train_y)
 if test_y is not None:
xgtest = xgb.DMatrix(test_X, label=test_y)
 watchlist = [(xgtrain,'train'),(xgtest, 'test')]
model = xgb.train(plst, xgtrain, num_rounds, watchlist, early_stopping_rounds=50, verbose_eval=10)
 else:
xgtest = xgb.DMatrix(test_X)
 model = xgb.train(plst, xgtrain, num_rounds)
 pred_test_y = model.predict(xgtest)
 return pred_test_y
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )<define_variables>
pred = runXGB(train_df, train_y, test_df)
 del train_df, test_df
cutoff = 0.2
pred[pred>=cutoff] = 1
pred[pred<cutoff] = 0
out_df["Pred"] = pred
out_df = out_df.ix[out_df["Pred"].astype('int')==1]
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE= 128<load_from_csv>
def merge_products(x):
return " ".join(list(x.astype('str')))
 out_df = out_df.groupby("order_id")["product_id"].aggregate(merge_products ).reset_index()
out_df.columns = ["order_id", "products"]
<concatenate><EOS> 
sub_df = pd.read_csv(data_path + "sample_submission.csv", usecols=["order_id"])
 sub_df = pd.merge(sub_df, out_df, how="left", on="order_id")
 sub_df["products"].fillna("None", inplace=True)
 sub_df.to_csv("xgb_starter_3450.csv", index=False )