Kaggle competition: Instacart Market Basket Analysis
SEED = [0, 1, 2, 3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions 
<merge>
uxp = pd.read_pickle('.. /input/ml-instacart-f1-0-38-part-one-features/uxp.pkl')
 uxp.head()
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )<save_to_csv>
orders = pd.read_csv('.. /input/instacart-market-basket-analysis/orders.csv')
 order_products_train = pd.read_csv('.. /input/instacart-market-basket-analysis/order_products__train.csv')
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )<install_modules>
orders_last = orders[(orders.eval_set=='train')|(orders.eval_set=='test')]
!pip install transformers 
<import_modules>
uxp = uxp.merge(orders_last, on='user_id', how='left')
 uxp.head(10 )
pd.set_option('max_colwidth', 500)
 pd.set_option('display.float_format', lambda x: '%.4f' % x)
 tf.random.set_seed(42)
 random.seed(42)
 warnings.filterwarnings("ignore" )<load_from_csv>
uxp_train = uxp[uxp.eval_set=='train']
uxp_train = uxp_train.merge(order_products_train, on=['product_id', 'order_id'], how='left')
 uxp_train = uxp_train.drop(['order_id','eval_set', 'add_to_cart_order'], axis=1)
 uxp_train = uxp_train.fillna(0)
 uxp_train.head(20 )
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 ids_with_target_error = [328,443,513,2619,3640,3900,4342,5781,6552,6554,6570,6701,6702,6729,6861,7226]
train.loc[train['id'].isin(ids_with_target_error),'target'] = 0<categorify>
uxp_test = uxp[uxp.eval_set=='test']
uxp_test = uxp_test.drop(['eval_set', 'order_id'], axis=1)
 uxp_test = uxp_test.fillna(0)
 uxp_test.head(20 )
%%time
tokenizer = BertTokenizer.from_pretrained('bert-large-uncased', do_lower_case=True)
 def greed_encode(data, max_len):
input_ids = []
attention_masks = []
for i in range(len(data.text)) :
encoded = tokenizer.encode_plus(data.text[i], add_special_tokens=True, max_length=max_len, pad_to_max_length=True)
 tok_len = sum(encoded['attention_mask'])
 if tok_len > max_len*.8:
all_encode = tokenizer.encode_plus(data.text[i], add_special_tokens=True)
 all_ids = all_encode['input_ids']
all_attention = all_encode['attention_mask']
max_len_half = int(max_len/2)
 input_ids.append(all_ids[:max_len_half] + all_ids[-max_len_half:])
 attention_masks.append(all_attention[:max_len_half] + all_attention[-max_len_half:])
 else:
input_ids.append(encoded['input_ids'])
 attention_masks.append(encoded['attention_mask'])
 return np.array(input_ids),np.array(attention_masks)
 train_input_ids,train_attention_masks = greed_encode(train,50)
 test_input_ids,test_attention_masks = greed_encode(test,50)
 y_train = train.target<choose_model_class>
del uxp
del orders_last
gc.collect()
N_SAMPLES = 8
def create_model(bert_model, MAX_LEN=50):
input_ids = layers.Input(shape=(MAX_LEN,), dtype=tf.int32, name='input_ids')
 attention_mask = layers.Input(shape=(MAX_LEN,), dtype=tf.int32, name='attention_mask')
 last_hidden_state, _ = bert_model({'input_ids': input_ids, 'attention_mask': attention_mask})
 last_hidden_state = Dropout(0.1 )(last_hidden_state)
 x_avg = layers.GlobalAveragePooling1D()(last_hidden_state)
 x_max = layers.GlobalMaxPooling1D()(last_hidden_state)
 x = layers.Concatenate()([x_avg, x_max])
 samples = []
for n in range(N_SAMPLES):
sample_mask = layers.Dense(64, activation='relu', name = f'dense_{n}')
 sample = layers.Dropout (.5 )(x)
 sample = sample_mask(sample)
 sample = layers.Dense(1, activation='sigmoid', name=f'sample_{n}' )(sample)
 samples.append(sample)
 output = layers.Average(name='output' )(samples)
 model = Model(inputs=[input_ids, attention_mask], outputs=output)
 model.compile(Adam(lr=1e-5), loss = BinaryCrossentropy(label_smoothing=0.1), metrics=['accuracy'])
 return model<load_pretrained>
uxp_train.loc[:, 'reordered'] = uxp_train.reordered.fillna(0)
 X_train, X_val, y_train, y_val = train_test_split(uxp_train.drop('reordered', axis=1), uxp_train.reordered,
test_size=0.2, random_state=42)

gc.collect()
d_train = xgboost.DMatrix(X_train, y_train)
 param = {'max_depth':10,
'eta':0.02,
'colsample_bytree':0.4,
'subsample':0.75,
'silent':1,
'nthread':27,
'eval_metric':'logloss',
'binary':'logistic',
'tree_method':'hist'
}
watchlist= [(d_train, "train")]
bst = xgboost.train(params=param, dtrain=d_train, num_boost_round=1000, evals=watchlist, early_stopping_rounds=40, verbose_eval=5)
 xgboost.plot_importance(bst )
bert_model = TFBertModel.from_pretrained('bert-large-uncased')
 model = create_model(bert_model)
 model.summary()<predict_on_test>
del [X_train, X_val, y_train, y_val]
gc.collect()
def lgb_svc_cv(model):
cls_layer_model = Model(model.input, outputs=model.get_layer(f'dense_1' ).output)
 X_train = cls_layer_model.predict([train_input_ids,train_attention_masks])
 X_test = cls_layer_model.predict([test_input_ids,test_attention_masks])
 y_train = train.target.values
N_FOLDS = 5
print(f'LGBM')
 folds = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=42)
 oof = np.zeros(len(X_train))
sub = np.zeros(len(X_test))
params = {'boosting_type': 'dart'}
for fold_,(train_idx, val_idx)in enumerate(folds.split(X_train, y_train)) :
X_train_cv, y_train_cv = pd.DataFrame(X_train ).loc[train_idx], pd.DataFrame(y_train ).loc[train_idx]
X_val, y_val = pd.DataFrame(X_train ).loc[val_idx], pd.DataFrame(y_train ).loc[val_idx]
train_data = lgb.Dataset(X_train_cv, label=y_train_cv)
 val_data = lgb.Dataset(X_val, label=y_val)
 watchlist = [train_data, val_data]
clf = lgb.train(params, train_set = train_data, valid_sets=watchlist)
 oof[val_idx] = clf.predict(X_val)
 sub += clf.predict(X_test)/folds.n_splits
sub_all_lgb = sub
print(accuracy_score(y_train, np.round(oof ).astype(int)) ,'
')
 print(f'SVC')
 folds = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=42)
 oof = np.zeros(len(X_train))
sub = np.zeros(len(X_test))
scores = [0 for _ in range(folds.n_splits)]
for fold_,(train_idx, val_idx)in enumerate(folds.split(X_train, y_train)) :
X_train_cv, y_train_cv = pd.DataFrame(X_train ).loc[train_idx], pd.DataFrame(y_train ).loc[train_idx]
X_val, y_val = pd.DataFrame(X_train ).loc[val_idx], pd.DataFrame(y_train ).loc[val_idx]
clf = SVC(kernel='rbf', C=1.75, gamma = 0.1, probability = True ).fit(X_train_cv, y_train_cv)
 oof[val_idx] = clf.predict_proba(X_val)[:,1]
sub += clf.predict_proba(X_test)[:,1]/folds.n_splits
sub_all_svc = sub
print(accuracy_score(y_train, np.round(oof ).astype(int)) ,'
')
 return sub_all_lgb, sub_all_svc<load_pretrained>
d_test = xgboost.DMatrix(uxp_test)
 uxp_test = uxp_test.reset_index()
uxp_test = uxp_test[['product_id', 'user_id']]
uxp_test["reordered"] = bst.predict(d_test)
 del bst
oof_preds = np.zeros(train_input_ids.shape[0])
 test_preds = np.zeros(test.shape[0])
 all_preds = pd.DataFrame()
fold_hist = {}
n_splits = 2
folds = KFold(n_splits=n_splits, shuffle=True, random_state=42)
 for i,(trn_idx, val_idx)in enumerate(folds.split(train_input_ids)) :
modelstart = time.time()
bert_model = TFBertModel.from_pretrained('bert-large-uncased')
 model = create_model(bert_model)
 es = callbacks.EarlyStopping(monitor='val_loss', min_delta=0.001, patience=5, verbose=1,
mode='min', baseline=None, restore_best_weights=True)
 def lr_sc(epoch):
return 1.5e-5/(epoch + 1)
 scheduler = LearningRateScheduler(lr_sc)
 history = model.fit(
 x=[train_input_ids[trn_idx], train_attention_masks[trn_idx]], y=y_train[trn_idx],
validation_data=([train_input_ids[val_idx], train_attention_masks[val_idx]], y_train[val_idx]),
epochs=3,
batch_size=16,
callbacks=[scheduler, es] 
)
 best_index = np.argmin(history.history['val_loss'])
 fold_hist[i] = history
oof_preds[val_idx] = model.predict([train_input_ids[val_idx], train_attention_masks[val_idx]] ).ravel()
all_preds[str(i)+ '_fold_NN'] = model.predict([test_input_ids,test_attention_masks] ).reshape(-1)
 sc = accuracy_score(y_train[val_idx],(oof_preds[val_idx] > 0.5 ).astype(int))
print("
FOLFD {} in {:.1f} min - Avg Acc NN {:.5f} - Best Epoch {}".format(i,(time.time() - modelstart)/60, sc, best_index + 1),'
')
 all_preds[str(i)+ '_fold_lgb'], all_preds[str(i)+ '_fold_svc'] = lgb_svc_cv(model)
 del model
K.clear_session()
gc.collect()<prepare_output>
orders_test = orders[orders.eval_set=='test']
test_pred = all_preds[['0_fold_NN','1_fold_NN']].mean(axis = 1)*.8 + all_preds[['0_fold_lgb','1_fold_lgb']].mean(axis = 1)*.1 + all_preds[['0_fold_svc','1_fold_svc']].mean(axis = 1)*.1<load_from_csv>
uxp_test = uxp_test.merge(orders_test[["user_id", "order_id"]], on='user_id', how='left' ).drop('user_id', axis=1)
 uxp_test.columns = ['product_id', 'prediction', 'order_id']
uxp_test.product_id = uxp_test.product_id.astype(int)
 uxp_test.order_id = uxp_test.order_id.astype(int)
 uxp_test.head()
submission = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 submission['target'] = np.round(test_pred ).astype(int)
 submission.head(10 )<save_to_csv>
del orders
del orders_test
gc.collect()
submission.to_csv('submission.csv', index=False )<save_to_csv>
class F1Optimizer() :
def __init__(self):
pass
@staticmethod
def get_expectations(P, pNone=None):
expectations = []
P = np.sort(P)[::-1]
n = np.array(P ).shape[0]
DP_C = np.zeros(( n + 2, n + 1))
if pNone is None:
pNone =(1.0 - P ).prod()
DP_C[0][0] = 1.0
for j in range(1, n):
DP_C[0][j] =(1.0 - P[j - 1])* DP_C[0, j - 1]
for i in range(1, n + 1):
DP_C[i, i] = DP_C[i - 1, i - 1] * P[i - 1]
for j in range(i + 1, n + 1):
DP_C[i, j] = P[j - 1] * DP_C[i - 1, j - 1] +(1.0 - P[j - 1])* DP_C[i, j - 1]
DP_S = np.zeros(( 2 * n + 1,))
DP_SNone = np.zeros(( 2 * n + 1,))
for i in range(1, 2 * n + 1):
DP_S[i] = 1./(1.* i)
 DP_SNone[i] = 1./(1.* i + 1)
 for k in range(n + 1)[::-1]:
f1 = 0
f1None = 0
for k1 in range(n + 1):
f1 += 2 * k1 * DP_C[k1][k] * DP_S[k + k1]
f1None += 2 * k1 * DP_C[k1][k] * DP_SNone[k + k1]
for i in range(1, 2 * k - 1):
DP_S[i] =(1 - P[k - 1])* DP_S[i] + P[k - 1] * DP_S[i + 1]
DP_SNone[i] =(1 - P[k - 1])* DP_SNone[i] + P[k - 1] * DP_SNone[i + 1]
expectations.append([f1None + 2 * pNone /(2 + k), f1])
 return np.array(expectations[::-1] ).T
@staticmethod
def maximize_expectation(P, pNone=None):
expectations = F1Optimizer.get_expectations(P, pNone)
 ix_max = np.unravel_index(expectations.argmax() , expectations.shape)
 max_f1 = expectations[ix_max]
predNone = True if ix_max[0] == 0 else False
best_k = ix_max[1]
return best_k, predNone, max_f1
@staticmethod
def _F1(tp, fp, fn):
return 2 * tp /(2 * tp + fp + fn)
 @staticmethod
def _Fbeta(tp, fp, fn, beta=1.0):
beta_squared = beta ** 2
return(1.0 + beta_squared)* tp /(( 1.0 + beta_squared)* tp + fp + beta_squared * fn)
 def get_best_prediction(items, preds, pNone=None):
items_preds = sorted(list(zip(items, preds)) , key=itemgetter(1), reverse=True)
 P = [p for i,p in items_preds]
L = [i for i,p in items_preds]
opt = F1Optimizer.maximize_expectation(P)
 best_prediction = []
best_prediction +=(L[:opt[0]])
 if best_prediction == []:
best_prediction = ['None']
".format(best_prediction, f1_max))
return ' '.join(list(map(str,best_prediction)) )
submission.to_csv('submission.csv', index=False )<install_modules>
sub_item = uxp_test.groupby(['order_id','product_id'] ).prediction.mean().reset_index()
sub = sub_item.groupby('order_id' ).product_id.apply(list ).to_frame()
sub['yhat'] = sub_item.groupby('order_id' ).prediction.apply(list)
 sub.reset_index(inplace=True)
 del uxp_test, sub_item
gc.collect()
def multi(i):
if i%1000==0:
print('{:.3f} min'.format(( time.time() -st_time)/60))
items = sub.loc[i,'product_id']
preds = sub.loc[i,'yhat']
ret = get_best_prediction(items, preds)
 return ret
st_time = time.time()
pool = mp.Pool(4)
 callback = pool.map(multi, range(sub.shape[0]))
sub['products'] = callback
sub.head()
!pip -q install pyspellchecker<import_modules>
sub.reset_index(inplace=True)
 sub = sub[['order_id', 'products']]
import os
import numpy as np
import pandas as pd
import random
import re
import string
from spellchecker import SpellChecker
from tqdm.notebook import tqdm
import warnings
from sklearn.metrics import roc_auc_score, f1_score, matthews_corrcoef
from sklearn.model_selection import StratifiedKFold
import tensorflow as tf
import tensorflow_addons as tfa
import tensorflow.keras.backend as K
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.layers import Dropout, Dense, Input
from tensorflow.keras.callbacks import EarlyStopping, LearningRateScheduler
from tensorflow.keras.models import Model
from tokenizers import BertWordPieceTokenizer
from transformers import TFBertModel, AdamWeightDecay, BertTokenizerFast<set_options>
warnings.filterwarnings("ignore" )<define_variables>

 print(sub.shape[0])
 print(sub.shape[0]==75000)
 sub.to_csv('sub.csv', index=False )
SEED = 42
MAX_LEN = 192
EPOCHS = 6
NUM_SPLITS = 5
LR = 3e-5<feature_engineering>
submission = pd.read_csv(".. /working/sub.csv")
 submission.head()