Kaggle competition: Instacart Market Basket Analysis
os.environ['PYTHONHASHSEED']=str(SEED)
 random.seed(SEED)
 np.random.seed(SEED)
 tf.random.set_seed(SEED )<load_from_csv>
IDIR = '.. /input/'
print('loading prior')
 priors = pd.read_csv(IDIR + 'order_products__prior.csv', dtype={
'order_id': np.int32,
'product_id': np.uint16,
'add_to_cart_order': np.int16,
'reordered': np.int8})
 print('loading train')
 train = pd.read_csv(IDIR + 'order_products__train.csv', dtype={
'order_id': np.int32,
'product_id': np.uint16,
'add_to_cart_order': np.int16,
'reordered': np.int8})
 print('loading orders')
 orders = pd.read_csv(IDIR + 'orders.csv', dtype={
'order_id': np.int32,
'user_id': np.int32,
'eval_set': 'category',
'order_number': np.int16,
'order_dow': np.int8,
'order_hour_of_day': np.int8,
'days_since_prior_order': np.float32})
 print('loading products')
 products = pd.read_csv(IDIR + 'products.csv', dtype={
'product_id': np.uint16,
'order_id': np.int32,
'aisle_id': np.uint8,
'department_id': np.uint8},
usecols=['product_id', 'aisle_id', 'department_id'])
 print('priors {}: {}'.format(priors.shape, ', '.join(priors.columns)))
 print('orders {}: {}'.format(orders.shape, ', '.join(orders.columns)))
 print('train {}: {}'.format(train.shape, ', '.join(train.columns)))
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv" )<feature_engineering>
print("Computing Product f")
 prods = pd.DataFrame()
prods['orders'] = priors.groupby(priors.product_id ).size().astype(np.int32)
 prods['reorders'] = priors['reordered'].groupby(priors.product_id ).sum().astype(np.int32)
 prods['reorder_rate'] =(prods.reorders/prods.orders ).astype(np.float32)
 products = products.join(prods, on='product_id')
 products.set_index('product_id', drop=False, inplace=True)
 products.head()
del prods
test["target"] = -1<concatenate>
orders.set_index('order_id', inplace=True, drop=False )
df = pd.concat([train, test] )<feature_engineering>
priors = priors.join(orders, on='order_id', rsuffix='_' )
PUNCT_TO_REMOVE = string.punctuation
def remove_punctuation(text):
return text.translate(str.maketrans('', '', PUNCT_TO_REMOVE))
df["text"] = df["text"].apply(lambda text: remove_punctuation(text))<drop_column>
priors.drop('order_id_', inplace=True, axis=1 )
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 df["text"] = df["text"].apply(lambda text: remove_emoji(text))<feature_engineering>
print("Computing User Features")
 usr = pd.DataFrame()
usr['average_days_between_orders'] = orders.groupby('user_id')['days_since_prior_order'].mean().astype(np.float32)
 usr['nb_orders'] = orders.groupby('user_id' ).size().astype(np.int16)
 usr.head()
def remove_urls(text):
url_pattern = re.compile(r'https?://\S+|www\.\S+')
 try:
return url_pattern.sub(r'', text)
 except:
print(text)
 df["text"] = df["text"].apply(lambda text: remove_urls(text))<feature_engineering>
users = pd.DataFrame()
users['total_items'] = priors.groupby('user_id' ).size().astype(np.int16)
 users['all_products'] = priors.groupby('user_id')['product_id'].apply(set)
 users['total_distinct_items'] = users.all_products.map(len ).astype(np.int16 )
def remove_html(text):
html_pattern = re.compile('<.*?>')
 return html_pattern.sub(r'', text)
 df["text"] = df["text"].apply(lambda text: remove_html(text))<feature_engineering>
users = users.join(usr)
 del usr
with open(".. /input/slangtext/slang.txt", "r")as file:
chat_words_str = file.read()
chat_words_map_dict = {}
chat_words_list = []
for line in chat_words_str.split("
"):
if line != "" and "=" in line:
cw = line.split("=")[0]
cw_expanded = line.split("=")[1]
chat_words_list.append(cw)
 chat_words_map_dict[cw] = cw_expanded
chat_words_list = set(chat_words_list )<feature_engineering>
users['average_basket'] = users.total_items/users.nb_orders
users.head()
def chat_words_conversion(text):
new_text = []
for w in text.split() :
if w.upper() in chat_words_list:
new_text.append(chat_words_map_dict[w.upper() ])
 else:
new_text.append(w)
 return " ".join(new_text)
 df["text"] = df["text"].apply(lambda text: chat_words_conversion(text))<feature_engineering>
priors['user_product'] = priors.product_id + priors.user_id * 100000
spell = SpellChecker()
def correct_spellings(text):
corrected_text = []
misspelled_words = spell.unknown(text.split())
 for word in text.split() :
if word in misspelled_words:
corrected_text.append(spell.correction(word))
else:
corrected_text.append(word)
 return " ".join(corrected_text)
 df["text"] = df["text"].apply(lambda text: chat_words_conversion(text))<split>
d = dict()
d= dict()
for row in priors.itertuples() :
z = row.user_product
if z not in d:
d[z] =(1,(row.order_number, row.order_id),row.add_to_cart_order)
 else:
d[z] =(d[z][0] + 1, max(d[z][1],(row.order_number, row.order_id)) , d[z][2] + row.add_to_cart_order)
train = df[df["target"]!=-1]
test = df[df["target"]==-1]<categorify>
userXproduct = pd.DataFrame.from_dict(d, orient='index')
 del d
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):
tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )<load_pretrained>
userXproduct.columns = ['nb_orders', 'last_order_id', 'sum_pos_in_cart']
userXproduct.nb_orders = userXproduct.nb_orders.astype(np.int16)
 userXproduct.last_order_id = userXproduct.last_order_id.map(lambda x: x[1] ).astype(np.int32)
 userXproduct.sum_pos_in_cart = userXproduct.sum_pos_in_cart.astype(np.int16)
 print('user X product f', len(userXproduct))
del priors
tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')
 save_path = 'distilbert_base_uncased/'
if not os.path.exists(save_path):
os.makedirs(save_path)
 tokenizer.save_pretrained(save_path)
 fast_tokenizer = BertWordPieceTokenizer('distilbert_base_uncased/vocab.txt', lowercase=True )<categorify>
print('split orders : train, test')
 test_orders = orders[orders.eval_set == 'test']
train_orders = orders[orders.eval_set == 'train']
train.set_index(['order_id', 'product_id'], inplace=True, drop=False )
X_test_tokenized = fast_encode(test["text"].astype(str), fast_tokenizer, maxlen=MAX_LEN )<compute_test_metric>
def features(selected_orders, labels_given=False):
print('build candidate list')
 order_list = []
product_list = []
labels = []
i=0
for row in selected_orders.itertuples() :
i+=1
if i%10000 == 0: print('order row',i)
 order_id = row.order_id
user_id = row.user_id
user_products = users.all_products[user_id]
product_list += user_products
order_list += [order_id] * len(user_products)
 if labels_given:
labels += [(order_id, product)in train.index for product in user_products]
df = pd.DataFrame({'order_id':order_list, 'product_id':product_list}, dtype=np.int32)
 labels = np.array(labels, dtype=np.int8)
 del order_list
del product_list
print('user related features')
 df['user_id'] = df.order_id.map(orders.user_id)
 df['user_total_orders'] = df.user_id.map(users.nb_orders)
 df['user_total_items'] = df.user_id.map(users.total_items)
 df['total_distinct_items'] = df.user_id.map(users.total_distinct_items)
 df['user_average_days_between_orders'] = df.user_id.map(users.average_days_between_orders)
 df['user_average_basket'] = df.user_id.map(users.average_basket)
 print('order related features')
 df['order_hour_of_day'] = df.order_id.map(orders.order_hour_of_day)
 df['days_since_prior_order'] = df.order_id.map(orders.days_since_prior_order)
 df['days_since_ratio'] = df.days_since_prior_order / df.user_average_days_between_orders
print('product related features')
 df['aisle_id'] = df.product_id.map(products.aisle_id)
 df['department_id'] = df.product_id.map(products.department_id)
 df['product_orders'] = df.product_id.map(products.orders ).astype(np.int32)
 df['product_reorders'] = df.product_id.map(products.reorders)
 df['product_reorder_rate'] = df.product_id.map(products.reorder_rate)
 print('user_X_product related features')
 df['z'] = df.user_id * 100000 + df.product_id
df.drop(['user_id'], axis=1, inplace=True)
 df['UP_orders'] = df.z.map(userXproduct.nb_orders)
 df['UP_orders_ratio'] =(df.UP_orders / df.user_total_orders ).astype(np.float32)
 df['UP_last_order_id'] = df.z.map(userXproduct.last_order_id)
 df['UP_average_pos_in_cart'] =(df.z.map(userXproduct.sum_pos_in_cart)/ df.UP_orders ).astype(np.float32)
 df['UP_reorder_rate'] =(df.UP_orders / df.user_total_orders ).astype(np.float32)
 df['UP_orders_since_last'] = df.user_total_orders - df.UP_last_order_id.map(orders.order_number)
 df['UP_delta_hour_vs_last'] = abs(df.order_hour_of_day - df.UP_last_order_id.map(orders.order_hour_of_day)).map(lambda x: min(x, 24-x)).astype(np.int8)
 df.drop(['UP_last_order_id', 'z'], axis=1, inplace=True)
 print(df.dtypes)
 print(df.memory_usage())
 return(df, labels )
def focal_loss(gamma=2., alpha=.25):
def focal_loss_fixed(y_true, y_pred):
pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
return -K.mean(alpha * K.pow(1.- pt_1, gamma)* K.log(pt_1)) - K.mean(( 1 - alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0))
return focal_loss_fixed<compute_test_metric>
df_train, labels = features(train_orders, labels_given=True)
def auc(y_true, y_pred):
def fallback_auc(y_true, y_pred):
try:
return roc_auc_score(y_true, y_pred)
 except:
return 0.5
return tf.py_function(fallback_auc,(y_true, y_pred), tf.double )<define_search_model>
f_to_use = ['user_total_orders', 'user_total_items', 'total_distinct_items',
'user_average_days_between_orders', 'user_average_basket',
'order_hour_of_day', 'days_since_prior_order', 'days_since_ratio',
'aisle_id', 'department_id', 'product_orders', 'product_reorders',
'product_reorder_rate', 'UP_orders', 'UP_orders_ratio',
'UP_average_pos_in_cart', 'UP_reorder_rate', 'UP_orders_since_last',
'UP_delta_hour_vs_last']
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 BATCH_SIZE = 16 * strategy.num_replicas_in_sync<choose_model_class>
d_train = lgb.Dataset(df_train[f_to_use],
label=labels,
categorical_feature=['aisle_id', 'department_id'])
 del df_train
def build_model(transformer, loss='binary_crossentropy', max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
x = Dropout(0.35 )(cls_token)
 out = Dense(2, activation='softmax' )(x)
 optimizer = tfa.optimizers.RectifiedAdam(lr=LR)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(optimizer=optimizer, loss=loss, metrics=[tf.keras.metrics.AUC() ])
 return model<choose_model_class>
params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': {'binary_logloss'},
'num_leaves': 96,
'max_depth': 10,
'feature_fraction': 0.9,
'bagging_fraction': 0.95,
'bagging_freq': 5
}
ROUNDS = 100
print('light GBM train :-)')
 bst = lgb.train(params, d_train, ROUNDS)
 del d_train
<choose_model_class><EOS> 
df_test, _ = features(test_orders)
 print('light GBM predict')
 preds = bst.predict(df_test[f_to_use])
 df_test['pred'] = preds
TRESHOLD = 0.22
d = dict()
for row in df_test.itertuples() :
if row.pred > TRESHOLD:
try:
d[row.order_id] += ' ' + str(row.product_id)
 except:
d[row.order_id] = str(row.product_id)
 for order in test_orders.order_id:
if order not in d:
d[order] = 'None'
sub = pd.DataFrame.from_dict(d, orient='index')
 sub.reset_index(inplace=True)
 sub.columns = ['order_id', 'products']
sub.to_csv('sub.csv', index=False )