Kaggle competition: Instacart Market Basket Analysis
<SOS> metric: MeanFScore Kaggle data source: instacart-market-basket-analysis<train_model>
!pip install turicreate
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
import os
import numpy as np
import pandas as pd
import turicreate
from zipfile import ZipFile
pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))))<predict_on_test>
ds_dir = '.. /input/instacart-market-basket-analysis'
pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
with ZipFile(os.path.join(ds_dir,"aisles.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"departments.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"order_products__prior.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"order_products__train.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"orders.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"products.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"sample_submission.csv.zip"), 'r')as zipObj:
zipObj.extractall()
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<statistical_test>
prior = pd.read_csv('order_products__prior.csv')
 train = pd.read_csv('order_products__train.csv')
 orders = pd.read_csv('orders.csv')
 products = pd.read_csv('products.csv')
 prior.fillna(0,inplace=True)
 train.fillna(0,inplace=True)
 orders.fillna(0,inplace=True )
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )<train_model>
data = pd.concat([train,prior])
 ou = orders[['order_id','user_id']]
data = pd.merge(data,ou,on='order_id' ).drop('order_id',axis=1 )
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
reordered = data.groupby(['user_id','product_id'] ).reordered.sum()
data = pd.merge(data,reordered, on=['user_id','product_id'], how='left')
 data = data[data.reordered_y>0]
data['target'] = np.log(data['reordered_y']+1)
 data.drop(['reordered_x','reordered_y','add_to_cart_order'],axis=1,inplace=True )
pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_fasttext_val_y>thresh ).astype(int))))<predict_on_test>
test = orders[orders.eval_set=='test']
test = test[['user_id']]
pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
relation = data[['user_id','product_id']].copy()
relation.drop_duplicates(inplace=True )
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<statistical_test>
sub = orders[orders.eval_set == 'test']
sub = sub[['order_id','user_id']]
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )<train_model>
train_data = turicreate.SFrame(data)
 test_data = turicreate.SFrame(test)
 relation_data = turicreate.SFrame(relation )
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
def clean_prediction(row):
data = row.products
data = str("".join(str(data)) [1:-1].replace(',',' '))
return data
pred_paragram_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_paragram_val_y>thresh ).astype(int))))<predict_on_test>
result = recommender.recommend(users=test_data, items=relation_data, exclude_known=False, k=len(products)).to_dataframe()
pred_paragram_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
for key,item in thresholds.items() :
print(key,':',str(item))
results[key] = result[result['score'] > item]
results[key] = results[key].groupby('user_id')['product_id'].apply(list ).reset_index(name='products')
 results[key]['products'] = results[key].apply(clean_prediction, axis=1 )
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<compute_test_metric>
for key,item in thresholds.items() :
results[key] = pd.merge(sub, results[key],how='outer',on='user_id' ).sort_values('user_id')
 results[key].fillna('None', inplace=True)
 results[key].drop('user_id',axis=1,inplace=True )
<save_to_csv><EOS> 
for key,item in thresholds.items() :
results[key].to_csv('submission_turi_ItemSim_pearson_pol'+str(key)+'.csv',index=False )