Kaggle competition: Instacart Market Basket Analysis
<SOS> metric: MeanFScore Kaggle data source: instacart-market-basket-analysis<import_modules>
gc.enable()
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from sklearn.metrics import f1_score, roc_auc_score
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate
from keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D, merge, Masking,TimeDistributed
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.layers import concatenate<define_variables>
PATH = '/kaggle/input/instacart-market-basket-analysis/'
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }
mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }
def clean_text(x):
for dic in [contraction_mapping, mispell_dict, punct_mapping]:
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x<load_from_csv>
print('loading files...')
 order_products_prior = pd.read_csv(PATH + 'order_products__prior.csv')
 order_products_train = pd.read_csv(PATH + 'order_products__train.csv')
 orders = pd.read_csv(PATH + 'orders.csv')
 products = pd.read_csv(PATH + 'products.csv', usecols=['product_id', 'aisle_id', 'department_id'])
 orders.eval_set = orders.eval_set.replace({'prior': 0, 'train': 1, 'test':2})
 orders.days_since_prior_order = orders.days_since_prior_order.fillna(30)
 print('done loading' )
def load_and_prec() :
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_df['question_text'] = train_df['question_text'].fillna("" ).apply(lambda x: clean_text(x))
test_df['question_text'] = test_df['question_text'].fillna("" ).apply(lambda x: clean_text(x))
train_X = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
np.random.seed(2018)
 trn_idx = np.random.permutation(len(train_X))
train_X = train_X[trn_idx]
train_y = train_y[trn_idx]
return train_X, test_X, train_y, tokenizer.word_index<statistical_test>
prior_orders = pd.merge(orders, order_products_prior, on='order_id', how='inner')
 prior_orders.head()
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<choose_model_class>
del order_products_prior
gc.collect()
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<define_variables>
users = prior_orders.groupby(by='user_id')['order_number'].aggregate('max' ).to_frame('u_num_of_orders' ).reset_index()
users.head()
embed_size = 300
max_features = 95000
maxlen = 40<define_search_model>
total_prd_per_order = prior_orders.groupby(by=['user_id', 'order_id'])['product_id'].aggregate('count' ).to_frame('total_products_per_order' ).reset_index()
avg_products = total_prd_per_order.groupby(by=['user_id'])['total_products_per_order'].mean().to_frame('u_avg_prd' ).reset_index()
avg_products.head()
del [total_prd_per_order]
gc.collect()
avg_products.head()
def model_bilstm_attention(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.1 )(x)
 l = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 g = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = concatenate([l, g])
 x = Bidirectional(CuDNNLSTM(64//2, return_sequences=True))(x)
 x1 = Attention(maxlen )(x)
 x2 = GlobalAveragePooling1D()(x)
 x3 = GlobalMaxPooling1D()(x)
 x = Concatenate()([x1, x2, x3])
 x = Dense(16, activation='relu' )(x)
 x = Dropout(0.1 )(x)
 outp = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<train_model>
reorder_u = prior_orders.groupby(by='user_id')['reordered'].aggregate('mean' ).to_frame('u_reorder_ratio' ).reset_index()
reorder_u['u_reorder_ratio'] = reorder_u['u_reorder_ratio'].astype(np.float16)
 reorder_u.head()
def train_pred(model, train_X, train_y, val_X, val_y, epochs=2):
for step in range(epochs):
model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y), verbose=1)
 pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)
 best_thresh = 0.5
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(val_y,(pred_val_y > thresh ).astype(int))
if score > best_score:
best_thresh = thresh
best_score = score
print("Epoch: ", step, "- Val F1 Score: {:.4f}".format(best_score))
pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)
 print('=' * 60)
 return pred_val_y, pred_test_y, best_score<split>
avg_days = prior_orders.groupby(by='user_id')['days_since_prior_order'].aggregate('mean' ).to_frame('average_days_between_orders')
 avg_days = avg_days.reset_index()
avg_days.head()
%%time
train_X, test_X, train_y, word_index = load_and_prec()<train_on_grid>
total_item = prior_orders.groupby(by='user_id' ).size().to_frame('u_total_items_bought' ).astype(np.int16)
 total_item.head()
%%time
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_3 = load_para(word_index )<train_model>
users = users.merge(avg_products, on='user_id', how='left')
 users = users.merge(dow, on='user_id', how='left')
 users = users.merge(hod, on='user_id', how='left')
 users = users.merge(reorder_u, on='user_id', how='left')
 users = users.merge(avg_days, on='user_id', how='left')
 users = users.merge(total_item, on='user_id', how='left')
 users.head()
%%time
embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_3], axis = 0)
 np.shape(embedding_matrix )<compute_test_metric>
del [reorder_u, dow, hod, avg_products, avg_days, total_item]
gc.collect()
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(100)]:
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result<find_best_model_class>
prd = prior_orders.groupby(by='product_id')['order_id'].aggregate('count' ).to_frame('p_num_of_times' ).reset_index()
prd.head()
DATA_SPLIT_SEED = 2018
train_meta = np.zeros(train_y.shape)
 test_meta = np.zeros(test_X.shape[0])
 splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=DATA_SPLIT_SEED ).split(train_X, train_y))
for idx,(train_idx, valid_idx)in enumerate(splits):
X_train = train_X[train_idx]
y_train = train_y[train_idx]
X_val = train_X[valid_idx]
y_val = train_y[valid_idx]
model = model_bilstm_attention(embedding_matrix)
 pred_val_y, pred_test_y, best_score = train_pred(model, X_train, y_train, X_val, y_val, epochs = 4)
 train_meta[valid_idx] = pred_val_y.reshape(-1)
 test_meta += pred_test_y.reshape(-1)/ len(splits )<save_to_csv>
reorder_p = prior_orders.groupby(by='product_id')['reordered'].aggregate('mean' ).to_frame('p_reorder_ratio' ).reset_index()
reorder_p.head()
search_result = threshold_search(train_y, train_meta)
 print(search_result)
 sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.prediction = test_meta > search_result['threshold']
sub.to_csv("submission.csv", index=False )<define_variables>
add_to_cart = prior_orders.groupby(by='product_id')['add_to_cart_order'].aggregate('mean' ).to_frame('p_avg_cart_position' ).reset_index()
add_to_cart.head()
embed_size = 300
max_features = 95000
maxlen = 70<import_modules>
prd = prd.merge(reorder_p, on='product_id', how='left')
 prd = prd.merge(add_to_cart, on='product_id', how='left')
 del [reorder_p, add_to_cart]
gc.collect()
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from sklearn.metrics import f1_score, roc_auc_score
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate
from keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.layers import concatenate
from keras.optimizers import SGD<load_from_csv>
uxp = prior_orders.groupby(by=['user_id', 'product_id'])['order_id'].aggregate('count' ).to_frame('uxp_times_bought')
 uxp = uxp.reset_index()
uxp.head()
def load_and_prec() :
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_X = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
np.random.seed(2018)
 trn_idx = np.random.permutation(len(train_X))
train_X = train_X[trn_idx]
train_y = train_y[trn_idx]
return train_X, test_X, train_y, tokenizer.word_index<statistical_test>
times = prior_orders.groupby(by=['user_id', 'product_id'])['order_id'].aggregate('count' ).to_frame('times_bought')
 times = times.reset_index()
times.head()
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<import_modules>
total_orders = prior_orders.groupby('user_id')['order_number'].max().to_frame('total_orders' ).reset_index()
total_orders.head()
class Snapshot(Callback):
def __init__(self, folder_path, nb_epochs, nb_cycles=5, verbose=0):
if nb_cycles > nb_epochs:
raise ValueError('nb_epochs has to be lower than nb_cycles.')
 super(Snapshot, self ).__init__()
self.verbose = verbose
self.folder_path = folder_path
self.nb_epochs = nb_epochs
self.nb_cycles = nb_cycles
self.period = self.nb_epochs // self.nb_cycles
self.nb_digits = len(str(self.nb_cycles))
self.path_format = os.path.join(self.folder_path, 'weights_cycle_{}.h5')
 def on_epoch_end(self, epoch, logs=None):
if epoch == 0 or(epoch + 1)% self.period != 0: return
if not os.path.exists(self.folder_path):
os.makedirs(self.folder_path)
 cycle = int(epoch / self.period)
 cycle_str = str(cycle ).rjust(self.nb_digits, '0')
 self.model.save_weights(self.path_format.format(cycle_str), overwrite=True)
 K.set_value(self.model.optimizer.lr, self.base_lr)
 if self.verbose > 0:
print('
Epoch %05d: Reached %d-th cycle, saving model.' %(epoch, cycle))
def on_epoch_begin(self, epoch, logs=None):
if epoch <= 0: return
lr = self.schedule(epoch)
 K.set_value(self.model.optimizer.lr, lr)
 if self.verbose > 0:
print('
Epoch %05d: Snapchot modifying learning '
'rate to %s.' %(epoch + 1, lr))
def set_model(self, model):
self.model = model
if not hasattr(self.model.optimizer, 'lr'):
raise ValueError('Optimizer must have a "lr" attribute.')
 self.base_lr = float(K.get_value(self.model.optimizer.lr))
def schedule(self, epoch):
lr = math.pi *(epoch % self.period)/ self.period
lr = self.base_lr / 2 *(math.cos(lr)+ 1)
 return lr
def load_ensemble(folder, keep_last=None):
paths = glob.glob(os.path.join(folder, 'weights_cycle_*.h5'))
print('Found:', ', '.join(paths))
if keep_last is not None:
paths = sorted(paths)[-keep_last:]
print('Loading:', ', '.join(paths))
x_in = Input(shape=(maxlen,))
outputs = []
for i, path in enumerate(paths):
m = get_model(x_in)
 m.load_weights(path)
 outputs.append(m.output)
 shape = outputs[0].get_shape().as_list()
x = Lambda(lambda x: K.mean(K.stack(x, axis=0), axis=0),
output_shape=lambda _: shape )(outputs)
 model = Model(inputs=x_in, outputs=x)
 return model 
<compute_test_metric>
span = pd.merge(total_orders, first_order_num, on='user_id', how='right')
 span.head()
def f1(y_true, y_pred):

 def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))<choose_model_class>
span['Order_Range_D'] = span.total_orders - span.first_order_num + 1
span.head()
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
uxp_ratio = pd.merge(times, span, on=['user_id', 'product_id'], how='left')
 uxp_ratio.head()
def get_model(inp):
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.1 )(x)
 x = Bidirectional(CuDNNLSTM(40, return_sequences=True))(x)
 y = Bidirectional(CuDNNGRU(40, return_sequences=True))(x)
 atten_1 = Attention(maxlen )(x)
 atten_2 = Attention(maxlen )(y)
 avg_pool = GlobalAveragePooling1D()(y)
 max_pool = GlobalMaxPooling1D()(y)
 conc = concatenate([atten_1, atten_2, avg_pool, max_pool])
 conc = Dense(16, activation="relu" )(conc)
 conc = Dropout(0.1 )(conc)
 outp = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 return model<train_model>
uxp_ratio['uxp_reorder_ratio'] = uxp_ratio.times_bought / uxp_ratio.Order_Range_D
uxp_ratio.head()
def train_pred(model, train_X, train_y, val_X, val_y, epochs=2):
cbs = [Snapshot('snapshots', nb_epochs=epochs, verbose=0, nb_cycles=3)]
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[f1])
 model.fit(train_X, train_y, batch_size=512, epochs=epochs, validation_data=(val_X, val_y), callbacks=cbs, verbose=0)
 del model
model = load_ensemble('snapshots', 2)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[f1])
 pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)
 best_thresh = 0.5
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(val_y,(pred_val_y > thresh ).astype(int))
if score > best_score:
best_thresh = thresh
best_score = score
print("Val F1 Score: {:.4f}".format(best_score))
pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)
 return pred_val_y, pred_test_y, best_score<split>
uxp_ratio.drop(['times_bought', 'total_orders', 'first_order_num', 'Order_Range_D'], axis=1, inplace=True)
 uxp_ratio.head()
train_X, test_X, train_y, word_index = load_and_prec()
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_3 = load_para(word_index )<normalization>
del [times, span, first_order_num, total_orders]
gc.collect()
embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_3], axis = 0)
 np.shape(embedding_matrix )<compute_train_metric>
uxp = uxp.merge(uxp_ratio, on=['user_id', 'product_id'], how='left')
 del uxp_ratio
gc.collect()
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(100)]:
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result<predict_on_test>
DATA_SPLIT_SEED = 2018
train_meta = np.zeros(train_y.shape)
 test_meta = np.zeros(test_X.shape[0])
 splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=DATA_SPLIT_SEED ).split(train_X, train_y))
for idx,(train_idx, valid_idx)in enumerate(splits):
X_train = train_X[train_idx]
y_train = train_y[train_idx]
X_val = train_X[valid_idx]
y_val = train_y[valid_idx]
model = get_model(Input(shape=(maxlen,)))
 pred_val_y, pred_test_y, best_score = train_pred(model, X_train, y_train, X_val, y_val, epochs = 6)
 train_meta[valid_idx] = pred_val_y.reshape(-1)
 test_meta += pred_test_y.reshape(-1)/ len(splits )<save_to_csv>
prior_orders['order_number_back'] = prior_orders.groupby(by=['user_id'])['order_number'].transform(max)- prior_orders.order_number + 1
prior_orders.head()
search_result = threshold_search(train_y, train_meta)
 print(search_result)
 sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.prediction = test_meta > search_result['threshold']
sub.to_csv("submission.csv", index=False )<import_modules>
last_five['uxp_ratio_last_five'] = last_five.uxp_last_five / 5.0
last_five.head()
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from gensim.models import KeyedVectors
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate
from keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers<categorify>
def clean_text(x):
x = str(x)
 for punct in "/-'":
x = x.replace(punct, ' ')
 for punct in '&':
x = x.replace(punct, f' {punct} ')
 for punct in '?!.,"
x = x.replace(punct, '')
 return x<categorify>
uxp = uxp.merge(last_five, on=['user_id', 'product_id'], how='left')
 del [last_five, temp]
gc.collect()
uxp.head()
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x<string_transform>
uxp.fillna(0, inplace=True)
 uxp.head(10 )
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispell_dict = {'colour':'color',
'centre':'center',
'didnt':'did not',
'doesnt':'does not',
'isnt':'is not',
'shouldnt':'should not',
'favourite':'favorite',
'travelling':'traveling',
'counselling':'counseling',
'theatre':'theater',
'cancelled':'canceled',
'labour':'labor',
'organisation':'organization',
'wwii':'world war 2',
'citicise':'criticize',
'instagram': 'social medium',
'whatsapp': 'social medium',
'snapchat': 'social medium'
}
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text )<feature_engineering>
data = uxp.merge(users, on='user_id', how='left')
 data.head()
def cleansing(text):
text["question_text"] = text["question_text"].progress_apply(lambda x: clean_text(x))
text["question_text"] = text["question_text"].progress_apply(lambda x: clean_numbers(x))
text["question_text"] = text["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
sentences = text["question_text"].progress_apply(lambda x: x.split())
 to_remove = ['a','to','of','and']
sentences = [[word for word in sentence if not word in to_remove] for sentence in tqdm(sentences)]
return sentences<define_variables>
data = data.merge(prd, on='product_id', how='left')
 data.head()
embed_size = 300
max_features = 95000
maxlen = 70<load_from_csv>
del [users, prd, uxp]
gc.collect()
def load_and_prec() :
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_df, val_df = train_test_split(train_df, test_size=0.08, random_state=2018)
 train_X = cleansing(train_df)
 val_X = cleansing(val_df)
 test_X = cleansing(test_df)
 tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
np.random.seed(2018)
 trn_idx = np.random.permutation(len(train_X))
val_idx = np.random.permutation(len(val_X))
train_X = train_X[trn_idx]
val_X = val_X[val_idx]
train_y = train_y[trn_idx]
val_y = val_y[val_idx]
return train_X, val_X, test_X, train_y, val_y, tokenizer.word_index<statistical_test>
orders_future = orders.loc[(( orders.eval_set == 1)|(orders.eval_set == 2)) , ['user_id', 'eval_set', 'order_id']]
orders_future.head()
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_google(word_index):
news_path = '.. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'
embeddings_index = KeyedVectors.load_word2vec_format(news_path, binary=True)
 vocabulary_size=min(max_features, len(word_index)+1)
 embedding_matrix = np.zeros(( vocabulary_size, embed_size))
for word, i in word_index.items() :
if i>=max_features:
continue
try:
embedding_vector = embeddings_index[word]
embedding_matrix[i] = embedding_vector
except KeyError:
embedding_matrix[i]=np.random.normal(0,np.sqrt(0.25),embed_size)
 return embedding_matrix<choose_model_class>
data = data.merge(orders_future, on='user_id', how='left')
 data.head()
def model_cnn(embedding_matrix):
filter_sizes = [1,2,3,5]
num_filters = 36
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Reshape(( maxlen, embed_size, 1))(x)
 maxpool_pool = []
for i in range(len(filter_sizes)) :
conv = Conv2D(num_filters, kernel_size=(filter_sizes[i], embed_size),
kernel_initializer='he_normal', activation='relu' )(x)
 maxpool_pool.append(MaxPool2D(pool_size=(maxlen - filter_sizes[i] + 1, 1))(conv))
z = Concatenate(axis=1 )(maxpool_pool)
 z = Flatten()(z)
 z = Dropout(0.1 )(z)
 outp = Dense(1, activation="sigmoid" )(z)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<choose_model_class>
data_train = data[data.eval_set == 1]
data_train.head()
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
data_train = data_train.merge(order_products_train[['product_id', 'order_id', 'reordered']], on=['product_id', 'order_id'], how='left')
 data_train.head()
def model_lstm_atten(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<choose_model_class>
data_train.drop(['eval_set', 'order_id'], axis=1, inplace=True )
def model_gru_srk_atten(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model 
<choose_model_class>
data_test = data[data.eval_set == 2]
data_test.head()
def model_lstm_du(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 conc = Dense(64, activation="relu" )(conc)
 conc = Dropout(0.1 )(conc)
 outp = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<choose_model_class>
data_test.drop(['eval_set', 'order_id'], axis=1, inplace=True )
def model_gru_atten_3(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(100, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<find_best_params>
products = pd.read_csv(PATH + 'products.csv', usecols=['product_id', 'aisle_id', 'department_id'])
 data_train = data_train.merge(products, on='product_id', how='left')
 data_test = data_test.merge(products, on='product_id', how='left')
 data_train = data_train.set_index(['user_id', 'product_id'])
 data_test = data_test.set_index(['user_id', 'product_id'] )
def train_pred(model, epochs=2):
for e in range(epochs):
model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y))
pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)
 best_thresh = 0.5
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(val_y,(pred_val_y > thresh ).astype(int))
if score > best_score:
best_thresh = thresh
best_score = score
print("Val F1 Score: {:.4f}".format(best_score))
pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)
 return pred_val_y, pred_test_y, best_score<load_pretrained>
columns_mean = ['aisle_id', 'department_id']
for col in columns_mean:
mean = data_train.groupby(col ).reordered.mean()
data_train[col] = data_train[col].map(mean)
 data_test[col] = data_test[col].map(mean )
train_X, val_X, test_X, train_y, val_y, word_index = load_and_prec()
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_2 = load_fasttext(word_index)
 embedding_matrix_3 = load_para(word_index)
 embedding_matrix_4 = load_google(word_index )<feature_engineering>
del [data, orders_future, products, order_products_train]
gc.collect()
embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_2, embedding_matrix_3, embedding_matrix_4], axis = 0)
 np.shape(embedding_matrix )<define_variables>
from sklearn.model_selection import train_test_split, GridSearchCV, KFold
from sklearn.metrics import f1_score, classification_report
from scikitplot.metrics import plot_confusion_matrix
from scikitplot.classifiers import plot_feature_importances
import xgboost as xgb
import lightgbm as lgb
outputs = []<predict_on_test>
X = data_train.drop(['reordered', 'uxp_ratio_last_five'], axis=1)
 y = data_train.reordered
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=10 )
pred_val_y, pred_test_y, best_score = train_pred(model_gru_atten_3(embedding_matrix), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, '3 GRU w/ atten'] )<predict_on_test>
del [data_train]
gc.collect()
pred_val_y, pred_test_y, best_score = train_pred(model_gru_srk_atten(embedding_matrix), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, 'gru atten srk'] )<predict_on_test>
parameters = {
'eavl_metric' : 'logloss',
'max_depth' : 5,
'colsample_bytree' : 0.4,
'subsample' : 0.8
}
pred_val_y, pred_test_y, best_score = train_pred(model_cnn(embedding_matrix), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, best_score, '2d CNN'] )<predict_on_test>
xgb.fit(data_train.drop(['reordered', 'uxp_ratio_last_five'], axis=1), data_train.reordered )
pred_val_y, pred_test_y, best_score = train_pred(model_cnn(embedding_matrix_1), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, '2d CNN GloVe'] )<predict_on_test>
y_pred_test =(xgb.predict_proba(data_test.drop('uxp_ratio_last_five', axis=1)) [:, 1] >= 0.21 ).astype('int' )
pred_val_y, pred_test_y, best_score = train_pred(model_lstm_du(embedding_matrix), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, 'LSTM DU'] )<predict_on_test>
data_test['prediction'] = y_pred_test
data_test.head()
pred_val_y, pred_test_y, best_score = train_pred(model_lstm_atten(embedding_matrix), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, '2 LSTM w/ attention'] )<predict_on_test>
orders = pd.read_csv(PATH + 'orders.csv')
 orders_test = orders.loc[orders.eval_set == 'test', ['user_id', 'order_id']]
orders_test.head()
pred_val_y, pred_test_y, best_score = train_pred(model_lstm_atten(embedding_matrix_1), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, '2 LSTM w/ attention GloVe'] )<predict_on_test>
final = final.merge(orders_test, on='user_id', how='left')
 final.head()
pred_val_y, pred_test_y, best_score = train_pred(model_lstm_atten(embedding_matrix_2), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, '2 LSTM w/ attention Wiki'] )<predict_on_test>
final = final.drop('user_id', axis=1 )
pred_val_y, pred_test_y, best_score = train_pred(model_lstm_atten(embedding_matrix_3), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, '2 LSTM w/ attention Para'] )<predict_on_test>
final['product_id'] = final.product_id.astype(int)
 del orders
del orders_test
gc.collect()
final.head()
pred_val_y, pred_test_y, best_score = train_pred(model_lstm_atten(embedding_matrix_4), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, '2 LSTM w/ attention Google'] )<sort_values>
d = dict()
for row in final.itertuples() :
if row.prediction== 1:
try:
d[row.order_id] += ' ' + str(row.product_id)
 except:
d[row.order_id] = str(row.product_id)
 for order in final.order_id:
if order not in d:
d[order] = 'None'
gc.collect()
outputs.sort(key=lambda x: x[2])
 weights = [i for i in range(1, len(outputs)+ 1)]
weights = [float(i)/ sum(weights)for i in weights]
print(weights )<compute_test_metric>
sub = pd.DataFrame.from_dict(d, orient='index')
 sub.reset_index(inplace=True)
 sub.columns = ['order_id', 'products']
sub.head()
pred_val_y = np.mean([outputs[i][0] for i in range(len(outputs)) ], axis = 0)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(val_y,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh )<save_to_csv>
sub.to_csv('sub.csv', index=False, header=True )
<define_variables><EOS> 
del [X, y, X_train, y_train, y_test, X_test, xgb, y_pred]
gc.collect()