Kaggle competition: Instacart Market Basket Analysis
<SOS> metric: MeanFScore Kaggle data source: instacart-market-basket-analysis<split>
gc.enable()
def train_val_split(df,frac=0.2):
df=df.sample(frac=1 ).reset_index(drop=True)
 df_sincere=df[df['target']==0]
df_taxic=df[df['target']==1]
sincere_border=int(len(df_sincere)*(1-frac))
taxic_border=int(len(df_taxic)*(1-frac))
df_train=pd.concat([df_sincere[:sincere_border],df_taxic[:taxic_border]])
 df_val=pd.concat([df_sincere[sincere_border:],df_taxic[taxic_border:]])
 print('Training data peppered(prepared)!')
 return df_train.sample(frac=1 ).reset_index(drop=True), df_val.sample(frac=1 ).reset_index(drop=True )<choose_model_class>
orders = pd.read_csv('.. /input/orders.csv')
 order_products_train = pd.read_csv('.. /input/order_products__train.csv')
 order_products_prior = pd.read_csv('.. /input/order_products__prior.csv')
 products = pd.read_csv('.. /input/products.csv')
 aisles = pd.read_csv('.. /input/aisles.csv')
 departments = pd.read_csv('.. /input/departments.csv' )
def train_model(train_df,val_df,n_epochs=8,batch_size=1024):
model = Sequential()
model.add(Bidirectional(CuDNNLSTM(64, return_sequences=True),
input_shape=(60, 300)))
 model.add(Bidirectional(CuDNNLSTM(32)))
 model.add(Dense(64))
model.add(Activation('relu'))
model.add(Dense(32))
model.add(Activation('relu'))
model.add(Dense(16))
model.add(Activation('relu'))
model.add(Dense(1,activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 model.fit_generator(batch_gen(train_df, min_batch_size=batch_size), epochs=n_epochs,
steps_per_epoch=math.ceil(len(train_df)/batch_size),
validation_data=batch_gen(val_df, min_batch_size=batch_size),
validation_steps=math.ceil(len(val_df)/batch_size),
verbose=True)
 return model<predict_on_test>
aisles['aisle'] = aisles['aisle'].astype('category')
 departments['department'] = departments['department'].astype('category')
 orders['eval_set'] = orders['eval_set'].astype('category')
 products['product_name'] = products['product_name'].astype('category' )
def get_predictions(df, model, min_batch_size=1024,transformation_func=str_prep):
predictions=np.ndarray(shape=(0,1))
print(f'Total values to predict: {len(df)}')
 n_batches=math.ceil(len(df)/min_batch_size)
 for batch in trange(n_batches):
start=batch*min_batch_size
end=start+min_batch_size
if batch==n_batches-1:
X=df['question_text'][start:].apply(transformation_func)
 X=pad_sequences(X,maxlen=60,dtype='float32',padding='post')
 y_predicted=model.predict(X)
 else:
X=df['question_text'][start:end].apply(transformation_func)
 X=pad_sequences(X,maxlen=60,dtype='float32',padding='post')
 y_predicted=model.predict(X)
 predictions=np.append(predictions,y_predicted,axis=0)
 return predictions<find_best_params>
op = orders.merge(order_products_prior, on='order_id', how='inner')
 op.head()
def select_best_threshold(y_true, y_predicted):
thresholds = []
for thresh in np.arange(0.0, 1, 0.001):
thresh = np.round(thresh, 3)
 res=metrics.f1_score(y_true,(y_predicted>thresh ).astype(int))
thresholds.append([thresh, res])
 thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh)
 return best_thresh<train_model>
user = op.groupby('user_id')['order_number'].max().to_frame('u_total_orders')
 user.head()
stack_size=3
model_stack={}
for ind in range(stack_size):
print(f'
=== Model number {ind+1} ===
')
 train_df, val_df=train_val_split(df, frac=0.1)
 model=train_model(train_df,val_df, n_epochs=5)
 y_predicted=get_predictions(val_df,model)
 y_true=val_df['target'].values
best_thresh=select_best_threshold(y_true, y_predicted)
 model_stack[ind]=(model, best_thresh)
 print(f'Model number {ind+1} finished!' )<load_from_csv>
dow=op.groupby('user_id')['order_dow'].mean().to_frame('average_dow')
 dow=dow.reset_index()
dow.head()
test_set=pd.read_csv('.. /input/test.csv' )<predict_on_test>
user = user.merge(u_reorder, on='user_id', how='left')
 del u_reorder
gc.collect()
user.head()
predictions={}
for key,(model, threshold)in model_stack.items() :
predictions[key]=(get_predictions(test_set,model)>threshold ).astype(int )<data_type_conversions>
prd = op.groupby('product_id')['order_id'].count().to_frame('p_total_purchases')
 prd.head()
preds=np.stack([value for value in predictions.values() ],axis=1 ).reshape(-1,stack_size)
 preds=np.average(preds,axis=1)
 preds=np.around(preds,decimals=0 ).astype(int )<save_to_csv>
p_reorder = op.groupby('product_id' ).filter(lambda x: x.shape[0] >40)
 p_reorder.head()
test_set['prediction']=preds
to_submit=test_set[['qid','prediction']]
to_submit.to_csv("submission.csv", index=False)
 print('Submissions saved to file!' )<set_options>
prd = prd.merge(p_reorder, on='product_id', how='left')
 del p_reorder
gc.collect()
prd.head()
tqdm.pandas()
%matplotlib inline
py.init_notebook_mode(connected=True)
 print(os.listdir(".. /input")) 
<load_from_csv>
prd=prd.merge(Addtocart, on='product_id',how='left')
 del Addtocart
gc.collect()
prd.head()
train=pd.read_csv(".. /input/train.csv")
 test=pd.read_csv(".. /input/test.csv" )<feature_engineering>
prd['p_reorder_ratio'] = prd['p_reorder_ratio'].fillna(value=0)
 prd.head()
def build_vocab(sentences,verbose=True):
vocab={}
for sentence in tqdm(sentences,disable=(not verbose)) :
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab 
<string_transform>
uxp = op.groupby(['user_id', 'product_id'])['order_id'].count().to_frame('uxp_total_bought')
 uxp.head()
sentences = train["question_text"].progress_apply(lambda x: x.split() ).values
vocab = build_vocab(sentences)
 print({k: vocab[k] for k in list(vocab)[:5]} )<load_pretrained>
times = op.groupby(['user_id', 'product_id'])[['order_id']].count()
times.columns = ['Times_Bought_N']
times.head()
news_path = '.. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'
word2vecDict= KeyedVectors.load_word2vec_format(news_path, binary=True)
<categorify>
total_orders = op.groupby('user_id')['order_number'].max().to_frame('total_orders')
 total_orders.head()
def check_coverage(vocab,word2vecDict):
a = {}
oov = {}
k = 0
i = 0
for word in tqdm(vocab):
try:
a[word] = word2vecDict[word]
k += vocab[word]
except:
oov[word] = vocab[word]
i += vocab[word]
pass
print('Found embeddings for {:.2%} of vocab'.format(len(a)/ len(vocab)))
 print('Found embeddings for {:.2%} of all text'.format(k /(k + i)))
 sorted_x = sorted(oov.items() , key=operator.itemgetter(1)) [::-1]
return sorted_x<compute_test_metric>
span = pd.merge(total_orders, first_order_no, on='user_id', how='right')
 span.head()
oov = check_coverage(vocab,word2vecDict )<categorify>
span['Order_Range_D'] = span.total_orders - span.first_order_number + 1
span.head()
def clean_text(x):
x = str(x)
 for punct in "/-'":
x = x.replace(punct, ' ')
 for punct in '&':
x = x.replace(punct, f' {punct} ')
 for punct in '?!.,"
x = x.replace(punct, '')
 return x 
<feature_engineering>
uxp_ratio = pd.merge(times, span, on=['user_id', 'product_id'], how='left')
 uxp_ratio.head()
train_df["question_text"] = train["question_text"].progress_apply(lambda x: clean_text(x))
test_df["question_text"] = test["question_text"].progress_apply(lambda x: clean_text(x))
sentences = train["question_text"].apply(lambda x: x.split())
 vocab = build_vocab(sentences )<compute_test_metric>
uxp_ratio['uxp_reorder_ratio'] = uxp_ratio.Times_Bought_N / uxp_ratio.Order_Range_D
uxp_ratio.head()
oov = check_coverage(vocab,word2vecDict )<define_variables>
uxp_ratio = uxp_ratio.drop(['Times_Bought_N', 'total_orders', 'first_order_number', 'Order_Range_D'], axis=1)
 uxp_ratio.head()
for i in range(10):
print(word2vecDict.index2entity[i] )<feature_engineering>
del [times, first_order_no, span]
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}',"
return x
train_df["question_text"] = train["question_text"].progress_apply(lambda x: clean_numbers(x))
test_df["question_text"] = test["question_text"].progress_apply(lambda x: clean_numbers(x))<string_transform>
uxp = uxp.merge(uxp_ratio, on=['user_id', 'product_id'], how='left')
 del uxp_ratio
uxp.head()
sentences = train_df["question_text"].progress_apply(lambda x: x.split())
 vocab = build_vocab(sentences )<compute_test_metric>
op['order_number_back'] = op.groupby('user_id')['order_number'].transform(max)- op.order_number +1
op.head(15 )
oov = check_coverage(vocab,word2vecDict )<string_transform>
last_five = op5.groupby(['user_id','product_id'])[['order_id']].count()
last_five.columns = ['times_last5']
last_five.head(10 )
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispell_dict = {'colour':'color',
'centre':'center',
'didnt':'did not',
'doesnt':'does not',
'isnt':'is not',
'shouldnt':'should not',
'favourite':'favorite',
'travelling':'traveling',
'counselling':'counseling',
'theatre':'theater',
'cancelled':'canceled',
'labour':'labor',
'organisation':'organization',
'wwii':'world war 2',
'citicise':'criticize',
'instagram': 'social medium',
'whatsapp': 'social medium',
'snapchat': 'social medium'
}
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text )<feature_engineering>
uxp = uxp.merge(last_five, on=['user_id', 'product_id'], how='left')
 del last_five
uxp.head()
train_df["question_text"] = train["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
test_df["question_text"] = test["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
sentences = train["question_text"].progress_apply(lambda x: x.split())
 to_remove = ['a','to','of','and']
sentences = [[word for word in sentence if not word in to_remove] for sentence in tqdm(sentences)]
vocab = build_vocab(sentences )<compute_test_metric>
uxp = uxp.fillna(0)
 uxp.head()
oov = check_coverage(vocab,word2vecDict )<string_transform>
data = uxp.merge(user, on='user_id', how='left')
 data.head()
embed_size = 300
maxlen = 200
max_features = 20000
tokenizer = Tokenizer(num_words=max_features)
 train_x,val_x=train_test_split(train_df, test_size=0.1, random_state=2018)
 train_X=train_x["question_text"].fillna("_na_" ).values
val_X=val_x["question_text"].fillna("_na_" ).values
test_X=test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_x['target'].values
val_y = val_x['target'].values<feature_engineering>
data = data.merge(prd, on='product_id', how='left')
 data.head()
embeddings_index = dict()
for word in word2vecDict.wv.vocab:
embeddings_index[word] = word2vecDict.word_vec(word )<feature_engineering>
del op, user, prd, uxp
gc.collect()
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix =(np.random.rand(nb_words, embed_size)- 0.5)/ 5.0
for word, i in word_index.items() :
if i >= max_features: continue
if word in word2vecDict:
embedding_vector = word2vecDict.get_vector(word)
 embedding_matrix[i] = embedding_vector
del word2vecDict; gc.collect()<choose_model_class>
orders_future = orders[(( orders.eval_set=='train')|(orders.eval_set=='test')) ]
orders_future = orders_future[ ['user_id', 'eval_set', 'order_id'] ]
orders_future.head(10)
embed_size = 300
max_features = 20000
maxlen = 200
S_DROPOUT = 0.4
DROPOUT = 0.1
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size , weights=[embedding_matrix_3] )(inp)
 x = SpatialDropout1D(S_DROPOUT )(x)
 x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 x = Dense(16, activation="relu" )(conc)
 x = Dropout(DROPOUT )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )<split>
data = data.merge(orders_future, on='user_id', how='left')
 data.head(10 )
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)) 
<string_transform>
data_train = data[data.eval_set=='train']
data_train.head()
embed_size = 300
max_features = 50000
maxlen = 100
train_x,val_x=train_test_split(train, test_size=0.1, random_state=42)
 train_X=train_x["question_text"].fillna("_na_" ).values
val_X=val_x["question_text"].fillna("_na_" ).values
test_X=test["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_x['target'].values
val_y = val_x['target'].values 
<feature_engineering>
data_train = data_train.merge(order_products_train[['product_id','order_id', 'reordered']], on=['product_id','order_id'], how='left')
 data_train.head(15 )
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )<train_model>
data_train['reordered'] = data_train['reordered'].fillna(0)
 data_train.head(15 )
model.fit(train_X, train_y, batch_size=1024, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
data_train = data_train.set_index(['user_id', 'product_id'])
 data_train.head(15 )
pred_test_y = model.predict([val_X], batch_size=512, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_test_y>thresh ).astype(int))))<predict_on_test>
data_train = data_train.drop(['eval_set', 'order_id'], axis=1)
 data_train.head(15 )
pred_y = model.predict([test_X], batch_size=512, verbose=1 )<save_to_csv>
data_test = data[data.eval_set=='test']
data_test.head()
pred_y =(pred_y>0.35 ).astype(int)
 out= pd.DataFrame({"qid":test["qid"].values})
 out['prediction'] = pred_y
out.to_csv("submission.csv", index=False )<define_variables>
data_test = data_test.set_index(['user_id', 'product_id'])
 data_test.head()
embed_size = 300
max_features = 95000
maxlen = 70<import_modules>
data_test = data_test.drop(['eval_set','order_id'], axis=1)
 data_test.head()
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
import string, re
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate
from keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers<categorify>
X_train, y_train = data_train.drop('reordered', axis=1), data_train.reordered
rfc = RandomForestClassifier(n_estimators=20,max_depth=6, n_jobs=-1 ,random_state=50)
 model = rfc.fit(X_train, y_train)
 feature_importances_df = pd.DataFrame(model.feature_importances_, index = X_train.columns, columns=['importance'] ).sort_values('importance',ascending=False)
 print(feature_importances_df)
 feat_importances = pd.Series(model.feature_importances_, index=X_train.columns ).sort_values()
feat_importances.plot(kind='barh')
 gc.collect()
def clean(text):
text = text.translate(string.punctuation)
 text = text.lower()
text = re.sub(r"[^A-Za-z0-9^,!.\/'+-=]", " ", text)
 text = re.sub(r"what's", "what is ", text)
 text = re.sub(r"'s", " ", text)
 text = re.sub(r"'ve", " have ", text)
 text = re.sub(r"n't", " not ", text)
 text = re.sub(r"i'm", "i am ", text)
 text = re.sub(r"'re", " are ", text)
 text = re.sub(r"'d", " would ", text)
 text = re.sub(r"'ll", " will ", text)
 text = re.sub(r",", " ", text)
 text = re.sub(r"\.", " ", text)
 text = re.sub(r"!", " ! ", text)
 text = re.sub(r"\/", " ", text)
 text = re.sub(r"\^", " ^ ", text)
 text = re.sub(r"\+", " + ", text)
 text = re.sub(r"\-", " - ", text)
 text = re.sub(r"\=", " = ", text)
 text = re.sub(r"'", " ", text)
 text = re.sub(r"(\d+ )(k)", r"\g<1>000", text)
 text = re.sub(r":", " : ", text)
 text = re.sub(r" e g ", " eg ", text)
 text = re.sub(r" b g ", " bg ", text)
 text = re.sub(r" u s ", " american ", text)
 text = re.sub(r"\0s", "0", text)
 text = re.sub(r" 9 11 ", "911", text)
 text = re.sub(r"e - mail", "email", text)
 text = re.sub(r"j k", "jk", text)
 text = re.sub(r"\s{2,}", " ", text)
 text = re.sub('[^a-zA-Z]',' ', text)
 text = re.sub(' +',' ',text)
 return text<load_from_csv>
model.get_params()
def load_and_prec() :
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_df['clean_text'] = train_df['question_text'].apply(clean)
 test_df['clean_text'] = test_df['question_text'].apply(clean)
 train_df, val_df = train_test_split(train_df, test_size=0.08, random_state=2018)
 train_X = train_df["clean_text"].fillna("_
val_X = val_df["clean_text"].fillna("_
test_X = test_df["clean_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
np.random.seed(2018)
 trn_idx = np.random.permutation(len(train_X))
val_idx = np.random.permutation(len(val_X))
train_X = train_X[trn_idx]
val_X = val_X[val_idx]
train_y = train_y[trn_idx]
val_y = val_y[val_idx]
return train_X, val_X, test_X, train_y, val_y, tokenizer.word_index<statistical_test>
param_grid = {
'max_depth': [4,5,6],
'max_features': [3, 4, 5],
'n_estimators': [14, 16, 20]
}
rfc = RandomForestClassifier()
grid_search = GridSearchCV(estimator = rfc, n_jobs=1, param_grid = param_grid,
cv = 3,)
 grid_search.fit(X_train, y_train)
 print("The best parameters are: /n", grid_search.best_params_)
 model = grid_search.best_estimator_
del [X_train, y_train, rfc]
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix 
<choose_model_class>
model.get_params()
def model_lstm_du(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 conc = Dense(64, activation="relu" )(conc)
 conc = Dropout(0.1 )(conc)
 outp = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<find_best_params>
X_train, y_train = data_train.drop('reordered', axis=1), data_train.reordered
d_train = xgboost.DMatrix(X_train, y_train)
 param = {'max_depth':10,
'eta':0.02,
'colsample_bytree':0.4,
'subsample':0.75,
'silent':1,
'nthread':27,
'eval_metric':'logloss',
'binary':'logistic',
'tree_method':'hist'
}
watchlist= [(d_train, "train")]
bst = xgboost.train(params=param, dtrain=d_train, num_boost_round=100, evals=watchlist, verbose_eval=10)
 xgboost.plot_importance(bst )
def train_pred(model, epochs=2):
for e in range(epochs):
model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y))
pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)
 best_thresh = 0.5
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(val_y,(pred_val_y > thresh ).astype(int))
if score > best_score:
best_thresh = thresh
best_score = score
print("Val F1 Score: {:.4f}".format(best_score))
pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)
 return pred_val_y, pred_test_y, best_score<split>
test_pred = model.predict(data_test ).astype(int)
 test_pred =(model.predict_proba(data_test)[:,1] >= 0.21 ).astype(int)
 test_pred[0:10]
%%time
train_X, val_X, test_X, train_y, val_y, word_index = load_and_prec() 
<predict_on_test>
data_test['prediction'] = test_pred
data_test.head(10 )
%%time
pred_val_y, pred_test_y, best_score = train_pred(model_lstm_du(embedding_matrix_1), epochs = 2 )<find_best_params>
orders_test = orders.loc[orders.eval_set=='test',("user_id", "order_id")]
orders_test.head()
thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(val_y,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh )<compute_test_metric>
final = final.merge(orders_test, on='user_id', how='left')
 final.head()
metrics.confusion_matrix(val_y,pred_val_y>best_thresh )<save_to_csv>
final = final.drop('user_id', axis=1 )
pred_test_y =(pred_test_y > best_thresh ).astype(int)
 test_df = pd.read_csv(".. /input/test.csv", usecols=["qid"])
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<define_variables>
final['product_id'] = final.product_id.astype(int)
 del orders
del orders_test
gc.collect()
final.head()
embed_size = 300
max_features = 95000
maxlen = 70<import_modules>
d = dict()
for row in final.itertuples() :
if row.prediction== 1:
try:
d[row.order_id] += ' ' + str(row.product_id)
 except:
d[row.order_id] = str(row.product_id)
 for order in final.order_id:
if order not in d:
d[order] = 'None'
gc.collect()
d
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate
from keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers<load_from_csv>
sub = pd.DataFrame.from_dict(d, orient='index')
 sub.reset_index(inplace=True)
 sub.columns = ['order_id', 'products']
sub.head()
<statistical_test><EOS> 
sub.to_csv('sub.csv', index=False )