Kaggle competition: Instacart Market Basket Analysis
<SOS> metric: MeanFScore Kaggle data source: instacart-market-basket-analysis<choose_model_class>
gc.enable()
def model_cnn(embedding_matrix):
filter_sizes = [1,2,3,5]
num_filters = 36
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Reshape(( maxlen, embed_size, 1))(x)
 maxpool_pool = []
for i in range(len(filter_sizes)) :
conv = Conv2D(num_filters, kernel_size=(filter_sizes[i], embed_size),
kernel_initializer='he_normal', activation='elu' )(x)
 maxpool_pool.append(MaxPool2D(pool_size=(maxlen - filter_sizes[i] + 1, 1))(conv))
z = Concatenate(axis=1 )(maxpool_pool)
 z = Flatten()(z)
 z = Dropout(0.1 )(z)
 outp = Dense(1, activation="sigmoid" )(z)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<choose_model_class>
orders = pd.read_csv('.. /input/orders.csv')
 order_products_train = pd.read_csv('.. /input/order_products__train.csv')
 order_products_prior = pd.read_csv('.. /input/order_products__prior.csv')
 products = pd.read_csv('.. /input/products.csv')
 aisles = pd.read_csv('.. /input/aisles.csv')
 departments = pd.read_csv('.. /input/departments.csv' )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
def model_lstm_atten(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<choose_model_class>
aisles['aisle'] = aisles['aisle'].astype('category')
 departments['department'] = departments['department'].astype('category')
 orders['eval_set'] = orders['eval_set'].astype('category')
 products['product_name'] = products['product_name'].astype('category' )
def model_gru_srk_atten(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model 
<choose_model_class>
op = orders.merge(order_products_prior, on='order_id', how='inner')
 op.head()
def model_lstm_du(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 conc = Dense(64, activation="relu" )(conc)
 conc = Dropout(0.1 )(conc)
 outp = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<find_best_params>
user = op.groupby('user_id')['order_number'].max().to_frame('u_total_orders')
 user.head()
def train_pred(model, epochs=2):
for e in range(epochs):
model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y))
pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)
 best_thresh = 0.5
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(val_y,(pred_val_y > thresh ).astype(int))
if score > best_score:
best_thresh = thresh
best_score = score
print("Val F1 Score: {:.4f}".format(best_score))
pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)
 return pred_val_y, pred_test_y, best_score<split>
user = user.merge(u_reorder, on='user_id', how='left')
 del u_reorder
gc.collect()
user.head()
train_X, val_X, test_X, train_y, val_y, word_index = load_and_prec()
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_2 = load_fasttext(word_index)
 embedding_matrix_3 = load_para(word_index )<predict_on_test>
prd = op.groupby('product_id')['order_id'].count().to_frame('p_total_purchases')
 prd.head()
outputs = []
pred_val_y, pred_test_y, best_score = train_pred(model_gru_srk_atten(embedding_matrix), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, 'gru atten srk'] )<predict_on_test>
p_reorder = op.groupby('product_id' ).filter(lambda x: x.shape[0] >120)
 p_reorder.head()
pred_val_y, pred_test_y, best_score = train_pred(model_cnn(embedding_matrix), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, '2d CNN'] )<predict_on_test>
prd = prd.merge(p_reorder, on='product_id', how='left')
 del p_reorder
gc.collect()
prd.head()
pred_val_y, pred_test_y, best_score = train_pred(model_cnn(embedding_matrix_1), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, '2d CNN GloVe'] )<predict_on_test>
prd['p_reorder_ratio'] = prd['p_reorder_ratio'].fillna(value=0)
 prd.head()
pred_val_y, pred_test_y, best_score = train_pred(model_lstm_du(embedding_matrix), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, 'LSTM DU'] )<predict_on_test>
uxp = op.groupby(['user_id', 'product_id'])['order_id'].count().to_frame('uxp_total_bought')
 uxp.head()
pred_val_y, pred_test_y, best_score = train_pred(model_lstm_atten(embedding_matrix), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, best_score, '2 LSTM w/ attention'] )<predict_on_test>
times = op.groupby(['user_id', 'product_id'])[['order_id']].count()
times.columns = ['Times_Bought_N']
times.head()
pred_val_y, pred_test_y, best_score = train_pred(model_lstm_atten(embedding_matrix_1), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, best_score, '2 LSTM w/ attention GloVe'] )<predict_on_test>
total_orders = op.groupby('user_id')['order_number'].max().to_frame('total_orders')
 total_orders.head()
pred_val_y, pred_test_y, best_score = train_pred(model_lstm_atten(embedding_matrix_3), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, best_score, '2 LSTM w/ attention Para'] )<sort_values>
span = pd.merge(total_orders, first_order_no, on='user_id', how='right')
 span.head()
outputs.sort(key=lambda x: x[2])
 weights = [i for i in range(1, len(outputs)+ 1)]
weights = [float(i)/ sum(weights)for i in weights]
print(weights )<compute_test_metric>
span['Order_Range_D'] = span.total_orders - span.first_order_number + 1
span.head()
pred_val_y = np.mean([outputs[i][0] for i in range(len(outputs)) ], axis = 0)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(val_y,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh )<save_to_csv>
uxp_ratio = pd.merge(times, span, on=['user_id', 'product_id'], how='left')
 uxp_ratio.head()
pred_test_y = np.mean([outputs[i][1] for i in range(len(outputs)) ], axis = 0)
 pred_test_y =(pred_test_y > best_thresh ).astype(int)
 test_df = pd.read_csv(".. /input/test.csv", usecols=["qid"])
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<load_from_csv>
uxp_ratio['uxp_reorder_ratio'] = uxp_ratio.Times_Bought_N / uxp_ratio.Order_Range_D
uxp_ratio.head()
train_df = pd.read_csv('.. /input/train.csv')
 X_train = train_df["question_text"].fillna("dieter" ).values
test_df = pd.read_csv('.. /input/test.csv')
 X_test = test_df["question_text"].fillna("dieter" ).values
y = train_df["target"]<import_modules>
uxp_ratio = uxp_ratio.drop(['Times_Bought_N', 'total_orders', 'first_order_number', 'Order_Range_D'], axis=1)
 uxp_ratio.head()
from keras.models import Model
from keras.layers import Input, Dense, Embedding, concatenate
from keras.layers import CuDNNGRU, Bidirectional, GlobalAveragePooling1D, GlobalMaxPooling1D, Conv1D
from keras.layers import Add, BatchNormalization, Activation, CuDNNLSTM, Dropout
from keras.layers import *
from keras.models import *
from keras.preprocessing import text, sequence
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
import gc
from sklearn import metrics
from keras.optimizers import *<string_transform>
del [times, first_order_no, span]
maxlen = 70
max_features = 50000
embed_size = 300
tokenizer = text.Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(X_train)+ list(X_test))
X_train = tokenizer.texts_to_sequences(X_train)
 X_test = tokenizer.texts_to_sequences(X_test)
 x_train = sequence.pad_sequences(X_train, maxlen=maxlen)
 x_test = sequence.pad_sequences(X_test, maxlen=maxlen )<categorify>
uxp = uxp.merge(uxp_ratio, on=['user_id', 'product_id'], how='left')
 del uxp_ratio
uxp.head()
def attention_3d_block(inputs):
TIME_STEPS = inputs.shape[1].value
SINGLE_ATTENTION_VECTOR = False
input_dim = int(inputs.shape[2])
 a = Permute(( 2, 1))(inputs)
 a = Reshape(( input_dim, TIME_STEPS))(a)
 a = Dense(TIME_STEPS, activation='softmax' )(a)
 if SINGLE_ATTENTION_VECTOR:
a = Lambda(lambda x: K.mean(x, axis=1))(a)
 a = RepeatVector(input_dim )(a)
 a_probs = Permute(( 2, 1))(a)
 output_attention_mul = Multiply()([inputs, a_probs])
 return output_attention_mul<feature_engineering>
data = uxp.merge(user, on='user_id', how='left')
 data.head()
class AttLayer(Layer):
def __init__(self, attention_dim):
self.init = initializers.get('normal')
 self.supports_masking = True
self.attention_dim = attention_dim
super(AttLayer, self ).__init__()
def build(self, input_shape):
assert len(input_shape)== 3
self.W = K.variable(self.init(( input_shape[-1], self.attention_dim)))
 self.b = K.variable(self.init(( self.attention_dim,)))
 self.u = K.variable(self.init(( self.attention_dim, 1)))
 self.trainable_weights = [self.W, self.b, self.u]
super(AttLayer, self ).build(input_shape)
 def compute_mask(self, inputs, mask=None):
return mask
def call(self, x, mask=None):
uit = K.tanh(K.bias_add(K.dot(x, self.W), self.b))
ait = K.dot(uit, self.u)
 ait = K.squeeze(ait, -1)
 ait = K.exp(ait)
 if mask is not None:
ait *= K.cast(mask, K.floatx())
 ait /= K.cast(K.sum(ait, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 ait = K.expand_dims(ait)
 weighted_input = x * ait
output = K.sum(weighted_input, axis=1)
 return output
def compute_output_shape(self, input_shape):
return(input_shape[0], input_shape[-1] )<categorify>
data = data.merge(prd, on='product_id', how='left')
 data.head()
def noise_measurement(train, noise_level):
noised_train = train.copy()
to_transform = np.random.random(train.shape)< noise_level
transform_y = np.random.randint(0, max_features, size=train.shape)
 noised_train[to_transform] = transform_y[to_transform]
return noised_train<statistical_test>
del op, user, prd, uxp
gc.collect()
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_1 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix_1[i] = embedding_vector
del embeddings_index; gc.collect()<statistical_test>
orders_future = orders[(( orders.eval_set=='train')|(orders.eval_set=='test')) ]
orders_future = orders_future[ ['user_id', 'eval_set', 'order_id'] ]
orders_future.head(10)
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_2 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix_2[i] = embedding_vector
del embeddings_index; gc.collect()<statistical_test>
data = data.merge(orders_future, on='user_id', how='left')
 data.head(10 )
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_3 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix_3[i] = embedding_vector
del embeddings_index; gc.collect()<feature_engineering>
data_train = data[data.eval_set=='train']
data_train.head()
embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_2, embedding_matrix_3], axis=0)
 del embedding_matrix_1, embedding_matrix_2, embedding_matrix_3
gc.collect()
np.shape(embedding_matrix )<split>
data_train = data_train.merge(order_products_train[['product_id','order_id', 'reordered']], on=['product_id','order_id'], how='left')
 data_train.head(15 )
X_tra, X_val, y_tra, y_val = train_test_split(x_train, y, test_size = 0.1, random_state=42 )<choose_model_class>
data_train['reordered'] = data_train['reordered'].fillna(0)
 data_train.head(15 )
def model1() :
inp = Input(shape=(maxlen,))
embed = Embedding(max_features, embed_size * 1, weights=[embedding_matrix], trainable=False )(inp)
 embed = SpatialDropout1D(0.1 )(embed)
 filter_sizes = [1,2,3,5]
num_filters = 64
conv_0 = Conv1D(num_filters, filter_sizes[0], padding='valid', kernel_initializer='normal', activation='relu' )(embed)
 conv_1 = Conv1D(num_filters, filter_sizes[1], padding='valid', kernel_initializer='normal', activation='relu' )(embed)
 conv_2 = Conv1D(num_filters, filter_sizes[2], padding='valid', kernel_initializer='normal', activation='relu' )(embed)
 conv_3 = Conv1D(num_filters, filter_sizes[3], padding='valid', kernel_initializer='normal', activation='relu' )(embed)
 maxpool_0 = MaxPool1D(pool_size=(maxlen - filter_sizes[0] + 1), strides=(1), padding='valid' )(conv_0)
 maxpool_1 = MaxPool1D(pool_size=(maxlen - filter_sizes[1] + 1), strides=(1), padding='valid' )(conv_1)
 maxpool_2 = MaxPool1D(pool_size=(maxlen - filter_sizes[2] + 1), strides=(1), padding='valid' )(conv_2)
 maxpool_3 = MaxPool1D(pool_size=(maxlen - filter_sizes[3] + 1), strides=(1), padding='valid' )(conv_3)
 concatenated_tensor = Concatenate(axis=1 )([maxpool_0, maxpool_1, maxpool_2, maxpool_3])
 flatten = Flatten()(concatenated_tensor)
 x = flatten
x = Dropout(0.5 )(x)
 x = Dense(128, activation='relu' )(x)
 outp = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy',
optimizer=Adam(lr=1e-3, decay=1e-6),
metrics=['accuracy'])
 return model<train_model>
data_train = data_train.set_index(['user_id', 'product_id'])
 data_train.head(15 )
MODEL1 = model1()
MODEL1.summary()
batch_size = 2048
epochs = 4
hist = MODEL1.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)
 MODEL1.save('./model1.h5' )<predict_on_test>
data_train = data_train.drop(['eval_set', 'order_id'], axis=1)
 data_train.head(15 )
pred_val_y_1 = MODEL1.predict([X_val], batch_size=1024, verbose=1)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(y_val,(pred_val_y_1 > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh_1 = thresholds[0][0]
print("Best threshold: ", best_thresh_1)
 y_pred_1 = MODEL1.predict(x_test, batch_size=1024, verbose=True )<choose_model_class>
data_test = data[data.eval_set=='test']
data_test.head()
def model2() :
inp = Input(shape=(maxlen,))
embed = Embedding(max_features, embed_size * 1, weights=[embedding_matrix], trainable=False )(inp)
 x = embed
x = SpatialDropout1D(0.1 )(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = attention_3d_block(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = AttLayer(maxlen )(x)
 x = Dropout(0.3 )(x)
 x = Dense(128, activation='relu' )(x)
 outp = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy',
optimizer=Adam(lr=1e-3, decay=1e-6),
metrics=['accuracy'])
 return model<train_model>
data_test = data_test.set_index(['user_id', 'product_id'])
 data_test.head()
MODEL2 = model2()
MODEL2.summary()
batch_size = 2048
epochs = 5
hist = MODEL2.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)
 MODEL2.save('./model2.h5' )<predict_on_test>
data_test = data_test.drop(['eval_set','order_id'], axis=1)
 data_test.head()
pred_val_y_2 = MODEL2.predict([X_val], batch_size=1024, verbose=1)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(y_val,(pred_val_y_2 > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh_2 = thresholds[0][0]
print("Best threshold: ", best_thresh_2)
 y_pred_2 = MODEL2.predict(x_test, batch_size=1024, verbose=True )<choose_model_class>
X_train, y_train = data_train.drop('reordered', axis=1), data_train.reordered
clf1=LogisticRegression(class_weight={0:1,1:15},random_state=5)
 clf2=RandomForestClassifier(class_weight={0:1,1:12},criterion='entropy',max_depth=8,max_features='auto',
min_samples_leaf=10,
n_estimators=30,n_jobs=-1,random_state=5)
 clf3=DecisionTreeClassifier(random_state=5,class_weight="balanced")
 rfc=VotingClassifier(estimators=[('lr',clf1),('rf',clf2),('dt',clf3)],voting='soft')
 model = rfc.fit(X_train, y_train )
def model3() :
filters = 64
inp = Input(shape=(maxlen,))
embed = Embedding(max_features, embed_size * 1, weights=[embedding_matrix], trainable=False )(inp)
 x = embed
x = SpatialDropout1D(0.1 )(x)
 x = Conv1D(filters, 1, activation='relu', padding='valid', kernel_initializer='normal' )(x)
 x = Dropout(0.1 )(x)
 x = Conv1D(filters, 2, activation='relu', padding='valid', kernel_initializer='normal' )(x)
 x = Dropout(0.1 )(x)
 x = Conv1D(filters, 3, activation='relu', padding='valid', kernel_initializer='normal' )(x)
 x = Dropout(0.1 )(x)
 x = Conv1D(filters, 5, activation='relu', padding='valid', kernel_initializer='normal' )(x)
 x = Dropout(0.1 )(x)
 x = Flatten()(x)
 x = Dropout(0.3 )(x)
 x = Dense(128, activation='relu' )(x)
 outp = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy',
optimizer=Adam(lr=1e-3, decay=1e-6),
metrics=['accuracy'])
 return model<train_model>
prediction = model.predict(data_test)
 prediction =(rfc.predict_proba(data_test)[:,1] >= 0.42 ).astype(int )
MODEL3 = model3()
MODEL3.summary()
batch_size = 2048
epochs = 10
hist = MODEL3.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)
 MODEL3.save('./model3.h5' )<find_best_params>
data_test['prediction'] = prediction
data_test.tail(50 )
pred_val_y_3 = MODEL3.predict([X_val], batch_size=1024, verbose=1)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(y_val,(pred_val_y_3 > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh_3 = thresholds[0][0]
print("Best threshold: ", best_thresh_3)
 y_pred_3 = MODEL3.predict(x_test, batch_size=1024, verbose=True )<choose_model_class>
orders_test = orders.loc[orders.eval_set=='test',("user_id", "order_id")]
orders_test.head()
def model4() :
inp = Input(shape=(maxlen,))
embed = Embedding(max_features, embed_size * 1, weights=[embedding_matrix], trainable=False )(inp)
 x = embed
x = SpatialDropout1D(0.1 )(x)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = attention_3d_block(x)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = AttLayer(64 )(x)
 x = Dropout(0.3 )(x)
 x = Dense(128, activation='relu' )(x)
 outp = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy',
optimizer=Adam(lr=1e-3, decay=1e-6),
metrics=['accuracy'])
 return model<train_model>
final = final.merge(orders_test, on='user_id', how='left')
 final.head()
MODEL4 = model4()
MODEL4.summary()
batch_size = 2048
epochs = 5
hist = MODEL4.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)
 MODEL4.save('./model4.h5' )<find_best_params>
final = final.drop('user_id', axis=1)
pred_val_y_4 = MODEL4.predict([X_val], batch_size=1024, verbose=1)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(y_val,(pred_val_y_4 > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh_4 = thresholds[0][0]
print("Best threshold: ", best_thresh_4)
 y_pred_4 = MODEL4.predict(x_test, batch_size=1024, verbose=True )<compute_test_metric>
final['product_id'] = final.product_id.astype(int)
 del orders
del orders_test
gc.collect()
final.head()
pred_val_y =(2.5*pred_val_y_1 + 2.5*pred_val_y_2 + 2.5*pred_val_y_3 + 2.5*pred_val_y_4)/10
thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(y_val,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh )<save_to_csv>
d = dict()
for row in final.itertuples() :
if row.prediction== 1:
try:
d[row.order_id] += ' ' + str(row.product_id)
 except:
d[row.order_id] = str(row.product_id)
 for order in final.order_id:
if order not in d:
d[order] = 'None'
gc.collect()
d
y_pred =(2.5*y_pred_1 + 2.5*y_pred_2 + 2.5*y_pred_3 + 2.5*y_pred_4)/10
y_te =(y_pred[:,0] > best_thresh ).astype(np.int)
 submit_df = pd.DataFrame({"qid": test_df["qid"], "prediction": y_te})
 submit_df.to_csv("submission.csv", index=False )<save_to_csv>
sub = pd.DataFrame.from_dict(d, orient='index')
 sub.reset_index(inplace=True)
 sub.columns = ['order_id', 'products']
sub.head()
<install_modules><EOS> 
sub.to_csv('sub.csv', index=False )