Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<choose_model_class>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from keras.models import Sequential
from keras.layers import Dense, Dropout, LSTM, GRU
from keras.layers.embeddings import Embedding
import lightgbm as lgb
from kaggle.competitions import twosigmanews
from tqdm import tqdm
import gc
def model2() :
inp = Input(shape=(maxlen,))
embed = Embedding(max_features, embed_size, weights=[embedding_matrix_1], trainable=False )(inp)
 x = embed
x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = attention_3d_block(x)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = AttLayer(64 )(x)
 x = Dropout(0.3 )(x)
 x = Dense(128, activation='relu' )(x)
 outp = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model<import_modules>
DEBUG = False
YEARMIN = 2011
from keras.models import Model
from keras.layers import Input, Dense, Embedding, concatenate
from keras.layers import CuDNNGRU, Bidirectional, GlobalAveragePooling1D, GlobalMaxPooling1D, Conv1D
from keras.layers import Add, BatchNormalization, Activation, CuDNNLSTM, Dropout
from keras.layers import *
from keras.models import *
from keras.preprocessing import text, sequence
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
import gc
from sklearn import metrics<define_variables>
np.random.seed(100)
 set_random_seed(150 )
F1scores_epochs = [0.6327, 0.6607, 0.6688]<define_variables>
env = twosigmanews.make_env() 
(market_train, news_train)= env.get_training_data()
F1scores_epochs = [0.6327, 0.6607, 0.6688, 0.6698]<train_model>
market_train = market_train.loc[market_train['time'].dt.year > YEARMIN]
MODEL2 = model2()
MODEL2.summary()
batch_size = 2048
epochs = 5
early_stopping = EarlyStopping(patience=3, verbose=1, monitor='val_loss', mode='min')
 model_checkpoint = ModelCheckpoint('./model2.model', save_best_only=True, verbose=1, monitor='val_loss', mode='min')
 reduce_lr = ReduceLROnPlateau(factor=0.5, patience=3, min_lr=0.0001, verbose=1)
 hist = MODEL2.fit(train_X, train_y, batch_size=batch_size, epochs=epochs, validation_data=(val_X, val_y), verbose=True)
 MODEL2.save('./model2.h5' )<find_best_params>
pred_val_y_2 = MODEL2.predict([val_X], batch_size=1024, verbose=1)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(val_y,(pred_val_y_2 > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh_2 = thresholds[0][0]
print("Best threshold: ", best_thresh_2)
 y_pred_2 = MODEL2.predict(test_X, batch_size=1024, verbose=True)
 print(y_pred_2 )<save_to_csv>
print('MARKET TRAIN:')
 for col in market_train.columns:
print(col+' has '+str(market_train[col].isna().sum())+' NAs')
 print('
NEWS TRAIN:')
 for col in news_train.columns:
print(col+' has '+str(news_train[col].isna().sum())+' NAs' )
y_te =(y_pred_2[:,0] > best_thresh_2 ).astype(np.int)
 submit_df = pd.DataFrame({"qid": dftest["qid"], "prediction": y_te})
 submit_df.to_csv("submission.csv", index=False)
<import_modules>
del news_train
gc.collect()
tqdm.pandas()<load_from_csv>
market_train['returnsClosePrevMktres1'] = market_train.groupby(['assetCode'])['returnsClosePrevMktres1'].transform(lambda x: x.fillna(x.median()))
market_train['returnsOpenPrevMktres1'] = market_train.groupby(['assetCode'])['returnsOpenPrevMktres1'].transform(lambda x: x.fillna(x.median()))
market_train['returnsClosePrevMktres10'] = market_train.groupby(['assetCode'])['returnsClosePrevMktres10'].transform(lambda x: x.fillna(x.median()))
market_train['returnsOpenPrevMktres10'] = market_train.groupby(['assetCode'])['returnsOpenPrevMktres10'].transform(lambda x: x.fillna(x.median()))
market_train = market_train.fillna(market_train.median() )
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print('Train data dimension: ', train_df.shape)
 display(train_df.head())
 print('Test data dimension: ', test_df.shape)
 display(test_df.head() )<set_options>
market_train = market_train.sort_values(['assetCode','time'])
 market_train.head()
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<compute_test_metric>
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result<compute_test_metric>
n_in = 5
n_out = 1
def sigmoid(x):
return 1 /(1 + np.exp(-x))<define_variables>
def series_to_supervised(data, extraCols, n_in=1, n_out=1, dropnan=True, pad=False):
if pad:
data = np.asarray([data[0].tolist() ]*n_in + data.tolist())
 n_vars = 1 if type(data)is list else data.shape[1]
df = pd.DataFrame(data)
 cols, names = list() , list()
for i in range(n_in, 0, -1):
cols.append(df.shift(i))
names += [('var%d(t-%d)' %(j+1, i)) for j in range(n_vars)]
for i in range(0, n_out):
cols.append(df.shift(-i))
if i == 0:
names += [('var%d(t)' %(j+1)) for j in range(n_vars)]
else:
names += [('var%d(t+%d)' %(j+1, i)) for j in range(n_vars)]
agg = pd.concat(cols, axis=1)
 agg.columns = names
if dropnan:
agg.dropna(inplace=True)
 agg = pd.concat([agg.reset_index(drop=True), extraCols.reset_index(drop=True)], axis=1)
 return agg
embed_size = 300
max_features = 95000
maxlen = 70<define_variables>
market_train['time'] = pd.to_datetime(market_train['time'].dt.date )
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x<feature_engineering>
market_train['close_minus_open'] = market_train['close'] - market_train['open']
market_train['margin1'] = market_train['open'] / market_train['close']
train_df["question_text"] = train_df["question_text"].str.lower()
test_df["question_text"] = test_df["question_text"].str.lower()
train_df["question_text"] = train_df["question_text"].apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_text(x))
x_train = train_df["question_text"].fillna("_
x_test = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(x_train))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = pad_sequences(x_train, maxlen=maxlen)
 x_test = pad_sequences(x_test, maxlen=maxlen)
 y_train = train_df['target'].values<statistical_test>
total_market_obs_df = [market_train.loc[(market_train['time'].dt.year >= 2016)&(market_train['time'].dt.month >= 9)].groupby('assetCode' ).tail(30 ).drop(['universe','returnsOpenNextMktres10'], axis=1)]
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')[:300]
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.005838499,0.48782197
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.0053247833,0.49346462
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<normalization>
ewma = pd.Series.ewm
seed_everything()
glove_embeddings = load_glove(tokenizer.word_index)
 paragram_embeddings = load_para(tokenizer.word_index)
 embedding_matrix = np.mean([glove_embeddings, paragram_embeddings], axis=0)
 np.shape(embedding_matrix )<split>
def create_lag2(df_code,n_lag=[5,],shift_size=1):
code = df_code['assetCode'].unique()
for col in return_features:
df_code['%s_macd'%(col)] = ewma(df_code[col], span=12 ).mean() - ewma(df_code[col], span=26 ).mean()
return df_code
def generate_lag_features2(df,n_lag = [5]):
assetCodes = df['assetCode'].unique()
all_df = []
df_codes = df.groupby('assetCode')
 df_codes = [df_code[1][['time','assetCode']+return_features] for df_code in df_codes]
pool = Pool(4)
 all_df = pool.map(create_lag2, df_codes)
 new_df = pd.concat(all_df)
 new_df.drop(return_features,axis=1,inplace=True)
 pool.close()
return new_df
splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=10 ).split(x_train, y_train))<normalization>
def create_lag3(df_code,n_lag=[5,],shift_size=1):
code = df_code['assetCode'].unique()
for col in return_features:
df_code['%s_bollingerband_high'%(col)] = df_code[col].rolling(window=7 ).mean() + 2 * df_code[col].rolling(window=7 ).std()
df_code['%s_bollingerband_low'%(col)] = df_code[col].rolling(window=7 ).mean() - 2 * df_code[col].rolling(window=7 ).std()
return df_code
def generate_lag_features3(df,n_lag = [5]):
assetCodes = df['assetCode'].unique()
all_df = []
df_codes = df.groupby('assetCode')
 df_codes = [df_code[1][['time','assetCode']+return_features] for df_code in df_codes]
pool = Pool(4)
 all_df = pool.map(create_lag3, df_codes)
 new_df = pd.concat(all_df)
 new_df.drop(return_features,axis=1,inplace=True)
 pool.close()
return new_df
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1)
 class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x<choose_model_class>
def create_lag4(df_code,n_lag=[5,],shift_size=1):
code = df_code['assetCode'].unique()
for col in return_features:
df_code['%s_ewma'%(col)] = ewma(df_code[col], span=9 ).mean()
return df_code
def generate_lag_features4(df,n_lag = [5]):
assetCodes = df['assetCode'].unique()
all_df = []
df_codes = df.groupby('assetCode')
 df_codes = [df_code[1][['time','assetCode']+return_features] for df_code in df_codes]
pool = Pool(4)
 all_df = pool.map(create_lag4, df_codes)
 new_df = pd.concat(all_df)
 new_df.drop(return_features,axis=1,inplace=True)
 pool.close()
return new_df
class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self ).__init__()
hidden_size = 40
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.1)
 self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)
 self.lstm_attention = Attention(hidden_size * 2, maxlen)
 self.gru_attention = Attention(hidden_size * 2, maxlen)
 self.linear = nn.Linear(320, 16)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.out = nn.Linear(16, 1)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 h_lstm_atten = self.lstm_attention(h_lstm)
 h_gru_atten = self.gru_attention(h_gru)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 conc = torch.cat(( h_lstm_atten, h_gru_atten, avg_pool, max_pool), 1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out<define_variables>
gc.collect()
batch_size = 512
n_epochs = 6<define_variables>
return_features = ['volume','returnsClosePrevRaw1',
'returnsOpenPrevRaw1', 'returnsClosePrevMktres1',
'returnsOpenPrevMktres1', 'returnsClosePrevRaw10',
'returnsOpenPrevRaw10', 'returnsClosePrevMktres10',
'returnsOpenPrevMktres10',
'open', 'close',
'margin1', 'close_minus_open']
n_lag = [5]
new_df = generate_lag_features(market_train,n_lag=n_lag)
 market_train = pd.merge(market_train,new_df,how='left',on=['time','assetCode'])
 del new_df
gc.collect()
train_preds = np.zeros(( len(train_df)))
 test_preds = np.zeros(( len(test_df)))
 seed_everything()
x_test_cuda = torch.tensor(x_test, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 for i,(train_idx, valid_idx)in enumerate(splits):
x_train_fold = torch.tensor(x_train[train_idx], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(y_train[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(x_train[valid_idx], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(y_train[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()
model = NeuralNet()
model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction='mean')
 optimizer = torch.optim.Adam(model.parameters())
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(n_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for x_batch, y_batch in tqdm(train_loader, disable=True):
y_pred = model(x_batch)
 loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(( len(test_df)))
 avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))
for i,(x_batch,)in enumerate(test_loader):
y_pred = model(x_batch ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits )<compute_test_metric>
return_features = ['open', 'close']
n_lag = [5]
new_df = generate_lag_features2(market_train,n_lag=n_lag)
 market_train = pd.merge(market_train,new_df,how='left',on=['time','assetCode'])
 del new_df
gc.collect()
search_result = threshold_search(y_train, train_preds)
 search_result<save_to_csv>
return_features = ['open', 'close']
n_lag = [5]
new_df = generate_lag_features3(market_train,n_lag=n_lag)
 market_train = pd.merge(market_train,new_df,how='left',on=['time','assetCode'])
 del new_df
gc.collect()
submission = test_df[['qid']].copy()
submission['prediction'] = test_preds > search_result['threshold']
submission.to_csv('submission.csv', index=False )<train_model>
return_features = ['open', 'close']
n_lag = [5]
new_df = generate_lag_features4(market_train,n_lag=n_lag)
 market_train = pd.merge(market_train,new_df,how='left',on=['time','assetCode'])
 del new_df
gc.collect()
def f1() :
return<set_options>
market_train['volume_diff'] = market_train['volume'] - market_train['volume_lag_5_mean']
market_train['returnsClosePrevRaw1_diff'] = market_train['returnsClosePrevRaw1'] - market_train['returnsClosePrevRaw1_lag_5_mean']
market_train['returnsOpenPrevRaw1_diff'] = market_train['returnsOpenPrevRaw1'] - market_train['returnsOpenPrevRaw1_lag_5_mean']
market_train['returnsClosePrevMktres1_diff'] = market_train['returnsClosePrevMktres1'] - market_train['returnsClosePrevMktres1_lag_5_mean']
market_train['returnsOpenPrevMktres1_diff'] = market_train['returnsOpenPrevMktres1'] - market_train['returnsOpenPrevMktres1_lag_5_mean']
market_train['returnsClosePrevRaw10_diff'] = market_train['returnsClosePrevRaw10'] - market_train['returnsClosePrevRaw10_lag_5_mean']
market_train['returnsOpenPrevRaw10_diff'] = market_train['returnsOpenPrevRaw10'] - market_train['returnsOpenPrevRaw10_lag_5_mean']
market_train['returnsClosePrevMktres10_diff'] = market_train['returnsClosePrevMktres10'] - market_train['returnsClosePrevMktres10_lag_5_mean']
market_train['returnsOpenPrevMktres10_diff'] = market_train['returnsOpenPrevMktres10'] - market_train['returnsOpenPrevMktres10_lag_5_mean']
market_train['open_diff'] = market_train['open'] - market_train['open_lag_5_mean']
market_train['close_diff'] = market_train['close'] - market_train['close_lag_5_mean']
market_train['margin1_diff'] = market_train['margin1'] - market_train['margin1_lag_5_mean']
market_train['close_minus_open_diff'] = market_train['close_minus_open'] - market_train['close_minus_open_lag_5_mean']
market_train['open_macd_diff'] = market_train['open_ewma'] - market_train['open_macd']
market_train['close_macd_diff'] = market_train['close_ewma'] - market_train['close_macd']
market_train['open_bb_high_diff'] = market_train['open'] - market_train['open_bollingerband_high']
market_train['open_bb_low_diff'] = market_train['open'] - market_train['open_bollingerband_low']
market_train['close_bb_high_diff'] = market_train['close'] - market_train['close_bollingerband_high']
market_train['close_bb_low_diff'] = market_train['close'] - market_train['close_bollingerband_low']
market_train['open_ewma_diff'] = market_train['open'] - market_train['open_ewma']
market_train['close_ewma_diff'] = market_train['close'] - market_train['close_ewma']
<set_options>
date = market_train.time
num_target = market_train.returnsOpenNextMktres10.astype('float32')
 bin_target =(market_train.returnsOpenNextMktres10 >= 0 ).astype('int8')
 universe = market_train.universe.astype('int8')
def set_all_parameters() :
global epoc_used
global mn
global batch_size
global patience
global nsplits
global stop_split
global num_lstm
global num_dense
global rate_drop_lstm
global rate_drop_dense
global rate_drop_spatial
global loss
global act
global opt
global met
global es_mon
global es_mode
global Trainable
global lr
global method
global model_list
global maxlen
global max_features
global pretext_proc
global bstart
sm ="6"
ssh = 50
ssd = 50
sdh = 0.1
sdd = 0.1
sds =0.1
loss_bin='binary_crossentropy'
opt_adam="adam"
opt_rmsprop="rmsprop"
met_acc=["accuracy"]
met_f1=[f1]
mon_acc = "val_acc"
mode_acc = "max"
mon_f1 = "val_f1"
mode_f1 = "max"
mon_loss ="val_loss"
mode_loss="min"
ns=4
ss=4
bs=1024
ne=8
pl=3
bstart = time.time()
model_list=[]
model_list.append(["18",1,"6","Concatenated",10,1,1024,4,4,50,50,0.1,0.1,0.15,'binary_crossentropy','rmsprop',["accuracy"],'val_acc','max'])
 maxlen = 75
max_features = None
pretext_proc=True
Trainable=False 
<set_options>
from sklearn.model_selection import train_test_split
set_all_parameters()<train_model>
from sklearn.model_selection import train_test_split
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):

 self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) , K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<import_modules>
train_index = market_train.index[market_train['time'].dt.year < 2016].tolist()
test_index = market_train.index[(market_train['time'].dt.year >= 2016)&(market_train['universe']==1)].tolist()
class AttentionWeightedAverage(Layer):

 def __init__(self, return_attention=False, **kwargs):
self.init = initializers.get('uniform')
 self.supports_masking = True
self.return_attention = return_attention
super(AttentionWeightedAverage, self ).__init__(** kwargs)
 def build(self, input_shape):
self.input_spec = [InputSpec(ndim=3)]
assert len(input_shape)== 3
self.W = self.add_weight(shape=(input_shape[2], 1),
name='{}_W'.format(self.name),
initializer=self.init)
 self.trainable_weights = [self.W]
super(AttentionWeightedAverage, self ).build(input_shape)
 def call(self, x, mask=None):
logits = K.dot(x, self.W)
 x_shape = K.shape(x)
 logits = K.reshape(logits,(x_shape[0], x_shape[1]))
ai = K.exp(logits - K.max(logits, axis=-1, keepdims=True))
if mask is not None:
mask = K.cast(mask, K.floatx())
 ai = ai * mask
att_weights = ai /(K.sum(ai, axis=1, keepdims=True)+ K.epsilon())
 weighted_input = x * K.expand_dims(att_weights)
 result = K.sum(weighted_input, axis=1)
 if self.return_attention:
return [result, att_weights]
return result
def get_output_shape_for(self, input_shape):
return self.compute_output_shape(input_shape)
 def compute_output_shape(self, input_shape):
output_len = input_shape[2]
if self.return_attention:
return [(input_shape[0], output_len),(input_shape[0], input_shape[1])]
return(input_shape[0], output_len)
 def compute_mask(self, input, input_mask=None):
if isinstance(input_mask, list):
return [None] * len(input_mask)
 else:
return None<choose_model_class>
lgbm_cols = ['volume', 'close', 'open', 'returnsClosePrevRaw1',
'returnsOpenPrevRaw1', 'returnsClosePrevMktres1',
'returnsOpenPrevMktres1', 'returnsClosePrevRaw10',
'returnsOpenPrevRaw10', 'returnsClosePrevMktres10',
'returnsOpenPrevMktres10', 'close_minus_open', 'margin1',
'volume_lag_5_mean', 'returnsClosePrevRaw1_lag_5_mean',
'returnsOpenPrevRaw1_lag_5_mean', 'returnsClosePrevMktres1_lag_5_mean',
'returnsOpenPrevMktres1_lag_5_mean', 'returnsClosePrevRaw10_lag_5_mean',
'returnsOpenPrevRaw10_lag_5_mean',
'returnsClosePrevMktres10_lag_5_mean',
'returnsOpenPrevMktres10_lag_5_mean', 'open_lag_5_mean',
'close_lag_5_mean', 'margin1_lag_5_mean', 'close_minus_open_lag_5_mean',
'open_macd', 'close_macd', 'open_bollingerband_high',
'open_bollingerband_low', 'close_bollingerband_high',
'close_bollingerband_low', 'open_ewma', 'close_ewma', 'volume_diff',
'returnsClosePrevRaw1_diff', 'returnsOpenPrevRaw1_diff',
'returnsClosePrevMktres1_diff', 'returnsOpenPrevMktres1_diff',
'returnsClosePrevRaw10_diff', 'returnsOpenPrevRaw10_diff',
'returnsClosePrevMktres10_diff', 'returnsOpenPrevMktres10_diff',
'open_diff', 'close_diff', 'margin1_diff', 'close_minus_open_diff',
'open_macd_diff', 'close_macd_diff', 'open_bb_high_diff',
'open_bb_low_diff', 'close_bb_high_diff', 'close_bb_low_diff',
'open_ewma_diff', 'close_ewma_diff']
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
<statistical_test>
d_train = lgb.Dataset(market_train.iloc[train_index].drop(['returnsOpenNextMktres10', 'time', 'universe', 'assetCode', 'time', 'assetName'], axis=1),
label = bin_target.iloc[train_index])
 d_valid = lgb.Dataset(market_train.iloc[test_index].drop(['returnsOpenNextMktres10', 'time', 'universe', 'assetCode', 'time', 'assetName'], axis=1),
label = bin_target.iloc[test_index] )
def check_coverage(vocab,embeddings_index):
try:
print("Len - embeddings_index:",len(embeddings_index))
except :
print("Len - embeddings_index:",len(embeddings_index.index2word))
print("Len -",len(vocab))
a = {}
oov = {}
k = 0
i = 0
for word in tqdm(vocab):
try:
a[word] = embeddings_index[word]
k += vocab[word]
except:
oov[word] = vocab[word]
i += vocab[word]
pass
print("Words Found:",k)
 print("Words Not Found:",i)
 print("Total Words:",i+k)
 print('Found embeddings for {:.2%} of vocab'.format(len(a)/ len(vocab)))
 print('Found embeddings for {:.2%} of all text'.format(k /(k + i)))
 sorted_x = sorted(oov.items() , key=operator.itemgetter(1)) [::-1]
return sorted_x<statistical_test>
t_valid = date.iloc[test_index]
y_valid = num_target.iloc[test_index]
u_valid = universe.iloc[test_index]
d_valid.params = {
'extra_time': t_valid.factorize() [0],
'mktres': y_valid.values,
'universe': u_valid.values
}
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
 scale = K.sqrt(s_squared_norm + K.epsilon())
 return x / scale
class Capsule(Layer):
def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = squash
else:
self.activation = Activation(activation)
 def build(self, input_shape):
super(Capsule, self ).build(input_shape)
 input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1, input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 def call(self, u_vecs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
 else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
 batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs,(0, 2, 1, 3))
b = K.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = K.permute_dimensions(b,(0, 2, 1))
c = K.softmax(b)
 c = K.permute_dimensions(c,(0, 2, 1))
b = K.permute_dimensions(b,(0, 2, 1))
outputs = self.activation(K.batch_dot(c, u_hat_vecs, [2, 2]))
if i < self.routings - 1:
b = K.batch_dot(outputs, u_hat_vecs, [2, 3])
 return outputs
def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule )<string_transform>
d_valid.params
def remove_urls(vTEXT):
vTEXT = re.sub(r'(https|http)?:\/\/(\w|\.|\/|\?|\=|\&|\%)*\b', '', vTEXT, flags=re.MULTILINE)
 return(vTEXT)
 def ReplaceThreeOrMore(s):
pattern = re.compile(r" (.) \1{2,}", re.DOTALL)
 return pattern.sub(r"\1", s)
 def splitstring(s):
proposed_pattern = s[0]
for i, c in enumerate(s[1:], 1):
if c != " ":
if proposed_pattern == s[i:(i+len(proposed_pattern)) ]:
break
else:
proposed_pattern += c
else:
exit(1)
 return proposed_pattern
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
special_character_removal=re.compile(r'[^a-z\d ]',re.IGNORECASE)
 replace_numbers=re.compile(r'\d+',re.IGNORECASE)
 def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispell_dict = {'colour':'color',
'centre':'center',
'didnt':'did not',
'doesnt':'does not',
'isnt':'is not',
'shouldnt':'should not',
'favourite':'favorite',
'travelling':'traveling',
'counselling':'counseling',
'theatre':'theater',
'cancelled':'canceled',
'labour':'labor',
'organisation':'organization',
'wwii':'world war 2',
'citicise':'criticize',
'instagram': 'social medium',
'whatsapp': 'social medium',
'snapchat': 'social medium'
}
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
 def clean_text(x):
x = str(x)
 for punct in "/-'":
x = x.replace(punct, ' ')
 for punct in '&':
x = x.replace(punct, f' {punct} ')
 for punct in '?!.,"
x = x.replace(punct, '')
 return x
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text_NOTUSED(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def text_to_wordlist(text,to_lower=True, rem_urls=False, rem_3plus=False,
clean_t=True, clean_num=True,mispelling=True,rem_specwords= False,
split_repeated=True, rem_special=False, rep_num=False,
man_adj=True, rem_stopwords=False, stem_snowball=False,
stem_porter=False, lemmatize=False):
if rem_urls:
text = remove_urls(text)
 if to_lower:
text = text.lower()
if rem_3plus:
text = ReplaceThreeOrMore(text)
 if clean_t:
text= clean_text(text)
 if clean_num:
text= clean_numbers(text)
 if mispelling:
text= replace_typical_misspell(text)
 if man_adj:
text = re.sub(r"[^A-Za-z0-9^,!.\/'+-=]", " ", text)
 text = re.sub(r"what's", "what is ", text)
 text = re.sub(r"'s", " ", text)
 text = re.sub(r"'ve", " have ", text)
 text = re.sub(r"can't", "cannot ", text)
 text = re.sub(r"n't", " not ", text)
 text = re.sub(r"i'm", "i am ", text)
 text = re.sub(r"'re", " are ", text)
 text = re.sub(r"'d", " would ", text)
 text = re.sub(r"'ll", " will ", text)
 text = re.sub(r",", " ", text)
 text = re.sub(r"\.", " ", text)
 text = re.sub(r"!", " ! ", text)
 text = re.sub(r"\/", " ", text)
 text = re.sub(r"\^", " ^ ", text)
 text = re.sub(r"\+", " + ", text)
 text = re.sub(r"\-", " - ", text)
 text = re.sub(r"\=", " = ", text)
 text = re.sub(r"'", " ", text)
 text = re.sub(r"(\d+ )(k)", r"\g<1>000", text)
 text = re.sub(r":", " : ", text)
 text = re.sub(r" e g ", " eg ", text)
 text = re.sub(r" b g ", " bg ", text)
 text = re.sub(r" u s ", " american ", text)
 text = re.sub(r"\0s", "0", text)
 text = re.sub(r" 9 11 ", "911", text)
 text = re.sub(r"e - mail", "email", text)
 text = re.sub(r"j k", "jk", text)
 text = re.sub(r"\s{2,}", " ", text)
 text = text.split()
if split_repeated:
for i, c in enumerate(text):
text[i]=splitstring(c)
 if rem_specwords:
to_remove = ['a','to','of','and']
text = [w for w in text if not w in to_remove]
if rem_stopwords:
stops = set(stopwords.words("english"))
text = [w for w in text if not w in stops]
text = " ".join(text)
 if rem_special:
text=special_character_removal.sub('',text)
 if rep_num:
text=replace_numbers.sub('n',text)
 if stem_snowball:
text = text.split()
stemmer = SnowballStemmer('english')
 stemmed_words = [stemmer.stem(word)for word in text]
text = " ".join(stemmed_words)
 if stem_porter:
st = PorterStemmer()
txt = " ".join([st.stem(w)for w in text.split() ])
 if lemmatize:
wordnet_lemmatizer = WordNetLemmatizer()
txt = " ".join([wordnet_lemmatizer.lemmatize(w)for w in text.split() ])
 return(text )<load_pretrained>
def sigma_score(preds, valid_data):
df_time = valid_data.params['extra_time']
df_mktres = valid_data.params['mktres']
df_universe = valid_data.params['universe']
x_t =(( preds * 2)- 1)* pd.Series(df_mktres)* pd.Series(df_universe)
 x_t_sum = x_t.groupby(df_time ).sum()
score = x_t_sum.mean() / x_t_sum.std()
return 'sigma_score', score, True
def save_tokenizer(file, tokenizer):
with open(file, 'wb')as handle:
pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)
 def load_tokenizer(file, tokenizer):
with open(file, 'rb')as handle:
tokenizer = pickle.load(handle)
 return tokenizer<load_from_csv>
params = {}
params['learning_rate'] = 0.3
params['boosting_type'] = 'gbdt'
params['objective'] = 'binary'
params['metric'] = 'None'
params['feature_fraction'] = 0.80
params['verbose'] = 0
params['max_depth'] = 7
params['lambda_l1'] = 3
params['lambda_l2'] = 3
def load_and_prec(PreProcess=False):
global max_features
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_X=train_df["question_text"].values
test_X=test_df["question_text"].values
tokenizer = Tokenizer(num_words=max_features)
 if PreProcess:
train_questions = []
for text in train_X:
train_questions.append(text_to_wordlist(text))
test_questions=[]
for text in test_X:
test_questions.append(text_to_wordlist(text))
tokenizer.fit_on_texts(train_questions+test_questions)
 train_X = tokenizer.texts_to_sequences(train_questions)
 test_X = tokenizer.texts_to_sequences(test_questions)
 else:
tokenizer.fit_on_texts(list(train_X)+list(test_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 print(len(train_X), 'train sequences')
 print(len(test_X), 'test sequences')
 print('Average train sequence length: {}'.format(np.mean(list(map(len, train_X)) , dtype=int)))
 print('Average test sequence length: {}'.format(np.mean(list(map(len, test_X)) , dtype=int)))
 print('Max train sequence length: {}'.format(np.max(list(map(len, train_X)))))
 print('Max test sequence length: {}'.format(np.max(list(map(len, test_X)))))
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
print(len(tokenizer.word_index))
print(len(tokenizer.word_counts))
if(max_features==None):
max_features = len(tokenizer.word_index)+1
return train_X, test_X, train_y, test_df, tokenizer.word_index<choose_model_class>
corey_lgbm_model = lgb.train(params=params,
train_set=d_train,
num_boost_round=1000,
valid_sets=d_valid,
early_stopping_rounds=50,
verbose_eval=10,
feval=sigma_score
)
def create_model0(embedding_matrix):
mdln="0-Bidirec(CuDNNGRU)-GlobalMax-Dense-Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = LSTM(num_lstm, return_sequences=True )(embedding_layer)
 x = GlobalMaxPool1D()(x)
 x = Dense(num_dense, activation="relu" )(x)
 x = Dropout(rate_drop_dense )(x)
 preds = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy', optimizer=opt, metrics=met)
 return model, mdln<choose_model_class>
import seaborn as sns
import matplotlib.pyplot as plt
def create_model1(embedding_matrix):
mdln="1-LSTM-Dropout-Attention-Dense-Dropout-BatchNormalization"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = LSTM(num_lstm, dropout=rate_drop_lstm, recurrent_dropout=rate_drop_lstm,return_sequences=True )(embedding_layer)
 x = Dropout(rate_drop_lstm )(x)
 x = Attention(maxlen )(x)
 x = Dense(num_dense, activation="relu" )(x)
 x = Dropout(rate_drop_dense )(x)
 x = BatchNormalization()(x)
 preds = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy',optimizer=opt, metrics=met)
 return model, mdln<choose_model_class>
yhat_lgbm = corey_lgbm_model.predict(market_train.iloc[test_index].drop(['returnsOpenNextMktres10', 'time', 'universe', 'assetCode', 'time', 'assetName'], axis=1))
yhat_lgbm =(yhat_lgbm * 2)- 1
def create_model2(embedding_matrix):
mdln="2-Bidirect(CuDNNLSTM)-GlobalMax-Dense-Dropout-Dense-Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = Bidirectional(CuDNNLSTM(num_lstm, return_sequences=True))(embedding_layer)
 x = GlobalMaxPool1D()(x)
 x = Dense(num_dense, activation="relu" )(x)
 x = Dropout(rate_drop_dense )(x)
 x = Dense(num_dense, activation="relu" )(x)
 x = Dropout(rate_drop_dense )(x)
 preds = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy',optimizer=opt, metrics=met)
 return model, mdln<choose_model_class>
params = {}
params['learning_rate'] = 0.3
params['boosting_type'] = 'goss'
params['objective'] = 'binary'
params['metric'] = 'None'
params['feature_fraction'] = 0.60
params['verbose'] = 0
params['max_depth'] = 4
params['lambda_l2'] = 1
def create_model3(embedding_matrix):
mdln="3-Bidirect(CuDNNLSTM)-Bidirect(CuDNNLSTM)-Attention-Dense"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 num_lstm2=int(num_lstm/2)
 x = Bidirectional(CuDNNLSTM(num_lstm, return_sequences=True))(embedding_layer)
 x = Bidirectional(CuDNNLSTM(num_lstm2, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(num_dense, activation="relu" )(x)
 preds = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3),metrics=met)
 return model, mdln<choose_model_class>
corey_goss_model = lgb.train(params=params,
train_set=d_train,
num_boost_round=3000,
valid_sets=d_valid,
early_stopping_rounds=25,
verbose_eval=10,
feval=sigma_score
)
def create_model4(embedding_matrix):
mdln="4-Reshape-Concat(Conv2D(Filters)+MaxPool2d)-Flatten-Dropout"
filter_sizes = [1,2,3,5]
num_filters = 36
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = Reshape(( maxlen, embedding_matrix.shape[1], 1))(embedding_layer)
 maxpool_pool = []
for i in range(len(filter_sizes)) :
conv = Conv2D(num_filters, kernel_size=(filter_sizes[i], embedding_matrix.shape[1]),
kernel_initializer='he_normal', activation='elu' )(x)
 maxpool_pool.append(MaxPool2D(pool_size=(maxlen - filter_sizes[i] + 1, 1))(conv))
z = Concatenate(axis=1 )(maxpool_pool)
 z = Flatten()(z)
 z = Dropout(0.1 )(z)
 preds = Dense(1, activation="sigmoid" )(z)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy',optimizer=opt, metrics=met)
 return model, mdln<choose_model_class>
yhat_goss = corey_goss_model.predict(market_train.iloc[test_index].drop(['returnsOpenNextMktres10', 'time', 'universe', 'assetCode', 'time', 'assetName'], axis=1))
yhat_goss =(yhat_goss * 2)- 1
def create_model5(embedding_matrix):
mdln="05-conv-max-conv-max-bulstm-max-dd"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = Dropout(0.2 )(embedding_layer)
 x = Conv1D(filters=embedding_matrix.shape[1], kernel_size=4, padding='same', activation='relu' )(x)
 x = MaxPooling1D(pool_size=2 )(x)
 x = Conv1D(filters=embedding_matrix.shape[1], kernel_size=4, padding='same', activation='relu' )(x)
 x = MaxPooling1D(pool_size=2 )(x)
 x = Bidirectional(CuDNNLSTM(num_lstm, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(num_dense, activation="relu" )(x)
 x = Dropout(rate_drop_dense )(x)
 x = Dense(num_dense, activation="relu" )(x)
 x = Dropout(rate_drop_dense )(x)
 preds = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy',optimizer=opt, metrics=met)
 return model, mdln<choose_model_class>
params = {}
params['learning_rate'] = 0.3
params['boosting_type'] = 'dart'
params['objective'] = 'binary'
params['metric'] = 'None'
params['feature_fraction'] = 0.60
params['verbose'] = 0
params['max_depth'] = 10
params['lambda_l2'] = 0.4
params['drop_rate'] = 0.2
params['drop_seed'] = 3230
def create_model6(embedding_matrix):
mdln="6-Bidirect(CuDNNGRU)-Cocat(GlobalMax+Global Avarage)- Dense - Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(embedding_layer)
 tower_1 = GlobalMaxPool1D()(x)
 tower_2 = GlobalAveragePooling1D()(x)
 output = concatenate([ tower_1, tower_2])
 x = Dense(num_dense, activation="relu" )(output)
 x = Dropout(rate_drop_dense )(x)
 preds = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy',optimizer=opt,metrics=met)
 return model, mdln<choose_model_class>
corey_dart_model = lgb.train(params=params,
train_set=d_train,
num_boost_round=3000,
valid_sets=d_valid,
early_stopping_rounds=50,
verbose_eval=10,
feval=sigma_score
)
def create_model7(embedding_matrix):
mdln="7-Bidirect(CuDNNGRU)-Attention-Dense-Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(embedding_layer)
 x = Attention(maxlen )(x)
 x = Dense(num_dense, activation="relu" )(x)
 x = Dropout(rate_drop_dense )(x)
 preds = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy', optimizer=opt, metrics=met)
 return model , mdln<choose_model_class>
yhat_dart = corey_dart_model.predict(market_train.iloc[test_index].drop(['returnsOpenNextMktres10', 'time', 'universe', 'assetCode', 'time', 'assetName'], axis=1))
yhat_dart =(yhat_dart * 2)- 1
def create_model8(embedding_matrix):
mdln="8-Bidirect(CuDNNGRU)-Concat(GlobalMax+GlobalAverage)-Dense-Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(embedding_layer)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 conc = Dense(num_dense, activation="relu" )(conc)
 conc = Dropout(rate_drop_dense )(conc)
 preds = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy', optimizer=opt, metrics=met)
 return model, mdln<choose_model_class>
del d_train, d_valid, params
gc.collect()
def create_model9(embedding_matrix):
mdln="9-Bidirect(CuDNNGRU)-Bidirect(CuDNNGRU)-Bidirect(CuDNNGRU)-Attention-Dense"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 num_lstm2=int(num_lstm)
 num_lstm12=num_lstm2+int(num_lstm2/2)
 x = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(embedding_layer)
 x = Bidirectional(CuDNNGRU(num_lstm12, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(num_lstm2, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 preds = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy', optimizer=opt, metrics=met)
 return model, mdln<choose_model_class>
LSTM_COLUMNS_TO_USE = ['time',
'assetCode',
'volume','returnsOpenPrevMktres10','returnsClosePrevMktres10','returnsOpenPrevRaw10','returnsClosePrevRaw10',
'returnsOpenNextMktres10',
'universe',
]
def create_model10(embedding_matrix):
mdln="10-Spatial-Bidirect(CuDNNGRU)-conv2-maxpool-attavr-average-concatenate-drop-dense"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 recurrent_units = 64
filter1_nums = 128
embedding_layer = SpatialDropout1D(rate_drop_spatial )(embedding_layer)
 rnn_1 = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(embedding_layer)
 conv_2 = Conv1D(filter1_nums, 2, kernel_initializer="normal", padding="valid", activation="relu", strides=1 )(rnn_1)
 maxpool = GlobalMaxPooling1D()(conv_2)
 attn = AttentionWeightedAverage()(conv_2)
 average = GlobalAveragePooling1D()(conv_2)
 concatenated = concatenate([maxpool, attn, average], axis=1)
 x = Dropout(rate_drop_dense )(concatenated)
 x = Dense(num_dense, activation="relu" )(x)
 output_layer = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=input_layer, outputs=output_layer)
 adam_optimizer = optimizers.Adam(lr=1e-3, clipvalue=5, decay=1e-5)
 model.compile(loss='binary_crossentropy', optimizer=adam_optimizer, metrics=met)
 return model, mdln<choose_model_class>
market_train = market_train[LSTM_COLUMNS_TO_USE]
def create_model11(embedding_matrix):
mdln="11-Capsule"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = SpatialDropout1D(rate=rate_drop_spatial )(embedding_layer)
 x = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True,
kernel_initializer=glorot_normal(seed=123000), recurrent_initializer=orthogonal(gain=1.0, seed=10000)) )(x)
 x = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True )(x)
 x = Flatten()(x)
 x = Dense(num_dense, activation='relu', kernel_initializer=glorot_normal(seed=123000))(x)
 x = Dropout(rate_drop_dense )(x)
 x = BatchNormalization()(x)
 preds = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy', optimizer=Adam() ,)
 return model,mdln<prepare_x_and_y>
print('MARKET TRAIN:')
 for col in market_train.columns:
print(col+' has '+str(market_train[col].isna().sum())+' NAs' )
def create_model12(embedding_matrix):
mdln="12-SpatialDropout+Bidirect(CudNNGRU)+Bidirect(CudNLSTM)+Attention+Attention+Dense+Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = SpatialDropout1D(rate_drop_spatial )(embedding_layer)
 x = Bidirectional(CuDNNLSTM(num_lstm, return_sequences=True))(x)
 y = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(x)
 atten_1 = AttentionWeightedAverage()(x)
 atten_2 = AttentionWeightedAverage()(y)
 avg_pool = GlobalAveragePooling1D()(y)
 max_pool = GlobalMaxPooling1D()(y)
 conc = concatenate([atten_1, atten_2, avg_pool, max_pool])
 conc = Dense(num_dense, activation="relu" )(conc)
 conc = Dropout(rate_drop_dense )(conc)
 preds = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy', optimizer=opt, metrics=met)
 return model,mdln<choose_model_class>
market_train = market_train.fillna(market_train.median() )
def create_model13(embedding_matrix):
mdln="13-SpatialDropout+Bidirect(CudNNGRU)+Bidirect(CudNLSTM)+Attention+Attention+Dense+Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = SpatialDropout1D(rate_drop_spatial )(embedding_layer)
 x = Bidirectional(CuDNNLSTM(num_lstm, kernel_initializer=glorot_uniform(seed = 2018), return_sequences=True))(x)
 y = Bidirectional(CuDNNGRU(num_lstm, kernel_initializer=glorot_uniform(seed = 2018),return_sequences=True))(x)
 atten_1 = Attention(maxlen )(x)
 atten_2 = Attention(maxlen )(y)
 avg_pool = GlobalAveragePooling1D()(y)
 max_pool = GlobalMaxPooling1D()(y)
 conc = concatenate([atten_1, atten_2, avg_pool, max_pool])
 conc = Dense(num_dense, kernel_initializer=he_uniform(seed=2018), activation="relu" )(conc)
 conc = Dropout(rate_drop_dense )(conc)
 preds = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy', optimizer=opt, metrics=met)
 return model,mdln<choose_model_class>
INFORMATION_COLS = ['assetCode','time','universe','returnsOpenNextMktres10']
INPUT_COLS = [f for f in market_train.columns if f not in INFORMATION_COLS]
def create_model14(embedding_matrix):
mdln="14-Bidirect(CuDNNLSTM)-SpatialDropout-GlobalMax-BatchNorm-Dense-Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = CuDNNLSTM(num_lstm, return_sequences=True )(embedding_layer)
 x = SpatialDropout1D(rate_drop_spatial )(x)
 x = GlobalMaxPool1D()(x)
 x = BatchNormalization()(x)
 x = Dense(num_dense )(x)
 x = Dropout(rate_drop_dense )(x)
 preds = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy',optimizer=opt, metrics=met)
 return model, mdln<choose_model_class>
scaler = StandardScaler()
market_train[INPUT_COLS] = scaler.fit_transform(market_train[INPUT_COLS] )
def create_model15(embedding_matrix):
mdln="15-SpatialDropout-Bidirect(CudNNGRU)+Concat(GlobalMax+GlobalAverage)+Dense+Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = SpatialDropout1D(rate_drop_spatial )(embedding_layer)
 x = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 x = Dense(num_dense, activation="relu" )(conc)
 x = Dropout(rate_drop_dense )(x)
 preds = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy',optimizer=opt, metrics=met)
 return model, mdln<choose_model_class>
del market_train
gc.collect()
def create_model16(embedding_matrix):
mdln="16-Bidirect(CudNNGRU)+GlobalMax+Dense+Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = Bidirectional(CuDNNLSTM(num_lstm, return_sequences=True))(embedding_layer)
 x = GlobalMaxPool1D()(x)
 x = Dense(num_dense, activation="relu" )(x)
 x = Dropout(rate_drop_dense )(x)
 preds = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy',optimizer=opt, metrics=met)
 return model, mdln<choose_model_class>
full_lstm_df = pd.concat(lstm_df_list, axis=0 ).reset_index(drop=True )
def create_model17(embedding_matrix):
mdln="17-Bidirec(CuDNNLSTM)-GlobalMax-Dense-Dropout-Dense-Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = Bidirectional(CuDNNLSTM(num_lstm, return_sequences=True))(embedding_layer)
 x = GlobalMaxPool1D()(x)
 x = Dense(num_dense, activation="relu" )(x)
 x = Dropout(rate_drop_dense )(x)
 x = Dense(num_dense, activation="relu" )(x)
 x = Dropout(rate_drop_dense )(x)
 preds = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy',optimizer=opt, metrics=met)
 return model, mdln<prepare_x_and_y>
del lstm_df_list
gc.collect()
def create_model18(embedding_matrix):
mdln="18-SpatialDropout+Bidirect(CudNNGRU)+Bidirect(CudNLSTM)+Attention+Attention+Dense+Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = SpatialDropout1D(rate_drop_spatial )(embedding_layer)
 x = Bidirectional(CuDNNLSTM(num_lstm, return_sequences=True))(x)
 y = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(x)
 atten_1 = Attention(maxlen )(x)
 atten_2 = Attention(maxlen )(y)
 avg_pool = GlobalAveragePooling1D()(y)
 max_pool = GlobalMaxPooling1D()(y)
 conc = concatenate([atten_1, atten_2, avg_pool, max_pool])
 conc = Dense(num_dense, activation="relu" )(conc)
 conc = Dropout(rate_drop_dense )(conc)
 preds = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy', optimizer=opt, metrics=met)
 return model,mdln<choose_model_class>
full_lstm_df['TARGET'] =(np.sign(full_lstm_df['returnsOpenNextMktres10'])+ 1)/ 2
def create_model19(embedding_matrix):
mdln="19-SpatialDropout+Bidirect(CudNNGRU)+Conv1D+Concat(GlobalMax+GlobalAverage)+Dense+Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = SpatialDropout1D(rate_drop_spatial )(embedding_layer)
 x = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(x)
 x = Conv1D(64, kernel_size = 3, padding = "valid", kernel_initializer = "glorot_uniform" )(x)
 tower_1 = GlobalMaxPool1D()(x)
 tower_2 = GlobalAveragePooling1D()(x)
 output = concatenate([ tower_1, tower_2])
 out = Dense(num_dense, activation="relu" )(output)
 out = Dropout(rate_drop_dense )(out)
 preds = Dense(1, activation="sigmoid" )(out)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy',optimizer=Adam(lr=1e-3),metrics=met)
 return model, mdln 
<choose_model_class>
train = full_lstm_df.loc[full_lstm_df['time'].dt.year < 2016]
valid = full_lstm_df.loc[full_lstm_df['time'].dt.year >= 2016]
valid = valid.loc[valid['universe']==1]
def create_model(label,embedding_matrix):
if(label=="0"):
return create_model0(embedding_matrix)
 if(label=="1"):
return create_model1(embedding_matrix)
 if(label=="2"):
return create_model2(embedding_matrix)
 if(label=="3"):
return create_model3(embedding_matrix)
 if(label=="4"):
return create_model4(embedding_matrix)
 if(label=="5"):
return create_model5(embedding_matrix)
 if(label=="6"):
return create_model6(embedding_matrix)
 if(label=="7"):
return create_model7(embedding_matrix)
 if(label=="8"):
return create_model8(embedding_matrix)
 if(label=="9"):
return create_model9(embedding_matrix)
 if(label=="10"):
return create_model10(embedding_matrix)
 if(label=="11"):
return create_model11(embedding_matrix)
 if(label=="12"):
return create_model12(embedding_matrix)
 if(label=="13"):
return create_model13(embedding_matrix)
 if(label=="14"):
return create_model14(embedding_matrix)
 if(label=="15"):
return create_model15(embedding_matrix)
 if(label=="16"):
return create_model16(embedding_matrix)
 if(label=="17"):
return create_model17(embedding_matrix)
 if(label=="18"):
return create_model18(embedding_matrix)
 if(label=="19"):
return create_model19(embedding_matrix)
 return None,"None" 
<compute_test_metric>
valid_X, valid_y = valid.values[:, :-5], valid['TARGET'].values
valid_returns = valid.values[:, -2]
valid_universe = valid.values[:, -3]
valid_time = valid.values[:, -4]
def best_F1(v_y ,t_y,show_plot=False):
bs=0
bt=0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(v_y,(t_y>=thresh ).astype(int))
if score >= bs:
bt = thresh
bs = score
if(show_plot):
print("
Best F1 score at threshold %2.4f is %2.4f
" %(bt, bs))
plot_confusion_matrix(v_y, np.array(pd.Series(t_y.reshape(-1,)).map(lambda x:1 if x>=bt else 0)))
 d=confusion_matrix(v_y,(t_y>=bt ).astype(int))
print("Total:",d.sum() ,"/Insincere:",d[1,:].sum() ,"/Sincere:",d[0,:].sum())
 print("Total Erros:",(d[0,1]+d[1,0]),"/Insincere: %2f"%(( d[0,1]/(d[0,1]+d[1,1]))),"/Sincere: %2f" %(( d[1,0]/(d[1,0]+d[0,0]))))
return bt, bs<compute_test_metric>
del valid
gc.collect()
def f1(y_true, y_pred):
def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon())) 
<create_dataframe>
%whos
def rec_results(Model,Embedding,Thresh,Score,StartTime):
global resultsdf
resultsdf = resultsdf.append({'Model':Model,
'Method':method,
'Embedding':Embedding,
'Pretext':pretext_proc,
'Weigth':wgt,
'F1':Score,
"Threshold":Thresh,
"Duration":str(round(( time.time() -StartTime)/60,0)) ,
'Epochs':epochs,
'EpochsUsed':epoc_used,
'Trainable':Trainable,
'Opt':opt,
'MaxLength':maxlen,
'MaxFeatures':max_features,
'batch_size':batch_size,
'patience':patience,
'num_lstm':num_lstm,
'rate_drop_lstm':rate_drop_lstm,
'num_dense':num_dense,
'rate_drop_dense':rate_drop_dense,
'rate_drop_spatial':rate_drop_spatial,
'opt':opt,
'met':met,
'es_mon':es_mon,
'es_mode':es_mode,
'Date':datetime.now().strftime("%d-%m-%Y %H:%M")}, ignore_index=True )<load_pretrained>
train_X, train_y = train.values[:, :-5], train['TARGET'].values
def load_embedding(emb,word_index):
estart = time.time()
print('Indexing '+emb+' vectors')
 if(emb=="Glove"):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
if(emb=="Google"):
EMBEDDING_FILE = '.. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'
embeddings_index = KeyedVectors.load_word2vec_format(EMBEDDING_FILE, binary=True)
 if(emb=="Paragram"):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 if(emb=="Wiki"):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 print("Vector",EMBEDDING_FILE)
 print("End Indexing:",(str(timedelta(seconds=(time.time() -estart)))))
 estart = time.time()
print('Preparing embedding matrix')
 out_of_features=0
out_of_embedding=0
in_embedding=0
if(emb=="Wiki" or emb=="Glove" or emb=="Paragram"):
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index)) +1
np.random.seed(2018)
 embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features:
out_of_features+=1
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
in_embedding+=1
else:
out_of_embedding+=1
else:
embed_size=300
nb_words = min(max_features, len(word_index)) +1
np.random.seed(2018)
 embedding_matrix=(np.random.rand(nb_words, embed_size)- 0.5)/ 5.0
for word, i in word_index.items() :
if i >= max_features:
out_of_features+=1
continue
if word in embeddings_index:
embedding_vector = embeddings_index.get_vector(word)
 embedding_matrix[i] = embedding_vector
in_embedding+=1
else:
out_of_embedding+=1
print("Total Vocabulary:",len(word_index))
print("Out of Features:", out_of_features)
 print("Out of Embeddings:",out_of_embedding)
 print("Mapped Embeddings:",in_embedding)
 print("End Preparing embedding matrix:",(str(timedelta(seconds=(time.time() -estart)))))
 del embeddings_index
return embedding_matrix<find_best_model_class>
del train
gc.collect()
def evaluate_model1(label,emb, embedding_matrix,train_X,train_y,t_X,t_y,v_X,v_y, epochs, batch_size, patience=0):
global resultsdf
global pred_val_y
global prev_test_y
global met
global epoc_used
best_thresh = 0.0
best_score = 0.0
mstart = time.time()
model=None
model,mdln = create_model(label,embedding_matrix)
 print("")
 print(mdln+'-'+emb)
 model_checkpoint = ModelCheckpoint('./model.hdf5', monitor=es_mon, mode=es_mode,
verbose=True, save_best_only=True,
save_weights_only=False)
 early_stopping = EarlyStopping(monitor=es_mon, mode=es_mode, patience=patience,verbose=True)
 callbacks = [ early_stopping, model_checkpoint]
print("Model Fitting:")
 hist= model.fit(t_X, t_y, batch_size=batch_size, epochs=epochs,
shuffle=True, verbose=True, validation_data=(v_X, v_y),
callbacks=callbacks)
 ",hist.history.keys())
 epoc_used=len(hist.history[es_mon])
 model.load_weights('./model.hdf5')
 print("Predicting Values:")
 pvy = model.predict([v_X], batch_size=batch_size, verbose=2)
 best_thresh,best_score=best_F1(v_y,pvy[:,0] , False)
 pty = model.predict([test_X], batch_size=1024, verbose=2)
 rec_results(mdln,emb,best_thresh,best_score,mstart)
 del model
gc.collect()
time.sleep(10)
 return pvy[:,0],pty[:,0],best_score,best_thresh<train_model>
USE_SIGMOID_ACTIVATION_LAYER = True
def evaluate_model2(label,emb, embedding_matrix,train_X,train_y, t_X, t_y, v_X, v_y, epochs, batch_size, patience=0):
global resultsdf
global pred_val_y
global prev_test_y
global met
best_thresh = 0.0
best_score = 0.0
mstart = time.time()
model=None
model,mdln = create_model(label, embedding_matrix)
 print("")
 print(mdln+'-'+emb)
 bs=0
pt=0
print("Model Fitting:")
 for e in range(epochs):
model.fit(t_X, t_y, batch_size=batch_size, epochs=1, validation_data=(v_X, v_y))
print("Predicting Values:")
 pvy = model.predict([v_X], batch_size=1024, verbose=0)
 best_thresh,best_score=best_F1(v_y,pvy[:,0], False)
 print("epoch ",e+1,"/",epochs)
 if(best_score > bs):
print("F1 Score Improved from %2.4f to %2.4f" %(bs,best_score))
bs = best_score
bt = best_thresh
epoc_used=e+1
model.save_weights('./model.hdf5')
 else:
print("F1 Score not improved")
 pt=pt+1
if(pt>patience):
break
model.load_weights('./model.hdf5')
 pty = model.predict([test_X], batch_size=1024, verbose=0)
 rec_results(mdln,emb,bt,bs,mstart)
 del model
gc.collect()
time.sleep(10)
 return pvy[:,0],pty[:,0],bs,bt<choose_model_class>
from keras import callbacks
def evaluate_model3(label,emb, embedding_matrix,train_X,train_y, t_X, t_y, v_X, v_y, epochs, batch_size, patience=0):
global resultsdf
global pred_val_y
global prev_test_y
global met
global clr
global epoc_used
clr = CyclicLR(base_lr=0.001, max_lr=0.01,
step_size=300., mode='exp_range',
gamma=0.99994)
 callback = [clr,]
best_thresh = 0.0
best_score = 0.0
mstart = time.time()
model=None
model,mdln = create_model(label, embedding_matrix)
 print("")
 print(mdln+'-'+emb)
 bs=0
pt=0
print("Model Fitting:")
 for e in range(epochs):
model.fit(t_X, t_y, batch_size=batch_size, epochs=1, validation_data=(v_X, v_y),callbacks = callback,verbose=2)
 print("Predicting Values:")
 pvy = model.predict([v_X], batch_size=1024, verbose=0)
 print("F1 - Score:")
 best_thresh,best_score=best_F1(v_y,pvy[:,0])
 print("epoch ",e+1,"/",epochs)
 if(best_score > bs):
print("F1 Score Improved from %2.4f to %2.4f" %(bs,best_score))
bs = best_score
bt = best_thresh
epoc_used=e+1
model.save_weights('./model.hdf5')
 else:
print("F1 Score not improved")
 pt=pt+1
if(pt>patience):
break
model.load_weights('./model.hdf5')
 pty = model.predict([test_X], batch_size=1024, verbose=0)
 rec_results(mdln,emb,bt,bs,mstart)
 del model
gc.collect()
time.sleep(10)
 return pvy[:,0],pty[:,0],bs,bt<train_model>
class Metrics(callbacks.Callback):
def on_train_begin(self, logs={}):
self._data = []
def on_epoch_end(self, batch, logs={}):
X_val, y_val = self.validation_data[0], self.validation_data[1]
y_predict =(pd.DataFrame(model.predict(X_val)) * 2)- 1
_sigmascore = sigma_scorelstm(y_val, y_predict)
 print(" — sigmascore: %f" %(_sigmascore))
self._data.append({
'val_sigmascore': _sigmascore
})
 return
def get_data(self):
return self._data
metrics = Metrics()
def evaluate_model4(label,emb, embedding_matrix,train_X,train_y,t_X,t_y,v_X,v_y, epochs, batch_size, patience=0):
global resultsdf
global pred_val_y
global prev_test_y
global met
global clr
global epoc_used
clr = CyclicLR(base_lr=0.001, max_lr=0.01,
step_size=300., mode='exp_range',
gamma=0.99994)
 best_thresh = 0.0
best_score = 0.0
mstart = time.time()
model=None
model,mdln = create_model(label,embedding_matrix)
 print("")
 print(mdln+'-'+emb)
 model_checkpoint = ModelCheckpoint('./model.hdf5', monitor=es_mon, mode=es_mode,
verbose=True, save_best_only=True,
save_weights_only=False)
 early_stopping = EarlyStopping(monitor=es_mon, mode=es_mode, patience=patience,verbose=True)
 callbacks = [ early_stopping, model_checkpoint,clr,]
print("Model Fitting:")
 hist= model.fit(t_X, t_y, batch_size=batch_size, epochs=epochs,
shuffle=True, verbose=True, validation_data=(v_X, v_y),
callbacks=callbacks)
 epoc_used=len(hist.history[es_mon])
 print("Getting the Best Model")
 model.load_weights('./model.hdf5')
 print("Predicting Values:")
 pvy = model.predict([v_X], batch_size=batch_size, verbose=2)
 best_thresh,best_score=best_F1(v_y,pvy[:,0], False)
 pty = model.predict([test_X], batch_size=1024, verbose=2)
 rec_results(mdln,emb,best_thresh,best_score,mstart)
 del model
gc.collect()
time.sleep(10)
 return pvy[:,0],pty[:,0],best_score,best_thresh<choose_model_class>
def sigma_scorelstm(y_true, y_pred):
x_t_i = y_pred * pd.DataFrame(valid_returns)* pd.DataFrame(valid_universe)
 data = pd.concat([pd.DataFrame(valid_time), x_t_i], axis=1)
 data.columns = ['day','x_t_i']
x_t = data.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_valid = mean / std
return score_valid
def evaluate_model5(label,emb, embedding_matrix,train_X,train_y, t_X, t_y, v_X, v_y, epochs, batch_size, patience=0):
global resultsdf
global pred_val_y
global prev_test_y
global met
global clr
global epoc_used
clr = CyclicLR(base_lr=0.001, max_lr=0.01,
step_size=300., mode='exp_range',
gamma=0.99994)
 callback = [clr,]
best_thresh = 0.0
best_score = 0.0
mstart = time.time()
model=None
model,mdln = create_model(label, embedding_matrix)
 print("")
 print(mdln+'-'+emb)
 bs=0
pt=0
print("Model Fitting:")
 for e in range(epochs):
model.fit(t_X, t_y, batch_size=batch_size, epochs=1, validation_data=(v_X, v_y),verbose=2)
 print("Predicting Values:")
 pvy = model.predict([v_X], batch_size=1024, verbose=0)
 best_thresh = 0.0
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(v_y,(pvy[:,0] > thresh ).astype(int))
if score > best_score:
print("F1COnf Score Improved from %2.4f to %2.4f" %(best_score,score))
best_thresh = thresh
best_score = score
best_thresh = 0.0
best_score = 0.0
print("F1 - Score:")
 best_thresh,best_score=best_F1(v_y,pvy[:,0])
 print("epoch ",e+1,"/",epochs)
 epoc_used=e+1
if(best_score > bs):
print("F1 Score Improved from %2.4f to %2.4f" %(bs,best_score))
bs = best_score
bt = best_thresh
pty = model.predict([test_X], batch_size=1024, verbose=0)
 rec_results(mdln,emb,bt,bs,mstart)
 del model
gc.collect()
time.sleep(10)
 return pvy[:,0],pty[:,0],bs,bt<train_model>
x_t_i = yhat_lstm * pd.DataFrame(valid_returns)* pd.DataFrame(valid_universe)
 data = pd.concat([pd.DataFrame(valid_time), x_t_i], axis=1)
 data.columns = ['day','x_t_i']
x_t = data.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_valid = mean / std
print(score_valid )
def evaluate_model6(label,emb, embedding_matrix,train_X,train_y,t_X,t_y,v_X,v_y, epochs, batch_size, patience=0):
global resultsdf
global pred_val_y
global prev_test_y
global met
global epoc_used
best_thresh = 0.0
best_score = 0.0
mstart = time.time()
model=None
model,mdln = create_model(label,embedding_matrix)
 print("")
 print(mdln+'-'+emb)
 model_checkpoint = ModelCheckpoint('./model.hdf5', monitor=es_mon, mode=es_mode,
verbose=True, save_best_only=True,
save_weights_only=False)
 reduce_lr = ReduceLROnPlateau(monitor=es_mon, factor=0.5, patience=1, min_lr=0.0001, verbose=True)
 early_stopping = EarlyStopping(monitor=es_mon,min_delta=0.0001, mode='auto', patience=patience,verbose=True)
 callbacks = [ early_stopping,model_checkpoint,reduce_lr]
print("Model Fitting:")
 hist= model.fit(t_X, t_y, batch_size=batch_size, epochs=epochs,
shuffle=True, verbose=True, validation_data=(v_X, v_y),
callbacks=callbacks)
 ",hist.history.keys())
 epoc_used=len(hist.history[es_mon])
 model.load_weights('./model.hdf5')
 print("Predicting Values:")
 pvy = model.predict([v_X], batch_size=batch_size, verbose=2)
 best_thresh,best_score=best_F1(v_y,pvy[:,0] , False)
 pty = model.predict([test_X], batch_size=1024, verbose=2)
 rec_results(mdln,emb,best_thresh,best_score,mstart)
 del model
gc.collect()
time.sleep(10)
 return pvy[:,0],pty[:,0],best_score,best_thresh<split>
del history, train_X, train_y, valid_X, valid_y
gc.collect()
def evaluate_stratifiedkfold(label,mt,emb,train_X,train_y, test_X, n_splits, stop_split, epochs, batch_size,patience):
global resultsdf
global allpred_val_y
global allprev_test_y
global mn
global method
method=mt
random_seed = 2018
train_stratified=np.zeros([len(train_X),(stop_split)])
 test_stratified = np.zeros([len(test_X),(stop_split)])
 msstart = time.time()
embedding_matrix=embedding_matrix_combined
if(emb=="Glove"):
embedding_matrix=embedding_matrix_1
if(emb=="Wiki"):
embedding_matrix=embedding_matrix_2
if(emb=="Paragram"):
embedding_matrix=embedding_matrix_3
if(emb=="Google"):
embedding_matrix=embedding_matrix_4
if(emb=="Concatenated"):
embedding_matrix=embedding_matrix_concatenated
splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=random_seed ).split(train_X, train_y))
mn=mn+1
bs=np.zeros([stop_split])
 bt=np.zeros([stop_split])
 for idx,(train_idx, valid_idx)in enumerate(splits):
K.clear_session()
X_split = train_X[train_idx]
y_split = train_y[train_idx]
print("Split:",idx)
 print("Total X-train:",len(y_split)," Insincere:",y_split.sum() ,y_split.sum() / len(y_split))
X_val = train_X[valid_idx]
y_val = train_y[valid_idx]
print("Total X-val:",len(y_val)," Insincere:",y_val.sum() , y_val.sum() /len(y_val))
if(mt=="0"):
pred_train_y, pred_test_y, best_score, best_thresh=evaluate_model(md,emb,embedding_matrix,train_X,train_y, X_split, y_split, X_val, y_val, epochs, batch_size ,patience)
 if(mt=="1"):
pred_train_y, pred_test_y, best_score, best_thresh=evaluate_model1(md,emb,embedding_matrix,train_X,train_y, X_split, y_split, X_val, y_val,epochs, batch_size ,patience)
 if(mt=="2"):
pred_train_y, pred_test_y, best_score, best_thresh=evaluate_model2(md,emb,embedding_matrix,train_X,train_y, X_split, y_split, X_val, y_val,epochs, batch_size ,patience)
 if(mt=="3"):
pred_train_y, pred_test_y, best_score, best_thresh=evaluate_model3(md,emb,embedding_matrix,train_X,train_y, X_split, y_split, X_val, y_val,epochs, batch_size ,patience)
 if(mt=="4"):
pred_train_y, pred_test_y, best_score, best_thresh=evaluate_model4(md,emb,embedding_matrix,train_X,train_y, X_split, y_split, X_val, y_val,epochs, batch_size ,patience)
 if(mt=="5"):
pred_train_y, pred_test_y, best_score, best_thresh=evaluate_model5(md,emb,embedding_matrix,train_X,train_y, X_split, y_split, X_val, y_val,epochs, batch_size ,patience)
 if(mt=="6"):
pred_train_y, pred_test_y, best_score, best_thresh=evaluate_model6(md,emb,embedding_matrix,train_X,train_y, X_split, y_split, X_val, y_val,epochs, batch_size ,patience)
 if(best_score > bs[idx]):
print("F1 Score Improved from %2.4f to %2.4f" %(bs[idx],best_score))
bs[idx] = best_score
bt[idx] = best_thresh
test_stratified[:,idx] = pred_test_y
if(( idx+1)==stop_split):
break
train_pred = train_stratified.mean(axis=1)
 test_pred = test_stratified.mean(axis=1)
 if(stop_split > 1):
rec_results("Model Mean:"+label+"-skfold- "+str(n_splits)+"Stop: "+str(stop_split),emb,bt.min() ,bs.mean() ,msstart)
 return train_pred,test_pred,bs.max() ,bt.min()<load_pretrained>
x_t_i = pd.DataFrame(( yhat_lgbm + yhat_goss + yhat_dart)/3)* pd.DataFrame(valid_returns)* pd.DataFrame(valid_universe)
 data = pd.concat([pd.DataFrame(valid_time), x_t_i], axis=1)
 data.columns = ['day','x_t_i']
x_t = data.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_valid = mean / std
print(score_valid )
embedding_matrix_1 = load_embedding("Glove",word_index)
 embedding_matrix_3 = load_embedding("Paragram",word_index)
 embedding_matrix_combined = np.mean([embedding_matrix_1, embedding_matrix_3], axis = 0)
 np.shape(embedding_matrix_combined)
 embedding_matrix_concatenated = np.concatenate(( embedding_matrix_1, embedding_matrix_3), axis=1)
 np.shape(embedding_matrix_concatenated )<define_variables>
x_t_i =(yhat_lstm + pd.DataFrame(yhat_lgbm + yhat_goss + yhat_dart)) /4 * pd.DataFrame(valid_returns)* pd.DataFrame(valid_universe)
 data = pd.concat([pd.DataFrame(valid_time), x_t_i], axis=1)
 data.columns = ['day','x_t_i']
x_t = data.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_valid = mean / std
print(score_valid )
set_all_parameters()
resultsdf = pd.DataFrame(columns=['Model','F1','Embedding','Pretext','Weigth','Duration','Method',
'Epochs','EpochsUsed','patience','batch_size'
,'MaxLength','MaxFeatures' ])
 allpred_train_y=np.zeros([len(train_X),(len(model_list)) ])
 allpred_test_y=np.zeros([len(test_X),(len(model_list)) ])
 allbest=np.zeros([(len(model_list)) ])
 allthresh=np.zeros([(len(model_list)) ])
 epoc_used=0
mn=-1
for ne,item in enumerate(model_list):
md=item[0]
wgt=item[1]
mt=item[2]
emb=item[3]
epochs=item[4]
patience=item[5]
batch_size=item[6]
nsplits=item[7]
stop_split=item[8]
num_lstm=item[9]
num_dense=item[10]
rate_drop_lstm = item[11]
rate_drop_dense = item[12]
rate_drop_spatial=item[13]
loss=item[14]
opt=item[15]
met=item[16]
es_mon=item[17]
es_mode=item[18]
print(mt)
 print("md-"+md)
 trainp,testp,bs,bt=evaluate_stratifiedkfold(md,mt,emb,train_X,train_y, test_X, nsplits, stop_split, epochs, batch_size,patience)
 allbest[mn]=bs
allthresh[mn]=bt
allpred_test_y[:,mn] = testp
print("FIM-"+md)
<categorify>
yhat_lgbm = pd.DataFrame(( yhat_lgbm + 1)/ 2)
 yhat_goss = pd.DataFrame(( yhat_goss + 1)/ 2)
 yhat_dart = pd.DataFrame(( yhat_dart + 1)/ 2)
 yhat_lstm =(yhat_lstm + 1)/ 2
twgt=0
wgt_test_y=0
wgt_train_y=0
essemblet="Essemble: "
for ne,item in enumerate(model_list):
wg=item[1]
twgt=twgt+wg
wgt_test_y =wgt_test_y +(allpred_test_y[:,ne]*wg)
 essemblet=essemblet+item[0]+" "
wgt_test_y=wgt_test_y/twgt 
<save_to_csv>
truth =(np.sign(valid_returns)+ 1)/ 2
print(allthresh)
 print(allthresh.min())
 print(allthresh.mean())
 y_te =(np.array(wgt_test_y)>= allthresh.min() ).astype(np.int)
 submit_df = pd.DataFrame({"qid": test_df["qid"], "prediction": y_te})
 submit_df.to_csv("./submission.csv",index=False)
 rec_results('Time Elapsed','',0,0,bstart)
 display(resultsdf )<define_variables>
ensemble = pd.concat([yhat_lgbm, yhat_goss, yhat_dart, yhat_lstm], axis=1)
 ensemble.columns = ['lgbm','goss','dart', 'lstm']
truth = pd.DataFrame(truth ).astype('int')
 truth.columns = ['y']
embed_size = 300
max_features = 95000
maxlen = 70<import_modules>
logreg = LogisticRegression()
logreg.fit(ensemble, truth)
 logreg.score(ensemble, truth )
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate
from keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers<define_variables>
yhat_ens =(pd.DataFrame(logreg.predict_proba(ensemble)[:,1])* 2)- 1
x_t_i = yhat_ens * pd.DataFrame(valid_returns)* pd.DataFrame(valid_universe)
 data = pd.concat([pd.DataFrame(valid_time), x_t_i], axis=1)
 data.columns = ['day','x_t_i']
x_t = data.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_valid = mean / std
print(score_valid )
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x<load_from_csv>
days = env.get_prediction_days()
def load_and_prec() :
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 train_df["question_text"] = train_df["question_text"].str.lower()
test_df["question_text"] = test_df["question_text"].str.lower()
train_df["question_text"] = train_df["question_text"].apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_text(x))
print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_df, val_df = train_test_split(train_df, test_size=0.001, random_state=2018)
 train_X = train_df["question_text"].fillna("_
val_X = val_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
np.random.seed(2018)
 trn_idx = np.random.permutation(len(train_X))
val_idx = np.random.permutation(len(val_X))
train_X = train_X[trn_idx]
val_X = val_X[val_idx]
train_y = train_y[trn_idx]
val_y = val_y[val_idx]
return train_X, val_X, test_X, train_y, val_y, tokenizer.word_index<statistical_test>
del active_history['universe']
del active_history['returnsOpenNextMktres10']
del active_history['TARGET']
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.005838499,0.48782197
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.0053247833,0.49346462
embed_size = all_embs.shape[1]
print(emb_mean,emb_std,"para")
 nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<choose_model_class>
LSTM_COLUMNS_TO_USE = [col for col in LSTM_COLUMNS_TO_USE if(col!='universe' and col!='returnsOpenNextMktres10')]
def model_cnn(embedding_matrix):
filter_sizes = [1,2,3,5]
num_filters = 36
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Reshape(( maxlen, embed_size, 1))(x)
 maxpool_pool = []
for i in range(len(filter_sizes)) :
conv = Conv2D(num_filters, kernel_size=(filter_sizes[i], embed_size),
kernel_initializer='he_normal', activation='elu' )(x)
 maxpool_pool.append(MaxPool2D(pool_size=(maxlen - filter_sizes[i] + 1, 1))(conv))
z = Concatenate(axis=1 )(maxpool_pool)
 z = Flatten()(z)
 z = Dropout(0.1 )(z)
 outp = Dense(1, activation="sigmoid" )(z)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<choose_model_class>
for(market_obs_df, _, predictions_template_df)in days:
active_history.drop([col for col in active_history.columns if '(t-'+str(n_in)+')' in col], axis=1, inplace=True)
 del active_history['time']
market_obs_df['time'] = pd.to_datetime(market_obs_df['time'].dt.date)
 market_obs_df['close_minus_open'] = market_obs_df['close'] - market_obs_df['open']
market_obs_df['margin1'] = market_obs_df['open'] / market_obs_df['close']
total_market_obs_df.append(market_obs_df)
 history_df = pd.concat(total_market_obs_df[-(np.max(30)+1):])
 return_features = ['volume','returnsClosePrevRaw1',
'returnsOpenPrevRaw1', 'returnsClosePrevMktres1',
'returnsOpenPrevMktres1', 'returnsClosePrevRaw10',
'returnsOpenPrevRaw10', 'returnsClosePrevMktres10',
'returnsOpenPrevMktres10',
'open', 'close',
'margin1', 'close_minus_open']
n_lag = [5]
new_df = generate_lag_features(history_df,n_lag=n_lag)
 market_obs_df = pd.merge(market_obs_df,new_df,how='left',on=['time','assetCode'])
 del new_df
gc.collect()
return_features = ['open', 'close']
n_lag = [5]
new_df = generate_lag_features2(history_df,n_lag=n_lag)
 market_obs_df = pd.merge(market_obs_df,new_df,how='left',on=['time','assetCode'])
 del new_df
gc.collect()
return_features = ['open', 'close']
n_lag = [5]
new_df = generate_lag_features3(history_df,n_lag=n_lag)
 market_obs_df = pd.merge(market_obs_df,new_df,how='left',on=['time','assetCode'])
 del new_df
gc.collect()
return_features = ['open', 'close']
n_lag = [5]
new_df = generate_lag_features4(history_df,n_lag=n_lag)
 market_obs_df = pd.merge(market_obs_df,new_df,how='left',on=['time','assetCode'])
 del new_df
gc.collect()
market_obs_df['volume_diff'] = market_obs_df['volume'] - market_obs_df['volume_lag_5_mean']
market_obs_df['returnsClosePrevRaw1_diff'] = market_obs_df['returnsClosePrevRaw1'] - market_obs_df['returnsClosePrevRaw1_lag_5_mean']
market_obs_df['returnsOpenPrevRaw1_diff'] = market_obs_df['returnsOpenPrevRaw1'] - market_obs_df['returnsOpenPrevRaw1_lag_5_mean']
market_obs_df['returnsClosePrevMktres1_diff'] = market_obs_df['returnsClosePrevMktres1'] - market_obs_df['returnsClosePrevMktres1_lag_5_mean']
market_obs_df['returnsOpenPrevMktres1_diff'] = market_obs_df['returnsOpenPrevMktres1'] - market_obs_df['returnsOpenPrevMktres1_lag_5_mean']
market_obs_df['returnsClosePrevRaw10_diff'] = market_obs_df['returnsClosePrevRaw10'] - market_obs_df['returnsClosePrevRaw10_lag_5_mean']
market_obs_df['returnsOpenPrevRaw10_diff'] = market_obs_df['returnsOpenPrevRaw10'] - market_obs_df['returnsOpenPrevRaw10_lag_5_mean']
market_obs_df['returnsClosePrevMktres10_diff'] = market_obs_df['returnsClosePrevMktres10'] - market_obs_df['returnsClosePrevMktres10_lag_5_mean']
market_obs_df['returnsOpenPrevMktres10_diff'] = market_obs_df['returnsOpenPrevMktres10'] - market_obs_df['returnsOpenPrevMktres10_lag_5_mean']
market_obs_df['open_diff'] = market_obs_df['open'] - market_obs_df['open_lag_5_mean']
market_obs_df['close_diff'] = market_obs_df['close'] - market_obs_df['close_lag_5_mean']
market_obs_df['margin1_diff'] = market_obs_df['margin1'] - market_obs_df['margin1_lag_5_mean']
market_obs_df['close_minus_open_diff'] = market_obs_df['close_minus_open'] - market_obs_df['close_minus_open_lag_5_mean']
market_obs_df['open_macd_diff'] = market_obs_df['open_ewma'] - market_obs_df['open_macd']
market_obs_df['close_macd_diff'] = market_obs_df['close_ewma'] - market_obs_df['close_macd']
market_obs_df['open_bb_high_diff'] = market_obs_df['open'] - market_obs_df['open_bollingerband_high']
market_obs_df['open_bb_low_diff'] = market_obs_df['open'] - market_obs_df['open_bollingerband_low']
market_obs_df['close_bb_high_diff'] = market_obs_df['close'] - market_obs_df['close_bollingerband_high']
market_obs_df['close_bb_low_diff'] = market_obs_df['close'] - market_obs_df['close_bollingerband_low']
market_obs_df['open_ewma_diff'] = market_obs_df['open'] - market_obs_df['open_ewma']
market_obs_df['close_ewma_diff'] = market_obs_df['close'] - market_obs_df['close_ewma']
yhat_lgbm = pd.DataFrame(corey_lgbm_model.predict(market_obs_df[lgbm_cols]))
yhat_goss = pd.DataFrame(corey_goss_model.predict(market_obs_df[lgbm_cols]))
yhat_dart = pd.DataFrame(corey_dart_model.predict(market_obs_df[lgbm_cols]))
active_history_cols = ['assetCode']
for step in range(0, n_in):
for varnum in range(1, len(INPUT_COLS)+1):
if step==0:
stepnum = ''
else:
stepnum = '-'+str(step)
 active_history_cols.append('var'+str(varnum)+'(t'+stepnum+')')
 active_history = active_history[active_history_cols]
market_obs_df = market_obs_df[LSTM_COLUMNS_TO_USE]
market_obs_df = market_obs_df.fillna(market_obs_df.median())
 market_obs_df[INPUT_COLS] = scaler.transform(market_obs_df[INPUT_COLS])
 market_obs_df = market_obs_df.fillna(0)
 active_history = market_obs_df.merge(right=active_history, how='outer', on='assetCode')
 active_history_cols = ['time','assetCode']
for step in range(0, n_in+1):
for varnum in range(1, len(INPUT_COLS)+1):
if step==0:
stepnum = ''
else:
stepnum = '-'+str(step)
 active_history_cols.append('var'+str(varnum)+'(t'+stepnum+')')
 active_history.columns = active_history_cols
active_history_cols = []
for step in range(0, n_in):
for varnum in range(1, len(INPUT_COLS)+1):
if step==0:
stepnum = ''
else:
stepnum = '-'+str(step)
 active_history_cols.append('var'+str(varnum)+'(t'+stepnum+')')
 active_history_cols2 = []
for step in range(1, n_in+1):
for varnum in range(1, len(INPUT_COLS)+1):
if step==0:
stepnum = ''
else:
stepnum = '-'+str(step)
 active_history_cols2.append('var'+str(varnum)+'(t'+stepnum+')')
 active_history.loc[active_history['time'].isnull() , active_history_cols] = active_history.loc[active_history['time'].isnull() , active_history_cols2].values
LAST_VAL_TO_BE_IMPUTED = 'var1(t-'+str(n_in)+')'
for step in range(1, n_in+1):
active_history.loc[active_history[LAST_VAL_TO_BE_IMPUTED].isnull() , [col for col in active_history.columns if '(t-'+str(step)+')' in col]] = active_history.loc[active_history[LAST_VAL_TO_BE_IMPUTED].isnull() , [col for col in active_history.columns if '(t)' in col]].values
to_predict = active_history.loc[active_history['time'].notnull() , [col for col in active_history.columns if 'var' in col]]
to_predict = to_predict.values
to_predict = to_predict.reshape(( to_predict.shape[0], 1, to_predict.shape[1]))
yhat_lstm = pd.DataFrame(model.predict(to_predict))
ensemble = pd.concat([yhat_lgbm, yhat_goss, yhat_dart, yhat_lstm], axis=1)
 ensemble.columns = ['lgbm','goss','dart', 'lstm']
preds = logreg.predict_proba(ensemble)[:,1]
preds =(preds * 2)- 1
predictions_template_df['confidenceValue'] = preds
env.predict(predictions_template_df )
<choose_model_class><EOS> 
env.write_submission_file()