Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<predict_on_test>
env = twosigmanews.make_env() 
(market_train, news_train)= env.get_training_data()
pred_val_y, pred_test_y = train_pred(model_lstm_du(embedding_matrix), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, 'LSTM DU'] )<predict_on_test>
pred_val_y, pred_test_y = train_pred(model_lstm_atten(embedding_matrix), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, '2 LSTM w/ attention'] )<predict_on_test>
market_train = market_train[market_train.returnsOpenNextMktres10<1][market_train.returnsOpenNextMktres10>-1]
pred_val_y, pred_test_y = train_pred(model_lstm_atten(embedding_matrix_1), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, '2 LSTM w/ attention GloVe'] )<predict_on_test>
def calcRsi(series, period = 14):

 try:
delta = series.diff().dropna()
u = delta * 0
d = u.copy()
u[delta > 0] = delta[delta > 0]
d[delta < 0] = -delta[delta < 0]
u[u.index[period-1]] = np.mean(u[:period])
 u = u.drop(u.index[:(period-1)])
 d[d.index[period-1]] = np.mean(d[:period])
 d = d.drop(d.index[:(period-1)])
 rs = u.ewm(com=period-1, adjust=False ).mean() \
/ d.ewm(com=period-1, adjust=False ).mean()
rsi = 100 - 100 /(1 + rs)
 except IndexError:
rsi = 0
return rsi
pred_val_y, pred_test_y = train_pred(model_lstm_atten(embedding_matrix_3), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, '2 LSTM w/ attention Para'] )<load_from_csv>
def addBollinger(df, period=20, col='close'):

 bbmid_series = df[col].rolling(window=period ).mean()
series_stdev = df[col].rolling(window=period ).std()
df['BBUpperBand'] = bbmid_series + 2*series_stdev
df['BBLowerBand'] = bbmid_series - 2*series_stdev
df['BBBandwidth'] = df['BBUpperBand'] - df['BBLowerBand']
df['BBMiddleBand'] = bbmid_series
return df
coefs = [0.20076554,0.07993707,0.11611663,0.14885248,0.15734404,0.17454667,0.14288361]
pred_test_y = np.sum([outputs[i][1]*coefs[i] for i in range(len(coefs)) ], axis = 0)
 pred_test_y =(pred_test_y > 0.34 ).astype(int)
 test_df = pd.read_csv(".. /input/test.csv", usecols=["qid"])
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<load_pretrained>
def addMACD(df):
ema_fast = df['close'].ewm(span=12 ).mean()
ema_slow = df['close'].ewm(span=26 ).mean()
signal_line = df['close'].ewm(span=9 ).mean()
df['macd'] = ema_fast - ema_slow
df['macd_signal'] = df.macd.ewm(span=9, adjust=False ).mean()
df['macdh'] = df['macd'] - df['macd_signal']
return df
def save_model(model, model_path):

 torch.save(model.state_dict() , model_path)
 def load_model(model, model_path, use_cuda=False):

 map_location = 'cpu'
if use_cuda and torch.cuda.is_available() :
map_location = 'cuda:0'
model.load_state_dict(torch.load(model_path, map_location))
return model<categorify>
def allindicator(market_train):
market_train['RSI'] = calcRsi(market_train['close'], 14)
 market_train = addBollinger(market_train)
 market_train = addMACD(market_train)
 return market_train
NEG_INF = -10000
TINY_FLOAT = 1e-6
def mask_mean(seq, mask=None):

 if mask is None:
return torch.mean(seq, dim=1)
 mask_sum = torch.sum(
 seq * mask.unsqueeze(-1 ).float() , dim=1)
 seq_len = torch.sum(mask, dim=-1)
 mask_mean = mask_sum /(seq_len.unsqueeze(-1 ).float() + TINY_FLOAT)
 return mask_mean
def mask_max(seq, mask=None):

 if mask is None:
return torch.mean(seq, dim=1)
 torch
mask_max, _ = torch.max(
 seq +(1 - mask.unsqueeze(-1 ).float())* NEG_INF,
dim=1)
 return mask_max
def seq_mask(seq_len, max_len):

 idx = torch.arange(max_len ).to(seq_len ).repeat(seq_len.size(0), 1)
 mask = torch.gt(seq_len.unsqueeze(1), idx ).to(seq_len)
 return mask
class DynamicLSTM(nn.Module):

 def __init__(self, input_size, hidden_size=100,
num_layers=1, dropout=0., bidirectional=False):
super(DynamicLSTM, self ).__init__()
self.lstm = nn.LSTM(
 input_size, hidden_size, num_layers, bias=True,
batch_first=True, dropout=dropout, bidirectional=bidirectional)
 def forward(self, x, seq_lens):
_, idx_sort = torch.sort(seq_lens, dim=0, descending=True)
 _, idx_unsort = torch.sort(idx_sort, dim=0)
 x_sort = torch.index_select(x, dim=0, index=idx_sort)
 seq_lens_sort = torch.index_select(seq_lens, dim=0, index=idx_sort)
 x_packed = pack_padded_sequence(
 x_sort, seq_lens_sort, batch_first=True)
 y_packed, _ = self.lstm(x_packed)
 y_sort, length = pad_packed_sequence(y_packed, batch_first=True)
 y = torch.index_select(y_sort, dim=0, index=idx_unsort)
 return y
class QuoraModel(nn.Module):

 def __init__(self, args):
super(QuoraModel, self ).__init__()
vocab_size = args["vocab_size"]
pretrained_embed = args["pretrained_embed"]
padding_idx = args["padding_idx"]
embed_dim = 300
num_classes = 1
num_layers = 2
hidden_dim = 50
dropout = 0.5
if pretrained_embed is None:
self.embed = nn.Embedding(vocab_size, embed_dim)
 else:
self.embed = nn.Embedding.from_pretrained(
 pretrained_embed, freeze=False)
 self.embed.padding_idx = padding_idx
self.rnn = DynamicLSTM(
 embed_dim, hidden_dim, num_layers=num_layers,
dropout=dropout, bidirectional=True)
 self.fc = nn.Linear(hidden_dim * 4, hidden_dim)
 self.act = nn.ReLU()
self.drop = nn.Dropout(dropout)
 self.out = nn.Linear(hidden_dim, num_classes)
 self.loss = nn.BCEWithLogitsLoss()
def forward(self, word_seq, seq_len):
max_seq_len = torch.max(seq_len)
 mask = seq_mask(seq_len, max_seq_len)
 e = self.drop(self.embed(word_seq))
r = self.rnn(e, seq_len)
 r_avg = mask_mean(r, mask)
 r_max = mask_max(r, mask)
 r = torch.cat([r_avg, r_max], dim=-1)
 f = self.drop(self.act(self.fc(r)))
 logits = self.out(f ).squeeze(-1)
 return logits
class Trainer(object):

 def __init__(self, **kwargs):
self.n_epochs = kwargs["epochs"]
self.batch_size = kwargs["batch_size"]
self.validate = kwargs["validate"]
self.save_best_dev = kwargs["save_best_dev"]
self.use_cuda = kwargs["use_cuda"]
self.print_every_step = kwargs["print_every_step"]
self.optimizer = kwargs["optimizer"]
self.model_path = kwargs["model_path"]
self.eval_metrics = kwargs["eval_metrics"]
self._best_accuracy = 0.0
self.device = 'cpu'
if torch.cuda.is_available() and self.use_cuda:
self.device = 'cuda:0'
def train(self, network, train_data, dev_data=None):
network = network.to(self.device)
 train_iter = torchtext.data.Iterator(
 dataset=train_data, batch_size=self.batch_size,
train=True, shuffle=True, sort=False,
device=self.device)
 if self.validate:
default_valid_args = {
"batch_size": max(8, self.batch_size // 10),
"use_cuda": self.use_cuda}
validator = Tester(**default_valid_args)
 start = time.time()
for epoch in range(1, self.n_epochs + 1):
network.train()
train_iter.init_epoch()
self._train_step(
 train_iter, network, start=start,
n_print=self.print_every_step, epoch=epoch)
 if self.validate:
if dev_data is None:
raise RuntimeError(
 "self.validate is True in trainer, "
"but dev_data is None."
" Please provide the validation data.")
 eval_results = validator.test(network, dev_data)
 if self.save_best_dev and self.best_eval_result(eval_results):
save_model(network, self.model_path)
 print("Saved better model selected by validation.")
 def _train_step(self, data_iterator, network, **kwargs):

 step = 0
for batch in data_iterator: 
(text, text_len), target = batch.text, batch.target
self.optimizer.zero_grad()
logits = network(text, text_len)
 loss = network.loss(logits, target.float())
 loss.backward()
self.optimizer.step()
if kwargs["n_print"] > 0 and step % kwargs["n_print"] == 0:
end = time.time()
diff = timedelta(seconds=round(end - kwargs["start"]))
print_output = "[epoch: {:>3} step: {:>4}]" \
" train loss: {:>4.6} time: {}".format(
 kwargs["epoch"], step, loss.item() , diff)
 print(print_output)
 step += 1
def best_eval_result(self, eval_results):

 assert self.eval_metrics in eval_results, \
"Evaluation doesn't contain metrics '{}'." \ 
.format(self.eval_metrics)
 accuracy = eval_results[self.eval_metrics]
if accuracy > self._best_accuracy:
self._best_accuracy = accuracy
return True
else:
return False
class Tester(object):

 def __init__(self, **kwargs):
self.batch_size = kwargs["batch_size"]
self.use_cuda = kwargs["use_cuda"]
self.device = 'cpu'
if torch.cuda.is_available() and self.use_cuda:
self.device = 'cuda:0'
def test(self, network, dev_data, threshold=0.33):
network = network.to(self.device)
 network.eval()
output_list = []
truth_list = []
data_iter = torchtext.data.Iterator(
 dataset=dev_data, batch_size=self.batch_size,
train=False, device=self.device, sort=False)
 for batch in data_iter:
text, target = batch.text, batch.target
with torch.no_grad() :
prediction = network(*text)
 output_list.append(prediction.detach())
 truth_list.append(target.detach())
 eval_results = self.evaluate(output_list, truth_list, threshold)
 print("[tester] {}".format(self.print_eval_results(eval_results)))
 return eval_results
def evaluate(self, predict, truth, threshold=0.33):

 y_trues, y_preds = [], []
for y_true, logit in zip(truth, predict):
y_pred =(torch.sigmoid(logit)> threshold ).long().cpu().numpy()
y_true = y_true.cpu().numpy()
y_trues.append(y_true)
 y_preds.append(y_pred)
 y_true = np.concatenate(y_trues, axis=0)
 y_pred = np.concatenate(y_preds, axis=0)
 precision = metrics.precision_score(y_true, y_pred, pos_label=1)
 recall = metrics.recall_score(y_true, y_pred, pos_label=1)
 f1 = metrics.f1_score(y_true, y_pred, pos_label=1)
 metrics_dict = {"precision": precision, "recall": recall, "f1": f1}
return metrics_dict
def print_eval_results(self, results):

 return ", ".join(
 [str(key)+ "=" + "{:.4f}".format(value)
 for key, value in results.items() ])
 class Predictor(object):

 def __init__(self, batch_size=8, use_cuda=False):
self.batch_size = batch_size
self.use_cuda = use_cuda
self.device = 'cpu'
if torch.cuda.is_available() and self.use_cuda:
self.device = 'cuda:0'
def predict(self, network, data, threshold=0.33):
network = network.to(self.device)
 network.eval()
batch_output = []
data_iter = torchtext.data.Iterator(
 dataset=data, batch_size=self.batch_size,
train=False, device=self.device, sort=False)
 for batch in data_iter:
text = batch.text
with torch.no_grad() :
prediction = network(*text)
 batch_output.append(prediction.detach())
 return self._post_processor(batch_output, threshold)
 def _post_processor(self, batch_output, threshold=0.33):

 y_preds = []
for logit in batch_output:
y_pred =(torch.sigmoid(logit)> threshold ).long().cpu().numpy()
y_preds.append(y_pred)
 y_pred = np.concatenate(y_preds, axis=0)
 return y_pred<define_variables>
from multiprocessing import Pool
train_path = '.. /input/train.csv'
test_path = '.. /input/test.csv'
embed_path = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
submission_path = './submission.csv'
model_path = './default_model.pkl'<create_dataframe>
def groupAsset(market_train):
all_df = []
df_codes = market_train.groupby('assetCode')
 df_codes = [df_code[1] for df_code in df_codes]
pool = Pool(4)
 all_df = pool.map(allindicator, df_codes)
 new_df = pd.concat(all_df)
 pool.close()
del all_df, df_codes, market_train
return new_df
def pre() :

 print("Pre-processing...")
 fix_length = 100
text = torchtext.data.Field(
 sequential=True, use_vocab=True, lower=True,
tokenize=nltk.word_tokenize, batch_first=True,
is_target=False, fix_length=fix_length,
include_lengths=True)
 target = torchtext.data.Field(
 sequential=False, use_vocab=False,
batch_first=True, is_target=True)
 dataset = torchtext.data.TabularDataset(
 train_path, format='csv',
fields={"question_text":('text', text),
"target":('target', target)})
 data_test = torchtext.data.TabularDataset(
 test_path, format='csv',
fields={"question_text":('text', text)})
 text.build_vocab(dataset, data_test, min_freq=3)
 text.vocab.load_vectors(torchtext.vocab.Vectors(embed_path))
vocab_size = len(text.vocab.itos)
 padding_idx = text.vocab.stoi[text.pad_token]
data_train, data_val = dataset.split(split_ratio=0.9)
 print("train set size:", len(data_train))
print("val set size:", len(data_val))
print("test set size:", len(data_test))
print("vocab size:", len(text.vocab.itos))
print("embed shape:", text.vocab.vectors.shape)
 print('')
 args_dict = {
"data_train": data_train, "data_val": data_val,
"data_test": data_test, "vocab_size": vocab_size,
"padding_idx": padding_idx}
return args_dict<categorify>
def data_prep(market_train,news_train):
market_train = groupAsset(market_train)
 market_train['formatdate']=market_train['time'].dt.date
news_train['formatdate'] = news_train.firstCreated.dt.date
for col_cat in ['headlineTag', 'provider', 'sourceId']:
news_train[col_cat], uniques = pd.factorize(news_train[col_cat])
 del uniques
news_train_sim = news_train.drop(columns=['time','sourceTimestamp','firstCreated','headline','subjects','audiences','assetCodes'])
 news_train_1 = news_train_sim.groupby(by=['assetName','formatdate'],as_index=False,observed=True ).mean().add_suffix('_mean')
 news_train_1.rename(columns={'assetName_mean': 'assetName', 'formatdate_mean': 'formatdate'}, inplace=True)
 merge_train = pd.merge(market_train, news_train_1, how='left', left_on=['formatdate', 'assetName'],
right_on=['formatdate', 'assetName'])
 merge_train.fillna(0,inplace=True)
 del market_train, news_train_sim, news_train_1
return merge_train
args = pre()<train_on_grid>
merge_train = data_prep(market_train,news_train)
 up = merge_train.returnsOpenNextMktres10 >= 0
up = up.values
fcol = [c for c in merge_train.columns if c not in ['assetCode','assetName','time',
'returnsOpenNextMktres10', 'formatdate', 'universe']]
X = merge_train[fcol].values
r = merge_train.returnsOpenNextMktres10.values
mins = np.min(X, axis=0)
 maxs = np.max(X, axis=0)
 rng = maxs - mins
X = 1 -(( maxs - X)/ rng )
def train(**args):

 print("Training...")
 data_train = args["data_train"]
pretrained_embed = data_train.fields["text"].vocab.vectors
model_args = {
"vocab_size": args["vocab_size"],
"padding_idx": args["padding_idx"],
"pretrained_embed": pretrained_embed,
}
model = QuoraModel(model_args)
 trainer_args = {
"epochs": 3,
"batch_size": 128,
"validate": True,
"save_best_dev": True,
"use_cuda": True,
"print_every_step": 1000,
"optimizer": torch.optim.Adam(model.parameters() , lr=1e-3),
"model_path": model_path,
"eval_metrics": "f1",
}
trainer = Trainer(**trainer_args)
 data_val = args["data_val"]
trainer.train(model, data_train, dev_data=data_val)
 print('' )<train_model>
X_train, X_test, up_train, up_test, r_train, r_test = model_selection.train_test_split(X, up, r, test_size=0.1, random_state=99)
train(**args )<choose_model_class>
params = {'learning_rate': 0.25, 'max_depth': 10, 'boosting': 'gbdt', 'objective': 'binary', 'metric': 'binary_logloss', 'is_training_metric': True, 'seed': 42}
model = lgb.train(params, train_set=lgb.Dataset(X_train, label=up_train), num_boost_round=2000,
valid_sets=[lgb.Dataset(X_train, label=up_train), lgb.Dataset(X_test, label=up_test)],
verbose_eval=50, early_stopping_rounds=30 )
def test(**args):

 print("Testing...")
 model_args = {
"vocab_size": args["vocab_size"],
"padding_idx": args["padding_idx"],
"pretrained_embed": None,
}
model = QuoraModel(model_args)
 load_model(model, model_path, use_cuda=True)
 tester_args = {
"batch_size": 128,
"use_cuda": True,
}
tester = Tester(**tester_args)
 data_val = args["data_val"]
best_thresh, best_f1 = 0., 0.
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 f1 = tester.test(model, data_val, threshold=thresh)["f1"]
print("threshold: {:>.2f} f1: {}".format(thresh, f1))
if f1 > best_f1:
best_thresh, best_f1 = thresh, f1
args["threshold"] = best_thresh
print("best f1 on dev: {} threshold: {:>.2f}".format(best_f1, best_thresh))
print('')
 return args<choose_model_class>
n_days = 0
for(market_obs_df, news_obs_df, predictions_template_df)in env.get_prediction_days() :
n_days +=1
if n_days % 50 == 0:
print(n_days,end=' ')
 market_obs_df = data_prep(market_obs_df, news_obs_df)
 X_live = market_obs_df[fcol].values
X_live = 1 -(( maxs - X_live)/ rng)
 lp = model.predict(X_live)
 confidence = 2 * lp -1
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence':confidence})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
<load_from_csv><EOS> 
env.write_submission_file()