Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<categorify>
pd.set_option('max_columns', 50 )
def create_examples(lines, set_type, labels=None):
guid=f'{set_type}'
examples = []
if set_type == 'train':
for line, label in zip(lines, labels):
text_a = line
label = str(label)
 examples.append(
 run_classifier.InputExample(guid=guid, text_a=text_a, text_b=None, label=label))
else:
for line in lines:
text_a = line
label = '0'
examples.append(
 run_classifier.InputExample(guid=guid, text_a=text_a, text_b=None, label=label))
return examples
TRAIN_BATCH_SIZE = 32
EVAL_BATCH_SIZE = 8
LEARNING_RATE = 1e-5
NUM_TRAIN_EPOCHS = 3.0
WARMUP_PROPORTION = 0.1
MAX_SEQ_LENGTH = 100
SAVE_CHECKPOINTS_STEPS = 1000
ITERATIONS_PER_LOOP = 100
NUM_TPU_CORES = 8
VOCAB_FILE = os.path.join(BERT_PRETRAINED_DIR, 'vocab.txt')
 CONFIG_FILE = os.path.join(BERT_PRETRAINED_DIR, 'bert_config.json')
 INIT_CHECKPOINT = os.path.join(BERT_PRETRAINED_DIR, 'bert_model.ckpt')
 DO_LOWER_CASE = BERT_MODEL.startswith('uncased')
 label_list = [str(num)for num in range(38)]
tokenizer = tokenization.FullTokenizer(vocab_file=VOCAB_FILE, do_lower_case=DO_LOWER_CASE)
 train_examples = create_examples(X_train, 'train', labels=y_train)
 tpu_cluster_resolver = None
run_config = tf.contrib.tpu.RunConfig(
 cluster=tpu_cluster_resolver,
model_dir=OUTPUT_DIR,
save_checkpoints_steps=SAVE_CHECKPOINTS_STEPS,
tpu_config=tf.contrib.tpu.TPUConfig(
 iterations_per_loop=ITERATIONS_PER_LOOP,
num_shards=NUM_TPU_CORES,
per_host_input_for_training=tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2))
num_train_steps = int(
 len(train_examples)/ TRAIN_BATCH_SIZE * NUM_TRAIN_EPOCHS)
 num_warmup_steps = int(num_train_steps * WARMUP_PROPORTION)
 model_fn = run_classifier.model_fn_builder(
 bert_config=modeling.BertConfig.from_json_file(CONFIG_FILE),
num_labels=len(label_list),
init_checkpoint=INIT_CHECKPOINT,
learning_rate=LEARNING_RATE,
num_train_steps=num_train_steps,
num_warmup_steps=num_warmup_steps,
use_tpu=False,
use_one_hot_embeddings=True)
 estimator = tf.contrib.tpu.TPUEstimator(
 use_tpu=False,
model_fn=model_fn,
config=run_config,
train_batch_size=TRAIN_BATCH_SIZE,
eval_batch_size=EVAL_BATCH_SIZE )<train_model>
env = twosigmanews.make_env()
market_train_df, news_train = env.get_training_data()
print('Please wait...')
 train_features = run_classifier.convert_examples_to_features(
 train_examples, label_list, MAX_SEQ_LENGTH, tokenizer)
 print('>> Started training at {} '.format(datetime.now()))
print(' Num examples = {}'.format(len(train_examples)))
 print(' Batch size = {}'.format(TRAIN_BATCH_SIZE))
tf.logging.info(" Num steps = %d", num_train_steps)
 train_input_fn = run_classifier.input_fn_builder(
 features=train_features,
seq_length=MAX_SEQ_LENGTH,
is_training=True,
drop_remainder=True)
 estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)
 print('>> Finished training at {}'.format(datetime.now()))<define_variables>
for column in market_train_df.select_dtypes(include='float64' ).columns:
market_train_df[column] = market_train_df[column].astype('float16' )
def input_fn_builder(features, seq_length, is_training, drop_remainder):

 all_input_ids = []
all_input_mask = []
all_segment_ids = []
all_label_ids = []
for feature in features:
all_input_ids.append(feature.input_ids)
 all_input_mask.append(feature.input_mask)
 all_segment_ids.append(feature.segment_ids)
 all_label_ids.append(feature.label_id)
 def input_fn(params):

 print(params)
 batch_size = 500
num_examples = len(features)
 d = tf.data.Dataset.from_tensor_slices({
"input_ids":
tf.constant(
 all_input_ids, shape=[num_examples, seq_length],
dtype=tf.int32),
"input_mask":
tf.constant(
 all_input_mask,
shape=[num_examples, seq_length],
dtype=tf.int32),
"segment_ids":
tf.constant(
 all_segment_ids,
shape=[num_examples, seq_length],
dtype=tf.int32),
"label_ids":
tf.constant(all_label_ids, shape=[num_examples], dtype=tf.int32),
})
 if is_training:
d = d.repeat()
d = d.shuffle(buffer_size=100)
 d = d.batch(batch_size=batch_size, drop_remainder=drop_remainder)
 return d
return input_fn<predict_on_test>
start = datetime(2010, 1, 1, 0, 0, 0 ).date()
market_train = market_train_df.loc[market_train_df['time'].dt.date >= start].reset_index(drop=True)
 news_train = news_train.loc[news_train['time'].dt.date >= start].reset_index(drop=True )
predict_examples = create_examples(X_test, 'test')
 predict_features = run_classifier.convert_examples_to_features(
 predict_examples, label_list, MAX_SEQ_LENGTH, tokenizer)
 predict_input_fn = input_fn_builder(
 features=predict_features,
seq_length=MAX_SEQ_LENGTH,
is_training=False,
drop_remainder=False)
 result = estimator.predict(input_fn=predict_input_fn )<prepare_output>
def preprocess_news(news_train):
drop_list = [
'audiences', 'subjects', 'assetName',
'headline', 'firstCreated', 'sourceTimestamp',
]
news_train.drop(drop_list, axis=1, inplace=True)
 for col in ['headlineTag', 'provider', 'sourceId']:
news_train[col], uniques = pd.factorize(news_train[col])
 del uniques
news_train['assetCodes'] = news_train['assetCodes'].apply(lambda x: x[1:-1].replace("'", ""))
return news_train
preds = []
for prediction in result:
preds.append(np.argmax(prediction['probabilities']))<import_modules>
news_train = preprocess_news(news_train )
from sklearn.metrics import accuracy_score<compute_test_metric>
def unstack_asset_codes(news_train):
codes = []
indexes = []
for i, values in news_train['assetCodes'].iteritems() :
explode = values.split(", ")
 codes.extend(explode)
 repeat_index = [int(i)]*len(explode)
 indexes.extend(repeat_index)
 index_df = pd.DataFrame({'news_index': indexes, 'assetCode': codes})
 del codes, indexes
gc.collect()
return index_df
print("Accuracy of BERT is:",accuracy_score(y_test,preds))<compute_test_metric>
index_df = unstack_asset_codes(news_train)
 index_df.head()
print(classification_report(y_test,preds))<feature_engineering>
def merge_news_on_index(news_train, index_df):
news_train['news_index'] = news_train.index.copy()
news_unstack = index_df.merge(news_train, how='left', on='news_index')
 news_unstack.drop(['news_index', 'assetCodes'], axis=1, inplace=True)
 return news_unstack
testpd = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 testpd['text'] = testpd['text'].apply(lambda x : re.sub(r'\b[A-Z]+\b', '',re.sub('\s+',' ',re.sub(r'[^a-zA-Z]', ' ',str(x)))))
 testpd['text'] = testpd['text'].apply(lambda x : x.lower())
 testln = pd.DataFrame({'id':testpd['id'],
'text':testpd['text'].replace(r'
',' ',regex=True)})
 testl = testln["text"].values<predict_on_test>
news_unstack = merge_news_on_index(news_train, index_df)
 del news_train, index_df
gc.collect()
news_unstack.head(3 )
predict_test = create_examples(testl, 'test')
 predict_features1 = run_classifier.convert_examples_to_features(
 predict_test, label_list, MAX_SEQ_LENGTH, tokenizer)
 predict_input_fn1 = input_fn_builder(
 features=predict_features1,
seq_length=MAX_SEQ_LENGTH,
is_training=False,
drop_remainder=False)
 result1 = estimator.predict(input_fn=predict_input_fn1 )<prepare_output>
def group_news(news_frame):
news_frame['date'] = news_frame.time.dt.date
aggregations = ['mean']
gp = news_frame.groupby(['assetCode', 'date'] ).agg(aggregations)
 gp.columns = pd.Index(["{}_{}".format(e[0], e[1])for e in gp.columns.tolist() ])
 gp.reset_index(inplace=True)
 gp.drop(['sourceId_mean'],axis=1, inplace=True)
 float_cols = {c: 'float32' for c in gp.columns if c not in ['assetCode', 'date']}
return gp.astype(float_cols )
preds1 = []
for prediction in result1 :
preds1.append(np.argmax(prediction['probabilities']))<prepare_output>
news_agg = group_news(news_unstack)
 del news_unstack; gc.collect()
news_agg.head(3 )
def create_output(predictions):
probabilities = []
for(i, prediction)in enumerate(predictions):
preds = prediction
probabilities.append(preds)
 dff = pd.DataFrame(probabilities)
 dff.head()
return dff<save_to_csv>
def process_date(df):
df['date'] = df['time'].dt.date
df['month'] = df['time'].dt.month
df['dayofweek'] = df['time'].dt.dayofweek
return df
predslab=[identity_columns[x] for x in preds1]
testln["target"]=predslab
merged_df = testln
submission = merged_df.drop(['text'], axis=1)
 submission.to_csv("submission.csv", index=False )<import_modules>
def process_ma(df,columns=['open','close','volume'], windows=[10, 20, 60]):
ma_columns = []
ma_dev_columns = []
drop_list = []
for col in columns:
for window in windows:
ma_column = 'ma_{0}_{1}'.format(col,window)
 ma_dev_column = 'ma_dev_{0}_{1}'.format(col,window)
 ma_lag_column = 'ma_lag_{0}_{1}'.format(col,window)
 ma_lag_rate_column = 'ma_lag_rate_{0}_{1}'.format(col,window)
 std_column = 'std_{0}_{1}'.format(col,window)
 ma_columns.append(ma_column)
 ma_dev_columns.append(ma_dev_column)
 df[ma_column] = df.groupby('assetCode')[col].apply(lambda x: x.rolling(window ).mean())
 df[ma_dev_column] = df[col] / df[ma_column] - 1
df[std_column] = df.groupby('assetCode')[col].apply(lambda x: x.rolling(window ).std())
 df[ma_lag_column] = df.groupby('assetCode')[ma_column].shift()
df[ma_lag_rate_column] =(df[ma_column] - df[ma_lag_column])/ df[ma_lag_column]
drop_list.append(ma_column)
 drop_list.append(ma_lag_column)
 df.drop(drop_list,axis=1, inplace=True)
 return df
import numpy as np
import pandas as pd
from fastai.text import *
from fastai.callbacks import *<load_from_csv>
market_train = process_date(market_train)
 market_train = process_ma(market_train )
train = pd.read_csv(path/'train.csv')
 test = pd.read_csv(path/'test.csv' )<concatenate>
df = market_train.merge(news_agg, how='left', on=['assetCode', 'date'])
 del market_train, news_agg
gc.collect()
df.head(3 )
data_lm =(TextList.from_df(pd.concat([train[['text']], test[['text']]], ignore_index=True, axis=0)) 
.split_by_rand_pct(0.15)
.label_for_lm() 
.databunch(bs=128))<define_variables>
for column in df.select_dtypes(include='float64' ).columns:
df[column] = df[column].astype('float16')
 for column in df.select_dtypes(include='int64' ).columns:
df[column] = df[column].astype('int16')
 df.dtypes
data_lm.show_batch()<load_pretrained>
def custom_metric(date, pred_proba, num_target, universe):
y = pred_proba*2 - 1
r = num_target.clip(-1,1)
 x = y * r * universe
result = pd.DataFrame({'day' : date, 'x' : x})
 x_t = result.groupby('day' ).sum().values
return np.mean(x_t)/ np.std(x_t )
learn = language_model_learner(data_lm, AWD_LSTM, drop_mult=0.5 )<train_model>
date = df.date
num_target = df.returnsOpenNextMktres10.astype('float32')
 bin_target =(df.returnsOpenNextMktres10 >= 0 ).astype('int8')
 universe = df.universe.astype('int8')
 df.drop(['returnsOpenNextMktres10', 'date', 'universe', 'assetCode', 'assetName', 'time'],
axis=1, inplace=True)
 df = df.astype('float32')
 gc.collect()
learn.fit_one_cycle(1, 1e-2, moms=(0.8,0.7))<choose_model_class>
train_index, test_index = train_test_split(df.index.values, test_size=0.1 )
callback = SaveModelCallback(learn,monitor="accuracy", mode="max", name="best_lang_model" )<train_model>
best_params = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'binary_logloss',
'learning_rate': 0.05,
'num_leaves': 63,
'n_estimators': 1000,
'min_child_samples': 20,
'colsample_bytree': 1,
'subsample': 1.0,
'reg_alpha': 0.1,
'reg_lambda': 0.8,
}
learn.unfreeze()
learn.fit_one_cycle(10, 1e-3, moms=(0.8,0.7), callbacks=[callback] )<categorify>
lgb_train = lgb.Dataset(df.iloc[train_index], bin_target.iloc[train_index])
 lgb_eval = lgb.Dataset(df.iloc[test_index], bin_target.iloc[test_index], reference=lgb_train )
learn.save('fine_tuned')
 learn.save_encoder('fine_tuned_enc' )<drop_column>
clf = lgb.train(
 best_params,
lgb_train,
valid_sets=lgb_eval,
early_stopping_rounds=5
)
df = train[['text', 'target']]<drop_column>
pred_y = clf.predict(df.iloc[test_index])
 pred_y[pred_y >= 0.5] = 1
pred_y[pred_y < 0.5] = 0
print(( pred_y == bin_target.iloc[test_index] ).sum() / len(pred_y))
df_test = test[['text']]<categorify>
test_df_columns = ['time', 'assetCode', 'assetName', 'volume', 'close', 'open',
'returnsClosePrevRaw1', 'returnsOpenPrevRaw1',
'returnsClosePrevMktres1', 'returnsOpenPrevMktres1',
'returnsClosePrevRaw10', 'returnsOpenPrevRaw10',
'returnsClosePrevMktres10', 'returnsOpenPrevMktres10']
base_df = market_train_df[market_train_df['time'] >= '2016-10-01']
base_df = base_df[test_df_columns]
base_df['id'] = -1
base_df.shape
data_clas =(TextList.from_df(df, vocab=data_lm.vocab)
.split_by_rand_pct(0.1)
.label_from_df('target')
.add_test(TextList.from_df(df_test, vocab=data_lm.vocab)) 
.databunch(bs=128))<load_pretrained>
def calc_target(df):
df['open_next10'] = df.groupby('assetCode')['open'].shift(-10)
 df['return_next10'] = df['open_next10'] / df['open'] - 1.0
df['target'] =(df.return_next10 >= 0 ).astype('int8')
 return df
learn = text_classifier_learner(data_clas, AWD_LSTM, drop_mult=0.5, metrics=[accuracy, FBeta(beta=1)])
 learn.load_encoder('fine_tuned_enc' )<train_model>
target_test_df = calc_target(market_train_df)
 target_test_df.head()
learn.fit_one_cycle(1, 1e-3, moms=(0.8,0.7))<train_model>
def online_train(model, df, train_day_id, feats):
best_params = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'binary_logloss',
'learning_rate': 0.05,
'num_leaves': 63,
'n_estimators': 1000,
'min_child_samples': 20,
'colsample_bytree': 1,
'subsample': 1.0,
'reg_alpha': 0.1,
'reg_lambda': 0.8,
}
train_df = calc_target(df)
 train_df = train_df[train_df['id'] == train_day_id]
train_df.reset_index(drop=True)
 target_df = train_df[['target']]
train_df = train_df[feats]
model.refit(train_df, target_df['target'])
 return model
learn.freeze_to(-2)
 learn.fit_one_cycle(1, slice(1e-3/(2.6**4),1e-2), moms=(0.8,0.7))<train_model>
def write_submission(model, env):
days = env.get_prediction_days()
day_id = 0
market_obs_df_append = None
online_train_df_append = None
for(market_obs_df, news_obs_df, predictions_template_df)in days:
news_obs_df = preprocess_news(news_obs_df)
 index_df = unstack_asset_codes(news_obs_df)
 news_unstack = merge_news_on_index(news_obs_df, index_df)
 news_obs_agg = group_news(news_unstack)
 market_obs_df['id'] = day_id
if market_obs_df_append is None:
market_obs_df_append = base_df
market_obs_df_append = pd.concat([market_obs_df_append,market_obs_df],
ignore_index=True,
sort=False)
 market_obs_process = process_date(market_obs_df_append)
 market_obs_process = process_ma(market_obs_process)
 market_obs_df = market_obs_process[market_obs_process['id']==day_id]
obs_df = market_obs_df.merge(news_obs_agg, how='left', on=['assetCode', 'date'])
 del market_obs_df, news_obs_agg, news_obs_df, news_unstack, index_df
gc.collect()
obs_df = obs_df[obs_df.assetCode.isin(predictions_template_df.assetCode)]
feats = [c for c in obs_df.columns if c not in ['date', 'assetCode', 'assetName', 'time', 'id']]
if online_train_df_append is None:
online_train_df_append = obs_df
else:
online_train_df_append = pd.concat([online_train_df_append, obs_df],
ignore_index=True,
sort=False)
 if day_id >= 10:
model = online_train(model, online_train_df_append, day_id - 10, feats)
 preds = model.predict(obs_df[feats])* 2 - 1
sub = pd.DataFrame({'assetCode': obs_df['assetCode'], 'confidence': preds})
 predictions_template_df = predictions_template_df.merge(sub, how='left' ).drop(
 'confidenceValue', axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 if day_id == 59:
market_obs_df_append.drop(
 market_obs_df_append.index[market_obs_df_append['id']==-1],
inplace=True)
 elif day_id >= 60:
market_obs_df_append.drop(
 market_obs_df_append.index[market_obs_df_append['id']==day_id-60],
inplace=True)
 day_id += 1
del obs_df, predictions_template_df, preds, sub
gc.collect()
env.write_submission_file()
print('day_count',day_id )
learn.freeze_to(-3)
 learn.fit_one_cycle(1, slice(5e-3/(2.6**4),5e-3), moms=(0.8,0.7))<choose_model_class>
write_submission(clf, env )