Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<load_from_csv>
import numpy as np
import pandas as pd
import random
from datetime import datetime, date
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
import time
import lightgbm as lgb
from kaggle.competitions import twosigmanews
!cp '/kaggle/input/nlp-getting-started/test.csv' '/kaggle/working/test.csv'
!cp '/kaggle/input/nlp-getting-started/train.csv' '/kaggle/working/train.csv'<install_modules>
env = twosigmanews.make_env() 
(market_train, news_train)= env.get_training_data()
!pip install fastai --upgrade<set_options>
for col in market_train.columns:
if(market_train[col].dtype == "int64" or market_train[col].dtype == "float64"):
market_train[col] = market_train[col].fillna(market_train[col].mean() )
%reload_ext autoreload
%autoreload 2
path = Path('/kaggle/working/')
 path.ls()<load_from_csv>
market_train['time'] = market_train.time.dt.date
market_train = market_train.loc[market_train['time'] >= date(2010, 1, 1)]
train_df = pd.read_csv(path/'train.csv')
 test_df = pd.read_csv(path/'test.csv')
 print(train_df.head())
<load_from_csv>
asset_code_dict = {k: v for v, k in enumerate(market_train['assetCode'].unique())}
columns_news = ['firstCreated','relevance','sentimentClass','sentimentNegative','sentimentNeutral',
'sentimentPositive','noveltyCount24H','noveltyCount7D','volumeCounts24H','volumeCounts7D','assetCodes','sourceTimestamp',
'assetName','audiences', 'urgency', 'takeSequence', 'bodySize', 'companyCount',
'sentenceCount', 'firstMentionSentence','time']
dls = TextDataLoaders.from_csv(path=path,
csv_fname='train.csv',
text_col='text',
label_col='target',
valid_pct=0.2)
 first(dls[0] )<choose_model_class>
def feature_engineering(df):
df['date'] = df['time']
df['price_diff'] = df['close'] - df['open']
df['close_to_open'] = np.abs(df['close'] / df['open'])
 df['assetName_mean_open'] = df.groupby('assetName')['open'].transform('mean')
 df['assetName_mean_close'] = df.groupby('assetName')['close'].transform('mean')
 df['assetCodeT'] = df['assetCode'].map(asset_code_dict)
 return df
df = feature_engineering(market_train )
learn = text_classifier_learner(dls, AWD_LSTM, drop_mult=0.5, metrics=F1Score())
 learn.model_dir = '/kaggle/working/'
learn.fine_tune(2, 1e-2 )<save_model>
def reduce_mem_usage(df):
for col in df.columns:
if df[col].dtype == np.float64 or df[col].dtype == np.float32:
df[col] = df[col].astype(np.float16)
 return df
df = reduce_mem_usage(df )
learn.save('simple_NLP' )<load_pretrained>
for col in df.columns:
if df[col].dtype == "int64" or df[col].dtype == "float16":
df[col] = df[col].fillna(df[col].mean() )
learn.load('simple_NLP' )<predict_on_test>
up = df['returnsOpenNextMktres10'] >= 0
universe = df['universe'].values
d = df['date']
fcol = [c for c in df if c not in ['date', 'assetCode', 'assetCodes', 'assetCodesLen', 'assetName', 'audiences',
'firstCreated', 'headline', 'headlineTag', 'marketCommentary', 'provider',
'returnsOpenNextMktres10', 'sourceId', 'subjects', 'time', 'time_x', 'universe','sourceTimestamp']]
print("Creating X")
 X = df[fcol].values
print("Creating up")
 up = up.values
print("Creating r")
 r = df.returnsOpenNextMktres10.values
mins = np.min(X, axis=0)
 maxs = np.max(X, axis=0)
 rng = maxs - mins
print("Reshaping X")
 X = 1 -(( maxs - X)/ rng)
 print("Rehaped and asseting")
 assert X.shape[0] == up.shape[0] == r.shape[0]
df['time'] = pd.to_datetime(df['date'])
 te = df['time']>date(2015, 1, 1)
 print("Getting index values")
 tt = 0
for tt,i in enumerate(te.values):
if i:
idx = tt
print(i,tt)
 break
print(idx)
 print("Creating X_train, X_test")
 X_train, X_test = X[:idx],X[idx:]
print("Creating up_train, up_test")
 up_train, up_test = up[:idx],up[idx:]
print("Creating r_train, r_test")
 r_train, r_test = r[:idx],r[idx:]
print("Creating u_train, u_test")
 u_train,u_test = universe[:idx],universe[idx:]
print("Creating d_train, d_test")
 d_train,d_test = d[:idx],d[idx:]
print("Creating Train Dataset")
 train_data = lgb.Dataset(X_train, label=up_train.astype(int))
print("Creating Test Dataset")
 test_data = lgb.Dataset(X_test, label=up_test.astype(int))
print(learn.predict('It was crazy'))
print(learn.predict('It was a disaster'))<predict_on_test>
DATA_SPLIT_SEED = 2018
splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=DATA_SPLIT_SEED ).split(X, up))
for idx,(train_idx, valid_idx)in enumerate(splits):
X_train = X[train_idx]
y_train = up[train_idx]
X_val = X[valid_idx]
y_val = up[valid_idx]
train_data = lgb.Dataset(X_train, label=y_train)
 test_data = lgb.Dataset(X_val, label=y_val)
 x_1 = [0.19000424246380565, 2452, 212, 328, 202]
x_2 = [0.19016805202090095, 2583, 213, 312, 220]
def exp_loss(p,y):
y = y.get_label()
grad = -y*(1.0-1.0/(1.0+np.exp(-y*p)))
 hess = -(np.exp(y*p)*(y*p-1)-1)/(( np.exp(y*p)+1)**2)
 return grad,hess
params_1 = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'binary',
'learning_rate': x_1[0],
'num_leaves': x_1[1],
'min_data_in_leaf': x_1[2],
'num_iteration': 239,
'max_bin': x_1[4],
'verbose': 1
}
params_2 = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'binary',
'learning_rate': x_2[0],
'num_leaves': x_2[1],
'min_data_in_leaf': x_2[2],
'num_iteration': 172,
'max_bin': x_2[4],
'verbose': 1
}
gbm_1 = lgb.train(params_1,
train_data,
num_boost_round=100,
valid_sets=test_data,
early_stopping_rounds=5, 
)
 gbm_2 = lgb.train(params_2,
train_data,
num_boost_round=100,
valid_sets=test_data,
early_stopping_rounds=5,
)
print('Total number of needed predictions:',len(test_df))
predictions = []
ids = []
for i in range(len(test_df)) :
print(i)
 pred, _, _ = learn.predict(test_df.loc[i, 'text'])
 ids.append(test_df.loc[i, 'id'])
 predictions.append(pred)
<save_to_csv>
confidence_test =(gbm_1.predict(X_test)+ gbm_2.predict(X_test)) /2
confidence_test =(confidence_test-confidence_test.min())/(confidence_test.max() -confidence_test.min())
 confidence_test = confidence_test*2-1
print(max(confidence_test),min(confidence_test))
r_test = r_test.clip(-1,1)
 x_t_i = confidence_test * r_test * u_test
data = {'day' : d_test, 'x_t_i' : x_t_i}
df = pd.DataFrame(data)
 x_t = df.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_test = mean / std
print(score_test )
submission = pd.DataFrame({'id': ids, 'target':predictions})
 print(submission)
 submission.to_csv("simple_submission.csv", index=False )<concatenate>
days = env.get_prediction_days()
n_days = 0
prep_time = 0
prediction_time = 0
packaging_time = 0
df = pd.concat([train_df, test_df] )<load_pretrained>
def post_scaling(df):
mean, std = np.mean(df), np.std(df)
 df =(df - mean)/(std * 8)
 return np.clip(df,-1,1 )
dls_lm = TextDataLoaders.from_df(df, text_col='text', is_lm=True, valid_pct=0.0)
 dls_lm.show_batch(max_n=3 )<load_pretrained>
for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
if(n_days%50==0):
print(n_days,end=' ')
 for col in market_obs_df.columns:
if(market_obs_df[col].dtype == "int64" or market_obs_df[col].dtype == "float64"):
market_obs_df[col] = market_obs_df[col].fillna(market_obs_df[col].mean())
 for col in market_obs_df.columns:
if(market_obs_df[col].dtype == "int64" or market_obs_df[col].dtype == "float64"):
market_obs_df[col] = market_obs_df[col].fillna(market_obs_df[col].mean())
 market_obs_df = feature_engineering(market_obs_df)
 fcol = [c for c in market_obs_df if c not in ['date', 'assetCode', 'time', 'returnsOpenNextMktres10', 'universe', 'assetName', 'Unnamed: 0']]
market_obs_df = market_obs_df[market_obs_df.assetCode.isin(predictions_template_df.assetCode)]
X_live = market_obs_df[fcol].values
lp =(gbm_1.predict(X_live)+ gbm_2.predict(X_live)) /2
confidence = lp
confidence = confidence * 2 - 1
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence': post_scaling(confidence)})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df )
<train_model><EOS> 
env.write_submission_file()
sub = pd.read_csv("submission.csv" )