Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<categorify>
pd.set_option('max_columns', 50 )
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )<merge>
env = twosigmanews.make_env()
market_train_df, news_train = env.get_training_data()
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base<data_type_conversions>
for column in market_train_df.select_dtypes(include='float64' ).columns:
market_train_df[column] = market_train_df[column].astype('float16' )
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
<feature_engineering>
start = datetime(2010, 1, 1, 0, 0, 0 ).date()
market_train = market_train_df.loc[market_train_df['time'].dt.date >= start].reset_index(drop=True)
 news_train = news_train.loc[news_train['time'].dt.date >= start].reset_index(drop=True )
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']<filter>
def preprocess_news(news_train):
drop_list = [
'audiences', 'subjects', 'assetName',
'headline', 'firstCreated', 'sourceTimestamp',
]
news_train.drop(drop_list, axis=1, inplace=True)
 for col in ['headlineTag', 'provider', 'sourceId']:
news_train[col], uniques = pd.factorize(news_train[col])
 del uniques
news_train['assetCodes'] = news_train['assetCodes'].apply(lambda x: x[1:-1].replace("'", ""))
return news_train
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data<compute_test_metric>
news_train = preprocess_news(news_train )
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer="adam", metrics=[score])
 return model<prepare_x_and_y>
def unstack_asset_codes(news_train):
codes = []
indexes = []
for i, values in news_train['assetCodes'].iteritems() :
explode = values.split(", ")
 codes.extend(explode)
 repeat_index = [int(i)]*len(explode)
 indexes.extend(repeat_index)
 index_df = pd.DataFrame({'news_index': indexes, 'assetCode': codes})
 del codes, indexes
gc.collect()
return index_df
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))<choose_model_class>
index_df = unstack_asset_codes(news_train)
 index_df.head()
NFOLD = 5
kf = KFold(n_splits=NFOLD )<find_best_model_class>
def merge_news_on_index(news_train, index_df):
news_train['news_index'] = news_train.index.copy()
news_unstack = index_df.merge(news_train, how='left', on='news_index')
 news_unstack.drop(['news_index', 'assetCodes'], axis=1, inplace=True)
 return news_unstack
%%time
cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=500,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD 
<compute_test_metric>
news_unstack = merge_news_on_index(news_train, index_df)
 del news_train, index_df
gc.collect()
news_unstack.head(3 )
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<prepare_output>
def group_news(news_frame):
news_frame['date'] = news_frame.time.dt.date
aggregations = ['mean']
gp = news_frame.groupby(['assetCode', 'date'] ).agg(aggregations)
 gp.columns = pd.Index(["{}_{}".format(e[0], e[1])for e in gp.columns.tolist() ])
 gp.reset_index(inplace=True)
 gp.drop(['sourceId_mean'],axis=1, inplace=True)
 float_cols = {c: 'float32' for c in gp.columns if c not in ['assetCode', 'date']}
return gp.astype(float_cols )
sub['FVC1'] = pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]<prepare_output>
news_agg = group_news(news_unstack)
 del news_unstack; gc.collect()
news_agg.head(3 )
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()<feature_engineering>
def process_date(df):
df['date'] = df['time'].dt.date
df['month'] = df['time'].dt.month
df['dayofweek'] = df['time'].dt.dayofweek
return df
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<load_from_csv>
def process_ma(df,columns=['open','close','volume'], windows=[10, 20, 60]):
ma_columns = []
ma_dev_columns = []
drop_list = []
for col in columns:
for window in windows:
ma_column = 'ma_{0}_{1}'.format(col,window)
 ma_dev_column = 'ma_dev_{0}_{1}'.format(col,window)
 ma_lag_column = 'ma_lag_{0}_{1}'.format(col,window)
 ma_lag_rate_column = 'ma_lag_rate_{0}_{1}'.format(col,window)
 std_column = 'std_{0}_{1}'.format(col,window)
 ma_columns.append(ma_column)
 ma_dev_columns.append(ma_dev_column)
 df[ma_column] = df.groupby('assetCode')[col].apply(lambda x: x.rolling(window ).mean())
 df[ma_dev_column] = df[col] / df[ma_column] - 1
df[std_column] = df.groupby('assetCode')[col].apply(lambda x: x.rolling(window ).std())
 df[ma_lag_column] = df.groupby('assetCode')[ma_column].shift()
df[ma_lag_rate_column] =(df[ma_column] - df[ma_lag_column])/ df[ma_lag_column]
drop_list.append(ma_column)
 drop_list.append(ma_lag_column)
 df.drop(drop_list,axis=1, inplace=True)
 return df
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1<save_to_csv>
market_train = process_date(market_train)
 market_train = process_ma(market_train )
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )<import_modules>
df = market_train.merge(news_agg, how='left', on=['assetCode', 'date'])
 del market_train, news_agg
gc.collect()
df.head(3 )
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold<import_modules>
for column in df.select_dtypes(include='float64' ).columns:
df[column] = df[column].astype('float16')
 for column in df.select_dtypes(include='int64' ).columns:
df[column] = df[column].astype('int16')
 df.dtypes
import tensorflow as tf 
<define_variables>
def custom_metric(date, pred_proba, num_target, universe):
y = pred_proba*2 - 1
r = num_target.clip(-1,1)
 x = y * r * universe
result = pd.DataFrame({'day' : date, 'x' : x})
 x_t = result.groupby('day' ).sum().values
return np.mean(x_t)/ np.std(x_t )
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )<define_variables>
date = df.date
num_target = df.returnsOpenNextMktres10.astype('float32')
 bin_target =(df.returnsOpenNextMktres10 >= 0 ).astype('int8')
 universe = df.universe.astype('int8')
 df.drop(['returnsOpenNextMktres10', 'date', 'universe', 'assetCode', 'assetName', 'time'],
axis=1, inplace=True)
 df = df.astype('float32')
 gc.collect()
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
DESIRED_SIZE = 256
BATCH_SIZE=128<load_from_csv>
train_index, test_index = train_test_split(df.index.values, test_size=0.1 )
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )<concatenate>
best_params = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'binary_logloss',
'learning_rate': 0.05,
'num_leaves': 63,
'n_estimators': 500,
'min_child_samples': 20,
'colsample_bytree': 1,
'subsample': 1.0,
'reg_alpha': 0.1,
'reg_lambda': 0.8,
}
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )<feature_engineering>
lgb_train = lgb.Dataset(df.iloc[train_index], bin_target.iloc[train_index])
 lgb_eval = lgb.Dataset(df.iloc[test_index], bin_target.iloc[test_index], reference=lgb_train )
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )<categorify>
clf = lgb.train(
 best_params,
lgb_train,
valid_sets=lgb_eval,
early_stopping_rounds=5
)
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )<merge>
pred_y = clf.predict(df.iloc[test_index])
 pred_y[pred_y >= 0.5] = 1
pred_y[pred_y < 0.5] = 0
print(( pred_y == bin_target.iloc[test_index] ).sum() / len(pred_y))
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base<data_type_conversions>
test_df_columns = ['time', 'assetCode', 'assetName', 'volume', 'close', 'open',
'returnsClosePrevRaw1', 'returnsOpenPrevRaw1',
'returnsClosePrevMktres1', 'returnsOpenPrevMktres1',
'returnsClosePrevRaw10', 'returnsOpenPrevRaw10',
'returnsClosePrevMktres10', 'returnsOpenPrevMktres10']
base_df = market_train_df[market_train_df['time'] >= '2016-10-01']
base_df = base_df[test_df_columns]
base_df['id'] = -1
base_df.shape
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
<feature_engineering>
def calc_target(df):
df['open_next10'] = df.groupby('assetCode')['open'].shift(-10)
 df['return_next10'] = df['open_next10'] / df['open'] - 1.0
df['target'] =(df.return_next10 >= 0 ).astype('int8')
 return df
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','week','BASE']<filter>
target_test_df = calc_target(market_train_df)
 target_test_df.head()
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data<compute_test_metric>
def online_train(model, df, train_day_id, feats):
best_params = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'binary_logloss',
'learning_rate': 0.05,
'num_leaves': 63,
'n_estimators': 500,
'min_child_samples': 20,
'colsample_bytree': 1,
'subsample': 1.0,
'reg_alpha': 0.1,
'reg_lambda': 0.8,
}
train_df = calc_target(df)
 train_df = train_df[train_df['id'] == train_day_id]
train_df.reset_index(drop=True)
 target_df = train_df[['target']]
train_df = train_df[feats]
model.refit(train_df, target_df['target'])
 return model
def metric(trueFVC, predFVC, predSTD):
clipSTD = np.clip(predSTD, 70 , 9e9)
 deltaFVC = np.clip(np.abs(trueFVC-predFVC), 0 , 1000)
 return np.mean(-1*(np.sqrt(2)*deltaFVC/clipSTD)- np.log(np.sqrt(2)*clipSTD)) 
<prepare_x_and_y>
def write_submission(model, env):
days = env.get_prediction_days()
day_id = 0
market_obs_df_append = None
online_train_df_append = None
for(market_obs_df, news_obs_df, predictions_template_df)in days:
news_obs_df = preprocess_news(news_obs_df)
 index_df = unstack_asset_codes(news_obs_df)
 news_unstack = merge_news_on_index(news_obs_df, index_df)
 news_obs_agg = group_news(news_unstack)
 market_obs_df['id'] = day_id
if market_obs_df_append is None:
market_obs_df_append = base_df
market_obs_df_append = pd.concat([market_obs_df_append,market_obs_df],
ignore_index=True,
sort=False)
 market_obs_process = process_date(market_obs_df_append)
 market_obs_process = process_ma(market_obs_process)
 market_obs_df = market_obs_process[market_obs_process['id']==day_id]
obs_df = market_obs_df.merge(news_obs_agg, how='left', on=['assetCode', 'date'])
 del market_obs_df, news_obs_agg, news_obs_df, news_unstack, index_df
gc.collect()
obs_df = obs_df[obs_df.assetCode.isin(predictions_template_df.assetCode)]
feats = [c for c in obs_df.columns if c not in ['date', 'assetCode', 'assetName', 'time', 'id']]
if online_train_df_append is None:
online_train_df_append = obs_df
else:
online_train_df_append = pd.concat([online_train_df_append, obs_df],
ignore_index=True,
sort=False)
 if day_id >= 10:
model = online_train(model, online_train_df_append, day_id - 10, feats)
 preds = model.predict(obs_df[feats])* 2 - 1
preds[np.where(abs(preds)< 0.05)] = 0.0
sub = pd.DataFrame({'assetCode': obs_df['assetCode'], 'confidence': preds})
 predictions_template_df = predictions_template_df.merge(sub, how='left' ).drop(
 'confidenceValue', axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 if day_id == 59:
market_obs_df_append.drop(
 market_obs_df_append.index[market_obs_df_append['id']==-1],
inplace=True)
 elif day_id >= 60:
market_obs_df_append.drop(
 market_obs_df_append.index[market_obs_df_append['id']==day_id-60],
inplace=True)
 day_id += 1
del obs_df, predictions_template_df, preds, sub
gc.collect()
env.write_submission_file()
print('day_count',day_id )
<import_modules><EOS> 
write_submission(clf, env )