Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<import_modules>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import time
from sklearn.metrics import accuracy_score
from kaggle.competitions import twosigmanews
from keras import initializers
from keras import regularizers
from keras import constraints
from keras.utils import conv_utils
from keras.utils.data_utils import get_file
from keras.engine.topology import get_source_inputs
from keras.engine import InputSpec
from keras import backend as K
from keras.layers import LeakyReLU
from keras.layers import ZeroPadding2D
from keras.losses import binary_crossentropy
import keras.callbacks as callbacks
from keras.callbacks import Callback
from keras.applications.xception import Xception
from keras.layers import multiply
import keras
from keras import optimizers
from keras.legacy import interfaces
from keras.utils.generic_utils import get_custom_objects
from keras.engine.topology import Input
from keras.engine.training import Model
from keras.layers.convolutional import Conv2D, UpSampling2D, Conv2DTranspose
from keras.layers.core import Activation, SpatialDropout2D
from keras.layers.merge import concatenate
from keras.layers.normalization import BatchNormalization
from keras.layers.pooling import MaxPooling2D
from keras.layers import Input,Dropout,BatchNormalization,Activation,Add
from keras.regularizers import l2
from keras.layers.core import Dense, Lambda
from keras.layers.merge import concatenate, add
from keras.layers import GlobalAveragePooling2D, Reshape, Dense, multiply, Permute
from keras.optimizers import SGD
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M<define_variables>
env = twosigmanews.make_env() 
(market_train, _)= env.get_training_data()
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42)
 ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128<load_from_csv>
cat_cols = ['assetCode']
num_cols = ['volume', 'close', 'open', 'returnsClosePrevRaw1', 'returnsOpenPrevRaw1', 'returnsClosePrevMktres1',
'returnsOpenPrevMktres1', 'returnsClosePrevRaw10', 'returnsOpenPrevRaw10', 'returnsClosePrevMktres10',
'returnsOpenPrevMktres10']
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient")
<count_unique_values>
train_indices, val_indices = train_test_split(market_train.index.values,test_size=0.25, random_state=23)
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub])
 print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )<feature_engineering>
def encode(encoder, x):
len_encoder = len(encoder)
 try:
id = encoder[x]
except KeyError:
id = len_encoder
return id
encoders = [{} for cat in cat_cols]
for i, cat in enumerate(cat_cols):
print('encoding %s...' % cat, end=' ')
 encoders[i] = {l: id for id, l in enumerate(market_train.loc[:, cat].astype(str ).unique())}
market_train[cat] = market_train[cat].astype(str ).apply(lambda x: encode(encoders[i], x))
print('Done')
 embed_sizes = [len(encoder)+ 1 for encoder in encoders]
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min')
 base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True)
 data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )<feature_engineering>
a=0
for type in types:
a=a+d[type].shape[0]
a
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE'] 
<filter>
low =.25
high =.75
bounds = {}
for type in types:
filt_df = d[type].loc[:, d[type].columns != 'assetCode']
filt_df = filt_df.loc[:, filt_df.columns != 'universe']
quant_df = filt_df.quantile([low, high])
 IQR = quant_df.iloc[1,:]- quant_df.iloc[0,:]
quant_df.iloc[0,:] = quant_df.iloc[0,:] - 1.5*IQR
quant_df.iloc[1,:] = quant_df.iloc[1,:] + 1.5*IQR
bounds[type] = quant_df
bounds[type] = bounds[type].reset_index()
bounds[type] = bounds[type].drop("index", axis=1)
 bounds[1]
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
tr.shape, chunk.shape, sub.shape<compute_test_metric>
bounds[1].loc[1,"volume"]
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="relu", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.005, amsgrad=False), metrics=[score])
 return model
net = make_model()
print(net.summary())
 print(net.count_params())
<prepare_x_and_y>
market_train1=pd.DataFrame()
for type in types:
for column in num_cols:
d[type] = d[type].loc[d[type][column]>=bounds[type].loc[0,column]]
d[type] = d[type].loc[d[type][column]<=bounds[type].loc[1,column]]
market_train1=pd.concat([market_train1,d[type]], ignore_index=True)
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3)) 
<choose_model_class>
orig_len=market_train.shape[0]
new_len = market_train1.shape[0]
rmv_len1 = np.abs(orig_len-new_len)
 print('There were %i lines removed' %rmv_len1 )
NFOLD = 5
kf = KFold(n_splits=NFOLD )<find_best_model_class>
market_train1['close_open_ratio'] = np.abs(market_train1['close']/market_train1['open'])
 market_train1 = market_train1.loc[market_train1['close_open_ratio'] < 1.4]
market_train1 = market_train1.loc[market_train1['close_open_ratio'] > 0.4]
newer_len=market_train1.shape[0]
rmv_len2 = np.abs(new_len-newer_len)
 print('There were %i lines removed additionally' %rmv_len2 )
%%time
cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=800,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD<compute_test_metric>
market_train1=market_train1.dropna()
rmv_len3 = np.abs(newer_len-market_train1.shape[0])
 print('There were %i lines removed additionally' %rmv_len3)
 deleted_rows=rmv_len1+rmv_len2+rmv_len3
print('There were %i lines in total removed' %deleted_rows )
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<feature_engineering>
market_train1['average'] =(market_train1['close'] + market_train1['open'])/2
market_train1['pricevolume'] = market_train1['volume'] * market_train1['close']
sub['FVC1'] = pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10)
 subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<save_to_csv>
num_cols = ['volume', 'close', 'open', 'returnsClosePrevRaw1', 'returnsOpenPrevRaw1', 'returnsClosePrevMktres1',
'returnsOpenPrevMktres1', 'returnsClosePrevRaw10', 'returnsOpenPrevRaw10', 'returnsClosePrevMktres10',
'returnsOpenPrevMktres10','close_open_ratio','average','pricevolume']
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False)
<import_modules>
market_train=market_train1.copy()
import numpy as np
import pandas as pd
import os
import random
import matplotlib.pyplot as plt
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
from tqdm import tqdm
from PIL import Image
import pydicom<define_variables>
train_indices, val_indices = train_test_split(market_train.index.values,test_size=0.25, random_state=23 )
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )<define_variables>
market_train[num_cols] = market_train[num_cols].fillna(0)
 print('scaling numerical columns')
 scaler = StandardScaler()
scaler = StandardScaler()
market_train[num_cols] = scaler.fit_transform(market_train[num_cols])
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE = 128<feature_engineering>
categorical_inputs = []
for cat in cat_cols:
categorical_inputs.append(Input(shape=[1], name=cat))
categorical_embeddings = []
for i, cat in enumerate(cat_cols):
categorical_embeddings.append(Embedding(embed_sizes[i], 10 )(categorical_inputs[i]))
categorical_logits = Flatten()(categorical_embeddings[0])
 categorical_logits = Dense(32,activation='relu' )(categorical_logits)
 a=len(num_cols)
 numerical_inputs = Input(shape=(a,), name='num')
 numerical_logits = numerical_inputs
numerical_logits = BatchNormalization()(numerical_logits)
 numerical_logits = Dense(128,activation='relu' )(numerical_logits)
 numerical_logits = Dense(64,activation='relu' )(numerical_logits)
 logits = Concatenate()([numerical_logits,categorical_logits])
 logits = Dense(64,activation='relu' )(logits)
 out = Dense(1, activation='sigmoid' )(logits)
 model = Model(inputs = categorical_inputs + [numerical_inputs], outputs=out)
 model.compile(optimizer='adam',loss=binary_crossentropy )
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )<concatenate>
def get_input(market_train, indices):
X_num = market_train.loc[indices, num_cols].values
X = {'num':X_num}
for cat in cat_cols:
X[cat] = market_train.loc[indices, cat_cols].values
y =(market_train.loc[indices,'returnsOpenNextMktres10']>=0 ).values
r = market_train.loc[indices,'returnsOpenNextMktres10'].values
u = market_train.loc[indices, 'universe']
d = market_train.loc[indices, 'time'].dt.date
return X,y,r,u,d
X_train,y_train,r_train,u_train,d_train = get_input(market_train, train_indices)
 X_valid,y_valid,r_valid,u_valid,d_valid = get_input(market_train, val_indices )
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )<feature_engineering>
class SnapshotCallbackBuilder:
def __init__(self, nb_epochs, nb_snapshots, init_lr=0.1):
self.T = nb_epochs
self.M = nb_snapshots
self.alpha_zero = init_lr
def get_callbacks(self, model_prefix='Model'):
callback_list = [
callbacks.ModelCheckpoint("model.hdf5",monitor='val_my_iou_metric',
mode = 'max', save_best_only=True, verbose=1),
swa,
callbacks.LearningRateScheduler(schedule=self._cosine_anneal_schedule)
 ]
return callback_list
def _cosine_anneal_schedule(self, t):
cos_inner = np.pi *(t %(self.T // self.M))
cos_inner /= self.T // self.M
cos_out = np.cos(cos_inner)+ 1
return float(self.alpha_zero / 2 * cos_out )
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )<categorify>

 check_point = ModelCheckpoint('model.hdf5',verbose=True, save_best_only=True)
 early_stop = EarlyStopping(patience=5,verbose=True)
 model.fit(X_train,y_train.astype(int),
validation_data=(X_valid,y_valid.astype(int)) ,
epochs=5,
verbose=True,
callbacks=[early_stop,check_point] )
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )<merge>
r_valid = r_valid.clip(-1,1)
 x_t_i = confidence_valid * r_valid * u_valid
data = {'day' : d_valid, 'x_t_i' : x_t_i}
df = pd.DataFrame(data)
 x_t = df.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_valid = mean / std
print(score_valid )
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base<data_type_conversions>
days = env.get_prediction_days()
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )<feature_engineering>
n_days = 0
prep_time = 0
prediction_time = 0
packaging_time = 0
predicted_confidences = np.array([])
 for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
print(n_days,end=' ')
 t = time.time()
market_obs_df['assetCode_encoded'] = market_obs_df[cat].astype(str ).apply(lambda x: encode(encoders[i], x))
market_obs_df['close_open_ratio'] = np.abs(market_obs_df['close']/market_obs_df['open'])
 market_obs_df['average'] =(market_obs_df['close'] + market_obs_df['open'])/2
market_obs_df['pricevolume'] = market_obs_df['volume'] * market_obs_df['close']
market_obs_df[num_cols] = market_obs_df[num_cols].fillna(0)
 market_obs_df[num_cols] = scaler.transform(market_obs_df[num_cols])
 X_num_test = market_obs_df[num_cols].values
X_test = {'num':X_num_test}
X_test['assetCode'] = market_obs_df['assetCode_encoded'].values
prep_time += time.time() - t
t = time.time()
market_prediction = model.predict(X_test)[:,0]*2 -1
predicted_confidences = np.concatenate(( predicted_confidences, market_prediction))
prediction_time += time.time() -t
t = time.time()
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence':market_prediction})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 packaging_time += time.time() - t
env.write_submission_file()
total = prep_time + prediction_time + packaging_time
print(f'Preparing Data: {prep_time:.2f}s')
 print(f'Making Predictions: {prediction_time:.2f}s')
 print(f'Packing: {packaging_time:.2f}s')
 print(f'Total: {total:.2f}s' )