Kaggle competition: Two Sigma: Using News to Predict Stock Movements
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']<filter>
env = twosigmanews.make_env() 
(market_train_df, news_train_df)= env.get_training_data()
print('data loaded' )
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data<compute_test_metric>
print('There are ', len(market_train_df['returnsOpenNextMktres10']), 'data before data selection')
 market_train_df = market_train_df.loc[market_train_df['time'] >= '2010-01-01 22:00:00+0000']
print('There are ', len(market_train_df['returnsOpenNextMktres10']), 'data after data selection' )
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss<init_hyperparams>
market_train_df.isna().sum()
LR = 0.1
BETA_1 = 0.9
BETA_2 = 0.999
DECAY = 0.01
AMSGRAD = False<choose_model_class>
print('There are ', len(market_train_df['returnsOpenNextMktres10']), 'data before dropping NaN')
 market_train_df = market_train_df.dropna()
print('There are ', len(market_train_df['returnsOpenNextMktres10']), 'data after dropping NaN' )
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="NN-Optimized")
 model.compile(loss=mloss(0.8), optimizer = tf.keras.optimizers.Adam(lr = LR, beta_1 = BETA_1, beta_2 = BETA_2, epsilon = None, decay = DECAY, amsgrad = AMSGRAD), metrics=[score])
 return model<prepare_x_and_y>
market_train_df.isna().sum()
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))<choose_model_class>
def remove_outlier_by_percentile(df,col_list,lower_p,upper_p):

 for i in range(len(col_list)) :
df =(df[(df[col_list[i]]<np.percentile(df[col_list[i]],upper_p)) &(df[col_list[i]]>np.percentile(df[col_list[i]],lower_p)) ])
 return df
outlier_removal_list = [ 'returnsClosePrevRaw1',
'returnsOpenPrevRaw1',
'returnsClosePrevRaw10',
'returnsOpenPrevRaw10',
'returnsOpenNextMktres10']
print('There are ', len(market_train_df['returnsOpenNextMktres10']), 'data before removing outliers')
 market_train_df = remove_outlier_by_percentile(market_train_df,outlier_removal_list,2,98)
 print('There are ', len(market_train_df['returnsOpenNextMktres10']), 'data after removing outliers' )
NFOLD = 10
kf = KFold(n_splits=NFOLD )<train_model>
def scale_data(df,features):
scaler = StandardScaler()
df[features]=scaler.fit_transform(df[features])
 return df
features = ['volume','close','open','returnsClosePrevRaw1','returnsOpenPrevRaw1',
'returnsClosePrevMktres1','returnsOpenPrevMktres1','returnsClosePrevRaw10',
'returnsOpenPrevRaw10','returnsClosePrevMktres10','returnsOpenPrevMktres10', 'returnsOpenNextMktres10']
market_train_df = scale_data(market_train_df,features)
 market_train_df[features].head()
%%time
count = 0
for train_idx, val_idx in kf.split(z):
count += 1
print(f"FOLD {count}:")
 net = make_model()
net.fit(z[train_idx], y[train_idx], batch_size=BATCH_SIZE, epochs=850,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("Train:", net.evaluate(z[train_idx], y[train_idx], verbose=0, batch_size=BATCH_SIZE))
print("Val:", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("Predicting Test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD<compute_test_metric>
valid_ratio = 0.20
features = ['volume','close','open','returnsClosePrevRaw1','returnsOpenPrevRaw1',
'returnsClosePrevMktres1','returnsOpenPrevMktres1','returnsClosePrevRaw10',
'returnsOpenPrevRaw10','returnsClosePrevMktres10','returnsOpenPrevMktres10']
X = market_train_df[features]
Y = market_train_df['returnsOpenNextMktres10']
X_train, X_valid, Y_train, Y_valid = train_test_split(X,Y,test_size=valid_ratio)
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<prepare_output>
input_dim = X_train.shape[1]
input_dim
sub['FVC1'] = pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]<prepare_output>
input_dim = X_train.shape[1]
output_dim = 1
dropout_rate = 0.2
sgd = optimizers.SGD(lr=0.01, clipnorm=1.)
model = Sequential()
model.add(Dense(11, input_dim=input_dim, activation = 'relu'))
model.add(Dense(6, activation = 'relu'))
model.add(Dropout(rate=dropout_rate))
model.add(Dense(output_dim))
model.compile(loss='mean_squared_error',optimizer=sgd)
 history = model.fit(X_train, Y_train,
epochs=20,
verbose=1,
validation_data =(X_valid, Y_valid),
callbacks=callbacks_list,
shuffle=True,
batch_size=128)
 print(model.summary())
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()<feature_engineering>
loss = model.evaluate(X_train, Y_train, verbose=False)
 print("Training Loss: {:.4f}".format(loss))
loss = model.evaluate(X_valid, Y_valid, verbose=False)
 print("Validation Loss: {:.4f}".format(loss))
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<load_from_csv>
data = {'y_real':Y_valid[0:15],'y_pred':(model.predict(X_valid.values[0:15])).reshape(1,-1)[0]}
pd.DataFrame(data )
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1<save_to_csv>
data = {'y_train':Y_train[0:15],'y_pred':(model.predict(X_train.values[0:15])).reshape(1,-1)[0]}
pd.DataFrame(data )
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )<import_modules>
_df = pd.DataFrame(( model.predict(X_train.values[0:15])))
 _df[0]
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold<import_modules>
all_Y = market_train_df[['returnsOpenNextMktres10']]
all_Y.head()
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M<define_variables>
scaler = MinMaxScaler(feature_range=(-1,1))
scaler.fit(all_Y)
 print('scaler fitting done' )
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )<define_variables>
def make_prediction(df):
pred = pd.DataFrame(model.predict(df))
scaled_pred = pd.DataFrame(scaler.transform(pred[[0]]))
scaled_pred[scaled_pred>0]=1
scaled_pred[scaled_pred<0]=-1
pred_arr = np.array(scaled_pred ).reshape(1,-1)[0]
return pred_arr
print('function loaded' )
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128<load_from_csv>
my_pred = make_prediction(X_train.head())
 my_pred
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )<concatenate>
temp_df = X_train.head().copy()
temp_df['confidenceValue'] = my_pred
temp_df
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )<count_unique_values>
days = env.get_prediction_days()
print('days loaded' )
<feature_engineering><EOS> 
n_days = 0
for(market_obs_df, _, predictions_template_df)in days:
n_days += 1
print(n_days)
 features = ['volume','close','open','returnsClosePrevRaw1','returnsOpenPrevRaw1',
'returnsClosePrevMktres1','returnsOpenPrevMktres1','returnsClosePrevRaw10',
'returnsOpenPrevRaw10','returnsClosePrevMktres10','returnsOpenPrevMktres10']
returns_features = ['returnsClosePrevRaw10','returnsClosePrevMktres1','returnsOpenPrevMktres1',
'returnsOpenPrevRaw10','returnsClosePrevMktres10','returnsOpenPrevMktres10']
market_obs_df_scaled = scale_data(market_obs_df,features)
 x_submission = market_obs_df_scaled[features].copy()
for i in range(len(returns_features)) :
x_submission[returns_features[i]]= x_submission[returns_features[i]].fillna(x_submission[returns_features[i]].mean())
 pred = make_prediction(x_submission)
 print(pred)
 predictions_template_df['confidenceValue'] = pred
env.predict(predictions_template_df)
 del x_submission
print('Done!')
 env.write_submission_file()