Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<count_missing_values>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from keras.models import Sequential
from keras.layers import Dense, Dropout, LSTM, GRU, SpatialDropout1D, GlobalMaxPool1D
from keras.layers.embeddings import Embedding
from kaggle.competitions import twosigmanews
from tqdm import tqdm
import gc
print("No.of null values in Color column"+str(train_df['Color'].isnull().sum()))
print("No.of null values in Color column"+str(test_df['Color'].isnull().sum()))<count_values>
DEBUG = False
YEARMIN = 2011
train_df['Transmission'].value_counts()<count_values>
np.random.seed(100)
 set_random_seed(150 )
test_df['Transmission'].value_counts()<filter>
env = twosigmanews.make_env()
train_df[train_df['Transmission']=='Manual']<rename_columns>
( market_train, _)= env.get_training_data()
train_df['Transmission'].replace("Manual","MANUAL",inplace=True )<count_values>
market_train = market_train.loc[market_train['time'].dt.year > YEARMIN]
train_df['Transmission'].value_counts()<data_type_conversions>
train_df['Transmission'].fillna(value="Transmission_unk",inplace=True)
 test_df['Transmission'].fillna(value="Transmission_unk",inplace=True )<count_values>
print('MARKET TRAIN:')
 for col in market_train.columns:
print(col+' has '+str(market_train[col].isna().sum())+' NAs' )
train_df['WheelTypeID'].value_counts()<drop_column>
market_train['returnsClosePrevMktres1'] = market_train.groupby(['assetCode'])['returnsClosePrevMktres1'].transform(lambda x: x.fillna(x.median()))
market_train['returnsOpenPrevMktres1'] = market_train.groupby(['assetCode'])['returnsOpenPrevMktres1'].transform(lambda x: x.fillna(x.median()))
market_train['returnsClosePrevMktres10'] = market_train.groupby(['assetCode'])['returnsClosePrevMktres10'].transform(lambda x: x.fillna(x.median()))
market_train['returnsOpenPrevMktres10'] = market_train.groupby(['assetCode'])['returnsOpenPrevMktres10'].transform(lambda x: x.fillna(x.median()))
market_train = market_train.fillna(market_train.median() )
train_df.drop('WheelTypeID',inplace=True,axis=1)
 test_df.drop('WheelTypeID',inplace=True,axis=1 )<count_values>
market_train = market_train.sort_values(['assetCode','time'])
 market_train.reset_index(drop=True,inplace=True)
 market_train.head()
train_df['WheelType'].value_counts()<count_values>
market_train['time'] = pd.to_datetime(market_train['time'].dt.date )
test_df['WheelType'].value_counts()<data_type_conversions>
market_train['margin1'] = market_train['open'] / market_train['close']
market_train['TARGET'] = np.sign(market_train['returnsOpenNextMktres10'] )
train_df['WheelType'].fillna(value="WheelType_unk",inplace=True)
 test_df['WheelType'].fillna(value="WheelType_unk",inplace=True )<count_values>
total_market_obs_df = [market_train.loc[(market_train['time'].dt.year >= 2016)&(market_train['time'].dt.month >= 9)].groupby('assetCode' ).tail(30 ).drop(['universe','returnsOpenNextMktres10'], axis=1)]
train_df['WheelType'].value_counts()<count_values>
LSTM_COLUMNS_TO_USE = ['time',
'assetCode',
'universe',
'returnsOpenNextMktres10',
'TARGET',
'volume',
'returnsClosePrevRaw1', 'returnsOpenPrevRaw1',
'returnsClosePrevMktres1', 'returnsOpenPrevMktres1',
'returnsClosePrevRaw10', 'returnsOpenPrevRaw10',
'returnsClosePrevMktres10', 'returnsOpenPrevMktres10',
'margin1',
]
train_df['Nationality'].value_counts()<count_values>
market_train = market_train[LSTM_COLUMNS_TO_USE]
test_df['Nationality'].value_counts()<data_type_conversions>
gc.collect()
train_df['Nationality'].fillna(value="Nationality_unk",inplace=True)
 test_df['Nationality'].fillna(value="Nationality_unk",inplace=True )<count_values>
INFORMATION_COLS = ['time','assetCode','universe','returnsOpenNextMktres10','TARGET']
INPUT_COLS = [f for f in market_train.columns if f not in INFORMATION_COLS]
train_df['Size'].value_counts()<count_values>
scaler = StandardScaler()
market_train[INPUT_COLS] = scaler.fit_transform(market_train[INPUT_COLS] )
test_df['Size'].value_counts()<data_type_conversions>
def series_to_supervised(dataset, n_in=1):
data_X = []
data_time = []
data_assetCode = []
data_universe = []
data_returns = []
data_TARGET = []
for i in range(0, len(dataset)) :
data_time.append(dataset[i][0])
 data_assetCode.append(dataset[i][1])
 data_universe.append(dataset[i][2])
 data_returns.append(dataset[i][3])
 data_TARGET.append(dataset[i][4])
 to_append = np.append(np.zeros(shape=(max(0,n_in - 1 - i), 10)) ,(dataset[max(0, i - n_in+1):i+1, len(INFORMATION_COLS):]), axis=0)
 data_X.append(to_append)
 return data_X, data_time, data_assetCode, data_universe, data_returns, data_TARGET
train_df['Size'].fillna(value="Size_unk",inplace=True)
 test_df['Size'].fillna(value="Size_unk",inplace=True )<count_values>
LOOK_BACK = 15
train_df['TopThreeAmericanName'].value_counts()<count_values>
del market_train
gc.collect()
test_df['TopThreeAmericanName'].value_counts()<data_type_conversions>
from keras import callbacks
train_df['TopThreeAmericanName'].fillna(value="Top_unk",inplace=True)
 test_df['TopThreeAmericanName'].fillna(value="Top_unk",inplace=True )<count_values>
class Metrics(callbacks.Callback):
def on_train_begin(self, logs={}):
self._data = []
def on_epoch_end(self, batch, logs={}):
X_val, y_val = self.validation_data[0], self.validation_data[1]
y_predict =(pd.DataFrame(model.predict(X_val)) * 2)- 1
_sigmascore = sigma_scorelstm(y_val, y_predict)
 print(" — sigmascore: %f" %(_sigmascore))
self._data.append({
'val_sigmascore': _sigmascore
})
 return
def get_data(self):
return self._data
metrics = Metrics()
train_df['PRIMEUNIT'].value_counts()<count_values>
train_index = [i for i in range(len(lstm_df_list)) ]
test_df['PRIMEUNIT'].value_counts()<data_type_conversions>
def sigma_scorelstm(y_true, y_pred):
x_t_i = y_pred * pd.DataFrame([the_returns[i] for i in train_index])* pd.DataFrame([the_universe[i] for i in train_index])
 data = pd.concat([pd.DataFrame([the_time[i] for i in train_index]), x_t_i], axis=1)
 data.columns = ['day','x_t_i']
x_t = data.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_valid = mean / std
return score_valid
train_df['PRIMEUNIT'].fillna(value="Prime_unk",inplace=True)
 test_df['PRIMEUNIT'].fillna(value="Prime_unk",inplace=True )<categorify>
trainX = np.array([lstm_df_list[i] for i in train_index])
 trainY =(np.array([the_TARGET[i] for i in train_index])+ 1)/ 2
train_df['AUCGUART'].replace("AGREEN","GREEN",inplace=True)
 test_df['AUCGUART'].replace("ARED","RED",inplace=True)
<data_type_conversions>
model = Sequential()
model.add(GRU(50, return_sequences=True, input_shape=(LOOK_BACK, trainX.shape[2])))
 model.add(SpatialDropout1D(0.5))
model.add(GlobalMaxPool1D())
 model.add(Dropout(0.50))
model.add(Dense(128, activation='relu'))
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.25))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='rmsprop')
 history = model.fit(trainX, trainY, epochs=2, batch_size=1028, validation_data=(trainX, trainY), verbose=2, shuffle=True, callbacks=[metrics] )
train_df['AUCGUART'].fillna(value="AUC_unk",inplace=True)
 test_df['AUCGUART'].fillna(value="AUC_unk",inplace=True )<drop_column>
del trainX, trainY
gc.collect()
train_df.drop(['MMRAcquisitionAuctionAveragePrice','MMRAcquisitionAuctionCleanPrice',
'MMRAcquisitionRetailAveragePrice','MMRAcquisitonRetailCleanPrice',
'MMRCurrentAuctionAveragePrice','MMRCurrentAuctionCleanPrice',
'MMRCurrentRetailAveragePrice','MMRCurrentRetailCleanPrice'],inplace=True,axis=1)
 test_df.drop(['MMRAcquisitionAuctionAveragePrice','MMRAcquisitionAuctionCleanPrice',
'MMRAcquisitionRetailAveragePrice','MMRAcquisitonRetailCleanPrice',
'MMRCurrentAuctionAveragePrice','MMRCurrentAuctionCleanPrice',
'MMRCurrentRetailAveragePrice','MMRCurrentRetailCleanPrice'],inplace=True,axis=1 )<drop_column>
days = env.get_prediction_days()
train_df.drop('PurchDate',axis=1,inplace=True)
 test_df.drop('PurchDate',axis=1,inplace=True )<drop_column>
LSTM_COLUMNS_TO_USE = [col for col in LSTM_COLUMNS_TO_USE if(col!='universe' and col!='returnsOpenNextMktres10' and col!='TARGET')]
train_df.drop(['RefId','IsBadBuy'],axis=1 ).dtypes!= 'object'<drop_column>
total_market_obs_df[0].drop('TARGET', axis=1, inplace=True )
not_cat=train_df.drop(['RefId','IsBadBuy'],axis=1 ).columns[train_df.drop(['RefId','IsBadBuy'],axis=1 ).dtypes!= 'object']<feature_engineering>
def series_to_supervised(dataset, curdate, n_in=1):
data_X = []
for i in range(0, len(dataset)) :
if dataset[i][0]==curdate:
to_append = np.append(np.zeros(shape=(max(0,n_in - 1 - i), 10)) ,(dataset[max(0, i - n_in+1):i+1, 2:]), axis=0)
 data_X.append(to_append)
 return data_X
for i in not_cat:
maximum=np.max(train_df[i])
 train_df[i]=train_df[i]/maximum
maximum_test=np.max(test_df[i])
 test_df[i]=test_df[i]/maximum_test<drop_column>
for(market_obs_df, _, predictions_template_df)in days:
market_obs_df = market_obs_df.fillna(market_obs_df.median())
 market_obs_df = market_obs_df.sort_values('assetCode')
 market_obs_df['time'] = pd.to_datetime(market_obs_df['time'].dt.date)
 market_obs_df['margin1'] = market_obs_df['open'] / market_obs_df['close']
total_market_obs_df.append(market_obs_df)
 history_df = pd.concat(total_market_obs_df[-(np.max(30)+1):])
 tmp = history_df[LSTM_COLUMNS_TO_USE]
tmp = tmp.fillna(tmp.median())
 tmp[INPUT_COLS] = scaler.fit_transform(tmp[INPUT_COLS])
 lstm_df_list = np.empty(shape=(predictions_template_df.shape[0],LOOK_BACK,10))
row_at = 0
for asset in market_obs_df['assetCode'].unique() :
res = series_to_supervised(tmp.loc[tmp['assetCode']==asset].values, curdate=tmp['time'].max() , n_in=LOOK_BACK)
 lstm_df_list[row_at] = np.array(res)
 row_at = row_at + 1
trainX = np.array([lstm_df_list[i] for i in range(len(lstm_df_list)) ])
 yhat_lstm = model.predict(trainX)
 yhat_lstm = yhat_lstm.flatten()
yhat_lstm = pd.DataFrame(yhat_lstm)
 preds =(yhat_lstm * 2)- 1
predictions_template_df['confidenceValue'] = preds
env.predict(predictions_template_df )
<filter><EOS> 
env.write_submission_file()