Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<count_values>
py.init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore')
train_data['Transmission'].value_counts()<count_values>
env = twosigmanews.make_env()
print('Done!' )
test_data['Transmission'].value_counts()<define_variables>
( market_train_df, news_train_df)= env.get_training_data()
train_data['Transmission'].replace("Manual","MANUAL",inplace = True )<count_values>
market_train, news_train = market_train_df.copy() , news_train_df.copy()
market_train_df1, news_train_df1 = market_train_df.copy() , news_train_df.copy()
train_data['Transmission'].value_counts()<data_type_conversions>
mis_value_graph(market_train_df)
 market_train_df = mis_impute(market_train_df)
 market_train_df.isna().sum().to_frame()
train_data['Transmission'].fillna(value = 'Transmission_unk',inplace = True)
 test_data['Transmission'].fillna(value = 'Transmission_unk',inplace = True )<count_missing_values>
best_asset_volume = market_train_df.groupby("assetCode")["close"].count().to_frame().sort_values(by=['close'],ascending= False)
 best_asset_volume = best_asset_volume.sort_values(by=['close'])
 largest_by_volume = list(best_asset_volume.nlargest(10, ['close'] ).index)
test_data['Transmission'].isnull().sum()<count_values>
for i in largest_by_volume:
asset1_df['high'] = asset1_df['open']
asset1_df['low'] = asset1_df['close']
for ind, row in asset1_df.iterrows() :
if row['close'] > row['open']:
asset1_df.loc[ind, 'high'] = row['close']
asset1_df.loc[ind, 'low'] = row['open']
trace1 = go.Candlestick(
 x = asset1_df['time'].dt.strftime(date_format='%Y-%m-%d' ).values,
open = asset1_df['open'].values,
low = asset1_df['low'].values,
high = asset1_df['high'].values,
close = asset1_df['close'].values,
increasing=dict(line=dict(color= generate_color())) ,
decreasing=dict(line=dict(color= generate_color())))
 layout = dict(title = "Candlestick chart for {}".format(i),
xaxis = dict(
 title = 'Month',
rangeslider = dict(visible = False)
),
yaxis = dict(title = 'Price(USD)')
)
 data = [trace1]
py.iplot(dict(data=data, layout=layout), filename='basic-line' )
train_data['WheelTypeID'].value_counts()<drop_column>
mis_value_graph(news_train_df)
 news_train_df = mis_impute(news_train_df)
 news_train_df.isna().sum().to_frame()
train_data.drop('WheelTypeID',axis =1, inplace=True)
 test_data.drop('WheelTypeID',axis =1, inplace=True )<count_values>
news_sentiment_count = news_train_df.groupby(["urgency","assetName"])[["sentimentNegative","sentimentNeutral","sentimentPositive"]].count()
news_sentiment_count = news_sentiment_count.reset_index()
train_data['WheelType'].value_counts()<data_type_conversions>
news_sentiment_urgency = news_train_df.groupby(["urgency"])[["sentimentNegative","sentimentNeutral","sentimentPositive"]].count()
news_sentiment_urgency = news_sentiment_urgency.reset_index()
train_data['WheelType'].fillna(value = 'WheelType_unk',inplace = True)
 test_data['WheelType'].fillna(value = 'WheelType_unk',inplace = True )<count_values>
cat_cols = ['assetCode']
num_cols = ['volume', 'close', 'open', 'returnsClosePrevRaw1', 'returnsOpenPrevRaw1', 'returnsClosePrevMktres1',
'returnsOpenPrevMktres1', 'returnsClosePrevRaw10', 'returnsOpenPrevRaw10', 'returnsClosePrevMktres10',
'returnsOpenPrevMktres10']
train_data['Nationality'].value_counts()<data_type_conversions>
train_indices, val_indices = train_test_split(market_train_df1.index.values,test_size=0.25, random_state=23 )
train_data['Nationality'].fillna(value = 'Nationality_unk',inplace = True)
 test_data['Nationality'].fillna(value = 'Nationality_unk',inplace = True )<count_values>
def encode(encoder, x):
len_encoder = len(encoder)
 try:
id = encoder[x]
except KeyError:
id = len_encoder
return id
encoders = [{} for cat in cat_cols]
for i, cat in enumerate(cat_cols):
print('encoding %s...' % cat, end=' ')
 encoders[i] = {l: id for id, l in enumerate(market_train_df1.loc[train_indices, cat].astype(str ).unique())}
market_train_df1[cat] = market_train_df1[cat].astype(str ).apply(lambda x: encode(encoders[i], x))
print('Done')
 embed_sizes = [len(encoder)+ 1 for encoder in encoders]
train_data['Size'].value_counts()<data_type_conversions>
market_train_df1[num_cols] = market_train_df1[num_cols].fillna(0)
 print('scaling numerical columns')
 scaler = StandardScaler()
scaler = StandardScaler()
market_train_df1[num_cols] = scaler.fit_transform(market_train_df1[num_cols] )
train_data['Size'].fillna(value = 'Size_unk',inplace = True)
 test_data['Size'].fillna(value = 'Size_unk',inplace = True )<count_values>
train_data['TopThreeAmericanName'].value_counts()<data_type_conversions>
train_data['TopThreeAmericanName'].fillna(value = 'TopThreeAmericanName_unk',inplace = True)
 test_data['TopThreeAmericanName'].fillna(value = 'TopThreeAmericanName_unk',inplace = True )<count_values>
train_data['PRIMEUNIT'].value_counts()<data_type_conversions>
train_data['PRIMEUNIT'].fillna(value = 'Prime_unk',inplace = True)
 test_data['PRIMEUNIT'].fillna(value = 'Prime_unk',inplace = True )<count_values>
categorical_inputs = []
for cat in cat_cols:
categorical_inputs.append(Input(shape=[1], name=cat))
categorical_embeddings = []
for i, cat in enumerate(cat_cols):
categorical_embeddings.append(Embedding(embed_sizes[i], 10 )(categorical_inputs[i]))
categorical_logits = Flatten()(categorical_embeddings[0])
 categorical_logits = Dense(32,activation='relu' )(categorical_logits)
 categorical_logits =Dropout(0.5 )(categorical_logits)
 categorical_logits =BatchNormalization()(categorical_logits)
 categorical_logits = Dense(32,activation='relu' )(categorical_logits)
 numerical_inputs = Input(shape=(11,), name='num')
 numerical_logits = numerical_inputs
numerical_logits = BatchNormalization()(numerical_logits)
 numerical_logits = Dense(128,activation='relu' )(numerical_logits)
 numerical_logits=Dropout(0.5 )(numerical_logits)
 numerical_logits = BatchNormalization()(numerical_logits)
 numerical_logits = Dense(128,activation='relu' )(numerical_logits)
 numerical_logits = Dense(64,activation='relu' )(numerical_logits)
 logits = Concatenate()([numerical_logits,categorical_logits])
 logits = Dense(64,activation='relu' )(logits)
 out = Dense(1, activation='sigmoid' )(logits)
 model = Model(inputs = categorical_inputs + [numerical_inputs], outputs=out)
 model.compile(optimizer='adam',loss=binary_crossentropy )
train_data['AUCGUART'].value_counts()<count_values>
def get_input(market_train, indices):
X_num = market_train.loc[indices, num_cols].values
X = {'num':X_num}
for cat in cat_cols:
X[cat] = market_train.loc[indices, cat_cols].values
y =(market_train.loc[indices,'returnsOpenNextMktres10'] >= 0 ).values
r = market_train.loc[indices,'returnsOpenNextMktres10'].values
u = market_train.loc[indices, 'universe']
d = market_train.loc[indices, 'time'].dt.date
return X,y,r,u,d
X_train,y_train,r_train,u_train,d_train = get_input(market_train_df1, train_indices)
 X_valid,y_valid,r_valid,u_valid,d_valid = get_input(market_train_df1, val_indices )
test_data['AUCGUART'].value_counts()<categorify>
class SWA(keras.callbacks.Callback):
def __init__(self, filepath, swa_epoch):
super(SWA, self ).__init__()
self.filepath = filepath
self.swa_epoch = swa_epoch
def on_train_begin(self, logs=None):
self.nb_epoch = self.params['epochs']
print('Stochastic weight averaging selected for last {} epochs.' 
.format(self.nb_epoch - self.swa_epoch))
def on_epoch_end(self, epoch, logs=None):
if epoch == self.swa_epoch:
self.swa_weights = self.model.get_weights()
elif epoch > self.swa_epoch:
for i in range(len(self.swa_weights)) :
self.swa_weights[i] =(self.swa_weights[i] * 
(epoch - self.swa_epoch)+ self.model.get_weights() [i])/(( epoch - self.swa_epoch)+ 1)
 else:
pass
def on_train_end(self, logs=None):
self.model.set_weights(self.swa_weights)
 print('Final model parameters set to stochastic weight average.')
 self.model.save_weights(self.filepath)
 print('Final stochastic averaged weights saved to file.')
 class SnapshotCallbackBuilder:
def __init__(self, nb_epochs, nb_snapshots, init_lr=0.1):
self.T = nb_epochs
self.M = nb_snapshots
self.alpha_zero = init_lr
def get_callbacks(self, model_prefix='Model'):
callback_list = [
callbacks.ModelCheckpoint("model.hdf5",monitor='val_my_iou_metric',
mode = 'max', save_best_only=True, verbose=1),
swa,
callbacks.LearningRateScheduler(schedule=self._cosine_anneal_schedule)
 ]
return callback_list
def _cosine_anneal_schedule(self, t):
cos_inner = np.pi *(t %(self.T // self.M))
cos_inner /= self.T // self.M
cos_out = np.cos(cos_inner)+ 1
return float(self.alpha_zero / 2 * cos_out )
train_data['AUCGUART'].replace('AGREEN','GREEN',inplace=True)
 test_data['AUCGUART'].replace('ARED','RED',inplace=True )<data_type_conversions>
check_point = ModelCheckpoint('model.hdf5',verbose=True, save_best_only=True)
 early_stop = EarlyStopping(patience=5,verbose=True)
 model.fit(X_train,y_train.astype(int),
validation_data=(X_valid,y_valid.astype(int)) ,
epochs=5,
verbose=True,
callbacks=[early_stop,check_point] )
train_data['AUCGUART'].fillna(value = 'AUC_unk',inplace = True)
 test_data['AUCGUART'].fillna(value = 'AUC_unk',inplace = True )<drop_column>
r_valid = r_valid.clip(-1,1)
 x_t_i = confidence_valid * r_valid * u_valid
data = {'day' : d_valid, 'x_t_i' : x_t_i}
df = pd.DataFrame(data)
 x_t = df.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_valid = mean / std
print(score_valid )
train_data.drop(['MMRAcquisitionAuctionAveragePrice','MMRAcquisitionAuctionCleanPrice',
'MMRAcquisitionRetailAveragePrice','MMRAcquisitonRetailCleanPrice',
'MMRCurrentAuctionAveragePrice','MMRCurrentAuctionCleanPrice',
'MMRCurrentRetailAveragePrice','MMRCurrentRetailCleanPrice'],
inplace=True,axis=1)
 test_data.drop(['MMRAcquisitionAuctionAveragePrice','MMRAcquisitionAuctionCleanPrice',
'MMRAcquisitionRetailAveragePrice','MMRAcquisitonRetailCleanPrice',
'MMRCurrentAuctionAveragePrice','MMRCurrentAuctionCleanPrice',
'MMRCurrentRetailAveragePrice','MMRCurrentRetailCleanPrice'],
inplace=True,axis=1 )<drop_column>
days = env.get_prediction_days()
<drop_column><EOS> 
n_days = 0
prep_time = 0
prediction_time = 0
packaging_time = 0
predicted_confidences = np.array([])
 for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
print(n_days,end=' ')
 t = time.time()
market_obs_df['assetCode_encoded'] = market_obs_df[cat].astype(str ).apply(lambda x: encode(encoders[i], x))
market_obs_df[num_cols] = market_obs_df[num_cols].fillna(0)
 market_obs_df[num_cols] = scaler.transform(market_obs_df[num_cols])
 X_num_test = market_obs_df[num_cols].values
X_test = {'num':X_num_test}
X_test['assetCode'] = market_obs_df['assetCode_encoded'].values
prep_time += time.time() - t
t = time.time()
market_prediction = model.predict(X_test)[:,0]*2 -1
predicted_confidences = np.concatenate(( predicted_confidences, market_prediction))
prediction_time += time.time() -t
t = time.time()
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence':market_prediction})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 packaging_time += time.time() - t
env.write_submission_file()
total = prep_time + prediction_time + packaging_time
print(f'Preparing Data: {prep_time:.2f}s')
 print(f'Making Predictions: {prediction_time:.2f}s')
 print(f'Packing: {packaging_time:.2f}s')
 print(f'Total: {total:.2f}s' )