Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<choose_model_class>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import time
from sklearn.metrics import accuracy_score
from kaggle.competitions import twosigmanews
with strategy.scope() :
desnet=tf.keras.applications.DenseNet201(
 weights='imagenet',
include_top=False 
)
 model2 = tf.keras.Sequential([
desnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model2.compile(
 optimizer=tf.keras.optimizers.Adam(lr=0.0001),
loss = 'categorical_crossentropy',
metrics=[tfa.metrics.F1Score(num_classes=104, average="macro")] 
)
 model2.summary()
model2.load_weights('.. /input/mytopmodel/dense_best.h5' )<choose_model_class>
env = twosigmanews.make_env() 
(market_train, news_train)= env.get_training_data()
with strategy.scope() :
desnet=tf.keras.applications.InceptionResNetV2(
 weights='imagenet',
include_top=False 
)
 model3 = tf.keras.Sequential([
desnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model3.compile(
 optimizer=tf.keras.optimizers.Adam(lr=0.0001),
loss = 'categorical_crossentropy',
metrics=[tfa.metrics.F1Score(num_classes=104, average="macro")] 
)
 model3.summary()
model3.load_weights('.. /input/mytopmodel/InceptionResV2_best.h5' )<choose_model_class>
cat_cols = ['assetCode']
num_cols = ['volume', 'close', 'open', 'returnsClosePrevRaw1', 'returnsOpenPrevRaw1', 'returnsClosePrevMktres1',
'returnsOpenPrevMktres1', 'returnsClosePrevRaw10', 'returnsOpenPrevRaw10', 'returnsClosePrevMktres10',
'returnsOpenPrevMktres10']
with strategy.scope() :
desnet=tf.keras.applications.VGG19(
 weights='imagenet',
include_top=False 
)
 model4 = tf.keras.Sequential([
desnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model4.compile(
 optimizer=tf.keras.optimizers.Adam(lr=0.0001),
loss = 'categorical_crossentropy',
metrics=[tfa.metrics.F1Score(num_classes=104, average="macro")] 
)
 model4.summary()
model4.load_weights('.. /input/mytopmodel/VGG_best.h5' )<choose_model_class>
market_train = market_train.loc[pd.to_datetime(market_train['time'])>= pd.to_datetime('2009-01-01' ).tz_localize('UTC')]
train_indices, val_indices = train_test_split(market_train.index.values,test_size=0.25, random_state=23)
with strategy.scope() :
desnet=efn.EfficientNetB5(
 weights='noisy-student',
include_top=False 
)
 model5 = tf.keras.Sequential([
desnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model5.compile(
 optimizer=tf.keras.optimizers.Adam(lr=0.0001),
loss = 'categorical_crossentropy',
metrics=[tfa.metrics.F1Score(num_classes=104, average="macro")] 
)
 model5.summary()
model5.load_weights('.. /input/mytopmodel/B5_best.h5' )<choose_model_class>
def encode(encoder, x):
len_encoder = len(encoder)
 try:
id = encoder[x]
except KeyError:
id = len_encoder
return id
encoders = [{} for cat in cat_cols]
for i, cat in enumerate(cat_cols):
print('encoding %s...' % cat, end=' ')
 encoders[i] = {l: id for id, l in enumerate(market_train.loc[train_indices, cat].astype(str ).unique())}
market_train[cat] = market_train[cat].astype(str ).apply(lambda x: encode(encoders[i], x))
print('Done')
 embed_sizes = [len(encoder)+ 1 for encoder in encoders]
with strategy.scope() :
desnet=tf.keras.applications.ResNet152V2(
 weights='imagenet',
include_top=False 
)
 model6 = tf.keras.Sequential([
desnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model6.compile(
 optimizer=tf.keras.optimizers.Adam(lr=0.0001),
loss = 'categorical_crossentropy',
metrics=[tfa.metrics.F1Score(num_classes=104, average="macro")] 
)
 model6.summary()
model6.load_weights('.. /input/mytopmodel/Res152_best.h5' )<define_variables>
market_train[num_cols] = market_train[num_cols].fillna(0)
 print('scaling numerical columns')
 scaler = StandardScaler()
scaler = StandardScaler()
print(market_train['time'].dtypes)
 market_train[num_cols] = scaler.fit_transform(market_train[num_cols])
models = [model1, model2, model3, model4, model5, model6]<save_to_csv>
def get_input(market_train, indices):
X_num = market_train.loc[indices, num_cols].values
X = {'num':X_num}
for cat in cat_cols:
X[cat] = market_train.loc[indices, cat_cols].values
y =(market_train.loc[indices,'returnsOpenNextMktres10'] >= 0 ).values
r = market_train.loc[indices,'returnsOpenNextMktres10'].values
u = market_train.loc[indices, 'universe']
d = market_train.loc[indices, 'time'].dt.date
return X,y,r,u,d
X_train,y_train,r_train,u_train,d_train = get_input(market_train, train_indices)
 X_valid,y_valid,r_valid,u_valid,d_valid = get_input(market_train, val_indices )
test_ds = get_test_dataset(ordered=True)
 test_images_ds = test_ds.map(lambda image, idnum: image)
 print('Computing predictions...')
 probabilities = np.average([models[i].predict(test_images_ds)for i in range(6)], axis = 0)
 predictions = np.argmax(probabilities, axis=-1)
 print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')
 return histories, models<create_dataframe>
warnings.filterwarnings(action ='ignore',category = DeprecationWarning)
 algo = partial(tpe.suggest, n_startup_jobs=10)
 def auto_turing(args):
model = XGBClassifier(n_jobs = 4, n_estimators = args['n_estimators'],max_depth=6)
 model.fit(X_train['num'],y_train.astype(int))
confidence_valid = model.predict(X_valid['num'])*2 -1
score = accuracy_score(confidence_valid>0,y_valid)
 print(args,score)
 return -score
model = RandomForestClassifier(n_estimators=50, criterion='gini', max_depth=None,min_samples_split=2, min_samples_leaf=1,
min_weight_fraction_leaf=0.0,max_features='auto',max_leaf_nodes=None,min_impurity_split=1e-07,
bootstrap=True, oob_score=False, n_jobs=1,random_state=None, verbose=0,warm_start=False, class_weight=None)
 model.fit(X_train['num'], y_train.astype(int))
confidence_valid = model.predict(X_valid['num'])*2 -1
score = accuracy_score(confidence_valid>0,y_valid)
 print(score)
<choose_model_class>
r_valid = r_valid.clip(-1,1)
 x_t_i = confidence_valid * r_valid * u_valid
data = {'day' : d_valid, 'x_t_i' : x_t_i}
df = pd.DataFrame(data)
 x_t = df.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_valid = mean / std
print(score_valid )
<choose_model_class>
days = env.get_prediction_days()
<choose_model_class><EOS> 
n_days = 0
prep_time = 0
prediction_time = 0
packaging_time = 0
predicted_confidences = np.array([])
 for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
print(n_days,end=' ')
 t = time.time()
market_obs_df['assetCode_encoded'] = market_obs_df[cat].astype(str ).apply(lambda x: encode(encoders[i], x))
market_obs_df[num_cols] = market_obs_df[num_cols].fillna(0)
 market_obs_df[num_cols] = scaler.transform(market_obs_df[num_cols])
 X_num_test = market_obs_df[num_cols].values
X_test = {'num':X_num_test}
X_test['assetCode'] = market_obs_df['assetCode_encoded'].values
prep_time += time.time() - t
t = time.time()
market_prediction = model.predict(X_test['num'])*2 -1
predicted_confidences = np.concatenate(( predicted_confidences, market_prediction))
prediction_time += time.time() -t
t = time.time()
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence':market_prediction})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 packaging_time += time.time() - t
env.write_submission_file()
total = prep_time + prediction_time + packaging_time
print(f'Preparing Data: {prep_time:.2f}s')
 print(f'Making Predictions: {prediction_time:.2f}s')
 print(f'Packing: {packaging_time:.2f}s')
 print(f'Total: {total:.2f}s' )