Kaggle competition: Two Sigma: Using News to Predict Stock Movements
bottom_10_roots = get_n(train_df_, 'grapheme_root', 10, False)
 bottom_10_roots<drop_column>
import numpy as np
import lightgbm as lgb
import pandas as pd
from kaggle.competitions import twosigmanews
import matplotlib.pyplot as plt
import random
from datetime import datetime, date
from xgboost import XGBClassifier
from sklearn import model_selection
from sklearn.metrics import mean_squared_error
import time
train_df_ = train_df_.drop(['grapheme'], axis=1, inplace=False )<data_type_conversions>
env = twosigmanews.make_env()
print('Done!' )
train_df_[['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']] = train_df_[['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']].astype('uint8' )<define_variables>
( market_train_df, news_train_df)= env.get_training_data()
IMG_SIZE=64
N_CHANNELS=1<define_variables>
market_train_df['time'] = market_train_df['time'].dt.date
market_train_df = market_train_df.loc[market_train_df['time']>=date(2010, 1, 1)]
HEIGHT = 137
WIDTH = 236
SIZE = 64
CROP_SIZE = 64<categorify>
def get_dummies(df):
cols = []
for col in df:
cols.append(pd.get_dummies(df[col].astype(str)))
 return pd.concat(cols, axis=1 )<choose_model_class>
return_features = ['returnsClosePrevMktres10','returnsClosePrevRaw10','open','close']
n_lag = [3,7,14]
new_df = generate_lag_features(market_train_df,n_lag=n_lag)
inputs = Input(shape =(IMG_SIZE, IMG_SIZE, 1))
model = Conv2D(filters=32, kernel_size=(3, 3), padding='SAME', activation='relu', input_shape=(IMG_SIZE, IMG_SIZE, 1))(inputs)
 model = Conv2D(filters=32, kernel_size=(3, 3), padding='SAME', activation='relu' )(model)
 model = Conv2D(filters=32, kernel_size=(3, 3), padding='SAME', activation='relu' )(model)
 model = BatchNormalization(momentum=0.15 )(model)
 model = MaxPool2D(pool_size=(2, 2))(model)
 model = Conv2D(filters=32, kernel_size=(5, 5), padding='SAME', activation='relu' )(model)
 model = Dropout(rate=0.3 )(model)
 model = Conv2D(filters=64, kernel_size=(3, 3), padding='SAME', activation='relu' )(model)
 model = Conv2D(filters=64, kernel_size=(3, 3), padding='SAME', activation='relu' )(model)
 model = Conv2D(filters=64, kernel_size=(3, 3), padding='SAME', activation='relu' )(model)
 model = BatchNormalization(momentum=0.15 )(model)
 model = MaxPool2D(pool_size=(2, 2))(model)
 model = Conv2D(filters=64, kernel_size=(5, 5), padding='SAME', activation='relu' )(model)
 model = BatchNormalization(momentum=0.15 )(model)
 model = Dropout(rate=0.3 )(model)
 model = Conv2D(filters=128, kernel_size=(3, 3), padding='SAME', activation='relu' )(model)
 model = Conv2D(filters=128, kernel_size=(3, 3), padding='SAME', activation='relu' )(model)
 model = Conv2D(filters=128, kernel_size=(3, 3), padding='SAME', activation='relu' )(model)
 model = BatchNormalization(momentum=0.15 )(model)
 model = MaxPool2D(pool_size=(2, 2))(model)
 model = Conv2D(filters=128, kernel_size=(5, 5), padding='SAME', activation='relu' )(model)
 model = BatchNormalization(momentum=0.15 )(model)
 model = Dropout(rate=0.3 )(model)
 model = Conv2D(filters=256, kernel_size=(3, 3), padding='SAME', activation='relu' )(model)
 model = Conv2D(filters=256, kernel_size=(3, 3), padding='SAME', activation='relu' )(model)
 model = Conv2D(filters=256, kernel_size=(3, 3), padding='SAME', activation='relu' )(model)
 model = BatchNormalization(momentum=0.15 )(model)
 model = MaxPool2D(pool_size=(2, 2))(model)
 model = Conv2D(filters=256, kernel_size=(5, 5), padding='SAME', activation='relu' )(model)
 model = BatchNormalization(momentum=0.15 )(model)
 model = Dropout(rate=0.3 )(model)
 model = Flatten()(model)
 model = Dense(1024, activation = "relu" )(model)
 model = Dropout(rate=0.3 )(model)
 dense = Dense(512, activation = "relu" )(model)
 head_root = Dense(168, activation = 'softmax' )(dense)
 head_vowel = Dense(11, activation = 'softmax' )(dense)
 head_consonant = Dense(7, activation = 'softmax' )(dense)
 model = Model(inputs=inputs, outputs=[head_root, head_vowel, head_consonant] )<choose_model_class>
market_train_df = pd.merge(market_train_df,new_df,how='left',on=['time','assetCode'])
 market_train_df.head()
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'] )<choose_model_class>
def mis_impute(data):
for i in data.columns:
if data[i].dtype == "object":
data[i] = data[i].fillna("other")
 elif(data[i].dtype == "int64" or data[i].dtype == "float64"):
data[i] = data[i].fillna(data[i].mean())
 else:
pass
return data
market_train_df = mis_impute(market_train_df )
learning_rate_reduction_root = ReduceLROnPlateau(monitor='dense_3_accuracy',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001)
 learning_rate_reduction_vowel = ReduceLROnPlateau(monitor='dense_4_accuracy',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001)
 learning_rate_reduction_consonant = ReduceLROnPlateau(monitor='dense_5_accuracy',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )<define_variables>
def data_prep(market_train):
market_train['bartrend'] = market_train['close'] / market_train['open']
market_train['average'] =(market_train['close'] + market_train['open'])/2
lbl = {k: v for v, k in enumerate(market_train['assetCode'].unique())}
market_train['assetCodeT'] = market_train['assetCode'].map(lbl)
 market_train = market_train.dropna(axis=0)
 return market_train
market_train_df = data_prep(market_train_df)
 print(market_train_df.shape,market_train_df.columns )
batch_size = 256
epochs = 30<set_options>
up = market_train_df['returnsOpenNextMktres10'] >= 0
universe = market_train_df['universe'].values
d = market_train_df['time']
fcol = [c for c in market_train_df if c not in ['assetCode', 'assetCodes', 'assetCodesLen', 'assetName', 'audiences',
'firstCreated', 'headline', 'headlineTag', 'marketCommentary', 'provider',
'returnsOpenNextMktres10', 'sourceId', 'subjects', 'time', 'time_x', 'universe','sourceTimestamp']]
X = market_train_df[fcol].values
up = up.values
r = market_train_df.returnsOpenNextMktres10.values
mins = np.min(X, axis=0)
 maxs = np.max(X, axis=0)
 rng = maxs - mins
X = 1 -(( maxs - X)/ rng)
 assert X.shape[0] == up.shape[0] == r.shape[0]
te = market_train_df['time']>date(2015, 1, 1)
 tt = 0
for tt,i in enumerate(te.values):
if i:
idx = tt
print(i,tt)
 break
print(idx)
 X_train, X_test = X[:idx],X[idx:]
up_train, up_test = up[:idx],up[idx:]
r_train, r_test = r[:idx],r[idx:]
u_train,u_test = universe[:idx],universe[idx:]
d_train,d_test = d[:idx],d[idx:]
train_data = lgb.Dataset(X_train, label=up_train.astype(int))
test_data = lgb.Dataset(X_test, label=up_test.astype(int))
del histories
gc.collect()<define_variables>
confidence_test =(gbm_1.predict(X_test)+ gbm_2.predict(X_test)) /2
confidence_test =(confidence_test-confidence_test.min())/(confidence_test.max() -confidence_test.min())
 confidence_test = confidence_test*2-1
print(max(confidence_test),min(confidence_test))
r_test = r_test.clip(-1,1)
 x_t_i = confidence_test * r_test * u_test
data = {'day' : d_test, 'x_t_i' : x_t_i}
df = pd.DataFrame(data)
 x_t = df.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_test = mean / std
print(score_test)
preds_dict = {
'grapheme_root': [],
'vowel_diacritic': [],
'consonant_diacritic': []
}<predict_on_test>
del X_train,X_test
gc.collect()
components = ['consonant_diacritic', 'grapheme_root', 'vowel_diacritic']
target=[]
row_id=[]
for i in range(4):
df_test_img = pd.read_parquet('/kaggle/input/bengaliai-cv19/test_image_data_{}.parquet'.format(i))
df_test_img.set_index('image_id', inplace=True)
 X_test = resize(df_test_img, need_progress_bar=False)/255
X_test = X_test.values.reshape(-1, IMG_SIZE, IMG_SIZE, N_CHANNELS)
 preds = model.predict(X_test)
 for i, p in enumerate(preds_dict):
preds_dict[p] = np.argmax(preds[i], axis=1)
 for k,id in enumerate(df_test_img.index.values):
for i,comp in enumerate(components):
id_sample=id+'_'+comp
row_id.append(id_sample)
 target.append(preds_dict[comp][k])
 del df_test_img
del X_test
gc.collect()
df_sample = pd.DataFrame(
 {
'row_id': row_id,
'target':target
},
columns = ['row_id','target'] 
)
 df_sample.to_csv('submission.csv',index=False)
 df_sample.head()<import_modules>
def post_scaling(df):
mean, std = np.mean(df), np.std(df)
 df =(df - mean)/(std * 8)
 return np.clip(df,-1,1 )
print(os.listdir(".. /input")) 
<load_from_csv>
days = env.get_prediction_days()
n_days = 0
prep_time = 0
prediction_time = 0
packaging_time = 0
total_market_obs_df = []
for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
if(n_days%50==0):
print(n_days,end=' ')
 t = time.time()
market_obs_df['time'] = market_obs_df['time'].dt.date
return_features = ['returnsClosePrevMktres10','returnsClosePrevRaw10','open','close']
total_market_obs_df.append(market_obs_df)
 if len(total_market_obs_df)==1:
history_df = total_market_obs_df[0]
else:
history_df = pd.concat(total_market_obs_df[-(np.max(n_lag)+1):])
 print(history_df)
 new_df = generate_lag_features(history_df,n_lag=[3,7,14])
 market_obs_df = pd.merge(market_obs_df,new_df,how='left',on=['time','assetCode'])
 market_obs_df = mis_impute(market_obs_df)
 market_obs_df = data_prep(market_obs_df)
 X_live = market_obs_df[fcol].values
X_live = 1 -(( maxs - X_live)/ rng)
 prep_time += time.time() - t
t = time.time()
lp =(gbm_1.predict(X_live)+ gbm_2.predict(X_live)) /2
prediction_time += time.time() -t
t = time.time()
confidence = lp
confidence =(confidence-confidence.min())/(confidence.max() -confidence.min())
 confidence = confidence * 2 - 1
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence': post_scaling(confidence)})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 packaging_time += time.time() - t
env.write_submission_file()
sub = pd.read_csv("submission.csv" )