Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<drop_column>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import time
import datetime
from sklearn.metrics import accuracy_score
from kaggle.competitions import twosigmanews
import gc
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )<define_variables>
env = twosigmanews.make_env() 
(market_train, news_train)= env.get_training_data()
gc.enable()
target_cols = [x for x in train_targets_scored.columns if x != 'sig_id']
aux_target_cols = [x for x in train_targets_nonscored.columns if x != 'sig_id']
all_target_cols = target_cols + aux_target_cols
num_targets = len(target_cols)
 num_aux_targets = len(aux_target_cols)
 num_all_targets = len(all_target_cols)
 print('num_targets: {}'.format(num_targets))
print('num_aux_targets: {}'.format(num_aux_targets))
print('num_all_targets: {}'.format(num_all_targets))<categorify>
news_train['time'] = news_train['time'].dt.floor('d')
 cols = ['sentimentNegative','sentimentNeutral','sentimentPositive','relevance','companyCount','bodySize','sentenceCount','wordCount','firstMentionSentence',
'sentimentWordCount','takeSequence','sentimentClass','noveltyCount12H', 'noveltyCount24H','noveltyCount3D', 'noveltyCount5D', 'noveltyCount7D',
'volumeCounts12H','volumeCounts24H', 'volumeCounts3D', 'volumeCounts5D','volumeCounts7D']
news_total = news_train[['time','assetName'] + cols].copy()
del news_train
gc.collect()
news_train = news_total
print(news_train.columns)
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct<train_model>
warnings.filterwarnings(action ='ignore',category = DeprecationWarning)
 def get_news_train(raw_data,days = 5):
news_last = pd.DataFrame()
rate = 1.0
for i in range(days):
cur_train = raw_data[cols] * rate
rate *= 0.7
cur_train['time'] = raw_data['time'] + datetime.timedelta(days = i,hours=22)
 cur_train['key'] = cur_train['time'].astype(str)+ raw_data['assetName'].astype(str)
 cur_train = cur_train[['key'] + cols].groupby('key' ).sum()
cur_train['key'] = cur_train.index.values
news_last = pd.concat([news_last, cur_train[['key'] + cols]])
 del cur_train
gc.collect()
print("after concat the shape is:",news_last.shape)
 news_last = news_last.groupby('key' ).sum()
news_last['key'] = news_last.index.values
print("the result shape is:",news_last.shape)
 del news_last['key']
return news_last
news_last = get_news_train(news_train)
 print(news_last.shape)
 print(news_last.head())
 print(news_last.dtypes)
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds<normalization>
market_train['key'] = market_train['time'].astype(str)+ market_train['assetName'].astype(str)
 market_train = market_train.join(news_last,on = 'key',how='left')
 print(market_train['sentimentNeutral'].isnull().value_counts())
 market_train.head()
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<choose_model_class>
cat_cols = ['assetCode','assetName']
num_cols = ['volume', 'close', 'open', 'returnsClosePrevRaw1', 'returnsOpenPrevRaw1', 'returnsClosePrevMktres1',
'returnsOpenPrevMktres1', 'returnsClosePrevRaw10', 'returnsOpenPrevRaw10', 'returnsClosePrevMktres10',
'returnsOpenPrevMktres10','sentimentNegative','sentimentNeutral','sentimentPositive','relevance','companyCount','bodySize',
'sentenceCount','wordCount','firstMentionSentence', 'sentimentWordCount','takeSequence','sentimentClass','noveltyCount12H',
'noveltyCount24H','noveltyCount3D', 'noveltyCount5D', 'noveltyCount7D', 'volumeCounts12H','volumeCounts24H', 'volumeCounts3D',
'volumeCounts5D','volumeCounts7D']
class Model(nn.Module):
def __init__(self, num_features, num_targets):
super(Model, self ).__init__()
self.hidden_size = [1500, 1250, 1000, 750]
self.dropout_value = [0.5, 0.35, 0.3, 0.25]
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, self.hidden_size[0])
 self.batch_norm2 = nn.BatchNorm1d(self.hidden_size[0])
 self.dropout2 = nn.Dropout(self.dropout_value[0])
 self.dense2 = nn.Linear(self.hidden_size[0], self.hidden_size[1])
 self.batch_norm3 = nn.BatchNorm1d(self.hidden_size[1])
 self.dropout3 = nn.Dropout(self.dropout_value[1])
 self.dense3 = nn.Linear(self.hidden_size[1], self.hidden_size[2])
 self.batch_norm4 = nn.BatchNorm1d(self.hidden_size[2])
 self.dropout4 = nn.Dropout(self.dropout_value[2])
 self.dense4 = nn.Linear(self.hidden_size[2], self.hidden_size[3])
 self.batch_norm5 = nn.BatchNorm1d(self.hidden_size[3])
 self.dropout5 = nn.Dropout(self.dropout_value[3])
 self.dense5 = nn.utils.weight_norm(nn.Linear(self.hidden_size[3], num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.leaky_relu(self.dense3(x))
x = self.batch_norm4(x)
 x = self.dropout4(x)
 x = F.leaky_relu(self.dense4(x))
x = self.batch_norm5(x)
 x = self.dropout5(x)
 x = self.dense5(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))<choose_model_class>
train_indices, val_indices = train_test_split(market_train.index.values,test_size=0.25, random_state=23 )
class FineTuneScheduler:
def __init__(self, epochs):
self.epochs = epochs
self.epochs_per_step = 0
self.frozen_layers = []
def copy_without_top(self, model, num_features, num_targets, num_targets_new):
self.frozen_layers = []
model_new = Model(num_features, num_targets)
 model_new.load_state_dict(model.state_dict())
 for name, param in model_new.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == 5:
continue
param.requires_grad = False
if layer_index not in self.frozen_layers:
self.frozen_layers.append(layer_index)
 self.epochs_per_step = self.epochs // len(self.frozen_layers)
 model_new.batch_norm5 = nn.BatchNorm1d(model_new.hidden_size[3])
 model_new.dropout5 = nn.Dropout(model_new.dropout_value[3])
 model_new.dense5 = nn.utils.weight_norm(nn.Linear(model_new.hidden_size[-1], num_targets_new))
model_new.to(DEVICE)
 return model_new
def step(self, epoch, model):
if len(self.frozen_layers)== 0:
return
if epoch % self.epochs_per_step == 0:
last_frozen_index = self.frozen_layers[-1]
for name, param in model.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == last_frozen_index:
param.requires_grad = True
del self.frozen_layers[-1]<categorify>
def encode(encoder, x):
len_encoder = len(encoder)
 try:
id = encoder[x]
except KeyError:
id = len_encoder
return id
encoders = [{} for i in range(len(cat_cols)) ]
for i, cat in enumerate(cat_cols):
print('encoding %s...' % cat, end=' ')
 encoders[i] = {l: id for id, l in enumerate(market_train.loc[train_indices, cat].unique())}
market_train[cat] = market_train[cat].astype(str ).apply(lambda x: encode(encoders[i], x))
print('Done')
 embed_sizes = [len(encoder)+ 1 for encoder in encoders]
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data<define_variables>
def get_input(market_train, indices):
X = market_train.loc[indices, num_cols]
for cat in cat_cols:
X[cat] = market_train.loc[indices, cat].values
y =(market_train.loc[indices,'returnsOpenNextMktres10'] >= 0 ).values
r = market_train.loc[indices,'returnsOpenNextMktres10'].values
u = market_train.loc[indices, 'universe']
d = market_train.loc[indices, 'time'].dt.date
return X,y,r,u,d
X_train,y_train,r_train,u_train,d_train = get_input(market_train, train_indices)
 X_valid,y_valid,r_valid,u_valid,d_valid = get_input(market_train, val_indices)
 X_train.shape
print(X_valid.shape )
feature_cols = [c for c in process_data(train ).columns if c not in all_target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold', 'sig_id', 'drug_id']]
num_features = len(feature_cols)
 num_features<define_variables>
algo = partial(tpe.suggest, n_startup_jobs=10)
 def run_lgb(train_X, train_y, val_X, val_y,args):
params = {
"objective" : "binary",
"metric" : "binary_logloss",
"num_leaves" : args['num_leaves'],
"min_child_samples" : args['min_child_samples'],
"learning_rate" : args['learning_rate'],
"bagging_fraction" : 0.7,
"feature_fraction" : 0.66,
"bagging_frequency" : 5,
"bagging_seed" : 2018,
"verbosity" : -1
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 model = lgb.train(params, lgtrain, args['n_estimators'], valid_sets=[lgval], early_stopping_rounds=50, verbose_eval=100)
 pred_val_y = model.predict(val_X, num_iteration=model.best_iteration)
 confidence_valid = model.predict(X_valid)*2 -1
score = accuracy_score(confidence_valid > 0 , y_valid)
 print(score)
 mse = mean_squared_error(confidence_valid > 0, y_valid.astype(float))
print("MSE", mse)
 print("args",args)
 return model, mse
args = {'learning_rate': 1.0958730495793214, 'max_depth': 7, 'min_child_samples': 301, 'n_estimators': 439, 'num_leaves': 43}
model, _ = run_lgb(X_train, y_train.astype(int), X_valid, y_valid.astype(int), args)
 gc.collect()
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 24
BATCH_SIZE = 128
WEIGHT_DECAY = {'ALL_TARGETS': 1e-5, 'SCORED_ONLY': 3e-6}
MAX_LR = {'ALL_TARGETS': 1e-2, 'SCORED_ONLY': 3e-3}
DIV_FACTOR = {'ALL_TARGETS': 1e3, 'SCORED_ONLY': 1e2}
PCT_START = 0.1<choose_model_class>
confidence_valid = model.predict(X_valid)*2 -1
r_valid = r_valid.clip(-1,1)
 x_t_i = confidence_valid * r_valid * u_valid
data = {'day' : d_valid, 'x_t_i' : x_t_i}
df = pd.DataFrame(data)
 x_t = df.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_valid = mean / std
print(score_valid)
 market_train.describe()
model = Model(num_features, num_all_targets)
 model<define_search_model>
days = env.get_prediction_days()
<train_model><EOS> 
n_days = 0
predicted_confidences = np.array([])
 news_pre = deque()
news_all = pd.DataFrame()
BaseMod = 50
for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
print(n_days,end=' ')
 news_all = pd.concat([news_all,news_obs_df])
 if n_days >= BaseMod and n_days % BaseMod >= 0 and n_days % BaseMod < 8:
news_pre.append(news_obs_df)
 elif n_days >= BaseMod and n_days % BaseMod == 8:
del news_all
gc.collect()
news_all = pd.DataFrame()
for item in news_pre:
news_all = pd.concat([news_all,item])
 news_pre.clear()
news_all['time'] = news_all['time'].dt.floor('d')
 news_last = pd.DataFrame()
news_last = get_news_train(news_all)
 market_obs_df['key'] = market_obs_df['time'].astype(str)+ market_obs_df['assetName'].astype(str)
 market_obs_df = market_obs_df.join(news_last,on = 'key',how='left')
 market_obs_df['close'].clip(upper = 1000, inplace = True)
 market_obs_df['open'].clip(upper = 1000, inplace = True)
 market_obs_df['volume'].clip(upper = 1e+08, inplace = True)
 market_obs_df[num_cols]=market_obs_df[num_cols].fillna(col_mean)
 market_obs_df[num_cols] = scaler.transform(market_obs_df[num_cols])
 X_test = market_obs_df[num_cols]
X_test['assetCode'] = market_obs_df['assetCode'].apply(lambda x: encode(encoders[0], x)).values
X_test['assetName'] = market_obs_df['assetName'].apply(lambda x: encode(encoders[1], x)).values
market_prediction = model.predict(X_test)*2 -1
predicted_confidences = np.concatenate(( predicted_confidences, market_prediction))
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence':market_prediction})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 del news_last
gc.collect()
env.write_submission_file()