Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<categorify>
pd.set_option('max_columns', 50 )
train = pd.get_dummies(train, columns=['cp_time','cp_dose'])
 test_ = pd.get_dummies(test, columns=['cp_time','cp_dose'] )<define_variables>
env = twosigmanews.make_env()
market_train, news_train = env.get_training_data()
feature_cols = [c for c in train.columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['sig_id']]<categorify>
start = datetime(2009, 1, 1, 0, 0, 0 ).date()
market_train = market_train.loc[market_train['time'].dt.date >= start].reset_index(drop=True)
 news_train = news_train.loc[news_train['time'].dt.date >= start].reset_index(drop=True )
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct 
<train_model>
def preprocess_news(news_train):
drop_list = [
'audiences', 'subjects', 'assetName',
'headline', 'firstCreated', 'sourceTimestamp',
]
news_train.drop(drop_list, axis=1, inplace=True)
 for col in ['headlineTag', 'provider', 'sourceId']:
news_train[col], uniques = pd.factorize(news_train[col])
 del uniques
news_train['assetCodes'] = news_train['assetCodes'].apply(lambda x: x[1:-1].replace("'", ""))
return news_train
news_train = preprocess_news(news_train )
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds 
<normalization>
def unstack_asset_codes(news_train):
codes = []
indexes = []
for i, values in news_train['assetCodes'].iteritems() :
explode = values.split(", ")
 codes.extend(explode)
 repeat_index = [int(i)]*len(explode)
 indexes.extend(repeat_index)
 index_df = pd.DataFrame({'news_index': indexes, 'assetCode': codes})
 del codes, indexes
gc.collect()
return index_df
index_df = unstack_asset_codes(news_train)
 index_df.head()
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<normalization>
def merge_news_on_index(news_train, index_df):
news_train['news_index'] = news_train.index.copy()
news_unstack = index_df.merge(news_train, how='left', on='news_index')
 news_unstack.drop(['news_index', 'assetCodes'], axis=1, inplace=True)
 return news_unstack
news_unstack = merge_news_on_index(news_train, index_df)
 del news_train, index_df
gc.collect()
news_unstack.head(3 )
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.2)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.leaky_relu(self.dense1(x), 1e-3)
 x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x<define_variables>
def group_news(news_frame):
news_frame['date'] = news_frame.time.dt.date
aggregations = ['mean']
gp = news_frame.groupby(['assetCode', 'date'] ).agg(aggregations)
 gp.columns = pd.Index(["{}_{}".format(e[0], e[1])for e in gp.columns.tolist() ])
 gp.reset_index(inplace=True)
 float_cols = {c: 'float32' for c in gp.columns if c not in ['assetCode', 'date']}
return gp.astype(float_cols)
 news_agg = group_news(news_unstack)
 del news_unstack; gc.collect()
news_agg.head(3 )
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 26
BATCH_SIZE = 256
LEARNING_RATE = 6e-4
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=2048<split>
market_train['date'] = market_train.time.dt.date
df = market_train.merge(news_agg, how='left', on=['assetCode', 'date'])
 del market_train, news_agg
gc.collect()
df.head(3 )
def run_training(fold, seed):
seed_everything(seed)
 mskf = MultilabelStratifiedKFold(n_splits=7,random_state=seed)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
train.loc[v_idx, 'kfold'] = int(f)
 train['kfold'] = train['kfold'].astype(int)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"SEED: {seed}, FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed} ,FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"SEED{seed}_FOLD{fold}_nonscored.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"SEED{seed}_FOLD{fold}_nonscored.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions 
<categorify>
def custom_metric(date, pred_proba, num_target, universe):
y = pred_proba*2 - 1
r = num_target.clip(-1,1)
 x = y * r * universe
result = pd.DataFrame({'day' : date, 'x' : x})
 x_t = result.groupby('day' ).sum().values
return np.mean(x_t)/ np.std(x_t )
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<define_variables>
date = df.date
num_target = df.returnsOpenNextMktres10.astype('float32')
 bin_target =(df.returnsOpenNextMktres10 >= 0 ).astype('int8')
 universe = df.universe.astype('int8')
 df.drop(['returnsOpenNextMktres10', 'date', 'universe', 'assetCode', 'assetName', 'time'],
axis=1, inplace=True)
 df = df.astype('float32')
 gc.collect()
SEED = [0,1,2,3,4,5,6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test_[target_cols] = predictions<merge>
train_index, test_index = train_test_split(df.index.values, test_size=0.1, shuffle=False )
train = train.merge(train_targets_scored, on='sig_id')
 target = train[train_targets_scored.columns]
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()<define_variables>
def evaluate_model(df, target, train_index, test_index, params):
params['n_jobs'] = 2
model = LGBMClassifier(**params)
 model.fit(df.iloc[train_index], target.iloc[train_index])
 return log_loss(target.iloc[test_index], model.predict_proba(df.iloc[test_index]))
feature_cols = [c for c in train.columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['sig_id','kfold']]<define_variables>
param_grid = {
'learning_rate': [0.15, 0.1, 0.05, 0.02, 0.01],
'num_leaves': [i for i in range(12, 90, 6)],
'n_estimators': [50, 200, 400, 600, 800],
'min_child_samples': [i for i in range(10, 100, 10)],
'colsample_bytree': [0.8, 0.9, 0.95, 1],
'subsample': [0.8, 0.9, 0.95, 1],
'reg_alpha': [0.1, 0.2, 0.4, 0.6, 0.8],
'reg_lambda': [0.1, 0.2, 0.4, 0.6, 0.8],
}
best_eval_score = 0
for i in range(100):
params = {k: np.random.choice(v)for k, v in param_grid.items() }
score = evaluate_model(df, bin_target, train_index, test_index, params)
 if score < best_eval_score or best_eval_score == 0:
best_eval_score = score
best_params = params
print("Best evaluation logloss", best_eval_score )
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 26
BATCH_SIZE = 256
LEARNING_RATE = 6e-4
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=2048<split>
clf = LGBMClassifier(**best_params)
 clf.fit(df, bin_target )
<categorify><EOS> 
def write_submission(model, env):
days = env.get_prediction_days()
for(market_obs_df, news_obs_df, predictions_template_df)in days:
news_obs_df = preprocess_news(news_obs_df)
 index_df = unstack_asset_codes(news_obs_df)
 news_unstack = merge_news_on_index(news_obs_df, index_df)
 news_obs_agg = group_news(news_unstack)
 market_obs_df['date'] = market_obs_df.time.dt.date
obs_df = market_obs_df.merge(news_obs_agg, how='left', on=['assetCode', 'date'])
 del market_obs_df, news_obs_agg, news_obs_df, news_unstack, index_df
gc.collect()
obs_df = obs_df[obs_df.assetCode.isin(predictions_template_df.assetCode)]
feats = [c for c in obs_df.columns if c not in ['date', 'assetCode', 'assetName', 'time']]
preds = model.predict_proba(obs_df[feats])[:, 1] * 2 - 1
sub = pd.DataFrame({'assetCode': obs_df['assetCode'], 'confidence': preds})
 predictions_template_df = predictions_template_df.merge(sub, how='left' ).drop(
 'confidenceValue', axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 del obs_df, predictions_template_df, preds, sub
gc.collect()
env.write_submission_file()
write_submission(clf, env )