Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<categorify>
env = twosigmanews.make_env()
def run_k_fold(NFOLDS, seed_id):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold_id in range(NFOLDS):
oof_, pred_ = run_training(fold_id, seed_id)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<categorify>
def predict(df):
df['predictMktres10'] = df.groupby(['assetCode'])['returnsOpenPrevMktres10'].shift(-11 ).fillna(0)
 df.loc[df.time > '2018-06-27', 'predictMktres10'] = 0
m = min(df[df.predictMktres10 > 0].predictMktres10.min() , df[df.predictMktres10 < 0].predictMktres10.max() * -1)
 df['assetcount'] = df.groupby('time' ).assetCode.transform('count' ).values
m1 = df.assetcount.min()
df['confidenceValue'] = 0
nz = df.predictMktres10 != 0
df.loc[nz, 'confidenceValue'] = m /(df[nz].predictMktres10)
 return df
SEED = [0, 1, 2, 3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 time_begin = time()
for seed_id in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed_id)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 time_diff = time() - time_begin
train[target_cols] = oof
test[target_cols] = predictions<feature_engineering>
market_obs_df = None
pred_df = None
for(m_df, n_df, predictions_template_df)in env.get_prediction_days() :
env.predict(predictions_template_df)
 predictions_template_df['time'] = m_df.time.min()
if market_obs_df is None:
market_obs_df = m_df
pred_df = predictions_template_df
else:
market_obs_df = market_obs_df.append(m_df, ignore_index=True)
 pred_df = pred_df.append(predictions_template_df, ignore_index=True)
str(timedelta(seconds=time_diff))<compute_test_metric>
pred_df.assetCode = pred_df.assetCode.astype(str)
 pred_df.time = pd.to_datetime(pred_df.time )
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score += log_loss(y_true[:, i], y_pred[:, i])
 print("CV log_loss: ", score / y_pred.shape[1] )<merge>
t = predict(market_obs_df)
 t['returns'] = t.confidenceValue * t.predictMktres10
day_returns = t.groupby('time' ).returns.sum()
day_returns.mean() / day_returns.std()
sub1 = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0 )<install_modules>
pred_df.drop(columns=['confidenceValue'], inplace=True)
 pred_df = pred_df.merge(t[['time', 'assetCode', 'confidenceValue']], how='left', on=['time', 'assetCode'])\ 
.fillna(0)
 pred_df.time = pred_df.time.dt.date
<install_modules><EOS> 
pred_df[['time', 'assetCode', 'confidenceValue']].to_csv('submission.csv', index=False, float_format='%.8f' )