Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<prepare_output>
env = twosigmanews.make_env() 
(marketdf, newsdf)= env.get_training_data()
print('preparing data...' )
weighted_y_pred = pd.DataFrame()
weighted_y_pred['sig_id'] = predictions[0]['sig_id']
for column in target_columns:
column_data = []
for i in range(len(predictions)) :
column_data.append(predictions[i][column] * weights[i])
 weighted_y_pred[column] = np.sum(column_data, axis=0)
 weighted_y_pred.shape<prepare_output>
def prepare_data(marketdf, newsdf):
marketdf['time'] = marketdf.time.dt.strftime("%Y%m%d" ).astype(int)
 marketdf['bartrend'] = marketdf['close'] / marketdf['open']
marketdf['average'] =(marketdf['close'] + marketdf['open'])/2
marketdf['pricevolume'] = marketdf['volume'] * marketdf['close']
newsdf['time'] = newsdf.time.dt.strftime("%Y%m%d" ).astype(int)
 newsdf['assetCode'] = newsdf['assetCodes'].map(lambda x: list(eval(x)) [0])
 newsdf['position'] = newsdf['firstMentionSentence'] / newsdf['sentenceCount']
newsdf['coverage'] = newsdf['sentimentWordCount'] / newsdf['wordCount']
marketdf = marketdf.loc[marketdf['time'] > 20120000]
droplist = ['sourceTimestamp','firstCreated','sourceId','headline','takeSequence','provider','firstMentionSentence',
'sentenceCount','bodySize','headlineTag','marketCommentary','subjects','audiences','sentimentClass',
'assetName', 'assetCodes','urgency','wordCount','sentimentWordCount']
newsdf.drop(droplist, axis=1, inplace=True)
 marketdf.drop(['assetName', 'volume'], axis=1, inplace=True)
 newsgp = newsdf.groupby(['time','assetCode'], sort=False ).aggregate(np.mean ).reset_index()
return pd.merge(marketdf, newsgp, how='left', on=['time', 'assetCode'], copy=False)
 cdf = prepare_data(marketdf, newsdf)
 del marketdf, newsdf
y_pred = weighted_y_pred<save_to_csv>
print('building training set...')
 targetcols = ['returnsOpenNextMktres10']
traincols = [col for col in cdf.columns if col not in ['time', 'assetCode', 'universe'] + targetcols]
dates = cdf['time'].unique()
train = range(len(dates)) [:int(0.85*len(dates)) ]
val = range(len(dates)) [int(0.85*len(dates)) :]
cdf[targetcols[0]] =(cdf[targetcols[0]] > 0 ).astype(int)
 Xt = cdf[traincols].fillna(0 ).loc[cdf['time'].isin(dates[train])].values
Yt = cdf[targetcols].fillna(0 ).loc[cdf['time'].isin(dates[train])].values
Xv = cdf[traincols].fillna(0 ).loc[cdf['time'].isin(dates[val])].values
Yv = cdf[targetcols].fillna(0 ).loc[cdf['time'].isin(dates[val])].values
print(Xt.shape, Xv.shape )
submission = pd.DataFrame(index = public_ids + private_ids, columns=target_columns)
 submission.index.name = 'sig_id'
submission[:] = 0
submission.loc[y_pred.sig_id,:] = y_pred[target_columns].values
submission.loc[test[test.cp_type == 'ctl_vehicle'].sig_id] = 0
submission.to_csv('submission.csv', index=True )<import_modules>
print('Training lightgbm')
 params = {"objective" : "binary",
"metric" : "binary_logloss",
"num_leaves" : 125,
"max_depth": -1,
"learning_rate" : 0.0005,
"bagging_fraction" : 0.9,
"feature_fraction" : 0.9,
"bagging_freq" : 5,
"bagging_seed" : 2018,
"verbosity" : -1 }
lgtrain, lgval = lgb.Dataset(Xt, Yt[:,0]), lgb.Dataset(Xv, Yv[:,0])
 lgbmodel = lgb.train(params, lgtrain, 2000, valid_sets=[lgtrain, lgval], early_stopping_rounds=100, verbose_eval=200)
Flatten, LeakyReLU, BatchNormalization)
 LearningRateScheduler,EarlyStopping,
ReduceLROnPlateau,CSVLogger)
 warnings.simplefilter('ignore')
 sns.set_style('whitegrid' )<load_from_csv>
def post_scaling(df):
mean, std = np.mean(df), np.std(df)
 df =(df - mean)/(std * 8)
 return np.clip(df,-1,1 )
<set_options><EOS> 
print("generating predictions...")
 preddays = env.get_prediction_days()
for marketdf, newsdf, predtemplatedf in preddays:
cdf = prepare_data(marketdf, newsdf)
 Xp = cdf[traincols].fillna(0 ).values
preds = lgbmodel.predict(Xp, num_iteration=lgbmodel.best_iteration)* 2 - 1
predsdf = pd.DataFrame({'ast':cdf['assetCode'],'conf':post_scaling(preds)})
 predtemplatedf['confidenceValue'][predtemplatedf['assetCode'].isin(predsdf.ast)] = predsdf['conf'].values
env.predict(predtemplatedf)
 env.write_submission_file()