Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<load_from_csv>
env = twosigmanews.make_env() 
(marketdf, newsdf)= env.get_training_data()
allpathes=[]
allpreds_root = []
allpreds_vowel = []
allpreds_consonant = []
tAllBegin = time.time()
for i in range(4):
test_csv = pd.read_parquet(os.path.join(root_path, f'test_image_data_{i}.parquet'))
tBegin = time.time()
dataloaders = make_loader(data_folder = test_csv,
batch_size=8,
num_workers = 2,
is_shuffle = False)
 pathes, logit1, logit2, logit3 = getmodeleval(model, dataloaders)
 preds_root = np.argmax(logit1, axis=1)
 preds_vowel = np.argmax(logit2, axis=1)
 preds_consonant = np.argmax(logit3, axis=1)
 allpathes.extend(pathes)
 allpreds_root.extend(preds_root.tolist())
 allpreds_vowel.extend(preds_vowel.tolist())
 allpreds_consonant.extend(preds_consonant.tolist())
 tEnd = time.time()
print(i, int(round(tEnd * 1000)) - int(round(tBegin * 1000)) , "ms")
 tAllEnd = time.time()
print(len(allpathes), len(allpreds_root), len(allpreds_vowel), len(allpreds_consonant), int(round(tAllEnd * 1000)) - int(round(tAllBegin * 1000)) , "ms" )<load_from_csv>
return_features = ['returnsClosePrevMktres10','returnsClosePrevRaw10','open','close' ]
add_features=['pricevolume' , 'bartrend' , 'average' , 'returnsOpenNextMktres10']
def create_lag(df_code,n_lag=[3,7,10], shift_size=1):
code = df_code['assetCode'].unique()
for col in return_features:
for window in n_lag:
rolled = df_code[col].shift(shift_size ).rolling(window=window)
 lag_mean = rolled.mean()
df_code['%s_lag_%s_mean'%(col,window)] = lag_mean
return df_code.fillna(-1)
 def generate_lag_features(df,n_lag = [3,7,10]):
features = ['time', 'assetCode', 'assetName', 'volume', 'close', 'open',
'returnsClosePrevRaw1', 'returnsOpenPrevRaw1',
'returnsClosePrevMktres1', 'returnsOpenPrevMktres1',
'returnsClosePrevRaw10', 'returnsOpenPrevRaw10',
'returnsClosePrevMktres10', 'returnsOpenPrevMktres10',
'returnsOpenNextMktres10', 'universe']
assetCodes = df['assetCode'].unique()
all_df = []
df['pricevolume'] = df['volume'] * df['close']
df['bartrend'] = df['close'] / df['open']
df['average'] =(df['close'] + df['open'])/2
df_codes = df.groupby('assetCode')
 df_codes = [df_code[1][['time','assetCode'] + add_features + return_features ] for df_code in df_codes]
pool = Pool(4)
 all_df = pool.map(create_lag, df_codes)
 new_df = pd.concat(all_df)
 new_df.drop(return_features,axis=1,inplace=True)
 pool.close()
return new_df
row_id=[]
target=[]
for idx, image_id in enumerate(allpathes):
target.extend([allpreds_consonant[idx]])
 target.extend([allpreds_root[idx]])
 target.extend([allpreds_vowel[idx]])
 row_id.extend([str(image_id)+ '_consonant_diacritic'])
 row_id.extend([str(image_id)+ '_grapheme_root'])
 row_id.extend([str(image_id)+ '_vowel_diacritic'])
 submission_df = pd.read_csv(root_path + '/sample_submission.csv')
 submission_df.target = np.hstack(np.array(target ).astype(np.int))
print(submission_df.head(10))
submission_df.to_csv('submission.csv', index=False)
 print("================end ======================" )<install_modules>
!pip install.. /input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null 
<set_options>
def prepare_data(marketdf, newsdf):
n_lag = [3,7,10]
marketdf = generate_lag_features(marketdf, n_lag = n_lag)
 marketdf['time'] = marketdf.time.dt.strftime("%Y%m%d" ).astype(int)
 newsdf['time'] = newsdf.time.dt.strftime("%Y%m%d" ).astype(int)
 newsdf['assetCode'] = newsdf['assetCodes'].map(lambda x: list(eval(x)) [0])
 newsdf['position'] = newsdf['firstMentionSentence'] / newsdf['sentenceCount']
newsdf['coverage'] = newsdf['sentimentWordCount'] / newsdf['wordCount']
droplist = ['sourceTimestamp','firstCreated','sourceId','headline','takeSequence','provider','firstMentionSentence',
'sentenceCount','bodySize','headlineTag','marketCommentary','subjects','audiences','sentimentClass',
'assetName', 'assetCodes','urgency','wordCount','sentimentWordCount']
newsdf.drop(droplist, axis=1, inplace=True)
 newsgp = newsdf.groupby(['time','assetCode'], sort=False ).aggregate(np.mean ).reset_index()
return pd.merge(marketdf, newsgp, how='left', on=['time', 'assetCode'], copy=False)
 cdf = prepare_data(marketdf, newsdf )
py.init_notebook_mode(connected=True)
 pd.set_option('max_columns', 50 )<define_variables>
print('building training set...')
 targetcols = ['returnsOpenNextMktres10']
traincols = [col for col in cdf.columns if col not in ['time', 'assetCode', 'universe'] + targetcols]
dates = cdf['time'].unique()
train = range(len(dates)) [:int(0.95*len(dates)) ]
val = range(len(dates)) [int(0.95*len(dates)) :]
cdf[targetcols[0]] =(cdf[targetcols[0]] > 0 ).astype(int)
 Xt = cdf[traincols].fillna(0 ).loc[cdf['time'].isin(dates[train])].values
Yt = cdf[targetcols].fillna(0 ).loc[cdf['time'].isin(dates[train])].values
Xv = cdf[traincols].fillna(0 ).loc[cdf['time'].isin(dates[val])].values
Yv = cdf[targetcols].fillna(0 ).loc[cdf['time'].isin(dates[val])].values
debug=False
submission=True
batch_size=256
device='cuda:0'
out='.'<define_variables>
print('Training lightgbm')
 params = {"objective" : "binary",
"metric" : "binary_logloss",
"num_leaves" : 60,
"max_depth": -1,
"learning_rate" : 0.01,
"bagging_fraction" : 0.9,
"feature_fraction" : 0.9,
"bagging_freq" : 5,
"bagging_seed" : 2333,
"verbosity" : -1 }
lgtrain, lgval = lgb.Dataset(Xt, Yt[:,0]), lgb.Dataset(Xv, Yv[:,0])
 lgbmodel = lgb.train(params, lgtrain, 2000, valid_sets=[lgtrain, lgval], early_stopping_rounds=100, verbose_eval=200)
datadir = Path('/kaggle/input/bengaliai-cv19')
 featherdir = Path('/kaggle/input/bengaliaicv19feather')
 outdir = Path('.' )<load_from_csv>
preddays = env.get_prediction_days()
<categorify><EOS> 
print("generating predictions...")
 add_features=['pricevolume' , 'bartrend' , 'average' ]
for(marketdf, newsdf, predtemplatedf)in preddays:
cdf = prepare_data(marketdf, newsdf)
 Xp = cdf[traincols].fillna(0 ).values
preds = lgbmodel.predict(Xp, num_iteration=lgbmodel.best_iteration)* 2 - 1
predsdf = pd.DataFrame({'ast':cdf['assetCode'],'conf':preds})
 predtemplatedf['confidenceValue'][predtemplatedf['assetCode'].isin(predsdf.ast)] = predsdf['conf'].values
env.predict(predtemplatedf)
 env.write_submission_file()
print('done!!!!!!!' )