Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<train_model>
stop = set(stopwords.words('english'))
lm_learn = lm_learn.load('1epoch')
 lm_learn.unfreeze()
lm_learn.fit_one_cycle(10, 1e-3)
 lm_learn.save_encoder('finetuned' )<load_from_csv>
env = twosigmanews.make_env() 
(market_train_df, news_train_df)= env.get_training_data()
dls_clas = TextDataLoaders.from_csv(path=path,
csv_fname='train.csv',
text_col='text',
label_col='target',
valid_pct=0.2,
text_vocab=dls_lm.vocab )<load_pretrained>
print(market_train_df.time.min())
 market_train_df_5years=market_train_df[market_train_df.time>'2011-12-30 22:00:00+0000']
adv_learn = text_classifier_learner(dls_clas, AWD_LSTM, drop_mult=0.5, metrics=F1Score())
 adv_learn.model_dir = '/kaggle/working/'
adv_learn = adv_learn.load_encoder('finetuned')
 adv_learn.fit_one_cycle(1, 2e-2 )<train_model>
market_train_df_5years.to_csv("market_2011_2016.csv",index=False )
adv_learn.freeze_to(-2)
 adv_learn.fit_one_cycle(1, slice(1e-2/(2.6**4),1e-2))<train_model>
%matplotlib inlinemarket_train_df.info()
py.init_notebook_mode(connected=True)
adv_learn.freeze_to(-3)
 adv_learn.fit_one_cycle(1, slice(5e-3/(2.6**4),5e-3))<train_model>
missing_cols=['returnsClosePrevMktres1',
'returnsOpenPrevMktres1',
'returnsClosePrevMktres10',
'returnsOpenPrevMktres10']
for col in missing_cols:
market_train_df[col]=market_train_df[col].fillna(market_train_df[col].mean() )
adv_learn.unfreeze()
adv_learn.fit_one_cycle(2, slice(1e-3/(2.6**4),1e-3))<filter>
import datetimeprint(f'The market data start from {market_train_df["time"].min() }, and end on {market_train_df["time"].max() }' )
test_df.loc[0, 'text']<predict_on_test>
return_terms=[
"returnsClosePrevRaw1",
"returnsOpenPrevRaw1",
"returnsClosePrevMktres1",
"returnsOpenPrevMktres1",
"returnsClosePrevRaw10",
"returnsOpenPrevRaw10",
"returnsClosePrevMktres10",
"returnsOpenPrevMktres10",
"returnsOpenNextMktres10"
]
for return_term in return_terms:
market_train_df[return_term]=market_train_df[return_term].clip(-1,1)
adv_learn.predict(test_df.loc[0, 'text'] )<predict_on_test>
print(f'The news data start from {news_train_df["time"].min() }, and end on {news_train_df["time"].max() }')
predictions = []
ids = []
for i in range(len(test_df)) :
print(i)
 pred, _, _ = adv_learn.predict(test_df.loc[i, 'text'])
 ids.append(test_df.loc[i, 'id'])
 predictions.append(pred )<save_to_csv>
def expand(df, expand_column):
lens = []
d = {}
expands_list=[]
for i,item in df[expand_column].items() :
expand=list(item[2:-2].split("', '"))
lens.append(len(expand))
expands_list.extend(expand)
 d[expand_column] = expands_list
for col in df.columns.values:
if col != expand_column:
d[col] = np.repeat(df[col].values, lens)
 return pd.DataFrame(d)
submission = pd.DataFrame({'id': ids, 'target':predictions})
 print(submission)
 submission.to_csv("complex_submission.csv", index=False )<load_pretrained>
def prep_market(market_df):
market_df=market_df.drop(['assetName'],axis=1)
 market_df["time"]=pd.to_datetime(market_df["time"])
 market_df['time'] = market_df.time.dt.date
market_df['close_to_open'] = market_df['close'] / market_df['open']
return market_df
def prep_news(news_df):
news_df['firstCreated']=pd.to_datetime(news_df['firstCreated'])
 news_df=news_df.drop(news_drop_col,axis=1)
 news_df['headlineTagT'] = news_df['headlineTag'].map(headlineTag_dic)
 news_df['provider'] = news_df['provider'].map(provider_dic)
 news_df['marketCommentary'] = news_df['marketCommentary'].map(marketCommentary_map)
 news_df=expand(news_df, "assetCodes")
 return news_df
def group_news(news_df):
news_df['firstCreated']=news_df['firstCreated'].dt.date
news_df = news_df.groupby(['firstCreated', 'assetCodes'], as_index=False ).mean()
return news_df
def merge_market_news(market_df,news_df):
market_and_news_df = pd.merge(market_df, news_df, left_on=['time', 'assetCode'], right_on=['firstCreated', 'assetCodes'])
 return market_and_news_df
dls_lm_back = TextDataLoaders.from_df(df, text_col='text', is_lm=True, valid_pct=0, backward = True)
 dls_lm_back.show_batch(max_n=3 )<load_pretrained>
market_train_df=prep_market(market_train_df)
 market_news_train_df = merge_market_news(market_train_df, news_train_df)
lm_learn_back = language_model_learner(dls_lm_back, AWD_LSTM, metrics=[accuracy, Perplexity() ], path=path, wd=0.1)
 lm_learn_back.model_dir = '/kaggle/working/'<train_model>
features_col=['volume', 'close', 'open', 'returnsClosePrevRaw1','returnsOpenPrevRaw1',
'returnsClosePrevMktres1','returnsOpenPrevMktres1', 'returnsClosePrevRaw10',
'returnsOpenPrevRaw10', 'returnsClosePrevMktres10','returnsOpenPrevMktres10',
'close_to_open', 'urgency', 'takeSequence', 'provider',
'bodySize', 'companyCount', 'marketCommentary', 'sentenceCount','wordCount',
'firstMentionSentence', 'relevance', 'sentimentClass','sentimentNegative',
'sentimentNeutral', 'sentimentPositive','sentimentWordCount', 'noveltyCount12H',
'noveltyCount24H','noveltyCount3D', 'noveltyCount5D', 'noveltyCount7D', 'volumeCounts12H',
'volumeCounts24H', 'volumeCounts3D', 'volumeCounts5D', 'volumeCounts7D','headlineTagT']
features_col_market=['volume', 'close', 'open', 'returnsClosePrevRaw1','returnsOpenPrevRaw1',
'returnsClosePrevMktres1','returnsOpenPrevMktres1', 'returnsClosePrevRaw10',
'returnsOpenPrevRaw10', 'returnsClosePrevMktres10','returnsOpenPrevMktres10', 'close_to_open']
lm_learn_back.fit_one_cycle(1, 1e-2)
 lm_learn_back.save('1epoch_back' )<categorify>
UpOrDown = market_news_train_df.returnsOpenNextMktres10 >= 0
UpOrDown = UpOrDown.values
returns = market_news_train_df.returnsOpenNextMktres10.values
X=market_news_train_df[features_col].values
mins_X = np.min(X, axis=0)
 maxs_X = np.max(X, axis=0)
 range_X = maxs_X - mins_X
X = 1 -(( maxs_X - X)/ range_X)
lm_learn_back = lm_learn_back.load('1epoch_back')
 lm_learn_back.unfreeze()
lm_learn_back.fit_one_cycle(10, 1e-3)
 lm_learn_back.save_encoder('finetuned_back' )<load_from_csv>
UpOrDown_market = market_train_df.returnsOpenNextMktres10 >= 0
UpOrDown_market = UpOrDown_market.values
returns_market = market_train_df.returnsOpenNextMktres10.values
X_market=market_train_df[features_col_market].values
mins_X_market = np.min(X_market, axis=0)
 maxs_X_market = np.max(X_market, axis=0)
 range_X_market = maxs_X_market - mins_X_market
X_market = 1 -(( maxs_X_market - X_market)/ range_X_market)
dls_clas_back = TextDataLoaders.from_csv(path=path,
csv_fname='train.csv',
text_col='text',
label_col='target',
valid_pct=0.2,
text_vocab=dls_lm_back.vocab,
backwards = True )<load_pretrained>
X_train, X_test, UpOrDown_train, UpOrDown_test, returns_train, returns_test = model_selection.train_test_split(X, UpOrDown, returns, test_size=0.1, random_state=99 )
back_learn = text_classifier_learner(dls_clas_back, AWD_LSTM, drop_mult=0.5, metrics=F1Score())
 back_learn.model_dir = '/kaggle/working/'
back_learn = back_learn.load_encoder('finetuned_back')
 back_learn.fit_one_cycle(1, 2e-2 )<train_model>
params = {'learning_rate': 0.05, 'max_depth': 5, 'boosting': 'gbdt', 'objective': 'binary', 'metric': 'auc', 'is_training_metric': True, 'seed': 42}
model = lgb.train(params, train_set=lgb.Dataset(X_train, label=UpOrDown_train), num_boost_round=2000,
valid_sets=[lgb.Dataset(X_train, label=UpOrDown_train), lgb.Dataset(X_test, label=UpOrDown_test)],
verbose_eval=50, early_stopping_rounds=30)
back_learn.freeze_to(-2)
 back_learn.fit_one_cycle(1, slice(1e-2/(2.6**4),1e-2))<train_model>
X_train_market, X_test_market, UpOrDown_train_market, UpOrDown_test_market, returns_train_market, returns_test_market = model_selection.train_test_split(X_market, UpOrDown_market, returns_market, test_size=0.1, random_state=99)
 params = {'learning_rate': 0.05, 'max_depth': 5, 'boosting': 'gbdt', 'objective': 'binary', 'metric': 'auc', 'is_training_metric': True, 'seed': 42}
model_market = lgb.train(params, train_set=lgb.Dataset(X_train_market, label=UpOrDown_train_market), num_boost_round=2000,
valid_sets=[lgb.Dataset(X_train_market, label=UpOrDown_train_market), lgb.Dataset(X_test_market, label=UpOrDown_test_market)],
verbose_eval=50, early_stopping_rounds=30 )
<train_model><EOS> 
days = env.get_prediction_days()
n_days = 0
for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
if n_days % 50 == 0:
print(n_days,end=' ')
 market_obs_df = market_obs_df[market_obs_df.assetCode.isin(predictions_template_df.assetCode)]
market_obs_df=prep_market(market_obs_df)
 news_obs_df=prep_news(news_obs_df)
 news_obs_df=group_news(news_obs_df)
 market_news_obs_df = merge_market_news(market_obs_df, news_obs_df)
 assetcode_set=set(market_news_obs_df['assetCode'].values)
 X_live = market_news_obs_df[features_col].values
X_live = 1 -(( maxs_X - X_live)/ range_X)
 lp = model.predict(X_live)
 confidence = 2 * lp -1
preds = pd.DataFrame({'assetCode':market_news_obs_df['assetCode'],'confidence':confidence})
 market_only_obs_df=market_obs_df[~market_obs_df['assetCode'].isin(assetcode_set)]
X_live_market_only=market_only_obs_df[features_col_market].values
X_live_market_only=1-(( maxs_X_market - X_live_market_only)/ range_X_market)
 lp_market_only = model_market.predict(X_live_market_only)
 confidence_market_only = 2 * lp_market_only -1
preds_market_only= pd.DataFrame({'assetCode':market_only_obs_df['assetCode'],'confidence':confidence_market_only})
 preds_all=pd.concat([preds,preds_market_only],sort='False')
 predictions_template_df = predictions_template_df.merge(preds_all,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 env.write_submission_file()