Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<predict_on_test>
env=twosigmanews.make_env()
back_learn.predict(test_df.loc[0, 'text'] )<predict_on_test>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import lightgbm as lgbm
import xgboost as xgb
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
predictions = []
ids = []
for i in range(len(test_df)) :
print(i)
 _, _, forward_predict = adv_learn.predict(test_df.loc[i, 'text'])
 _, _, backwards_predict = back_learn.predict(test_df.loc[i, 'text'])
 pred = bool(forward_predict[1]+backwards_predict[1] > 1)
 ids.append(test_df.loc[i, 'id'])
 predictions.append(int(pred))<save_to_csv>
market_data,news_data=env.get_training_data()
print(market_data.shape,news_data.shape)
 market_data['universe']=market_data['universe'].astype(int)
 display(market_data.head())
 display(news_data.head() )
submission = pd.DataFrame({'id': ids, 'target':predictions})
 print(submission)
 submission.to_csv("ensemble_submission.csv", index=False )<load_from_csv>
about=pd.DataFrame(index=market_data.columns)
 about['types']=market_data.dtypes.values
about['unique_values']=market_data.nunique().values
about['missing_values']=market_data.isnull().sum().values
about['missing_values_percentage']=about['missing_values']/market_data.shape[0]
about
train=pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test=pd.read_csv('/kaggle/input/nlp-getting-started/test.csv' )<prepare_output>
print("Training data starts at",market_data['time'].dt.date.min() ,
"and it ends at",market_data['time'].dt.date.max() )
occurences['Percentage(%)']=(occurences['Number of Occurences']/occurences['Number of Occurences'].sum())*100
occurences.set_index('Class' )<prepare_x_and_y>
market_data=market_data.loc[market_data['time'].dt.date>=pd.datetime(2009,1,1 ).date() ]
news_data=news_data.loc[news_data['time'].dt.date>=pd.datetime(2009,1,1 ).date() ]
print(market_data.shape,news_data.shape )
traindata = list(np.array(train.iloc[:,3]))
testdata = list(np.array(test.iloc[:,3]))
y = np.array(train.iloc[:,4] ).astype(int)
 X_all = traindata + testdata
lentrain = len(traindata )<feature_engineering>
gblabel_encoder_asset_code={name:idx for idx,name in enumerate(market_data['assetCode'].unique())}
gblabel_encoder_asset_name={name:idx for idx,name in enumerate(market_data['assetName'].unique())}
tfidf = TfidfVectorizer(min_df=3, max_features=None, strip_accents='unicode',
analyzer='word',token_pattern=r'\w{1,}',ngram_range=(1, 2), use_idf=1,smooth_idf=1,sublinear_tf=1)
 print("Implementing TFIDF to both the test and train data")
 tfidf.fit(X_all)
 print("Transforming the data")
 X_all = tfidf.transform(X_all )<train_model>
def make_the_dataframe(df,assetcode_encoder,assetname_encoder,is_train):
unique_values_asset_code=df['assetCode'].unique()
for name in unique_values_asset_code:
if name not in assetcode_encoder.keys() :
assetcode_encoder[name]=max(assetcode_encoder.values())+1
df['assetCode_encoding']=df['assetCode'].map(assetcode_encoder)
 unique_values_asset_name=df['assetName'].unique()
for name in unique_values_asset_name:
if name not in assetname_encoder.keys() :
assetname_encoder[name]=max(assetname_encoder.values())+1
df['assetName_encoding']=df['assetName'].map(assetname_encoder)
 df['weekofyear']=df['time'].dt.weekofyear
df['dayofweek']=df['time'].dt.dayofweek
df['close_to_open_ratio']=df['close']/df['open']
if is_train:
df['label']=(df['returnsOpenNextMktres10']>0 ).astype(int)
 df['time']=df['time'].dt.date
necessary_df=df[['time','assetCode','assetName','returnsOpenNextMktres10','universe']].copy()
df.drop(columns=['time','assetName','assetCode','returnsOpenNextMktres10','universe'],inplace=True)
 return df,necessary_df
else:
df.drop(columns=['time','assetName','assetCode'],inplace=True)
 return df
def make_the_numpy_array(market_data_train,market_data_val):
X_train=market_data_train.iloc[:,:-1].values
X_val=market_data_val.iloc[:,:-1].values
y_train=market_data_train.iloc[:,-1].values
y_val=market_data_val.iloc[:,-1].values
return X_train,X_val,y_train,y_val
def competitoin_metric(extract,probs):
extract['confidence']=2*probs-1
extract['score']=extract['universe']*extract['returnsOpenNextMktres10']*extract['confidence']
x=extract.groupby('time')['score'].sum().values
return np.mean(x)/np.std(x )
X = X_all[:lentrain]
X_test = X_all[lentrain:]
log = LogisticRegression(penalty='l2',dual=False, tol=0.0001,
C=1, fit_intercept=True, intercept_scaling=1.0,
class_weight=None, random_state=None)
 print("Training on the train data")
 log.fit(X,y)
 y_pred_X=log.predict(X)
 print('Training accuracy is {}'.format(accuracy_score(y, y_pred_X)))
 predictions = log.predict(X_test)
 predictions<save_to_csv>
splits=[2012,2013,2014,2015]
models=[]
for idx,year in enumerate(splits):
print("Year = ",year)
 yr,mo,dte=year,1,1
print("Splitting the data with respect to time")
 market_data_val=market_data.loc[(market_data['time'].dt.date>=pd.datetime(yr,mo+2,dte ).date())]
market_data_train=market_data.loc[market_data['time'].dt.date<pd.datetime(yr,mo,dte ).date() ]
market_data_train,extract_train=make_the_dataframe(market_data_train.copy() ,gblabel_encoder_asset_code.copy() ,
gblabel_encoder_asset_name.copy() ,is_train=True)
 market_data_val,extract_val=make_the_dataframe(market_data_val.copy() ,gblabel_encoder_asset_code.copy() ,
gblabel_encoder_asset_name.copy() ,is_train=True)
 X_train,X_val,y_train,y_val=make_the_numpy_array(market_data_train,market_data_val)
 print("The training data shape is",X_train.shape,y_train.shape)
 print("The validation data shape is",X_val.shape,y_val.shape)
 gc.enable()
del market_data_train,market_data_val
gc.collect()
clf=xgb.XGBClassifier(n_jobs=4,max_depth=7)
 clf.fit(X_train,y_train,eval_set=[(X_train,y_train),(X_val,y_val)],eval_metric='logloss',
early_stopping_rounds=25,verbose=10)
 print("The Training Competition metric is:",competitoin_metric(extract_train,
clf.predict_proba(X_train)[:,1]))
print("The validation competition metric is:",competitoin_metric(extract_val,
clf.predict_proba(X_val)[:,1]))
models.append(clf)
 gc.enable()
del X_train,X_val,y_train,y_val
gc.collect()
test_ids=test['id']
submission = pd.DataFrame(predictions,index=test_ids,columns=['target'])
 submission.to_csv('submission_nlprnot.csv')
 print("submission file created.. " )<import_modules>
print(pd.__version__)


 print(os.listdir(".. /input")) 
<install_modules>
def make_the_predictions(models,test_market_data,test_news_data,predictions_template_df):
test=make_the_dataframe(test_market_data,gblabel_encoder_asset_code.copy() ,
gblabel_encoder_asset_name.copy() ,is_train=False)
 X_test=test.iloc[:,:].values
gc.enable()
del test
gc.collect()
predictions_template_df['confidenceValue']=0
for clf in models:
confidences=2*clf.predict_proba(X_test)[:,1]-1
predictions_template_df['confidenceValue']=predictions_template_df['confidenceValue']+confidences
predictions_template_df['confidenceValue']=predictions_template_df['confidenceValue']/len(models)
 return predictions_template_df
!pip install bert-tensorflow 
<load_from_url>
for day,(market_obs_df, news_obs_df, predictions_template_df)in enumerate(env.get_prediction_days()):
if(day==0):
print("The columns of market are",market_obs_df.columns)
 predictions=make_the_predictions(models,market_obs_df, news_obs_df, predictions_template_df)
 env.predict(predictions)
 if(( day+1)%20==0):
print("Predictions for",day+1,"days done" )
!wget https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip 
<load_pretrained>
env.write_submission_file()