Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<choose_model_class>
import pandas as pd
import numpy as np
import gc
NFOLD = 10
kf = KFold(n_splits=NFOLD )<predict_on_test>
env = twosigmanews.make_env()
cnt = 0
clf = Ridge(alpha=0.05)
 pe = np.zeros(( ze.shape[0], 2))
pred = np.zeros(( z.shape[0], 2))
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 clf.fit(z[tr_idx], y[tr_idx])
 pred[val_idx, 0] = clf.predict(z[val_idx])
 pred_std = np.mean(np.abs(y[val_idx] - pred[val_idx, 0])) * np.sqrt(2)
 pred[val_idx, 1] = pred_std
print("val", metric(y[val_idx], pred[val_idx, 0], pred[val_idx, 1]))
pe[:, 0] += clf.predict(ze)/ NFOLD
pe[:, 1] += pred_std / NFOLD
print("oof", metric(y, pred[:, 0], pred[:, 1]))<compute_test_metric>
( market_train_df, news_train_df)= env.get_training_data()
print("OOF uncertainty", np.unique(pred[:, 1]))
print("TEST uncertainty", np.unique(pe[:, 1]))<prepare_output>
print(market_train_df.isna().sum())
 print("---------------")
 print(market_train_df.shape[0] )
sub['FVC1'] = pe[:, 0]
sub['Confidence1'] = pe[:, 1]<prepare_output>
market_train_df = market_train_df[market_train_df['universe'] == 1]
market_train_df = market_train_df[market_train_df['time'].dt.year >= 2012]
market_train_df.reset_index(drop=True, inplace=True )
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()<feature_engineering>
print(market_train_df.isna().sum())
 print("---------------")
 print(market_train_df.shape[0] )
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<load_from_csv>
market_train_df['returnsClosePrevMktres1'].fillna(market_train_df['returnsClosePrevMktres1'].mean() , inplace=True)
 market_train_df['returnsOpenPrevMktres1'].fillna(market_train_df['returnsOpenPrevMktres1'].mean() , inplace=True)
 market_train_df['returnsClosePrevMktres10'].fillna(market_train_df['returnsClosePrevMktres10'].mean() , inplace=True)
 market_train_df['returnsOpenPrevMktres10'].fillna(market_train_df['returnsOpenPrevMktres10'].mean() , inplace=True)
 print(market_train_df.isna().sum())
 print("---------------")
 print(market_train_df.shape[0] )
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1<save_to_csv>
print(news_train_df.isna().sum())
 print("---------------")
 print(news_train_df.shape[0] )
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )<import_modules>
news_var = ['time','assetName', 'bodySize','companyCount','sentenceCount','wordCount',
'firstMentionSentence','relevance','sentimentClass','sentimentNegative',
'sentimentNeutral','sentimentPositive','sentimentWordCount','noveltyCount12H',
'noveltyCount24H','noveltyCount3D','noveltyCount5D','noveltyCount7D',
'volumeCounts12H','volumeCounts24H','volumeCounts3D','volumeCounts5D','volumeCounts7D'
]
news_train_df = news_train_df[news_var]
news_train_df = news_train_df[news_train_df.time.dt.year >= 2012]
news_train_df['date'] = news_train_df.time.dt.date
news_train_df.info()
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns<load_from_csv>
market_train_df['date'] = market_train_df.time.dt.date
market_train_df = pd.merge(market_train_df, news_train_df,how='left',on = ['assetName','date'])
 train_df = market_train_df
train_df.head()
df_train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 df_test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 print('Train shape: ', df_train.shape)
 print('Number of unique customers in train: {}'.format(df_train['Patient'].nunique()))
print('Test shape:', df_test.shape )<remove_duplicates>
del news_train_df
del market_train_df
gc.collect()
df_base = df_train.drop_duplicates(subset='Patient', keep='first')
 df_base = df_base[['Patient', 'Weeks', 'FVC',
'Percent', 'Age']].rename(columns={'Weeks': 'base_week',
'Percent': 'base_percent',
'Age': 'base_age',
'FVC': 'base_FVC'})
 df_base.head(3 )<groupby>
print(train_df.isna().sum() / train_df.shape[0])
 print("-------------")
 print(train_df.shape[0] )
df_train['visit'] = 1
df_train['visit'] = df_train[['Patient', 'visit']].groupby('Patient' ).cumsum()
df_train = df_train.loc[df_train['visit'] > 1, :]<merge>
train_df['bodySize'].fillna(train_df['bodySize'].mean() , inplace=True)
 train_df['companyCount'].fillna(train_df['companyCount'].mean() , inplace=True)
 train_df['sentenceCount'].fillna(train_df['sentenceCount'].mean() , inplace=True)
 train_df['wordCount'].fillna(train_df['wordCount'].mean() , inplace=True)
 train_df['firstMentionSentence'].fillna(train_df['firstMentionSentence'].mean() , inplace=True)
 train_df['relevance'].fillna(train_df['relevance'].mean() , inplace=True)
 train_df['sentimentClass'].fillna(train_df['sentimentClass'].mean() , inplace=True)
 train_df['sentimentNegative'].fillna(train_df['sentimentNegative'].mean() , inplace=True)
 train_df['sentimentNeutral'].fillna(train_df['sentimentNeutral'].mean() , inplace=True)
 train_df['sentimentPositive'].fillna(train_df['sentimentPositive'].mean() , inplace=True)
 train_df['sentimentWordCount'].fillna(train_df['sentimentWordCount'].mean() , inplace=True)
 train_df['noveltyCount12H'].fillna(train_df['noveltyCount12H'].mean() , inplace=True)
 train_df['noveltyCount24H'].fillna(train_df['noveltyCount24H'].mean() , inplace=True)
 train_df['noveltyCount3D'].fillna(train_df['noveltyCount3D'].mean() , inplace=True)
 train_df['noveltyCount5D'].fillna(train_df['noveltyCount5D'].mean() , inplace=True)
 train_df['noveltyCount7D'].fillna(train_df['noveltyCount7D'].mean() , inplace=True)
 train_df['volumeCounts12H'].fillna(train_df['volumeCounts12H'].mean() , inplace=True)
 train_df['volumeCounts24H'].fillna(train_df['volumeCounts24H'].mean() , inplace=True)
 train_df['volumeCounts3D'].fillna(train_df['volumeCounts3D'].mean() , inplace=True)
 train_df['volumeCounts5D'].fillna(train_df['volumeCounts5D'].mean() , inplace=True)
 train_df['volumeCounts7D'].fillna(train_df['volumeCounts7D'].mean() , inplace=True)
 print(train_df.isna().sum() / train_df.shape[0])
 print("-------------")
 print(train_df.shape[0] )
df_train = pd.merge(df_train,
df_base,
on='Patient',
how='left')
 print(df_train.shape)
 df_train.head(3 )<categorify>
id_train = train_df.time.dt.year != 2016
id_test = train_df.time.dt.year == 2016
dep_var = 'returnsOpenNextMktres10'
ind_var = ['volume', 'close', 'open', 'returnsClosePrevRaw1',
'returnsOpenPrevRaw1', 'returnsClosePrevMktres1',
'returnsOpenPrevMktres1', 'returnsClosePrevRaw10',
'returnsOpenPrevRaw10', 'returnsClosePrevMktres10',
'returnsOpenPrevMktres10','bodySize', 'companyCount', 'wordCount',
'firstMentionSentence', 'relevance', 'sentimentClass',
'sentimentWordCount', 'noveltyCount7D', 'volumeCounts7D']
df_train = train_df.loc[id_train,ind_var]
df_test = train_df.loc[id_test,ind_var]
print("{0} training rows and {1} testing rows".format(df_train.shape[0],df_test.shape[0]))
y_train = train_df.loc[id_train,dep_var]
y_test = train_df.loc[id_test,dep_var]
df_train['weeks_passed'] = df_train['Weeks'] - df_train['base_week']
df_train = pd.get_dummies(df_train, columns=['Sex', 'SmokingStatus'] )<data_type_conversions>
xg_reg = xgb.XGBRegressor(objective ='reg:linear', colsample_bytree = 0.3, learning_rate = 0.1,
max_depth = 5, alpha = 10, n_estimators = 100)
 xg_reg.fit(df_train,y_train )
sub['Patient'] = sub['Patient_Week'].apply(lambda x: x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: x.split('_')[1] ).astype(int)
 sub.head()<categorify>
pred_train = xg_reg.predict(df_train)
 rms_train = sqrt(mean_squared_error(y_train, pred_train))
pred_test = xg_reg.predict(df_test)
 rms_test = sqrt(mean_squared_error(y_test, pred_test))
print('Train RMSE: {0} Test RMSE: {1}'.format(rms_train,rms_test))
df_test = df_test.rename(columns={'Weeks': 'base_week',
'Percent': 'base_percent',
'Age': 'base_age',
'FVC': 'base_FVC'})
 df_test = pd.merge(sub,
df_test,
on='Patient',
how='right')
 df_test = pd.get_dummies(df_test, columns=['Sex', 'SmokingStatus'])
 df_test['weeks_passed'] = df_test['Weeks'] - df_test['base_week']
df_test.head()<feature_engineering>
pred_test_df = train_df.loc[id_test,['time','assetCode','universe','returnsOpenNextMktres10']]
pred_test_df['dayofyear'] = pred_test_df.time.dt.dayofyear
pred_test_df['confidence'] = [1 if pred >=0 else -1 for pred in pred_test]
pred_test_df['score'] = pred_test_df.universe * pred_test_df.returnsOpenNextMktres10 * pred_test_df.confidence
print(pred_test_df.confidence.value_counts())
 score_1 = pred_test_df.groupby(['dayofyear'] ).score.sum()
score_2 = score_1.mean() / score_1.std()
print("
Competition Score: ",np.round(score_2,4))
missing_columns = np.setdiff1d(df_train.drop(['Patient', 'FVC', 'Percent', 'Age', 'visit'], axis = 1 ).columns, df_test.columns)
 if len(missing_columns)> 0:
print('/!\ Missing columns in test: ', missing_columns)
 for col in missing_columns:
df_test[col] = 0<compute_test_metric>
def make_predictions(market_obs_df,news_obs_df,predictions_df,ind_var=ind_var,news_var=news_var,xg_reg=xg_reg):
news_obs_df = news_obs_df.loc[:,news_var]
news_obs_df['date'] = news_obs_df.time.dt.date
news_train_df_grp = news_obs_df.groupby(['date','assetName'] ).mean().reset_index()
market_obs_df['date'] = market_obs_df.time.dt.date
market_obs_df = pd.merge(market_obs_df,news_train_df_grp,how='left',on = ['assetName','date'])
 market_obs_df.fillna(0,inplace = True)
 test = market_obs_df.loc[:,ind_var]
predictions_df.confidenceValue = [1 if pred >=0 else -1 for pred in xg_reg.predict(test)]
def OSIC_metric(y_true, y_pred, y_pred_std):
delta = np.clip(abs(y_true - y_pred), 0, 1000)
 std_clipped = np.clip(y_pred_std, 70, np.inf)
 return np.mean(-(np.sqrt(2)*delta/std_clipped)- np.log(np.sqrt(2)*std_clipped))<choose_model_class>
days = env.get_prediction_days()
for(market_obs_df, news_obs_df, predictions_template_df)in days:
make_predictions(market_obs_df, news_obs_df, predictions_template_df)
 env.predict(predictions_template_df)
 print('Done!' )
class Model() :
def __init__(self, model=BayesianRidge() , n_splits=2):
self.regressor = model
self.n_splits = n_splits
self.gkf = GroupKFold(n_splits=n_splits)
 self.train_cols = ['Weeks', 'base_week', 'base_FVC',
'base_percent', 'base_age', 'weeks_passed', 'Sex_Female',
'Sex_Male', 'SmokingStatus_Currently smokes',
'SmokingStatus_Ex-smoker', 'SmokingStatus_Never smoked']
def fit(self, X, y):
self.regressor.fit(X, y)
 def predict(self, X):
pred = self.regressor.predict(X, return_std=True)
 return pred
def fit_predict_cv(self, df, df_test=pd.DataFrame()):
scores = np.zeros(( self.n_splits,))
oof = np.zeros(( len(df),))
oof_std = np.zeros_like(oof)
 if len(df_test)> 0:
pred_sub = np.zeros(( len(df_test), self.n_splits))
pred_sub_std = np.zeros_like(pred_sub)
 target = 'FVC'
for i,(train_idx, val_idx)in enumerate(self.gkf.split(df, groups=df['Patient'])) :
X_train = df.loc[train_idx, self.train_cols]
y_train = df.loc[train_idx, target]
X_val = df.loc[val_idx, self.train_cols]
y_val = df.loc[val_idx, target]
self.fit(X_train, y_train)
 pred_train, pred_train_std = self.predict(X_train)
 pred_val, pred_val_std = self.predict(X_val)
 if len(df_test)> 0:
pred_sub[:, i], pred_sub_std[:, i] = self.predict(df_test[self.train_cols])
 oof[val_idx] = pred_val
oof_std[val_idx] = pred_val_std
print('Train score: {0:.2f} | Test score: {1:.2f}'.format(OSIC_metric(y_train, pred_train, pred_train_std),
OSIC_metric(y_val, pred_val, pred_val_std)))
 print('OOF score: {0:.4f}'.format(OSIC_metric(df[target], oof, oof_std)))
 res = dict()
res['oof'] = oof
res['oof_std'] = oof_std
if len(df_test)> 0:
res['pred_sub'] = pred_sub.mean(axis=1)
 res['pred_sub_std'] = pred_sub_std.mean(axis=1)
 return res<train_model>
env.write_submission_file()