Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<define_variables>
env = twosigmanews.make_env()
SEED = [0,1,2,3,4,5,6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions<compute_train_metric>
( market_train_df, news_train_df)= env.get_training_data()
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
cv_score = 0
roc_score = 0
for i in range(len(target_cols)) :
cv_score += log_loss(y_true[:, i], y_pred[:, i])
 roc_score +=roc_auc_score(y_true[:, i], y_pred[:, i], average='micro')
 print("CV log_loss: ", cv_score / y_pred.shape[1])
 print("Overall AUC: ", roc_score / y_pred.shape[1] )<save_to_csv>
print(" ---------------- Before removing missing values")
 print(market_train_df.isna().sum())
 market_train_df.dropna(inplace= True)
 print("
---------------- After removing missing values ")
 print(market_train_df.isna().sum() )
oof_pretrain= valid_results[target_cols]
oof_pretrain.to_csv('off_pretrain_last.csv', index=False )<save_to_csv>
market_train_df_1 = market_train_df[market_train_df.time.dt.year >= 2013]
public_id = list(df['sig_id'].values)
 test_id = list(test_features['sig_id'].values)
 private_id = list(set(test_id)-set(public_id))
df_submit = pd.DataFrame(index = public_id+private_id, columns=target_cols)
 df_submit.index.name = 'sig_id'
df_submit[:] = 0
df_submit.loc[test.sig_id,:] = test[target_cols].values
df_submit.loc[test_features[test_features.cp_type=='ctl_vehicle'].sig_id]= 0
df_submit.to_csv('submission.csv',index=True )<set_options>
print("No.of observations(2013 to 2016): ", market_train_df_1.shape[0])
 print(market_train_df_1.time.dt.year.value_counts() )
warnings.filterwarnings('ignore' )<install_modules>
del market_train_df
gc.collect()
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/ 
<load_from_csv>
news_train_df_1 = news_train_df[news_train_df.time.dt.year >= 2013].copy()
train_features = pd.read_csv(data_dir + 'train_features.csv')
 train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(data_dir + 'train_targets_nonscored.csv')
 train_drug = pd.read_csv(data_dir + 'train_drug.csv')
 test_features = pd.read_csv(data_dir + 'test_features.csv')
 sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 print('train_features: {}'.format(train_features.shape))
print('train_targets_scored: {}'.format(train_targets_scored.shape))
print('train_targets_nonscored: {}'.format(train_targets_nonscored.shape))
print('train_drug: {}'.format(train_drug.shape))
print('test_features: {}'.format(test_features.shape))
print('sample_submission: {}'.format(sample_submission.shape))<create_dataframe>
del news_train_df
gc.collect()
train_features2=train_features.copy()
test_features2=test_features.copy()<define_variables>
news_train_df_1.time.dt.year.value_counts()
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<create_dataframe>
news_var = ['time','assetName', 'bodySize','companyCount','sentenceCount','wordCount',
'firstMentionSentence','relevance','sentimentClass','sentimentNegative',
'sentimentNeutral','sentimentPositive','sentimentWordCount','noveltyCount12H',
'noveltyCount24H','noveltyCount3D','noveltyCount5D','noveltyCount7D',
'volumeCounts12H','volumeCounts24H','volumeCounts3D','volumeCounts5D','volumeCounts7D'
]
qt = QuantileTransformer(n_quantiles=100,random_state=42,output_distribution='normal')
 data = pd.concat([pd.DataFrame(train_features[GENES+CELLS]), pd.DataFrame(test_features[GENES+CELLS])])
 data2 = qt.fit_transform(data[GENES+CELLS])
 train_features[GENES+CELLS] = pd.DataFrame(data2[:train_features.shape[0]])
 test_features[GENES+CELLS] = pd.DataFrame(data2[-test_features.shape[0]:] )<set_options>
news_train_df_1 = news_train_df_1[news_var]
news_train_df_1['date'] = news_train_df_1.time.dt.date
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42 )<create_dataframe>
news_train_df_grp = news_train_df_1.groupby(['date','assetName'] ).mean().reset_index()
n_comp = 600
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]];
test2 = data2[-test_features.shape[0]:]
train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gpca), axis=1)
 test_features = pd.concat(( test_features, test_gpca), axis=1 )<create_dataframe>
market_train_df_1['date'] = market_train_df_1.time.dt.date
market_train_df_1 = pd.merge(market_train_df_1,news_train_df_grp,how='left',on = ['assetName','date'])
 market_train_df_1.head()
n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_cpca), axis=1)
 test_features = pd.concat(( test_features, test_cpca), axis=1 )<create_dataframe>
market_train_df_1.isna().sum() /market_train_df_1.shape[0]
var_thresh = VarianceThreshold(0.85)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
<concatenate>
del news_train_df_grp
gc.collect()
def fe_cluster(train, test, n_clusters_g = 22, n_clusters_c = 4, SEED = 42):
features_g = GENES
features_c = CELLS
def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_{kind}'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_{kind}'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features2 ,test_features2=fe_cluster(train_features2,test_features2 )<concatenate>
market_train_df_1 = market_train_df_1[market_train_df_1.universe == 1].copy()
train_pca=pd.concat(( train_gpca,train_cpca),axis=1)
 test_pca=pd.concat(( test_gpca,test_cpca),axis=1 )<categorify>
id_train = market_train_df_1.time.dt.year != 2016
id_test = market_train_df_1.time.dt.year == 2016
dep_var = 'returnsOpenNextMktres10'
ind_var = ['volume', 'close', 'open', 'returnsClosePrevRaw1',
'returnsOpenPrevRaw1', 'returnsClosePrevMktres1',
'returnsOpenPrevMktres1', 'returnsClosePrevRaw10',
'returnsOpenPrevRaw10', 'returnsClosePrevMktres10',
'returnsOpenPrevMktres10','bodySize', 'companyCount', 'sentenceCount', 'wordCount',
'firstMentionSentence', 'relevance', 'sentimentClass',
'sentimentNegative', 'sentimentNeutral', 'sentimentPositive',
'sentimentWordCount', 'noveltyCount12H', 'noveltyCount24H',
'noveltyCount3D', 'noveltyCount5D', 'noveltyCount7D', 'volumeCounts12H',
'volumeCounts24H', 'volumeCounts3D', 'volumeCounts5D',
'volumeCounts7D']
df_train = market_train_df_1.loc[id_train,ind_var]
df_test = market_train_df_1.loc[id_test,ind_var]
print("{0} training rows and {1} testing rows".format(df_train.shape[0],df_test.shape[0]))
y_train = market_train_df_1.loc[id_train,dep_var]
y_test = market_train_df_1.loc[id_test,dep_var]
def fe_cluster_pca(train, test,n_clusters=5,SEED = 42):
data=pd.concat([train,test],axis=0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_pca'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_pca'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_pca'])
 test = pd.get_dummies(test, columns = [f'clusters_pca'])
 return train, test
train_cluster_pca ,test_cluster_pca = fe_cluster_pca(train_pca,test_pca )<split>
xg_reg = xgb.XGBRegressor(objective ='reg:linear', colsample_bytree = 0.3, learning_rate = 0.1,
max_depth = 5, alpha = 10, n_estimators = 100)
 xg_reg.fit(df_train,y_train )
train_cluster_pca = train_cluster_pca.iloc[:,650:]
test_cluster_pca = test_cluster_pca.iloc[:,650:]<split>
pred_train = xg_reg.predict(df_train)
 rms_train = sqrt(mean_squared_error(y_train, pred_train))
pred_test = xg_reg.predict(df_test)
 rms_test = sqrt(mean_squared_error(y_test, pred_test))
print('Train RMSE: {0} Test RMSE: {1}'.format(rms_train,rms_test))
train_features_cluster=train_features2.iloc[:,876:]
test_features_cluster=test_features2.iloc[:,876:]<define_variables>
pred_test_df = market_train_df_1.loc[id_test,['time','assetCode','universe','returnsOpenNextMktres10']]
pred_test_df['dayofyear'] = pred_test_df.time.dt.dayofyear
pred_test_df['confidence'] = [1 if pred >=0 else -1 for pred in pred_test]
pred_test_df['score'] = pred_test_df.universe * pred_test_df.returnsOpenNextMktres10 * pred_test_df.confidence
print(pred_test_df.confidence.value_counts())
 score_1 = pred_test_df.groupby(['dayofyear'] ).score.sum()
score_2 = score_1.mean() / score_1.std()
print("
Competition Score: ",np.round(score_2,4))
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']<feature_engineering>
days = env.get_prediction_days()
def fe_stats(train, test):
features_g = GENES
features_c = CELLS
for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
train_features2,test_features2=fe_stats(train_features2,test_features2 )<split>
def make_predictions(market_obs_df,news_obs_df,predictions_df,ind_var,news_var,xg_reg):
news_obs_df = news_obs_df.loc[:,news_var]
news_obs_df['date'] = news_obs_df.time.dt.date
news_train_df_grp = news_obs_df.groupby(['date','assetName'] ).mean().reset_index()
market_obs_df['date'] = market_obs_df.time.dt.date
market_obs_df = pd.merge(market_obs_df,news_train_df_grp,how='left',on = ['assetName','date'])
 market_obs_df.fillna(0,inplace = True)
 test = market_obs_df.loc[:,ind_var]
predictions_df.confidenceValue = [1 if pred >=0 else -1 for pred in xg_reg.predict(test)]
train_features_stats=train_features2.iloc[:,902:]
test_features_stats=test_features2.iloc[:,902:] 
<concatenate>
for(market_obs_df, news_obs_df, predictions_template_df)in days:
make_predictions(market_obs_df,news_obs_df,predictions_template_df,ind_var,news_var,xg_reg)
 env.predict(predictions_template_df)
 print('Done!' )
<merge><EOS> 
env.write_submission_file()