Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<create_dataframe>
env = twosigmanews.make_env() 
(market_train, news_train)= env.get_training_data()
predictions_clip = predictions
predictions_clip = pd.DataFrame(predictions_clip, columns=target_cols )<load_pretrained>
def data_prep(market_train,news_train):
market_train.time = market_train.time.dt.date
news_train.time = news_train.time.dt.hour
news_train.sourceTimestamp= news_train.sourceTimestamp.dt.hour
news_train.firstCreated = news_train.firstCreated.dt.date
news_train['assetCodesLen'] = news_train['assetCodes'].map(lambda x: len(eval(x)))
 news_train['assetCodes'] = news_train['assetCodes'].map(lambda x: list(eval(x)) [0])
 kcol = ['firstCreated', 'assetCodes']
news_train = news_train.groupby(kcol, as_index=False ).mean()
market_train = pd.merge(market_train, news_train, how='left', left_on=['time', 'assetCode'],
right_on=['firstCreated', 'assetCodes'])
 lbl = {k: v for v, k in enumerate(market_train['assetCode'].unique())}
market_train['assetCodeT'] = market_train['assetCode'].map(lbl)
 market_train = market_train.dropna(axis=0)
 return market_train
oof_multi_heads_all = np.load('.. /input/jp-multiheads/multi_heads_oof.npy' )<save_to_csv>
market_train = data_prep(market_train,news_train )
test = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sig_id = test[test['cp_type']!='ctl_vehicle'].sig_id.reset_index(drop=True)
 predictions_clip['sig_id'] = sig_id
sub_mh = pd.merge(test[['sig_id']], predictions_clip, on='sig_id', how='left')
 sub_mh.fillna(0, inplace=True)
 sub_mh.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub_mh.to_csv('submission_mh.csv', index=False )<create_dataframe>
up = market_train.returnsOpenNextMktres10 >= 0
%who DataFrame
del folds, predictions_clip,targets, test,gen_cols, pca_gen_cols, ohe_cols, cell_cols, pca_cell_cols
gc.collect()
%who DataFrame<compute_train_metric>
fcol = [c for c in market_train if c not in ['assetCode', 'assetCodes', 'assetCodesLen', 'assetName', 'audiences',
'firstCreated', 'headline', 'headlineTag', 'marketCommentary', 'provider',
'returnsOpenNextMktres10', 'sourceId', 'subjects', 'time', 'time_x', 'universe','sourceTimestamp']]
def log_loss_numpy(y_pred):
y_true_ravel = np.asarray(y_true ).ravel()
y_pred = np.asarray(y_pred ).ravel()
y_pred = np.clip(y_pred, 1e-15, 1 - 1e-15)
 loss = np.where(y_true_ravel == 1, - np.log(y_pred), - np.log(1 - y_pred))
return loss.mean()
def func_numpy_metric(weights):
oof_blend = np.tensordot(weights, oof, axes =(( 0),(0)))
 return log_loss_numpy(oof_blend)
 def grad_func(weights):
oof_clip = np.clip(oof, 1e-15, 1 - 1e-15)
 gradients = np.zeros(oof.shape[0])
 for i in range(oof.shape[0]):
a, b, c = y_true, oof_clip[i], np.zeros(( oof.shape[1], oof.shape[2]))
for j in range(oof.shape[0]):
if j != i:
c += weights[j] * oof_clip[j]
gradients[i] = -np.mean(( -a*b+(b**2)*weights[i]+b*c)/(( b**2)*(weights[i]**2)+2*b*c*weights[i]-b*weights[i]+(c**2)-c))
return gradients
@njit
def grad_func_jit(weights):
oof_clip = np.minimum(1 - 1e-15, np.maximum(oof, 1e-15))
gradients = np.zeros(oof.shape[0])
 for i in range(oof.shape[0]):
a, b, c = y_true, oof_clip[i], np.zeros(( oof.shape[1], oof.shape[2]))
for j in range(oof.shape[0]):
if j != i:
c += weights[j] * oof_clip[j]
gradients[i] = -np.mean(( -a*b+(b**2)*weights[i]+b*c)/(( b**2)*(weights[i]**2)+2*b*c*weights[i]-b*weights[i]+(c**2)-c))
return gradients<load_from_csv>
X = market_train[fcol].values
up = up.values
r = market_train.returnsOpenNextMktres10.values
y_true = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv', index_col = 'sig_id' ).values
oof_dict = {'NN-Transfer-Learning': '.. /input/nn-transferlearning-7seeds4248-10folds/oof_nn_transfer_all.npy',
'TabNet': '.. /input/tabnetfe7seeds424810folds/oof_TabNet_all.npy',
'ResNet': '.. /input/resent-1841/resnet_oof.npy',
'Multi-Heads': '.. /input/jp-multiheads/multi_heads_oof.npy'
}
sub = {}
sub['NN-Transfer-Learning']= sub_nn_transfer
sub['TabNet'] = submission_TabNet
sub['ResNet'] = sub_rs
sub['Multi-Heads'] = sub_mh
oof = []
sub_all = []
oof_tmp = np.zeros([y_true.shape[0], y_true.shape[1]])
 for i in oof_dict:
sub_tmp = sub[i].iloc[:,1:].values
if sub_tmp.sum() > 0.0000000001:
oof.append(np.load(oof_dict[i]))
sub_all.append(sub_tmp)
 oof = np.asarray(oof)
 sub_all = np.asarray(sub_all)
<compute_test_metric>
mins = np.min(X, axis=0)
 maxs = np.max(X, axis=0)
 rng = maxs - mins
X = 1 -(( maxs - X)/ rng )
%%time
log_loss_scores = {}
for n, key in enumerate(oof_dict.keys()):
score_oof = log_loss_numpy(oof[n])
 log_loss_scores[key] = score_oof
print(f'{key} CV:\t', score_oof)
 print('-' * 50 )<define_variables>
X_train, X_test, up_train, up_test, r_train, r_test\
= model_selection.train_test_split(X, up, r, test_size=0.25, random_state=99 )
test_weights = np.array([1 / oof.shape[0]] * oof.shape[0] )<train_on_grid>
from xgboost import XGBClassifier
import time
tol = 1e-10
init_guess = [1 / oof.shape[0]] * oof.shape[0]
bnds = [(0, 1)for _ in range(oof.shape[0])]
cons = {'type': 'eq',
'fun': lambda x: np.sum(x)- 1,
'jac': lambda x: [1] * len(x)}
print('Inital Blend OOF:', func_numpy_metric(init_guess))
start_time = time()
res_scipy = minimize(fun = func_numpy_metric,
x0 = init_guess,
method = 'SLSQP',
jac = grad_func_jit,
bounds = bnds,
constraints = cons,
tol = tol)
 print(f'[{str(datetime.timedelta(seconds = time() - start_time)) [2:7]}] Optimised Blend OOF:', res_scipy.fun)
 print('Optimised Weights:', res_scipy.x )<normalization>
xgb_up = XGBClassifier(n_jobs=4,n_estimators=250,max_depth=8,eta=0.1 )
print('Check the sum of all weights:', np.sum(res_scipy.x))
if np.sum(res_scipy.x)- 1 <= tol:
print('Great! The sum of all weights equals to 1!')
 else:
print('Manual adjustion is needed to modify the weights.' )<load_from_csv>
t = time.time()
print('Fitting Up')
 xgb_up.fit(X_train,up_train)
 print(f'Done, time = {time.time() - t}' )
sub = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 tmp_sub = np.zeros([sub.shape[0], sub.shape[1]-1])
 for i in range(len(sub_all)) :
tmp_sub += res_scipy.x[i]*sub_all[i]
sub.iloc[:,1:] = np.clip(tmp_sub, 0.00005,0.99995)
 sub.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub.to_csv('submission.csv', index=False )<load_from_csv>
accuracy_score(xgb_up.predict(X_test),up_test )
!rm submission_TabNet.csv<set_options>
days = env.get_prediction_days()
warnings.filterwarnings('ignore' )<import_modules>
n_days = 0
prep_time = 0
prediction_time = 0
packaging_time = 0
for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
print(n_days,end=' ')
 t = time.time()
market_obs_df = data_prep(market_obs_df, news_obs_df)
 market_obs_df = market_obs_df[market_obs_df.assetCode.isin(predictions_template_df.assetCode)]
X_live = market_obs_df[fcol].values
X_live = 1 -(( maxs - X_live)/ rng)
 prep_time += time.time() - t
t = time.time()
lp = xgb_up.predict_proba(X_live)
 prediction_time += time.time() -t
t = time.time()
confidence = 2* lp[:,1] -1
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence':confidence})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 packaging_time += time.time() - t
from sklearn.preprocessing import QuantileTransformer<load_from_csv>
env.write_submission_file()
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 df = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<create_dataframe>
total = prep_time + prediction_time + packaging_time
print(f'Preparing Data: {prep_time:.2f}s')
 print(f'Making Predictions: {prediction_time:.2f}s')
 print(f'Packing: {packaging_time:.2f}s')
 print(f'Total: {total:.2f}s' )
<define_variables><EOS> 
%matplotlib inline