Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<save_to_csv>
import numpy as np
import pandas as pd
from kaggle.competitions import twosigmanews
from datetime import datetime, date
import numpy as np
from sklearn import model_selection
submission.to_csv('submission.csv', index=False )<import_modules>
env = twosigmanews.make_env() 
(market_train_orig, news_train_orig)= env.get_training_data()
market_train_df = market_train_orig.copy()
news_train_df = news_train_orig.copy()
<set_options>
market_train_df['time'] = market_train_df['time'].dt.date
market_train_df = market_train_df.loc[market_train_df['time']>=date(2009, 1, 1)]
jovian.commit(project='osic-bayesian' )<load_from_csv>
news_train_df['time'] = news_train_df['time'].dt.date
news_train_df = news_train_df.loc[news_train_df['time']>=date(2009, 1, 1)]
df_train_file = ".. /input/kkbox-churn-scala-label/user_label_201703.csv"
df_train = pd.read_csv(df_train_file, dtype = {'is_churn': 'int8'})
 df_train.info()<load_from_csv>
del market_train_orig
del news_train_orig
df_test_file = ".. /input/kkbox-churn-prediction-challenge/sample_submission_v2.csv"
df_test = pd.read_csv(df_test_file, dtype = {'is_churn': 'int8'})
 df_test.info()<load_from_csv>
return_features = ['returnsClosePrevMktres10','returnsClosePrevRaw10','open','close']
n_lag = [3,7,14]
new_df = generate_lag_features(market_train_df,n_lag=n_lag)
 market_train_df = pd.merge(market_train_df,new_df,how='left',on=['time','assetCode'] )
df_userlogs_file = ".. /input/kkbox-churn-prediction-challenge/user_logs.csv"
df_userlogs_file_2 = ".. /input/kkbox-churn-prediction-challenge/user_logs_v2.csv"
df_userlogs = pd.read_csv(df_userlogs_file, nrows = 36000000)
 df_userlogs_2 = pd.read_csv(df_userlogs_file_2)
 df_userlogs = df_userlogs.append(df_userlogs_2, ignore_index = True)
 df_userlogs.info()
del df_userlogs_2<merge>
del new_df
gc.collect()
del df_userlogs['date']
counts = df_userlogs.groupby('msno')['total_secs'].count().reset_index()
counts.columns = ['msno', 'days_listened']
sums = df_userlogs.groupby('msno' ).sum().reset_index()
df_userlogs = sums.merge(counts, how = 'inner', on = 'msno' )<feature_engineering>
def prepare_news_data(news_df):
news_df['position'] = news_df['firstMentionSentence'] / news_df['sentenceCount']
news_df['coverage'] = news_df['sentimentWordCount'] / news_df['wordCount']
droplist = ['sourceTimestamp','firstCreated','sourceId','headline','takeSequence','provider','firstMentionSentence',
'sentenceCount','bodySize','headlineTag','marketCommentary','subjects','audiences','sentimentClass',
'assetName', 'urgency','wordCount','noveltyCount12H','sentimentWordCount','noveltyCount24H','noveltyCount3D','noveltyCount5D','noveltyCount7D','volumeCounts12H','volumeCounts24H','volumeCounts3D','volumeCounts5D','volumeCounts7D',]
news_df.drop(droplist, axis=1, inplace=True)
 assets = []
indices = []
for i, values in news_df['assetCodes'].iteritems() :
assetCodes = eval(values)
 assets.extend(assetCodes)
 indices.extend([i]*len(assetCodes))
mapping_df = pd.DataFrame({'news_index': indices, 'assetCode': assets})
 del assets, indices
news_df['news_index'] = news_df.index.copy()
expanded_news_df = mapping_df.merge(news_df, how='left', on='news_index')
 del mapping_df, news_df
expanded_news_df.drop(['news_index', 'assetCodes'], axis=1, inplace=True)
 return expanded_news_df.groupby(['time', 'assetCode'] ).mean().reset_index()
df_userlogs['secs_per_song'] = df_userlogs['total_secs'].div(df_userlogs['num_25'] + df_userlogs['num_50'] + df_userlogs['num_75'] + df_userlogs['num_985'] + df_userlogs['num_100'] )<load_from_csv>
news_train_df = prepare_news_data(news_train_df )
df_members_file = ".. /input/kkbox-churn-prediction-challenge/members_v3.csv"
df_members = pd.read_csv(df_members_file)
 df_members.info()<feature_engineering>
market_train_df = market_train_df.merge(news_train_df, how='left', on=['assetCode', 'time'])
 market_train_df
df_members['city'] = df_members.city.apply(lambda x: int(x)if pd.notnull(x)else "NAN")
 df_members['bd'] = df_members.bd.apply(lambda x: -99999 if float(x)<= 1 else x)
 df_members['bd'] = df_members.bd.apply(lambda x: -99999 if float(x)>= 100 else x)
 df_members['gender'] = df_members['gender'].fillna("others")
 df_members['registered_via'] = df_members.registered_via.apply(lambda x: int(x)if pd.notnull(x)else "NAN")
 current = datetime.strptime('20170331', "%Y%m%d" ).date()
df_members['num_days'] = df_members.registration_init_time.apply(lambda x:(current - datetime.strptime(str(int(x)) , "%Y%m%d" ).date() ).days if pd.notnull(x)else "NAN")
 del df_members['registration_init_time']<drop_column>
del news_train_df
gc.collect
del df_members['city']<define_search_space>
def fill_in(data):
for i in data.columns:
if data[i].dtype == "object":
data[i] = data[i].fillna("")
 elif(data[i].dtype == "int64" or data[i].dtype == "float64"):
data[i] = data[i].fillna(data[i].mean())
 else:
pass
return data
market_train_df = fill_in(market_train_df )
df_members['registered_via'].replace([-1, 1, 2, 5, 6, 8, 10, 11, 13, 14, 16, 17, 18, 19], 1, inplace = True )<drop_column>
def data_other(market_train_df):
lbl = {k: v for v, k in enumerate(market_train_df['assetCode'].unique())}
market_train_df['assetCodeT'] = market_train_df['assetCode'].map(lbl)
 market_train_df = market_train_df.dropna(axis=0)
 return market_train_df
market_train_df = data_other(market_train_df )
del df_members['bd']<load_from_csv>
green = market_train_df.returnsOpenNextMktres10 > 0
green = green.values
df_transactions_file = ".. /input/kkbox-churn-prediction-challenge/transactions.csv"
df_transactions_file_2 = ".. /input/kkbox-churn-prediction-challenge/transactions_v2.csv"
df_transactions = pd.read_csv(df_transactions_file)
 df_transactions_2 = pd.read_csv(df_transactions_file_2)
 df_transactions = df_transactions.append(df_transactions_2, ignore_index = True)
 df_transactions.describe()
del df_transactions_2<drop_column>
fcol = [c for c in market_train_df if c not in ['assetCode', 'assetCodes', 'assetCodesLen', 'assetName', 'audiences',
'firstCreated', 'headline', 'headlineTag', 'marketCommentary', 'provider',
'returnsOpenNextMktres10', 'sourceId', 'subjects', 'time', 'time_x', 'universe','sourceTimestamp']]
del df_transactions['payment_plan_days']<remove_duplicates>
X = market_train_df[fcol].values
mins = np.min(X, axis=0)
 maxs = np.max(X, axis=0)
 rng = maxs - mins
X = 1 -(( maxs - X)/ rng )
del df_transactions['membership_expire_date']
del df_transactions['transaction_date']
df_transactions = df_transactions.drop_duplicates()
df_transactions = df_transactions.groupby('msno' ).mean().reset_index()
df_transactions.head()<define_variables>
all_10day = market_train_df.returnsOpenNextMktres10.values
gender = {'male': 0, 'female': 1, 'others' :2}<merge>
X_train, X_test, green_train, green_test, all_train, all_test = model_selection.train_test_split(
 X, green, all_10day, test_size=0.20, random_state=59 )
df_training = pd.merge(left = df_train, right = df_members, how = 'left', on = ['msno'])
 df_training = pd.merge(left = df_training, right = df_transactions , how = 'left', on = ['msno'])
 df_training = pd.merge(left = df_training, right = df_userlogs, how = 'left', on = ['msno'])
 df_training['gender'] = df_training['gender'].map(gender )<merge>
train_data = lgb.Dataset(X_train, label=green_train.astype(int))
test_data = lgb.Dataset(X_test, label=green_test.astype(int))
df_testing = pd.merge(left = df_test, right = df_members, how = 'left', on = ['msno'])
 df_testing = pd.merge(left = df_testing, right = df_transactions , how = 'left', on = ['msno'])
 df_testing = pd.merge(left = df_testing, right = df_userlogs, how = 'left', on = ['msno'])
 df_testing['gender'] = df_testing['gender'].map(gender )<drop_column>
x_1 = [0.19000424246380565, 2452, 212, 328, 202]
x_2 = [0.19016805202090095, 2583, 213, 312, 220]
del df_members
del df_userlogs
del df_transactions<categorify>
params_1 = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'binary',
'learning_rate': x_1[0],
'num_leaves': x_1[1],
'min_data_in_leaf': x_1[2],
'num_iteration': x_1[3],
'max_bin': x_1[4],
'verbose': 1
}
params_2 = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'binary',
'learning_rate': x_2[0],
'num_leaves': x_2[1],
'min_data_in_leaf': x_2[2],
'num_iteration': x_2[3],
'max_bin': x_2[4],
'verbose': 1
}
gbm_1 = lgb.train(params_1,
train_data,
num_boost_round=1000,
valid_sets=test_data,
early_stopping_rounds=100)
 gbm_2 = lgb.train(params_2,
train_data,
num_boost_round=1000,
valid_sets=test_data,
early_stopping_rounds=100 )
<import_modules>
confidence_test =(gbm_1.predict(X_test)+ gbm_2.predict(X_test)) /2
confidence_test =(confidence_test-confidence_test.min())/(confidence_test.max() -confidence_test.min())
 confidence_test = confidence_test*2-1
print(max(confidence_test),min(confidence_test))
r_test = r_test.clip(-1,1)
 x_t_i = confidence_test * r_test * u_test
data = {'day' : d_test, 'x_t_i' : x_t_i}
df = pd.DataFrame(data)
 x_t = df.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_test = mean / std
print(score_test )
import sklearn as sl
from sklearn import model_selection
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.linear_model import LogisticRegression
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
import lightgbm as lgb<split>
days = env.get_prediction_days()
n_days = 0
prep_time = 0
prediction_time = 0
packaging_time = 0
total_market_obs_df = []
cols = [c for c in df_training.columns if c not in ['is_churn','msno']]
X = df_training[cols]
Y = df_training['is_churn']
validation_size = 0.20
seed = 7
scoring = 'roc_auc'
X_train, X_validation, Y_train, Y_validation = model_selection.train_test_split(X, Y, test_size = validation_size, random_state = seed )<train_model>
fcol1 = [c for c in market_train_df if c not in ['assetCode', 'assetCodes', 'assetCodesLen', 'assetName', 'audiences',
'firstCreated', 'headline', 'headlineTag', 'marketCommentary', 'provider',
'returnsOpenNextMktres10', 'sourceId', 'subjects', 'time', 'time_x', 'universe','sourceTimestamp','companyCount','relevance','sentimentNegative','sentimentNeutral','sentimentPositive','position','coverage']]
lgb_params = { 'learning_rate': 0.02,
'application': 'binary',
'max_depth': 35,
'num_leaves': 3500,
'verbosity': -1,
'metric': 'binary_logloss'
}
d_trainl = lgb.Dataset(X_train, label = Y_train)
 d_validl = lgb.Dataset(X_validation, label = Y_validation)
 watchlistl = [d_trainl, d_validl]
lgb_model = lgb.train(lgb_params,
train_set = d_trainl,
num_boost_round = 1000,
valid_sets = watchlistl,
early_stopping_rounds = 50,
verbose_eval = 10)
 lgb_pred = lgb_model.predict(df_testing[cols])
 lgb_testing = df_testing.copy()
lgb_testing['is_churn'] = lgb_pred.clip(0.+1e-15, 1-1e-15)
 lgb_testing[['msno','is_churn']].to_csv('lgb_result.csv', index = False )<load_from_csv>
X = market_train_df[fcol1].values
mins = np.min(X, axis=0)
 maxs = np.max(X, axis=0)
 rng = maxs - mins
X = 1 -(( maxs - X)/ rng )
train = pd.read_csv('.. /input/kkbox-churn-scala-label/user_label_201703.csv', dtype={'is_churn': 'int8'})
<load_from_csv>
for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
if(n_days%50==0):
print(n_days,end=' ')
 t = time.time()
market_obs_df['time'] = market_obs_df['time'].dt.date
return_features = ['returnsClosePrevMktres10','returnsClosePrevRaw10','open','close']
total_market_obs_df.append(market_obs_df)
 if len(total_market_obs_df)==1:
history_df = total_market_obs_df[0]
else:
history_df = pd.concat(total_market_obs_df[-(np.max(n_lag)+1):])
 print(history_df)
 new_df = generate_lag_features(history_df,n_lag=[3,7,14])
 market_obs_df = pd.merge(market_obs_df,new_df,how='left',on=['time','assetCode'])
 market_obs_df = fill_in(market_obs_df)
 market_obs_df = data_other(market_obs_df)
 X_live = market_obs_df[fcol1].values
X_live = 1 -(( maxs - X_live)/ rng)
 prep_time += time.time() - t
t = time.time()
lp =(gbm_1.predict(X_live)+ gbm_2.predict(X_live)) /2
prediction_time += time.time() -t
t = time.time()
confidence = lp
confidence =(confidence-confidence.min())/(confidence.max() -confidence.min())
 confidence = confidence * 2 - 1
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence':confidence})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 min_max_scaler = preprocessing.MinMaxScaler(feature_range=(-1, 1))
predictions_template_df[['confidenceValue']] = min_max_scaler.fit_transform(predictions_template_df[['confidenceValue']])
 env.predict(predictions_template_df)
 packaging_time += time.time() - t
<train_model><EOS> 
env.write_submission_file()
sub = pd.read_csv("submission.csv" )