Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<data_type_conversions>
warnings.filterwarnings("ignore",category=DeprecationWarning)
 env = twosigmanews.make_env() 
(market_train, news_train)= env.get_training_data()
test_data["Transmission"].fillna(value="Transmission_unk",inplace=True )<count_values>
def show_mem_usage() :
ipython_vars = ['In', 'Out', 'exit', 'quit', 'get_ipython', 'ipython_vars']
list_objects=[]
mem = 0
for x in globals() :
if x.startswith('_'): continue
if x in sys.modules: continue
if x in ipython_vars: continue
if isinstance(globals().get(x), pd.DataFrame):
mem = sys.getsizeof(globals().get(x)) /1e+6
if mem > 1:
list_objects.append([x, mem ])
 else:
for o in dir(globals().get(x)) :
if o.startswith('__'): continue
mem = sys.getsizeof(getattr(globals().get(x), o)) /1e+6
if mem > 1:
list_objects.append(['.'.join([x,o]), mem ])
 return sorted(list_objects, key=lambda x: x[1], reverse=True )
train_data["WheelType"].value_counts()<data_type_conversions>
def timeit(method):
def timed(*args, **kw):
ts = time.time()
result = method(*args, **kw)
 te = time.time()
if TIME_FUNCTIONS:
dt =(te - ts)*1000
if dt < 1000:
print('{:<40} {:>20.2f} ms'.format(method.__name__, dt))
else:
print('{:<40} {:>20.2f} s'.format(method.__name__, dt/1000))
return result
return timed
train_data["WheelType"].fillna(value="WheelType_unk",inplace=True )<data_type_conversions>
class model2SigmaStockPrizes() :
@timeit
def __init__(self, market, news, verbose=False):
self.market = market.sort_values('time')
 self.news = news.sort_values('time')
 self.market = reduce_mem_usage(self.market, verbose)
 self.news = reduce_mem_usage(self.news, verbose)
 self.verbose = verbose
self._format_dates()
self._convert_booleans()
@timeit
def _format_dates(self):
for df in [self.market, self.news]:
datetime_cols = [c for c in df.columns if 'date' in str(df[c].dtypes)]
for col in datetime_cols:
df[col] = df[col].dt.normalize()
if self.verbose:
print("Content of column:'{}' set as date".format(col))
@timeit
def _convert_booleans(self):
for col in self.news.columns:
if self.news[col].dtype == bool:
self.news[col] = self.news[col].astype(int )
test_data["WheelType"].fillna(value="WheelType_unk",inplace=True )<data_type_conversions>
TIME_FUNCTIONS = True
two_sigma_model = model2SigmaStockPrizes(market_train, news_train, verbose=True )
train_data["Nationality"].fillna(value="Nationality_unk",inplace=True )<data_type_conversions>
del news_train
del market_train
gc.collect()
show_mem_usage()
test_data["Nationality"].fillna(value="Nationality_unk",inplace=True )<count_values>
@timeit
def select_dates(self, first_date):
self.market = self.market.loc[self.market['time'] >= first_date]
self.news = self.news.loc[self.news['time'] >= first_date]
if self.verbose:
print("data before '{}' has been removed".format(first_date))
two_sigma_model.select_dates = select_dates.__get__(two_sigma_model)
 two_sigma_model.select_dates(datetime.datetime(2009, 1, 1, 0, 0, 0, 0, pytz.UTC))
train_data["TopThreeAmericanName"].value_counts()<data_type_conversions>
gc.collect()
show_mem_usage()
train_data["Size"].fillna(value="Size_unk",inplace=True )<data_type_conversions>
@timeit
def column_combinations(self):
self.market['price_diff'] = self.market['close'] - self.market['open']
@timeit
def asset_codes_encoding(self):
self.news['assetCodesLen'] = self.news['assetCodes'].map(lambda x: len(eval(x)))
 self.news['assetCodes'] = self.news['assetCodes'].str.findall(f"'([\w\./]+)'" ).apply(lambda x: x[0])
 @timeit
def news_count(self):
t = self.news.groupby(['time', 'assetName'] ).size().reset_index(name='news_count')
 self.news = pd.merge(self.news, t, on=['time', 'assetName'])
 two_sigma_model.column_combinations = column_combinations.__get__(two_sigma_model)
 two_sigma_model.asset_codes_encoding = asset_codes_encoding.__get__(two_sigma_model)
 two_sigma_model.news_count = news_count.__get__(two_sigma_model)
 two_sigma_model.column_combinations()
two_sigma_model.asset_codes_encoding()
two_sigma_model.news_count()
test_data["Size"].fillna(value="Size_unk",inplace=True )<data_type_conversions>
@timeit
def aggregate_numericals(self):
numerics = ['int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64']
self.news_numerical = self.news.select_dtypes(include=numerics ).copy()
self.news_numerical_columns = self.news_numerical.columns
self.news_numerical.loc[:, 'assetCodes'] = self.news['assetCodes']
self.news_numerical.loc[:, 'time'] = self.news['time']
agg_func = {
'takeSequence': ['sum', 'mean', 'max', 'min', 'std'],
'bodySize': ['sum', 'mean', 'max', 'min', 'std'],
'marketCommentary': ['sum', 'mean'],
'sentenceCount': ['sum', 'mean', 'max', 'min', 'std'],
'wordCount': ['sum', 'mean', 'max', 'min', 'std'],
'relevance': ['sum', 'mean', 'max', 'min', 'std'],
'firstMentionSentence': ['sum', 'mean', 'max', 'min', 'std'],
'sentimentNegative': ['sum', 'mean', 'max', 'min', 'std'],
'sentimentNeutral': ['sum', 'mean', 'max', 'min', 'std'],
'sentimentPositive': ['sum', 'mean', 'max', 'min', 'std'],
'sentimentWordCount': ['sum', 'mean', 'max', 'min', 'std'],
'noveltyCount12H': ['sum', 'mean', 'max', 'min', 'std'],
'noveltyCount24H': ['sum', 'mean', 'max', 'min', 'std'],
'noveltyCount3D': ['sum', 'mean', 'max', 'min', 'std'],
'noveltyCount5D': ['sum', 'mean', 'max', 'min', 'std'],
'noveltyCount7D': ['sum', 'mean', 'max', 'min', 'std'],
'volumeCounts12H': ['sum', 'mean', 'max', 'min', 'std'],
'volumeCounts24H': ['sum', 'mean', 'max', 'min', 'std'],
'volumeCounts3D': ['sum', 'mean', 'max', 'min', 'std'],
'volumeCounts5D': ['sum', 'mean', 'max', 'min', 'std'],
'volumeCounts7D': ['sum', 'mean', 'max', 'min', 'std'],
'news_count': ['max']
}
self.news_numerical = self.news_numerical.groupby(['assetCodes', 'time'] ).agg(agg_func)
 self.news_numerical.columns = ['_'.join(col ).strip()
for col in self.news_numerical.columns.values]
self.news_numerical.reset_index(inplace=True)
 two_sigma_model.aggregate_numericals = aggregate_numericals.__get__(two_sigma_model)
train_data["TopThreeAmericanName"].fillna(value="TopThreeAmericanName_unk",inplace=True )<data_type_conversions>
@timeit
def aggregate_categoricals(self):
categ_columns = [col for col in self.news.columns
if col not in self.news_numerical_columns]
if self.verbose:
print("news's categorical columns:
", categ_columns)
 temp = self.news[self.news['news_count'] > 1][categ_columns].copy()
multiple_articles = temp.groupby(['assetCodes', 'time'] ).tail(1)
 single_articles = self.news[self.news['news_count'] == 1][categ_columns]
self.news_categ = pd.concat([multiple_articles, single_articles])
 two_sigma_model.aggregate_categoricals = aggregate_categoricals.__get__(two_sigma_model)
test_data["TopThreeAmericanName"].fillna(value="TopThreeAmericanName_unk",inplace=True )<data_type_conversions>
@timeit
def merge_news(self):
self.news = pd.merge(self.news_numerical,
self.news_categ,
on = ['assetCodes', 'time'],
how='left')
 self.news_categ = 0
self.news_numerical = 0
@timeit
def merge_market_news(self, keep_news=False):
if keep_news:
self.merged_df = pd.merge(self.market,
self.news,
left_on=['time', 'assetCode'],
right_on=['time', 'assetCodes'],
how='left')
 else:
self.merged_df = self.market
self.news = 0
self.market = 0
self.merged_df = reduce_mem_usage(self.merged_df, self.verbose)
 if self.verbose:
print("merged_df's shape is: {}".format(self.merged_df.shape))
two_sigma_model.merge_news = merge_news.__get__(two_sigma_model)
 two_sigma_model.merge_market_news = merge_market_news.__get__(two_sigma_model)
 two_sigma_model.merge_market_news()
train_data["PRIMEUNIT"].fillna(value="PRIMEUNIT_unk",inplace=True )<data_type_conversions>
gc.collect()
show_mem_usage()
test_data["PRIMEUNIT"].fillna(value="PRIMEUNIT_unk",inplace=True )<data_type_conversions>
@timeit
def set_labels(self, labeled_columns, define_index=False):
if define_index:
self.indexer = {}
for col in labeled_columns:
_, self.indexer[col] = pd.factorize(self.merged_df[col])
 self.categorical_columns = labeled_columns
if self.verbose:
print("categorical variables: {}".format(labeled_columns))
for col in labeled_columns:
self.merged_df[col] = self.indexer[col].get_indexer(self.merged_df[col])
 two_sigma_model.set_labels = set_labels.__get__(two_sigma_model)
 two_sigma_model.set_labels([], define_index=True )
train_data["AUCGUART"].fillna(value="AUCGUART_unk",inplace=True )<data_type_conversions>
gc.collect()
show_mem_usage()
test_data["AUCGUART"].fillna(value="AUCGUART_unk",inplace=True )<drop_column>
def run_parallel(grouped, method, verbose):
with parallel.Parallel(n_jobs=-1, verbose=verbose)as par:
segments = par(delayed(method )(df_seg)for df_seg in grouped)
 return segments
def account_for_past(df_code, n_lag=[7, 14, 21], shift_size=1):
features = [c for c in df_code.columns if c not in ['assetCode']]
lag_columns = []
for col in features:
for window in n_lag:
if LAST_DATE_ONLY:
tmp = df_code[df_code.index.max() - 2*timedelta(window):df_code.index.max() ]
rolled = tmp[col].shift(shift_size, freq='D' ).rolling(window=window)
 else:
rolled = df_code[col].shift(shift_size, freq='D' ).rolling(window=window)
 col_name = '{}_median_past{}'.format(col, window)
 lag_columns.append(col_name)
 df_code[col_name] = rolled.median()
col_name = '{}_max_past{}'.format(col, window)
 lag_columns.append(col_name)
 df_code[col_name] = rolled.max()
col_name = '{}_min_past{}'.format(col, window)
 lag_columns.append(col_name)
 df_code[col_name] = rolled.min()
return lag_columns, df_code.drop(features, axis=1)
 def create_returns(df_code, n_days=[3, 5, 7, 12]):
features = [c for c in df_code.columns if c not in ['assetCode']]
lag_columns = []
for col in features:
for days in n_days:
col_shift = 'shift_{}_{}'.format(col, days)
 col_name = 'returns_{}_PrevRaw{}'.format(col, days)
 lag_columns.append(col_name)
 df_code.loc[:, col_shift] = df_code[col].shift(days, freq='D')
 df_code[col_name] =(df_code[col_shift] - df_code[col])/ df_code[col_shift]
df_code.drop(col_shift, axis=1, inplace=True)
 return lag_columns, df_code.drop(features, axis=1)
 @timeit
def define_lagged_var(self):
start_time = time.time()
list_df = [d[1][self.common_lag_features + ['price_diff']].set_index('time' ).copy()
for d in self.history_df.groupby('assetCode', sort=False)]
if self.verbose:
print('assetCodes: {}, time to group: {:<5.2f}s'.format(len(list_df), time.time() - start_time))
grouped = run_parallel(list_df, account_for_past, int(self.verbose))
self.lag_columns = grouped[0][0]
self.merged_df = pd.merge(self.merged_df,
pd.concat([d[1] for d in grouped] ).reset_index() ,
on=self.common_lag_features,
how='left')
 @timeit
def define_lagged_return(self):
start_time = time.time()
list_df = [d[1][self.common_lag_features + ['close']].set_index('time' ).copy()
for d in self.history_df.groupby('assetCode', sort=False)]
if self.verbose:
print('assetCodes: {}, time to group: {:<5.2f}s'.format(len(list_df), time.time() - start_time))
grouped = run_parallel(list_df, create_returns, int(self.verbose))
self.lag_columns = grouped[0][0]
self.merged_df = pd.merge(self.merged_df,
pd.concat([d[1] for d in grouped] ).reset_index() ,
on=self.common_lag_features,
how='left')
 two_sigma_model.define_lagged_var = define_lagged_var.__get__(two_sigma_model)
 two_sigma_model.define_lagged_return = define_lagged_return.__get__(two_sigma_model)
 LAST_DATE_ONLY = False
two_sigma_model.define_lagged_var()
two_sigma_model.define_lagged_return()
train_data.drop(["Model","Trim","SubModel","WheelTypeID","MMRAcquisitionAuctionAveragePrice","MMRAcquisitionAuctionCleanPrice","MMRAcquisitionRetailAveragePrice","MMRAcquisitonRetailCleanPrice","MMRCurrentAuctionAveragePrice","MMRCurrentAuctionCleanPrice","MMRCurrentRetailAveragePrice","MMRCurrentRetailCleanPrice","PurchDate"],axis=1,inplace=True )<drop_column>
@timeit
def news_moving_averages(self, col, window):
col_name = 'news_mva_'+ col + '_' + str(window)+ 'days'
if self.verbose:
print("column '{}' has been created".format(col_name))
avg_col = \
self.history_df[self.common_lag_features+[col]].set_index('time' ).\
groupby('assetCode' ).rolling(window=window, freq='D' ).mean().reset_index()
avg_col.rename(columns = {col: col_name}, inplace = True)
 self.merged_df = pd.merge(self.merged_df,
avg_col[self.common_lag_features + [col_name]],
on=self.common_lag_features,
how='left')
 return col_name
@timeit
def calc_news_moving_averages(self):
self.news_lag_columns = []
for col in ['sentimentNegative_mean']:
new_col = self.news_moving_averages(col, 7)
 self.news_lag_columns.append(new_col)
 two_sigma_model.news_moving_averages = news_moving_averages.__get__(two_sigma_model)
 two_sigma_model.calc_news_moving_averages = calc_news_moving_averages.__get__(two_sigma_model)
test_data.drop(["Model","Trim","SubModel","WheelTypeID","MMRAcquisitionAuctionAveragePrice","MMRAcquisitionAuctionCleanPrice","MMRAcquisitionRetailAveragePrice","MMRAcquisitonRetailCleanPrice","MMRCurrentAuctionAveragePrice","MMRCurrentAuctionCleanPrice","MMRCurrentRetailAveragePrice","MMRCurrentRetailCleanPrice","PurchDate"],axis=1,inplace=True )<drop_column>
@timeit
def select_variables(self):
removed_columns = [
'assetCode', 'assetCodes', 'assetCodesLen', 'assetName_x', 'assetName_y', 'assetName',
'audiences', 'firstCreated', 'headline', 'returnsOpenNextMktres10',
'sourceId', 'subjects', 'time', 'universe','sourceTimestamp']
self.selected_variables = [c for c in self.merged_df.columns if c not in removed_columns]
if self.verbose:
print("variables kept: {}".format(self.selected_variables))
two_sigma_model.select_variables = select_variables.__get__(two_sigma_model)
 two_sigma_model.select_variables()
not_categorical=train_data.drop(["RefId","IsBadBuy"],axis=1 ).columns[train_data.drop(["RefId","IsBadBuy"],axis=1 ).dtypes!='object']<feature_engineering>
@timeit
def define_time_cv(self):
X = self.merged_df[self.selected_variables]
target = self.merged_df['returnsOpenNextMktres10']
time = self.merged_df['time']
universe = self.merged_df['universe']
n_train = int(X.shape[0] * 0.8)
 self.X_train, self.y_train = X.iloc[:n_train], target[:n_train]
self.X_valid, self.y_valid = X.iloc[n_train:], target[n_train:]
self.t_valid = time.iloc[n_train:]
u_valid =(universe.iloc[n_train:] > 0)
 self.t_valid = time.iloc[n_train:]
self.X_valid = self.X_valid[u_valid]
self.y_valid = self.y_valid[u_valid]
self.t_valid = self.t_valid[u_valid]
del u_valid
two_sigma_model.define_time_cv = define_time_cv.__get__(two_sigma_model)
 two_sigma_model.define_time_cv()
for i in not_categorical:
maximum=np.max(train_data[i])
 train_data[i]=train_data[i]/maximum
max_test=np.max(test_data[i])
 test_data[i]=test_data[i]/max_test<count_values>
@timeit
def prep_lgbm_data(self):
self.dtrain = lgb.Dataset(
 self.X_train.values, self.y_train,
feature_name = self.selected_variables,
categorical_feature = self.categorical_columns,
free_raw_data = False)
 self.dvalid = lgb.Dataset(
 self.X_valid.values, self.y_valid,
feature_name = self.selected_variables,
categorical_feature = self.categorical_columns,
free_raw_data = False)
 self.dvalid.params = {'extra_time': self.t_valid.factorize() [0]}
two_sigma_model.prep_lgbm_data = prep_lgbm_data.__get__(two_sigma_model)
 two_sigma_model.prep_lgbm_data()
test_data["VehicleAge"].value_counts()<drop_column>
def sigma_score(preds, valid_data):
df_time = valid_data.params['extra_time']
labels = valid_data.get_label()
val = pd.DataFrame()
val['time'] = df_time
val['y'] = preds * labels.values
output = val.groupby('time' ).sum()
score = output['y'].mean() / output['y'].std()
return 'sigma_score', score, True
categorical=train_data.drop(["RefId","IsBadBuy"],axis=1 ).columns[train_data.drop(["RefId","IsBadBuy"],axis=1 ).dtypes=='object']<categorify>
def clean_frames(self):
del self.merged_df
two_sigma_model.clean_frames = clean_frames.__get__(two_sigma_model)
 two_sigma_model.clean_frames()
pd.get_dummies(train_data[categorical[0]] )<categorify>
gc.collect()
show_mem_usage()
for i in categorical:
dum=pd.get_dummies(train_data[i])
 dum.columns=str(i)+"_"+dum.columns
train_data=pd.concat([train_data,dum],axis=1)
 train_data.drop(i,inplace=True,axis=1)
 dum=pd.get_dummies(test_data[i])
 dum.columns=str(i)+"_"+dum.columns
test_data=pd.concat([test_data,dum],axis=1)
 test_data.drop(i,inplace=True,axis=1 )<feature_engineering>
@timeit
def train_model(self, lgb_params):
evals_result = {}
self.model = lgb.train(
 lgb_params,
self.dtrain,
num_boost_round= 10000,
valid_sets=(self.dvalid,),
valid_names=('valid',),
verbose_eval=100,
early_stopping_rounds=200,
feval=sigma_score,
evals_result=evals_result 
)
 df_result = pd.DataFrame(evals_result['valid'])
 self.num_boost_round, valid_score = \
df_result['sigma_score'].idxmax() +1, df_result['sigma_score'].max()
print(f'Best score was {valid_score:.5f} on round {self.num_boost_round}')
 del self.X_train
del self.y_train
del self.X_valid
del self.y_valid
two_sigma_model.train_model = train_model.__get__(two_sigma_model)
 lgb_params = dict(
 objective = 'regression_l1',
learning_rate = 0.01,
num_leaves = 51,
max_depth = 8,
bagging_fraction = 0.9,
bagging_freq = 1,
feature_fraction = 0.9,
lambda_l1 = 0.0,
lambda_l2 = 1.0,
metric = 'None',
seed = 42)
 two_sigma_model.train_model(lgb_params )
for i in train_data.drop("IsBadBuy",axis=1 ).columns:
if i not in test_data.columns:
test_data[i]=np.zeros(len(test_data))
for i in test_data.columns:
if i not in train_data.columns:
train_data[i]=np.zeros(len(train_data)) 
<drop_column>
@timeit
def data_prep(self, market, news):
self.verbose = False
self.market = market
self.news = news
self._format_dates()
self._convert_booleans()
self.column_combinations()
self.asset_codes_encoding()
self.news_count()
self.merge_market_news()
self.set_labels([])
 two_sigma_model.data_prep = data_prep.__get__(two_sigma_model )
test_data=test_data[train_data.drop("IsBadBuy",axis=1 ).columns]<import_modules>
days = env.get_prediction_days()
from sklearn.model_selection import train_test_split<import_modules>
@timeit
def add_new_date(self):
self.history_df = pd.concat([self.history_df,
self.merged_df[
self.common_lag_features +
self.used_for_lag +
self.used_for_news_lag]])
 if self.verbose:
print("data for lagged quantities from {} to {}".format(
 self.lag_df['time'].min().date() ,
self.lag_df['time'].max().date()))
two_sigma_model.add_new_date = add_new_date.__get__(two_sigma_model )
from sklearn.model_selection import train_test_split<prepare_x_and_y>
@timeit
def predict(self, date, pred_template):
df = self.merged_df[self.merged_df['time'] == date]
predictions = self.model.predict(
 df[self.selected_variables].values,
ntree_limit = self.num_boost_round 
)
 preds = pd.DataFrame({'assetCode': df['assetCode'],
'confidence': np.clip(predictions, -1, 1)})
 self.formated_pred = \
pred_template.merge(preds, how='left' ).\
drop('confidenceValue', axis=1 ).\
fillna(0 ).\
rename(columns={'confidence': 'confidenceValue'})
 two_sigma_model.predict = predict.__get__(two_sigma_model )
x=train_data.drop(["RefId","IsBadBuy"],axis=1)
 y=train_data["IsBadBuy"]<split>
two_sigma_model.history_df = \
two_sigma_model.history_df[two_sigma_model.history_df['time'] >(two_sigma_model.history_df['time'].max() - timedelta(30)) ]
x_train,x_val,y_train,y_val=train_test_split(x,y,random_state=42 )<import_modules>
TIME_FUNCTIONS = False
n_days = 0
for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
start_time = time.time()
print(n_days, end=' ')
 two_sigma_model.data_prep(market_obs_df, news_obs_df)
 date = two_sigma_model.merged_df['time'].max()
two_sigma_model.add_new_date()
two_sigma_model.define_lagged_var()
two_sigma_model.define_lagged_return()
two_sigma_model.select_variables()
two_sigma_model.predict(date, predictions_template_df)
 env.predict(two_sigma_model.formated_pred )
<import_modules><EOS> 
env.write_submission_file()