Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<categorify>
env = twosigmanews.make_env()
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<set_options>
sampling = True
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42 )<create_dataframe>
( market_train_df, news_train_df)= env.get_training_data()
if sampling:
market_train_df = market_train_df.tail(40_000)
 news_train_df = news_train_df.tail(100_000)
 else:
market_train_df = market_train_df.tail(1_500_000)
 news_train_df = news_train_df.tail(4_000_000 )
n_comp = 600
pca_g = PCA(n_components=n_comp, random_state=42)
 data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 gpca=(pca_g.fit(data[GENES]))
train2=(gpca.transform(train_features[GENES]))
test2 =(gpca.transform(test_features[GENES]))
train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gpca), axis=1)
 test_features = pd.concat(( test_features, test_gpca), axis=1)
 dump(gpca, open('gpca.pkl', 'wb'))<create_dataframe>
news_agg_cols = [f for f in news_train_df.columns if 'novelty' in f or
'volume' in f or
'sentiment' in f or
'bodySize' in f or
'Count' in f or
'marketCommentary' in f or
'relevance' in f]
news_agg_dict = {}
for col in news_agg_cols:
news_agg_dict[col] = ['mean', 'sum', 'max', 'min']
news_agg_dict['urgency'] = ['min', 'count']
news_agg_dict['takeSequence'] = ['max']
n_comp = 50
pca_c = PCA(n_components=n_comp, random_state=42)
 data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 cpca=(pca_c.fit(data[CELLS]))
train2=(cpca.transform(train_features[CELLS]))
test2 =(cpca.transform(test_features[CELLS]))
train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_cpca), axis=1)
 test_features = pd.concat(( test_features, test_cpca), axis=1)
 dump(cpca, open('cpca.pkl', 'wb'))<concatenate>
def check_index(index, indecies):
if index in indecies:
return True
else:
return False
def fillnulls(X):
X['headline'] = X['headline'].fillna('null')
 def generalize_time(X):
X['time'] = X['time'].dt.strftime('%Y-%m-%d %H:%M:%S' ).str.slice(0,16)
 def partial_groupby(market_df, news_df, df_assetCodes):
temp_news_df_expanded = pd.merge(df_assetCodes, news_df[['time', 'assetCodes']], left_on='level_0', right_index=True, suffixes=(['','_old']))
temp_news_df = temp_news_df_expanded.copy() [['time', 'assetCode']]
temp_market_df = market_df.copy() [['time', 'assetCode']]
temp_news_df['news_index'] = temp_news_df.index.values
temp_market_df['market_index'] = temp_market_df.index.values
temp_news_df.set_index(['time', 'assetCode'], inplace=True)
 temp_market_df_2 = temp_market_df.join(temp_news_df, on=['time', 'assetCode'])
 del temp_market_df, temp_news_df
temp_market_df_2 = temp_market_df_2.dropna()
market_valid_indecies = temp_market_df_2['market_index'].tolist()
news_valid_indecies = temp_market_df_2['news_index'].tolist()
del temp_market_df_2
market_df = market_df.loc[market_valid_indecies]
return news_valid_indecies
def join_market_news(market_df, news_df, nulls=False):
generalize_time(market_df)
 generalize_time(news_df)
 news_df['assetCodes'] = news_df['assetCodes'].str.findall(f"'([\w\./]+)'")
 assetCodes_expanded = list(chain(*news_df['assetCodes']))
assetCodes_index = news_df.index.repeat(news_df['assetCodes'].apply(len))
assert len(assetCodes_index)== len(assetCodes_expanded)
 df_assetCodes = pd.DataFrame({'level_0': assetCodes_index, 'assetCode': assetCodes_expanded})
 if not nulls:
news_valid_indecies = partial_groupby(market_df, news_df, df_assetCodes)
 news_col = ['time', 'assetCodes', 'headline'] + sorted(list(news_agg_dict.keys()))
news_df_expanded = pd.merge(df_assetCodes, news_df[news_col], left_on='level_0', right_index=True, suffixes=(['','_old']))
if not nulls:
news_df_expanded = news_df_expanded.loc[news_valid_indecies]
def news_df_feats(x):
if x.name == 'headline':
return list(x)
 news_df_expanded = news_df_expanded.reset_index()
news_groupby = news_df_expanded.groupby(['time', 'assetCode'])
 news_df_aggregated = news_groupby.agg(news_agg_dict ).apply(np.float32 ).reset_index()
news_df_aggregated.columns = ['_'.join(col ).strip() for col in news_df_aggregated.columns.values]
news_df_cat = news_groupby.transform(lambda x: news_df_feats(x)) ['headline'].to_frame()
new_news_df = pd.concat([news_df_aggregated, news_df_cat], axis=1)
 del news_df_aggregated
del news_df_cat
del news_df
new_news_df.rename(columns={'time_': 'time', 'assetCode_': 'assetCode'}, inplace=True)
 new_news_df.set_index(['time', 'assetCode'], inplace=True)
 market_df = market_df.join(new_news_df, on=['time', 'assetCode'])
 fillnulls(market_df)
 return market_df
c_n = [f for f in list(train_features.columns)if f not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
mask =(train_features[c_n].var() >= 0.85 ).values
tmp = train_features[c_n].loc[:, mask]
train_features = pd.concat([train_features[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1)
 tmp = test_features[c_n].loc[:, mask]
test_features = pd.concat([test_features[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1 )<define_search_model>
%%time
X_train = join_market_news(market_train_df, news_train_df, nulls=False )
def fe_cluster_genes(train, test, n_clusters_g = 22, SEED = 42):
features_g = GENES
def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans_genes = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 dump(kmeans_genes, open('kmeans_genes.pkl', 'wb'))
train[f'clusters_{kind}'] = kmeans_genes.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans_genes.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 return train, test
train_features2 ,test_features2=fe_cluster_genes(train_features2,test_features2 )<find_best_model_class>
def round_scores(x):
if x >= 0:
return 1
else:
return 0
def get_headline_df(X_train):
headlines_lst = []
target_lst = []
for row in range(0,len(X_train.index)) :
for sentence in X_train['headline'].iloc[row]:
headlines_lst.append(sentence)
 target_lst.append(round_scores(X_train['returnsOpenNextMktres10'].iloc[row]))
return pd.DataFrame({'headline':pd.Series(headlines_lst), 'returnsOpenNextMktres10':pd.Series(target_lst)})
 def get_headline(headlines_df):
headlines_lst = []
for row in range(0,len(headlines_df.index)) :
headlines_lst.append(headlines_df.iloc[row])
 basicvectorizer = CountVectorizer()
headlines_vectorized = basicvectorizer.fit_transform(headlines_lst)
 print(headlines_vectorized.shape)
 return headlines_vectorized, basicvectorizer
def headline_mapping(target, headlines_vectored, headline_vectorizer):
print(np.asarray(target ).shape)
 headline_model = LogisticRegression()
headline_model = headline_model.fit(headlines_vectored, target)
 basicwords = headline_vectorizer.get_feature_names()
basiccoeffs = headline_model.coef_.tolist() [0]
coeff_df = pd.DataFrame({'Word' : basicwords,
'Coefficient' : basiccoeffs})
 coefficient_dict = dict(zip(coeff_df.Word, coeff_df.Coefficient))
return coefficient_dict, coeff_df['Coefficient'].mean()
def get_coeff_col(X, coeff_dict, coeff_default):
def get_coeff(word_lst):
coeff_sum = 0
for word in word_lst:
if word in coeff_dict:
coeff_sum += coeff_dict[word]
else:
coeff_sum += coeff_default
coeff_score = coeff_sum / len(word_lst)
 return coeff_score
basicvectorizer = CountVectorizer()
headlines_coeff_lst = []
for row in range(0,len(X['headline'].index)) :
coeff_score = 0
for i in range(0,len(X['headline'].iloc[row])) :
coeff_score += get_coeff(str(X['headline'].iloc[row][i] ).split(' '))
headlines_coeff_lst.append(coeff_score / len(X['headline'].iloc[row]))
coeff_mean_df = pd.DataFrame({'headline_coeff_mean': pd.Series(headlines_coeff_lst)})
 X = pd.concat([X.reset_index() , coeff_mean_df], axis=1)
 return X
def fe_cluster_cells(train, test, n_clusters_c = 4, SEED = 42):
features_c = CELLS
def create_cluster(train, test, features, kind = 'c', n_clusters = n_clusters_c):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans_cells = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 dump(kmeans_cells, open('kmeans_cells.pkl', 'wb'))
train[f'clusters_{kind}'] = kmeans_cells.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans_cells.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features2 ,test_features2=fe_cluster_cells(train_features2,test_features2 )<concatenate>
headline_df = get_headline_df(X_train)
 coefficient_dict, coefficient_default = headline_mapping(headline_df['returnsOpenNextMktres10'],
*get_headline(headline_df['headline']))
train_pca=pd.concat(( train_gpca,train_cpca),axis=1)
 test_pca=pd.concat(( test_gpca,test_cpca),axis=1 )<categorify>
X_train = get_coeff_col(X_train, coefficient_dict, coefficient_default )
def fe_cluster_pca(train, test,n_clusters=5,SEED = 42):
data=pd.concat([train,test],axis=0)
 kmeans_pca = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 dump(kmeans_pca, open('kmeans_pca.pkl', 'wb'))
train[f'clusters_pca'] = kmeans_pca.predict(train.values)
 test[f'clusters_pca'] = kmeans_pca.predict(test.values)
 train = pd.get_dummies(train, columns = [f'clusters_pca'])
 test = pd.get_dummies(test, columns = [f'clusters_pca'])
 return train, test
train_cluster_pca ,test_cluster_pca = fe_cluster_pca(train_pca,test_pca )<split>
def extra_features(X):
new_col = X["close"] - X["open"]
X.insert(loc=6, column="daily_diff", value=new_col)
 X['close_to_open'] = np.abs(X['close'] / X['open'] )
train_cluster_pca = train_cluster_pca.iloc[:,650:]
test_cluster_pca = test_cluster_pca.iloc[:,650:]<split>
extra_features(X_train )
train_features_cluster=train_features2.iloc[:,876:]
test_features_cluster=test_features2.iloc[:,876:]<define_variables>
def split_time(df):
df['time_day'] = df['time'].str.slice(8,10)
 df['time_month'] = df['time'].str.slice(5,7)
 df['time_year'] = df['time'].str.slice(0,4)
 df['time_hour'] = df['time'].str.slice(11,13)
 df['time_minute'] = df['time'].str.slice(14,16)
 df['temp_time'] = df['time'].str.replace(" UTC", "")
 df['temp_time'] = pd.to_datetime(df['temp_time'], format='%Y-%m-%d %H')
 df['time_day_of_year'] = df.temp_time.dt.dayofyear
df['time_week_of_year'] = df.temp_time.dt.weekofyear
df["time_weekday"] = df.temp_time.dt.weekday
df["time_quarter"] = df.temp_time.dt.quarter
del df['temp_time']
gc.collect()
time_feats = ['time_day', 'time_month', 'time_year']
df[time_feats] = df[time_feats].apply(pd.to_numeric)
 cal = USFederalHolidayCalendar()
holidays = cal.holidays(start='2007-01-01', end='2018-09-27' ).to_pydatetime()
df['on_holiday'] = df['time'].str.slice(0,10 ).apply(lambda x: 1 if x in holidays else 0 )
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']<feature_engineering>
split_time(X_train )
def fe_stats(train, test):
features_g = GENES
features_c = CELLS
for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-26'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
df['c26_c38'] = df['c-26'] * df['c-38']
df['c90_c13'] = df['c-90'] * df['c-13']
df['c85_c31'] = df['c-85'] * df['c-31']
df['c63_c42'] = df['c-63'] * df['c-42']
df['c94_c11'] = df['c-94'] * df['c-11']
df['c94_c60'] = df['c-94'] * df['c-60']
df['c55_c42'] = df['c-55'] * df['c-42']
df['g37_c50'] = df['g-37'] * df['g-50']
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
train_features2,test_features2=fe_stats(train_features2,test_features2 )<split>
def remove_cols(X):
del_cols = [f for f in X.columns if X[f].dtype == 'object'] + ['assetName', 'index']
for f in del_cols:
del X[f]
train_features_stats=train_features2.iloc[:,902:]
test_features_stats=test_features2.iloc[:,902:]<concatenate>
remove_cols(X_train )
train_features = pd.concat(( train_features, train_features_cluster,train_cluster_pca,train_features_stats), axis=1)
 test_features = pd.concat(( test_features, test_features_cluster,test_cluster_pca,test_features_stats), axis=1 )<count_unique_values>
def get_X(market_df, news_df):
X_test = join_market_news(market_df, news_df, nulls=True)
 X_test = get_coeff_col(X_test, coefficient_dict, coefficient_default)
 extra_features(X_test)
 split_time(X_test)
 remove_cols(X_test)
 return X_test
cols2 = train_targets_nonscored.columns.to_list()
uniques2 = {col: train_targets_nonscored[col].nunique() for col in cols2}
uniques2=pd.DataFrame(uniques2, index=[0] ).T
uniques2=uniques2.rename(columns={0:'count'})
 uniques2= uniques2.drop('sig_id', axis=0)
 print(f"{len(uniques2[uniques2['count']==1])} targets without ANY mechanism of action in the nonscored dataset" )<filter>
def set_data(X_train):
y_train = X_train['returnsOpenNextMktres10']
del X_train['returnsOpenNextMktres10'], X_train['universe']
x1, x2, y1, y2 = train_test_split(X_train,
y_train,
test_size=0.25,
random_state=99)
 return x1, x2, y1, y2
def lgbm_training(X_train):
params = {'learning_rate': 0.02,
'boosting': 'gbdt',
'objective': 'regression',
'seed': 2018}
x1, x2, y1, y2 = set_data(X_train)
 lgb_model = lgb.train(params,
lgb.Dataset(x1, label=y1),
5000,
lgb.Dataset(x2, label=y2),
verbose_eval=100,
early_stopping_rounds=200)
 return lgb_model
nonmoacols=uniques2[uniques2['count']==1].index
train_targets_nonscored_columns = [col for col in list(train_targets_nonscored.columns)if col not in nonmoacols]
train_targets_nonscored=train_targets_nonscored[train_targets_nonscored_columns]<merge>
lgb_model = lgbm_training(X_train )
train = train_features.merge(train_targets_nonscored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_nonscored.columns]<drop_column>
def make_predictions(market_obs_df, news_obs_df):
X_test = get_X(market_obs_df, news_obs_df)
 prediction_values = np.clip(lgb_model.predict(X_test), -1, 1)
 return prediction_values
for(market_obs_df, news_obs_df, predictions_template_df)in env.get_prediction_days() :
predictions_template_df['confidenceValue'] = make_predictions(market_obs_df, news_obs_df)
 env.predict(predictions_template_df)
<drop_column><EOS> 
env.write_submission_file()
print('finished!' )