Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<feature_engineering>
pd.set_option('max_columns', 50 )
for i in not_categorical:
maximum = np.max(train_data[i])
 train_data[i] = train_data[i]/maximum
maximum_test = np.max(test_data[i])
 test_data[i] = test_data[i]/maximum_test<drop_column>
env = twosigmanews.make_env()
market_train_df, news_train = env.get_training_data()
categorical = train_data.drop(['RefId','IsBadBuy'],axis=1 ).columns[train_data.drop(['RefId','IsBadBuy'],axis=1 ).dtypes=='object']<categorify>
start = datetime(2009, 1, 1, 0, 0, 0 ).date()
market_train = market_train_df.loc[market_train_df['time'].dt.date >= start].reset_index(drop=True)
 news_train = news_train.loc[news_train['time'].dt.date >= start].reset_index(drop=True )
for i in categorical:
dummies = pd.get_dummies(train_data[i])
 dummies.columns = str(i)+'_'+dummies.columns
train_data = pd.concat([train_data,dummies],axis=1)
 train_data.drop(i,inplace=True,axis=1)
 dummies = pd.get_dummies(test_data[i])
 dummies.columns = str(i)+'_'+dummies.columns
test_data = pd.concat([test_data,dummies],axis=1)
 test_data.drop(i,inplace=True,axis=1 )<feature_engineering>
def preprocess_news(news_train):
drop_list = [
'audiences', 'subjects', 'assetName',
'headline', 'firstCreated', 'sourceTimestamp',
]
news_train.drop(drop_list, axis=1, inplace=True)
 for col in ['headlineTag', 'provider', 'sourceId']:
news_train[col], uniques = pd.factorize(news_train[col])
 del uniques
news_train['assetCodes'] = news_train['assetCodes'].apply(lambda x: x[1:-1].replace("'", ""))
return news_train
for i in train_data.drop('IsBadBuy' ,axis=1 ).columns:
if i not in test_data.columns:
test_data[i]=np.zeros(len(test_data))<feature_engineering>
news_train = preprocess_news(news_train )
for i in test_data.columns:
if i not in train_data.columns:
train_data[i]=np.zeros(len(train_data))<drop_column>
def unstack_asset_codes(news_train):
codes = []
indexes = []
for i, values in news_train['assetCodes'].iteritems() :
explode = values.split(", ")
 codes.extend(explode)
 repeat_index = [int(i)]*len(explode)
 indexes.extend(repeat_index)
 index_df = pd.DataFrame({'news_index': indexes, 'assetCode': codes})
 del codes, indexes
gc.collect()
return index_df
test_data = test_data[train_data.drop('IsBadBuy',axis=1 ).columns]<prepare_x_and_y>
index_df = unstack_asset_codes(news_train)
 index_df.head()
X = train_data.drop(['RefId','IsBadBuy'],axis=1)
 y = train_data['IsBadBuy']<split>
def merge_news_on_index(news_train, index_df):
news_train['news_index'] = news_train.index.copy()
news_unstack = index_df.merge(news_train, how='left', on='news_index')
 news_unstack.drop(['news_index', 'assetCodes'], axis=1, inplace=True)
 return news_unstack
X_train ,X_test,y_train,y_test = train_test_split(X,y,random_state = 32 )<train_model>
news_unstack = merge_news_on_index(news_train, index_df)
 del news_train, index_df
gc.collect()
news_unstack.head(3 )
KNN_model = KNeighborsClassifier(n_neighbors = 11)
 KNN_model.fit(X_train,y_train )<compute_test_metric>
def group_news(news_frame):
news_frame['date'] = news_frame.time.dt.date
aggregations = ['mean']
gp = news_frame.groupby(['assetCode', 'date'] ).agg(aggregations)
 gp.columns = pd.Index(["{}_{}".format(e[0], e[1])for e in gp.columns.tolist() ])
 gp.reset_index(inplace=True)
 float_cols = {c: 'float32' for c in gp.columns if c not in ['assetCode', 'date']}
return gp.astype(float_cols )
KNN_model.score(X_test,y_test )<predict_on_test>
news_agg = group_news(news_unstack)
 del news_unstack; gc.collect()
news_agg.head(3 )
predict=KNN_model.predict(test_data.drop('RefId',axis=1))<prepare_output>
def process_date(df):
df['date'] = df['time'].dt.date
df['month'] = df['time'].dt.month
df['dayofweek'] = df['time'].dt.dayofweek
return df
Submission=pd.DataFrame(data=predict,columns=['IsBadBuy'])
 Submission.head()<prepare_output>
def process_ma(df,columns=['open','close','volume'], windows=[5, 20, 60]):
ma_columns = []
ma_dev_columns = []
for col in columns:
for window in windows:
ma_column = 'ma_{0}_{1}'.format(col,window)
 ma_dev_column = 'ma_dev_{0}_{1}'.format(col,window)
 std_column = 'std_{0}_{1}'.format(col,window)
 ma_columns.append(ma_column)
 ma_dev_columns.append(ma_dev_column)
 df[ma_column] = df.groupby('assetCode')[col].apply(lambda x: x.rolling(window ).mean())
 df[ma_dev_column] = df[col] / df[ma_column] - 1
df[std_column] = df.groupby('assetCode')[col].apply(lambda x: x.rolling(window ).std())
 return df
Submission['RefId']=test_data['RefId']
Submission.set_index('RefId',inplace=True )<save_to_csv>
market_train = process_date(market_train)
 market_train = process_ma(market_train)
Submission.head()
Submission.to_csv('Submission.csv' )<load_from_csv>
df = market_train.merge(news_agg, how='left', on=['assetCode', 'date'])
 del market_train, news_agg
gc.collect()
df.head(3 )
train_data=pd.read_csv('/kaggle/input/DontGetKicked/training.csv')
 train_data.head()<load_from_csv>
def custom_metric(date, pred_proba, num_target, universe):
y = pred_proba*2 - 1
r = num_target.clip(-1,1)
 x = y * r * universe
result = pd.DataFrame({'day' : date, 'x' : x})
 x_t = result.groupby('day' ).sum().values
return np.mean(x_t)/ np.std(x_t )
test_data=pd.read_csv('/kaggle/input/DontGetKicked/test.csv')
 test_data.head()<count_missing_values>
date = df.date
num_target = df.returnsOpenNextMktres10.astype('float32')
 bin_target =(df.returnsOpenNextMktres10 >= 0 ).astype('int8')
 universe = df.universe.astype('int8')
 df.drop(['returnsOpenNextMktres10', 'date', 'universe', 'assetCode', 'assetName', 'time'],
axis=1, inplace=True)
 df = df.astype('float32')
 gc.collect()
train_data.isnull().sum()<count_missing_values>
train_index, test_index = train_test_split(df.index.values, test_size=0.1 )
test_data.isnull().sum()<count_values>
def evaluate_model(df, target, train_index, test_index, params):
model = LGBMClassifier(**params)
 model.fit(df.iloc[train_index], target.iloc[train_index])
 return log_loss(target.iloc[test_index], model.predict_proba(df.iloc[test_index]))
test_data["Transmission"].value_counts()<rename_columns>
param_grid = {
'learning_rate': [0.05, 0.02, 0.01],
'num_leaves': [25, 38, 63],
'n_estimators': [100, 200, 400],
'min_child_samples': [5, 10, 20, 40, 100],
'colsample_bytree': [0.8, 0.9, 1],
'subsample': [0.8, 0.9, 1],
'reg_alpha': [0.1, 0.2, 0.4, 0.6, 0.8],
'reg_lambda': [0.1, 0.2, 0.4, 0.6, 0.8],
}
best_eval_score = 0
for i in range(50):
params = {k: np.random.choice(v)for k, v in param_grid.items() }
score = evaluate_model(df, bin_target, train_index, test_index, params)
 if score < best_eval_score or best_eval_score == 0:
best_eval_score = score
best_params = params
print(best_eval_score)
 print("Best evaluation logloss", best_eval_score )
train_data["Transmission"].replace("Manual","MANUAL",inplace=True )<data_type_conversions>
clf = LGBMClassifier(**best_params)
 clf.fit(df, bin_target )
train_data["Color"].fillna(value="Color_unk",inplace=True )<data_type_conversions>
test_df_columns = ['time', 'assetCode', 'assetName', 'volume', 'close', 'open',
'returnsClosePrevRaw1', 'returnsOpenPrevRaw1',
'returnsClosePrevMktres1', 'returnsOpenPrevMktres1',
'returnsClosePrevRaw10', 'returnsOpenPrevRaw10',
'returnsClosePrevMktres10', 'returnsOpenPrevMktres10']
base_df = market_train_df[market_train_df['time'] >= '2016-10-01']
base_df = base_df[test_df_columns]
base_df['id'] = -1
base_df.shape
test_data["Color"].fillna(value="Color_unk",inplace=True )<count_values>
def write_submission(model, env):
days = env.get_prediction_days()
day_id = 0
market_obs_df_append = None
for(market_obs_df, news_obs_df, predictions_template_df)in days:
news_obs_df = preprocess_news(news_obs_df)
 index_df = unstack_asset_codes(news_obs_df)
 news_unstack = merge_news_on_index(news_obs_df, index_df)
 news_obs_agg = group_news(news_unstack)
 market_obs_df['id'] = day_id
if market_obs_df_append is None:
market_obs_df_append = base_df
market_obs_df_append = pd.concat([market_obs_df_append,market_obs_df],
ignore_index=True,
sort=False)
 market_obs_process = process_date(market_obs_df_append)
 market_obs_process = process_ma(market_obs_process)
 market_obs_df = market_obs_process[market_obs_process['id']==day_id]
obs_df = market_obs_df.merge(news_obs_agg, how='left', on=['assetCode', 'date'])
 del market_obs_df, news_obs_agg, news_obs_df, news_unstack, index_df
gc.collect()
obs_df = obs_df[obs_df.assetCode.isin(predictions_template_df.assetCode)]
feats = [c for c in obs_df.columns if c not in ['date', 'assetCode', 'assetName', 'time', 'id']]
preds = model.predict_proba(obs_df[feats])[:, 1] * 2 - 1
sub = pd.DataFrame({'assetCode': obs_df['assetCode'], 'confidence': preds})
 predictions_template_df = predictions_template_df.merge(sub, how='left' ).drop(
 'confidenceValue', axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 if day_id == 59:
market_obs_df_append.drop(
 market_obs_df_append.index[market_obs_df_append['id']==-1],
inplace=True)
 elif day_id >= 60:
market_obs_df_append.drop(
 market_obs_df_append.index[market_obs_df_append['id']==day_id-60],
inplace=True)
 day_id += 1
del obs_df, predictions_template_df, preds, sub
gc.collect()
env.write_submission_file()
print('day_count',day_id )
<data_type_conversions><EOS> 
write_submission(clf, env )