Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<merge>
env = twosigmanews.make_env()
ts = time.time()
group = train.groupby(['date_block_num','shop_id'] ).agg({'revenue': ['sum']})
 group.columns = ['date_shop_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(['shop_id'] ).agg({'date_shop_revenue': ['mean']})
 group.columns = ['shop_avg_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['shop_id'], how='left')
 matrix['shop_avg_revenue'] = matrix['shop_avg_revenue'].astype(np.float32)
 matrix['delta_revenue'] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix['delta_revenue'] = matrix['delta_revenue'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'delta_revenue')
 matrix.drop(['date_shop_revenue','shop_avg_revenue','delta_revenue'], axis=1, inplace=True)
 time.time() - ts<feature_engineering>
def prepare_market_data(market_df):
market_df['ratio'] = market_df['close'] / market_df['open']
market_df['average'] =(market_df['close'] + market_df['open'])/2
market_df['pricevolume'] = market_df['volume'] * market_df['close']
market_df.drop(['assetName', 'volume'], axis=1, inplace=True)
 return market_df
matrix['month'] = matrix['date_block_num'] % 12<categorify>
def prepare_news_data(news_df):
news_df['position'] = news_df['firstMentionSentence'] / news_df['sentenceCount']
news_df['coverage'] = news_df['sentimentWordCount'] / news_df['wordCount']
droplist = ['sourceTimestamp','firstCreated','sourceId','headline','takeSequence','provider','firstMentionSentence',
'sentenceCount','bodySize','headlineTag','marketCommentary','subjects','audiences','sentimentClass',
'assetName', 'urgency','wordCount','sentimentWordCount']
news_df.drop(droplist, axis=1, inplace=True)
 assets = []
indices = []
for i, values in news_df['assetCodes'].iteritems() :
assetCodes = eval(values)
 assets.extend(assetCodes)
 indices.extend([i]*len(assetCodes))
mapping_df = pd.DataFrame({'news_index': indices, 'assetCode': assets})
 del assets, indices
news_df['news_index'] = news_df.index.copy()
expanded_news_df = mapping_df.merge(news_df, how='left', on='news_index')
 del mapping_df, news_df
expanded_news_df.drop(['news_index', 'assetCodes'], axis=1, inplace=True)
 return expanded_news_df.groupby(['time', 'assetCode'] ).mean().reset_index()
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 matrix['days'] = matrix['month'].map(days ).astype(np.int8 )<data_type_conversions>
def prepare_data(market_df, news_df, start=None):
market_df['time'] = market_df['time'].dt.date
news_df['time'] = news_df['time'].dt.date
if start is not None:
market_df = market_df[market_df['time'] >= start].reset_index(drop=True)
 news_df = news_df[news_df['time'] >= start].reset_index(drop=True)
 market_df = prepare_market_data(market_df)
 news_df = prepare_news_data(news_df)
 return market_df.merge(news_df, how='left', on=['assetCode', 'time'] ).fillna(0 )
ts = time.time()
cache = {}
matrix['item_shop_last_sale'] = -1
matrix['item_shop_last_sale'] = matrix['item_shop_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = str(row.item_id)+' '+str(row.shop_id)
 if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
matrix.at[idx, 'item_shop_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts<data_type_conversions>
( market_df, news_df)= env.get_training_data()
print('preparing data...')
 start = datetime.date(2009,1,1)
 merged_df = prepare_data(market_df, news_df, start)
 print('Ready!' )
ts = time.time()
cache = {}
matrix['item_last_sale'] = -1
matrix['item_last_sale'] = matrix['item_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = row.item_id
if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
if row.date_block_num>last_date_block_num:
matrix.at[idx, 'item_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts<feature_engineering>
train_columns = [x for x in merged_df.columns if x not in ['assetCode', 'time', 'returnsOpenNextMktres10', 'universe']]
X_train = merged_df[train_columns].values
y_train =(merged_df.returnsOpenNextMktres10 >= 0 ).astype(int ).values
ts = time.time()
matrix['item_shop_first_sale'] = matrix['date_block_num'] - matrix.groupby(['item_id','shop_id'])['date_block_num'].transform('min')
 matrix['item_first_sale'] = matrix['date_block_num'] - matrix.groupby('item_id')['date_block_num'].transform('min')
 time.time() - ts<filter>
print('Training XGBoost')
 t = time.time()
xgb_market = XGBClassifier(n_jobs=4, n_estimators=200, max_depth=8, eta=0.05)
 xgb_market.fit(X_train, y_train)
 print(f'Done, time = {time.time() - t}s' )
ts = time.time()
matrix = matrix[matrix.date_block_num > 11]
time.time() - ts<correct_missing_values>
print("generating predictions...")
 days = env.get_prediction_days()
for market_df, news_df, pred_template_df in days:
test_df = prepare_data(market_df, news_df, start)
 X_test = test_df[train_columns].values
preds = xgb_market.predict_proba(X_test)[:,1] * 2 - 1
preds_df = pd.DataFrame({'assetCode':test_df['assetCode'],'confidenceValue':preds})
 env.predict(preds_df)
 env.write_submission_file()