Kaggle competition: Two Sigma: Using News to Predict Stock Movements
<SOS> metric: custom Kaggle data source: two-sigma-financial-news<feature_engineering>
from kaggle.competitions import twosigmanews
from datetime import datetime, date
import numpy as np
from sklearn import model_selection
submission["RefId"]=test_data["RefId"]<prepare_output>
env = twosigmanews.make_env() 
(market_train_df, news_train_df)= env.get_training_data()
market_train, news_train = market_train_df.copy() , news_train_df.copy()
submission.set_index("RefId",inplace=True )<save_to_csv>
def fill_in(data):
for i in data.columns:
if data[i].dtype == "object":
data[i] = data[i].fillna("")
 elif(data[i].dtype == "int64" or data[i].dtype == "float64"):
data[i] = data[i].fillna(data[i].mean())
 else:
pass
return data
market_train_df = fill_in(market_train_df )
submission.to_csv("submission.csv" )<load_from_csv>
def data_other(market_train_df):
market_train_df.time = market_train_df.time.dt.date
lbl = {k: v for v, k in enumerate(market_train_df['assetCode'].unique())}
market_train_df['assetCodeT'] = market_train_df['assetCode'].map(lbl)
 market_train_df = market_train_df.dropna(axis=0)
 return market_train_df
market_train_df = data_other(market_train_df )
items = pd.read_csv('.. /input/items.csv')
 shops = pd.read_csv('.. /input/shops.csv')
 cats = pd.read_csv('.. /input/item_categories.csv')
 train = pd.read_csv('.. /input/sales_train.csv')
 test = pd.read_csv('.. /input/test.csv' ).set_index('ID' )<filter>
market_train_df = market_train_df.loc[market_train_df['time']>=date(2009, 1, 1)]
train = train[train.item_price<100000]
train = train[train.item_cnt_day<1001]<feature_engineering>
green = market_train_df.returnsOpenNextMktres10 > 0
green = green.values
median = train[(train.shop_id==32)&(train.item_id==2973)&(train.date_block_num==4)&(train.item_price>0)].item_price.median()
train.loc[train.item_price<0, 'item_price'] = median<feature_engineering>
fcol = [c for c in market_train_df if c not in ['assetCode', 'assetCodes', 'assetCodesLen', 'assetName', 'audiences',
'firstCreated', 'headline', 'headlineTag', 'marketCommentary', 'provider',
'returnsOpenNextMktres10', 'sourceId', 'subjects', 'time', 'time_x', 'universe','sourceTimestamp']]
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11<categorify>
X = market_train_df[fcol].values
mins = np.min(X, axis=0)
 maxs = np.max(X, axis=0)
 rng = maxs - mins
X = 1 -(( maxs - X)/ rng )
shops.loc[shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 shops = shops[['shop_id','city_code']]
cats['split'] = cats['item_category_name'].str.split('-')
 cats['type'] = cats['split'].map(lambda x: x[0].strip())
 cats['type_code'] = LabelEncoder().fit_transform(cats['type'])
 cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id','type_code', 'subtype_code']]
items.drop(['item_name'], axis=1, inplace=True )<concatenate>
all_10day = market_train_df.returnsOpenNextMktres10.values
len(list(set(test.item_id)- set(test.item_id ).intersection(set(train.item_id)))) , len(list(set(test.item_id))), len(test )<data_type_conversions>
X_train, X_test, green_train, green_test, all_train, all_test = model_selection.train_test_split(
 X, green, all_10day, test_size=0.20, random_state=59 )
ts = time.time()
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True)
 time.time() - ts<feature_engineering>
train_data = lgb.Dataset(X_train, label=green_train.astype(int))
test_data = lgb.Dataset(X_test, label=green_test.astype(int))
train['revenue'] = train['item_price'] * train['item_cnt_day']<merge>
x_1 = [0.19000424246380565, 2452, 212, 328, 202]
x_2 = [0.19016805202090095, 2583, 213, 312, 220]
ts = time.time()
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
time.time() - ts<data_type_conversions>
params_1 = {
'task': 'train',
'boosting_type': 'dart',
'objective': 'binary',
'learning_rate': x_1[0],
'num_leaves': x_1[1],
'min_data_in_leaf': x_1[2],
'num_iteration': x_1[3],
'max_bin': x_1[4],
'verbose': 1
}
params_2 = {
'task': 'train',
'boosting_type': 'dart',
'objective': 'binary',
'learning_rate': x_2[0],
'num_leaves': x_2[1],
'min_data_in_leaf': x_2[2],
'num_iteration': x_2[3],
'max_bin': x_2[4],
'verbose': 1
}
gbm_1 = lgb.train(params_1,
train_data,
num_boost_round=100,
valid_sets=test_data,
early_stopping_rounds=5)
 gbm_2 = lgb.train(params_2,
train_data,
num_boost_round=100,
valid_sets=test_data,
early_stopping_rounds=5 )
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16 )<concatenate>
days = env.get_prediction_days()
n_days = 0
prep_time = 0
prediction_time = 0
packaging_time = 0
ts = time.time()
matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True)
 time.time() - ts<data_type_conversions>
for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
if(n_days%50==0):
print(n_days,end=' ')
 market_obs_df = fill_in(market_obs_df)
 market_obs_df = data_other(market_obs_df)
 market_obs_df = market_obs_df[market_obs_df.assetCode.isin(predictions_template_df.assetCode)]
X_live = market_obs_df[fcol].values
X_live = 1 -(( maxs - X_live)/ rng)
 lp =(gbm_1.predict(X_live)+ gbm_2.predict(X_live)) /2
confidence = lp
confidence =(confidence-confidence.min())/(confidence.max() -confidence.min())
 confidence = confidence * 2 - 1
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence':confidence})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df )
<merge><EOS> 
env.write_submission_file()