Kaggle competition: ASHRAE - Great Energy Predictor III
df_sub.to_csv('submission.csv',index=False )<set_options>
py.init_notebook_mode(connected=True)
 pd.set_option('max_columns', 150)
 py.init_notebook_mode(connected=True)
 init_notebook_mode(connected=True)
env = twosigmanews.make_env()<feature_engineering>
metadata_dtype = {'site_id':"uint8",'building_id':'uint16','square_feet':'float32','year_built':'float32','floor_count':"float16"}
weather_dtype = {"site_id":"uint8",'air_temperature':"float16",'cloud_coverage':"float16",'dew_temperature':"float16",'precip_depth_1_hr':"float16",
'sea_level_pressure':"float32",'wind_direction':"float16",'wind_speed':"float16"}
train_dtype = {'meter':"uint8",'building_id':'uint16'}
def prepare_market_data(market_df):
market_df['ratio'] = market_df['close'] / market_df['open']
market_df['average'] =(market_df['close'] + market_df['open'])/2
market_df['pricevolume'] = market_df['volume'] * market_df['close']
market_df.drop(['assetName', 'volume'], axis=1, inplace=True)
 return market_df<drop_column>
%%time
weather_train = pd.read_csv(".. /input/ashrae-energy-prediction/weather_train.csv", parse_dates=['timestamp'], dtype=weather_dtype)
 weather_test = pd.read_csv(".. /input/ashrae-energy-prediction/weather_test.csv", parse_dates=['timestamp'], dtype=weather_dtype)
 metadata = pd.read_csv(".. /input/ashrae-energy-prediction/building_metadata.csv", dtype=metadata_dtype)
 train = pd.read_csv(".. /input/ashrae-energy-prediction/train.csv", parse_dates=['timestamp'], dtype=train_dtype)
 test = pd.read_csv(".. /input/ashrae-energy-prediction/test.csv", parse_dates=['timestamp'], usecols=['building_id','meter','timestamp'], dtype=train_dtype)
 print('Size of train_df data', train.shape)
 print('Size of weather_train_df data', weather_train.shape)
 print('Size of weather_test_df data', weather_test.shape)
 print('Size of building_meta_df data', metadata.shape )
def prepare_news_data(news_df):
news_df['position'] = news_df['firstMentionSentence'] / news_df['sentenceCount']
news_df['coverage'] = news_df['sentimentWordCount'] / news_df['wordCount']
droplist = ['sourceTimestamp','firstCreated','sourceId','headline','takeSequence','provider','firstMentionSentence',
'sentenceCount','bodySize','headlineTag','marketCommentary','subjects','audiences','sentimentClass',
'assetName', 'urgency','wordCount','sentimentWordCount']
news_df.drop(droplist, axis=1, inplace=True)
 assets = []
indices = []
for i, values in news_df['assetCodes'].iteritems() :
assetCodes = eval(values)
 assets.extend(assetCodes)
 indices.extend([i]*len(assetCodes))
mapping_df = pd.DataFrame({'news_index': indices, 'assetCode': assets})
 del assets, indices
news_df['news_index'] = news_df.index.copy()
expanded_news_df = mapping_df.merge(news_df, how='left', on='news_index')
 del mapping_df, news_df
expanded_news_df.drop(['news_index', 'assetCodes'], axis=1, inplace=True)
 return expanded_news_df.groupby(['time', 'assetCode'] ).mean().reset_index()<merge>
train['meter'].replace({0:"Electricity",1:"ChilledWater",2:"Steam",3:"HotWater"},inplace=True)
 test['meter'].replace({0:"Electricity",1:"ChilledWater",2:"Steam",3:"HotWater"},inplace=True )
def prepare_data(market_df, news_df, start=None):
market_df['time'] = market_df['time'].dt.date
news_df['time'] = news_df['time'].dt.date
if start is not None:
market_df = market_df[market_df['time'] >= start].reset_index(drop=True)
 news_df = news_df[news_df['time'] >= start].reset_index(drop=True)
 market_df = prepare_market_data(market_df)
 news_df = prepare_news_data(news_df)
 return market_df.merge(news_df, how='left', on=['assetCode', 'time'] ).fillna(0 )<categorify>
metadata.drop('floor_count',axis=1,inplace=True )
( market_df, news_df)= env.get_training_data()
print('preparing data...')
 start = datetime.date(2009,1,1)
 merged_df = prepare_data(market_df, news_df, start)
 print('Ready!' )<split>
for df in [train, test]:
df['Month'] = df['timestamp'].dt.month.astype("uint8")
 df['DayOfMonth'] = df['timestamp'].dt.day.astype("uint8")
 df['DayOfWeek'] = df['timestamp'].dt.dayofweek.astype("uint8")
 df['Hour'] = df['timestamp'].dt.hour.astype("uint8" )
train_columns = [x for x in merged_df.columns if x not in ['assetCode', 'time', 'returnsOpenNextMktres10', 'universe']]
X_train = merged_df[train_columns].values
y_train =(merged_df.returnsOpenNextMktres10 >= 0 ).astype(int ).values 
<train_model>
train['meter_reading'] = np.log1p(train['meter_reading'] )
print('Training XGBoost')
 t = time.time()
xgb_market = XGBClassifier(n_jobs=4, n_estimators=200, max_depth=8, eta=0.05)
 xgb_market.fit(X_train, y_train)
 print(f'Done, time = {time.time() - t}s' )<predict_on_test>
metadata['primary_use'].replace({"Healthcare":"Other","Parking":"Other","Warehouse/storage":"Other","Manufacturing/industrial":"Other",
"Retail":"Other","Services":"Other","Technology/science":"Other","Food sales and service":"Other",
"Utility":"Other","Religious worship":"Other"},inplace=True)
 metadata['square_feet'] = np.log1p(metadata['square_feet'])
 metadata['year_built'].fillna(-999, inplace=True)
 metadata['year_built'] = metadata['year_built'].astype('int16' )
print("generating predictions...")
 days = env.get_prediction_days()
for market_df, news_df, pred_template_df in days:
test_df = prepare_data(market_df, news_df, start)
 X_test = test_df[train_columns].values
preds = xgb_market.predict_proba(X_test)[:,1] * 2 - 1
preds_df = pd.DataFrame({'assetCode':test_df['assetCode'],'confidenceValue':preds})
 env.predict(preds_df)
 env.write_submission_file()<import_modules>
%%time
train = pd.merge(train,metadata,on='building_id',how='left')
 test = pd.merge(test,metadata,on='building_id',how='left')
 print("Training Data+Metadata Shape {}".format(train.shape))
print("Testing Data+Metadata Shape {}".format(test.shape))
gc.collect()
train = pd.merge(train,weather_train,on=['site_id','timestamp'],how='left')
 test = pd.merge(test,weather_test,on=['site_id','timestamp'],how='left')
 print("Training Data+Metadata+Weather Shape {}".format(train.shape))
print("Testing Data+Metadata+Weather Shape {}".format(test.shape))
gc.collect()
import numpy as np
import lightgbm as lgb
import pandas as pd
from kaggle.competitions import twosigmanews
import matplotlib.pyplot as plt
import random
from datetime import datetime, date
from xgboost import XGBClassifier
from sklearn import model_selection
from sklearn.metrics import mean_squared_error
import time
import math
import gc
from statsmodels.nonparametric.smoothers_lowess import lowess
from statsmodels.graphics.tsaplots import plot_acf, plot_pacf
import inspect, sys
from multiprocessing import Pool<define_variables>
for df in [train,test]:
df['square_feet'] = df['square_feet'].astype('float16')
 cols = ['air_temperature','cloud_coverage','dew_temperature','precip_depth_1_hr','sea_level_pressure','wind_direction','wind_speed']
for col in cols:
train[col].fillna(train[col].mean() ,inplace=True)
 test[col].fillna(test[col].mean() ,inplace=True)
 idx_to_drop = list(( train[(train['site_id'] == 0)&(train['timestamp'] < "2016-05-21 00:00:00")] ).index)
 print(len(idx_to_drop))
train.drop(idx_to_drop,axis='rows',inplace=True)
 idx_to_drop = list(train[(train['meter'] == "Electricity")&(train['meter_reading'] == 0)].index)
 print(len(idx_to_drop))
train.drop(idx_to_drop,axis='rows',inplace=True )
dic_assetName={}
dic_assetNameSNo={}
dic_assetCodes={}
dic_assetCode={}
dic_assetCodeSNo={}
dic_rng = {}
dic_min = {}
set_sector = {"TECH","ENER","BMAT","INDS","CYCS","NCYC","FINS","HECA","TECH","TCOM","UTIL"}
dic_sector_code_name = {"ENER":"Energy", "BMAT":"Basic Materials", "INDS":"Industrials", "CYCS":"Cyclical Consumer Goods & Services", \
"NCYC":"Non Cyclical Consumer Goods & Services", "FINS": "Financials", "HECA":"Healthcare","TECH":"Technology", \
"TCOM":"Telecommunication Services", "UTIL":"Utilities"}
sector_col = list(set_sector)
 sector_col.append("assetNameSNo")
 dic_sectorSNo = {v: k for v,k in enumerate(list(set_sector)) }
dic_sector = {k: v for v,k in enumerate(list(set_sector)) }
dic_sectorSNo["-1"] = "Unknown"
dic_sector["Unknown"] = -1
dic_asset_sector = {}
n_days = 0
prep_time = 0
prediction_time = 0
packaging_time = 0
total_market_obs_df = []
total_news_obs_df = []
return_features = ['returnsClosePrevMktres10','returnsClosePrevRaw10',"SenSR"]
sector_return_features = ["weighted_SenSR"]
n_lag = [3,7,14]
debug_func = set()
prv_mkt_data = []
prv_news_data = []<set_options>
%%time
number_unique_meter_per_building = train.groupby('building_id')['meter'].nunique()
train['number_unique_meter_per_building'] = train['building_id'].map(number_unique_meter_per_building)
 mean_meter_reading_per_building = train.groupby('building_id')['meter_reading'].mean()
train['mean_meter_reading_per_building'] = train['building_id'].map(mean_meter_reading_per_building)
 median_meter_reading_per_building = train.groupby('building_id')['meter_reading'].median()
train['median_meter_reading_per_building'] = train['building_id'].map(median_meter_reading_per_building)
 std_meter_reading_per_building = train.groupby('building_id')['meter_reading'].std()
train['std_meter_reading_per_building'] = train['building_id'].map(std_meter_reading_per_building)
 mean_meter_reading_on_year_built = train.groupby('year_built')['meter_reading'].mean()
train['mean_meter_reading_on_year_built'] = train['year_built'].map(mean_meter_reading_on_year_built)
 median_meter_reading_on_year_built = train.groupby('year_built')['meter_reading'].median()
train['median_meter_reading_on_year_built'] = train['year_built'].map(median_meter_reading_on_year_built)
 std_meter_reading_on_year_built = train.groupby('year_built')['meter_reading'].std()
train['std_meter_reading_on_year_built'] = train['year_built'].map(std_meter_reading_on_year_built)
 mean_meter_reading_per_meter = train.groupby('meter')['meter_reading'].mean()
train['mean_meter_reading_per_meter'] = train['meter'].map(mean_meter_reading_per_meter)
 median_meter_reading_per_meter = train.groupby('meter')['meter_reading'].median()
train['median_meter_reading_per_meter'] = train['meter'].map(median_meter_reading_per_meter)
 std_meter_reading_per_meter = train.groupby('meter')['meter_reading'].std()
train['std_meter_reading_per_meter'] = train['meter'].map(std_meter_reading_per_meter)
 mean_meter_reading_per_primary_usage = train.groupby('primary_use')['meter_reading'].mean()
train['mean_meter_reading_per_primary_usage'] = train['primary_use'].map(mean_meter_reading_per_primary_usage)
 median_meter_reading_per_primary_usage = train.groupby('primary_use')['meter_reading'].median()
train['median_meter_reading_per_primary_usage'] = train['primary_use'].map(median_meter_reading_per_primary_usage)
 std_meter_reading_per_primary_usage = train.groupby('primary_use')['meter_reading'].std()
train['std_meter_reading_per_primary_usage'] = train['primary_use'].map(std_meter_reading_per_primary_usage)
 mean_meter_reading_per_site_id = train.groupby('site_id')['meter_reading'].mean()
train['mean_meter_reading_per_site_id'] = train['site_id'].map(mean_meter_reading_per_site_id)
 median_meter_reading_per_site_id = train.groupby('site_id')['meter_reading'].median()
train['median_meter_reading_per_site_id'] = train['site_id'].map(median_meter_reading_per_site_id)
 std_meter_reading_per_site_id = train.groupby('site_id')['meter_reading'].std()
train['std_meter_reading_per_site_id'] = train['site_id'].map(std_meter_reading_per_site_id)
 test['number_unique_meter_per_building'] = test['building_id'].map(number_unique_meter_per_building)
 test['mean_meter_reading_per_building'] = test['building_id'].map(mean_meter_reading_per_building)
 test['median_meter_reading_per_building'] = test['building_id'].map(median_meter_reading_per_building)
 test['std_meter_reading_per_building'] = test['building_id'].map(std_meter_reading_per_building)
 test['mean_meter_reading_on_year_built'] = test['year_built'].map(mean_meter_reading_on_year_built)
 test['median_meter_reading_on_year_built'] = test['year_built'].map(median_meter_reading_on_year_built)
 test['std_meter_reading_on_year_built'] = test['year_built'].map(std_meter_reading_on_year_built)
 test['mean_meter_reading_per_meter'] = test['meter'].map(mean_meter_reading_per_meter)
 test['median_meter_reading_per_meter'] = test['meter'].map(median_meter_reading_per_meter)
 test['std_meter_reading_per_meter'] = test['meter'].map(std_meter_reading_per_meter)
 test['mean_meter_reading_per_primary_usage'] = test['primary_use'].map(mean_meter_reading_per_primary_usage)
 test['median_meter_reading_per_primary_usage'] = test['primary_use'].map(median_meter_reading_per_primary_usage)
 test['std_meter_reading_per_primary_usage'] = test['primary_use'].map(std_meter_reading_per_primary_usage)
 test['mean_meter_reading_per_site_id'] = test['site_id'].map(mean_meter_reading_per_site_id)
 test['median_meter_reading_per_site_id'] = test['site_id'].map(median_meter_reading_per_site_id)
 test['std_meter_reading_per_site_id'] = test['site_id'].map(std_meter_reading_per_site_id )
env = twosigmanews.make_env()
days = env.get_prediction_days()<filter>
%%time
for df in [train, test]:
df['mean_meter_reading_per_building'] = df['mean_meter_reading_per_building'].astype("float16")
 df['median_meter_reading_per_building'] = df['mean_meter_reading_per_building'].astype("float16")
 df['std_meter_reading_per_building'] = df['std_meter_reading_per_building'].astype("float16")
 df['mean_meter_reading_on_year_built'] = df['mean_meter_reading_on_year_built'].astype("float16")
 df['median_meter_reading_on_year_built'] = df['median_meter_reading_on_year_built'].astype("float16")
 df['std_meter_reading_on_year_built'] = df['std_meter_reading_on_year_built'].astype("float16")
 df['mean_meter_reading_per_meter'] = df['mean_meter_reading_per_meter'].astype("float16")
 df['median_meter_reading_per_meter'] = df['median_meter_reading_per_meter'].astype("float16")
 df['std_meter_reading_per_meter'] = df['std_meter_reading_per_meter'].astype("float16")
 df['mean_meter_reading_per_primary_usage'] = df['mean_meter_reading_per_primary_usage'].astype("float16")
 df['median_meter_reading_per_primary_usage'] = df['median_meter_reading_per_primary_usage'].astype("float16")
 df['std_meter_reading_per_primary_usage'] = df['std_meter_reading_per_primary_usage'].astype("float16")
 df['mean_meter_reading_per_site_id'] = df['mean_meter_reading_per_site_id'].astype("float16")
 df['median_meter_reading_per_site_id'] = df['median_meter_reading_per_site_id'].astype("float16")
 df['std_meter_reading_per_site_id'] = df['std_meter_reading_per_site_id'].astype("float16")
 df['number_unique_meter_per_building'] = df['number_unique_meter_per_building'].astype('uint8')
 gc.collect()
( market_train_df, news_train_df)= env.get_training_data() 
<feature_engineering>
train.drop('timestamp',axis=1,inplace=True)
 test.drop('timestamp',axis=1,inplace=True)
 le = LabelEncoder()
train['meter']= le.fit_transform(train['meter'] ).astype("uint8")
 test['meter']= le.fit_transform(test['meter'] ).astype("uint8")
 train['primary_use']= le.fit_transform(train['primary_use'] ).astype("uint8")
 test['primary_use']= le.fit_transform(test['primary_use'] ).astype("uint8")
 print(train.shape, test.shape )
if 1==2:
prv_time_list = market_train_df["time"].dt.date.unique() [-14:]
prv_time_list.sort()
prv_time_list
for prv_time in prv_time_list:
prv_mkt_data.append(market_train_df[market_train_df["time"].dt.date == prv_time])
 prv_news_data.append(news_train_df[news_train_df["time"].dt.date == prv_time] )<feature_engineering>
%%time
y = train['meter_reading']
train.drop('meter_reading',axis=1,inplace=True)
 categorical_cols = ['building_id','Month','meter','Hour','primary_use','DayOfWeek','DayOfMonth']
def fixassetopenclose(in_asset_df):
in_asset_df["final_close"] = in_asset_df["close"]
in_asset_df["final_open"] = in_asset_df["open"]
in_asset_df["adjusted_close"] = in_asset_df["close"].shift(-1)/(1 + in_asset_df["returnsClosePrevRaw1"].shift(-1))
in_asset_df["ratio"] = in_asset_df["close"]/in_asset_df["adjusted_close"]
idx_list = in_asset_df[(in_asset_df["ratio"].round(1)<0.9)|(in_asset_df["ratio"].round(1)>1.1)].index
for idx in idx_list:
affected_idx = in_asset_df[in_asset_df.index <= idx].index
ratio = in_asset_df.loc[idx,"ratio"]
in_asset_df.loc[affected_idx,["final_close","final_open"]] = in_asset_df.loc[affected_idx,["final_close","final_open"]] / ratio
del affected_idx
changed_recs = in_asset_df[in_asset_df["final_close"] != in_asset_df["close"]].index
return in_asset_df.loc[changed_recs,["final_close","final_open"]]
def fixopenclose(in_mkt_df):
assetlist = []
changedlist = []
df_group = in_mkt_df.groupby(["assetCode"])
 for group in df_group:
assetlist.append(group[1])
 pool = Pool(4)
 changedlist = pool.map(fixassetopenclose, assetlist)
 pool.close()
changed_df = pd.concat(changedlist)
 changed_df.dropna(inplace=True)
 in_mkt_df.loc[changed_df.index,["close"]] = list(changed_df["final_close"])
 in_mkt_df.loc[changed_df.index,["open"]] = list(changed_df["final_open"])
 fixopenclose(market_train_df )<define_variables>
x_train,x_test,y_train,y_test = train_test_split(train,y,test_size=0.2,random_state=0, stratify=meter_cut)
 print(x_train.shape)
 print(y_train.shape)
 print(x_test.shape)
 print(y_test.shape )
def findnulls(in_mkt_df, in_news_df):
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_mkt_cols = set(in_mkt_df.columns)
 org_news_cols = set(in_news_df.columns)
 for col in in_mkt_df:
cols = in_mkt_df[col]
if fname in debug_func:
print("Market df: Num nulls in " + col + " is " + str(cols[cols.isna() ].shape[0])+ " out of " + str(cols.shape[0]))
for col in in_news_df:
cols = in_news_df[col]
if fname in debug_func:
print("Market df: Num nulls in " + col + " is " + str(cols[cols.isna() ].shape[0])+ " out of " + str(cols.shape[0]))
if fname in debug_func:
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------" )<feature_engineering>
from sklearn.ensemble import RandomForestRegressor as RF
import lightgbm as lgb
dic_time = {}
def fix_time_col(in_mkt_df, in_news_df):
global dic_time
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_mkt_cols = set(in_mkt_df.columns)
 org_news_cols = set(in_news_df.columns)
 if "datetime" in str(in_mkt_df["time"].dtype):
in_mkt_df["time"] = in_mkt_df["time"].dt.date
if "datetime" in str(in_news_df["time"].dtype):
in_news_df["time"] = in_news_df["time"].dt.date
set_time = set(in_mkt_df["time"].unique())| set(in_news_df["time"].unique())
 lst_time = list(set_time)
 if len(dic_time)== 0:
dic_time= {k:v for v,k in enumerate(lst_time)}
else:
set_time_new = set_time - set(dic_time.keys())
 if fname in debug_func:
print("Need to get sno for " + str(len(set_time_new)) + " news time.")
 time_max_SNo = len(dic_time)
 dic_time_new = {k:v+time_max_SNo for v,k in enumerate(list(set_time_new)) }
dic_time.update(dic_time_new)
 in_mkt_df["timeSNo"] = in_mkt_df["time"].map(dic_time)
 in_news_df["timeSNo"] = in_news_df["time"].map(dic_time)
 if fname in debug_func:
print("mkt cols dropped: " + str(org_mkt_cols - set(in_mkt_df.columns)))
 print("new mkt cols added: " + str(set(in_mkt_df.columns)- org_mkt_cols))
print("news cols dropped: " + str(org_news_cols - set(in_news_df.columns)))
 print("new news cols added: " + str(set(in_news_df.columns)- org_news_cols))
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------")
 def clean_assetName_col(in_mkt_df, in_news_df):
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_mkt_cols = set(in_mkt_df.columns)
 org_news_cols = set(in_news_df.columns)
 in_mkt_df["assetName"] = in_mkt_df["assetName"].str.lower()
in_mkt_df["assetName"].replace({"inc":"","llc":"","ltd":""}, inplace=True)
 in_mkt_df["assetName"] = in_mkt_df["assetName"].str.strip()
in_news_df["assetName"] = in_news_df["assetName"].str.lower()
in_news_df["assetName"].replace({"inc":"","llc":"","ltd":""}, inplace=True)
 in_news_df["assetName"] = in_news_df["assetName"].str.strip()
if fname in debug_func:
print("mkt cols dropped: " + str(org_mkt_cols - set(in_mkt_df.columns)))
 print("new mkt cols added: " + str(set(in_mkt_df.columns)- org_mkt_cols))
print("news cols dropped: " + str(org_news_cols - set(in_news_df.columns)))
 print("new news cols added: " + str(set(in_news_df.columns)- org_news_cols))
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------" )<define_variables>
lgb_train = lgb.Dataset(x_train, y_train,categorical_feature=categorical_cols)
 lgb_test = lgb.Dataset(x_test, y_test,categorical_feature=categorical_cols)
def get_final_assetName(row):
min_mkt_assetNameSNo = row["min_mkt_assetNameSNo"]
max_mkt_assetNameSNo = row["max_mkt_assetNameSNo"]
min_assetNameSNo = row["min_assetNameSNo"]
max_assetNameSNo = row["max_assetNameSNo"]
if(( not math.isnan(min_mkt_assetNameSNo)) &(math.isnan(max_mkt_assetNameSNo))):
return min_assetNameSNo
elif(( not math.isnan(max_mkt_assetNameSNo)) &(math.isnan(min_mkt_assetNameSNo))):
return max_assetNameSNo
elif(( math.isnan(max_mkt_assetNameSNo)) &(math.isnan(min_mkt_assetNameSNo))):
return max_assetNameSNo
else:
return -1
def get_replace_assetName(row):
min_mkt_assetNameSNo = row["min_mkt_assetNameSNo"]
max_mkt_assetNameSNo = row["max_mkt_assetNameSNo"]
min_assetNameSNo = row["min_assetNameSNo"]
max_assetNameSNo = row["max_assetNameSNo"]
if(( math.isnan(min_mkt_assetNameSNo)) &(not math.isnan(max_mkt_assetNameSNo))):
return min_assetNameSNo
elif(( math.isnan(max_mkt_assetNameSNo)) &(not math.isnan(min_mkt_assetNameSNo))):
return max_assetNameSNo
elif(( math.isnan(max_mkt_assetNameSNo)) &(math.isnan(min_mkt_assetNameSNo))):
return min_assetNameSNo
else:
return -1
def replace_assetName(in_mkt_df, in_news_df):
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_mkt_cols = set(in_mkt_df.columns)
 org_news_cols = set(in_news_df.columns)
 in_news_df["num_rows"] = 1
assetName_df = in_news_df[["assetCodesSNo","assetNameSNo","num_rows"]].groupby(\
["assetCodesSNo","assetNameSNo"], as_index=False ).count()
if fname in debug_func:
print("Distinct assetcodes, assetname combination: " + str(assetName_df.shape[0])+ " out of " + str(in_news_df.shape[0]))
assetCode_df = assetName_df[["assetCodesSNo", \
"assetNameSNo","num_rows"]].groupby(["assetCodesSNo"], \
as_index=False ).agg({"assetNameSNo": ["max","min"], "num_rows": "sum"})
 assetCode_df.columns = ["assetCodesSNo", "max_assetNameSNo", "min_assetNameSNo", "count_num_rows"]
assetCode_df = assetCode_df[assetCode_df["max_assetNameSNo"] != assetCode_df["min_assetNameSNo"]]
num_with_duplicate_assetName = assetCode_df.shape[0]
if fname in debug_func:
print("Rows with different assetname for same assetcode : " + str(num_with_duplicate_assetName)+ " out of " + str(assetName_df.shape[0]))
in_news_df.drop(["num_rows"], axis=1, inplace=True)
 if assetCode_df.shape[0] > 0:
assetCode_df["max_assetName"] = assetCode_df["max_assetNameSNo"].map(dic_assetNameSNo)
 assetCode_df["min_assetName"] = assetCode_df["min_assetNameSNo"].map(dic_assetNameSNo)
 list_mkt_asset = list(in_mkt_df["assetName"].unique())
 dic_mkt_assetName= {k:v for v,k in enumerate(list_mkt_asset)}
assetCode_df["min_mkt_assetNameSNo"] = assetCode_df["min_assetName"].map(dic_mkt_assetName)
 assetCode_df["max_mkt_assetNameSNo"] = assetCode_df["max_assetName"].map(dic_mkt_assetName)
 assetCode_df["final_assetNameSNo"] = assetCode_df.apply(get_final_assetName, axis=1)
 assetCode_df["replace_assetNameSNo"] = assetCode_df.apply(get_replace_assetName, axis=1)
 missing_in_mkt = assetCode_df[assetCode_df["final_assetNameSNo"]==-1].shape[0]
if fname in debug_func:
print("None of the asset names exist in market df for " + str(missing_in_mkt)+ " entries")
 if num_with_duplicate_assetName == missing_in_mkt:
df1 = assetCode_df[["assetCodesSNo","max_assetNameSNo"]]
df1.columns = ["assetCodesSNo","assetNameSNo"]
df2 = assetCode_df[["assetCodesSNo","min_assetNameSNo"]]
df2.columns = ["assetCodesSNo","assetNameSNo"]
tot_df = pd.concat([df1,df2])
 rows_to_be_dropped = pd.merge(in_news_df.reset_index() , tot_df, how="inner", on=["assetCodesSNo", "assetNameSNo"])
 print(rows_to_be_dropped.index.unique())
 print("dropping rows:" + str(rows_to_be_dropped.shape[0]))
in_news_df.drop(rows_to_be_dropped["index"].unique() , axis=0, inplace=True)
 assetCode_df["final_assetName"] = assetCode_df["final_assetNameSNo"].map(dic_assetNameSNo)
 assetCode_df = assetCode_df[assetCode_df["final_assetNameSNo"]!=-1]
df_temp = pd.merge(in_news_df.reset_index() [["index","assetCodesSNo","assetNameSNo"]], \
assetCode_df[["replace_assetNameSNo", "final_assetNameSNo","assetCodesSNo","final_assetName"]], \
how="inner", left_on=["assetCodesSNo","assetNameSNo"], right_on=["assetCodesSNo","replace_assetNameSNo"], left_index=True ).set_index("index")
 in_news_df.loc[df_temp.index, "assetNameSNo"] = list(df_temp["final_assetNameSNo"])
 in_news_df.loc[df_temp.index, "assetName"] = list(df_temp["final_assetName"])
 in_news_df["num_rows"] = 1
assetName_df = in_news_df[["assetCodesSNo","assetNameSNo","num_rows"]].groupby(\
["assetCodesSNo","assetNameSNo"], as_index=False ).count()
if fname in debug_func:
print("After replacement, Distinct assetcodes, assetname combination: " + str(assetName_df.shape[0])+ " out of " + str(in_news_df.shape[0]))
assetCode_df = assetName_df[["assetCodesSNo", \
"assetNameSNo","num_rows"]].groupby(["assetCodesSNo"], \
as_index=False ).agg({"assetNameSNo": ["max","min"], "num_rows": "sum"})
 assetCode_df.columns = ["assetCodesSNo", "max_assetNameSNo", "min_assetNameSNo", "count_num_rows"]
assetCode_df = assetCode_df[assetCode_df["max_assetNameSNo"] != assetCode_df["min_assetNameSNo"]]
if fname in debug_func:
print("After replacement, Rows with different assetname for same assetcode : " + str(assetCode_df.shape[0])+ " out of " + str(assetName_df.shape[0]))
in_news_df.drop(["num_rows"], axis=1, inplace=True)
 in_news_df.drop(["assetCodes","assetCodesSNo"], inplace=True, axis=1)
 if fname in debug_func:
print("mkt cols dropped: " + str(org_mkt_cols - set(in_mkt_df.columns)))
 print("new mkt cols added: " + str(set(in_mkt_df.columns)- org_mkt_cols))
print("news cols dropped: " + str(org_news_cols - set(in_news_df.columns)))
 print("new news cols added: " + str(set(in_news_df.columns)- org_news_cols))
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------")
<normalization>
del x_train, y_train
params = {'feature_fraction': 0.75,
'bagging_fraction': 0.75,
'objective': 'regression_l2',
'max_depth': -1,
'learning_rate': 0.01,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'rmse',
"verbosity": -1,
'reg_alpha': 0.5,
'reg_lambda': 0.5,
'random_state': 47
}
reg = lgb.train(params, lgb_train, num_boost_round=3000, valid_sets=[lgb_train, lgb_test], early_stopping_rounds=100, verbose_eval = 100 )
def scale_col(in_news_df, col):
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_news_cols = set(in_news_df.columns)
 global dic_rng
global dic_min
if col not in dic_rng.keys() :
col_max = in_news_df[col].max()
col_min = in_news_df[col].min()
col_rng = col_max - col_min
dic_rng[col] = col_rng
dic_min[col] = col_min
else:
col_min = dic_min[col]
col_rng = dic_rng[col]
in_news_df[col] =(in_news_df[col]-col_min)/col_rng
if col_min < 0:
in_news_df[col] = 2*in_news_df[col] - 1
print(col, in_news_df[col].min() , in_news_df[col].max())
 if fname in debug_func:
print("news cols dropped: " + str(org_news_cols - set(in_news_df.columns)))
 print("new news cols added: " + str(set(in_news_df.columns)- org_news_cols))
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------")
 def drop_extra_news_cols(in_news_df):
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_news_cols = set(in_news_df.columns)
 news_drop_cols = ["sourceTimestamp","firstCreated","sourceId","headline","urgency","takeSequence","provider","audiences","bodySize",\
"headlineTag","sentenceCount","wordCount","firstMentionSentence","sentimentClass","sentimentWordCount","companyCount", "marketCommentary",
"volumeCounts12H","volumeCounts24H","volumeCounts3D","volumeCounts5D","volumeCounts7D"]
news_drop_cols = [col for col in news_drop_cols if col in in_news_df.columns ]
in_news_df.drop(news_drop_cols, axis=1, inplace=True)
 if fname in debug_func:
print("news cols dropped: " + str(org_news_cols - set(in_news_df.columns)))
 print("new news cols added: " + str(set(in_news_df.columns)- org_news_cols))
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------")
 def drop_irrelevant_news(in_news_df):
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_news_cols = set(in_news_df.columns)
 if "relevance" in in_news_df.columns:
irrelevant_news = in_news_df[in_news_df["relevance"] < 0.3].index
print("Num low relevance news:", len(irrelevant_news))
in_news_df.drop(irrelevant_news, axis=0, inplace=True)
 del irrelevant_news
gc.collect()
if fname in debug_func:
print("news cols dropped: " + str(org_news_cols - set(in_news_df.columns)))
 print("new news cols added: " + str(set(in_news_df.columns)- org_news_cols))
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------")
 def drop_neutral_news(in_news_df):
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_news_cols = set(in_news_df.columns)
 if "sentimentPositive" in in_news_df.columns:
in_news_df["netSentimentPositive"] =(in_news_df["sentimentPositive"] - in_news_df["sentimentNegative"])
 neutral_news = in_news_df[in_news_df["netSentimentPositive"].abs() < 0.05].index
in_news_df.drop(neutral_news, axis=0, inplace=True)
 in_news_df["netSentimentPositive"] = in_news_df["netSentimentPositive"] *(1 - in_news_df["sentimentNeutral"])
 print("Num neutral news:", len(neutral_news))
del neutral_news
gc.collect()
in_news_df.drop(["sentimentNegative","sentimentNeutral","sentimentPositive"], axis=1, inplace=True)
 if fname in debug_func:
print("news cols dropped: " + str(org_news_cols - set(in_news_df.columns)))
 print("new news cols added: " + str(set(in_news_df.columns)- org_news_cols))
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------")
 def gen_netnovelty_col(in_news_df):
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_news_cols = set(in_news_df.columns)
 if "noveltyCount12H" in in_news_df.columns:
arr_novelty = np.array(in_news_df[["noveltyCount12H","noveltyCount24H","noveltyCount3D","noveltyCount5D","noveltyCount7D"]])
 in_news_df["inv_netNovelty"] = list(1/(np.argmax(arr_novelty, axis=1)+2))
col = "inv_netNovelty"
scale_col(in_news_df, col)
 in_news_df.drop(["noveltyCount12H","noveltyCount24H","noveltyCount3D","noveltyCount5D","noveltyCount7D"], axis=1, inplace=True)
 in_news_df["SenSR"] = in_news_df["netSentimentPositive"] * in_news_df["relevance"] *(in_news_df["inv_netNovelty"])
 in_news_df.drop(["relevance","netSentimentPositive","inv_netNovelty"],axis=1, inplace=True)
 if fname in debug_func:
print("news cols dropped: " + str(org_news_cols - set(in_news_df.columns)))
 print("new news cols added: " + str(set(in_news_df.columns)- org_news_cols))
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------")
<feature_engineering>
del train
def create_sector(in_asset_df):
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_news_cols = set(in_asset_df.columns)
 global dic_asset_sector
global set_sector
global sector_col
global dic_sectorSNo
in_asset_df["sector"] = in_asset_df["subjects"].apply(lambda x: str(eval(x)& set_sector))
for col in set_sector:
in_asset_df[col] = in_asset_df["sector"].apply(lambda x: col in eval(x))
asset_df = in_asset_df[sector_col].groupby(["assetNameSNo"] ).sum()
asset_df1 = asset_df.sum(axis=1)
 asset_unknown_sector = list(asset_df1[asset_df1==0].index)
 if len(asset_unknown_sector)> 0:
print("Could not find sector of ", len(asset_unknown_sector), " assets")
 asset_df["sectorNameSNo"] = asset_df.values.argmax(axis=1)
 asset_df["sectorName"] = asset_df["sectorNameSNo"].map(dic_sectorSNo)
 asset_df.loc[asset_unknown_sector,["sectorNameSNo","sectorName"]] = [-1,"Unknown"]
asset_df = asset_df[["sectorName","sectorNameSNo"]]
asset_df.reset_index(inplace=True)
 if fname in debug_func:
print("news cols dropped: " + str(org_news_cols - set(in_asset_df.columns)))
 print("new news cols added: " + str(set(in_asset_df.columns)- org_news_cols))
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------")
 return asset_df
def get_sector_name(in_news_df, forpred=False):
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_news_cols = set(in_news_df.columns)
 global dic_asset_sector
if "sectorName" not in in_news_df.columns:
in_news_df["sectorName"] = ""
in_news_df["sectorNameSNo"] = -1
if len(dic_asset_sector)> 0:
in_news_df["sectorName"] = in_news_df["assetNameSNo"].map(dic_asset_sector)
 in_news_df["sectorName"].fillna("", inplace=True)
 in_news_df["sectorNameSNo"] = in_news_df["sectorName"].map(dic_sector)
 if 1==1:
rec_index = in_news_df[in_news_df["sectorName"] == ""].index
lst_subjects = in_news_df.loc[rec_index]["subjects"].unique()
dic_subjects = {k:v for v,k in enumerate(list(lst_subjects)) }
dic_subjectsSNo = {v:k for v,k in enumerate(list(lst_subjects)) }
in_news_df["subjectsSNo"] = -1
in_news_df.loc[rec_index,["subjectsSNo"]] = in_news_df.loc[rec_index]["subjects"].map(dic_subjects)
 in_asset_df = in_news_df.loc[rec_index]
in_asset_df["tempCol"] = 1
in_asset_df = in_asset_df[["assetNameSNo","subjectsSNo","tempCol"]].groupby(["assetNameSNo","subjectsSNo"], as_index=False ).first()
in_asset_df.drop(["tempCol"], inplace=True, axis=1)
 in_asset_df["subjects"] = in_asset_df["subjectsSNo"].map(dic_subjectsSNo)
 if fname in debug_func:
print("New asset records in news : " + str(in_asset_df.shape[0])+ " out of " + str(in_news_df.shape[0]))
print("***************************")
 all_news = []
if in_asset_df.shape[0] > 0:
if forpred==False:
maxSNo = in_asset_df["assetNameSNo"].max()
partition_size =(maxSNo // 4)+ 1
for i in range(4):
minr = i*partition_size
maxr =(i+1)*partition_size
all_news.append(in_asset_df[(in_asset_df["assetNameSNo"] >= minr)&(in_asset_df["assetNameSNo"] < maxr)])
 pool = Pool(4)
 all_df = pool.map(create_sector, all_news)
 asset_df = pd.concat(all_df)
 pool.close()
else:
asset_df = create_sector(in_asset_df)
 asset_df1 = asset_df.set_index(["assetNameSNo"])[["sectorName"]]
new_dict = asset_df1.to_dict() ["sectorName"]
dic_asset_sector.update(new_dict)
 merged_df = pd.merge(in_news_df.reset_index() [["assetNameSNo","index"]], asset_df[["assetNameSNo","sectorName","sectorNameSNo"]], \
on=["assetNameSNo"], how="left" ).set_index("index")
 in_news_df.loc[merged_df.index,["sectorName","sectorNameSNo"]] = merged_df.loc[merged_df.index,["sectorName","sectorNameSNo"]]
if 1==1:
if "subjects" in in_news_df.columns:
in_news_df.drop("subjects", axis=1, inplace=True)
 if fname in debug_func:
print("news cols dropped: " + str(org_news_cols - set(in_news_df.columns)))
 print("new news cols added: " + str(set(in_news_df.columns)- org_news_cols))
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------" )<merge>
predictions = []
step = 50000
for i in range(0, len(x_test), step):
predictions.extend(np.expm1(reg.predict(x_test.iloc[i: min(i+step, len(x_test)) , :], num_iteration=reg.best_iteration)) )
def merge_mkt_news(in_mkt_df, in_news_df):
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_mkt_cols = set(in_mkt_df.columns)
 org_news_cols = set(in_news_df.columns)
 global dic_asset_sector
in_mkt_df["sectorName"] = in_mkt_df["assetNameSNo"].map(dic_asset_sector)
 df_unknown_sector = in_mkt_df[in_mkt_df["sectorName"].isna() ]
in_mkt_df["sectorName"].fillna("Unknown", inplace=True)
 print("no of market recs with unresolved sector: " + str(len(df_unknown_sector["assetNameSNo"].unique())))
 in_mkt_df["sectorNameSNo"] = in_mkt_df["sectorName"].map(dic_sector)
 in_mkt_df["mkt_cap"] =(( in_mkt_df["close"] + in_mkt_df["open"])/2)* in_mkt_df["volume"]
in_news_df1 = in_news_df[["timeSNo","assetNameSNo","SenSR"]].groupby(["assetNameSNo","timeSNo"], as_index=False ).sum()
if "SenSR" in in_mkt_df.columns:
in_mkt_df.drop(["SenSR"], axis=1, inplace=True)
 merged_df = pd.merge(in_mkt_df.reset_index() , in_news_df1, on= ["assetNameSNo","timeSNo"], how="inner" ).set_index("index")
 in_mkt_df["SenSR"] = 0
in_mkt_df.loc[merged_df.index, ["SenSR"]] = merged_df.loc[merged_df.index, ["SenSR"]]
in_mkt_df["weighted_SenSR"] = in_mkt_df["SenSR"] * in_mkt_df["mkt_cap"]
if fname in debug_func:
print("mkt cols dropped: " + str(org_mkt_cols - set(in_mkt_df.columns)))
 print("new mkt cols added: " + str(set(in_mkt_df.columns)- org_mkt_cols))
print("news cols dropped: " + str(org_news_cols - set(in_news_df.columns)))
 print("new news cols added: " + str(set(in_news_df.columns)- org_news_cols))
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------" )<define_search_space>
def RMSLE_Func(actual, predicted):
actual = np.array(actual)
 predicted = np.array(predicted)
 return(np.sqrt(np.sum(np.power(np.log(predicted+1)-np.log(actual+1),2)) /actual.shape[0]))
def create_sector_lag(df_code,n_lag=[3,7,14,],shift_size=1):
for col in sector_return_features:
for window in n_lag:
if df_code["sectorNameSNo"].unique() [0] == -1:
df_code['%s_lag_%s_mean'%(col,window)] = -1
df_code['%s_lag_%s_max'%(col,window)] = -1
df_code['%s_lag_%s_min'%(col,window)] = -1
else:
rolled = df_code[[col,"mkt_cap"]].shift(shift_size ).rolling(window=window)
 lag_mean = rolled[col].sum() / rolled["mkt_cap"].sum()
lag_max = rolled[col].max() / rolled["mkt_cap"].sum()
lag_min = rolled[col].mean() / rolled["mkt_cap"].sum()
df_code['%s_lag_%s_mean'%(col,window)] = lag_mean
df_code['%s_lag_%s_max'%(col,window)] = lag_max
df_code['%s_lag_%s_min'%(col,window)] = lag_min
return df_code.fillna(-1)
 def generate_sector_lag_features(df,n_lag = [3,7,14]):
fname = sys._getframe().f_code.co_name
if fname in debug_func:
print("Start " + fname)
 start = datetime.now()
org_mkt_cols = set(df.columns)
 all_df = []
df_codes = df[['timeSNo','sectorNameSNo',"mkt_cap"] + sector_return_features].groupby(["timeSNo",'sectorNameSNo'], as_index=False ).sum()
df_codes = df_codes.groupby('sectorNameSNo')
 df_codes = [df_code[1][['timeSNo','sectorNameSNo',"mkt_cap"] + sector_return_features] for df_code in df_codes]
pool = Pool(4)
 all_df = pool.map(create_sector_lag, df_codes)
 new_df = pd.concat(all_df)
 new_df.drop(sector_return_features + ["mkt_cap"],axis=1,inplace=True)
 pool.close()
if fname in debug_func:
print("mkt cols dropped: " + str(org_mkt_cols - set(df.columns)))
 print("new mkt cols added: " + str(set(df.columns)- org_mkt_cols))
end = datetime.now()
print("End " + fname + ": " + str(end-start)+ "---------------------")
 return new_df.reset_index()<feature_engineering>
RMSLE = RMSLE_Func(y_test, predictions)
 print(RMSLE)
 del x_test, y_test, predictions
fix_time_col(market_train_df, news_train_df)
 market_train_df = market_train_df.loc[market_train_df['time']>=date(2010, 1, 1)]
news_train_df = news_train_df.loc[news_train_df['time']>=date(2010, 1, 1)]
clean_assetName_col(market_train_df, news_train_df)
 dic_assetName, dic_assetNameSNo, dic_assetCodes, dic_assetCode, dic_assetCodeSNo = set_assetNameCodes_SNo(market_train_df, news_train_df,dic_assetName, \
dic_assetNameSNo, dic_assetCodes, dic_assetCode, dic_assetCodeSNo)
 replace_assetName(market_train_df, news_train_df)
 drop_extra_news_cols(news_train_df)
 drop_irrelevant_news(news_train_df)
 drop_neutral_news(news_train_df)
 gen_netnovelty_col(news_train_df)
<merge>
%%time
predictions = []
step = 50000
for i in range(0, len(test), step):
predictions.extend(np.expm1(reg.predict(test.iloc[i: min(i+step, len(test)) , :], num_iteration=reg.best_iteration)) )
<set_options><EOS> 
%%time
Submission = pd.DataFrame(test.index,columns=['row_id'])
 Submission['meter_reading'] = predictions
Submission['meter_reading'].clip(lower=0,upper=None,inplace=True)
 Submission.to_csv("lgbm.csv",index=None )