Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<merge>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
new_df = generate_lag_features(market_train_df,n_lag=n_lag)
 market_train_df = pd.merge(market_train_df,new_df,how='left',on=['timeSNo','assetCodeSNo'])
<merge>
SEED = 42
FOLDS = 3
def seed_env(seed=SEED):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 pd.set_option('display.max_columns', 380)
 pd.set_option('display.max_rows', 500)
 seed_env()
sns.set()
%matplotlib inline
new_df = generate_sector_lag_features(market_train_df,n_lag=n_lag)
 market_train_df = pd.merge(market_train_df,new_df,how='left',on=['timeSNo','sectorNameSNo'])
<categorify>
def air_temp_fe(df):
df['min_day_temp'] = df.groupby(['building_id','month','day'])['air_temperature'].transform('min' ).astype(np.float16)
 df['max_day_temp'] = df.groupby(['building_id','month','day'])['air_temperature'].transform('max' ).astype(np.float16)
 df['mean_day_temp'] = df.groupby(['building_id','month','day'])['air_temperature'].transform('mean' ).astype(np.float16)
 df['site_id_mean_temp'] = df.groupby(['site_id','month'])['air_temperature'].transform('mean' ).astype(np.float16)
 nominal_degree = 18
df['degree_day'] =(np.round(( df['max_day_temp'] + df['min_day_temp'])/2)- nominal_degree ).astype(np.float16)
 df.drop(['min_day_temp','max_day_temp'], axis = 1, inplace = True)
 def sea_level_pressure_fe(df):
df['sea_level_pressure_month_mean'] = df.groupby(['building_id','month'])['sea_level_pressure'].transform('mean' ).astype(np.float16)
 def add_lag_feature(df, window=3):
group_df = df.groupby('site_id')
 cols = ['air_temperature']
rolled = group_df[cols].rolling(window=window, min_periods=0)
 lag_mean = rolled.mean().reset_index().astype(np.float16)
 for col in cols:
df[f'{col}_mean_lag{window}'] = lag_mean[col].astype(np.float32)
 del lag_mean,rolled,group_df
gc.collect()
def mis_impute(data):
for i in data.columns:
if data[i].dtype == "object":
data[i] = data[i].fillna("other")
 elif(data[i].dtype == "int64" or data[i].dtype == "float64"):
data[i] = data[i].fillna(data[i].mean())
 else:
pass
return data
market_train_df = mis_impute(market_train_df )<define_variables>
def set_localtime(df):
zone_dict={0:4,1:0,2:7,3:4,4:7,5:0,6:4,7:4,8:4,9:5,10:7,11:4,12:0,13:5,14:4,15:4}
for sid, zone in zone_dict.items() :
sids = df.site_id == sid
df.loc[sids, 'timestamp'] = df[sids].timestamp - pd.offsets.Hour(zone)
 def time_features(df):
df['month'] = df['timestamp'].dt.month.astype(np.int8)
 df['day'] = df['timestamp'].dt.day.astype(np.int8)
 df['hour'] = df['timestamp'].dt.hour.astype(np.int8)
 df['weekday'] = df['timestamp'].dt.weekday.astype(np.int8 )
up = market_train_df['returnsOpenNextMktres10'] >= 0
universe = market_train_df['universe'].values
d = market_train_df['time']
fcol = [c for c in market_train_df if c not in ['assetCode', 'assetCodes', 'assetCodesLen', 'assetName', 'audiences',
'firstCreated', 'headline', 'headlineTag', 'marketCommentary', 'provider',
'returnsOpenNextMktres10', 'sourceId', 'subjects', 'time', 'time_x', "timeSNo",
'universe','sourceTimestamp', "sectorName","index","mkt_cap","open","close"]]
X = market_train_df[fcol].values
up = up.values
r = market_train_df.returnsOpenNextMktres10.values
del market_train_df
gc.collect()
mins = np.min(X, axis=0)
 maxs = np.max(X, axis=0)
 rng = maxs - mins
X = 1 -(( maxs - X)/ rng)
 assert X.shape[0] == up.shape[0] == r.shape[0]
X_train, X_test, up_train, up_test, r_train, r_test,u_train,u_test,d_train,d_test = \
model_selection.train_test_split(X, up, r,universe,d,test_size=0.25, random_state=99)
 print(X_train.shape)
 train_data = lgb.Dataset(X, label=up.astype(int), feature_name=fcol, categorical_feature=["assetCodeSNo","assetNameSNo","sectorNameSNo"], free_raw_data=False)
 test_data = lgb.Dataset(X_test, label=up_test.astype(int), feature_name=fcol, categorical_feature=["assetCodeSNo","assetNameSNo","sectorNameSNo"], free_raw_data=False )<set_options>
def building_features(df,df_group):
building_mean = df_group.mean().astype(np.float16)
 building_median = df_group.median().astype(np.float16)
 building_std = df_group.std().astype(np.float16)
 df['building_median'] = df['building_id'].map(building_median ).astype(np.float16)
 df['building_mean'] = df['building_id'].map(building_mean ).astype(np.float16)
 df['building_std'] = df['building_id'].map(building_std ).astype(np.float16)
 del building_mean,building_median,building_std
gc.collect()
del X, X_train
gc.collect()<compute_train_metric>
categorical_features = ["primary_use","building_id","month","day","hour","weekday","site_id"]
confidence_test =(gbm_1.predict(X_test)+ gbm_2.predict(X_test)) /2
confidence_test =(confidence_test-confidence_test.min())/(confidence_test.max() -confidence_test.min())
 confidence_test = confidence_test*2-1
print(max(confidence_test),min(confidence_test))
r_test = r_test.clip(-1,1)
 x_t_i = confidence_test * r_test * u_test
data = {'day' : d_test, 'x_t_i' : x_t_i}
df = pd.DataFrame(data)
 x_t = df.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_test = mean / std
print(score_test)
<set_options>
def train(train_df, target, splits):
params = {
"objective": "regression",
"boosting": "gbdt",
"num_leaves": 41,
"max_depth": 6,
"learning_rate": 0.05,
"feature_fraction": 0.8,
"bagging_fraction": 0.33,
"bagging_freq": 5,
"metric": "rmse",
"reg_lambda": 2
}
lgb_models = list()
for split in splits:
train_index = split[0]
val_index = split[1]
train_X = train_df.iloc[train_index]
val_X = train_df.iloc[val_index]
train_y = target.iloc[train_index]
val_y = target.iloc[val_index]
lgb_train = lgb.Dataset(train_X, train_y, categorical_feature=categorical_features)
 lgb_eval = lgb.Dataset(val_X, val_y, categorical_feature=categorical_features)
 lgb_model = lgb.train(params,
lgb_train,
valid_sets=(lgb_train, lgb_eval),
valid_names=["train","valid"],
num_boost_round=1600,
early_stopping_rounds=200,
verbose_eval = 200)
 lgb_models.append(lgb_model)
 del train_X,val_X,train_y,val_y,lgb_train,lgb_eval
gc.collect()
return lgb_models
del X_test, d_train
gc.collect()<feature_engineering>
def df_transformation(df, group):
df['timestamp'] = pd.to_datetime(df['timestamp'], format="%Y-%m-%d %H:%M:%S")
 df["meter"] = df["meter"].astype(np.int8)
 df["building_id"] = df["building_id"].astype(np.int16)
 time_features(df)
 building_features(df,group)
 def weather_transformation(weather):
weather['timestamp'] = pd.to_datetime(weather['timestamp'], format="%Y-%m-%d %H:%M:%S")
 set_localtime(weather)
 def building_transformation(building):
building["primary_use"] = building["primary_use"].astype(np.int8)
 building["floor_count"] = building["floor_count"].astype(np.int8)
 building["year_built"] = building["year_built"].astype(np.int16)
 def memory_adj(df):
df['site_id'] = df['site_id'].astype(np.int8)
 df['square_feet'] = df['square_feet'].astype(np.int32 )
def process_mkt_news_data(market_obs_df, news_obs_df):
global dic_assetName, dic_assetNameSNo, dic_assetCodes, dic_assetCode, dic_assetCodeSNo
fix_time_col(market_obs_df, news_obs_df)
 curtime = market_obs_df["time"].unique()
print(curtime)
 found=1
if len(total_market_obs_df)>1:
while found==1:
prev_df = total_market_obs_df[-1]
if prev_df["time"].unique() == curtime:
print("removing------------------------")
 total_market_obs_df.pop(-1)
 total_news_obs_df.pop(-1)
 else:
found=0
clean_assetName_col(market_obs_df, news_obs_df)
 dic_assetName, dic_assetNameSNo, dic_assetCodes, dic_assetCode, dic_assetCodeSNo = set_assetNameCodes_SNo(market_obs_df, news_obs_df,dic_assetName, dic_assetNameSNo, \
dic_assetCodes,dic_assetCode, dic_assetCodeSNo)
 drop_extra_news_cols(news_obs_df)
 drop_irrelevant_news(news_obs_df)
 drop_neutral_news(news_obs_df)
 gen_netnovelty_col(news_obs_df)
 get_sector_name(news_obs_df, True)
 return dic_assetName, dic_assetNameSNo, dic_assetCodes, dic_assetCode, dic_assetCodeSNo, curtime<predict_on_test>
train_df = pd.read_feather('.. /input/ashre-data-clean-leak-for-train/train_no_nan_s.feather')
 train_df['meter_reading'] = np.log1p(train_df['meter_reading'])
 df_group = train_df.groupby('building_id')['meter_reading']
df_transformation(train_df,df_group)
 weather_df = pd.read_feather('.. /input/ashre-data-clean-leak-for-train/weather_intr_no_nan_s.feather')
 building_metadata_df = pd.read_feather('.. /input/ashre-data-clean-leak-for-train/building_no_nan_s.feather')
 weather_transformation(weather_df)
 building_transformation(building_metadata_df )
if len(total_market_obs_df)== 0:
for i in range(len(prv_mkt_data)) :
mkt_data = prv_mkt_data[i]
news_data = prv_news_data[i]
mkt_data.drop(["universe","returnsOpenNextMktres10"], inplace=True, axis=1)
 dic_assetName, dic_assetNameSNo, dic_assetCodes, dic_assetCode, dic_assetCodeSNo, curtime = process_mkt_news_data(mkt_data, news_data)
 total_market_obs_df.append(mkt_data)
 total_news_obs_df.append(news_data)
 def make_random_predictions_new(market_obs_df, news_obs_df,predictions_template_df):
global n_days
global prep_time
global prediction_time
global packaging_time
global total_market_obs_df
global total_news_obs_df
global dic_assetName, dic_assetNameSNo, dic_assetCodes, dic_assetCode, dic_assetCodeSNo
n_days +=1
if(n_days%50==0):
print(n_days,end=' ')
 t = time.time()
dic_assetName, dic_assetNameSNo, dic_assetCodes, dic_assetCode, dic_assetCodeSNo, curtime = process_mkt_news_data(market_obs_df, news_obs_df)
 total_market_obs_df.append(market_obs_df)
 total_news_obs_df.append(news_obs_df)
 if len(total_market_obs_df)==1:
history_df = total_market_obs_df[0]
history_news_df = total_news_obs_df[0]
else:
history_df = pd.concat(total_market_obs_df[-(np.max(n_lag)+1):], sort=False)
 history_news_df = pd.concat(total_news_obs_df[-(np.max(n_lag)+1):], sort=False)
 history_df.sort_values(["timeSNo"], inplace=True)
 history_df.index = list(range(history_df.shape[0]))
history_news_df.sort_values(["timeSNo"], inplace=True)
 history_news_df.index = list(range(history_news_df.shape[0]))
fixopenclose(history_df)
 merge_mkt_news(history_df, history_news_df)
 new_df = generate_lag_features(history_df,n_lag=[3,7,14])
 history_df = pd.merge(history_df,new_df,how='left',on=['timeSNo','assetCodeSNo'])
 new_df = generate_sector_lag_features(history_df,n_lag=[3,7,14])
 history_df = pd.merge(history_df,new_df,how='left',on=['timeSNo','sectorNameSNo'])
 history_df = mis_impute(history_df)
 history_df.dropna(inplace=True)
 market_obs_df = history_df[history_df["time"].isin(curtime)]
X_live = market_obs_df[fcol].values
X_live = 1 -(( maxs - X_live)/ rng)
 prep_time += time.time() - t
t = time.time()
lp =(gbm_1.predict(X_live)+ gbm_2.predict(X_live)) /2
prediction_time += time.time() -t
t = time.time()
confidence = lp
confidence =(confidence-confidence.min())/(confidence.max() -confidence.min())
 confidence = confidence * 2 - 1
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence':confidence})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 packaging_time += time.time() - t
return market_obs_df<define_variables>
def df_for_meter_type(meter):
target_train_df = train_df[train_df['meter'] == meter]
target_train_df = target_train_df.merge(building_metadata_df, on='building_id', how='left')
 target_train_df = target_train_df.merge(weather_df, on=['site_id', 'timestamp'], how='left')
 air_temp_fe(target_train_df)
 sea_level_pressure_fe(target_train_df)
 target_train_df.drop(['air_temperature','sea_level_pressure','wind_direction','wind_speed'], axis = 1, inplace = True)
 train_idx_f1 = target_train_df[(target_train_df.timestamp.dt.year == 2016)].index
test_idx_f1 = target_train_df[(target_train_df.timestamp.dt.year == 2017)|(target_train_df.timestamp.dt.year == 2018)].index
train_idx_f2 = target_train_df[(target_train_df.timestamp.dt.year == 2016)|(target_train_df.timestamp.dt.year == 2017)].index
test_idx_f2 = target_train_df[(target_train_df.timestamp.dt.year == 2018)].index
splits = [(train_idx_f1,test_idx_f1),(train_idx_f2,test_idx_f2)]
target = target_train_df['meter_reading']
meter_df = target_train_df.drop(['meter_reading','timestamp','meter'],axis = 1,inplace = False)
 memory_adj(meter_df)
 del target_train_df,train_idx_f1,test_idx_f1,train_idx_f2,test_idx_f2
return meter_df, target, splits
market_obs_df, news_obs_df, predictions_template_df = next(days )<predict_on_test>
%%time
meter_0_df, target, splits = df_for_meter_type(0)
 meter_0_df
market_obs_df = make_random_predictions_new(market_obs_df, news_obs_df,predictions_template_df )<predict_on_test>
%%time
meter_1_df, target, splits = df_for_meter_type(1)
 meter_1_df
for(market_obs_df, news_obs_df, predictions_template_df)in days:
market_obs_df = make_random_predictions_new(market_obs_df, news_obs_df,predictions_template_df)
 print('Done!' )<load_from_csv>
%%time
meter_2_df, target, splits = df_for_meter_type(2)
 meter_2_df
env.write_submission_file()
sub = pd.read_csv("submission.csv" )<import_modules>
%%time
meter_3_df, target, splits = df_for_meter_type(2)
 meter_3_df
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import time
from sklearn.metrics import accuracy_score
from kaggle.competitions import twosigmanews
from keras import initializers
from keras import regularizers
from keras import constraints
from keras.utils import conv_utils
from keras.utils.data_utils import get_file
from keras.engine.topology import get_source_inputs
from keras.engine import InputSpec
from keras import backend as K
from keras.layers import LeakyReLU
from keras.layers import ZeroPadding2D
from keras.losses import binary_crossentropy
import keras.callbacks as callbacks
from keras.callbacks import Callback
from keras.applications.xception import Xception
from keras.layers import multiply
import keras
from keras import optimizers
from keras.legacy import interfaces
from keras.utils.generic_utils import get_custom_objects
from keras.engine.topology import Input
from keras.engine.training import Model
from keras.layers.convolutional import Conv2D, UpSampling2D, Conv2DTranspose
from keras.layers.core import Activation, SpatialDropout2D
from keras.layers.merge import concatenate
from keras.layers.normalization import BatchNormalization
from keras.layers.pooling import MaxPooling2D
from keras.layers import Input,Dropout,BatchNormalization,Activation,Add
from keras.regularizers import l2
from keras.layers.core import Dense, Lambda
from keras.layers.merge import concatenate, add
from keras.layers import GlobalAveragePooling2D, Reshape, Dense, multiply, Permute
from keras.optimizers import SGD<split>
del train_df, weather_df
gc.collect()
env = twosigmanews.make_env() 
(market_train, _)= env.get_training_data()<define_variables>
test_df = pd.read_feather('.. /input/ashre-simple-no-missing-for-test/test_no_nan_s.feather')
 df_transformation(test_df,df_group)
 weather_test_df = pd.read_feather('.. /input/ashre-simple-no-missing-for-test/weather_test_intr_no_nan_s.feather')
 building_metadata_df = pd.read_feather('.. /input/ashre-simple-no-missing-for-test/building_test_no_nan_s.feather')
 weather_transformation(weather_test_df)
 building_transformation(building_metadata_df )
cat_cols = ['assetCode']
num_cols = ['volume', 'close', 'open', 'returnsClosePrevRaw1', 'returnsOpenPrevRaw1', 'returnsClosePrevMktres1',
'returnsOpenPrevMktres1', 'returnsClosePrevRaw10', 'returnsOpenPrevRaw10', 'returnsClosePrevMktres10',
'returnsOpenPrevMktres10'] 
<split>
def df_for_meter_type_test(meter):
target_test_df = test_df[test_df['meter'] == meter]
target_test_df = target_test_df.merge(building_metadata_df, on='building_id', how='left')
 target_test_df = target_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left')
 air_temp_fe(target_test_df)
 sea_level_pressure_fe(target_test_df)
 target_test_df.drop(['air_temperature','sea_level_pressure','wind_direction','wind_speed'], axis = 1, inplace = True)
 meter_df = target_test_df.drop(['timestamp','meter','row_id'],axis = 1,inplace = False)
 memory_adj(meter_df)
 del target_test_df
return meter_df
train_indices, val_indices = train_test_split(market_train.index.values,test_size=0.25, random_state=23)
<categorify>
X_test = df_for_meter_type_test(0)
 y_test0 = sum([model.predict(X_test, num_iteration=model.best_iteration)for model in meter_0_models])/2
del X_test,meter_0_models
gc.collect()
def encode(encoder, x):
len_encoder = len(encoder)
 try:
id = encoder[x]
except KeyError:
id = len_encoder
return id
encoders = [{} for cat in cat_cols]
for i, cat in enumerate(cat_cols):
print('encoding %s...' % cat, end=' ')
 encoders[i] = {l: id for id, l in enumerate(market_train.loc[:, cat].astype(str ).unique())}
market_train[cat] = market_train[cat].astype(str ).apply(lambda x: encode(encoders[i], x))
print('Done')
 embed_sizes = [len(encoder)+ 1 for encoder in encoders] 
<define_variables>
X_test = df_for_meter_type_test(1)
 y_test1 = sum([model.predict(X_test, num_iteration=model.best_iteration)for model in meter_1_models])/2
del X_test,meter_1_models
gc.collect()
a=0
for type in types:
a=a+d[type].shape[0]
a<feature_engineering>
X_test = df_for_meter_type_test(2)
 y_test2 = sum([model.predict(X_test, num_iteration=model.best_iteration)for model in meter_2_models])/2
del X_test,meter_2_models
gc.collect()
low =.25
high =.75
bounds = {}
for type in types:
filt_df = d[type].loc[:, d[type].columns != 'assetCode']
filt_df = filt_df.loc[:, filt_df.columns != 'universe']
quant_df = filt_df.quantile([low, high])
 IQR = quant_df.iloc[1,:]- quant_df.iloc[0,:]
quant_df.iloc[0,:] = quant_df.iloc[0,:] - 1.5*IQR
quant_df.iloc[1,:] = quant_df.iloc[1,:] + 1.5*IQR
bounds[type] = quant_df
bounds[type] = bounds[type].reset_index()
bounds[type] = bounds[type].drop("index", axis=1)
 bounds[1]<filter>
X_test = df_for_meter_type_test(3)
 y_test3 = sum([model.predict(X_test, num_iteration=model.best_iteration)for model in meter_3_models])/2
del X_test,meter_3_models
gc.collect()
bounds[1].loc[1,"volume"]<feature_engineering>
sample_submission = pd.read_csv('.. /input/ashrae-energy-prediction/sample_submission.csv' )
market_train1=pd.DataFrame()
for type in types:
for column in num_cols:
d[type] = d[type].loc[d[type][column]>=bounds[type].loc[0,column]]
d[type] = d[type].loc[d[type][column]<=bounds[type].loc[1,column]]
market_train1=pd.concat([market_train1,d[type]], ignore_index=True)
<define_variables>
sample_submission.loc[test_df[test_df['meter'] == 0]['row_id'], 'meter_reading'] = np.clip(np.expm1(y_test0),0, a_max=None)
 sample_submission.loc[test_df[test_df['meter'] == 1]['row_id'], 'meter_reading'] = np.clip(np.expm1(y_test1),0, a_max=None)
 sample_submission.loc[test_df[test_df['meter'] == 2]['row_id'], 'meter_reading'] = np.clip(np.expm1(y_test2),0, a_max=None)
 sample_submission.loc[test_df[test_df['meter'] == 3]['row_id'], 'meter_reading'] = np.clip(np.expm1(y_test3),0, a_max=None )
<feature_engineering><EOS> 
sample_submission.to_csv('submission_Dec8_1.csv', index=False, float_format='%.4f' )