Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<prepare_output>
nround = 1000
debug = False
black_day =10
data = {'y_train':Y_train[0:15],'y_pred':(model.predict(X_train.values[0:15])).reshape(1,-1)[0]}
pd.DataFrame(data )<predict_on_test>
DATA_PATH = ".. /input/ashrae-energy-prediction/"
_df = pd.DataFrame(( model.predict(X_train.values[0:15])))
 _df[0]<prepare_x_and_y>
train_df = pd.read_csv(DATA_PATH + 'train.csv')
 train_df = train_df [ train_df['building_id'] != 1099 ]
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")')
 building_df = pd.read_csv(DATA_PATH + 'building_metadata.csv')
 weather_df = pd.read_csv(DATA_PATH + 'weather_train.csv' )
all_Y = market_train_df[['returnsOpenNextMktres10']]
all_Y.head()<train_model>
root_black = '.. /input/ashrae-local-datatime-and-black-count/'
train_df_black = pd.read_feather(root_black + 'train_black.feather')
 train_df = train_df[train_df_black.black_count < 24*black_day]
del train_df_black
gc.collect()
scaler = MinMaxScaler(feature_range=(-1,1))
scaler.fit(all_Y)
 print('scaler fitting done' )<predict_on_test>
site_0_bids = building_df[building_df.site_id == 0].building_id.unique()
train_df.loc[(train_df.building_id.isin(site_0_bids)) &(train_df.meter==0), 'meter_reading'] = train_df[(train_df.building_id.isin(site_0_bids)) &(train_df.meter==0)]['meter_reading'] * 0.2931
def make_prediction(df):
pred = pd.DataFrame(model.predict(df))
scaled_pred = pd.DataFrame(scaler.transform(pred[[0]]))
scaled_pred[scaled_pred>0]=1
scaled_pred[scaled_pred<0]=-1
pred_arr = np.array(scaled_pred ).reshape(1,-1)[0]
return pred_arr
print('function loaded' )<predict_on_test>
def fill_weather_dataset(weather_df):
time_format = "%Y-%m-%d %H:%M:%S"
start_date = datetime.datetime.strptime(weather_df['timestamp'].min() ,time_format)
 end_date = datetime.datetime.strptime(weather_df['timestamp'].max() ,time_format)
 total_hours = int(((end_date - start_date ).total_seconds() + 3600)/ 3600)
 hours_list = [(end_date - datetime.timedelta(hours=x)).strftime(time_format)for x in range(total_hours)]
missing_hours = []
for site_id in range(16):
site_hours = np.array(weather_df[weather_df['site_id'] == site_id]['timestamp'])
 new_rows = pd.DataFrame(np.setdiff1d(hours_list,site_hours),columns=['timestamp'])
 new_rows['site_id'] = site_id
weather_df = pd.concat([weather_df,new_rows])
 weather_df = weather_df.reset_index(drop=True)
 weather_df["datetime"] = pd.to_datetime(weather_df["timestamp"])
 weather_df["day"] = weather_df["datetime"].dt.day
weather_df["week"] = weather_df["datetime"].dt.week
weather_df["month"] = weather_df["datetime"].dt.month
weather_df = weather_df.set_index(['site_id','day','month'])
 air_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['air_temperature'].mean() ,columns=["air_temperature"])
 weather_df.update(air_temperature_filler,overwrite=False)
 cloud_coverage_filler = weather_df.groupby(['site_id','day','month'])['cloud_coverage'].mean()
cloud_coverage_filler = pd.DataFrame(cloud_coverage_filler.fillna(method='ffill'),columns=["cloud_coverage"])
 weather_df.update(cloud_coverage_filler,overwrite=False)
 due_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['dew_temperature'].mean() ,columns=["dew_temperature"])
 weather_df.update(due_temperature_filler,overwrite=False)
 sea_level_filler = weather_df.groupby(['site_id','day','month'])['sea_level_pressure'].mean()
sea_level_filler = pd.DataFrame(sea_level_filler.fillna(method='ffill'),columns=['sea_level_pressure'])
 weather_df.update(sea_level_filler,overwrite=False)
 wind_direction_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_direction'].mean() ,columns=['wind_direction'])
 weather_df.update(wind_direction_filler,overwrite=False)
 wind_speed_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_speed'].mean() ,columns=['wind_speed'])
 weather_df.update(wind_speed_filler,overwrite=False)
 precip_depth_filler = weather_df.groupby(['site_id','day','month'])['precip_depth_1_hr'].mean()
precip_depth_filler = pd.DataFrame(precip_depth_filler.fillna(method='ffill'),columns=['precip_depth_1_hr'])
 weather_df.update(precip_depth_filler,overwrite=False)
 weather_df = weather_df.reset_index()
weather_df = weather_df.drop(['datetime','day','week','month'],axis=1)
 return weather_df
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print("Memory usage of dataframe is {:.2f} MB".format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == "int":
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype("category")
 end_mem = df.memory_usage().sum() / 1024**2
print("Memory usage after optimization is: {:.2f} MB".format(end_mem))
print("Decreased by {:.1f}%".format(100 *(start_mem - end_mem)/ start_mem))
return df
def features_engineering(df):
df.sort_values("timestamp")
 df.reset_index(drop=True)
 df["timestamp"] = pd.to_datetime(df["timestamp"],format="%Y-%m-%d %H:%M:%S")
 df["hour"] = df["timestamp"].dt.hour
df["weekend"] = df["timestamp"].dt.weekday
holidays = ["2016-01-01", "2016-01-18", "2016-02-15", "2016-05-30", "2016-07-04",
"2016-09-05", "2016-10-10", "2016-11-11", "2016-11-24", "2016-12-26",
"2017-01-02", "2017-01-16", "2017-02-20", "2017-05-29", "2017-07-04",
"2017-09-04", "2017-10-09", "2017-11-10", "2017-11-23", "2017-12-25",
"2018-01-01", "2018-01-15", "2018-02-19", "2018-05-28", "2018-07-04",
"2018-09-03", "2018-10-08", "2018-11-12", "2018-11-22", "2018-12-25",
"2019-01-01"]
df["is_holiday"] =(df.timestamp.isin(holidays)).astype(int)
 df['square_feet'] = np.log1p(df['square_feet'])
 drop = ["timestamp","sea_level_pressure", "wind_direction", "wind_speed","year_built","floor_count"]
df = df.drop(drop, axis=1)
 gc.collect()
le = LabelEncoder()
df["primary_use"] = le.fit_transform(df["primary_use"])
 return df
my_pred = make_prediction(X_train.head())
 my_pred<prepare_output>
def add_sg(df):
w = 11
p = 2
for si in df.site_id.unique() :
index = df.site_id == si
df.loc[index, 'air_smooth'] = sg(df[index].air_temperature, w, p)
 df.loc[index, 'dew_smooth'] = sg(df[index].dew_temperature, w, p)
 df.loc[index, 'air_diff'] = sg(df[index].air_temperature, w, p, 1)
 df.loc[index, 'dew_diff'] = sg(df[index].dew_temperature, w, p, 1)
 df.loc[index, 'air_diff2'] = sg(df[index].air_temperature, w, p, 2)
 df.loc[index, 'dew_diff2'] = sg(df[index].dew_temperature, w, p, 2 )
temp_df = X_train.head().copy()
temp_df['confidenceValue'] = my_pred
temp_df<define_variables>
weather_df = fill_weather_dataset(weather_df)
 add_sg(weather_df )
days = env.get_prediction_days()
print('days loaded' )<define_variables>
train_df = reduce_mem_usage(train_df,use_float16=True)
 building_df = reduce_mem_usage(building_df,use_float16=True)
 weather_df = reduce_mem_usage(weather_df,use_float16=True )
n_days = 0
for(market_obs_df, _, predictions_template_df)in days:
n_days += 1
print(n_days)
 features = ['volume','close','open','returnsClosePrevRaw1','returnsOpenPrevRaw1',
'returnsClosePrevMktres1','returnsOpenPrevMktres1','returnsClosePrevRaw10',
'returnsOpenPrevRaw10','returnsClosePrevMktres10','returnsOpenPrevMktres10']
returns_features = ['returnsClosePrevRaw10','returnsClosePrevMktres1','returnsOpenPrevMktres1',
'returnsOpenPrevRaw10','returnsClosePrevMktres10','returnsOpenPrevMktres10']
market_obs_df_scaled = scale_data(market_obs_df,features)
 x_submission = market_obs_df_scaled[features].copy()
for i in range(len(returns_features)) :
x_submission[returns_features[i]]= x_submission[returns_features[i]].fillna(x_submission[returns_features[i]].mean())
 pred = make_prediction(x_submission)
 print(pred)
 predictions_template_df['confidenceValue'] = pred
env.predict(predictions_template_df)
 del x_submission
print('Done!')
 env.write_submission_file()<set_options>
train_df = train_df.merge(building_df, left_on='building_id',right_on='building_id',how='left')
 train_df = train_df.merge(weather_df,how='left',left_on=['site_id','timestamp'],right_on=['site_id','timestamp'])
 del weather_df
gc.collect()
pd.set_option('max_columns', 50 )<split>
train_df = features_engineering(train_df )
env = twosigmanews.make_env()
market_train_df, news_train = env.get_training_data()<data_type_conversions>
target = np.log1p(train_df["meter_reading"])
 features = train_df.drop('meter_reading', axis = 1)
 del train_df
gc.collect()
for column in market_train_df.select_dtypes(include='float64' ).columns:
market_train_df[column] = market_train_df[column].astype('float16' )<feature_engineering>
categorical_features = ["building_id", "site_id", "meter", "primary_use", "is_holiday", "weekend"]
params = {
"objective": "regression",
"boosting": "gbdt",
"num_leaves": 1280,
"learning_rate": 0.05,
"feature_fraction": 0.85,
"reg_lambda": 2,
"metric": "rmse",
}
kf = KFold(n_splits=3)
 models = []
for train_index,test_index in kf.split(features):
train_features = features.loc[train_index]
train_target = target.loc[train_index]
test_features = features.loc[test_index]
test_target = target.loc[test_index]
d_training = lgb.Dataset(train_features, label=train_target,categorical_feature=categorical_features, free_raw_data=False)
 d_test = lgb.Dataset(test_features, label=test_target,categorical_feature=categorical_features, free_raw_data=False)
 model = lgb.train(params, train_set=d_training, num_boost_round=nround, valid_sets=[d_training,d_test], verbose_eval=25, early_stopping_rounds=50)
 models.append(model)
 del train_features, train_target, test_features, test_target, d_training, d_test
gc.collect()
start = datetime(2010, 1, 1, 0, 0, 0 ).date()
market_train = market_train_df.loc[market_train_df['time'].dt.date >= start].reset_index(drop=True)
 news_train = news_train.loc[news_train['time'].dt.date >= start].reset_index(drop=True )<drop_column>
del features, target
gc.collect()
def preprocess_news(news_train):
drop_list = [
'audiences', 'subjects', 'assetName',
'headline', 'firstCreated', 'sourceTimestamp',
]
news_train.drop(drop_list, axis=1, inplace=True)
 for col in ['headlineTag', 'provider', 'sourceId']:
news_train[col], uniques = pd.factorize(news_train[col])
 del uniques
news_train['assetCodes'] = news_train['assetCodes'].apply(lambda x: x[1:-1].replace("'", ""))
return news_train<string_transform>
test_df = pd.read_csv(DATA_PATH + 'test.csv')
 row_ids = test_df["row_id"]
test_df.drop("row_id", axis=1, inplace=True)
 test_df = reduce_mem_usage(test_df )
news_train = preprocess_news(news_train )<drop_column>
test_df = test_df.merge(building_df,left_on='building_id',right_on='building_id',how='left')
 del building_df
gc.collect()
def unstack_asset_codes(news_train):
codes = []
indexes = []
for i, values in news_train['assetCodes'].iteritems() :
explode = values.split(", ")
 codes.extend(explode)
 repeat_index = [int(i)]*len(explode)
 indexes.extend(repeat_index)
 index_df = pd.DataFrame({'news_index': indexes, 'assetCode': codes})
 del codes, indexes
gc.collect()
return index_df<drop_column>
weather_df = pd.read_csv(DATA_PATH + 'weather_test.csv')
 weather_df = fill_weather_dataset(weather_df)
 add_sg(weather_df)
 weather_df = reduce_mem_usage(weather_df)
index_df = unstack_asset_codes(news_train)
 index_df.head()<merge>
test_df = test_df.merge(weather_df,how='left',on=['timestamp','site_id'])
 del weather_df
gc.collect()
def merge_news_on_index(news_train, index_df):
news_train['news_index'] = news_train.index.copy()
news_unstack = index_df.merge(news_train, how='left', on='news_index')
 news_unstack.drop(['news_index', 'assetCodes'], axis=1, inplace=True)
 return news_unstack<merge>
test_df = features_engineering(test_df)
 test_df = reduce_mem_usage(test_df )
news_unstack = merge_news_on_index(news_train, index_df)
 del news_train, index_df
gc.collect()
news_unstack.head(3 )<data_type_conversions>
results = []
for model in models:
if results == []:
results = np.expm1(model.predict(test_df, num_iteration=model.best_iteration)) / len(models)
 else:
results += np.expm1(model.predict(test_df, num_iteration=model.best_iteration)) / len(models)
 del model
gc.collect()
def group_news(news_frame):
news_frame['date'] = news_frame.time.dt.date
aggregations = ['mean']
gp = news_frame.groupby(['assetCode', 'date'] ).agg(aggregations)
 gp.columns = pd.Index(["{}_{}".format(e[0], e[1])for e in gp.columns.tolist() ])
 gp.reset_index(inplace=True)
 gp.drop(['sourceId_mean'],axis=1, inplace=True)
 float_cols = {c: 'float32' for c in gp.columns if c not in ['assetCode', 'date']}
return gp.astype(float_cols )<groupby>
sample_submission = pd.DataFrame({"row_id": row_ids, "meter_reading": np.clip(results, 0, a_max=None)})
 del row_ids,results
gc.collect()
news_agg = group_news(news_unstack)
 del news_unstack; gc.collect()
news_agg.head(3 )<feature_engineering>
sample_submission.loc[(test_df.building_id.isin(site_0_bids)) &(test_df.meter==0), 'meter_reading'] = sample_submission[(test_df.building_id.isin(site_0_bids)) &(test_df.meter==0)]['meter_reading'] * 3.4118
def process_date(df):
df['date'] = df['time'].dt.date
df['month'] = df['time'].dt.month
df['dayofweek'] = df['time'].dt.dayofweek
return df<define_variables>
del test_df, models
gc.collect()
def process_ma(df,columns=['open','close','volume'], windows=[10, 20, 60]):
ma_columns = []
ma_dev_columns = []
drop_list = []
for col in columns:
for window in windows:
ma_column = 'ma_{0}_{1}'.format(col,window)
 ma_dev_column = 'ma_dev_{0}_{1}'.format(col,window)
 ma_lag_column = 'ma_lag_{0}_{1}'.format(col,window)
 ma_lag_rate_column = 'ma_lag_rate_{0}_{1}'.format(col,window)
 std_column = 'std_{0}_{1}'.format(col,window)
 ma_columns.append(ma_column)
 ma_dev_columns.append(ma_dev_column)
 df[ma_column] = df.groupby('assetCode')[col].apply(lambda x: x.rolling(window ).mean())
 df[ma_dev_column] = df[col] / df[ma_column] - 1
df[std_column] = df.groupby('assetCode')[col].apply(lambda x: x.rolling(window ).std())
 df[ma_lag_column] = df.groupby('assetCode')[ma_column].shift()
df[ma_lag_rate_column] =(df[ma_column] - df[ma_lag_column])/ df[ma_lag_column]
drop_list.append(ma_column)
 drop_list.append(ma_lag_column)
 df.drop(drop_list,axis=1, inplace=True)
 return df<categorify>
if not debug:
sample_submission.to_csv("submission.csv", index=False )
market_train = process_date(market_train)
 market_train = process_ma(market_train )<merge>
leak_df = pd.read_feather('.. /input/ashrae-leak-data-station/leak.feather')
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[(leak_df.timestamp.dt.year > 2016)&(leak_df.timestamp.dt.year < 2019)]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
leak_df = leak_df[leak_df.building_id!=245]
root = '.. /input/ashrae-feather-format-for-fast-loading/'
test_df = pd.read_feather(root + 'test.feather')
 building_meta_df = pd.read_feather(root + 'building_metadata.feather')
 test_df['timestamp'] = pd.to_datetime(test_df.timestamp )
df = market_train.merge(news_agg, how='left', on=['assetCode', 'date'])
 del market_train, news_agg
gc.collect()
df.head(3 )<data_type_conversions>
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0
test_df['pred'] = sample_submission.meter_reading
leak_df = leak_df.merge(test_df[['building_id', 'meter', 'timestamp', 'pred', 'row_id']], left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 leak_df = leak_df.merge(building_meta_df[['building_id', 'site_id']], on='building_id', how='left' )
for column in df.select_dtypes(include='float64' ).columns:
df[column] = df[column].astype('float16')
 for column in df.select_dtypes(include='int64' ).columns:
df[column] = df[column].astype('int16')
 df.dtypes<compute_test_metric>
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.metrics import mean_squared_error
def custom_metric(date, pred_proba, num_target, universe):
y = pred_proba*2 - 1
r = num_target.clip(-1,1)
 x = y * r * universe
result = pd.DataFrame({'day' : date, 'x' : x})
 x_t = result.groupby('day' ).sum().values
return np.mean(x_t)/ np.std(x_t )<data_type_conversions>
leak_df['pred_l1p'] = np.log1p(leak_df.pred)
 leak_df['meter_reading_l1p'] = np.log1p(leak_df.meter_reading)
 sns.distplot(leak_df.pred_l1p)
 sns.distplot(leak_df.meter_reading_l1p)
 leak_score = np.sqrt(mean_squared_error(leak_df.pred_l1p, leak_df.meter_reading_l1p))
date = df.date
num_target = df.returnsOpenNextMktres10.astype('float32')
 bin_target =(df.returnsOpenNextMktres10 >= 0 ).astype('int8')
 universe = df.universe.astype('int8')
 df.drop(['returnsOpenNextMktres10', 'date', 'universe', 'assetCode', 'assetName', 'time'],
axis=1, inplace=True)
 df = df.astype('float32')
 gc.collect()<split>
print('total score=', leak_score )
<init_hyperparams><EOS> 
leak_df = leak_df[['meter_reading', 'row_id']].set_index('row_id' ).dropna()
sample_submission.loc[leak_df.index, 'meter_reading'] = leak_df['meter_reading']
if not debug:
sample_submission.to_csv('submission_ucf_replaced.csv', index=False, float_format='%.4f' )