Kaggle competition: ASHRAE - Great Energy Predictor III
yhat_new = yhat*W_ho
error = rmspe(np.expm1(ho_ytest), np.expm1(yhat_new))
print('RMSPE for weight corretion {:6f}'.format(error))<save_to_csv>
black_day = 10
outlier = False
rescale = False
debug=False
num_rounds = 200
clip0=False
folds = 3
use_ucf=False
ucf_clip=False
ucf_year = [2017, 2018]
predmode='all'
print("Make predictions on the test set")
 dtest = xgb.DMatrix(xtest)
 test_probs = gbm.predict(dtest)
 result = pd.DataFrame({"Id": test['Id'], 'Sales': np.expm1(test_probs)})
 result.to_csv("Rossmann_submission_1.csv", index=False)
 result = pd.DataFrame({"Id": test['Id'], 'Sales': np.expm1(test_probs*0.995)})
 result.to_csv("Rossmann_submission_2.csv", index=False)
 result = pd.DataFrame({"Id": test['Id'], 'Sales': np.expm1(test_probs*W_test)})
 result.to_csv("Rossmann_submission_3.csv", index=False)
<find_best_params>
import gc
import os
from pathlib import Path
import random
import sys
from tqdm import tqdm_notebook as tqdm
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from IPython.core.display import display, HTML
from sklearn import preprocessing
from sklearn.model_selection import KFold
from sklearn.preprocessing import LabelEncoder
import lightgbm as lgb
from sklearn.metrics import mean_squared_error
<save_to_csv>
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df
<set_options>
zone_dict={0:4,1:0,2:7,3:4,4:7,5:0,6:4,7:4,8:4,9:5,10:7,11:4,12:0,13:5,14:4,15:4}
def set_local(df):
for sid, zone in zone_dict.items() :
sids = df.site_id == sid
df.loc[sids, 'timestamp'] = df[sids].timestamp - pd.offsets.Hour(zone )
%matplotlib inline<compute_test_metric>
%%time
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 train_df = pd.read_feather(root/'train.feather')
 weather_train_df = pd.read_feather(root/'weather_train.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather' )
def to_weight(y):
w = np.zeros(y.shape, dtype=float)
 ind = y != 0
w[ind] = 1./(y[ind]**2)
 return w
def rmspe(y, yhat):
w = to_weight(y)
 rmspe = np.sqrt(np.mean(w *(y - yhat)**2))
return rmspe
def str_date_to_ordinal(date):
return datetime.strptime(date, '%Y-%m-%d' ).date().toordinal()
def date_to_ordinal(date):
return date.toordinal()
def str_to_date(date):
return datetime.strptime(date, '%Y-%m-%d' ).date()
SEED = 42<load_from_csv>
building_meta_df = building_meta_df.merge(train_df[['building_id','meter']].drop_duplicates() , on='building_id' )
train_df = pd.read_csv('.. /input/train.csv', sep=',', parse_dates=['Date'], date_parser=str_to_date,
low_memory=False)
 test_df = pd.read_csv('.. /input/test.csv', sep=',', parse_dates=['Date'], date_parser=str_to_date,
low_memory=False )<filter>
building_meta_df['groupNum_train'] = building_meta_df['site_id'].astype('int')*10 + building_meta_df['meter'].astype('int' )
test_df[pd.isnull(test_df.Open)]<feature_engineering>
train_df = train_df [ train_df['building_id'] != 1099 ]
test_df.loc[pd.isnull(test_df.Open), 'Open'] = 1<concatenate>
building_meta_df['floor_area'] = building_meta_df.square_feet / building_meta_df.floor_count
all_df = pd.concat([train_df, test_df], axis=0)
 all_df.head()<data_type_conversions>
en_holidays = holidays.England()
ir_holidays = holidays.Ireland()
ca_holidays = holidays.Canada()
us_holidays = holidays.UnitedStates()
def add_holiyday(df_weather):
en_idx = df_weather.query('site_id == 1 or site_id == 5' ).index
ir_idx = df_weather.query('site_id == 12' ).index
ca_idx = df_weather.query('site_id == 7 or site_id == 11' ).index
us_idx = df_weather.query('site_id == 0 or site_id == 2 or site_id == 3 or site_id == 4 or site_id == 6 or site_id == 8 or site_id == 9 or site_id == 10 or site_id == 13 or site_id == 14 or site_id == 15' ).index
df_weather['IsHoliday'] = 0
df_weather.loc[en_idx, 'IsHoliday'] = df_weather.loc[en_idx, 'timestamp'].apply(lambda x: en_holidays.get(x, default=0))
df_weather.loc[ir_idx, 'IsHoliday'] = df_weather.loc[ir_idx, 'timestamp'].apply(lambda x: ir_holidays.get(x, default=0))
df_weather.loc[ca_idx, 'IsHoliday'] = df_weather.loc[ca_idx, 'timestamp'].apply(lambda x: ca_holidays.get(x, default=0))
df_weather.loc[us_idx, 'IsHoliday'] = df_weather.loc[us_idx, 'timestamp'].apply(lambda x: us_holidays.get(x, default=0))
holiday_idx = df_weather['IsHoliday'] != 0
df_weather.loc[holiday_idx, 'IsHoliday'] = 1
df_weather['IsHoliday'] = df_weather['IsHoliday'].astype(np.uint8 )
all_df['StateHoliday'] = all_df['StateHoliday'].astype('category' ).cat.codes<feature_engineering>
set_local(weather_train_df)
 add_holiyday(weather_train_df )
feature_name = 'Date'
dispatcher = {'DayOfMonth': pd.Index(all_df[feature_name] ).day,
'WeekOfYear': pd.Index(all_df[feature_name] ).week,
'MonthOfYear': pd.Index(all_df[feature_name] ).month,
'Year': pd.Index(all_df[feature_name] ).year,
'DayOfYear': pd.Index(all_df[feature_name] ).dayofyear
}
for new_feat_suffx, mapping in dispatcher.items() :
all_df[feature_name + new_feat_suffx] = mapping
all_df[feature_name] = all_df[feature_name].apply(date_to_ordinal)
 all_df.head()<load_from_csv>
train_df_black = train_df.copy()
train_df_black = train_df_black.merge(building_meta_df, on=['building_id', 'meter'], how='left')
 train_df_black = train_df_black.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')
 train_df_black['black_count']=0
for bid in train_df_black.building_id.unique() :
df = train_df_black[train_df_black.building_id==bid]
for meter in df.meter.unique() :
dfm = df[df.meter == meter]
b =(dfm.meter_reading == 0 ).astype(int)
 train_df_black.loc[(train_df_black.building_id==bid)&(train_df_black.meter == meter), 'black_count'] = b.groupby(( ~b.astype(bool)).cumsum() ).cumsum()
store_df = pd.read_csv('.. /input/store.csv', sep=',')
 store_df.head()<data_type_conversions>
train_df = train_df.merge(train_df_black[['timestamp','building_id','meter','black_count']], on=['timestamp','building_id','meter'])
 train_df = train_df[train_df['black_count'] < 24*black_day].drop('black_count', axis=1 )
def CompetitionOpenSince_to_ordinal(row):
try:
date = '%d-%d' %(int(row['CompetitionOpenSinceYear']), int(row['CompetitionOpenSinceMonth']))
return datetime.strptime(date, '%Y-%m' ).date().toordinal()
except:
return np.nan<data_type_conversions>
del train_df_black
gc.collect()
store_df['CompetitionOpenSinceDate'] = store_df.apply(CompetitionOpenSince_to_ordinal, axis=1)
 mean_CompetitionOpenSince = store_df['CompetitionOpenSinceDate'].mean()
store_df['CompetitionOpenSinceDate'] = store_df['CompetitionOpenSinceDate'].fillna(
 mean_CompetitionOpenSince ).astype(np.int64)
 store_df.head()<data_type_conversions>
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")' )
def Promo2Since_to_ordinal(row):
try:
date = '%d-W%d' %(int(row['Promo2SinceYear']), int(row['Promo2SinceWeek']))
return datetime.strptime(date + '-1', '%Y-W%W-%w' ).date().toordinal()
except:
return np.nan<data_type_conversions>
( train_df[train_df.building_id == 954].meter_reading==0 ).astype(int ).plot()
store_df['Promo2SinceDate'] = store_df.apply(Promo2Since_to_ordinal, axis=1)
 mean_Promo2SinceDate = store_df['Promo2SinceDate'].mean()
store_df['Promo2SinceDate'] = store_df['Promo2SinceDate'].fillna(mean_Promo2SinceDate ).astype(np.int64)
 store_df.loc[store_df['Promo2'] == 0, 'Promo2SinceDate'] = 0
store_df.head()<categorify>
train_df = train_df.query('not(building_id == 954 & meter_reading == 0)')
 train_df = train_df.query('not(building_id == 1221 & meter_reading == 0)' )
mode_PromoInterval = store_df[store_df['PromoInterval'].notnull() ]['PromoInterval'].mode() [0]
promo_intervals = {'Jan,Apr,Jul,Oct':(1,4,7,10),
'Feb,May,Aug,Nov':(2,5,8,11),
'Mar,Jun,Sept,Dec':(3,6,9,12)
 }
promo_intervals[np.nan] = promo_intervals[mode_PromoInterval]
store_df['PromoInterval'] = store_df['PromoInterval'].apply(lambda x: promo_intervals[x])
 store_df.head()<drop_column>
funny_bids = [993, 1168, 904, 954, 778, 1021]
print('before', len(train_df))
if outlier:
train_df.loc[(train_df.building_id == 993)&(train_df.meter == 0)&(train_df.meter_reading > 30000), 'meter_reading'] = 31921
train_df.loc[(train_df.building_id == 993)&(train_df.meter == 1)&(train_df.meter_reading > 90000), 'meter_reading'] = 96545.5
train_df = train_df[(( train_df.building_id == 1168)&(train_df.meter == 0)&(train_df.meter_reading >10000)) == False]
train_df.loc[(train_df.building_id == 904)&(train_df.meter == 0)&(train_df.meter_reading >10000), 'meter_reading'] = 11306
train_df = train_df[(( train_df.building_id == 954)&(train_df.meter_reading >10000)) ==False]
if rescale:
train_df.loc[(train_df.building_id == 778)&(train_df.meter == 1), 'meter_reading'] = train_df.loc[(train_df.building_id == 778)&(train_df.meter == 1), 'meter_reading']/ 100
train_df.loc[(train_df.building_id == 1021)&(train_df.meter == 3), 'meter_reading'] = train_df.loc[(train_df.building_id == 1021)&(train_df.meter == 3), 'meter_reading']/ 1000
train_df = train_df.reset_index()
print('after', len(train_df))
gc.collect()
columns_to_drop = ['CompetitionOpenSinceMonth', 'CompetitionOpenSinceYear',
'Promo2SinceWeek', 'Promo2SinceYear']
store_df.drop(columns_to_drop, axis=1, inplace=True)
 store_df.head()<feature_engineering>
train_df.loc[(train_df.building_id.isin(site_0_bids)) &(train_df.meter==0), 'meter_reading'] = train_df[(train_df.building_id.isin(site_0_bids)) &(train_df.meter==0)]['meter_reading'] * 0.2931
mean_CompetitionDistance = store_df.CompetitionDistance.mean()
store_df['CompetitionDistance'] = store_df['CompetitionDistance'].fillna(mean_CompetitionDistance)
 store_df.info()<merge>
train_df['date'] = train_df['timestamp'].dt.date
train_df['meter_reading_log1p'] = np.log1p(train_df['meter_reading'] )
df = pd.merge(all_df, store_df, how='left', on=['Store'])
 df.head().T<feature_engineering>
def preprocess(df):
df["hour"] = df["timestamp"].dt.hour
df["day"] = df["timestamp"].dt.day
df["weekend"] = df["timestamp"].dt.weekday
df["month"] = df["timestamp"].dt.month
df["dayofweek"] = df["timestamp"].dt.dayofweek
def is_promo2_today(row):
return int(row['Promo2'] == 1 and row['Promo2SinceDate'] <= row['Date'] \
and row['DateMonthOfYear'] in row['PromoInterval'])
 def is_competition_open(row):
return int(row['CompetitionOpenSinceDate'] <= row['Date'] )<data_type_conversions>
preprocess(train_df )
df['Promo2Today'] = df.apply(is_promo2_today, axis=1)
 df['CompetitionIsOpen'] = df.apply(is_competition_open, axis=1)
 avg_checks = pd.DataFrame(df.groupby('Store')['Sales'].sum().astype(np.float64)\
/ df.groupby('Store')['Customers'].sum().astype(np.float64),
columns=['AverageCheck'] ).reset_index()
df = df.merge(avg_checks, on='Store', how='left')
 df.head().T<drop_column>
if use_ucf:
train_df = train_df.sort_values('month')
 train_df = train_df.reset_index()
columns_to_drop = ['PromoInterval', 'Date', 'CompetitionOpenSinceDate', 'Promo2SinceDate']
df.drop(columns_to_drop, axis=1, inplace=True )<prepare_x_and_y>
X_train = train_df[train_df.columns.drop(['Customers', 'Sales'])].values
y_train = train_df['Sales'].values
print(X_train.shape, y_train.shape )<drop_column>
weather_train_df.isna().sum()
columns = train_df.columns.drop(['Sales', 'Customers'])
 del train_df
gc.collect()<define_search_space>
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )
<train_model>
weather_train_df = weather_train_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
rf = RandomForestRegressor(n_estimators=50, random_state=SEED)
 rf.fit(X_train, y_train )<compute_test_metric>
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )
<predict_on_test>
def add_lag_feature(weather_df, window=3):
group_df = weather_df.groupby('site_id')
 cols = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction', 'wind_speed']
rolled = group_df[cols].rolling(window=window, min_periods=0)
 lag_mean = rolled.mean().reset_index().astype(np.float16)
 lag_max = rolled.max().reset_index().astype(np.float16)
 lag_min = rolled.min().reset_index().astype(np.float16)
 lag_std = rolled.std().reset_index().astype(np.float16)
 for col in cols:
weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]
weather_df[f'{col}_max_lag{window}'] = lag_max[col]
weather_df[f'{col}_min_lag{window}'] = lag_min[col]
weather_df[f'{col}_std_lag{window}'] = lag_std[col]
test_predicted = rf.predict(X_test)
 test_predicted.shape<data_type_conversions>
add_lag_feature(weather_train_df, window=3)
 add_lag_feature(weather_train_df, window=72 )
submission = pd.concat([test_df.Id.astype(int), pd.DataFrame(test_predicted,
columns=['Sales'], index=test_df.index)], axis=1)
 submission.head()<save_to_csv>
year_map = building_meta_df.year_built.value_counts()
building_meta_df['year_cnt'] = building_meta_df.year_built.map(year_map)
 bid_map = train_df.building_id.value_counts()
train_df['bid_cnt'] = train_df.building_id.map(bid_map )
submission[['Id', 'Sales']].to_csv('submission_6.csv', index=False )<import_modules>
primary_use_list = building_meta_df['primary_use'].unique()
primary_use_dict = {key: value for value, key in enumerate(primary_use_list)}
print('primary_use_dict: ', primary_use_dict)
 building_meta_df['primary_use'] = building_meta_df['primary_use'].map(primary_use_dict)
 gc.collect()
import pandas as pd
import numpy as np<import_modules>
train_df = reduce_mem_usage(train_df, use_float16=True)
 building_meta_df = reduce_mem_usage(building_meta_df, use_float16=True)
 weather_train_df = reduce_mem_usage(weather_train_df, use_float16=True )
import pandas as pd
import numpy as np<load_from_csv>
def add_sg(df):
w = 11
p = 2
for si in df.site_id.unique() :
index = df.site_id == si
df.loc[index, 'air_smooth'] = sg(df[index].air_temperature, w, p)
 df.loc[index, 'dew_smooth'] = sg(df[index].dew_temperature, w, p)
 df.loc[index, 'air_diff'] = sg(df[index].air_temperature, w, p, 1)
 df.loc[index, 'dew_diff'] = sg(df[index].dew_temperature, w, p, 1)
 df.loc[index, 'air_diff2'] = sg(df[index].air_temperature, w, p, 2)
 df.loc[index, 'dew_diff2'] = sg(df[index].dew_temperature, w, p, 2 )
data = pd.read_csv('.. /input/train.csv', sep=',', low_memory=False)
 data['StateHoliday'] = data['StateHoliday'].apply(lambda x: str(x))<groupby>
add_sg(weather_train_df )
grouppedByStoreDayPromo = data[data['Sales'] > 0].groupby(by=['Store', 'DayOfWeek', 'Promo'] )<load_from_csv>
category_cols = ['building_id', 'primary_use', 'groupNum_train']
feature_cols = ['square_feet', 'year_built'] + [
'hour', 'weekend',
] + [
'air_temperature', 'cloud_coverage',
'dew_temperature', 'precip_depth_1_hr',
'sea_level_pressure',
'air_temperature_mean_lag72',
'air_temperature_max_lag72', 'air_temperature_min_lag72',
'air_temperature_std_lag72', 'cloud_coverage_mean_lag72',
'dew_temperature_mean_lag72', 'precip_depth_1_hr_mean_lag72',
'sea_level_pressure_mean_lag72',
'wind_speed_mean_lag72',
'air_temperature_mean_lag3',
'air_temperature_max_lag3',
'air_temperature_min_lag3', 'cloud_coverage_mean_lag3',
'dew_temperature_mean_lag3',
'precip_depth_1_hr_mean_lag3',
'sea_level_pressure_mean_lag3',
]
test = pd.read_csv('.. /input/test.csv', sep=',' )<groupby>
train_df = train_df.merge(building_meta_df, on=['building_id','meter'], how='left')
 train_df = train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')
 train_df = reduce_mem_usage(train_df, use_float16=True)
 gc.collect()
mn = data[data['Sales'] > 0].groupby(['Store', 'DayOfWeek', 'Promo'])['Sales'].mean().reset_index()<merge>
def create_X_y(train_df, groupNum_train):
target_train_df = train_df[train_df['groupNum_train'] == groupNum_train].copy()
X_train = target_train_df[feature_cols + category_cols]
y_train = target_train_df['meter_reading_log1p'].values
del target_train_df
return X_train, y_train
res = pd.merge(left=test, right=mn, on=['Store', 'DayOfWeek', 'Promo'], how='left')
 res.ix[res['Open'] == 0, 'Sales'] = 0<save_to_csv>
def fit_lgbm(train, val, devices=(-1,), seed=None, cat_features=None, num_rounds=1500, lr=0.1, bf=0.1):

 X_train, y_train = train
X_valid, y_valid = val
metric = 'l2'
params = {'num_leaves': 31,
'objective': 'regression',
'learning_rate': lr,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": bf,
"feature_fraction": 0.9,
"metric": metric,
}
device = devices[0]
if device == -1:
pass
else:
print(f'using gpu device_id {device}...')
 params.update({'device': 'gpu', 'gpu_device_id': device})
 params['seed'] = seed
early_stop = 20
verbose_eval = 50
d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)
 d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 print('best_score', model.best_score)
 log = {'train/mae': model.best_score['training']['l2'],
'valid/mae': model.best_score['valid_1']['l2']}
return model, y_pred_valid, log
res[['Id', 'Sales']].to_csv('result_mean.csv', sep=',', index=None )<set_options>
seed = 666
shuffle = False
kf = StratifiedKFold(n_splits=folds )
%matplotlib inline<load_from_csv>
for groupNum_train in building_meta_df['groupNum_train'].unique() :
X_train, y_train = create_X_y(train_df, groupNum_train=groupNum_train)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('groupNum_train', groupNum_train, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 exec('models' +str(groupNum_train)+ '=[]')
 train_df_site = train_df[train_df['groupNum_train']==groupNum_train].copy()
for train_idx, valid_idx in kf.split(train_df_site, train_df_site['building_id']):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
mindex = train_df_site.iloc[valid_idx,:].month.unique()
print(mindex)
 print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=num_rounds, lr=0.05, bf=0.7)
 y_valid_pred_total[valid_idx] = y_pred_valid
exec('models' +str(groupNum_train)+ '.append([mindex, model])')
 gc.collect()
if debug:
break
exec('oof' +str(groupNum_train)+ '= mean_squared_error(y_train, y_valid_pred_total)')
 try:
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 plt.show()
except:
pass
del X_train, y_train
gc.collect()
print('-------------------------------------------------------------' )
df_train = pd.read_csv(".. /input/train.csv", parse_dates=["Date"], date_parser=pd.to_datetime, low_memory=False)
 df_test = pd.read_csv(".. /input/test.csv", parse_dates=["Date"], date_parser=pd.to_datetime, low_memory=False)
 df_store = pd.read_csv(".. /input/store.csv" )<merge>
print('loading...')
 test_df = pd.read_feather(root/'test.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 weather_test_df = weather_test_df.drop_duplicates(['timestamp', 'site_id'])
 set_local(weather_test_df)
 add_holiyday(weather_test_df)
 print('preprocessing building...')
 test_df['date'] = test_df['timestamp'].dt.date
preprocess(test_df)
 print('preprocessing weather...')
 weather_test_df = weather_test_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
weather_test_df.groupby('site_id' ).apply(lambda group: group.isna().sum())
 add_sg(weather_test_df)
 add_lag_feature(weather_test_df, window=3)
 add_lag_feature(weather_test_df, window=72)
 test_df['bid_cnt'] = test_df.building_id.map(bid_map)
 test_df = test_df.merge(building_meta_df[['building_id','meter','groupNum_train']], on=['building_id','meter'], how='left')
 print('reduce mem usage...')
 test_df = reduce_mem_usage(test_df, use_float16=True)
 weather_test_df = reduce_mem_usage(weather_test_df, use_float16=True)
 gc.collect()
print(test_df.shape )
train = df_train.merge(df_store)
 test = df_test.merge(df_store )<merge>
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
reduce_mem_usage(sample_submission)
 print(sample_submission.shape )
train = df_train.merge(df_store)
 test = df_test.merge(df_store )<feature_engineering>
def create_X(test_df, groupNum_train):
target_test_df = test_df[test_df['groupNum_train'] == groupNum_train].copy()
target_test_df = target_test_df.merge(building_meta_df, on=['building_id','meter','groupNum_train'], how='left')
 target_test_df = target_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left')
 X_test = target_test_df[feature_cols + category_cols]
return X_test
def get_Promo2Active(df):
months_map = {v:i+1 for i, v in enumerate(["Jan", "Feb", "Mar", "Apr", "May", "Jun",
"Jul", "Aug", "Sept", "Oct", "Nov", "Dec"])}
def is_Promo2_active(row):
if row.Promo2 == 0:
return 0
current_week, current_month, current_year = row.Date.week, row.Date.month, row.Date.year
start_week, start_year = row.Promo2SinceWeek, row.Promo2SinceYear
active_months = set([months_map[m] for m in row.PromoInterval.split(",")])
 has_started =(current_year == start_year and current_week >= start_week)or current_year > start_year
return int(has_started and current_month in active_months)
 return df.apply(is_Promo2_active, axis=1 )<feature_engineering>
def pred_all(X_test, models, batch_size=1000000):
iterations =(X_test.shape[0] + batch_size -1)// batch_size
print('iterations', iterations)
 y_test_pred_total = np.zeros(X_test.shape[0])
 for i,(mindex, model)in enumerate(models):
print(f'predicting {i}-th model')
 for k in tqdm(range(iterations)) :
y_pred_test = model.predict(X_test[k*batch_size:(k+1)*batch_size], num_iteration=model.best_iteration)
 y_test_pred_total[k*batch_size:(k+1)*batch_size] += y_pred_test
y_test_pred_total /= len(models)
 return y_test_pred_total
def pred(X_test, models, batch_size=1000000):
if predmode == 'valid':
print('valid pred')
 return pred_valid(X_test, models, batch_size=1000000)
 elif predmode == 'train':
print('train pred')
 return pred_train(X_test, models, batch_size=1000000)
 else:
print('all pred')
 return pred_all(X_test, models, batch_size=1000000 )
def get_CompetitionActive(df):
def is_competition_active(row):
if np.isnan(row.CompetitionDistance):
return 0
if np.isnan(row.CompetitionOpenSinceMonth)and np.isnan(row.CompetitionOpenSinceYear):
return 1
current_month, current_year = row.Date.month, row.Date.year
opened_month, opened_year = row.CompetitionOpenSinceMonth, row.CompetitionOpenSinceYear
return int(( current_year == opened_year and current_month >= opened_month)or current_year > opened_year)
 return df.apply(is_competition_active, axis=1 )<feature_engineering>
sample_submission.loc[(test_df.building_id.isin(site_0_bids)) &(test_df.meter==0), 'meter_reading'] = sample_submission[(test_df.building_id.isin(site_0_bids)) &(test_df.meter==0)]['meter_reading'] * 3.4118
train["Promo2Active"] = get_Promo2Active(train)
 train["CompetitionActive"] = get_CompetitionActive(train )<groupby>
if clip0:
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0
train[train.Promo2 == 1].groupby("Promo2Active" ).agg({"Sales": "mean"} ).plot(kind="bar" )<merge>
if not debug:
sample_submission.to_csv('submission_no_leak.csv', index=False, float_format='%.4f' )
train = df_train.merge(df_store)
 test = df_test.merge(df_store)
 train["Promo2Active"] = get_Promo2Active(train)
 train["CompetitionActive"] = get_CompetitionActive(train)
 test["Promo2Active"] = get_Promo2Active(test)
 test["CompetitionActive"] = get_CompetitionActive(test )<data_type_conversions>
leak_df = pd.read_feather('.. /input/ashrae-leak-data-station/leak.feather')
 print(leak_df.duplicated().sum())
 print(leak_df.meter.value_counts())
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[(leak_df.timestamp.dt.year > 2016)&(leak_df.timestamp.dt.year < 2019)]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
leak_df = leak_df[leak_df.building_id!=245]
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0
test_df['pred'] = sample_submission.meter_reading
leak_df = leak_df.merge(test_df[['building_id', 'meter', 'timestamp', 'pred', 'row_id']], left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 leak_df = leak_df.merge(building_meta_df[['building_id', 'site_id']].drop_duplicates() , on='building_id', how='left' )
train["DayOfYear"] = train.Date.apply(lambda x: x.timetuple().tm_yday)
 test["DayOfYear"] = test.Date.apply(lambda x: x.timetuple().tm_yday )<feature_engineering>
leak_df['pred_l1p'] = np.log1p(leak_df.pred)
 leak_df['meter_reading_l1p'] = np.log1p(leak_df.meter_reading)
 sns.distplot(leak_df.pred_l1p)
 sns.distplot(leak_df.meter_reading_l1p)
 leak_score = np.sqrt(mean_squared_error(leak_df.pred_l1p, leak_df.meter_reading_l1p))
min_date = train.Date.min()
def date_to_day_number(df):
return(df.Date - min_date ).apply(lambda x: x.days )<feature_engineering>
leak_df = leak_df[['meter_reading', 'row_id']].set_index('row_id' ).dropna()
sample_submission.loc[leak_df.index, 'meter_reading'] = leak_df['meter_reading']
train["Day"] = date_to_day_number(train)
 test["Day"] = date_to_day_number(test )<sort_values>
if not debug:
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )
<compute_test_metric><EOS> 
print('LV score=', leak_score )