Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<count_values>
folds = 5
debug = False
num_rounds=1000
use_ucf = True
use_sort = True
del_2016 = False
ucf_year = [2017, 2018]
df_play["Position"].value_counts()<feature_engineering>
py.init_notebook_mode(connected=True)
def get_team_score(df_play):
df_play.loc[df_play["Team"]=="home", "rusherTeamScore"] = df_play["HomeScoreBeforePlay"]
df_play.loc[df_play["Team"]=="away", "rusherTeamScore"] = df_play["VisitorScoreBeforePlay"]
df_play.loc[df_play["Team"]=="home", "defenceTeamScore"] = df_play["VisitorScoreBeforePlay"]
df_play.loc[df_play["Team"]=="away", "defenceTeamScore"] = df_play["HomeScoreBeforePlay"]
df_play.loc[:, "diffScore"] = df_play["rusherTeamScore"] - df_play["defenceTeamScore"]
return df_play<compute_test_metric>
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df
df_play = get_team_score(df_play )<feature_engineering>
%%time
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 train_df = pd.read_feather(root/'train.feather')
 weather_train_df = pd.read_feather(root/'weather_train.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather' )
def player_team_name(df_play):
df_play.loc[:, "PlayerTeamAbbr"] = df_play["HomeTeamAbbr"]
df_play.loc[ df_play["Team"]=="away", "PlayerTeamAbbr"] = df_play["VisitorTeamAbbr"]
return df_play
def team_name(df_play):
df_play.loc[:, "PlayerTeamAbbr"] = df_play["HomeTeamAbbr"]
df_play.loc[ df_play["Team"]=="away", "PlayerTeamAbbr"] = df_play["VisitorTeamAbbr"]
df_play.loc[:, "DefenceTeamAbbr"] = df_play["HomeTeamAbbr"]
df_play.loc[ df_play["Team"]=="home", "DefenceTeamAbbr"] = df_play["VisitorTeamAbbr"]
return df_play<define_variables>
ucf_root = Path('.. /input/ashrae-ucf-spider-and-eda-full-test-labels')
 leak0_df = pd.read_pickle(ucf_root/'site0.pkl')
 leak0_df['meter_reading'] = leak0_df.meter_reading_scraped
leak0_df.drop(['meter_reading_original','meter_reading_scraped'], axis=1, inplace=True)
 leak0_df.fillna(0, inplace=True)
 leak0_df.loc[leak0_df.meter_reading < 0, 'meter_reading'] = 0
leak0_df = leak0_df[leak0_df.timestamp.dt.year > 2016]
print(len(leak0_df))
<feature_engineering>
ucl_root = Path('.. /usr/lib/ucl_data_leakage_episode_2')
 leak1_df = pd.read_pickle(ucl_root/'site1.pkl')
 leak1_df['meter_reading'] = leak1_df.meter_reading_scraped
leak1_df.drop(['meter_reading_scraped'], axis=1, inplace=True)
 leak1_df.fillna(0, inplace=True)
 leak1_df.loc[leak1_df.meter_reading < 0, 'meter_reading'] = 0
leak1_df = leak1_df[leak1_df.timestamp.dt.year > 2016]
print(len(leak1_df))
df_all.loc[:, "offense"] =(df_all["PlayerTeamAbbr"]==df_all["PossessionTeam"])*1<feature_engineering>
leak2_df = pd.read_csv('/kaggle/input/asu-buildings-energy-consumption/asu_2016-2018.csv')
 leak2_df['timestamp'] = pd.to_datetime(leak2_df['timestamp'])
 leak2_df.fillna(0, inplace=True)
 leak2_df.loc[leak2_df.meter_reading < 0, 'meter_reading'] = 0
leak2_df = leak2_df[leak2_df.timestamp.dt.year > 2016]
print(len(leak2_df))
def count_position(df_play):
df_play.loc[:, "OffensePersonnel"] = df_play["OffensePersonnel"].apply(
 lambda x : { i.split(" ")[-1]:int(i.split(" ")[-2])for i in x.split(",")})
 df_play.loc[:, "DefensePersonnel"] = df_play["DefensePersonnel"].apply(
 lambda x : { i.split(" ")[-1]:int(i.split(" ")[-2])for i in x.split(",")})
 for position in ["LB", "DB"]:
df_play.loc[:, position] = [ d[position] for d in df_play["DefensePersonnel"]]
for position in ["RB", "TE", "WR"]:
df_play.loc[:, position] = [ d[position] for d in df_play["OffensePersonnel"]]
return df_play<filter>
leak5_df = pd.read_csv('/kaggle/input/ashrae-site15-cornell/site15_leakage.csv')
 leak5_df['timestamp'] = pd.to_datetime(leak5_df['timestamp'])
 leak5_df.fillna(0, inplace=True)
 leak5_df.loc[leak5_df.meter_reading < 0, 'meter_reading'] = 0
leak5_df = leak5_df[leak5_df.timestamp.dt.year > 2016]
print(len(leak5_df))
df_play[df_play["Position"].isin(["CB", "DT", "DE"])][[ "PlayId", "DisplayName", "Position",
"Team", "PossessionTeam", "HomeTeamAbbr", "VisitorTeamAbbr"]] 
<filter>
print('New leak rate = {}'.format(( len(leak0_df)+ len(leak1_df)+ len(leak2_df)+ len(leak5_df)) / len(train_df)) )
df_all[df_all["PlayId"]==20181007081701][[ "PlayId", "DisplayName",
"Team", "PossessionTeam", "HomeTeamAbbr", "VisitorTeamAbbr", "Position"]]<normalization>
if use_ucf:
if del_2016:
print('delete all buildings site0 in 2016')
 bids = leak_df.building_id.unique()
train_df = train_df[train_df.building_id.isin(bids)== False]
leak0_df = leak0_df[leak0_df.timestamp.dt.year.isin(ucf_year)]
leak1_df = leak1_df[leak1_df.timestamp.dt.year.isin(ucf_year)]
leak2_df = leak2_df[leak2_df.timestamp.dt.year.isin(ucf_year)]
leak5_df = leak5_df[leak5_df.timestamp.dt.year.isin(ucf_year)]
train_df = pd.concat([train_df, leak0_df, leak1_df, leak2_df,leak5_df])
 train_df.reset_index(inplace=True)
 weather_train_df = pd.concat([weather_train_df, weather_test_df])
 weather_train_df.reset_index(inplace=True )
team_yards_df, defence_yards_df, personal_yards = average_yards(df_play )<define_variables>
del weather_test_df, leak0_df, leak1_df, leak2_df, leak5_df
gc.collect()
position_rename = {
"SAF":"S", "FS":"S", "SS":"S", "S":"S", "DB":"S",
"CB":"CB",
"ILB":"MLB", "MLB":"MLB",
"OLB":"OLB", "LB":"OLB",
"NT":"DT", "DT":"DT",
"DE":"DE", "DL":"DE",
"C":"C",
"G":"OG", "OG":"OG",
"T":"OT", "OT":"OT",
"WR":"WR",
"TE":"TE",
"QB":"QB",
"FB":"FB",
"HB":"HB",
"RB":"RB",
} 
<categorify>
train_df['date'] = train_df['timestamp'].dt.date
train_df['meter_reading_log1p'] = np.log1p(train_df['meter_reading'] )
df_all["position"] = df_all["Position"].map(position_rename)
 df_play["position"] = df_play["Position"].map(position_rename )<feature_engineering>
bad_rows = pd.read_csv('/kaggle/input/rows-to-drop/rows_to_drop.csv')
 train_df.drop(bad_rows.loc[:, '0'], inplace = True)
 train_df.reset_index(drop = True, inplace = True )
df_all.loc[df_all["offense"]==1, "position"] = df_all.loc[df_all["offense"]==1, "position"].apply(group_position_offense)
 df_all.loc[df_all["offense"]==0, "position"] = df_all.loc[df_all["offense"]==0, "position"].apply(group_position_defense)
<define_variables>
zone_dict={0:4,1:0,2:7,3:4,4:7,5:0,6:4,7:4,8:4,9:5,10:7,11:4,12:0,13:5,14:4,15:4}
def set_localtime(df):
for sid, zone in zone_dict.items() :
sids = df.site_id == sid
df.loc[sids, 'timestamp'] = df[sids].timestamp - pd.offsets.Hour(zone )
position_stats_columms=['X', 'Y', 'A', 'Dis','PlayerHeight', 'PlayerWeight', 'Sx', 'Sy', 'Orientation']
position_corrY_columms=['X', 'A', 'Dis','PlayerHeight', 'PlayerWeight', 'Sx', 'Sy', 'Orientation']<groupby>
def preprocess(df):
df["hour"] = df["timestamp"].dt.hour
df["weekend"] = df["timestamp"].dt.weekday
df["month"] = df["timestamp"].dt.month
def position_stats(df):
groups = df[(df["NflId"]!=df["NflIdRusher"])][
["PlayId", "position", *position_stats_columms]].groupby(["PlayId", "position"])
 position_mean = groups.agg(["count", "mean", "std", "skew"] ).unstack(level=1)
 position_mean.columns = [ col[2]+"_"+col[0]+"_"+col[1] for col in position_mean.columns]
cols = [col for col in position_mean.columns
if("X_count" in col)
 or("mean" in col)
 or("std" in col and re.findall("OL|WR|DL|LB|S_|CB", col))
or("skew" in col and re.findall("OL|DL", col)) ]
position_corr = groups.corr() ["Y"].unstack().drop("Y", axis=1 ).reset_index()
position_corr = position_corr.loc[position_corr["position"].isin(["OL", "DL", "LB", "WR"])]
position_corr = position_corr.pivot(index="PlayId", columns="position", values=position_corrY_columms)
 position_corr.columns = [ col[1]+"_"+col[0]+"_corrY" for col in position_corr.columns]
position_stats = position_mean[cols].join(position_corr)
 position_stats.fillna(0, inplace=True)
 return position_stats<groupby>
preprocess(train_df )
def average_personal_data(df_play, df_all):
offence_av = df_all.loc[df_all["offence"]==1, ["PlayerHeight", "PlayerWeight", "S", "A", "PlayId"]].groupby("PlayId" ).mean()
defence_av = df_all.loc[df_all["offence"]==0, ["PlayerHeight", "PlayerWeight", "S", "A", "PlayId"]].groupby("PlayId" ).mean()
offence_av.columns = ['PlayerHeight_offence', 'PlayerWeight_offence', 'S_offence', 'A_offence']
defence_av.columns = ['PlayerHeight_defence', 'PlayerWeight_defence', 'S_defence', 'A_defence']
df_play = df_play.merge(offence_av, on="PlayId", how="left" ).merge(defence_av, on="PlayId", how="left")
 return df_play<feature_engineering>
if use_ucf and use_sort:
train_df = train_df.sort_values('month')
 train_df = train_df.reset_index()
def map_position(df):
df = df.loc[df["NflId"]!=df["NflIdRusher"], ["Position", *position_stats_columms]]
df.loc[:,"offset"] =(df["Y"]-df["Y"].mean())**2 + df["X"]**2
OL = df[df["Position"]=="OL"].sort_values("offset")
 OB = df[df["Position"].isin(["QB", "RB", "TE", "WR"])].sort_values("offset")
 DL = df[df["Position"].isin(["DL", "LB"])].sort_values("offset")
 DB = df[df["Position"]=="DB"].sort_values("offset")
 if len(OL)>5:
OB = pd.concat([OB, OL.iloc[5:]])
 OL = OL.iloc[:5]
elif len(OB)>5:
OL = pd.concat([OL, OB.iloc[:-5]])
 OB = OL.iloc[-5:]
if len(DL)>7:
DB = pd.concat([DB, DL.iloc[7:]])
 DL = DL.iloc[:7]
elif len(DB)>4:
DL = pd.concat([DL, DB.iloc[:-4]])
 DB = DB.iloc[-4:]
OB, OL, DB, DL = OB.sort_values("Y"), OL.sort_values("Y"), DB.sort_values("Y"), DL.sort_values("Y")
 return pd.concat([OB, OL, DB, DL] ).drop("offset", axis=1 ).reset_index(drop=True)
 def process_positiondata(df_all):
map_position_df = df_all.groupby("PlayId" ).apply(map_position)
 positiondata_df = map_position_df.drop("Position",axis=1 ).unstack(level=1 ).fillna(0)
 positiondata_df.columns = [ col[0]+"_"+str(col[1])for col in positiondata_df.columns]
return positiondata_df<merge>
df_group = train_df.groupby('building_id')['meter_reading_log1p']
building_median = df_group.median().astype(np.float16)
 train_df['building_median'] = train_df['building_id'].map(building_median)
 del df_group
df_play = df_play.merge(position_stats(df_all), on="PlayId", how="left")
<merge>
weather_train_df.isna().sum()
def merge_lastYearAv(df_play, team_yards_df, defence_yards_df, personal_yards, year):
last_year = year-1
df = team_yards_df[[last_year]]
df.columns = ["team_yards_av", "team_yards_std"]
df_year = df_play[df_play["Season"]==year].merge(df, on=["Team","PlayerTeamAbbr"], how="left")
 df = defence_yards_df[[last_year]]
df.columns = ["def_yards_av", "def_yards_std"]
df_year = df_year.merge(df, on=["Team","DefenceTeamAbbr"], how="left")
 df_year = df_year.merge(
 personal_yards[last_year].rename(columns={'mean': 'player_yards_av', 'std': 'player_yards_std',
'count': 'player_yards_count'}),
on="NflId", how="left")
 return df_year<merge>
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )
def merge_twoYearAv(df_play, team_yards_df, defence_yards_df, personal_yards):
df =(team_yards_df[2017]+team_yards_df[2018])/2
df.columns = ["team_yards_av", "team_yards_std"]
df_year = df_play.merge(df, on=["Team","PlayerTeamAbbr"], how="left")
 df =(defence_yards_df[2017]+defence_yards_df[2018])/2
df.columns = ["def_yards_av", "def_yards_std"]
df_year = df_year.merge(df, on=["Team","DefenceTeamAbbr"], how="left")
 df =(personal_yards[2017]+personal_yards[2018])/2
df_year = df_year.merge(
 df.rename(columns={'mean': 'player_yards_av', 'std': 'player_yards_std',
'count': 'player_yards_count'}),
on="NflId", how="left")
 return df_year<merge>
weather_train_df = weather_train_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
def merge_anotherYearAv(df_play, team_yards_df, defence_yards_df, personal_yards):
df = team_yards_df[[2017]]
df.columns = ["team_yards_av", "team_yards_std"]
df_2018 = df_play[df_play["Season"]==2018].merge(df, on=["Team","PlayerTeamAbbr"], how="left")
 df = defence_yards_df[[2017]]
df.columns = ["def_yards_av", "def_yards_std"]
df_2018 = df_2018.merge(df, on=["Team","DefenceTeamAbbr"], how="left")
 df_2018 = df_2018.merge(
 personal_yards[2017].rename(columns={'mean': 'player_yards_av', 'std': 'player_yards_std',
'count': 'player_yards_count'}),
on="NflId", how="left")
 df = team_yards_df[[2018]]
df.columns = ["team_yards_av", "team_yards_std"]
df_2017 = df_play[df_play["Season"]==2017].merge(df, on=["Team","PlayerTeamAbbr"], how="left")
 df = defence_yards_df[[2018]]
df.columns = ["def_yards_av", "def_yards_std"]
df_2017 = df_2017.merge(df, on=["Team","DefenceTeamAbbr"], how="left")
 df_2017 = df_2017.merge(
 personal_yards[2018].rename(columns={'mean': 'player_yards_av', 'std': 'player_yards_std',
'count': 'player_yards_count'}),
on="NflId", how="left")
 return pd.concat([df_2017, df_2018] )<import_modules>
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )
import pickle<load_pretrained>
def add_lag_feature(weather_df, window=3):
group_df = weather_df.groupby('site_id')
 cols = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction', 'wind_speed']
rolled = group_df[cols].rolling(window=window, min_periods=0)
 lag_mean = rolled.mean().reset_index().astype(np.float16)
 lag_max = rolled.max().reset_index().astype(np.float16)
 lag_min = rolled.min().reset_index().astype(np.float16)
 lag_std = rolled.std().reset_index().astype(np.float16)
 for col in cols:
weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]
weather_df[f'{col}_max_lag{window}'] = lag_max[col]
weather_df[f'{col}_min_lag{window}'] = lag_min[col]
weather_df[f'{col}_std_lag{window}'] = lag_std[col]
df_year.to_pickle("df_year.pickle" )<count_missing_values>
set_localtime(weather_train_df )
df_year.isnull().sum().sort_values().tail(15 )<feature_engineering>
primary_use_list = building_meta_df['primary_use'].unique()
primary_use_dict = {key: value for value, key in enumerate(primary_use_list)}
print('primary_use_dict: ', primary_use_dict)
 building_meta_df['primary_use'] = building_meta_df['primary_use'].map(primary_use_dict)
 gc.collect()
def select_columns(df):
df_u = df.drop(["FieldPosition", "StadiumType", "GameWeather", "WindDirection", "DefendersInTheBox"], axis=1)
 df_u = df_u.fillna(df_u.mean())
 df_X = df_u.drop([
"DisplayName", "PossessionTeam",
"OffensePersonnel", "DefensePersonnel", "PlayDirection",
"PlayerBirthDate", "PlayerCollegeName",
"HomeTeamAbbr", "VisitorTeamAbbr",
"Location", "Turf", "Position",
"OffenseFormation",
"GameClock", "TimeHandoff",
"GameId", "PlayId", "JerseyNumber", "NflIdRusher",
"YardLine", "VisitorScoreBeforePlay", "HomeScoreBeforePlay", "defenceTeamScore",
], axis=1)
 return df_X<prepare_x_and_y>
train_df = reduce_mem_usage(train_df, use_float16=True)
 building_meta_df = reduce_mem_usage(building_meta_df, use_float16=True)
 weather_train_df = reduce_mem_usage(weather_train_df, use_float16=True )
df_X = select_columns(df_year)
 df_y = df_X["Yards"]
df_X = df_X.drop(["Yards"], axis=1 )<define_variables>
category_cols = ['building_id', 'site_id', 'primary_use']
feature_cols = ['square_feet', 'year_built'] + [
'hour', 'weekend',
'building_median'] + [
'air_temperature', 'cloud_coverage',
'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
'wind_direction', 'wind_speed' ]
cat_int = ['stadiumtype', 'weather', 'natural_turf',]
cat_num = ['NflId', 'Season', ]
cat_str = ['Team', 'position', "Stadium", "PlayerTeamAbbr", "DefenceTeamAbbr"]
categorical_features_indices = sorted([list(df_X.columns ).index(n)for n in cat_int+cat_num+cat_str] )<create_dataframe>
def create_X_y(train_df, target_meter):
target_train_df = train_df[train_df['meter'] == target_meter]
target_train_df = target_train_df.merge(building_meta_df, on='building_id', how='left')
 target_train_df = target_train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')
 X_train = target_train_df[feature_cols + category_cols]
y_train = target_train_df['meter_reading_log1p'].values
del target_train_df
return X_train, y_train
df_ycopy = df_y.copy()<define_search_space>
def fit_lgbm(train, val, devices=(-1,), seed=None, cat_features=None, num_rounds=1500, lr=0.1, bf=0.1):

 X_train, y_train = train
X_valid, y_valid = val
metric = 'l2'
params = {'num_leaves': 31,
'objective': 'regression',
'learning_rate': lr,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": bf,
"feature_fraction": 0.9,
"metric": metric,
}
device = devices[0]
if device == -1:
pass
else:
print(f'using gpu device_id {device}...')
 params.update({'device': 'gpu', 'gpu_device_id': device})
 params['seed'] = seed
early_stop = 20
verbose_eval = 20
d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)
 d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 print('best_score', model.best_score)
 log = {'train/mae': model.best_score['training']['l2'],
'valid/mae': model.best_score['valid_1']['l2']}
return model, y_pred_valid, log
max_yards = 20
min_yards = -5<filter>
seed = 666
shuffle = False
kf = KFold(n_splits=folds, shuffle=shuffle, random_state=seed)
 oof_total = 0
print(len(df_y.loc[df_y>=max_yards]))
print(len(df_y.loc[df_y<=min_yards]))<count_values>
target_meter = 0
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models0 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=num_rounds, lr=0.05, bf=0.7)
 y_valid_pred_total[valid_idx] = y_pred_valid
models0.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof0 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof0 * len(y_train)
 del X_train, y_train
gc.collect()
df_count = df_ycopy.value_counts().sort_index()<import_modules>
target_meter = 1
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models1 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=num_rounds,
lr=0.05, bf=0.5)
 y_valid_pred_total[valid_idx] = y_pred_valid
models1.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof1 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof1 * len(y_train)
 del X_train, y_train
gc.collect()
from scipy.optimize import curve_fit<train_model>
target_meter = 2
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models2 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=num_rounds, lr=0.05, bf=0.8)
 y_valid_pred_total[valid_idx] = y_pred_valid
models2.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof2 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof2 * len(y_train)
 del X_train, y_train
gc.collect()
def linear_fit(x, a, b):
return a*x+b
array_x= df_count.loc[max_yards:37].index
array_y= np.log(df_count.loc[max_yards:37].values)
 param, cov = curve_fit(linear_fit, array_x, array_y)
 large_probs = pd.DataFrame(np.exp(np.array(range(max_yards,100)) *param[0]),
columns=["value"], index=np.array(range(max_yards,100)))
 large_probs = large_probs/large_probs.sum()<feature_engineering>
target_meter = 3
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models3 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=num_rounds,
lr=0.03, bf=0.9)
 y_valid_pred_total[valid_idx] = y_pred_valid
models3.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof3 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof3 * len(y_train)
 del X_train, y_train
gc.collect()
df_y.loc[df_y>=max_yards] = max_yards
df_y.loc[df_y<=min_yards] = min_yards<import_modules>
print('oof score meter0 =', np.sqrt(oof0))
print('oof score meter1 =', np.sqrt(oof1))
print('oof score meter2 =', np.sqrt(oof2))
print('oof score meter3 =', np.sqrt(oof3))
print('oof score total =', np.sqrt(oof_total / len(train_df)) )
from tqdm import tqdm
from sklearn.model_selection import train_test_split 
<import_modules>
del train_df, weather_train_df, building_meta_df
gc.collect()
from catboost import CatBoostClassifier, Pool 
<split>
print('loading...')
 test_df = pd.read_feather(root/'test.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather')
 set_localtime(weather_test_df)
 print('preprocessing building...')
 test_df['date'] = test_df['timestamp'].dt.date
preprocess(test_df)
 test_df['building_median'] = test_df['building_id'].map(building_median)
 print('preprocessing weather...')
 weather_test_df = weather_test_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
weather_test_df.groupby('site_id' ).apply(lambda group: group.isna().sum())
 print('reduce mem usage...')
 reduce_mem_usage(test_df, use_float16=True)
 reduce_mem_usage(weather_test_df, use_float16=True)
 gc.collect()
train_X, test_X, train_y, test_y, train_ycopy, test_ycopy = train_test_split(df_X, df_y, df_ycopy, test_size=0.3 )<define_variables>
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
reduce_mem_usage(sample_submission )
df_y0 = df_y-min_yards
train_y0 = train_y-min_yards
test_y0 = test_y-min_yards<import_modules>
def create_X(test_df, target_meter):
target_test_df = test_df[test_df['meter'] == target_meter]
target_test_df = target_test_df.merge(building_meta_df, on='building_id', how='left')
 target_test_df = target_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left')
 X_test = target_test_df[feature_cols + category_cols]
return X_test
from sklearn.model_selection import KFold 
<create_dataframe>
def pred(X_test, models, batch_size=1000000):
iterations =(X_test.shape[0] + batch_size -1)// batch_size
print('iterations', iterations)
 y_test_pred_total = np.zeros(X_test.shape[0])
 for i, model in enumerate(models):
print(f'predicting {i}-th model')
 for k in tqdm(range(iterations)) :
y_pred_test = model.predict(X_test[k*batch_size:(k+1)*batch_size], num_iteration=model.best_iteration)
 y_test_pred_total[k*batch_size:(k+1)*batch_size] += y_pred_test
y_test_pred_total /= len(models)
 return y_test_pred_total
def calc_score(pred_y, test_y, yardsToTouchdown):
pred_df = pd.DataFrame(pred_y)
 pred_df.columns = [ "Yards"+str(n)for n in range(min_yards,max_yards+1)]
pred_df_all = pd.DataFrame(pred_df,columns=[ "Yards"+str(n)for n in range(-99,100)])
 pred_df_all.loc[:, "Yards"+str(max_yards):] = \ 
(pred_df_all.loc[:, ["Yards"+str(max_yards)]] ).dot(large_probs.values.T ).values
pred_df_all.loc[:, "Yards-14":"Yards"+str(min_yards)] = \ 
(pred_df_all.loc[:, ["Yards"+str(min_yards)]] ).dot(small_probs.values.T ).values
pred_df_all.fillna(0, inplace=True)
 pred_df_all = pred_df_all.cumsum(axis=1)
 for index, row in pred_df_all.iterrows() :
pred_df_all.loc[index, "Yards"+str(yardsToTouchdown[index]):] = 1
test_df = pd.DataFrame(np.array([(i >= test_y)*1 for i in range(-99,100)] ).T,
columns=pred_df_all.columns)
 CRPS=(( pred_df_all-test_df)**2 ).mean().mean()
return CRPS, pred_df_all, test_df<import_modules>
sample_submission.loc[test_df['meter'] == 0, 'meter_reading'] = np.expm1(y_test0)
 sample_submission.loc[test_df['meter'] == 1, 'meter_reading'] = np.expm1(y_test1)
 sample_submission.loc[test_df['meter'] == 2, 'meter_reading'] = np.expm1(y_test2)
 sample_submission.loc[test_df['meter'] == 3, 'meter_reading'] = np.expm1(y_test3 )
from six.moves import xrange<import_modules>
if not debug:
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )
import math<compute_train_metric>
leak_score0 = 0
leak_df = pd.read_pickle(ucf_root/'site0.pkl')
 leak_df['meter_reading'] = leak_df.meter_reading_scraped
leak_df.drop(['meter_reading_original','meter_reading_scraped'], axis=1, inplace=True)
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[leak_df.timestamp.dt.year > 2016]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0
for bid in leak_df.building_id.unique() :
temp_df = leak_df[(leak_df.building_id == bid)]
for m in temp_df.meter.unique() :
v0 = sample_submission.loc[(test_df.building_id == bid)&(test_df.meter==m), 'meter_reading'].values
v1 = temp_df[temp_df.meter==m].meter_reading.values
leak_score0 += mean_squared_error(np.log1p(v0), np.log1p(v1)) * len(v0)
 sample_submission.loc[(test_df.building_id == bid)&(test_df.meter==m), 'meter_reading'] = temp_df[temp_df.meter==m].meter_reading.values
leak_score0 /= len(leak_df )
class CRPSMetric(object):
def get_final_error(self, error, weight):
return error /(weight + 1e-38)
 def is_max_optimal(self):
return True
def evaluate(self, approxes, target, weight):
n_class =(max_yards-min_yards+1)
 assert len(approxes)== n_class
assert len(target)== len(approxes[0])
 error_sum = 0
weight_sum = 0
for i in xrange(len(target)) :
w = 1.0 if weight is None else weight[i]
weight_sum += w
for j in xrange(len(approxes)) :
target01 =(target[i]>=j)*1
prob = 1/(1+(1/math.exp(approxes[j][i])))
 error_sum +=(( prob - target01)**2)/n_class
return error_sum, len(target )<compute_test_metric>
leak_score1 = 0
leak_df = pd.read_pickle(ucl_root/'site1.pkl')
 leak_df['meter_reading'] = leak_df.meter_reading_scraped
leak_df.drop(['meter_reading_scraped'], axis=1, inplace=True)
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[leak_df.timestamp.dt.year > 2016]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
for bid in leak_df.building_id.unique() :
temp_df = leak_df[(leak_df.building_id == bid)]
for m in temp_df.meter.unique() :
v0 = sample_submission.loc[(test_df.building_id == bid)&(test_df.meter==m), 'meter_reading'].values
v1 = temp_df[temp_df.meter==m].meter_reading.values
leak_score1 += mean_squared_error(np.log1p(v0), np.log1p(v1)) * len(v0)
 sample_submission.loc[(test_df.building_id == bid)&(test_df.meter==m), 'meter_reading'] = temp_df[temp_df.meter==m].meter_reading.values
leak_score1 /= len(leak_df )
<import_modules>
leak_score2 = 0
leak_df = pd.read_csv('/kaggle/input/asu-buildings-energy-consumption/asu_2016-2018.csv')
 leak_df['timestamp'] = pd.to_datetime(leak_df['timestamp'])
 leak_df.fillna(0, inplace=True)
 leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
leak_df = leak_df[leak_df.timestamp.dt.year > 2016]
for bid in leak_df.building_id.unique() :
temp_df = leak_df[(leak_df.building_id == bid)]
for m in temp_df.meter.unique() :
v0 = sample_submission.loc[(test_df.building_id == bid)&(test_df.meter==m), 'meter_reading'].values
v1 = temp_df[temp_df.meter==m].meter_reading.values
leak_score2 += mean_squared_error(np.log1p(v0), np.log1p(v1)) * len(v0)
 sample_submission.loc[(test_df.building_id == bid)&(test_df.meter==m), 'meter_reading'] = temp_df[temp_df.meter==m].meter_reading.values
leak_score2 /= len(leak_df )
import optuna<train_on_grid>
if not debug:
sample_submission.to_csv('submission_ucf_replaced.csv', index=False, float_format='%.4f' )
<train_on_grid><EOS> 
print('UCF score = ', np.sqrt(leak_score0))
print('UCL score = ', np.sqrt(leak_score1))
print('ASU score = ', np.sqrt(leak_score2))