Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<categorify>
folds = 3
debug = False
num_rounds=1000
use_ucf = False
use_sort = False
del_2016 = False
copy_site2 = False
ucf_year = [2017, 2018]
def formation_numeric(txt):
QB = 0
RB = 0
WR = 0
TE = 0
OL = 0
DL = 0
LB = 0
DB = 0
if "QB" in txt:
QB = 1 - int(txt[txt.find("QB")-2:txt.find("QB")-1])
 if "RB" in txt:
RB = 1 - int(txt[txt.find("RB")-2:txt.find("RB")-1])
 if "WR" in txt:
WR = 3 - int(txt[txt.find("WR")-2:txt.find("WR")-1])
 if "TE" in txt:
TE = int(txt[txt.find("TE")-2:txt.find("TE")-1])- 1
if "OL" in txt:
OL = int(txt[txt.find("OL")-2:txt.find("OL")-1])- 5
if "DL" in txt:
DL = int(txt[txt.find("DL")-2:txt.find("DL")-1])
 if "LB" in txt:
LB = int(txt[txt.find("LB")-2:txt.find("LB")-1])
 if "DB" in txt:
DB = -int(txt[txt.find("DB")-2:txt.find("DB")-1])
 return QB + RB + WR + TE + OL + DL + LB + DB<feature_engineering>
py.init_notebook_mode(connected=True)
train['Formation_Heavy_Score'] = train["OffensePersonnel"].apply(formation_numeric )<count_values>
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df
train['Formation_Heavy_Score'].value_counts()<data_type_conversions>
%%time
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 train_df = pd.read_feather(root/'train.feather')
 weather_train_df = pd.read_feather(root/'weather_train.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather' )
formation_numeric("1 TE, 9 OL" )<count_values>
ucf_root = Path('.. /input/ashrae-ucf-spider-and-eda-full-test-labels')
 leak0_df = pd.read_pickle(ucf_root/'site0.pkl')
 leak0_df['meter_reading'] = leak0_df.meter_reading_scraped
leak0_df.drop(['meter_reading_original','meter_reading_scraped'], axis=1, inplace=True)
 leak0_df.fillna(0, inplace=True)
 leak0_df.loc[leak0_df.meter_reading < 0, 'meter_reading'] = 0
leak0_df = leak0_df[leak0_df.timestamp.dt.year > 2016]
print(len(leak0_df))
train['GameClock'].value_counts()<feature_engineering>
ucl_root = Path('.. /usr/lib/ucl_data_leakage_episode_2')
 leak1_df = pd.read_pickle(ucl_root/'site1.pkl')
 leak1_df['meter_reading'] = leak1_df.meter_reading_scraped
leak1_df.drop(['meter_reading_scraped'], axis=1, inplace=True)
 leak1_df.fillna(0, inplace=True)
 leak1_df.loc[leak1_df.meter_reading < 0, 'meter_reading'] = 0
leak1_df = leak1_df[leak1_df.timestamp.dt.year > 2016]
print(len(leak1_df))
train['GameClock'] = train['GameClock'].apply(strtoseconds )<feature_engineering>
leak2_df = pd.read_csv('/kaggle/input/asu-buildings-energy-consumption/asu_2016-2018.csv')
 leak2_df['timestamp'] = pd.to_datetime(leak2_df['timestamp'])
 leak2_df.fillna(0, inplace=True)
 leak2_df.loc[leak2_df.meter_reading < 0, 'meter_reading'] = 0
if copy_site2:
train_df = train_df.merge(leak2_df, left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 train_df.loc[train_df.meter_reading_y.isnull() == False, 'meter_reading_x'] = train_df[train_df.meter_reading_y.isnull() == False].meter_reading_y
train_df.drop('meter_reading_y', axis=1, inplace=True)
 train_df.rename(columns={'meter_reading_x': 'meter_reading'}, inplace=True)
 leak2_df = leak2_df[leak2_df.timestamp.dt.year > 2016]
leak2_df = leak2_df[leak2_df.building_id!=245]
print(len(leak2_df))
train['PlayerHeight'] = train['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))<feature_engineering>
print('leak rate = {}'.format(( len(leak0_df)+ len(leak1_df)+ len(leak2_df)) / len(train_df)) )
train['PlayerBMI'] = 703*(train['PlayerWeight']/(train['PlayerHeight'])**2 )<data_type_conversions>
if use_ucf:
leak0_df = leak0_df[leak0_df.timestamp.dt.year.isin(ucf_year)]
leak1_df = leak1_df[leak1_df.timestamp.dt.year.isin(ucf_year)]
leak2_df = leak2_df[leak2_df.timestamp.dt.year.isin(ucf_year)]
train_df = pd.concat([train_df, leak0_df, leak1_df, leak2_df])
 train_df.reset_index(inplace=True)
 weather_train_df = pd.concat([weather_train_df, weather_test_df])
 weather_train_df.reset_index(inplace=True )
train['TimeHandoff'] = train['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeSnap'] = train['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))<feature_engineering>
del weather_test_df, leak0_df, leak1_df, leak2_df
gc.collect()
train['TimeDelta'] = train.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1 )<feature_engineering>
train_df['date'] = train_df['timestamp'].dt.date
train_df['meter_reading_log1p'] = np.log1p(train_df['meter_reading'] )
train['PlayerBirthDate'] = train['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))<feature_engineering>
building_meta_df[building_meta_df.site_id == 0]
seconds_in_year = 60*60*24*365.25
train['PlayerAge'] = train.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1 )<drop_column>
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")' )
train = train.drop(['TimeHandoff', 'TimeSnap', 'PlayerBirthDate'], axis=1 )<count_values>
zone_dict={0:4,1:0,2:7,3:4,4:7,5:0,6:4,7:4,8:4,9:5,10:7,11:4,12:0,13:5,14:4,15:4}
def set_localtime(df):
for sid, zone in zone_dict.items() :
sids = df.site_id == sid
df.loc[sids, 'timestamp'] = df[sids].timestamp - pd.offsets.Hour(zone )
train['WindSpeed'].value_counts()<feature_engineering>
def preprocess(df):
df["hour"] = df["timestamp"].dt.hour
df["weekend"] = df["timestamp"].dt.weekday
df["month"] = df["timestamp"].dt.month
df["dayofweek"] = df["timestamp"].dt.dayofweek
train['WindSpeed'] = train['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x )<count_values>
preprocess(train_df )
train['WindSpeed'].value_counts()<feature_engineering>
if use_ucf and use_sort:
train_df = train_df.sort_values('month')
 train_df = train_df.reset_index()
train['WindSpeed'] = train['WindSpeed'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 train['WindSpeed'] = train['WindSpeed'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x )<data_type_conversions>
df_group = train_df.groupby('building_id')['meter_reading_log1p']
building_median = df_group.median().astype(np.float16)
 train_df['building_median'] = train_df['building_id'].map(building_median)
 del df_group
def str_to_float(txt):
try:
return float(txt)
 except:
return -1<feature_engineering>
weather_train_df.isna().sum()
train['WindSpeed'] = train['WindSpeed'].apply(str_to_float )<count_values>
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )
train['WindDirection'].value_counts()<categorify>
weather_train_df = weather_train_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
def clean_WindDirection(txt):
if pd.isna(txt):
return np.nan
txt = txt.lower()
txt = ''.join([c for c in txt if c not in punctuation])
 txt = txt.replace('from', '')
 txt = txt.replace(' ', '')
 txt = txt.replace('north', 'n')
 txt = txt.replace('south', 's')
 txt = txt.replace('west', 'w')
 txt = txt.replace('east', 'e')
 return txt<feature_engineering>
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )
train['WindDirection'] = train['WindDirection'].apply(clean_WindDirection )<count_values>
def add_lag_feature(weather_df, window=3):
group_df = weather_df.groupby('site_id')
 cols = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction', 'wind_speed']
rolled = group_df[cols].rolling(window=window, min_periods=0)
 lag_mean = rolled.mean().reset_index().astype(np.float16)
 lag_max = rolled.max().reset_index().astype(np.float16)
 lag_min = rolled.min().reset_index().astype(np.float16)
 lag_std = rolled.std().reset_index().astype(np.float16)
 for col in cols:
weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]
weather_df[f'{col}_max_lag{window}'] = lag_max[col]
weather_df[f'{col}_min_lag{window}'] = lag_min[col]
weather_df[f'{col}_std_lag{window}'] = lag_std[col]
train['WindDirection'].value_counts()<categorify>
set_localtime(weather_train_df )
def transform_WindDirection(txt):
if pd.isna(txt):
return np.nan
if txt=='n':
return 0
if txt=='nne' or txt=='nen':
return 1/8
if txt=='ne':
return 2/8
if txt=='ene' or txt=='nee':
return 3/8
if txt=='e':
return 4/8
if txt=='ese' or txt=='see':
return 5/8
if txt=='se':
return 6/8
if txt=='ses' or txt=='sse':
return 7/8
if txt=='s':
return 8/8
if txt=='ssw' or txt=='sws':
return 9/8
if txt=='sw':
return 10/8
if txt=='sww' or txt=='wsw':
return 11/8
if txt=='w':
return 12/8
if txt=='wnw' or txt=='nww':
return 13/8
if txt=='nw':
return 14/8
if txt=='nwn' or txt=='nnw':
return 15/8
return np.nan<feature_engineering>
primary_use_list = building_meta_df['primary_use'].unique()
primary_use_dict = {key: value for value, key in enumerate(primary_use_list)}
print('primary_use_dict: ', primary_use_dict)
 building_meta_df['primary_use'] = building_meta_df['primary_use'].map(primary_use_dict)
 gc.collect()
train['WindDirection'] = train['WindDirection'].apply(transform_WindDirection )<count_values>
train_df = reduce_mem_usage(train_df, use_float16=True)
 building_meta_df = reduce_mem_usage(building_meta_df, use_float16=True)
 weather_train_df = reduce_mem_usage(weather_train_df, use_float16=True )
train['PlayDirection'].value_counts()<feature_engineering>
category_cols = ['building_id', 'site_id', 'primary_use']
feature_cols = ['square_feet', 'year_built'] + [
'hour', 'weekend',
'building_median'] + [
'air_temperature', 'cloud_coverage',
'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
'wind_direction', 'wind_speed',]
train['PlayDirection'] = train['PlayDirection'].apply(lambda x: x.strip() == 'right' )<feature_engineering>
def create_X_y(train_df, target_meter):
target_train_df = train_df[train_df['meter'] == target_meter]
target_train_df = target_train_df.merge(building_meta_df, on='building_id', how='left')
 target_train_df = target_train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')
 X_train = target_train_df[feature_cols + category_cols]
y_train = target_train_df['meter_reading_log1p'].values
del target_train_df
return X_train, y_train
train['Team'] = train['Team'].apply(lambda x: x.strip() =='home' )<feature_engineering>
def fit_lgbm(train, val, devices=(-1,), seed=None, cat_features=None, num_rounds=1500, lr=0.1, bf=0.1):

 X_train, y_train = train
X_valid, y_valid = val
metric = 'l2'
params = {'num_leaves': 31,
'objective': 'regression',
'learning_rate': lr,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": bf,
"feature_fraction": 0.9,
"metric": metric,
}
device = devices[0]
if device == -1:
pass
else:
print(f'using gpu device_id {device}...')
 params.update({'device': 'gpu', 'gpu_device_id': device})
 params['seed'] = seed
early_stop = 20
verbose_eval = 20
d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)
 d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 print('best_score', model.best_score)
 log = {'train/mae': model.best_score['training']['l2'],
'valid/mae': model.best_score['valid_1']['l2']}
return model, y_pred_valid, log
train['GameWeather'] = train['GameWeather'].str.lower()
indoor = "indoor"
train['GameWeather'] = train['GameWeather'].apply(lambda x: indoor if not pd.isna(x)and indoor in x else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x )<count_values>
seed = 666
shuffle = False
kf = KFold(n_splits=folds, shuffle=shuffle, random_state=seed)
 oof_total = 0
weather_count = Counter()
for weather in train['GameWeather']:
if pd.isna(weather):
continue
for word in weather.split() :
weather_count[word]+=1
weather_count.most_common() [:15]<categorify>
target_meter = 0
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models0 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=num_rounds, lr=0.05, bf=0.7)
 y_valid_pred_total[valid_idx] = y_pred_valid
models0.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof0 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof0 * len(y_train)
 del X_train, y_train
gc.collect()
def map_weather(txt):
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt:
ans*=0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*3
if 'sunny' in txt or 'sun' in txt:
return ans*2
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt:
return -2*ans
if 'snow' in txt:
return -3*ans
return 0<feature_engineering>
target_meter = 1
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models1 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=num_rounds,
lr=0.05, bf=0.5)
 y_valid_pred_total[valid_idx] = y_pred_valid
models1.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof1 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof1 * len(y_train)
 del X_train, y_train
gc.collect()
train['GameWeather'] = train['GameWeather'].apply(map_weather )<feature_engineering>
target_meter = 2
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models2 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=num_rounds, lr=0.05, bf=0.8)
 y_valid_pred_total[valid_idx] = y_pred_valid
models2.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof2 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof2 * len(y_train)
 del X_train, y_train
gc.collect()
train['IsRusher'] = train['NflId'] == train['NflIdRusher']<drop_column>
target_meter = 3
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models3 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=num_rounds,
lr=0.03, bf=0.9)
 y_valid_pred_total[valid_idx] = y_pred_valid
models3.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof3 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof3 * len(y_train)
 del X_train, y_train
gc.collect()
train.drop(['NflId', 'NflIdRusher'], axis=1, inplace=True )<feature_engineering>
oof_total = oof_total / len(train_df )
train['X'] = train.apply(lambda row: row['X'] if row['PlayDirection'] else 120-row['X'], axis=1 )<feature_engineering>
del train_df, weather_train_df, building_meta_df
gc.collect()
def new_orientation(angle, play_direction):
if play_direction == 0:
new_angle = 360.0 - angle
if new_angle == 360.0:
new_angle = 0.0
return new_angle
else:
return angle
train['Orientation'] = train.apply(lambda row: new_orientation(row['Orientation'], row['PlayDirection']), axis=1)
 train['Dir'] = train.apply(lambda row: new_orientation(row['Dir'], row['PlayDirection']), axis=1 )<feature_engineering>
print('loading...')
 test_df = pd.read_feather(root/'test.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather')
 set_localtime(weather_test_df)
 print('preprocessing building...')
 test_df['date'] = test_df['timestamp'].dt.date
preprocess(test_df)
 test_df['building_median'] = test_df['building_id'].map(building_median)
 print('preprocessing weather...')
 weather_test_df = weather_test_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
weather_test_df.groupby('site_id' ).apply(lambda group: group.isna().sum())
 print('reduce mem usage...')
 test_df = reduce_mem_usage(test_df, use_float16=True)
 weather_test_df = reduce_mem_usage(weather_test_df, use_float16=True)
 gc.collect()
train['YardsLeft'] = train.apply(lambda row: 100-row['YardLine'] if row['HomeField'] else row['YardLine'], axis=1)
 train['YardsLeft'] = train.apply(lambda row: row['YardsLeft'] if row['PlayDirection'] else 100-row['YardsLeft'], axis=1 )<filter>
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
sample_submission = reduce_mem_usage(sample_submission )
((train['YardsLeft']<train['Yards'])|(train['YardsLeft']-100>train['Yards'])).mean()<drop_column>
def create_X(test_df, target_meter):
target_test_df = test_df[test_df['meter'] == target_meter]
target_test_df = target_test_df.merge(building_meta_df, on='building_id', how='left')
 target_test_df = target_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left')
 X_test = target_test_df[feature_cols + category_cols]
return X_test
train.drop(train.index[(train['YardsLeft']<train['Yards'])|(train['YardsLeft']-100>train['Yards'])], inplace=True )<sort_values>
def pred(X_test, models, batch_size=1000000):
iterations =(X_test.shape[0] + batch_size -1)// batch_size
print('iterations', iterations)
 y_test_pred_total = np.zeros(X_test.shape[0])
 for i, model in enumerate(models):
print(f'predicting {i}-th model')
 for k in tqdm(range(iterations)) :
y_pred_test = model.predict(X_test[k*batch_size:(k+1)*batch_size], num_iteration=model.best_iteration)
 y_test_pred_total[k*batch_size:(k+1)*batch_size] += y_pred_test
y_test_pred_total /= len(models)
 return y_test_pred_total
train = train.sort_values(by=['PlayId', 'Team', 'IsRusher', 'JerseyNumber'] ).reset_index()<drop_column>
sample_submission.loc[test_df['meter'] == 0, 'meter_reading'] = np.expm1(y_test0)
 sample_submission.loc[test_df['meter'] == 1, 'meter_reading'] = np.expm1(y_test1)
 sample_submission.loc[test_df['meter'] == 2, 'meter_reading'] = np.expm1(y_test2)
 sample_submission.loc[test_df['meter'] == 3, 'meter_reading'] = np.expm1(y_test3 )
train.drop(['GameId', 'PlayId', 'index', 'IsRusher', 'Team'], axis=1, inplace=True )<drop_column>
if not debug:
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )
cat_features = []
for col in train.columns:
if train[col].dtype =='object':
cat_features.append(col)
 train = train.drop(cat_features, axis=1 )<correct_missing_values>
leak_score0 = 0
leak_df = pd.read_pickle(ucf_root/'site0.pkl')
 leak_df['meter_reading'] = leak_df.meter_reading_scraped
leak_df.drop(['meter_reading_original','meter_reading_scraped'], axis=1, inplace=True)
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[leak_df.timestamp.dt.year > 2016]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0
for bid in leak_df.building_id.unique() :
temp_df = leak_df[(leak_df.building_id == bid)]
for m in temp_df.meter.unique() :
v0 = sample_submission.loc[(test_df.building_id == bid)&(test_df.meter==m), 'meter_reading'].values
v1 = temp_df[temp_df.meter==m].meter_reading.values
leak_score0 += mean_squared_error(np.log1p(v0), np.log1p(v1)) * len(v0)
 sample_submission.loc[(test_df.building_id == bid)&(test_df.meter==m), 'meter_reading'] = temp_df[temp_df.meter==m].meter_reading.values
leak_score0 /= len(leak_df )
train.fillna(-999, inplace=True )<prepare_x_and_y>
leak_score1 = 0
leak_df = pd.read_pickle(ucl_root/'site1.pkl')
 leak_df['meter_reading'] = leak_df.meter_reading_scraped
leak_df.drop(['meter_reading_scraped'], axis=1, inplace=True)
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[leak_df.timestamp.dt.year > 2016]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
for bid in leak_df.building_id.unique() :
temp_df = leak_df[(leak_df.building_id == bid)]
for m in temp_df.meter.unique() :
v0 = sample_submission.loc[(test_df.building_id == bid)&(test_df.meter==m), 'meter_reading'].values
v1 = temp_df[temp_df.meter==m].meter_reading.values
leak_score1 += mean_squared_error(np.log1p(v0), np.log1p(v1)) * len(v0)
 sample_submission.loc[(test_df.building_id == bid)&(test_df.meter==m), 'meter_reading'] = temp_df[temp_df.meter==m].meter_reading.values
leak_score1 /= len(leak_df )
X_train = np.array(train[players_col] ).reshape(-1, len(players_col)*22 )<prepare_x_and_y>
leak_score2 = 0
leak_df = pd.read_csv('/kaggle/input/asu-buildings-energy-consumption/asu_2016-2018.csv')
 leak_df['timestamp'] = pd.to_datetime(leak_df['timestamp'])
 leak_df.fillna(0, inplace=True)
 leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
leak_df = leak_df[leak_df.timestamp.dt.year > 2016]
leak_df = leak_df[leak_df.building_id!=245]
for bid in leak_df.building_id.unique() :
temp_df = leak_df[(leak_df.building_id == bid)]
for m in temp_df.meter.unique() :
v0 = sample_submission.loc[(test_df.building_id == bid)&(test_df.meter==m), 'meter_reading'].values
v1 = temp_df[temp_df.meter==m].meter_reading.values
leak_score2 += mean_squared_error(np.log1p(v0), np.log1p(v1)) * len(v0)
 sample_submission.loc[(test_df.building_id == bid)&(test_df.meter==m), 'meter_reading'] = temp_df[temp_df.meter==m].meter_reading.values
leak_score2 /= len(leak_df )
play_col = train.drop(players_col+['Yards'], axis=1 ).columns
X_play_col = np.zeros(shape=(X_train.shape[0], len(play_col)))
 for i, col in enumerate(play_col):
X_play_col[:, i] = train[col][::22]<prepare_x_and_y>
if not debug:
sample_submission.to_csv('submission_ucf_replaced.csv', index=False, float_format='%.4f' )
X_train = np.concatenate([X_train, X_play_col], axis=1)
 y_train = np.zeros(shape=(X_train.shape[0], 199))
for i,yard in enumerate(train['Yards'][::22]):
y_train[i, yard+99:] = np.ones(shape=(1, 100-yard))<normalization>
print('oof score meter0 =', np.sqrt(oof0))
print('oof score meter1 =', np.sqrt(oof1))
print('oof score meter2 =', np.sqrt(oof2))
print('oof score meter3 =', np.sqrt(oof3))
print('oof score total =', np.sqrt(oof_total))
<define_variables><EOS> 
print('UCF score = ', np.sqrt(leak_score0))
print('UCL score = ', np.sqrt(leak_score1))
print('ASU score = ', np.sqrt(leak_score2))