Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<define_variables>
py.init_notebook_mode(connected=True)
env = nflrush.make_env()
for(test_df, sample_prediction_df)in env.iter_test() :
dfb = prepare_data(test_df)
 feature_names = ['Team', 'X', 'Y', 'S', 'A', 'Dis', 'Orientation', 'Dir', 'NflId',
'Season', 'YardLine', 'Quarter', 'PossessionTeam', 'Down', 'Distance',
'FieldPosition', 'HomeScoreBeforePlay', 'VisitorScoreBeforePlay',
'NflIdRusher', 'OffenseFormation', 'DefendersInTheBox', 'PlayDirection',
'PlayerHeight', 'PlayerWeight', 'Position', 'HomeTeamAbbr',
'VisitorTeamAbbr', 'Week', 'Stadium', 'Location', 'StadiumType', 'Turf',
'GameWeather', 'Temperature', 'Humidity', 'WindSpeed', 'WindDirection',
'DefensePersonnel_DL', 'DefensePersonnel_LB', 'DefensePersonnel_DB',
'OffensePersonnel_Flag', 'OffensePersonnel_QB', 'OffensePersonnel_RB',
'OffensePersonnel_TE', 'OffensePersonnel_WR', 'OffensePersonnel_OL',
'TimeSnapHandoff', 'GameYear', 'GameMonth', 'GameDay', 'PlayerAgeYears',
'GameClockSeconds']
dfb_sc = transform_data(dfb)
 dfb_sc_final = dfb[feature_names]
y_pred = np.mean([model.predict(dfb_sc_final)],axis=0)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1 ).tolist() [0]
preds_df = pd.DataFrame(data=[y_pred], columns=sample_prediction_df.columns)
 env.predict(preds_df)
 env.write_submission_file()<set_options>
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df
sns.set_style('darkgrid')
 mpl.rcParams['figure.figsize'] = [15,10]<find_best_params>
%%time
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 train_df = pd.read_feather(root/'train.feather')
 test_df = pd.read_feather(root/'test.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather' )
env = nflrush.make_env()<load_from_csv>
leak_df = pd.read_feather('.. /input/ashrae-leak-data-station/leak.feather')
 leak_df.fillna(0, inplace=True)
 print(leak_df.timestamp.min() , leak_df.timestamp.max())
 leak_df = leak_df[(leak_df.timestamp.dt.year > 2016)&(leak_df.timestamp.dt.year < 2019)]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
leak_df = leak_df[leak_df.building_id!=245]
train = pd.read_csv('.. /input/nfl-big-data-bowl-2020/train.csv', dtype={'WindSpeed': 'object'} )<count_values>
leak_df.meter.value_counts()
train['A'].value_counts()<count_missing_values>
print(leak_df.duplicated().sum() )
train['A'].isnull().sum()<feature_engineering>
print(len(leak_df)/ len(train_df))
train['DefendersInTheBox_vs_Distance'] = train['DefendersInTheBox'] / train['Distance']<count_values>
del train_df
gc.collect()
train['StadiumType'].value_counts()<categorify>
sample_submission1 = pd.read_csv('.. /input/ashrae-kfold-lightgbm-without-leak-1-08/submission.csv', index_col=0)
 sample_submission2 = pd.read_csv('.. /input/ashrae-half-and-half/submission.csv', index_col=0)
 sample_submission3 = pd.read_csv('.. /input/ashrae-highway-kernel-route4/submission.csv', index_col=0 )
def clean_StadiumType(txt):
if pd.isna(txt):
return np.nan
txt = txt.lower()
txt = ''.join([c for c in txt if c not in punctuation])
 txt = re.sub(' +', ' ', txt)
 txt = txt.strip()
txt = txt.replace('outside', 'outdoor')
 txt = txt.replace('outdor', 'outdoor')
 txt = txt.replace('outddors', 'outdoor')
 txt = txt.replace('outdoors', 'outdoor')
 txt = txt.replace('oudoor', 'outdoor')
 txt = txt.replace('indoors', 'indoor')
 txt = txt.replace('ourdoor', 'outdoor')
 txt = txt.replace('retractable', 'rtr.')
 return txt<feature_engineering>
test_df['pred1'] = sample_submission1.meter_reading
test_df['pred2'] = sample_submission2.meter_reading
test_df['pred3'] = sample_submission3.meter_reading
test_df.loc[test_df.pred3<0, 'pred3'] = 0
del sample_submission1, sample_submission2, sample_submission3
gc.collect()
test_df = reduce_mem_usage(test_df)
 leak_df = reduce_mem_usage(leak_df )
train['StadiumType'] = train['StadiumType'].apply(clean_StadiumType )<categorify>
leak_df = leak_df.merge(test_df[['building_id', 'meter', 'timestamp', 'pred1', 'pred2', 'pred3', 'row_id']], left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 leak_df = leak_df.merge(building_meta_df[['building_id', 'site_id']], on='building_id', how='left' )
def transform_StadiumType(txt):
if pd.isna(txt):
return np.nan
if 'outdoor' in txt or 'open' in txt:
return 1
if 'indoor' in txt or 'closed' in txt:
return 0
return np.nan<feature_engineering>
leak_df['pred1_l1p'] = np.log1p(leak_df.pred1)
 leak_df['pred2_l1p'] = np.log1p(leak_df.pred2)
 leak_df['pred3_l1p'] = np.log1p(leak_df.pred3)
 leak_df['meter_reading_l1p'] = np.log1p(leak_df.meter_reading )
train['StadiumType'] = train['StadiumType'].apply(transform_StadiumType )<define_variables>
leak_df[leak_df.pred1_l1p.isnull() ]
Turf = {'Field Turf':'Artificial', 'A-Turf Titan':'Artificial', 'Grass':'Natural', 'UBU Sports Speed S5-M':'Artificial',
'Artificial':'Artificial', 'DD GrassMaster':'Artificial', 'Natural Grass':'Natural',
'UBU Speed Series-S5-M':'Artificial', 'FieldTurf':'Artificial', 'FieldTurf 360':'Artificial', 'Natural grass':'Natural', 'grass':'Natural',
'Natural':'Natural', 'Artifical':'Artificial', 'FieldTurf360':'Artificial', 'Naturall Grass':'Natural', 'Field turf':'Artificial',
'SISGrass':'Artificial', 'Twenty-Four/Seven Turf':'Artificial', 'natural grass':'Natural'}
train['Turf'] = train['Turf'].map(Turf)
 train['Turf'] = train['Turf'] == 'Natural'<filter>
leak_df['mean_pred'] = np.mean(leak_df[['pred1', 'pred2', 'pred3']].values, axis=1)
 leak_df['mean_pred_l1p'] = np.log1p(leak_df.mean_pred)
 leak_score = np.sqrt(mean_squared_error(leak_df.mean_pred_l1p, leak_df.meter_reading_l1p))
sns.distplot(leak_df.mean_pred_l1p)
 sns.distplot(leak_df.meter_reading_l1p)
 print('mean score=', leak_score )
train[(train['PossessionTeam']!=train['HomeTeamAbbr'])&(train['PossessionTeam']!=train['VisitorTeamAbbr'])][['PossessionTeam', 'HomeTeamAbbr', 'VisitorTeamAbbr']]<categorify>
N = 10
scores = np.zeros(N,)
 for i in range(N):
p = i * 1./N
v = p * leak_df['pred1'].values +(1.-p)* leak_df ['pred3'].values
vl1p = np.log1p(v)
 scores[i] = np.sqrt(mean_squared_error(vl1p, leak_df.meter_reading_l1p))
map_abbr = {'ARI': 'ARZ', 'BAL': 'BLT', 'CLE': 'CLV', 'HOU': 'HST'}
for abb in train['PossessionTeam'].unique() :
map_abbr[abb] = abb<categorify>
best_weight = np.argmin(scores)* 1./N
print(scores.min() , best_weight )
train['PossessionTeam'] = train['PossessionTeam'].map(map_abbr)
 train['HomeTeamAbbr'] = train['HomeTeamAbbr'].map(map_abbr)
 train['VisitorTeamAbbr'] = train['VisitorTeamAbbr'].map(map_abbr )<feature_engineering>
scores = np.zeros(N,)
 for i in range(N):
p = i * 1./N
v = p *(best_weight * leak_df['pred1'].values +(1.-best_weight)* leak_df ['pred3'].values)+(1.-p)* leak_df ['pred2'].values
vl1p = np.log1p(v)
 scores[i] = np.sqrt(mean_squared_error(vl1p, leak_df.meter_reading_l1p))
train['HomePossesion'] = train['PossessionTeam'] == train['HomeTeamAbbr']<feature_engineering>
best_weight2 = np.argmin(scores)* 1./N
print(scores.min() , best_weight2)
train['Field_eq_Possession'] = train['FieldPosition'] == train['PossessionTeam']
train['HomeField'] = train['FieldPosition'] == train['HomeTeamAbbr']<count_values>
v = 0.33 * leak_df['pred1'].values + 0.33 * leak_df['pred3'].values + 0.3 * leak_df['pred2'].values
vl1p = np.log1p(v)
 print(np.sqrt(mean_squared_error(vl1p, leak_df.meter_reading_l1p)) )
off_form = train['OffenseFormation'].unique()
train['OffenseFormation'].value_counts()<categorify>
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
sample_submission['meter_reading'] = 0.33 * test_df.pred1 + 0.33 * test_df.pred3 + 0.3 * test_df.pred2
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0
train = pd.concat([train.drop(['OffenseFormation'], axis=1), pd.get_dummies(train['OffenseFormation'], prefix='Formation')], axis=1)
 dummy_col = train.columns<count_values>
leak_df = leak_df[['meter_reading', 'row_id']].set_index('row_id' ).dropna()
sample_submission.loc[leak_df.index, 'meter_reading'] = leak_df['meter_reading']
<count_values><EOS> 
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )