Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<define_variables>
py.init_notebook_mode(connected=True)
training_columns =[
'Down',
'Age',
'Offense_DL',
'Offense_QB',
'Offense_RB',
'Offense_LB',
'Offense_DB',
'Offense_TE',
'Defense_DL',
'Defense_OL',
'min_dist_rusher_to_defender',
'min_dist_qb_to_defender',
'dist_rusher_QB',
'A',
'predict_defense',
'predict_offense',
'predict_player',
'YardLine_adj',
'S',
'DefendersInTheBox',
'Dis',
'Distance',
'X_adj',
'DisplayName_Count',
'Defense_DB',
'DisplayName_Bin_Count_(1.72, 4.21]',
'DisplayName_Bin_Count_(6.7, 99.0]',
'DisplayName_Yards_Bin_Rate_(-99.0, -0.01]',
'DisplayName_Bin_Count_(0.01, 1.72]',
'VisitorScoreBeforePlay',
'DisplayName_Bin_Count_(4.21, 6.7]',
'clock_bin',
'Defense_Team_Yards_Bin_Rate_(6.7, 99.0]',
'Offense_WR',
'Team_side',
'Offense_score',
'Deffense_score',
'Defense_LB',
'DisplayName_Yards_Bin_Rate_(0.01, 1.72]',
'Defense_Team_Yards_Bin_Rate_(-0.01, 0.01]',
'Defense_Team_Bin_Count_(1.72, 4.21]',
'Defense_Team_Avg_Yards',
'Offense_OL',
'DisplayName_Yards_Bin_Rate_(-0.01, 0.01]',
'Defense_Team_Yards_Bin_Rate_(-99.0, -0.01]',
'Defense_Team_Bin_Count_(-99.0, -0.01]',
'dis_from_center',
'dir_adj',
'Orientation_adj',
'dir_to_Orientation',
'v_x',
'v_y',
'arc_tan',
'arc_tan_cat',
't1_distance',
't2_distance',
't3_distance',
'v_x_a1',
'v_y_a1',
'v_x_a2',
'v_y_a2',
'v_x_a3',
'v_y_a3'
] 
<categorify>
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df
categorical_columns = ['DisplayName_rusher_performance_bin', 'Position', 'Stadium','GameWeather','OffensePersonnel','Turf_type','OffenseFormation']
one_hot = pd.get_dummies(train_df[categorical_columns])
 train_df = train_df.join(one_hot )<define_variables>
%%time
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 train_df = pd.read_feather(root/'train.feather')
 test_df = pd.read_feather(root/'test.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather' )
training_columns = training_columns + list(one_hot.columns )<prepare_x_and_y>
leak_df = pd.read_feather('.. /input/ashrae-leak-data-station/leak.feather')
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[(leak_df.timestamp.dt.year > 2016)&(leak_df.timestamp.dt.year < 2019)]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
leak_df = leak_df[leak_df.building_id!=245]
lead_site_id_0 = leak_df[leak_df['meter']==0]
train_df_temp=train_df[training_columns].fillna(0)
 X = train_df_temp
yards = train_df.Yards
y = np.zeros(( yards.shape[0], 37))
for idx, target in enumerate(list(yards)) :
if target > 22:
y[idx][36] = 1
elif target < -14:
y[idx][0] = 1
else:
y[idx][14 + target] = 1
X = np.array(X )<prepare_x_and_y>
sample_submission3 = pd.read_csv(".. /input/ashrae-final-submission/fe2_lgbm_tz.csv", index_col=0)
 sample_submission2 = pd.read_csv(".. /input/ashrae-final-submission/submission.csv", index_col=0)
 sample_submission1 = pd.read_csv(".. /input/ashrae-final-submission/submission_final_day.csv", index_col=0 )
<compute_test_metric>
test_df['pred1'] = sample_submission1.meter_reading
test_df['pred2'] = sample_submission2.meter_reading
test_df['pred3'] = sample_submission3.meter_reading
del sample_submission1, sample_submission2, sample_submission3
gc.collect()
test_df = reduce_mem_usage(test_df)
 leak_df = reduce_mem_usage(leak_df )
<train_model>
leak_df = leak_df.merge(test_df[['building_id', 'meter', 'timestamp', 'pred1', 'pred2', 'pred3','row_id']], left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 leak_df = leak_df.merge(building_meta_df[['building_id', 'site_id']], on='building_id', how='left' )
final_model = RandomForestRegressor(bootstrap=False, max_features=0.3, min_samples_leaf=10,
min_samples_split=20, n_estimators=200, n_jobs=-1, random_state=42)
 final_model.fit(X, y )<compute_train_metric>
leak_df['pred1_l1p'] = np.log1p(leak_df.pred1)
 leak_df['pred2_l1p'] = np.log1p(leak_df.pred2)
 leak_df['pred3_l1p'] = np.log1p(leak_df.pred3)
 leak_df['meter_reading_l1p'] = np.log1p(leak_df.meter_reading )
def make_my_predictions(test_df_all, sample_prediction_df, models, player_and_team_stats, personnel_columns_to_add, final_model, training_columns, categorical_columns, one_hot_columns):
test_df_all = clean_TeamAbbreviations(test_df_all)
 test_df_all = imperial_to_metric(test_df_all)
 test_df_all = tag_rusher(test_df_all)
 test_df_all = get_player_age(test_df_all)
 test_df_all = get_adjusted_yardline(test_df_all)
 test_df_all=norm_x(test_df_all)
 test_df_all['dis_from_center']=abs(train_df_all['X_adj'])
 test_df_all=Direction_orientation_adj(test_df_all)
 test_df_all=X_Y_Velocity(test_df_all)
 test_df_all = qb_and_rusher_to_closest_defender(test_df_all)
 test_df=test_df_all[test_df_all['NflId']==test_df_all['NflIdRusher']]
test_df = test_df.fillna(0)
 test_df = get_direction(test_df)
 test_df = get_leading_team_features(test_df)
 test_df = get_GameClock_features(test_df)
 test_df = get_split_personnel_types(personnel_columns_to_add, test_df)
 for key,value in player_and_team_stats.items() :
test_df = test_df.merge(value, on=key, how='left' ).fillna(0)
 if key == 'DisplayName' and test_df['DisplayName_rusher_performance_bin'].iloc[0] == 0:
test_df['DisplayName_rusher_performance_bin'] = 'few_plays'
test_df=set_Turf(test_df)
 test_df['StadiumType'] = test_df['StadiumType'].apply(clean_StadiumType)
 test_df['WindDirection']=test_df['WindDirection'].apply(clean_WindDirection)
 test_df['WindDirection']=test_df['WindDirection'].apply(transform_WindDirection)
 test_df=GameWeather(test_df)
 for model_type,model in models.items() :
test_df_curr = h2o.H2OFrame(test_df)
 pred=model.predict(test_df_curr)
 test_df = test_df.join(pred.as_data_frame())
 test_df.rename(columns={'predict':'predict_' + model_type}, inplace=True)
 one_hot_test = pd.get_dummies(test_df[categorical_columns])
 one_hot_test_cols = [x for x in one_hot_test if x in one_hot_columns]
one_hot_test = one_hot_test[one_hot_test_cols]
test_df = test_df.join(one_hot_test)
 more_to_add = [x for x in one_hot_columns if x not in list(one_hot_test.columns)]
for column_to_add in more_to_add:
test_df[column_to_add] = 0
test_df=test_df[training_columns]
test_X = np.array(test_df)
 pred = final_model.predict(test_X)
 pred = np.array([np.concatenate(( np.zeros(85), pred[0], np.zeros(77)) , axis=0)])
 pred = np.clip(np.cumsum(pred, axis=1), 0, 1)
 pred = pd.DataFrame(data=[list(pred[0])], columns=sample_prediction_df.columns)
 return pred<load_from_csv>
v = 0.1 * leak_df['pred1'].values + 0.6 * leak_df['pred2'].values + 0.2 * leak_df['pred3'].values
vl1p = np.log1p(v)
 curr_score = np.sqrt(mean_squared_error(vl1p, leak_df.meter_reading_l1p))
print(curr_score )
env = nflrush.make_env()
for(test_df, sample_prediction_df)in env.iter_test() :
predictions_df = make_my_predictions(test_df, sample_prediction_df, models, player_and_team_stats, personnel_columns_to_add, final_model, training_columns, categorical_columns, list(one_hot.columns))
env.predict(predictions_df)
 env.write_submission_file()<create_dataframe>
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
w1 = 0.1
w2 = 0.6
w3 = 0.2
print("The weights are: w1=" + str(w1)+ ", w2=" + str(w2)+ ", w3=" + str(w3))
sample_submission['meter_reading'] = w1 * test_df.pred1 + w2 * test_df.pred2 + w3 * test_df.pred3
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0
<train_model>
leak_df = leak_df[['meter_reading', 'row_id']].set_index('row_id' ).dropna()
sample_submission.loc[leak_df.index, 'meter_reading'] = leak_df['meter_reading']
<categorify><EOS> 
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )