Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<load_from_csv>
DATA_PATH = ".. /input/ashrae-energy-prediction/"
<load_from_csv>
train_df = pd.read_csv(DATA_PATH + 'train.csv')
 train_df = train_df [ train_df['building_id'] != 1099 ]
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")')
 building_df = pd.read_csv(DATA_PATH + 'building_metadata.csv')
 weather_df = pd.read_csv(DATA_PATH + 'weather_train.csv' )
<count_unique_values>
def fill_weather_dataset(weather_df):
time_format = "%Y-%m-%d %H:%M:%S"
start_date = datetime.datetime.strptime(weather_df['timestamp'].min() ,time_format)
 end_date = datetime.datetime.strptime(weather_df['timestamp'].max() ,time_format)
 total_hours = int(((end_date - start_date ).total_seconds() + 3600)/ 3600)
 hours_list = [(end_date - datetime.timedelta(hours=x)).strftime(time_format)for x in range(total_hours)]
missing_hours = []
for site_id in range(16):
site_hours = np.array(weather_df[weather_df['site_id'] == site_id]['timestamp'])
 new_rows = pd.DataFrame(np.setdiff1d(hours_list,site_hours),columns=['timestamp'])
 new_rows['site_id'] = site_id
weather_df = pd.concat([weather_df,new_rows])
 weather_df = weather_df.reset_index(drop=True)
 weather_df["datetime"] = pd.to_datetime(weather_df["timestamp"])
 weather_df["day"] = weather_df["datetime"].dt.day
weather_df["week"] = weather_df["datetime"].dt.week
weather_df["month"] = weather_df["datetime"].dt.month
weather_df = weather_df.set_index(['site_id','day','month'])
 air_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['air_temperature'].mean() ,columns=["air_temperature"])
 weather_df.update(air_temperature_filler,overwrite=False)
 cloud_coverage_filler = weather_df.groupby(['site_id','day','month'])['cloud_coverage'].mean()
cloud_coverage_filler = pd.DataFrame(cloud_coverage_filler.fillna(method='ffill'),columns=["cloud_coverage"])
 weather_df.update(cloud_coverage_filler,overwrite=False)
 due_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['dew_temperature'].mean() ,columns=["dew_temperature"])
 weather_df.update(due_temperature_filler,overwrite=False)
 sea_level_filler = weather_df.groupby(['site_id','day','month'])['sea_level_pressure'].mean()
sea_level_filler = pd.DataFrame(sea_level_filler.fillna(method='ffill'),columns=['sea_level_pressure'])
 weather_df.update(sea_level_filler,overwrite=False)
 wind_direction_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_direction'].mean() ,columns=['wind_direction'])
 weather_df.update(wind_direction_filler,overwrite=False)
 wind_speed_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_speed'].mean() ,columns=['wind_speed'])
 weather_df.update(wind_speed_filler,overwrite=False)
 precip_depth_filler = weather_df.groupby(['site_id','day','month'])['precip_depth_1_hr'].mean()
precip_depth_filler = pd.DataFrame(precip_depth_filler.fillna(method='ffill'),columns=['precip_depth_1_hr'])
 weather_df.update(precip_depth_filler,overwrite=False)
 weather_df = weather_df.reset_index()
weather_df = weather_df.drop(['datetime','day','week','month'],axis=1)
 return weather_df
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print("Memory usage of dataframe is {:.2f} MB".format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == "int":
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype("category")
 end_mem = df.memory_usage().sum() / 1024**2
print("Memory usage after optimization is: {:.2f} MB".format(end_mem))
print("Decreased by {:.1f}%".format(100 *(start_mem - end_mem)/ start_mem))
return df
def features_engineering(df):
df.sort_values("timestamp")
 df.reset_index(drop=True)
 df["timestamp_formatted"] = pd.to_datetime(df["timestamp"],format="%Y-%m-%d %H:%M:%S")
 df["hour"] =(df["timestamp_formatted"].dt.hour ).fillna(-1 ).astype(np.int8)
 df["weekend"] =(df["timestamp_formatted"].dt.weekday ).fillna(-1 ).astype(np.int8)
 df["day"] =(df["timestamp_formatted"].dt.day ).fillna(-1 ).astype(np.int8)
 df["month"] =(df["timestamp_formatted"].dt.month ).fillna(-1 ).astype(np.int8)
 df['square_feet_log'] = np.log1p(df['square_feet'])
 df['years'] =(df["year_built"] - 1900 ).fillna(-1 ).astype(np.int8)
 df['wind_compass'] = -1
df.loc[(df['wind_direction'] >= 315)|(df['wind_direction'] < 45), 'wind_compass'] = 0
df.loc[(df['wind_direction'] >= 45)&(df['wind_direction'] < 135), 'wind_compass'] = 1
df.loc[(df['wind_direction'] >= 135)&(df['wind_direction'] < 225), 'wind_compass'] = 2
df.loc[(df['wind_direction'] >= 225)&(df['wind_direction'] < 315), 'wind_compass'] = 3
df['wind_compass'] = df['wind_compass'].astype(np.int8)
 df['wind_scale'] = -1
df.loc[df['wind_speed'] < 0.5, 'wind_scale'] = 0
df.loc[(df['wind_speed'] >= 0.5)&(df['wind_speed'] < 1.5), 'wind_scale'] = 1
df.loc[(df['wind_speed'] >= 1.5)&(df['wind_speed'] < 3.5), 'wind_scale'] = 2
df.loc[(df['wind_speed'] >= 3.5)&(df['wind_speed'] < 5.5), 'wind_scale'] = 3
df.loc[(df['wind_speed'] >= 5.5)&(df['wind_speed'] < 8.5), 'wind_scale'] = 4
df.loc[df['wind_speed'] >= 8.5, 'wind_scale'] = 5
df['wind_scale'] = df['wind_scale'].astype(np.int8)
 drop = ["timestamp",'square_feet',"timestamp_formatted","year_built","sea_level_pressure", "wind_direction", "wind_speed"]
df = df.drop(drop, axis=1)
 gc.collect()
le = LabelEncoder()
df["primary_use"] = le.fit_transform(df["primary_use"])
 end_mem = df.memory_usage().sum() / 1024**2
print("Memory usage is: {:.2f} MB".format(end_mem))
return df
<set_options>
weather_df = fill_weather_dataset(weather_df )
pd.set_option('display.max_columns', 50)
 pd.set_option('display.max_rows', 150)
 sns.set_style('darkgrid')
 mpl.rcParams['figure.figsize'] = [15,10]<find_best_params>
train_df = reduce_mem_usage(train_df,use_float16=True)
 building_df = reduce_mem_usage(building_df,use_float16=True)
 weather_df = reduce_mem_usage(weather_df,use_float16=True )
env = nflrush.make_env()<load_from_csv>
train_df = train_df.merge(building_df, left_on='building_id',right_on='building_id',how='left')
 train_df = train_df.merge(weather_df,how='left',left_on=['site_id','timestamp'],right_on=['site_id','timestamp'])
 del weather_df
gc.collect()
train = pd.read_csv('.. /input/nfl-big-data-bowl-2020/train.csv', dtype={'WindSpeed': 'object'} )<count_values>
train_df = features_engineering(train_df )
train['Position'].value_counts()<count_unique_values>
target = np.log1p(train_df["meter_reading"])
 features = train_df.drop('meter_reading', axis = 1)
 del train_df, sample
gc.collect()
cat_features = []
int_features = []
float_features = []
bool_features = []
uint8_features = []
for col in train.columns:
if train[col].dtype =='object':
cat_features.append(( col, len(train[col].unique())))
 if train[col].dtype =='int64':
int_features.append(( col, len(train[col].unique())))
 if train[col].dtype =='float64':
float_features.append(( col, len(train[col].unique())))
 if train[col].dtype =='bool':
bool_features.append(( col, len(train[col].unique())))
 if train[col].dtype =='uint8':
uint8_features.append(( col, len(train[col].unique())))

print("{} Categorical Features found.".format(len(cat_features)))
 print("{} Integer Features found.".format(len(int_features)))
 print("{} Float Features found.".format(len(float_features)))
 print("{} Boolean Features found.".format(len(bool_features)))
 print("{} uint8 OneHot Features found.".format(len(uint8_features)) )<categorify>
categorical_features = ["building_id", "site_id", "meter", "primary_use", "weekend", "day", "month", 'wind_compass', 'wind_scale']
params = {
"objective": "regression",
"boosting": "gbdt",
"num_leaves": 1280,
"learning_rate": 0.05,
"feature_fraction": 0.85,
"reg_lambda": 2,
"metric": "rmse",
}
kf = KFold(n_splits=3)
 models = []
for train_index,test_index in kf.split(features):
train_features = features.loc[train_index]
train_target = target.loc[train_index]
test_features = features.loc[test_index]
test_target = target.loc[test_index]
d_training = lgb.Dataset(train_features, label=train_target,categorical_feature=categorical_features, free_raw_data=False)
 d_test = lgb.Dataset(test_features, label=test_target,categorical_feature=categorical_features, free_raw_data=False)
 model = lgb.train(params, train_set=d_training, num_boost_round=1000, valid_sets=[d_training,d_test], verbose_eval=25, early_stopping_rounds=50)
 models.append(model)
 del train_features, train_target, test_features, test_target, d_training, d_test
gc.collect()
def convert_to_seconds(txt):
txt = txt.split(':')
 ans = int(txt[0])*60 + int(txt[1])+ int(txt[2])/60
return ans<categorify>
del features, target
gc.collect()
def classify_offense_formation(txt):
if pd.isna(txt):
return "unknown"
txt = txt.lower()
if(txt.find("single")!= -1):
return "singleback"
elif(txt.find("shot")!= -1):
return "shotgun"
elif(txt.find("i_form")!= -1):
return "i_form"
elif(txt.find("pistol")!= -1):
return "pistol"
elif(txt.find("jumbo")!= -1):
return "jumbo"
elif(txt.find("wildcat")!= -1):
return "wildcat"
elif(txt.find("empty")!= -1):
return "empty"
elif(txt.find("ace")!= -1):
return "ace"
else:
return "unknown"<string_transform>
test_df = pd.read_csv(DATA_PATH + 'test.csv')
 row_ids = test_df["row_id"]
test_df.drop("row_id", axis=1, inplace=True)
 test_df = reduce_mem_usage(test_df )
offense_dict = {'DB' : "offense_DB", 'DL' : "offense_DL",
'LB' : "offense_LB", 'OL' : "offense_OL",
'QB' : "offense_QB", 'RB' : "offense_RB",
'TE' : "offense_TE", 'WR' : "offense_WR"}
def ProcessOffensePersonnel(row):
for whole_offense in row["OffensePersonnel"].split(","):
offense_item = whole_offense.strip().split(" ")
 if(offense_item[1] in offense_dict.keys()):
if offense_item[0].isdigit() :
row[offense_dict[offense_item[1]]] = int(offense_item[0])
 else:
row[offense_item[offense_item[1]]] = 0
else:
if offense_item[0].isdigit() :
row["offense_UNKNOWN"] += int(offense_item[0])
 else:
row["offense_UNKNOWN"] = 0
return row<feature_engineering>
test_df = test_df.merge(building_df,left_on='building_id',right_on='building_id',how='left')
 del building_df
gc.collect()
defense_dict = {'DB' : "defense_DB", 'DL' : "defense_DL",
'LB' : "defense_LB", 'OL' : "defense_OL"}
def ProcessDefensePersonnel(row):
for whole_defense in row["DefensePersonnel"].split(","):
defense_item = whole_defense.strip().split(" ")
 if(defense_item[1] in defense_dict.keys()):
if defense_item[0].isdigit() :
row[defense_dict[defense_item[1]]] = int(defense_item[0])
 else:
row[defense_dict[defense_item[1]]] = 0
else:
if defense_item[0].isdigit() :
row["defense_UNKNOWN"] += int(defense_item[0])
 else:
row["defense_UNKNOWN"] = 0
return row<categorify>
weather_df = pd.read_csv(DATA_PATH + 'weather_test.csv')
 weather_df = fill_weather_dataset(weather_df)
 weather_df = reduce_mem_usage(weather_df)
def classify_stadium_type(txt):
if pd.isna(txt):
return True
txt = txt.lower()
if(txt.find("ou")!= -1):
return True
if(txt.find("in")!= -1):
return False
if(txt.find("op")!= -1):
return True
if(txt.find("cl")!= -1):
return False
return True<categorify>
test_df = test_df.merge(weather_df,how='left',on=['timestamp','site_id'])
 del weather_df
gc.collect()
def classify_turf_type(txt):
if pd.isna(txt):
print("isna")
 return True
txt = txt.lower()
if(txt.find("gras")!= -1):
return True
if(txt.find("nat")!= -1):
return True
return False<normalization>
test_df = features_engineering(test_df )
def new_X(x_coordinate, play_direction):
if play_direction == False:
return 120.0 - x_coordinate
else:
return x_coordinate
def new_orientation(angle, play_direction):
if play_direction == False:
new_angle = 360.0 - angle
if new_angle == 360.0:
return 0.0
else:
return new_angle
else:
if angle == 360.0:
return 0.0
else:
return angle
def line_of_scrimmage_in_coordinates(rush_team, field_position, yardline):
if rush_team == field_position:
return 10.0 + yardline
else:
return 60.0 +(50 - yardline)
 def euclidean_distance(x1,y1,x2,y2):
x_diff =(x1-x2)**2
y_diff =(y1-y2)**2
return np.sqrt(x_diff + y_diff)
 def is_looking_back(orientation):
if orientation > 180.0:
return True
else:
return False
def recalculate_yardline(df):
new_yardline = df[df['NflId'] == df['NflIdRusher']].copy()
new_yardline['YardLine'] = new_yardline[['PossessionTeam','FieldPosition','YardLine']].apply(lambda x: line_of_scrimmage_in_coordinates(x[0],x[1],x[2]), axis=1)
 new_yardline = new_yardline[['GameId','PlayId','YardLine']]
df = df.drop('YardLine', axis=1)
 df = pd.merge(df, new_yardline, on=['GameId','PlayId'], how='inner')
 return df
def recalculate_pos_and_dir(df):
df['X'] = df[['X','PlayDirection']].apply(lambda x: new_X(x[0],x[1]), axis=1)
 df['Orientation'] = df[['Orientation','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)
 df['Dir'] = df[['Dir','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)
 return df 
<feature_engineering>
results = np.zeros(len(test_df))
step = 50000
for model in models:
for i in tqdm(range(0, len(test_df), step)) :
results[i: min(i+step, len(test_df)) ] += np.expm1(model.predict(test_df.iloc[i: min(i+step, len(test_df)) , :], num_iteration=model.best_iteration))
del model
gc.collect()
results = results / len(models )
def features_relative_coordinates(df):
rusher_features = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','NflIdRusher','X','Y','Orientation','Dir','YardLine']].copy()
rusher_features['rusher_yards_to_go'] = rusher_features['YardLine'] - rusher_features['X']
rusher_features['rusher_oriented_back'] = rusher_features['Orientation'].apply(lambda x: is_looking_back(x))
rusher_features['rusher_moving_back'] = rusher_features['Dir'].apply(lambda x: is_looking_back(x))
rusher_features = rusher_features.rename(columns={'X':'rusher_X', 'Y':'rusher_Y'})
 rusher_features = rusher_features[['GameId','PlayId','NflIdRusher','rusher_X','rusher_Y','rusher_yards_to_go',
'rusher_oriented_back','rusher_moving_back']]
player_distance = df[['GameId','PlayId','NflId','X','Y']].copy()
player_distance = pd.merge(player_distance, rusher_features, on=['GameId','PlayId'], how='inner')
 player_distance['dist_to_rusher'] = player_distance[['X','Y','rusher_X','rusher_Y']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 player_distance = player_distance.groupby(['GameId','PlayId','rusher_yards_to_go','rusher_oriented_back','rusher_moving_back'])\ 
.agg({'dist_to_rusher':['min','max','mean','std']})\ 
.reset_index()
player_distance.columns = ['GameId','PlayId','rusher_yards_to_go','rusher_oriented_back','rusher_moving_back',
'min_dist','max_dist','mean_dist','std_dist']
df = pd.merge(player_distance,df,on=['GameId','PlayId'],how='inner')
 return df
def defense_features(df):
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','Team','X','Y']].copy()
rusher.columns = ['GameId','PlayId','RusherTeam','RusherX','RusherY']
defense = pd.merge(df,rusher,on=['GameId','PlayId'],how='inner')
 defense = defense[defense['Team'] != defense['RusherTeam']][['GameId','PlayId','X','Y','RusherX','RusherY']]
defense['def_dist_to_rusher'] = defense[['X','Y','RusherX','RusherY']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 defense = defense.groupby(['GameId','PlayId'])\ 
.agg({'def_dist_to_rusher':['min','max','mean','std']})\ 
.reset_index()
defense.columns = ['GameId','PlayId','def_min_dist','def_max_dist','def_mean_dist','def_std_dist']
df = pd.merge(defense,df,on=['GameId','PlayId'],how='inner')
 return df
def dist_to_rusher_features(df):
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','X','Y']].copy()
rusher.columns = ['GameId','PlayId','RusherX','RusherY']
defense = pd.merge(df,rusher,on=['GameId','PlayId'],how='inner')
 defense = defense[['GameId','PlayId',"NflId",'X','Y','RusherX','RusherY']]
defense['dist_to_rusher'] = defense[['X','Y','RusherX','RusherY']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 defense.drop(["X", "Y", "RusherX", "RusherY"], axis=1, inplace=True)
 df = pd.merge(defense,df,on=['GameId','PlayId',"NflId"],how='inner')
 return df
def calc_new_position(X, Y, S, A, Dir):
D1 = S + A/2
D3 = D1 + 2*S + 2*A
X1 = X + D1 * np.sin(Dir/180*np.pi)
 Y1 = Y + D1 * np.cos(Dir/180*np.pi)
 X3 = X + D3 * np.sin(Dir/180*np.pi)
 Y3 = Y + D3 * np.cos(Dir/180*np.pi)
 return X1, Y1, X3, Y3
def additional_time_series_features(df):
features = df[['GameId','PlayId','NflId','X','Y','S','A','Dir']].copy()
features[['X1sec', 'Y1sec', 'X3sec', 'Y3sec']] = features[['X','Y','S','A', 'Dir']].apply(lambda x: calc_new_position(x[0],x[1],x[2],x[3],x[4]), axis=1, result_type="expand")
 features.drop(["X", "Y", "S", "A", "Dir"], axis=1, inplace=True)
 df = pd.merge(features,df,on=['GameId','PlayId','NflId'],how='inner')
 return df
def line(p1, p2):
A =(p1[1] - p2[1])
 B =(p2[0] - p1[0])
 C =(p1[0]*p2[1] - p2[0]*p1[1])
 return A, B, -C
def calc_intersection(L1P1X, L1P1Y, L1P2X, L1P2Y, L2P1X, L2P1Y, L2P2X, L2P2Y):
L1 = line([L1P1X,L1P1Y],[L1P2X,L1P2Y])
 L2 = line([L2P1X,L2P1Y],[L2P2X,L2P2Y])
 D = L1[0] * L2[1] - L1[1] * L2[0]
Dx = L1[2] * L2[1] - L1[1] * L2[2]
Dy = L1[0] * L2[2] - L1[2] * L2[0]
if D != 0:
x = Dx / D
y = Dy / D
if(( x > L1P1X)and(x > L1P2X)) :
return False
elif(( x < L1P1X)and(x < L1P2X)) :
return False
elif(( x < L2P1X)and(x < L2P2X)) :
return False
elif(( x > L2P1X)and(x > L2P2X)) :
return False
else:
if(( y > L1P1Y)and(y > L1P2Y)) :
return False
elif(( y < L1P1Y)and(y < L1P2Y)) :
return False
if(( y > L2P1Y)and(y > L2P2Y)) :
return False
elif(( y < L2P1Y)and(y < L2P2Y)) :
return False
else:
return True
else:
return False
def calc_crossing_movements(df):
rusher_features = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','X','Y','X1sec','Y1sec','X3sec','Y3sec']].copy()
rusher_features = rusher_features.rename(columns={'X':'rusher_X', 'Y':'rusher_Y', 'X1sec':'rusher_X1sec', 'Y1sec':'rusher_Y1sec', 'X3sec':'rusher_X3sec', 'Y3sec':'rusher_Y3sec'})
 features = df[['GameId','PlayId','NflId','X','Y','X1sec','Y1sec','X3sec','Y3sec']].copy()
features = pd.merge(features,rusher_features,on=['GameId','PlayId'],how='inner')
 df["CrossesWithin_1"] = features[['X','Y','X1sec','Y1sec','rusher_X','rusher_Y','rusher_X1sec','rusher_Y1sec']].apply(lambda x: calc_intersection(x[0],x[1],x[2],x[3],x[4],x[5],x[6],x[7]), axis=1)
 df["CrossesWithin_3"] = features[['X','Y','X3sec','Y3sec','rusher_X','rusher_Y','rusher_X3sec','rusher_Y3sec']].apply(lambda x: calc_intersection(x[0],x[1],x[2],x[3],x[4],x[5],x[6],x[7]), axis=1)
 return df 
<categorify>
del test_df, models
gc.collect()
<categorify><EOS> 
results_df = pd.DataFrame({"row_id": row_ids, "meter_reading": np.clip(results, 0, a_max=None)})
 del row_ids,results
gc.collect()
results_df.to_csv("submission.csv", index=False)