Kaggle competition: ASHRAE - Great Energy Predictor III
warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', 150)
 env = nflrush.make_env()<load_from_csv>
black_day = 10
outlier = False
rescale = False
debug=False
num_rounds = 1000
clip0=False
folds = 3
use_ucf=False
ucf_clip=False
ucf_year = [2017, 2018]
predmode='all'
train = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2020/train.csv', dtype={'WindSpeed': 'object'})
 outcomes = train[['GameId','PlayId','Yards']].drop_duplicates()
train_2 = train.copy()
train_3 = train.copy()<categorify>
py.init_notebook_mode(connected=True)
def strtoseconds(txt):
txt = txt.split(':')
 ans = int(txt[0])*60 + int(txt[1])+ int(txt[2])/60
return ans
def strtofloat(x):
try:
return float(x)
 except:
return -1
def map_weather(txt):
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt:
ans*=0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*3
if 'sunny' in txt or 'sun' in txt:
return ans*2
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt:
return -2*ans
if 'snow' in txt:
return -3*ans
return 0
def OffensePersonnelSplit(x):
dic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0, 'QB' : 0, 'RB' : 0, 'TE' : 0, 'WR' : 0}
for xx in x.split(","):
xxs = xx.split(" ")
 dic[xxs[-1]] = int(xxs[-2])
 return dic
def DefensePersonnelSplit(x):
dic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0}
for xx in x.split(","):
xxs = xx.split(" ")
 dic[xxs[-1]] = int(xxs[-2])
 return dic
def orientation_to_cat(x):
x = np.clip(x, 0, 360 - 1)
 try:
return str(int(x/15))
except:
return "nan"
def euclidean_distance(x1,y1,x2,y2):
x_diff =(x1-x2)**2
y_diff =(y1-y2)**2
return np.sqrt(x_diff + y_diff)
 def standardize_speed(df):
speed_std = np.where(df.Season == '2017',(( df['S'] - 2.4355)/ 1.2930),
np.where(df.Season == '2018',(( df['S'] - 2.7570)/ 1.4551), 
(( df['S'] - 2.7456)/ 1.4501)))
 return(speed_std)
<feature_engineering>
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df
def get_o_line_stats(df):
ol_df = df.loc[(df.Position.isin(['C','G','T','OG','OT','TE'])) &(df.IsOnOffense),
['PlayId','A','Dir_std_1','Dir_std_2','ToLeft', 'Orientation']]
ol_df['Dir_std'] = np.where(ol_df.ToLeft, ol_df['Dir_std_2'], ol_df['Dir_std_1'])
 ol_df['OlMidDir'] = abs(ol_df['Dir_std'] - 90)
 ol_df['Dir_cat'] = np.where(( ol_df.Dir_std < 150)&(ol_df.Dir_std > 30), 10,
np.where(ol_df.Dir_std > 150, 1,
np.where(( ol_df.Dir_std > 0)&(ol_df.Dir_std < 30), 1, -1)))
 ol_df['OlMovingDf'] = np.where(( ol_df.Dir_std < 150)&(ol_df.Dir_std > 30), 1, 0)
 ol_df['OlATimesDir'] = ol_df['A'] * ol_df['Dir_cat']
ol_df_grouped = ol_df.groupby('PlayId', as_index=False)[['A','OlATimesDir','OlMovingDf', 'OlMidDir', 'Orientation']].mean()
ol_df_grouped.rename(columns={'A':'OlAMean', 'Orientation': 'OlOrientationMean'}, inplace=True)
 return(ol_df_grouped)
 ol_agg_df = get_o_line_stats(standardized_dir_df )<feature_engineering>
zone_dict={0:4,1:0,2:7,3:4,4:7,5:0,6:4,7:4,8:4,9:5,10:7,11:4,12:0,13:5,14:4,15:4}
def set_local(df):
for sid, zone in zone_dict.items() :
sids = df.site_id == sid
df.loc[sids, 'timestamp'] = df[sids].timestamp - pd.offsets.Hour(zone )
%time train_basetable = create_features(train, False )<feature_engineering>
%%time
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 root_black = Path('.. /input/ashrae-local-datatime-and-black-count')
 train_df = pd.read_feather(root/'train.feather')
 train_df_black = pd.read_feather(root_black/'train_black.feather')
 weather_train_df = pd.read_feather(root/'weather_train.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather' )
pos_dict= {'DefensePersonnel': ['DL','LB','DB'],
'OffensePersonnel': ['RB','TE','WR','QB','OL']}
def get_personnel(row, position):
filtered_row = [pos_group for pos_group in row if position in pos_group]
if len(filtered_row)> 0:
return(int(filtered_row[0][0]))
else:
return(0)
 def ball_carrier_features(df):
bc_df = df.loc[df['IsBallCarrier'], ['PlayId','ToLeft','X_std','Y_std','X_std_end','Y_std_end','Dir_std_1','Dir_std_2','Position', 'A']]
bc_df.rename(columns={'X_std': 'X_std_bc','Y_std':'Y_std_bc',
'X_std_end':'X_std_end_bc','Y_std_end':'Y_std_end_bc',
'Dir_std_1':'Dir_std_1_bc','Dir_std_2':'Dir_std_2_bc',
'Position':'BcPosition',
}, inplace=True)
 bc_df['BcRunRight'] = np.where(((bc_df.ToLeft)&(bc_df.Dir_std_2_bc > 90)) | 
(( bc_df.ToLeft == False)&(bc_df.Dir_std_1_bc > 90)) ,True, False)
 bc_df['BcDir'] = np.where(bc_df.ToLeft, bc_df.Dir_std_2_bc, bc_df.Dir_std_1_bc)
 bc_df['BcFast'] = np.where(( bc_df.BcPosition == 'WR')|(bc_df.BcPosition == 'CB'), 1, 0)
 bc_df['BcSlow'] = np.where(( bc_df.BcPosition == 'QB')|(bc_df.BcPosition == 'FB')| 
(bc_df.BcPosition == 'DE')|(bc_df.BcPosition == 'DT'), 1, 0)
 bc_df['BcDirCat'] = np.where(( bc_df.BcDir < 150)&(bc_df.BcDir > 30), 10,
np.where(bc_df.BcDir > 150, 1,
np.where(( bc_df.BcDir > 0)&(bc_df.BcDir < 30), 1, -1)))
 bc_df['BcMovingDf'] = np.where(( bc_df.BcDir < 150)&(bc_df.BcDir > 30), 1, 0)
 bc_df['BcATimesDir'] = bc_df['A'] * bc_df['BcDirCat']
bc_df.drop('A', axis= 1, inplace=True)
 return(bc_df)
 def calculate_defender_loc(df, ball_carrier_df):
def_df = df.loc[df['IsOnOffense']==False]
dir_df = def_df[['PlayId','DisplayName','JerseyNumber','X_std','Y_std']].merge(ball_carrier_df, on='PlayId')
 dir_df['DefenderTowardsPlayDir'] = np.where(((dir_df['BcRunRight'])&(dir_df['Y_std'] <= dir_df['Y_std_bc'])) | 
(( dir_df['BcRunRight'] == False)&(dir_df['Y_std'] >= dir_df['Y_std_bc'])) ,
1, 0)
 dir_agg_df = dir_df.groupby(['PlayId','X_std_bc','Y_std_bc','X_std_end_bc',
'Y_std_end_bc','BcRunRight','BcDir','BcFast','BcSlow',
'BcMovingDf','BcATimesDir',
], as_index=False)['DefenderTowardsPlayDir'].sum()
spread_x_df = dir_df.groupby('PlayId', as_index = False)['X_std'].agg({'DefenseMaxX': np.max,
'DefenseMinX': np.min})
 spread_y_df = dir_df.groupby('PlayId', as_index = False)['Y_std'].agg({'DefenseMaxY': np.max,
'DefenseMinY': np.min})
 spread_x_df['DefenseXSpread'] = spread_x_df.DefenseMaxX - spread_x_df.DefenseMinX
spread_y_df['DefenseYSpread'] = spread_y_df.DefenseMaxY - spread_y_df.DefenseMinY
dir_agg_df = dir_agg_df.merge(spread_x_df[['PlayId','DefenseXSpread']], on = 'PlayId' ).merge(spread_y_df[['PlayId','DefenseYSpread']], on ='PlayId')
 return(dir_agg_df)
 print('Creating some more features...')
 def more_feature_engineering(df, positions_schema):
df['GameClockAdj'] = df['GameClock'].apply(strtoseconds)
 for k, v in pos_dict.items() :
df[k + 'Split'] = df[k].str.split(', ')
 for pos in v:
df[pos + 'Count'] = df[k + 'Split'].apply(lambda row: get_personnel(row, pos))
df = df.drop(list(pos_dict.keys())+ [x + 'Split' for x in pos_dict.keys() ], axis = 1)
 df['Home'] = np.where(df['TeamOnOffense'] == 'home', 1, 0)
 df['Shotgun'] = np.where(df['OffenseFormation'] == 'SHOTGUN', 1, 0)
 df['TeamAbbrOnDefense'] = np.where(df['TeamOnOffense'] == 'home', df['VisitorTeamAbbr'], df['HomeTeamAbbr'])
 df['GameWeatherPro'] = df['GameWeather'].str.lower().apply(lambda x: "indoor" if not pd.isna(x)and "indoor" in x else x)
 df['GameWeatherPro'] = df['GameWeatherPro'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 df['GameWeatherPro'] = df['GameWeatherPro'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 df['GameWeatherPro'] = df['GameWeatherPro'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 df['GameWeatherPro'] = df['GameWeatherPro'].apply(map_weather)
 df["DiffScoreBeforePlay"] = df["HomeScoreBeforePlay"] - df["VisitorScoreBeforePlay"]
df["DiffScoreBeforePlayBinary"] =(df["HomeScoreBeforePlay"] > df["VisitorScoreBeforePlay"] ).astype("object")
 df = df.drop(['GameWeather','Season','TimeHandoff','TimeSnap','PlayerBirthDate','Orientation','Dir','WindSpeed','GameClock'], axis =1)
 bc_df = ball_carrier_features(df)
 merged_bc_def_df = calculate_defender_loc(df, bc_df)
 final_fe_df = df.merge(merged_bc_def_df, on ='PlayId')
 final_fe_df['DistFromBc'] = np.sqrt(( final_fe_df.X_std-final_fe_df.X_std_bc)**2
+(final_fe_df.Y_std-final_fe_df.Y_std_bc)**2)
 final_fe_df['S'] = np.where(final_fe_df['S'] == 0,.1, final_fe_df['S'])
 final_fe_df['TimeToBc'] = final_fe_df['DistFromBc'] / final_fe_df['S']
final_fe_df['DistFromBcEnd'] = np.sqrt(( final_fe_df.X_std_end-final_fe_df.X_std_end_bc)**2
+(final_fe_df.Y_std_end-final_fe_df.Y_std_end_bc)**2)
 final_fe_df['TimeToBcEnd'] = final_fe_df['DistFromBcEnd'] / final_fe_df['S']
final_fe_df['Dir_std'] = np.where(final_fe_df.ToLeft, final_fe_df.Dir_std_2, final_fe_df.Dir_std_1)
 final_fe_df['DefendersInTheBox'] = final_fe_df['DefendersInTheBox'].fillna(int(final_fe_df.DefendersInTheBox.median()))
final_fe_df['DefendersInTheBox_vs_Distance'] = final_fe_df['DefendersInTheBox'] / final_fe_df['Distance']
final_fe_df['InsideRun'] = np.where(( final_fe_df['BcDir'] > 65)&(final_fe_df['BcDir'] < 115), 1, 0)
 final_fe_df['OffTackle'] = np.where(((final_fe_df['BcDir'] <= 65)&(final_fe_df['BcDir'] > 30)) | \ 
(( final_fe_df['BcDir'] >= 115)&(final_fe_df['BcDir'] < 150)) , 1, 0)
 return(final_fe_df)
 fe_df = more_feature_engineering(standardized_dir_df, pos_dict )<remove_duplicates>
train_df = train_df [ train_df['building_id'] != 1099 ]
fe_feature_lst = ['GameId','PlayId','GameClockAdj','Shotgun','DefenseXSpread','DefenseYSpread',
'X_std_bc','Y_std_bc','BcDir','InsideRun','OffTackle','TeamOnOffense','YardsFromOwnGoal',
'BcFast','BcSlow','BcMovingDf','BcATimesDir',
]
fe_feature_df = fe_df[fe_feature_lst].drop_duplicates()<prepare_x_and_y>
building_meta_df['floor_area'] = building_meta_df.square_feet / building_meta_df.floor_count
def get_bc_voronoi_area(df, point_type='current'):
df = df.sort_values(['IsOnOffense','PlayId','IsBallCarrier','Y_std'])
 bc_v_area_lst = []
if point_type == 'current':
X_col, Y_col = 'X_std', 'Y_std'
elif point_type == 'end':
X_col, Y_col = 'X_std_end', 'Y_std_end'
else:
print('Invalid point type')
 for play_id in df.PlayId.unique() :
try:
v_fe_df = df.loc[df.PlayId == play_id]
v_fe_df = v_fe_df.loc[(v_fe_df.IsOnOffense == False)|(v_fe_df.IsBallCarrier)]
points = np.c_[v_fe_df[X_col], v_fe_df[Y_col]]
vor = Voronoi(points)
 vertices = vor.vertices
regions = vor.regions
point_regions = vor.point_region
v_perm = [vertices[r] for r in regions[point_regions[-1]]]
rusher_x = v_fe_df.loc[v_fe_df.IsBallCarrier, X_col].values[0]
rusher_y = v_fe_df.loc[v_fe_df.IsBallCarrier, Y_col].values[0]
vor_df = pd.DataFrame(np.vstack(v_perm), columns = [X_col, Y_col])
 vor_df[X_col] = np.where(vor_df[X_col] < rusher_x, rusher_x,
np.where(vor_df[X_col] > rusher_x + 20, rusher_x + 20, vor_df[X_col]))
vor_df[Y_col] = np.where(vor_df[Y_col] <(rusher_y / 2), rusher_y / 2,
np.where(vor_df[Y_col] > rusher_y * 2, rusher_y * 2, vor_df[Y_col]))
vor_area = ConvexHull(vor_df, qhull_options = 'QJ' ).area
if vor_area > 1000:
print(play_id)
 bc_v_area_lst.append(vor_area)
 except:
print('Voronoi Error for play {}'.format(play_id))
bc_v_area_lst.append(42)
 return(bc_v_area_lst )<feature_engineering>
en_holidays = holidays.England()
ir_holidays = holidays.Ireland()
ca_holidays = holidays.Canada()
us_holidays = holidays.UnitedStates()
def add_holiyday(df_weather):
en_idx = df_weather.query('site_id == 1 or site_id == 5' ).index
ir_idx = df_weather.query('site_id == 12' ).index
ca_idx = df_weather.query('site_id == 7 or site_id == 11' ).index
us_idx = df_weather.query('site_id == 0 or site_id == 2 or site_id == 3 or site_id == 4 or site_id == 6 or site_id == 8 or site_id == 9 or site_id == 10 or site_id == 13 or site_id == 14 or site_id == 15' ).index
df_weather['IsHoliday'] = 0
df_weather.loc[en_idx, 'IsHoliday'] = df_weather.loc[en_idx, 'timestamp'].apply(lambda x: en_holidays.get(x, default=0))
df_weather.loc[ir_idx, 'IsHoliday'] = df_weather.loc[ir_idx, 'timestamp'].apply(lambda x: ir_holidays.get(x, default=0))
df_weather.loc[ca_idx, 'IsHoliday'] = df_weather.loc[ca_idx, 'timestamp'].apply(lambda x: ca_holidays.get(x, default=0))
df_weather.loc[us_idx, 'IsHoliday'] = df_weather.loc[us_idx, 'timestamp'].apply(lambda x: us_holidays.get(x, default=0))
holiday_idx = df_weather['IsHoliday'] != 0
df_weather.loc[holiday_idx, 'IsHoliday'] = 1
df_weather['IsHoliday'] = df_weather['IsHoliday'].astype(np.uint8 )
%time bc_v_area = get_bc_voronoi_area(fe_df, 'current' )<concatenate>
set_local(weather_train_df)
 add_holiyday(weather_train_df )
bc_v_area_end = get_bc_voronoi_area(fe_df, 'end' )<feature_engineering>
train_df = train_df[train_df_black.black_count < 24*black_day]
fe_feature_df['bc_v_area'] = bc_v_area
fe_feature_df['bc_v_area_end'] = bc_v_area_end<feature_engineering>
del train_df_black
gc.collect()
def standardize_dataset(train):
train['ToLeft'] = train.PlayDirection == "left"
train['IsBallCarrier'] = train.NflId == train.NflIdRusher
train['TeamOnOffense'] = "home"
train.loc[train.PossessionTeam != train.HomeTeamAbbr, 'TeamOnOffense'] = "away"
train['IsOnOffense'] = train.Team == train.TeamOnOffense
train['YardLine_std'] = 100 - train.YardLine
train.loc[train.FieldPosition.fillna('')== train.PossessionTeam,
'YardLine_std'
] = train.loc[train.FieldPosition.fillna('')== train.PossessionTeam,
'YardLine']
train['X_std'] = train.X
train.loc[train.ToLeft, 'X_std'] = 120 - train.loc[train.ToLeft, 'X']
train['Y_std'] = train.Y
train.loc[train.ToLeft, 'Y_std'] = 53.3 - train.loc[train.ToLeft, 'Y']
train['Orientation_std'] = train.Orientation
train.loc[train.ToLeft, 'Orientation_std'] = np.mod(180 + train.loc[train.ToLeft, 'Orientation_std'], 360)
 train['Dir_std'] = train.Dir
train.loc[train.ToLeft, 'Dir_std'] = np.mod(180 + train.loc[train.ToLeft, 'Dir_std'], 360)
 train.loc[train['Season'] == 2017, 'Orientation'] = np.mod(90 + train.loc[train['Season'] == 2017, 'Orientation'], 360)
 return train
dominance_df = standardize_dataset(train_3 )<train_on_grid>
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")' )
def radius_calc(dist_to_ball):

 return 4 + 6 *(dist_to_ball >= 15)+(dist_to_ball ** 3)/ 560 *(dist_to_ball < 15)
 @np.vectorize
def pitch_control(x_point, y_point):

 offense_ids = my_play[my_play['IsOnOffense']].index
offense_control = compute_influence(x_point, y_point, offense_ids)
 offense_score = np.sum(offense_control)
 defense_ids = my_play[~my_play['IsOnOffense']].index
defense_control = compute_influence(x_point, y_point, defense_ids)
 defense_score = np.sum(defense_control)
 return expit(offense_score - defense_score)
 class Controller:

 def __init__(self, play):
self.play = play
self.vec_influence = np.vectorize(self.compute_influence)
 self.vec_control = np.vectorize(self.pitch_control)
 def compute_influence(self, x_point, y_point, player_id):

 point = np.array([x_point, y_point])
 player_row = self.play.loc[player_id]
theta = math.radians(player_row[56])
 speed = player_row[5]
player_coords = player_row[54:56].values
ball_coords = self.play[self.play['IsBallCarrier']].iloc[:, 54:56].values
dist_to_ball = euclidean(player_coords, ball_coords)
 S_ratio =(speed / 13)** 2
RADIUS = radius_calc(dist_to_ball)
 S_matrix = np.matrix([[RADIUS *(1 + S_ratio), 0], [0, RADIUS *(1 - S_ratio)]])
 R_matrix = np.matrix([[np.cos(theta), - np.sin(theta)], [np.sin(theta), np.cos(theta)]])
 COV_matrix = np.dot(np.dot(np.dot(R_matrix, S_matrix), S_matrix), np.linalg.inv(R_matrix))
norm_fact =(1 / 2 * np.pi)*(1 / np.sqrt(np.linalg.det(COV_matrix)))
 mu_play = player_coords + speed * np.array([np.cos(theta), np.sin(theta)])/ 2
intermed_scalar_player = np.dot(np.dot(( player_coords - mu_play),
np.linalg.inv(COV_matrix)) ,
np.transpose(( player_coords - mu_play)))
 player_influence = norm_fact * np.exp(- 0.5 * intermed_scalar_player[0, 0])
 intermed_scalar_point = np.dot(np.dot(( point - mu_play),
np.linalg.inv(COV_matrix)) ,
np.transpose(( point - mu_play)))
 point_influence = norm_fact * np.exp(- 0.5 * intermed_scalar_point[0, 0])
 return point_influence / player_influence
def pitch_control(self, x_point, y_point):

 offense_ids = self.play[self.play['IsOnOffense']].index
offense_control = self.vec_influence(x_point, y_point, offense_ids)
 offense_score = np.sum(offense_control)
 defense_ids = self.play[~self.play['IsOnOffense']].index
defense_control = self.vec_influence(x_point, y_point, defense_ids)
 defense_score = np.sum(defense_control)
 return expit(offense_score - defense_score)
 def display_control(self, grid_size=(30, 15), figsize=(11, 7)) :
front, behind = 15, 5
left, right = 20, 20
colorm = ['purple'] * 11 + ['orange'] * 11
colorm[np.where(self.play.Rusher.values)[0][0]] = 'black'
player_coords = self.play[self.play['Rusher']][['X_std', 'Y_std']].values[0]
X, Y = np.meshgrid(np.linspace(player_coords[0] - behind,
player_coords[0] + front,
grid_size[0]),
np.linspace(player_coords[1] - left,
player_coords[1] + right,
grid_size[1]))
infl = self.vec_control(X, Y)
 plt.figure(figsize=figsize)
 plt.contourf(X, Y, infl, 12, cmap='bwr')
 plt.scatter(self.play['X'].values, self.play['Y'].values, c=colorm)
 plt.title('Yards gained = {}, play_id = {}'.format(self.play['Yards'].values[0],
self.play['PlayId'].unique() [0]))
plt.show()
control_lst = []
for play_id in tqdm(dominance_df.PlayId.unique()):
my_play = dominance_df[dominance_df['PlayId']==play_id]
control = Controller(my_play)
 bc_coords = list(*my_play.loc[(my_play.IsBallCarrier), ['X_std','Y_std']].values)
 pitch_control = control.vec_control(*bc_coords)
 control_lst.append(pitch_control )<feature_engineering>
( train_df[train_df.building_id == 954].meter_reading==0 ).astype(int ).plot()
train_basetable['BcInfl'] = [np.asscalar(i)for i in control_lst]<statistical_test>
train_df = train_df.query('not(building_id == 954 & meter_reading == 0)')
 train_df = train_df.query('not(building_id == 1221 & meter_reading == 0)' )
def centeroidnp(arr):
length = arr.shape[0]
sum_x = np.sum(arr[:, 0])
 sum_y = np.sum(arr[:, 1])
 return(sum_x/length, sum_y/length)
 def get_defense_centroid(df):
centroid_lst = []
defense_df = df.loc[df.IsOnOffense == False]
for play_id in defense_df.PlayId.unique() :
def_points = defense_df.loc[defense_df.PlayId == play_id, ['X_std','Y_std']]
cp = centeroidnp(np.array(def_points))
centroid_lst.append(cp)
 return(centroid_lst)
 %time train_basetable['def_centroid'] = get_defense_centroid(fe_df)
 train_basetable['def_centroid_X_std'] = train_basetable.def_centroid.apply(lambda x: x[0])
 train_basetable['def_centroid_Y_std'] = train_basetable.def_centroid.apply(lambda x: x[1] )<feature_engineering>
funny_bids = [993, 1168, 904, 954, 778, 1021]
print('before', len(train_df))
if outlier:
train_df.loc[(train_df.building_id == 993)&(train_df.meter == 0)&(train_df.meter_reading > 30000), 'meter_reading'] = 31921
train_df.loc[(train_df.building_id == 993)&(train_df.meter == 1)&(train_df.meter_reading > 90000), 'meter_reading'] = 96545.5
train_df = train_df[(( train_df.building_id == 1168)&(train_df.meter == 0)&(train_df.meter_reading >10000)) == False]
train_df.loc[(train_df.building_id == 904)&(train_df.meter == 0)&(train_df.meter_reading >10000), 'meter_reading'] = 11306
train_df = train_df[(( train_df.building_id == 954)&(train_df.meter_reading >10000)) ==False]
if rescale:
train_df.loc[(train_df.building_id == 778)&(train_df.meter == 1), 'meter_reading'] = train_df.loc[(train_df.building_id == 778)&(train_df.meter == 1), 'meter_reading']/ 100
train_df.loc[(train_df.building_id == 1021)&(train_df.meter == 3), 'meter_reading'] = train_df.loc[(train_df.building_id == 1021)&(train_df.meter == 3), 'meter_reading']/ 1000
train_df = train_df.reset_index()
print('after', len(train_df))
gc.collect()
def get_qb_location(df):
qb_fe_df = df.loc[(df.Position == 'QB')].sort_values(['PlayId','S'])
 qb_fe_df['QbRank'] = qb_fe_df.groupby(by=['PlayId'])['S'].transform(lambda x: x.rank())
 qb_fe_df = qb_fe_df.loc[qb_fe_df['QbRank'] == 1, ['PlayId', 'X_std', 'Y_std']]
qb_fe_df.rename(columns={'X_std':'X_std_qb', 'Y_std': 'Y_std_qb'}, inplace=True)
 return(qb_fe_df)
 qb_location_df = get_qb_location(fe_df )<groupby>
train_df.loc[(train_df.building_id.isin(site_0_bids)) &(train_df.meter==0), 'meter_reading'] = train_df[(train_df.building_id.isin(site_0_bids)) &(train_df.meter==0)]['meter_reading'] * 0.2931
def get_defense_end(df):
def_df = df.loc[df.IsOnOffense==False]
def_df['DefYardsTowardsBc'] = def_df['DistFromBc'] - def_df['DistFromBcEnd']
def_dist_end_df = def_df.groupby('PlayId', as_index=False)['DistFromBcEnd'] \ 
.agg({'MinDistFromBcEnd':np.min,'MeanDistFromBcEnd':np.mean,'MedDistFromBcEnd':np.median,'StdDistFromBcEnd':np.std})
 def_toward_bc_df = def_df.groupby('PlayId', as_index = False)['DefYardsTowardsBc'] \ 
.agg({'SumDefYardsTowardsBc':sum, 'StdDefYardsTowardsBc':np.std, 'MaxDefYardsTowardsBc': np.max})
 return(def_dist_end_df.merge(def_toward_bc_df, on = 'PlayId'))
def_dist_end_df = get_defense_end(fe_df)
<groupby>
train_df['date'] = train_df['timestamp'].dt.date
train_df['meter_reading_log1p'] = np.log1p(train_df['meter_reading'] )
def_time_to_spot_df = fe_df.loc[fe_df.IsOnOffense == False] \ 
.groupby('PlayId', as_index=False)[['TimeToBc','TimeToBcEnd']] \ 
.agg([np.min, np.mean, np.median, np.std])
 def_time_to_spot_df.columns = def_time_to_spot_df.columns.droplevel(0)
 def_time_to_spot_df.columns = ['TimeToBc' + c for c in def_time_to_spot_df.columns[0:4]] \
+ ['TimeToBcEnd' + c for c in def_time_to_spot_df.columns[4:]]
def_time_to_spot_df.reset_index(inplace=True )<groupby>
def preprocess(df):
df["hour"] = df["timestamp"].dt.hour
df["day"] = df["timestamp"].dt.day
df["weekend"] = df["timestamp"].dt.weekday
df["month"] = df["timestamp"].dt.month
df["dayofweek"] = df["timestamp"].dt.dayofweek
fe_defenders = fe_df.loc[fe_df.IsOnOffense == False]
fe_defenders['DistFromBcRank'] = fe_defenders.groupby('PlayId')['DistFromBc'].transform(lambda x: x.rank())
 three_def_df = fe_defenders.loc[fe_defenders.DistFromBcRank <= 3]
grouped_closest_dist_df = three_def_df.groupby('PlayId', as_index=False)['DistFromBc'].mean()
grouped_closest_dist_df.rename(columns={'DistFromBc':'Top3DistFromBcMean'}, inplace=True )<merge>
preprocess(train_df )
merged_basetable = train_basetable.merge(def_dist_end_df, on='PlayId')\ 
.merge(ol_agg_df, on = 'PlayId')\ 
.merge(fe_feature_df, on=['GameId','PlayId'])\ 
.merge(qb_location_df, how = 'left', on = 'PlayId')\ 
.merge(def_time_to_spot_df, how = 'left', on = 'PlayId')\ 
.merge(grouped_closest_dist_df, how = 'left', on = 'PlayId')
 merged_basetable['Shotgun_Down_Distance'] = merged_basetable['Shotgun'] * merged_basetable['Down'] * merged_basetable['Distance']
merged_basetable['BcDistFromDefCent'] = np.sqrt(( merged_basetable.X_std_bc - merged_basetable.def_centroid_X_std)**2
+(merged_basetable.Y_std_bc - merged_basetable.def_centroid_Y_std)**2)
 merged_basetable['BcDistFromQb'] = np.sqrt(( merged_basetable.X_std_bc - merged_basetable.X_std_qb)**2
+(merged_basetable.Y_std_bc - merged_basetable.Y_std_qb)**2)
 merged_basetable['BcDistFromQb'] = merged_basetable['BcDistFromQb'].fillna(merged_basetable.BcDistFromQb.mean())
 merged_basetable['BcDirFromMid'] = abs(merged_basetable['BcDir'] - 90 )<categorify>
if use_ucf:
train_df = train_df.sort_values('month')
 train_df = train_df.reset_index()
<feature_engineering>
merged_basetable['Quarter4'] = np.where(merged_basetable.Quarter == 4, 1, 0)
 merged_basetable['Down4'] = np.where(merged_basetable.Down == 4, 1, 0 )<drop_column>
weather_train_df.isna().sum()
cols_to_drop = ['def_centroid','X','Y','X_std_qb','Y_std_qb','def_centroid_X_std','def_centroid_Y_std',
'Dir','TeamOnOffense','Quarter','Down','YardsFromOwnGoal',
]
merged_basetable.drop(cols_to_drop, axis = 1, inplace=True )<feature_engineering>
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )
def cleanabv(train):
train['ToLeft'] = train.PlayDirection == "left"
train['IsBallCarrier'] = train.NflId == train.NflIdRusher
train.loc[train.VisitorTeamAbbr == "ARI", 'VisitorTeamAbbr'] = "ARZ"
train.loc[train.HomeTeamAbbr == "ARI", 'HomeTeamAbbr'] = "ARZ"
train.loc[train.VisitorTeamAbbr == "BAL", 'VisitorTeamAbbr'] = "BLT"
train.loc[train.HomeTeamAbbr == "BAL", 'HomeTeamAbbr'] = "BLT"
train.loc[train.VisitorTeamAbbr == "CLE", 'VisitorTeamAbbr'] = "CLV"
train.loc[train.HomeTeamAbbr == "CLE", 'HomeTeamAbbr'] = "CLV"
train.loc[train.VisitorTeamAbbr == "HOU", 'VisitorTeamAbbr'] = "HST"
train.loc[train.HomeTeamAbbr == "HOU", 'HomeTeamAbbr'] = "HST"
train['Dir_std_1'] = np.where(( train['ToLeft'])&(train['Dir'] < 90), train['Dir'] + 360, train['Dir'])
 train['Dir_std_1'] = np.where(( train['ToLeft']==False)&(train['Dir'] > 270), train['Dir'] - 360, train['Dir_std_1'])
 train['Dir_std_2'] = np.where(train['ToLeft'], train['Dir_std_1'] - 180, train['Dir_std_1'])
 train['Dir_std'] = np.where(train.ToLeft, train.Dir_std_2, train.Dir_std_1)
 train.loc[train.DisplayName == "Bradley Sowell", "Position"] = "T"
return train
def getolinesix(play):
oline = play.loc[(play.loc[:,"Position"]=="T")|(play.loc[:,"Position"]=="G")|(play.loc[:,"Position"]=="C")|(play.loc[:,"Position"]=="OT")|(play.loc[:,"Position"]=="OG"),:]
oline = oline.sort_values(by = 'Y')
 oline = oline.reset_index()
firstoline = oline.loc[0,"level_0"]
lastoline = oline.loc[4,"level_0"]
onetwo = 0
if firstoline>10:
offense = play[11:22]
onetwo = 2
else:
offense = play[0:11]
onetwo = 1
offense = offense.sort_values(by = 'Y')
 offense = offense.reset_index()
ox1 = offense.loc[offense.loc[:,"level_0"]==firstoline,:]
ox1 = ox1.index
ox5 = offense.loc[offense.loc[:,"level_0"]==lastoline,:]
ox5 = ox5.index
if len(ox1)== 0:
ox1 = pd.Series([1])
 if len(ox5)== 0:
ox5 = pd.Series([10])
 lh = 0
li = -1
rh = 0
ri = -1
for i in range(ox1[0]-1,-1,-1):
tempx = offense.loc[i,"Y"]
tempy = offense.loc[i,"X"]
oxx = offense.loc[ox1,"Y"]
oxy = offense.loc[ox1,"X"]
tempd = distance(tempx,oxx,tempy,oxy)
 if tempd <= 4:
if offense.loc[i,"Position"]=="TE":
lh = 1
li = i
break
else:
lh = 0
li = -1
for i in range(ox5[0]+1,11,1):
tempx = offense.loc[i,"Y"]
tempy = offense.loc[i,"X"]
oxx = offense.loc[ox5,"Y"]
oxy = offense.loc[ox5,"X"]
tempd = distance(tempx,oxx,tempy,oxy)
 if tempd <= 4:
if offense.loc[i,"Position"]=="TE":
rh = 1
ri = i
break
else:
rh = 0
ri = -1
play.reset_index()
play.sort_values(by = 'Y')
 return oline,ri,rh,li,lh,offense
def encodepos(play,uniqueteams):
possession = np.zeros(22)
 for i in range(0,22):
for j in range(0,32):
if play.loc[i,"PossessionTeam"]==uniqueteams[j]:
possession[i]=j
return possession
def encodefpos(play,uniqueteams):
fposition = np.zeros(22)
 for i in range(0,22):
for j in range(0,32):
if play.loc[i,"FieldPosition"]==uniqueteams[j]:
fposition[i]=j
return fposition
def changedirection(play,pos,fpos):
ha = play.loc[play.loc[:,"NflId"]==play.loc[0,"NflIdRusher"],"Team"]
bc = np.where(play.loc[:,"NflId"]==play.loc[0,"NflIdRusher"])[0]
bcy = play.loc[bc,"X"]
bcy = bcy.reset_index()
if bc > 10:
meandy = stats.mean(play.loc[11:,'X'])
 else:
meandy = stats.mean(play.loc[0:10,'X'])
 postemp = pos[0]
fpostemp = fpos[0]
if bcy.loc[0,"X"]>meandy:
play.loc[:,"X"] = 100 - play.loc[:,"X"] + 20
else:
play.loc[:,"X"] = play.loc[:,"X"]
play.loc[:,"Y"] = 53.3 - play.loc[:,"Y"]
if postemp != fpostemp:
play.loc[:,"YardLine"] = 100 - play.loc[0,"YardLine"]
ydline = play.loc[0,"YardLine"]
return play, ha, ydline, bc
def distance(x1,x2,y1,y2):
dist = math.sqrt(( x1-x2)**2+(y1-y2)**2)
 return dist
def linear(x1,x2,y1,y2):
slope =(y1-y2)/(x1-x2)
 temp = slope*x1
b = y1-temp
return slope, b
def hmengagedextra(play,ha,ydline,bc):
oline,ri,rh,li,lh,offense = getolinesix(play)
 play = play.reset_index()
if rh == 1:
rol = offense.loc[[ri]]
oline = pd.concat([oline,rol])
 else:
temp = oline.loc[[4]]
temp.loc[4,"Y"]=oline.loc[4,"Y"]+2
oline = pd.concat([oline,temp])
 if lh == 1:
lol = offense.loc[[li]]
oline = pd.concat([lol,oline])
 else:
temp = oline.loc[[0]]
temp.loc[0,"Y"] = temp.loc[0,"Y"]-2
oline = pd.concat([temp,oline])
 olx = oline.loc[:,"Y"]
olx = olx.reset_index()
oly = oline.loc[:,"X"]
oly = oly.reset_index()
bcy = play.loc[bc,"X"]
bcy = bcy.reset_index()
if ha.all() == "away":
defense = play.loc[11:21,:]
else:
defense = play.loc[0:10,:]
defense = defense.drop(columns="level_0")
 defense = defense.reset_index()
olnum = np.zeros(6)
 olnumeng = np.zeros(6)
 numdefbf = np.zeros(6)
 for i in range(0,11):
dx = defense.loc[i,"Y"]
dy = defense.loc[i,"X"]
for j in range(0,6):
olx1 = olx.loc[j,"Y"]
olx2 = olx.loc[j+1,"Y"]
oly1 = oly.loc[j,"X"]
oly2 = oly.loc[j+1,"X"]
if(dx > olx1)&(dx <= olx2):
dist1 = distance(olx1,dx,oly1,dy)
 dist2 = distance(olx2,dx,oly2,dy)
 if(dy < ydline+15)&(dy > bcy.loc[0,"X"]):
slope,b = linear(olx1,olx2,oly1,oly2)
 tempy = dx*slope+b
if dy<=tempy:
numdefbf[j] = numdefbf[j] + 1
if dist1 <=1.5 or dist2 <=1.5:
olnumeng[j] = olnumeng[j]+1
olnum[j] = olnum[j]+1
return olnum,olnumeng,numdefbf,oline
train = pd.read_csv(".. /input/nfl-big-data-bowl-2020/train.csv", low_memory=False)
 train = cleanabv(train)
 uniqueplays = np.unique(train.loc[:,"PlayId"])
 uniqueteams = np.unique(train.loc[:,"PossessionTeam"] )<categorify>
weather_train_df = weather_train_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
holes_spread = []
for i in tqdm(range(0, len(uniqueplays))):
play = train.loc[train.loc[:,"PlayId"]==uniqueplays[i],:]
play = play.reset_index()
playid = play.PlayId[0]
pos = encodepos(play,uniqueteams)
 fpos = encodefpos(play,uniqueteams)
 play,ha,ydline,bc = changedirection(play,pos,fpos)
 olnum,olnumeng,numdefbf,oline = hmengagedextra(play,ha,ydline,bc)
 bc_dir = play.loc[play.IsBallCarrier, 'Dir_std'].values[0]
oline_y = oline.Y
holes = np.diff(oline_y)
 holes_spread.append(np.mean(holes))
if i == 0:
olnums = olnum
olnumseng = olnumeng
numdefsbf = numdefbf
else:
olnums = np.vstack(( olnums,olnum))
olnumseng = np.vstack(( olnumseng,olnumeng))
numdefsbf = np.vstack(( numdefsbf,numdefbf))<feature_engineering>
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )
los_df = outcomes.copy().drop('Yards', axis = 1)
 los_df['olnums'] = [sum(i)for i in olnums]
los_df['olnumseng'] = [sum(i)for i in olnumseng]
los_df['numdefsbf'] = [sum(i)for i in numdefsbf]
los_df['avgholesize'] = holes_spread<merge>
def add_lag_feature(weather_df, window=3):
group_df = weather_df.groupby('site_id')
 cols = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction', 'wind_speed']
rolled = group_df[cols].rolling(window=window, min_periods=0)
 lag_mean = rolled.mean().reset_index().astype(np.float16)
 lag_max = rolled.max().reset_index().astype(np.float16)
 lag_min = rolled.min().reset_index().astype(np.float16)
 lag_std = rolled.std().reset_index().astype(np.float16)
 for col in cols:
weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]
weather_df[f'{col}_max_lag{window}'] = lag_max[col]
weather_df[f'{col}_min_lag{window}'] = lag_min[col]
weather_df[f'{col}_std_lag{window}'] = lag_std[col]
merged_basetable = merged_basetable.merge(los_df, on=['GameId', 'PlayId'])
 merged_basetable['numdefsb_cube'] = merged_basetable['numdefsbf'] ** 3
merged_basetable['numdefsbf_cube_div_min_dist'] = merged_basetable['numdefsb_cube'] / merged_basetable['def_min_dist']<data_type_conversions>
add_lag_feature(weather_train_df, window=3)
 add_lag_feature(weather_train_df, window=72 )
merged_basetable['BcInfl'] = merged_basetable['BcInfl'].fillna (.8)
 merged_basetable['v_infl'] =(merged_basetable['BcInfl'] * merged_basetable['bc_v_area_end'] ).fillna(24 )<filter>
year_map = building_meta_df.year_built.value_counts()
building_meta_df['year_cnt'] = building_meta_df.year_built.map(year_map)
 bid_map = train_df.building_id.value_counts()
train_df['bid_cnt'] = train_df.building_id.map(bid_map )
lower_bound = merged_basetable.Yards.quantile(0.002)
 upper_bound = merged_basetable.Yards.quantile(0.998)
 merged_basetable = merged_basetable.loc[(merged_basetable.Yards > lower_bound)&(merged_basetable.Yards < upper_bound)]<create_dataframe>
primary_use_list = building_meta_df['primary_use'].unique()
primary_use_dict = {key: value for value, key in enumerate(primary_use_list)}
print('primary_use_dict: ', primary_use_dict)
 building_meta_df['primary_use'] = building_meta_df['primary_use'].map(primary_use_dict)
 gc.collect()
cluster_features = ['back_from_scrimmage','S','A','Shotgun','BcDistFromQb','InsideRun','OffTackle','TimeDelta','BcDirFromMid',]
cluster_df = merged_basetable[cluster_features]<predict_on_test>
train_df = reduce_mem_usage(train_df, use_float16=True)
 building_meta_df = reduce_mem_usage(building_meta_df, use_float16=True)
 weather_train_df = reduce_mem_usage(weather_train_df, use_float16=True )
cluster_fit = fit_clusters(data_transformed, 5)
 cluster_labels = cluster_fit.predict(data_transformed)
 merged_basetable['RunCluster'] = cluster_labels<prepare_x_and_y>
def add_sg(df):
w = 11
p = 2
for si in df.site_id.unique() :
index = df.site_id == si
df.loc[index, 'air_smooth'] = sg(df[index].air_temperature, w, p)
 df.loc[index, 'dew_smooth'] = sg(df[index].dew_temperature, w, p)
 df.loc[index, 'air_diff'] = sg(df[index].air_temperature, w, p, 1)
 df.loc[index, 'dew_diff'] = sg(df[index].dew_temperature, w, p, 1)
 df.loc[index, 'air_diff2'] = sg(df[index].air_temperature, w, p, 2)
 df.loc[index, 'dew_diff2'] = sg(df[index].dew_temperature, w, p, 2 )
X = merged_basetable.copy()
yards = X.Yards
y = np.zeros(( yards.shape[0], 199))
for idx, target in enumerate(list(yards)) :
y[idx][99 + target] = 1
X.drop(['GameId','PlayId','Yards'], axis=1, inplace=True)
 scaler = StandardScaler()
binary_features = ['back_oriented_down_field', 'back_moving_down_field', 'Shotgun', 'InsideRun', 'OffTackle',
'BcFast','BcSlow','BcMovingDf','Quarter4', 'Down4','RunCluster',
]
binary_array = np.array(X.loc[:, binary_features])
 numeric_df = X.drop(binary_features, axis = 1)
 numeric_scaled = scaler.fit_transform(numeric_df)
 X_transformed = np.hstack(( numeric_scaled, binary_array))
X_train, X_val, y_train, y_val = train_test_split(X_transformed, y, test_size=0.25, random_state=12345)
 print(X_train.shape, X_val.shape)
 print(y_train.shape, y_val.shape )<compute_train_metric>
add_sg(weather_train_df )
def get_rf(x_tr, y_tr, x_val, y_val, shape):
model = RandomForestRegressor(bootstrap=False, max_features=0.3, min_samples_leaf=15,
min_samples_split=8, n_estimators=25, n_jobs=-1, random_state=42)
 model.fit(x_tr, y_tr)
 y_pred = model.predict(x_val)
 y_valid = y_val
crps = crps_score(y_pred, y_valid, shape=shape)
 return model, crps, y_pred
def crps_score(y_prediction, y_valid, shape=X.shape[0]):
y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_prediction, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * shape)
 crps = np.round(val_s, 6)
 return crps
rf, crps_rf, y_pred = get_rf(X_train, y_train, X_val, y_val, shape=X_val.shape[0])
 print("the crps(RF)is %f"%(crps_rf))<create_dataframe>
category_cols = ['building_id', 'site_id', 'primary_use', 'IsHoliday']
feature_cols = ['square_feet', 'year_built'] + [
'hour', 'weekend',
'day',
] + [
'air_temperature', 'cloud_coverage',
'dew_temperature', 'precip_depth_1_hr',
'sea_level_pressure',
'air_temperature_mean_lag72',
'air_temperature_max_lag72', 'air_temperature_min_lag72',
'air_temperature_std_lag72', 'cloud_coverage_mean_lag72',
'dew_temperature_mean_lag72', 'precip_depth_1_hr_mean_lag72',
'sea_level_pressure_mean_lag72',
'wind_speed_mean_lag72',
'air_temperature_mean_lag3',
'air_temperature_max_lag3',
'air_temperature_min_lag3', 'cloud_coverage_mean_lag3',
'dew_temperature_mean_lag3',
'precip_depth_1_hr_mean_lag3',
'sea_level_pressure_mean_lag3',
'year_cnt', 'bid_cnt',
'dew_smooth', 'air_smooth',
'dew_diff', 'air_diff',
'dew_diff2', 'air_diff2',
]
pd.DataFrame(list(zip(X.columns, rf.feature_importances_)) ,
columns = ['Feature', 'importances'] ).sort_values('importances', ascending = False )<set_options>
def create_X_y(train_df, target_meter):
target_train_df = train_df[train_df['meter'] == target_meter]
target_train_df = target_train_df.merge(building_meta_df, on='building_id', how='left')
 target_train_df = target_train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')
 X_train = target_train_df[feature_cols + category_cols]
y_train = target_train_df['meter_reading_log1p'].values
del target_train_df
return X_train, y_train
class CRPSCallback(Callback):
def __init__(self,validation, predict_batch_size=20, include_on_batch=False):
super(CRPSCallback, self ).__init__()
self.validation = validation
self.predict_batch_size = predict_batch_size
self.include_on_batch = include_on_batch
print('validation shape',len(self.validation))
def on_batch_begin(self, batch, logs={}):
pass
def on_train_begin(self, logs={}):
if not('CRPS_score_val' in self.params['metrics']):
self.params['metrics'].append('CRPS_score_val')
 def on_batch_end(self, batch, logs={}):
if(self.include_on_batch):
logs['CRPS_score_val'] = float('-inf')
 def on_epoch_end(self, epoch, logs={}):
logs['CRPS_score_val'] = float('-inf')
 if(self.validation):
X_valid, y_valid = self.validation[0], self.validation[1]
y_pred = self.model.predict(X_valid)
 y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * X_valid.shape[0])
 val_s = np.round(val_s, 6)
 logs['CRPS_score_val'] = val_s<compute_test_metric>
def fit_lgbm(train, val, devices=(-1,), seed=None, cat_features=None, num_rounds=1500, lr=0.1, bf=0.1):

 X_train, y_train = train
X_valid, y_valid = val
metric = 'l2'
params = {'num_leaves': 31,
'objective': 'regression',
'learning_rate': lr,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": bf,
"feature_fraction": 0.9,
"metric": metric,
}
device = devices[0]
if device == -1:
pass
else:
print(f'using gpu device_id {device}...')
 params.update({'device': 'gpu', 'gpu_device_id': device})
 params['seed'] = seed
early_stop = 20
verbose_eval = 20
d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)
 d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 print('best_score', model.best_score)
 log = {'train/mae': model.best_score['training']['l2'],
'valid/mae': model.best_score['valid_1']['l2']}
return model, y_pred_valid, log
def get_model(x_tr,y_tr,x_val,y_val):
inp = Input(shape =(x_tr.shape[1],))
x = Dense(1024, input_dim=X.shape[1], activation='relu' )(inp)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(512, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(256, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 out = Dense(199, activation='softmax' )(x)
 model = Model(inp,out)
 model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=[])
 es = EarlyStopping(monitor='CRPS_score_val',
mode='min',
restore_best_weights=True,
verbose=1,
patience=15)
 mc = ModelCheckpoint('best_model.h5',monitor='CRPS_score_val',mode='min',
save_best_only=True, verbose=1, save_weights_only=True)
 bsz = 1024
steps = x_tr.shape[0]/bsz
model.fit(x_tr, y_tr,callbacks=[CRPSCallback(validation =(x_val,y_val)) ,es,mc], epochs=20, batch_size=bsz,verbose=0)
 model.load_weights("best_model.h5")
 y_pred = model.predict(x_val)
 y_valid = y_val
y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * x_val.shape[0])
 crps = np.round(val_s, 6)
 return model,crps, y_true, y_pred<split>
seed = 666
shuffle = False
kf = GroupKFold(n_splits=folds )
losses = []
models = []
crps_csv = []
s_time = time.time()
for k in range(1):
kfold = KFold(5, random_state = 42 + k, shuffle = True)
 for k_fold,(tr_inds, val_inds)in enumerate(kfold.split(yards)) :
print("-----------")
 print("-----------")
 tr_x,tr_y = X_transformed[tr_inds],y[tr_inds]
val_x,val_y = X_transformed[val_inds],y[val_inds]
model,crps,y_true,y_preds = get_model(tr_x,tr_y,val_x,val_y)
 models.append(model)
 print("the %d fold crps is %f"%(( k_fold+1),crps))
crps_csv.append(crps)
 print("mean crps is %f"%np.mean(crps_csv))<compute_test_metric>
def get_groups(df, meter):
if folds == 12:
return df[df.meter==meter].month -1
elif folds == 6:
return(df[df.meter==meter].month -1)// 2
elif folds == 3:
return(df[df.meter==meter].month -1)// 4
print('mean crps is {} and std is {}'.format(np.mean(crps_csv), np.std(crps_csv)) )<predict_on_test>
oof_total = 0
target_meter = 0
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models0 = []
for train_idx, valid_idx in kf.split(X_train, y_train, groups=get_groups(train_df, target_meter)) :
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
mindex = train_df[train_df.meter==target_meter].iloc[valid_idx,:].month.unique()
print(mindex)
 print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=num_rounds, lr=0.05, bf=0.7)
 y_valid_pred_total[valid_idx] = y_pred_valid
models0.append([mindex, model])
 gc.collect()
if debug:
break
oof0 = mean_squared_error(y_train, y_valid_pred_total)
 sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof_total += oof0 * len(y_train)
 del X_train, y_train
gc.collect()
def nn_predict(x_te):
model_num = len(models)
 for k,m in enumerate(models):
if k==0:
y_pred = m.predict(x_te,batch_size=1024)
 else:
y_pred+=m.predict(x_te,batch_size=1024)
 y_pred = y_pred / model_num
return y_pred<split>
target_meter = 1
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models1 = []
for train_idx, valid_idx in kf.split(X_train, y_train, groups=get_groups(train_df, target_meter)) :
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
mindex = train_df[train_df.meter==target_meter].iloc[valid_idx,:].month.unique()
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=num_rounds,
lr=0.05, bf=0.5)
 y_valid_pred_total[valid_idx] = y_pred_valid
models1.append([mindex, model])
 gc.collect()
if debug:
break
oof1 = mean_squared_error(y_train, y_valid_pred_total)
 sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof_total += oof1 * len(y_train)
 del X_train, y_train
gc.collect()
iter_test = env.iter_test()<merge>
target_meter = 2
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models2 = []
for train_idx, valid_idx in kf.split(X_train, y_train, groups=get_groups(train_df, target_meter)) :
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
mindex = train_df[train_df.meter==target_meter].iloc[valid_idx,:].month.unique()
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=num_rounds, lr=0.05, bf=0.8)
 y_valid_pred_total[valid_idx] = y_pred_valid
models2.append([mindex, model])
 gc.collect()
if debug:
break
oof2 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof2 * len(y_train)
 sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 del X_train, y_train
gc.collect()
%%time
test_x_lst = []
test_df_2 = test_df.copy()
test_df_3 = test_df.copy()
test_df_4 = test_df.copy()
test_basetable = create_features(test_df, deploy=True)
 test_adjusted_df = adjust_players_and_teams(test_df_2, player_schema, team_schema)
 test_standardized_coor_df = standardize_coordinates(test_adjusted_df)
 test_standardized_dir_df = standardize_direction(test_standardized_coor_df)
 test_ol_agg_df = get_o_line_stats(test_standardized_dir_df)
 test_merged_basetable = test_basetable.merge(test_ol_agg_df, on='PlayId')
 test_fe_df = more_feature_engineering(test_standardized_dir_df, pos_dict)
 test_fe_feature_df = test_fe_df[fe_feature_lst].drop_duplicates()
test_bc_v_area = get_bc_voronoi_area(test_fe_df, 'current')
 test_bc_v_area_end = get_bc_voronoi_area(test_fe_df, 'end')
 test_fe_feature_df['bc_v_area'] = test_bc_v_area[0]
test_fe_feature_df['bc_v_area_end'] = test_bc_v_area_end[0]
test_def_dist_end_df = get_defense_end(test_fe_df)
 test_fe_df[['X_std_end','Y_std_end']] = test_fe_df[['X_std_end','Y_std_end']].fillna(25)
 test_qb_location_df = get_qb_location(test_fe_df)
 test_def_time_to_spot_df = test_fe_df.loc[test_fe_df.IsOnOffense == False] \ 
.groupby('PlayId', as_index=False)[['TimeToBc','TimeToBcEnd']] \ 
.agg([np.min, np.mean, np.median, np.std])
 test_def_time_to_spot_df.columns = test_def_time_to_spot_df.columns.droplevel(0)
 test_def_time_to_spot_df.columns = ['TimeToBc' + c for c in test_def_time_to_spot_df.columns[0:4]] \
+ ['TimeToBcEnd' + c for c in test_def_time_to_spot_df.columns[4:]]
test_def_time_to_spot_df.reset_index(inplace=True)
 test_fe_defenders = test_fe_df.loc[test_fe_df.IsOnOffense == False]
test_fe_defenders['DistFromBcRank'] = test_fe_defenders.groupby('PlayId')['DistFromBc'].transform(lambda x: x.rank())
 test_three_def_df = test_fe_defenders.loc[test_fe_defenders.DistFromBcRank <= 3]
test_grouped_closest_dist_df = test_three_def_df.groupby('PlayId', as_index=False)['DistFromBc'].mean()
test_grouped_closest_dist_df.rename(columns={'DistFromBc':'Top3DistFromBcMean'}, inplace=True)
 test_merged_basetable = test_merged_basetable.merge(test_def_dist_end_df, how = 'left', on='PlayId')\ 
.merge(test_fe_feature_df, how = 'left', on=['GameId','PlayId'])\ 
.merge(test_qb_location_df, how = 'left', on = 'PlayId')\ 
.merge(test_def_time_to_spot_df, how = 'left', on = 'PlayId')\ 
.merge(test_grouped_closest_dist_df, how = 'left', on = 'PlayId' ).fillna(25)
 test_merged_basetable['Shotgun_Down_Distance'] = test_merged_basetable['Shotgun'] * test_merged_basetable['Down'] * test_merged_basetable['Distance']
test_merged_basetable['def_centroid'] = get_defense_centroid(test_fe_df)
 test_merged_basetable['def_centroid_X_std'] = test_merged_basetable.def_centroid.apply(lambda x: x[0])
 test_merged_basetable['def_centroid_Y_std'] = test_merged_basetable.def_centroid.apply(lambda x: x[1])
 test_merged_basetable['BcDistFromDefCent'] = np.sqrt(( test_merged_basetable.X_std_bc - test_merged_basetable.def_centroid_X_std)**2
+(test_merged_basetable.Y_std_bc - test_merged_basetable.def_centroid_Y_std)**2)
 test_merged_basetable['BcDistFromQb'] = np.sqrt(( test_merged_basetable.X_std_bc - test_merged_basetable.X_std_qb)**2
+(test_merged_basetable.Y_std_bc - test_merged_basetable.Y_std_qb)**2)
 test_merged_basetable['BcDirFromMid'] = abs(test_merged_basetable['BcDir'] - 90)
 test_merged_basetable['Down4'] = np.where(test_merged_basetable.Down == 4, 1, 0)
 test_merged_basetable['Quarter4'] = np.where(test_merged_basetable.Quarter == 4, 1, 0)
 test_merged_basetable.drop(['GameId','PlayId'] + cols_to_drop, axis=1, inplace=True)
 test_cluster_df = test_merged_basetable[cluster_features]
test_data_transformed = scale_data(test_cluster_df)
 test_merged_basetable['RunCluster'] = cluster_fit.predict(test_data_transformed)
 test_merged_basetable = test_merged_basetable[list(merged_basetable.drop(['GameId','PlayId','Yards'], axis=1 ).columns)].fillna(1)
 test_x_lst.append(test_merged_basetable)
 test_binary_array = np.array(test_merged_basetable.loc[:, binary_features])
 test_numeric_df = test_merged_basetable.drop(binary_features, axis = 1)
 test_numeric_scaled = scaler.transform(test_numeric_df)
 test_scaled_basetable = np.hstack(( test_numeric_scaled, test_binary_array))
nn_y_pred = nn_predict(test_scaled_basetable)
 nn_y_pred = np.clip(np.cumsum(nn_y_pred, axis=1), 0, 1 ).tolist() [0]
rf_y_pred = rf.predict(test_scaled_basetable)
 rf_y_pred = np.clip(np.cumsum(rf_y_pred, axis=1), 0, 1)
 blended_y_pred =(nn_y_pred + rf_y_pred)/ 2
preds_df = pd.DataFrame(data=[nn_y_pred], columns=sample_prediction_df.columns)
 env.predict(preds_df)
<set_options>
target_meter = 3
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models3 = []
for train_idx, valid_idx in kf.split(X_train, y_train, groups=get_groups(train_df, target_meter)) :
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
mindex = train_df[train_df.meter==target_meter].iloc[valid_idx,:].month.unique()
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=num_rounds,
lr=0.03, bf=0.9)
 y_valid_pred_total[valid_idx] = y_pred_valid
models3.append([mindex, model])
 gc.collect()
if debug:
break
oof3 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof3 * len(y_train)
 oof_total = oof_total / len(train_df)
 sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 del X_train, y_train
gc.collect()
pd.set_option('display.max_columns', 250)
 pd.set_option('display.max_rows', 20 )<load_from_csv>
print('oof0=', np.sqrt(oof0))
print('oof1=', np.sqrt(oof1))
print('oof2=', np.sqrt(oof2))
print('oof3=', np.sqrt(oof3))
print('total=', np.sqrt(oof_total))
train_df = pd.read_csv('.. /input/nfl-big-data-bowl-2020/train.csv', low_memory=False)
 print(train_df.shape)
<count_values>
del train_df, weather_train_df
gc.collect()
train_df.Season.value_counts()<feature_engineering>
print('loading...')
 test_df = pd.read_feather(root/'test.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 weather_test_df = weather_test_df.drop_duplicates(['timestamp', 'site_id'])
 set_local(weather_test_df)
 add_holiyday(weather_test_df)
 print('preprocessing building...')
 test_df['date'] = test_df['timestamp'].dt.date
preprocess(test_df)
 print('preprocessing weather...')
 weather_test_df = weather_test_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
weather_test_df.groupby('site_id' ).apply(lambda group: group.isna().sum())
 add_sg(weather_test_df)
 add_lag_feature(weather_test_df, window=3)
 add_lag_feature(weather_test_df, window=72)
 test_df['bid_cnt'] = test_df.building_id.map(bid_map)
 print('reduce mem usage...')
 test_df = reduce_mem_usage(test_df, use_float16=True)
 weather_test_df = reduce_mem_usage(weather_test_df, use_float16=True)
 gc.collect()
print(test_df.shape )
train_df.loc[train_df.HomeTeamAbbr.values == "ARI", 'HomeTeamAbbr'] = "ARZ"
train_df.loc[train_df.HomeTeamAbbr.values == "BAL", 'HomeTeamAbbr'] = "BLT"
train_df.loc[train_df.HomeTeamAbbr.values == "CLE", 'HomeTeamAbbr'] = "CLV"
train_df.loc[train_df.HomeTeamAbbr.values == "HOU", 'HomeTeamAbbr'] = "HST"
train_df.loc[train_df['Season'] == 2017, 'S'] =(train_df['S'][train_df['Season'] == 2017] - 2.4355)/1.2930 * 1.4551 + 2.7570
train_df.loc[train_df['Season'] == 2017, 'A'] =(train_df['A'][train_df['Season'] == 2017] - 1.5896)/0.8795 * 1.0603 + 1.7820<categorify>
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
reduce_mem_usage(sample_submission)
 print(sample_submission.shape )
def strtoseconds(txt):
txt = txt.split(':')
 ans = int(txt[0])*60 + int(txt[1])+ int(txt[2])/60
return ans
def map_weather(txt):
ans = 1
if pd.isna(txt):
ans = 0
if 'partly' in txt:
ans *= 0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*3
if 'sunny' in txt or 'sun' in txt:
return ans*2
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt or 'shower' in txt:
return -2*ans
if 'snow' in txt:
return -3*ans
return 0
def clean_wind_speed(windspeed):
ws = str(windspeed)
 if ws.isdigit() :
return int(ws)
 if '-' in ws:
return np.mean([int(i)for i in ws.split('-')])
 if ws.split(' ')[0].isdigit() :
return int(ws.split(' ')[0])
 if 'mph' in ws.lower() :
return int(ws.lower().split('mph')[0])
 else:
return 0
def clean_wind_direction(wind_direction):
wd = str(wind_direction ).upper()
if wd == 'N' or 'FROM S' in wd:
return 'north'
if wd == 'S' or 'FROM N' in wd:
return 'south'
if wd == 'W' or 'FROM E' in wd:
return 'west'
if wd == 'E' or 'FROM W' in wd:
return 'east'
if 'FROM SW' in wd or 'FROM SSW' in wd or 'FROM WSW' in wd:
return 'north east'
if 'FROM SE' in wd or 'FROM SSE' in wd or 'FROM ESE' in wd:
return 'north west'
if 'FROM NW' in wd or 'FROM NNW' in wd or 'FROM WNW' in wd:
return 'south east'
if 'FROM NE' in wd or 'FROM NNE' in wd or 'FROM ENE' in wd:
return 'south west'
if 'NW' in wd or 'NORTHWEST' in wd:
return 'north west'
if 'NE' in wd or 'NORTH EAST' in wd:
return 'north east'
if 'SW' in wd or 'SOUTHWEST' in wd:
return 'south west'
if 'SE' in wd or 'SOUTHEAST' in wd:
return 'south east'
return 'none'
def clean_turf(turf):
natural = ['Grass', 'Natural Grass', 'DD GrassMaster', 'SISGrass',
'Natural grass', 'Natural', 'Naturall Grass', 'natural grass', 'grass', 'natural']
artificial = ['Field Turf', 'Artificial', 'FieldTurf', 'UBU Speed Series-S5-M',
'A-Turf Titan', 'UBU Sports Speed S5-M', 'FieldTurf360', 'FieldTurf 360',
'Twenty-Four/Seven Turf', 'Artifical', 'Field turf', 'artificial']
if turf in natural:
return 0
elif turf in artificial:
return 1
else:
return 0.5
def euclid_distance(X, Y):
X = np.array(X)
 Y = np.array(Y)
 distance =(( X - Y)**2 ).sum()
return np.sqrt(distance)
 def get_height(txt):
feet = int(txt.split('-')[0])
 inch = int(txt.split('-')[1])
 return feet + np.round(inch/12, 2)
 def centeroidnp(arr):
length = arr.shape[0]
x = arr[:, 0]
y = arr[:, 1]
x_mean = x.mean()
y_mean = y.mean()
if not np.isfinite(x_mean):
x_mean = 0
if not np.isfinite(y_mean):
y_mean = 0
return x_mean, y_mean
def vertices_std(arr):
length = arr.shape[0]
x = arr[:, 0]
y = arr[:, 1]
x_std = x.std()
y_std = y.std()
if not np.isfinite(x_std):
x_std = 0
if not np.isfinite(y_std):
y_std = 0
return x_std, y_std
def cos_similarity(ax, ay, bx, by):
a = np.array([ax, ay] ).reshape(1, -1)
 b = np.array([bx, by] ).reshape(1, -1)
 return cosine_similarity(a, b)[0][0]
def area_triangle(a, b, c):
a = np.array(a)
 b = np.array(b)
 c = np.array(c)
 ab = b - a
ac = c - a
return 0.5*abs(ab[0]*ac[1] - ab[1]*ac[0])
 def standardize_vor(vertices):
v_std = []
for v in vertices:
if v[0] > 120 or v[0] < 0:
continue
if v[1] > 53.3 or v[1] < 0:
continue
v_std.append(list(v))
return np.array(v_std )<feature_engineering>
def create_X(test_df, target_meter):
target_test_df = test_df[test_df['meter'] == target_meter]
target_test_df = target_test_df.merge(building_meta_df, on='building_id', how='left')
 target_test_df = target_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left')
 X_test = target_test_df[feature_cols + category_cols + ['month']]
return X_test
def OffensePersonnelSplit(x):
dic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0, 'QB' : 0, 'RB' : 0, 'TE' : 0, 'WR' : 0, 'IRR': 0}
for xx in x.split(","):
xxs = xx.split(" ")
 if xxs[-1] not in dic.keys() :
dic['IRR'] = int(xxs[-2])
 else:
dic[xxs[-1]] = int(xxs[-2])
 return dic
def OffenseFormationMap(formation):
dic = {'SINGLEBACK': 0, 'SHOTGUN': 0, 'I_FORM': 0, 'PISTOL': 0, 'JUMBO': 0, 'WILDCAT': 0, 'EMPTY': 0}
if pd.isna(formation):
dic['EMPTY'] = 1
else:
fm = str(formation)
 if fm not in dic.keys() :
dic['EMPTY'] = 1
else:
dic[fm] = 1
return dic 
<feature_engineering>
def pred_all(X_test, models, batch_size=1000000):
iterations =(X_test.shape[0] + batch_size -1)// batch_size
print('iterations', iterations)
 y_test_pred_total = np.zeros(X_test.shape[0])
 for i,(mindex, model)in enumerate(models):
print(f'predicting {i}-th model')
 for k in tqdm(range(iterations)) :
y_pred_test = model.predict(X_test[k*batch_size:(k+1)*batch_size], num_iteration=model.best_iteration)
 y_test_pred_total[k*batch_size:(k+1)*batch_size] += y_pred_test
y_test_pred_total /= len(models)
 return y_test_pred_total
def pred(X_test, models, batch_size=1000000):
if predmode == 'valid':
print('valid pred')
 return pred_valid(X_test, models, batch_size=1000000)
 elif predmode == 'train':
print('train pred')
 return pred_train(X_test, models, batch_size=1000000)
 else:
print('all pred')
 return pred_all(X_test, models, batch_size=1000000 )
def DefensePersonnelSplit(x):
dic = {'DB': 0, 'DL': 0, 'LB': 0, 'OL': 0, 'IRR': 0}
for xx in x.split(','):
xxs = xx.split(' ')
 if xxs[-1] not in dic.keys() :
dic['IRR'] = int(xxs[-2])
 else:
dic[xxs[-1]] = int(xxs[-2])
 return dic<feature_engineering>
print(sample_submission.loc[test_df['meter'] == 0, 'meter_reading'].shape,np.expm1(y_test0 ).shape)
 print(sample_submission.loc[test_df['meter'] == 1, 'meter_reading'].shape,np.expm1(y_test1 ).shape)
 print(sample_submission.loc[test_df['meter'] == 2, 'meter_reading'].shape,np.expm1(y_test2 ).shape)
 print(sample_submission.loc[test_df['meter'] == 3, 'meter_reading'].shape,np.expm1(y_test3 ).shape)
 sample_submission.loc[test_df['meter'] == 0, 'meter_reading'] = np.expm1(y_test0)
 sample_submission.loc[test_df['meter'] == 1, 'meter_reading'] = np.expm1(y_test1)
 sample_submission.loc[test_df['meter'] == 2, 'meter_reading'] = np.expm1(y_test2)
 sample_submission.loc[test_df['meter'] == 3, 'meter_reading'] = np.expm1(y_test3 )
def get_data(train, mode='train'):
data_df = pd.DataFrame()
df = pd.Series()
playid_list = train['PlayId'].unique()
for playid in tqdm(playid_list):
df_playid = train.query('PlayId == @playid')
 df_playid['ToLeft'] = df_playid.PlayDirection == 'left'
for c in df_playid.columns:
if df_playid[c].isnull().any() :
df_playid[c] = df_playid[c].fillna(df_playid[c].mean())

df['GameId'] = df_playid['GameId'].iloc[0]
df['PlayId'] = int(playid)
 play_direction = df_playid['PlayDirection'].iloc[0]
if play_direction == 'left':
df_playid['X'] = 120.0 - df_playid['X']
df_playid['Y'] = 53.3 - df_playid['Y']
df_playid['Dir_rad'] = np.mod(90 - df_playid.Dir, 360)* math.pi/180.0
if play_direction == 'left':
df_playid['Dir_rad'] = np.mod(np.pi + df_playid['Dir_rad'], 2*np.pi)
 df_playid['Orientation_rad'] = np.mod(df_playid.Orientation, 360)* math.pi/180.0
df_playid.loc[df_playid.Season >= 2018, 'Orientation_rad'] = np.mod(df_playid.loc[df_playid.Season >= 2018, 'Orientation']-90, 360)* math.pi/180.0
if play_direction == 'left':
df_playid['Orientation_rad'] = np.mod(np.pi + df_playid['Orientation_rad'], 2*np.pi)
 if df_playid['PossessionTeam'].iloc[0] == df_playid['FieldPosition'].iloc[0]:
df_playid['YardLine'] = df_playid['YardLine']
else :
df_playid['YardLine'] = 100 - df_playid['YardLine']

 df['GameWeather'] = map_weather(str(df_playid['GameWeather'].iloc[0] ).lower())
 df['Turf'] = clean_turf(df_playid['Turf'].iloc[0])
 if pd.isna(df_playid['Humidity'].iloc[0]):
df['Humidity'] = humidity_avg
else:
df['Humidity'] = df_playid['Humidity'].iloc[0]
if pd.isna(df_playid['Temperature'].iloc[0]):
df['Temperature'] = temp_avg
else:
df['Temperature'] = df_playid['Temperature'].iloc[0]
df['GameClock'] = strtoseconds(df_playid['GameClock'].iloc[0])
 if df_playid['Season'].iloc[0] < 2018:
df['Season'] = 0
else:
df['Season'] = 1
df['Quarter'] = df_playid['Quarter'].iloc[0]
df['Down'] = df_playid['Down'].iloc[0]

 rusher = df_playid['NflIdRusher'].iloc[0]
offense_team = df_playid[df_playid['NflId'] == rusher]['Team'].iloc[0]
df_offense = df_playid.query('Team == @offense_team')
 if offense_team == 'home':
df['ScoreDiff'] = df_playid['HomeScoreBeforePlay'].iloc[0] - df_playid['VisitorScoreBeforePlay'].iloc[0]
else:
df['ScoreDiff'] = df_playid['VisitorScoreBeforePlay'].iloc[0] - df_playid['HomeScoreBeforePlay'].iloc[0]
off_centroid =(df_offense['X'].mean() , df_offense['Y'].mean())
 off_Ori = df_offense['Orientation_rad'].mean()
off_Dir = df_offense['Dir_rad'].mean()
rusher_Ori = df_offense[df_offense['NflId'] == rusher]['Orientation_rad'].iloc[0]
rusher_Dir = df_offense[df_offense['NflId'] == rusher]['Dir_rad'].iloc[0]
rusher_X = df_playid[df_playid['NflId'] == rusher]['X'].iloc[0]
rusher_Y = df_playid[df_playid['NflId'] == rusher]['Y'].iloc[0]
rusher_S = df_offense[df_offense['NflId'] == rusher]['S'].iloc[0]
rusher_A = df_offense[df_offense['NflId'] == rusher]['A'].iloc[0]
rusher_Dis = df_playid[df_playid['NflId'] == rusher]['Dis'].iloc[0]
rusher_Age = df_playid['Season'].iloc[0] \
- int(df_playid[df_playid['NflId'] == rusher]['PlayerBirthDate'].iloc[0].split('/')[-1])
 df_offense['EuclidDistanceFromRusher'] = [euclid_distance([rusher_X, rusher_Y],
[df_offense['X'].iloc[i],
df_offense['Y'].iloc[i]])for i in range(len(df_offense)) ]
off_closest_EuclidDis_from_rusher = df_offense[df_offense['NflId'] != rusher]['EuclidDistanceFromRusher'].min()
closest_Off_from_rusher = df_offense[df_offense['EuclidDistanceFromRusher'] == off_closest_EuclidDis_from_rusher]['NflId'].iloc[0]
closest_Off_from_rusher_Dir = df_offense[df_offense['EuclidDistanceFromRusher'] == off_closest_EuclidDis_from_rusher]['Dir_rad'].iloc[0]
df['RusherClosestOffenseX'] = df_offense[df_offense['NflId'] == closest_Off_from_rusher]['X'].iloc[0]
df['RusherClosestOffenseY'] = df_offense[df_offense['NflId'] == closest_Off_from_rusher]['Y'].iloc[0]
df['RusherClosestOffenseS'] = df_offense[df_offense['NflId'] == closest_Off_from_rusher]['S'].iloc[0]
df['RusherClosestOffenseA'] = df_offense[df_offense['NflId'] == closest_Off_from_rusher]['A'].iloc[0]
df['RusherClosestOffenseDir'] = df_offense[df_offense['NflId'] == closest_Off_from_rusher]['Dir_rad'].iloc[0]
df['RusherClosestOffenseDis'] = df_offense[df_offense['NflId'] == closest_Off_from_rusher]['Dis'].iloc[0]
df['RusherClosestOffenseS_Horizontal'] = df['RusherClosestOffenseS'] * np.cos(closest_Off_from_rusher_Dir)
 df['RusherClosestOffenseS_Vertical'] = df['RusherClosestOffenseS'] * np.sin(closest_Off_from_rusher_Dir)
 df['RusherClosestOffenseA_Horizontal'] = df['RusherClosestOffenseA'] * np.cos(closest_Off_from_rusher_Dir)
 df['RusherClosestOffenseA_Vertical'] = df['RusherClosestOffenseA'] * np.sin(closest_Off_from_rusher_Dir)
 closest_top3_id_off = df_offense.sort_values(by = 'EuclidDistanceFromRusher')['NflId'][:3]
df['RusherSecondClosestOffenseX'] = df_offense[df_offense['NflId'] == closest_top3_id_off.iloc[1]]['X'].iloc[0]
df['RusherSecondClosestOffenseY'] = df_offense[df_offense['NflId'] == closest_top3_id_off.iloc[1]]['Y'].iloc[0]
df['RusherSecondClosestOffenseS'] = df_offense[df_offense['NflId'] == closest_top3_id_off.iloc[1]]['S'].iloc[0]
df['RusherSecondClosestOffenseA'] = df_offense[df_offense['NflId'] == closest_top3_id_off.iloc[1]]['A'].iloc[0]
df['RusherSecondClosestOffenseDir'] = df_offense[df_offense['NflId'] == closest_top3_id_off.iloc[1]]['Dir_rad'].iloc[0]
df['RusherSecondClosestOffenseDis'] = df_offense[df_offense['NflId'] == closest_top3_id_off.iloc[1]]['Dis'].iloc[0]
df['RusherThirdClosestOffenseX'] = df_offense[df_offense['NflId'] == closest_top3_id_off.iloc[2]]['X'].iloc[0]
df['RusherThirdClosestOffenseY'] = df_offense[df_offense['NflId'] == closest_top3_id_off.iloc[2]]['Y'].iloc[0]
df['RusherThirdClosestOffenseS'] = df_offense[df_offense['NflId'] == closest_top3_id_off.iloc[2]]['S'].iloc[0]
df['RusherThirdClosestOffenseA'] = df_offense[df_offense['NflId'] == closest_top3_id_off.iloc[2]]['A'].iloc[0]
df['RusherThirdClosestOffenseDir'] = df_offense[df_offense['NflId'] == closest_top3_id_off.iloc[2]]['Dir_rad'].iloc[0]
df['RusherThirdClosestOffenseDis'] = df_offense[df_offense['NflId'] == closest_top3_id_off.iloc[2]]['Dis'].iloc[0]
off_formation_dic = OffenseFormationMap(df_offense['OffenseFormation'].iloc[0])
 for fm in off_formation_dic.items() :
df['Formation_' + fm[0]] = fm[1]
df['AvgOffenseWeight'] = df_offense['PlayerWeight'].mean()
df['AvgOffenseHeight'] = df_offense['PlayerHeight'].apply(get_height ).mean()
df['AvgOffenseDis'] = df_offense['Dis'].mean()
df['StdOffenseDis'] = df_offense['Dis'].std()
df['MedOffenseDis'] = df_offense['Dis'].median()
df['AvgOffenseX'] = df_offense['X'].mean()
df['AvgOffenseY'] = df_offense['Y'].mean()
df['StdOffenseX'] = df_offense['X'].std()
df['StdOffenseY'] = df_offense['Y'].std()
df['AvgOffenseSpeed'] = df_offense['S'].mean()
df['AvgOffenseA'] = df_offense['A'].mean()
df['StdOffenseSpeed'] = df_offense['S'].std()
df['StdOffenseA'] = df_offense['A'].std()
df['MedOffenseSpeed'] = df_offense['S'].median()
df['MedOffenseA'] = df_offense['A'].median()
df['OffenseSVertical'] = df['AvgOffenseSpeed'] * np.sin(off_Dir)
 df['OffenseSHorizontal'] = df['AvgOffenseSpeed'] * np.cos(off_Dir)
 df['OffenseSVerticalMed'] = df['MedOffenseSpeed'] * np.sin(off_Dir)
 df['OffenseSHorizontalMed'] = df['MedOffenseSpeed'] * np.cos(off_Dir)
 df['OffenseAVertical'] = df['AvgOffenseA'] * np.sin(off_Dir)
 df['OffenseAHorizontal'] = df['AvgOffenseA'] * np.cos(off_Dir)
 df['OffenseAVerticalMed'] = df['MedOffenseA'] * np.sin(off_Dir)
 df['OffenseAHorizontalMed'] = df['MedOffenseA'] * np.cos(off_Dir)
 df['AvgOffenseDir'] = off_Dir
df['StdOffenseDir'] = df_offense['Dir_rad'].std()
df['OffenseSpreadX'] = df_offense['X'].max() - df_offense['X'].min()
df['OffenseSpreadY'] = df_offense['Y'].max() - df_offense['Y'].min()
df['OffDistanceFromYardline'] = abs(off_centroid[0] - df_playid['YardLine'].iloc[0])
 df['OffDistanceFromYardlineStd'] = abs(df_offense['X'] - df_playid['YardLine'].iloc[0] ).std()
df['OffDistanceFromYardlineMin'] = abs(df_offense['X'] - df_playid['YardLine'].iloc[0] ).min()
df['OffDistanceFromYardlineMed'] = abs(df_offense['X'] - df_playid['YardLine'].iloc[0] ).median()
offense_points = []
for idx, nflid in enumerate(df_offense['NflId']):
x = df_offense[df_offense['NflId'] == nflid]['X'].iloc[0]
y = df_offense[df_offense['NflId'] == nflid]['Y'].iloc[0]
offense_points.append([x, y])
 if nflid == rusher:
rusher_vor_region_idx = idx
offense_points = np.array(offense_points)
 offense_vor = Voronoi(offense_points)
 offense_vor_region = offense_vor.regions
rusher_vor_region = offense_vor_region[rusher_vor_region_idx]
rusher_vor_region = np.clip(np.array(rusher_vor_region), 0, None)
 rusher_vor_region_vertices = [offense_vor.vertices[v] for v in rusher_vor_region]
rusher_vor_region_vertices = standardize_vor(rusher_vor_region_vertices)
 if len(rusher_vor_region_vertices)< 1:
rusher_vor_closeX = rusher_X
rusher_vor_farX = rusher_X
rusher_vor_est_radius = 0
else:
rusher_vor_closeX = rusher_vor_region_vertices[np.argmin([abs(rusher_X - v[0])for v in rusher_vor_region_vertices])][0]
rusher_vor_farX = rusher_vor_region_vertices[np.argmax([abs(rusher_X - v[0])for v in rusher_vor_region_vertices])][0]
rusher_vor_est_radius = np.array([euclid_distance([rusher_X, rusher_Y], [v[0], v[1]])for v in rusher_vor_region_vertices] ).mean()
df['RusherVorRegionCloseX'] = rusher_vor_closeX
df['RusherVorRegionFarX'] = rusher_vor_farX
df['RusherVorCloseX_Yardline'] = rusher_vor_closeX - df_offense['YardLine'].iloc[0]
df['RusherVorFarX_Yardline'] = rusher_vor_farX - df_offense['YardLine'].iloc[0]
df['RusherVorArea'] = 2*np.pi*(rusher_vor_est_radius)**2
off_vor_v = standardize_vor(offense_vor.vertices)
 df['OffenseVorVerticesLen'] = len(off_vor_v)
 df['OffenseVorVerticesCentX'] = centeroidnp(off_vor_v)[0]
df['OffenseVorVerticesCentY'] = centeroidnp(off_vor_v)[1]
df['OffenseVorVerticesStdX'] = vertices_std(off_vor_v)[0]
df['OffenseVorVerticesStdY'] = vertices_std(off_vor_v)[1]
rusher_off_vor_distance = [euclid_distance([rusher_X, rusher_Y], [v[0], v[1]])for v in off_vor_v]
rusher_off_vor_distance.sort()
df['RusherVorVMinDistance'] = rusher_off_vor_distance[0]
df['RusherVorV2ndMinDistance'] = rusher_off_vor_distance[1]
df['IsRusherRB'] = int(df_offense[df_offense['NflId'] == rusher]['Position'] == 'RB')
 df['RusherPosFromYardLine'] = df_playid['YardLine'].iloc[0] - rusher_X
df['RusherDir'] = rusher_Dir
df['RusherOri'] = rusher_Ori
df['RusherOriDirDiff'] = abs(rusher_Ori - rusher_Dir)
 df['RusherSpeed'] = rusher_S
df['RusherAcceleration'] = rusher_A
df['RusherSVertical'] = df_offense[df_offense['NflId'] == rusher]['S'].iloc[0] * np.sin(rusher_Dir)
 df['RusherSHorizontal'] = df_offense[df_offense['NflId'] == rusher]['S'].iloc[0] * np.cos(rusher_Dir)
 df['RusherAVertical'] = df_offense[df_offense['NflId'] == rusher]['A'].iloc[0] * np.sin(rusher_Dir)
 df['RusherAHorizontal'] = df_offense[df_offense['NflId'] == rusher]['A'].iloc[0] * np.cos(rusher_Dir)
 df['RusherDis'] = rusher_Dis
df['RusherAge'] = rusher_Age
df['RusherWeight'] = df_offense[df_offense['NflId'] == rusher]['PlayerWeight'].iloc[0]
df['RusherHeight'] = get_height(df_offense[df_offense['NflId'] == rusher]['PlayerHeight'].iloc[0])
 df['RusherPower'] = 0.5 * df['RusherWeight'] *(rusher_S**2)
 df['RusherPowerHorizontal'] = 0.5 * df['RusherWeight'] *(rusher_S * np.cos(rusher_Dir)) **2
off_pos = OffensePersonnelSplit(df_playid['OffensePersonnel'].iloc[0])
 for pos in off_pos.items() :
df['Offense_' + pos[0]] = pos[1]
df_playid['TimeHandoff'] = pd.to_datetime(df_playid['TimeHandoff'])
 df_playid['TimeSnap'] = pd.to_datetime(df_playid['TimeSnap'])
 thts =(df_playid['TimeHandoff'].dt.second - df_playid['TimeSnap'].dt.second ).iloc[0]
if thts < 0:
thts = thts + 60
df['TimeTillHandoff'] = thts
df['Distance'] = df_playid['Distance'].iloc[0]
df['YardLine'] = df_playid['YardLine'].iloc[0]
month = int(df_playid['TimeSnap'].dt.month.iloc[0])
 month_dic = {9:0, 10:0, 11:0, 12:0, 'else':0}
if month in month_dic.keys() :
month_dic[month] = 1
else:
month_dic['else'] = 1
for mon in month_dic.keys() :
df['Month'+str(mon)] = month_dic[mon]

 df_defense = df_playid.query('Team != @offense_team')
 df_defense['DisXFromRusher'] = df_defense['X'] - rusher_X
df_defense['DisYFromRusher'] = df_defense['Y'] - rusher_Y
df_defense['DisFromRusher'] = np.sqrt(df_defense['DisXFromRusher']**2 + df_defense['DisYFromRusher']**2)
 def_centroid = [df_defense['X'].mean() , df_defense['Y'].mean() ]
def_Ori = df_defense['Orientation_rad'].mean()
def_Dir = df_defense['Dir_rad'].mean()
df['AvgDefenseWeight'] = df_defense['PlayerWeight'].mean()
df['AvgDefenseHeight'] = df_defense['PlayerHeight'].apply(get_height ).mean()
df['AvgDefenseX'] = df_defense['X'].mean()
df['AvgDefenseY'] = df_defense['Y'].mean()
df['StdDefenseX'] = df_defense['X'].std()
df['StdDefenseY'] = df_defense['Y'].std()
df['AvgDefenseSpeed'] = df_defense['S'].mean()
df['AvgDefenseA'] = df_defense['A'].mean()
df['StdDefenseSpeed'] = df_defense['S'].std()
df['StdDefenseA'] = df_defense['A'].std()
df['MedDefenseSpeed'] = df_defense['S'].median()
df['MedDefenseA'] = df_defense['A'].median()
df['DefenseSVertical'] = df['AvgDefenseSpeed'] * np.sin(def_Dir)
 df['DefenseSHorizontal'] = df['AvgDefenseSpeed'] * np.cos(def_Dir)
 df['DefenseSVertical'] = df['MedDefenseSpeed'] * np.sin(def_Dir)
 df['DefenseSHorizontal'] = df['MedDefenseSpeed'] * np.cos(def_Dir)
 df['DefenseAVertical'] = df['AvgDefenseA'] * np.sin(def_Dir)
 df['DefenseAHorizontal'] = df['AvgDefenseA'] * np.cos(def_Dir)
 df['DefenseAVertical'] = df['MedDefenseA'] * np.sin(def_Dir)
 df['DefenseAHorizontal'] = df['MedDefenseA'] * np.cos(def_Dir)
 df['AvgDefenseDis'] = df_defense['Dis'].mean()
df['StdDefenseDis'] = df_defense['Dis'].std()
df['MedDefenseDis'] = df_defense['Dis'].median()
df['AvgDefenseDir'] = def_Dir
df['StdDefenseDir'] = df_defense['Dir_rad'].std()
df['MedDefenseDir'] = df_defense['Dir_rad'].median()
df['AvgDisFromRusher'] = df_defense['DisFromRusher'].mean()
df['StdDisFromRusher'] = df_defense['DisFromRusher'].std()
df['MinDisFromRusher'] = df_defense['DisFromRusher'].min()
df['MaxDisFromRusher'] = df_defense['DisFromRusher'].max()
df['MedDisFromRusher'] = df_defense['DisFromRusher'].median()
df['DefDistanceFromYardline'] = abs(def_centroid[0] - df_playid['YardLine'].iloc[0])
 df['DefDistanceFromYardlineStd'] = abs(df_defense['X'] - df_playid['YardLine'].iloc[0] ).std()
df['DefDistanceFromYardlineMin'] = abs(df_defense['X'] - df_playid['YardLine'].iloc[0] ).min()
df['DefDistanceFromYardlineMed'] = abs(df_defense['X'] - df_playid['YardLine'].iloc[0] ).median()
def_pos = DefensePersonnelSplit(df_playid['DefensePersonnel'].iloc[0])
 for pos in def_pos.items() :
df['Def_' + pos[0]] = pos[1]
d_in_box = df_defense['DefendersInTheBox'].iloc[0]
if pd.isna(d_in_box):
df['DefendersInTheBox'] = 7
else:
df['DefendersInTheBox'] = d_in_box
df['DefenseSpreadX'] = df_defense['X'].max() - df_defense['X'].min()
df['DefenseSpreadY'] = df_defense['Y'].max() - df_defense['Y'].min()
defense_points = []
for x, y in zip(df_defense['X'], df_defense['Y']):
defense_points.append([x, y])
 defense_points = np.array(defense_points)
 defense_vor = Voronoi(defense_points)
 def_vor_v = standardize_vor(defense_vor.vertices)
 df['DefenseVorVerticesLen'] = len(def_vor_v)
 df['DefenseVorVerticesCentX'] = centeroidnp(def_vor_v)[0]
df['DefenseVorVerticesCentY'] = centeroidnp(def_vor_v)[1]
df['DefenseVorVerticesStdX'] = vertices_std(def_vor_v)[0]
df['DefenseVorVerticesStdY'] = vertices_std(def_vor_v)[1]

 if rusher_Dir > 0.5*math.pi:
num_off_on_rusher_dir = len(df_offense[df_offense['NflId'] != rusher][df_offense['Y'] < rusher_Y])
 num_def_on_rusher_dir = len(df_defense[df_defense['NflId'] != rusher][df_defense['Y'] < rusher_Y])
 else:
num_off_on_rusher_dir = len(df_offense[df_offense['NflId'] != rusher][df_offense['Y'] >= rusher_Y])
 num_def_on_rusher_dir = len(df_defense[df_defense['NflId'] != rusher][df_defense['Y'] >= rusher_Y])
 df['NumOffonRusherDir'] = num_off_on_rusher_dir
df['NumDefonRusherDir'] = num_def_on_rusher_dir
df['RusherVSDefCentroid'] = euclid_distance([rusher_X, rusher_Y], def_centroid)
 df['RusherVSOffVoronoiCentroid'] = euclid_distance([rusher_X, rusher_Y], [centeroidnp(off_vor_v)[0], centeroidnp(off_vor_v)[1]])
 rusher_vs_offvor = []
for i,v in enumerate(off_vor_v):
rusher_vs_offvor.append(euclid_distance([rusher_X, rusher_Y], off_vor_v[i]))
df['RusherVSOffVoronoiStd'] = np.array(rusher_vs_offvor ).std()
df['RusherVSOffVoronoiMed'] = np.median(np.array(rusher_vs_offvor))
df['RusherVSOffVoronoiMin'] = np.array(rusher_vs_offvor ).min()
df['RusherVSOffVoronoiMax'] = np.array(rusher_vs_offvor ).max()
df['RusherVSDefCentroid'] = euclid_distance([rusher_X, rusher_Y], def_centroid)
 df['RusherVSDefVoronoiCentroid'] = euclid_distance([rusher_X, rusher_Y], [centeroidnp(def_vor_v)[0], centeroidnp(def_vor_v)[1]])
 rusher_vs_defvor = []
for i,v in enumerate(def_vor_v):
rusher_vs_defvor.append(euclid_distance([rusher_X, rusher_Y], def_vor_v[i]))
df['RusherVSDefVoronoiStd'] = np.array(rusher_vs_defvor ).std()
df['RusherVSDefVoronoiMed'] = np.median(np.array(rusher_vs_defvor))
df['RusherVSDefVoronoiMin'] = np.array(rusher_vs_defvor ).min()
df['RusherVSDefVoronoiMax'] = np.array(rusher_vs_defvor ).max()
df_defense['EuclidDistanceFromRusher'] = [euclid_distance([rusher_X, rusher_Y],
[df_defense[df_defense['NflId'] != rusher]['X'].iloc[i],
df_defense[df_defense['NflId'] != rusher]['Y'].iloc[i]])for i in range(len(df_defense[df_defense['NflId'] != rusher])) ]
df_defense['DisYfromRusher'] = abs(df_defense[df_defense['NflId'] != rusher]['Y'] - rusher_Y)
 closest_EuclidDis_from_rusher = df_defense['EuclidDistanceFromRusher'].min()
closest_Def_from_rusher = df_defense[df_defense['EuclidDistanceFromRusher'] == closest_EuclidDis_from_rusher]['NflId'].iloc[0]
closest_Def_from_rusher_Dir = df_defense[df_defense['EuclidDistanceFromRusher'] == closest_EuclidDis_from_rusher]['Dir_rad'].iloc[0]
df['RusherClosestDefenseX'] = df_defense[df_defense['NflId'] == closest_Def_from_rusher]['X'].iloc[0]
df['RusherClosestDefenseY'] = df_defense[df_defense['NflId'] == closest_Def_from_rusher]['Y'].iloc[0]
df['RusherClosestDefenseS'] = df_defense[df_defense['NflId'] == closest_Def_from_rusher]['S'].iloc[0]
df['RusherClosestDefenseA'] = df_defense[df_defense['NflId'] == closest_Def_from_rusher]['A'].iloc[0]
df['RusherClosestDefenseDir'] = df_defense[df_defense['NflId'] == closest_Def_from_rusher]['Dir_rad'].iloc[0]
df['RusherClosestDefenseDis'] = df_defense[df_defense['NflId'] == closest_Def_from_rusher]['Dis'].iloc[0]
df['RusherClosestDefenseS_Horizontal'] = df['RusherClosestDefenseS'] * np.cos(closest_Def_from_rusher_Dir)
 df['RusherClosestDefenseS_Vertical'] = df['RusherClosestDefenseS'] * np.sin(closest_Def_from_rusher_Dir)
 df['RusherClosestDefenseA_Horizontal'] = df['RusherClosestDefenseA'] * np.cos(closest_Def_from_rusher_Dir)
 df['RusherClosestDefenseA_Vertical'] = df['RusherClosestDefenseA'] * np.sin(closest_Def_from_rusher_Dir)
 closest_top3_id = df_defense.sort_values(by = 'DisFromRusher')['NflId'][:3]
df['RusherSecondClosestDefenseX'] = df_defense[df_defense['NflId'] == closest_top3_id.iloc[1]]['X'].iloc[0]
df['RusherSecondClosestDefenseY'] = df_defense[df_defense['NflId'] == closest_top3_id.iloc[1]]['Y'].iloc[0]
df['RusherSecondClosestDefenseS'] = df_defense[df_defense['NflId'] == closest_top3_id.iloc[1]]['S'].iloc[0]
df['RusherSecondClosestDefenseA'] = df_defense[df_defense['NflId'] == closest_top3_id.iloc[1]]['A'].iloc[0]
df['RusherSecondClosestDefenseDir'] = df_defense[df_defense['NflId'] == closest_top3_id.iloc[1]]['Dir_rad'].iloc[0]
df['RusherSecondClosestDefenseDis'] = df_defense[df_defense['NflId'] == closest_top3_id.iloc[1]]['Dis'].iloc[0]
df['RusherThirdClosestDefenseX'] = df_defense[df_defense['NflId'] == closest_top3_id.iloc[2]]['X'].iloc[0]
df['RusherThirdClosestDefenseY'] = df_defense[df_defense['NflId'] == closest_top3_id.iloc[2]]['Y'].iloc[0]
df['RusherThirdClosestDefenseS'] = df_defense[df_defense['NflId'] == closest_top3_id.iloc[2]]['S'].iloc[0]
df['RusherThirdClosestDefenseA'] = df_defense[df_defense['NflId'] == closest_top3_id.iloc[2]]['A'].iloc[0]
df['RusherThirdClosestDefenseDir'] = df_defense[df_defense['NflId'] == closest_top3_id.iloc[2]]['Dir_rad'].iloc[0]
df['RusherThirdClosestDefenseDis'] = df_defense[df_defense['NflId'] == closest_top3_id.iloc[2]]['Dis'].iloc[0]
first_closestDef_coordinate = [df['RusherClosestDefenseX'], df['RusherClosestDefenseY']]
second_closestDef_coordinate = [df['RusherSecondClosestDefenseX'], df['RusherSecondClosestDefenseY']]
third_closestDef_coordinate = [df['RusherThirdClosestDefenseX'], df['RusherThirdClosestDefenseY']]
df['ClosestDefenceTriangle'] = area_triangle(first_closestDef_coordinate, second_closestDef_coordinate, third_closestDef_coordinate)
 closest_def_Dir = df_defense[df_defense['NflId'] == closest_Def_from_rusher]['Dir_rad'].iloc[0]
closest_def_S = df_defense[df_defense['NflId'] == closest_Def_from_rusher]['S'].iloc[0]
df['RusherClosestDefenseDir'] = closest_def_Dir
df['Rusher-DefenderMinDistance'] = euclid_distance([rusher_X, rusher_Y],
[df['RusherClosestDefenseX'], df['RusherClosestDefenseY']])
 if closest_def_S < 0.01:
df['EstTimeTackled'] = 10
else:
df['EstTimeTackled'] = df['Rusher-DefenderMinDistance']/(closest_def_S)
 df_defense['SVertical'] = df_defense['S'] * np.sin(def_Dir)
 df_defense['SHorizontal'] = df_defense['S'] * np.cos(def_Dir)
 df['RusherDefender_CosSimilarity'] = cos_similarity(df['RusherSHorizontal'], df['RusherSVertical'],
df['RusherClosestDefenseS']*np.sin(closest_def_Dir), df['RusherClosestDefenseS']*np.cos(closest_def_Dir))

 if mode == 'train':
df['Yards'] = df_playid['Yards'].iloc[0]
data_df = data_df.append(df, ignore_index=True)
 if mode == 'train':
data_df = data_df.astype({'PlayId': int, 'GameId': int, 'Yards': int})
 else:
data_df = data_df.astype({'PlayId': int, 'GameId': int})
 return data_df<prepare_x_and_y>
sample_submission.loc[(test_df.building_id.isin(site_0_bids)) &(test_df.meter==0), 'meter_reading'] = sample_submission[(test_df.building_id.isin(site_0_bids)) &(test_df.meter==0)]['meter_reading'] * 3.4118
warnings.filterwarnings('ignore')
 X = get_data(train_df )<data_type_conversions>
if clip0:
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0
for c in X.columns:
if(X[c].dtype != int):
X[c] = X[c].astype('float32')
 for c in X.columns:
if X[c].isnull().any() :
print(c)
 print(X[c].mean())
 X[c] = X[c].fillna(X[c].mean() )<count_missing_values>
if not debug:
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )
X.isnull().any()<drop_column>
leak_df = pd.read_feather('.. /input/ashrae-leak-data-station/leak.feather')
 print(leak_df.duplicated().sum())
 print(leak_df.meter.value_counts())
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[(leak_df.timestamp.dt.year > 2016)&(leak_df.timestamp.dt.year < 2019)]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
leak_df = leak_df[leak_df.building_id!=245]
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0
test_df['pred'] = sample_submission.meter_reading
leak_df = leak_df.merge(test_df[['building_id', 'meter', 'timestamp', 'pred', 'row_id']], left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 leak_df = leak_df.merge(building_meta_df[['building_id', 'site_id']], on='building_id', how='left' )
yards_train = X['Yards']
X_gameid = pd.DataFrame(X['GameId'], columns=['GameId'])
 X_features = X.drop(['Yards', 'PlayId', 'GameId'], axis=1)
 print(X_features.shape)
 print(yards_train.shape)
 X_gameid.head()<prepare_x_and_y>
leak_df['pred_l1p'] = np.log1p(leak_df.pred)
 leak_df['meter_reading_l1p'] = np.log1p(leak_df.meter_reading)
 sns.distplot(leak_df.pred_l1p)
 sns.distplot(leak_df.meter_reading_l1p)
 leak_score = np.sqrt(mean_squared_error(leak_df.pred_l1p, leak_df.meter_reading_l1p))
y_train = np.zeros(( yards_train.shape[0], 199))
for idx, target in enumerate(list(yards_train)) :
y_train[idx][99 + int(target)] = 1
y_train.shape<normalization>
leak_df = leak_df[['meter_reading', 'row_id']].set_index('row_id' ).dropna()
sample_submission.loc[leak_df.index, 'meter_reading'] = leak_df['meter_reading']
scaler = StandardScaler()
X_features_sc = scaler.fit_transform(X_features)
 X_features = pd.DataFrame(X_features_sc, index = X_features.index, columns = X_features.columns)
 X_features.head()<init_hyperparams>
if not debug:
sample_submission.to_csv('submission_ucf_replaced.csv', index=False, float_format='%.4f' )
best_params_lgb = {'lambda_l1': 0.13413394854686794,
'lambda_l2': 0.0009122197743451751,
'num_leaves': 44,
'feature_fraction': 0.4271070738920401,
'bagging_fraction': 0.9999128827046064,
'bagging_freq': 3,
"learning_rate": 0.005,
'min_child_samples': 43,
'objective': 'regression',
'metric': 'mae',
'verbosity': -1,
'boosting_type': 'gbdt',
"boost_from_average" : False,
'random_state': 42}<train_model>
print('oof score meter0 =', np.sqrt(oof0))
print('oof score meter1 =', np.sqrt(oof1))
print('oof score meter2 =', np.sqrt(oof2))
print('oof score meter3 =', np.sqrt(oof3))
print('oof score total =', np.sqrt(oof_total))
<define_variables><EOS> 
print('LV score=', leak_score )