Kaggle competition: CareerCon 2019 - Help Navigate Robots
<SOS> metric: categorizationaccuracy Kaggle data source: career-con-2019<create_dataframe>
warnings.filterwarnings('ignore' )
<train_model>
%%time
X_train = pd.read_csv(os.path.join(PATH, 'X_train.csv'))
X_test = pd.read_csv(os.path.join(PATH, 'X_test.csv'))
y_train = pd.read_csv(os.path.join(PATH, 'y_train.csv'))
<categorify>
print("Train X: {}
Train y: {}
Test X: {}".format(X_train.shape, y_train.shape, X_test.shape))
<load_from_csv>
def missing_data(data):
total = data.isnull().sum()
percent =(data.isnull().sum() /data.isnull().count() *100)
 tt = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 types = []
for col in data.columns:
dtype = str(data[col].dtype)
 types.append(dtype)
 tt['Types'] = types
return(np.transpose(tt))
<load_from_csv>
missing_data(X_train )
<count_unique_values>
missing_data(X_test )
<import_modules>
missing_data(y_train )
import numpy as np
import matplotlib.pyplot as plt
from scipy.spatial import Voronoi
from shapely.ops import polygonize,unary_union
from shapely.geometry import LineString, MultiPolygon, MultiPoint, Point, Polygon
import pandas as pd
from scipy.spatial import Voronoi, voronoi_plot_2d
import math
import matplotlib.patches as patches<set_options>
def quaternion_to_euler(x, y, z, w):
t0 = +2.0 *(w * x + y * z)
 t1 = +1.0 - 2.0 *(x * x + y * y)
 X = math.atan2(t0, t1)
 t2 = +2.0 *(w * y - z * x)
 t2 = +1.0 if t2 > +1.0 else t2
t2 = -1.0 if t2 < -1.0 else t2
Y = math.asin(t2)
 t3 = +2.0 *(w * z + x * y)
 t4 = +1.0 - 2.0 *(y * y + z * z)
 Z = math.atan2(t3, t4)
 return X, Y, Z
TRAIN_ABLE_FALSE=True
if TRAIN_ABLE_FALSE:
os.environ['CUDA_VISIBLE_DEVICES'] = "1"
TRAIN_OFFLINE = False
pd.set_option('display.max_columns', 50)
 pd.set_option('display.max_rows', 150 )<set_options>
def perform_euler_factors_calculation(df):
df['total_angular_velocity'] = np.sqrt(np.square(df['angular_velocity_X'])+ np.square(df['angular_velocity_Y'])+ np.square(df['angular_velocity_Z']))
df['total_linear_acceleration'] = np.sqrt(np.square(df['linear_acceleration_X'])+ np.square(df['linear_acceleration_Y'])+ np.square(df['linear_acceleration_Z']))
df['total_xyz'] = np.sqrt(np.square(df['orientation_X'])+ np.square(df['orientation_Y'])+
np.square(df['orientation_Z']))
df['acc_vs_vel'] = df['total_linear_acceleration'] / df['total_angular_velocity']
x, y, z, w = df['orientation_X'].tolist() , df['orientation_Y'].tolist() , df['orientation_Z'].tolist() , df['orientation_W'].tolist()
nx, ny, nz = [], [], []
for i in range(len(x)) :
xx, yy, zz = quaternion_to_euler(x[i], y[i], z[i], w[i])
 nx.append(xx)
 ny.append(yy)
 nz.append(zz)
 df['euler_x'] = nx
df['euler_y'] = ny
df['euler_z'] = nz
df['total_angle'] = np.sqrt(np.square(df['euler_x'])+ np.square(df['euler_y'])+ np.square(df['euler_z']))
df['angle_vs_acc'] = df['total_angle'] / df['total_linear_acceleration']
df['angle_vs_vel'] = df['total_angle'] / df['total_angular_velocity']
return df
tqdm.pandas()<feature_engineering>
def perform_feature_engineering(df):
df_out = pd.DataFrame()
def mean_change_of_abs_change(x):
return np.mean(np.diff(np.abs(np.diff(x))))
def mean_abs_change(x):
return np.mean(np.abs(np.diff(x)))
 for col in df.columns:
if col in ['row_id', 'series_id', 'measurement_number']:
continue
df_out[col + '_mean'] = df.groupby(['series_id'])[col].mean()
df_out[col + '_min'] = df.groupby(['series_id'])[col].min()
df_out[col + '_max'] = df.groupby(['series_id'])[col].max()
df_out[col + '_std'] = df.groupby(['series_id'])[col].std()
df_out[col + '_mad'] = df.groupby(['series_id'])[col].mad()
df_out[col + '_med'] = df.groupby(['series_id'])[col].median()
df_out[col + '_skew'] = df.groupby(['series_id'])[col].skew()
df_out[col + '_range'] = df_out[col + '_max'] - df_out[col + '_min']
df_out[col + '_max_to_min'] = df_out[col + '_max'] / df_out[col + '_min']
df_out[col + '_mean_abs_change'] = df.groupby('series_id')[col].apply(mean_abs_change)
 df_out[col + '_mean_change_of_abs_change'] = df.groupby('series_id')[col].apply(mean_change_of_abs_change)
 df_out[col + '_abs_max'] = df.groupby('series_id')[col].apply(lambda x: np.max(np.abs(x)))
 df_out[col + '_abs_min'] = df.groupby('series_id')[col].apply(lambda x: np.min(np.abs(x)))
 df_out[col + '_abs_mean'] = df.groupby('series_id')[col].apply(lambda x: np.mean(np.abs(x)))
 df_out[col + '_abs_std'] = df.groupby('series_id')[col].apply(lambda x: np.std(np.abs(x)))
 df_out[col + '_abs_avg'] =(df_out[col + '_abs_min'] + df_out[col + '_abs_max'])/2
df_out[col + '_abs_range'] = df_out[col + '_abs_max'] - df_out[col + '_abs_min']
return df_out
def standardise_features(train):
train['ToLeft'] = train.PlayDirection == "left"
train['IsBallCarrier'] = train.NflId == train.NflIdRusher
train.loc[train.VisitorTeamAbbr == "ARI", 'VisitorTeamAbbr'] = "ARZ"
train.loc[train.HomeTeamAbbr == "ARI", 'HomeTeamAbbr'] = "ARZ"
train.loc[train.VisitorTeamAbbr == "BAL", 'VisitorTeamAbbr'] = "BLT"
train.loc[train.HomeTeamAbbr == "BAL", 'HomeTeamAbbr'] = "BLT"
train.loc[train.VisitorTeamAbbr == "CLE", 'VisitorTeamAbbr'] = "CLV"
train.loc[train.HomeTeamAbbr == "CLE", 'HomeTeamAbbr'] = "CLV"
train.loc[train.VisitorTeamAbbr == "HOU", 'VisitorTeamAbbr'] = "HST"
train.loc[train.HomeTeamAbbr == "HOU", 'HomeTeamAbbr'] = "HST"
train['Dir_rad'] = np.mod(90 - train.Dir, 360)* math.pi/180.0
train['Rusher'] = train['NflIdRusher'] == train['NflId']
train['TeamOnOffense'] = "home"
train.loc[train.PossessionTeam != train.HomeTeamAbbr, 'TeamOnOffense'] = "away"
train['IsOnOffense'] = train.Team == train.TeamOnOffense
train['YardLine_std'] = 100 - train.YardLine
train.loc[train.FieldPosition.fillna('')== train.PossessionTeam,
'YardLine_std'
] = train.loc[train.FieldPosition.fillna('')== train.PossessionTeam,
'YardLine']
train['X_std'] = train.X
train.loc[train.ToLeft, 'X_std'] = 120 - train.loc[train.ToLeft, 'X']
train['Y_std'] = train.Y
train.loc[train.ToLeft, 'Y_std'] = 160/3 - train.loc[train.ToLeft, 'Y']
train['Dir_std'] = train.Dir_rad
train.loc[train.ToLeft, 'Dir_std'] = np.mod(np.pi + train.loc[train.ToLeft, 'Dir_rad'], 2*np.pi)
 train['Orientation_rad'] = np.mod(train.Orientation, 360)* math.pi/180.0
train.loc[train.Season >= 2018, 'Orientation_rad'
] = np.mod(train.loc[train.Season >= 2018, 'Orientation'] - 90, 360)* math.pi/180.0
train['Orientation_std'] = train.Orientation_rad
train.loc[train.ToLeft, 'Orientation_std'] = np.mod(np.pi + train.loc[train.ToLeft, 'Orientation_rad'], 2*np.pi)
 return train
def euclidean_distance(x1,y1,x2,y2):
x_diff =(x1-x2)**2
y_diff =(y1-y2)**2
return np.sqrt(x_diff + y_diff)
 def back_direction(orientation):
if orientation > 180.0:
return 1
else:
return 0<categorify>
%%time
X_train = perform_euler_factors_calculation(X_train )
def strtoseconds(txt):
txt = txt.split(':')
 ans = int(txt[0])*60 + int(txt[1])+ int(txt[2])/60
return ans
def strtofloat(x):
try:
return float(x)
 except:
return -1
def map_weather(txt):
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt:
ans*=0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*3
if 'sunny' in txt or 'sun' in txt:
return ans*2
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt:
return -2*ans
if 'snow' in txt:
return -3*ans
return 0
def OffensePersonnelSplit(x):
dic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0, 'QB' : 0, 'RB' : 0, 'TE' : 0, 'WR' : 0}
for xx in x.split(","):
xxs = xx.split(" ")
 dic[xxs[-1]] = int(xxs[-2])
 return dic
def DefensePersonnelSplit(x):
dic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0}
for xx in x.split(","):
xxs = xx.split(" ")
 dic[xxs[-1]] = int(xxs[-2])
 return dic
def orientation_to_cat(x):
x = np.clip(x, 0, 360 - 1)
 try:
return str(int(x/15))
except:
return "nan"
def defense_features_next_second(df, next_duration=1):
def get_dx_dy_row(row):
t= next_duration
dist= t*row.S + 0.5*row.A*(t)**2
radian_angle = row.Dir_std
dx = dist * math.cos(radian_angle)
 dy = dist * math.sin(radian_angle)
 return dx, dy
df['X_std_n'],df['Y_std_n'] = zip(*df.apply(get_dx_dy_row,axis=1))
df['X_std_n'] += df.X_std
df['Y_std_n'] += df.Y_std
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','Team','X_std_n','Y_std_n']]
rusher.columns = ['GameId','PlayId','RusherTeam','RusherX','RusherY']
defense = pd.merge(df,rusher,on=['GameId','PlayId'],how='inner')
 defense = defense[defense['Team'] != defense['RusherTeam']][['GameId','PlayId','X_std_n','Y_std_n','RusherX','RusherY']]
defense['def_dist_to_back'] = defense[['X_std_n','Y_std_n','RusherX','RusherY']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 defense['change_x'] = defense.RusherX - defense.X_std_n
defense['change_y'] = defense.RusherY - defense.Y_std_n
defense = defense.groupby(['GameId','PlayId'])\ 
.agg({'def_dist_to_back':['min','max','mean','std'],
'change_x':['min','max','mean','std']})\ 
.reset_index()
defense.columns = ['GameId','PlayId',f'def_min_dist_next_{next_duration}',
f'def_max_dist_next_{next_duration}',
f'def_mean_dist_{next_duration}',f'def_std_dist_{next_duration}',
f'def_min_chx_{next_duration}',f'def_max_chx_{next_duration}',
f'def_mean_chx_{next_duration}',f'def_std_chx_{next_duration}']
return defense
def back_features(df):
carriers = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','NflIdRusher',
'X_std','Y_std','Orientation_std','Dir_std','YardLine_std']]
carriers['back_from_scrimmage'] = carriers['YardLine_std'] - carriers['X_std']
carriers['back_oriented_down_field'] = carriers['Orientation_std'].apply(lambda x: back_direction(x))
carriers['back_moving_down_field'] = carriers['Dir_std'].apply(lambda x: back_direction(x))
carriers = carriers.rename(columns={'X_std':'back_X',
'Y_std':'back_Y'})
 carriers = carriers[['GameId','PlayId','NflIdRusher','back_X','back_Y',
'back_from_scrimmage','back_oriented_down_field',
'back_moving_down_field']]
return carriers
def features_relative_to_back(df, carriers):
player_distance = df[['GameId','PlayId','NflId','X_std','Y_std']]
player_distance = pd.merge(player_distance, carriers, on=['GameId','PlayId'], how='inner')
 player_distance = player_distance[player_distance['NflId'] != player_distance['NflIdRusher']]
player_distance['dist_to_back'] = player_distance[['X_std','Y_std','back_X','back_Y']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 player_distance = player_distance.groupby(['GameId','PlayId','back_from_scrimmage',
'back_oriented_down_field','back_moving_down_field'])\ 
.agg({'dist_to_back':['min','max','mean','std']})\ 
.reset_index()
player_distance.columns = ['GameId','PlayId','back_from_scrimmage',
'back_oriented_down_field','back_moving_down_field',
'min_dist','max_dist','mean_dist','std_dist']
return player_distance
def defense_features(df):
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','Team','X_std','Y_std']]
rusher.columns = ['GameId','PlayId','RusherTeam','RusherX','RusherY']
defense = pd.merge(df,rusher,on=['GameId','PlayId'],how='inner')
 defense = defense[defense['Team'] != defense['RusherTeam']][['GameId','PlayId','X_std','Y_std','RusherX','RusherY']]
defense['def_dist_to_back'] = defense[['X_std','Y_std','RusherX','RusherY']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 defense['change_x'] = defense.RusherX - defense.X_std
defense = defense.groupby(['GameId','PlayId'])\ 
.agg({'def_dist_to_back':['min','max','mean','std'],
'change_x':['min','max','mean','std']})\ 
.reset_index()
defense.columns = ['GameId','PlayId','def_min_dist','def_max_dist','def_mean_dist','def_std_dist',
'def_min_chx','def_max_chx','def_mean_chx','def_std_chx',]
return defense
def offense_features(df):
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','Team','X_std','Y_std']]
rusher.columns = ['GameId','PlayId','RusherTeam','RusherX','RusherY']
offense = pd.merge(df,rusher,on=['GameId','PlayId'],how='inner')
 offense = offense[(offense.IsOnOffense)&(~offense.Rusher)][['GameId','PlayId','X_std','Y_std','RusherX','RusherY']]
offense['def_dist_to_back'] = offense[['X_std','Y_std','RusherX','RusherY']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 offense['change_x'] = offense.RusherX - offense.X_std
offense = offense.groupby(['GameId','PlayId'])\ 
.agg({'def_dist_to_back':['min','max','mean','std'],
'change_x':['min','max','mean','std']})\ 
.reset_index()
offense.columns = ['GameId','PlayId','off_min_dist','off_max_dist','off_mean_dist','off_std_dist',
'off_min_chx','off_max_chx','off_mean_chx','off_std_chx']
return offense
def static_features(df):
add_new_feas = []
df['PlayerHeight_dense'] = df['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
add_new_feas.append('PlayerHeight_dense')
 df['WindSpeed_ob'] = df['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 df['WindSpeed_ob'] = df['WindSpeed_ob'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 df['WindSpeed_ob'] = df['WindSpeed_ob'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 df['WindSpeed_dense'] = df['WindSpeed_ob'].apply(strtofloat)
 add_new_feas.append('WindSpeed_dense')
 df['GameWeather_process'] = df['GameWeather'].str.lower()
df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: "indoor" if not pd.isna(x)and "indoor" in x else x)
 df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 df['GameWeather_dense'] = df['GameWeather_process'].apply(map_weather)
 add_new_feas.append('GameWeather_dense')
 df["Orientation_ob"] = df["Orientation_std"].apply(lambda x : orientation_to_cat(x)).astype("object")
 df["Dir_ob"] = df["Dir_std"].apply(lambda x : orientation_to_cat(x)).astype("object")
 df["Orientation_sin"] = df["Orientation_std"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
df["Orientation_cos"] = df["Orientation_std"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
df["Dir_sin"] = df["Dir_std"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
df["Dir_cos"] = df["Dir_std"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
add_new_feas.append("Dir_sin")
 add_new_feas.append("Dir_cos")
 df["diffScoreBeforePlay"] = df["HomeScoreBeforePlay"] - df["VisitorScoreBeforePlay"]
add_new_feas.append("diffScoreBeforePlay")
 static_features = df[df['NflId'] == df['NflIdRusher']][add_new_feas+['GameId','PlayId','X_std','Y_std',
'S','A','Dis',
'Orientation_std','Dir_std','Dir_rad',
'YardLine_std','Quarter','Down','Distance',
'DefendersInTheBox']].drop_duplicates()
static_features.fillna(-999,inplace=True)
 return static_features
def combine_features(relative_to_back, defense, static, offense, deploy=False):
df = pd.merge(relative_to_back,defense,on=['GameId','PlayId'],how='inner')
 df = pd.merge(df,offense,on=['GameId','PlayId'],how='inner')
 df = pd.merge(df,static,on=['GameId','PlayId'],how='inner')
 if not deploy:
df = pd.merge(df, outcomes, on=['GameId','PlayId'], how='inner')
 return df
def getFarthestPointDefenseFeatures(df,farthest_points_df):
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','Team']]
rusher = rusher.merge(farthest_points_df,on='PlayId',how='left')
 rusher.columns = ['GameId','PlayId','RusherTeam','RusherX','RusherY']
defense = pd.merge(df,rusher,on=['GameId','PlayId'],how='inner')
 defense = defense[defense['Team'] != defense['RusherTeam']][['GameId','PlayId','X_std','Y_std','RusherX','RusherY']]
defense['def_dist_to_back'] = defense[['X_std','Y_std','RusherX','RusherY']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 defense = defense.groupby(['GameId','PlayId'])\ 
.agg({'def_dist_to_back':['min','max','mean','std']})\ 
.reset_index()
defense.columns = ['GameId','PlayId','def_min_dist_far','def_max_dist_far','def_mean_dist_far','def_std_dist_far']
return defense

 print()<feature_engineering>
%%time
X_test = perform_euler_factors_calculation(X_test )
def getFeaturesAfterRemovingNearestDefense(play1,th1=0.5,th2=0.5):
def get_dx_dy_row(row):
t= th1
dist= t*row.S + 0.5*row.A*(t)**2
radian_angle = row.Dir_std
dx = dist * math.cos(radian_angle)
 dy = dist * math.sin(radian_angle)
 return dx, dy
play1['X_std_n'],play1['Y_std_n'] = zip(*play1.apply(get_dx_dy_row,axis=1))
play1['X_std_n'] += play1.X_std
play1['Y_std_n'] += play1.Y_std
play1 =play1[['PlayId','NflId','NflIdRusher','S','A','X_std','Y_std',
'IsOnOffense','Rusher',
'X_std_n','Y_std_n']]
onDefense_play1 = play1[~play1.IsOnOffense].copy()
onOffense_play1 = play1[(play1.IsOnOffense)].copy()
rusher_id = onOffense_play1[onOffense_play1.Rusher].NflId.values[0]
del onDefense_play1['IsOnOffense'],onDefense_play1['NflIdRusher']
del onOffense_play1['IsOnOffense'],onOffense_play1['NflIdRusher']
def_odd_cp = onOffense_play1.merge(onDefense_play1,on='PlayId')
 def_odd_cp['dist_to_defense'] = def_odd_cp[['X_std_n_x','Y_std_n_x','X_std_n_y','Y_std_n_y']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 rusher_def_cp = def_odd_cp[def_odd_cp.NflId_x== rusher_id]
def_odd_cp = def_odd_cp[def_odd_cp.NflId_x!= rusher_id]
to_rem_def_players = def_odd_cp[def_odd_cp.dist_to_defense<=th2].NflId_y.values
play2 = play1[(play1.Rusher)|(~play1.IsOnOffense)]
play2 = play2[~play2.NflId.isin(to_rem_def_players)]
play3 = play2[~play1.Rusher].copy()
rusher_def_cp = rusher_def_cp[~rusher_def_cp.NflId_y.isin(to_rem_def_players)]
points=play2[['X_std_n','Y_std_n']].values
rusher = onOffense_play1[onOffense_play1.Rusher]
play3['RusherX'] = rusher.X_std.values[0]
play3['RusherY'] = rusher.Y_std.values[0]
play3['ls1_theta'] = np.arctan(((play3.Y_std_n - play3.Y_std)/(play3.X_std_n - play3.X_std)))
 play3['ls2_theta'] = np.arctan(((play3.Y_std - play3.RusherY)/(play3.X_std - play3.RusherX)))
 play3['bc_dir_target_end_zone'] = np.abs(play3['ls1_theta']-play3['ls2_theta'])
 bc_dir_target_end_zone_mean = play3.bc_dir_target_end_zone.mean()
bc_dir_target_end_zone_min = play3.bc_dir_target_end_zone.min()
bc_dir_target_end_zone_max = play3.bc_dir_target_end_zone.max()
bc_dir_target_end_zone_std = play3.bc_dir_target_end_zone.std()
points = np.array(points)
 try:
vor = Voronoi(points)
 except :
print('QhullError:',play1.PlayId.values[0])
 return -1
n_points = points.shape[0]
offense = play2.IsOnOffense.values
regions,vertices=voronoi_finite_polygons_2d(vor)
 for r in range(n_points):
region = regions[r]
sp = points[r]
if offense[r]:
pg = vertices[region]
less_xix = np.where(pg[:,0]>=sp[0])[0]
mp = MultiPoint([Point(i)for i in pg])
 pgn = Polygon(mp)
 lp1 = [sp[0],sp[1]-100]
lp2 = [sp[0],sp[1]+100]
line = [lp1,lp2]
shapely_line = LineString(line)
 intersection_line = list(pgn.intersection(shapely_line ).coords)
 pg = pg[less_xix]
pg = np.vstack(( intersection_line[0],pg,intersection_line[1]))
mp = MultiPoint([Point(i)for i in pg])
 pgn = Polygon(mp)
 return pgn.area,rusher_def_cp.dist_to_defense.mean() ,\
rusher_def_cp.dist_to_defense.min() ,rusher_def_cp.dist_to_defense.max() ,\
rusher_def_cp.dist_to_defense.std() , bc_dir_target_end_zone_mean,\
bc_dir_target_end_zone_min,bc_dir_target_end_zone_max,bc_dir_target_end_zone_std
def getMoreFeatures(basetable,df):
basetable['SpeedForward'] = basetable.S*np.cos(basetable.Dir_std)
 basetable['hA'] = basetable.A*np.cos(basetable.Dir_std)
 basetable['hD'] = basetable.SpeedForward + 0.5*basetable.hA
basetable['Distance'] = basetable.X_std-90
basetable['dis_forward'] = basetable.Dis*np.cos(basetable.Dir_std)
 df_def = df[(~df.IsOnOffense)].copy()
rusher_df = df[df.Rusher][['PlayId','X_std','Y_std','S']].copy()
rusher_df.columns = ['PlayId','RusherX','RusherY','RusherS']
df_def =df_def.merge(rusher_df,on='PlayId',how='left')
 df_def['rusherDistance'] = np.sqrt(( df_def.X_std - df_def.Y_std)**2 +(df_def.X_std - df_def.RusherX)**2)
 df_def['time_tackle'] = df_def['rusherDistance']/df_def.S
time_tackle_df = df_def.groupby(['PlayId'])['time_tackle'].min().reset_index()
basetable = basetable.merge(time_tackle_df,on='PlayId',how='left')
 basetable=basetable.fillna(-1)
 return basetable
def create_features(df, deploy=False):
df = standardise_features(df)
 back_feats = back_features(df)
 rel_back = features_relative_to_back(df, back_feats)
 def_feats = defense_features(df)
 off_feats = offense_features(df)
 static_feats = static_features(df)
 basetable = combine_features(rel_back, def_feats, static_feats,off_feats, deploy=deploy)
 for dur in [0.5,1,2,2.5,3,4]:
def_f_n = defense_features_next_second(df,next_duration=dur)
 basetable = basetable.merge(def_f_n,on=['GameId','PlayId'],how='left')
 vor_area_ruhser_arr = []
vor_area_ruhser_after_half_second_arr = []
vor_area_ruhser_after_1_second_arr = []
playids = []
farthest_points_x = []
farthest_points_y = []
dist_to_defense_area=[]
dist_to_defense_min=[]
dist_to_defense_max=[]
dist_to_defense_mean=[]
dist_to_defense_std=[]
bc_dir_target_end_zone_mean= []
bc_dir_target_end_zone_min= []
bc_dir_target_end_zone_max= []
bc_dir_target_end_zone_std= []
def_area = []
playcnt=0
for playid in df.PlayId.unique() :
playcnt+=1
if playcnt%1000==0:
print('Porcessed Plays',playcnt)
 play1 =df[(df.PlayId ==playid)].copy()
try:
vor_area_ruhser,fpx,fpy = getVornoiAreaRusher(play1)
 except:
vor_area_ruhser=0
fpx=X_std
fpy=Y_std
try:
vor_area_ruhser_after_half_second = getVornoiAreaRusher_after_half_second(play1)
 except:
vor_area_ruhser_after_half_second=0
try:
vor_area_ruhser_after_1_second = getVornoiAreaRusher_after_1_second(play1)
 except:
vor_area_ruhser_after_1_second=0
try:
a1,a2,a3,a4,a5,a6,a7,a8,a9 = getFeaturesAfterRemovingNearestDefense(play1,th1=1,th2=0.4)
 except:
a1=0
a2=0
a3=0
a4=0
a5=0
a6=0
a7=0
a8=0
a9=0
dist_to_defense_area.append(a1)
 dist_to_defense_mean.append(a2)
 dist_to_defense_min.append(a3)
 dist_to_defense_max.append(a4)
 dist_to_defense_std.append(a5)
 bc_dir_target_end_zone_mean.append(a6)
 bc_dir_target_end_zone_min.append(a7)
 bc_dir_target_end_zone_max.append(a8)
 bc_dir_target_end_zone_std.append(a9)
 vor_area_ruhser_arr.append(vor_area_ruhser)
 vor_area_ruhser_after_half_second_arr.append(vor_area_ruhser_after_half_second)
 vor_area_ruhser_after_1_second_arr.append(vor_area_ruhser_after_1_second)
 farthest_points_x.append(fpx)
 farthest_points_y.append(fpy)
 playids.append(playid)

vor_df = pd.DataFrame({'PlayId':playids, 'vor_area_ruhser':vor_area_ruhser_arr,
'vor_area_ruhser_after_half_second':vor_area_ruhser_after_half_second_arr,
'vor_area_ruhser_after_1_second':vor_area_ruhser_after_1_second_arr,
'dist_to_defense_area':dist_to_defense_area,
'dist_to_defense_mean':dist_to_defense_mean,
'dist_to_defense_min':dist_to_defense_min,
'dist_to_defense_max':dist_to_defense_max,
'dist_to_defense_std':dist_to_defense_std,
'bc_dir_target_end_zone_mean':bc_dir_target_end_zone_mean,
'bc_dir_target_end_zone_min':bc_dir_target_end_zone_min,
'bc_dir_target_end_zone_max':bc_dir_target_end_zone_max,
'bc_dir_target_end_zone_std':bc_dir_target_end_zone_std
})
 basetable = basetable.merge(vor_df,on='PlayId',how='left')
 farthest_points_df = pd.DataFrame({'PlayId':playids,'farthest_point_x':farthest_points_x,
'farthest_point_y':farthest_points_y
})
 defensedf = getFarthestPointDefenseFeatures(df,farthest_points_df)
 basetable = basetable.merge(defensedf,on='PlayId',how='left')
 basetable = getMoreFeatures(basetable ,df)
 basetable = basetable.fillna(-199)
 return basetable<load_from_csv>
%%time
X_train = perform_feature_engineering(X_train[features] )
if TRAIN_OFFLINE:
train = pd.read_csv('./input/train.csv', dtype={'WindSpeed': 'object'})
 else:
train = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2020/train.csv', dtype={'WindSpeed': 'object'} )<feature_engineering>
%%time
X_test = perform_feature_engineering(X_test[features] )
train.loc[train['Season'] == 2017, 'S'] =(train['S'][train['Season'] == 2017] - 2.4355)/ 1.2930 * 1.4551 + 2.7570<remove_duplicates>
print("Train X: {}
Train y: {}
Test X: {}".format(X_train.shape, y_train.shape, X_test.shape))
outcomes = train[['GameId','PlayId','Yards']].drop_duplicates()<feature_engineering>
%%time
correlations = X_train.corr().abs().unstack().sort_values(kind="quicksort" ).reset_index()
correlations = correlations[correlations['level_0'] != correlations['level_1']]
%time train_basetable = create_features(train, False )<merge>
n_top_corr = correlations[correlations[0]==1.0].shape[0]
print("There are {} different features pairs with correlation factor 1.0.".format(n_top_corr))
wt= train[train.Rusher][['PlayId','PlayerWeight']].copy()
train_basetable = train_basetable.merge(wt,left_on=['PlayId'],right_on=['PlayId'],how='left')
 train_basetable = train_basetable.fillna(-99)
 train_basetable['AW'] = train_basetable.A / train_basetable.PlayerWeight<correct_missing_values>
drop_features = list(correlations.head(n_top_corr)['level_0'].unique())
 X_train = X_train.drop(drop_features,axis=1)
 X_test = X_test.drop(drop_features,axis=1 )
train_basetable.fillna(-199,inplace=True )<prepare_x_and_y>
print("Train X: {}
Train y: {}
Test X: {}".format(X_train.shape, y_train.shape, X_test.shape))
X = train_basetable.copy()
yards = X.Yards
y = np.zeros(( yards.shape[0], 199))
for idx, target in enumerate(list(yards)) :
y[idx][99 + target] = 1
X.drop(['GameId_x','GameId_y'], axis=1, inplace=True )<define_variables>
le = LabelEncoder()
y_train['surface'] = le.fit_transform(y_train['surface'] )
crm = ['min_dist',
'max_dist', 'mean_dist', 'std_dist', 'def_max_dist', 'def_std_dist',
'def_max_chx', 'def_mean_chx', 'WindSpeed_dense', 'GameWeather_dense', 'diffScoreBeforePlay',
'Y_std', 'Orientation_std', 'Dir_std', 'Quarter', 'Down', 'def_max_dist_next_0.5',
'def_std_dist_0.5', 'def_mean_chx_0.5', 'def_max_dist_next_1', 'def_std_dist_1',
'def_mean_chx_1', 'def_max_dist_next_2', 'def_std_dist_2', 'def_min_chx_2',
'def_min_dist_next_2.5', 'def_max_dist_next_2.5', 'def_std_dist_2.5',
'def_min_chx_2.5', 'def_std_chx_2.5', 'def_min_dist_next_3', 'def_max_dist_next_3',
'def_std_chx_3', 'vor_area_ruhser', 'def_max_dist_far', 'def_mean_dist_far',
'def_std_dist_far'] 
<define_variables>
X_train.fillna(0, inplace = True)
 X_train.replace(-np.inf, 0, inplace = True)
 X_train.replace(np.inf, 0, inplace = True)
 X_test.fillna(0, inplace = True)
 X_test.replace(-np.inf, 0, inplace = True)
 X_test.replace(np.inf, 0, inplace = True )
tormcols=['back_oriented_down_field', 'back_moving_down_field', 'std_dist', 'def_std_dist',
'def_min_chx', 'off_max_dist', 'PlayerHeight_dense', 'Dir_sin', 'Dir_cos',
'diffScoreBeforePlay', 'X_std', 'Y_std', 'Distance', 'DefendersInTheBox',
'def_max_dist_next_0.5', 'def_std_chx_1', 'def_min_chx_2', 'def_std_chx_2',
'def_mean_dist_2.5', 'def_min_chx_2.5', 'def_max_chx_2.5', 'def_min_dist_next_3',
'def_mean_dist_3', 'def_min_chx_3', 'def_max_chx_3', 'def_std_chx_3',
'def_min_dist_next_4', 'def_std_dist_4', 'def_std_chx_4',
'bc_dir_target_end_zone_max', 'bc_dir_target_end_zone_std',
'def_mean_dist_far', 'def_std_dist_far', 'max_dist', 'mean_dist',
'off_min_chx', 'off_max_chx', 'GameWeather_dense', 'Dir_std', 'Down',
'def_min_chx_0.5', 'def_std_dist_2.5', 'def_max_dist_next_4',
'def_min_chx_4', 'bc_dir_target_end_zone_min']
X=X.drop(tormcols,axis=1 )<feature_engineering>
scaler = StandardScaler()
X_train_scaled = pd.DataFrame(scaler.fit_transform(X_train))
X_test_scaled = pd.DataFrame(scaler.transform(X_test))
print("Scaled !" )
for c in X.columns:
X[c] = X[c].replace(-np.inf,199)
 X[c] = X[c].replace(np.inf,199 )<normalization>
folds = StratifiedKFold(n_splits=49, shuffle=True, random_state=2018 )
scaler = StandardScaler()
X1 = scaler.fit_transform(X.drop(['PlayId','Yards'],axis=1))<set_options>
sub_preds_rf = np.zeros(( X_test_scaled.shape[0], 9))
oof_preds_rf = np.zeros(( X_train_scaled.shape[0]))
score = 0
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X_train_scaled, y_train['surface'])) :
clf = RandomForestClassifier(n_estimators = 500, n_jobs = -1)
 clf.fit(X_train_scaled.iloc[trn_idx], y_train['surface'][trn_idx])
 oof_preds_rf[val_idx] = clf.predict(X_train_scaled.iloc[val_idx])
 sub_preds_rf += clf.predict_proba(X_test_scaled)/ folds.n_splits
score += clf.score(X_train_scaled.iloc[val_idx], y_train['surface'][val_idx])
 print('Fold: {} score: {}'.format(fold_,clf.score(X_train_scaled.iloc[val_idx], y_train['surface'][val_idx])))
 print('Avg Accuracy', score / folds.n_splits )
class CRPSCallback(Callback):
def __init__(self,validation, predict_batch_size=20, include_on_batch=False):
super(CRPSCallback, self ).__init__()
self.validation = validation
self.predict_batch_size = predict_batch_size
self.include_on_batch = include_on_batch
print('validation shape',len(self.validation))
def on_batch_begin(self, batch, logs={}):
pass
def on_train_begin(self, logs={}):
if not('CRPS_score_val' in self.params['metrics']):
self.params['metrics'].append('CRPS_score_val')
 def on_batch_end(self, batch, logs={}):
if(self.include_on_batch):
logs['CRPS_score_val'] = float('-inf')
 def on_epoch_end(self, epoch, logs={}):
logs['CRPS_score_val'] = float('-inf')
 if(self.validation):
X_valid, y_valid = self.validation[0], self.validation[1]
y_pred = self.model.predict(X_valid)
 y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * X_valid.shape[0])
 val_s = np.round(val_s, 6)
 logs['CRPS_score_val'] = val_s 
<choose_model_class>
USE_LGB = True
if(USE_LGB):
sub_preds_lgb = np.zeros(( X_test.shape[0], 9))
oof_preds_lgb = np.zeros(( X_train.shape[0]))
score = 0
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X_train, y_train['surface'])) :
train_x, train_y = X_train.iloc[trn_idx], y_train['surface'][trn_idx]
valid_x, valid_y = X_train.iloc[val_idx], y_train['surface'][val_idx]
clf = LGBMClassifier(
 nthread=-1,
n_estimators=2000,
learning_rate=0.01,
boosting_type='gbdt',
is_unbalance=True,
objective='multiclass',
numclass=9,
silent=-1,
verbose=-1,
feval=None)
 clf.fit(train_x, train_y, eval_set=[(train_x, train_y),(valid_x, valid_y)],
verbose= 1000, early_stopping_rounds= 200)
 oof_preds_lgb[val_idx] = clf.predict(valid_x)
 sub_preds_lgb += clf.predict_proba(X_test)/ folds.n_splits
score += clf.score(valid_x, valid_y)
 print('Fold: {} score: {}'.format(fold_,clf.score(valid_x, valid_y)))
 print('Avg Accuracy', score / folds.n_splits )
def get_model(x_tr,y_tr,x_val,y_val):
inp = Input(shape =(x_tr.shape[1],))
x = Dense(1024, input_dim=X.shape[1], activation='relu' )(inp)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(512, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(256, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 out = Dense(199, activation='softmax' )(x)
 model = Model(inp,out)
 model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=[])
 es = EarlyStopping(monitor='CRPS_score_val',
mode='min',
restore_best_weights=True,
verbose=1,
patience=10)
 mc = ModelCheckpoint('best_model.h5',monitor='CRPS_score_val',mode='min',
save_best_only=True, verbose=1, save_weights_only=True)
 bsz = 512
steps = x_tr.shape[0]/bsz
model.fit(x_tr, y_tr,callbacks=[CRPSCallback(validation =(x_val,y_val)) ,es,mc], epochs=100, batch_size=bsz,verbose=1)
 model.load_weights("best_model.h5")
 y_pred1 = model.predict(x_val)
 y_valid = y_val
y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred1, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * x_val.shape[0])
 crps = np.round(val_s, 6)
 return model,crps,y_pred1<create_dataframe>
submission = pd.read_csv(os.path.join(PATH,'sample_submission.csv'))
submission['surface'] = le.inverse_transform(sub_preds_rf.argmax(axis=1))
submission.to_csv('submission_rf.csv', index=False)
 submission.head(10 )
<init_hyperparams><EOS> 
USE_LGB = True
if(USE_LGB):
submission['surface'] = le.inverse_transform(sub_preds_lgb.argmax(axis=1))
submission.to_csv('submission_lgb.csv', index=False)
 submission.head(10 )