Kaggle competition: CareerCon 2019 - Help Navigate Robots
outcomes = train[['GameId','PlayId','Yards']].drop_duplicates()<merge>
df_train = pd.read_csv(".. /input/X_train.csv")
 df_train.head()
<merge>
df_test = pd.read_csv(".. /input/X_test.csv")
 df_test.head()
def get_play_control(df):
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','Orientation', 'S', 'A', 'X', 'Y']]
rusher.columns = ['GameId','PlayId', 'RusherDir', 'RusherS', 'RusherA', 'RusherX', 'RusherY']
df = df.merge(rusher,on=['GameId','PlayId'], how='inner')
 df["theta"] = df["Orientation"].apply(math.radians)
 df["dist_to_ball"] =(( df["RusherX"]-df["X"])**2+(df["RusherY"]-df["Y"])**2)**0.5
df["RADIUS"] = df["dist_to_ball"].apply(radius_calc)
 df["S_ratio"] = df["S"].apply(lambda x:(x/13)**2)
 df["influence"] = df[["RADIUS", "S_ratio", "theta", "RusherX", "RusherY", "S", "X", "Y" ]].apply(lambda x: influence(*x), axis=1)
 df["OffenceControl"] = df[df["IsOnOffense"]].groupby(['GameId','PlayId'])["influence"].transform("sum")
 df["DefenceControl"] = df[~df["IsOnOffense"]].groupby(['GameId','PlayId'])["influence"].transform("sum")
 df["OffenceControl"]= df.groupby(['GameId','PlayId'])["OffenceControl"].fillna(method="ffill")
 df["OffenceControl"]= df.groupby(['GameId','PlayId'])["OffenceControl"].fillna(method="bfill")
 df["DefenceControl"]= df.groupby(['GameId','PlayId'])["DefenceControl"].fillna(method="ffill")
 df["DefenceControl"]= df.groupby(['GameId','PlayId'])["DefenceControl"].fillna(method="bfill")
 df["play_control"] = df["OffenceControl"] - df["DefenceControl"]
df = new_def_feats(df)
 df.drop(rusher.columns, axis=1, inplace=True)
 df.drop(["theta", "dist_to_ball", "RADIUS", "S_ratio", "OffenceControl", "DefenceControl"], axis=1, inplace=True)
 return df 
<prepare_x_and_y>
df_train["series_id"].is_monotonic
@np.vectorize
def influence(RADIUS, S_ratio, theta, RusherX, RusherY, speed, X_std, Y_std):
player_coords = np.array([X_std, Y_std])
 point = np.array([RusherX, RusherY])
 S_matrix = np.matrix([[RADIUS *(1 + S_ratio), 0], [0, RADIUS *(1 - S_ratio)]])
 R_matrix = np.matrix([[np.cos(theta), - np.sin(theta)], [np.sin(theta), np.cos(theta)]])
 COV_matrix = np.dot(np.dot(np.dot(R_matrix, S_matrix), S_matrix), np.linalg.inv(R_matrix))
norm_fact =(1 / 2 * np.pi)*(1 / np.sqrt(np.linalg.det(COV_matrix)))
 mu_play = player_coords + speed * np.array([np.cos(theta), np.sin(theta)])/ 2
intermed_scalar_player = np.dot(np.dot(( player_coords - mu_play),
np.linalg.inv(COV_matrix)) ,
np.transpose(( player_coords - mu_play)))
 player_influence = norm_fact * np.exp(- 0.5 * intermed_scalar_player[0, 0])
 intermed_scalar_point = np.dot(np.dot(( point - mu_play),
np.linalg.inv(COV_matrix)) ,
np.transpose(( point - mu_play)))
 point_influence = norm_fact * np.exp(- 0.5 * intermed_scalar_point[0, 0])
 return point_influence / player_influence<categorify>
df_y = pd.read_csv(".. /input/y_train.csv")
 df_y.head()
def strtoseconds(txt):
txt = txt.split(':')
 ans = int(txt[0])*60 + int(txt[1])+ int(txt[2])/60
return ans
def strtofloat(x):
try:
return float(x)
 except:
return -1
def map_weather(txt):
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt:
ans*=0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*3
if 'sunny' in txt or 'sun' in txt:
return ans*2
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt:
return -2*ans
if 'snow' in txt:
return -3*ans
return 0
def OffensePersonnelSplit(x):
dic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0, 'QB' : 0, 'RB' : 0, 'TE' : 0, 'WR' : 0}
for xx in x.split(","):
xxs = xx.split(" ")
 dic[xxs[-1]] = int(xxs[-2])
 return dic
def DefensePersonnelSplit(x):
dic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0}
for xx in x.split(","):
xxs = xx.split(" ")
 dic[xxs[-1]] = int(xxs[-2])
 return dic
def orientation_to_cat(x):
x = np.clip(x, 0, 360 - 1)
 try:
return str(int(x/15))
except:
return "nan"
def preprocess(train):
train['GameClock_sec'] = train['GameClock'].apply(strtoseconds)
 train["GameClock_minute"] = train["GameClock"].apply(lambda x : x.split(":")[0] ).astype("object")
 train['PlayerHeight_dense'] = train['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
train['TimeHandoff'] = train['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeSnap'] = train['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeDelta'] = train.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 train['PlayerBirthDate'] = train['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
train['PlayerAge'] = train.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 train["PlayerAge_ob"] = train['PlayerAge'].astype(np.int ).astype("object")
 train['GameWeather_process'] = train['GameWeather'].str.lower()
train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: "indoor" if not pd.isna(x)and "indoor" in x else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 train['GameWeather_dense'] = train['GameWeather_process'].apply(map_weather)
 train['IsRusher'] =(train['NflId'] == train['NflIdRusher'])
 train['IsRusher_ob'] =(train['NflId'] == train['NflIdRusher'] ).astype("object")
 temp = train[train["IsRusher"]][["Team", "PlayId"]].rename(columns={"Team":"RusherTeam"})
 train = train.merge(temp, on = "PlayId")
 train["IsRusherTeam"] = train["Team"] == train["RusherTeam"]
train["Quarter_ob"] = train["Quarter"].astype("object")
 train["Down_ob"] = train["Down"].astype("object")
 train["JerseyNumber_ob"] = train["JerseyNumber"].astype("object")
 train["YardLine_ob"] = train["YardLine"].astype("object")
 train["Orientation_ob"] = train["Orientation"].apply(lambda x : orientation_to_cat(x)).astype("object")
 train["Dir_ob"] = train["Dir"].apply(lambda x : orientation_to_cat(x)).astype("object")
 train["Orientation_sin"] = train["Orientation"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
train["Orientation_cos"] = train["Orientation"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
train["Dir_sin"] = train["Dir"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
train["Dir_cos"] = train["Dir"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
train["diffScoreBeforePlay"] = train["HomeScoreBeforePlay"] - train["VisitorScoreBeforePlay"]
train["diffScoreBeforePlay_binary_ob"] =(train["HomeScoreBeforePlay"] > train["VisitorScoreBeforePlay"] ).astype("object")
 Turf = {'Field Turf':'Artificial', 'A-Turf Titan':'Artificial', 'Grass':'Natural', 'UBU Sports Speed S5-M':'Artificial', 'Artificial':'Artificial', 'DD GrassMaster':'Artificial', 'Natural Grass':'Natural', 'UBU Speed Series-S5-M':'Artificial', 'FieldTurf':'Artificial', 'FieldTurf 360':'Artificial', 'Natural grass':'Natural', 'grass':'Natural', 'Natural':'Natural', 'Artifical':'Artificial', 'FieldTurf360':'Artificial', 'Naturall Grass':'Natural', 'Field turf':'Artificial', 'SISGrass':'Artificial', 'Twenty-Four/Seven Turf':'Artificial', 'natural grass':'Natural'}
train['Turf'] = train['Turf'].map(Turf)
 temp = train["OffensePersonnel"].iloc[np.arange(0, len(train), 22)].apply(lambda x : pd.Series(OffensePersonnelSplit(x)))
 temp.columns = ["Offense" + c for c in temp.columns]
temp["PlayId"] = train["PlayId"].iloc[np.arange(0, len(train), 22)]
train = train.merge(temp, on = "PlayId")
 temp = train["DefensePersonnel"].iloc[np.arange(0, len(train), 22)].apply(lambda x : pd.Series(DefensePersonnelSplit(x)))
 temp.columns = ["Defense" + c for c in temp.columns]
temp["PlayId"] = train["PlayId"].iloc[np.arange(0, len(train), 22)]
train = train.merge(temp, on = "PlayId")
 train = train.sort_values(by = ['X'] ).sort_values(by = ['Dis'] ).sort_values(by=['PlayId', 'IsRusherTeam', 'IsRusher'] ).reset_index(drop = True)
 return train<feature_engineering>
df_y.nunique()
def clean_features(train):

 try:
train.loc[train.VisitorTeamAbbr == "ARI", 'VisitorTeamAbbr'] = "ARZ"
train.loc[train.HomeTeamAbbr == "ARI", 'HomeTeamAbbr'] = "ARZ"
train.loc[train.VisitorTeamAbbr == "BAL", 'VisitorTeamAbbr'] = "BLT"
train.loc[train.HomeTeamAbbr == "BAL", 'HomeTeamAbbr'] = "BLT"
train.loc[train.VisitorTeamAbbr == "CLE", 'VisitorTeamAbbr'] = "CLV"
train.loc[train.HomeTeamAbbr == "CLE", 'HomeTeamAbbr'] = "CLV"
train.loc[train.VisitorTeamAbbr == "HOU", 'VisitorTeamAbbr'] = "HST"
train.loc[train.HomeTeamAbbr == "HOU", 'HomeTeamAbbr'] = "HST"
except:
pass
print("...| Wind Direction")
 try:
train['WindDirection_std'] = train['WindDirection'].fillna(0)
 except:
train['WindDirection_std'] = 0
train['WindDirection'] = "unknown"
north = ['N','From S','North']
south = ['S','From N','South','s']
west = ['W','From E','West']
east = ['E','From W','from W','EAST','East']
north_east = ['FROM SW','NE','NORTH EAST','North East','NorthEast','Northeast','From SW']
north_west = ['E','NW','NORTHWEST', 'Northwest']
south_east = ['SE','SOUTHEAST','Southeast']
south_west = ['SW','SOUTHWEST','SouthWest','Southwest']
nne = ["NNE", "From SSW", "N-NE", "North Northeast"]
ene = ["East North East", 'ENE','From WSW']
ese = ['East Southeast', 'ESE','From WNW']
sse = [ 'From NNW', 'South Southeast','SSE']
ssw = [ 'From NNE', 'South Southwest','SSW']
wsw = [ 'From ENE', 'W-SW','West-Southwest','WSW']
wnw = [ 'From ESE', 'W-NW','WNW','West Northwest']
nnw = [ 'From SSE','North/Northwest','NNW']
def clean_wind_dir(x):
if x in north:
return 0
elif x in south:
return 180
elif x in west:
return 270
elif x in east:
return 90
elif x in north_east:
return 45
elif x in north_west:
return 315
elif x in south_east:
return 135
elif x in south_west:
return 225
elif x in nne:
return 20
elif x in ene:
return 70
elif x in sse:
return 110
elif x in ssw:
return 200
elif x in wsw:
return 250
elif x in wnw:
return 290
elif x in nnw:
return 340
else:
return 0
train['WindDirection_std'] = train['WindDirection_std'].apply(clean_wind_dir)

print("...| Wind Speed")
 def give_me_WindSpeed(txt):
txt = str(txt ).lower().replace('mph', '' ).strip()
if '-' in txt:
txt =(int(txt.split('-')[0])+ int(txt.split('-')[1])) / 2
try:
return float(txt)
 except:
return -1
train['WindSpeed'] = train['WindSpeed'].apply(give_me_WindSpeed)
 print("...| Weather")
 def give_me_GameWeather(txt):
txt = str(txt ).lower().replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly' ).replace('clear and sunny', 'sunny and clear')
 txt = txt.replace('skies', '' ).replace("mostly", "" ).strip()
if "indoor" in txt:
txt = "indoor"
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt:
ans*=0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*5
if 'sunny' in txt or 'sun' in txt:
return ans*3
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt:
return -3*ans
if 'snow' in txt:
return -5*ans
return 0
train['GameWeather_std'] = train['GameWeather'].apply(give_me_GameWeather)
 rain = ['Rainy', 'Rain Chance 40%', 'Showers','Cloudy with periods of rain, thunder possible.Winds shifting to WNW, 10-20 mph.',
'Scattered Showers', 'Cloudy, Rain', 'Rain shower', 'Light Rain', 'Rain']
overcast = ['Cloudy, light snow accumulating 1-3"', 'Party Cloudy', 'Cloudy, chance of rain',
'Coudy', 'Cloudy, 50% change of rain', 'Rain likely, temps in low 40s.',
'Cloudy and cold', 'Cloudy, fog started developing in 2nd quarter',
'Partly Clouidy', '30% Chance of Rain', 'Mostly Coudy', 'Cloudy and Cool',
'cloudy', 'Partly cloudy', 'Overcast', 'Hazy', 'Mostly cloudy', 'Mostly Cloudy',
'Partly Cloudy', 'Cloudy']
clear = ['Partly clear', 'Sunny and clear', 'Sun & clouds', 'Clear and Sunny',
'Sunny and cold', 'Sunny Skies', 'Clear and Cool', 'Clear and sunny',
'Sunny, highs to upper 80s', 'Mostly Sunny Skies', 'Cold',
'Clear and warm', 'Sunny and warm', 'Clear and cold', 'Mostly sunny',
'T: 51; H: 55; W: NW 10 mph', 'Clear Skies', 'Clear skies', 'Partly sunny',
'Fair', 'Partly Sunny', 'Mostly Sunny', 'Clear', 'Sunny', 'Sunny, Windy']
snow = ['Heavy lake effect snow', 'Snow']
none = ['N/A Indoor', 'Indoors', 'Indoor', 'N/A(Indoors)', 'Controlled Climate']
def clean_weather(x):
if x in rain:
return "rain"
elif x in overcast:
return "overcast"
elif x in clear:
return "clear"
elif x in snow:
return "snow"
else:
return "other"
train['GameWeather'] = train['GameWeather'].apply(clean_weather)
 print("...| Turf")
 def clean_turf(x):
if "art" in x:
return 0
if "grass" in x:
return 1
if "natural" in x:
return 1
if "turf" in x:
return 0
else:
return 0
train["Turf_std"] = train["Turf"].apply(lambda x: clean_turf(x.lower()))
print("...| Stadium")
 stadium_replace = {
"Twickenham Stadium":"Twickenham",
"Los Angeles Memorial Coliesum":"Los Angeles Memorial Coliseum",
"FirstEnergy Stadium":"FirstEnergyStadium",
"FirstEnergy":"FirstEnergyStadium",
"First Energy Stadium":"FirstEnergyStadium",
"M&T Bank Stadium":"M & T Bank Stadium",
"M&T Stadium": "M & T Bank Stadium",
"Broncos Stadium at Mile High":"Broncos Stadium At Mile High",
"Mercedes-Benz Dome":"Mercedes-Benz Stadium",
"Mercedes-Benz Superdome": "Mercedes-Benz Stadium",
"MetLife Stadium":"Metlife Stadium",
"MetLife":"Metlife Stadium",
}
for key, value in stadium_replace.items() :
train["Stadium"] = train["Stadium"].replace(key, value)
 outdoor = [
'Outdoor', 'Outdoors', 'Cloudy', 'Heinz Field',
'Outdor', 'Ourdoor', 'Outside', 'Outddors',
'Outdoor Retr Roof-Open', 'Oudoor', 'Bowl'
]
indoor_closed = [
'Indoors', 'Indoor', 'Indoor, Roof Closed', 'Indoor, Roof Closed',
'Retractable Roof', 'Retr.Roof-Closed', 'Retr.Roof - Closed', 'Retr.Roof Closed',
]
indoor_open = ['Indoor, Open Roof', 'Open', 'Retr.Roof-Open', 'Retr.Roof - Open']
dome_closed = ['Dome', 'Domed, closed', 'Closed Dome', 'Domed', 'Dome, closed']
dome_open = ['Domed, Open', 'Domed, open']
train['StadiumType'] = train['StadiumType'].replace(outdoor, "outdoor")
 train['StadiumType'] = train['StadiumType'].replace(indoor_closed, 'indoor closed')
 train['StadiumType'] = train['StadiumType'].replace(indoor_open, 'indoor open')
 train['StadiumType'] = train['StadiumType'].replace(dome_closed, 'dome closed')
 train['StadiumType'] = train['StadiumType'].replace(dome_open, 'dome open')
 def timesnap2day(x):
days = x.split("-")
 return 365 * int(days[0])+ 30 * int(days[1])+ int(days[2][:2])
 train['MatchDay'] = train['TimeSnap'].apply(timesnap2day)
 print("...| Time")
 def strtoseconds(txt):
txt = txt.split(':')
 return int(txt[0])*60 + int(txt[1])+ int(txt[2])/60
train['GameClock_std'] = train['GameClock'].apply(strtoseconds)
 train['Height_feet'] = train['PlayerHeight'].apply(lambda x: int(x.split('-')[0]))
return train<categorify>
encoder = LabelEncoder()
y = encoder.fit_transform(df_y["surface"])
 y_count = len(list(encoder.classes_))
def remove_NAs(train):
def na_categorical(x):
try:
return x.fillna(x.mode() [0])
 except:
return x.fillna(method="ffill")
 train["OffenseFormation"] = train.OffenseFormation.transform(lambda x: na_categorical(x))
train["StadiumType"] = train.groupby("Stadium")["StadiumType"].transform(lambda x: na_categorical(x))
train["DefendersInTheBox"] = train.groupby("DefensePersonnel")["DefendersInTheBox"].transform(lambda x: x.fillna(x.median()))
try:
train["GameWeather_std"] = train.groupby("Week")["GameWeather_std"].transform(lambda x: x.median())
 except:
train["GameWeather_std"] = train["GameWeather_std"].transform(lambda x: x.fillna(x.median()))
try:
train["Temperature"] = train.groupby("Week")["Temperature"].transform(lambda x: x.fillna(x.mean()))
except:
train['Temperature'] = train['Temperature'].fillna(train['Temperature'].median() , inplace=True)
 try:
train["WindSpeed"] = train.groupby("Week")["WindSpeed"].transform(lambda x: x.fillna(x.mean()))
except:
train["WindSpeed"] = train["WindSpeed"].transform(lambda x: x.fillna(x.mean()))
def wind_speed_closed(x):
if "closed" in x[0]:
return 0
train["StadiumClosed"] = train["StadiumType"].fillna("" ).apply(lambda x: "closed" in x)
 train['Temperature'] = train['Temperature'].fillna(75)
 train['Humidity'] = train['Humidity'].fillna(65)
 train["WindSpeed"] = train["WindSpeed"].fillna(10)
 return train<feature_engineering>
label_mapping = {i: l for i, l in enumerate(encoder.classes_)}
def fix_sides(train):

 train['ToLeft'] = train.PlayDirection == "left"
train['IsBallCarrier'] = train.NflId == train.NflIdRusher
train.loc[train.VisitorTeamAbbr == "ARI", 'VisitorTeamAbbr'] = "ARZ"
train.loc[train.HomeTeamAbbr == "ARI", 'HomeTeamAbbr'] = "ARZ"
train.loc[train.VisitorTeamAbbr == "BAL", 'VisitorTeamAbbr'] = "BLT"
train.loc[train.HomeTeamAbbr == "BAL", 'HomeTeamAbbr'] = "BLT"
train.loc[train.VisitorTeamAbbr == "CLE", 'VisitorTeamAbbr'] = "CLV"
train.loc[train.HomeTeamAbbr == "CLE", 'HomeTeamAbbr'] = "CLV"
train.loc[train.VisitorTeamAbbr == "HOU", 'VisitorTeamAbbr'] = "HST"
train.loc[train.HomeTeamAbbr == "HOU", 'HomeTeamAbbr'] = "HST"

 train['TeamOnOffense'] = "home"
train.loc[train.PossessionTeam != train.HomeTeamAbbr, 'TeamOnOffense'] = "away"
train['IsOnOffense'] = train.Team == train.TeamOnOffense
train['YardLine_std'] = 100 - train.YardLine
train.loc[train.FieldPosition.fillna('')== train.PossessionTeam,
'YardLine_std'
] = train.loc[train.FieldPosition.fillna('')== train.PossessionTeam,
'YardLine']
train['X_std'] = train.X
train.loc[train.ToLeft, 'X_std'] = 120 - train.loc[train.ToLeft, 'X']
train['Y_std'] = train.Y
train.loc[train.ToLeft, 'Y_std'] = 160/3 - train.loc[train.ToLeft, 'Y']
train['Orientation_std'] = train.Orientation
train.loc[train.ToLeft, 'Orientation_std'] = np.mod(180 + train.loc[train.ToLeft, 'Orientation_std'], 360)
 train['Dir_std'] = train.Dir
train.loc[train.ToLeft, 'Dir_std'] = np.mod(180 + train.loc[train.ToLeft, 'Dir_std'], 360)
 train.loc[train.ToLeft, 'WindDirection_std'] = np.mod(180 + train.loc[train.ToLeft, 'WindDirection_std'], 360)
 return train<prepare_x_and_y>
3810*128
def voronoi_feats(train):
def shoelace(corners):
n = len(corners)
 area = 0.0
for i in range(n):
j =(i + 1)% n
area += corners[i][0] * corners[j][1]
area -= corners[j][0] * corners[i][1]
area = abs(area)/ 2.0
return area
def get_voronoi(play_id, train):
df = train[train.PlayId == play_id]
xy = df[['X_std', 'Y_std']].values
n_points = xy.shape[0]
xy1 = xy.copy()
xy1[:,1] = - xy[:,1]
xy2 = xy.copy()
xy2[:,1] = 320/3 - xy[:,1]
xy3 = xy.copy()
xy3[:,0] = 20 - xy[:,0]
xy4 = xy.copy()
xy4[:,0] = 220 - xy[:,0]
xy = np.concatenate(( xy, xy1, xy2, xy3, xy4), axis=0)
 offense = df.IsOnOffense.values
vor = Voronoi(xy)
 areas = []
for r in range(n_points):
region = vor.regions[vor.point_region[r]]
if not -1 in region:
polygon = [vor.vertices[i] for i in region]
polygon_area = shoelace(polygon)
 areas.append(polygon_area)
 fractions = [a/sum(areas)for a in areas]
return areas, fractions
plays = train.PlayId.unique()
train["Voronoi"] = 0
train["Voronoi_fraction"] = 0
for play in plays:
area, frac = get_voronoi(play, train)
 n = len(train[train.PlayId == play])
 while len(area)<n:
area.append(0)
 frac.append(0)
 train.loc[train.PlayId == play,"Voronoi"] = area[:n]
train.loc[train.PlayId == play,"Voronoi_fraction"] = frac[:n]
return train<feature_engineering>
df_train["target"] = y.repeat(128 )
def feature_engineering(train):

 train['OffenseTeam'] = train['PossessionTeam']
train['OffenseHome'] = train[['OffenseTeam','HomeTeamAbbr']].apply(lambda x: 1 if x[0] == x[1] else 0, axis = 1)
 train['DefenseTeam'] = train[['OffenseHome','HomeTeamAbbr','VisitorTeamAbbr']].apply(lambda x: x[2] if x[0] == 1 else x[1], axis = 1)
 train['OffenseLead'] = train[['OffenseHome','HomeScoreBeforePlay','VisitorScoreBeforePlay']].apply(lambda x: x[1]-x[2] if x[0] == 1 else x[2]-x[1], axis = 1)
 train['YardsToGo'] = train[['FieldPosition','OffenseTeam','YardLine']].apply(\
lambda x:(50-x['YardLine'])+50 if x['OffenseTeam']==x['FieldPosition'] else x['YardLine'],1)
 train["BMI"] = train["Height_feet"]/train["PlayerWeight"]
def dotpr(x):
return x[0]*x[1]*np.cos(x[2]-x[3])
 train["Speed_and_Wind_dot"] = train[["S",'WindSpeed',"Dir_std","WindDirection_std"]].apply(dotpr, axis=1)
 train["isRB"] = train["Position"].apply(lambda x: x == "RB")
 train['DefendersInTheBox_vs_Distance'] = train['DefendersInTheBox'] / train['Distance']

 train['Morning'] = train['GameClock'].apply(lambda x : 1 if(int(x[0:2])>=0 and int(x[0:2])<12)else 0)
 train['Afternoon'] = train['GameClock'].apply(lambda x : 1 if(int(x[0:2])<18 and int(x[0:2])>=12)else 0)
 train['Evening'] = train['GameClock'].apply(lambda x : 1 if(int(x[0:2])>= 18 and int(x[0:2])< 24)else 0)
 train['Hours'] = train['GameClock'].apply(lambda x : int(x[0:2]))
train.loc[train.FieldPosition == train.PossessionTeam,'YardsFromOwnGoal'] = train.loc[train.FieldPosition == train.PossessionTeam,'YardLine']
train.loc[train.FieldPosition != train.PossessionTeam,'YardsFromOwnGoal'] = 50 - train.loc[train.FieldPosition != train.PossessionTeam,'YardLine']
train["Snowing"] = train["GameWeather"].apply(lambda x: x == "snow")
 train['U'] = train['S'] * 1.75 * np.cos(train['Dir_std'])+ train['X_std']
train['W'] = train['S'] * 1.75 * np.sin(train['Dir_std'])+ train['Y_std']
train['Uo'] = train['S'] * 1.25 * np.cos(train['Orientation_std'])+ train['X_std']
train['Wo'] = train['S'] * 1.25 * np.sin(train['Orientation_std'])+ train['Y_std']
train = train.drop(['OffenseTeam',"DefenseTeam"], axis=1)
 def projection_features(df):
rad = 2 * np.pi *(90 - df[['Orientation_std']])/ 360
v_0 = df['S'].values * np.cos(rad ).values.reshape(-1)
 v_1 = np.sin(rad ).values.reshape(-1)
 a_0 = df['A'].values * np.cos(rad ).values.reshape(-1)
 a_1 = np.sin(rad)
 df['v_0'] = v_0
df['v_1'] = v_1
df['a_0'] = a_0
df['a_1'] = a_1
return df
train = projection_features(train)
 def euclidean_distance(x1,y1,x2,y2):
x_diff =(x1-x2)**2
y_diff =(y1-y2)**2
return np.sqrt(x_diff + y_diff)
 return train<prepare_x_and_y>
def process_outliers(df, outlier_cols):
def fz_raw(x):
zscore =(x - x.mean())/x.std()
z_upper = x.mean() +3*x.std()
z_upper = np.repeat(z_upper, 128)
 z_lower = x.mean() -3*x.std()
z_lower = np.repeat(z_lower, 128)
 return np.array(( zscore, z_upper, z_lower))
cols_to_drop = []
for col in tqdm(outlier_cols):
a = df[col].values
series_groups = df.groupby("series_id")
 xyz = series_groups[col].apply(fz_raw)
 xyz_c = np.concatenate(xyz, axis=1)
 z = xyz_c[0]
mask = z >= 3
a[mask] = xyz_c[1][mask]
mask = z <= -3
a[mask] = xyz_c[2][mask]
def create_features(df, deploy=False):
def new_X(x_coordinate, play_direction):
if play_direction == 'left':
return 120.0 - x_coordinate
else:
return x_coordinate
def new_line(rush_team, field_position, yardline):
if rush_team == field_position:
return 10.0 + yardline
else:
return 60.0 +(50 - yardline)
 def new_orientation(angle, play_direction):
if play_direction == 'left':
new_angle = 360.0 - angle
if new_angle == 360.0:
new_angle = 0.0
return new_angle
else:
return angle
def euclidean_distance(x1,y1,x2,y2):
x_diff =(x1-x2)**2
y_diff =(y1-y2)**2
return np.sqrt(x_diff + y_diff)
 def back_direction(orientation):
if orientation > 180.0:
return 1
else:
return 0
def update_yardline(df):
new_yardline = df[df['NflId'] == df['NflIdRusher']]
new_yardline['YardLine'] = new_yardline[['PossessionTeam','FieldPosition','YardLine']].apply(lambda x: new_line(x[0],x[1],x[2]), axis=1)
 new_yardline = new_yardline[['GameId','PlayId','YardLine']]
return new_yardline
def update_orientation(df, yardline):
df['X'] = df[['X','PlayDirection']].apply(lambda x: new_X(x[0],x[1]), axis=1)
 df['Orientation'] = df[['Orientation','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)
 df['Dir'] = df[['Dir','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)
 df = df.drop('YardLine', axis=1)
 df = pd.merge(df, yardline, on=['GameId','PlayId'], how='inner')
 return df
def back_features(df):
carriers = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','NflIdRusher','X','Y','Orientation','Dir','YardLine']]
carriers['back_from_scrimmage'] = carriers['YardLine'] - carriers['X']
carriers['back_oriented_down_field'] = carriers['Orientation'].apply(lambda x: back_direction(x))
carriers['back_moving_down_field'] = carriers['Dir'].apply(lambda x: back_direction(x))
carriers = carriers.rename(columns={'X':'back_X',
'Y':'back_Y'})
 carriers = carriers[['GameId','PlayId','NflIdRusher','back_X','back_Y','back_from_scrimmage','back_oriented_down_field','back_moving_down_field']]
return carriers
def features_relative_to_back(df, carriers):
player_distance = df[['GameId','PlayId','NflId','X','Y']]
player_distance = pd.merge(player_distance, carriers, on=['GameId','PlayId'], how='inner')
 player_distance = player_distance[player_distance['NflId'] != player_distance['NflIdRusher']]
player_distance['dist_to_back'] = player_distance[['X','Y','back_X','back_Y']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 player_distance = player_distance.groupby(['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field'])\ 
.agg({'dist_to_back':['min','max','mean','std']})\ 
.reset_index()
player_distance.columns = ['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field',
'min_dist','max_dist','mean_dist','std_dist']
return player_distance
def defense_features(df):
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','Team','X','Y']]
rusher.columns = ['GameId','PlayId','RusherTeam','RusherX','RusherY']
defense = pd.merge(df,rusher,on=['GameId','PlayId'],how='inner')
 defense = defense[defense['Team'] != defense['RusherTeam']][['GameId','PlayId','X','Y','RusherX','RusherY']]
defense['def_dist_to_back'] = defense[['X','Y','RusherX','RusherY']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 defense = defense.groupby(['GameId','PlayId'])\ 
.agg({'def_dist_to_back':['min','max','mean','std']})\ 
.reset_index()
defense.columns = ['GameId','PlayId','def_min_dist','def_max_dist','def_mean_dist','def_std_dist']
return defense
def static_features(df, p_cols):
add_new_feas = []
df['PlayerHeight_dense'] = df['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
add_new_feas.append('PlayerHeight_dense')
 df['TimeHandoff'] = df['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeSnap'] = df['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeDelta'] = df.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 df['PlayerBirthDate'] =df['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
df['PlayerAge'] = df.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 add_new_feas.append('PlayerAge')
 df['GameWeather_process'] = df['GameWeather'].str.lower()
df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: "indoor" if not pd.isna(x)and "indoor" in x else x)
 df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 df['GameWeather_dense'] = df['GameWeather_process'].apply(map_weather)
 add_new_feas.append('GameWeather_dense')
 df["Orientation_ob"] = df["Orientation"].apply(lambda x : orientation_to_cat(x)).astype("object")
 df["Dir_ob"] = df["Dir"].apply(lambda x : orientation_to_cat(x)).astype("object")
 df["Orientation_sin"] = df["Orientation"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
df["Orientation_cos"] = df["Orientation"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
add_new_feas.append("Orientation_cos")
 add_new_feas.append("Orientation_sin")
 df["Dir_sin"] = df["Dir"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
df["Dir_cos"] = df["Dir"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
add_new_feas.append("Dir_sin")
 add_new_feas.append("Dir_cos")
 df["WindDir_sin"] = df["WindDirection_std"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
df["WindDir_cos"] = df["WindDirection_std"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
add_new_feas.append("WindDir_sin")
 add_new_feas.append("WindDir_cos")
 df["Force"] = df["PlayerWeight"]*df["A"]
df["diffScoreBeforePlay"] = df["HomeScoreBeforePlay"] - df["VisitorScoreBeforePlay"]
add_new_feas.append("diffScoreBeforePlay")
 add_new_feas.append("TimeDelta")
 fe = ['v_0', 'v_1',
"BMI",
"Force",
"Speed_and_Wind_dot",
"Voronoi", "Voronoi_fraction",
"play_control"]
def_f = [ 'secondsLeftInHalf',
'numericFormation',
"Defense_X_stdiv",
"Defense_Y_stdiv",
"Defense_Y_spread",
"Defense_X_spread",
"min_time_to_tackle",
"mean_time_to_tackle",
"dist_to_offense_centroid",
"dist_to_defense_centroid"]
static_features = df[df['NflId'] == df['NflIdRusher']][add_new_feas+fe+p_cols + def_f +['GameId','PlayId','X','Y','S','A','Dis','Orientation','Dir',
'YardLine','Quarter','Down','Distance','DefendersInTheBox']].drop_duplicates()
static_features.fillna(-999,inplace=True)
 return static_features
def split_personnel(s):
splits = s.split(',')
 for i in range(len(splits)) :
splits[i] = splits[i].strip()
return splits
def defense_formation(l):
dl = 0
lb = 0
db = 0
other = 0
for position in l:
sub_string = position.split(' ')
 if sub_string[1] == 'DL':
dl += int(sub_string[0])
 elif sub_string[1] in ['LB','OL']:
lb += int(sub_string[0])
 else:
db += int(sub_string[0])
 counts =(dl,lb,db,other)
 return counts
def offense_formation(l):
qb = 0
rb = 0
wr = 0
te = 0
ol = 0
sub_total = 0
qb_listed = False
for position in l:
sub_string = position.split(' ')
 pos = sub_string[1]
cnt = int(sub_string[0])
 if pos == 'QB':
qb += cnt
sub_total += cnt
qb_listed = True
elif pos in ['RB','LB']:
rb += cnt
sub_total += cnt
elif pos in ['WR','DB']:
wr += cnt
sub_total += cnt
elif pos == 'TE':
te += cnt
sub_total += cnt
else:
ol += cnt
sub_total += cnt
if sub_total < 11:
diff = 11 - sub_total
if not qb_listed:
qb += 1
diff -= 1
ol += diff
counts =(qb,rb,wr,te,ol)
 return counts
def personnel_features(df):
personnel = df[['GameId','PlayId','OffensePersonnel','DefensePersonnel']].drop_duplicates()
personnel['DefensePersonnel'] = personnel['DefensePersonnel'].apply(lambda x: split_personnel(x))
personnel['DefensePersonnel'] = personnel['DefensePersonnel'].apply(lambda x: defense_formation(x))
personnel['num_DL'] = personnel['DefensePersonnel'].apply(lambda x: x[0])
 personnel['num_LB'] = personnel['DefensePersonnel'].apply(lambda x: x[1])
 personnel['num_DB'] = personnel['DefensePersonnel'].apply(lambda x: x[2])
 personnel['OffensePersonnel'] = personnel['OffensePersonnel'].apply(lambda x: split_personnel(x))
personnel['OffensePersonnel'] = personnel['OffensePersonnel'].apply(lambda x: offense_formation(x))
personnel['num_QB'] = personnel['OffensePersonnel'].apply(lambda x: x[0])
 personnel['num_RB'] = personnel['OffensePersonnel'].apply(lambda x: x[1])
 personnel['num_WR'] = personnel['OffensePersonnel'].apply(lambda x: x[2])
 personnel['num_TE'] = personnel['OffensePersonnel'].apply(lambda x: x[3])
 personnel['num_OL'] = personnel['OffensePersonnel'].apply(lambda x: x[4])
 personnel['OL_diff'] = personnel['num_OL'] - personnel['num_DL']
personnel['OL_TE_diff'] =(personnel['num_OL'] + personnel['num_TE'])- personnel['num_DL']
personnel['run_def'] =(personnel['num_DL'] + personnel['num_LB'] > 6 ).astype(int)
 personnel.drop(['OffensePersonnel','DefensePersonnel'], axis=1, inplace=True)
 return personnel
def rusher_features(df):
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','Dir', 'S', 'A', 'X', 'Y']]
rusher.columns = ['GameId','PlayId', 'RusherDir', 'RusherS', 'RusherA', 'RusherX', 'RusherY']
radian_angle =(90 - rusher['RusherDir'])* np.pi / 180.0
v_horizontal = np.abs(rusher['RusherS'] * np.cos(radian_angle))
v_vertical = np.abs(rusher['RusherS'] * np.sin(radian_angle))
rusher['v_horizontal'] = v_horizontal
rusher['v_vertical'] = v_vertical
rusher.columns = ['GameId','PlayId', 'RusherDir',
'RusherS',
'RusherA','RusherX', 'RusherY','v_horizontal', 'v_vertical']
return rusher
def dist_to_rusher(x):
a =(x[0],x[1])
 b =(x[2],x[3])
 return euclidean(a,b)
 def combine_features(relative_to_back, defense,rushing, static, deploy=deploy):
df = pd.merge(relative_to_back,defense,on=['GameId','PlayId'],how='inner')
 df = pd.merge(df,rushing,on=['GameId','PlayId'],how='inner')
 df = pd.merge(df,static,on=['GameId','PlayId'],how='inner')
 if not deploy:
df = pd.merge(df, outcomes, on=['GameId','PlayId'], how='inner')
 return df
df = clean_features(df)
 df = fix_sides(df)
 df = feature_engineering(df)
 df = voronoi_feats(df)
 pc = get_play_control(df)
 df["play_control"] = pc["play_control"].values
for col in [ 'secondsLeftInHalf',
'numericFormation',
"Defense_X_stdiv",
"Defense_Y_stdiv",
"Defense_Y_spread",
"Defense_X_spread",
"min_time_to_tackle",
"mean_time_to_tackle",
"dist_to_offense_centroid",
"dist_to_defense_centroid"]:
df[col] = pc[col].values
pf, p_cols = positional(df)
 for col in p_cols:
df[col] = pf[col].values
yardline = update_yardline(df)
 df = update_orientation(df, yardline)
 back_feats = back_features(df)
 rel_back = features_relative_to_back(df, back_feats)
 def_feats = defense_features(df)
 static_feats = static_features(df, p_cols)
 rush_feats = rusher_features(df)
 personnel = personnel_features(df)
 basetable = combine_features(rel_back, def_feats,rush_feats,static_feats, deploy=deploy)
 basetable["s_delta_0"] = basetable["v_0"]-basetable['v_horizontal']
basetable["s_delta_1"] = basetable["v_1"]-basetable['v_vertical']
basetable.drop(["v_0","v_1","X","Y","S","A", "Force"],axis=1, inplace=True)
 return basetable<feature_engineering>
outlier_cols = [col_name for col_name in df_train.columns if col_name not in ["row_id", "series_id", "measurement_number"]]
outlier_cols.remove("orientation_X")
 outlier_cols.remove("orientation_Y")
 outlier_cols.remove("orientation_Z")
 outlier_cols.remove("orientation_W")
 outlier_cols.remove("target")
%time train_basetable = create_features(train, False )<prepare_x_and_y>
process_outliers(df_train, outlier_cols )
X = train_basetable.copy()
yards = X.Yards
y = np.zeros(( yards.shape[0], 199))
for idx, target in enumerate(list(yards)) :
y[idx][99 + target] = 1
X.drop(['GameId','PlayId','Yards'], axis=1, inplace=True )<feature_engineering>
process_outliers(df_test, outlier_cols )
columns = np.full(( corr.shape[0],), True, dtype=bool)
 for i in range(corr.shape[0]):
for j in range(i+1, corr.shape[0]):
if corr.iloc[i,j] >= 0.99:
if columns[j]:
columns[j] = False
feature_columns = X.columns.values
drop_columns = X.columns[columns == False].values
print(feature_columns)
 print(drop_columns )<normalization>
def quaternion_to_euler(x, y, z, w):
t0 = +2.0 *(w * x + y * z)
 t1 = +1.0 - 2.0 *(x * x + y * y)
 X = math.atan2(t0, t1)
 t2 = +2.0 *(w * y - z * x)
 t2 = +1.0 if t2 > +1.0 else t2
t2 = -1.0 if t2 < -1.0 else t2
Y = math.asin(t2)
 t3 = +2.0 *(w * z + x * y)
 t4 = +1.0 - 2.0 *(y * y + z * z)
 Z = math.atan2(t3, t4)
 return X, Y, Z
def fe(actual):
new = pd.DataFrame()
actual['total_angular_velocity'] =(actual['angular_velocity_X'] ** 2 + actual['angular_velocity_Y'] ** 2 + actual['angular_velocity_Z'] ** 2)** 0.5
actual['total_linear_acceleration'] =(actual['linear_acceleration_X'] ** 2 + actual['linear_acceleration_Y'] ** 2 + actual['linear_acceleration_Z'] ** 2)** 0.5
actual['acc_vs_vel'] = actual['total_linear_acceleration'] / actual['total_angular_velocity']
df = actual
xyz = np.vectorize(quaternion_to_euler )(df['orientation_X'], df['orientation_Y'], df['orientation_Z'], df['orientation_W'])
 actual['euler_x'] = xyz[0]
actual['euler_y'] = xyz[1]
actual['euler_z'] = xyz[2]
actual['total_angle'] =(actual['euler_x'] ** 2 + actual['euler_y'] ** 2 + actual['euler_z'] ** 2)** 5
actual['angle_vs_acc'] = actual['total_angle'] / actual['total_linear_acceleration']
actual['angle_vs_vel'] = actual['total_angle'] / actual['total_angular_velocity']
series_groups = df.groupby("series_id")
 def f1(x):
return np.mean(np.diff(np.abs(np.diff(x))))
def f2(x):
return np.mean(np.abs(np.diff(x)))
 def fx_raw(x):
diff = np.diff(x)
 diff = np.concatenate([[np.nan], diff])
 abs_diff = np.abs(diff)
 abs_diff_diff = np.abs(np.diff(abs_diff))
abs_diff_diff = np.concatenate([[np.nan], abs_diff_diff])
 raw_array = np.array(( abs_diff, abs_diff_diff, diff))
return raw_array
def fx_sum(x):
abs_max = np.max(np.abs(x))
abs_min = np.min(np.abs(x))
abs_diff = np.abs(np.diff(x))
abs_diff_avg = np.mean(abs_diff)
 abs_diff_diff = np.abs(np.diff(abs_diff))
abs_diff_diff_avg = np.mean(abs_diff_diff)
 sum_diff = np.sum(np.diff(x))
sum_abs_diff = np.sum(abs_diff)
 count_diffs = np.count_nonzero(x)
 sum_array = np.array(( abs_max, abs_min, abs_diff_avg, abs_diff_diff_avg,
sum_diff, sum_abs_diff, count_diffs))
return sum_array
for col in tqdm(actual.columns):
if col in ['row_id', 'series_id', 'measurement_number', 'target']:
continue
new[col + '_mean'] = series_groups[col].mean()
new[col + '_min'] = series_groups[col].min()
new[col + '_max'] = series_groups[col].max()
new[col + '_std'] = series_groups[col].std()
new[col + '_max_to_min'] = new[col + '_max'] / new[col + '_min']
xyz = series_groups[col].apply(fx_sum)
 rows = len(series_groups)
 col_count = len(xyz[0])
 xyz_sum = np.concatenate(xyz ).reshape(rows, col_count)
 column_names = [col+"_abs_max", col+"_abs_min",
col+"_abs_diff_avg", col+"_abs_diff_diff_avg",
col+"_sum_diff", col+"_sum_abs_diff",
col+"_count_diff"
]
df_xyz_sum = pd.DataFrame(xyz_sum, columns=column_names)
 new = pd.concat([new, df_xyz_sum], axis=1)
 xyz = series_groups[col].apply(fx_raw)
 xyz_c = np.concatenate(xyz, axis=1)
 df_xyz_raw = pd.DataFrame({col+"_abs_diff": xyz_c[0],
col+"_abs_diff_diff": xyz_c[1],
col+"_diff": xyz_c[2]
})
 actual = pd.concat([actual, df_xyz_raw], axis=1)
 return new, actual
scaler = StandardScaler()
X = scaler.fit_transform(X[feature_columns] )<split>
%%time
df_train_sum, df_train = fe(df_train )
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.15, random_state=12345 )<compute_test_metric>
%%time
df_test_sum, df_test = fe(df_test )
def crps_score(y_prediction, y_valid, shape=X.shape[0]):
y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_prediction, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * shape)
 crps = np.round(val_s, 6)
 return crps<import_modules>
corr_matrix = df_train.corr().abs()
sol =(corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool)) 
.stack() 
.sort_values(ascending=False))
top_df = pd.DataFrame(sol ).reset_index()
class CRPSCallback(Callback):
def __init__(self,validation, predict_batch_size=20, include_on_batch=False):
super(CRPSCallback, self ).__init__()
self.validation = validation
self.predict_batch_size = predict_batch_size
self.include_on_batch = include_on_batch
print('validation shape',len(self.validation))
def on_batch_begin(self, batch, logs={}):
pass
def on_train_begin(self, logs={}):
if not('CRPS_score_val' in self.params['metrics']):
self.params['metrics'].append('CRPS_score_val')
 def on_batch_end(self, batch, logs={}):
if(self.include_on_batch):
logs['CRPS_score_val'] = float('-inf')
 def on_epoch_end(self, epoch, logs={}):
logs['CRPS_score_val'] = float('-inf')
 if(self.validation):
X_valid, y_valid = self.validation[0], self.validation[1]
y_pred = self.model.predict(X_valid)
 y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * X_valid.shape[0])
 val_s = np.round(val_s, 6)
 logs['CRPS_score_val'] = val_s 
<compute_train_metric>
top_df.columns = ["var1", "var2", "corr"]
top_df.head(10 )
def get_model(x_tr,y_tr,x_val,y_val):
inp = Input(shape =(x_tr.shape[1],))
x = Dense(1024, input_dim=X.shape[1] )(inp)
 x = LeakyReLU(alpha=0.3 )(x)
 x = Dropout(0.2 )(x)
 x = BatchNormalization()(x)
 x = Dense(512 )(x)
 x = LeakyReLU(alpha=0.2 )(x)
 x = Dropout(0.2 )(x)
 x = BatchNormalization()(x)
 x = Dense(256 )(x)
 x = LeakyReLU(alpha=0.1 )(x)
 x = Dropout(0.2 )(x)
 x = BatchNormalization()(x)
 out = Dense(199, activation='softmax' )(x)
 model = Model(inp,out)
 model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=[])
 es = EarlyStopping(monitor='CRPS_score_val',
mode='min',
restore_best_weights=True,
verbose=1,
patience=20)
 mc = ModelCheckpoint('best_model.h5',monitor='CRPS_score_val',mode='min',
save_best_only=True, verbose=1, save_weights_only=True)
 bsz = 1024
steps = x_tr.shape[0]/bsz
model.fit(x_tr, y_tr,callbacks=[CRPSCallback(validation =(x_val,y_val)) ,es,mc], epochs=100, batch_size=bsz,verbose=1)
 model.load_weights("best_model.h5")
 y_pred = model.predict(x_val)
 y_valid = y_val
y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * x_val.shape[0])
 crps = np.round(val_s, 6)
 return model,crps<split>
for x in range(len(top_df)) :
var1 = top_df.iloc[x]["var1"]
var2 = top_df.iloc[x]["var2"]
corr = cor[var1][var2]
top_df.at[x, "corr2"] = corr
top_df.head(20 )
losses = []
models = []
crps_csv = []
s_time = time.time()
for k in range(2):
kfold = KFold(5, random_state = 42 + k, shuffle = True)
 for k_fold,(tr_inds, val_inds)in enumerate(kfold.split(yards)) :
print("-----------")
 print("-----------")
 tr_x,tr_y = X[tr_inds],y[tr_inds]
val_x,val_y = X[val_inds],y[val_inds]
model,crps = get_model(tr_x,tr_y,val_x,val_y)
 models.append(model)
 print("the %d fold crps is %f"%(( k_fold+1),crps))
crps_csv.append(crps)
 print("mean crps is %f"%np.mean(crps_csv))
def predict(x_te):
model_num = len(models)
 for k,m in enumerate(models):
if k==0:
y_pred = m.predict(x_te,batch_size=1024)
 else:
y_pred+=m.predict(x_te,batch_size=1024)
 y_pred = y_pred / model_num
return y_pred 
<init_hyperparams>
df_train.groupby("target" ).mean()
<statistical_test>
feature_cols = list(df_train.columns)
 feature_cols.remove('row_id')
 feature_cols.remove('series_id')
 feature_cols.remove('measurement_number')
 feature_cols.remove('target')
def get_model(x_tr,y_tr,x_val,y_val):
inp = Input(shape =(x_tr.shape[1],))
x = Dense(1024, input_dim=X.shape[1] )(inp)
 x = LeakyReLU(alpha=0.3 )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(512 )(x)
 x = LeakyReLU(alpha=0.2 )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(512 )(x)
 x = LeakyReLU(alpha=0.2 )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(256 )(x)
 x = LeakyReLU(alpha=0.1 )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 out = Dense(199, activation='softmax' )(x)
 model = Model(inp,out)
 model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=[])
 es = EarlyStopping(monitor='CRPS_score_val',
mode='min',
restore_best_weights=True,
verbose=1,
patience=10)
 mc = ModelCheckpoint('best_model1.h5',monitor='CRPS_score_val',mode='min',
save_best_only=True, verbose=1, save_weights_only=True)
 bsz = 1024
steps = x_tr.shape[0]/bsz
model.fit(x_tr, y_tr,callbacks=[CRPSCallback(validation =(x_val,y_val)) ,es,mc], epochs=100, batch_size=bsz,verbose=1)
 model.load_weights("best_model1.h5")
 y_pred = model.predict(x_val)
 y_valid = y_val
y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * x_val.shape[0])
 crps = np.round(val_s, 6)
 return model,crps<split>
df_train.to_csv("features_train_raw.csv")
 df_test.to_csv("features_test_raw.csv")
 df_train_sum.to_csv("features_train_sum.csv")
 df_test_sum.to_csv("features_test_sum.csv" )
s_time = time.time()
for k in range(1):
kfold = KFold(5, random_state = 42 + k, shuffle = True)
 for k_fold,(tr_inds, val_inds)in enumerate(kfold.split(yards)) :
print("-----------")
 print("-----------")
 tr_x,tr_y = X[tr_inds],y[tr_inds]
val_x,val_y = X[val_inds],y[val_inds]
model,crps = get_model(tr_x,tr_y,val_x,val_y)
 models.append(model)
 print("the %d fold crps is %f"%(( k_fold+1),crps))
crps_csv.append(crps)
 print("mean crps is %f"%np.mean(crps_csv))
def predict(x_te):
model_num = len(models)
 for k,m in enumerate(models):
if k==0:
y_pred = m.predict(x_te,batch_size=1024)
 else:
y_pred+=m.predict(x_te,batch_size=1024)
 y_pred = y_pred / model_num
return y_pred 
<choose_model_class>
def scale_df(df1, df2, feature_cols):
scaler = StandardScaler()
scaled_features = scaler.fit_transform(df1[feature_cols])
 df1[feature_cols] = scaled_features
scaled_test_features = scaler.transform(df2[feature_cols])
 df2[feature_cols] = scaled_test_features
return df1, df2
def get_model(x_tr,y_tr,x_val,y_val):
inp = Input(shape =(x_tr.shape[1],))
x = Dense(512, input_dim=X.shape[1] )(inp)
 x = LeakyReLU(alpha=0.2 )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(512 )(x)
 x = LeakyReLU(alpha=0.2 )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 out = Dense(199, activation='softmax' )(x)
 model = Model(inp,out)
 model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=[])
 es = EarlyStopping(monitor='CRPS_score_val',
mode='min',
restore_best_weights=True,
verbose=1,
patience=10)
 mc = ModelCheckpoint('best_model2.h5',monitor='CRPS_score_val',mode='min',
save_best_only=True, verbose=1, save_weights_only=True)
 bsz = 1024
steps = x_tr.shape[0]/bsz
model.fit(x_tr, y_tr,callbacks=[CRPSCallback(validation =(x_val,y_val)) ,es,mc], epochs=100, batch_size=bsz,verbose=1)
 model.load_weights("best_model2.h5")
 y_pred = model.predict(x_val)
 y_valid = y_val
y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * x_val.shape[0])
 crps = np.round(val_s, 6)
 return model,crps<split>
df_train, df_test = scale_df(df_train, df_test, feature_cols )
s_time = time.time()
for k in range(1):
kfold = KFold(5, random_state = 42 + k, shuffle = True)
 for k_fold,(tr_inds, val_inds)in enumerate(kfold.split(yards)) :
print("-----------")
 print("-----------")
 tr_x,tr_y = X[tr_inds],y[tr_inds]
val_x,val_y = X[val_inds],y[val_inds]
model,crps = get_model(tr_x,tr_y,val_x,val_y)
 models.append(model)
 print("the %d fold crps is %f"%(( k_fold+1),crps))
crps_csv.append(crps)
 print("mean crps is %f"%np.mean(crps_csv))
def predict(x_te):
model_num = len(models)
 for k,m in enumerate(models):
if k==0:
y_pred = m.predict(x_te,batch_size=1024)
 else:
y_pred+=m.predict(x_te,batch_size=1024)
 y_pred = y_pred / model_num
return y_pred 
<compute_test_metric>
df_train_sum, df_test_sum = scale_df(df_train_sum, df_test_sum, df_train_sum.columns )
<compute_test_metric>
df_train.to_csv("features_train_scaled_raw.csv")
 df_test.to_csv("features_test_scaled_raw.csv")
 df_train_sum.to_csv("features_train_scaled_sum.csv")
 df_test_sum.to_csv("features_test_scaled_sum.csv" )
<predict_on_test>
df_features = df_train[feature_cols]
<predict_on_test>
folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=69)
 sub_preds_rf = np.zeros(( df_test_sum.shape[0], 9))
oof_preds_rf = np.zeros(( df_train_sum.shape[0]))
score = 0
misclassified_indices = []
misclassified_tuples_all = []
for i,(train_index, test_index)in enumerate(folds.split(df_train_sum, y)) :
print('-'*20, i, '-'*20)
 clf = RandomForestClassifier(n_estimators = 200, n_jobs = -1)
 clf.fit(df_train_sum.iloc[train_index], y[train_index])
 oof_preds_rf[test_index] = clf.predict(df_train_sum.iloc[test_index])
 sub_preds_rf += clf.predict_proba(df_test_sum)/ folds.n_splits
score += clf.score(df_train_sum.iloc[test_index], y[test_index])
 print('score ', clf.score(df_train_sum.iloc[test_index], y[test_index]))
importances = clf.feature_importances_
features = df_train_sum.columns
feat_importances = pd.Series(importances, index=features)
 feat_importances.nlargest(30 ).sort_values().plot(kind='barh', color='
plt.show()
missed = y[test_index] != oof_preds_rf[test_index]
misclassified_indices.append(test_index[missed])
 misclassified_samples1 = y[test_index][missed]
misclassified_samples2 = oof_preds_rf[test_index][missed].astype("int")
 m1 = encoder.inverse_transform(misclassified_samples1)
 m2 = encoder.inverse_transform(misclassified_samples2)
 misclassified_tuples = [(a, b)for a, b in zip(m1, m2)]
misclassified_tuples_all.append(misclassified_tuples)
 print('Avg Accuracy', score / folds.n_splits)
<predict_on_test>
ss = pd.read_csv('.. /input/sample_submission.csv')
 ss['surface'] = encoder.inverse_transform(sub_preds_rf.argmax(axis=1))
ss.to_csv('rf.csv', index=False)
 ss.head(10 )
%%time
if TRAIN_OFFLINE==False:
env = nflrush.make_env()
iter_test = env.iter_test()
for(test_df, sample_prediction_df)in iter_test:
basetable = create_features(test_df, deploy=True)
 basetable.drop(['GameId','PlayId'], axis=1, inplace=True)
 scaled_basetable = scaler.transform(basetable)
 y_pred = predict(scaled_basetable)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1 ).tolist() [0]
preds_df = pd.DataFrame(data=[y_pred], columns=sample_prediction_df.columns)
 env.predict(preds_df)
 env.write_submission_file()<feature_engineering>
df_missed_all = None
total_counter = collections.Counter()
def counter_to_df(df_missed_all, counter, new_col_name):
sorted_freqs = counter.most_common()
rows = [[row[0][0], row[0][1], row[1]] for row in sorted_freqs]
df_missed = pd.DataFrame(rows, columns=["expected", "actual", "count"])
 if df_missed_all is None:
df_missed_all = df_missed
else:
df_missed_all = df_missed_all.merge(df_missed, how='left', on=["expected", "actual"])
 created_col = df_missed_all.columns[-1]
print("renaming: "+new_col_name)
 df_missed_all.rename(columns={created_col: new_col_name}, inplace=True)
 df_missed_all[new_col_name].fillna(0, inplace=True)
 df_missed_all[new_col_name] = df_missed_all[new_col_name].astype(int)
 return df_missed_all
i = 1
for misclassified_tuples in misclassified_tuples_all:
counter = collections.Counter(misclassified_tuples)
 total_counter.update(misclassified_tuples)
 df_missed_all = counter_to_df(df_missed_all, counter, "count"+str(i))
i += 1
df_missed_all = counter_to_df(df_missed_all, total_counter, "all")
TRAIN_ABLE_FALSE=True
if TRAIN_ABLE_FALSE:
os.environ['CUDA_VISIBLE_DEVICES'] = "1"
TRAIN_OFFLINE = False
pd.set_option('display.max_columns', 50)
 pd.set_option('display.max_rows', 150 )<load_from_csv>
df_missed_all.sort_values(by="all", ascending=False )
if TRAIN_OFFLINE:
train = pd.read_csv('.. /input/train.csv', dtype={'WindSpeed': 'object'})
 else:
train = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2020/train.csv', dtype={'WindSpeed': 'object'})
<remove_duplicates>
miss_map = {}
hit_map = {}
ids, counts = np.unique(y, return_counts=True)
 totals = dict(zip(ids, counts))
total_misses = 0
total_hits = 0
for label_id in label_mapping.keys() :
label_name = label_mapping[label_id]
misses = df_missed_all[df_missed_all["expected"] == label_name]["all"].sum()
total_misses += misses
miss_map[label_name] = misses
total = totals[label_id]
hits = total - misses
total_hits += hits
hit_map[label_name] = hits
print(miss_map)
 print(hit_map)
 print(total_misses)
 print(total_hits)
 print(total_misses+total_hits )
outcomes = train[['GameId','PlayId','Yards']].drop_duplicates()<merge>
df_hit_miss = pd.DataFrame.from_dict(hit_map, orient="index")
 df_miss = pd.DataFrame.from_dict(miss_map, orient="index")
 df_hit_miss = df_hit_miss.join(df_miss, how='outer', lsuffix='_left', rsuffix='_right')
 df_hit_miss.columns = ["hits", "misses"]
df_hit_miss.sort_values(by="hits", ascending=False )
<merge><EOS> 
idx_counter = collections.Counter()
for missed_idx in misclassified_indices:
idx_counter.update(missed_idx )