Kaggle competition: CareerCon 2019 - Help Navigate Robots
<SOS> metric: categorizationaccuracy Kaggle data source: career-con-2019<prepare_x_and_y>
%matplotlib inline
le = preprocessing.LabelEncoder()
style.use('ggplot')
 warnings.filterwarnings('ignore')
 gc.enable()
!ls.. /input/
!ls.. /input/robots-best-submission
print("Ready !" )
@np.vectorize
def influence(RADIUS, S_ratio, theta, RusherX, RusherY, speed, X_std, Y_std):
player_coords = np.array([X_std, Y_std])
 point = np.array([RusherX, RusherY])
 S_matrix = np.matrix([[RADIUS *(1 + S_ratio), 0], [0, RADIUS *(1 - S_ratio)]])
 R_matrix = np.matrix([[np.cos(theta), - np.sin(theta)], [np.sin(theta), np.cos(theta)]])
 COV_matrix = np.dot(np.dot(np.dot(R_matrix, S_matrix), S_matrix), np.linalg.inv(R_matrix))
norm_fact =(1 / 2 * np.pi)*(1 / np.sqrt(np.linalg.det(COV_matrix)))
 mu_play = player_coords + speed * np.array([np.cos(theta), np.sin(theta)])/ 2
intermed_scalar_player = np.dot(np.dot(( player_coords - mu_play),
np.linalg.inv(COV_matrix)) ,
np.transpose(( player_coords - mu_play)))
 player_influence = norm_fact * np.exp(- 0.5 * intermed_scalar_player[0, 0])
 intermed_scalar_point = np.dot(np.dot(( point - mu_play),
np.linalg.inv(COV_matrix)) ,
np.transpose(( point - mu_play)))
 point_influence = norm_fact * np.exp(- 0.5 * intermed_scalar_point[0, 0])
 return point_influence / player_influence<categorify>
data = pd.read_csv('.. /input/career-con-2019/X_train.csv')
 tr = pd.read_csv('.. /input/career-con-2019/X_train.csv')
 sub = pd.read_csv('.. /input/career-con-2019/sample_submission.csv')
 test = pd.read_csv('.. /input/career-con-2019/X_test.csv')
 target = pd.read_csv('.. /input/career-con-2019/y_train.csv')
 print("Data is ready !!" )
def strtoseconds(txt):
txt = txt.split(':')
 ans = int(txt[0])*60 + int(txt[1])+ int(txt[2])/60
return ans
def strtofloat(x):
try:
return float(x)
 except:
return -1
def map_weather(txt):
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt:
ans*=0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*3
if 'sunny' in txt or 'sun' in txt:
return ans*2
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt:
return -2*ans
if 'snow' in txt:
return -3*ans
return 0
def OffensePersonnelSplit(x):
dic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0, 'QB' : 0, 'RB' : 0, 'TE' : 0, 'WR' : 0}
for xx in x.split(","):
xxs = xx.split(" ")
 dic[xxs[-1]] = int(xxs[-2])
 return dic
def DefensePersonnelSplit(x):
dic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0}
for xx in x.split(","):
xxs = xx.split(" ")
 dic[xxs[-1]] = int(xxs[-2])
 return dic
def orientation_to_cat(x):
x = np.clip(x, 0, 360 - 1)
 try:
return str(int(x/15))
except:
return "nan"
def preprocess(train):
train['GameClock_sec'] = train['GameClock'].apply(strtoseconds)
 train["GameClock_minute"] = train["GameClock"].apply(lambda x : x.split(":")[0] ).astype("object")
 train['PlayerHeight_dense'] = train['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
train['TimeHandoff'] = train['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeSnap'] = train['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeDelta'] = train.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 train['PlayerBirthDate'] = train['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
train['PlayerAge'] = train.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 train["PlayerAge_ob"] = train['PlayerAge'].astype(np.int ).astype("object")
 train['GameWeather_process'] = train['GameWeather'].str.lower()
train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: "indoor" if not pd.isna(x)and "indoor" in x else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 train['GameWeather_dense'] = train['GameWeather_process'].apply(map_weather)
 train['IsRusher'] =(train['NflId'] == train['NflIdRusher'])
 train['IsRusher_ob'] =(train['NflId'] == train['NflIdRusher'] ).astype("object")
 temp = train[train["IsRusher"]][["Team", "PlayId"]].rename(columns={"Team":"RusherTeam"})
 train = train.merge(temp, on = "PlayId")
 train["IsRusherTeam"] = train["Team"] == train["RusherTeam"]
train["Quarter_ob"] = train["Quarter"].astype("object")
 train["Down_ob"] = train["Down"].astype("object")
 train["JerseyNumber_ob"] = train["JerseyNumber"].astype("object")
 train["YardLine_ob"] = train["YardLine"].astype("object")
 train["Orientation_ob"] = train["Orientation"].apply(lambda x : orientation_to_cat(x)).astype("object")
 train["Dir_ob"] = train["Dir"].apply(lambda x : orientation_to_cat(x)).astype("object")
 train["Orientation_sin"] = train["Orientation"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
train["Orientation_cos"] = train["Orientation"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
train["Dir_sin"] = train["Dir"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
train["Dir_cos"] = train["Dir"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
train["diffScoreBeforePlay"] = train["HomeScoreBeforePlay"] - train["VisitorScoreBeforePlay"]
train["diffScoreBeforePlay_binary_ob"] =(train["HomeScoreBeforePlay"] > train["VisitorScoreBeforePlay"] ).astype("object")
 Turf = {'Field Turf':'Artificial', 'A-Turf Titan':'Artificial', 'Grass':'Natural', 'UBU Sports Speed S5-M':'Artificial', 'Artificial':'Artificial', 'DD GrassMaster':'Artificial', 'Natural Grass':'Natural', 'UBU Speed Series-S5-M':'Artificial', 'FieldTurf':'Artificial', 'FieldTurf 360':'Artificial', 'Natural grass':'Natural', 'grass':'Natural', 'Natural':'Natural', 'Artifical':'Artificial', 'FieldTurf360':'Artificial', 'Naturall Grass':'Natural', 'Field turf':'Artificial', 'SISGrass':'Artificial', 'Twenty-Four/Seven Turf':'Artificial', 'natural grass':'Natural'}
train['Turf'] = train['Turf'].map(Turf)
 temp = train["OffensePersonnel"].iloc[np.arange(0, len(train), 22)].apply(lambda x : pd.Series(OffensePersonnelSplit(x)))
 temp.columns = ["Offense" + c for c in temp.columns]
temp["PlayId"] = train["PlayId"].iloc[np.arange(0, len(train), 22)]
train = train.merge(temp, on = "PlayId")
 temp = train["DefensePersonnel"].iloc[np.arange(0, len(train), 22)].apply(lambda x : pd.Series(DefensePersonnelSplit(x)))
 temp.columns = ["Defense" + c for c in temp.columns]
temp["PlayId"] = train["PlayId"].iloc[np.arange(0, len(train), 22)]
train = train.merge(temp, on = "PlayId")
 train = train.sort_values(by = ['X'] ).sort_values(by = ['Dis'] ).sort_values(by=['PlayId', 'IsRusherTeam', 'IsRusher'] ).reset_index(drop = True)
 return train<feature_engineering>
len(data.measurement_number.value_counts() )
def clean_features(train):

 try:
train.loc[train.VisitorTeamAbbr == "ARI", 'VisitorTeamAbbr'] = "ARZ"
train.loc[train.HomeTeamAbbr == "ARI", 'HomeTeamAbbr'] = "ARZ"
train.loc[train.VisitorTeamAbbr == "BAL", 'VisitorTeamAbbr'] = "BLT"
train.loc[train.HomeTeamAbbr == "BAL", 'HomeTeamAbbr'] = "BLT"
train.loc[train.VisitorTeamAbbr == "CLE", 'VisitorTeamAbbr'] = "CLV"
train.loc[train.HomeTeamAbbr == "CLE", 'HomeTeamAbbr'] = "CLV"
train.loc[train.VisitorTeamAbbr == "HOU", 'VisitorTeamAbbr'] = "HST"
train.loc[train.HomeTeamAbbr == "HOU", 'HomeTeamAbbr'] = "HST"
except:
pass
print("...| Wind Direction")
 try:
train['WindDirection_std'] = train['WindDirection'].fillna(0)
 except:
train['WindDirection_std'] = 0
train['WindDirection'] = "unknown"
north = ['N','From S','North']
south = ['S','From N','South','s']
west = ['W','From E','West']
east = ['E','From W','from W','EAST','East']
north_east = ['FROM SW','NE','NORTH EAST','North East','NorthEast','Northeast','From SW']
north_west = ['E','NW','NORTHWEST', 'Northwest']
south_east = ['SE','SOUTHEAST','Southeast']
south_west = ['SW','SOUTHWEST','SouthWest','Southwest']
nne = ["NNE", "From SSW", "N-NE", "North Northeast"]
ene = ["East North East", 'ENE','From WSW']
ese = ['East Southeast', 'ESE','From WNW']
sse = [ 'From NNW', 'South Southeast','SSE']
ssw = [ 'From NNE', 'South Southwest','SSW']
wsw = [ 'From ENE', 'W-SW','West-Southwest','WSW']
wnw = [ 'From ESE', 'W-NW','WNW','West Northwest']
nnw = [ 'From SSE','North/Northwest','NNW']
def clean_wind_dir(x):
if x in north:
return 0
elif x in south:
return 180
elif x in west:
return 270
elif x in east:
return 90
elif x in north_east:
return 45
elif x in north_west:
return 315
elif x in south_east:
return 135
elif x in south_west:
return 225
elif x in nne:
return 20
elif x in ene:
return 70
elif x in sse:
return 110
elif x in ssw:
return 200
elif x in wsw:
return 250
elif x in wnw:
return 290
elif x in nnw:
return 340
else:
return 0
train['WindDirection_std'] = train['WindDirection_std'].apply(clean_wind_dir)

print("...| Wind Speed")
 def give_me_WindSpeed(txt):
txt = str(txt ).lower().replace('mph', '' ).strip()
if '-' in txt:
txt =(int(txt.split('-')[0])+ int(txt.split('-')[1])) / 2
try:
return float(txt)
 except:
return -1
train['WindSpeed'] = train['WindSpeed'].apply(give_me_WindSpeed)
 print("...| Weather")
 def give_me_GameWeather(txt):
txt = str(txt ).lower().replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly' ).replace('clear and sunny', 'sunny and clear')
 txt = txt.replace('skies', '' ).replace("mostly", "" ).strip()
if "indoor" in txt:
txt = "indoor"
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt:
ans*=0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*5
if 'sunny' in txt or 'sun' in txt:
return ans*3
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt:
return -3*ans
if 'snow' in txt:
return -5*ans
return 0
train['GameWeather_std'] = train['GameWeather'].apply(give_me_GameWeather)
 rain = ['Rainy', 'Rain Chance 40%', 'Showers','Cloudy with periods of rain, thunder possible.Winds shifting to WNW, 10-20 mph.',
'Scattered Showers', 'Cloudy, Rain', 'Rain shower', 'Light Rain', 'Rain']
overcast = ['Cloudy, light snow accumulating 1-3"', 'Party Cloudy', 'Cloudy, chance of rain',
'Coudy', 'Cloudy, 50% change of rain', 'Rain likely, temps in low 40s.',
'Cloudy and cold', 'Cloudy, fog started developing in 2nd quarter',
'Partly Clouidy', '30% Chance of Rain', 'Mostly Coudy', 'Cloudy and Cool',
'cloudy', 'Partly cloudy', 'Overcast', 'Hazy', 'Mostly cloudy', 'Mostly Cloudy',
'Partly Cloudy', 'Cloudy']
clear = ['Partly clear', 'Sunny and clear', 'Sun & clouds', 'Clear and Sunny',
'Sunny and cold', 'Sunny Skies', 'Clear and Cool', 'Clear and sunny',
'Sunny, highs to upper 80s', 'Mostly Sunny Skies', 'Cold',
'Clear and warm', 'Sunny and warm', 'Clear and cold', 'Mostly sunny',
'T: 51; H: 55; W: NW 10 mph', 'Clear Skies', 'Clear skies', 'Partly sunny',
'Fair', 'Partly Sunny', 'Mostly Sunny', 'Clear', 'Sunny', 'Sunny, Windy']
snow = ['Heavy lake effect snow', 'Snow']
none = ['N/A Indoor', 'Indoors', 'Indoor', 'N/A(Indoors)', 'Controlled Climate']
def clean_weather(x):
if x in rain:
return "rain"
elif x in overcast:
return "overcast"
elif x in clear:
return "clear"
elif x in snow:
return "snow"
else:
return "other"
train['GameWeather'] = train['GameWeather'].apply(clean_weather)
 print("...| Turf")
 def clean_turf(x):
if "art" in x:
return 0
if "grass" in x:
return 1
if "natural" in x:
return 1
if "turf" in x:
return 0
else:
return 0
train["Turf_std"] = train["Turf"].apply(lambda x: clean_turf(x.lower()))
print("...| Stadium")
 stadium_replace = {
"Twickenham Stadium":"Twickenham",
"Los Angeles Memorial Coliesum":"Los Angeles Memorial Coliseum",
"FirstEnergy Stadium":"FirstEnergyStadium",
"FirstEnergy":"FirstEnergyStadium",
"First Energy Stadium":"FirstEnergyStadium",
"M&T Bank Stadium":"M & T Bank Stadium",
"M&T Stadium": "M & T Bank Stadium",
"Broncos Stadium at Mile High":"Broncos Stadium At Mile High",
"Mercedes-Benz Dome":"Mercedes-Benz Stadium",
"Mercedes-Benz Superdome": "Mercedes-Benz Stadium",
"MetLife Stadium":"Metlife Stadium",
"MetLife":"Metlife Stadium",
}
for key, value in stadium_replace.items() :
train["Stadium"] = train["Stadium"].replace(key, value)
 outdoor = [
'Outdoor', 'Outdoors', 'Cloudy', 'Heinz Field',
'Outdor', 'Ourdoor', 'Outside', 'Outddors',
'Outdoor Retr Roof-Open', 'Oudoor', 'Bowl'
]
indoor_closed = [
'Indoors', 'Indoor', 'Indoor, Roof Closed', 'Indoor, Roof Closed',
'Retractable Roof', 'Retr.Roof-Closed', 'Retr.Roof - Closed', 'Retr.Roof Closed',
]
indoor_open = ['Indoor, Open Roof', 'Open', 'Retr.Roof-Open', 'Retr.Roof - Open']
dome_closed = ['Dome', 'Domed, closed', 'Closed Dome', 'Domed', 'Dome, closed']
dome_open = ['Domed, Open', 'Domed, open']
train['StadiumType'] = train['StadiumType'].replace(outdoor, "outdoor")
 train['StadiumType'] = train['StadiumType'].replace(indoor_closed, 'indoor closed')
 train['StadiumType'] = train['StadiumType'].replace(indoor_open, 'indoor open')
 train['StadiumType'] = train['StadiumType'].replace(dome_closed, 'dome closed')
 train['StadiumType'] = train['StadiumType'].replace(dome_open, 'dome open')
 def timesnap2day(x):
days = x.split("-")
 return 365 * int(days[0])+ 30 * int(days[1])+ int(days[2][:2])
 train['MatchDay'] = train['TimeSnap'].apply(timesnap2day)
 print("...| Time")
 def strtoseconds(txt):
txt = txt.split(':')
 return int(txt[0])*60 + int(txt[1])+ int(txt[2])/60
train['GameClock_std'] = train['GameClock'].apply(strtoseconds)
 train['Height_feet'] = train['PlayerHeight'].apply(lambda x: int(x.split('-')[0]))
return train<categorify>
totalt = data.isnull().sum().sort_values(ascending=False)
 percent =(data.isnull().sum() /data.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([totalt, percent], axis=1, keys=['Total', 'Percent'])
 print("Missing Data at Training")
 missing_data.tail()
def remove_NAs(train):
def na_categorical(x):
try:
return x.fillna(x.mode() [0])
 except:
return x.fillna(method="ffill")
 train["OffenseFormation"] = train.OffenseFormation.transform(lambda x: na_categorical(x))
train["StadiumType"] = train.groupby("Stadium")["StadiumType"].transform(lambda x: na_categorical(x))
train["DefendersInTheBox"] = train.groupby("DefensePersonnel")["DefendersInTheBox"].transform(lambda x: x.fillna(x.median()))
try:
train["GameWeather_std"] = train.groupby("Week")["GameWeather_std"].transform(lambda x: x.median())
 except:
train["GameWeather_std"] = train["GameWeather_std"].transform(lambda x: x.fillna(x.median()))
try:
train["Temperature"] = train.groupby("Week")["Temperature"].transform(lambda x: x.fillna(x.mean()))
except:
train['Temperature'] = train['Temperature'].fillna(train['Temperature'].median() , inplace=True)
 try:
train["WindSpeed"] = train.groupby("Week")["WindSpeed"].transform(lambda x: x.fillna(x.mean()))
except:
train["WindSpeed"] = train["WindSpeed"].transform(lambda x: x.fillna(x.mean()))
def wind_speed_closed(x):
if "closed" in x[0]:
return 0
train["StadiumClosed"] = train["StadiumType"].fillna("" ).apply(lambda x: "closed" in x)
 train['Temperature'] = train['Temperature'].fillna(75)
 train['Humidity'] = train['Humidity'].fillna(65)
 train["WindSpeed"] = train["WindSpeed"].fillna(10)
 return train<feature_engineering>
totalt = test.isnull().sum().sort_values(ascending=False)
 percent =(test.isnull().sum() /data.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([totalt, percent], axis=1, keys=['Total', 'Percent'])
 print("Missing Data at Test")
 missing_data.tail()
def fix_sides(train):

 train['ToLeft'] = train.PlayDirection == "left"
train['IsBallCarrier'] = train.NflId == train.NflIdRusher
train.loc[train.VisitorTeamAbbr == "ARI", 'VisitorTeamAbbr'] = "ARZ"
train.loc[train.HomeTeamAbbr == "ARI", 'HomeTeamAbbr'] = "ARZ"
train.loc[train.VisitorTeamAbbr == "BAL", 'VisitorTeamAbbr'] = "BLT"
train.loc[train.HomeTeamAbbr == "BAL", 'HomeTeamAbbr'] = "BLT"
train.loc[train.VisitorTeamAbbr == "CLE", 'VisitorTeamAbbr'] = "CLV"
train.loc[train.HomeTeamAbbr == "CLE", 'HomeTeamAbbr'] = "CLV"
train.loc[train.VisitorTeamAbbr == "HOU", 'VisitorTeamAbbr'] = "HST"
train.loc[train.HomeTeamAbbr == "HOU", 'HomeTeamAbbr'] = "HST"

 train['TeamOnOffense'] = "home"
train.loc[train.PossessionTeam != train.HomeTeamAbbr, 'TeamOnOffense'] = "away"
train['IsOnOffense'] = train.Team == train.TeamOnOffense
train['YardLine_std'] = 100 - train.YardLine
train.loc[train.FieldPosition.fillna('')== train.PossessionTeam,
'YardLine_std'
] = train.loc[train.FieldPosition.fillna('')== train.PossessionTeam,
'YardLine']
train['X_std'] = train.X
train.loc[train.ToLeft, 'X_std'] = 120 - train.loc[train.ToLeft, 'X']
train['Y_std'] = train.Y
train.loc[train.ToLeft, 'Y_std'] = 160/3 - train.loc[train.ToLeft, 'Y']
train['Orientation_std'] = train.Orientation
train.loc[train.ToLeft, 'Orientation_std'] = np.mod(180 + train.loc[train.ToLeft, 'Orientation_std'], 360)
 train['Dir_std'] = train.Dir
train.loc[train.ToLeft, 'Dir_std'] = np.mod(180 + train.loc[train.ToLeft, 'Dir_std'], 360)
 train.loc[train.ToLeft, 'WindDirection_std'] = np.mod(180 + train.loc[train.ToLeft, 'WindDirection_std'], 360)
 return train<prepare_x_and_y>
target.groupby('group_id' ).surface.nunique().max()
def voronoi_feats(train):
def shoelace(corners):
n = len(corners)
 area = 0.0
for i in range(n):
j =(i + 1)% n
area += corners[i][0] * corners[j][1]
area -= corners[j][0] * corners[i][1]
area = abs(area)/ 2.0
return area
def get_voronoi(play_id, train):
df = train[train.PlayId == play_id]
xy = df[['X_std', 'Y_std']].values
n_points = xy.shape[0]
xy1 = xy.copy()
xy1[:,1] = - xy[:,1]
xy2 = xy.copy()
xy2[:,1] = 320/3 - xy[:,1]
xy3 = xy.copy()
xy3[:,0] = 20 - xy[:,0]
xy4 = xy.copy()
xy4[:,0] = 220 - xy[:,0]
xy = np.concatenate(( xy, xy1, xy2, xy3, xy4), axis=0)
 offense = df.IsOnOffense.values
vor = Voronoi(xy)
 areas = []
for r in range(n_points):
region = vor.regions[vor.point_region[r]]
if not -1 in region:
polygon = [vor.vertices[i] for i in region]
polygon_area = shoelace(polygon)
 areas.append(polygon_area)
 fractions = [a/sum(areas)for a in areas]
return areas, fractions
plays = train.PlayId.unique()
train["Voronoi"] = 0
train["Voronoi_fraction"] = 0
for play in plays:
area, frac = get_voronoi(play, train)
 n = len(train[train.PlayId == play])
 while len(area)<n:
area.append(0)
 frac.append(0)
 train.loc[train.PlayId == play,"Voronoi"] = area[:n]
train.loc[train.PlayId == play,"Voronoi_fraction"] = frac[:n]
return train<feature_engineering>
target['group_id'].nunique()
def feature_engineering(train):

 train['OffenseTeam'] = train['PossessionTeam']
train['OffenseHome'] = train[['OffenseTeam','HomeTeamAbbr']].apply(lambda x: 1 if x[0] == x[1] else 0, axis = 1)
 train['DefenseTeam'] = train[['OffenseHome','HomeTeamAbbr','VisitorTeamAbbr']].apply(lambda x: x[2] if x[0] == 1 else x[1], axis = 1)
 train['OffenseLead'] = train[['OffenseHome','HomeScoreBeforePlay','VisitorScoreBeforePlay']].apply(lambda x: x[1]-x[2] if x[0] == 1 else x[2]-x[1], axis = 1)
 train['YardsToGo'] = train[['FieldPosition','OffenseTeam','YardLine']].apply(\
lambda x:(50-x['YardLine'])+50 if x['OffenseTeam']==x['FieldPosition'] else x['YardLine'],1)
 train["BMI"] = train["Height_feet"]/train["PlayerWeight"]
def dotpr(x):
return x[0]*x[1]*np.cos(x[2]-x[3])
 train["Speed_and_Wind_dot"] = train[["S",'WindSpeed',"Dir_std","WindDirection_std"]].apply(dotpr, axis=1)
 train["isRB"] = train["Position"].apply(lambda x: x == "RB")
 train['DefendersInTheBox_vs_Distance'] = train['DefendersInTheBox'] / train['Distance']

 train['Morning'] = train['GameClock'].apply(lambda x : 1 if(int(x[0:2])>=0 and int(x[0:2])<12)else 0)
 train['Afternoon'] = train['GameClock'].apply(lambda x : 1 if(int(x[0:2])<18 and int(x[0:2])>=12)else 0)
 train['Evening'] = train['GameClock'].apply(lambda x : 1 if(int(x[0:2])>= 18 and int(x[0:2])< 24)else 0)
 train['Hours'] = train['GameClock'].apply(lambda x : int(x[0:2]))
train.loc[train.FieldPosition == train.PossessionTeam,'YardsFromOwnGoal'] = train.loc[train.FieldPosition == train.PossessionTeam,'YardLine']
train.loc[train.FieldPosition != train.PossessionTeam,'YardsFromOwnGoal'] = 50 - train.loc[train.FieldPosition != train.PossessionTeam,'YardLine']
train["Snowing"] = train["GameWeather"].apply(lambda x: x == "snow")
 train['U'] = train['S'] * 1.75 * np.cos(train['Dir_std'])+ train['X_std']
train['W'] = train['S'] * 1.75 * np.sin(train['Dir_std'])+ train['Y_std']
train['Uo'] = train['S'] * 1.25 * np.cos(train['Orientation_std'])+ train['X_std']
train['Wo'] = train['S'] * 1.25 * np.sin(train['Orientation_std'])+ train['Y_std']
train = train.drop(['OffenseTeam',"DefenseTeam"], axis=1)
 def projection_features(df):
rad = 2 * np.pi *(90 - df[['Orientation_std']])/ 360
v_0 = df['S'].values * np.cos(rad ).values.reshape(-1)
 v_1 = np.sin(rad ).values.reshape(-1)
 a_0 = df['A'].values * np.cos(rad ).values.reshape(-1)
 a_1 = np.sin(rad)
 df['v_0'] = v_0
df['v_1'] = v_1
df['a_0'] = a_0
df['a_1'] = a_1
return df
train = projection_features(train)
 def euclidean_distance(x1,y1,x2,y2):
x_diff =(x1-x2)**2
y_diff =(y1-y2)**2
return np.sqrt(x_diff + y_diff)
 return train<prepare_x_and_y>
del serie1
gc.collect()
def create_features(df, deploy=False):
def new_X(x_coordinate, play_direction):
if play_direction == 'left':
return 120.0 - x_coordinate
else:
return x_coordinate
def new_line(rush_team, field_position, yardline):
if rush_team == field_position:
return 10.0 + yardline
else:
return 60.0 +(50 - yardline)
 def new_orientation(angle, play_direction):
if play_direction == 'left':
new_angle = 360.0 - angle
if new_angle == 360.0:
new_angle = 0.0
return new_angle
else:
return angle
def euclidean_distance(x1,y1,x2,y2):
x_diff =(x1-x2)**2
y_diff =(y1-y2)**2
return np.sqrt(x_diff + y_diff)
 def back_direction(orientation):
if orientation > 180.0:
return 1
else:
return 0
def update_yardline(df):
new_yardline = df[df['NflId'] == df['NflIdRusher']]
new_yardline['YardLine'] = new_yardline[['PossessionTeam','FieldPosition','YardLine']].apply(lambda x: new_line(x[0],x[1],x[2]), axis=1)
 new_yardline = new_yardline[['GameId','PlayId','YardLine']]
return new_yardline
def update_orientation(df, yardline):
df['X'] = df[['X','PlayDirection']].apply(lambda x: new_X(x[0],x[1]), axis=1)
 df['Orientation'] = df[['Orientation','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)
 df['Dir'] = df[['Dir','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)
 df = df.drop('YardLine', axis=1)
 df = pd.merge(df, yardline, on=['GameId','PlayId'], how='inner')
 return df
def back_features(df):
carriers = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','NflIdRusher','X','Y','Orientation','Dir','YardLine']]
carriers['back_from_scrimmage'] = carriers['YardLine'] - carriers['X']
carriers['back_oriented_down_field'] = carriers['Orientation'].apply(lambda x: back_direction(x))
carriers['back_moving_down_field'] = carriers['Dir'].apply(lambda x: back_direction(x))
carriers = carriers.rename(columns={'X':'back_X',
'Y':'back_Y'})
 carriers = carriers[['GameId','PlayId','NflIdRusher','back_X','back_Y','back_from_scrimmage','back_oriented_down_field','back_moving_down_field']]
return carriers
def features_relative_to_back(df, carriers):
player_distance = df[['GameId','PlayId','NflId','X','Y']]
player_distance = pd.merge(player_distance, carriers, on=['GameId','PlayId'], how='inner')
 player_distance = player_distance[player_distance['NflId'] != player_distance['NflIdRusher']]
player_distance['dist_to_back'] = player_distance[['X','Y','back_X','back_Y']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 player_distance = player_distance.groupby(['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field'])\ 
.agg({'dist_to_back':['min','max','mean','std']})\ 
.reset_index()
player_distance.columns = ['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field',
'min_dist','max_dist','mean_dist','std_dist']
return player_distance
def defense_features(df):
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','Team','X','Y']]
rusher.columns = ['GameId','PlayId','RusherTeam','RusherX','RusherY']
defense = pd.merge(df,rusher,on=['GameId','PlayId'],how='inner')
 defense = defense[defense['Team'] != defense['RusherTeam']][['GameId','PlayId','X','Y','RusherX','RusherY']]
defense['def_dist_to_back'] = defense[['X','Y','RusherX','RusherY']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 defense = defense.groupby(['GameId','PlayId'])\ 
.agg({'def_dist_to_back':['min','max','mean','std']})\ 
.reset_index()
defense.columns = ['GameId','PlayId','def_min_dist','def_max_dist','def_mean_dist','def_std_dist']
return defense
def static_features(df):
add_new_feas = []
df['PlayerHeight_dense'] = df['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
add_new_feas.append('PlayerHeight_dense')
 df['TimeHandoff'] = df['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeSnap'] = df['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeDelta'] = df.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 df['PlayerBirthDate'] =df['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
df['PlayerAge'] = df.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 add_new_feas.append('PlayerAge')
 df['GameWeather_process'] = df['GameWeather'].str.lower()
df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: "indoor" if not pd.isna(x)and "indoor" in x else x)
 df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 df['GameWeather_dense'] = df['GameWeather_process'].apply(map_weather)
 add_new_feas.append('GameWeather_dense')
 df["Orientation_ob"] = df["Orientation"].apply(lambda x : orientation_to_cat(x)).astype("object")
 df["Dir_ob"] = df["Dir"].apply(lambda x : orientation_to_cat(x)).astype("object")
 df["Orientation_sin"] = df["Orientation"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
df["Orientation_cos"] = df["Orientation"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
add_new_feas.append("Orientation_cos")
 add_new_feas.append("Orientation_sin")
 df["Dir_sin"] = df["Dir"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
df["Dir_cos"] = df["Dir"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
add_new_feas.append("Dir_sin")
 add_new_feas.append("Dir_cos")
 df["WindDir_sin"] = df["WindDirection_std"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
df["WindDir_cos"] = df["WindDirection_std"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
add_new_feas.append("WindDir_sin")
 add_new_feas.append("WindDir_cos")
 df["Force"] = df["PlayerWeight"]*df["A"]
df["diffScoreBeforePlay"] = df["HomeScoreBeforePlay"] - df["VisitorScoreBeforePlay"]
add_new_feas.append("diffScoreBeforePlay")
 add_new_feas.append("TimeDelta")
 fe = ['v_0', 'v_1',
"BMI",
"Force",
"Speed_and_Wind_dot",
"Voronoi", "Voronoi_fraction",
"play_control"]
def_f = [ 'secondsLeftInHalf',
'numericFormation',
"Defense_X_stdiv",
"Defense_Y_stdiv",
"Defense_Y_spread",
"Defense_X_spread",
"min_time_to_tackle",
"mean_time_to_tackle",
"dist_to_offense_centroid",
"dist_to_defense_centroid"]
static_features = df[df['NflId'] == df['NflIdRusher']][add_new_feas+fe+def_f +['GameId','PlayId','X','Y','S','A','Dis','Orientation','Dir',
'YardLine','Quarter','Down','Distance','DefendersInTheBox']].drop_duplicates()
static_features.fillna(-999,inplace=True)
 return static_features
def split_personnel(s):
splits = s.split(',')
 for i in range(len(splits)) :
splits[i] = splits[i].strip()
return splits
def defense_formation(l):
dl = 0
lb = 0
db = 0
other = 0
for position in l:
sub_string = position.split(' ')
 if sub_string[1] == 'DL':
dl += int(sub_string[0])
 elif sub_string[1] in ['LB','OL']:
lb += int(sub_string[0])
 else:
db += int(sub_string[0])
 counts =(dl,lb,db,other)
 return counts
def offense_formation(l):
qb = 0
rb = 0
wr = 0
te = 0
ol = 0
sub_total = 0
qb_listed = False
for position in l:
sub_string = position.split(' ')
 pos = sub_string[1]
cnt = int(sub_string[0])
 if pos == 'QB':
qb += cnt
sub_total += cnt
qb_listed = True
elif pos in ['RB','LB']:
rb += cnt
sub_total += cnt
elif pos in ['WR','DB']:
wr += cnt
sub_total += cnt
elif pos == 'TE':
te += cnt
sub_total += cnt
else:
ol += cnt
sub_total += cnt
if sub_total < 11:
diff = 11 - sub_total
if not qb_listed:
qb += 1
diff -= 1
ol += diff
counts =(qb,rb,wr,te,ol)
 return counts
def personnel_features(df):
personnel = df[['GameId','PlayId','OffensePersonnel','DefensePersonnel']].drop_duplicates()
personnel['DefensePersonnel'] = personnel['DefensePersonnel'].apply(lambda x: split_personnel(x))
personnel['DefensePersonnel'] = personnel['DefensePersonnel'].apply(lambda x: defense_formation(x))
personnel['num_DL'] = personnel['DefensePersonnel'].apply(lambda x: x[0])
 personnel['num_LB'] = personnel['DefensePersonnel'].apply(lambda x: x[1])
 personnel['num_DB'] = personnel['DefensePersonnel'].apply(lambda x: x[2])
 personnel['OffensePersonnel'] = personnel['OffensePersonnel'].apply(lambda x: split_personnel(x))
personnel['OffensePersonnel'] = personnel['OffensePersonnel'].apply(lambda x: offense_formation(x))
personnel['num_QB'] = personnel['OffensePersonnel'].apply(lambda x: x[0])
 personnel['num_RB'] = personnel['OffensePersonnel'].apply(lambda x: x[1])
 personnel['num_WR'] = personnel['OffensePersonnel'].apply(lambda x: x[2])
 personnel['num_TE'] = personnel['OffensePersonnel'].apply(lambda x: x[3])
 personnel['num_OL'] = personnel['OffensePersonnel'].apply(lambda x: x[4])
 personnel['OL_diff'] = personnel['num_OL'] - personnel['num_DL']
personnel['OL_TE_diff'] =(personnel['num_OL'] + personnel['num_TE'])- personnel['num_DL']
personnel['run_def'] =(personnel['num_DL'] + personnel['num_LB'] > 6 ).astype(int)
 personnel.drop(['OffensePersonnel','DefensePersonnel'], axis=1, inplace=True)
 return personnel
def rusher_features(df):
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','Dir', 'S', 'A', 'X', 'Y']]
rusher.columns = ['GameId','PlayId', 'RusherDir', 'RusherS', 'RusherA', 'RusherX', 'RusherY']
radian_angle =(90 - rusher['RusherDir'])* np.pi / 180.0
v_horizontal = np.abs(rusher['RusherS'] * np.cos(radian_angle))
v_vertical = np.abs(rusher['RusherS'] * np.sin(radian_angle))
rusher['v_horizontal'] = v_horizontal
rusher['v_vertical'] = v_vertical
rusher.columns = ['GameId','PlayId', 'RusherDir',
'RusherS',
'RusherA','RusherX', 'RusherY','v_horizontal', 'v_vertical']
return rusher
def dist_to_rusher(x):
a =(x[0],x[1])
 b =(x[2],x[3])
 return euclidean(a,b)
 def combine_features(relative_to_back, defense,rushing, static, deploy=deploy):
df = pd.merge(relative_to_back,defense,on=['GameId','PlayId'],how='inner')
 df = pd.merge(df,rushing,on=['GameId','PlayId'],how='inner')
 df = pd.merge(df,static,on=['GameId','PlayId'],how='inner')
 if not deploy:
df = pd.merge(df, outcomes, on=['GameId','PlayId'], how='inner')
 return df
print("Cleaning features...")
 df = clean_features(df)
 print("Feature Engineering...")
 df = fix_sides(df)
 df = feature_engineering(df)
 print("Calculating Voronoi Areas")
 df = voronoi_feats(df)
 print("Calculating Play Control")
 pc = get_play_control(df)
 df["play_control"] = pc["play_control"].values
for col in [ 'secondsLeftInHalf',
'numericFormation',
"Defense_X_stdiv",
"Defense_Y_stdiv",
"Defense_Y_spread",
"Defense_X_spread",
"min_time_to_tackle",
"mean_time_to_tackle",
"dist_to_offense_centroid",
"dist_to_defense_centroid"]:
df[col] = pc[col].values
print("Yardline features...")
 yardline = update_yardline(df)
 print("Fixing orientation...")
 df = update_orientation(df, yardline)
 print("Backline features...")
 back_feats = back_features(df)
 print("Features relative to backline...")
 rel_back = features_relative_to_back(df, back_feats)
 print("Defence features...")
 def_feats = defense_features(df)
 print("Static features...")
 static_feats = static_features(df)
 print("Rusher features...")
 rush_feats = rusher_features(df)
 print("Personnel features...")
 personnel = personnel_features(df)
 basetable = combine_features(rel_back, def_feats,rush_feats,static_feats, deploy=deploy)
 basetable["distance_to_rusher"] = 0
basetable["distance_to_rusher"] = basetable[["X","Y","RusherX","RusherY"]].apply(lambda x: dist_to_rusher(x), axis=1)
 basetable["s_delta_0"] = basetable["v_0"]-basetable['v_horizontal']
basetable["s_delta_1"] = basetable["v_1"]-basetable['v_vertical']
basetable.drop(["v_0","v_1", "Force"],axis=1, inplace=True)
 print("Complete.")
 return basetable<feature_engineering>
series_dict = {}
for series in(data['series_id'].unique()):
series_dict[series] = data[data['series_id'] == series]
%time train_basetable = create_features(train, False )<prepare_x_and_y>
del series_dict
gc.collect()
X = train_basetable.copy()
yards = X.Yards
y = np.zeros(( yards.shape[0], 199))
for idx, target in enumerate(list(yards)) :
y[idx][99 + target] = 1
X.drop(['GameId','PlayId','Yards'], axis=1, inplace=True )<normalization>
train_x = pd.read_csv('.. /input/career-con-2019/X_train.csv')
 train_y = pd.read_csv('.. /input/career-con-2019/y_train.csv' )
scaler = StandardScaler()
X = scaler.fit_transform(X )<split>
def prepare_data(t):
def f(d):
d=d.sort_values(by=['measurement_number'])
 return pd.DataFrame({
'lx':[ d['linear_acceleration_X'].values ],
'ly':[ d['linear_acceleration_Y'].values ],
'lz':[ d['linear_acceleration_Z'].values ],
'ax':[ d['angular_velocity_X'].values ],
'ay':[ d['angular_velocity_Y'].values ],
'az':[ d['angular_velocity_Z'].values ],
})
 t= t.groupby('series_id' ).apply(f)
 def mfft(x):
return [ x/math.sqrt(128.0)for x in np.absolute(np.fft.fft(x)) ][1:65]
t['lx_f']=[ mfft(x)for x in t['lx'].values ]
t['ly_f']=[ mfft(x)for x in t['ly'].values ]
t['lz_f']=[ mfft(x)for x in t['lz'].values ]
t['ax_f']=[ mfft(x)for x in t['ax'].values ]
t['ay_f']=[ mfft(x)for x in t['ay'].values ]
t['az_f']=[ mfft(x)for x in t['az'].values ]
return t
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.15, random_state=12345 )<import_modules>
t=prepare_data(train_x)
 t=pd.merge(t,train_y[['series_id','surface','group_id']],on='series_id')
 t=t.rename(columns={"surface": "y"} )
class CRPSCallback(Callback):
def __init__(self,validation, predict_batch_size=20, include_on_batch=False):
super(CRPSCallback, self ).__init__()
self.validation = validation
self.predict_batch_size = predict_batch_size
self.include_on_batch = include_on_batch
print('validation shape',len(self.validation))
def on_batch_begin(self, batch, logs={}):
pass
def on_train_begin(self, logs={}):
if not('CRPS_score_val' in self.params['metrics']):
self.params['metrics'].append('CRPS_score_val')
 def on_batch_end(self, batch, logs={}):
if(self.include_on_batch):
logs['CRPS_score_val'] = float('-inf')
 def on_epoch_end(self, epoch, logs={}):
logs['CRPS_score_val'] = float('-inf')
 if(self.validation):
X_valid, y_valid = self.validation[0], self.validation[1]
y_pred = self.model.predict(X_valid)
 y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * X_valid.shape[0])
 val_s = np.round(val_s, 6)
 logs['CRPS_score_val'] = val_s 
<compute_train_metric>
def aggf(d, feature):
va= np.array(d[feature].tolist())
 mean= sum(va)/va.shape[0]
var= sum([(va[i,:]-mean)**2 for i in range(va.shape[0])])/va.shape[0]
dev= [ math.sqrt(x)for x in var ]
return pd.DataFrame({
'mean': [ mean ],
'dev' : [ dev ],
})
 display={
'hard_tiles_large_space':'r-.',
'concrete':'g-.',
'tiled':'b-.',
'fine_concrete':'r-',
'wood':'g-',
'carpet':'b-',
'soft_pvc':'y-',
'hard_tiles':'r--',
'soft_tiles':'g--',
}
def get_model(x_tr,y_tr,x_val,y_val):
inp = Input(shape =(x_tr.shape[1],))
x = Dense(1024, input_dim=X.shape[1] )(inp)
 x = LeakyReLU(alpha=0.3 )(x)
 x = Dropout(0.2 )(x)
 x = BatchNormalization()(x)
 x = Dense(512 )(x)
 x = LeakyReLU(alpha=0.2 )(x)
 x = Dropout(0.2 )(x)
 x = BatchNormalization()(x)
 x = Dense(256 )(x)
 x = LeakyReLU(alpha=0.1 )(x)
 x = Dropout(0.2 )(x)
 x = BatchNormalization()(x)
 out = Dense(199, activation='softmax' )(x)
 model = Model(inp,out)
 model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=[])
 es = EarlyStopping(monitor='CRPS_score_val',
mode='min',
restore_best_weights=True,
verbose=1,
patience=20)
 mc = ModelCheckpoint('best_model.h5',monitor='CRPS_score_val',mode='min',
save_best_only=True, verbose=1, save_weights_only=True)
 bsz = 1024
steps = x_tr.shape[0]/bsz
model.fit(x_tr, y_tr,callbacks=[CRPSCallback(validation =(x_val,y_val)) ,es,mc], epochs=100, batch_size=bsz,verbose=1)
 model.load_weights("best_model.h5")
 y_pred = model.predict(x_val)
 y_valid = y_val
y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * x_val.shape[0])
 crps = np.round(val_s, 6)
 return model,crps<split>
del train_x, train_y
gc.collect()
losses = []
models = []
crps_csv = []
s_time = time.time()
for k in range(2):
kfold = KFold(5, random_state = 42 + k, shuffle = True)
 for k_fold,(tr_inds, val_inds)in enumerate(kfold.split(yards)) :
print("-----------")
 print("-----------")
 tr_x,tr_y = X[tr_inds],y[tr_inds]
val_x,val_y = X[val_inds],y[val_inds]
model,crps = get_model(tr_x,tr_y,val_x,val_y)
 models.append(model)
 print("the %d fold crps is %f"%(( k_fold+1),crps))
crps_csv.append(crps)
 print("mean crps is %f"%np.mean(crps_csv))
def predict(x_te):
model_num = len(models)
 for k,m in enumerate(models):
if k==0:
y_pred = m.predict(x_te,batch_size=1024)
 else:
y_pred+=m.predict(x_te,batch_size=1024)
 y_pred = y_pred / model_num
return y_pred 
<statistical_test>
def quaternion_to_euler(x, y, z, w):
t0 = +2.0 *(w * x + y * z)
 t1 = +1.0 - 2.0 *(x * x + y * y)
 X = math.atan2(t0, t1)
 t2 = +2.0 *(w * y - z * x)
 t2 = +1.0 if t2 > +1.0 else t2
t2 = -1.0 if t2 < -1.0 else t2
Y = math.asin(t2)
 t3 = +2.0 *(w * z + x * y)
 t4 = +1.0 - 2.0 *(y * y + z * z)
 Z = math.atan2(t3, t4)
 return X, Y, Z
def get_model(x_tr,y_tr,x_val,y_val):
inp = Input(shape =(x_tr.shape[1],))
x = Dense(1024, input_dim=X.shape[1] )(inp)
 x = LeakyReLU(alpha=0.3 )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(512 )(x)
 x = LeakyReLU(alpha=0.2 )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(512 )(x)
 x = LeakyReLU(alpha=0.2 )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(256 )(x)
 x = LeakyReLU(alpha=0.1 )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 out = Dense(199, activation='softmax' )(x)
 model = Model(inp,out)
 model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=[])
 es = EarlyStopping(monitor='CRPS_score_val',
mode='min',
restore_best_weights=True,
verbose=1,
patience=10)
 mc = ModelCheckpoint('best_model1.h5',monitor='CRPS_score_val',mode='min',
save_best_only=True, verbose=1, save_weights_only=True)
 bsz = 1024
steps = x_tr.shape[0]/bsz
model.fit(x_tr, y_tr,callbacks=[CRPSCallback(validation =(x_val,y_val)) ,es,mc], epochs=100, batch_size=bsz,verbose=1)
 model.load_weights("best_model1.h5")
 y_pred = model.predict(x_val)
 y_valid = y_val
y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * x_val.shape[0])
 crps = np.round(val_s, 6)
 return model,crps<split>
def fe_step0(actual):
actual['norm_quat'] =(actual['orientation_X']**2 + actual['orientation_Y']**2 + actual['orientation_Z']**2 + actual['orientation_W']**2)
 actual['mod_quat'] =(actual['norm_quat'])**0.5
actual['norm_X'] = actual['orientation_X'] / actual['mod_quat']
actual['norm_Y'] = actual['orientation_Y'] / actual['mod_quat']
actual['norm_Z'] = actual['orientation_Z'] / actual['mod_quat']
actual['norm_W'] = actual['orientation_W'] / actual['mod_quat']
return actual
s_time = time.time()
for k in range(1):
kfold = KFold(5, random_state = 42 + k, shuffle = True)
 for k_fold,(tr_inds, val_inds)in enumerate(kfold.split(yards)) :
print("-----------")
 print("-----------")
 tr_x,tr_y = X[tr_inds],y[tr_inds]
val_x,val_y = X[val_inds],y[val_inds]
model,crps = get_model(tr_x,tr_y,val_x,val_y)
 models.append(model)
 print("the %d fold crps is %f"%(( k_fold+1),crps))
crps_csv.append(crps)
 print("mean crps is %f"%np.mean(crps_csv))
def predict(x_te):
model_num = len(models)
 for k,m in enumerate(models):
if k==0:
y_pred = m.predict(x_te,batch_size=1024)
 else:
y_pred+=m.predict(x_te,batch_size=1024)
 y_pred = y_pred / model_num
return y_pred 
<choose_model_class>
def fe_step1(actual):

 x, y, z, w = actual['norm_X'].tolist() , actual['norm_Y'].tolist() , actual['norm_Z'].tolist() , actual['norm_W'].tolist()
nx, ny, nz = [], [], []
for i in range(len(x)) :
xx, yy, zz = quaternion_to_euler(x[i], y[i], z[i], w[i])
 nx.append(xx)
 ny.append(yy)
 nz.append(zz)
 actual['euler_x'] = nx
actual['euler_y'] = ny
actual['euler_z'] = nz
return actual
def get_model(x_tr,y_tr,x_val,y_val):
inp = Input(shape =(x_tr.shape[1],))
x = Dense(512, input_dim=X.shape[1] )(inp)
 x = LeakyReLU(alpha=0.2 )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(512 )(x)
 x = LeakyReLU(alpha=0.2 )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 out = Dense(199, activation='softmax' )(x)
 model = Model(inp,out)
 model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=[])
 es = EarlyStopping(monitor='CRPS_score_val',
mode='min',
restore_best_weights=True,
verbose=1,
patience=10)
 mc = ModelCheckpoint('best_model2.h5',monitor='CRPS_score_val',mode='min',
save_best_only=True, verbose=1, save_weights_only=True)
 bsz = 1024
steps = x_tr.shape[0]/bsz
model.fit(x_tr, y_tr,callbacks=[CRPSCallback(validation =(x_val,y_val)) ,es,mc], epochs=100, batch_size=bsz,verbose=1)
 model.load_weights("best_model2.h5")
 y_pred = model.predict(x_val)
 y_valid = y_val
y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * x_val.shape[0])
 crps = np.round(val_s, 6)
 return model,crps<split>
def feat_eng(data):
df = pd.DataFrame()
data['totl_anglr_vel'] =(data['angular_velocity_X']**2 + data['angular_velocity_Y']**2 + data['angular_velocity_Z']**2)** 0.5
data['totl_linr_acc'] =(data['linear_acceleration_X']**2 + data['linear_acceleration_Y']**2 + data['linear_acceleration_Z']**2)**0.5
data['totl_xyz'] =(data['orientation_X']**2 + data['orientation_Y']**2 + data['orientation_Z']**2)**0.5
data['acc_vs_vel'] = data['totl_linr_acc'] / data['totl_anglr_vel']
def mean_change_of_abs_change(x):
return np.mean(np.diff(np.abs(np.diff(x))))
for col in data.columns:
if col in ['row_id','series_id','measurement_number']:
continue
df[col + '_mean'] = data.groupby(['series_id'])[col].mean()
df[col + '_median'] = data.groupby(['series_id'])[col].median()
df[col + '_max'] = data.groupby(['series_id'])[col].max()
df[col + '_min'] = data.groupby(['series_id'])[col].min()
df[col + '_std'] = data.groupby(['series_id'])[col].std()
df[col + '_range'] = df[col + '_max'] - df[col + '_min']
df[col + '_maxtoMin'] = df[col + '_max'] / df[col + '_min']
df[col + '_mean_abs_chg'] = data.groupby(['series_id'])[col].apply(lambda x: np.mean(np.abs(np.diff(x))))
df[col + '_mean_change_of_abs_change'] = data.groupby('series_id')[col].apply(mean_change_of_abs_change)
 df[col + '_abs_max'] = data.groupby(['series_id'])[col].apply(lambda x: np.max(np.abs(x)))
 df[col + '_abs_min'] = data.groupby(['series_id'])[col].apply(lambda x: np.min(np.abs(x)))
 df[col + '_abs_avg'] =(df[col + '_abs_min'] + df[col + '_abs_max'])/2
return df
s_time = time.time()
for k in range(1):
kfold = KFold(5, random_state = 42 + k, shuffle = True)
 for k_fold,(tr_inds, val_inds)in enumerate(kfold.split(yards)) :
print("-----------")
 print("-----------")
 tr_x,tr_y = X[tr_inds],y[tr_inds]
val_x,val_y = X[val_inds],y[val_inds]
model,crps = get_model(tr_x,tr_y,val_x,val_y)
 models.append(model)
 print("the %d fold crps is %f"%(( k_fold+1),crps))
crps_csv.append(crps)
 print("mean crps is %f"%np.mean(crps_csv))
def predict(x_te):
model_num = len(models)
 for k,m in enumerate(models):
if k==0:
y_pred = m.predict(x_te,batch_size=1024)
 else:
y_pred+=m.predict(x_te,batch_size=1024)
 y_pred = y_pred / model_num
return y_pred 
<compute_test_metric>
def _kurtosis(x):
return kurtosis(x)
 def CPT5(x):
den = len(x)*np.exp(np.std(x))
return sum(np.exp(x)) /den
def skewness(x):
return skew(x)
 def SSC(x):
x = np.array(x)
 x = np.append(x[-1], x)
 x = np.append(x,x[1])
 xn = x[1:len(x)-1]
xn_i2 = x[2:len(x)]
xn_i1 = x[0:len(x)-2]
ans = np.heaviside(( xn-xn_i1)*(xn-xn_i2),0)
 return sum(ans[1:])
 def wave_length(x):
x = np.array(x)
 x = np.append(x[-1], x)
 x = np.append(x,x[1])
 xn = x[1:len(x)-1]
xn_i2 = x[2:len(x)]
return sum(abs(xn_i2-xn))
def norm_entropy(x):
tresh = 3
return sum(np.power(abs(x),tresh))
def SRAV(x):
SRA = sum(np.sqrt(abs(x)))
 return np.power(SRA/len(x),2)
 def mean_abs(x):
return sum(abs(x)) /len(x)
 def zero_crossing(x):
x = np.array(x)
 x = np.append(x[-1], x)
 x = np.append(x,x[1])
 xn = x[1:len(x)-1]
xn_i2 = x[2:len(x)]
return sum(np.heaviside(-xn*xn_i2,0))
print("mean crps is %f"%np.mean(crps_csv)) 
<predict_on_test>
def fe_advanced_stats(data):
df = pd.DataFrame()
for col in data.columns:
if col in ['row_id','series_id','measurement_number']:
continue
if 'orientation' in col:
continue
print("FE on column ", col, "...")
 df[col + '_skew'] = data.groupby(['series_id'])[col].skew()
df[col + '_mad'] = data.groupby(['series_id'])[col].mad()
df[col + '_q25'] = data.groupby(['series_id'])[col].quantile(0.25)
 df[col + '_q75'] = data.groupby(['series_id'])[col].quantile(0.75)
 df[col + '_q95'] = data.groupby(['series_id'])[col].quantile(0.95)
 df[col + '_iqr'] = df[col + '_q75'] - df[col + '_q25']
df[col + '_CPT5'] = data.groupby(['series_id'])[col].apply(CPT5)
 df[col + '_SSC'] = data.groupby(['series_id'])[col].apply(SSC)
 df[col + '_skewness'] = data.groupby(['series_id'])[col].apply(skewness)
 df[col + '_wave_lenght'] = data.groupby(['series_id'])[col].apply(wave_length)
 df[col + '_norm_entropy'] = data.groupby(['series_id'])[col].apply(norm_entropy)
 df[col + '_SRAV'] = data.groupby(['series_id'])[col].apply(SRAV)
 df[col + '_kurtosis'] = data.groupby(['series_id'])[col].apply(_kurtosis)
 df[col + '_zero_crossing'] = data.groupby(['series_id'])[col].apply(zero_crossing)
 return df
%%time
if TRAIN_OFFLINE==False:
env = nflrush.make_env()
iter_test = env.iter_test()
for(test_df, sample_prediction_df)in iter_test:
basetable = create_features(test_df, deploy=True)
 basetable.drop(['GameId','PlayId'], axis=1, inplace=True)
 scaled_basetable = scaler.transform(basetable)
 y_pred = predict(scaled_basetable)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1 ).tolist() [0]
preds_df = pd.DataFrame(data=[y_pred], columns=sample_prediction_df.columns)
 env.predict(preds_df)
 env.write_submission_file()<categorify>
basic_fe = ['linear_acceleration_X','linear_acceleration_Y','linear_acceleration_Z',
'angular_velocity_X','angular_velocity_Y','angular_velocity_Z']
env = nflrush.make_env()<set_options>
def fe_plus(data):
aux = pd.DataFrame()
for serie in data.index:
aux = X_train[X_train['series_id']==serie]
for col in basic_fe:
data.loc[serie,col + '_unq'] = aux[col].round(3 ).nunique()
data.loc[serie,col + 'ratio_unq'] = aux[col].round(3 ).nunique() /18
try:
data.loc[serie,col + '_freq'] = aux[col].value_counts().idxmax()
except:
data.loc[serie,col + '_freq'] = 0
data.loc[serie,col + '_max_freq'] = aux[aux[col] == aux[col].max() ].shape[0]
data.loc[serie,col + '_min_freq'] = aux[aux[col] == aux[col].min() ].shape[0]
data.loc[serie,col + '_pos_freq'] = aux[aux[col] >= 0].shape[0]
data.loc[serie,col + '_neg_freq'] = aux[aux[col] < 0].shape[0]
data.loc[serie,col + '_nzeros'] =(aux[col]==0 ).sum(axis=0 )
pd.set_option('mode.chained_assignment', None)
<load_from_csv>
data.fillna(0,inplace=True)
 test.fillna(0,inplace=True)
 data.replace(-np.inf,0,inplace=True)
 data.replace(np.inf,0,inplace=True)
 test.replace(-np.inf,0,inplace=True)
 test.replace(np.inf,0,inplace=True )
tracking_df_train = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2020/train.csv', low_memory=False )<define_variables>
target['surface'] = le.fit_transform(target['surface'] )
Yard_bins=['Yards'+str(i)for i in range(-99,100)]<train_model>
target['surface'].value_counts()
def preprocess_tracking_data(tracking_df,verbose=0, suppress_warnings=True, for_test=True):

 if suppress_warnings:
pd.set_option('mode.chained_assignment', None)
 else:
pd.set_option('mode.chained_assignment', 'warn')
 if verbose:
print("cleaned up")
 start = time.time()
def adjust_2017_motion_metrics(metric):
metric_mean_2018=tracking_df.query('Season==2018')[metric].mean()
tracking_df[metric].loc[tracking_df.Season==2017]=tracking_df.query('Season==2017')[metric].transform(lambda x:(x - x.mean())/ x.std())+metric_mean_2018
if not for_test:
adjust_2017_motion_metrics('S')
 adjust_2017_motion_metrics('A')
 tracking_df['DefendersInTheBox'].fillna(7, inplace=True)
 tracking_df['Temperature'].fillna(62, inplace=True)
 tracking_df.loc[tracking_df.VisitorTeamAbbr == "ARI", 'VisitorTeamAbbr'] = "ARZ"
tracking_df.loc[tracking_df.HomeTeamAbbr == "ARI", 'HomeTeamAbbr'] = "ARZ"
tracking_df.loc[tracking_df.VisitorTeamAbbr == "BAL", 'VisitorTeamAbbr'] = "BLT"
tracking_df.loc[tracking_df.HomeTeamAbbr == "BAL", 'HomeTeamAbbr'] = "BLT"
tracking_df.loc[tracking_df.VisitorTeamAbbr == "CLE", 'VisitorTeamAbbr'] = "CLV"
tracking_df.loc[tracking_df.HomeTeamAbbr == "CLE", 'HomeTeamAbbr'] = "CLV"
tracking_df.loc[tracking_df.VisitorTeamAbbr == "HOU", 'VisitorTeamAbbr'] = "HST"
tracking_df.loc[tracking_df.HomeTeamAbbr == "HOU", 'HomeTeamAbbr'] = "HST"
def side_of_ball(x):
if x['Team']=='away':
if x['VisitorTeamAbbr']==x['PossessionTeam']:
return 'OFF'
else:
return 'DEF'
elif x['Team']=='home':
if x['VisitorTeamAbbr']==x['PossessionTeam']:
return 'DEF'
else:
return 'OFF'
else:
return 'UNK'
tracking_df['side_of_ball']=tracking_df.apply(side_of_ball, axis=1)
 def flip_x_same_direction(x):
if x['PlayDirection']=='left':
return 120.0 - x['X']
else:
return x['X']
tracking_df['X_same_way']=tracking_df.apply(flip_x_same_direction, axis=1)
 def flip_y_same_direction(x):
if x['PlayDirection']=='left':
return 53.3 - x['Y']
else:
return x['Y']
tracking_df['Y_same_way']=tracking_df.apply(flip_y_same_direction, axis=1)
 tracking_df['Y_dist_from_center']=tracking_df['Y_same_way']-53.3/2
tracking_df['Y_abs_dist_from_center']=np.abs(tracking_df['Y_dist_from_center'])
 def flip_Dir_same_direction(x):
if x['PlayDirection']=='left':
return 360.0 - x['Dir']
else:
return x['Dir']
tracking_df['Dir_same_way']=tracking_df.apply(flip_Dir_same_direction, axis=1)
 def shift_Dir_to_endzone(x):
if x['Dir_same_way']<=270:
return x['Dir_same_way']-90
else:
return -1.0*x['Dir_same_way']+180
def fill_missing_Dir(x):
if np.isnan(x['Dir_same_way']):
if x['side_of_ball']=='OFF':
return 0
else:
return 180
else:
return x['Dir_0Deg_to_endzone']
tracking_df['Dir_0Deg_to_endzone']=tracking_df.apply(shift_Dir_to_endzone, axis=1)
 tracking_df['Dir_0Deg_to_endzone']=tracking_df.apply(fill_missing_Dir,axis=1)
 tracking_df['Dir_abs_0Deg_to_endzone']=np.abs(tracking_df['Dir_0Deg_to_endzone'])
 def corrected_2017_orientation(x):
if x['Season']!=2017:
return x['Orientation']
else:
return np.mod(90+x['Orientation'],360)
 tracking_df['Orientation_corrected']=tracking_df.apply(corrected_2017_orientation, axis=1)
 def flip_Orientation_same_direction(x):
if x['PlayDirection']=='left':
return 360.0 - x['Orientation_corrected']
else:
return x['Orientation_corrected']
tracking_df['Orientation_same_way']=tracking_df.apply(flip_Orientation_same_direction, axis=1)
 def shift_Orientation_to_endzone(x):
if x['Orientation_same_way']<=270:
return x['Orientation_same_way']-90
else:
return -1.0*x['Orientation_same_way']+180
tracking_df['Orientation_0Deg_to_endzone']=tracking_df.apply(shift_Orientation_to_endzone, axis=1)
 tracking_df['Orientation_abs_0Deg_to_endzone']=np.abs(tracking_df['Orientation_0Deg_to_endzone'])
 def calculate_yards_to_end_zone(x):
if x['PossessionTeam']==x['FieldPosition']:
return 100-x['YardLine']
else:
return x['YardLine']
tracking_df['Yards_to_end_zone']=tracking_df.apply(calculate_yards_to_end_zone, axis=1)
 tracking_df['X_to_YardLine']=tracking_df['X_same_way']-(110-tracking_df['Yards_to_end_zone'])
 tracking_df['X_to_1stDown']=tracking_df['X_to_YardLine']-tracking_df['Distance']
tracking_df['speed_in_direction_of_EZ']=tracking_df['S']*np.cos(tracking_df['Dir_0Deg_to_endzone']/180*np.pi)
 tracking_df['X_to_YardLine_sec_momentum']=tracking_df['X_to_YardLine']\
+tracking_df['speed_in_direction_of_EZ']
tracking_df['Y_dist_from_center_sec_momentum']=tracking_df['Y_dist_from_center']\
+tracking_df['S']*np.sin(tracking_df['Dir_0Deg_to_endzone']/180*np.pi)
 if verbose:
print("going the same way")
 end = time.time()
print(end - start)
 start = time.time()
tracking_df['ball_carrier']=(tracking_df['NflId']==tracking_df['NflIdRusher'])
 tracking_df['Position'].loc[tracking_df['Position']=='SAF']='S'
tracking_df['Position'].loc[tracking_df['Position']=='HB']='RB'
tracking_df['NflIdRusher']=tracking_df['NflIdRusher'].astype(str)
 tracking_df['DefendingTeam']=tracking_df[['HomeTeamAbbr','VisitorTeamAbbr','PossessionTeam']]\ 
.apply(lambda x: x['HomeTeamAbbr'] if x['PossessionTeam']==x['VisitorTeamAbbr'] else x['VisitorTeamAbbr']
, axis=1)
 if verbose:
print("Minor context and features done")
 end = time.time()
print(end - start)
 global team_off_yard_reference, team_def_yard_reference, rusher_yard_reference, rusher_low_vol_reference
if not for_test:
start = time.time()
team_off_yard_reference=tracking_df.groupby('PlayId' ).first() \ 
.groupby(['PossessionTeam'])['Yards']\ 
.describe(include='all')\ 
.query('count>=20')\ 
.sort_values(by='75%',ascending=False)
 team_off_yard_reference.columns=['PossessionTeam_Yards_'+col for col in team_off_yard_reference.columns]
team_def_yard_reference=tracking_df.groupby('PlayId' ).first() \ 
.groupby('DefendingTeam')\
['Yards']\ 
.describe(include='all')\ 
.query('count>=20')\ 
.sort_values(by='75%',ascending=False)
 team_def_yard_reference.columns=['DefendingTeam_Yards_'+col for col in team_def_yard_reference.columns]
rusher_yard_reference=tracking_df.groupby('PlayId' ).first() \ 
.groupby(['NflIdRusher'])['Yards']\ 
.describe(include='all')\ 
.query('count>=20')
 rusher_yard_reference.columns=['Rusher_Yards_'+col for col in rusher_yard_reference.columns]
rusher_yard_reference.reset_index(inplace=True)
 rusher_yard_reference.NflIdRusher=rusher_yard_reference.NflIdRusher.astype(str)
 rusher_low_vol_reference=tracking_df.groupby('PlayId' ).first().merge(rusher_yard_reference
, on='NflIdRusher'
, how='left')\ 
.query('Rusher_Yards_count!=Rusher_Yards_count')['Yards']\ 
.describe(include='all')
 rusher_low_vol_reference.index=['Rusher_Yards_'+col for col in rusher_low_vol_reference.index]
if verbose:
print("Player and Team Yard Summary reference tables")
 end = time.time()
print(end - start)
 start = time.time()
ball_carrier_df=tracking_df.query('ball_carrier==True' ).copy()
def strtoseconds(txt):
txt = txt.split(':')
 ans = int(txt[0])*60 + int(txt[1])+ int(txt[2])/60
return ans
ball_carrier_df['GameClock'] = ball_carrier_df['GameClock'].apply(strtoseconds)
 ball_carrier_df['TimeHandoff'] = ball_carrier_df['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
ball_carrier_df['TimeSnap'] = ball_carrier_df['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
ball_carrier_df['TimefromSnap'] = ball_carrier_df.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 ball_carrier_df['PlayerBirthDate'] = ball_carrier_df['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
ball_carrier_df['PlayerAge'] = ball_carrier_df.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 ball_carrier_df['PlayerHeight'] = ball_carrier_df['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
ball_carrier_df['NflIdRusher']=ball_carrier_df['NflIdRusher'].astype(str)
 Turf = {'Field Turf':'Artificial', 'A-Turf Titan':'Artificial', 'Grass':'Natural', 'UBU Sports Speed S5-M':'Artificial',
'Artificial':'Artificial', 'DD GrassMaster':'Artificial', 'Natural Grass':'Natural',
'UBU Speed Series-S5-M':'Artificial', 'FieldTurf':'Artificial', 'FieldTurf 360':'Artificial', 'Natural grass':'Natural', 'grass':'Natural',
'Natural':'Natural', 'Artifical':'Artificial', 'FieldTurf360':'Artificial', 'Naturall Grass':'Natural', 'Field turf':'Artificial',
'SISGrass':'Artificial', 'Twenty-Four/Seven Turf':'Artificial', 'natural grass':'Natural'}
ball_carrier_df['Turf_type'] = ball_carrier_df['Turf'].map(Turf)
 def map_weather(txt):
txt=str(txt ).lower()
if pd.isna(txt):
return 'other'
if 'rain' in txt or 'shower' in txt:
return 'rain'
if 'snow' in txt:
return 'snow'
return 'other'
ball_carrier_df['GameWeather_simple'] = ball_carrier_df['GameWeather'].apply(map_weather)
 ball_carrier_df['score_diff']=(ball_carrier_df["HomeScoreBeforePlay"] - ball_carrier_df["VisitorScoreBeforePlay"])*(ball_carrier_df['Team']=='away')
 ball_carrier_df['OffenseFormation'].loc[ball_carrier_df['OffenseFormation']=='ACE']='SINGLEBACK'
ball_carrier_df=ball_carrier_df.merge(rusher_yard_reference,
on='NflIdRusher',
how='left')\ 
.fillna(rusher_low_vol_reference)
 if verbose:
print("ball carrier measures done")
 end = time.time()
print(end - start)
 start = time.time()
position_cnts=tracking_df.groupby('PlayId')['Position'].value_counts().unstack(level=-1 ).fillna(0)
 position_cnts.columns=['Position_cnt_'+pos for pos in position_cnts.columns]
tracking_df['Position_group']=tracking_df['Position']
tracking_df['Position_group'].loc[tracking_df.side_of_ball == 'DEF']=tracking_df['Position_group'].replace(['SAF','HB','RB','QB','TE','WR','CB','SS','FS','S','DB'],'DB')
 tracking_df['Position_group'].loc[tracking_df.side_of_ball == 'DEF']=tracking_df['Position_group'].replace(['DL','OG','OT','NT','C','G','T','DT','DE'],'DL')
 tracking_df['Position_group'].loc[tracking_df.side_of_ball == 'DEF']=tracking_df['Position_group'].replace(['LB','MLB','FB','OLB','ILB'],'LB')
 tracking_df['Position_group'].loc[tracking_df.side_of_ball == 'OFF']=tracking_df['Position_group'].replace(['WR','SS','FS','CB','DB','S','SAF'],'WR')
 tracking_df['Position_group'].loc[tracking_df.side_of_ball == 'OFF']=tracking_df['Position_group'].replace(['OG','OT','T','G','C','DE','DT','NT','DL'],'OL')
 tracking_df['Position_group'].loc[tracking_df.side_of_ball == 'OFF']=tracking_df['Position_group'].replace(['RB','FB','MLB','HB'],'RB')
 tracking_df['Position_group'].loc[tracking_df.side_of_ball == 'OFF']=tracking_df['Position_group'].replace(['TE','ILB','OLB','LB'],'TE')
 position_grp_cnts=tracking_df.groupby('PlayId')['Position_group'].value_counts().unstack(level=-1 ).fillna(0)
 position_grp_cnts.columns=['Position_grp_cnt_'+pos for pos in position_grp_cnts.columns]
if 'Position_grp_cnt_OL' not in position_grp_cnts.columns:
position_grp_cnts['Position_grp_cnt_OL']=0
if 'Position_grp_cnt_DL' not in position_grp_cnts.columns:
position_grp_cnts['Position_grp_cnt_DL']=0
if 'Position_grp_cnt_TE' not in position_grp_cnts.columns:
position_grp_cnts['Position_grp_cnt_TE']=0
if 'Position_grp_cnt_LB' not in position_grp_cnts.columns:
position_grp_cnts['Position_grp_cnt_LB']=0
position_grp_cnts['Position_grp_cnt_Trench_diff'] = position_grp_cnts['Position_grp_cnt_OL']\
- position_grp_cnts['Position_grp_cnt_DL']
position_grp_cnts['Position_grp_cnt_Trench_w_TE_diff'] =(position_grp_cnts['Position_grp_cnt_OL']\
+ position_grp_cnts['Position_grp_cnt_TE'])\
- position_grp_cnts['Position_grp_cnt_DL']
position_grp_cnts['Position_grp_cnt_Block_vs_runstuff_diff'] = position_grp_cnts['Position_grp_cnt_OL']\
+ position_grp_cnts['Position_grp_cnt_TE']\
- position_grp_cnts['Position_grp_cnt_DL']\
- position_grp_cnts['Position_grp_cnt_LB']
position_grp_cnts['Position_grp_cnt_RunStuff_def'] =(( position_grp_cnts['Position_grp_cnt_DL'] + position_grp_cnts['Position_grp_cnt_LB'])> 6 ).astype(int)
 if verbose:
print("position counts done")
 end = time.time()
print(end - start)
 start = time.time()
def distance_based_metrics(X='X_to_YardLine',Y='Y_dist_from_center',momentum=False):

 if momentum:
momentum_suffix='_sec_momentum'
else:
momentum_suffix=''
X_field=X+momentum_suffix
Y_field=Y+momentum_suffix
start = time.time()
off_dist_to_def=tracking_df[['PlayId','NflId','JerseyNumber',X_field,Y_field,'Orientation_0Deg_to_endzone','side_of_ball','ball_carrier']].query("side_of_ball=='OFF'")\ 
.merge(tracking_df[['PlayId','NflId','JerseyNumber',X_field,Y_field,'side_of_ball']].query("side_of_ball=='DEF'"),
on='PlayId', how='inner', suffixes=("_off","_def"))
def calc_distance(x1,y1,x2,y2):
loc1 = np.asarray([x1,y1])
 loc2 = np.asarray([x2,y2])
 return np.linalg.norm(loc1-loc2)
 def calc_dist_off_to_def(x):
return calc_distance(x[X_field+'_off'],x[Y_field+'_off'],x[X_field+'_def'],x[Y_field+'_def'])
 off_dist_to_def['off_distance_to_def']=off_dist_to_def.apply(calc_dist_off_to_def,axis=1)
 off_dist_to_def['engaged_dist']=(off_dist_to_def['off_distance_to_def']<=4)
 engaged=off_dist_to_def.query('engaged_dist==True')
 def player_to_player_direction(x1,y1,x2,y2):
try:
return np.arctan(( y2-y1)/(x2-x1)) *180/np.pi
except:
return 0
def blocker_to_def_dir(x):
return player_to_player_direction(x[X_field+'_off'],x[Y_field+'_off'],x[X_field+'_def'],x[Y_field+'_def'])
 engaged['blocker_to_def_dir']=engaged.apply(blocker_to_def_dir, axis=1)
 engaged['engaged_orientation']=(np.abs(engaged['blocker_to_def_dir'] - engaged['Orientation_0Deg_to_endzone'])<=45)
 engaged=engaged.merge(ball_carrier_df[['PlayId',X_field,Y_field]]
,on='PlayId')\ 
.rename(columns={X_field:X_field+'_bc',Y_field:Y_field+'_bc'})
 def calc_dist_off_to_bc(x):
return calc_distance(x[X_field+'_off'],x[Y_field+'_off'],x[X_field+'_bc'],x[Y_field+'_bc'])
 def calc_dist_def_to_bc(x):
return calc_distance(x[X_field+'_def'],x[Y_field+'_def'],x[X_field+'_bc'],x[Y_field+'_bc'])
 engaged['off_distance_to_bc']=engaged.apply(calc_dist_off_to_bc,axis=1)
 engaged['def_distance_to_bc']=engaged.apply(calc_dist_def_to_bc,axis=1)
 engaged['engaged_blocker_between_bc_and_def']=(engaged['off_distance_to_bc']<engaged['def_distance_to_bc'])
 engaged['engaged']=engaged[['engaged_dist','engaged_orientation','engaged_blocker_between_bc_and_def']].all(axis='columns')
 engaged=engaged.query('engaged==True')
 engaged['Off_leverage_left']=(engaged[Y_field+'_off']>=engaged[Y_field+'_def'])
 engaged['Off_leverage_right']=(engaged[Y_field+'_off']<=engaged[Y_field+'_def'])
 off_dist_to_def=off_dist_to_def.merge(engaged[['PlayId','NflId_off','NflId_def','engaged']],
on=['PlayId','NflId_off','NflId_def'], how='left')\ 
.fillna(False)
 bc_dist_to_def=off_dist_to_def[['PlayId','NflId_def','off_distance_to_def','ball_carrier',X_field+'_def',Y_field+'_def']].query('ball_carrier==True' ).copy() \ 
.rename(columns={'off_distance_to_def':'bc_distance_to_def'})
 bc_dist_to_def['bc_distance'+momentum_suffix+'_to_def_rank']=bc_dist_to_def.groupby('PlayId')['bc_distance_to_def'].rank(method='first' ).astype(int)
 bc_dist_to_def['def_left_to_right_rank']=bc_dist_to_def.groupby('PlayId')[Y_field+'_def'].rank(method='first',ascending=False ).astype(int)
 bc_dist_to_def=bc_dist_to_def.merge(engaged.groupby(['PlayId','NflId_def'])['NflId_off'].count() ,
left_on=['PlayId','NflId_def'],right_index=True,
how='left')\ 
.rename(columns={'NflId_off':'engaged_by'})\ 
.fillna(0)
 bc_dist_to_def['engaged']=(bc_dist_to_def['engaged_by']>0)
 play_def_left_to_right_rank_df=bc_dist_to_def.pivot(index='PlayId',columns='def_left_to_right_rank',
values=[
'bc_distance_to_def',X_field+'_def',Y_field+'_def'] ).astype('float64')
 play_def_left_to_right_rank_df.columns = ['def_left_to_right'+momentum_suffix+'_rank'+str(col[1])+'_'+col[0] for col in play_def_left_to_right_rank_df.columns.values]
play_bc_dist_to_def_rank_df=bc_dist_to_def.query('bc_distance'+momentum_suffix+'_to_def_rank<=11')[['PlayId','bc_distance_to_def','engaged','engaged_by', 'bc_distance'+momentum_suffix+'_to_def_rank']]\ 
.pivot(index='PlayId',columns='bc_distance'+momentum_suffix+'_to_def_rank',values=['bc_distance_to_def',
'engaged_by'])\ 
.fillna(bc_dist_to_def.bc_distance_to_def.max())
 play_bc_dist_to_def_rank_df.columns = ['bc_distance'+momentum_suffix+'_to_def_rank'+str(col[1])+'_'+col[0] for col in play_bc_dist_to_def_rank_df.columns.values]
bc_dist_to_def_summaries=bc_dist_to_def.groupby('PlayId')['bc_distance_to_def'].agg(['mean','median','min', 'max'])
 bc_dist_to_def_summaries.columns=['bc_dist'+momentum_suffix+'_to_def_'+col for col in bc_dist_to_def_summaries.columns]
bc_dist_to_unblocked_def=bc_dist_to_def.query('engaged==False')[['PlayId','NflId_def','bc_distance_to_def']]
bc_dist_to_unblocked_def['bc_distance'+momentum_suffix+'_to_unblocked_def_rank']=bc_dist_to_unblocked_def.groupby('PlayId')['bc_distance_to_def'].rank(method='first' ).astype(int)
 keep_N_unblocked_def=3
base_unblocked_def_table=pd.DataFrame()
rank_N_table=ball_carrier_df[['PlayId','Yards_to_end_zone',X_field,'Y_abs_dist_from_center']]
for rank in range(1,keep_N_unblocked_def+1):
rank_N_table['bc_distance'+momentum_suffix+'_to_unblocked_def_rank']=rank
base_unblocked_def_table=pd.concat([base_unblocked_def_table,rank_N_table])
 def calc_dist_bc_to_nearest_EZ_corner(x):
return calc_distance(x[X_field+'_bc'],x['Y_abs_dist_from_center'],x[X_field+'_def'],26.65)
 bc_dist_to_unblocked_def=base_unblocked_def_table\ 
.merge(bc_dist_to_unblocked_def,
on=['PlayId','bc_distance'+momentum_suffix+'_to_unblocked_def_rank'],
how='left' 
)\ 
.merge(tracking_df\
[['PlayId','NflId',X_field,Y_field]],
left_on=['PlayId','NflId_def'], right_on=['PlayId','NflId'],
how='left', suffixes=('_bc','_def')) \ 
.rename(columns={'bc_distance_to_def':'distance', Y_field:Y_field+'_def'})\ 
.reset_index(drop=True)
 bc_dist_to_unblocked_def.fillna({'NflId_def':9999999,
X_field+'_def':bc_dist_to_unblocked_def\
['Yards_to_end_zone'].reset_index(drop=True)+10,
Y_field+'_def':26.65
},
inplace=True)
 bc_dist_to_unblocked_def.fillna({'distance':bc_dist_to_unblocked_def\ 
.apply(calc_dist_bc_to_nearest_EZ_corner,axis=1)\ 
.reset_index(drop=True)},
inplace=True)
 bc_dist_to_unblocked_def_summaries=bc_dist_to_unblocked_def.groupby('PlayId')['distance'].agg(['mean','median','min', 'max'])
 bc_dist_to_unblocked_def_summaries.columns=['bc_dist'+momentum_suffix+'_to_unblocked_def_'+col for col in bc_dist_to_unblocked_def_summaries.columns]
play_bc_dist_to_unblocked_def_rank_df=bc_dist_to_unblocked_def[['PlayId','distance', 'bc_distance'+momentum_suffix+'_to_unblocked_def_rank',X_field+'_def',Y_field+'_def']]\ 
.pivot(index='PlayId',columns='bc_distance'+momentum_suffix+'_to_unblocked_def_rank',values=['distance',X_field+'_def',Y_field+'_def'])
 play_bc_dist_to_unblocked_def_rank_df.columns = ['bc_distance'+momentum_suffix+'_to_unblocked_def_rank'+str(col[1])+'_'+col[0] for col in play_bc_dist_to_unblocked_def_rank_df.columns.values]
if verbose:
print("Distances to ball carrier"+momentum_suffix+"done")
 end = time.time()
print(end - start)
 start = time.time()
def calc_def_hull_measures(runner_v_def_points):
hull_values=[]
for layer_of_def in range(0,3):
try:
layer_hull = ConvexHull(runner_v_def_points[[X_field,Y_field]])
 hull_area=layer_hull.area
hull_expected_gain=runner_v_def_points[X_field].max()
hull_width=runner_v_def_points[Y_field].max() -runner_v_def_points[Y_field].min()
hull_defenders=len(layer_hull.vertices)-1
runner_vertice=runner_v_def_points.query("side_of_ball=='OFF'")[[X_field,Y_field]]
runner_vertice_backstop=runner_vertice.copy()
runner_vertice_backstop2=runner_vertice_backstop.copy()
runner_vertice_backstop[X_field]=runner_vertice_backstop[X_field]-1.0
runner_vertice_backstop[Y_field]=runner_vertice_backstop[Y_field]-1.0
runner_vertice_backstop2[X_field]=runner_vertice_backstop2[X_field]-1.0
runner_vertice_backstop2[Y_field]=runner_vertice_backstop2[Y_field]+1.0
FD_stick_left=runner_v_def_points.query("side_of_ball=='OFF'")[[X_field,Y_field]]
FD_stick_right=FD_stick_left.copy()
FD_stick_left[X_field]=runner_v_def_points.query("side_of_ball=='OFF'")['Distance']
FD_stick_left[Y_field]=26.65
FD_stick_right[X_field]=runner_v_def_points.query("side_of_ball=='OFF'")['Distance']
FD_stick_right[Y_field]=-26.65
Goal_Post=runner_v_def_points.query("side_of_ball=='OFF'")[[X_field,Y_field]]
Goal_Post[X_field]=runner_v_def_points.query("side_of_ball=='OFF'")['Yards_to_end_zone']+10
Goal_Post[Y_field]=0
voronoi_points=runner_v_def_points[[X_field,Y_field]].iloc[layer_hull.vertices]\ 
.append(runner_vertice_backstop ).append(runner_vertice_backstop2)\ 
.append(Goal_Post)
 if ~(runner_v_def_points[['side_of_ball']].iloc[layer_hull.vertices]=='OFF' ).any() [0]:
voronoi_points=voronoi_points.append(runner_vertice)
 vor = Voronoi(voronoi_points)
 RB_point_ix = np.where(( vor.points==np.array(runner_vertice)).all(axis=1)) [0][0]
RB_point_region_ix=vor.point_region[RB_point_ix]
RB_point_region_vertice_ixs= vor.regions[RB_point_region_ix]
RB_point_region_vertices=vor.vertices[RB_point_region_vertice_ixs]
voronoi_expected_gain=RB_point_region_vertices[:,0].max()
voronoi_width=RB_point_region_vertices[:,1].max() -RB_point_region_vertices[:,1].min()
hull_values.append(hull_area)
 hull_values.append(hull_expected_gain)
 hull_values.append(hull_width)
 hull_values.append(hull_defenders)
 hull_values.append(voronoi_expected_gain)
 hull_values.append(voronoi_width)
 runner_v_def_points = runner_v_def_points.drop(index=runner_v_def_points.iloc[layer_hull.vertices].query("side_of_ball=='DEF'" ).index)
 except:
hull_values.append(hull_area)
 hull_values.append(hull_expected_gain)
 hull_values.append(hull_width)
 hull_values.append(0)
 hull_values.append(voronoi_expected_gain)
 hull_values.append(voronoi_width)
 hull_columns=[
'hull_secondary'+momentum_suffix+'_area', 'hull_secondary'+momentum_suffix+'_expected_gain',
'hull_secondary'+momentum_suffix+'_width','hull_secondary'+momentum_suffix+'_defenders',
'voronoi_secondary'+momentum_suffix+'_expected_gain','voronoi_secondary'+momentum_suffix+'_width',
'hull_contain'+momentum_suffix+'_area', 'hull_contain'+momentum_suffix+'_expected_gain',
'hull_contain'+momentum_suffix+'_width','hull_contain'+momentum_suffix+'_defenders',
'voronoi_contain'+momentum_suffix+'_expected_gain','voronoi_contain'+momentum_suffix+'_width',
'hull_2nd_attack'+momentum_suffix+'_area', 'hull_2nd_attack'+momentum_suffix+'_expected_gain',
'hull_2nd_attack'+momentum_suffix+'_width','hull_2nd_attack'+momentum_suffix+'_defenders',
'voronoi_2nd_attack'+momentum_suffix+'_expected_gain','voronoi_2nd_attack'+momentum_suffix+'_width',
]
return pd.DataFrame([hull_values],columns=hull_columns)
 defender_hulls_df=tracking_df.query("(side_of_ball=='DEF' or ball_carrier==True)")[['PlayId',X_field,Y_field,'side_of_ball','Distance','Yards_to_end_zone']].groupby('PlayId' ).apply(calc_def_hull_measures ).droplevel(level=1)
 if verbose:
print("defensive hulls"+momentum_suffix+" done")
 end = time.time()
print(end - start)
 start = time.time()
blockers_df=tracking_df.query("side_of_ball=='OFF' and Position not in('WR','RB','QB','FB')")[['PlayId','Position',Y_field,X_field,'NflId']].sort_values(by=['PlayId',Y_field], ascending=False)
 blockers_df['blocker_left_to_right_rank']=blockers_df.groupby(['PlayId'])[Y_field].rank(method='first',ascending=False ).astype(int)
 blockers_df['pair_rank']=blockers_df['blocker_left_to_right_rank']+1
left_sideline=blockers_df.query("blocker_left_to_right_rank==1" ).assign(Position='LS',Y_field=53.3, X_field=0, NflId=0000000,
blocker_left_to_right_rank=0, pair_rank=1)
 gap_left_edges_df=pd.concat([blockers_df,left_sideline], sort=True ).sort_values(by=['PlayId','blocker_left_to_right_rank'])
 gaps_df=gap_left_edges_df.merge(gap_left_edges_df, how='left'
, left_on=['PlayId','pair_rank'],right_on=['PlayId','blocker_left_to_right_rank'], suffixes=['_left','_right'])\ 
.fillna(value={'Position_right':'RS',Y_field+'_right':0,X_field+'_right':0,'NflId_right':1111111})
 gaps_df=gaps_df.merge(tracking_df.query('ball_carrier==True')[['PlayId',Y_field,X_field,'Orientation_0Deg_to_endzone','Dir_0Deg_to_endzone']], on='PlayId')\ 
.rename(columns={Y_field:Y_field+'_ball_carrier',X_field:X_field+'_ball_carrier','Orientation_0Deg_to_endzone':'Orientation_ball_carrier','Dir_0Deg_to_endzone':'Dir_ball_carrier'})
 gaps_df['gap_width']=np.linalg.norm(gaps_df[[X_field+'_left', Y_field+'_left']].values - gaps_df[[X_field+'_right', Y_field+'_right']].values, axis=1)
 gaps_df['gap_center_X']=(gaps_df[X_field+'_left']+gaps_df[X_field+'_right'])/2.0
gaps_df['gap_center_Y']=(gaps_df[Y_field+'_left']+gaps_df[Y_field+'_right'])/2.0
gaps_df['gap_distance_from_ball_carrier']=np.linalg.norm(gaps_df[['gap_center_X', 'gap_center_Y']].values - gaps_df[[X_field+'_ball_carrier', Y_field+'_ball_carrier']].values, axis=1)
 gaps_df=gaps_df.merge(engaged[['PlayId','NflId_off','Off_leverage_right']].query('Off_leverage_right==False' ).drop_duplicates()
, left_on=['PlayId','NflId_left']
, right_on=['PlayId','NflId_off']
, how='left')\ 
.drop(columns='NflId_off')\ 
.fillna(True)\ 
.merge(engaged[['PlayId','NflId_off','Off_leverage_left']].query('Off_leverage_left==False' ).drop_duplicates()
, left_on=['PlayId','NflId_right']
, right_on=['PlayId','NflId_off']
, how='left')\ 
.drop(columns='NflId_off')\ 
.fillna(True)
 gaps_df['Off_leverage_right'].loc[gaps_df['Position_left']=='LS']=False
gaps_df['Off_leverage_left'].loc[gaps_df['Position_right']=='RS']=False
gaps_df['Off_full_leverage']=gaps_df[['Off_leverage_right','Off_leverage_left']].all(axis=1)
 gaps_df['Off_no_leverage']=~gaps_df[['Off_leverage_right','Off_leverage_left']].any(axis=1)
 def ball_carrier_to_gap_direction(x):
if x['gap_center_Y']>x[Y_field+'_ball_carrier']:
to_center_direction = np.arctan(( x['gap_center_X']-x[X_field+'_ball_carrier'])/(x['gap_center_Y']-x[Y_field+'_ball_carrier'])) /np.pi*180.0
elif x['gap_center_Y']<x[Y_field+'_ball_carrier']:
to_center_direction =180.0 + np.arctan(( x['gap_center_X']-x[X_field+'_ball_carrier'])/(x['gap_center_Y']-x[Y_field+'_ball_carrier'])) /np.pi*180.0
else:
to_center_direction = 90.0
return to_center_direction
gaps_df['ball_carrier_to_gap_direction']=gaps_df.apply(ball_carrier_to_gap_direction, axis=1)
 gaps_df['ball_carrier_Orientation_to_gap_ang_sim']=gaps_df.apply(lambda x: 1 - np.abs(x['Orientation_ball_carrier']-x['ball_carrier_to_gap_direction'])/180.0, axis=1)
 gaps_df['ball_carrier_Orientation'+momentum_suffix+'_to_gap_ang_sim_rank']=gaps_df.groupby(['PlayId'])['ball_carrier_Orientation_to_gap_ang_sim'].rank(method='first',ascending=True ).astype(int)
 gap_dist_to_unblocked_def=gaps_df[['PlayId','NflId_left','gap_center_X','gap_center_Y']]\ 
.merge(bc_dist_to_unblocked_def[['PlayId',X_field+'_def',Y_field+'_def']],
on='PlayId',
how='inner')
 def calc_dist_gap_to_def(x):
return calc_distance(x['gap_center_X'],x['gap_center_Y'],x[X_field+'_def'],x[Y_field+'_def'])
 gap_dist_to_unblocked_def['gap_distance_from_unblocked_def']=gap_dist_to_unblocked_def.apply(calc_dist_gap_to_def,axis=1)
 gap_min_dist_to_unblocked_def=gap_dist_to_unblocked_def.groupby(['PlayId','NflId_left'])[['gap_distance_from_unblocked_def']].min()
gaps_df=gaps_df.merge(gap_min_dist_to_unblocked_def,
left_on=['PlayId','NflId_left'],
right_index=True,
how='left')\ 
.fillna(55)
 gap_float_metrics=['gap_distance_from_ball_carrier',
'gap_width',
'gap_distance_from_unblocked_def']
gap_bool_metrics=['Off_full_leverage',
'Off_no_leverage'
]
def pivot_ranked_gaps(gap_rank_col,gap_values):
play_gaps_rank_df= gaps_df.query('{gap_rank_col}<=6'.format(gap_rank_col=gap_rank_col)) \
[['PlayId',gap_rank_col]+gap_values]\ 
.pivot(index='PlayId',columns=gap_rank_col
,values=gap_values)
 play_gaps_rank_df.columns = [gap_rank_col+str(col[1])+'_'+col[0] for col in play_gaps_rank_df.columns.values]
return play_gaps_rank_df
play_gaps_Orientation_ang_rank_df=pivot_ranked_gaps('ball_carrier_Orientation'+momentum_suffix+'_to_gap_ang_sim_rank',gap_float_metrics)
 play_gaps_Orientation_ang_rank_df_bools=pivot_ranked_gaps('ball_carrier_Orientation'+momentum_suffix+'_to_gap_ang_sim_rank',gap_bool_metrics)
 play_gaps_df=pd.concat([
play_gaps_Orientation_ang_rank_df,
play_gaps_Orientation_ang_rank_df_bools,
], axis=1)
 if verbose:
print("gaps"+momentum_suffix+"done")
 end = time.time()
print(end - start)
 start = time.time()
return play_gaps_df,\
defender_hulls_df,\
bc_dist_to_def_summaries,\
play_def_left_to_right_rank_df,\
play_bc_dist_to_def_rank_df,\
bc_dist_to_unblocked_def_summaries,\
play_bc_dist_to_unblocked_def_rank_df
play_gaps_df,\
defender_hulls_df,\
bc_dist_to_def_summaries,\
play_def_left_to_right_rank_df,\
play_bc_dist_to_def_rank_df,\
bc_dist_to_unblocked_def_summaries,\
play_bc_dist_to_unblocked_def_rank_df=distance_based_metrics(X='X_to_YardLine',Y='Y_dist_from_center',momentum=False)
 play_gaps_sec_momentum_df,\
defender_hulls_sec_momentum_df,\
bc_dist_to_def_sec_momentum_summaries,\
play_def_left_to_right_sec_momentum_rank_df,\
play_bc_dist_to_def_sec_momentum_rank_df,\
bc_dist_to_unblocked_def_sec_momentum_summaries,\
play_bc_dist_to_unblocked_def_sec_momentum_rank_df=distance_based_metrics(X='X_to_YardLine',Y='Y_dist_from_center',momentum=True)
 modeling_df=pd.get_dummies(
 ball_carrier_df.merge(team_off_yard_reference,left_on='PossessionTeam',right_index=True, how='inner')\ 
.merge(team_def_yard_reference,left_on='DefendingTeam',right_index=True, how='inner')\ 
.merge(position_cnts,left_on='PlayId',right_index=True, how='inner')\ 
.merge(position_grp_cnts,left_on='PlayId',right_index=True, how='inner')\ 
.merge(play_gaps_df,left_on='PlayId',right_index=True, how='inner')\ 
.merge(defender_hulls_df, left_on='PlayId',right_index=True, how='inner')\ 
.merge(bc_dist_to_def_summaries,left_on='PlayId',right_index=True, how='inner')\ 
.merge(play_def_left_to_right_rank_df,left_on='PlayId',right_index=True, how='inner')\ 
.merge(play_bc_dist_to_def_rank_df,left_on='PlayId',right_index=True, how='inner')\ 
.merge(bc_dist_to_unblocked_def_summaries,left_on='PlayId',right_index=True, how='inner')\ 
.merge(play_bc_dist_to_unblocked_def_rank_df,left_on='PlayId',right_index=True, how='inner')\ 
.merge(play_gaps_sec_momentum_df,left_on='PlayId',right_index=True, how='inner')\ 
.merge(defender_hulls_sec_momentum_df, left_on='PlayId',right_index=True, how='inner')\ 
.merge(bc_dist_to_def_sec_momentum_summaries,left_on='PlayId',right_index=True, how='inner')\ 
.merge(play_def_left_to_right_sec_momentum_rank_df,left_on='PlayId',right_index=True, how='inner')\ 
.merge(play_bc_dist_to_def_sec_momentum_rank_df,left_on='PlayId',right_index=True, how='inner')\ 
.merge(bc_dist_to_unblocked_def_sec_momentum_summaries,left_on='PlayId',right_index=True, how='inner')\ 
.merge(play_bc_dist_to_unblocked_def_sec_momentum_rank_df,left_on='PlayId',right_index=True, how='inner')\ 
.drop(columns=['GameId','PlayId','X','Y','Orientation','Dir','NflId','YardLine','Yards','Orientation_corrected','ball_carrier','side_of_ball',
'TimeHandoff','TimeSnap',
'StadiumType','Stadium','Turf','GameWeather','FieldPosition','PlayerBirthDate','HomeTeamAbbr','VisitorTeamAbbr','Humidity',
"HomeScoreBeforePlay", "VisitorScoreBeforePlay",'OffensePersonnel','DefensePersonnel',
'NflIdRusher','PossessionTeam', 'DefendingTeam',
'bc_dist_to_def_min','bc_dist_to_unblocked_def_min',
'PlayerCollegeName','Location','WindSpeed', 'WindDirection','JerseyNumber','DisplayName',
'X_same_way','Y_same_way','Dir_same_way','Orientation_same_way',
'Position','OffenseFormation', 'PlayDirection'
])
)\ 
.drop(columns=['Position_cnt_QB','Position_cnt_WR','Position_cnt_TE',
'Position_CB', 'Position_DE', 'Position_DT',
'Team_away','PlayDirection_right','Turf_type_Artificial'], errors='ignore')
 Yard_bins=['Yards'+str(i)for i in range(-99,100)]
Yard_bin_target_df=pd.cut(ball_carrier_df['Yards'],bins=range(-100,100),labels=Yard_bins)
 if verbose:
print("combine features done")
 end = time.time()
print(end - start)
 print("preprocessing done")
 return modeling_df, Yard_bin_target_df, team_off_yard_reference, team_def_yard_reference, rusher_yard_reference, rusher_low_vol_reference<define_variables>
folds = StratifiedKFold(n_splits=10, shuffle=True, random_state=59 )
must_keep_features=['A','Yards_to_end_zone',
'X_to_YardLine',
'S',
'bc_dist_to_def_median',
'Orientation_abs_0Deg_to_endzone',
'GameClock',
'bc_dist_sec_momentum_to_def_median',
'X_to_1stDown',
'bc_dist_to_def_mean',
'Dir_0Deg_to_endzone',
'Y_abs_dist_from_center',
'PlayerAge',
'Y_dist_from_center_sec_momentum',
'bc_dist_sec_momentum_to_def_min',
'bc_dist_sec_momentum_to_def_median',
'hull_secondary_expected_gain',
'hull_contain_expected_gain',
'hull_2nd_attack_expected_gain',
'voronoi_secondary_sec_momentum_expected_gain',
'voronoi_contain_sec_momentum_expected_gain',
'voronoi_2nd_attack_sec_momentum_expected_gain',
'ball_carrier_Orientation_to_gap_ang_sim_rank1_gap_width',
'ball_carrier_Orientation_to_gap_ang_sim_rank2_gap_width',
'ball_carrier_Orientation_to_gap_ang_sim_rank3_gap_width',
'ball_carrier_Orientation_to_gap_ang_sim_rank4_gap_width',
'ball_carrier_Orientation_to_gap_ang_sim_rank5_gap_width',
'ball_carrier_Orientation_to_gap_ang_sim_rank6_gap_width',
'ball_carrier_Orientation_to_gap_ang_sim_rank1_gap_distance_from_ball_carrier',
'ball_carrier_Orientation_to_gap_ang_sim_rank2_gap_distance_from_ball_carrier',
'ball_carrier_Orientation_to_gap_ang_sim_rank3_gap_distance_from_ball_carrier',
'ball_carrier_Orientation_to_gap_ang_sim_rank4_gap_distance_from_ball_carrier',
'ball_carrier_Orientation_to_gap_ang_sim_rank5_gap_distance_from_ball_carrier',
'ball_carrier_Orientation_to_gap_ang_sim_rank6_gap_distance_from_ball_carrier',
'ball_carrier_Orientation_to_gap_ang_sim_rank1_gap_distance_from_unblocked_def',
'ball_carrier_Orientation_to_gap_ang_sim_rank2_gap_distance_from_unblocked_def',
'ball_carrier_Orientation_to_gap_ang_sim_rank3_gap_distance_from_unblocked_def',
'ball_carrier_Orientation_to_gap_ang_sim_rank4_gap_distance_from_unblocked_def',
'ball_carrier_Orientation_to_gap_ang_sim_rank5_gap_distance_from_unblocked_def',
'ball_carrier_Orientation_to_gap_ang_sim_rank6_gap_distance_from_unblocked_def',
'def_left_to_right_rank1_X_to_YardLine_def',
'def_left_to_right_rank2_X_to_YardLine_def',
'def_left_to_right_rank3_X_to_YardLine_def',
'def_left_to_right_rank4_X_to_YardLine_def',
'def_left_to_right_rank5_X_to_YardLine_def',
'def_left_to_right_rank6_X_to_YardLine_def',
'def_left_to_right_rank7_X_to_YardLine_def',
'def_left_to_right_rank8_X_to_YardLine_def',
'def_left_to_right_rank9_X_to_YardLine_def',
'def_left_to_right_rank10_X_to_YardLine_def',
'def_left_to_right_rank11_X_to_YardLine_def',
'def_left_to_right_rank1_bc_distance_to_def',
'def_left_to_right_rank2_bc_distance_to_def',
'def_left_to_right_rank3_bc_distance_to_def',
'def_left_to_right_rank4_bc_distance_to_def',
'def_left_to_right_rank5_bc_distance_to_def',
'def_left_to_right_rank6_bc_distance_to_def',
'def_left_to_right_rank7_bc_distance_to_def',
'def_left_to_right_rank8_bc_distance_to_def',
'def_left_to_right_rank9_bc_distance_to_def',
'def_left_to_right_rank10_bc_distance_to_def',
'def_left_to_right_rank11_bc_distance_to_def',
'bc_distance_to_def_rank1_bc_distance_to_def',
'bc_distance_to_def_rank2_bc_distance_to_def',
'bc_distance_to_def_rank3_bc_distance_to_def',
'bc_distance_to_def_rank4_bc_distance_to_def',
'bc_distance_to_def_rank5_bc_distance_to_def',
'bc_distance_to_def_rank6_bc_distance_to_def',
'bc_distance_to_def_rank7_bc_distance_to_def',
'bc_distance_to_def_rank8_bc_distance_to_def',
'bc_distance_to_def_rank9_bc_distance_to_def',
'bc_distance_to_def_rank10_bc_distance_to_def',
'bc_distance_to_def_rank11_bc_distance_to_def',
'bc_distance_to_unblocked_def_rank1_Y_dist_from_center_def',
'bc_distance_to_unblocked_def_rank2_Y_dist_from_center_def',
'bc_distance_to_unblocked_def_rank3_Y_dist_from_center_def',
'bc_distance_to_unblocked_def_rank1_X_to_YardLine_def',
'bc_distance_to_unblocked_def_rank2_X_to_YardLine_def',
'bc_distance_to_unblocked_def_rank3_X_to_YardLine_def',
'ball_carrier_Orientation_sec_momentum_to_gap_ang_sim_rank6_gap_distance_from_ball_carrier',
'ball_carrier_Orientation_sec_momentum_to_gap_ang_sim_rank1_gap_width',
'ball_carrier_Orientation_sec_momentum_to_gap_ang_sim_rank2_gap_width',
'ball_carrier_Orientation_sec_momentum_to_gap_ang_sim_rank3_gap_width',
'ball_carrier_Orientation_sec_momentum_to_gap_ang_sim_rank4_gap_width',
'ball_carrier_Orientation_sec_momentum_to_gap_ang_sim_rank5_gap_width',
'ball_carrier_Orientation_sec_momentum_to_gap_ang_sim_rank6_gap_width',
'ball_carrier_Orientation_sec_momentum_to_gap_ang_sim_rank1_gap_distance_from_unblocked_def',
'ball_carrier_Orientation_sec_momentum_to_gap_ang_sim_rank2_gap_distance_from_unblocked_def',
'ball_carrier_Orientation_sec_momentum_to_gap_ang_sim_rank3_gap_distance_from_unblocked_def',
'ball_carrier_Orientation_sec_momentum_to_gap_ang_sim_rank4_gap_distance_from_unblocked_def',
'ball_carrier_Orientation_sec_momentum_to_gap_ang_sim_rank5_gap_distance_from_unblocked_def',
'ball_carrier_Orientation_sec_momentum_to_gap_ang_sim_rank6_gap_distance_from_unblocked_def',
'ball_carrier_Orientation_sec_momentum_to_gap_ang_sim_rank1_gap_distance_from_ball_carrier',
'ball_carrier_Orientation_sec_momentum_to_gap_ang_sim_rank2_gap_distance_from_ball_carrier',
'ball_carrier_Orientation_sec_momentum_to_gap_ang_sim_rank3_gap_distance_from_ball_carrier',
'ball_carrier_Orientation_sec_momentum_to_gap_ang_sim_rank4_gap_distance_from_ball_carrier',
'ball_carrier_Orientation_sec_momentum_to_gap_ang_sim_rank5_gap_distance_from_ball_carrier',
'ball_carrier_Orientation_sec_momentum_to_gap_ang_sim_rank6_gap_distance_from_ball_carrier',
'def_left_to_right_sec_momentum_rank1_bc_distance_to_def',
'def_left_to_right_sec_momentum_rank2_bc_distance_to_def',
'def_left_to_right_sec_momentum_rank3_bc_distance_to_def',
'def_left_to_right_sec_momentum_rank4_bc_distance_to_def',
'def_left_to_right_sec_momentum_rank5_bc_distance_to_def',
'def_left_to_right_sec_momentum_rank6_bc_distance_to_def',
'def_left_to_right_sec_momentum_rank7_bc_distance_to_def',
'def_left_to_right_sec_momentum_rank8_bc_distance_to_def',
'def_left_to_right_sec_momentum_rank9_bc_distance_to_def',
'def_left_to_right_sec_momentum_rank10_bc_distance_to_def',
'def_left_to_right_sec_momentum_rank11_bc_distance_to_def',
'def_left_to_right_sec_momentum_rank1_X_to_YardLine_sec_momentum_def',
'def_left_to_right_sec_momentum_rank2_X_to_YardLine_sec_momentum_def',
'def_left_to_right_sec_momentum_rank3_X_to_YardLine_sec_momentum_def',
'def_left_to_right_sec_momentum_rank4_X_to_YardLine_sec_momentum_def',
'def_left_to_right_sec_momentum_rank5_X_to_YardLine_sec_momentum_def',
'def_left_to_right_sec_momentum_rank6_X_to_YardLine_sec_momentum_def',
'def_left_to_right_sec_momentum_rank7_X_to_YardLine_sec_momentum_def',
'def_left_to_right_sec_momentum_rank8_X_to_YardLine_sec_momentum_def',
'def_left_to_right_sec_momentum_rank9_X_to_YardLine_sec_momentum_def',
'def_left_to_right_sec_momentum_rank10_X_to_YardLine_sec_momentum_def',
'def_left_to_right_sec_momentum_rank11_X_to_YardLine_sec_momentum_def',
'bc_distance_sec_momentum_to_def_rank1_bc_distance_to_def',
'bc_distance_sec_momentum_to_def_rank2_bc_distance_to_def',
'bc_distance_sec_momentum_to_def_rank3_bc_distance_to_def',
'bc_distance_sec_momentum_to_def_rank4_bc_distance_to_def',
'bc_distance_sec_momentum_to_def_rank5_bc_distance_to_def',
'bc_distance_sec_momentum_to_def_rank6_bc_distance_to_def',
'bc_distance_sec_momentum_to_def_rank7_bc_distance_to_def',
'bc_distance_sec_momentum_to_def_rank8_bc_distance_to_def',
'bc_distance_sec_momentum_to_def_rank9_bc_distance_to_def',
'bc_distance_sec_momentum_to_def_rank10_bc_distance_to_def',
'bc_distance_sec_momentum_to_def_rank11_bc_distance_to_def',
'bc_distance_sec_momentum_to_unblocked_def_rank1_X_to_YardLine_sec_momentum_def',
'bc_distance_sec_momentum_to_unblocked_def_rank2_X_to_YardLine_sec_momentum_def',
'bc_distance_sec_momentum_to_unblocked_def_rank3_X_to_YardLine_sec_momentum_def',
'bc_distance_sec_momentum_to_unblocked_def_rank1_Y_dist_from_center_sec_momentum_def',
'bc_distance_sec_momentum_to_unblocked_def_rank2_Y_dist_from_center_sec_momentum_def',
'bc_distance_sec_momentum_to_unblocked_def_rank3_Y_dist_from_center_sec_momentum_def',
]<define_variables>
predicted = np.zeros(( test.shape[0],9))
measured= np.zeros(( data.shape[0]))
score = 0
%time predictive_df, target_df, team_off_yard_reference, team_def_yard_reference, rusher_yard_reference, rusher_low_vol_reference =preprocess_tracking_data(tracking_df_train, verbose=1, for_test=False)
 model_features=must_keep_features<train_model>
for times,(trn_idx, val_idx)in enumerate(folds.split(data.values,target['surface'].values)) :
model = RandomForestClassifier(n_estimators=500, n_jobs = -1)
 model.fit(data.iloc[trn_idx],target['surface'][trn_idx])
 measured[val_idx] = model.predict(data.iloc[val_idx])
 predicted += model.predict_proba(test)/folds.n_splits
score += model.score(data.iloc[val_idx],target['surface'][val_idx])
 print("Fold: {} score: {}".format(times,model.score(data.iloc[val_idx],target['surface'][val_idx])))
 importances = model.feature_importances_
indices = np.argsort(importances)
 features = data.columns
if model.score(data.iloc[val_idx],target['surface'][val_idx])> 0.92000:
hm = 30
plt.figure(figsize=(7, 10))
plt.title('Feature Importances')
 plt.barh(range(len(indices[:hm])) , importances[indices][:hm], color='b', align='center')
 plt.yticks(range(len(indices[:hm])) , [features[i] for i in indices])
 plt.xlabel('Relative Importance')
 plt.show()
gc.collect()
model=ExtraTreesClassifier(n_estimators=400,
max_features=0.8,
criterion='gini',
n_jobs=-1,
bootstrap=False,
min_samples_leaf=10,
verbose=0)
 model.fit(predictive_df[model_features], pd.get_dummies(target_df ).T.cumsum().T )<split>
print('Avg Accuracy RF', score / folds.n_splits )
iter_test = env.iter_test()<predict_on_test>
confusion_matrix(measured,target['surface'] )
def make_my_predictions_on_cdf_model(test_df, model):

 test_df['Yards']=0
predictive_df, target_df, _, _ , _ , _=preprocess_tracking_data(test_df)
 missing_features=list(set(model_features)-set(predictive_df.columns))
missing_features_dict={}
for feature in missing_features:
missing_features_dict[feature]=[0]
predictive_df=pd.concat([predictive_df.reset_index(drop=True),pd.DataFrame(missing_features_dict)], axis=1)
 predicted_probas=model.predict_proba(predictive_df[model_features])
 prediction_cdf=pd.DataFrame([1-a[:,0] for a in predicted_probas])
 prediction_cdf=prediction_cdf
prediction_cdf.iloc[:predictive_df['Yards_to_end_zone'][0]]=0.0
prediction_cdf.iloc[99+predictive_df['Yards_to_end_zone'][0]:]=1.0
prediction_cdf.where(prediction_cdf<=1,1,inplace=True)
 prediction_cdf=prediction_cdf.T
prediction_cdf.columns=pd.get_dummies(target_df ).columns
return prediction_cdf<create_dataframe>
sub['surface'] = le.inverse_transform(predicted.argmax(axis=1))
sub.to_csv('submission.csv', index=False)
 sub.head()
<create_dataframe>
best_sub = pd.read_csv('.. /input/robots-best-submission/final_submission.csv')
 best_sub.to_csv('best_submission.csv', index=False)
 best_sub.head(10 )
count=0
predictions_test=pd.DataFrame()
predictions_orig_test=pd.DataFrame()
for(test_df, _)in iter_test:
count+=1
if count%100==0:
print(count)
 predictions_df = make_my_predictions_on_cdf_model(test_df, model)
 predictions_df.columns=list(predictions_df.columns)
 env.predict(predictions_df )<save_to_csv>
sub073 = pd.read_csv('.. /input/robots-best-submission/mybest0.73.csv')
 sub072 = pd.read_csv('.. /input/robots-best-submission/sub_0.72.csv')
 sub072_2 = pd.read_csv('.. /input/robots-best-submission/sub_0.72_2.csv')
 sub071 = pd.read_csv('.. /input/robots-best-submission/sub_0.71.csv')
 sub06 = pd.read_csv('.. /input/robots-best-submission/sub_0.6.csv')
 sub073 = sub073.rename(columns = {'surface':'surface073'})
 sub072 = sub072.rename(columns = {'surface':'surface072'})
 sub072_2 = sub072_2.rename(columns = {'surface':'surface072_2'})
 sub071 = sub071.rename(columns = {'surface':'surface071'})
 sub06 = sub06.rename(columns = {'surface':'surface06'})
 print("Submission data is ready" )
env.write_submission_file()<set_options>
subtest = pd.concat([sub073['series_id'], sub073['surface073'], sub072['surface072'], sub071['surface071'], sub06['surface06']], axis=1)
 subtest.head()
torch.manual_seed(0)
 np.random.seed(0)
 class Data(object):
def __init__(self, players: torch.Tensor, rusher: torch.Tensor, meta: torch.Tensor, y: Optional[torch.Tensor],
yardLine: np.ndarray, year: np.ndarray, player_cols: List[str], rusher_cols: List[str],
meta_cols: List[str]):
self.players = players
self.rusher = rusher
self.meta = meta
self.y = y
self.yardLine = yardLine
self.year = year
self.player_cols = player_cols
self.rusher_cols = rusher_cols
self.meta_cols = meta_cols
assert self.players.size(0)== self.rusher.size(0)
 if yardLine is not None:
assert len(yardLine)== self.players.size(0)
 def len(self):
return self.players.size(0)
 def y_soft(self, sigma: float = 1.0):
return torch.from_numpy(gaussian_filter1d(self.y.numpy() , sigma=sigma))
def slice(self, begin: int, end: int)-> 'Data':
p = self.players[begin:end] if self.players is not None else None
r = self.rusher[begin:end] if self.rusher is not None else None
m = self.meta[begin:end] if self.meta is not None else None
y = self.y[begin:end] if self.y is not None else None
yd = self.yardLine[begin:end].copy() if self.yardLine is not None else None
yr = self.year[begin:end].copy() if self.year is not None else None
return Data(p, r, m, y, yd, yr, self.player_cols, self.rusher_cols, self.meta_cols)
 def _sample_by_mask(self, mask):
mask_tensor = torch.from_numpy(mask)
 p = self.players[mask_tensor] if self.players is not None else None
r = self.rusher[mask_tensor] if self.rusher is not None else None
m = self.meta[mask_tensor] if self.meta is not None else None
y = self.y[mask_tensor] if self.y is not None else None
yd = self.yardLine[mask].copy() if self.yardLine is not None else None
yr = self.year[mask].copy() if self.year is not None else None
return Data(p, r, m, y, yd, yr, self.player_cols, self.rusher_cols, self.meta_cols)
 def downsample_2017(self, dropout_rate: float):
assert 0 <= dropout_rate <= 1.0
dropout = np.random.choice([True, False], size=len(self.year), p=[1 - dropout_rate, dropout_rate])
 mask =(self.year != 2017)| dropout
return self._sample_by_mask(mask)
 def shuffled(self):
indices = np.random.permutation(self.players.shape[0])
 p = np.take(self.players, indices, axis=0)
 r = np.take(self.rusher, indices, axis=0)if self.rusher is not None else None
m = np.take(self.meta, indices, axis=0)if self.meta is not None else None
y = np.take(self.y, indices, axis=0)if self.y is not None else None
yd = np.take(self.yardLine, indices, axis=0 ).copy() if self.yardLine is not None else None
yr = np.take(self.year, indices, axis=0 ).copy() if self.year is not None else None
return Data(p, r, m, y, yd, yr, self.player_cols, self.rusher_cols, self.meta_cols)
 def random_split(self, p: float):
mask = np.random.choice([True, False], p=[p, 1 - p], size=self.meta.size(0))
d1 = self._sample_by_mask(mask)
 d2 = self._sample_by_mask(~mask)
 return d1, d2
def concat_dataset(l: Data, r: Data):
p = torch.cat(( l.players, r.players))
rs = torch.cat(( l.rusher, r.rusher))
m = torch.cat(( l.meta, r.meta))
y = torch.cat(( l.y, r.y)) if l.y is not None else None
yd = np.concatenate([l.yardLine, r.yardLine])if l.yardLine is not None else None
yr = np.concatenate([l.year, r.year])if l.year is not None else None
print(p.size())
 print(yd.shape)
 return Data(p, rs, m, y, yd, yr, l.player_cols, l.rusher_cols, l.meta_cols)
 def _get_clones(module, N):
return ModuleList([copy.deepcopy(module)for i in range(N)])
 class MultiheadAttention(Module):
__constants__ = ['q_proj_weight', 'k_proj_weight', 'v_proj_weight', 'in_proj_weight']
def __init__(self, embed_dim, num_heads, dropout=0., bias=True, add_bias_kv=False, add_zero_attn=False, kdim=None,
vdim=None):
super(MultiheadAttention, self ).__init__()
self.embed_dim = embed_dim
self.kdim = kdim if kdim is not None else embed_dim
self.vdim = vdim if vdim is not None else embed_dim
self._qkv_same_embed_dim = self.kdim == embed_dim and self.vdim == embed_dim
self.num_heads = num_heads
self.dropout = dropout
self.head_dim = embed_dim // num_heads
assert self.head_dim * num_heads == self.embed_dim, "embed_dim must be divisible by num_heads"
if self._qkv_same_embed_dim is False:
self.q_proj_weight = Parameter(torch.Tensor(embed_dim, embed_dim))
self.k_proj_weight = Parameter(torch.Tensor(embed_dim, self.kdim))
self.v_proj_weight = Parameter(torch.Tensor(embed_dim, self.vdim))
self.register_parameter('in_proj_weight', None)
 else:
self.in_proj_weight = Parameter(torch.empty(3 * embed_dim, embed_dim))
self.register_parameter('q_proj_weight', None)
 self.register_parameter('k_proj_weight', None)
 self.register_parameter('v_proj_weight', None)
 if bias:
self.in_proj_bias = Parameter(torch.empty(3 * embed_dim))
else:
self.register_parameter('in_proj_bias', None)
 self.out_proj = Linear(embed_dim, embed_dim, bias=bias)
 if add_bias_kv:
self.bias_k = Parameter(torch.empty(1, 1, embed_dim))
self.bias_v = Parameter(torch.empty(1, 1, embed_dim))
else:
self.bias_k = self.bias_v = None
self.add_zero_attn = add_zero_attn
self._reset_parameters()
def _reset_parameters(self):
if self._qkv_same_embed_dim:
xavier_uniform_(self.in_proj_weight)
 else:
xavier_uniform_(self.q_proj_weight)
 xavier_uniform_(self.k_proj_weight)
 xavier_uniform_(self.v_proj_weight)
 if self.in_proj_bias is not None:
constant_(self.in_proj_bias, 0.)
constant_(self.out_proj.bias, 0.)
if self.bias_k is not None:
xavier_normal_(self.bias_k)
 if self.bias_v is not None:
xavier_normal_(self.bias_v)
 def forward(self, query, key, value, key_padding_mask=None,
need_weights=True, attn_mask=None):
if not self._qkv_same_embed_dim:
return multi_head_attention_forward(
 query, key, value, self.embed_dim, self.num_heads,
self.in_proj_weight, self.in_proj_bias,
self.bias_k, self.bias_v, self.add_zero_attn,
self.dropout, self.out_proj.weight, self.out_proj.bias,
training=self.training,
key_padding_mask=key_padding_mask, need_weights=need_weights,
attn_mask=attn_mask, use_separate_proj_weight=True,
q_proj_weight=self.q_proj_weight, k_proj_weight=self.k_proj_weight,
v_proj_weight=self.v_proj_weight)
 else:
return multi_head_attention_forward(
 query, key, value, self.embed_dim, self.num_heads,
self.in_proj_weight, self.in_proj_bias,
self.bias_k, self.bias_v, self.add_zero_attn,
self.dropout, self.out_proj.weight, self.out_proj.bias,
training=self.training,
key_padding_mask=key_padding_mask, need_weights=need_weights,
attn_mask=attn_mask)
 class TransformerEncoder(Module):
def __init__(self, encoder_layer, num_layers, norm=None):
super(TransformerEncoder, self ).__init__()
self.layers = _get_clones(encoder_layer, num_layers)
 self.num_layers = num_layers
self.norm = norm
def forward(self, src, mask=None, src_key_padding_mask=None):
r
output = src
for i in range(self.num_layers):
output = self.layers[i](output, src_mask=mask,
src_key_padding_mask=src_key_padding_mask)
 if self.norm:
output = self.norm(output)
 return output
class TransformerEncoderLayer(Module):
def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1, dropout_attn=0.1, pre_LN=False):
super(TransformerEncoderLayer, self ).__init__()
self.self_attn = MultiheadAttention(d_model, nhead, dropout=dropout_attn)
 self.linear1 = Linear(d_model, dim_feedforward)
 self.dropout = Dropout(dropout)
 self.linear2 = Linear(dim_feedforward, d_model)
 self.norm1 = LayerNorm(d_model)
 self.norm2 = LayerNorm(d_model)
 self.dropout1 = Dropout(dropout)
 self.dropout2 = Dropout(dropout)
 self.activation = F.relu
self.pre_LN = pre_LN
def forward(self, src, src_mask=None, src_key_padding_mask=None):
r
if self.pre_LN:
src2 = self.norm1(src)
 src2 = self.self_attn(src2, src2, src2, attn_mask=src_mask,
key_padding_mask=src_key_padding_mask)[0]
src = src + self.dropout1(src2)
 else:
src2 = self.self_attn(src, src, src, attn_mask=src_mask,
key_padding_mask=src_key_padding_mask)[0]
src = src + self.dropout1(src2)
 src = self.norm1(src)
 if hasattr(self, "activation"):
src2 = self.linear2(self.dropout(self.activation(self.linear1(src))))
else:
src2 = self.linear2(self.dropout(F.relu(self.linear1(src))))
src = src + self.dropout2(src2)
 src = self.norm2(src)
 return src
def multi_head_attention_forward(query,
key,
value,
embed_dim_to_check,
num_heads,
in_proj_weight,
in_proj_bias,
bias_k,
bias_v,
add_zero_attn,
dropout_p,
out_proj_weight,
out_proj_bias,
training=True,
key_padding_mask=None,
need_weights=True,
attn_mask=None,
use_separate_proj_weight=False,
q_proj_weight=None,
k_proj_weight=None,
v_proj_weight=None,
static_k=None,
static_v=None 
):
r
qkv_same = torch.equal(query, key)and torch.equal(key, value)
 kv_same = torch.equal(key, value)
 tgt_len, bsz, embed_dim = query.size()
assert embed_dim == embed_dim_to_check
assert list(query.size())== [tgt_len, bsz, embed_dim]
assert key.size() == value.size()
head_dim = embed_dim // num_heads
assert head_dim * num_heads == embed_dim, "embed_dim must be divisible by num_heads"
scaling = float(head_dim)** -0.5
if use_separate_proj_weight is not True:
if qkv_same:
q, k, v = F.linear(query, in_proj_weight, in_proj_bias ).chunk(3, dim=-1)
 elif kv_same:
_b = in_proj_bias
_start = 0
_end = embed_dim
_w = in_proj_weight[_start:_end, :]
if _b is not None:
_b = _b[_start:_end]
q = F.linear(query, _w, _b)
 if key is None:
assert value is None
k = None
v = None
else:
_b = in_proj_bias
_start = embed_dim
_end = None
_w = in_proj_weight[_start:, :]
if _b is not None:
_b = _b[_start:]
k, v = F.linear(key, _w, _b ).chunk(2, dim=-1)
 else:
_b = in_proj_bias
_start = 0
_end = embed_dim
_w = in_proj_weight[_start:_end, :]
if _b is not None:
_b = _b[_start:_end]
q = F.linear(query, _w, _b)
 _b = in_proj_bias
_start = embed_dim
_end = embed_dim * 2
_w = in_proj_weight[_start:_end, :]
if _b is not None:
_b = _b[_start:_end]
k = F.linear(key, _w, _b)
 _b = in_proj_bias
_start = embed_dim * 2
_end = None
_w = in_proj_weight[_start:, :]
if _b is not None:
_b = _b[_start:]
v = F.linear(value, _w, _b)
 else:
q_proj_weight_non_opt = torch.jit._unwrap_optional(q_proj_weight)
 len1, len2 = q_proj_weight_non_opt.size()
assert len1 == embed_dim and len2 == query.size(-1)
 k_proj_weight_non_opt = torch.jit._unwrap_optional(k_proj_weight)
 len1, len2 = k_proj_weight_non_opt.size()
assert len1 == embed_dim and len2 == key.size(-1)
 v_proj_weight_non_opt = torch.jit._unwrap_optional(v_proj_weight)
 len1, len2 = v_proj_weight_non_opt.size()
assert len1 == embed_dim and len2 == value.size(-1)
 if in_proj_bias is not None:
q = F.linear(query, q_proj_weight_non_opt, in_proj_bias[0:embed_dim])
 k = F.linear(key, k_proj_weight_non_opt, in_proj_bias[embed_dim:(embed_dim * 2)])
 v = F.linear(value, v_proj_weight_non_opt, in_proj_bias[(embed_dim * 2):])
 else:
q = F.linear(query, q_proj_weight_non_opt, in_proj_bias)
 k = F.linear(key, k_proj_weight_non_opt, in_proj_bias)
 v = F.linear(value, v_proj_weight_non_opt, in_proj_bias)
 q = q * scaling
if bias_k is not None and bias_v is not None:
if static_k is None and static_v is None:
k = torch.cat([k, bias_k.repeat(1, bsz, 1)])
 v = torch.cat([v, bias_v.repeat(1, bsz, 1)])
 if attn_mask is not None:
attn_mask = torch.cat([attn_mask,
torch.zeros(( attn_mask.size(0), 1),
dtype=attn_mask.dtype,
device=attn_mask.device)], dim=1)
 if key_padding_mask is not None:
key_padding_mask = torch.cat(
 [key_padding_mask, torch.zeros(( key_padding_mask.size(0), 1),
dtype=key_padding_mask.dtype,
device=key_padding_mask.device)], dim=1)
 else:
assert static_k is None, "bias cannot be added to static key."
assert static_v is None, "bias cannot be added to static value."
else:
assert bias_k is None
assert bias_v is None
q = q.contiguous().view(tgt_len, bsz * num_heads, head_dim ).transpose(0, 1)
 if k is not None:
k = k.contiguous().view(-1, bsz * num_heads, head_dim ).transpose(0, 1)
 if v is not None:
v = v.contiguous().view(-1, bsz * num_heads, head_dim ).transpose(0, 1)
 if static_k is not None:
assert static_k.size(0)== bsz * num_heads
assert static_k.size(2)== head_dim
k = static_k
if static_v is not None:
assert static_v.size(0)== bsz * num_heads
assert static_v.size(2)== head_dim
v = static_v
src_len = k.size(1)
 if key_padding_mask is not None:
assert key_padding_mask.size(0)== bsz
assert key_padding_mask.size(1)== src_len
if add_zero_attn:
src_len += 1
k = torch.cat([k, torch.zeros(( k.size(0), 1)+ k.size() [2:], dtype=k.dtype, device=k.device)], dim=1)
 v = torch.cat([v, torch.zeros(( v.size(0), 1)+ v.size() [2:], dtype=v.dtype, device=v.device)], dim=1)
 if attn_mask is not None:
attn_mask = torch.cat([attn_mask, torch.zeros(( attn_mask.size(0), 1),
dtype=attn_mask.dtype,
device=attn_mask.device)], dim=1)
 if key_padding_mask is not None:
key_padding_mask = torch.cat(
 [key_padding_mask, torch.zeros(( key_padding_mask.size(0), 1),
dtype=key_padding_mask.dtype,
device=key_padding_mask.device)], dim=1)
 attn_output_weights = torch.bmm(q, k.transpose(1, 2))
assert list(attn_output_weights.size())== [bsz * num_heads, tgt_len, src_len]
if attn_mask is not None:
attn_mask = attn_mask.unsqueeze(0)
 attn_output_weights += attn_mask
if key_padding_mask is not None:
attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len)
 attn_output_weights = attn_output_weights.masked_fill(
 key_padding_mask.unsqueeze(1 ).unsqueeze(2),
float('-inf'), 
)
 attn_output_weights = attn_output_weights.view(bsz * num_heads, tgt_len, src_len)
 attn_output_weights = F.softmax(
 attn_output_weights, dim=-1)
 attn_output_weights = F.dropout(attn_output_weights, p=dropout_p, training=training)
 attn_output = torch.bmm(attn_output_weights, v)
 assert list(attn_output.size())== [bsz * num_heads, tgt_len, head_dim]
attn_output = attn_output.transpose(0, 1 ).contiguous().view(tgt_len, bsz, embed_dim)
 attn_output = F.linear(attn_output, out_proj_weight, out_proj_bias)
 if need_weights:
attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len)
 return attn_output, attn_output_weights.sum(dim=1)/ num_heads
else:
return attn_output, None
class TransformerModel(nn.Module):
def __init__(self, ninp: int, nemb: int = 1, nhead: int = 1, nhid: int = 32, nlayers: int = 4, nfinal: int = 1024,
dropout_encoder: float = 0.1, dropout_embed: float = 0.0, dropout_classifier: float = 0.0,
n_class: int = 199, ninp_rusher: int = 16, pre_LN: bool = False, rusher_emb: int = 32,
n_emb_layers: int = 2,
ninp_meta: int = 8, meta_emb: int = 32, gauss_noise: float = 0.0,
gauss_xy_noise: float = 0.0,
n_fin_layers: int = 3, dropout_attn: float = 0):

 super(TransformerModel, self ).__init__()
self.model_type = 'Transformer'
encoder_layers = TransformerEncoderLayer(nemb, nhead, nhid, dropout_encoder, pre_LN=pre_LN, dropout_attn=dropout_attn)
 self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
 self.n_emb_layers = n_emb_layers
self.conv1 = nn.Conv1d(in_channels=ninp, out_channels=nemb, kernel_size=1)
 self.conv2 = nn.Conv1d(in_channels=nemb, out_channels=nemb, kernel_size=1)
 self.conv3 = nn.Conv1d(in_channels=nemb, out_channels=nemb, kernel_size=1)
 self.relu1 = nn.PReLU()
self.relu2 = nn.PReLU()
self.relu3 = nn.PReLU()
self.relu4 = nn.PReLU()
self.relu5 = nn.ReLU()
if dropout_embed > 0:
self.dropout1 = nn.Dropout(dropout_embed)
 else:
self.dropout1 = None
self.avgpool = nn.AvgPool1d(kernel_size=22)
 assert n_fin_layers == 3
self.linear = nn.Sequential(
 nn.Dropout(dropout_classifier),
nn.Linear(nemb + rusher_emb + meta_emb, nfinal),
nn.ReLU() ,
nn.Dropout(dropout_classifier),
nn.Linear(nfinal, nfinal),
nn.ReLU() ,
nn.Dropout(dropout_classifier),
nn.Linear(nfinal, n_class)
)
 self.activation = nn.Softmax(dim=1)
 self.linear2 = nn.Linear(ninp_rusher, rusher_emb)
 self.gauss_noise = gauss_noise
self.gauss_xy_noise = gauss_xy_noise
if meta_emb > 0:
self.linear3 = nn.Linear(ninp_meta, meta_emb)
 else:
self.linear3 = None
def forward(self, x: Data, with_clip: bool = True):
src = x.players
src_rusher = x.rusher
src = src.permute([0, 2, 1])
 if self.training:
if self.gauss_noise > 0.0:
noise = torch.randn_like(src)* self.gauss_noise
src = src + noise
if self.gauss_xy_noise > 0.0:
dy = torch.randn(src.size(0)) * self.gauss_xy_noise
src[:, 1, :] += dy.reshape(src.size(0), 1 ).expand(src.size(0), src.size(2))
src_rusher[:, 1] += dy
src = self.relu1(self.conv1(src))
src = self.dropout1(src)
 src = self.relu2(self.conv2(src))
src = self.dropout1(src)
 src = self.relu3(self.conv3(src))
src = src.permute([2, 0, 1])
 output = self.transformer_encoder(src)
 output = output.permute([1, 2, 0])
 output = torch.squeeze(self.avgpool(output), dim=2)
 if self.linear3 is not None:
output = torch.cat(( output, self.relu4(self.linear2(src_rusher)) , self.relu5(self.linear3(x.meta))), dim=1)
 else:
output = torch.cat(( output, self.relu4(self.linear2(src_rusher))), dim=1)
 output = self.linear(output)
 output = self.activation(output)
 if not self.training and x.yardLine is not None:
output = torch.cumsum(output, dim=1 ).numpy()
output = np.clip(output, 0.0, 1.0)
 if with_clip:
left = 99 - x.yardLine
right = 199 - x.yardLine
for k in range(len(output)) :
output[k, :left[k] + 1] = 0.0
output[k, right[k]:] = 1.0
return output
def prep(df: pd.DataFrame)-> Tuple[pd.DataFrame, pd.DataFrame]:
df['ToLeft'] = df.PlayDirection == "left"
df['IsBallCarrier'] = df.NflId == df.NflIdRusher
df.loc[df.VisitorTeamAbbr == "ARI", 'VisitorTeamAbbr'] = "ARZ"
df.loc[df.HomeTeamAbbr == "ARI", 'HomeTeamAbbr'] = "ARZ"
df.loc[df.VisitorTeamAbbr == "BAL", 'VisitorTeamAbbr'] = "BLT"
df.loc[df.HomeTeamAbbr == "BAL", 'HomeTeamAbbr'] = "BLT"
df.loc[df.VisitorTeamAbbr == "CLE", 'VisitorTeamAbbr'] = "CLV"
df.loc[df.HomeTeamAbbr == "CLE", 'HomeTeamAbbr'] = "CLV"
df.loc[df.VisitorTeamAbbr == "HOU", 'VisitorTeamAbbr'] = "HST"
df.loc[df.HomeTeamAbbr == "HOU", 'HomeTeamAbbr'] = "HST"
df['TeamOnOffense'] = "home"
df.loc[df.PossessionTeam != df.HomeTeamAbbr, 'TeamOnOffense'] = "away"
df['IsOnOffense'] = df.Team == df.TeamOnOffense
df['YardLine_std'] = 100 - df.YardLine
df.loc[df.FieldPosition.fillna('')== df.PossessionTeam, 'YardLine_std'] = df.loc[
df.FieldPosition.fillna('')== df.PossessionTeam, 'YardLine']
df['X_std'] = df.X
df.loc[df.ToLeft, 'X_std'] = 120 - df.loc[df.ToLeft, 'X']
df['Y_std'] = df.Y
df.loc[df.ToLeft, 'Y_std'] = 160 / 3 - df.loc[df.ToLeft, 'Y']
df['Orientation_std'] = df.Orientation
df.loc[df.ToLeft, 'Orientation_std'] = np.mod(180 + df.loc[df.ToLeft, 'Orientation_std'], 360)
 df['Dir_std'] = df.Dir
df.loc[df.ToLeft, 'Dir_std'] = np.mod(180 + df.loc[df.ToLeft, 'Dir_std'], 360)
 df['IsOffence'] = df['Team'] == df['TeamOnOffense']
df['OffenceScoreBeforePlay'] = df['HomeScoreBeforePlay']
df.loc[df.TeamOnOffense == "away", 'OffenceScoreBeforePlay'] = df.loc[
df.TeamOnOffense == "away", 'VisitorScoreBeforePlay']
df['DefenseScoreBeforePlay'] = df['VisitorScoreBeforePlay']
df.loc[df.TeamOnOffense == "away", 'DefenseScoreBeforePlay'] = df.loc[
df.TeamOnOffense == "away", 'HomeScoreBeforePlay']
df['OffenceTeamAbbr'] = df['HomeTeamAbbr']
df.loc[df.TeamOnOffense == "away", 'OffenceTeamAbbr'] = df.loc[df.TeamOnOffense == "away", 'VisitorTeamAbbr']
df['DefenseTeamAbbr'] = df['VisitorTeamAbbr']
df.loc[df.TeamOnOffense == "away", 'DefenseTeamAbbr'] = df.loc[df.TeamOnOffense == "away", 'HomeTeamAbbr']
df['Year'] = pd.to_datetime(df.TimeSnap ).dt.year
df.loc[df['Year'] == 2017, 'Orientation_std'] = np.mod(90 + df.loc[df['Year'] == 2017, 'Orientation_std'], 360)
 player_features = ['X_std', 'Y_std', 'S', 'A', 'Dis', 'Orientation_std', 'Dir_std', 'NflId', 'JerseyNumber',
'PlayerHeight',
'PlayerWeight', 'PlayerBirthDate', 'PlayerCollegeName', 'Position', 'IsBallCarrier', 'IsOffence']
play_features = ['YardLine_std', 'Quarter', 'GameClock', 'PossessionTeam', 'Down', 'Distance', 'FieldPosition',
'OffenceScoreBeforePlay', 'DefenseScoreBeforePlay', 'OffenseFormation', 'OffensePersonnel',
'DefendersInTheBox', 'DefensePersonnel', 'TimeHandoff', 'TimeSnap',
'OffenceTeamAbbr', 'DefenseTeamAbbr', 'Week', 'Stadium', 'Location',
'Turf', 'GameWeather', 'Temperature', 'Humidity', 'WindSpeed', 'WindDirection',
'TeamOnOffense', 'HomeTeamAbbr', 'Year']
if 'Yards' in df:
play_features.append('Yards')
 players = df[['PlayId', 'GameId'] + player_features].copy()
play = df[['GameId', 'PlayId'] + play_features].copy().drop_duplicates(subset=['PlayId'])
 return play, players
def prep_players_nn(play, players, scaler=None, scaler_meta=None):
p = players.drop(['NflId', 'JerseyNumber', 'PlayerHeight', 'PlayerBirthDate', 'PlayerCollegeName', 'GameId'],
axis=1 ).copy()
if scaler is None:
is_training = True
else:
assert scaler_meta is not None
is_training = False
if 'YardLine_std' not in p:
p = pd.merge(p, play[['YardLine_std', 'PlayId']], on='PlayId', how='left')
 p['IsBallCarrier'] = p['IsBallCarrier'].astype(int)
 p['IsOffence'] = p['IsOffence'].astype(int)
 p['X_std'] -= p['YardLine_std']
p['Dir_cos'] = np.cos(np.deg2rad(90 - p['Dir_std']))
p['Dir_sin'] = np.sin(np.deg2rad(90 - p['Dir_std']))
p = p.fillna(0)
 rb = p[p['IsBallCarrier'] == 1][['X_std', 'Y_std', 'PlayId', 'S', 'Dir_sin', 'Dir_cos', 'Dir_std']]
rb.columns = ['X_', 'Y_', 'PlayId', 'S_', 'Dir_sin_', 'Dir_cos_', 'Dir_std_']
p = pd.merge(p, rb, how='left')
 p['DX'] = p['X_std'] - p['X_']
p['DY'] = p['Y_std'] - p['Y_']
angles = 90.0 - np.rad2deg(np.arctan2(p['DY'], p['DX']))
p['AngleFromRB'] = angles - p['Dir_std_']
p['AngleFromRB'] = np.mod(p['AngleFromRB'] + 360, 360)
 p.loc[p['AngleFromRB'] > 180, 'AngleFromRB'] -= 360
dt = 0.001
p['DX2'] =(p['X_std'] + dt * p['Dir_cos'] * p['S'])-(p['X_'] + dt * p['Dir_cos_'] * p['S_'])
 p['DY2'] =(p['Y_std'] + dt * p['Dir_sin'] * p['S'])-(p['Y_'] + dt * p['Dir_sin_'] * p['S_'])
 angles = 90.0 - np.rad2deg(np.arctan2(p['DY2'], p['DX2']))
p['AngleFromRB2'] = angles - p['Dir_std_']
p['AngleFromRB2'] = np.mod(p['AngleFromRB2'] + 360, 360)
 p.loc[p['AngleFromRB2'] > 180, 'AngleFromRB2'] -= 360
p['AngleFromRB2'] = p['AngleFromRB2'] - p['AngleFromRB']
p.loc[p['AngleFromRB2'] > 180, 'AngleFromRB2'] -= 360
p.loc[p['AngleFromRB2'] < -180, 'AngleFromRB2'] += 360
p.loc[p['IsBallCarrier'] == 1, 'AngleFromRB'] = 0
p.loc[p['IsBallCarrier'] == 1, 'AngleFromRB2'] = 0
p.drop(['DX2', 'DY2'], axis=1, inplace=True)
 p['AngleTan'] = np.arctan2(p['DY'], p['DX'])
 p = p.replace([np.inf, -np.inf], np.nan)
 p = p.fillna(0)
 p.drop(['X_', 'Y_', 'Dir_sin_', 'Dir_cos_', 'S_', 'Dir_std_', 'Orientation_std', 'Dir_std'], axis=1, inplace=True)
 concat = p.drop(['Position', 'PlayId', 'YardLine_std'], axis=1)
 if is_training:
scaler = StandardScaler()
scaled = scaler.fit_transform(concat.values)
 else:
scaled = scaler.transform(concat.values)
 df = pd.DataFrame(scaled, columns=concat.columns)
 df_rusher = df[p['IsBallCarrier'] == 1].reset_index(drop=True)
 drop_cols = ['IsBallCarrier', 'IsOffence', 'OffenseDist0', 'PlayerWeight', 'DX', 'DY', 'AngleFromRB', 'AngleToRB',
'DistDelta', 'AngleTan', 'AngleFromRB2', 'delaunay_adj']
drop_cols = [c for c in drop_cols if c in df_rusher.columns]
df_rusher.drop(drop_cols, axis=1, inplace=True)
 df_rusher = df_rusher.set_index(players['PlayId'].iloc[np.arange(0, len(players), 22)])
 meta = play[['YardLine_std', 'Distance']].copy()
meta = meta.replace([np.inf, -np.inf], np.nan)
 meta = meta.fillna(0)
 if scaler_meta is not None:
scaled_meta = scaler_meta.transform(meta.values)
 else:
scaler_meta = StandardScaler()
scaled_meta = scaler_meta.fit_transform(meta.values)
 scaled_meta = pd.DataFrame(scaled_meta, columns=meta.columns)
 return df.set_index(players['PlayId']), df_rusher, scaled_meta, scaler, scaler_meta
class EnsembleModel(object):
def __init__(self):
self.models = []
def add_model(self, model):
self.models.append(model)
 def train(self):
for m in self.models:
m.train()
def eval(self):
for m in self.models:
m.eval()
def __call__(self, *input, **kwargs):
assert len(self.models)>= 1
base = self.models[0](*input, **kwargs)
 for m in self.models[1:]:
base += m(*input, **kwargs)
 return base / len(self.models)
 class SnapShot(object):
def __init__(self, model: nn.Module, epoch: int, loss: float):
self.state = copy.deepcopy(model.state_dict())
 self.epoch = epoch
self.loss = loss
torch.save(self.state, f'snapshot_{epoch}_{loss}')
 class SnapShots(object):
def __init__(self, interval: int = 3, torelance: float = 1.01, verbose: bool = True):
self.best_model = None
self.snap_shots = []
self.best_val_loss = 1.0
self.interval = interval
self.torelance = torelance
self.verbose = verbose
def add(self, model: nn.Module, epoch: int, loss: float):
if loss < self.best_val_loss:
if self.verbose:
print(f'best model is updated.epoch{epoch}, loss={loss:.7f}')
 self.best_model = SnapShot(model, epoch, loss)
 self.best_val_loss = loss
if epoch % self.interval == 0:
if self.verbose:
print(f'Add snapshot.epoch{epoch}, loss={loss:.7f}')
 self.snap_shots.append(SnapShot(model, epoch, loss))
def load_best_single_model(self, model: nn.Module):
if self.best_model is not None:
model.load_state_dict(self.best_model.state)
 return model
def load_ensemble_model(self, cls: Type, params: Dict, max_models: int = 5):
model = EnsembleModel()
best = cls(**params)
 self.load_best_single_model(best)
 model.add_model(best)
 candidates = []
for i, s in enumerate(self.snap_shots):
if s.loss > self.torelance * self.best_model.loss:
continue
if s.epoch == self.best_model.epoch:
continue
candidates.append(( i, s.loss))
candidates = sorted(candidates, key = lambda x: x[1])
 if len(candidates)> max_models - 1:
candidates = candidates[:max_models - 1]
for i, _ in candidates:
s = self.snap_shots[i]
sub = cls(**params)
 sub.load_state_dict(s.state)
 model.add_model(sub)
 if self.verbose:
print(f'add {s.epoch}-th epoch to ensemble(loss:{s.loss:.7f})')
 return model
def train_model(model: nn.Module, scheduler, batch_size: int,
train_data: Data, valid_data: Data, writer, epochs: int,
downsample_2017: float,
calc_train_loss: bool = True,
params: Dict = None):
np.random.seed(0)
 std_batch_length = 12000 // batch_size + 1
n_total_batch = 0
epoch = 1
epoch_start_time = time.time()
ensemble = None
model.eval()
snapshots = SnapShots(torelance=1.007, interval=3)
 snapshots.add(model, 0, evaluate(model, valid_data))
model.train()
while epoch < epochs:
data = train_data.shuffled()
if downsample_2017 > 0.0:
data = data.downsample_2017(downsample_2017)
 for i in range(0, data.len() - 1, batch_size):
iend = min(i + batch_size, data.len())
 batch_data = data.slice(i, iend)
 optimizer.zero_grad()
output = model(batch_data)
 loss = crps(output, batch_data.y)
 loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 0.5)
 optimizer.step()
n_total_batch += 1
if n_total_batch > std_batch_length:
model.eval()
if calc_train_loss and epoch % 10 == 0:
train_loss = evaluate(model, data)
 else:
train_loss = -1
if epoch % 4 == 0 or epoch > 30:
val_loss = evaluate(model, valid_data)
 snapshots.add(model, epoch, val_loss)
 else:
val_loss = -1
print('| end of epoch {:3d} | lr: {:2.5f} | time: {:5.2f}s | train loss {:5.6f} | '
'valid loss {:5.9f}'.format(epoch, scheduler.get_lr() [0],(time.time() - epoch_start_time),
train_loss, val_loss))
scheduler.step()
if writer is not None:
writer.writerow([epoch, scheduler.get_lr() [0], train_loss, val_loss])
 f.flush()
n_total_batch = 0
epoch += 1
epoch_start_time = time.time()
model.train()
if epoch in [40]:
batch_size *= 2
std_batch_length = 12000 // batch_size + 1
model = snapshots.load_best_single_model(model)
 eval_single = evaluate(model, valid_data)
 try:
ensemble = snapshots.load_ensemble_model(TransformerModel, params)
 eval_ensemble = evaluate(ensemble, valid_data)
 print(f'final loss: {eval_single:.8f}(single)/ {eval_ensemble:.8f}({len(ensemble.models)} models)')
 except:
pass
return ensemble if ensemble is not None else model
def evaluate(eval_model: nn.Module, data: Data, with_clip: bool = True):
eval_model.eval()
assert data.y is not None
with torch.no_grad() :
y_actual = data.y.numpy()
y_predicted = eval_model(data, with_clip)
 loss = np.sum(( y_predicted - y_actual)** 2)/(199 * len(y_predicted))
return loss
def return_delta(x):
temp = np.zeros(199)
 temp[x + 99:] = 1
return temp
def NFL_validation_split(df: pd.DataFrame, game_set=None):
games = df[['GameId', 'PossessionTeam']].drop_duplicates()
games = games.sort_values(['PossessionTeam', 'GameId'], ascending=[True, False])
 games['row_number'] = games.groupby(['PossessionTeam'] ).cumcount() + 1
game_set = game_set or {1, 2, 3, 4, 5}
game_ids = set(games[games['row_number'].isin(game_set)]['GameId'].unique().tolist())
 return game_ids
def NFL_group_split(df: pd.DataFrame, nfolds: int, nidx: int):
kf = GroupKFold(nfolds)
 train_idx, valid_idx = list(kf.split(df, groups=df['GameId'])) [nidx]
return set(df['GameId'].iloc[valid_idx].unique())
 def load_data_nn_test(test_df: pd.DataFrame, scaler: StandardScaler, scaler_meta: StandardScaler)-> Data:
play, players = prep(test_df)
 players_nn, rusher_nn, meta_nn, _, _ = prep_players_nn(play, players, scaler, scaler_meta)
 X = players_nn.values.astype(np.float32 ).reshape(( -1, 22, len(players_nn.columns)))
 Xr = rusher_nn.values.astype(np.float32)
 Xm = meta_nn.values.astype(np.float32)
 year = play['Year'].values.copy()
return Data(torch.from_numpy(X), torch.from_numpy(Xr), torch.from_numpy(Xm),
None, play['YardLine_std'].values.copy() , year,
list(players_nn.columns), list(rusher_nn.columns), list(meta_nn.columns))
def load_data_nn(n_plays_sample=None,
nfolds=None, nidx=None,
skiprows=None,
game_set=None)-> Tuple[Data, Data, StandardScaler, StandardScaler]:
train = pd.read_csv('.. /input/nfl-big-data-bowl-2020/train.csv', nrows=n_plays_sample, skiprows=skiprows)
 play, players = prep(train)
 players_nn, rusher_nn, meta_nn, scaler, scaler_meta = prep_players_nn(play, players, None, None)
 assert len(players_nn)== len(rusher_nn)* 22
assert len(rusher_nn)== len(meta_nn)
 assert len(rusher_nn)== len(play)
 play_ids = np.array(players_nn.index[np.arange(0, len(players_nn), 22)])
 game_ids = np.array(players['GameId'].values[np.arange(0, len(players_nn), 22)])
 assert len(play_ids)== len(game_ids)
 if nfolds is not None:
assert nidx is not None
game_ids_valid = NFL_group_split(train, nfolds, nidx)
 else:
game_ids_valid = NFL_validation_split(train, game_set)
 X = players_nn.values.astype(np.float32 ).reshape(( -1, 22, len(players_nn.columns)))
 Xr = rusher_nn.values.astype(np.float32)
 Xm = meta_nn.values.astype(np.float32)
 y = np.vstack(play['Yards'].apply(return_delta ).values ).astype(np.float32)
 yardLine = play['YardLine_std'].values
year = play['Year'].values
valid_mask = np.isin(game_ids, np.array(list(game_ids_valid)))
 X_valid = torch.from_numpy(X[valid_mask])
 Xr_valid = torch.from_numpy(Xr[valid_mask])
 Xm_valid = torch.from_numpy(Xm[valid_mask])
 y_valid = torch.from_numpy(y[valid_mask])
 yd_valid = yardLine[valid_mask].copy()
yr_valid = year[valid_mask].copy()
X_train = torch.from_numpy(X[~valid_mask])
 Xr_train = torch.from_numpy(Xr[~valid_mask])
 Xm_train = torch.from_numpy(Xm[~valid_mask])
 y_train = torch.from_numpy(y[~valid_mask])
 yd_train = yardLine[~valid_mask].copy()
yr_train = year[~valid_mask].copy()
print(f'X_train: {X_train.shape}, {Xr_train.shape}, {Xm_train.shape}')
 print(f'X_valid: {X_valid.shape}, {Xr_valid.shape}, {Xm_valid.shape}')
 print(f'y_train: {y_train.shape}, y_valid: {y_valid.shape}')
 print(f'players: {list(players_nn.columns)}')
 print(f'rusher: {list(rusher_nn.columns)}')
 print(f'meta: {list(meta_nn.columns)}')
 dtrain = Data(X_train, Xr_train, Xm_train, y_train, yd_train, yr_train,
list(players_nn.columns), list(rusher_nn.columns), list(meta_nn.columns))
dvalid = Data(X_valid, Xr_valid, Xm_valid, y_valid, yd_valid, yr_valid,
list(players_nn.columns), list(rusher_nn.columns), list(meta_nn.columns))
assert len(X)== len(y)
 return dtrain, dvalid, scaler, scaler_meta
def crps(y_pred, y_true):
loss = torch.mean(( torch.cumsum(y_pred, dim=1)- y_true)** 2)
 return loss
n_holdout = 0
n_train = None
train_data, valid_data, scaler, scaler_meta = load_data_nn(None)
 n_inp = train_data.players.shape[2]
nemb = 128
nhead = 1
nhid = 96
nlayers = 4
nfinal = 512
lr = 0.0001
gamma = 0.976
dropout_encoder = 0.0
n_emb_layers = 3
dropout_classifier = 0.3
dropout_embed = 0.15
dropout_attn = 0.35
batch_size = 16
epochs = 100
meta_emb = 8
downsample_2017 = 0.4
gauss_noise = 0.15
gauss_xy_noise = 0.1
n_fin_layers = 3
mode = 'ensemble'
log_filename = 'kernel_v92_encoder_0_attn_02'
params = {
'ninp': n_inp,
'nemb': nemb,
'nhead': nhead,
'nhid': nhid,
'nlayers': nlayers,
'nfinal': nfinal,
'ninp_rusher': train_data.rusher.shape[1],
'pre_LN': True,
'dropout_encoder': dropout_encoder,
'dropout_embed': dropout_embed,
'dropout_classifier': dropout_classifier,
'n_emb_layers': n_emb_layers,
'ninp_meta': train_data.meta.shape[1],
'meta_emb': meta_emb,
'gauss_noise': gauss_noise,
'gauss_xy_noise': gauss_xy_noise,
'n_fin_layers': n_fin_layers,
'dropout_attn': dropout_attn
}
no_decay = ['bias', '.norm']
if mode == 'grid':
for dropout_embed in [0.05, 0.1, 0.15]:
for dropout_encoder in [0.2, 0.3, 0.4]:
log_f = f'{log_filename}_embed{dropout_embed}_encoder{dropout_encoder}.csv'
params['dropout_encoder'] = dropout_encoder
params['dropout_embed'] = dropout_embed
model = TransformerModel(**params)
 f = open(log_f, 'w+', newline='')
 writer = csv.writer(f)
 writer.writerow([dropout_embed, dropout_classifier])
 writer.writerow(['epoch', 'lr', 'train_loss', 'valid_loss'])
 optimizer = torch.optim.Adam(model.parameters() , lr=lr)
 scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=gamma)
 model = train_model(model, scheduler, batch_size, train_data, valid_data, writer, epochs, downsample_2017, params=params)
 elif mode == 'ensemble':
models = EnsembleModel()
game_sets = [
{1, 3, 5, 7, 9},
{2, 4, 6, 8, 10}
]
for i in range(len(game_sets)) :
model = TransformerModel(**params)
 train_data, valid_data, scaler, scaler_meta = load_data_nn(None, nfolds=8, nidx=i)
 optimizer = torch.optim.Adam(model.parameters() , lr=lr)
 scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=gamma)
 model = train_model(model, scheduler, batch_size, train_data, valid_data, None, epochs, downsample_2017,
calc_train_loss=False, params=params)
 models.add_model(model)
 model = models
else:
assert mode == 'single'
model = TransformerModel(**params)
 f = open(log_filename + '.csv', 'w+', newline='')
 writer = csv.writer(f)
 writer.writerow(train_data.player_cols)
 writer.writerow(train_data.rusher_cols)
 writer.writerow(['epoch', 'lr', 'train_loss', 'valid_loss'])
 optimizer = torch.optim.Adam(model.parameters() , lr=lr)
 scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=gamma)
 model = train_model(model, scheduler, batch_size, train_data, valid_data, writer, epochs, downsample_2017, params=params)
<feature_engineering>
differents = []
for i in range(0,subtest.shape[0]):
labels = list(subtest.iloc[i,1:])
 result = len(set(labels)) >1
if result:
differents.append(( i, str(labels)))
 differents = pd.DataFrame(differents, columns=['idx','group'])
 differents.head()
def tta(test_df, sigma_dir=1.0, sigma_y=1.0):
n_aug = 10
test_df_aug = pd.concat([test_df]*n_aug)
 test_df_aug['Dir'] += np.random.normal(0, sigma_dir, size=len(test_df_aug))
test_df_aug['Y'] += np.repeat(np.random.normal(0, sigma_y, size=n_aug), 22)
 test_df_aug['PlayId'] += np.repeat(np.arange(0, n_aug), 22)
 return test_df_aug<set_options>
differents['group'].nunique()
env = nflrush.make_env()<predict_on_test>
differents['count'] = differents.groupby('group')['group'].transform('count')
 differents = differents.sort_values(by=['count'], ascending=False)
 differents = differents.drop(['idx'],axis=1)
 differents = differents.drop_duplicates()
model.eval()
n_prev = 0
original_data = concat_dataset(train_data, valid_data)
 for(test_df, sample_prediction_df)in env.iter_test() :
try:
test_df = tta(test_df)
 X_test = load_data_nn_test(test_df, scaler, scaler_meta)
 except Exception as e:
print(f'
env.predict(sample_prediction_df)
 continue
with torch.no_grad() :
predicted = model(X_test ).mean(axis=0)
 sample_prediction_df.iloc[0,:] = np.squeeze(predicted)
 env.predict(sample_prediction_df)
<save_to_csv>
style.use('ggplot' )
env.write_submission_file()<set_options>
X_train = pd.read_csv('.. /input/career-con-2019/X_train.csv')
 X_test = pd.read_csv('.. /input/career-con-2019/X_test.csv')
 target = pd.read_csv('.. /input/career-con-2019/y_train.csv' )
pd.options.display.max_rows = 4000
warnings.filterwarnings('ignore')
 sns.set()
%matplotlib inline
pd.set_option('display.max_columns', 50)
 pd.set_option('display.max_rows', 150 )<categorify>
series_dict = {}
for series in(X_train['series_id'].unique()):
series_dict[series] = X_train[X_train['series_id'] == series]
def clean_WindDirection(txt):
if isinstance(txt, int):
return('NA')
 elif pd.isna(txt):
return('NA')
 else:
if pd.isna(txt):
return np.nan
txt = txt.lower()
txt = ''.join([c for c in txt if c not in punctuation])
 txt = txt.replace('from', '')
 txt = txt.replace(' ', '')
 txt = txt.replace('north', 'n')
 txt = txt.replace('south', 's')
 txt = txt.replace('west', 'w')
 txt = txt.replace('east', 'e')
 return txt
def transform_WindDirection(txt):
if pd.isna(txt):
return('-1')
 if txt=='n':
return 0
elif txt=='nne' or txt=='nen':
return 1/8
elif txt=='ne':
return 2/8
elif txt=='ene' or txt=='nee':
return 3/8
elif txt=='e':
return 4/8
elif txt=='ese' or txt=='see':
return 5/8
elif txt=='se':
return 6/8
elif txt=='ses' or txt=='sse':
return 7/8
elif txt=='s':
return 8/8
elif txt=='ssw' or txt=='sws':
return 9/8
elif txt=='sw':
return 10/8
elif txt=='sww' or txt=='wsw':
return 11/8
elif txt=='w':
return 12/8
elif txt=='wnw' or txt=='nww':
return 13/8
elif txt=='nw':
return 14/8
elif txt=='nwn' or txt=='nnw':
return 15/8
else: return -1
return np.nan
def clean_StadiumType(txt):
if pd.isna(txt):
return('NA')
 if isinstance(txt, int):
return('NA')
 else:
txt = txt.lower()
txt = ''.join([c for c in txt if c not in punctuation])
 txt = re.sub(' +', ' ', txt)
 txt = txt.strip()
txt = txt.replace('outside', 'outdoor')
 txt = txt.replace('outdor', 'outdoor')
 txt = txt.replace('outddors', 'outdoor')
 txt = txt.replace('outdoors', 'outdoor')
 txt = txt.replace('oudoor', 'outdoor')
 txt = txt.replace('indoors', 'indoor')
 txt = txt.replace('ourdoor', 'outdoor')
 txt = txt.replace('retractable', 'rtr.')
 return txt
def set_Turf(df):
Turf = {'Field Turf':'Artificial', 'A-Turf Titan':'Artificial', 'Grass':'Natural',
'UBU Sports Speed S5-M':'Artificial', 'Artificial':'Artificial',
'DD GrassMaster':'Artificial', 'Natural Grass':'Natural', 'UBU Speed Series-S5-M':'Artificial',
'FieldTurf':'Artificial', 'FieldTurf 360':'Artificial', 'Natural grass':'Natural', 'grass':'Natural',
'Natural':'Natural', 'Artifical':'Artificial', 'FieldTurf360':'Artificial', 'Naturall Grass':'Natural',
'Field turf':'Artificial', 'SISGrass':'Artificial', 'Twenty-Four/Seven Turf':'Artificial',
'natural grass':'Natural'}
df['Turf_type']=df['Turf'].map(Turf)
 return df
def GameWeather(df):
if isinstance(df['GameWeather'].iloc[0], np.integer):
df['GameWeather']='NA'
return(df)
 else:
df['GameWeather'] = df['GameWeather'].str.lower()
indoor = "indoor"
df['GameWeather'] = df['GameWeather'].apply(lambda x: indoor if not pd.isna(x)and indoor in x else x)
 df['GameWeather'] = df['GameWeather'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 df['GameWeather'] = df['GameWeather'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 df['GameWeather'] = df['GameWeather'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 return df<feature_engineering>
def filter_signal(signal, threshold=1e3):
fourier = rfft(signal)
 frequencies = rfftfreq(signal.size, d=20e-3/signal.size)
 fourier[frequencies > threshold] = 0
return irfft(fourier )
def clean_TeamAbbreviations(train_df):
train_df.loc[train_df.VisitorTeamAbbr == "ARI", 'VisitorTeamAbbr'] = "ARZ"
train_df.loc[train_df.HomeTeamAbbr == "ARI", 'HomeTeamAbbr'] = "ARZ"
train_df.loc[train_df.VisitorTeamAbbr == "BAL", 'VisitorTeamAbbr'] = "BLT"
train_df.loc[train_df.HomeTeamAbbr == "BAL", 'HomeTeamAbbr'] = "BLT"
train_df.loc[train_df.VisitorTeamAbbr == "CLE", 'VisitorTeamAbbr'] = "CLV"
train_df.loc[train_df.HomeTeamAbbr == "CLE", 'HomeTeamAbbr'] = "CLV"
train_df.loc[train_df.VisitorTeamAbbr == "HOU", 'VisitorTeamAbbr'] = "HST"
train_df.loc[train_df.HomeTeamAbbr == "HOU", 'HomeTeamAbbr'] = "HST"
return train_df<feature_engineering>
X_train_denoised = X_train.copy()
X_test_denoised = X_test.copy()
for col in X_train.columns:
if col[0:3] == 'ang' or col[0:3] == 'lin':
denoised_data = X_train.groupby(['series_id'])[col].apply(lambda x: filter_signal(x))
list_denoised_data = []
for arr in denoised_data:
for val in arr:
list_denoised_data.append(val)
 X_train_denoised[col] = list_denoised_data
for col in X_test.columns:
if col[0:3] == 'ang' or col[0:3] == 'lin':
denoised_data = X_test.groupby(['series_id'])[col].apply(lambda x: filter_signal(x))
list_denoised_data = []
for arr in denoised_data:
for val in arr:
list_denoised_data.append(val)
 X_test_denoised[col] = list_denoised_data
def tag_rusher(train_df):
train_df['IsBallCarrier'] = train_df.NflId == train_df.NflIdRusher
return train_df<feature_engineering>
series_dict = {}
for series in(X_train_denoised['series_id'].unique()):
series_dict[series] = X_train_denoised[X_train_denoised['series_id'] == series]
<feature_engineering><EOS> 
X_train_denoised.to_csv('test_denoised.csv', index=False)
 X_test_denoised.to_csv('train_denoised.csv', index=False )