Kaggle competition: NFL Big Data Bowl
<SOS> metric: CRPS Kaggle data source: nfl-big-data-bowl<filter>
pd.options.display.max_rows = 4000
warnings.filterwarnings('ignore')
 sns.set()
%matplotlib inline
pd.set_option('display.max_columns', 50)
 pd.set_option('display.max_rows', 150 )
orders_test = orders[orders.eval_set=='test']<merge>
def clean_WindDirection(txt):
if isinstance(txt, int):
return('NA')
 elif pd.isna(txt):
return('NA')
 else:
if pd.isna(txt):
return np.nan
txt = txt.lower()
txt = ''.join([c for c in txt if c not in punctuation])
 txt = txt.replace('from', '')
 txt = txt.replace(' ', '')
 txt = txt.replace('north', 'n')
 txt = txt.replace('south', 's')
 txt = txt.replace('west', 'w')
 txt = txt.replace('east', 'e')
 return txt
def transform_WindDirection(txt):
if pd.isna(txt):
return('-1')
 if txt=='n':
return 0
elif txt=='nne' or txt=='nen':
return 1/8
elif txt=='ne':
return 2/8
elif txt=='ene' or txt=='nee':
return 3/8
elif txt=='e':
return 4/8
elif txt=='ese' or txt=='see':
return 5/8
elif txt=='se':
return 6/8
elif txt=='ses' or txt=='sse':
return 7/8
elif txt=='s':
return 8/8
elif txt=='ssw' or txt=='sws':
return 9/8
elif txt=='sw':
return 10/8
elif txt=='sww' or txt=='wsw':
return 11/8
elif txt=='w':
return 12/8
elif txt=='wnw' or txt=='nww':
return 13/8
elif txt=='nw':
return 14/8
elif txt=='nwn' or txt=='nnw':
return 15/8
else: return -1
return np.nan
def clean_StadiumType(txt):
if pd.isna(txt):
return('NA')
 if isinstance(txt, int):
return('NA')
 else:
txt = txt.lower()
txt = ''.join([c for c in txt if c not in punctuation])
 txt = re.sub(' +', ' ', txt)
 txt = txt.strip()
txt = txt.replace('outside', 'outdoor')
 txt = txt.replace('outdor', 'outdoor')
 txt = txt.replace('outddors', 'outdoor')
 txt = txt.replace('outdoors', 'outdoor')
 txt = txt.replace('oudoor', 'outdoor')
 txt = txt.replace('indoors', 'indoor')
 txt = txt.replace('ourdoor', 'outdoor')
 txt = txt.replace('retractable', 'rtr.')
 return txt
def set_Turf(df):
Turf = {'Field Turf':'Artificial', 'A-Turf Titan':'Artificial', 'Grass':'Natural',
'UBU Sports Speed S5-M':'Artificial', 'Artificial':'Artificial',
'DD GrassMaster':'Artificial', 'Natural Grass':'Natural', 'UBU Speed Series-S5-M':'Artificial',
'FieldTurf':'Artificial', 'FieldTurf 360':'Artificial', 'Natural grass':'Natural', 'grass':'Natural',
'Natural':'Natural', 'Artifical':'Artificial', 'FieldTurf360':'Artificial', 'Naturall Grass':'Natural',
'Field turf':'Artificial', 'SISGrass':'Artificial', 'Twenty-Four/Seven Turf':'Artificial',
'natural grass':'Natural'}
df['Turf_type']=df['Turf'].map(Turf)
 return df
def GameWeather(df):
if isinstance(df['GameWeather'].iloc[0], np.integer):
df['GameWeather']='NA'
return(df)
 else:
df['GameWeather'] = df['GameWeather'].str.lower()
indoor = "indoor"
df['GameWeather'] = df['GameWeather'].apply(lambda x: indoor if not pd.isna(x)and indoor in x else x)
 df['GameWeather'] = df['GameWeather'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 df['GameWeather'] = df['GameWeather'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 df['GameWeather'] = df['GameWeather'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 return df
data_test = data_test.merge(orders_test[["user_id", "order_id"]], on='user_id', how='left')
 data_test.head()<set_options>
def clean_TeamAbbreviations(train_df):
train_df.loc[train_df.VisitorTeamAbbr == "ARI", 'VisitorTeamAbbr'] = "ARZ"
train_df.loc[train_df.HomeTeamAbbr == "ARI", 'HomeTeamAbbr'] = "ARZ"
train_df.loc[train_df.VisitorTeamAbbr == "BAL", 'VisitorTeamAbbr'] = "BLT"
train_df.loc[train_df.HomeTeamAbbr == "BAL", 'HomeTeamAbbr'] = "BLT"
train_df.loc[train_df.VisitorTeamAbbr == "CLE", 'VisitorTeamAbbr'] = "CLV"
train_df.loc[train_df.HomeTeamAbbr == "CLE", 'HomeTeamAbbr'] = "CLV"
train_df.loc[train_df.VisitorTeamAbbr == "HOU", 'VisitorTeamAbbr'] = "HST"
train_df.loc[train_df.HomeTeamAbbr == "HOU", 'HomeTeamAbbr'] = "HST"
return train_df
del orders
del orders_test
gc.collect()<data_type_conversions>
def tag_rusher(train_df):
train_df['IsBallCarrier'] = train_df.NflId == train_df.NflIdRusher
return train_df
data_test['product_id'] = data_test.product_id.astype(int)
 data_test = data_test.drop('user_id', axis=1)
 gc.collect() 
<feature_engineering>
def get_direction(train_df):
train_df['ToLeft'] = train_df.PlayDirection == "left"
return train_df
d = dict()
for row in data_test.itertuples() :
if row.reordered == 1:
try:
d[row.order_id] += ' ' + str(row.product_id)
 except:
d[row.order_id] = str(row.product_id)
 for order in data_test.order_id:
if order not in d:
d[order] = 'None'
gc.collect()<save_to_csv>
def feet_inch_to_cm(height_fi):
foot = 30.48
inch = 2.54
return int(height_fi.split('-')[0])* foot + int(height_fi.split('-')[1])* inch
def imperial_to_metric(train_df):
train_df['PlayerHeight_cm']= train_df['PlayerHeight'].apply(feet_inch_to_cm)
 train_df['PlayerWeight_kg']= train_df['PlayerWeight']*0.453592
return train_df
sub = pd.DataFrame.from_dict(d, orient='index')
 sub.reset_index(inplace=True)
 sub.columns = ['order_id', 'products']
sub.to_csv('sub.csv', index=False )<load_from_csv>
def get_player_age(train_df):
train_df['Age']=2019 - pd.DatetimeIndex(train_df['PlayerBirthDate'] ).year
return train_df
submission = pd.read_csv(".. /working/sub.csv")
 submission.head()<install_modules>
def possession_team(possessionteam, hometeam, visitorteam):
if possessionteam == hometeam:
return 'home'
else:
return 'visitor'
def leading_team_home_visitor(homescore, visitorscore):
if homescore == visitorscore:
return 'draw'
else:
if homescore > visitorscore:
return 'home'
else:
return 'visitor'
def leading_team_def_off(team, posessionteam, hometeam, visitorteam, homescore, visitorscore):
if homescore == visitorscore:
return 'draw'
else:
if posessionteam == hometeam:
if homescore > visitorscore:
return 'offense'
else:
return 'defense'
else:
if visitorscore > homescore:
return 'offense'
else:
return 'defense'
def get_leading_team_features(train_df):
train_df['possession_team_home_visitor'] = train_df.apply(lambda row: possession_team(row.PossessionTeam,row.HomeTeamAbbr,row.VisitorTeamAbbr), axis = 1)
 train_df['leading_by'] = abs(train_df['HomeScoreBeforePlay'] - train_df['VisitorScoreBeforePlay'])
 train_df['leading_team_defense_offense'] = train_df.apply(lambda row: leading_team_def_off(row.Team,
row.PossessionTeam,
row.HomeTeamAbbr,
row.VisitorTeamAbbr,
row.HomeScoreBeforePlay,
row.VisitorScoreBeforePlay), axis = 1)
 train_df['leading_team_home_visitor'] = train_df.apply(lambda row: leading_team_home_visitor(row.HomeScoreBeforePlay,row.VisitorScoreBeforePlay), axis = 1)
 train_df['Offense_score']=np.where(train_df['leading_team_defense_offense']=='offense', train_df['leading_by'], -train_df['leading_by'])
 train_df['Deffense_score']=np.where(train_df['leading_team_defense_offense']=='defense', train_df['leading_by'], -train_df['leading_by'])
 train_df['Offense_Team'] = train_df['PossessionTeam']
train_df['Defense_Team'] = train_df.apply(lambda row: row['HomeTeamAbbr'] if row['PossessionTeam'] != row['HomeTeamAbbr'] else row['VisitorTeamAbbr'],axis=1)
 return train_df
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index<import_modules>
def get_adjusted_yardline(train_df):
train_df['Home_team_play']=np.where(train_df['PossessionTeam']==train_df['HomeTeamAbbr'], 1, 0)
 train_df['Team_side']=np.where(train_df['PossessionTeam']==train_df['FieldPosition'], 1, 0)
 train_df['YardLine_adj']=np.where(train_df['Team_side']==1,train_df['YardLine'],100-train_df['YardLine'])
 return train_df
import os
import cv2
import pydicom
import pandas as pd
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import random
from tqdm.notebook import tqdm
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import mean_absolute_error
from tensorflow_addons.optimizers import RectifiedAdam
from tensorflow.keras import Model
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
from tensorflow.keras.optimizers import Nadam
import seaborn as sns
import plotly.express as px
import plotly.graph_objects as go
from PIL import Image<define_variables>
def get_GameClock_features(train_df):
train_df['snap_to_handoff']=(pd.to_datetime(train_df['TimeHandoff'])-pd.to_datetime(train_df['TimeSnap'])).dt.total_seconds()
sec=pd.to_timedelta(train_df['GameClock'])
 sec=sec.dt.total_seconds() /60
train_df['GameClock_sec']=sec
train_df['clock_bin']=np.where(train_df['Quarter']==1,1,np.where(train_df['Quarter']==2,2,np.where(train_df['Quarter']==3,3,
np.where(train_df['Quarter']==4,np.where(train_df['GameClock_sec']<=564,4,5),np.where(train_df['GameClock_sec']<=333,6,7)))))
 return train_df
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed )<define_variables>
def get_personnel_types(dataset):
setups = list(dataset['OffensePersonnel'].unique())
 setups = ','.join(setups)
 setups = re.sub(r'\d+', '', setups ).replace(' ','')
 mylist = list(set(setups.split(',')))
 offense_column_names = ['Offense_' + s for s in mylist]
setups = list(dataset['DefensePersonnel'].unique())
 setups = ','.join(setups)
 setups = re.sub(r'\d+', '', setups ).replace(' ','')
 mylist = list(set(setups.split(',')))
 defense_column_names = ['Defense_' + s for s in mylist]
return offense_column_names + defense_column_names
def extract_personnel_count(whole_personnel, specific_personnel):
specific_personnel = specific_personnel.split('_')[1]
if specific_personnel in whole_personnel:
whole_personnel = whole_personnel.split(',')
 types = [re.sub(r'\d+', '', x ).replace(' ','')for x in whole_personnel]
counts = [[int(s)for s in x.split() if s.isdigit() ][0] for x in whole_personnel]
dictionary = dict(zip(types, counts))
return dictionary[specific_personnel]
else:
return 0
def get_split_personnel_types(columns_to_add, train_df):
for col in columns_to_add:
if col.split('_')[0] == 'Defense':
train_df[col] = train_df['DefensePersonnel'].apply(lambda x: extract_personnel_count(x, col))
else:
train_df[col] = train_df['OffensePersonnel'].apply(lambda x: extract_personnel_count(x, col))
return train_df
seed_everything(42 )<set_options>
def add_bins_and_generate_stats(train_df):
bins=[-99,-0.01,0.01,1.72,4.21,6.7,99]
train_df['Yards_Bin'] = pd.cut(train_df['Yards'], bins)
 stats = {}
for my_type in ['DisplayName', 'Offense_Team', 'Defense_Team']:
per_my_type = train_df[[my_type,'Yards','Yards_Bin']].groupby([my_type,'Yards_Bin'] ).count().fillna(0 ).reset_index()
per_my_type = per_my_type.merge(per_my_type.groupby(my_type)['Yards'].sum().reset_index() , on=my_type, how='left')
 per_my_type['Yards_Bin_Rate'] = per_my_type['Yards_x'] / per_my_type['Yards_y']
per_my_type.rename(columns={'Yards_x':'Bin_Count', 'Yards_y': 'Total_Plays'}, inplace=True)
 per_my_type = pd.pivot_table(per_my_type, values=['Bin_Count', 'Yards_Bin_Rate'], index=[my_type], columns=['Yards_Bin'])
 per_my_type.columns = per_my_type.columns.map('{0[0]}_{0[1]}'.format)
 per_my_type = per_my_type.reset_index()
avg_yards = train_df.groupby(my_type)['Yards'].mean().reset_index()
std_yards = train_df.groupby(my_type)['Yards'].std().reset_index().fillna(0)
 counts = train_df.groupby(my_type)['Yards'].count().reset_index()
per_type = avg_yards.merge(counts, on=my_type, how='inner' ).merge(std_yards, on=my_type, how='inner')
 per_type.rename(columns={'Yards_x':'Avg_Yards', 'Yards_y':'Count', 'Yards':'Std_Yards'}, inplace= True)
 per_my_type= per_my_type.merge(per_type, on=my_type, how='left')
 if my_type == 'DisplayName':
per_player_enough_plays = per_type[per_type['Count'] >= 10]
per_player_not_enough_plays = per_type[per_type['Count'] < 10]
per_player_not_enough_plays['rusher_performance_bin'] = 'few_plays'
per_player_enough_plays['rusher_performance_bin'] = pd.qcut(per_player_enough_plays['Avg_Yards'], q=[0, 0.2,0.5,0.9,1], labels= ['bottom_20','20-50','50-90','top_10'])
 per_player = pd.concat([per_player_enough_plays, per_player_not_enough_plays])
 per_my_type= per_my_type.merge(per_player[[my_type,'rusher_performance_bin']], on=my_type, how='left')
 per_my_type.columns = [per_my_type.columns[0]] + [my_type + '_' + str(col)for col in per_my_type.columns[1:]]
stats[my_type] = per_my_type
train_df = train_df.merge(per_my_type, on=my_type, how='left')
 return stats, train_df
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config )<init_hyperparams>
def convert_to_cat(df):
for col in ['DefensePersonnel','Home_team_play','Team_side','Down','Defense_Team','OffenseFormation','OffensePersonnel',
'leading_team_defense_offense','PlayDirection','clock_bin','Defense_DB','Defense_LB','Defense_DL','Offense_WR']:
df[col] = df[col].astype('category')
 return df
Dropout_model = 0.3856
FVC_weight = 0.2
Confidence_weight = 0.2<load_from_csv>
def euclidean_distance(x1,x2,y1,y2):
return np.sqrt(( x1-x2)**2 +(y1-y2)**2)
 def norm_x(df):
df['Player_o_d']=np.where(( df["Home_team_play"] ==1)&(df["Team"] == 'home')|(df["Home_team_play"] ==0)&(df["Team"] == 'away'),'Offense_Player','Defense_Player')
 df['YardLine_Direction']=np.where(df["PlayDirection"] =='right',df['YardLine_adj'],100-df['YardLine_adj'])
 df['X_adj']=np.where(df["Player_o_d"] =='Offense_Player',abs(df['YardLine_Direction']+10-df['X']),-abs(df['YardLine_Direction']+10-df['X']))
return(df)
 def qb_and_rusher_to_closest_defender(df):
play_ids = list(df.PlayId.unique())
 rows = []
for play_id in play_ids:
play = df[df['PlayId'] == play_id]
rusher = play[play['IsBallCarrier']==True][['X', 'Y']]
QB_offense=play[play['Position']=='QB'][['X','Y']]
defenders = play[play['Player_o_d']=='Defense_Player'][['X', 'Y']]
defenders['rusher_X'] = rusher['X'].iloc[0]
defenders['rusher_Y'] = rusher['Y'].iloc[0]
try:
defenders['qb_X'] = QB_offense['X'].iloc[0]
defenders['qb_Y'] = QB_offense['Y'].iloc[0]
except:
defenders['qb_X'] = 50
defenders['qb_Y'] = 50
defenders['dist_rusher_to_defender'] = defenders.apply(lambda row: euclidean_distance(row.X,row.rusher_X,row.Y,row.rusher_Y), axis = 1)
 defenders['dist_qb_to_defender'] = defenders.apply(lambda row: euclidean_distance(row.X,row.qb_X,row.Y,row.qb_Y), axis = 1)
 rows.append([play_id, defenders['dist_rusher_to_defender'].min() , defenders['dist_qb_to_defender'].min() , euclidean_distance(rusher['X'].iloc[0], defenders['qb_X'].iloc[0], rusher['Y'].iloc[0], defenders['qb_Y'].iloc[0])])
 rows = pd.DataFrame(rows, columns=['PlayId', 'min_dist_rusher_to_defender', 'min_dist_qb_to_defender', 'dist_rusher_QB'])
 df = df.merge(rows, on='PlayId', how='left')
 return df
def Direction_orientation_adj(df):
df['dir_adj']=np.where(df["Player_o_d"] =='Offense_Player',np.where(df["PlayDirection"] =='right',df["Dir"],360-df["Dir"]),np.where(df["PlayDirection"] =='right',360-df["Dir"],df["Dir"]))
df['Orientation_adj']=np.where(df["Player_o_d"] =='Offense_Player',np.where(df["PlayDirection"] =='right',df["Orientation"],360-df["Orientation"]),np.where(df["PlayDirection"] =='right',360-df["Orientation"],df["Orientation"]))
df['dir_to_Orientation']=180 - abs(abs(df['dir_adj'] - df['Orientation_adj'])- 180)
 return(df)
 def X_Y_Velocity(df):
df['v_x']=np.sin(df['dir_adj'] * np.pi/180)*df['S']
df['v_y']=np.cos(df['dir_adj'] * np.pi/180)*df['S']
df['arc_tan']=np.arctan(( df['v_y']/df['v_x'])) *180/np.pi
df['arc_tan_cat']=pd.cut(df['arc_tan'],bins=[-90, -45, 0, 45, 90],labels=False)
 df['t1_distance']=df['Dis']+df['S']*1+0.5*df['A']*1
df['t2_distance']=df['Dis']+df['S']*2+0.5*df['A']*4
df['t3_distance']=df['Dis']+df['S']*3+0.5*df['A']*9
df['v_x_a1']=np.sin(df['dir_adj'] * np.pi/180)*(df['S']+df['A']*1)
 df['v_y_a1']=np.cos(df['dir_adj'] * np.pi/180)*(df['S']+df['A']*1)
 df['v_x_a2']=np.sin(df['dir_adj'] * np.pi/180)*(df['S']+df['A']*2)
 df['v_y_a2']=np.cos(df['dir_adj'] * np.pi/180)*(df['S']+df['A']*2)
 df['v_x_a3']=np.sin(df['dir_adj'] * np.pi/180)*(df['S']+df['A']*3)
 df['v_y_a3']=np.cos(df['dir_adj'] * np.pi/180)*(df['S']+df['A']*3)
 df['a_x']=np.sin(df['dir_adj'] * np.pi/180)*df['A']
df['a_y']=np.cos(df['dir_adj'] * np.pi/180)*df['A']
return(df )
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv' )<concatenate>
h2o.init(min_mem_size='16G' )
def get_tab(df):
vector = [(df.Age.values[0] - 30)/ 30]
if df.Sex.values[0] == 'male':
vector.append(0)
 else:
vector.append(1)
 if df.SmokingStatus.values[0] == 'Never smoked':
vector.extend([0,0])
 elif df.SmokingStatus.values[0] == 'Ex-smoker':
vector.extend([1,1])
 elif df.SmokingStatus.values[0] == 'Currently smokes':
vector.extend([0,1])
 else:
vector.extend([1,0])
 return np.array(vector )<feature_engineering>
def build_and_return_model(train_df, training_columns, name, ntrees, max_depth, min_rows):
train = h2o.H2OFrame(train_df)
 model = H2ORandomForestEstimator(ntrees=ntrees, max_depth=max_depth, min_rows=min_rows, keep_cross_validation_predictions=True, nfolds=10, seed=1)
 model.train(x=training_columns, y='Yards', training_frame=train)
 cv_predictions = model.cross_validation_holdout_predictions()
train_df = train_df.join(cv_predictions.as_data_frame())
 train_df.rename(columns={'predict':('predict_' + name)}, inplace=True)
 return model, train_df
A = {}
TAB = {}
P = []
for i, p in tqdm(enumerate(train.Patient.unique())) :
sub = train.loc[train.Patient == p, :]
fvc = sub.FVC.values
weeks = sub.Weeks.values
c = np.vstack([weeks, np.ones(len(weeks)) ] ).T
a, b = np.linalg.lstsq(c, fvc)[0]
A[p] = a
TAB[p] = get_tab(sub)
 P.append(p )<feature_engineering>
train_df_all = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2020/train.csv', low_memory=False)
 train_df_all = clean_TeamAbbreviations(train_df_all)
 train_df_all = imperial_to_metric(train_df_all)
 train_df_all = tag_rusher(train_df_all)
 train_df_all = get_player_age(train_df_all)
 train_df_all = get_adjusted_yardline(train_df_all)
 train_df_all=norm_x(train_df_all)
 train_df_all['dis_from_center']= abs(train_df_all['X_adj'])
 train_df_all=Direction_orientation_adj(train_df_all)
 train_df_all=X_Y_Velocity(train_df_all)
 train_df_all = qb_and_rusher_to_closest_defender(train_df_all)
 train_df=train_df_all[train_df_all['NflId']==train_df_all['NflIdRusher']]
train_df.FieldPosition = train_df.FieldPosition.fillna('NA')
 train_df = get_direction(train_df)
 train_df = get_leading_team_features(train_df)
 train_df = get_GameClock_features(train_df)
 train_df['StadiumType'] = train_df['StadiumType'].apply(clean_StadiumType)
 train_df['WindDirection']=train_df['WindDirection'].apply(clean_WindDirection)
 train_df['WindDirection']=train_df['WindDirection'].apply(transform_WindDirection)
 train_df=set_Turf(train_df)
 train_df=GameWeather(train_df)
 personnel_columns_to_add = get_personnel_types(train_df)
 train_df = get_split_personnel_types(personnel_columns_to_add, train_df)
 player_and_team_stats, train_df = add_bins_and_generate_stats(train_df)
 models_columns=get_models_columns()
model_player,train_df = build_and_return_model(train_df, models_columns['player'], 'player', 500, 5, 27)
 model_offense,train_df = build_and_return_model(train_df, models_columns['offense'], 'offense', 500, 5, 15)
 model_defense,train_df = build_and_return_model(train_df, models_columns['defense'], 'defense', 500, 6, 25)
 models = {'player': model_player, 'offense': model_offense,'defense': model_defense}
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a<choose_model_class>
training_columns =[
'Down',
'Age',
'Offense_DL',
'Offense_QB',
'Offense_RB',
'Offense_LB',
'Offense_DB',
'Offense_TE',
'Defense_DL',
'Defense_OL',
'min_dist_rusher_to_defender',
'min_dist_qb_to_defender',
'dist_rusher_QB',
'A',
'predict_defense',
'predict_offense',
'predict_player',
'YardLine_adj',
'S',
'DefendersInTheBox',
'Dis',
'Distance',
'X_adj',
'DisplayName_Count',
'Defense_DB',
'DisplayName_Bin_Count_(1.72, 4.21]',
'DisplayName_Bin_Count_(6.7, 99.0]',
'DisplayName_Yards_Bin_Rate_(-99.0, -0.01]',
'DisplayName_Bin_Count_(0.01, 1.72]',
'VisitorScoreBeforePlay',
'DisplayName_Bin_Count_(4.21, 6.7]',
'clock_bin',
'Defense_Team_Yards_Bin_Rate_(6.7, 99.0]',
'Offense_WR',
'Team_side',
'Offense_score',
'Deffense_score',
'Defense_LB',
'DisplayName_Yards_Bin_Rate_(0.01, 1.72]',
'Defense_Team_Yards_Bin_Rate_(-0.01, 0.01]',
'Defense_Team_Bin_Count_(1.72, 4.21]',
'Defense_Team_Avg_Yards',
'Offense_OL',
'DisplayName_Yards_Bin_Rate_(-0.01, 0.01]',
'Defense_Team_Yards_Bin_Rate_(-99.0, -0.01]',
'Defense_Team_Bin_Count_(-99.0, -0.01]',
'dis_from_center',
'dir_adj',
'Orientation_adj',
'dir_to_Orientation',
'v_x',
'v_y',
'arc_tan',
'arc_tan_cat',
't1_distance',
't2_distance',
't3_distance',
'v_x_a1',
'v_y_a1',
'v_x_a2',
'v_y_a2',
'v_x_a3',
'v_y_a3'
]
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(512, 512, 1), model_class=None):
inp = Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(4,))
x2 = tf.keras.layers.GaussianNoise(0.2 )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(Dropout_model )(x)
 x = Dense(1 )(x)
 model = Model([inp, inp2] , x)
 weights = [w for w in os.listdir('.. /input/osic-model-weights')if model_class in w][0]
model.load_weights('.. /input/osic-model-weights/' + weights)
 return model
model_classes = ['b5']
models = [build_model(shape=(512, 512, 1), model_class=m)for m in model_classes]
print('Number of models: ' + str(len(models)) )<split>
categorical_columns = ['DisplayName_rusher_performance_bin', 'Position', 'Stadium','GameWeather','OffensePersonnel','Turf_type','OffenseFormation']
one_hot = pd.get_dummies(train_df[categorical_columns])
 train_df = train_df.join(one_hot )
tr_p, vl_p = train_test_split(P, shuffle=True, train_size = 0.9 )<compute_test_metric>
training_columns = training_columns + list(one_hot.columns )
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 69)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric )<define_variables>
train_df_temp=train_df[training_columns].fillna(0)
 X = train_df_temp
yards = train_df.Yards
y = np.zeros(( yards.shape[0], 37))
for idx, target in enumerate(list(yards)) :
if target > 22:
y[idx][36] = 1
elif target < -14:
y[idx][0] = 1
else:
y[idx][14 + target] = 1
X = np.array(X )
subs = []
for model in models:
metric = []
for q in tqdm(range(1, 10)) :
m = []
for p in vl_p:
x = []
tab = []
if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:
continue
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/{i}'))
tab.append(get_tab(train.loc[train.Patient == p, :]))
if len(x)< 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q / 10)
 percent_true = train.Percent.values[train.Patient == p]
fvc_true = train.FVC.values[train.Patient == p]
weeks_true = train.Weeks.values[train.Patient == p]
fvc = a *(weeks_true - weeks_true[0])+ fvc_true[0]
percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])
 m.append(score(fvc_true, fvc, percent))
print(np.mean(m))
metric.append(np.mean(m))
q =(np.argmin(metric)+ 1)/ 10
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub )<prepare_output>
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N )<save_to_csv>
sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False )<prepare_output>
final_model = RandomForestRegressor(bootstrap=False, max_features=0.3, min_samples_leaf=10,
min_samples_split=20, n_estimators=200, n_jobs=-1, random_state=42)
 final_model.fit(X, y )
img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()<load_from_csv>
def make_my_predictions(test_df_all, sample_prediction_df, models, player_and_team_stats, personnel_columns_to_add, final_model, training_columns, categorical_columns, one_hot_columns):
test_df_all = clean_TeamAbbreviations(test_df_all)
 test_df_all = imperial_to_metric(test_df_all)
 test_df_all = tag_rusher(test_df_all)
 test_df_all = get_player_age(test_df_all)
 test_df_all = get_adjusted_yardline(test_df_all)
 test_df_all=norm_x(test_df_all)
 test_df_all['dis_from_center']=abs(train_df_all['X_adj'])
 test_df_all=Direction_orientation_adj(test_df_all)
 test_df_all=X_Y_Velocity(test_df_all)
 test_df_all = qb_and_rusher_to_closest_defender(test_df_all)
 test_df=test_df_all[test_df_all['NflId']==test_df_all['NflIdRusher']]
test_df = test_df.fillna(0)
 test_df = get_direction(test_df)
 test_df = get_leading_team_features(test_df)
 test_df = get_GameClock_features(test_df)
 test_df = get_split_personnel_types(personnel_columns_to_add, test_df)
 for key,value in player_and_team_stats.items() :
test_df = test_df.merge(value, on=key, how='left' ).fillna(0)
 if key == 'DisplayName' and test_df['DisplayName_rusher_performance_bin'].iloc[0] == 0:
test_df['DisplayName_rusher_performance_bin'] = 'few_plays'
test_df=set_Turf(test_df)
 test_df['StadiumType'] = test_df['StadiumType'].apply(clean_StadiumType)
 test_df['WindDirection']=test_df['WindDirection'].apply(clean_WindDirection)
 test_df['WindDirection']=test_df['WindDirection'].apply(transform_WindDirection)
 test_df=GameWeather(test_df)
 for model_type,model in models.items() :
test_df_curr = h2o.H2OFrame(test_df)
 pred=model.predict(test_df_curr)
 test_df = test_df.join(pred.as_data_frame())
 test_df.rename(columns={'predict':'predict_' + model_type}, inplace=True)
 one_hot_test = pd.get_dummies(test_df[categorical_columns])
 one_hot_test_cols = [x for x in one_hot_test if x in one_hot_columns]
one_hot_test = one_hot_test[one_hot_test_cols]
test_df = test_df.join(one_hot_test)
 more_to_add = [x for x in one_hot_columns if x not in list(one_hot_test.columns)]
for column_to_add in more_to_add:
test_df[column_to_add] = 0
test_df=test_df[training_columns]
test_X = np.array(test_df)
 pred = final_model.predict(test_X)
 pred = np.array([np.concatenate(( np.zeros(85), pred[0], np.zeros(77)) , axis=0)])
 pred = np.clip(np.cumsum(pred, axis=1), 0, 1)
 pred = pd.DataFrame(data=[list(pred[0])], columns=sample_prediction_df.columns)
 return pred
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )<concatenate>
env = nflrush.make_env()
for(test_df, sample_prediction_df)in env.iter_test() :
predictions_df = make_my_predictions(test_df, sample_prediction_df, models, player_and_team_stats, personnel_columns_to_add, final_model, training_columns, categorical_columns, list(one_hot.columns))
env.predict(predictions_df)
 env.write_submission_file()
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )<count_unique_values>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )<feature_engineering>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )<categorify>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )<merge>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base<data_type_conversions>
<feature_engineering><EOS>