Kaggle competition: NFL Big Data Bowl
<SOS> metric: CRPS Kaggle data source: nfl-big-data-bowl<create_dataframe>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import datetime
from kaggle.competitions import nflrush
from tqdm import tqdm
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import LabelEncoder
import keras
from keras.callbacks import EarlyStopping
from keras import backend as K
from collections import Counter
from sklearn.model_selection import RepeatedKFold
from keras.models import Sequential, Model
from keras.layers import Input, Dense, Embedding, Concatenate, Flatten, BatchNormalization, Dropout
from keras.callbacks import ModelCheckpoint
from keras.utils.vis_utils import plot_model
from keras import metrics
VERBOSE = 1
PLOT = 1
histories = []
df_oof = pd.DataFrame() ; df_res = pd.DataFrame()
t_start = time.time()
for fold,(idTrain, idValid)in enumerate(folds):
print("
print(( "
print("
if DEVICE == 'TPU':
if tpu:
tf.tpu.experimental.initialize_tpu_system(tpu)
 fold_valid_files = [f for f in train_files if any([int(re.match("^train([0-9]+)", f.split("/")[-1] ).group(1)) % 15 == i for i in idValid])]
fold_valid_files = [f for f in fold_valid_files if GCS_PATH1 in f]
fold_train_files = [f for f in train_files if f not in fold_valid_files]
np.random.shuffle(fold_train_files)
 print("Train files: %d\t\t Valid files: %d" %(len(fold_train_files), len(fold_valid_files)))
 K.clear_session()
if DEVICE == 'TPU':
keras.mixed_precision.experimental.set_policy('mixed_bfloat16')
 with strategy.scope() :
model = build_model(dim=IMG_SIZE, ef=EFF_NET)
 FOLD_CKPT_FOLDER = os.path.join(CKPT_FOLDER, "fold%d" % fold)
 if not os.path.exists(FOLD_CKPT_FOLDER):
os.mkdir(FOLD_CKPT_FOLDER)
 callbacks =[
keras.callbacks.ModelCheckpoint(os.path.join(FOLD_CKPT_FOLDER, "model_fold%d_e{epoch:02d}.h5" % fold), save_weights_only=True),
keras.callbacks.LearningRateScheduler(lrfn)
 ]
if BALANCE_POS_RATIO:
print("Using balanced dataset with pos_ratio = %d%%" % int(100 * BALANCE_POS_RATIO))
ds_train = get_balanced_dataset(fold_train_files, repeat=True, augment=[dropout_aug], drop_remainder=True, shuffle=True,
pos_ratio=BALANCE_POS_RATIO,
dim=IMG_SIZE, read_dim=IMG_READ_SIZE, batch_size=GLOBAL_BATCH_SIZE)
 FOLD_POS = TOTAL_POS *(FOLDS - 1)/ FOLDS
STEPS = int(FOLD_POS / BALANCE_POS_RATIO / GLOBAL_BATCH_SIZE)
 else:
print("Using unbalanced dataset")
 ds_train = get_dataset(fold_train_files, repeat=True, augment=[dropout_aug], drop_remainder=True, shuffle=True,
dim=IMG_SIZE, read_dim=IMG_READ_SIZE, batch_size=GLOBAL_BATCH_SIZE)
 STEPS = int(count_data_items(fold_train_files)/ GLOBAL_BATCH_SIZE)
 ds_valid = get_dataset(fold_valid_files, repeat=False, augment=False, drop_remainder=False, shuffle=False,
dim=IMG_SIZE, read_dim=IMG_READ_SIZE, batch_size=GLOBAL_BATCH_SIZE * TBM)
 if fold==4:
model.load_weights('/kaggle/input/pretraineffi65fold/model_fold%s.h5'%fold)
 else:
model.load_weights('/kaggle/input/pretrained/model_fold%s.h5'%fold)
 print("Training...")
 history = model.fit(
 ds_train,
validation_data = ds_valid,
epochs = EPOCHS,
steps_per_epoch = STEPS,
verbose = VERBOSE,
callbacks = callbacks,
validation_freq = VALID_FREQ 
)
 histories.append(history)
 ckpt_files = np.sort(tf.io.gfile.glob(os.path.join(FOLD_CKPT_FOLDER, "*.h5")))
 ckpt_files_fow_swa = ckpt_files[-N_SWA:]
if len(ckpt_files_fow_swa)> 1:
with strategy.scope() :
model = average.average_weights(ckpt_files_fow_swa, decay=SWA_DECAY, model=model)
 for f in ckpt_files:
os.remove(f)
 model.save(os.path.join(CKPT_FOLDER, "model_fold%d.h5" % fold))
ds_valid = get_dataset(fold_valid_files, augment=TTA >= 1, repeat=True, dim=IMG_SIZE, read_dim=IMG_READ_SIZE, batch_size=GLOBAL_BATCH_SIZE * TBM, drop_remainder=True)
 ct_valid = count_data_items(fold_valid_files); STEPS = int(np.ceil(TTA * ct_valid / GLOBAL_BATCH_SIZE / TBM))
fold_valid_pred = model.predict(ds_valid, steps=STEPS, verbose=1)
 fold_valid_pred = fold_valid_pred[:ct_valid * TTA,]
ds_valid = get_dataset(fold_valid_files, augment=False, repeat=False, dim=IMG_SIZE, batch_size=GLOBAL_BATCH_SIZE * TBM, drop_remainder=False, labeled=False)
 fold_valid_names = np.concatenate([np.array([ni.decode("utf-8")for ni in n.numpy() ])for n in ds_valid.map(lambda i, n: n)], 0)
 fold_df = pd.DataFrame({'image_name': np.tile(fold_valid_names, [TTA]), 'pred': fold_valid_pred.squeeze() , 'fold': fold})
 df_oof = pd.concat([df_oof, fold_df])
 fold_df['image_name'] = fold_df['image_name'].str.replace('_downsampled', '')
 fold_df = fold_df.groupby('image_name' ).mean().reset_index()
fold_df = fold_df.merge(df_base_train[['image_name', 'patient_id', 'target']], on='image_name' ).merge(testiness, on='image_name')
 fold_df['fold'] = fold
auc = metrics.roc_auc_score(fold_df.target, fold_df.pred)
 ds_test = get_dataset(test_files, augment=TTA >= 1, repeat=True, dim=IMG_SIZE, read_dim=IMG_READ_SIZE, batch_size=GLOBAL_BATCH_SIZE * TBM, drop_remainder=True, labeled=False)
 ct_test = count_data_items(test_files); STEPS = int(np.ceil(TTA * ct_test / GLOBAL_BATCH_SIZE / TBM))
fold_test_pred = model.predict(ds_test.map(lambda i, l: i), steps=STEPS, verbose=1)
 fold_test_pred = fold_test_pred[:ct_test * TTA,]
ds_test = get_dataset(test_files, augment=False, repeat=False, dim=IMG_SIZE, batch_size=GLOBAL_BATCH_SIZE * TBM, drop_remainder=False, labeled=False)
 fold_test_names = np.concatenate([np.array([ni.decode("utf-8")for ni in n.numpy() ])for n in ds_test.map(lambda i, n: n)], 0)
 fold_res = pd.DataFrame({'image_name': np.tile(fold_test_names, [TTA]), 'pred': fold_test_pred.squeeze() , 'fold': fold})
 df_res = pd.concat([df_res, fold_res])
 used_time_till_now = time.time() - t_start
time_per_fold = used_time_till_now /(fold + 1)
 print("Validation AUC last epoch = %.4f" % history.history['val_auc'][-1])
 print("Validation AUC(TTA %2d)= %.4f" %(TTA, auc))
print("Total time = %ds\t\tTime per fold = %ds" %(int(used_time_till_now), int(time_per_fold)))
 if PLOT:
plt.figure(figsize=(16, 4))
plt.subplot(1, 2, 1)
 plt.plot(history.history['loss'], color='tab:blue', marker='o')
 plt.plot(range(VALID_FREQ - 1, EPOCHS, VALID_FREQ), history.history['val_loss'], color='tab:blue', marker='x', linestyle=':')
 plt.yscale('log')
 plt.subplot(1, 2, 2)
 plt.plot(history.history['auc'], color='tab:red', marker='o')
 plt.plot(range(VALID_FREQ - 1, EPOCHS, VALID_FREQ), history.history['val_auc'], color='tab:red', marker='x', linestyle=':')
 plt.show()
del model, ds_train, ds_valid, ds_test
print("
")
 if(fold + 1)< FOLDS:
time_till_next_fold = used_time_till_now + time_per_fold
if time_till_next_fold > TIME_BUDGET:
print("Forecasted total time till next fold completion = %d(budget = %d)" %(time_till_next_fold, TIME_BUDGET))
break
pass<merge>
train = pd.read_csv('.. /input/nfl-big-data-bowl-2020/train.csv', dtype={'WindSpeed': 'object'} )
xxx = df_oof.groupby('image_name' ).mean().reset_index().merge(df_base_train, on='image_name')
 print("OOF AUC(TTA %d)= %.4f" %(TTA, metrics.roc_auc_score(xxx.target, xxx.pred)) )<save_to_csv>
def fix_play_direction(df):
df.loc[df['PlayDirection'] == 'left', 'X'] = 120 - df.loc[df['PlayDirection'] == 'left', 'X']
df.loc[df['PlayDirection'] == 'left', 'Y'] =(160 / 3)- df.loc[df['PlayDirection'] == 'left', 'Y']
df.loc[df['PlayDirection'] == 'left', 'Orientation'] = np.mod(180 + df.loc[df['PlayDirection'] == 'left', 'Orientation'], 360)
 df.loc[df['PlayDirection'] == 'left', 'Dir'] = np.mod(180 + df.loc[df['PlayDirection'] == 'left', 'Dir'], 360)
 df['FieldPosition'].fillna('', inplace=True)
 df.loc[df['PossessionTeam'] != df['FieldPosition'], 'YardLine'] = 100 - df.loc[df['PossessionTeam'] != df['FieldPosition'], 'YardLine']
return df
train = fix_play_direction(train )
df_res.to_csv('.. /working/test_res_all.csv', index=False)
 df_oof.to_csv('.. /working/oof_res_all.csv', index=False )<save_to_csv>
train['GameClock'] = train['GameClock'].apply(strtoseconds )
df_res[['image_name', 'pred']].groupby('image_name' ).mean().reset_index().rename({'pred': 'target'}, axis=1 ).to_csv("submission.csv", index=False )<save_to_csv>
train['PlayerHeight'] = train['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
effnet6 = pd.read_csv('/kaggle/input/good-submission/submission_effi6.csv')
 effnet6.to_csv('.. /working/effnet6.csv',index=False )<set_options>
train['TimeHandoff'] = train['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeSnap'] = train['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
sns.set()
py.init_notebook_mode(connected=True)
 os.listdir(".. /input/" )<define_variables>
train['TimeDelta'] = train.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1 )
basepath = ".. /input/siim-isic-melanoma-classification/"
modelspath = ".. /input/pytorch-pretrained-image-models/"
imagestatspath = ".. /input/siimisic-melanoma-classification-image-stats/"<load_from_csv>
train['PlayerBirthDate'] = train['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
train_info = pd.read_csv(basepath + "train.csv")
 train_info.head()<load_from_csv>
seconds_in_year = 60*60*24*365.25
train['PlayerAge'] = train.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1 )
test_info = pd.read_csv(basepath + "test.csv")
 test_info.head()<sort_values>
train = train.drop(['TimeHandoff', 'TimeSnap', 'PlayerBirthDate'], axis=1 )
missing_vals_train = train_info.isnull().sum() / train_info.shape[0]
missing_vals_train[missing_vals_train > 0].sort_values(ascending=False )<sort_values>
train['WindSpeed'] = train['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x )
missing_vals_test = test_info.isnull().sum() / test_info.shape[0]
missing_vals_test[missing_vals_test > 0].sort_values(ascending=False )<count_values>
train['WindSpeed'].value_counts()
train_info.image_name.value_counts().max()<count_values>
train['WindSpeed'] = train['WindSpeed'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 train['WindSpeed'] = train['WindSpeed'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x )
test_info.image_name.value_counts().max()<count_values>
def str_to_float(txt):
try:
return float(txt)
 except:
return -1
train_info.patient_id.value_counts().max()<count_values>
train['WindSpeed'] = train['WindSpeed'].apply(str_to_float )
test_info.patient_id.value_counts().max()<feature_engineering>
train.drop('WindDirection', axis=1, inplace=True )
200/test_info.shape[0] * 100<count_unique_values>
train['PlayDirection'] = train['PlayDirection'].apply(lambda x: x is 'right' )
train_info.groupby("benign_malignant" ).target.nunique()<groupby>
train['Team'] = train['Team'].apply(lambda x: x.strip() =='home' )
patient_ages_table_train = train_info.groupby(["patient_id", "age_approx"] ).size() / train_info.groupby("patient_id" ).size()
patient_ages_table_train = patient_ages_table_train.unstack().transpose()
patient_ages_table_test = test_info.groupby(["patient_id", "age_approx"] ).size() / test_info.groupby("patient_id" ).size()
patient_ages_table_test = patient_ages_table_test.unstack().transpose()
patient_with_known_ages_train = train_info[train_info.patient_id.isin(patient_ages_table_train.columns.values)]
sorted_patients_train = patient_with_known_ages_train.patient_id.value_counts().index.values
patient_with_known_ages_test = test_info[test_info.patient_id.isin(patient_ages_table_test.columns.values)]
sorted_patients_test = patient_with_known_ages_test.patient_id.value_counts().index.values
fig, ax = plt.subplots(2,1, figsize=(20,20))
sns.heatmap(patient_ages_table_train[sorted_patients_train], cmap="Reds", ax=ax[0], cbar=False);
ax[0].set_title("Image coverage in % per patient and age in train data");
sns.heatmap(patient_ages_table_test[sorted_patients_test], cmap="Blues", ax=ax[1], cbar=False);
ax[1].set_title("Image coverage in % per patient and age in test data");
ax[0].set_xlabel("")
 ax[1].set_xlabel("");<feature_engineering>
train['GameWeather'] = train['GameWeather'].str.lower()
indoor = "indoor"
train['GameWeather'] = train['GameWeather'].apply(lambda x: indoor if not pd.isna(x)and indoor in x else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x )
patient_gender_train = train_info.groupby("patient_id" ).sex.unique().apply(lambda l: l[0])
 patient_gender_test = test_info.groupby("patient_id" ).sex.unique().apply(lambda l: l[0])
 train_patients = pd.DataFrame(index=patient_gender_train.index.values, data=patient_gender_train.values, columns=["sex"])
 test_patients = pd.DataFrame(index=patient_gender_test.index.values, data=patient_gender_test.values, columns=["sex"])
 train_patients.loc[:, "num_images"] = train_info.groupby("patient_id" ).size()
test_patients.loc[:, "num_images"] = test_info.groupby("patient_id" ).size()
train_patients.loc[:, "min_age"] = train_info.groupby("patient_id" ).age_approx.min()
train_patients.loc[:, "max_age"] = train_info.groupby("patient_id" ).age_approx.max()
test_patients.loc[:, "min_age"] = test_info.groupby("patient_id" ).age_approx.min()
test_patients.loc[:, "max_age"] = test_info.groupby("patient_id" ).age_approx.max()
train_patients.loc[:, "age_span"] = train_patients["max_age"] - train_patients["min_age"]
test_patients.loc[:, "age_span"] = test_patients["max_age"] - test_patients["min_age"]
train_patients.loc[:, "benign_cases"] = train_info.groupby(["patient_id", "benign_malignant"] ).size().loc[:, "benign"]
train_patients.loc[:, "malignant_cases"] = train_info.groupby(["patient_id", "benign_malignant"] ).size().loc[:, "malignant"]
train_patients["min_age_malignant"] = train_info.groupby(["patient_id", "benign_malignant"] ).age_approx.min().loc[:, "malignant"]
train_patients["max_age_malignant"] = train_info.groupby(["patient_id", "benign_malignant"] ).age_approx.max().loc[:, "malignant"]<sort_values>
weather_count = Counter()
for weather in train['GameWeather']:
if pd.isna(weather):
continue
for word in weather.split() :
weather_count[word]+=1
weather_count.most_common() [:15]
train_patients.sort_values(by="malignant_cases", ascending=False ).head()<define_variables>
def map_weather(txt):
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt:
ans*=0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*3
if 'sunny' in txt or 'sun' in txt:
return ans*2
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt:
return -2*ans
if 'snow' in txt:
return -3*ans
return 0
example_files = os.listdir(basepath + "train/")[0:2]
example_files<feature_engineering>
train['GameWeather'] = train['GameWeather'].apply(map_weather )
train_info["dcm_path"] = basepath + "train/" + train_info.image_name + ".dcm"
test_info["dcm_path"] = basepath + "test/" + test_info.image_name + ".dcm"<load_from_csv>
train['IsRusher'] = train['NflId'] == train['NflIdRusher']
train.drop(['NflId', 'NflIdRusher'], axis=1, inplace=True )
example_dcm = pydicom.dcmread(train_info.dcm_path[2])
 example_dcm<feature_engineering>
train = train.sort_values(by=['PlayId', 'Team', 'IsRusher'] ).reset_index()
train_info["image_path"] = basepath + "jpeg/train/" + train_info.image_name + ".jpg"
test_info["image_path"] = basepath + "jpeg/test/" + test_info.image_name + ".jpg"<load_from_csv>
play_features = []
pos_features = ["X", "Y"]
player_features = ["S", "A", "Dis", "Orientation", "Dir", "PlayerHeight", "PlayerWeight", "Position", "PlayerAge", "IsRusher", "Team"]
game_features = ["YardLine", "Quarter", "GameClock",
"Down", "Distance", "HomeScoreBeforePlay", "VisitorScoreBeforePlay", "DefendersInTheBox", "Temperature", "Humidity", "TimeDelta"]
test_image_stats = pd.read_csv(imagestatspath + "test_image_stats.csv")
 test_image_stats.head(1 )<load_from_csv>
game_train = train[game_features + ["PlayId"]].groupby("PlayId" ).max() [game_features].values
train_image_stats_1 = pd.read_csv(imagestatspath + "train_image_stats_10000.csv")
 train_image_stats_2 = pd.read_csv(imagestatspath + "train_image_stats_20000.csv")
 train_image_stats_3 = pd.read_csv(imagestatspath + "train_image_stats_toend.csv")
 train_image_stats_4 = train_image_stats_1.append(train_image_stats_2)
 train_image_stats = train_image_stats_4.append(train_image_stats_3)
 train_image_stats.shape<define_variables>
scaling_factor = 2
X_train = np.zeros(( len(train)//22, 120//scaling_factor, 58//scaling_factor, len(player_features)+2), dtype = np.float16 )
plot_test = True<groupby>
x_field = np.arange(-(120//scaling_factor)/2,(120//scaling_factor)/2, 1)
 y_field = np.abs(np.arange(-(58//scaling_factor)/2,(58//scaling_factor)/2, 1))
test_image_stats.groupby(["rows", "columns"] ).size().sort_values(ascending=False ).iloc[0:10] / test_image_stats.shape[0]<groupby>
field_array = np.dot(x_field[:, None],y_field[None,:])
 field_array = field_array/field_array.max()
train_image_stats.groupby(["rows", "columns"] ).size().sort_values(ascending=False ).iloc[0:10] / train_image_stats.shape[0]<define_variables>
X_train[:, :, :, -2] = field_array
examples1 = {"rows": 1080, "columns": 1920}
examples2 = {"rows": 4000, "columns": 6000}<load_from_csv>
plt.imshow(X_train[0, :, :, -2].astype(np.float32 ).T )
train = pd.read_csv('.. /input/siim-isic-melanoma-classification/train.csv')
 print('Examples WITH Melanoma')
 imgs = train.loc[train.target==1].sample(10 ).image_name.values
plt.figure(figsize=(20,8))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()
print('Examples WITHOUT Melanoma')
 imgs = train.loc[train.target==0].sample(10 ).image_name.values
plt.figure(figsize=(20,8))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()<install_modules>
le = LabelEncoder()
train["Position"] = le.fit_transform(train["Position"])
 train[["X", "Y"]] = train[["X", "Y"]].astype(int )
!pip install -q efficientnet >> /dev/null<import_modules>
scalers = {}
for feature in player_features:
if feature == "X" or feature == "Y" or feature == "IsRusher":
continue
else:
ss = StandardScaler()
train[feature] = ss.fit_transform(train[feature].values.reshape(-1, 1))
scalers[feature] = ss
import pandas as pd, numpy as np
from kaggle_datasets import KaggleDatasets
import tensorflow as tf, re, math
import tensorflow.keras.backend as K
import efficientnet.tfkeras as efn
from sklearn.model_selection import KFold
from sklearn.metrics import roc_auc_score
import matplotlib.pyplot as plt<define_variables>
for i, playid in enumerate(tqdm(range(len(train)))) :
player_df = train.iloc[i, :]
X_train[i//22, player_df["X"]//scaling_factor, player_df["Y"]//scaling_factor, 0:len(player_features)] += player_df[player_features].astype(np.float16)
 X_train[i//22, player_df["YardLine"]//2, :, -1] = 1
DEVICE = "TPU"
SEED = 42
FOLDS = 5
IMG_SIZES = [384,384,384,384,384]
INC2019 = [0,0,0,0,0]
INC2018 = [1,1,1,1,1]
BATCH_SIZES = [32]*FOLDS
EPOCHS = [12]*FOLDS
EFF_NETS = [6,6,6,6,6]
WGTS = [1/FOLDS]*FOLDS
TTA = 11<choose_model_class>
X_train[np.isnan(X_train)] = 0
game_train[np.isnan(game_train)] = 0
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<feature_engineering>
y_train = np.zeros(shape=(train.shape[0]//22, 199))
for i,yard in enumerate(train['Yards'][::22]):
y_train[i, yard+99:] = np.ones(shape=(1, 100-yard))
GCS_PATH = [None]*FOLDS; GCS_PATH2 = [None]*FOLDS
for i,k in enumerate(IMG_SIZES):
GCS_PATH[i] = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(k,k))
GCS_PATH2[i] = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(k,k))
files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/train*.tfrec')))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec')) )<define_variables>
for i in range(len(player_features)+2):
plt.imshow(X_train[0, :, :, i].astype(np.float32 ).T)
 try:
print(player_features[i])
 except:
pass
plt.show()
ROT_ = 180.0
SHR_ = 2.0
HZOOM_ = 8.0
WZOOM_ = 8.0
HSHIFT_ = 8.0
WSHIFT_ = 8.0<normalization>
__all__ = ['RAdam']
class RAdam(keras.optimizers.Optimizer):

 def __init__(self, learning_rate=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., weight_decay=0., amsgrad=False,
total_steps=0, warmup_proportion=0.1, min_lr=0., **kwargs):
learning_rate = kwargs.pop('lr', learning_rate)
 super(RAdam, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.learning_rate = K.variable(learning_rate, name='learning_rate')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.weight_decay = K.variable(weight_decay, name='weight_decay')
 self.total_steps = K.variable(total_steps, name='total_steps')
 self.warmup_proportion = K.variable(warmup_proportion, name='warmup_proportion')
 self.min_lr = K.variable(min_lr, name='min_lr')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.initial_weight_decay = weight_decay
self.initial_total_steps = total_steps
self.amsgrad = amsgrad
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * K.cast(self.iterations, K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx())+ 1
if self.initial_total_steps > 0:
warmup_steps = self.total_steps * self.warmup_proportion
decay_steps = K.maximum(self.total_steps - warmup_steps, 1)
 decay_rate =(self.min_lr - lr)/ decay_steps
lr = K.switch(
 t <= warmup_steps,
lr *(t / warmup_steps),
lr + decay_rate * K.minimum(t - warmup_steps, decay_steps), 
)
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for(i, p)in enumerate(params)]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for(i, p)in enumerate(params)]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for(i, p)in enumerate(params)]
else:
vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params)) ]
self.weights = [self.iterations] + ms + vs + vhats
beta_1_t = K.pow(self.beta_1, t)
 beta_2_t = K.pow(self.beta_2, t)
 sma_inf = 2.0 /(1.0 - self.beta_2)- 1.0
sma_t = sma_inf - 2.0 * t * beta_2_t /(1.0 - beta_2_t)
 for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 m_corr_t = m_t /(1.0 - beta_1_t)
 if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 v_corr_t = K.sqrt(vhat_t /(1.0 - beta_2_t))
self.updates.append(K.update(vhat, vhat_t))
else:
v_corr_t = K.sqrt(v_t /(1.0 - beta_2_t))
r_t = K.sqrt(( sma_t - 4.0)/(sma_inf - 4.0)* 
(sma_t - 2.0)/(sma_inf - 2.0)*
sma_inf / sma_t)
 p_t = K.switch(sma_t >= 5, r_t * m_corr_t /(v_corr_t + self.epsilon), m_corr_t)
 if self.initial_weight_decay > 0:
p_t += self.weight_decay * p
p_t = p - lr * p_t
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
@property
def lr(self):
return self.learning_rate
@lr.setter
def lr(self, learning_rate):
self.learning_rate = learning_rate
def get_config(self):
config = {
'learning_rate': float(K.get_value(self.learning_rate)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.weight_decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad,
'total_steps': float(K.get_value(self.total_steps)) ,
'warmup_proportion': float(K.get_value(self.warmup_proportion)) ,
'min_lr': float(K.get_value(self.min_lr)) ,
}
base_config = super(RAdam, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, DIM=256):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )<prepare_x_and_y>
class Mish(Activation):

 def __init__(self, activation, **kwargs):
super(Mish, self ).__init__(activation, **kwargs)
 self.__name__ = 'Mish'
def mish(inputs):
return inputs * tf.math.tanh(tf.math.softplus(inputs))
get_custom_objects().update({'Mish': Mish(mish)} )
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img, augment=True, dim=256):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img,DIM=dim)
 img = tf.image.random_flip_up_down(img)
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 img = tf.reshape(img, [dim,dim, 3])
 return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )<create_dataframe>
def crps(y_true, y_pred):
return K.mean(K.square(y_true - K.cumsum(y_pred, axis=1)) , axis=1 )
def get_dataset(files, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True, batch_size=16, dim=256):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, imgname_or_label:(prepare_image(img, augment=augment, dim=dim),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(batch_size * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds<choose_model_class>
def create_model() :
image_inputs = Input(shape=(X_train.shape[1],X_train.shape[2], X_train.shape[3]))
game_inputs = Input(shape=(game_train.shape[1],))
x = BatchNormalization()(image_inputs)
 x = Conv2D(16,(3,3), activation='Mish' )(x)
 x = MaxPooling2D(pool_size=(2, 2))(x)
 x = Conv2D(16,(3,3), activation='Mish' )(x)
 x = MaxPooling2D(pool_size=(2, 2))(x)
 x = Conv2D(32,(3,3), activation='Mish' )(x)
 x = Conv2D(64,(3,3), activation='Mish' )(x)
 x = GlobalMaxPooling2D()(x)
 game_feats = BatchNormalization()(game_inputs)
 game_feats = Dense(25, activation = 'Mish' )(game_feats)
 x = Concatenate()([x, game_feats])
 x = Dropout (.2 )(x)
 out = Dense(199, activation='softmax' )(x)
 model = Model(inputs = [image_inputs,game_inputs], outputs=out)
 return model
EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3,
efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6]
def build_model(dim=128, ef=0):
inp = tf.keras.layers.Input(shape=(dim,dim,3))
base = EFNS[ef](input_shape=(dim,dim,3),weights='imagenet',include_top=False)
 x = base(inp)
 x = tf.keras.layers.GlobalAveragePooling2D()(x)
 x = tf.keras.layers.Dense(256, activation = 'relu',kernel_regularizer = tf.keras.regularizers.l2() )(x)
 x = tf.keras.layers.Dropout(0.2 )(x)
 x = tf.keras.layers.Dense(128, activation = 'relu' )(x)
 x = tf.keras.layers.Dropout(0.1 )(x)
 x = tf.keras.layers.Dense(1,activation='sigmoid' )(x)
 model = tf.keras.Model(inputs=inp,outputs=x)
 opt = tf.keras.optimizers.Adam(learning_rate=0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05)
 model.compile(optimizer=opt,loss=loss,metrics=['AUC'])
 return model<init_hyperparams>
def train_model(x_tr, y_tr, x_vl, y_vl):
model = create_model()
print(model.summary())
 er = EarlyStopping(patience=5, min_delta=1e-4, restore_best_weights=True, monitor='val_loss')
 model.compile(optimizer=RAdam(warmup_proportion=0.1, min_lr=1e-5), loss=crps, metrics=[metrics.mae])
 model.fit(x_tr, y_tr, epochs=100, callbacks=[er], validation_data=[x_vl, y_vl])
 return model
def get_lr_callback(batch_size=8):
lr_start = 0.000005
lr_max = 0.00000125 * REPLICAS * batch_size
lr_min = 0.000001
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.8
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback<define_variables>
rkf = RepeatedKFold(n_splits=4, n_repeats=1)
 models = []
for tr_idx, vl_idx in rkf.split(X_train, y_train):
x_tr, y_tr = X_train[tr_idx].astype(np.float16), y_train[tr_idx].astype(np.float16)
 x_vl, y_vl = X_train[vl_idx].astype(np.float16), y_train[vl_idx].astype(np.float16)
 game_tr, game_vl = game_train[tr_idx], game_train[vl_idx]
model = train_model([x_tr,game_tr], y_tr, [x_vl,game_vl], y_vl)
 models.append(model )
VERBOSE = 0
DISPLAY_PLOT = True
skf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)
 oof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = []
preds = np.zeros(( count_data_items(files_test),1))
for fold,(idxT,idxV)in enumerate(skf.split(np.arange(15))):
if DEVICE=='TPU':
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 print('
print(' 
(IMG_SIZES[fold],EFF_NETS[fold],BATCH_SIZES[fold]*REPLICAS))
files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxT])
 if INC2019[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1])
 print('
if INC2018[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2])
 print('
np.random.shuffle(files_train); print('
files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxV])
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/test*.tfrec')))
 K.clear_session()
with strategy.scope() :
model = build_model(dim=IMG_SIZES[fold],ef=EFF_NETS[fold])
 sv = tf.keras.callbacks.ModelCheckpoint(
 'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min', save_freq='epoch')
 print('Training...')
 history = model.fit(
 get_dataset(files_train, augment=True, shuffle=True, repeat=True,
dim=IMG_SIZES[fold], batch_size = BATCH_SIZES[fold]),
epochs=EPOCHS[fold], callbacks = [sv,get_lr_callback(BATCH_SIZES[fold])],
steps_per_epoch=count_data_items(files_train)/BATCH_SIZES[fold]//REPLICAS,
validation_data=get_dataset(files_valid,augment=False,shuffle=False,
repeat=False,dim=IMG_SIZES[fold]),
verbose=VERBOSE 
)
 print('Loading best model...')
 model.load_weights('fold-%i.h5'%fold)
 print('Predicting OOF with TTA...')
 ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_valid = count_data_items(files_valid); STEPS = TTA * ct_valid/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,]
oof_pred.append(np.mean(pred.reshape(( ct_valid,TTA),order='F'),axis=1))
ds_valid = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=True, return_image_names=True)
 oof_tar.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]))
oof_folds.append(np.ones_like(oof_tar[-1],dtype='int8')*fold)
 ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 oof_names.append(np.array([img_name.numpy().decode("utf-8")for img, img_name in iter(ds.unbatch())]))
print('Predicting Test with TTA...')
 ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,]
preds[:,0] += np.mean(pred.reshape(( ct_test,TTA),order='F'),axis=1)* WGTS[fold]
auc = roc_auc_score(oof_tar[-1],oof_pred[-1])
 oof_val.append(np.max(history.history['val_auc']))
print('
if DISPLAY_PLOT:
plt.figure(figsize=(15,5))
plt.plot(np.arange(EPOCHS[fold]),history.history['auc'],'-o',label='Train AUC',color='
plt.plot(np.arange(EPOCHS[fold]),history.history['val_auc'],'-o',label='Val AUC',color='
x = np.argmax(history.history['val_auc']); y = np.max(history.history['val_auc'])
 xdist = plt.xlim() [1] - plt.xlim() [0]; ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
%.2f'%y,size=14)
 plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)
 plt.legend(loc=2)
 plt2 = plt.gca().twinx()
plt2.plot(np.arange(EPOCHS[fold]),history.history['loss'],'-o',label='Train Loss',color='
plt2.plot(np.arange(EPOCHS[fold]),history.history['val_loss'],'-o',label='Val Loss',color='
x = np.argmin(history.history['val_loss']); y = np.min(history.history['val_loss'])
 ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
plt.ylabel('Loss',size=14)
 plt.title('FOLD %i - Image Size %i, EfficientNet B%i, inc2019=%i, inc2018=%i'% 
(fold+1,IMG_SIZES[fold],EFF_NETS[fold],INC2019[fold],INC2018[fold]),size=18)
 plt.legend(loc=3)
 plt.show()<save_to_csv>
preds = models[0].predict([x_vl,game_vl] )
oof = np.concatenate(oof_pred); true = np.concatenate(oof_tar);
names = np.concatenate(oof_names); folds = np.concatenate(oof_folds)
 auc = roc_auc_score(true,oof)
 print('Overall OOF AUC with TTA = %.3f'%auc)
 df_oof = pd.DataFrame(dict(
 image_name = names, target=true, pred = oof, fold=folds))
df_oof.to_csv('oof.csv',index=False)
 df_oof.head()<create_dataframe>
del train, X_train
ds = get_dataset(files_test, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )<save_to_csv>
def make_pred(df, sample, env, models):
train = df
train = fix_play_direction(train)
 train['GameClock'] = train['GameClock'].apply(strtoseconds)
 train['PlayerHeight'] = train['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
train['TimeHandoff'] = train['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeSnap'] = train['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeDelta'] = train.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 train['PlayerBirthDate'] = train['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
train['PlayerAge'] = train.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 train = train.drop(['TimeHandoff', 'TimeSnap', 'PlayerBirthDate'], axis=1)
 train['WindSpeed'] = train['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 train['WindSpeed'] = train['WindSpeed'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 train['WindSpeed'] = train['WindSpeed'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 train['WindSpeed'] = train['WindSpeed'].apply(str_to_float)
 train.drop('WindDirection', axis=1, inplace=True)
 train['PlayDirection'] = train['PlayDirection'].apply(lambda x: x is 'right')
 train['Team'] = train['Team'].apply(lambda x: x.strip() =='home')
 train['GameWeather'] = train['GameWeather'].str.lower()
indoor = "indoor"
train['GameWeather'] = train['GameWeather'].apply(lambda x: indoor if not pd.isna(x)and indoor in x else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 train['GameWeather'] = train['GameWeather'].apply(map_weather)
 train['IsRusher'] = train['NflId'] == train['NflIdRusher']
train.drop(['NflId', 'NflIdRusher'], axis=1, inplace=True)
 train = train.sort_values(by=['PlayId', 'Team', 'IsRusher'] ).reset_index()
X_train = np.zeros(( 1, 120//scaling_factor, 58//scaling_factor, len(player_features)+2), dtype = np.float16)
 X_train[:, :, :, -2] = field_array
train["Position"] = le.transform(train["Position"])
 train[["X", "Y"]] = train[["X", "Y"]].astype(int)
 for feature in player_features:
if feature == "X" or feature == "Y" or feature == "IsRusher":
continue
else:
ss = scalers[feature]
train[feature] = ss.transform(train[feature].values.reshape(-1, 1))
for row in range(22):
player_df = train.iloc[row]
X_train[0, player_df["X"]//scaling_factor, player_df["Y"]//scaling_factor, 0:len(player_features)] = player_df[player_features].astype(np.float16)
 X_train[0, train["YardLine"]//2, :, -1] = 1
game_train = train[game_features + ["PlayId"]].groupby("PlayId" ).max() [game_features].values
game_train[np.isnan(game_train)] = 0
X_train[np.isnan(X_train)] = 0
y_pred = np.mean([model.predict([X_train, game_train])for model in models], axis=0)
 y_pred[0] = np.cumsum(y_pred[0], axis=0 ).clip(0, 1)
 y_pred[:, -1] = np.ones(shape=(y_pred.shape[0], 1))
y_pred[:, 0] = np.zeros(shape=(y_pred.shape[0], 1))
env.predict(pd.DataFrame(data=y_pred,columns=sample.columns))
return y_pred
submission = pd.DataFrame(dict(image_name=image_names, target=preds[:,0]))
submission = submission.sort_values('image_name')
 submission.to_csv('submission.csv', index=False)
 submission.head()<set_options>
env = nflrush.make_env()
pd.set_option('display.max_columns', 1000)
 kaggle=True
if kaggle:
dirs="/kaggle/input/data-science-bowl-2019/"
else:
dirs="./"<feature_engineering>
for test, sample in tqdm(env.iter_test()):
make_pred(test, sample, env, models )
def SeedEverything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 return
SeedEverything(1993 )<define_variables>
env.write_submission_file()