Kaggle competition: NFL Big Data Bowl
<SOS> metric: CRPS Kaggle data source: nfl-big-data-bowl<choose_model_class>
sns.set_style('darkgrid')
 mpl.rcParams['figure.figsize'] = [15,10]
with strategy.scope() :
enet = efn.EfficientNetB7(
 input_shape=(512, 512, 3),
weights='imagenet',
include_top=False 
)
 model = tf.keras.Sequential([
enet,
tf.keras.layers.GlobalMaxPooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model.compile(
 optimizer=tf.keras.optimizers.Adam(lr=0.0001),
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 model.summary()<train_model>
env = nflrush.make_env()
history = model.fit(
 get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[lr_callback],
validation_data=None if SKIP_VALIDATION else get_validation_dataset()
)<choose_model_class>
train = pd.read_csv('.. /input/nfl-big-data-bowl-2020/train.csv', dtype={'WindSpeed': 'object'} )
with strategy.scope() :
res = ResNet152V2(
 input_shape=(512, 512, 3),
weights='imagenet',
include_top=False 
)
 model2 = tf.keras.Sequential([
res,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model2.compile(
 optimizer=tf.keras.optimizers.Adam(lr=0.0001),
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 model2.summary()<train_model>
train['A'].value_counts()
history2 = model2.fit(
 get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[lr_callback],
validation_data=None if SKIP_VALIDATION else get_validation_dataset()
)<choose_model_class>
train['A'].isnull().sum()
with strategy.scope() :
Inct=InceptionResNetV2(
 input_shape=(512, 512, 3),
weights='imagenet',
include_top=False 
)
 model3 = tf.keras.Sequential([
Inct,
tf.keras.layers.GlobalMaxPooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model3.compile(
 optimizer=tf.keras.optimizers.Adam(lr=0.0001),
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 model3.summary()<train_model>
train['DefendersInTheBox_vs_Distance'] = train['DefendersInTheBox'] / train['Distance']
history3 = model3.fit(
 get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[lr_callback],
validation_data=None if SKIP_VALIDATION else get_validation_dataset()
)<choose_model_class>
train['StadiumType'].value_counts()
with strategy.scope() :
den=DenseNet201(
 input_shape=(512, 512, 3),
weights='imagenet',
include_top=False 
)
 model4 = tf.keras.Sequential([
den,
tf.keras.layers.GlobalMaxPooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model4.compile(
 optimizer=tf.keras.optimizers.Adam(lr=0.0001),
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 model4.summary()<train_model>
def clean_StadiumType(txt):
if pd.isna(txt):
return np.nan
txt = txt.lower()
txt = ''.join([c for c in txt if c not in punctuation])
 txt = re.sub(' +', ' ', txt)
 txt = txt.strip()
txt = txt.replace('outside', 'outdoor')
 txt = txt.replace('outdor', 'outdoor')
 txt = txt.replace('outddors', 'outdoor')
 txt = txt.replace('outdoors', 'outdoor')
 txt = txt.replace('oudoor', 'outdoor')
 txt = txt.replace('indoors', 'indoor')
 txt = txt.replace('ourdoor', 'outdoor')
 txt = txt.replace('retractable', 'rtr.')
 return txt
history4 = model4.fit(
 get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[lr_callback],
validation_data=None if SKIP_VALIDATION else get_validation_dataset()
)<predict_on_test>
train['StadiumType'] = train['StadiumType'].apply(clean_StadiumType )
if not SKIP_VALIDATION:
cmdataset = get_validation_dataset(ordered=True)
 images_ds = cmdataset.map(lambda image, label: image)
 labels_ds = cmdataset.map(lambda image, label: label ).unbatch()
cm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES)) ).numpy()
m = model.predict(images_ds)
 m2 = model2.predict(images_ds)
 m3 = model3.predict(images_ds)
 scores = []
best_score,best_alpha,best_beta=0,0,0
for alpha in np.linspace(0,1,100):
for beta in np.linspace(0,1,100):
cm_probabilities = alpha*m+(beta)*m2+(1-alpha-beta)*m3
cm_predictions = np.argmax(cm_probabilities, axis=-1)
 score = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 if(score > best_score):
best_alpha = alpha
best_beta = beta
best_score = score
scores.append(best_score)
 plt.plot(scores)
 print("best_alha:",best_alpha,"best_beta:",best_beta)
 cm_probabilities = best_alpha*m+best_beta*m2+(1-best_alpha)*m3
cm_predictions = np.argmax(cm_probabilities, axis=-1)
 else:
best_alpha = 0.505
best_beta = 0.303<compute_test_metric>
def transform_StadiumType(txt):
if pd.isna(txt):
return np.nan
if 'outdoor' in txt or 'open' in txt:
return 1
if 'indoor' in txt or 'closed' in txt:
return 0
return np.nan
if not SKIP_VALIDATION:
cmat = confusion_matrix(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)))
 score = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 precision = precision_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 recall = recall_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 display_confusion_matrix(cmat, score, precision, recall)
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))<predict_on_test>
train['StadiumType'] = train['StadiumType'].apply(transform_StadiumType )
test_ds = get_test_dataset(ordered=True)
 print('Computing predictions...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilities =(model.predict(test_images_ds)+ model2.predict(test_images_ds)+model3.predict(test_images_ds)+model4.predict(test_images_ds)) /4
predictions = np.argmax(probabilities, axis=-1)
 print(predictions)
 print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )<install_modules>
Turf = {'Field Turf':'Artificial', 'A-Turf Titan':'Artificial', 'Grass':'Natural', 'UBU Sports Speed S5-M':'Artificial',
'Artificial':'Artificial', 'DD GrassMaster':'Artificial', 'Natural Grass':'Natural',
'UBU Speed Series-S5-M':'Artificial', 'FieldTurf':'Artificial', 'FieldTurf 360':'Artificial', 'Natural grass':'Natural', 'grass':'Natural',
'Natural':'Natural', 'Artifical':'Artificial', 'FieldTurf360':'Artificial', 'Naturall Grass':'Natural', 'Field turf':'Artificial',
'SISGrass':'Artificial', 'Twenty-Four/Seven Turf':'Artificial', 'natural grass':'Natural'}
train['Turf'] = train['Turf'].map(Turf)
 train['Turf'] = train['Turf'] == 'Natural'
!pip install -q efficientnet<import_modules>
train[(train['PossessionTeam']!=train['HomeTeamAbbr'])&(train['PossessionTeam']!=train['VisitorTeamAbbr'])][['PossessionTeam', 'HomeTeamAbbr', 'VisitorTeamAbbr']]
print("Tensorflow version " + tf.__version__ )<set_options>
map_abbr = {'ARI': 'ARZ', 'BAL': 'BLT', 'CLE': 'CLV', 'HOU': 'HST'}
for abb in train['PossessionTeam'].unique() :
map_abbr[abb] = abb
AUTO = tf.data.experimental.AUTOTUNE
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
 GCS_DS_PATH = KaggleDatasets().get_gcs_path()
IMAGE_SIZE = [512, 512]
EPOCHS = 25
BATCH_SIZE = 16 * strategy.num_replicas_in_sync<choose_model_class>
train['PossessionTeam'] = train['PossessionTeam'].map(map_abbr)
 train['HomeTeamAbbr'] = train['HomeTeamAbbr'].map(map_abbr)
 train['VisitorTeamAbbr'] = train['VisitorTeamAbbr'].map(map_abbr )
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<define_variables>
train['HomePossesion'] = train['PossessionTeam'] == train['HomeTeamAbbr']
GCS_PATH_SELECT = {
192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',
224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',
331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',
512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'
}
GCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')
 VALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')
 SKIP_VALIDATION = True
if SKIP_VALIDATION:
TRAINING_FILENAMES = TRAINING_FILENAMES + VALIDATION_FILENAMES<set_options>
train['Field_eq_Possession'] = train['FieldPosition'] == train['PossessionTeam']
train['HomeField'] = train['FieldPosition'] == train['HomeTeamAbbr']
MIXED_PRECISION = True
XLA_ACCELERATE = True
if MIXED_PRECISION:
if tpu:
policy = tf.keras.mixed_precision.experimental.Policy('mixed_bfloat16')
 else:
policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')
 mixed_precision.set_policy(policy)
 print('Mixed precision enabled')
 if XLA_ACCELERATE:
tf.config.optimizer.set_jit(True)
 print('Accelerated Linear Algebra enabled' )<define_variables>
off_form = train['OffenseFormation'].unique()
train['OffenseFormation'].value_counts()
CLASSES = ['pink primrose', 'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea', 'wild geranium', 'tiger lily', 'moon orchid', 'bird of paradise', 'monkshood', 'globe thistle',
'snapdragon', "colt's foot", 'king protea', 'spear thistle', 'yellow iris', 'globe-flower', 'purple coneflower', 'peruvian lily', 'balloon flower', 'giant white arum lily',
'fire lily', 'pincushion flower', 'fritillary', 'red ginger', 'grape hyacinth', 'corn poppy', 'prince of wales feathers', 'stemless gentian', 'artichoke', 'sweet william',
'carnation', 'garden phlox', 'love in the mist', 'cosmos', 'alpine sea holly', 'ruby-lipped cattleya', 'cape flower', 'great masterwort', 'siam tulip', 'lenten rose',
'barberton daisy', 'daffodil', 'sword lily', 'poinsettia', 'bolero deep blue', 'wallflower', 'marigold', 'buttercup', 'daisy', 'common dandelion',
'petunia', 'wild pansy', 'primula', 'sunflower', 'lilac hibiscus', 'bishop of llandaff', 'gaura', 'geranium', 'orange dahlia', 'pink-yellow dahlia',
'cautleya spicata', 'japanese anemone', 'black-eyed susan', 'silverbush', 'californian poppy', 'osteospermum', 'spring crocus', 'iris', 'windflower', 'tree poppy',
'gazania', 'azalea', 'water lily', 'rose', 'thorn apple', 'morning glory', 'passion flower', 'lotus', 'toad lily', 'anthurium',
'frangipani', 'clematis', 'hibiscus', 'columbine', 'desert-rose', 'tree mallow', 'magnolia', 'cyclamen ', 'watercress', 'canna lily',
'hippeastrum ', 'bee balm', 'pink quill', 'foxglove', 'bougainvillea', 'camellia', 'mallow', 'mexican petunia', 'bromelia', 'blanket flower',
'trumpet creeper', 'blackberry lily', 'common tulip', 'wild rose']
len(CLASSES )<categorify>
train = pd.concat([train.drop(['OffenseFormation'], axis=1), pd.get_dummies(train['OffenseFormation'], prefix='Formation')], axis=1)
 dummy_col = train.columns
np.set_printoptions(threshold=15, linewidth=80)
 def batch_to_numpy_images_and_labels(data):
images, labels = data
numpy_images = images.numpy()
numpy_labels = labels.numpy()
if numpy_labels.dtype == object:
numpy_labels = [None for _ in enumerate(numpy_images)]
return numpy_images, numpy_labels
def title_from_label_and_target(label, correct_label):
if correct_label is None:
return CLASSES[label], True
correct =(label == correct_label)
 return "{} [{}{}{}]".format(CLASSES[label], 'OK' if correct else 'NO', u"\u2192" if not correct else '',
CLASSES[correct_label] if not correct else ''), correct
def display_one_flower(image, title, subplot, red=False, titlesize=16):
plt.subplot(*subplot)
 plt.axis('off')
 plt.imshow(image)
 if len(title)> 0:
plt.title(title, fontsize=int(titlesize)if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))
return(subplot[0], subplot[1], subplot[2]+1)
 def display_batch_of_images(databatch, predictions=None):

 images, labels = batch_to_numpy_images_and_labels(databatch)
 if labels is None:
labels = [None for _ in enumerate(images)]
rows = int(math.sqrt(len(images)))
 cols = len(images)//rows
FIGSIZE = 13.0
SPACING = 0.1
subplot=(rows,cols,1)
 if rows < cols:
plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))
else:
plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))
for i,(image, label)in enumerate(zip(images[:rows*cols], labels[:rows*cols])) :
title = '' if label is None else CLASSES[label]
correct = True
if predictions is not None:
title, correct = title_from_label_and_target(predictions[i], label)
 dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3
subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)
 plt.tight_layout()
if label is None and predictions is None:
plt.subplots_adjust(wspace=0, hspace=0)
 else:
plt.subplots_adjust(wspace=SPACING, hspace=SPACING)
 plt.show()
def display_confusion_matrix(cmat, score, precision, recall):
plt.figure(figsize=(15,15))
ax = plt.gca()
ax.matshow(cmat, cmap='Reds')
 ax.set_xticks(range(len(CLASSES)))
 ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})
 plt.setp(ax.get_xticklabels() , rotation=45, ha="left", rotation_mode="anchor")
 ax.set_yticks(range(len(CLASSES)))
 ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})
 plt.setp(ax.get_yticklabels() , rotation=45, ha="right", rotation_mode="anchor")
 titlestring = ""
if score is not None:
titlestring += 'f1 = {:.3f} '.format(score)
 if precision is not None:
titlestring += '
precision = {:.3f} '.format(precision)
 if recall is not None:
titlestring += '
recall = {:.3f} '.format(recall)
 if len(titlestring)> 0:
ax.text(101, 1, titlestring, fontdict={'fontsize': 18, 'horizontalalignment':'right', 'verticalalignment':'top', 'color':'
plt.show()
def display_training_curves(training, validation, title, subplot):
if subplot%10==1:
plt.subplots(figsize=(10,10), facecolor='
plt.tight_layout()
ax = plt.subplot(subplot)
 ax.set_facecolor('
ax.plot(training)
 ax.plot(validation)
 ax.set_title('model '+ title)
 ax.set_ylabel(title)
 ax.set_xlabel('epoch')
 ax.legend(['train', 'valid.'] )<categorify>
off_personnel = train['OffensePersonnel'].unique()
train['OffensePersonnel'].value_counts()
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def data_augment(image, label):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 return image, label
def get_training_dataset(do_aug=True):
dataset = load_dataset(TRAINING_FILENAMES, labeled=True)
 dataset = dataset.repeat()
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 if do_aug:
dataset = dataset.map(transform, num_parallel_calls=AUTO)
 dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(ordered=False):
dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)
 NUM_VALIDATION_IMAGES =(1 - SKIP_VALIDATION)* count_data_items(VALIDATION_FILENAMES)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))<normalization>
off_personnel = train['OffensePersonnel'].unique()
train['OffensePersonnel'].value_counts('OL' )
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))<categorify>
def formation_numeric(txt):
QB = 0
RB = 0
WR = 0
TE = 0
OL = 0
DL = 0
LB = 0
DB = 0
if "QB" in txt:
QB = 1 - int(txt[txt.find("QB")-2:txt.find("QB")-1])
 if "RB" in txt:
RB = 1 - int(txt[txt.find("RB")-2:txt.find("RB")-1])
 if "WR" in txt:
WR = 3 - int(txt[txt.find("WR")-2:txt.find("WR")-1])
 if "TE" in txt:
TE = int(txt[txt.find("TE")-2:txt.find("TE")-1])- 1
if "OL" in txt:
OL = int(txt[txt.find("OL")-2:txt.find("OL")-1])- 5
if "DL" in txt:
DL = int(txt[txt.find("DL")-2:txt.find("DL")-1])
 if "LB" in txt:
LB = int(txt[txt.find("LB")-2:txt.find("LB")-1])
 if "DB" in txt:
DB = -int(txt[txt.find("DB")-2:txt.find("DB")-1])
 return QB + RB + WR + TE + OL + DL + LB + DB
def transform(image,label):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
rot = 15.* tf.random.normal([1],dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image,tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3]),label<choose_model_class>
train['Formation_Heavy_Score'] = train["OffensePersonnel"].apply(formation_numeric )
with strategy.scope() :
enet = efn.EfficientNetB7(
 input_shape=(512, 512, 3),
weights='imagenet',
include_top=False 
)
 model = tf.keras.Sequential([
enet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax', dtype='float32')
 ] )<choose_model_class>
train['Formation_Heavy_Score'].value_counts()
model.compile(
 optimizer=tf.keras.optimizers.Adam() ,
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 model.summary()<train_model>
formation_numeric("1 TE, 9 OL" )
history = model.fit(
 get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[lr_callback],
validation_data= None if SKIP_VALIDATION else get_validation_dataset()
)<choose_model_class>
train['GameClock'].value_counts()
with strategy.scope() :
rnet = tf.keras.applications.DenseNet201(
 input_shape=(512, 512, 3),
weights='imagenet',
include_top=False 
)
 model2 = tf.keras.Sequential([
rnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax', dtype='float32')
 ])
<choose_model_class>
train['GameClock'] = train['GameClock'].apply(strtoseconds )
model2.compile(
 optimizer=tf.keras.optimizers.Adam() ,
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 model2.summary()<train_model>
train['PlayerHeight'] = train['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
history2 = model2.fit(
 get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[lr_callback],
validation_data=None if SKIP_VALIDATION else get_validation_dataset()
)<predict_on_test>
train['PlayerBMI'] = 703*(train['PlayerWeight']/(train['PlayerHeight'])**2 )
if not SKIP_VALIDATION:
cmdataset = get_validation_dataset(ordered=True)
 images_ds = cmdataset.map(lambda image, label: image)
 labels_ds = cmdataset.map(lambda image, label: label ).unbatch()
cm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES)) ).numpy()
m = model.predict(images_ds)
 m2 = model2.predict(images_ds)
 scores = []
for alpha in np.linspace(0,1,100):
cm_probabilities = alpha*m+(1-alpha)*m2
cm_predictions = np.argmax(cm_probabilities, axis=-1)
 scores.append(f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro'))
print("Correct labels: ", cm_correct_labels.shape, cm_correct_labels)
 print("Predicted labels: ", cm_predictions.shape, cm_predictions)
 plt.plot(scores)
 best_alpha = np.argmax(scores)/100
cm_probabilities = best_alpha*m+(1-best_alpha)*m2
cm_predictions = np.argmax(cm_probabilities, axis=-1)
 else:
best_alpha = 0.54<compute_test_metric>
train['TimeHandoff'] = train['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeSnap'] = train['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
if not SKIP_VALIDATION:
cmat = confusion_matrix(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)))
 score = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 precision = precision_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 recall = recall_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 display_confusion_matrix(cmat, score, precision, recall)
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))<save_to_csv>
train['TimeDelta'] = train.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1 )
test_ds = get_test_dataset(ordered=True)
 print('Computing predictions...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilities = best_alpha*model.predict(test_images_ds)+(1-best_alpha)*model2.predict(test_images_ds)
 predictions = np.argmax(probabilities, axis=-1)
 print(predictions)
 print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )<define_variables>
train['PlayerBirthDate'] = train['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
GCS_DS_PATH = KaggleDatasets().get_gcs_path()<set_options>
seconds_in_year = 60*60*24*365.25
train['PlayerAge'] = train.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1 )
print('TensorFlow version: %s' % tf.__version__)
 print('Keras version: %s' % tf.keras.__version__)
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("TPU is not available")
 tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
train = train.drop(['TimeHandoff', 'TimeSnap', 'PlayerBirthDate'], axis=1 )
IMAGE_SIZE = [512, 512]
IMAGE_CHANNELS = 3
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
GCS_PATH_SELECT = {
192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',
224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',
331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',
512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'
}
GCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')
 VALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec' )<define_variables>
train['WindSpeed'].value_counts()
TENSOR_BOARD = False
VERBOSE = 0
IMAGE_AUGMENTATION = True
GRID_MASK = False
USE_CLASS_WEIGHTS = False
MODEL_NAME = 6 * ['InceptionV3']
WEIGHTS_NAME = 6 * ['imagenet']
selected_optimizer = 'adam'
selected_loss = 'sparse_categorical_crossentropy'
selected_metrics = 'sparse_categorical_accuracy'
EPOCHS = 30
FOLDS = len(MODEL_NAME)
 SEED = 92
MODEL_FILENAMES = TRAINING_FILENAMES + VALIDATION_FILENAMES<define_variables>
train['WindSpeed'] = train['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x )
CLASSES = [
'pink primrose', 'hard-leaved pocket orchid',
'canterbury bells', 'sweet pea',
'wild geranium', 'tiger lily',
'moon orchid', 'bird of paradise',
'monkshood', 'globe thistle',
'snapdragon', "colt's foot",
'king protea', 'spear thistle',
'yellow iris', 'globe-flower',
'purple coneflower', 'peruvian lily',
'balloon flower', 'giant white arum lily',
'fire lily', 'pincushion flower',
'fritillary', 'red ginger',
'grape hyacinth', 'corn poppy',
'prince of wales feathers', 'stemless gentian',
'artichoke', 'sweet william',
'carnation', 'garden phlox',
'love in the mist', 'cosmos',
'alpine sea holly', 'ruby-lipped cattleya',
'cape flower', 'great masterwort',
'siam tulip', 'lenten rose',
'barberton daisy', 'daffodil',
'sword lily', 'poinsettia',
'bolero deep blue', 'wallflower',
'marigold', 'buttercup',
'daisy', 'common dandelion',
'petunia', 'wild pansy',
'primula', 'sunflower',
'lilac hibiscus', 'bishop of llandaff',
'gaura', 'geranium',
'orange dahlia', 'pink-yellow dahlia',
'cautleya spicata', 'japanese anemone',
'black-eyed susan', 'silverbush',
'californian poppy', 'osteospermum',
'spring crocus', 'iris',
'windflower', 'tree poppy',
'gazania', 'azalea',
'water lily', 'rose',
'thorn apple', 'morning glory',
'passion flower', 'lotus',
'toad lily', 'anthurium',
'frangipani', 'clematis',
'hibiscus', 'columbine',
'desert-rose', 'tree mallow',
'magnolia', 'cyclamen ',
'watercress', 'canna lily',
'hippeastrum ', 'bee balm',
'pink quill', 'foxglove',
'bougainvillea', 'camellia',
'mallow', 'mexican petunia',
'bromelia', 'blanket flower',
'trumpet creeper', 'blackberry lily',
'common tulip', 'wild rose']<normalization>
train['WindSpeed'].value_counts()
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 rotation_matrix = tf.reshape(
 tf.concat([
c1, s1, zero,
-s1, c1, zero,
zero,zero, one],
axis=0),
[3,3] 
)
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(
 tf.concat([
one, s2, zero,
zero, c2, zero,
zero,zero, one],
axis=0),
[3,3] 
)
 zoom_matrix = tf.reshape(
 tf.concat([
one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one],
axis=0),
[3,3] 
)
 shift_matrix = tf.reshape(
 tf.concat([
one, zero, height_shift,
zero, one, width_shift,
zero, zero, one],
axis=0),
[3,3] 
)
 return K.dot(
 K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def data_rotate(image,label):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
rot = 15.* tf.random.normal([1],dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image,tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3]),label<categorify>
train['WindSpeed'] = train['WindSpeed'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 train['WindSpeed'] = train['WindSpeed'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x )
AUTO = tf.data.experimental.AUTOTUNE
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, IMAGE_CHANNELS])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else
read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def data_augment(image, label):
image = tf.image.random_flip_left_right(image)
 return image, label
def get_training_dataset(dataset, do_aug=False, do_grid=False):
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 if do_aug: dataset = dataset.map(data_rotate, num_parallel_calls=AUTO)
 if do_grid: dataset = dataset.map(data_gridmask, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(dataset):
dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)
 NUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.
 format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))<categorify>
def str_to_float(txt):
try:
return float(txt)
 except:
return -1
def read_tfrecord_label(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 label = tf.cast(example['class'], tf.int32)
 return label
def get_class_weights(filenames):
count = count_data_items(filenames)
 dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.map(read_tfrecord_label, num_parallel_calls=AUTO)
 labels = next(iter(dataset.batch(count)) ).numpy()
return class_weight.compute_class_weight('balanced', [x for x in range(len(CLASSES)) ], labels )<choose_model_class>
train['WindSpeed'] = train['WindSpeed'].apply(str_to_float )
def model_select(name, weights = 'imagenet'):
if name == 'EfficientNetB7':
try:
except ImportError:
!pip install -q efficientnet
pretrained_model = efn.EfficientNetB7(
 weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, IMAGE_CHANNELS])
 pretrained_model.trainable = True
return tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 elif name == 'InceptionV3':
pretrained_model = tf.keras.applications.InceptionV3(
 weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, IMAGE_CHANNELS])
 pretrained_model.trainable = True
return tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(4096, activation='relu'),
tf.keras.layers.Dense(4096, activation='relu'),
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 elif name == 'DenseNet201':
pretrained_model = tf.keras.applications.DenseNet201(
 weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, IMAGE_CHANNELS])
 pretrained_model.trainable = True
return tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 elif name == 'ResNet152V2':
pretrained_model = tf.keras.applications.ResNet152V2(
 weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, IMAGE_CHANNELS])
 pretrained_model.trainable = True
return tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(4096, activation='relu'),
tf.keras.layers.Dense(4096, activation='relu'),
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 elif name == 'VGG16':
pretrained_model = tf.keras.applications.VGG16(
 weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, IMAGE_CHANNELS])
 pretrained_model.trainable = True
return tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(4096, activation='relu'),
tf.keras.layers.Dense(4096, activation='relu'),
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 else:
print('Unknown model: %s' % name)
 raise
def create_model(name, weights = 'imagenet'):
with strategy.scope() :
model = model_select(name, weights)
 model.compile(
 optimizer = selected_optimizer,
loss = selected_loss,
metrics=[selected_metrics] 
)
 return model<set_options>
train['WindDirection'].value_counts()
def save_config() :
config = {
"IMAGE_SIZE" : IMAGE_SIZE,
"IMAGE_CHANNELS" : IMAGE_CHANNELS,
"BATCH_SIZE" : BATCH_SIZE,
"MODEL_NAME" : MODEL_NAME,
"WEIGHTS_NAME" : WEIGHTS_NAME,
"EPOCHS" : EPOCHS,
"FOLDS" : FOLDS,
"TENSOR_BOARD" : TENSOR_BOARD,
"VERBOSE" : VERBOSE,
"IMAGE_AUGMENTATION" : IMAGE_AUGMENTATION,
"GRID_MASK" : GRID_MASK,
"USE_CLASS_WEIGHTS" : USE_CLASS_WEIGHTS,
"SEED" : SEED
}
with open('config.json', 'w')as outfile:
json.dump(config, outfile)
 save_config()<init_hyperparams>
def clean_WindDirection(txt):
if pd.isna(txt):
return np.nan
txt = txt.lower()
txt = ''.join([c for c in txt if c not in punctuation])
 txt = txt.replace('from', '')
 txt = txt.replace(' ', '')
 txt = txt.replace('north', 'n')
 txt = txt.replace('south', 's')
 txt = txt.replace('west', 'w')
 txt = txt.replace('east', 'e')
 return txt
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 4
LR_SUSTAIN_EPOCHS = 6
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch -
LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
!rm -rf./logs/
log_dir = "logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
 def callbacks(ifold, board = False):
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = False)
 if board:
tensorboard_callback = tf.keras.callbacks.TensorBoard(
 log_dir=log_dir + f"/fold{ifold}", histogram_freq=1)
 return [lr_callback, tensorboard_callback]
else:
return [lr_callback]
kfold = KFold(FOLDS, shuffle = True, random_state = SEED)
 CLASS_WEIGHTS = get_class_weights(MODEL_FILENAMES)if USE_CLASS_WEIGHTS else None
test_ds = get_test_dataset(ordered=True)
 def train_cross_validate_predict(filenames):
print(f'Start training {FOLDS} folds {EPOCHS} epochs, img_aug {IMAGE_AUGMENTATION} grid {GRID_MASK} class weights {USE_CLASS_WEIGHTS}')
 histories = []
for ifold,(trn_ind, val_ind)in enumerate(kfold.split(filenames)) :
tdf = list(pd.DataFrame({'FILENAMES': filenames} ).loc[trn_ind]['FILENAMES'])
 vdf = list(pd.DataFrame({'FILENAMES': filenames} ).loc[val_ind]['FILENAMES'])
 steps_per_epoch = count_data_items(tdf)// BATCH_SIZE
train_dataset = load_dataset(tdf, labeled = True)
 val_dataset = load_dataset(vdf, labeled = True, ordered = True)
 validation_data = get_validation_dataset(val_dataset)
 print_and_log(f'
model = create_model(MODEL_NAME[ifold], WEIGHTS_NAME[ifold])
 history = model.fit(
 get_training_dataset(train_dataset, IMAGE_AUGMENTATION, GRID_MASK),
steps_per_epoch = steps_per_epoch,
epochs = EPOCHS,
callbacks = callbacks(ifold, TENSOR_BOARD),
validation_data = validation_data,
class_weight = CLASS_WEIGHTS,
verbose = VERBOSE 
)
 test_images_ds = test_ds.map(lambda image, idnum: image)
 predict = model.predict(test_images_ds)
 np.save(f'fold{ifold+1}.npy', predict)
 cm_predict = model.predict(validation_data)
 np.save(f'cm_fold{ifold+1}.npy', cm_predict)
 histories.append(history)
 del model, train_dataset, val_dataset, validation_data, predict, cm_predict
gc.collect()
if tpu:
tf.tpu.experimental.initialize_tpu_system(tpu)
 return histories<predict_on_test>
train['WindDirection'] = train['WindDirection'].apply(clean_WindDirection )
histories = train_cross_validate_predict(MODEL_FILENAMES )<save_to_csv>
train['WindDirection'].value_counts()
def get_predictions(preds):
print('Computing predictions...')
 probabilities = np.average([preds[i] for i in range(len(preds)) ], axis = 0)
 predictions = np.argmax(probabilities, axis=-1)
 print(predictions)
 return predictions
def save_prediction(predictions):
print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]),
fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )<define_variables>
def transform_WindDirection(txt):
if pd.isna(txt):
return np.nan
if txt=='n':
return 0
if txt=='nne' or txt=='nen':
return 1/8
if txt=='ne':
return 2/8
if txt=='ene' or txt=='nee':
return 3/8
if txt=='e':
return 4/8
if txt=='ese' or txt=='see':
return 5/8
if txt=='se':
return 6/8
if txt=='ses' or txt=='sse':
return 7/8
if txt=='s':
return 8/8
if txt=='ssw' or txt=='sws':
return 9/8
if txt=='sw':
return 10/8
if txt=='sww' or txt=='wsw':
return 11/8
if txt=='w':
return 12/8
if txt=='wnw' or txt=='nww':
return 13/8
if txt=='nw':
return 14/8
if txt=='nwn' or txt=='nnw':
return 15/8
return np.nan
GCS_DS_PATH = KaggleDatasets().get_gcs_path()<set_options>
train['WindDirection'] = train['WindDirection'].apply(transform_WindDirection )
print('TensorFlow version: %s' % tf.__version__)
 print('Keras version: %s' % tf.keras.__version__)
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("TPU is not available")
 tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
train['PlayDirection'].value_counts()
IMAGE_SIZE = [512, 512]
IMAGE_CHANNELS = 3
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
GCS_PATH_SELECT = {
192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',
224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',
331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',
512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'
}
GCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')
 VALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec' )<define_variables>
train['PlayDirection'] = train['PlayDirection'].apply(lambda x: x.strip() == 'right' )
TENSOR_BOARD = False
VERBOSE = 0
IMAGE_AUGMENTATION = True
GRID_MASK = False
USE_CLASS_WEIGHTS = False
MODEL_NAME = 6 * ['InceptionV3']
WEIGHTS_NAME = 6 * ['imagenet']
selected_optimizer = 'adam'
selected_loss = 'sparse_categorical_crossentropy'
selected_metrics = 'sparse_categorical_accuracy'
EPOCHS = 30
FOLDS = len(MODEL_NAME)
 SEED = 92
MODEL_FILENAMES = TRAINING_FILENAMES + VALIDATION_FILENAMES<define_variables>
train['Team'] = train['Team'].apply(lambda x: x.strip() =='home' )
CLASSES = [
'pink primrose', 'hard-leaved pocket orchid',
'canterbury bells', 'sweet pea',
'wild geranium', 'tiger lily',
'moon orchid', 'bird of paradise',
'monkshood', 'globe thistle',
'snapdragon', "colt's foot",
'king protea', 'spear thistle',
'yellow iris', 'globe-flower',
'purple coneflower', 'peruvian lily',
'balloon flower', 'giant white arum lily',
'fire lily', 'pincushion flower',
'fritillary', 'red ginger',
'grape hyacinth', 'corn poppy',
'prince of wales feathers', 'stemless gentian',
'artichoke', 'sweet william',
'carnation', 'garden phlox',
'love in the mist', 'cosmos',
'alpine sea holly', 'ruby-lipped cattleya',
'cape flower', 'great masterwort',
'siam tulip', 'lenten rose',
'barberton daisy', 'daffodil',
'sword lily', 'poinsettia',
'bolero deep blue', 'wallflower',
'marigold', 'buttercup',
'daisy', 'common dandelion',
'petunia', 'wild pansy',
'primula', 'sunflower',
'lilac hibiscus', 'bishop of llandaff',
'gaura', 'geranium',
'orange dahlia', 'pink-yellow dahlia',
'cautleya spicata', 'japanese anemone',
'black-eyed susan', 'silverbush',
'californian poppy', 'osteospermum',
'spring crocus', 'iris',
'windflower', 'tree poppy',
'gazania', 'azalea',
'water lily', 'rose',
'thorn apple', 'morning glory',
'passion flower', 'lotus',
'toad lily', 'anthurium',
'frangipani', 'clematis',
'hibiscus', 'columbine',
'desert-rose', 'tree mallow',
'magnolia', 'cyclamen ',
'watercress', 'canna lily',
'hippeastrum ', 'bee balm',
'pink quill', 'foxglove',
'bougainvillea', 'camellia',
'mallow', 'mexican petunia',
'bromelia', 'blanket flower',
'trumpet creeper', 'blackberry lily',
'common tulip', 'wild rose']<normalization>
train['GameWeather'] = train['GameWeather'].str.lower()
indoor = "indoor"
train['GameWeather'] = train['GameWeather'].apply(lambda x: indoor if not pd.isna(x)and indoor in x else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x )
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 rotation_matrix = tf.reshape(
 tf.concat([
c1, s1, zero,
-s1, c1, zero,
zero,zero, one],
axis=0),
[3,3] 
)
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(
 tf.concat([
one, s2, zero,
zero, c2, zero,
zero,zero, one],
axis=0),
[3,3] 
)
 zoom_matrix = tf.reshape(
 tf.concat([
one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one],
axis=0),
[3,3] 
)
 shift_matrix = tf.reshape(
 tf.concat([
one, zero, height_shift,
zero, one, width_shift,
zero, zero, one],
axis=0),
[3,3] 
)
 return K.dot(
 K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def data_rotate(image,label):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
rot = 15.* tf.random.normal([1],dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image,tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3]),label<categorify>
weather_count = Counter()
for weather in train['GameWeather']:
if pd.isna(weather):
continue
for word in weather.split() :
weather_count[word]+=1
weather_count.most_common() [:15]
AUTO = tf.data.experimental.AUTOTUNE
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, IMAGE_CHANNELS])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else
read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def data_augment(image, label):
image = tf.image.random_flip_left_right(image)
 return image, label
def get_training_dataset(dataset, do_aug=False, do_grid=False):
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 if do_aug: dataset = dataset.map(data_rotate, num_parallel_calls=AUTO)
 if do_grid: dataset = dataset.map(data_gridmask, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(dataset):
dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)
 NUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.
 format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))<categorify>
def map_weather(txt):
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt:
ans*=0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*3
if 'sunny' in txt or 'sun' in txt:
return ans*2
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt:
return -2*ans
if 'snow' in txt:
return -3*ans
return 0
def read_tfrecord_label(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 label = tf.cast(example['class'], tf.int32)
 return label
def get_class_weights(filenames):
count = count_data_items(filenames)
 dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.map(read_tfrecord_label, num_parallel_calls=AUTO)
 labels = next(iter(dataset.batch(count)) ).numpy()
return class_weight.compute_class_weight('balanced', [x for x in range(len(CLASSES)) ], labels )<choose_model_class>
train['GameWeather'] = train['GameWeather'].apply(map_weather )
def model_select(name, weights = 'imagenet'):
if name == 'EfficientNetB7':
try:
except ImportError:
!pip install -q efficientnet
pretrained_model = efn.EfficientNetB7(
 weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, IMAGE_CHANNELS])
 pretrained_model.trainable = True
return tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 elif name == 'InceptionV3':
pretrained_model = tf.keras.applications.InceptionV3(
 weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, IMAGE_CHANNELS])
 pretrained_model.trainable = True
return tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(4096, activation='relu'),
tf.keras.layers.Dense(4096, activation='relu'),
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 elif name == 'DenseNet201':
pretrained_model = tf.keras.applications.DenseNet201(
 weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, IMAGE_CHANNELS])
 pretrained_model.trainable = True
return tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 elif name == 'ResNet152V2':
pretrained_model = tf.keras.applications.ResNet152V2(
 weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, IMAGE_CHANNELS])
 pretrained_model.trainable = True
return tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(4096, activation='relu'),
tf.keras.layers.Dense(4096, activation='relu'),
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 elif name == 'VGG16':
pretrained_model = tf.keras.applications.VGG16(
 weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, IMAGE_CHANNELS])
 pretrained_model.trainable = True
return tf.keras.Sequential([
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(4096, activation='relu'),
tf.keras.layers.Dense(4096, activation='relu'),
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 else:
print('Unknown model: %s' % name)
 raise
def create_model(name, weights = 'imagenet'):
with strategy.scope() :
model = model_select(name, weights)
 model.compile(
 optimizer = selected_optimizer,
loss = selected_loss,
metrics=[selected_metrics] 
)
 return model<set_options>
train['IsRusher'] = train['NflId'] == train['NflIdRusher']
def save_config() :
config = {
"IMAGE_SIZE" : IMAGE_SIZE,
"IMAGE_CHANNELS" : IMAGE_CHANNELS,
"BATCH_SIZE" : BATCH_SIZE,
"MODEL_NAME" : MODEL_NAME,
"WEIGHTS_NAME" : WEIGHTS_NAME,
"EPOCHS" : EPOCHS,
"FOLDS" : FOLDS,
"TENSOR_BOARD" : TENSOR_BOARD,
"VERBOSE" : VERBOSE,
"IMAGE_AUGMENTATION" : IMAGE_AUGMENTATION,
"GRID_MASK" : GRID_MASK,
"USE_CLASS_WEIGHTS" : USE_CLASS_WEIGHTS,
"SEED" : SEED
}
with open('config.json', 'w')as outfile:
json.dump(config, outfile)
 save_config()<init_hyperparams>
train.drop(['NflId', 'NflIdRusher'], axis=1, inplace=True )
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 4
LR_SUSTAIN_EPOCHS = 6
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch -
LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
!rm -rf./logs/
log_dir = "logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
 def callbacks(ifold, board = False):
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = False)
 if board:
tensorboard_callback = tf.keras.callbacks.TensorBoard(
 log_dir=log_dir + f"/fold{ifold}", histogram_freq=1)
 return [lr_callback, tensorboard_callback]
else:
return [lr_callback]
kfold = KFold(FOLDS, shuffle = True, random_state = SEED)
 CLASS_WEIGHTS = get_class_weights(MODEL_FILENAMES)if USE_CLASS_WEIGHTS else None
test_ds = get_test_dataset(ordered=True)
 def train_cross_validate_predict(filenames):
print(f'Start training {FOLDS} folds {EPOCHS} epochs, img_aug {IMAGE_AUGMENTATION} grid {GRID_MASK} class weights {USE_CLASS_WEIGHTS}')
 histories = []
for ifold,(trn_ind, val_ind)in enumerate(kfold.split(filenames)) :
tdf = list(pd.DataFrame({'FILENAMES': filenames} ).loc[trn_ind]['FILENAMES'])
 vdf = list(pd.DataFrame({'FILENAMES': filenames} ).loc[val_ind]['FILENAMES'])
 steps_per_epoch = count_data_items(tdf)// BATCH_SIZE
train_dataset = load_dataset(tdf, labeled = True)
 val_dataset = load_dataset(vdf, labeled = True, ordered = True)
 validation_data = get_validation_dataset(val_dataset)
 print_and_log(f'
model = create_model(MODEL_NAME[ifold], WEIGHTS_NAME[ifold])
 history = model.fit(
 get_training_dataset(train_dataset, IMAGE_AUGMENTATION, GRID_MASK),
steps_per_epoch = steps_per_epoch,
epochs = EPOCHS,
callbacks = callbacks(ifold, TENSOR_BOARD),
validation_data = validation_data,
class_weight = CLASS_WEIGHTS,
verbose = VERBOSE 
)
 test_images_ds = test_ds.map(lambda image, idnum: image)
 predict = model.predict(test_images_ds)
 np.save(f'fold{ifold+1}.npy', predict)
 cm_predict = model.predict(validation_data)
 np.save(f'cm_fold{ifold+1}.npy', cm_predict)
 histories.append(history)
 del model, train_dataset, val_dataset, validation_data, predict, cm_predict
gc.collect()
if tpu:
tf.tpu.experimental.initialize_tpu_system(tpu)
 return histories<predict_on_test>
train['X'] = train.apply(lambda row: row['X'] if row['PlayDirection'] else 120-row['X'], axis=1 )
histories = train_cross_validate_predict(MODEL_FILENAMES )<save_to_csv>
def new_orientation(angle, play_direction):
if play_direction == 0:
new_angle = 360.0 - angle
if new_angle == 360.0:
new_angle = 0.0
return new_angle
else:
return angle
train['Orientation'] = train.apply(lambda row: new_orientation(row['Orientation'], row['PlayDirection']), axis=1)
 train['Dir'] = train.apply(lambda row: new_orientation(row['Dir'], row['PlayDirection']), axis=1 )
def get_predictions(preds):
print('Computing predictions...')
 probabilities = np.average([preds[i] for i in range(len(preds)) ], axis = 0)
 predictions = np.argmax(probabilities, axis=-1)
 print(predictions)
 return predictions
def save_prediction(predictions):
print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]),
fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )<import_modules>
train['YardsLeft'] = train.apply(lambda row: 100-row['YardLine'] if row['HomeField'] else row['YardLine'], axis=1)
 train['YardsLeft'] = train.apply(lambda row: row['YardsLeft'] if row['PlayDirection'] else 100-row['YardsLeft'], axis=1 )
print(tf.__version__)
<load_from_csv>
((train['YardsLeft']<train['Yards'])|(train['YardsLeft']-100>train['Yards'])).mean()
train_data = pd.read_csv('.. /input/digit-recognizer/train.csv')
 test_data = pd.read_csv('.. /input/digit-recognizer/test.csv')
 print("train data shape: {}".format(train_data.shape))
print("test data shape: {}".format(test_data.shape))
X, Y = train_data.drop(['label'], axis=1), train_data['label']
del train_data<categorify>
train.drop(train.index[(train['YardsLeft']<train['Yards'])|(train['YardsLeft']-100>train['Yards'])], inplace=True )
X = X/255.0
test_features = test_data/255.0
Y = keras.utils.to_categorical(Y, num_classes=10)
 X = X.values.reshape(-1,28,28,1)
 test_features = test_features.values.reshape(-1,28,28,1 )<split>
train = train.sort_values(by=['PlayId', 'Team', 'IsRusher', 'JerseyNumber'] ).reset_index()
X_train, X_val, y_train, y_val = train_test_split(X, Y, test_size = 0.1, random_state=2 )<train_model>
train.drop(['GameId', 'PlayId', 'index', 'IsRusher', 'Team'], axis=1, inplace=True )
datagen = keras.preprocessing.image.ImageDataGenerator(rotation_range=15,
width_shift_range=0.1,
zoom_range=0.1,
height_shift_range=0.1)
 datagen.fit(X_train )<choose_model_class>
cat_features = []
for col in train.columns:
if train[col].dtype =='object':
cat_features.append(col)
 train = train.drop(cat_features, axis=1 )
input_shape =(28, 28, 1)
 model = keras.Sequential()
model.add(keras.layers.Conv2D(filters=32, kernel_size=(5, 5), activation="relu", input_shape=input_shape, padding="Same"))
model.add(keras.layers.Conv2D(filters=32, kernel_size=(5, 5), activation="relu", padding="Same"))
model.add(keras.layers.Dropout(0.25))
model.add(keras.layers.MaxPooling2D(( 2, 2)))
 model.add(keras.layers.Conv2D(filters=64, kernel_size=(3, 3), activation="relu", padding="Same"))
model.add(keras.layers.Conv2D(filters=64, kernel_size=(3, 3), activation="relu", padding="Same"))
model.add(keras.layers.Dropout(0.25))
model.add(keras.layers.MaxPooling2D(( 2, 2)))
 model.add(keras.layers.Flatten())
 model.add(keras.layers.Dense(256, activation='relu'))
model.add(keras.layers.Dropout(0.2))
model.add(keras.layers.Dense(10))
model.summary()<train_model>
train.fillna(-999, inplace=True )
adam_opt = keras.optimizers.Adam(learning_rate=0.0008, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=False)
 model.compile(optimizer=adam_opt,
loss=tf.keras.losses.CategoricalCrossentropy(from_logits=True),
metrics = ['accuracy'])
 history = model.fit_generator(datagen.flow(X_train, y_train, batch_size=96),
steps_per_epoch=len(X_train)/96,
epochs=30,
validation_data=(X_val, y_val),
validation_freq=1,
shuffle=True )<save_to_csv>
X_train = np.array(train[players_col] ).reshape(-1, len(players_col)*22 )
submission = pd.DataFrame()
submission['ImageId'] = pd.Series(range(1,28001))
submission['Label'] = results
submission.to_csv('cnn_submission.csv',index=False )<choose_model_class>
play_col = train.drop(players_col+['Yards'], axis=1 ).columns
X_play_col = np.zeros(shape=(X_train.shape[0], len(play_col)))
 for i, col in enumerate(play_col):
X_play_col[:, i] = train[col][::22]
conv = Conv2D(filters=32, kernel_size=3, strides=1,
padding="SAME" )<choose_model_class>
X_train = np.concatenate([X_train, X_play_col], axis=1)
 y_train = np.zeros(shape=(X_train.shape[0], 199))
for i,yard in enumerate(train['Yards'][::22]):
y_train[i, yard+99:] = np.ones(shape=(1, 100-yard))
max_pool = MaxPool2D(pool_size=2 )<choose_model_class>
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train )
global_avg_pool = GlobalAvgPool2D()<import_modules>
batch_size=64
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, confusion_matrix
from keras.utils.np_utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D
from keras.optimizers import SGD
from keras.callbacks import ReduceLROnPlateau
from functools import partial<load_from_csv>
class RAdam(keras.optimizers.Optimizer):

 def __init__(self, learning_rate=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., weight_decay=0., amsgrad=False,
total_steps=0, warmup_proportion=0.1, min_lr=0., **kwargs):
learning_rate = kwargs.pop('lr', learning_rate)
 super(RAdam, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.learning_rate = K.variable(learning_rate, name='learning_rate')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.weight_decay = K.variable(weight_decay, name='weight_decay')
 self.total_steps = K.variable(total_steps, name='total_steps')
 self.warmup_proportion = K.variable(warmup_proportion, name='warmup_proportion')
 self.min_lr = K.variable(min_lr, name='min_lr')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.initial_weight_decay = weight_decay
self.initial_total_steps = total_steps
self.amsgrad = amsgrad
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * K.cast(self.iterations, K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx())+ 1
if self.initial_total_steps > 0:
warmup_steps = self.total_steps * self.warmup_proportion
decay_steps = K.maximum(self.total_steps - warmup_steps, 1)
 decay_rate =(self.min_lr - lr)/ decay_steps
lr = K.switch(
 t <= warmup_steps,
lr *(t / warmup_steps),
lr + decay_rate * K.minimum(t - warmup_steps, decay_steps), 
)
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for(i, p)in enumerate(params)]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for(i, p)in enumerate(params)]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for(i, p)in enumerate(params)]
else:
vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params)) ]
self.weights = [self.iterations] + ms + vs + vhats
beta_1_t = K.pow(self.beta_1, t)
 beta_2_t = K.pow(self.beta_2, t)
 sma_inf = 2.0 /(1.0 - self.beta_2)- 1.0
sma_t = sma_inf - 2.0 * t * beta_2_t /(1.0 - beta_2_t)
 for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 m_corr_t = m_t /(1.0 - beta_1_t)
 if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 v_corr_t = K.sqrt(vhat_t /(1.0 - beta_2_t))
self.updates.append(K.update(vhat, vhat_t))
else:
v_corr_t = K.sqrt(v_t /(1.0 - beta_2_t))
r_t = K.sqrt(( sma_t - 4.0)/(sma_inf - 4.0)* 
(sma_t - 2.0)/(sma_inf - 2.0)*
sma_inf / sma_t)
 p_t = K.switch(sma_t >= 5, r_t * m_corr_t /(v_corr_t + self.epsilon), m_corr_t)
 if self.initial_weight_decay > 0:
p_t += self.weight_decay * p
p_t = p - lr * p_t
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
@property
def lr(self):
return self.learning_rate
@lr.setter
def lr(self, learning_rate):
self.learning_rate = learning_rate
def get_config(self):
config = {
'learning_rate': float(K.get_value(self.learning_rate)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.weight_decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad,
'total_steps': float(K.get_value(self.total_steps)) ,
'warmup_proportion': float(K.get_value(self.warmup_proportion)) ,
'min_lr': float(K.get_value(self.min_lr)) ,
}
base_config = super(RAdam, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
train = pd.read_csv(".. /input/digit-recognizer/train.csv")
 test = pd.read_csv(".. /input/digit-recognizer/test.csv")
 print(train.shape)
 print(test.shape )<prepare_x_and_y>
def crps(y_true, y_pred):
return K.mean(K.square(y_true - K.cumsum(y_pred, axis=1)) , axis=1 )
Y_train = train["label"]
X_train = train.drop(labels = ["label"],axis = 1)
 X_test = test
Y_train.value_counts()<feature_engineering>
def get_model() :
x = keras.layers.Input(shape=[X_train.shape[1]])
 fc1 = keras.layers.Dense(units=450, input_shape=[X_train.shape[1]] )(x)
 act1 = keras.layers.PReLU()(fc1)
 bn1 = keras.layers.BatchNormalization()(act1)
 dp1 = keras.layers.Dropout(0.55 )(bn1)
 gn1 = keras.layers.GaussianNoise(0.15 )(dp1)
 concat1 = keras.layers.Concatenate()([x, gn1])
 fc2 = keras.layers.Dense(units=600 )(concat1)
 act2 = keras.layers.PReLU()(fc2)
 bn2 = keras.layers.BatchNormalization()(act2)
 dp2 = keras.layers.Dropout(0.55 )(bn2)
 gn2 = keras.layers.GaussianNoise(0.15 )(dp2)
 concat2 = keras.layers.Concatenate()([concat1, gn2])
 fc3 = keras.layers.Dense(units=400 )(concat2)
 act3 = keras.layers.PReLU()(fc3)
 bn3 = keras.layers.BatchNormalization()(act3)
 dp3 = keras.layers.Dropout(0.55 )(bn3)
 gn3 = keras.layers.GaussianNoise(0.15 )(dp3)
 concat3 = keras.layers.Concatenate([concat2, gn3])
 output = keras.layers.Dense(units=199, activation='softmax' )(concat2)
 model = keras.models.Model(inputs=[x], outputs=[output])
 return model
def train_model(X_train, y_train, X_val, y_val):
model = get_model()
model.compile(optimizer=RAdam(warmup_proportion=0.1, min_lr=1e-7), loss=crps)
 er = EarlyStopping(patience=20, min_delta=1e-4, restore_best_weights=True, monitor='val_loss')
 model.fit(X_train, y_train, epochs=200, callbacks=[er], validation_data=[X_val, y_val], batch_size=batch_size)
 return model
X_train = X_train / 255.0
X_test = X_test / 255.0<categorify>
rkf = RepeatedKFold(n_splits=5, n_repeats=5)
 models = []
for tr_idx, vl_idx in rkf.split(X_train, y_train):
x_tr, y_tr = X_train[tr_idx], y_train[tr_idx]
x_vl, y_vl = X_train[vl_idx], y_train[vl_idx]
model = train_model(x_tr, y_tr, x_vl, y_vl)
 models.append(model )
print(Y_train[0:5])
 Y_train = to_categorical(Y_train, num_classes = 10)
 print(Y_train[0:5] )<split>
def make_pred(df, sample, env, models):
df['StadiumType'] = df['StadiumType'].apply(clean_StadiumType)
 df['StadiumType'] = df['StadiumType'].apply(transform_StadiumType)
 df['DefendersInTheBox_vs_Distance'] = df['DefendersInTheBox'] / df['Distance']
df['OffenseFormation'] = df['OffenseFormation'].apply(lambda x: x if x in off_form else np.nan)
 df = pd.concat([df.drop(['OffenseFormation'], axis=1), pd.get_dummies(df['OffenseFormation'], prefix='Formation')], axis=1)
 missing_cols = set(dummy_col)- set(df.columns)-set('Yards')
 for c in missing_cols:
df[c] = 0
df = df[dummy_col]
df.drop(['Yards'], axis=1, inplace=True)
 df['Turf'] = df['Turf'].map(Turf)
 df['Turf'] = df['Turf'] == 'Natural'
df['PossessionTeam'] = df['PossessionTeam'].map(map_abbr)
 df['HomeTeamAbbr'] = df['HomeTeamAbbr'].map(map_abbr)
 df['VisitorTeamAbbr'] = df['VisitorTeamAbbr'].map(map_abbr)
 df['HomePossesion'] = df['PossessionTeam'] == df['HomeTeamAbbr']
df['Field_eq_Possession'] = df['FieldPosition'] == df['PossessionTeam']
df['HomeField'] = df['FieldPosition'] == df['HomeTeamAbbr']
df['Formation_Heavy_Score'] = df["OffensePersonnel"].apply(formation_numeric)
 df['GameClock'] = df['GameClock'].apply(strtoseconds)
 df['PlayerHeight'] = df['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
df['PlayerBMI'] = 703*(df['PlayerWeight']/(df['PlayerHeight'])**2)
 df['TimeHandoff'] = df['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeSnap'] = df['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeDelta'] = df.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 df['PlayerBirthDate'] = df['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
df['PlayerAge'] = df.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 df['WindSpeed'] = df['WindSpeed'].apply(str_to_float)
 df['WindDirection'] = df['WindDirection'].apply(clean_WindDirection)
 df['WindDirection'] = df['WindDirection'].apply(transform_WindDirection)
 df['PlayDirection'] = df['PlayDirection'].apply(lambda x: x.strip() == 'right')
 df['Team'] = df['Team'].apply(lambda x: x.strip() =='home')
 indoor = "indoor"
df['GameWeather'] = df['GameWeather'].apply(lambda x: indoor if not pd.isna(x)and indoor in x else x)
 df['GameWeather'] = df['GameWeather'].apply(lambda x: x.lower().replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly' ).replace('clear and sunny', 'sunny and clear' ).replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 df['GameWeather'] = df['GameWeather'].apply(map_weather)
 df['IsRusher'] = df['NflId'] == df['NflIdRusher']
df['X'] = df.apply(lambda row: row['X'] if row['PlayDirection'] else 120-row['X'], axis=1)
 df['Orientation'] = df.apply(lambda row: new_orientation(row['Orientation'], row['PlayDirection']), axis=1)
 df['Dir'] = df.apply(lambda row: new_orientation(row['Dir'], row['PlayDirection']), axis=1)
 df['YardsLeft'] = df.apply(lambda row: 100-row['YardLine'] if row['HomeField'] else row['YardLine'], axis=1)
 df['YardsLeft'] = df.apply(lambda row: row['YardsLeft'] if row['PlayDirection'] else 100-row['YardsLeft'], axis=1)
 df = df.sort_values(by=['PlayId', 'Team', 'IsRusher', 'JerseyNumber'] ).reset_index()
df = df.drop(['TimeHandoff', 'TimeSnap', 'PlayerBirthDate', 'NflId', 'NflIdRusher', 'GameId', 'PlayId', 'index', 'IsRusher', 'Team'], axis=1)
 cat_features = []
for col in df.columns:
if df[col].dtype =='object':
cat_features.append(col)
 df = df.drop(cat_features, axis=1)
 df.fillna(-999, inplace=True)
 X = np.array(df[players_col] ).reshape(-1, len(players_col)*22)
 play_col = df.drop(players_col, axis=1 ).columns
X_play_col = np.zeros(shape=(X.shape[0], len(play_col)))
 for i, col in enumerate(play_col):
X_play_col[:, i] = df[col][::22]
X = np.concatenate([X, X_play_col], axis=1)
 X = scaler.transform(X)
 y_pred = np.mean([np.cumsum(model.predict(X), axis=1)for model in models], axis=0)
 yardsleft = np.array(df['YardsLeft'][::22])
 for i in range(len(yardsleft)) :
y_pred[i, :yardsleft[i]-1] = 0
y_pred[i, yardsleft[i]+100:] = 1
env.predict(pd.DataFrame(data=y_pred.clip(0,1),columns=sample.columns))
return y_pred
random_seed = 2
X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.15, random_state=random_seed )<choose_model_class>
for test, sample in tqdm.tqdm(env.iter_test()):
make_pred(test, sample, env, models )
<choose_model_class><EOS> 
env.write_submission_file()