Kaggle competition: Petals to the Metal - Flower Classification on TPU
dftest.ForecastId = dftest.ForecastId.astype('int')
 dftest['ConfirmedCases'] = dftest['ConfirmedCases'].round().astype(int)
 dftest['Fatalities'] = dftest['Fatalities'].round().astype(int)
 dftest.to_csv('submission.csv', index=False)
<import_modules>
import math, re, os
import numpy as np
import tensorflow as tf
from sklearn.preprocessing import StandardScaler, MinMaxScaler, OneHotEncoder
from sklearn.model_selection import train_test_split, KFold, cross_val_score, GridSearchCV
from sklearn import preprocessing
import lightgbm as lgb
import optuna
import glob<define_variables>
print(tf.__version__ )
path = '.. /input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/'
Files = 'WDataFiles_Stage1/'<load_from_csv>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
TourneyCompactResults = pd.read_csv(path + Files + 'WNCAATourneyCompactResults.csv')
 GameCities = pd.read_csv(path+Files+'WGameCities.csv')
 Seasons = pd.read_csv(path+Files+'WSeasons.csv')
 TourneySeeds = pd.read_csv(path+Files+'WNCAATourneySeeds.csv')
 RegularSeasonCompactResults = pd.read_csv(path+Files+'WRegularSeasonCompactResults.csv' )<load_from_csv>
GCS_DS_PATH=KaggleDatasets().get_gcs_path('tpu-getting-started')
 print(GCS_DS_PATH )
test= pd.read_csv(path +'WSampleSubmissionStage1_2020.csv' )<feature_engineering>
IMAGE_SIZE=[512,512]
GCS_PATH=GCS_DS_PATH+'/tfrecords-jpeg-512x512'
AUTO=tf.data.experimental.AUTOTUNE
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')
 VALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')
 CLASSES = ['pink primrose', 'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea', 'wild geranium', 'tiger lily', 'moon orchid', 'bird of paradise', 'monkshood', 'globe thistle',
'snapdragon', "colt's foot", 'king protea', 'spear thistle', 'yellow iris', 'globe-flower', 'purple coneflower', 'peruvian lily', 'balloon flower', 'giant white arum lily',
'fire lily', 'pincushion flower', 'fritillary', 'red ginger', 'grape hyacinth', 'corn poppy', 'prince of wales feathers', 'stemless gentian', 'artichoke', 'sweet william',
'carnation', 'garden phlox', 'love in the mist', 'cosmos', 'alpine sea holly', 'ruby-lipped cattleya', 'cape flower', 'great masterwort', 'siam tulip', 'lenten rose',
'barberton daisy', 'daffodil', 'sword lily', 'poinsettia', 'bolero deep blue', 'wallflower', 'marigold', 'buttercup', 'daisy', 'common dandelion',
'petunia', 'wild pansy', 'primula', 'sunflower', 'lilac hibiscus', 'bishop of llandaff', 'gaura', 'geranium', 'orange dahlia', 'pink-yellow dahlia',
'cautleya spicata', 'japanese anemone', 'black-eyed susan', 'silverbush', 'californian poppy', 'osteospermum', 'spring crocus', 'iris', 'windflower', 'tree poppy',
'gazania', 'azalea', 'water lily', 'rose', 'thorn apple', 'morning glory', 'passion flower', 'lotus', 'toad lily', 'anthurium',
'frangipani', 'clematis', 'hibiscus', 'columbine', 'desert-rose', 'tree mallow', 'magnolia', 'cyclamen ', 'watercress', 'canna lily',
'hippeastrum ', 'bee balm', 'pink quill', 'foxglove', 'bougainvillea', 'camellia', 'mallow', 'mexican petunia', 'bromelia', 'blanket flower',
'trumpet creeper', 'blackberry lily', 'common tulip', 'wild rose']
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data,channels=3)
 image = tf.cast(image,tf.float32)/255.0
image=tf.reshape(image,[*IMAGE_SIZE,3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([],tf.string),
"class": tf.io.FixedLenFeature([],tf.int64)
 }
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image,label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([],tf.string),
"id": tf.io.FixedLenFeature([],tf.string)
 }
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order=tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic =False
dataset=tf.data.TFRecordDataset(filenames,num_parallel_reads=AUTO)
 dataset=dataset.with_options(ignore_order)
 dataset=dataset.map(read_labeled_tfrecord if labeled else
read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
TourneySeeds['Seed'] = TourneySeeds['Seed'].apply(lambda x: int(x[1:3]))
print(TourneySeeds.shape)
 TourneySeeds.head()<categorify>
BATCH_SIZE = 16*strategy.num_replicas_in_sync
ds_train = get_training_dataset()
ds_valid = get_validation_dataset()
ds_test = get_test_dataset()
print("Training: ",ds_train)
 print("Validation: ",ds_valid)
 print("Test: ",ds_test )
train = train.replace({'H':0,'A':1,'N':2})
 train.head()<categorify>
np.set_printoptions(threshold=15, linewidth=80)
 print("Training data shapes:")
 for image,label in ds_train.take(3):
print(image.numpy().shape,label.numpy().shape)
 print("Training data label examples:",label.numpy() )
le = preprocessing.LabelEncoder()
for column in ['CRType']:
le.fit(GameCities[column])
 GameCities[column] = le.transform(GameCities[column])
 GameCities.head()<categorify>
print("Test data shapes:")
 for image,idnum in ds_test.take(3):
print(image.numpy().shape,idnum.numpy().shape)
 print("Test data IDs:",idnum.numpy().astype('U'))
le = preprocessing.LabelEncoder()
for column in ['RegionW','RegionX','RegionY','RegionZ']:
le.fit(Seasons[column])
 Seasons[column] = le.transform(Seasons[column])
 for i in range(0,23):
print(Seasons['DayZero'][i].split('/'))
Seasons['ZeroMonth'] = Seasons['DayZero'][i].split('/')[0]
Seasons['ZeroDay'] = Seasons['DayZero'][i].split('/')[1]
Seasons['ZeroYear'] = Seasons['DayZero'][i].split('/')[2]
Seasons = Seasons.drop('DayZero',axis=1)
 Seasons['ZeroMonth'] = Seasons['ZeroMonth'].astype(int)
 Seasons['ZeroDay'] = Seasons['ZeroMonth'].astype(int)
 Seasons['ZeroYear'] = Seasons['ZeroMonth'].astype(int)
 Seasons.head()<merge>
def batch_to_numpy_images_and_labels(data):
images, labels = data
numpy_images = images.numpy()
numpy_labels = labels.numpy()
if numpy_labels.dtype ==object:
numpy_labels = [None for _ in enumerate(numpy_images)]
return numpy_images, numpy_labels
def title_from_label_and_target(label, correct_label):
if correct_label is None:
return CLASSES[label], True
correct =(label == correct_label)
 return "{} [{}{}{}]".format(CLASSES[label], 'OK' if correct else
'NO', u"\u2192" if not correct else '',
CLASSES[correct_label] if not correct else ''), correct
def display_one_flower(image, title, subplot, red=False, titlesize=16):
plt.subplot(*subplot)
 plt.axis('off')
 plt.imshow(image)
 if len(title)>0:
plt.title(title, fontsize=int(titlesize)if not red
else int(titlesize/1.2), color ='red' if red else
'black',fontdict={'verticalalignment':'center'},
pad = int(titlesize/1.5))
return(subplot[0],subplot[1],subplot[2]+1)
 def display_batch_of_images(databatch, predictions=None):

 images,labels = batch_to_numpy_images_and_labels(databatch)
 if labels is None:
labels = [None for _ in enumerate(images)]
rows=int(math.sqrt(len(images)))
 cols =len(images)//rows
FIGSIZE = 13.0
SPACING = 0.1
subplot =(rows,cols,1)
 if rows < cols:
plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))
else:
plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))
for i,(image, label)in enumerate(zip(images[:rows*cols],labels[:rows*cols])) :
title = '' if label is None else CLASSES[label]
correct = True
if predictions is not None:
title, correct =title_from_label_and_target(predictions[i],label)
 dynamic_titlesize=FIGSIZE*SPACING/max(rows,cols)*40+3
subplot=display_one_flower(image, title, subplot,
not correct, titlesize=dynamic_titlesize)
 plt.tight_layout()
if label is None and predictions is None:
plt.subplots_adjust(wspce=0, hspace=0)
 else:
plt.subplots_adjust(wspace=0, hspace=SPACING)
 plt.show()
def display_training_curves(training, validation, title, subplot):
if subplot%10==1:
plt.subplots(figsize=(10,10),facecolor='
plt.tight_layout()
ax=plt.subplot(subplot)
 ax.set_facecolor('
ax.plot(training)
 ax.plot(validation)
 ax.set_title('model' + title)
 ax.set_ylabel(title)
 ax.set_xlabel('epoch')
 ax.legend(['train','valid'] )
train = train.merge(Seasons, how='left',on=['Season'])
 train.head()<merge>
ds_iter = iter(ds_train.unbatch().batch(20))
train = train.merge(TourneySeeds, how='left', left_on=['Season', 'WTeamID'], right_on=['Season','TeamID'])
 train = train.drop('TeamID',axis=1)
 train = train.rename(columns={'Seed': 'WSeed'})
 train = train.merge(TourneySeeds, how='left', left_on=['Season', 'LTeamID'], right_on=['Season','TeamID'])
 train = train.drop('TeamID',axis=1)
 train = train.rename(columns={'Seed': 'LSeed'})
 train.head()<feature_engineering>
dataset = load_dataset(TRAINING_FILENAMES, labeled=True)
 one_element = tf.data.Dataset.from_tensors(next(iter(dataset)))
 augmented_element = one_element.repeat().map(data_augment ).batch(25)
 display_batch_of_images(next(iter(augmented_element)) )
test = test.drop(['Pred'], axis=1)
 test['Season'] = test['ID'].apply(lambda x: int(x.split('_')[0]))
test['WTeamID'] = test['ID'].apply(lambda x: int(x.split('_')[1]))
test['LTeamID'] = test['ID'].apply(lambda x: int(x.split('_')[2]))
test.head()<merge>
EPOCHS=20
def exponential_lr(epoch,
start_lr=0.00001,min_lr=0.00001,max_lr=0.00005,
rampup_epochs = 5, sustain_epochs = 0,
exp_decay = 0.8):
def lr(epoch, start_lr, min_lr,max_lr,rampup_epochs,sustain_epochs,
exp_decay):
if epoch < rampup_epochs:
lr=(( max_lr-start_lr)/
rampup_epochs * epoch + start_lr)
 elif epoch < rampup_epochs + sustain_epochs:
lr = max_lr
else:
lr =(( max_lr - min_lr)* exp_decay **(epoch-rampup_epochs-sustain_epochs)
 + min_lr)
 return lr
return lr(epoch,start_lr,min_lr,max_lr,rampup_epochs,sustain_epochs,exp_decay)
 lr_callback = tf.keras.callbacks.LearningRateScheduler(exponential_lr,verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [exponential_lr(x)for x in rng]
plt.plot(rng,y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0],
max(y),
y[-1]))
test = test.merge(TourneyCompactResults,how='left',on=['Season','WTeamID','LTeamID'] )<merge>
EPOCHS = 20
with strategy.scope() :
pretained_model=tf.keras.applications.EfficientNetB7(
 weights='imagenet',
include_top=False,
input_shape=[*IMAGE_SIZE,3] 
)
 pretained_model.trainable = True
model = tf.keras.Sequential([
pretained_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(len(CLASSES),activation='softmax')] )
test = test.replace({'H':0,'A':1,'N':2})
 test = test.merge(Seasons, how='left',on=['Season'])
 test = test.merge(TourneySeeds, how='left', left_on=['Season', 'WTeamID'], right_on=['Season','TeamID'])
 test = test.drop('TeamID',axis=1)
 test = test.rename(columns={'Seed': 'WSeed'})
 test = test.merge(TourneySeeds, how='left', left_on=['Season', 'LTeamID'], right_on=['Season','TeamID'])
 test = test.drop('TeamID',axis=1)
 test = test.rename(columns={'Seed': 'LSeed'})
<merge>
model.compile(loss='sparse_categorical_crossentropy',optimizer='adam',metrics=['sparse_categorical_accuracy'])
 model.summary()
test.merge(test,how='left',on=['ID','Season','WTeamID','LTeamID'] )<drop_column>
earlystopping=EarlyStopping(monitor='val_loss',patience=2 )
not_exist_in_test = [c for c in train.columns.values.tolist() if c not in test.columns.values.tolist() ]
print(not_exist_in_test)
 train = train.drop(not_exist_in_test, axis=1)
 train.head()<groupby>
EPOCHS = 20
STEPS_PER_EPOCH = NUM_TRAINING_IMAGES//BATCH_SIZE
history = model.fit(ds_train,validation_data=ds_valid,
epochs=EPOCHS,
steps_per_epoch=STEPS_PER_EPOCH,callbacks=[lr_callback,earlystopping]
)
team_win_score = RegularSeasonCompactResults.groupby(['Season', 'WTeamID'] ).agg({'WScore':['sum', 'count', 'var']} ).reset_index()
team_win_score.columns = [' '.join(col ).strip() for col in team_win_score.columns.values]
team_loss_score = RegularSeasonCompactResults.groupby(['Season', 'LTeamID'] ).agg({'LScore':['sum', 'count', 'var']} ).reset_index()
team_loss_score.columns = [' '.join(col ).strip() for col in team_loss_score.columns.values]
print(team_win_score.shape)
 team_win_score.head()<merge>
cmdataset=get_validation_dataset(ordered=True)
 image_ds=cmdataset.map(lambda image, label:image)
 labels_ds=cmdataset.map(lambda image, label:label ).unbatch()
cm_correct_labels=next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES)) ).numpy()
cm_probabilities=model.predict(image_ds)
 cm_predictions = np.argmax(cm_probabilities,axis=-1)
 labels = range(len(CLASSES))
cmat = confusion_matrix(
 cm_correct_labels,
cm_predictions,
labels=labels,)
 cmat=(cmat.T/cmat.sum(axis=1)).T
train = pd.merge(train, team_win_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'WTeamID'])
 train = pd.merge(train, team_loss_score, how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'LTeamID'])
 train = pd.merge(train, team_loss_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'LTeamID'])
 train = pd.merge(train, team_win_score, how='left', left_on=['Season', 'LTeamID_x'], right_on=['Season', 'WTeamID'])
 train.drop(['LTeamID_y', 'WTeamID_y'], axis=1, inplace=True)
 train.head()<merge>
score=f1_score(cm_correct_labels,cm_predictions,labels=labels,average='macro')
 precision=precision_score(cm_correct_labels,cm_predictions,labels=labels,average='macro')
 recall=recall_score(cm_correct_labels,cm_predictions,labels=labels,average='macro')
 display_confusion_matrix(cmat,score,precision,recall )
test = pd.merge(test, team_win_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'WTeamID'])
 test = pd.merge(test, team_loss_score, how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'LTeamID'])
 test = pd.merge(test, team_loss_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'LTeamID'])
 test = pd.merge(test, team_win_score, how='left', left_on=['Season', 'LTeamID_x'], right_on=['Season', 'WTeamID'])
 test.drop(['LTeamID_y', 'WTeamID_y'], axis=1, inplace=True)
 test.head() 
<feature_engineering>
dataset=get_validation_dataset()
dataset=dataset.unbatch().batch(20)
 batch=iter(dataset )
def preprocess(df):
df['x_score'] = df['WScore sum_x'] + df['LScore sum_y']
df['y_score'] = df['WScore sum_y'] + df['LScore sum_x']
df['x_count'] = df['WScore count_x'] + df['LScore count_y']
df['y_count'] = df['WScore count_y'] + df['WScore count_x']
df['x_var'] = df['WScore var_x'] + df['LScore count_y']
df['y_var'] = df['WScore var_y'] + df['WScore var_x']
return df
train = preprocess(train)
 test = preprocess(test)
 test.shape<drop_column>
images, labels=next(batch)
 probabilities= model.predict(images)
 predictions=np.argmax(probabilities,axis=-1)
 display_batch_of_images(( images,labels),predictions )
train_win = train.copy()
train_los = train.copy()
train_win = train_win[['WSeed', 'LSeed',
'x_score', 'y_score', 'x_count', 'y_count', 'x_var', 'y_var']]
train_los = train_los[['LSeed', 'WSeed',
'y_score', 'x_score', 'x_count', 'y_count', 'x_var', 'y_var']]
train_win.columns = ['Seed_1', 'Seed_2',
'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']
train_los.columns = ['Seed_1', 'Seed_2',
'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']
test = test[['ID', 'WSeed', 'LSeed',
'x_score', 'y_score', 'x_count', 'y_count', 'x_var', 'y_var']]
test.columns = ['ID', 'Seed_1', 'Seed_2',
'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']<feature_engineering>
test_ds = get_test_dataset(ordered=True)
 print('Computing predictions...')
 test_images_ds=test_ds.map(lambda image, idnum: image)
 probabilities = model.predict(test_images_ds)
 predictions=np.argmax(probabilities,axis=-1)
 print(predictions )
def feature_engineering(df):
df['Seed_diff'] = df['Seed_1'] - df['Seed_2']
df['Score_diff'] = df['Score_1'] - df['Score_2']
df['Count_diff'] = df['Count_1'] - df['Count_2']
df['Var_diff'] = df['Var_1'] - df['Var_2']
df['Mean_score1'] = df['Score_1'] / df['Count_1']
df['Mean_score2'] = df['Score_2'] / df['Count_2']
df['Mean_score_diff'] = df['Mean_score1'] - df['Mean_score2']
df['FanoFactor_1'] = df['Var_1'] / df['Mean_score1']
df['FanoFactor_2'] = df['Var_2'] / df['Mean_score2']
return df
train_win = feature_engineering(train_win)
 train_los = feature_engineering(train_los)
 test = feature_engineering(test)
 test.shape<feature_engineering>
print('Generating submission.csv file...')
 test_ids_ds =test_ds.map(lambda image,idnum:idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv',np.rec.fromarrays([test_ids,predictions]),
fmt=['%s', '%d'],
delimiter=',',
header='id,label',
comments='',)
 !head submssion.csv
<concatenate><EOS> 
save_model = model.save('Efficienet_B7_model.h5' )