Kaggle competition: Petals to the Metal - Flower Classification on TPU
<SOS> metric: macrofscore Kaggle data source: petals-to-the-metal-flower-classification-on-tpu<drop_column>
import numpy as np
import pandas as pd
import tensorflow as tf
import matplotlib.pyplot as plt
from kaggle_datasets import KaggleDatasets
train_df = train_df.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
 test_df = test_df.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
 combine = [train_df, test_df]
train_df.head()<feature_engineering>
!pip install -q efficientnet
for dataset in combine:
dataset['Age*Class'] = dataset.Age * dataset.Pclass
train_df.loc[:, ['Age*Class', 'Age', 'Pclass']].head(10 )<correct_missing_values>
GCS_DS_PATH = KaggleDatasets().get_gcs_path('tpu-getting-started')
 print(GCS_DS_PATH )
freq_port = train_df.Embarked.dropna().mode() [0]
freq_port<sort_values>
IMAGE_SIZE = 512
EPOCHS = 35
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
NUM_TRAINING_IMAGES = 12753
NUM_TEST_IMAGES = 7382
STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].fillna(freq_port)
 train_df[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<data_type_conversions>
train_data = tf.data.TFRecordDataset(
 tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-' + str(IMAGE_SIZE)+ 'x' + str(IMAGE_SIZE)+ '/train/*.tfrec'),
num_parallel_reads = tf.data.experimental.AUTOTUNE
)
for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 train_df.head()<data_type_conversions>
ignore_order = tf.data.Options()
ignore_order.experimental_deterministic = False
train_data = train_data.with_options(ignore_order )
test_df['Fare'].fillna(test_df['Fare'].dropna().median() , inplace=True)
 test_df.head()<data_type_conversions>
def read_labeled_tfrecord(example):
tfrec_format = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, tfrec_format)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [IMAGE_SIZE, IMAGE_SIZE, 3])
 return image
for dataset in combine:
dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 31, 'Fare'] = 3
dataset['Fare'] = dataset['Fare'].astype(int)
 combine = [train_df, test_df]
train_df.head(10 )<prepare_x_and_y>
train_data = train_data.map(read_labeled_tfrecord )
X_train = train_df.drop("Survived", axis=1)
 Y_train = train_df["Survived"]
X_test = test_df.drop("PassengerId", axis=1 ).copy()
X_train.shape, Y_train.shape, X_test.shape<compute_train_metric>
train_data = train_data.map(augment, num_parallel_calls=tf.data.experimental.AUTOTUNE )
logreg = LogisticRegression()
logreg.fit(X_train, Y_train)
 Y_pred = logreg.predict(X_test)
 acc_log = round(logreg.score(X_train, Y_train)* 100, 2)
 acc_log<predict_on_test>
train_data = train_data.repeat()
train_data = train_data.shuffle(2048)
 train_data = train_data.batch(BATCH_SIZE)
 train_data = train_data.prefetch(tf.data.experimental.AUTOTUNE )
knn = KNeighborsClassifier(n_neighbors = 3)
 knn.fit(X_train, Y_train)
 Y_pred = knn.predict(X_test)
 acc_knn = round(knn.score(X_train, Y_train)* 100, 2)
 acc_knn<train_model>
val_data = tf.data.TFRecordDataset(
 tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-' + str(IMAGE_SIZE)+ 'x' + str(IMAGE_SIZE)+ '/val/*.tfrec'),
num_parallel_reads = tf.data.experimental.AUTOTUNE 
)
 val_data = val_data.with_options(ignore_order)
 val_data = val_data.map(read_labeled_tfrecord, num_parallel_calls = tf.data.experimental.AUTOTUNE)
 val_data = val_data.batch(BATCH_SIZE)
 val_data = val_data.cache()
val_data = val_data.prefetch(tf.data.experimental.AUTOTUNE )
model = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=1)
 model.fit(X_train, Y_train)
 predictions = model.predict(X_test)
 model.score(X_train, Y_train)
 acc_random_forest = round(model.score(X_train, Y_train)* 100, 2)
 acc_random_forest<save_to_csv>
with strategy.scope() :
enet = efn.EfficientNetB7(
 input_shape=(IMAGE_SIZE, IMAGE_SIZE, 3),
weights='imagenet',
include_top=False 
)
 enet.trainable = True
model = tf.keras.Sequential([
enet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(104, activation='softmax', dtype='float32')
 ] )
output = pd.DataFrame({'PassengerId': test_df.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Your second submission was successfully saved!" )<define_variables>
model.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy']
)
warnings.filterwarnings('ignore')
 daybasecount = 4
baseday = 98 - float(daybasecount-1)/2.
 exponent = 1./float(daybasecount)
 fatalityBaseDayShift = 10
maxincrease = 140
maxDeadPrDay = 1500<load_from_csv>
callbacks = [
tf.keras.callbacks.ReduceLROnPlateau(monitor='loss', patience=2, verbose=1),
tf.keras.callbacks.EarlyStopping(monitor='loss', patience=5, verbose=1, restore_best_weights=True),
]
dftrain = pd.read_csv('.. /input/covid19-global-forecasting-week-3/train.csv', parse_dates=['Date'] ).sort_values(by=['Country_Region', 'Date'])
 dftest = pd.read_csv('.. /input/covid19-global-forecasting-week-3/test.csv', parse_dates=['Date'] ).sort_values(by=['Country_Region', 'Date'])
 ppp_tabel = pd.read_csv('.. /input/country-ppp/Country_PPP.csv', sep='\s+')
 ppp_tabel.drop('Id', 1,inplace=True)
 ppp_tabel = ppp_tabel.append({'Country' : 'Burma' , 'ppp' : 8000} , ignore_index=True)
 ppp_tabel = ppp_tabel.append({'Country' : 'MS_Zaandam' , 'ppp' : 40000} , ignore_index=True)
 ppp_tabel = ppp_tabel.append({'Country' : 'West_Bank_and_Gaza' , 'ppp' : 20000} , ignore_index=True)
 ppp_tabel["Country"].replace('_',' ', regex=True,inplace=True)
 ppp_tabel["Country"].replace('United States','US', regex=True,inplace=True)
 ppp_tabel.rename(columns={'Country':'Country_Region'},inplace=True)
 ppp_tabel.sort_values('Country_Region',inplace=True )<merge>
history = model.fit(
 train_data,
validation_data = val_data,
steps_per_epoch = STEPS_PER_EPOCH,
epochs = EPOCHS,
callbacks = callbacks,
)
dftrain['Dayofyear'] = dftrain['Date'].dt.dayofyear
dftest['Dayofyear'] = dftest['Date'].dt.dayofyear
dftest['Expo'] = dftest['Dayofyear']-baseday
print(dftrain.tail(5))
dftest = dftest.merge(dftrain[['Country_Region','Province_State','Date','ConfirmedCases','Fatalities']]
, on=['Country_Region','Province_State','Date'], how='left', indicator=True)
<merge>
def read_unlabeled_tfrecord(example):
tfrec_format = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
grouped=dftrain.groupby(['Country_Region','Province_State'] ).tail(daybasecount*2)
 grouped=grouped.groupby(['Country_Region','Province_State'] ).head(daybasecount)
 grouped.drop(['FatalityBasis'],axis=1,inplace=True)
 to_sum = ['NewCases','NewFatalities']
grouped1 = grouped.groupby(['Country_Region'])[to_sum].sum()
grouped1.rename(columns={'NewCases':'NewCases1','NewFatalities':'NewFatalities1'}, inplace=True)
 grouped = pd.merge(grouped1, grouped2, on=['Country_Region'])
 grouped['CasesIncreasePct'] = 100*(grouped['NewCases2']/grouped['NewCases1']-1)
 mask = grouped['CasesIncreasePct'] > maxincrease
grouped.loc[mask,'CasesIncreasePct'] = maxincrease
mask = grouped['CasesIncreasePct'] < 0
grouped.loc[mask,'CasesIncreasePct'] = 0
mask = grouped['CasesIncreasePct'].isnull()
grouped.loc[mask,'CasesIncreasePct'] = 0
grouped['Factor'] =(grouped['CasesIncreasePct']/100+1)**exponent
grouped = pd.merge(grouped, ppp_tabel, on=['Country_Region'])
 grouped['ppp'] = grouped['ppp']/10000.
 if False:
mask =(grouped['FatalityPct2'] > 9)&(grouped['ppp'] <= 1)
 grouped.loc[mask,'FatalityPct2'] = 5
mask =(grouped['FatalityPct2'] < 5)&(grouped['ppp'] <= 1)
 grouped.loc[mask,'FatalityPct2'] = 5
mask =(grouped['FatalityPct2'] > 6)&(grouped['ppp'] >= 7)
 grouped.loc[mask,'FatalityPct2'] = 6
mask =(grouped['FatalityPct2'] < 1.5)&(grouped['ppp'] >= 7)
 grouped.loc[mask,'FatalityPct2'] = 1.5
mask =(grouped['FatalityPct2'] >(9.5 - 0.43*grouped['ppp'])) &(grouped['ppp'] > 1)&(grouped['ppp'] < 7)
 grouped.loc[mask,'FatalityPct2'] =(9.5 - 0.43*grouped['ppp'])
 mask =(grouped['FatalityPct2'] <(5.6 - 0.5*grouped['ppp'])) &(grouped['ppp'] > 1)&(grouped['ppp'] < 7)
 grouped.loc[mask,'FatalityPct2'] =(5.6 - 0.5*grouped['ppp'])
 mask =(grouped['FatalityPct2'].isnull())&(grouped['ppp'] <= 1)
 grouped.loc[mask,'FatalityPct2'] = 7
mask =(grouped['FatalityPct2'].isnull())&(grouped['ppp'] >= 7)
 grouped.loc[mask,'FatalityPct2'] = 4
mask =(grouped['FatalityPct2'].isnull())&(grouped['ppp'] > 1)&(grouped['ppp'] < 7)
 grouped.loc[mask,'FatalityPct2'] =(7.5 - 0.5*grouped['ppp'])
 else:
mask =(grouped['FatalityPct2'] > 4)&(grouped['ppp'] <= 1)
 grouped.loc[mask,'FatalityPct2'] = 3
mask =(grouped['FatalityPct2'] < 1)&(grouped['ppp'] <= 1)
 grouped.loc[mask,'FatalityPct2'] = 2
mask =(grouped['FatalityPct2'] > 1.5)&(grouped['ppp'] >= 7)
 grouped.loc[mask,'FatalityPct2'] = 1.5
mask =(grouped['FatalityPct2'] < 0.5)&(grouped['ppp'] >= 7)
 grouped.loc[mask,'FatalityPct2'] = 0.5
mask =(grouped['FatalityPct2'] >(4.5 - 0.43*grouped['ppp'])) &(grouped['ppp'] > 1)&(grouped['ppp'] < 7)
 grouped.loc[mask,'FatalityPct2'] =(4.5 - 0.43*grouped['ppp'])
 mask =(grouped['FatalityPct2'] <(1.1 - 0.1*grouped['ppp'])) &(grouped['ppp'] > 1)&(grouped['ppp'] < 7)
 grouped.loc[mask,'FatalityPct2'] =(1.1 - 0.1*grouped['ppp'])
 mask =(grouped['FatalityPct2'].isnull())&(grouped['ppp'] <= 1)
 grouped.loc[mask,'FatalityPct2'] = 3
mask =(grouped['FatalityPct2'].isnull())&(grouped['ppp'] >= 7)
 grouped.loc[mask,'FatalityPct2'] = 1
mask =(grouped['FatalityPct2'].isnull())&(grouped['ppp'] > 1)&(grouped['ppp'] < 7)
 grouped.loc[mask,'FatalityPct2'] =(2.6 - 0.23*grouped['ppp'] )<merge>
test_data = tf.data.TFRecordDataset(
 tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-' + str(IMAGE_SIZE)+ 'x' + str(IMAGE_SIZE)+ '/test/*.tfrec'),
num_parallel_reads = tf.data.experimental.AUTOTUNE 
)
 test_data = test_data.with_options(tf.data.Options())
 test_data = test_data.map(read_unlabeled_tfrecord, num_parallel_calls = tf.data.experimental.AUTOTUNE)
 test_data = test_data.batch(BATCH_SIZE)
 test_data = test_data.prefetch(tf.data.experimental.AUTOTUNE )
dftest.drop('_merge',axis=1,inplace= True)
 dftest = dftest.merge(grouped[['Country_Region','FatalityPct2','Factor']], on=['Country_Region'], how='left')
 dftest = dftest.merge(grouped_gem[['Province_State','Country_Region','ConfirmedCases_base','ConfirmedCases_init','NewCases_base','Fatalities_init','FatalityBasis']], on=['Province_State','Country_Region'], how='left')
<feature_engineering>
test_images = test_data.map(lambda image, idnum: image)
 probabilities = model.predict(test_images)
 predictions = np.argmax(probabilities, axis=-1 )
dftest['ConfirmedCases_shift'] = dftest.groupby(['Country_Region', 'Province_State'])[['ConfirmedCases']].transform(lambda x: x.shift(1))
mask = dftest['ConfirmedCases'].isnull()
dftest.loc[mask,'NewCases'] = dftest.loc[mask,'NewCases_base']*(dftest.loc[mask,'Factor']**dftest.loc[mask,'Expo'])
 dftest['NewCases_cum'] = dftest.groupby(['Country_Region', 'Province_State'])[['NewCases']].cumsum()
dftest.loc[mask,'ConfirmedCases'] = dftest.loc[mask,'ConfirmedCases_init'] + dftest.loc[mask,'NewCases_cum']
mask3 = dftest['ConfirmedCases'] > 400000
dftest.loc[mask3,'FatalityPct2'] = dftest.loc[mask3,'FatalityPct2']*0.7
mask4 = dftest['ConfirmedCases'] > 800000
dftest.loc[mask4,'FatalityPct2'] = dftest.loc[mask4,'FatalityPct2']*0.7
dftest['FatalityBasis'] = dftest.groupby(['Country_Region', 'Province_State'])[
['ConfirmedCases']].transform(lambda x: x.shift(10))
dftest.loc[mask,'NewFatalities'] = dftest.loc[mask,'FatalityBasis'] * dftest.loc[mask,'FatalityPct2']/100
mask2 = dftest['NewFatalities'] > maxDeadPrDay
dftest.loc[mask2,'NewFatalities'] = maxDeadPrDay
dftest['NewFatalities_cum'] = dftest.groupby(['Country_Region', 'Province_State'])[['NewFatalities']].cumsum()
dftest.loc[mask,'Fatalities'] = dftest.loc[mask,'Fatalities_init'] + dftest.loc[mask,'NewFatalities_cum'] 
<drop_column>
ids = []
for image, image_ids in test_data.take(NUM_TEST_IMAGES):
ids.append(image_ids.numpy())
 ids = np.concatenate(ids, axis=None ).astype(str )
dftest.drop(['Dayofyear',
'Expo','FatalityPct2', 'Factor',
'ConfirmedCases_base', 'ConfirmedCases_init',
'NewCases_base', 'Fatalities_init', 'FatalityBasis',
'ConfirmedCases_shift',
'NewCases', 'NewCases_cum', 'NewFatalities','NewFatalities_cum'],axis=1,inplace=True)
 final = dftest.groupby(['Country_Region','Province_State'] ).tail(1)
 dftest.drop(['Province_State'],axis=1,inplace=True)
 dftest.rename(columns={'Province_State_orig':'Province_State'},inplace=True )<count_missing_values>
submission = pd.DataFrame(data={'id': ids, 'label': predictions})
 submission.to_csv('submission.csv', index=False )
mask = dftest["ConfirmedCases"].isnull()
print(mask.sum())
 errors = dftest.loc[mask]
print(errors)
 mask = dftest["Fatalities"].isnull()
print(mask.sum())
 errors = dftest.loc[mask]
print(errors)
 dftest.drop(['Province_State','Country_Region','Date'],axis=1,inplace=True)
 print("dftest columns =",dftest.columns)
<save_to_csv>
model.save('model.h5' )