Kaggle competition: Petals to the Metal - Flower Classification on TPU
<SOS> metric: macrofscore Kaggle data source: petals-to-the-metal-flower-classification-on-tpu<categorify>
!pip install -q efficientnet
filterwarnings('ignore')
 le = preprocessing.LabelEncoder()
countries = X_Train.Country.unique()
df_out = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in countries:
states = X_Train.loc[X_Train.Country == country, :].State.unique()
for state in states:
X_Train_CS = X_Train.loc[(X_Train.Country == country)&(X_Train.State == state), ['State', 'Country', 'Date', 'ConfirmedCases', 'Fatalities']]
y1_Train_CS = X_Train_CS.loc[:, 'ConfirmedCases']
y2_Train_CS = X_Train_CS.loc[:, 'Fatalities']
X_Train_CS = X_Train_CS.loc[:, ['State', 'Country', 'Date']]
X_Train_CS.Country = le.fit_transform(X_Train_CS.Country)
 X_Train_CS['State'] = le.fit_transform(X_Train_CS['State'])
 X_Test_CS = X_Test.loc[(X_Test.Country == country)&(X_Test.State == state), ['State', 'Country', 'Date', 'ForecastId']]
X_Test_CS_Id = X_Test_CS.loc[:, 'ForecastId']
X_Test_CS = X_Test_CS.loc[:, ['State', 'Country', 'Date']]
X_Test_CS.Country = le.fit_transform(X_Test_CS.Country)
 X_Test_CS['State'] = le.fit_transform(X_Test_CS['State'])
 model1 = XGBRegressor(n_estimators=1000)
 model1.fit(X_Train_CS, y1_Train_CS)
 y1_pred = model1.predict(X_Test_CS)
 model2 = XGBRegressor(n_estimators=1000)
 model2.fit(X_Train_CS, y2_Train_CS)
 y2_pred = model2.predict(X_Test_CS)
 df = pd.DataFrame({'ForecastId': X_Test_CS_Id, 'ConfirmedCases': y1_pred, 'Fatalities': y2_pred})
 df_out = pd.concat([df_out, df], axis=0)
<save_to_csv>
from tensorflow.keras.applications.vgg16 import VGG16
from tensorflow.keras.applications.vgg19 import VGG19
from tensorflow.keras.applications.inception_v3 import InceptionV3
from tensorflow.keras.applications.inception_resnet_v2 import InceptionResNetV2
from tensorflow.keras.applications.densenet import DenseNet121, DenseNet169, DenseNet201
from tensorflow.keras.applications.xception import Xception
from tensorflow.keras.applications.resnet50 import ResNet50
from tensorflow.keras.applications.resnet_v2 import ResNet50V2, ResNet101V2, ResNet152V2
from tensorflow.keras.applications.nasnet import NASNetLarge
from efficientnet.tfkeras import EfficientNetB7, EfficientNetL2, EfficientNetB0
df_out.ForecastId = df_out.ForecastId.astype('int')
 df_out.tail()
df_out.to_csv('submission.csv', index=False )<load_from_csv>
%matplotlib inline
print("Tensorflow version " + tf.__version__ )
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/train.csv')
 test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/test.csv')
 submission = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/test.csv' )<count_unique_values>
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
print("Number of Country_Region: ", train['Country_Region'].nunique())
 print("Dates go from day", max(train['Date']), "to day", min(train['Date']), ", a total of", train['Date'].nunique() , "days")
 print("Countries with Province/State informed: ", train[train['Province_State'].isna() ==False]['Country_Region'].unique() )<feature_engineering>
IMAGE_SIZE = [512, 512]
EPOCHS = 30
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
SEED = 2020
NUM_TRAINING_IMAGES = 12753
NUM_TEST_IMAGES = 7382
STEPS_PER_EPOCH =(NUM_TRAINING_IMAGES // BATCH_SIZE)* 2
EMPTY_VAL = "EMPTY_VAL"
def fillState(state, country):
if state == EMPTY_VAL: return country
return state
train['Province_State'].fillna(EMPTY_VAL, inplace=True)
 train['Province_State'] = train.loc[:, ['Province_State', 'Country_Region']].apply(lambda x : fillState(x['Province_State'], x['Country_Region']), axis=1)
 test['Province_State'].fillna(EMPTY_VAL, inplace=True)
 test['Province_State'] = test.loc[:, ['Province_State', 'Country_Region']].apply(lambda x : fillState(x['Province_State'], x['Country_Region']), axis=1)
 test.head()<prepare_x_and_y>
def decode_image(image_data):

 image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):

 ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord)
 return dataset
def data_augment(image, label):
flag = random.randint(1,3)
 coef_1 = random.randint(75, 95)* 0.01
coef_2 = random.randint(75, 95)* 0.01
if flag == 1:
image = tf.image.random_flip_left_right(image, seed=SEED)
 elif flag == 2:
image = tf.image.random_flip_up_down(image, seed=SEED)
 else:
image = tf.image.random_crop(image, [int(IMAGE_SIZE[0]*coef_1), int(IMAGE_SIZE[0]*coef_2), 3],seed=SEED)
 return image, label
def get_training_dataset() :
dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + f'/tfrecords-jpeg-{IMAGE_SIZE[0]}x{IMAGE_SIZE[0]}/train/*.tfrec'), labeled=True)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(-1)
 return dataset
def get_validation_dataset() :
dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + f'/tfrecords-jpeg-{IMAGE_SIZE[0]}x{IMAGE_SIZE[0]}/val/*.tfrec'), labeled=True, ordered=False)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(-1)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + f'/tfrecords-jpeg-{IMAGE_SIZE[0]}x{IMAGE_SIZE[0]}/test/*.tfrec'), labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(-1)
 return dataset
training_dataset = get_training_dataset()
validation_dataset = get_validation_dataset()
train['row_number'] = train.groupby(['Country_Region', 'Province_State'] ).cumcount()
x = train[train["Country_Region"] == 'China'][train["Province_State"] == 'Hubei']['row_number']
y = train[train["Country_Region"] == 'China'][train["Province_State"] == 'Hubei']['ConfirmedCases']
y_ = train[train["Country_Region"] == 'China'][train["Province_State"] == 'Hubei']['Fatalities']
def f(x, L, b, k, x_0):
return L /(1.+ np.exp(-k *(x - x_0)))+ b
def logistic(xs, L, k, x_0):
result = []
for x in xs:
xp = k*(x-x_0)
 if xp >= 0:
result.append(L /(1.+ np.exp(-xp)))
 else:
result.append(L * np.exp(xp)/(1.+ np.exp(xp)))
 return result
p0 = [max(y), 0.0,max(x)]
p0_ = [max(y_), 0.0,max(x)]
x_ = np.arange(0, 100, 1 ).tolist()
try:
popt, pcov = opt.curve_fit(logistic, x, y,p0)
 yfit = logistic(x_, *popt)
 popt_, pcov_ = opt.curve_fit(logistic, x, y_,p0_)
 yfit_ = logistic(x_, *popt_)
 except:
popt, pcov = opt.curve_fit(f, x, y, method="lm", maxfev=5000)
 yfit = f(x_, *popt)
 popt_, pcov_ = opt.curve_fit(f, x, y_, method="lm", maxfev=5000)
 yfit_ = f(x_, *popt_)
 fig, ax = plt.subplots(1, 1, figsize=(10, 8))
ax.plot(x, y, 'o', label ='Actual Cases')
 ax.plot(x_, yfit, '-', label ='Fitted Cases')
 ax.plot(x, y_, 'o', label ='Actual Fatalities')
 ax.plot(x_, yfit_, '-', label ='Fitted fatalities')
 ax.title.set_text('China - Hubei province')
 plt.legend(loc="center right")
 plt.show()<prepare_output>
LR_START = 0.00001
LR_MAX = 0.0001
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 8
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))
unique = pd.DataFrame(train.groupby(['Country_Region', 'Province_State'],as_index=False ).count())
 unique.head()<prepare_x_and_y>
def get_model(use_model):
base_model = use_model(weights='noisy-student',
include_top=False, pooling='avg',
input_shape=(*IMAGE_SIZE, 3))
x = base_model.output
predictions = Dense(104, activation='softmax' )(x)
 return Model(inputs=base_model.input, outputs=predictions)
 with strategy.scope() :
model = get_model(EfficientNetB0)
 model.compile(
 optimizer='nadam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 tf.keras.utils.plot_model(
 model, to_file='model.png', show_shapes=True, show_layer_names=True,
)
def date_day_diff(d1, d2):
delta = dt.datetime.strptime(d1, "%Y-%m-%d")- dt.datetime.strptime(d2, "%Y-%m-%d")
 return delta.days
log_regions = []
for index, region in unique.iterrows() :
st = region['Province_State']
co = region['Country_Region']
rdata = train[(train['Province_State']==st)&(train['Country_Region']==co)]
t = rdata['Date'].values
t = [float(date_day_diff(d, t[0])) for d in t]
y = rdata['ConfirmedCases'].values
y_ = rdata['Fatalities'].values
p0 = [max(y), 0.0, max(t)]
p0_ = [max(y_), 0.0, max(t)]
try:
popt, pcov = opt.curve_fit(logistic, t, y, p0, maxfev=10000)
 try:
popt_, pcov_ = opt.curve_fit(logistic, t, y_, p0_, maxfev=10000)
 except:
popt_, pcov_ = opt.curve_fit(f, t, y_,method="trf", maxfev=10000)
 log_regions.append(( co,st,popt,popt_))
except:
popt, pcov = opt.curve_fit(f, t, y,method="trf", maxfev=10000)
 popt_, pcov_ = opt.curve_fit(f, t, y_,method="trf", maxfev=10000)
 log_regions.append(( co,st,popt,popt_))<prepare_output>
history = model.fit(training_dataset,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[lr_callback, ModelCheckpoint(filepath='my_ef_net_b7.h5', monitor='val_loss',
save_best_only=True)],
validation_data=validation_dataset )
log_regions = pd.DataFrame(log_regions)
 log_regions.head()<rename_columns>
model = tf.keras.models.load_model('my_ef_net_b7.h5' )
<define_variables><EOS> 
test_ds = get_test_dataset(ordered=True)
 print('Вычисляем предсказания...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilities = model.predict(test_images_ds)
 predictions = np.argmax(probabilities, axis=-1)
 print(predictions)
 print('Создание файла submission.csv...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )