Kaggle competition: Petals to the Metal - Flower Classification on TPU
<SOS> metric: macrofscore Kaggle data source: petals-to-the-metal-flower-classification-on-tpu<prepare_output>
!pip install -q efficientnet
sample['target'] = preds<save_to_csv>
from tensorflow.keras.applications.vgg16 import VGG16
from tensorflow.keras.applications.vgg19 import VGG19
from tensorflow.keras.applications.inception_v3 import InceptionV3
from tensorflow.keras.applications.inception_resnet_v2 import InceptionResNetV2
from tensorflow.keras.applications.densenet import DenseNet121, DenseNet169, DenseNet201
from tensorflow.keras.applications.xception import Xception
from tensorflow.keras.applications.resnet50 import ResNet50
from tensorflow.keras.applications.resnet_v2 import ResNet50V2, ResNet101V2, ResNet152V2
from tensorflow.keras.applications.nasnet import NASNetLarge
from efficientnet.tfkeras import EfficientNetB7, EfficientNetL2, EfficientNetB0, EfficientNetB1
sample.to_csv('submission.csv', index=False )<load_from_csv>
%matplotlib inline
print("Tensorflow version " + tf.__version__ )
df = pd.read_csv('.. /input/44352/training_solutions_rev1.csv')
 df_train, df_test = train_test_split(df, test_size=.2)
 df_train.shape, df_test.shape<prepare_x_and_y>
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
%matplotlib inline
ORIG_SHAPE =(424,424)
 CROP_SIZE =(256,256)
 IMG_SHAPE =(64,64)
 def get_image(path, x1,y1, shape, crop_size):
x = plt.imread(path)
 x = x[x1:x1+crop_size[0], y1:y1+crop_size[1]]
x = resize(x, shape)
 x = x/255.
 return x
def get_all_images(dataframe, shape=IMG_SHAPE, crop_size=CROP_SIZE):
x1 =(ORIG_SHAPE[0]-CROP_SIZE[0])//2
y1 =(ORIG_SHAPE[1]-CROP_SIZE[1])//2
sel = dataframe.values
ids = sel[:,0].astype(int ).astype(str)
 y_batch = sel[:,1:]
x_batch = []
for i in tqdm(ids):
x = get_image('.. /input/44352/images_training_rev1/'+i+'.jpg', x1,y1, shape=shape, crop_size=crop_size)
 x_batch.append(x)
 x_batch = np.array(x_batch)
 return x_batch, y_batch
X_train, y_train = get_all_images(df_train)
 X_test, y_test = get_all_images(df_test )<choose_model_class>
IMAGE_SIZE = [512, 512]
EPOCHS = 30
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
GCS_PATH_SELECT = {
192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',
224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',
331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',
512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'
}
GCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')
 VALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')
 SEED = 2020
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true)))
 model = Sequential()
model.add(Conv2D(512,(3, 3), input_shape=(IMG_SHAPE[0], IMG_SHAPE[1], 3)))
 model.add(Conv2D(256,(3, 3)))
 model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(256,(3, 3)))
 model.add(Conv2D(128,(3, 3)))
 model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(128,(3, 3)))
 model.add(Conv2D(128,(3, 3)))
 model.add(Activation('relu'))
model.add(GlobalMaxPooling2D())
 model.add(Dropout(0.25))
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.25))
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.25))
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.25))
model.add(Dense(37))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adamax', metrics=[root_mean_squared_error])
 model.summary()<train_model>
def decode_image(image_data):

 image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):

 ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def data_augment(image, label):
flag = random.randint(1,3)
 coef_1 = random.randint(70, 90)* 0.01
coef_2 = random.randint(70, 90)* 0.01
if flag == 1:
image = tf.image.random_flip_left_right(image, seed=SEED)
 elif flag == 2:
image = tf.image.random_flip_up_down(image, seed=SEED)
 else:
image = tf.image.random_crop(image, [int(IMAGE_SIZE[0]*coef_1), int(IMAGE_SIZE[0]*coef_2), 3],seed=SEED)
 return image, label
def get_training_dataset() :
dataset = load_dataset(TRAINING_FILENAMES, labeled=True)
 dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset() :
dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=False)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)
 NUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))
batch_size = 128
model.fit(X_train, y_train, epochs=30, validation_data=(X_test, y_test))<define_variables>
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 8
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))
def test_image_generator(ids, shape=IMG_SHAPE):
x1 =(ORIG_SHAPE[0]-CROP_SIZE[0])//2
y1 =(ORIG_SHAPE[1]-CROP_SIZE[1])//2
x_batch = []
for i in ids:
x = get_image('.. /input/44352/images_test_rev1/'+i, x1, y1, shape=IMG_SHAPE, crop_size=CROP_SIZE)
 x_batch.append(x)
 x_batch = np.array(x_batch)
 return x_batch
val_files = os.listdir('.. /input/44352/images_test_rev1/')
 val_predictions = []
N_val = len(val_files)
 for i in tqdm(np.arange(0, N_val, batch_size)) :
if i+batch_size > N_val:
upper = N_val
else:
upper = i+batch_size
X = test_image_generator(val_files[i:upper])
 y_pred = model.predict(X)
 val_predictions.append(y_pred)
 val_predictions = np.array(val_predictions)
 Y_pred = np.vstack(val_predictions)
 ids = np.array([v.split('.')[0] for v in val_files] ).reshape(len(val_files),1)
 submission_df = pd.DataFrame(np.hstack(( ids, Y_pred)) , columns=df.columns)
 submission_df = submission_df.sort_values(by=['GalaxyID'])
 submission_df.to_csv('sample_submission.csv', index=False )<import_modules>
def get_model(use_model):
base_model = use_model(weights='imagenet',
include_top=False, pooling='avg',
input_shape=(*IMAGE_SIZE, 3))
x = base_model.output
predictions = Dense(104, activation='softmax' )(x)
 return Model(inputs=base_model.input, outputs=predictions)
 with strategy.scope() :
model = get_model(DenseNet201)
 model.compile(
 optimizer='nadam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
print("Read in libraries")
<load_from_csv>
history = model.fit(get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[lr_callback, ModelCheckpoint(filepath='my_densenet_201.h5', monitor='val_loss',
save_best_only=True)],
validation_data=get_validation_dataset() ,
workers = 3 )
PATH_WEEK2='/kaggle/input/covid19-global-forecasting-week-2'
df_train = pd.read_csv(f'{PATH_WEEK2}/train.csv')
 df_test = pd.read_csv(f'{PATH_WEEK2}/test.csv')
 df_train.head()
df_test.head()
df_train.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_test.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_train.rename(columns={'Province_State':'State'}, inplace=True)
 df_test.rename(columns={'Province_State':'State'}, inplace=True)
 df_train['Date'] = pd.to_datetime(df_train['Date'], infer_datetime_format=True)
 df_test['Date'] = pd.to_datetime(df_test['Date'], infer_datetime_format=True)
 df_train.info()
df_test.info()
y1_Train = df_train.iloc[:, -2]
y1_Train.head()
y2_Train = df_train.iloc[:, -1]
y2_Train.head()
EMPTY_VAL = "EMPTY_VAL"
def fillState(state, country):
if state == EMPTY_VAL: return country
return state<data_type_conversions>
model = tf.keras.models.load_model('my_densenet_201.h5' )
<categorify><EOS> 
test_ds = get_test_dataset(ordered=True)
 print('Вычисляем предсказания...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilities = model.predict(test_images_ds)
 predictions = np.argmax(probabilities, axis=-1)
 print(predictions)
 print('Создание файла submission.csv...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )