Kaggle competition: Petals to the Metal - Flower Classification on TPU
<SOS> metric: macrofscore Kaggle data source: petals-to-the-metal-flower-classification-on-tpu<data_type_conversions>
!pip install -U efficientnet
output=pd.DataFrame(output ).rename(index=str, columns={"index": "PassengerId", 0: "Survived"})
 output=output.reset_index()
output=output.rename(columns={"index":"PassengerId"})
 output["PassengerId"]=output["PassengerId"].astype("int")+892
output["Survived"]=(output[output.columns[output.shape[1]-1]]>0.5 ).apply(int)
 output=output[["PassengerId","Survived"]]<set_options>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from keras import backend as K
from efficientnet import keras as efn
from kaggle_datasets import KaggleDatasets
import re, math
shutil.rmtree("kaggle_pystacknet" )<save_to_csv>
tf.__version__
output.to_csv("results.csv",index=False,header=True )<import_modules>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
print("tpu")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
REPLICAS = strategy.num_replicas_in_sync
print("REPLICAS: ", REPLICAS )
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import Perceptron
from sklearn.linear_model import SGDClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPClassifier<load_from_csv>
BATCH_SIZE = 16 * REPLICAS
EPOCHS = 20
IMAGE_SIZE =(512, 512)
 TRAIN_IMG_NUM = 12753
VAL_IMG_NUM = 3712
TEST_IMG_NUM = 7382
STEPS_PER_EPOCHS = TRAIN_IMG_NUM // BATCH_SIZE
STEPS_PER_EPOCHS_FULL =(TRAIN_IMG_NUM + VAL_IMG_NUM)// BATCH_SIZE
AUTO = tf.data.experimental.AUTOTUNE
AugParams = {
'd1' : 100,
'd2': 160,
'rotate' : 45,
'ratio' : 0.5
}
data = pd.read_csv('.. /input/train.csv')
 data_test = pd.read_csv('.. /input/test.csv' )<define_variables>
GCS_DATA_PATH = KaggleDatasets().get_gcs_path()
def get_filenames(path):
return tf.io.gfile.glob(GCS_DATA_PATH + path)
 train_filenames = get_filenames("/tfrecords-jpeg-{0}x{1}/train/*.tfrec".format(*IMAGE_SIZE))
val_filenames = get_filenames("/tfrecords-jpeg-{0}x{1}/val/*.tfrec".format(*IMAGE_SIZE))
test_filenames = get_filenames("/tfrecords-jpeg-{0}x{1}/test/*.tfrec".format(*IMAGE_SIZE))
ID = data_test.PassengerId<prepare_x_and_y>
def load_dataset(filenames, ordered=True):
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=-1)
 ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
return dataset.with_options(ignore_order)
 train_records = load_dataset(train_filenames, False)
 val_records = load_dataset(val_filenames, False)
 test_records = load_dataset(test_filenames, True )
y_train = data.Survived<count_missing_values>
def decode_image(raw):

 decoded = tf.io.decode_jpeg(raw)
 image = tf.cast(decoded, tf.float32)/ 255.
 image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
print("Nan'ы присутсвуют в:")
 for x in data:
if data[x].isnull().sum() > 0:
print(x )<count_missing_values>
config = tf.io.FixedLenFeature
feature_train = {"class": config([], tf.int64),
"id" : config([], tf.string),
"image": config([], tf.string),}
feature_test = {"id" : config([], tf.string),
"image": config([], tf.string),}
def read_tfrecord(example_single, features):

 parsed = tf.io.parse_single_example(example_single, features)
 idm = parsed.get("id")
 label = parsed.get("class")
 image = decode_image(parsed.get("image"))
if not features.get("class"):
return image, idm
return image, label
parse_train = partial(read_tfrecord, features=feature_train)
 parse_test = partial(read_tfrecord, features=feature_test )
data.isnull().sum()<count_missing_values>
train_dataset = train_records.map(parse_train, num_parallel_calls=AUTO)
 val_dataset = val_records.map(parse_train, num_parallel_calls=AUTO)
 test_dataset = test_records.map(parse_test )
data.isnull().sum()<count_values>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))
print("А у нас ", int(data.Survived.value_counts(normalize=True)[1] * 100),'% выжило',sep = "")
 print("С родителями или детьми ", int(data.Parch[data.Parch > 0].count() /data.Parch.count() * 100),'%',sep = "")
 print("С братьями или сестрами ", int(data.SibSp[data.SibSp > 0].count() /data.SibSp.count() * 100),'%',sep = "")
 print("Пенсионеров ", int(data.Age[data.Age > 65].count() /data.Age.count() * 100),'%',sep = "")
<groupby>
def rot_shift_zoom_shear(image, DIM = IMAGE_SIZE[0]):
XDIM = DIM % 2
rot = 15.* tf.random.normal([1],dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image,tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3] )
print("Посмотрим распределение выживших по полу")
 data.groupby('Sex' ).Survived.mean()<groupby>
def transform(image, inv_mat, image_shape):
h, w, c = image_shape
cx, cy = w//2, h//2
new_xs = tf.repeat(tf.range(-cx, cx, 1), h)
 new_ys = tf.tile(tf.range(-cy, cy, 1), [w])
 new_zs = tf.ones([h*w], dtype=tf.int32)
 old_coords = tf.matmul(inv_mat, tf.cast(tf.stack([new_xs, new_ys, new_zs]), tf.float32))
old_coords_x, old_coords_y = tf.round(old_coords[0, :] + w//2), tf.round(old_coords[1, :] + h//2)
 clip_mask_x = tf.logical_or(old_coords_x<0, old_coords_x>w-1)
 clip_mask_y = tf.logical_or(old_coords_y<0, old_coords_y>h-1)
 clip_mask = tf.logical_or(clip_mask_x, clip_mask_y)
 old_coords_x = tf.boolean_mask(old_coords_x, tf.logical_not(clip_mask))
old_coords_y = tf.boolean_mask(old_coords_y, tf.logical_not(clip_mask))
new_coords_x = tf.boolean_mask(new_xs+cx, tf.logical_not(clip_mask))
new_coords_y = tf.boolean_mask(new_ys+cy, tf.logical_not(clip_mask))
old_coords = tf.cast(tf.stack([old_coords_y, old_coords_x]), tf.int32)
 new_coords = tf.cast(tf.stack([new_coords_y, new_coords_x]), tf.int64)
 rotated_image_values = tf.gather_nd(image, tf.transpose(old_coords))
rotated_image_channel = list()
for i in range(c):
vals = rotated_image_values[:,i]
sparse_channel = tf.SparseTensor(tf.transpose(new_coords), vals, [h, w])
 rotated_image_channel.append(tf.sparse.to_dense(sparse_channel, default_value=0, validate_indices=False))
return tf.transpose(tf.stack(rotated_image_channel), [1,2,0])
 def random_rotate(image, angle, image_shape):
def get_rotation_mat_inv(angle):
angle = math.pi * angle / 180
cos_val = tf.math.cos(angle)
 sin_val = tf.math.sin(angle)
 one = tf.constant([1], tf.float32)
 zero = tf.constant([0], tf.float32)
 rot_mat_inv = tf.concat([cos_val, sin_val, zero,
-sin_val, cos_val, zero,
zero, zero, one], axis=0)
 rot_mat_inv = tf.reshape(rot_mat_inv, [3,3])
 return rot_mat_inv
angle = float(angle)* tf.random.normal([1],dtype='float32')
 rot_mat_inv = get_rotation_mat_inv(angle)
 return transform(image, rot_mat_inv, image_shape)
 def GridMask(image_height, image_width, d1, d2, rotate_angle=1, ratio=0.5):
h, w = image_height, image_width
hh = int(np.ceil(np.sqrt(h*h+w*w)))
 hh = hh+1 if hh%2==1 else hh
d = tf.random.uniform(shape=[], minval=d1, maxval=d2, dtype=tf.int32)
 l = tf.cast(tf.cast(d,tf.float32)*ratio+0.5, tf.int32)
 st_h = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
 st_w = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
 y_ranges = tf.range(-1 * d + st_h, -1 * d + st_h + l)
 x_ranges = tf.range(-1 * d + st_w, -1 * d + st_w + l)
 for i in range(0, hh//d+1):
s1 = i * d + st_h
s2 = i * d + st_w
y_ranges = tf.concat([y_ranges, tf.range(s1,s1+l)], axis=0)
 x_ranges = tf.concat([x_ranges, tf.range(s2,s2+l)], axis=0)
 x_clip_mask = tf.logical_or(x_ranges <0 , x_ranges > hh-1)
 y_clip_mask = tf.logical_or(y_ranges <0 , y_ranges > hh-1)
 clip_mask = tf.logical_or(x_clip_mask, y_clip_mask)
 x_ranges = tf.boolean_mask(x_ranges, tf.logical_not(clip_mask))
y_ranges = tf.boolean_mask(y_ranges, tf.logical_not(clip_mask))
hh_ranges = tf.tile(tf.range(0,hh), [tf.cast(tf.reduce_sum(tf.ones_like(x_ranges)) , tf.int32)])
 x_ranges = tf.repeat(x_ranges, hh)
 y_ranges = tf.repeat(y_ranges, hh)
 y_hh_indices = tf.transpose(tf.stack([y_ranges, hh_ranges]))
x_hh_indices = tf.transpose(tf.stack([hh_ranges, x_ranges]))
y_mask_sparse = tf.SparseTensor(tf.cast(y_hh_indices, tf.int64), tf.zeros_like(y_ranges), [hh, hh])
 y_mask = tf.sparse.to_dense(y_mask_sparse, 1, False)
 x_mask_sparse = tf.SparseTensor(tf.cast(x_hh_indices, tf.int64), tf.zeros_like(x_ranges), [hh, hh])
 x_mask = tf.sparse.to_dense(x_mask_sparse, 1, False)
 mask = tf.expand_dims(tf.clip_by_value(x_mask + y_mask, 0, 1), axis=-1)
 mask = random_rotate(mask, rotate_angle, [hh, hh, 1])
 mask = tf.image.crop_to_bounding_box(mask,(hh-h)//2,(hh-w)//2, image_height, image_width)
 return mask
def apply_grid_mask(image, image_shape):
mask = GridMask(image_shape[0],
image_shape[1],
AugParams['d1'],
AugParams['d2'],
AugParams['rotate'],
AugParams['ratio'])
 if image_shape[-1] == 3:
mask = tf.concat([mask, mask, mask], axis=-1)
 return image * tf.cast(mask, tf.float32 )
data.groupby(data.Age[data.Age<18] ).Survived.mean()<drop_column>
def random_blockout(img, sl=0.1, sh=0.2, rl=0.4):
h, w, c = IMAGE_SIZE[0], IMAGE_SIZE[1], 3
origin_area = tf.cast(h*w, tf.float32)
 e_size_l = tf.cast(tf.round(tf.sqrt(origin_area * sl * rl)) , tf.int32)
 e_size_h = tf.cast(tf.round(tf.sqrt(origin_area * sh / rl)) , tf.int32)
 e_height_h = tf.minimum(e_size_h, h)
 e_width_h = tf.minimum(e_size_h, w)
 erase_height = tf.random.uniform(shape=[], minval=e_size_l, maxval=e_height_h, dtype=tf.int32)
 erase_width = tf.random.uniform(shape=[], minval=e_size_l, maxval=e_width_h, dtype=tf.int32)
 erase_area = tf.zeros(shape=[erase_height, erase_width, c])
 erase_area = tf.cast(erase_area, tf.uint8)
 pad_h = h - erase_height
pad_top = tf.random.uniform(shape=[], minval=0, maxval=pad_h, dtype=tf.int32)
 pad_bottom = pad_h - pad_top
pad_w = w - erase_width
pad_left = tf.random.uniform(shape=[], minval=0, maxval=pad_w, dtype=tf.int32)
 pad_right = pad_w - pad_left
erase_mask = tf.pad([erase_area], [[0,0],[pad_top, pad_bottom], [pad_left, pad_right], [0,0]], constant_values=1)
 erase_mask = tf.squeeze(erase_mask, axis=0)
 erased_img = tf.multiply(tf.cast(img,tf.float32), tf.cast(erase_mask, tf.float32))
return tf.cast(erased_img, img.dtype )
data.drop(["Cabin"],axis=1, inplace=True)
 data_test.drop(["Cabin"],axis=1, inplace=True)
 data.drop(["Ticket"],axis=1, inplace=True)
 data_test.drop(["Ticket"],axis=1, inplace=True)
 data.drop(["PassengerId"],axis=1, inplace=True)
 data_test.drop(["PassengerId"],axis=1, inplace=True )<import_modules>
def augmentation(image, label):
prob = tf.random.uniform(shape=[], minval=0.0, maxval=1.0)
 if prob <= 0.25: image = apply_grid_mask(image,(*IMAGE_SIZE,3))
elif prob <= 0.50: image = rot_shift_zoom_shear(image)
 elif prob <= 0.75: image = random_blockout(image)
 else: pass
return tf.cast(image, tf.float32), label
import seaborn as sns
import matplotlib.pyplot as plt<feature_engineering>
train_input =(
 train_dataset 
.repeat() 
.map(augmentation, num_parallel_calls=AUTO)
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 val_input =(
 val_dataset 
.batch(BATCH_SIZE)
.cache() 
)
 test_input =(
 test_dataset.batch(BATCH_SIZE)
.prefetch(AUTO)
 )
data["Status"] = data.Name.str.extract(r', (.*)\.', expand=False)
 data_test["Status"] = data_test.Name.str.extract(r', (.*)\.', expand=False )<groupby>
params = {
"include_top":False,
"input_shape":[*IMAGE_SIZE, 3],
"classes":104,
"pooling":"avg"}
def create_dnet() :
model = keras.applications.DenseNet201(**params)
 compiler = re.compile(r"conv5_block")
 for layer in model.layers:
if not compiler.search(layer.name):
layer.trainable = False
return model
def create_efn() :
model = efn.EfficientNetB6(**params)
 compiler = re.compile(r"block7")
 for layer in model.layers:
if not compiler.search(layer.name):
layer.trainable = False
return model
data.groupby('Status' ).Survived.count()<groupby>
loss = "sparse_categorical_crossentropy"
metric = "sparse_categorical_accuracy"
with strategy.scope() :
efficient = create_efn()
dense_net = create_dnet()
input_image = keras.Input(shape=[*IMAGE_SIZE, 3])
 efficient = efficient(input_image)
 dense_net = dense_net(input_image)
 dropout_1 = keras.layers.Dropout(0.3 )(efficient)
 dropout_2 = keras.layers.Dropout(0.3 )(dense_net)
 concat = keras.layers.concatenate([dropout_1, dropout_2])
 dense_2 = keras.layers.Dense(104, activation="softmax" )(concat)
 merged_model = keras.Model(input_image, dense_2)
 merged_model.compile(optimizer=keras.optimizers.Adam(lr_start),
loss=loss,
metrics=[metric])
 full_model = merged_model
data.groupby('Status' ).Survived.count()
for x in data.Status:
if data[data.Status == x].Survived.count() < 10:
data.Status.replace(x, 'OTHER', inplace= True)
 data.groupby('Status' ).Survived.count() 
<drop_column>
early_stop = keras.callbacks.EarlyStopping(monitor=metric, patience=5)
 check_point = keras.callbacks.ModelCheckpoint(
 filepath="best_model.h5",
monitor="val_sparse_categorical_accuracy",
save_best_only=True)
 learning_schedule = keras.callbacks.LearningRateScheduler(scheduler)
 callbacks_list = [early_stop, check_point, learning_schedule]
data.drop(["Name"],axis=1, inplace=True)
 data_test.drop(["Name"],axis=1, inplace=True)
 data.groupby('Status' ).Survived.mean()<set_options>
data.Embarked.mode()<data_type_conversions>
full_input =(
 train_dataset.concatenate(val_dataset)
.repeat() 
.map(augmentation, num_parallel_calls=AUTO)
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
 )
data.Embarked.fillna('S', inplace=True)
 data_test.Embarked.fillna('S', inplace=True )<filter>
check_point.monitor = metric
history = full_model.fit(full_input,
epochs=EPOCHS,
steps_per_epoch=STEPS_PER_EPOCHS_FULL,
callbacks=callbacks_list,
)
data.Age[data.Age.notnull() ]<import_modules>
best_model = keras.models.load_model("./best_model.h5" )
from sklearn.preprocessing import OneHotEncoder<categorify>
best_weights = best_model.get_weights()
merged_model.set_weights(best_weights )
data = pd.get_dummies(data)
 data_test = pd.get_dummies(data_test )<import_modules>
def aug_gridmask(image):
image = apply_grid_mask(image,(*IMAGE_SIZE, 3))
return tf.cast(image, tf.float32)
 def aug_rot(image):
image = rot_shift_zoom_shear(image)
 return tf.cast(image, tf.float32)
 def aug_blockout(image):
image = random_blockout(image)
 return tf.cast(image, tf.float32)
 def aug_none(image):
return image
aug_list = [aug_gridmask, aug_rot, aug_blockout, aug_none]
from sklearn.linear_model import LinearRegression<drop_column>
def predict_tta(model, test_image, aug_list=aug_list):
probs = []
for index, aug in enumerate(aug_list):
copy_image =(
 test_image.unbatch() 
.map(aug)
.batch(BATCH_SIZE))
probs.append(model.predict(copy_image))
return np.mean(probs, axis=0 )
testforfilna = data.drop(['Age', 'Survived'], axis= 1 )<filter>
test_image = test_input.map(lambda image, _id: image, num_parallel_calls=AUTO)
 test_id = test_input.map(lambda image, _id: _id ).unbatch().batch(TEST_IMG_NUM)
 image_id = next(iter(test_id)).numpy().astype("U")
 probabilities = predict_tta(merged_model, test_image)
 predictions = np.argmax(probabilities, axis=-1 )
<import_modules><EOS> 
submission = pd.DataFrame({"id": image_id, "label": predictions})
 submission.to_csv("submission.csv", index=False )