Kaggle competition: I’m Something of a Painter Myself
<SOS> metric: MiFID Kaggle data source: i’m-something-of-a-painter-myself<categorify>
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
SEED = 0
seed_everything(SEED )
titanic.Title = titanic.Title.map(normalized_title )<count_values>
HEIGHT = 256
WIDTH = 256
HEIGHT_RESIZE = 128
WIDTH_RESIZE = 128
CHANNELS = 3
BATCH_SIZE = 16
EPOCHS = 120
TRANSFORMER_BLOCKS = 6
GENERATOR_LR = 2e-4
DISCRIMINATOR_LR = 2e-4
print(titanic.Title.value_counts() )<count_missing_values>
def data_augment(image):
p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 if p_rotate >.8:
image = tf.image.rot90(image, k=3)
 elif p_rotate >.6:
image = tf.image.rot90(image, k=2)
 elif p_rotate >.4:
image = tf.image.rot90(image, k=1)
 image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if p_spatial >.75:
image = tf.image.transpose(image)
 image = tf.image.random_crop(image, size=[HEIGHT_RESIZE, WIDTH_RESIZE, CHANNELS])
 return image
titanic.Age = grouped.Age.apply(lambda x: x.fillna(x.median()))
titanic.isnull().sum()<data_type_conversions>
def normalize_img(img):
img = tf.cast(img, dtype=tf.float32)
 return(img / 127.5)- 1.0
def decode_image(image):
image = tf.image.decode_jpeg(image, channels=CHANNELS)
 image = tf.reshape(image, [HEIGHT, WIDTH, CHANNELS])
 return image
def read_tfrecord(example):
tfrecord_format = {
'image': tf.io.FixedLenFeature([], tf.string)
 }
example = tf.io.parse_single_example(example, tfrecord_format)
 image = decode_image(example['image'])
 return image
def load_dataset(filenames):
dataset = tf.data.TFRecordDataset(filenames)
 dataset = dataset.map(read_tfrecord, num_parallel_calls=AUTO)
 return dataset
def get_dataset(filenames, augment=None, repeat=True, shuffle=True, batch_size=1):
dataset = load_dataset(filenames)
 if augment:
dataset = dataset.map(augment, num_parallel_calls=AUTO)
 dataset = dataset.map(normalize_img, num_parallel_calls=AUTO)
 if repeat:
dataset = dataset.repeat()
if shuffle:
dataset = dataset.shuffle(512)
 dataset = dataset.batch(batch_size)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def display_samples(ds, n_samples):
ds_iter = iter(ds)
 for n_sample in range(n_samples):
example_sample = next(ds_iter)
 plt.subplot(121)
 plt.imshow(example_sample[0] * 0.5 + 0.5)
 plt.axis('off')
 plt.show()
def display_generated_samples(ds, model, n_samples):
ds_iter = iter(ds)
 for n_sample in range(n_samples):
example_sample = next(ds_iter)
 generated_sample = model.predict(example_sample)
 f = plt.figure(figsize=(12, 12))
plt.subplot(121)
 plt.title('Input image')
 plt.imshow(example_sample[0] * 0.5 + 0.5)
 plt.axis('off')
 plt.subplot(122)
 plt.title('Generated image')
 plt.imshow(generated_sample[0] * 0.5 + 0.5)
 plt.axis('off')
 plt.show()
def evaluate_cycle(ds, generator_a, generator_b, n_samples=1):
fig, axes = plt.subplots(n_samples, 3, figsize=(22,(n_samples*6)))
 axes = axes.flatten()
ds_iter = iter(ds)
 for n_sample in range(n_samples):
idx = n_sample*3
example_sample = next(ds_iter)
 generated_a_sample = generator_a.predict(example_sample)
 generated_b_sample = generator_b.predict(generated_a_sample)
 axes[idx].set_title('Input image', fontsize=18)
 axes[idx].imshow(example_sample[0] * 0.5 + 0.5)
 axes[idx].axis('off')
 axes[idx+1].set_title('Generated image', fontsize=18)
 axes[idx+1].imshow(generated_a_sample[0] * 0.5 + 0.5)
 axes[idx+1].axis('off')
 axes[idx+2].set_title('Cycled image', fontsize=18)
 axes[idx+2].imshow(generated_b_sample[0] * 0.5 + 0.5)
 axes[idx+2].axis('off')
 plt.show()
def create_gif(images_path, gif_path):
images = []
filenames = glob.glob(images_path)
 filenames.sort(key=lambda x: int(''.join(filter(str.isdigit, x))))
for epoch, filename in enumerate(filenames):
img = PIL.ImageDraw.Image.open(filename)
 ImageDraw.Draw(img ).text(( 0, 0),
f'Epoch {epoch+1}')
 images.append(img)
 imageio.mimsave(gif_path, images, fps=2)
 def predict_and_save(input_ds, generator_model, output_path):
i = 1
for img in input_ds:
prediction = generator_model(img, training=False)[0].numpy()
prediction =(prediction * 127.5 + 127.5 ).astype(np.uint8)
 im = PIL.Image.fromarray(prediction)
 im.save(f'{output_path}{str(i)}.jpg')
 i += 1
titanic.Fare = titanic.Fare.fillna(titanic.Fare.mean())
 titanic.Cabin = titanic.Cabin.fillna('U')
 most_Embarked = titanic.Embarked.value_counts().index[0]
titanic.Embarked = titanic.Embarked.fillna(most_Embarked )<count_missing_values>
conv_initializer = tf.random_normal_initializer(mean=0.0, stddev=0.02)
 gamma_initializer = tf.keras.initializers.RandomNormal(mean=0.0, stddev=0.02)
 def encoder_block(input_layer, filters, size=3, strides=2, apply_instancenorm=True, activation=L.ReLU() , name='block_x'):
block = L.Conv2D(filters, size,
strides=strides,
padding='same',
use_bias=False,
kernel_initializer=conv_initializer,
name=f'encoder_{name}' )(input_layer)
 if apply_instancenorm:
block = tfa.layers.InstanceNormalization(gamma_initializer=gamma_initializer )(block)
 block = activation(block)
 return block
def transformer_block(input_layer, size=3, strides=1, name='block_x'):
filters = input_layer.shape[-1]
block = L.Conv2D(filters, size, strides=strides, padding='same', use_bias=False,
kernel_initializer=conv_initializer, name=f'transformer_{name}_1' )(input_layer)
 block = L.ReLU()(block)
 block = L.Conv2D(filters, size, strides=strides, padding='same', use_bias=False,
kernel_initializer=conv_initializer, name=f'transformer_{name}_2' )(block)
 block = L.Add()([block, input_layer])
 return block
def decoder_block(input_layer, filters, size=3, strides=2, apply_instancenorm=True, name='block_x'):
block = L.Conv2DTranspose(filters, size,
strides=strides,
padding='same',
use_bias=False,
kernel_initializer=conv_initializer,
name=f'decoder_{name}' )(input_layer)
 if apply_instancenorm:
block = tfa.layers.InstanceNormalization(gamma_initializer=gamma_initializer )(block)
 block = L.ReLU()(block)
 return block
def decoder_rc_block(input_layer, filters, size=3, strides=1, apply_instancenorm=True, name='block_x'):
block = tf.image.resize(images=input_layer, method='bilinear',
size=(input_layer.shape[1]*2, input_layer.shape[2]*2))
block = L.Conv2D(filters, size,
strides=strides,
padding='same',
use_bias=False,
kernel_initializer=conv_initializer,
name=f'decoder_{name}' )(block)
 if apply_instancenorm:
block = tfa.layers.InstanceNormalization(gamma_initializer=gamma_initializer )(block)
 block = L.ReLU()(block)
 return block
titanic.isnull().sum()<feature_engineering>
def generator_fn(height=HEIGHT, width=WIDTH, channels=CHANNELS, transformer_blocks=TRANSFORMER_BLOCKS):
OUTPUT_CHANNELS = 3
inputs = L.Input(shape=[height, width, channels], name='input_image')
 enc_1 = encoder_block(inputs, 64, 7, 1, apply_instancenorm=False, activation=L.ReLU() , name='block_1')
 enc_2 = encoder_block(enc_1, 128, 3, 2, apply_instancenorm=True, activation=L.ReLU() , name='block_2')
 enc_3 = encoder_block(enc_2, 256, 3, 2, apply_instancenorm=True, activation=L.ReLU() , name='block_3')
 x = enc_3
for n in range(transformer_blocks):
x = transformer_block(x, 3, 1, name=f'block_{n+1}')
 x_skip = L.Concatenate(name='enc_dec_skip_1' )([x, enc_3])
 dec_1 = decoder_block(x_skip, 128, 3, 2, apply_instancenorm=True, name='block_1')
 x_skip = L.Concatenate(name='enc_dec_skip_2' )([dec_1, enc_2])
 dec_2 = decoder_block(x_skip, 64, 3, 2, apply_instancenorm=True, name='block_2')
 x_skip = L.Concatenate(name='enc_dec_skip_3' )([dec_2, enc_1])
 outputs = last = L.Conv2D(OUTPUT_CHANNELS, 7,
strides=1, padding='same',
kernel_initializer=conv_initializer,
use_bias=False,
activation='tanh',
name='decoder_output_block' )(x_skip)
 generator = Model(inputs, outputs)
 return generator
sample_generator = generator_fn()
sample_generator.summary()
titanic['FamilySize'] = titanic['SibSp'] + titanic['Parch'] + 1<count_values>
def discriminator_fn(height=HEIGHT, width=WIDTH, channels=CHANNELS):
inputs = L.Input(shape=[height, width, channels], name='input_image')
 x = encoder_block(inputs, 64, 4, 2, apply_instancenorm=False, activation=L.LeakyReLU(0.2), name='block_1')
 x = encoder_block(x, 128, 4, 2, apply_instancenorm=True, activation=L.LeakyReLU(0.2), name='block_2')
 x = encoder_block(x, 256, 4, 2, apply_instancenorm=True, activation=L.LeakyReLU(0.2), name='block_3')
 x = encoder_block(x, 512, 4, 1, apply_instancenorm=True, activation=L.LeakyReLU(0.2), name='block_4')
 outputs = L.Conv2D(1, 4, strides=1, padding='valid', kernel_initializer=conv_initializer )(x)
 discriminator = Model(inputs, outputs)
 return discriminator
sample_discriminator = discriminator_fn()
sample_discriminator.summary()
titanic['Cabin'].value_counts()<categorify>
with strategy.scope() :
monet_generator = generator_fn(height=HEIGHT_RESIZE, width=WIDTH_RESIZE, transformer_blocks=TRANSFORMER_BLOCKS)
 photo_generator = generator_fn(height=HEIGHT_RESIZE, width=WIDTH_RESIZE, transformer_blocks=TRANSFORMER_BLOCKS)
 monet_discriminator = discriminator_fn(height=HEIGHT_RESIZE, width=WIDTH_RESIZE)
 photo_discriminator = discriminator_fn(height=HEIGHT_RESIZE, width=WIDTH_RESIZE)
 class CycleGan(Model):
def __init__(
 self,
monet_generator,
photo_generator,
monet_discriminator,
photo_discriminator,
lambda_cycle=10, 
):
super(CycleGan, self ).__init__()
self.m_gen = monet_generator
self.p_gen = photo_generator
self.m_disc = monet_discriminator
self.p_disc = photo_discriminator
self.lambda_cycle = lambda_cycle
def compile(
 self,
m_gen_optimizer,
p_gen_optimizer,
m_disc_optimizer,
p_disc_optimizer,
gen_loss_fn,
disc_loss_fn,
cycle_loss_fn,
identity_loss_fn 
):
super(CycleGan, self ).compile()
self.m_gen_optimizer = m_gen_optimizer
self.p_gen_optimizer = p_gen_optimizer
self.m_disc_optimizer = m_disc_optimizer
self.p_disc_optimizer = p_disc_optimizer
self.gen_loss_fn = gen_loss_fn
self.disc_loss_fn = disc_loss_fn
self.cycle_loss_fn = cycle_loss_fn
self.identity_loss_fn = identity_loss_fn
def train_step(self, batch_data):
real_monet, real_photo = batch_data
with tf.GradientTape(persistent=True)as tape:
fake_monet = self.m_gen(real_photo, training=True)
 cycled_photo = self.p_gen(fake_monet, training=True)
 fake_photo = self.p_gen(real_monet, training=True)
 cycled_monet = self.m_gen(fake_photo, training=True)
 same_monet = self.m_gen(real_monet, training=True)
 same_photo = self.p_gen(real_photo, training=True)
 disc_real_monet = self.m_disc(real_monet, training=True)
 disc_real_photo = self.p_disc(real_photo, training=True)
 disc_fake_monet = self.m_disc(fake_monet, training=True)
 disc_fake_photo = self.p_disc(fake_photo, training=True)
 monet_gen_loss = self.gen_loss_fn(disc_fake_monet)
 photo_gen_loss = self.gen_loss_fn(disc_fake_photo)
 total_cycle_loss = self.cycle_loss_fn(real_monet, cycled_monet, self.lambda_cycle)+ self.cycle_loss_fn(real_photo, cycled_photo, self.lambda_cycle)
 total_monet_gen_loss = monet_gen_loss + total_cycle_loss + self.identity_loss_fn(real_monet, same_monet, self.lambda_cycle)
 total_photo_gen_loss = photo_gen_loss + total_cycle_loss + self.identity_loss_fn(real_photo, same_photo, self.lambda_cycle)
 monet_disc_loss = self.disc_loss_fn(disc_real_monet, disc_fake_monet)
 photo_disc_loss = self.disc_loss_fn(disc_real_photo, disc_fake_photo)
 monet_generator_gradients = tape.gradient(total_monet_gen_loss,
self.m_gen.trainable_variables)
 photo_generator_gradients = tape.gradient(total_photo_gen_loss,
self.p_gen.trainable_variables)
 monet_discriminator_gradients = tape.gradient(monet_disc_loss,
self.m_disc.trainable_variables)
 photo_discriminator_gradients = tape.gradient(photo_disc_loss,
self.p_disc.trainable_variables)
 self.m_gen_optimizer.apply_gradients(zip(monet_generator_gradients,
self.m_gen.trainable_variables))
self.p_gen_optimizer.apply_gradients(zip(photo_generator_gradients,
self.p_gen.trainable_variables))
self.m_disc_optimizer.apply_gradients(zip(monet_discriminator_gradients,
self.m_disc.trainable_variables))
self.p_disc_optimizer.apply_gradients(zip(photo_discriminator_gradients,
self.p_disc.trainable_variables))
return {'monet_gen_loss': total_monet_gen_loss,
'photo_gen_loss': total_photo_gen_loss,
'monet_disc_loss': monet_disc_loss,
'photo_disc_loss': photo_disc_loss
}
titanic.Cabin = titanic.Cabin.map(lambda x: x[0])
 titanic.Cabin.head()<categorify>
with strategy.scope() :
def discriminator_loss(real, generated):
real_loss = losses.BinaryCrossentropy(from_logits=True, reduction=losses.Reduction.NONE )(tf.ones_like(real), real)
 generated_loss = losses.BinaryCrossentropy(from_logits=True, reduction=losses.Reduction.NONE )(tf.zeros_like(generated), generated)
 total_disc_loss = real_loss + generated_loss
return total_disc_loss * 0.5
def generator_loss(generated):
return losses.BinaryCrossentropy(from_logits=True, reduction=losses.Reduction.NONE )(tf.ones_like(generated), generated)
 with strategy.scope() :
def calc_cycle_loss(real_image, cycled_image, LAMBDA):
loss1 = tf.reduce_mean(tf.abs(real_image - cycled_image))
return LAMBDA * loss1
with strategy.scope() :
def identity_loss(real_image, same_image, LAMBDA):
loss = tf.reduce_mean(tf.abs(real_image - same_image))
return LAMBDA * 0.5 * loss
titanic.Sex = titanic.Sex.map({"male":0,"female":1})
<categorify>
with strategy.scope() :
lr_monet_gen = lambda: linear_schedule_with_warmup(tf.cast(monet_generator_optimizer.iterations, tf.float32))
lr_photo_gen = lambda: linear_schedule_with_warmup(tf.cast(photo_generator_optimizer.iterations, tf.float32))
monet_generator_optimizer = optimizers.Adam(learning_rate=lr_monet_gen, beta_1=0.5)
 photo_generator_optimizer = optimizers.Adam(learning_rate=lr_photo_gen, beta_1=0.5)
 lr_monet_disc = lambda: linear_schedule_with_warmup(tf.cast(monet_discriminator_optimizer.iterations, tf.float32))
lr_photo_disc = lambda: linear_schedule_with_warmup(tf.cast(photo_discriminator_optimizer.iterations, tf.float32))
monet_discriminator_optimizer = optimizers.Adam(learning_rate=lr_monet_disc, beta_1=0.5)
 photo_discriminator_optimizer = optimizers.Adam(learning_rate=lr_photo_disc, beta_1=0.5)
 gan_model = CycleGan(monet_generator, photo_generator,
monet_discriminator, photo_discriminator)
 gan_model.compile(m_gen_optimizer=monet_generator_optimizer,
p_gen_optimizer=photo_generator_optimizer,
m_disc_optimizer=monet_discriminator_optimizer,
p_disc_optimizer=photo_discriminator_optimizer,
gen_loss_fn=generator_loss,
disc_loss_fn=discriminator_loss,
cycle_loss_fn=calc_cycle_loss,
identity_loss_fn=identity_loss )
enc_lst = ['Title','Cabin','Pclass','Embarked']
le = LabelEncoder()
for i in enc_lst:
titanic[i] = le.fit_transform(titanic[i] )<drop_column>
monet_ds = get_dataset(MONET_FILENAMES, augment=data_augment, batch_size=BATCH_SIZE)
 photo_ds = get_dataset(PHOTO_FILENAMES, augment=data_augment, batch_size=BATCH_SIZE)
 gan_ds = tf.data.Dataset.zip(( monet_ds, photo_ds))
photo_ds_eval = get_dataset(PHOTO_FILENAMES, repeat=False, shuffle=False, batch_size=1)
 monet_ds_eval = get_dataset(MONET_FILENAMES, repeat=False, shuffle=False, batch_size=1)
 class GANMonitor(Callback):

 def __init__(self, num_img=1, monet_path='monet', photo_path='photo'):
self.num_img = num_img
self.monet_path = monet_path
self.photo_path = photo_path
if not os.path.exists(self.monet_path):
os.makedirs(self.monet_path)
 if not os.path.exists(self.photo_path):
os.makedirs(self.photo_path)
 def on_epoch_end(self, epoch, logs=None):
for i, img in enumerate(photo_ds_eval.take(self.num_img)) :
prediction = monet_generator(img, training=False)[0].numpy()
prediction =(prediction * 127.5 + 127.5 ).astype(np.uint8)
 prediction = PIL.Image.fromarray(prediction)
 prediction.save(f'{self.monet_path}/generated_{i}_{epoch+1}.png')
 for i, img in enumerate(monet_ds_eval.take(self.num_img)) :
prediction = photo_generator(img, training=False)[0].numpy()
prediction =(prediction * 127.5 + 127.5 ).astype(np.uint8)
 prediction = PIL.Image.fromarray(prediction)
 prediction.save(f'{self.photo_path}/generated_{i}_{epoch+1}.png' )
titanic.drop(['Name', 'Ticket'],axis = 1,inplace = True )<prepare_x_and_y>
history = gan_model.fit(gan_ds,
epochs=EPOCHS,
callbacks=[GANMonitor() ],
steps_per_epoch=(max(n_monet_samples, n_photo_samples)//BATCH_SIZE),
verbose=2 ).history
train_x = titanic[:train_index]
test_x = titanic[train_index:]<data_type_conversions>
create_gif('/kaggle/working/monet/*.png', 'monet.gif' )
train_x.Survived = train_x.Survived.astype(int )<prepare_x_and_y>
create_gif('/kaggle/working/photo/*.png', 'photo.gif' )
X = train_x.drop('Survived', axis=1 ).values
y = train_x.Survived.values<drop_column>
evaluate_cycle(photo_ds_eval.take(2), monet_generator, photo_generator, n_samples=2 )
X_test = test_x.drop('Survived', axis=1 ).values<split>
evaluate_cycle(monet_ds_eval.take(2), photo_generator, monet_generator, n_samples=2 )
label = train_x.Survived
train_X , val_X , train_Y , val_Y = train_test_split(train_x , label,test_size = 0.2,shuffle = True )<drop_column>
display_generated_samples(photo_ds_eval.take(8), monet_generator, 8 )
dtrain_X = train_X.drop('Survived',axis = 1 )<drop_column>
%%time
os.makedirs('.. /images/')
 predict_and_save(photo_ds_eval, monet_generator, '.. /images/' )
dval_X = val_X.drop('Survived',axis = 1 )<choose_model_class>
shutil.make_archive('/kaggle/working/images/', 'zip', '.. /images')
 print(f"Generated samples: {len([name for name in os.listdir('.. /images/')if os.path.isfile(os.path.join('.. /images/', name)) ])}" )
<compute_test_metric><EOS> 
monet_generator.save('monet_generator.h5')
 photo_generator.save('photo_generator.h5')
 monet_discriminator.save('monet_discriminator.h5')
 photo_discriminator.save('photo_discriminator.h5' )