Kaggle competition: I’m Something of a Painter Myself
<SOS> metric: MiFID Kaggle data source: i’m-something-of-a-painter-myself<count_values>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Device:', tpu.master())
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 except:
strategy = tf.distribute.get_strategy()
print('Number of replicas:', strategy.num_replicas_in_sync)
 AUTO = tf.data.experimental.AUTOTUNE
print(tf.__version__ )
df_train['Embarked'].value_counts()<feature_engineering>
GCS_PATH_MONET = KaggleDatasets().get_gcs_path('monet-tfrecords-extdata')
 GCS_PATH = KaggleDatasets().get_gcs_path('gan-getting-started')
 GCS_PATH_MONET, GCS_PATH
def age_cat(x):
if x<10:
return 0
elif x<20:
return 1
elif x<30:
return 2
elif x<40:
return 3
elif x<50:
return 4
elif x<60:
return 5
elif x<70:
return 6
else:
return 7
df_train['Age_cat']=df_train['Age'].apply(age_cat )<feature_engineering>
MONET_FILENAMES = tf.io.gfile.glob(str(GCS_PATH + '/monet_tfrec/*.tfrec'))
print('Monet TFRecord Files:', len(MONET_FILENAMES))
PHOTO_FILENAMES = tf.io.gfile.glob(str(GCS_PATH + '/photo_tfrec/*.tfrec'))
print('Photo TFRecord Files:', len(PHOTO_FILENAMES))
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 n_monet_samples = count_data_items(MONET_FILENAMES)
 n_photo_samples = count_data_items(PHOTO_FILENAMES)
 BATCH_SIZE = 4
EPOCHS_NUM = 30
print(f'Monet TFRecord files: {len(MONET_FILENAMES)}')
 print(f'Monet image files: {n_monet_samples}')
 print(f'Photo TFRecord files: {len(PHOTO_FILENAMES)}')
 print(f'Photo image files: {n_photo_samples}')
 print(f"Batch_size: {BATCH_SIZE}")
 print(f"Epochs number: {EPOCHS_NUM}" )
df_test['Age_cat']=df_train['Age'].apply(age_cat )<drop_column>
IMAGE_SIZE = [256, 256]
def decode_image(image):
image = tf.image.decode_jpeg(image, channels=3)
 image =(tf.cast(image, tf.float32)/ 127.5)- 1
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_tfrecord(example):
tfrecord_format = {
"image_name": tf.io.FixedLenFeature([], tf.string),
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.string)
 }
example = tf.io.parse_single_example(example, tfrecord_format)
 image = decode_image(example['image'])
 return image
df_train.drop(['Age'],axis=1,inplace=True)
 df_test.drop(['Age'],axis=1,inplace=True )<categorify>
def data_augment(image):
p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 if p_crop >.5:
image = tf.image.resize(image, [286, 286])
 image = tf.image.random_crop(image, size=[256, 256, 3])
 if p_crop >.9:
image = tf.image.resize(image, [300, 300])
 image = tf.image.random_crop(image, size=[256, 256, 3])
 if p_rotate >.9:
image = tf.image.rot90(image, k=3)
 elif p_rotate >.7:
image = tf.image.rot90(image, k=2)
 elif p_rotate >.5:
image = tf.image.rot90(image, k=1)
 if p_spatial >.6:
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if p_spatial >.9:
image = tf.image.transpose(image)
 return image
def load_dataset(filenames):
dataset = tf.data.TFRecordDataset(filenames)
 dataset = dataset.map(read_tfrecord, num_parallel_calls=AUTO)
 return dataset
def get_gan_dataset(monet_files, photo_files, augment=None, repeat=True, shuffle=True, batch_size=1):
monet_ds = load_dataset(monet_files)
 photo_ds = load_dataset(photo_files)
 if augment:
monet_ds = monet_ds.map(augment, num_parallel_calls=AUTO)
 photo_ds = photo_ds.map(augment, num_parallel_calls=AUTO)
 if repeat:
monet_ds = monet_ds.repeat()
photo_ds = photo_ds.repeat()
if shuffle:
monet_ds = monet_ds.shuffle(2048)
 photo_ds = photo_ds.shuffle(2048)
 monet_ds = monet_ds.batch(batch_size, drop_remainder=True)
 photo_ds = photo_ds.batch(batch_size, drop_remainder=True)
 monet_ds = monet_ds.cache()
photo_ds = photo_ds.cache()
monet_ds = monet_ds.prefetch(AUTO)
 photo_ds = photo_ds.prefetch(AUTO)
 gan_ds = tf.data.Dataset.zip(( monet_ds, photo_ds))
return gan_ds
df_train['Initial']=df_train['Initial'].map({'Master':0,'Miss':1,'Mr':2,'Mrs':3,'Other':4} )<categorify>
full_dataset = get_gan_dataset(MONET_FILENAMES, PHOTO_FILENAMES, augment=data_augment, repeat=True, shuffle=True, batch_size=BATCH_SIZE)
df_test['Initial']=df_test['Initial'].map({'Master':0,'Miss':1,'Mr':2,'Mrs':3,'Other':4} )<categorify>
example_monet , example_photo = next(iter(full_dataset))
df_train['Embarked']=df_train['Embarked'].map({'C':0,'Q':1,'S':2} )<categorify>
OUTPUT_CHANNELS = 3
def downsample(filters, size, apply_instancenorm=True):
initializer = tf.random_normal_initializer(0., 0.02)
 gamma_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.02)
 result = keras.Sequential()
result.add(layers.Conv2D(filters, size, strides=2, padding='same',
kernel_initializer=initializer, use_bias=False))
if apply_instancenorm:
result.add(tfa.layers.InstanceNormalization(gamma_initializer=gamma_init))
result.add(layers.LeakyReLU())
 return result
df_test['Embarked']=df_test['Embarked'].map({'C':0,'Q':1,'S':2} )<count_missing_values>
def upsample(filters, size, apply_dropout=False):
initializer = tf.random_normal_initializer(0., 0.02)
 gamma_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.02)
 result = keras.Sequential()
result.add(layers.Conv2DTranspose(filters, size, strides=2,
padding='same',
kernel_initializer=initializer,
use_bias=False))
result.add(tfa.layers.InstanceNormalization(gamma_initializer=gamma_init))
if apply_dropout:
result.add(layers.Dropout(0.5))
result.add(layers.ReLU())
 return result
df_train.isnull().any()<categorify>
def Generator() :
inputs = layers.Input(shape=[256,256,3])
 down_stack = [
downsample(64, 4, apply_instancenorm=False),
downsample(128, 4),
downsample(256, 4),
downsample(512, 4),
downsample(512, 4),
downsample(512, 4),
downsample(512, 4),
downsample(512, 4),
]
up_stack = [
upsample(512, 4, apply_dropout=True),
upsample(512, 4, apply_dropout=True),
upsample(512, 4, apply_dropout=True),
upsample(512, 4),
upsample(256, 4),
upsample(128, 4),
upsample(64, 4),
]
initializer = tf.random_normal_initializer(0., 0.02)
 last = layers.Conv2DTranspose(OUTPUT_CHANNELS, 4,
strides=2,
padding='same',
kernel_initializer=initializer,
activation='tanh')
 x = inputs
skips = []
for down in down_stack:
x = down(x)
 skips.append(x)
 skips = reversed(skips[:-1])
 for up, skip in zip(up_stack, skips):
x = up(x)
 x = layers.Concatenate()([x, skip])
 x = last(x)
 return keras.Model(inputs=inputs, outputs=x )
df_train['Sex']=df_train['Sex'].map({'female':0,'male':1})
 df_test['Sex']=df_test['Sex'].map({'female':0,'male':1} )<categorify>
def Discriminator() :
initializer = tf.random_normal_initializer(0., 0.02)
 gamma_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.02)
 inp = layers.Input(shape=[256, 256, 3], name='input_image')
 x = inp
down1 = downsample(64, 4, False )(x)
 down2 = downsample(128, 4 )(down1)
 down3 = downsample(256, 4 )(down2)
 zero_pad1 = layers.ZeroPadding2D()(down3)
 conv = layers.Conv2D(512, 4, strides=1,
kernel_initializer=initializer,
use_bias=False )(zero_pad1)
 norm1 = tfa.layers.InstanceNormalization(gamma_initializer=gamma_init )(conv)
 leaky_relu = layers.LeakyReLU()(norm1)
 zero_pad2 = layers.ZeroPadding2D()(leaky_relu)
 last = layers.Conv2D(1, 4, strides=1,
kernel_initializer=initializer )(zero_pad2)
 return tf.keras.Model(inputs=inp, outputs=last )
df_train=pd.get_dummies(df_train,columns=['Initial'],prefix='Initial' )<categorify>
with strategy.scope() :
monet_generator = Generator()
photo_generator = Generator()
monet_discriminator = Discriminator()
photo_discriminator = Discriminator()
df_test=pd.get_dummies(df_test,columns=['Initial'],prefix='Initial' )<categorify>
class CycleGan(keras.Model):
def __init__(
 self,
monet_generator,
photo_generator,
monet_discriminator,
photo_discriminator,
lambda_cycle=10, 
):
super(CycleGan, self ).__init__()
self.m_gen = monet_generator
self.p_gen = photo_generator
self.m_disc = monet_discriminator
self.p_disc = photo_discriminator
self.lambda_cycle = lambda_cycle
def compile(
 self,
m_gen_optimizer,
p_gen_optimizer,
m_disc_optimizer,
p_disc_optimizer,
gen_loss_fn,
disc_loss_fn,
cycle_loss_fn,
identity_loss_fn 
):
super(CycleGan, self ).compile()
self.m_gen_optimizer = m_gen_optimizer
self.p_gen_optimizer = p_gen_optimizer
self.m_disc_optimizer = m_disc_optimizer
self.p_disc_optimizer = p_disc_optimizer
self.gen_loss_fn = gen_loss_fn
self.disc_loss_fn = disc_loss_fn
self.cycle_loss_fn = cycle_loss_fn
self.identity_loss_fn = identity_loss_fn
def train_step(self, batch_data):
real_monet, real_photo = batch_data
with tf.GradientTape(persistent=True)as tape:
fake_monet = self.m_gen(real_photo, training=True)
 cycled_photo = self.p_gen(fake_monet, training=True)
 fake_photo = self.p_gen(real_monet, training=True)
 cycled_monet = self.m_gen(fake_photo, training=True)
 same_monet = self.m_gen(real_monet, training=True)
 same_photo = self.p_gen(real_photo, training=True)
 disc_real_monet = self.m_disc(real_monet, training=True)
 disc_real_photo = self.p_disc(real_photo, training=True)
 disc_fake_monet = self.m_disc(fake_monet, training=True)
 disc_fake_photo = self.p_disc(fake_photo, training=True)
 monet_gen_loss = self.gen_loss_fn(disc_fake_monet)
 photo_gen_loss = self.gen_loss_fn(disc_fake_photo)
 total_cycle_loss = self.cycle_loss_fn(real_monet, cycled_monet, self.lambda_cycle)+ self.cycle_loss_fn(real_photo, cycled_photo, self.lambda_cycle)
 total_monet_gen_loss = monet_gen_loss + total_cycle_loss + self.identity_loss_fn(real_monet, same_monet, self.lambda_cycle)
 total_photo_gen_loss = photo_gen_loss + total_cycle_loss + self.identity_loss_fn(real_photo, same_photo, self.lambda_cycle)
 monet_disc_loss = self.disc_loss_fn(disc_real_monet, disc_fake_monet)
 photo_disc_loss = self.disc_loss_fn(disc_real_photo, disc_fake_photo)
 monet_generator_gradients = tape.gradient(total_monet_gen_loss,
self.m_gen.trainable_variables)
 photo_generator_gradients = tape.gradient(total_photo_gen_loss,
self.p_gen.trainable_variables)
 monet_discriminator_gradients = tape.gradient(monet_disc_loss,
self.m_disc.trainable_variables)
 photo_discriminator_gradients = tape.gradient(photo_disc_loss,
self.p_disc.trainable_variables)
 self.m_gen_optimizer.apply_gradients(zip(monet_generator_gradients,
self.m_gen.trainable_variables))
self.p_gen_optimizer.apply_gradients(zip(photo_generator_gradients,
self.p_gen.trainable_variables))
self.m_disc_optimizer.apply_gradients(zip(monet_discriminator_gradients,
self.m_disc.trainable_variables))
self.p_disc_optimizer.apply_gradients(zip(photo_discriminator_gradients,
self.p_disc.trainable_variables))
return {
"monet_gen_loss": total_monet_gen_loss,
"photo_gen_loss": total_photo_gen_loss,
"monet_disc_loss": monet_disc_loss,
"photo_disc_loss": photo_disc_loss
}
df_train=pd.get_dummies(df_train,columns=['Embarked'],prefix='Embarked')
 df_test=pd.get_dummies(df_test,columns=['Embarked'],prefix='Embarked' )<drop_column>
with strategy.scope() :
def discriminator_loss(real, generated):
real_loss = tf.keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE )(tf.ones_like(real), real)
 generated_loss = tf.keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE )(tf.zeros_like(generated), generated)
 total_disc_loss = real_loss + generated_loss
return total_disc_loss * 0.5
df_train.drop(['PassengerId', 'Name', 'SibSp', 'Parch', 'Ticket', 'Cabin'], axis=1, inplace=True)
 df_test.drop(['PassengerId', 'Name', 'SibSp', 'Parch', 'Ticket', 'Cabin'], axis=1, inplace=True )<import_modules>
with strategy.scope() :
def generator_loss(generated):
return tf.keras.losses.BinaryCrossentropy(from_logits=True,
reduction=tf.keras.losses.Reduction.NONE )(tf.ones_like(generated), generated )
from sklearn.linear_model import LogisticRegression
from sklearn import svm
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.metrics import confusion_matrix<prepare_x_and_y>
with strategy.scope() :
def calc_cycle_loss(real_image, cycled_image, LAMBDA):
loss1 = tf.reduce_mean(tf.abs(real_image - cycled_image))
return LAMBDA * loss1
train,valid=train_test_split(df_train,test_size=0.3,random_state=0,stratify=df_train['Survived'])
 train_X=train[train.columns[1:]]
train_Y=train[train.columns[:1]]
vld_X=valid[valid.columns[1:]]
vld_Y=valid[valid.columns[:1]]
X=df_train[df_train.columns[1:]]
Y=df_train['Survived']<compute_train_metric>
with strategy.scope() :
def identity_loss(real_image, same_image, LAMBDA):
loss = tf.reduce_mean(tf.abs(real_image - same_image))
return LAMBDA * 0.5 * loss
model=svm.SVC(kernel='rbf',C=1,gamma=0.1)
 model.fit(train_X,train_Y)
 prediction1=model.predict(vld_X)
 print('Accuracy for rbf SVM is ',metrics.accuracy_score(prediction1,vld_Y))<compute_train_metric>
with strategy.scope() :
monet_generator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)
 photo_generator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)
 monet_discriminator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)
 photo_discriminator_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5 )
model=svm.SVC(kernel='linear',C=0.1,gamma=0.1)
 model.fit(train_X,train_Y)
 prediction2=model.predict(vld_X)
 print('Accuracy for linear SVM is',metrics.accuracy_score(prediction2,vld_Y))<choose_model_class>
with strategy.scope() :
cycle_gan_model = CycleGan(
 monet_generator, photo_generator, monet_discriminator, photo_discriminator 
)
 cycle_gan_model.compile(
 m_gen_optimizer = monet_generator_optimizer,
p_gen_optimizer = photo_generator_optimizer,
m_disc_optimizer = monet_discriminator_optimizer,
p_disc_optimizer = photo_discriminator_optimizer,
gen_loss_fn = generator_loss,
disc_loss_fn = discriminator_loss,
cycle_loss_fn = calc_cycle_loss,
identity_loss_fn = identity_loss 
)
model = LogisticRegression()
model.fit(train_X,train_Y)
 prediction3=model.predict(vld_X)
 print('The accuracy of the Logistic Regression is',metrics.accuracy_score(prediction3,vld_Y))<choose_model_class>
cycle_gan_model.fit(
 full_dataset,
epochs=EPOCHS_NUM,
steps_per_epoch=(max(n_monet_samples, n_photo_samples)//BATCH_SIZE), 
)
model=DecisionTreeClassifier()
model.fit(train_X,train_Y)
 prediction4=model.predict(vld_X)
 print('The accuract of Decision Tree is' ,metrics.accuracy_score(prediction4,vld_Y))<choose_model_class>
def predict_and_save(input_ds, generator_model, output_path):
i = 1
for img in input_ds:
prediction = generator_model(img, training=False)[0].numpy()
prediction =(prediction * 127.5 + 127.5 ).astype(np.uint8)
 im = PIL.Image.fromarray(prediction)
 im.save(f'{output_path}{str(i)}.jpg')
 i += 1
model=KNeighborsClassifier()
model.fit(train_X,train_Y)
 prediction5=model.predict(vld_X)
 print('The accuracy of KNN is ',metrics.accuracy_score(prediction5,vld_Y))<compute_train_metric>
os.makedirs('.. /images/')
 predict_and_save(load_dataset(PHOTO_FILENAMES ).batch(1), monet_generator, '.. /images/')
<train_model><EOS> 
shutil.make_archive('/kaggle/working/images/', 'zip', '.. /images')
 print(f"Generated samples: {len([name for name in os.listdir('.. /images/')if os.path.isfile(os.path.join('.. /images/', name)) ])}" )