Kaggle competition: Generative Dog Images
<SOS> metric: PostProcessorKernel Kaggle data source: generative-dog-images<define_variables>
from keras.models import Sequential, Model, load_model
from keras.layers import UpSampling2D, Conv2D, Activation, BatchNormalization, Reshape, Dense, Input, LeakyReLU, Dropout, Flatten, ZeroPadding2D
from keras.optimizers import Adam
import glob
from PIL import Image
import numpy as np
import os
from ast import literal_eval
from scipy.misc import imsave
import xml.etree.ElementTree as ET
from tqdm import tqdm
import zipfile
import time
WALL = ['paredblolad','paredzocalo','paredpreb','pareddes','paredmad','paredzinc','paredfibras','paredother']
FLOOR = ['pisomoscer','pisocemento','pisoother','pisonatur','pisonotiene','pisomadera']
ROOF = ['techozinc','techoentrepiso','techocane','techootro']
WATER =['abastaguadentro','abastaguafuera','abastaguano']
ELEC = ['public','planpri','noelec','coopele']
TOILET = ['sanitario1','sanitario2','sanitario3','sanitario5','sanitario6']
COOK = ['energcocinar1','energcocinar2','energcocinar3','energcocinar4']
RUBBISH = ['elimbasu1','elimbasu2','elimbasu3','elimbasu4','elimbasu5','elimbasu6']
WALL_QLTY = ['epared1', 'epared1','epared3']
ROOF_QLTY = ['etecho1','etecho2','etecho3']
FLOOR_QLTY = ['eviv1','eviv2','eviv3']
SEX = ['male', 'female']
CIVIL = ['estadocivil1','estadocivil2','estadocivil3','estadocivil4','estadocivil5','estadocivil6','estadocivil7']
H_OWNER = ['tipovivi1','tipovivi2','tipovivi3','tipovivi4','tipovivi5']
REGION = ['lugar1','lugar2','lugar3','lugar4','lugar5','lugar6']
AREA = ['area1', 'area2']
ALL_LISTS = [AREA, REGION, H_OWNER, CIVIL, SEX, FLOOR_QLTY, ROOF_QLTY, WALL_QLTY, RUBBISH, COOK, TOILET, ELEC, WATER, ROOF, FLOOR, WALL]
LIST_NAMES = ['AREA', 'REGION', 'H_OWNER', 'CIVIL', 'SEX', 'FLOOR_QLTY', 'ROOF_QLTY', 'WALL_QLTY', 'RUBBISH', 'COOK', 'TOILET', 'ELEC', 'WATER', 'ROOF', 'FLOOR', 'WALL'] 
<feature_engineering>
class DataLoader:
def __init__(self, root='.. /input/', image_size=(64, 64)) :
self.root = root
self.breeds = os.listdir(self.root + 'annotation/Annotation/')
 self.idxIn = 0
self.namesIn = []
self.image_size = image_size
self.imagesIn = np.zeros(( 25000, image_size[0], image_size[1], 3))
def load_images(self):
start = time.time()
for breed in self.breeds:
for dog in os.listdir(self.root + 'annotation/Annotation/' + breed):
try:
img = Image.open(self.root + 'all-dogs/all-dogs/' + dog + '.jpg')
 except:
continue
tree = ET.parse(self.root + 'annotation/Annotation/' + breed + '/' + dog)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
img2 = img.crop(( xmin, ymin, xmin+w, ymin+w))
img2 = img2.resize(( self.image_size[0], self.image_size[1]), Image.ANTIALIAS)
 self.imagesIn[self.idxIn,:,:,:] = np.asarray(img2)
 if self.idxIn % 1000 == 0:
print(self.idxIn)
 self.namesIn.append(breed)
 self.idxIn += 1
idx = np.arange(self.idxIn)
 np.random.shuffle(idx)
 self.imagesIn = self.imagesIn[idx,:,:,:]
self.namesIn = np.array(self.namesIn)[idx]
end = time.time()
print(end - start)
 return self.imagesIn
def OHE_to_cat(df, lists_of_cols, names):
for cols, name in zip(lists_of_cols, names):
df[name] =(df[cols] == 1 ).idxmax(1)
 df[name] = df[name].apply(lambda x: cols.index(x))
df = df.drop(columns=cols)
 return df<concatenate>
class ImageZipper:
def __init__(self, folder='./', name='images.zip'):
self.folder = folder
self.z = zipfile.PyZipFile(self.folder + name, mode='w')
 def add_image(self, image, image_name):
file_name = self.folder + image_name + '.png'
imsave(file_name, image)
 self.z.write(file_name)
 os.remove(file_name)
 def close(self):
self.z.close()
train = OHE_to_cat(train, ALL_LISTS, LIST_NAMES)
 test = OHE_to_cat(test, ALL_LISTS, LIST_NAMES )<define_variables>
class DCGAN:
def __init__(self, discriminator_path, generator_path, output_directory, data_loader):
self.data_loader = data_loader
self.img_size = self.data_loader.image_size
self.upsample_layers = 5
self.starting_filters = 64
self.kernel_size = 3
self.channels = 3
self.discriminator_path = discriminator_path
self.generator_path = generator_path
self.output_directory = output_directory
self.generated_folder = f"{self.output_directory}/generated_{self.img_size[0]}x{self.img_size[1]}"
def build_generator(self):
noise_shape =(100,)
 model = Sequential()
model.add(
 Dense(self.starting_filters *(self.img_size[0] //(2 ** self.upsample_layers)) *(self.img_size[1] //(2 ** self.upsample_layers)) ,
activation="relu", input_shape=noise_shape))
model.add(Reshape(((self.img_size[0] //(2 ** self.upsample_layers)) , 
(self.img_size[1] //(2 ** self.upsample_layers)) ,
self.starting_filters)))
 model.add(BatchNormalization(momentum=0.8))
model.add(UpSampling2D())
 model.add(Conv2D(1024, kernel_size=self.kernel_size, padding="same"))
model.add(Activation("relu"))
model.add(BatchNormalization(momentum=0.8))
model.add(UpSampling2D())
 model.add(Conv2D(512, kernel_size=self.kernel_size, padding="same"))
model.add(Activation("relu"))
model.add(BatchNormalization(momentum=0.8))
model.add(UpSampling2D())
 model.add(Conv2D(256, kernel_size=self.kernel_size, padding="same"))
model.add(Activation("relu"))
model.add(BatchNormalization(momentum=0.8))
model.add(UpSampling2D())
 model.add(Conv2D(128, kernel_size=self.kernel_size, padding="same"))
model.add(Activation("relu"))
model.add(BatchNormalization(momentum=0.8))
model.add(UpSampling2D())
 model.add(Conv2D(64, kernel_size=self.kernel_size, padding="same"))
model.add(Activation("relu"))
model.add(BatchNormalization(momentum=0.8))
model.add(Conv2D(32, kernel_size=self.kernel_size, padding="same"))
model.add(Activation("relu"))
model.add(BatchNormalization(momentum=0.8))
model.add(Conv2D(self.channels, kernel_size=self.kernel_size, padding="same"))
model.add(Activation("tanh"))
noise = Input(shape=noise_shape)
 img = model(noise)
 return Model(noise, img)
 def build_discriminator(self):
img_shape =(self.img_size[0], self.img_size[1], self.channels)
 model = Sequential()
model.add(Conv2D(32, kernel_size=self.kernel_size, strides=2, input_shape=img_shape, padding="same"))
model.add(LeakyReLU(alpha=0.2))
model.add(Dropout(0.25))
model.add(Conv2D(64, kernel_size=self.kernel_size, strides=2, padding="same"))
model.add(ZeroPadding2D(padding=(( 0, 1),(0, 1))))
model.add(LeakyReLU(alpha=0.2))
model.add(Dropout(0.25))
model.add(BatchNormalization(momentum=0.8))
model.add(Conv2D(128, kernel_size=self.kernel_size, strides=2, padding="same"))
model.add(LeakyReLU(alpha=0.2))
model.add(Dropout(0.25))
model.add(BatchNormalization(momentum=0.8))
model.add(Conv2D(256, kernel_size=self.kernel_size, strides=1, padding="same"))
model.add(LeakyReLU(alpha=0.2))
model.add(Dropout(0.25))
model.add(Conv2D(512, kernel_size=self.kernel_size, strides=1, padding="same"))
model.add(LeakyReLU(alpha=0.2))
model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(1, activation='sigmoid'))
img = Input(shape=img_shape)
 validity = model(img)
 return Model(img, validity)
 def build_gan(self):
optimizer = Adam(0.0002, 0.5)
 if self.discriminator_path is not None and self.generator_path is not None and os.path.exists(self.discriminator_path)and os.path.exists(self.generator_path):
self.discriminator = load_model(self.discriminator_path)
 self.generator = load_model(self.generator_path)
 print("Loaded models...")
 else:
self.discriminator = self.build_discriminator()
self.discriminator.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
 self.generator = self.build_generator()
self.generator.compile(loss='binary_crossentropy', optimizer=optimizer)
 z = Input(shape=(100,))
img = self.generator(z)
 self.discriminator.trainable = False
valid = self.discriminator(img)
 self.combined = Model(z, valid)
 self.combined.compile(loss='binary_crossentropy', optimizer=optimizer)
 def train(self, epochs, batch_size=32, save_interval=50):
self.build_gan()
X_train = self.data_loader.load_images()
print("Training Data Shape: ", X_train.shape)
 X_train =(X_train.astype(np.float32)- 127.5)/ 127.5
half_batch = batch_size // 2
for epoch in range(epochs):
noise = np.random.normal(0, 1,(batch_size, 100))
g_loss = self.combined.train_on_batch(noise, np.ones(( batch_size, 1)))
 idx = np.random.randint(0, X_train.shape[0], half_batch)
 imgs = X_train[idx]
noise = np.random.normal(0, 1,(half_batch, 100))
gen_imgs = self.generator.predict(noise)
 d_loss_real = self.discriminator.train_on_batch(imgs, np.ones(( half_batch, 1)))
 d_loss_fake = self.discriminator.train_on_batch(gen_imgs, np.zeros(( half_batch, 1)))
 d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
 if epoch %(save_interval)== 0:
print(f"{epoch} [D loss: {d_loss[0]} | D Accuracy: {100 * d_loss[1]}] [G loss: {g_loss}]")
 def gene_imgs(self, count):
noise = np.random.normal(0, 1,(count, 100))
return self.generator.predict(noise)
 def generate_imgs(self, count, threshold):
imgs = []
for i in tqdm(range(count), desc='Generating images'):
score = [0]
while not(threshold[0] < score[0] < threshold[1]):
img = self.gene_imgs(1)
 score = self.discriminator.predict(img)
 imgs.append(img)
 imgs = np.asarray(imgs ).squeeze()
imgs = 0.5 * imgs + 0.5
return imgs
REPEAT_COLS = ['v18q','tamhog', 'tamviv', 'rez_esc','hhsize','parentesco1',
'parentesco2','parentesco3','parentesco4','parentesco5','parentesco6','parentesco7',
'parentesco8','parentesco9','parentesco10','parentesco11', 'parentesco12',
'idhogar','mobilephone', 'edjefa', 'edjefe', 'escolari', 'instlevel1',
'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7',
'instlevel8', 'instlevel9']<drop_column>
epochs = 530000
batch_size = 24
save_interval = 1000
output_directory = './'
data_loader = DataLoader()
dcgan = DCGAN(None, None, output_directory, data_loader)
 start = time.time()
dcgan.train(epochs=epochs, batch_size=batch_size, save_interval=save_interval)
 end = time.time()
print(end - start )
<count_values><EOS> 
load_generator = "./generat.h5"
load_discriminator = "./discrim.h5"
sample = 10000
sample_thresholds =(0.0, 1.0)
 dcgan = DCGAN(load_discriminator, load_generator, output_directory, data_loader)
 dcgan.build_gan()
zipper = ImageZipper(folder=output_directory)
 for i in range(0, sample, 100):
print('Images generated', i)
 images = dcgan.generate_imgs(100, sample_thresholds)
 for index, image in enumerate(images):
zipper.add_image(image, str(i + index))
zipper.close()