Kaggle competition: Generative Dog Images
<SOS> metric: PostProcessorKernel Kaggle data source: generative-dog-images<feature_engineering>
import numpy as np
import os
import shutil
import matplotlib.pyplot as plt
import zipfile
import tensorflow as tf
import xml.etree.ElementTree as ET
from tqdm import tqdm
from keras.models import *
from keras.layers.core import *
from keras.layers import *
from keras.layers.advanced_activations import LeakyReLU
from keras.initializers import RandomNormal
from keras.optimizers import Adam
from keras import backend as K
from PIL import Image
df_macro["timestamp"] = pd.to_datetime(df_macro["timestamp"])
 df_macro["year"] = df_macro["timestamp"].dt.year
df_macro["month"] = df_macro["timestamp"].dt.month
df_macro["yearmonth"] = 100*df_macro.year + df_macro.month
macmeds = df_macro.groupby("yearmonth" ).median()
df_train["timestamp"] = pd.to_datetime(df_train["timestamp"])
 df_train["year"] = df_train["timestamp"].dt.year
df_train["month"] = df_train["timestamp"].dt.month
df_train["yearmonth"] = 100*df_train.year + df_train.month
prices = pd.DataFrame({"yearmonth":df_train.yearmonth.values,
"price_doc":y_train })
 p = prices.groupby("yearmonth" ).median()
df = macmeds.join(p )<categorify>
NOISE_SIZE = 64
LR_D = 0.0001
LR_G = 0.0002
BATCH_SIZE = 64
EPOCHS = 650
BETA1 = 0.5
SEED = 4250
random_dim = 128
img_rows = 64
img_cols = 64
channels = 3
SEED = 4250
np.random.seed(SEED)
 random_dim = 128
def almonZmatrix(X, maxlag, maxdeg):

 n = len(X)
 Z = ml.zeros(( len(X)-maxlag, maxdeg+1))
for t in range(maxlag, n):
Z[t-maxlag,0] = sum([X[t-lag] for lag in range(maxlag+1)])
 for j in range(1, maxdeg+1):
s = 0.0
for i in range(1, maxlag+1):
s +=(i)**j * X[t-i]
Z[t-maxlag,j] = s
return Z<prepare_x_and_y>
ROOT_DIR = '.. /input/'
IMAGES_DIR = ROOT_DIR + 'all-dogs/all-dogs/'
BREEDS_DIR = ROOT_DIR + 'annotation/Annotation/'
IMAGES = os.listdir(IMAGES_DIR)
 BREEDS = os.listdir(BREEDS_DIR)
 print('Total Images: {}'.format(len(IMAGES)))
 print('Total Annotations: {}'.format(len(BREEDS)) )
y_macro = df.price_doc.div(df.cpi ).apply(np.log ).loc[201108:201506]
nobs = 47
tblags = 5
mrlags = 5
ztb = almonZmatrix(df.balance_trade.loc[201103:201506].as_matrix() , tblags, 1)
 zmr = almonZmatrix(df.mortgage_rate.loc[201103:201506].as_matrix() , mrlags, 1)
 columns = ['tb0', 'tb1', 'mr0', 'mr1']
z = pd.DataFrame(np.concatenate(( ztb, zmr), axis=1), y_macro.index.values, columns)
 X_macro = sm.add_constant(z )<train_model>
def load_images() :
all_images = np.zeros(( 22250, 64, 64, 3))
index = 0
for breed in BREEDS:
for dog in os.listdir(BREEDS_DIR + breed):
try: img = Image.open(IMAGES_DIR + dog + '.jpg')
 except: continue
tree = ET.parse(BREEDS_DIR + breed + '/' + dog)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 xdelta = xmax - xmin
ydelta = ymax - ymin
if xdelta >= 64 and ydelta >= 64:
img2 = img.crop(( xmin, ymin, xmax, ymax))
img2 = img2.resize(( 64, 64), Image.ANTIALIAS)
 image = np.asarray(img2)
 all_images[index,:] =(image.astype(np.float32)- 127.5)/127.5
index += 1
if index % 1000 == 0:
print('Processed Images: {}'.format(index))
print('Total Processed Images: {}'.format(index))
return all_images
macro_fit = sm.OLS(y_macro, X_macro ).fit()<predict_on_test>
class AdamWithWeightnorm(Adam):
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr *=(1./(1.+ self.decay * K.cast(self.iterations, K.floatx())))
 t = K.cast(self.iterations + 1, K.floatx())
 lr_t = lr * K.sqrt(1.- K.pow(self.beta_2, t)) /(1.- K.pow(self.beta_1, t))
shapes = [K.get_variable_shape(p)for p in params]
ms = [K.zeros(shape)for shape in shapes]
vs = [K.zeros(shape)for shape in shapes]
self.weights = [self.iterations] + ms + vs
for p, g, m, v in zip(params, grads, ms, vs):
ps = K.get_variable_shape(p)
 if len(ps)>1:
V, V_norm, V_scaler, g_param, grad_g, grad_V = get_weightnorm_params_and_grads(p, g)
 V_scaler_shape = K.get_variable_shape(V_scaler)
 m_g = K.zeros(V_scaler_shape)
 v_g = K.zeros(V_scaler_shape)
 m_g_t =(self.beta_1 * m_g)+(1.- self.beta_1)* grad_g
v_g_t =(self.beta_2 * v_g)+(1.- self.beta_2)* K.square(grad_g)
 new_g_param = g_param - lr_t * m_g_t /(K.sqrt(v_g_t)+ self.epsilon)
 self.updates.append(K.update(m_g, m_g_t))
self.updates.append(K.update(v_g, v_g_t))
m_t =(self.beta_1 * m)+(1.- self.beta_1)* grad_V
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(grad_V)
 new_V_param = V - lr_t * m_t /(K.sqrt(v_t)+ self.epsilon)
 self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
if getattr(p, 'constraint', None)is not None:
new_V_param = p.constraint(new_V_param)
 add_weightnorm_param_updates(self.updates, new_V_param, new_g_param, p, V_scaler)
 else:
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 p_t = p - lr_t * m_t /(K.sqrt(v_t)+ self.epsilon)
 self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
def get_weightnorm_params_and_grads(p, g):
ps = K.get_variable_shape(p)
 V_scaler_shape =(ps[-1],)
 V_scaler = K.ones(V_scaler_shape)
 norm_axes = [i for i in range(len(ps)- 1)]
V = p / tf.reshape(V_scaler, [1] * len(norm_axes)+ [-1])
 V_norm = tf.sqrt(tf.reduce_sum(tf.square(V), norm_axes))
g_param = V_scaler * V_norm
grad_g = tf.reduce_sum(g * V, norm_axes)/ V_norm
grad_V = tf.reshape(V_scaler, [1] * len(norm_axes)+ [-1])* \ 
(g - tf.reshape(grad_g / V_norm, [1] * len(norm_axes)+ [-1])* V)
 return V, V_norm, V_scaler, g_param, grad_g, grad_V
def add_weightnorm_param_updates(updates, new_V_param, new_g_param, W, V_scaler):
ps = K.get_variable_shape(new_V_param)
 norm_axes = [i for i in range(len(ps)- 1)]
new_V_norm = tf.sqrt(tf.reduce_sum(tf.square(new_V_param), norm_axes))
new_V_scaler = new_g_param / new_V_norm
new_W = tf.reshape(new_V_scaler, [1] * len(norm_axes)+ [-1])* new_V_param
updates.append(K.update(W, new_W))
updates.append(K.update(V_scaler, new_V_scaler))
def data_based_init(model, input):
if type(input)is dict:
feed_dict = input
elif type(input)is list:
feed_dict = {tf_inp: np_inp for tf_inp,np_inp in zip(model.inputs,input)}
else:
feed_dict = {model.inputs[0]: input}
if model.uses_learning_phase and K.learning_phase() not in feed_dict:
feed_dict.update({K.learning_phase() : 1})
 layer_output_weight_bias = []
for l in model.layers:
trainable_weights = l.trainable_weights
if len(trainable_weights)== 2:
W,b = trainable_weights
assert(l.built)
 layer_output_weight_bias.append(( l.name,l.get_output_at(0),W,b))
sess = K.get_session()
for l,o,W,b in layer_output_weight_bias:
print('Performing data dependent initialization for layer ' + l)
 m,v = tf.nn.moments(o, [i for i in range(len(o.get_shape())-1)])
 s = tf.sqrt(v + 1e-10)
 updates = tf.group(W.assign(W/tf.reshape(s,[1]*(len(W.get_shape())-1)+[-1])) , b.assign(( b-m)/s))
sess.run(updates, feed_dict )
test_cpi = df.cpi.loc[201507:201605]
test_index = test_cpi.index
ztb_test = almonZmatrix(df.balance_trade.loc[201502:201605].as_matrix() , tblags, 1)
 zmr_test = almonZmatrix(df.mortgage_rate.loc[201502:201605].as_matrix() , mrlags, 1)
 z_test = pd.DataFrame(np.concatenate(( ztb_test, zmr_test), axis=1), test_index, columns)
 X_macro_test = sm.add_constant(z_test)
 pred_lnrp = macro_fit.predict(X_macro_test)
 pred_p = np.exp(pred_lnrp)* test_cpi<create_dataframe>
def create_generator_model() :
init = RandomNormal(mean = 0.0, stddev = 0.02)
 model = Sequential()
start_shape = NOISE_SIZE * 4 * 4
model.add(Dense(start_shape, kernel_initializer = init, input_dim = random_dim))
model.add(Reshape(( 4, 4, 64)))
 model.add(UpSampling2D())
 model.add(Conv2D(128, kernel_size = 3, padding = "same", kernel_initializer = init))
model.add(ReLU())
 model.add(UpSampling2D())
 model.add(Conv2D(128, kernel_size = 3, padding = "same", kernel_initializer = init))
model.add(ReLU())
 model.add(UpSampling2D())
 model.add(Conv2D(128, kernel_size = 3, padding = "same", kernel_initializer = init))
model.add(ReLU())
 model.add(UpSampling2D())
 model.add(Conv2D(128, kernel_size = 3, padding = "same", kernel_initializer = init))
model.add(ReLU())
 model.add(Conv2D(3, kernel_size = 3, activation = 'tanh', padding = 'same', kernel_initializer=init))
model.compile(loss = 'binary_crossentropy', optimizer = AdamWithWeightnorm(lr = LR_G, beta_1 = BETA1))
print(model.summary())
 return model
adjust = pd.DataFrame(pred_p/test_p.price_doc, columns=["adjustment"])
 adjust<prepare_output>
def create_discriminator_model() :
input_shape =(img_rows, img_cols, channels)
 init = RandomNormal(mean = 0.0, stddev = 0.02)
 model = Sequential()
model.add(Conv2D(128, kernel_size = 3, strides = 2, padding = 'same', kernel_initializer = init, input_shape = input_shape))
model.add(LeakyReLU(0.2))
model.add(Dropout(0.25))
model.add(Conv2D(128, kernel_size = 3, strides = 2, padding = 'same', kernel_initializer = init))
model.add(LeakyReLU(0.2))
model.add(Dropout(0.25))
model.add(Conv2D(128, kernel_size = 3, strides = 2, padding = 'same', kernel_initializer = init))
model.add(LeakyReLU(0.2))
model.add(Dropout(0.25))
model.add(Conv2D(128, kernel_size = 3, strides = 2, padding = 'same', kernel_initializer = init))
model.add(LeakyReLU(0.2))
model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(1, activation = 'sigmoid', kernel_initializer = init))
model.compile(loss = 'binary_crossentropy', optimizer = AdamWithWeightnorm(lr = LR_D , beta_1 = BETA1))
print(model.summary())
 return model
combo = test_data.join(adjust, on='yearmonth')
 combo['adjusted'] = combo.price_doc * combo.adjustment
adjxgb_df = pd.DataFrame()
adjxgb_df['id'] = combo.id
adjxgb_df['price_doc'] = combo.adjusted
adjxgb_df.head()<save_to_csv>
def create_gan_model(discriminator, random_dim, generator):
discriminator.trainable = False
gan_input = Input(shape =(random_dim,))
generator_output = generator(gan_input)
 gan_output = discriminator(generator_output)
 gan_model = Model(inputs = gan_input, outputs = gan_output)
 gan_model.compile(loss = 'binary_crossentropy', optimizer = AdamWithWeightnorm(lr = LR_G, beta_1 = BETA1))
print(gan_model.summary())
 return gan_model
adjxgb_df.to_csv('adjusted_xgb_predicitons.csv', index=False )<load_from_csv>
def generator_input(latent_dim, n_samples):
input = np.random.randn(latent_dim * n_samples)
 input = input.reshape(( n_samples, latent_dim))
return input
stemmer = PorterStemmer()
random.seed(1966)
 df_train = pd.read_csv('.. /input/train.csv', encoding="ISO-8859-1")
 df_test = pd.read_csv('.. /input/test.csv', encoding="ISO-8859-1")
 df_pro_desc = pd.read_csv('.. /input/product_descriptions.csv')
 df_attr = pd.read_csv('.. /input/attributes.csv')
 df_brand = df_attr[df_attr.name == "MFG Brand Name"][["product_uid", "value"]].rename(columns={"value": "brand"})
 num_train = df_train.shape[0]
def str_stem(s):
if isinstance(s, str):
s = s.replace(" in.","in.")
 s = s.replace(" inch","in.")
 s = s.replace("inch","in.")
 s = s.replace(" in ","in.")
 s = s.replace(" ft.","ft.")
 s = s.replace(" feet","ft.")
 return " ".join([stemmer.stem(re.sub('[^A-Za-z0-9-./]', ' ', word)) for word in s.lower().split() ])
 else:
return "null"
def str_common_word(str1, str2):
str1, str2 = str1.lower() , str2.lower()
words, cnt = str1.split() , 0
for word in words:
if str2.find(word)>=0:
cnt+=1
return cnt
def str_whole_word(str1, str2, i_):
str1, str2 = str1.lower().strip() , str2.lower().strip()
cnt = 0
while i_ < len(str2):
i_ = str2.find(str1, i_)
 if i_ == -1:
return cnt
else:
cnt += 1
i_ += len(str1)
 return cnt
def fmean_squared_error(ground_truth, predictions):
fmean_squared_error_ = mean_squared_error(ground_truth, predictions)**0.5
return fmean_squared_error_
RMSE = make_scorer(fmean_squared_error, greater_is_better=False)
 df_all = pd.concat(( df_train, df_test), axis=0, ignore_index=True)
 df_all = pd.merge(df_all, df_pro_desc, how='left', on='product_uid')
 df_all = pd.merge(df_all, df_brand, how='left', on='product_uid')
 df_all['search_term'] = df_all['search_term'].map(lambda x:str_stem(x))
df_all['product_title'] = df_all['product_title'].map(lambda x:str_stem(x))
df_all['product_description'] = df_all['product_description'].map(lambda x:str_stem(x)) 
<feature_engineering>
def train_model(epochs = 1, batch_size = 128):
x_train = load_images()
batch_count = x_train.shape[0] / batch_size
generator = create_generator_model()
discriminator = create_discriminator_model()
gan_model = create_gan_model(discriminator, random_dim, generator)
 discriminator_fake_hist, discriminator_real_hist, generator_hist = [], [], []
for e in range(epochs):
script_stopper_counter = 0
print('======================== Epoch {} ============================='.format(e))
for _ in tqdm(range(int(batch_count))):
discriminator_fake_loss, discriminator_real_loss = [], []
for _ in range(2):
X_fake = generator.predict(generator_input(random_dim, batch_size))
y_fake = np.zeros(batch_size)
 y_fake[:] = 0
discriminator.trainable = True
d_fake_loss = discriminator.train_on_batch(X_fake, y_fake)
 X_real = x_train[np.random.randint(0, x_train.shape[0], size = batch_size)]
y_real = np.zeros(batch_size)
 y_real[:] = 0.9
discriminator.trainable = True
d_real_loss = discriminator.train_on_batch(X_real, y_real)
 discriminator_fake_loss.append(d_fake_loss)
 discriminator_real_loss.append(d_real_loss)
 noise = generator_input(random_dim, batch_size)
 y_gen = np.ones(batch_size)
 discriminator.trainable = False
generator_loss = gan_model.train_on_batch(noise, y_gen)
 d_fake_loss = %.4f, d_real_loss = %.4f g_loss = %.4f' % \
discriminator_fake_hist.append(np.mean(discriminator_fake_loss))
discriminator_real_hist.append(np.mean(discriminator_real_loss))
generator_hist.append(generator_loss)
 if np.mean(discriminator_fake_loss)> 10:
script_stopper_counter += 1
if e % 100 == 0:
plot_generated_images(e, generator)
 if script_stopper_counter > 160:
plot_generated_images(e, generator)
 break
plot_loss(discriminator_fake_hist, discriminator_real_hist, generator_hist)
 z = zipfile.PyZipFile('images.zip', mode = 'w')
 for k in range(10000):
generated_images = generator.predict(np.random.normal(0, 1, size = [1, random_dim]))
image = Image.fromarray(((generated_images + 1)* 127.5 ).astype('uint8' ).reshape(64, 64, 3))
f = str(k)+'.png'
image.save(f, 'PNG')
 z.write(f)
 os.remove(f)
 if k % 1000 == 0:
print(k)
 z.close()
df_all['brand'] = df_all['brand'].map(lambda x:str_stem(x))<data_type_conversions>
train_model(EPOCHS, BATCH_SIZE )