Kaggle competition: Generative Dog Images
<SOS> metric: PostProcessorKernel Kaggle data source: generative-dog-images<choose_model_class>
EPOCHS = 574
BATCH = 32
SIZE = 64
LRG = 0.0004
LRD = 0.0008
NZ = 64
FEATURES_G = 80
FEATURES_D = 112
LRFACTOR_G = 0.9
LRFACTOR_D = 0.9
REAL_LABEL = [0.9, 0.9]
FAKE_LABEL = [0.00, 0.00]
REAL_FAKE_LABEL = 0.9
SEED = 1234
RATIO = 1.50
PATH_ANNOTATIONS = '.. /input/annotation/Annotation'
PATH_ALL_DOGS = '.. /input/all-dogs/all-dogs'
PATH_DOGS = '.. /cropped-dogs'
n_steps = 100
cb = [ModelCheckpoint("model.hdf5", save_best_only=True, period=3)]
model = Sequential()
model.add(CuDNNGRU(48, input_shape=(None, n_features)))
 model.add(Dense(10, activation='relu'))
model.add(Dense(10, activation='relu'))
model.add(Dense(1))
model.summary()<train_model>
DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 def random_seed(seed):
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 tf.set_random_seed(seed)
 def R4(f): return F'{f:0.4}'.ljust(6, '0' ).ljust(8, ' ')
 def R6(f): return F'{f:0.6}'.ljust(8, '0' ).ljust(8, ' ')
 random_seed(SEED)
model.compile(optimizer=adam(lr=0.001), loss="mae")
 hist = model.fit_generator(train_gen,
steps_per_epoch=1000,
epochs=30,
verbose=0,
callbacks=cb,
shuffle=True,
validation_data=valid_gen,
validation_steps=200 )<save_to_csv>
%%time

 def enumerate_images(path):
for fn in os.listdir(path):
with Image.open(os.path.join(path, fn)) as image:
yield image.copy()
def enumerate_annotation_images(path_annotation, path_dogs):
breed_id = 0
breeds = os.listdir(path_annotation)
 for breed in breeds:
dog_id = 0
breed_id += 1
for dog in os.listdir(os.path.join(path_annotation, F'{breed}')) :
try:
image = Image.open(os.path.join(path_dogs, F'{dog}.jpg'))
except :
print(dog, 'not found!')
 continue
tree = ET.parse(os.path.join(path_annotation, F'{breed}', F'{dog}'))
root = tree.getroot()
objects = root.findall('object')
 for o in objects:
dog_id += 1
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 img = image.crop(( xmin, ymin, xmax, ymax))
yield img, breed_id, dog_id
def save_cropped_images(path_annotation, path_dogs, path_target, size=64):
if not os.path.exists(path_target):
os.mkdir(path_target)
 n = 0
current_id = 0
with tqdm(enumerate_annotation_images(path_annotation, path_dogs)) as t:
for image, breed_id, dog_id in t:
if current_id != breed_id:
current_id = breed_id
t.set_postfix({'id': breed_id})
 w, h = image.size
if w < size or h < size: continue
ratio = w / h
if h > w: ratio = h / w
sz = math.ceil(size * ratio)
 if ratio < RATIO:
image.thumbnail(( sz, sz), Image.ANTIALIAS)
 image.save(os.path.join(path_target, F'{breed_id:03}_{dog_id:03}.png'))
n += 1
return n
save_cropped_images(PATH_ANNOTATIONS, PATH_ALL_DOGS, PATH_DOGS)
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id', dtype={"time_to_failure": np.float32})
 for i, seg_id in enumerate(tqdm(submission.index)) :
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 x = seg['acoustic_data'].values
submission.time_to_failure[i] = model.predict(np.expand_dims(create_X(x), 0))
submission.head()
submission.to_csv('submission.csv' )<define_variables>

 class Generator(nn.Module):
def __init__(self, nz, nfeats):
super(Generator, self ).__init__()
self.conv1 = nn.ConvTranspose2d(nz, nfeats * 8, 4, 1, 0, bias=False)
 self.bn1 = nn.BatchNorm2d(nfeats * 8)
 self.conv2 = nn.ConvTranspose2d(nfeats * 8, nfeats * 8, 4, 2, 1, bias=False)
 self.bn2 = nn.BatchNorm2d(nfeats * 8)
 self.conv3 = nn.ConvTranspose2d(nfeats * 8, nfeats * 4, 4, 2, 1, bias=False)
 self.bn3 = nn.BatchNorm2d(nfeats * 4)
 self.conv4 = nn.ConvTranspose2d(nfeats * 4, nfeats * 2, 4, 2, 1, bias=False)
 self.bn4 = nn.BatchNorm2d(nfeats * 2)
 self.conv5 = nn.ConvTranspose2d(nfeats * 2, nfeats, 4, 2, 1, bias=False)
 self.bn5 = nn.BatchNorm2d(nfeats)
 self.conv6 = nn.ConvTranspose2d(nfeats, 3, 3, 1, 1, bias=False)
 def forward(self, x):
x = F.leaky_relu(self.bn1(self.conv1(x)))
 x = F.leaky_relu(self.bn2(self.conv2(x)))
 x = F.leaky_relu(self.bn3(self.conv3(x)))
 x = F.leaky_relu(self.bn4(self.conv4(x)))
 x = F.leaky_relu(self.bn5(self.conv5(x)))
 x = torch.tanh(self.conv6(x))
return x
def load(self, path, device=None):
self.load_state_dict(torch.load(path, map_location=device))
def save(self, path):
torch.save(self.state_dict() , path)

class Discriminator(nn.Module):
def __init__(self, nfeats):
super(Discriminator, self ).__init__()
self.conv1 = nn.Conv2d(3, nfeats, 4, 2, 1, bias=False)
 self.conv2 = nn.Conv2d(nfeats, nfeats * 2, 4, 2, 1, bias=False)
 self.bn2 = nn.BatchNorm2d(nfeats * 2)
 self.conv3 = nn.Conv2d(nfeats * 2, nfeats * 4, 4, 2, 1, bias=False)
 self.bn3 = nn.BatchNorm2d(nfeats * 4)
 self.conv4 = nn.Conv2d(nfeats * 4, nfeats * 8, 4, 2, 1, bias=False)
 self.bn4 = nn.BatchNorm2d(nfeats * 8)
 self.conv5 = nn.Conv2d(nfeats * 8, 1, 4, 1, 0, bias=False)
 def forward(self, x):
x = F.leaky_relu(self.conv1(x), 0.1)
 x = F.leaky_relu(self.bn2(self.conv2(x)) , 0.1)
 x = F.leaky_relu(self.bn3(self.conv3(x)) , 0.1)
 x = F.leaky_relu(self.bn4(self.conv4(x)) , 0.1)
 x = torch.sigmoid(self.conv5(x))
x = x.view(-1, 1)
 return x
def load(self, path, device=None):
self.load_state_dict(torch.load(path, map_location=device))
def save(self, path):
torch.save(self.state_dict() , path)
random_seed = 4126
cast = {
'acoustic_data': 'int',
'time_to_failure': 'float'
}
def df_fragments(path, length, skip=1):
with open(path, 'r')as f:
m = {}
cols = []
count = 0
index = 0
for line in f:
if len(cols)== 0:
for col in line.strip("
\r " ).split(','):
cols.append(col)
 continue
if count == 0:
for col in cols:
m[col] = []
if index % skip == 0:
for j, cell in enumerate(line.strip("
\r " ).split(',')) :
col = cols[j]
m[col].append(getattr(builtins, cast[col] )(cell))
count += 1
if count == length:
if index % skip == 0:
yield pd.DataFrame(m)
 index += 1
count = 0
def count_rows(path):
with open(path, 'r')as f:
i = -1
for _ in f:
i += 1
return i<define_variables>

 def image_resize(image, width=None, height=None, inter=cv2.INTER_AREA):
dim = None 
(h, w)= image.shape[:2]
if width is None and height is None:
return image
if width is None:
r = height / float(h)
 width = max(height, int(w * r))
else:
r = width / float(w)
 height = max(width, int(h * r))
dim =(width, height)
 resized = cv2.resize(image, dim, interpolation=inter)
 return resized
def random_crop(arr_image, width, height):
x = random.randint(0, arr_image.shape[1] - width)
 y = random.randint(0, arr_image.shape[0] - height)
 arr_image = arr_image[y:y + height, x:x + width]
return arr_image
def random_fliplr(arr_image):
r = random.randint(0, 2)
 if r == 1:
return np.fliplr(arr_image)
 return arr_image
class ArrayDataset(Dataset):
def __init__(self, path, size=64, margin=0, mean=None, std=None, count=None):
self.size = size
self.margin = margin
files = os.listdir(path)
 if count is not None:
files = files[:count]
self.arrange(path, files)
 if mean is None or std is None:
m, s = self.calc_norm()
if mean is None: mean = m
if std is None: std = s
self.mean = np.array(mean ).reshape(-1, 1, 1)
 self.std = np.array(std ).reshape(-1, 1, 1)
 def arrange(self, path, files):
self.images = []
self.labels = []
sizem = self.size + self.margin
for fn in files:
image = cv2.imread(os.path.join(path, fn))
if image.shape[0] > image.shape[1]:
image = image_resize(image, width=sizem)
 else:
image = image_resize(image, height=sizem)
 image = image[..., ::-1].copy()
self.images.append(image)
 lbl = fn[:3]
if str.isdigit(lbl):
self.labels.append(int(lbl))
else:
self.labels.append(0)
 def calc_norm(self):
mean = np.zeros(3)
 std = np.zeros(3)
 for x in self.images:
x = np.transpose(x,(2, 0, 1))
x = x.reshape(3, -1)
 mean += x.mean(1)
 std += x.std(1)
 m =(mean / len(self.images)) / 255.0
s =(std / len(self.images)) / 255.0
return m, s
def __len__(self):
return len(self.images)
 def __getitem__(self, idx):
x = self.images[idx]
x = random_crop(x, self.size, self.size)
 x = random_fliplr(x ).copy()
x = np.transpose(x,(2, 0, 1))
x = x / 255.0
x = x - self.mean
x = x / self.std
x = x.astype(np.float32)
 return x, self.labels[idx]
print('counting total...')
 total = count_rows('.. /input/train.csv')
 print('total: {}'.format(total))<load_from_csv>
%%time

 train_data = ArrayDataset(PATH_DOGS, size=SIZE, margin=0)
 train_loader = torch.utils.data.DataLoader(train_data, shuffle=True, batch_size=BATCH, num_workers=4, drop_last=True)
 netG = Generator(NZ, FEATURES_G ).to(DEVICE)
 netD = Discriminator(FEATURES_D ).to(DEVICE)
 criterion = nn.BCELoss()
optimizerG = optim.Adam(netG.parameters() , lr=LRG, betas=(0.5, 0.999))
optimizerD = optim.Adam(netD.parameters() , lr=LRD, betas=(0.5, 0.999))
schedulerG = op.lr_scheduler.LambdaLR(optimizerG, lambda epoch: LRFACTOR_G ** epoch)
 schedulerD = op.lr_scheduler.LambdaLR(optimizerD, lambda epoch: LRFACTOR_D ** epoch)
 def noise_generator(batch_size):
return torch.randn(batch_size, NZ, 1, 1)
 def label_generator(batch_size, mode):
if mode == 'real':
label = np.random.uniform(*REAL_LABEL, batch_size ).reshape(batch_size, 1)
 return torch.tensor(label, dtype=torch.float32)
 if mode == 'fake':
label = np.random.uniform(*FAKE_LABEL, batch_size ).reshape(batch_size, 1)
 return torch.tensor(label, dtype=torch.float32)
 if mode == 'real-fake':
return torch.full(( batch_size, 1), REAL_FAKE_LABEL)
 raise Exception('Invalid parameter mode')
print('generating train data...')
 fragment_size = 150000
skip = 1
mfcc_ttf_map = {}
for df in tqdm(df_fragments('.. /input/train.csv', length=fragment_size, skip=skip), total=(total//fragment_size)//skip):
mfcc = librosa.feature.mfcc(df['acoustic_data'].values.astype('float32'))
mfcc_mean = mfcc.mean(axis=1)
 for i, each_mfcc_mean in enumerate(mfcc_mean):
key = 'mfcc_{}'.format(i)
 if key not in mfcc_ttf_map:
mfcc_ttf_map[key] = []
mfcc_ttf_map[key].append(each_mfcc_mean)
 key = 'time_to_failure'
if key not in mfcc_ttf_map:
mfcc_ttf_map[key] = []
mfcc_ttf_map[key].append(df.iloc[-1][key])
 mfcc_ttf_df = pd.DataFrame(mfcc_ttf_map)
 fname = 'mfcc_train.csv'
mfcc_ttf_df.to_csv(fname, index=False)
 print('saved {}.'.format(fname)) 
<compute_train_metric>
def needs_scheduler(epoch):
if epoch > 0:
if epoch % 50 == 0:
print('Scheduler update')
 return True
return False
def report_cv(model):
X = mfcc_ttf_df.drop(['time_to_failure'], axis=1 ).values
y = mfcc_ttf_df['time_to_failure'].values
scores = cross_val_score(model, X, y, scoring='neg_mean_absolute_error', cv=10)
 print('Cross Validation scores: {}'.format(abs(scores)))
 print('Average score: {}'.format(abs(scores.mean())))
 report_cv(LinearRegression() )<set_options>
torch.save(netG.state_dict() , 'generator.pth')
 torch.save(netD.state_dict() , 'discriminator.pth')
report_cv(XGBRegressor(random_state=random_seed))<feature_engineering>
SEEDS = [
1525, 491, 1117, 1509, 1538, 1478, 16, 1251, 1126, 81, 254, 704, 1740, 2081, 79, 980, 1770, 795, 1317, 1471, 2198, 153, 1488, 1456, 454, 573, 690, 2289, 1753, 2075, 772, 301, 1240, 83, 686, 932, 826, 27, 24, 1744, 880, 2029, 1421, 1490, 667, 1387, 1948, 2130, 15, 133, 2151, 602, 76, 2090, 1277, 717, 2175, 2168, 1313, 1562, 1526, 1092, 800, 1804, 1605, 1120, 2320, 2372, 2080, 460, 126, 2302, 1940, 983, 668, 773, 1788, 565, 1540, 2337, 140, 2173, 2010, 1784, 1248, 1106, 1722, 208, 1757, 2371, 449, 481, 1914, 2217, 437, 1161, 169, 693, 807, 1839, 217, 1520, 1102, 30, 1199, 239, 603, 730, 569, 2309, 1434, 1679, 3, 313, 532, 35, 825, 824, 388, 255, 1758, 763, 87, 751, 2073, 585, 407, 1510, 365, 739, 417, 463, 1412, 1636, 1237, 1439, 278, 227, 1916, 2261, 154, 1622, 1833, 2213, 1738, 74, 892, 1203, 2001, 1227, 815, 872, 1769, 699, 2374, 1050, 1073, 620, 202, 1420, 2211, 840, 568, 1715, 1337, 1651, 1591, 878, 1445, 1100, 1702, 551, 72, 1917, 1256, 1479, 152, 2192, 559, 2031, 1920, 23, 2304, 2153, 1848, 2032, 1976, 578, 650, 1464, 1706, 820, 2232, 762, 33, 1627, 1141, 2262, 781, 174,
]
print('generating test features...')
 test_dir = '.. /input/test'
test_map = {}
for fname in tqdm(os.listdir(test_dir)) :
path = test_dir + '/' + fname
df = pd.read_csv(path)
 mfcc = librosa.feature.mfcc(df['acoustic_data'].values.astype('float32'))
mfcc_mean = mfcc.mean(axis=1)
 for i, each_mfcc_mean in enumerate(mfcc_mean):
key = 'mfcc_{}'.format(i)
 if key not in test_map:
test_map[key] = []
test_map[key].append(each_mfcc_mean)
 key = 'seg_id'
if key not in test_map:
test_map[key] = []
test_map[key].append(re.sub('.csv$', '', fname))
test_df = pd.DataFrame(test_map)
 test_csv = 'mfcc_test.csv'
test_df.to_csv(test_csv, index=False)
 print('saved {}'.format(test_csv))
def submit(model, file_path):
X = mfcc_ttf_df.drop(['time_to_failure'], axis=1 ).values
y = mfcc_ttf_df['time_to_failure'].values
model.fit(X, y)
 X_submit = test_df.drop(['seg_id'], axis=1 ).values
y_submit = model.predict(X_submit)
 submit_df = pd.DataFrame({
'seg_id': test_df['seg_id'].values,
'time_to_failure': y_submit
})
 submit_df.to_csv(file_path, index=False)
 print('saved {}'.format(file_path)) 
<set_options>
state_dict = torch.load('generator.pth')
 def load_model(model):
model.load_state_dict(state_dict)
 def generate_numpy_images(netG, count, batch_size, noise_gen):
netG.eval()
images_set = []
with torch.no_grad() :
for n in range(count // batch_size):
gen_z = noise_gen(batch_size ).to(DEVICE)
 gen_images = netG(gen_z)
 aimages = gen_images.detach().cpu()
aimages =(aimages + 1.0)* 0.5 * 255
images = aimages.numpy().transpose(0, 2, 3, 1)
 images = images.astype(int)
 images_set.append(images)
 images = np.concatenate(images_set)
 return images
def generate_tensors(netG, count, batch_size, noise_gen):
netG.eval()
images_set = []
with torch.no_grad() :
for n in range(count // batch_size):
gen_z = noise_gen(batch_size ).to(DEVICE)
 gen_images = netG(gen_z)
 images = gen_images.detach().cpu()
images_set.append(images)
 images = torch.cat(images_set)
 netG.train()
return images
def generate_images(count, batch, seed):
netG = Generator(NZ, FEATURES_G ).to(DEVICE)
 load_model(netG)
 random_seed(seed)
 images = generate_tensors(netG, count, batch, noise_generator)
 return images
<load_from_csv><EOS> 
%%time
PATH_GENERATED = '.. /output_images'
if not os.path.exists(PATH_GENERATED):
os.mkdir(PATH_GENERATED)
 n = 0
for seed in SEEDS:
images = generate_images(50, 50, seed)
 images =(images + 1.0)* 0.5
for ix in range(50):
image = images[ix]
save_image(image, os.path.join(PATH_GENERATED, f'image_{n:05d}.png'), normalize=False)
 n += 1
print(n)
 shutil.make_archive('images', 'zip', PATH_GENERATED)