Insignificant GAN training output

Henry_Chibueze: <p>After training a GAN for 40 epochs it still generates random noise as output. Can anyone pls tell me what’s wrong here coz i can’t see whats wrong.<br>
I mean even if 40 epochs is not enough it should still be able to generate sth better than random noise don’t you think ? <img src="https://discuss.pytorch.org/images/emoji/apple/disappointed_relieved.png?v=9" title=":disappointed_relieved:" class="emoji" alt=":disappointed_relieved:"> <img src="https://discuss.pytorch.org/images/emoji/apple/sob.png?v=9" title=":sob:" class="emoji" alt=":sob:"> <img src="https://discuss.pytorch.org/images/emoji/apple/cry.png?v=9" title=":cry:" class="emoji" alt=":cry:"> <img src="https://discuss.pytorch.org/images/emoji/apple/cold_sweat.png?v=9" title=":cold_sweat:" class="emoji" alt=":cold_sweat:"></p>
<pre><code class="lang-auto">import numpy as np
from tqdm import tqdm
import os, cv2, torch
import torch.nn as nn
from torchvision.utils import save_image
import random
import matplotlib.pyplot as plt

#Data preprocessing
data = []
def dataPreprocessor(raw_data_path, save_data_path):

    for single_data in tqdm(os.listdir(raw_data_path)):

        try:
            img = os.path.join(raw_data_path, single_data)
            img = cv2.imread(img)
            img = cv2.resize(img, (64, 64))
            img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
            img = np.array(img)
            data.append([img])

        except Exception as e:
            pass
   
    np.random.shuffle(data)
    np.save(save_data_path, data, allow_pickle=True)

device = torch.device('cuda' if torch.cuda.is_available() == True else 'cpu')

raw_data_path = 'raw_data/training_data'
train_path = 'processed_data/training_dataset/train_dataset.npy'
saved_model_path = 'saved_model/DCGAN_network.pth.tar'
#Hyper-parameters
in_channels_D = 1
out_channels_D = 1
in_noise_channels_G = 256
out_channels_G = in_channels_D
lr = 0.0002
EPOCHS = 40
fake_label = 0 #~0
real_label = 1 #~1
num_of_eval_data = 3
real_desc_loss = []
fake_desc_loss = []
Generator_loss = []

def save_model(model, path=saved_model_path):
    print('Saving model.......')
    torch.save(model, path)
    print('AI model has been saved.....')


def trainData():
    if not os.path.isfile(train_path):
        print("Preprocessing Data....... ")
        dataPreprocessor(raw_data_path=raw_data_path, save_data_path=train_path)
    else:
        pass


trainData()


from neuralNets import Descriminator, Generator
DescNet = Descriminator(in_channels_D, out_channels_D).to(device=device)
GenNet = Generator(in_noise_channels_G, out_channels_G).to(device=device)


import torch.optim as optim
optimizer_D = optim.Adam(DescNet.parameters(), lr=lr, betas=(0.5, 0.999))
optimizer_G = optim.Adam(GenNet.parameters(), lr=lr, betas=(0.5, 0.999))
lossFunc = nn.BCELoss()


def dataLoader(path):
    data = np.load(path, allow_pickle=True)
    X = np.array([i[0] for i in data])
    X = X.reshape(len(X), 64, 64, -1)
    X = X.transpose(0, 3, 1, 2)
    X = torch.Tensor(X)
    X = X/255
    return X


def Save_training_instances(save_data, directory, path):
    cv2.imwrite(f'{os.path.join(directory, path)}', save_data)


def trainProcess(epochs, BATCH_SIZE = 60):

    if not os.path.isfile(saved_model_path):

        epoch_tab = []
        Gen_loss = np.array([])
        D_real_loss = np.array([])
        D_fake_loss = np.array([])

        X = dataLoader(path=train_path)

        for epoch in range(epochs):
            epoch_tab.append(epochs)
            for imgs in tqdm(range(0, len(X), BATCH_SIZE)):
                X_batch = X[imgs:imgs+BATCH_SIZE].to(device=device)
                noise = torch.randn(X_batch.shape[0], in_noise_channels_G, 1, 1).to(device=device)
                real_target = torch.Tensor(X_batch.shape[0], 1).fill_(real_label).to(device=device)
                fake_target = torch.Tensor(X_batch.shape[0], 1).fill_(fake_label).to(device=device)

                #Train Descriminator
                DescNet.zero_grad()
                real_output = DescNet(X_batch)
                real_loss = lossFunc(real_output, real_target*0.9)

                r'the gradient of the output of the generator is detached,\
                  while training the Descriminator to prevent back_prop on it'
                G_output = GenNet(noise)
                fake_output = DescNet(G_output.detach())
                fake_loss = lossFunc(fake_output, fake_target*0.1)

                D_loss = real_loss + fake_loss
                D_loss.backward()
                optimizer_D.step()
                
                #Train Generator
                GenNet.zero_grad()
                D_output = DescNet(G_output)
                G_loss = lossFunc(D_output, real_target)
                optimizer_G.step()

                #append losses
                D_real_loss = np.append(D_real_loss, real_loss.data)
                D_fake_loss = np.append(D_fake_loss, fake_loss.data)
                Gen_loss = np.append(Gen_loss, G_loss.data)

                if epoch % 2 == 0:
                    save_image(G_output[:20], f'output/{epoch}.jpg', nrow=5, normalize=True)

            real_desc_loss.append(np.mean(D_real_loss))
            fake_desc_loss.append(np.mean(D_fake_loss))
            Generator_loss.append(np.mean(Gen_loss))
            print(f'epoch: {epoch+1}\t desc_real_loss:{np.mean(D_real_loss)}\t \
                desc_fake_loss: {np.mean(D_fake_loss)}\t Gen_loss: {np.mean(Gen_loss)}')
            D_real_loss = np.array([])
            D_fake_loss = np.array([])
            Gen_loss = np.array([])

        model_state = {'Gen_model':GenNet.state_dict(), 'Gen_optimizer':optimizer_D.state_dict(),
                        'Desc_model':DescNet.state_dict(),'Desc_optimizer':optimizer_D.state_dict(),
                        'loss_func':lossFunc.state_dict()}
        save_model(model_state)

    else:
        pass


def evalProcess(no_img):
    fixed_input_noise = torch.randn(no_img, in_noise_channels_G, 1, 1).to(device=device)
    with torch.no_grad():
        model_data = torch.load(saved_model_path)
        Gen_model = Generator(in_noise_channels_G, out_channels_G)
        Gen_model.load_state_dict(model_data['Gen_model'])
        Gen_model.eval().to(device=device)
        for num, img in zip(range(no_img), fixed_input_noise):
            img = img.to(device=device)
            img = img.view(1, in_noise_channels_G, 1, 1)
            output_img = Gen_model(img)
            output_img = output_img.detach()
            output_img = np.array(output_img*255)
            output_img = output_img.transpose(0, 2, 3, 1)
            #output_img = cv2.cvtColor(output_img, cv2.COLOR_GRAY2RGB)
            cv2.imshow(f'data{num}', output_img[0])
        cv2.waitKey(0)
        

trainProcess(EPOCHS)
evalProcess(num_of_eval_data)
</code></pre>
<p>Also here’s how my neural network looks like.</p>
<pre><code class="lang-auto">import torch
import torch.nn as nn

class Descriminator(nn.Module):
    def __init__(self, in_channels, out_channels):
        super(Descriminator, self).__init__()
        self.in_channels = in_channels
        self.out_channels = out_channels

        self.layer1 = nn.Sequential(
                        nn.Conv2d(self.in_channels, 256, 
                            kernel_size=4, stride=2, padding=1,
                            bias=False),
                        nn.LeakyReLU(0.2)
                    )

        self.layer2 = nn.Sequential(
                        nn.Conv2d(256, 512,
                            kernel_size=4, stride=2, padding=1,
                            bias=False),
                        nn.BatchNorm2d(512),
                        nn.LeakyReLU(0.2)
                    )

        self.layer3 = nn.Sequential(
                        nn.Conv2d(512, 1024,
                            kernel_size=4, stride=2, padding=1,
                            bias=False),
                        nn.BatchNorm2d(1024),
                        nn.LeakyReLU(0.2)
                    )

        self.layer4 = nn.Sequential(
                        nn.Conv2d(1024, 2048,
                            kernel_size=4, stride=2, padding=1,
                            bias=False),
                        nn.BatchNorm2d(2048),
                        nn.LeakyReLU(0.2)
                    )

        self.layer5 = nn.Sequential(
                        nn.Conv2d(2048, 4096,
                            kernel_size=4, stride=1, padding=0,
                            bias=False),
                        nn.BatchNorm2d(4096),
                        nn.LeakyReLU(0.2)
                    )

        self._linear_shape = 4096*1*1

        self.layer6 = nn.Sequential(
                        nn.Linear(self._linear_shape, self.out_channels),
                        nn.Sigmoid()
                    )

    def convs_fwd(self, x):
        output = self.layer1(x)
        output = self.layer2(output)
        output = self.layer3(output)
        output = self.layer4(output)
        output = self.layer5(output)

        return output

        
    def forward(self, input):
        output = self.convs_fwd(input)
        output = output.view(-1, self._linear_shape)
        output = self.layer6(output)

        return output
            


class Generator(nn.Module):
    def __init__(self, noise_channels, out_channels):
        super(Generator, self).__init__()
        self.noise_channel = noise_channels
        self.out_channels = out_channels

        self.layer1 = nn.Sequential(
                        nn.ConvTranspose2d(noise_channels, 1024,
                            kernel_size=4, stride=1, padding=0,
                            bias=False),
                        nn.BatchNorm2d(1024),
                        nn.ReLU(True)
                    )

        self.layer2 = nn.Sequential(
                        nn.ConvTranspose2d(1024, 512,
                            kernel_size=4, stride=2, padding=1,
                            bias=False),
                        nn.BatchNorm2d(512),
                        nn.ReLU(True)
                    )

        self.layer3 = nn.Sequential(
                        nn.ConvTranspose2d(512, 256,
                            kernel_size=4, stride=2, padding=1,
                            bias=False),
                        nn.BatchNorm2d(256),
                        nn.ReLU(True)
                    )

        self.layer4 = nn.Sequential(
                        nn.ConvTranspose2d(256, 128,
                            kernel_size=4, stride=2, padding=1,
                            bias=False),
                        nn.BatchNorm2d(128),
                        nn.ReLU(True)
                    )

        self.layer5 = nn.Sequential(
                        nn.ConvTranspose2d(128, self.out_channels,
                            kernel_size=4, stride=2, padding=1,
                            bias=False),
                        nn.Tanh()
                    )

    def forward(self, input):
        output = self.layer1(input)
        output = self.layer2(output)
        output = self.layer3(output)
        output = self.layer4(output)
        output = self.layer5(output)

        return output
</code></pre>
<p>Btw my dataset is .jpg format of the mnist handwritten digits quantized to a .npy file</p>
<p>Someone should pls help me out here <img src="https://discuss.pytorch.org/images/emoji/apple/pray.png?v=9" title=":pray:" class="emoji" alt=":pray:"></p>

Henry_Chibueze: <p>Well after looking at my code for another time, I’ve just discovered that I made a mistake in training the generator as I forgot to back propagate on the generator loss ( G_loss.barkward()).</p>
<p>Silly me <img src="https://discuss.pytorch.org/images/emoji/apple/stuck_out_tongue.png?v=9" title=":stuck_out_tongue:" class="emoji" alt=":stuck_out_tongue:"></p>