Gan loss tiny, discriminator loss huge? - deep-learning

Question

I can't find the problem in my code - I'm training a GAN and the gan loss and discriminator loss are very low, 0.04 and it seems like it's converging well but a - the pictures don't look very good but the actual problem is that, b - somehow when I do gan.predict(noise) it's very close to 1, but when I do discriminator.predict(gan(noise)), it's very close to 0 although it's supposed to be identical. Here's my code:
Generator code:
def create_generator():
generator=tf.keras.Sequential()
#generator.add(layers.Dense(units=50176,input_dim=25))
generator.add(layers.Dense(units=12544,input_dim=100))
#generator.add(layers.Dropout(0.2))
generator.add(layers.Reshape([112,112,1])) #112,112
generator.add(layers.Conv2D(32, kernel_size=3,padding='same',activation='relu'))
generator.add(layers.UpSampling2D()) #224,224
generator.add(layers.Conv2D(1, kernel_size=4,padding='same',activation='tanh'))
generator.compile(loss='binary_crossentropy', optimizer=adam_optimizer())
return generator
g=create_generator()
g.summary()
Discriminator code:
#IMAGE DISCRIMINATOR
def create_discriminator():
discriminator=tf.keras.Sequential()
discriminator.add(layers.Conv2D(64, kernel_size=2,padding='same',activation='relu',input_shape=[224,224,1]))
discriminator.add(layers.Dropout(0.5))
discriminator.add(layers.Conv2D(32,kernel_size=2,padding='same',activation='relu'))
discriminator.add(layers.Dropout(0.5))
discriminator.add(layers.Conv2D(16,kernel_size=2,padding='same',activation='relu'))
discriminator.add(layers.Dropout(0.5))
discriminator.add(layers.Conv2D(8,kernel_size=2,padding='same',activation='relu'))
discriminator.add(layers.Dropout(0.5))
discriminator.add(layers.Conv2D(1,kernel_size=2,padding='same',activation='relu'))
discriminator.add(layers.Dropout(0.5))
discriminator.add(layers.Flatten())
discriminator.add(layers.Dense(units=1, activation='sigmoid'))
discriminator.compile(loss='binary_crossentropy', optimizer=tf.optimizers.Adam(lr=0.0002))
return discriminator
d =create_discriminator()
d.summary()
Gan code:
def create_gan(discriminator, generator):
discriminator.trainable=False
gan_input = tf.keras.Input(shape=(100,))
x = generator(gan_input)
gan_output= discriminator(x)
gan= tf.keras.Model(inputs=gan_input, outputs=gan_output)
#gan.compile(loss='binary_crossentropy', optimizer='adam')
gan.compile(loss='binary_crossentropy', optimizer=adam_optimizer())
return gan
gan = create_gan(d,g)
gan.summary()
Training code (I purposely don't do train_on_batch with the gan cause I wanted to see if the gradients zero out.)
##tf.function
def training(epochs=1, batch_size=128, rounds=50):
batch_count = X_bad.shape[0] / batch_size
# Creating GAN
generator = create_generator()
discriminator = create_discriminator()
###########if you want to continue training an already trained gan
#discriminator.set_weights(weights)
gan = create_gan(discriminator, generator)
start = time.time()
for e in range(1,epochs+1 ):
#print("Epoch %d" %e)
#for _ in tqdm(range(batch_size)):
#generate random noise as an input to initialize the generator
noise= np.random.normal(0,1, [batch_size, 100])
# Generate fake MNIST images from noised input
generated_images = generator.predict(noise)
#print('gen im shape: ',np.shape(generated_images))
# Get a random set of real images
image_batch = X_bad[np.random.randint(low=0,high=X_bad.shape[0],size=batch_size)]
#print('im batch shape: ',image_batch.shape)
#Construct different batches of real and fake data
X= np.concatenate([image_batch, generated_images])
# Labels for generated and real data
y_dis=np.zeros(2*batch_size)
y_dis[:batch_size]=0.99
#Pre train discriminator on fake and real data before starting the gan.
discriminator.trainable=True
discriminator.train_on_batch(X, y_dis)
#Tricking the noised input of the Generator as real data
noise= np.random.normal(0,1, [batch_size, 100])
y_gen = np.ones(batch_size)
# During the training of gan,
# the weights of discriminator should be fixed.
#We can enforce that by setting the trainable flag
discriminator.trainable=False
#training the GAN by alternating the training of the Discriminator
#and training the chained GAN model with Discriminator’s weights freezed.
#gan.train_on_batch(noise, y_gen)
with tf.GradientTape() as tape:
pred=gan(noise)
loss_val=tf.keras.losses.mean_squared_error(y_gen,pred)
# loss_val=gan.test_on_batch(noise,y_gen)
# loss_val=tf.cast(loss_val,dtype=tf.float32)
grads=tape.gradient(loss_val,gan.trainable_variables)
optimizer.apply_gradients(zip(grads, gan.trainable_variables))
if e == 1 or e % rounds == 0:
end = time.time()
loss_value=discriminator.test_on_batch(X, y_dis)
print("Epoch {:03d}: Loss: {:.3f}".format(e,loss_value))
gen_loss=gan.test_on_batch(noise,y_gen)
print('gan loss: ',gen_loss)
#print('Epoch: ',e,' Loss: ',)
print('Time for ',rounds,' epochs: ',end-start,' seconds')
local_time = time.ctime(end)
print('Printing time: ',local_time)
plot_generated_images(e, generator,examples=5)
start = time.time()
return discriminator,generator,grads
Now, the final losses after around 2000 epochs are 0.039 for the generator and 0.034 for the discriminator. The thing is that when I do the following, look what I get
print('disc for bad train: ',np.mean(discriminator(X_bad[:50])))
#output
disc for bad train: 0.9995248
noise= np.random.normal(0,1, [500, 100])
generated_images = generator.predict(noise)
print('disc for gen: ',np.mean(discriminator(generated_images[:50])))
print('gan for gen: ',np.mean(gan(noise[:50])))
#output
disc for gen: 0.0018724388
gan for gen: 0.96554756
Can anyone find the problem?
Thanks!

Answer 1

If anyone stumbles upon this, then I figured it out (although I haven't fixed it yet). What's happening here is that my Gan object is training only the generator weights. The discriminator is a different object that is training itself but when I train the gan, I don't update the discriminator weights in the gan, I only updated the generator weights and therefore when I had the noise as the input to the gan, the output was as I wanted, close to one, however when I took the generated images as the input to the discriminator, since the discriminator was being trained separately from the generator (the discriminator weights were not being changed at all inside the gan), the result was close to zero. A better approach is to create a GAN as a class and be able to approach the discriminator within the gan and the generator within the gan in order to update both of their weights inside the gan.