将类信息添加到 keras 中的生成器模型

Question

我想使用条件GANs，以便为一个域生成图像(标记为domain A(，并从第二个域输入图像(标记为domain B(和类信息。两个域都使用相同的标签信息链接(域 A 的每个图像都链接到域 B 的图像和特定标签(。到目前为止，我在 Keras 中的生成器如下：

def generator_model_v2():
global BATCH_SIZE
inputs = Input((IN_CH, img_cols, img_rows))   
e1 = BatchNormalization(mode=0)(inputs)
e2 = Flatten()(e1)
e3 = BatchNormalization(mode=0)(e2)
e4 = Dense(1024, activation="relu")(e3)
e5 = BatchNormalization(mode=0)(e4)
e6 = Dense(512, activation="relu")(e5)
e7 = BatchNormalization(mode=0)(e6)
e8 = Dense(512, activation="relu")(e7)
e9 = BatchNormalization(mode=0)(e8)
e10 = Dense(IN_CH * img_cols *img_rows, activation="relu")(e9)
e11  = Reshape((3, 28, 28))(e10)
e12 = BatchNormalization(mode=0)(e11)
e13 = Activation('tanh')(e12)
model = Model(input=inputs, output=e13)
return model

到目前为止，我的生成器将来自domain A的图像作为输入(以及从domain B输出图像的范围(。我想以某种方式输入输入域 A 的类信息，范围是为域 B 生成相同类的图像。如何在拼合后添加标签信息。因此，例如，与其具有输入大小1x10241x1025。我可以对生成器中的类信息使用第二个输入吗？如果是，我如何从 GAN 的训练过程中调用生成器？

培训程序：

discriminator_and_classifier_on_generator = generator_containing_discriminator_and_classifier(
generator, discriminator, classifier)
generator.compile(loss=generator_l1_loss, optimizer=g_optim)
discriminator_and_classifier_on_generator.compile(
loss=[generator_l1_loss, discriminator_on_generator_loss, "categorical_crossentropy"],
optimizer="rmsprop")
discriminator.compile(loss=discriminator_loss, optimizer=d_optim) # rmsprop
classifier.compile(loss="categorical_crossentropy", optimizer=c_optim)
for epoch in range(30):
for index in range(int(X_train.shape[0] / BATCH_SIZE)):
image_batch = Y_train[index * BATCH_SIZE:(index + 1) * BATCH_SIZE]
label_batch = LABEL_train[index * BATCH_SIZE:(index + 1) * BATCH_SIZE]  # replace with your data here
generated_images = generator.predict(X_train[index * BATCH_SIZE:(index + 1) * BATCH_SIZE])
real_pairs = np.concatenate((X_train[index * BATCH_SIZE:(index + 1) * BATCH_SIZE, :, :, :], image_batch),axis=1)
fake_pairs = np.concatenate((X_train[index * BATCH_SIZE:(index + 1) * BATCH_SIZE, :, :, :], generated_images), axis=1)
X = np.concatenate((real_pairs, fake_pairs))
y = np.concatenate((np.ones((100, 1, 64, 64)), np.zeros((100, 1, 64, 64))))
d_loss = discriminator.train_on_batch(X, y)
discriminator.trainable = False
c_loss = classifier.train_on_batch(image_batch, label_batch)
classifier.trainable = False
g_loss = discriminator_and_classifier_on_generator.train_on_batch(
X_train[index * BATCH_SIZE:(index + 1) * BATCH_SIZE, :, :, :], 
[image_batch, np.ones((100, 1, 64, 64)), label_batch])
discriminator.trainable = True
classifier.trainable = True

代码是条件 dcgan 的实现(在鉴别器上添加分类器(。网络的功能是：

def generator_containing_discriminator_and_classifier(generator, discriminator, classifier):
inputs = Input((IN_CH, img_cols, img_rows))
x_generator = generator(inputs)
merged = merge([inputs, x_generator], mode='concat', concat_axis=1)
discriminator.trainable = False
x_discriminator = discriminator(merged)
classifier.trainable = False
x_classifier = classifier(x_generator)
model = Model(input=inputs, output=[x_generator, x_discriminator, x_classifier])
return model
def generator_containing_discriminator(generator, discriminator):
inputs = Input((IN_CH, img_cols, img_rows))
x_generator = generator(inputs)
merged = merge([inputs, x_generator], mode='concat',concat_axis=1)
discriminator.trainable = False
x_discriminator = discriminator(merged)
model = Model(input=inputs, output=[x_generator,x_discriminator])
return model

Answer 1

首先，按照条件生成对抗网络中给出的建议，您必须定义第二个输入。然后，只需连接两个输入向量并处理此级联向量。

def generator_model_v2():
input_image = Input((IN_CH, img_cols, img_rows)) 
input_conditional = Input((n_classes))  
e0 = Flatten()(input_image) 
e1 = Concatenate()([e0, input_conditional])   
e2 = BatchNormalization(mode=0)(e1)
e3 = BatchNormalization(mode=0)(e2)
e4 = Dense(1024, activation="relu")(e3)
e5 = BatchNormalization(mode=0)(e4)
e6 = Dense(512, activation="relu")(e5)
e7 = BatchNormalization(mode=0)(e6)
e8 = Dense(512, activation="relu")(e7)
e9 = BatchNormalization(mode=0)(e8)
e10 = Dense(IN_CH * img_cols *img_rows, activation="relu")(e9)
e11  = Reshape((3, 28, 28))(e10)
e12 = BatchNormalization(mode=0)(e11)
e13 = Activation('tanh')(e12)
model = Model(input=[input_image, input_conditional] , output=e13)
return model

然后，您需要在训练期间将类标签传递给网络：

classifier.train_on_batch((image_batch, class_batch), label_batch)