我在尝试从https://arxiv.org/pdf/1610.07629.pdf训练神经风格转移模型时遇到了nan。我试过降低学习率和使用不同的初始化，但它似乎不起作用。我怀疑这与我使用tape.Gradient的训练循环有关。
这是我的图像变换网络:
import tensorflow as tf
from tensorflow.keras.layers import Conv2D, Input, BatchNormalization, Add, ReLU, Reshape, UpSampling2D
from tensorflow.keras.models import Model
import numpy as np
from tensorflow.keras import initializers
# initializers
initializer = tf.random_normal_initializer(mean=0.0, stddev=0.01, seed=None)
betaInitializer = initializers.constant(0.)
gammaInitializer = initializers.constant(1.)
# Conditional Instance Normalisation layer
class ConditionalInstanceNorm(tf.keras.layers.Layer):
def __init__(self, scope_bn, y1, y2, alpha):
super(ConditionalInstanceNorm, self).__init__()
self.scope_bn = scope_bn
self.y1 = y1
self.y2 = y2
self.alpha = alpha
def build(self, input_shape):
self.beta = self.add_weight(name="beta"+self.scope_bn, shape=(self.y1.shape[-1], input_shape[-1]), initializer=betaInitializer, trainable=True)
self.gamma = self.add_weight(name="gamma"+self.scope_bn, shape=(self.y1.shape[-1], input_shape[-1]), initializer=gammaInitializer, trainable=True)
def call(self, inputs):
mean, var = tf.nn.moments(x=inputs, axes=[1,2], keepdims=True)
beta1 = tf.matmul(self.y1, self.beta)
gamma1 = tf.matmul(self.y1, self.gamma)
beta2 = tf.matmul(self.y2, self.beta)
gamma2 = tf.matmul(self.y2, self.gamma)
beta = self.alpha*beta1 + (1. - self.alpha)*beta2
gamma = self.alpha*gamma1 + (1. - self.alpha)*gamma2
x = tf.nn.batch_normalization(x=inputs, mean=mean, variance=var, offset=beta, scale=gamma, variance_epsilon=1e-10)
return x
# Applies upsampling if stride = 0.5, includes mirror padding, conv layer and conditional instance Norm layer
def PadConvBatch(x, filters=32, kernel_size=3, strides=1, activation='relu', scope_bn="", y1=None, y2=None, alpha=1):
if isinstance(strides, float):
x = UpSampling2D(size=2, interpolation='nearest')(x)
strides=1
padding = tf.cast((kernel_size-1)/2, tf.int32)
x = tf.pad(x, [[0,0], [padding,padding], [padding,padding], [0,0]], "REFLECT")
x = Conv2D(filters=filters, kernel_size=kernel_size, strides=strides, activation=activation, kernel_initializer=initializer)(x)
conditionalInstanceNorm = ConditionalInstanceNorm(scope_bn=scope_bn, y1=y1, y2=y2, alpha=alpha)
x = conditionalInstanceNorm(x)
return x
# Implementation of resnet block according to paper
def resblock(x, scope_bn1="", scope_bn2="", y1=None, y2=None, alpha=1):
fx = PadConvBatch(x=x, filters=128, activation="linear", scope_bn=scope_bn1, y1=y1, y2=y2, alpha=alpha)
fx = ReLU()(fx)
fx = PadConvBatch(x=fx, filters=128, activation="linear", scope_bn=scope_bn2, y1=y1, y2=y2, alpha=alpha)
out = Add()([x,fx])
return out
# Overall network
def ImageTransformNetwork(shape=(256,256,3), y1=None, y2=None, alpha=1):
inputs = Input(shape=shape) #1
x = PadConvBatch(x=inputs, kernel_size=9, scope_bn="1", y1=y1, y2=y2, alpha=alpha) #3
x = PadConvBatch(x=x, filters=64, strides=2, scope_bn="2", y1=y1, y2=y2, alpha=alpha) #5
x = PadConvBatch(x=x, filters=128, strides=2, scope_bn="3", y1=y1, y2=y2, alpha=alpha) #7
x = resblock(x=x, scope_bn1="4", scope_bn2="5", y1=y1, y2=y2, alpha=alpha) #13
x = resblock(x=x, scope_bn1="6", scope_bn2="7", y1=y1, y2=y2, alpha=alpha) #19
x = resblock(x=x, scope_bn1="8", scope_bn2="9", y1=y1, y2=y2, alpha=alpha) #25
x = resblock(x=x, scope_bn1="10", scope_bn2="11", y1=y1, y2=y2, alpha=alpha) #31
x = resblock(x=x, scope_bn1="12", scope_bn2="13", y1=y1, y2=y2, alpha=alpha) #37
x = PadConvBatch(x=x, filters=64, strides=0.5, scope_bn="14", y1=y1, y2=y2, alpha=alpha) #39
x = PadConvBatch(x=x, filters=32, strides=0.5, scope_bn="15", y1=y1, y2=y2, alpha=alpha) #41
x = tf.pad(x, [[0,0], [4,4], [4,4], [0,0]], "REFLECT")
x = Conv2D(filters=3, kernel_size=9, strides=1, activation='sigmoid', kernel_initializer=initializer)(x) #42
x = x*255
model = Model(inputs=inputs, outputs=x)
return model
import tensorflow as tf
import os
#config
os.environ['TFHUB_MODEL_LOAD_FORMAT'] = 'COMPRESSED'
#Specify layers for content and style representation
content_layers = ['block4_conv2']
style_layers = ['block1_conv1', 'block2_conv1', 'block3_conv1', 'block4_conv1', 'block5_conv1']
# Find number of layers
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)
# Grab vgg19 network without dense layers
def vgg_layers(layer_names):
vgg = tf.keras.applications.VGG19(include_top=False, weights='imagenet')
outputs = [vgg.get_layer(name).output for name in layer_names]
model = tf.keras.Model([vgg.input], outputs)
return model
# Calculate normalised gram matrix
def gram_matrix(input_tensor):
result = tf.linalg.einsum('bijc,bijd->bcd', input_tensor, input_tensor)
input_shape = tf.shape(input_tensor)
num_locations = tf.cast(input_shape[1]*input_shape[2], tf.float32)
return result/(num_locations)
# Setting up loss network model class + forward method
class LossNetwork(tf.keras.models.Model):
def __init__(self, style_layers, content_layers):
super(LossNetwork, self).__init__()
self.vgg = vgg_layers(style_layers+content_layers)
self.style_layers = style_layers
self.content_layers = content_layers
self.num_style_layers = len(style_layers)
self.vgg.trainable = False
def call(self, inputs):
inputs = inputs*255.0
preprocessed_input = tf.keras.applications.vgg19.preprocess_input(inputs)
outputs = self.vgg(preprocessed_input)
style_outputs, content_outputs = (outputs[:self.num_style_layers], outputs[self.num_style_layers:])
style_outputs = [gram_matrix(style_output) for style_output in style_outputs]
content_dict = {content_name: value for content_name, value in zip(self.content_layers, content_outputs)}
style_dict = {style_name: value for style_name, value in zip(self.style_layers, style_outputs)}
return {'content': content_dict, 'style': style_dict}
# returns network
def loadLossNetwork():
return LossNetwork(style_layers, content_layers)
至于训练，我目前只使用批大小为1并且只使用1张图像来测试它。以下是我的训练代码:
import tensorflow as tf
import numpy as np
import time
import functools
import ImageTransformNetwork
import ImagePreProcessing
import LossNetwork
import PIL.Image
"""Utility and Loss Functions START"""
#Converts a tensor into an image
def tensor_to_image(tensor):
#Multiplies every element in tensor by 255
tensor = tensor*255
#Converts tensor into a numpy array
tensor = np.array(tensor, dtype=np.uint8)
if np.ndim(tensor)>3:
#Ensures only 1 image is being sent
assert tensor.shape[0] == 1
#Takes out the first element so you just get a 3 dim np array
tensor = tensor[0]
#Converts numpy array into PIL image
return PIL.Image.fromarray(tensor)
# Calculates style and content loss in loss network
def style_content_loss(outputs):
style_outputs = outputs['style']
content_outputs = outputs['content']
style_loss_total = tf.constant(0.0, tf.float32)
for name in style_outputs.keys():
style_loss_total += tf.reduce_sum((style_outputs[name]-style_targets[name])**2)*style_weight/batch_size
content_loss_total = 0
for name in content_outputs.keys():
content_loss_total += ((content_outputs[name] - content_targets[name])**2)
content_loss_total *= content_weight/(content_outputs[name].shape[1]*content_outputs[name].shape[2])
loss = style_loss_total + content_loss_total
return loss
"""Utility and Loss Functions END"""
"""Setup START"""
#Loads images
content_image, style_image = ImagePreProcessing.loadImage()
content_image = tf.image.resize(content_image, [256,256])
content_image = tf.reshape(content_image, [1,256,256,3])
y1 = tf.Variable(tf.zeros([1,10]))
y2 = tf.Variable(tf.zeros([1,10]))
alpha = tf.constant([1.])
#Calls instance of image transform network and loss network
imageTransformNetwork = ImageTransformNetwork.ImageTransformNetwork(y1=y1, y2=y2, alpha=alpha)
lossNetwork = LossNetwork.loadLossNetwork()
# Obtain features of each layer of loss network model
style_targets = lossNetwork(style_image)['style']
content_targets = lossNetwork(content_image)['content']
#Sets optimiser, LBFGS is better
opt = tf.keras.optimizers.Adam(learning_rate=1e-3, beta_1=0.9, beta_2=0.999, epsilon=1e-1)
"""Setup END"""
"""Train START"""
#Training step
style_weight = 1e-2
content_weight = 1e4
total_variation_weight = 30
@tf.function()
def train_step(image):
with tf.GradientTape() as tape:
stylised = imageTransformNetwork(image, training=True)
outputs = lossNetwork(stylised)
loss = style_content_loss(outputs)
grad = tape.gradient(loss, imageTransformNetwork.trainable_variables)
opt.apply_gradients(zip(grad, imageTransformNetwork.trainable_variables))
#Optimisation loop
epochs = 1
steps_per_epoch = 5
batch_size = 1
start = time.time()
step = 0
for n in range(epochs):
for m in range(steps_per_epoch):
step += 1
train_step(content_image)
print(".", end='', flush=True)
print('Train step: {}'.format(step))
end=time.time()
print('Total time: {:.1f}'.format(end-start))
print(imageTransformNetwork.layers[15].weights)
stylised_tensor = imageTransformNetwork(content_image)
print(stylised_tensor)
stylised_image = tensor_to_image(stylised_tensor)
stylised_image.show()
"""Train END"""
以下是5个epoch后我的Action Transform Network的权重:
[& lt; tf。变量'conv2d_4/kernel:0' shape=(3,3,128, 128)Dtype =float32, numpy= array([[[[nan, nan, nan，…)， nan, nan, nan]，不，不，不，…， nan, nan, nan]，不，不，不，…， nan, nan, nan]，…，不，不，不，…， nan, nan, nan]，不，不，不，…， nan, nan, nan]，不，不，不，…， nan, nan, nan]]，
[[nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan], ..., [nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan]], [[nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan], ..., [nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan]]], [[[nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan], ..., [nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan]], [[nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan], ..., [nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan]], [[nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan], ..., [nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan]]], [[[nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan], ..., [nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan]], [[nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan], ..., [nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan]], [[nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan], ..., [nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan], [nan, nan, nan, ..., nan, nan, nan]]]], dtype=float32)>, <tf.Variable 'conv2d_4/bias:0' shape=(128,) dtype=float32, numpy=array([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan，南,不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，不，南，南，南，南，南，南，南，南，南，南，南，南]，dtype = float32)祝辞]
我的代码有什么问题可能导致这种情况吗?
修复了它，我正在重新编程我的训练代码，使其更清晰，并意外地修复了它。问题是我的y1张量都是0，因此实例归一化层中的beta和gamma值都是0，从而在某种程度上导致NaN值的传播。我也意识到我不应该在我的图像变换网络的末尾添加x = x*255，在vgg网络的开始添加另一个input = 255*input，这会使我的值膨胀。
import tensorflow as tf
import ImageProcessing
import LossNetwork
import ImageTransformNetwork
def train(epochs=1, ylabel=0, style_weight=1e-2, content_weight=1e4, batch_size=16):
train_dataset, style_image = ImageProcessing.loadDataset("Picture/", "chosen_pictures/candle_dancer,nolde.jpeg", batch_size)
y1 = [[0 for i in range(10)]]
y1[0][ylabel] = 1
y1 = tf.convert_to_tensor(y1, dtype=tf.float32)
y2 = tf.zeros([1,10])
alpha = tf.constant([1.])
imageTransformNetwork = ImageTransformNetwork.ImageTransformNetwork(y1=y1, y2=y2, alpha=alpha)
lossNetwork = LossNetwork.loadLossNetwork()
style_targets = lossNetwork(style_image)["style"]
opt = tf.keras.optimizers.Adam(learning_rate=1e-3, beta_1=0.9, beta_2=0.999, epsilon=1e-1)
step = 0
for epochs in range(epochs):
for i, batch in enumerate(train_dataset):
step += 1
batch = rescaling(batch)
with tf.GradientTape() as tape:
stylised = imageTransformNetwork(batch, training=True)
outputs = lossNetwork(stylised)
content_targets = lossNetwork(batch)["content"]
loss = LossNetwork.style_content_loss(outputs, style_targets, content_targets, style_weight, content_weight, batch_size)
grad = tape.gradient(loss, imageTransformNetwork.trainable_weights)
opt.apply_gradients(zip(grad, imageTransformNetwork.trainable_weights))
tf.print(loss)
if step%500 == 0:
model.save("savedModels/model"+step//500)
rescaling = tf.keras.layers.Rescaling(1./255, input_shape=(None, None, 3), dtype=tf.float32)
train()
结果权重不再包含任何nan:
[& lt; tf。变量'conv2d_4/kernel:0' shape=(3,3,128, 128)Dtype =float32, numpy= array([[[[-9.22914653e-04， -3.25754564e-03，2.19115964 e 03,…-1.54448478e-02, 6.85744826e-03， -3.45157599e-03]，[1.11031579e-02， -1.67974327e-02， - 9.001014400e -04，…][02]， [1.03713870e-02， [7.727846500e -03]，[6.38803188e-03, 1.52107864e-03, 3.26261576e-03，…]-1.68494694e-03， -1.20439203e-02， -3.61325294e-02]，…，[1.20753944e-02, 3.71512910e-03, 2.29522269e-02，…][5.69997064e-04， -9.27594118e-03， -2.40900833e-02]，[-4.25596721e-03, 3.20109446e-03， -1.33239599e-02，…]-9.32860561e-03, 1.42052788e-02， -1.05757872e-03]，[1.85675547e-02, 1.12123555e-02， -7.40887132e-03，…]-5.26969973e-03， -4.57865652e-03, 2.60398141e-03]]，
[[ 6.01717457e-03, -1.17905010e-02, -1.26012340e-02, ..., -2.16883258e-03, 1.40141053e-02, -4.93765983e-04], [-1.18666200e-03, -6.67788927e-03, 5.93844941e-03, ..., 1.05758598e-02, -1.33344317e-02, 4.41704318e-03], [ 4.38539824e-03, -9.45210271e-03, -1.96239129e-02, ..., 1.58801023e-02, 1.40289860e-02, -9.41994600e-03], ..., [ 1.37740280e-05, 6.21070713e-03, 1.13504520e-03, ..., -7.07426807e-03, 1.91432852e-02, -1.09282834e-02], [ 2.73628104e-02, -1.65116577e-03, -1.41831292e-02, ..., -1.61790382e-03, -7.22122053e-03, -1.38549581e-02], [ 3.37673048e-03, 1.00006070e-02, 2.99341255e-03, ..., 8.13197810e-03, -1.22140273e-02, -2.28559617e-02]], [[-4.94419318e-03, -1.06641259e-02, -1.19099990e-02, ..., 7.04542780e-03, 1.10173412e-02, -1.00985430e-02], [-1.07495757e-02, -1.03038019e-02, 2.26744954e-02, ..., 2.91943387e-03, -2.27406379e-02, -1.16430689e-02], [ 8.73126276e-03, 1.89025106e-03, 8.01201630e-03, ..., 5.62170707e-03, 6.12628879e-03, -2.53114733e-04], ..., [-9.01912060e-03, 1.86426658e-02, 7.42244720e-03, ..., -7.43635232e-03, 1.84447784e-02, 5.29437512e-03], [-2.77710403e-03, -1.11136436e-02, 4.61787265e-03, ..., 3.43568996e-03, 1.10518862e-03, 9.76805110e-03], [ 1.84791256e-02, 1.40596041e-02, 2.54741148e-03, ..., -3.77570093e-03, 6.19729050e-03, 2.31865281e-03]]], [[[ 2.15795785e-02, -1.19263222e-02, -2.69088056e-03, ..., 1.32677099e-03, 8.45701247e-03, -1.88063979e-02], [-9.04210471e-03, -5.87769412e-03, -5.84315648e-03, ..., 8.99547432e-03, 7.23270513e-03, 4.29085980e-04], [ 6.83306810e-03, 1.70376035e-03, 1.41868247e-02, ..., -2.99190381e-03, 1.33085134e-03, -1.67587642e-02], ..., [-1.36112515e-02, 5.13256202e-03, 5.86730288e-03, ..., 4.40224679e-03, 1.41993053e-02, -2.21067611e-02], [ 6.86844671e-03, -3.06997774e-03, -1.56005612e-02, ..., 2.00642040e-03, 1.37336999e-02, -7.43707130e-03], [-3.60132020e-04, 1.27417007e-02, -6.24217000e-03, ..., 8.11805599e-04, -6.68468559e-03, 1.66701805e-02]], [[ 6.16356591e-03, -1.51706664e-02, -4.45271842e-03, ..., -1.61965843e-03, -3.41256475e-03, 4.15274641e-03], [ 7.08530797e-03, 3.43623164e-04, 3.02305375e-03, ..., -1.34551339e-02, -7.74222892e-03, -9.40223970e-03], [-3.96022515e-04, -1.76352914e-02, 4.82104952e-03, ..., 1.61363333e-02, 1.36647765e-02, -4.76737216e-04], ..., [ 3.33301607e-03, -4.36515687e-03, -1.00332387e-02, ..., -7.81015400e-03, -9.20389965e-03, -1.65066053e-03], [-4.91148187e-03, -1.87809812e-03, -5.74924750e-03, ..., 5.51467016e-03, -4.86751553e-03, 4.92513832e-03], [ 2.10825708e-02, 6.21355977e-03, -3.15705035e-03, ..., 1.32449274e-03, 1.09589389e-02, -1.67893188e-03]], [[ 7.25688133e-03, -8.45121965e-03, -8.33619572e-03, ..., 4.51847119e-03, -3.63183790e-03, 9.71599482e-03], [-1.14116399e-03, -9.45648877e-04, 8.83929897e-03, ..., -7.76065839e-03, -1.52589316e-02, -8.26635305e-03], [-3.16444691e-03, 3.19679384e-03, 3.55893001e-03, ..., 7.63914781e-03, -3.52958916e-03, -1.72572676e-03], ..., [ 4.08169115e-03, -1.09134587e-02, -6.46053767e-03, ..., -2.58600842e-02, 9.29399393e-03, 1.65797323e-02], [ 1.06143337e-02, -1.58559270e-02, -1.03117265e-02, ..., -1.52223487e-03, -6.32383861e-03, -3.65341373e-04], [-4.81503829e-03, -4.38905088e-03, -4.73908149e-03, ..., -5.55378292e-03, 2.71180947e-03, -5.92879718e-03]]], [[[ 9.67705343e-03, 3.40853585e-03, 2.47258402e-04, ..., 1.45784477e-02, -1.67573560e-02, -2.56466400e-02], [ 3.59606324e-03, 1.20630264e-02, -1.37810390e-02, ..., -1.66904740e-03, -4.24776971e-03, -9.79586504e-03], [ 4.21262113e-03, -9.41874983e-04, 6.73639216e-03, ..., 2.38019545e-02, 4.90003964e-03, -4.52585239e-03], ..., [ 4.01812606e-04, 8.53525288e-03, -1.20494002e-02, ..., 1.56634264e-02, 1.45185292e-02, 1.58445258e-02], [ 1.92584179e-04, -4.28026915e-03, 8.10027216e-03, ..., 1.14922663e-02, 8.66387971e-03, 1.64663978e-03], [ 1.83315924e-03, 1.86281260e-02, 5.20413416e-03, ..., 8.63858406e-03, -2.18354282e-04, -1.01757115e-02]], [[ 2.81637791e-03, -4.76985564e-03, -1.39078144e-02, ..., -1.58296956e-04, 1.62388422e-02, 2.44920887e-03], [-2.65886029e-03, -1.00718942e-02, -9.38246213e-03, ..., -7.73472525e-03, -7.73774413e-03, 1.22246624e-03], [-9.11037438e-03, -2.70083174e-03, -1.20385522e-02, ..., -1.91840120e-02, 8.58602673e-03, 4.00405657e-03], ..., [ 1.86355747e-02, 3.25039215e-03, 1.14641823e-02, ..., 2.52336729e-02, 7.14283716e-03, -1.11020561e-02], [ 3.38363531e-03, -2.28321627e-02, -5.52309211e-03, ..., 9.30515677e-03, -7.97424931e-04, 7.66106136e-03], [ 2.67876014e-02, -9.12602246e-03, -3.73603543e-04, ..., -4.90523782e-03, 4.98087937e-03, -6.29857299e-04]], [[ 2.37794518e-02, -1.74591187e-02, -1.02144917e-02, ..., 8.63332115e-03, -7.53870059e-04, 1.45421876e-02], [-3.04296939e-03, -1.50832683e-02, 1.51605168e-02, ..., -6.79229852e-03, 1.01285530e-02, -1.75108504e-03], [-4.07854374e-03, -1.88519584e-03, 1.14850025e-03, ..., -2.26586498e-02, -3.04668397e-03, 1.14008989e-02], ..., [ 1.17204599e-02, 2.86580436e-03, -1.76627021e-02, ..., 3.94910201e-03, -2.89543215e-02, 2.58394983e-03], [ 1.01766353e-02, -2.09429841e-02, -1.05689308e-02, ..., -3.20194196e-03, -3.15695629e-02, 3.79671482e-03], [-4.92773578e-03, -4.51938109e-03, -2.68228631e-03, ..., 5.92946680e-03, -4.65792231e-03, -5.96702285e-03]]]], dtype=float32)>, <tf.Variable 'conv2d_4/bias:0' shape=(128,) dtype=float32, numpy= array([-0.00168076, 0.00166215, -0.00167152,-0.00130922, -0.00164626,0.00166992, -0.00168149, -0.00057945, -0.00169018, -0.00143871,0.00167122, 0.00168618, 0.00167541, 0.00169117， -0.00165321，0.00167541， -0.00166706, 0.00165939, 0.00165955, 0.00162764，0.00166588、0.00168612、0.00167851、0.00167884、0.0016743、0.00171701, 0.00165294, 0.00167435, 0.00167905, 0.00165572，0.00167356， -0.00166708， -0.00168088， -0.00166456， -0.00167135，-0.00167006, 0.00167656, 0.00170659, 0.00167328, 0.00166088，-0.0016864， -0.00166293， -0.00176472， -0.00166444, 0.00167372，0.00166756、0.00168708、0.00167746、-0.00167006、0.00166973、-0.00167239, 0.00163416， -0.00165197， -0.00167195, 0.00166292，0.00165982、0.00167608、-0.00168473、0.00166501、0.00167176、0.00171711, 0.00167139, 0.00167133， -0.00167868, 0.00160735，0.00166808、0.00167025、-0.00161975、0.00167196、-0.00167264、-0.00166804, 0.00174853, 0.00169298, 0.00165326， -0.00166989，0.00166873、0.0016681、-0.00164396、-0.00170325、0.00167879、-0.00166963， -0.00166775， -0.00166631, 0.00168302, 0.0016176，-0.00166655, 0.00166208， -0.00171519, 0.00167442， -0.00166131，0.00167713, 0.0016614, 0.00164063, 0.00160119， -0.00166949，-0.00167486, 0.0016609， -0.00165791, 0.00167266， -0.00156529，0.00164661， -0.00167219, 0.00167742, 0.00167498, 0.00166841，-0.00168209， -0.00166704, 0.00167016, 0.00168282， -0.00163766，-0.0016084, 0.00168005， -0.00166668， -0.00167182, 0.00141747，0.00157016、0.00168845、0.00166997、-0.00167921、0.00166262、0.00167095, 0.00165517， -0.00167311， -0.00168072, 0.00167287，[-0.00164948, 0.00166803, 0.00160261]， dtype=float32)>]