我正在尝试为我使用预训练的tensorflow XceptionNet创建的模型生成输入图像的热图。
我的模型结构是:
from tensorflow.keras import Model
from tensorflow.keras.layers import Input, Conv2D, MaxPool2D, Dense, Flatten, Dropout, AveragePooling2D, Concatenate, GlobalAveragePooling2D, BatchNormalization, ReLU, Add, SeparableConv2D
from tensorflow.keras.applications import Xception
def xception(img_shape, n_classes):
xceptionnet = Xception(input_shape=img_shape, include_top=False, weights='imagenet')
xceptionnet.trainable = False
input = Input(img_shape)
x = xceptionnet(input, training=False)
x = GlobalAveragePooling2D()(x)
x = Dropout(rate = 0.2)(x)
output = Dense(n_classes, activation='softmax')(x)
model = Model(input, output)
return model
input_shape = (256, 256, 3)
n_classes = 3
model = xception(input_shape, n_classes)
model.compile('Adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
model.summary()
模型结构[Model .summary()的输出]
我尝试使用与keras文档(https://keras.io/examples/vision/grad_cam/)中提到的相同格式来生成数据集中图像的热图。
所以根据文档，我的模型的显示图像部分是:
from IPython.display import Image, display
import matplotlib.pyplot as plt
import matplotlib.cm as cm
img_size = (256, 256, 3)
preprocess_input = keras.applications.xception.preprocess_input
decode_predictions = keras.applications.xception.decode_predictions
last_conv_layer_name = "xception"
# The local path to our target image
img_path = '/content/drive/My Drive/Colab Notebooks/data/Malignant/Malignant case (1).jpg'
display(Image(img_path))
以上部分运行良好。
但是现在当我执行部分:
def make_gradcam_heatmap(img_array, model, last_conv_layer_name, pred_index=None):
# First, we create a model that maps the input image to the activations
# of the last conv layer as well as the output predictions
last_conv_layer = model.get_layer(last_conv_layer_name)
new_model = tf.keras.models.Sequential()
for layer in model.layers[:model.layers.index(last_conv_layer)+1]:
new_model.add(layer)
new_model.add(tf.keras.layers.Flatten())
grad_model = tf.keras.models.Model(inputs=[new_model.input], outputs=[new_model.output, model.output])
with tf.GradientTape() as tape:
last_conv_layer_output, preds = grad_model(img_array)
last_conv_layer_output = tf.reshape(last_conv_layer_output, shape=(8, 8, 2048))
if pred_index is None:
pred_index = tf.argmax(preds[0])
class_channel = preds[:, pred_index]
# This is the gradient of the output neuron (top predicted or chosen)
# with regard to the output feature map of the last conv layer
grads = tape.gradient(class_channel, last_conv_layer_output)
# This is a vector where each entry is the mean intensity of the gradient
# over a specific feature map channel
pooled_grads = tf.reduce_mean(grads, axis=(0, 1, 2))
# We multiply each channel in the feature map array
# by "how important this channel is" with regard to the top predicted class
# then sum all the channels to obtain the heatmap class activation
last_conv_layer_output = last_conv_layer_output[0]
heatmap = last_conv_layer_output @ pooled_grads[..., tf.newaxis]
heatmap = tf.squeeze(heatmap)
# For visualization purpose, we will also normalize the heatmap between 0 & 1
heatmap = tf.maximum(heatmap, 0) / tf.math.reduce_max(heatmap)
return heatmap.numpy()
from tensorflow.keras.models import load_model
# Prepare image
img_array = preprocess_input(get_img_array(img_path, size=img_size))
# Make model
model = load_model('/content/drive/My Drive/Colab Notebooks/models/imageclassifier1.h5')
# Remove last layer's softmax
model.layers[-1].activation = None
# Print what the top predicted class is
preds = model.predict(img_array)
preds = np.argmax(preds[0])
labels = { 0 : "Cat",
1 : "Dog",
2 : "Human"}
# print("Predicted:", decode_predictions(preds, top=1)[0]) # For pre-trained models
print("Predicted:", labels[preds])
# Generate class activation heatmap
heatmap = make_gradcam_heatmap(img_array, model, last_conv_layer_name)
# Display heatmap
plt.matshow(heatmap)
plt.show()
我在pooled_grads = tf.reduce_mean(grads, axis=(0, 1, 2))行中得到错误，说grad是None。
下面是我收到的错误信息:
ValueError Traceback (most recent call last)
<ipython-input-86-e8872c6548f7> in <cell line: 25>()
23
24 # Generate class activation heatmap
---> 25 heatmap = make_gradcam_heatmap(img_array, model, last_conv_layer_name)
26
27 # Display heatmap
<ipython-input-84-cf963b881b8a> in make_gradcam_heatmap(img_array, model, last_conv_layer_name, pred_index)
50 # This is a vector where each entry is the mean intensity of the gradient
51 # over a specific feature map channel
---> 52 pooled_grads = tf.reduce_mean(grads, axis=(0, 1, 2))
53 print(pooled_grads)
54
/usr/local/lib/python3.9/dist-packages/tensorflow/python/util/traceback_utils.py in error_handler(*args, **kwargs)
151 except Exception as e:
152 filtered_tb = _process_traceback_frames(e.__traceback__)
--> 153 raise e.with_traceback(filtered_tb) from None
154 finally:
155 del filtered_tb
/usr/local/lib/python3.9/dist-packages/tensorflow/python/framework/constant_op.py in convert_to_eager_tensor(value, ctx, dtype)
101 dtype = dtypes.as_dtype(dtype).as_datatype_enum
102 ctx.ensure_initialized()
--> 103 return ops.EagerTensor(value, ctx.device_name, dtype)
104
105
ValueError: Attempt to convert a value (None) with an unsupported type (<class 'NoneType'>) to a Tensor.
我对这些东西很陌生，所以有人能帮帮我吗?
抛出一个错误，因为你的模型有一个嵌套结构(异常模型在更大的分类模型中)。
您可以通过再次在各个层传递输入来绕过这个问题来构建模型。将异常层的输出作为最后一个卷积输出，将张量通过层的末端传递得到最终输出。最后将输出合并到一个新模型中，如下所示:
inputs = keras.Input((256, 256, 3))
xception = model.get_layer("xception")
last_conv_output = xception(inputs)
x = last_conv_output
for idx in range(2, len(model.layers)):
x = model.layers[idx](x)
output = x
grad_model = keras.Model(inputs, [last_conv_output, output])
通过重构上面的代码生成热图:
import tensorflow as tf
def make_gradcam_heatmap(inputs, grad_model, pred_index=None):
# First, we create a model that maps the input image to the activations
# of the last conv layer as well as the output predictions
with tf.GradientTape() as tape:
last_conv_layer_output, preds = grad_model(inputs)
#print(preds)
if pred_index is None:
pred_index = tf.argmax(preds[0])
#print(pred_index)
class_channel = preds[:, pred_index]
#print(class_channel)
# This is the gradient of the output neuron (top predicted or chosen)
# with regard to the output feature map of the last conv layer
grads = tape.gradient(class_channel, last_conv_layer_output)
# This is a vector where each entry is the mean intensity of the gradient
# over a specific feature map channel
pooled_grads = tf.reduce_mean(grads, axis=(0, 1, 2))
# We multiply each channel in the feature map array
# by "how important this channel is" with regard to the top predicted class
# then sum all the channels to obtain the heatmap class activation
last_conv_layer_output = last_conv_layer_output[0]
heatmap = last_conv_layer_output @ pooled_grads[..., tf.newaxis]
heatmap = tf.squeeze(heatmap)
# For visualization purpose, we will also normalize the heatmap between 0 & 1
heatmap = tf.maximum(heatmap, 0) / tf.math.reduce_max(heatmap)
return heatmap.numpy()
输出:
inputs = tf.random.uniform((1, 256, 256, 3))
make_gradcam_heatmap(inputs, grad_model)
array([[0. , 0. , 0. , 0.0019973 , 0.00224069,
0. , 0. , 0. ],
[0. , 0. , 0.01608142, 0.13859619, 0.19170615,
0.03831227, 0. , 0. ],
[0.00344855, 0.15640435, 0.39062408, 0.57898533, 0.72229344,
0.18632776, 0.08909718, 0.00205518],
[0.05994121, 0.41158128, 0.55284446, 0.8489698 , 0.96675164,
0.34517574, 0.30315596, 0.05326002],
[0.07081833, 0.4438232 , 0.6151547 , 0.9064342 , 0.9261135 ,
0.41782287, 0.34709153, 0.09646279],
[0.00530773, 0.22800735, 0.52887404, 0.8523431 , 1. ,
0.5120882 , 0.23707563, 0. ],
[0. , 0.03709193, 0.20877707, 0.5426089 , 0.53451735,
0.24202193, 0. , 0. ],
[0. , 0.01366277, 0.03030644, 0.14712998, 0.19128165,
0. , 0. , 0. ]], dtype=float32)