keras tf.GradientTape().gradient()返回None

ifmq2ha2  于 2023-04-12  发布在  其他
关注(0)|答案(1)|浏览(130)

我正在尝试为我使用预训练的tensorflow XceptionNet创建的模型生成输入图像的热图。
我的模型结构是:

from tensorflow.keras import Model
from tensorflow.keras.layers import Input, Conv2D, MaxPool2D, Dense, Flatten, Dropout, AveragePooling2D, Concatenate, GlobalAveragePooling2D, BatchNormalization, ReLU, Add, SeparableConv2D
from tensorflow.keras.applications import Xception

def xception(img_shape, n_classes):
    xceptionnet = Xception(input_shape=img_shape, include_top=False, weights='imagenet')
    xceptionnet.trainable = False

    input = Input(img_shape)
    x = xceptionnet(input, training=False)
    x = GlobalAveragePooling2D()(x)
    x = Dropout(rate = 0.2)(x)

    output = Dense(n_classes, activation='softmax')(x)

    model = Model(input, output)
    return model

input_shape = (256, 256, 3)
n_classes = 3

model = xception(input_shape, n_classes)
model.compile('Adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
model.summary()

Model Structure [Output of model.summary()]
我尝试使用Keras文档中提到的相同格式(https://keras.io/examples/vision/grad_cam/)来生成数据集中图像的热图。
因此,根据文档,我的模型的显示图像部分是:

from IPython.display import Image, display
import matplotlib.pyplot as plt
import matplotlib.cm as cm

img_size = (256, 256, 3)
preprocess_input = keras.applications.xception.preprocess_input
decode_predictions = keras.applications.xception.decode_predictions

last_conv_layer_name = "xception"

# The local path to our target image
img_path = '/content/drive/My Drive/Colab Notebooks/data/Malignant/Malignant case (1).jpg'

display(Image(img_path))

上面的部分工作得很好。
但现在当我执行该部分时:

def make_gradcam_heatmap(img_array, model, last_conv_layer_name, pred_index=None):
    # First, we create a model that maps the input image to the activations
    # of the last conv layer as well as the output predictions

    last_conv_layer = model.get_layer(last_conv_layer_name)
    new_model = tf.keras.models.Sequential()
    for layer in model.layers[:model.layers.index(last_conv_layer)+1]:
        new_model.add(layer)
    new_model.add(tf.keras.layers.Flatten())
    grad_model = tf.keras.models.Model(inputs=[new_model.input], outputs=[new_model.output, model.output])

    
    with tf.GradientTape() as tape:
        last_conv_layer_output, preds = grad_model(img_array)
        last_conv_layer_output = tf.reshape(last_conv_layer_output, shape=(8, 8, 2048))
        if pred_index is None:
            pred_index = tf.argmax(preds[0])
        class_channel = preds[:, pred_index]

    # This is the gradient of the output neuron (top predicted or chosen)
    # with regard to the output feature map of the last conv layer
    grads = tape.gradient(class_channel, last_conv_layer_output)

    # This is a vector where each entry is the mean intensity of the gradient
    # over a specific feature map channel
    pooled_grads = tf.reduce_mean(grads, axis=(0, 1, 2))

    # We multiply each channel in the feature map array
    # by "how important this channel is" with regard to the top predicted class
    # then sum all the channels to obtain the heatmap class activation
    last_conv_layer_output = last_conv_layer_output[0]
    heatmap = last_conv_layer_output @ pooled_grads[..., tf.newaxis]
    heatmap = tf.squeeze(heatmap)

    # For visualization purpose, we will also normalize the heatmap between 0 & 1
    heatmap = tf.maximum(heatmap, 0) / tf.math.reduce_max(heatmap)
    return heatmap.numpy()
from tensorflow.keras.models import load_model

# Prepare image
img_array = preprocess_input(get_img_array(img_path, size=img_size))

# Make model
model = load_model('/content/drive/My Drive/Colab Notebooks/models/imageclassifier1.h5')

# Remove last layer's softmax
model.layers[-1].activation = None

# Print what the top predicted class is
preds = model.predict(img_array)
preds = np.argmax(preds[0])

labels = { 0 : "Cat",
           1 : "Dog",
           2 : "Human"}

# print("Predicted:", decode_predictions(preds, top=1)[0])  # For pre-trained models
print("Predicted:", labels[preds])

# Generate class activation heatmap
heatmap = make_gradcam_heatmap(img_array, model, last_conv_layer_name)

# Display heatmap
plt.matshow(heatmap)
plt.show()

我在pooled_grads = tf.reduce_mean(grads, axis=(0, 1, 2))行中得到错误,说grad是None。
下面是我收到的错误消息:

ValueError                                Traceback (most recent call last)
<ipython-input-86-e8872c6548f7> in <cell line: 25>()
     23 
     24 # Generate class activation heatmap
---> 25 heatmap = make_gradcam_heatmap(img_array, model, last_conv_layer_name)
     26 
     27 # Display heatmap

<ipython-input-84-cf963b881b8a> in make_gradcam_heatmap(img_array, model, last_conv_layer_name, pred_index)
     50     # This is a vector where each entry is the mean intensity of the gradient
     51     # over a specific feature map channel
---> 52     pooled_grads = tf.reduce_mean(grads, axis=(0, 1, 2))
     53     print(pooled_grads)
     54 

/usr/local/lib/python3.9/dist-packages/tensorflow/python/util/traceback_utils.py in error_handler(*args, **kwargs)
    151     except Exception as e:
    152       filtered_tb = _process_traceback_frames(e.__traceback__)
--> 153       raise e.with_traceback(filtered_tb) from None
    154     finally:
    155       del filtered_tb

/usr/local/lib/python3.9/dist-packages/tensorflow/python/framework/constant_op.py in convert_to_eager_tensor(value, ctx, dtype)
    101       dtype = dtypes.as_dtype(dtype).as_datatype_enum
    102   ctx.ensure_initialized()
--> 103   return ops.EagerTensor(value, ctx.device_name, dtype)
    104 
    105 

ValueError: Attempt to convert a value (None) with an unsupported type (<class 'NoneType'>) to a Tensor.

我对这个东西很陌生,所以有人可以帮助我。

wljmcqd8

wljmcqd81#

它抛出一个错误,因为你的模型有一个嵌套结构(更大的分类模型中的xception模型)。
你可以通过在层中再次传递输入来构建模型。将xception层的输出作为最后一个卷积输出,并将Tensor传递到层的末尾以获得最终输出。然后最终将输出合并到一个新模型中,如下所示:

inputs = keras.Input((256, 256, 3))
xception = model.get_layer("xception")
last_conv_output = xception(inputs)
x = last_conv_output
for idx in range(2, len(model.layers)):
    x = model.layers[idx](x)
output = x

grad_model = keras.Model(inputs, [last_conv_output, output])

通过重构上面的代码来生成热图:

import tensorflow as tf

def make_gradcam_heatmap(inputs, grad_model, pred_index=None):
    # First, we create a model that maps the input image to the activations
    # of the last conv layer as well as the output predictions

    
    with tf.GradientTape() as tape:
        last_conv_layer_output, preds = grad_model(inputs)
        #print(preds)
        if pred_index is None:
            pred_index = tf.argmax(preds[0])
        #print(pred_index)
        class_channel = preds[:, pred_index]
        #print(class_channel)

    # This is the gradient of the output neuron (top predicted or chosen)
    # with regard to the output feature map of the last conv layer
    grads = tape.gradient(class_channel, last_conv_layer_output)

    # This is a vector where each entry is the mean intensity of the gradient
    # over a specific feature map channel
    pooled_grads = tf.reduce_mean(grads, axis=(0, 1, 2))

    # We multiply each channel in the feature map array
    # by "how important this channel is" with regard to the top predicted class
    # then sum all the channels to obtain the heatmap class activation
    last_conv_layer_output = last_conv_layer_output[0]
    heatmap = last_conv_layer_output @ pooled_grads[..., tf.newaxis]
    heatmap = tf.squeeze(heatmap)

    # For visualization purpose, we will also normalize the heatmap between 0 & 1
    heatmap = tf.maximum(heatmap, 0) / tf.math.reduce_max(heatmap)
    return heatmap.numpy()

输出:

inputs = tf.random.uniform((1, 256, 256, 3))
make_gradcam_heatmap(inputs, grad_model)

array([[0.        , 0.        , 0.        , 0.0019973 , 0.00224069,
        0.        , 0.        , 0.        ],
       [0.        , 0.        , 0.01608142, 0.13859619, 0.19170615,
        0.03831227, 0.        , 0.        ],
       [0.00344855, 0.15640435, 0.39062408, 0.57898533, 0.72229344,
        0.18632776, 0.08909718, 0.00205518],
       [0.05994121, 0.41158128, 0.55284446, 0.8489698 , 0.96675164,
        0.34517574, 0.30315596, 0.05326002],
       [0.07081833, 0.4438232 , 0.6151547 , 0.9064342 , 0.9261135 ,
        0.41782287, 0.34709153, 0.09646279],
       [0.00530773, 0.22800735, 0.52887404, 0.8523431 , 1.        ,
        0.5120882 , 0.23707563, 0.        ],
       [0.        , 0.03709193, 0.20877707, 0.5426089 , 0.53451735,
        0.24202193, 0.        , 0.        ],
       [0.        , 0.01366277, 0.03030644, 0.14712998, 0.19128165,
        0.        , 0.        , 0.        ]], dtype=float32)

相关问题