请提出你的问题 Please ask your question

推理库编译完后，按 https://www.paddlepaddle.org.cn/inference/v2.4/guides/nv_gpu_infer/gpu_native_infer.html的步骤利用编译好的paddle_inference_install_dir库建立工程，并用Paddle-Inference-Demo\c++\gpu\yolov3的yolov3_test.cc来测试，VC采用2019社区版，运行情况如下：
不打开TRT可正常运行，运行信息如下：
inference.exe --model_file ./model/yolov3_r50vd_dcn_270e_coco/model.pdmodel --params_file ./model/yolov3_r50vd_dcn_270e_coco/model.pdiparams
e[1me[35m--- Running analysis [ir_graph_build_pass]e[0m
WARNING: Logging before InitGoogleLogging() is written to STDERR
I0217 15:12:32.518298 6720 executor.cc:186] Old Executor is Running.
e[1me[35m--- Running analysis [ir_analysis_pass]e[0m
e[32m--- Running IR pass [map_op_to_another_pass]e[0m
e[32m--- Running IR pass [identity_scale_op_clean_pass]e[0m
I0217 15:12:32.691711 6720 fuse_pass_base.cc:59] --- detected 2 subgraphs
e[32m--- Running IR pass [is_test_pass]e[0m
e[32m--- Running IR pass [simplify_with_basic_ops_pass]e[0m
e[32m--- Running IR pass [delete_quant_dequant_linear_op_pass]e[0m
e[32m--- Running IR pass [delete_weight_dequant_linear_op_pass]e[0m
e[32m--- Running IR pass [constant_folding_pass]e[0m
e[32m--- Running IR pass [silu_fuse_pass]e[0m
e[32m--- Running IR pass [conv_bn_fuse_pass]e[0m
I0217 15:12:32.939219 6720 fuse_pass_base.cc:59] --- detected 72 subgraphs
e[32m--- Running IR pass [conv_eltwiseadd_bn_fuse_pass]e[0m
e[32m--- Running IR pass [embedding_eltwise_layernorm_fuse_pass]e[0m
e[32m--- Running IR pass [multihead_matmul_fuse_pass_v2]e[0m
e[32m--- Running IR pass [vit_attention_fuse_pass]e[0m
e[32m--- Running IR pass [fused_multi_transformer_encoder_pass]e[0m
e[32m--- Running IR pass [fused_multi_transformer_decoder_pass]e[0m
e[32m--- Running IR pass [fused_multi_transformer_encoder_fuse_qkv_pass]e[0m
e[32m--- Running IR pass [fused_multi_transformer_decoder_fuse_qkv_pass]e[0m
e[32m--- Running IR pass [multi_devices_fused_multi_transformer_encoder_pass]e[0m
e[32m--- Running IR pass [multi_devices_fused_multi_transformer_encoder_fuse_qkv_pass]e[0m
e[32m--- Running IR pass [multi_devices_fused_multi_transformer_decoder_fuse_qkv_pass]e[0m
e[32m--- Running IR pass [fuse_multi_transformer_layer_pass]e[0m
e[32m--- Running IR pass [gpu_cpu_squeeze2_matmul_fuse_pass]e[0m
e[32m--- Running IR pass [gpu_cpu_reshape2_matmul_fuse_pass]e[0m
e[32m--- Running IR pass [gpu_cpu_flatten2_matmul_fuse_pass]e[0m
e[32m--- Running IR pass [gpu_cpu_map_matmul_v2_to_mul_pass]e[0m
e[32m--- Running IR pass [gpu_cpu_map_matmul_v2_to_matmul_pass]e[0m
e[32m--- Running IR pass [matmul_scale_fuse_pass]e[0m
e[32m--- Running IR pass [multihead_matmul_fuse_pass_v3]e[0m
e[32m--- Running IR pass [gpu_cpu_map_matmul_to_mul_pass]e[0m
e[32m--- Running IR pass [fc_fuse_pass]e[0m
e[32m--- Running IR pass [fc_elementwise_layernorm_fuse_pass]e[0m
e[32m--- Running IR pass [conv_elementwise_add_act_fuse_pass]e[0m
e[32m--- Running IR pass [conv_elementwise_add2_act_fuse_pass]e[0m
e[32m--- Running IR pass [conv_elementwise_add_fuse_pass]e[0m
I0217 15:12:34.549715 6720 fuse_pass_base.cc:59] --- detected 42 subgraphs
e[32m--- Running IR pass [transpose_flatten_concat_fuse_pass]e[0m
e[32m--- Running IR pass [auto_mixed_precision_pass]e[0m
e[32m--- Running IR pass [inplace_op_var_pass]e[0m
e[1me[35m--- Running analysis [ir_params_sync_among_devices_pass]e[0m
I0217 15:12:34.565307 6720 ir_params_sync_among_devices_pass.cc:94] Sync params from CPU to GPU
e[1me[35m--- Running analysis [adjust_cudnn_workspace_size_pass]e[0m
e[1me[35m--- Running analysis [inference_op_replace_pass]e[0m
e[1me[35m--- Running analysis [memory_optimize_pass]e[0m
I0217 15:12:34.705691 6720 memory_optimize_pass.cc:220] Cluster name : batch_norm_50.tmp_2 size: 739328
I0217 15:12:34.705691 6720 memory_optimize_pass.cc:220] Cluster name : batch_norm_9.tmp_2 size: 23658496
I0217 15:12:34.705691 6720 memory_optimize_pass.cc:220] Cluster name : relu_5.tmp_0 size: 23658496
I0217 15:12:34.705691 6720 memory_optimize_pass.cc:220] Cluster name : elementwise_add_1 size: 23658496
I0217 15:12:34.705691 6720 memory_optimize_pass.cc:220] Cluster name : yolo_box_1.tmp_0 size: 69312
I0217 15:12:34.705691 6720 memory_optimize_pass.cc:220] Cluster name : image size: 4435968
I0217 15:12:34.705691 6720 memory_optimize_pass.cc:220] Cluster name : elementwise_add_11 size: 5914624
I0217 15:12:34.705691 6720 memory_optimize_pass.cc:220] Cluster name : elementwise_add_15 size: 2957312
I0217 15:12:34.705691 6720 memory_optimize_pass.cc:220] Cluster name : yolo_box_0.tmp_0 size: 17328
I0217 15:12:34.705691 6720 memory_optimize_pass.cc:220] Cluster name : scale_factor size: 8
I0217 15:12:34.705691 6720 memory_optimize_pass.cc:220] Cluster name : cast_0.tmp_0 size: 8
I0217 15:12:34.705691 6720 memory_optimize_pass.cc:220] Cluster name : im_shape size: 8
e[1me[35m--- Running analysis [ir_graph_to_program_pass]e[0m
I0217 15:12:34.799851 6720 analysis_predictor.cc:1385] ======= optimize end =======
I0217 15:12:34.799851 6720 naive_executor.cc:151] --- skip [feed], feed -> scale_factor
I0217 15:12:34.799851 6720 naive_executor.cc:151] --- skip [feed], feed -> image
I0217 15:12:34.799851 6720 naive_executor.cc:151] --- skip [feed], feed -> im_shape
I0217 15:12:34.799851 6720 naive_executor.cc:151] --- skip [save_infer_model/scale_0.tmp_1], fetch -> fetch
I0217 15:12:34.799851 6720 naive_executor.cc:151] --- skip [save_infer_model/scale_1.tmp_1], fetch -> fetch
W0217 15:12:34.799851 6720 gpu_resources.cc:85] Please NOTE: device: 0, GPU Compute Capability: 8.6, Driver API Version: 11.7, Runtime API Version: 11.6
W0217 15:12:34.815106 6720 gpu_resources.cc:115] device: 0, cuDNN Version: 8.4.
WARNING: Logging before InitGoogleLogging() is written to STDERR
I0217 15:12:37.205616 6720 inference.cc:319] output num is 12

打开TRT后无法正常运行，运行信息如下：
inference.exe --model_file ./model/yolov3_r50vd_dcn_270e_coco/model.pdmodel --params_file ./model/yolov3_r50vd_dcn_270e_coco/model.pdiparams --run_mode=trt_fp32
WARNING: Logging before InitGoogleLogging() is written to STDERR
I0217 14:43:12.707584 9860 analysis_predictor.cc:1131] TensorRT subgraph engine is enabled
e[1me[35m--- Running analysis [ir_graph_build_pass]e[0m
I0217 14:43:12.721602 9860 executor.cc:186] Old Executor is Running.
e[1me[35m--- Running analysis [ir_analysis_pass]e[0m
e[32m--- Running IR pass [trt_support_nhwc_pass]e[0m
e[32m--- Running IR pass [adaptive_pool2d_convert_global_pass]e[0m
e[32m--- Running IR pass [shuffle_channel_detect_pass]e[0m
e[32m--- Running IR pass [quant_conv2d_dequant_fuse_pass]e[0m
e[32m--- Running IR pass [delete_fill_constant_op_pass]e[0m
e[32m--- Running IR pass [delete_quant_dequant_op_pass]e[0m
e[32m--- Running IR pass [delete_quant_dequant_filter_op_pass]e[0m
e[32m--- Running IR pass [trt_delete_weight_dequant_linear_op_pass]e[0m
e[32m--- Running IR pass [delete_quant_dequant_linear_op_pass]e[0m
e[32m--- Running IR pass [identity_scale_op_clean_pass]e[0m
I0217 14:43:12.940196 9860 fuse_pass_base.cc:59] --- detected 2 subgraphs
e[32m--- Running IR pass [add_support_int8_pass]e[0m
I0217 14:43:13.096343 9860 fuse_pass_base.cc:59] --- detected 295 subgraphs
e[32m--- Running IR pass [simplify_with_basic_ops_pass]e[0m
e[32m--- Running IR pass [trt_embedding_eltwise_layernorm_fuse_pass]e[0m
e[32m--- Running IR pass [preln_embedding_eltwise_layernorm_fuse_pass]e[0m
e[32m--- Running IR pass [delete_c_identity_op_pass]e[0m
e[32m--- Running IR pass [trt_multihead_matmul_fuse_pass_v2]e[0m
e[32m--- Running IR pass [trt_multihead_matmul_fuse_pass_v3]e[0m
e[32m--- Running IR pass [multihead_matmul_roformer_fuse_pass]e[0m
e[32m--- Running IR pass [constant_folding_pass]e[0m
e[32m--- Running IR pass [trt_flash_multihead_matmul_fuse_pass]e[0m
e[32m--- Running IR pass [trt_cross_multihead_matmul_fuse_pass]e[0m
e[32m--- Running IR pass [vit_attention_fuse_pass]e[0m
e[32m--- Running IR pass [layernorm_shift_partition_fuse_pass]e[0m
e[32m--- Running IR pass [merge_layernorm_fuse_pass]e[0m
e[32m--- Running IR pass [preln_residual_bias_fuse_pass]e[0m
e[32m--- Running IR pass [preln_layernorm_x_fuse_pass]e[0m
e[32m--- Running IR pass [reverse_roll_fuse_pass]e[0m
e[32m--- Running IR pass [conv_bn_fuse_pass]e[0m
I0217 14:43:13.424523 9860 fuse_pass_base.cc:59] --- detected 72 subgraphs
e[32m--- Running IR pass [unsqueeze2_eltwise_fuse_pass]e[0m
e[32m--- Running IR pass [trt_squeeze2_matmul_fuse_pass]e[0m
e[32m--- Running IR pass [trt_flatten2_matmul_fuse_pass]e[0m
e[32m--- Running IR pass [trt_map_matmul_v2_to_mul_pass]e[0m
e[32m--- Running IR pass [trt_map_matmul_v2_to_matmul_pass]e[0m
e[32m--- Running IR pass [trt_map_matmul_to_mul_pass]e[0m
e[32m--- Running IR pass [fc_fuse_pass]e[0m
e[32m--- Running IR pass [conv_elementwise_add_fuse_pass]e[0m
I0217 14:43:13.486861 9860 fuse_pass_base.cc:59] --- detected 78 subgraphs
e[32m--- Running IR pass [remove_padding_recover_padding_pass]e[0m
e[32m--- Running IR pass [delete_remove_padding_recover_padding_pass]e[0m
e[32m--- Running IR pass [dense_fc_to_sparse_pass]e[0m
e[32m--- Running IR pass [dense_multihead_matmul_to_sparse_pass]e[0m
e[32m--- Running IR pass [tensorrt_subgraph_pass]e[0m
I0217 14:43:13.502879 9860 tensorrt_subgraph_pass.cc:232] --- detect a sub-graph with 195 nodes
I0217 14:43:13.582355 9860 tensorrt_subgraph_pass.cc:594] Prepare TRT engine (Optimize model structure, Select OP kernel etc). This process may cost a lot of time.
W0217 14:43:15.112041 9860 helper.h:123] The implicit batch dimension mode has been deprecated. Please create the network with NetworkDefinitionCreationFlag::kEXPLICIT_BATCH flag whenever possible.
W0217 14:43:15.112041 9860 place.cc:164] The paddle::PlaceType::kCPU/kGPU is deprecated since version 2.3, and will be removed in version 2.4! Please use Tensor::is_cpu()/is_gpu() method to determine the type of place.

程序如下：
#include
#include
#include
#include

#include <gflags/gflags.h>
#include <glog/logging.h>

#include "paddle/include/paddle_inference_api.h"

using paddle_infer::Config;
using paddle_infer::Predictor;
using paddle_infer::CreatePredictor;
using paddle_infer::PrecisionType;

DEFINE_string(model_file, "", "Directory of the inference model.");
DEFINE_string(params_file, "", "Directory of the inference model.");
DEFINE_string(model_dir, "", "Directory of the inference model.");
DEFINE_int32(batch_size, 1, "Directory of the inference model.");
DEFINE_int32(warmup, 0, "warmup.");
DEFINE_int32(repeats, 1, "repeats.");
DEFINE_string(run_mode, "paddle_gpu", "run_mode which can be: trt_fp32, trt_fp16, trt_int8 and paddle_gpu");
DEFINE_bool(use_dynamic_shape, false, "use trt dynaminc shape.");

const int img_height = 512; //608
const int img_width = 512;

using Time = decltype(std::chrono::high_resolution_clock::now());
Time time() { return std::chrono::high_resolution_clock::now(); };
double time_diff(Time t1, Time t2) {
typedef std::chrono::microseconds ms;
auto diff = t2 - t1;
ms counter = std::chrono::duration_cast(diff);
return counter.count() / 1000.0;
}

std::shared_ptr InitPredictor() {
Config config;
if (FLAGS_model_dir != "") {
config.SetModel(FLAGS_model_dir);
}
config.SetModel(FLAGS_model_file, FLAGS_params_file);
config.EnableUseGpu(500, 0);

if (FLAGS_run_mode == "trt_fp32") {
config.EnableTensorRtEngine(1 << 30, FLAGS_batch_size, 5,
PrecisionType::kFloat32, false, false);
} else if (FLAGS_run_mode == "trt_fp16") {
config.EnableTensorRtEngine(1 << 30, FLAGS_batch_size, 5,
PrecisionType::kHalf, false, false);
} else if (FLAGS_run_mode == "trt_int8") {
config.EnableTensorRtEngine(1 << 30, FLAGS_batch_size, 5,
PrecisionType::kInt8, false, true);
}

if(FLAGS_use_dynamic_shape){
std::map<std::string, std::vector> min_input_shape = {
{"im_shape", {FLAGS_batch_size, 2}},
{"image", {FLAGS_batch_size, 3, 112, 112}},
{"scale_factor", {FLAGS_batch_size, 2}}
};
std::map<std::string, std::vector> max_input_shape = {
{"im_shape", {FLAGS_batch_size, 2}},
{"image", {FLAGS_batch_size, 3, img_height, img_width}},
{"scale_factor", {FLAGS_batch_size, 2}}
};
std::map<std::string, std::vector> opt_input_shape = {
{"im_shape", {FLAGS_batch_size, 2}},
{"image", {FLAGS_batch_size, 3, img_height, img_width}},
{"scale_factor", {FLAGS_batch_size, 2}}
};
config.SetTRTDynamicShapeInfo(min_input_shape, max_input_shape,
opt_input_shape);
}

// Open the memory optim.
config.EnableMemoryOptim();
return CreatePredictor(config);
}

void run(Predictor *predictor, const std::vector &input,
const std::vector &input_shape, const std::vector &input_im,
const std::vector &input_im_shape, std::vector *out_data) {
auto input_names = predictor->GetInputNames();
auto im_shape_handle = predictor->GetInputHandle(input_names[0]);
im_shape_handle->Reshape(input_im_shape);
im_shape_handle->CopyFromCpu(input_im.data());

auto image_handle = predictor->GetInputHandle(input_names[1]);
image_handle->Reshape(input_shape);
image_handle->CopyFromCpu(input.data());

auto scale_factor_handle = predictor->GetInputHandle(input_names[2]);
scale_factor_handle->Reshape(input_im_shape);
scale_factor_handle->CopyFromCpu(input_im.data());

CHECK(predictor->Run());

auto output_names = predictor->GetOutputNames();
auto output_t = predictor->GetOutputHandle(output_names[0]);
std::vector output_shape = output_t->shape();
int out_num = std::accumulate(output_shape.begin(), output_shape.end(), 1,
std::multiplies());

out_data->resize(out_num);
output_t->CopyToCpu(out_data->data());
}

int main(int argc, char *argv[]) {
google::ParseCommandLineFlags(&argc, &argv, true);
auto predictor = InitPredictor();

const int height = img_height;
const int width = img_width;
const int channels = 3;
std::vector input_shape = {FLAGS_batch_size, channels, height, width};
std::vector input_data(FLAGS_batch_size * channels * height * width);
for (size_t i = 0; i < input_data.size(); ++i) {
input_data[i] = i % 255 * 0.13f;
}
std::vector input_im_shape = {FLAGS_batch_size, 2};
std::vector input_im_data(FLAGS_batch_size * 2, img_height);

std::vector out_data;
run(predictor.get(), input_data, input_shape, input_im_data, input_im_shape,
&out_data);
LOG(INFO) << "output num is " << out_data.size();
return 0;
}

编译环境信息：
GIT COMMIT ID: 16986d6
WITH_MKL: ON
WITH_MKLDNN: ON
WITH_GPU: ON
WITH_ROCM: OFF
WITH_ASCEND_CL: OFF
WITH_ASCEND_CXX11: OFF
WITH_IPU: OFF
CUDA version: 11.6
CUDNN version: v8.4
CXX compiler version: 19.29.30148.0
WITH_TENSORRT: ON
TensorRT version: v8.4.1.5

大家有遇到类似问题吗？如何解决啊？