Paddle 预测出core，能帮忙看下啥原因不？paddle训练和预测的版本都是v1.3.0

nuypyhwy  于 2021-12-07  发布在  Java
关注(0)|答案(2)|浏览(335)
  2. # 0  tcmalloc::CentralFreeList::FetchFromSpans (this=0x189eac0 <tcmalloc::Static::central_cache_+2304>) at src/central_freelist.cc:262
  4. # 1  0x0000000000858311 in tcmalloc::CentralFreeList::FetchFromSpansSafe (this=0x189eac0 <tcmalloc::Static::central_cache_+2304>)
  6.     at src/central_freelist.cc:247
  8. # 2  0x00000000008583a1 in tcmalloc::CentralFreeList::RemoveRange (this=0x189eac0 <tcmalloc::Static::central_cache_+2304>,
  10.     start=0x7f57adce8878, end=0x7f57adce8880, N=6) at src/central_freelist.cc:227
  12. # 3  0x000000000085957b in tcmalloc::ThreadCache::FetchFromCentralCache (this=0x1c97000, cl=<optimized out>, byte_size=16)
  14.     at src/static_vars.h:59
  16. # 4  0x00000000008529de in tcmalloc::ThreadCache::Allocate (this=0x189eac0 <tcmalloc::Static::central_cache_+2304>, size=0,
  18.     cl=140014554876024) at src/thread_cache.h:329
  20. # 5  0x00000000008529de in CheckedMallocResult (result=<optimized out>)
  22. # 6  0x00000000008529de in (anonymous namespace)::do_malloc (size=16)
  24. # 7  0x00000000008529de in (anonymous namespace)::cpp_alloc (size=<optimized out>, nothrow=false)
  26. # 8  0x00000000008de4da in tc_new (size=25815744, size@entry=16) at src/tcmalloc.cc:1521
  28. # 9  0x00007f5954e0aa81 in __gnu_cxx::new_allocator<std::__detail::_Hash_node_base*>::allocate (this=<synthetic pointer>, __n=<optimized out>)
  30.     at /home/opt/gcc-4.8.2.bpkg-r4/gcc-4.8.2.bpkg-r4/include/c++/4.8.2/ext/new_allocator.h:104
  32. # 10 std::_Hashtable<std::string, std::pair<std::string const, Xbyak::JmpLabel const>, std::allocator<std::pair<std::string const, Xbyak::JmpLabel const> >, std::__detail::_Select1st, std::equal_to<std::string>, std::hash<std::string>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, false> >::_M_allocate_buckets (
  34.     this=0x7f57adce8990, __n=<optimized out>) at /home/opt/gcc-4.8.2.bpkg-r4/gcc-4.8.2.bpkg-r4/include/c++/4.8.2/bits/hashtable.h:779
  36. # 11 std::_Hashtable<std::string, std::pair<std::string const, Xbyak::JmpLabel const>, std::allocator<std::pair<std::string const, Xbyak::JmpLabel const> >, std::__detail::_Select1st, std::equal_to<std::string>, std::hash<std::string>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, false> >::_Hashtable(std::_Hashtable<std::string, std::pair<std::string const, Xbyak::JmpLabel const>, std::allocator<std::pair<std::string const, Xbyak::JmpLabel const> >, std::__detail::_Select1st, std::equal_to<std::string>, std::hash<std::string>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, false> >&&) (this=<optimized out>,
  38.     __ht=<unknown type in ./fluid_inference/paddle/lib/libpaddle_fluid.so, CU 0x798c45d, DIE 0x79e7754>)
  39.     at /home/opt/gcc-4.8.2.bpkg-r4/gcc-4.8.2.bpkg-r4/include/c++/4.8.2/bits/hashtable.h:945
  41. # 12 0x00007f5954e0ab55 in std::unordered_multimap<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, Xbyak::JmpLabel const, std::hash<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::basic_string<char, std::char_traits<---Type <return> to continue, or q <return> to quit---
  43. char>, std::allocator<char> > >, std::allocator<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, Xbyak::JmpLabel const> > >::unordered_multimap (this=0x46f31e40)
  44.     at /home/opt/gcc-4.8.2.bpkg-r4/gcc-4.8.2.bpkg-r4/include/c++/4.8.2/bits/unordered_map.h:852
  46. # 13 Xbyak::LabelManager::SlabelState::SlabelState (this=0x46f31e10)
  48.     at /home/work/chenxiue/Paddle/build/third_party/install/xbyak/include/xbyak/xbyak.h:1146
  50. # 14 std::_List_node<Xbyak::LabelManager::SlabelState>::_List_node<Xbyak::LabelManager::SlabelState> (this=<optimized out>)
  52.     at /home/opt/gcc-4.8.2.bpkg-r4/gcc-4.8.2.bpkg-r4/include/c++/4.8.2/bits/stl_list.h:114
  54. # 15 __gnu_cxx::new_allocator<std::_List_node<Xbyak::LabelManager::SlabelState> >::construct<std::_List_node<Xbyak::LabelManager::SlabelState>, Xbyak::LabelManager::SlabelState> (__p=<optimized out>, this=0x472cc090)
  56.     at /home/opt/gcc-4.8.2.bpkg-r4/gcc-4.8.2.bpkg-r4/include/c++/4.8.2/ext/new_allocator.h:120
  58. # 16 std::list<Xbyak::LabelManager::SlabelState, std::allocator<Xbyak::LabelManager::SlabelState> >::_M_create_node<Xbyak::LabelManager::SlabelState> (this=0x472cc090) at /home/opt/gcc-4.8.2.bpkg-r4/gcc-4.8.2.bpkg-r4/include/c++/4.8.2/bits/stl_list.h:505
  60. # 17 std::list<Xbyak::LabelManager::SlabelState, std::allocator<Xbyak::LabelManager::SlabelState> >::_M_insert<Xbyak::LabelManager::SlabelState> (__position=..., this=0x472cc090) at /home/opt/gcc-4.8.2.bpkg-r4/gcc-4.8.2.bpkg-r4/include/c++/4.8.2/bits/stl_list.h:1561
  62. # 18 std::list<Xbyak::LabelManager::SlabelState, std::allocator<Xbyak::LabelManager::SlabelState> >::push_back(Xbyak::LabelManager::SlabelState&&) (__x=<unknown type in ./fluid_inference/paddle/lib/libpaddle_fluid.so, CU 0x798c45d, DIE 0x79e7959>, this=0x472cc090)
  64.     at /home/opt/gcc-4.8.2.bpkg-r4/gcc-4.8.2.bpkg-r4/include/c++/4.8.2/bits/stl_list.h:1021
  66. # 19 Xbyak::LabelManager::reset (this=0x472cc088) at /home/work/chenxiue/Paddle/build/third_party/install/xbyak/include/xbyak/xbyak.h:1245
  68. # 20 0x00007f5954e35a5d in Xbyak::LabelManager::LabelManager (this=0x472cc088)
  70.     at /home/work/chenxiue/Paddle/build/third_party/install/xbyak/include/xbyak/xbyak.h:1238
  72. # 21 Xbyak::CodeGenerator::CodeGenerator (allocator=0x0, userPtr=0x0, maxSize=<optimized out>, this=0x472cc008)
  74.     at /home/work/chenxiue/Paddle/build/third_party/install/xbyak/include/xbyak/xbyak.h:2427
  76. # 22 paddle::operators::jit::gen::JitCode::JitCode (code_ptr=0x0, code_size=<optimized out>, this=0x472cc000)
  78.     at /home/work/chenxiue/Paddle/paddle/fluid/operators/jit/gen/jitcode.h:70
  80. # 23 paddle::operators::jit::gen::SeqPoolJitCode::SeqPoolJitCode (code_ptr=0x0, code_size=<optimized out>, attr=..., this=0x472cc000)
  82.     at /home/work/chenxiue/Paddle/paddle/fluid/operators/jit/gen/seqpool.h:32
  84. # 24 paddle::operators::jit::make_unique<paddle::operators::jit::gen::SeqPoolJitCode, paddle::operators::jit::seq_pool_attr_s const&, unsigned long> () at /home/work/chenxiue/Paddle/paddle/fluid/operators/jit/registry.h:33
  86. # 25 paddle::operators::jit::gen::SeqPoolCreator::CreateJitCode (this=<optimized out>, attr=...)
  88. ---Type <return> to continue, or q <return> to quit---
  89.     at /home/work/chenxiue/Paddle/paddle/fluid/operators/jit/gen/seqpool.cc:74
  91. # 26 0x00007f5954c63114 in paddle::operators::jit::GetJitCode<(paddle::operators::jit::KernelType)21, paddle::operators::jit::SeqPoolTuples<float>, paddle::platform::CPUPlace> (attr=...) at /home/work/chenxiue/Paddle/paddle/fluid/operators/jit/helper.h:54
  93. # 27 0x00007f5954c632eb in paddle::operators::jit::Get<(paddle::operators::jit::KernelType)21, paddle::operators::jit::SeqPoolTuples<float>, paddle::platform::CPUPlace> (attr=...) at /home/work/chenxiue/Paddle/paddle/fluid/operators/jit/helper.h:98
  95. # 28 0x00007f59554d87ca in paddle::operators::math::SequencePoolFunctor<paddle::platform::CPUDeviceContext, float>::operator() (
  97.     this=this@entry=0x7f57adce8f9e, context=..., pooltype="SUM", input=..., output=output@entry=0x7f0da68, is_test=is_test@entry=true, 
  98.     index=index@entry=0x0) at /home/work/chenxiue/Paddle/paddle/fluid/operators/math/sequence_pooling.cc:260
  100. # 29 0x00007f5954de2155 in paddle::operators::SequencePoolKernel<paddle::platform::CPUDeviceContext, float>::Compute (
  102.     this=this@entry=0x7f57adce9640, context=...) at /home/work/chenxiue/Paddle/paddle/fluid/operators/sequence_ops/sequence_pool_op.h:61
  104. # 30 0x00007f5954de2a93 in paddle::framework::OpKernelRegistrarFunctor<paddle::platform::CPUPlace, false, 0ul, paddle::operators::SequencePoolKernel<paddle::platform::CPUDeviceContext, float> >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}::operator()(paddle::framework::ExecutionContext const&) const (ctx=..., __closure=<optimized out>)
  106.     at /home/work/chenxiue/Paddle/paddle/fluid/framework/op_registry.h:107
  108. # 31 std::_Function_handler<void (paddle::framework::ExecutionContext const&), paddle::framework::OpKernelRegistrarFunctor<paddle::platform::CPUPlace, false, 0ul, paddle::operators::SequencePoolKernel<paddle::platform::CPUDeviceContext, float> >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) (__functor=..., __args#0=...) at /home/opt/gcc-4.8.2.bpkg-r4/gcc-4.8.2.bpkg-r4/include/c++/4.8.2/functional:2071
  110. # 32 0x00007f5955554c53 in std::function<void (paddle::framework::ExecutionContext const&)>::operator()(paddle::framework::ExecutionContext const&) const (__args#0=..., this=0x1e01e70) at /home/opt/gcc-4.8.2.bpkg-r4/gcc-4.8.2.bpkg-r4/include/c++/4.8.2/functional:2471
  112. # 33 paddle::framework::OperatorWithKernel::RunImpl (this=0x7f9cdc0, scope=..., place=...)
  114.     at /home/work/chenxiue/Paddle/paddle/fluid/framework/operator.cc:963
  116. # 34 0x00007f595555377b in paddle::framework::OperatorBase::Run (this=0x7f9cdc0, scope=..., place=...)
  118.     at /home/work/chenxiue/Paddle/paddle/fluid/framework/operator.cc:185
  120. # 35 0x00007f59549dd04a in paddle::framework::Executor::RunPreparedContext (this=0x77f31d0, ctx=0x7f32120, scope=scope@entry=0x7efbcc0,
  122.     create_local_scope=create_local_scope@entry=false, create_vars=create_vars@entry=false, keep_kids=keep_kids@entry=false)
  123.     at /home/work/chenxiue/Paddle/paddle/fluid/framework/executor.cc:433
  125. # 36 0x00007f595489ca41 in paddle::NativePaddlePredictor::Run (this=0x7d48540, inputs=..., output_data=0x7f57adce9fe0,
  127.     batch_size=<optimized out>) at /home/work/chenxiue/Paddle/paddle/fluid/inference/api/api_impl.cc:148
  128. ---Type <return> to continue, or q <return> to quit---
  130. # 37 0x00000000004a0647 in baidu::pmallrec::GbdtModel::predict (this=0x1e85240, user_id=18446744073709550617, user_feature=0x46fdc7b0,
  132.     recallpidlist=std::vector of length 110, capacity 128 = {...}, pidlist=std::vector of length 0, capacity 0, get_dict=true, 
  133.     log_id="40534463", statusstr="") at baidu/netdisk/dp-pmall-recapi/src/baidu/pmallrec/gbdt_model.cpp:475
Paddle

2条答案

按热度按时间
1qczuiv0

1qczuiv01#

predictor的创建是在线程外创建的么

赞(0）分享  回复(0）举报  2021-12-07
niwlg2el

niwlg2el2#

是在线程外创建的，补充一下，调整了brpc的线程数是1的时候，压测都没问题，可以正常预测。但是压测的情况下会出core，信息：

terminate called after throwing an instance of 'paddle::platform::EnforceNotMet'
what(): Invoke operator lookup_table error.
Python Callstacks:
File "/home/work/gongxijun/anaconda2/lib/python2.7/site-packages/paddle/fluid/framework.py", line 1317, in append_op
attrs=kwargs.get("attrs", None))
File "/home/work/gongxijun/anaconda2/lib/python2.7/site-packages/paddle/fluid/layer_helper.py", line 56, in append_op
return self.main_program.current_block().append_op(*args,**kwargs)
File "/home/work/gongxijun/anaconda2/lib/python2.7/site-packages/paddle/fluid/layers/nn.py", line 364, in embedding
'padding_idx': padding_idx
File "/home/work/gongxijun/online/rank/huangdao/huangdao/training/feature/extra_feature.py", line 118, in get_item_embedding
is_sparse=is_sparse)
File "/home/work/gongxijun/online/rank/huangdao/huangdao/training/feature/extra_feature.py", line 303, in get_item_combined_features_lr
is_sparse=True))
File "/home/work/gongxijun/online/rank/huangdao/huangdao/training/algorithm/gbdt_lr_trainer.py", line 266, in net_program
usr_combined_features = extra_feature.get_item_combined_features_lr(self.user_feature_col)
File "/home/work/gongxijun/online/rank/huangdao/huangdao/training/algorithm/gbdt_lr_trainer.py", line 163, in load
[inference, auc_var, avg_cost, acc] = self.net_program()
File "/home/work/gongxijun/online/rank/huangdao/huangdao/training/algorithm/gbdt_lr_trainer.py", line 299, in train
[inference, auc_var, avg_cost, acc] = self.load()
File "/home/work/gongxijun/online/rank/huangdao/huangdao/training/train_factory.py", line 55, in start
_train.train()
File "huangdao/huangdao_main.py", line 42, in run
self.__trainer.start()
File "huangdao/huangdao_main.py", line 47, in
Huangdao().run()

C++ Callstacks:
Enforce failed. Expected ids[i] < row_number, but received ids[i]:4607182418800017408 >= row_number:21.
at [/home/work/chenxiue/Paddle/paddle/fluid/operators/lookup_table_op.h:86]
PaddlePaddle Call Stacks:
0 0x7ffcd6982471p void paddle::platform::EnforceNotMet::Initstd::string(std::string, char const*, int) + 1537
1 0x7ffcd6983c1ap paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) + 90
2 0x7ffcd72a6a48p paddle::operators::LookupTableKernel::Compute(paddle::framework::ExecutionContext const&) const + 2824
3 0x7ffcd72a6ef3p std::Function_handler<void (paddle::framework::ExecutionContext const&), paddle::framework::OpKernelRegistrarFunctor<paddle::platform::CPUPlace, false, 0ul, paddle::operators::LookupTableKernel, paddle::operators::LookupTableKernel >::operator()(char const, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::*M_invoke(std::Any_data const&, paddle::framework::ExecutionContext const&) + 35
4 0x7ffcd763ec53p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void, boost::detail::variant::void*, boost::detail::variant::void*, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 659
5 0x7ffcd763d77bp paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) + 267
6 0x7ffcd6ac704ap paddle::framework::Executor::RunPreparedContext(paddle::framework::ExecutorPrepareContext*, paddle::framework::Scope*, bool, bool, bool) + 218
7 0x7ffcd6986a41p paddle::NativePaddlePredictor::Run(std::vector<paddle::PaddleTensor, std::allocatorpaddle::PaddleTensor > const&, std::vector<paddle::PaddleTensor, std::allocatorpaddle::PaddleTensor >, int) + 273
8 0x4a0697p
9 0x4c44c8p
10 0x495a20p
11 0x594116p baidu::rpc::policy::ProcessHttpRequest(baidu::rpc::InputMessageBase) + 3686
12 0x4eae1ap baidu::rpc::ProcessInputMessage(void*) + 10
13 0x4ec27fp baidu::rpc::InputMessenger::OnNewMessages(baidu::rpc::Socket*) + 383
14 0x51518dp baidu::rpc::Socket::ProcessEvent(void*) + 13
15 0x67bc6ap bthread::TaskGroup::task_runner(long) + 266
16 0x6713a1p

展开查看全部
赞(0）分享  回复(0）举报  2021-12-07
我来回答

相关问题

    查看更多

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新问答

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com