看到类似的段错误，有时与0.5.0.post1一起出现：

model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/vllm/engine/async_llm_engine.py", line 776, in _process_request
model_server-1  |     |     async for request_output in stream:
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/vllm/engine/async_llm_engine.py", line 89, in __anext__
model_server-1  |     |     raise result
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/starlette/responses.py", line 261, in wrap
model_server-1  |     |     await func()
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/starlette/responses.py", line 250, in stream_response
model_server-1  |     |     async for chunk in self.body_iterator:
model_server-1  |     |   File "/vllm-workspace/src/api_v2.py", line 267, in stream_results
model_server-1  |     |     raise e
model_server-1  |     |   File "/vllm-workspace/src/api_v2.py", line 223, in stream_results
model_server-1  |     |     async for request_output in results_generator:
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/vllm/engine/async_llm_engine.py", line 673, in generate
model_server-1  |     |     async for output in self._process_request(
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/vllm/engine/async_llm_engine.py", line 780, in _process_request
model_server-1  |     |     raise e
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/vllm/engine/async_llm_engine.py", line 776, in _process_request
model_server-1  |     |     async for request_output in stream:
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/vllm/engine/async_llm_engine.py", line 89, in __anext__
model_server-1  |     |     raise result
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/starlette/responses.py", line 261, in wrap
model_server-1  |     |     await func()
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/starlette/responses.py", line 250, in stream_response
model_server-1  |     |     async for chunk in self.body_iterator:
model_server-1  |     |   File "/vllm-workspace/src/api_v2.py", line 267, in stream_results
model_server-1  |     |     raise e
model_server-1  |     |   File "/vllm-workspace/src/api_v2.py", line 223, in stream_results
model_server-1  |     |     async for request_output in results_generator:
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/vllm/engine/async_llm_engine.py", line 673, in generate
model_server-1  |     |     async for output in self._process_request(
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/vllm/engine/async_llm_engine.py", line 780, in _process_request
model_server-1  |     |     raise e
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/vllm/engine/async_llm_engine.py", line 776, in _process_request
model_server-1  |     |     async for request_output in stream:
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/vllm/engine/async_llm_engine.py", line 89, in __anext__
model_server-1  |     |     raise result
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/vllm/engine/async_llm_engine.py", line 42, in _log_task_completion
model_server-1  |     |     return_value = task.result()
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/vllm/engine/async_llm_engine.py", line 532, in run_engine_loop
model_server-1  |     |     has_requests_in_progress = await asyncio.wait_for(
model_server-1  |     |   File "/usr/lib/python3.10/asyncio/tasks.py", line 445, in wait_for
model_server-1  |     |     return fut.result()
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/vllm/engine/async_llm_engine.py", line 506, in engine_step
model_server-1  |     |     request_outputs = await self.engine.step_async()
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/vllm/engine/async_llm_engine.py", line 235, in step_async
model_server-1  |     |     output = await self.model_executor.execute_model_async(
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/vllm/executor/distributed_gpu_executor.py", line 166, in execute_model_async
model_server-1  |     |     return await self._driver_execute_model_async(execute_model_req)
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/vllm/executor/multiproc_gpu_executor.py", line 149, in _driver_execute_model_async
model_server-1  |     |     return await self.driver_exec_model(execute_model_req)
model_server-1  |     |   File "/usr/lib/python3.10/concurrent/futures/thread.py", line 58, in run
model_server-1  |     |     result = self.fn(*self.args, **self.kwargs)
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 115, in decorate_context
model_server-1  |     |     return func(*args, **kwargs)
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/vllm/worker/worker.py", line 280, in execute_model
model_server-1  |     |     output = self.model_runner.execute_model(seq_group_metadata_list,
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 115, in decorate_context
model_server-1  |     |     return func(*args, **kwargs)
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/vllm/worker/model_runner.py", line 765, in execute_model
model_server-1  |     |     output = self.model.sample(
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/models/llama.py", line 386, in sample
model_server-1  |     |     next_tokens = self.sampler(logits, sampling_metadata)
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
model_server-1  |     |     return self._call_impl(*args, **kwargs)
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1541, in _call_impl
model_server-1  |     |     return forward_call(*args, **kwargs)
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/layers/sampler.py", line 96, in forward
model_server-1  |     |     sample_results, maybe_sampled_tokens_tensor = _sample(
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/layers/sampler.py", line 655, in _sample
model_server-1  |     |     return _sample_with_torch(
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/layers/sampler.py", line 544, in _sample_with_torch
model_server-1  |     |     sample_results = _random_sample(seq_groups,
model_server-1  |     |   File "/usr/local/lib/python3.10/dist-packages/vllm/model_executor/layers/sampler.py", line 324, in _random_sample
model_server-1  |     |     random_samples = random_samples.cpu()
model_server-1  |     | RuntimeError: CUDA error: device-side assert triggered
model_server-1  |     | CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
model_server-1  |     | For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
model_server-1  |     | Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
model_server-1  |     |
model_server-1  |     +------------------------------------
model_server-1  | [rank0]:[E ProcessGroupNCCL.cpp:1414] [PG 2 Rank 0] Process group watchdog thread terminated with exception: CUDA error: device-side assert triggered
model_server-1  | CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
model_server-1  | For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
model_server-1  | Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.

model_server-1  | Exception raised from c10_cuda_check_implementation at ../c10/cuda/CUDAException.cpp:43 (most recent call first):
model_server-1  | frame #0: c10::Error::Error(c10::SourceLocation, std::string) + 0x57 (0x7f34c72cf897 in /usr/local/lib/python3.10/dist-packages/torch/lib/libc10.so)
model_server-1  | frame #1: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::string const&) + 0x64 (0x7f34c727fb25 in /usr/local/lib/python3.10/dist-packages/torch/lib/libc10.so)
model_server-1  | frame #2: c10::cuda::c10_cuda_check_implementation(int, char const*, char const*, int, bool) + 0x118 (0x7f34c73a7718 in /usr/local/lib/python3.10/dist-packages/torch/lib/libc10_cuda.so)
model_server-1  | frame #3: c10d::ProcessGroupNCCL::WorkNCCL::finishedGPUExecutionInternal() const + 0x56 (0x7f347b276e36 in /usr/local/lib/python3.10/dist-packages/torch/lib/libtorch_cuda.so)
model_server-1  | frame #4: c10d::ProcessGroupNCCL::WorkNCCL::isCompleted() + 0x58 (0x7f347b27af38 in /usr/local/lib/python3.10/dist-packages/torch/lib/libtorch_cuda.so)
model_server-1  | frame #5: c10d::ProcessGroupNCCL::watchdogHandler() + 0x77c (0x7f347b2805ac in /usr/local/lib/python3.10/dist-packages/torch/lib/libtorch_cuda.so)
model_server-1  | frame #6: c10d::ProcessGroupNCCL::ncclCommWatchdog() + 0x10c (0x7f347b28131c in /usr/local/lib/python3.10/dist-packages/torch/lib/libtorch_cuda.so)
model_server-1  | frame #7: <unknown function> + 0xdc253 (0x7f34c7b8b253 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6)
model_server-1  | frame #8: <unknown function> + 0x94ac3 (0x7f34c9af4ac3 in /usr/lib/x86_64-linux-gnu/libc.so.6)
model_server-1  | frame #9: clone + 0x44 (0x7f34c9b85a04 in /usr/lib/x86_64-linux-gnu/libc.so.6)
model_server-1  |
model_server-1  | [2024-07-25 18:12:35,893 E 1 7228] logging.cc:108: Unhandled exception: N3c1016DistBackendErrorE. what(): [PG 2 Rank 0] Process group watchdog thread terminated with exception: CUDA error: device-side assert triggered
model_server-1  | CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
model_server-1  | For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
model_server-1  | Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
model_server-1  |
model_server-1  | Exception raised from c10_cuda_check_implementation at ../c10/cuda/CUDAException.cpp:43 (most recent call first):
model_server-1  | frame #0: c10::Error::Error(c10::SourceLocation, std::string) + 0x57 (0x7f34c72cf897 in /usr/local/lib/python3.10/dist-packages/torch/lib/libc10.so)
model_server-1  | frame #1: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::string const&) + 0x64 (0x7f34c727fb25 in /usr/local/lib/python3.10/dist-packages/torch/lib/libc10.so)
model_server-1  | frame #2: c10::cuda::c10_cuda_check_implementation(int, char const*, char const*, int, bool) + 0x118 (0x7f34c73a7718 in /usr/local/lib/python3.10/dist-packages/torch/lib/libc10_cuda.so)
model_server-1  | frame #3: c10d::ProcessGroupNCCL::WorkNCCL::finishedGPUExecutionInternal() const + 0x56 (0x7f347b276e36 in /usr/local/lib/python3.10/dist-packages/torch/lib/libtorch_cuda.so)
model_server-1  | frame #4: c10d::ProcessGroupNCCL::WorkNCCL::isCompleted() + 0x58 (0x7f347b27af38 in /usr/local/lib/python3.10/dist-packages/torch/lib/libtorch_cuda.so)
model_server-1  | frame #5: c10d::ProcessGroupNCCL::watchdogHandler() + 0x77c (0x7f347b2805ac in /usr/local/lib/python3.10/dist-packages/torch/lib/libtorch_cuda.so)
model_server-1  | frame #6: c10d::ProcessGroupNCCL::ncclCommWatchdog() + 0x10c (0x7f347b28131c in /usr/local/lib/python3.10/dist-packages/torch/lib/libtorch_cuda.so)
model_server-1  | frame #7: <unknown function> + 0xdc253 (0x7f34c7b8b253 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6)
model_server-1  | frame #8: <unknown function> + 0x94ac3 (0x7f34c9af4ac3 in /usr/lib/x86_64-linux-gnu/libc.so.6)
model_server-1  | frame #9: clone + 0x44 (0x7f34c9b85a04 in /usr/lib/x86_64-linux-gnu/libc.so.6)
model_server-1  |
model_server-1  | Exception raised from ncclCommWatchdog at ../torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:1418 (most recent call first):
model_server-1  | frame #0: c10::Error::Error(c10::SourceLocation, std::string) + 0x57 (0x7f34c72cf897 in /usr/local/lib/python3.10/dist-packages/torch/lib/libc10.so)
model_server-1  | frame #1: <unknown function> + 0xe32e33 (0x7f347af03e33 in /usr/local/lib/python3.10/dist-packages/torch/lib/libtorch_cuda.so)
model_server-1  | frame #2: <unknown function> + 0xdc253 (0x7f34c7b8b253 in /usr/lib/x86_64-linux-gnu/libstdc++.so.6)
model_server-1  | frame #3: <unknown function> + 0x94ac3 (0x7f34c9af4ac3 in /usr/lib/x86_64-linux-gnu/libc.so.6)
model_server-1  | frame #4: clone + 0x44 (0x7f34c9b85a04 in /usr/lib/x86_64-linux-gnu/libc.so.6)
model_server-1  |
model_server-1  | [2024-07-25 18:12:35,899 E 1 7228] logging.cc:115: Stack trace:
model_server-1  |  /usr/local/lib/python3.10/dist-packages/ray/_raylet.so(+0x1033f2a) [0x7f3286731f2a] ray::operator<<()
model_server-1  | /usr/local/lib/python3.10/dist-packages/ray/_raylet.so(+0x1036f72) [0x7f3286734f72] ray::TerminateHandler()
model_server-1  | /usr/lib/x86_64-linux-gnu/libstdc++.so.6(+0xae20c) [0x7f34c7b5d20c]
model_server-1  | /usr/lib/x86_64-linux-gnu/libstdc++.so.6(+0xae277) [0x7f34c7b5d277]
model_server-1  | /usr/lib/x86_64-linux-gnu/libstdc++.so.6(+0xae1fe) [0x7f34c7b5d1fe]
model_server-1  | /usr/local/lib/python3.10/dist-packages/torch/lib/libtorch_cuda.so(+0xe32ee4) [0x7f347af03ee4] c10d::ProcessGroupNCCL::ncclCommWatchdog()
model_server-1  | /usr/lib/x86_64-linux-gnu/libstdc++.so.6(+0xdc253) [0x7f34c7b8b253]
model_server-1  | /usr/lib/x86_64-linux-gnu/libc.so.6(+0x94ac3) [0x7f34c9af4ac3]
model_server-1  | /usr/lib/x86_64-linux-gnu/libc.so.6(clone+0x44) [0x7f34c9b85a04] __clone
model_server-1  |
model_server-1  | *** SIGABRT received at time=1721931155 on cpu 107 ***
model_server-1  | PC: @     0x7f34c9af69fc  (unknown)  pthread_kill
model_server-1  |     @     0x7f34c9aa2520  (unknown)  (unknown)
model_server-1  | [2024-07-25 18:12:35,900 E 1 7228] logging.cc:343: *** SIGABRT received at time=1721931155 on cpu 107 ***
model_server-1  | [2024-07-25 18:12:35,900 E 1 7228] logging.cc:343: PC: @     0x7f34c9af69fc  (unknown)  pthread_kill
model_server-1  | [2024-07-25 18:12:35,900 E 1 7228] logging.cc:343:     @     0x7f34c9aa2520  (unknown)  (unknown)
model_server-1  | Fatal Python error: Aborted
model_server-1  |
model_server-1  |
model_server-1  | Extension modules: ujson, numpy.core._multiarray_umath, numpy.core._multiarray_tests, numpy.linalg._umath_linalg, numpy.fft._pocketfft_internal, numpy.random._common, numpy.random.bit_generator, numpy.random._bounded_integers, numpy.random._mt19937, numpy.random.mtrand, numpy.random._philox, numpy.random._pcg64, numpy.random._
sfc64, numpy.random._generator, torch._C, torch._C._fft, torch._C._linalg, torch._C._nested, torch._C._nn, torch._C._sparse, torch._C._special, charset_normalizer.md, simplejson._speedups, requests.packages.charset_normalizer.md, requests.packages.chardet.md, yaml._yaml, psutil._psutil_linux, psutil._psutil_posix, msgpack._cmsgpack, google._upb
._message, setproctitle, uvloop.loop, ray._raylet, sentencepiece._sentencepiece, PIL._imaging, httptools.parser.parser, httptools.parser.url_parser, websockets.speedups (total: 38)
model_server-1  | [failure_signal_handler.cc : 332] RAW: Signal 11 raised at PC=0x7f34c9a88898 while already in AbslFailureSignalHandler()
model_server-1  | *** SIGSEGV received at time=1721931155 on cpu 107 ***
model_server-1  | PC: @     0x7f34c9a88898  (unknown)  abort
model_server-1  |     @     0x7f34c9aa2520  633865568  (unknown)
model_server-1  |     @     0x7f0ba3ffd640  (unknown)  (unknown)
model_server-1  | [2024-07-25 18:12:35,901 E 1 7228] logging.cc:343: *** SIGSEGV received at time=1721931155 on cpu 107 ***
model_server-1  | [2024-07-25 18:12:35,901 E 1 7228] logging.cc:343: PC: @     0x7f34c9a88898  (unknown)  abort
model_server-1  | [2024-07-25 18:12:35,902 E 1 7228] logging.cc:343:     @     0x7f34c9aa2520  633865568  (unknown)
model_server-1  | [2024-07-25 18:12:35,903 E 1 7228] logging.cc:343:     @     0x7f0ba3ffd640  (unknown)  (unknown)
model_server-1  | Fatal Python error: Segmentation fault