当前环境

# reproduce codes:

from vllm import LLM, SamplingParams
import datasets
raw_datasets = datasets.load_dataset( "truthful_qa", 'generation')

questions = [i['question'] for i in raw_datasets['validation']]

llm = LLM(model="mistralai/Mistral-7B-Instruct-v0.2" ,  dtype='bfloat16', trust_remote_code=True)

sampling_params = SamplingParams(
    temperature=0,
    max_tokens=256
)

for batch_size in [32, 64, 256]:

    outputs = llm.generate(
            questions[:batch_size],
            sampling_params
        )

    for o in outputs[:5]:
        print(o.outputs[0].text)
        print()

    print('------------------------------')

🐛 描述bug

当dtype='bfloat16'时，将batch_size更改为不同的数字，会导致上述outputs中出现明显的差异。这个问题在float16和32位时不存在。