From 2981d3ea863040308b878d6109c6e8c036fb8567 Mon Sep 17 00:00:00 2001 From: gongjy <2474590974@qq.com> Date: Fri, 27 Sep 2024 22:34:30 +0800 Subject: [PATCH] Update data preprocessing methods --- 0-eval_pretrain.py | 1 + 1 file changed, 1 insertion(+) diff --git a/0-eval_pretrain.py b/0-eval_pretrain.py index 570d42d..d54e505 100644 --- a/0-eval_pretrain.py +++ b/0-eval_pretrain.py @@ -107,6 +107,7 @@ if __name__ == "__main__": print('问题:', prompt) qa_index += 1 + prompt = tokenizer.bos_token + prompt x = tokenizer(prompt).data['input_ids'] x = (torch.tensor(x, dtype=torch.long, device=device)[None, ...])