Update data preprocessing methods

2024-09-27 22:34:30 +08:00 · 2024-09-27 22:34:30 +08:00 · 2981d3ea86
commit 2981d3ea86
parent 75753ea765
1 changed files with 1 additions and 0 deletions
--- a/0-eval_pretrain.py
+++ b/0-eval_pretrain.py
@ -107,6 +107,7 @@ if __name__ == "__main__":
            print('问题：', prompt)
            qa_index += 1

+        prompt = tokenizer.bos_token + prompt
        x = tokenizer(prompt).data['input_ids']
        x = (torch.tensor(x, dtype=torch.long, device=device)[None, ...])