From e34d4e9371cb682c92d370a5bf08d7150484615a Mon Sep 17 00:00:00 2001
From: gongjy <2474590974@qq.com>
Date: Wed, 19 Feb 2025 23:24:29 +0800
Subject: [PATCH] update tokenizer load

---
 eval_model.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/eval_model.py b/eval_model.py
index a3d0eaf..fc71257 100644
--- a/eval_model.py
+++ b/eval_model.py
@@ -33,10 +33,9 @@ def init_model(args):
             apply_lora(model)
             load_lora(model, f'./{args.out_dir}/lora/{args.lora_name}_{args.dim}.pth')
     else:
-        model = AutoModelForCausalLM.from_pretrained(
-            './MiniMind2',
-            trust_remote_code=True
-        )
+        transformers_model_path = './MiniMind2'
+        tokenizer = AutoTokenizer.from_pretrained(transformers_model_path)
+        model = AutoModelForCausalLM.from_pretrained(transformers_model_path, trust_remote_code=True)
     print(f'MiniMind模型参数量: {sum(p.numel() for p in model.parameters() if p.requires_grad) / 1e6:.2f}M(illion)')
     return model.eval().to(args.device), tokenizer