diff --git a/model/tokenizer.model b/model/tokenizer.model deleted file mode 100644 index 85c0803..0000000 Binary files a/model/tokenizer.model and /dev/null differ diff --git a/model/tokenizer_config.json b/model/tokenizer_config.json deleted file mode 100644 index eb538f9..0000000 --- a/model/tokenizer_config.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "add_bos_token": false, - "add_eos_token": false, - "add_prefix_space": true, - "added_tokens_decoder": { - "0": { - "content": "", - "lstrip": false, - "normalized": false, - "rstrip": false, - "single_word": false, - "special": true - }, - "1": { - "content": "", - "lstrip": false, - "normalized": false, - "rstrip": false, - "single_word": false, - "special": true - }, - "2": { - "content": "", - "lstrip": false, - "normalized": false, - "rstrip": false, - "single_word": false, - "special": true - } - }, - "additional_special_tokens": [], - "bos_token": "", - "clean_up_tokenization_spaces": false, - "eos_token": "", - "legacy": true, - "model_max_length": 1000000000000000019884624838656, - "pad_token": null, - "sp_model_kwargs": {}, - "spaces_between_special_tokens": false, - "tokenizer_class": "LlamaTokenizer", - "unk_token": "", - "use_default_system_prompt": false, - "chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ 'user\\n' + content + '\\nassistant\\n' }}{% elif message['role'] == 'assistant' %}{{ content + '' + '\\n' }}{% endif %}{% endfor %}" -}