From 3f67e53088d49f366df08ab0789a5ab2ec79aa2e Mon Sep 17 00:00:00 2001 From: ViperEkura <3081035982@qq.com> Date: Mon, 6 Apr 2026 09:22:46 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8Dtokenizer=20=E5=8F=82?= =?UTF-8?q?=E6=95=B0=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- astrai/tokenize/tokenizer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/astrai/tokenize/tokenizer.py b/astrai/tokenize/tokenizer.py index f717025..6c449af 100644 --- a/astrai/tokenize/tokenizer.py +++ b/astrai/tokenize/tokenizer.py @@ -57,17 +57,16 @@ class AutoTokenizer: instance = cls(path) return instance - def save_pretrained(self, tokenizer, save_path: str): + def save_pretrained(self, save_path: str): """ Save tokenizer to pretrained directory. Args: - tokenizer: Tokenizer instance to save save_path: Path to save the tokenizer """ save_path = Path(save_path) save_path.mkdir(parents=True, exist_ok=True) - self._tokenizer.save(tokenizer, save_path) + self._tokenizer.save(str(save_path / "tokenizer.json")) @classmethod def register_tokenizer(cls, name: str, tokenizer_class: type):