ViperEkura
/
AstrAI
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: 修复tokenizer 参数问题

This commit is contained in:
ViperEkura 2026-04-06 09:22:46 +08:00
parent bf7adb35b3
commit 3f67e53088
1 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
astrai/tokenize/tokenizer.py
View File

@ -57,17 +57,16 @@ class AutoTokenizer:
instance = cls(path) instance = cls(path)
return instance return instance
def save_pretrained(self, tokenizer, save_path: str): def save_pretrained(self, save_path: str):
""" """
Save tokenizer to pretrained directory. Save tokenizer to pretrained directory.
Args: Args:
tokenizer: Tokenizer instance to save
save_path: Path to save the tokenizer save_path: Path to save the tokenizer
""" """
save_path = Path(save_path) save_path = Path(save_path)
save_path.mkdir(parents=True, exist_ok=True) save_path.mkdir(parents=True, exist_ok=True)
self._tokenizer.save(tokenizer, save_path) self._tokenizer.save(str(save_path / "tokenizer.json"))
@classmethod @classmethod
def register_tokenizer(cls, name: str, tokenizer_class: type): def register_tokenizer(cls, name: str, tokenizer_class: type):