fix(tokenizer): 修复stop_ids属性返回错误的token ID列表

2025-10-31 19:19:38 +08:00 · 2025-10-31 19:19:38 +08:00 · e86328b753
parent 5d3799b715
commit e86328b753
1 changed files with 2 additions and 1 deletions
--- a/khaosz/data/tokenizer.py
+++ b/khaosz/data/tokenizer.py
@ -93,7 +93,8 @@ class BpeTokenizer:
    
    @property
    def stop_ids(self) -> List[int]:
-        stop_ids = self._control_tokens + self._special_tokens
+        stop_token = self._control_tokens + self._special_tokens
+        stop_ids = [self._tokenizer.token_to_id(token) for token in stop_token]
        return stop_ids
    
    @property