diff --git a/khaosz/data/tokenizer.py b/khaosz/data/tokenizer.py index 935027d..214c293 100644 --- a/khaosz/data/tokenizer.py +++ b/khaosz/data/tokenizer.py @@ -93,7 +93,8 @@ class BpeTokenizer: @property def stop_ids(self) -> List[int]: - stop_ids = self._control_tokens + self._special_tokens + stop_token = self._control_tokens + self._special_tokens + stop_ids = [self._tokenizer.token_to_id(token) for token in stop_token] return stop_ids @property