chore: 重命名目录

astrai/__init__.py

@@ -7,7 +7,7 @@ from astrai.config import (
 )
 from astrai.factory import BaseFactory
 from astrai.dataset import DatasetFactory
-from astrai.tokenizer import BpeTokenizer
+from astrai.tokenize import BpeTokenizer
 from astrai.inference.generator import (
     BatchGenerator,
     EmbeddingEncoder,

astrai/config/param_config.py

@@ -7,7 +7,7 @@ import safetensors.torch as st
 import torch.nn as nn
 
 from astrai.config.model_config import ModelConfig
-from astrai.tokenizer import BpeTokenizer
+from astrai.tokenize import BpeTokenizer
 from astrai.model.transformer import Transformer
 
 

astrai/inference/generator.py

@@ -7,7 +7,7 @@ from torch import Tensor
 from astrai.config.param_config import ModelParameter
 from astrai.factory import BaseFactory
 from astrai.inference.core import EmbeddingEncoderCore, GeneratorCore, KVCacheManager
-from astrai.tokenizer.chat_template import HistoryType, build_prompt
+from astrai.tokenize.chat_template import HistoryType, build_prompt
 
 
 def pad_sequence(ids_list: List[List[int]], pad_id: int) -> Tuple[List[List[int]], int]:

astrai/tokenize/__init__.py

@@ -1,10 +1,10 @@
-from astrai.tokenizer.tokenizer import (
+from astrai.tokenize.tokenizer import (
     BaseTokenizer,
     BpeTokenizer,
     BaseTrainer,
     BpeTrainer,
 )
-from astrai.tokenizer.chat_template import (
+from astrai.tokenize.chat_template import (
     HistoryType,
     MessageType,
     build_prompt,

tests/conftest.py

@@ -11,7 +11,7 @@ from tokenizers import pre_tokenizers
 from torch.utils.data import Dataset
 
 from astrai.config.model_config import ModelConfig
-from astrai.tokenizer import BpeTokenizer, BpeTrainer
+from astrai.tokenize import BpeTokenizer, BpeTrainer
 from astrai.model.transformer import Transformer