diff --git a/astrai/__init__.py b/astrai/__init__.py index 664f129..3f5bd84 100644 --- a/astrai/__init__.py +++ b/astrai/__init__.py @@ -7,7 +7,7 @@ from astrai.config import ( ) from astrai.factory import BaseFactory from astrai.dataset import DatasetFactory -from astrai.tokenizer import BpeTokenizer +from astrai.tokenize import BpeTokenizer from astrai.inference.generator import ( BatchGenerator, EmbeddingEncoder, diff --git a/astrai/config/param_config.py b/astrai/config/param_config.py index 7afc9bb..9f15acf 100644 --- a/astrai/config/param_config.py +++ b/astrai/config/param_config.py @@ -7,7 +7,7 @@ import safetensors.torch as st import torch.nn as nn from astrai.config.model_config import ModelConfig -from astrai.tokenizer import BpeTokenizer +from astrai.tokenize import BpeTokenizer from astrai.model.transformer import Transformer diff --git a/astrai/inference/generator.py b/astrai/inference/generator.py index 68c2e2e..46331dd 100644 --- a/astrai/inference/generator.py +++ b/astrai/inference/generator.py @@ -7,7 +7,7 @@ from torch import Tensor from astrai.config.param_config import ModelParameter from astrai.factory import BaseFactory from astrai.inference.core import EmbeddingEncoderCore, GeneratorCore, KVCacheManager -from astrai.tokenizer.chat_template import HistoryType, build_prompt +from astrai.tokenize.chat_template import HistoryType, build_prompt def pad_sequence(ids_list: List[List[int]], pad_id: int) -> Tuple[List[List[int]], int]: diff --git a/astrai/tokenizer/__init__.py b/astrai/tokenize/__init__.py similarity index 77% rename from astrai/tokenizer/__init__.py rename to astrai/tokenize/__init__.py index c93a5a0..5c7fe0c 100644 --- a/astrai/tokenizer/__init__.py +++ b/astrai/tokenize/__init__.py @@ -1,10 +1,10 @@ -from astrai.tokenizer.tokenizer import ( +from astrai.tokenize.tokenizer import ( BaseTokenizer, BpeTokenizer, BaseTrainer, BpeTrainer, ) -from astrai.tokenizer.chat_template import ( +from astrai.tokenize.chat_template import ( HistoryType, MessageType, build_prompt, diff --git a/astrai/tokenizer/chat_template.py b/astrai/tokenize/chat_template.py similarity index 100% rename from astrai/tokenizer/chat_template.py rename to astrai/tokenize/chat_template.py diff --git a/astrai/tokenizer/tokenizer.py b/astrai/tokenize/tokenizer.py similarity index 100% rename from astrai/tokenizer/tokenizer.py rename to astrai/tokenize/tokenizer.py diff --git a/tests/conftest.py b/tests/conftest.py index cd1f09c..0b6073e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,7 +11,7 @@ from tokenizers import pre_tokenizers from torch.utils.data import Dataset from astrai.config.model_config import ModelConfig -from astrai.tokenizer import BpeTokenizer, BpeTrainer +from astrai.tokenize import BpeTokenizer, BpeTrainer from astrai.model.transformer import Transformer