From c94a246c715de21dc44a3f7ea45341fa4b6a2f29 Mon Sep 17 00:00:00 2001
From: ViperEkura <3081035982@qq.com>
Date: Sat, 4 Apr 2026 17:03:22 +0800
Subject: [PATCH] =?UTF-8?q?chore:=20=E9=87=8D=E5=91=BD=E5=90=8D=E7=9B=AE?=
 =?UTF-8?q?=E5=BD=95?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 astrai/__init__.py                              | 2 +-
 astrai/config/param_config.py                   | 2 +-
 astrai/inference/generator.py                   | 2 +-
 astrai/{tokenizer => tokenize}/__init__.py      | 4 ++--
 astrai/{tokenizer => tokenize}/chat_template.py | 0
 astrai/{tokenizer => tokenize}/tokenizer.py     | 0
 tests/conftest.py                               | 2 +-
 7 files changed, 6 insertions(+), 6 deletions(-)
 rename astrai/{tokenizer => tokenize}/__init__.py (77%)
 rename astrai/{tokenizer => tokenize}/chat_template.py (100%)
 rename astrai/{tokenizer => tokenize}/tokenizer.py (100%)

diff --git a/astrai/__init__.py b/astrai/__init__.py
index 664f129..3f5bd84 100644
--- a/astrai/__init__.py
+++ b/astrai/__init__.py
@@ -7,7 +7,7 @@ from astrai.config import (
 )
 from astrai.factory import BaseFactory
 from astrai.dataset import DatasetFactory
-from astrai.tokenizer import BpeTokenizer
+from astrai.tokenize import BpeTokenizer
 from astrai.inference.generator import (
     BatchGenerator,
     EmbeddingEncoder,
diff --git a/astrai/config/param_config.py b/astrai/config/param_config.py
index 7afc9bb..9f15acf 100644
--- a/astrai/config/param_config.py
+++ b/astrai/config/param_config.py
@@ -7,7 +7,7 @@ import safetensors.torch as st
 import torch.nn as nn
 
 from astrai.config.model_config import ModelConfig
-from astrai.tokenizer import BpeTokenizer
+from astrai.tokenize import BpeTokenizer
 from astrai.model.transformer import Transformer
 
 
diff --git a/astrai/inference/generator.py b/astrai/inference/generator.py
index 68c2e2e..46331dd 100644
--- a/astrai/inference/generator.py
+++ b/astrai/inference/generator.py
@@ -7,7 +7,7 @@ from torch import Tensor
 from astrai.config.param_config import ModelParameter
 from astrai.factory import BaseFactory
 from astrai.inference.core import EmbeddingEncoderCore, GeneratorCore, KVCacheManager
-from astrai.tokenizer.chat_template import HistoryType, build_prompt
+from astrai.tokenize.chat_template import HistoryType, build_prompt
 
 
 def pad_sequence(ids_list: List[List[int]], pad_id: int) -> Tuple[List[List[int]], int]:
diff --git a/astrai/tokenizer/__init__.py b/astrai/tokenize/__init__.py
similarity index 77%
rename from astrai/tokenizer/__init__.py
rename to astrai/tokenize/__init__.py
index c93a5a0..5c7fe0c 100644
--- a/astrai/tokenizer/__init__.py
+++ b/astrai/tokenize/__init__.py
@@ -1,10 +1,10 @@
-from astrai.tokenizer.tokenizer import (
+from astrai.tokenize.tokenizer import (
     BaseTokenizer,
     BpeTokenizer,
     BaseTrainer,
     BpeTrainer,
 )
-from astrai.tokenizer.chat_template import (
+from astrai.tokenize.chat_template import (
     HistoryType,
     MessageType,
     build_prompt,
diff --git a/astrai/tokenizer/chat_template.py b/astrai/tokenize/chat_template.py
similarity index 100%
rename from astrai/tokenizer/chat_template.py
rename to astrai/tokenize/chat_template.py
diff --git a/astrai/tokenizer/tokenizer.py b/astrai/tokenize/tokenizer.py
similarity index 100%
rename from astrai/tokenizer/tokenizer.py
rename to astrai/tokenize/tokenizer.py
diff --git a/tests/conftest.py b/tests/conftest.py
index cd1f09c..0b6073e 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -11,7 +11,7 @@ from tokenizers import pre_tokenizers
 from torch.utils.data import Dataset
 
 from astrai.config.model_config import ModelConfig
-from astrai.tokenizer import BpeTokenizer, BpeTrainer
+from astrai.tokenize import BpeTokenizer, BpeTrainer
 from astrai.model.transformer import Transformer