26 lines
467 B
Python
26 lines
467 B
Python
"""Inference module for continuous batching."""
|
|
|
|
from astrai.inference.engine import (
|
|
GenerationRequest,
|
|
InferenceEngine,
|
|
)
|
|
from astrai.inference.scheduler import (
|
|
InferenceScheduler,
|
|
Task,
|
|
TaskStatus,
|
|
apply_sampling_strategies,
|
|
)
|
|
|
|
__all__ = [
|
|
# Engine
|
|
"InferenceEngine",
|
|
# Scheduler
|
|
"InferenceScheduler",
|
|
"Task",
|
|
"TaskStatus",
|
|
# Request
|
|
"GenerationRequest",
|
|
# Sampling
|
|
"apply_sampling_strategies",
|
|
]
|