如何为本地模型构建高并发模块?

请求1 \
请求2  \
请求3   →  队列 → 批处理 → 一次 model.encode()
请求4  /
请求5 /

├── inference_engine/          高并发批量处理请求模块
│   ├── queue_manager.py       # 队列管理
│   ├── batch_scheduler.py     # 批处理调度
│   ├── worker.py              # worker执行器
│   ├── task.py                # 请求封装
│   └── config.py              # 配置（batch size等）

class InferenceTask:
    def __init__(self, data, future):
        self.data = data
        self.future = future "future": ...
}

import asyncio

class QueueManager:
    def __init__(self):
        self.queue = asyncio.Queue()

    async def put(self, task):
        await self.queue.put(task)

    async def get_batch(self, max_batch_size, timeout=0.01):
        .....
        return batch

import asyncio

class BatchScheduler:
    def __init__(self, queue_manager, process_fn, batch_size=32):
        self.queue_manager = queue_manager
        self.process_fn = process_fn
        self.batch_size = batch_size

    async def start(self):
        while True:
            batch = await self.queue_manager.get_batch(self.batch_size)

            if not batch:
                continue

            inputs = [task.data for task in batch]

            # 👉 核心：统一处理
            results = self.process_fn(inputs)

            for task, result in zip(batch, results):
                task.future.set_result(result)

import asyncio

class InferenceWorker:
    def __init__(self, scheduler):
        self.scheduler = scheduler

    def start(self):
        asyncio.create_task(self.scheduler.start())

BATCH_SIZE = 32
TIMEOUT = 0.01