diff --git a/llmage/init.py b/llmage/init.py index 6c60405..26a429b 100644 --- a/llmage/init.py +++ b/llmage/init.py @@ -23,6 +23,9 @@ from .utils import ( invalidate_uapi_cache, get_llmusage_by_id, read_ioinfo_content, + _warm_llmid_cache, + get_llmid_cached, + invalidate_llmid_cache, ) from .llmclient import ( @@ -122,10 +125,11 @@ order by lc.name, a.name""" def _on_hot_reload(data=None): - """Event handler for hot_reload — wraps invalidate_uapi_cache to accept dispatcher's data arg.""" + """Event handler for hot_reload — invalidate caches.""" from appPublic.log import debug - debug(f'[llmage] on_hot_reload called, invalidating uapi cache (data={data})') + debug(f'[llmage] on_hot_reload called, invalidating caches (data={data})') invalidate_uapi_cache() + invalidate_llmid_cache() def load_llmage(): @@ -148,6 +152,10 @@ def load_llmage(): env.get_tpac_balance = get_tpac_balance env.inference_generator = inference_generator env.get_llms_by_catelog = get_llms_by_catelog + env.get_llmid_cached = get_llmid_cached + env.invalidate_llmid_cache = invalidate_llmid_cache + # 启动时预热 llmid 缓存 + asyncio.ensure_future(_warm_llmid_cache(env)) env.get_llmcatelogs = get_llmcatelogs env.checkCustomerBalance = checkCustomerBalance env.get_llmproviders = get_llmproviders diff --git a/llmage/utils.py b/llmage/utils.py index b88b73a..709399e 100644 --- a/llmage/utils.py +++ b/llmage/utils.py @@ -63,6 +63,55 @@ def invalidate_uapi_cache(upappid=None, apiname=None): _uapiio_cache.clear() +# ============================================================= +# Process-level cache for llmid lookup (model+catelogid -> llmid) +# ============================================================= +_llmid_cache = {} # key: "model_name:catelogid" -> llmid + + +async def _warm_llmid_cache(env): + """启动时全量加载 model+catelogid -> llmid 映射""" + global _llmid_cache + try: + async with get_sor_context(env, 'llmage') as sor: + sql = """SELECT a.name, b.id as catelogid, m.llmid + FROM llm_api_map m + JOIN llm a ON a.id = m.llmid AND a.status = 'published' + JOIN llmcatelog b ON b.id = m.llmcatelogid""" + recs = await sor.sqlExe(sql, {}) + for r in recs: + key = f"{r.name}:{r.catelogid}" + _llmid_cache[key] = r.llmid + debug(f'[llmage] llmid cache warmed: {len(_llmid_cache)} entries') + except Exception as e: + exception(f'[llmage] llmid cache warm failed: {e}') + _llmid_cache = {} + + +async def get_llmid_cached(env, model_name, catelogid): + """从缓存获取 llmid,未命中则查 DB 并缓存""" + global _llmid_cache + key = f"{model_name}:{catelogid}" + if key in _llmid_cache: + return _llmid_cache[key] + # 缓存未命中,查 DB(兼容模型在缓存预热后新增的场景) + async with get_sor_context(env, 'llmage') as sor: + sql = """SELECT m.llmid + FROM llm_api_map m + JOIN llm a ON a.id = m.llmid AND a.name = ${model}$ AND a.status = 'published' + JOIN llmcatelog b ON b.id = m.llmcatelogid AND (b.id = ${catelogid}$ OR b.name = ${catelogid}$)""" + recs = await sor.sqlExe(sql, {'model': model_name, 'catelogid': catelogid}) + llmid = recs[0].llmid if recs else None + if llmid: + _llmid_cache[key] = llmid + return llmid + + +def invalidate_llmid_cache(): + global _llmid_cache + _llmid_cache.clear() + + async def update_llmusage(ns): env = ServerEnv() async with get_sor_context(env, 'llmage') as sor: diff --git a/wwwroot/v1/chat/completions/index.dspy b/wwwroot/v1/chat/completions/index.dspy index 1f743da..5105c21 100644 --- a/wwwroot/v1/chat/completions/index.dspy +++ b/wwwroot/v1/chat/completions/index.dspy @@ -29,21 +29,11 @@ if not params_kw.prompt and not params_kw.messages: d = return_error('Missing need data(prompt or messages)') return json_response(d, status=400) env = request._run_ns -async with get_sor_context(env, 'llmage') as sor: - sql = """select distinct a.* from llm a -join llm_api_map m on a.id = m.llmid -join llmcatelog b on m.llmcatelogid = b.id -where (b.id = ${catelogid}$ OR b.name = ${catelogid}$) - and a.name=${model}$ - and a.status = 'published'""" - recs = await sor.sqlExe(sql, { - 'catelogid': catelogid, - 'model': params_kw.model or 'qwen3-max' - }) - if len(recs) == 0: - debug(f'{params_kw.model=} not found') - return openai_400() - params_kw.llmid = recs[0].id +llmid = await env.get_llmid_cached(env, params_kw.model or 'qwen3-max', catelogid) +if not llmid: + debug(f'{params_kw.model=} not found for catelogid={catelogid}') + return openai_400() +params_kw.llmid = llmid params_kw.llmcatelogid = catelogid debug(f'{params_kw.llmid=}')