yumoqing
e3f61f446d
- balance.py: module-level redis.asyncio singleton (works in all processes without env.redis injection), centralized tpac/self-org skip policy, userid param, extend_reserve, invalidate_balance_cache, finalize updates max_cost on cold start (no reserve) - llmclient.py: unified reserve injection in _inference_generator (covers all 22 dspy entries); fix luid mismatch (reserve key == llmusage id == finalize key); stream exception refund - syncinference.py / asyncinference.py: luid reuse + failure refunds (submit failure, outer exception, query_task_status FAILED) - product_interface.py: reserve + refund on product stream/non-stream - accounting.py: finalize after llm_accounting (was missing for direct llmage accounting path -> reserve leaked until TTL) - init.py: lambda signatures synced - v1/chat/completions/index.dspy: pass userid to reserve
63 lines
1.9 KiB
Plaintext
63 lines
1.9 KiB
Plaintext
async def gen():
|
|
env = request._run_ns.copy()
|
|
f = partial(inference_generator, request, params_kw=params_kw)
|
|
try:
|
|
if params_kw.stream:
|
|
async for l in f():
|
|
yield f'data: {l}\n'
|
|
yield 'data: [DONE]\n\n'
|
|
else:
|
|
async for l in f():
|
|
yield l
|
|
except Exception as e:
|
|
luid = params_kw.get('_luid')
|
|
if luid:
|
|
try:
|
|
await env.refund_balance(luid)
|
|
except:
|
|
pass
|
|
import json as _json
|
|
err = _json.dumps({"error": str(e), "status": "FAILED"}, ensure_ascii=False)
|
|
yield f'data: {err}\n' if params_kw.stream else err
|
|
|
|
debug_params('params_kw', params_kw)
|
|
catelogid = params_kw.catelogid or 't2t'
|
|
if params_kw.off_peak:
|
|
off_peak = params_kw.off_peak
|
|
if off_peak in [True, "Y" "y", 1, "1"]:
|
|
off_peak = True
|
|
else:
|
|
off_peak = False
|
|
params_kw.off_peak = off_peak
|
|
userid = await get_user()
|
|
userorgid = await get_userorgid()
|
|
if userid is None:
|
|
debug(f'need login')
|
|
return openai_403()
|
|
|
|
if not params_kw.prompt and not params_kw.messages:
|
|
debug(f'missing prompt and messages, model={params_kw.model}')
|
|
d = return_error('Missing need data(prompt or messages)')
|
|
return json_response(d, status=400)
|
|
env = request._run_ns
|
|
llmid = await env.get_llmid_cached(env, params_kw.model or 'qwen3-max', catelogid)
|
|
if not llmid:
|
|
debug(f'{params_kw.model=} not found for catelogid={catelogid}')
|
|
return openai_400()
|
|
params_kw.llmid = llmid
|
|
params_kw.llmcatelogid = catelogid
|
|
|
|
debug(f'{params_kw.llmid=}')
|
|
luid = getID()
|
|
reserved = await env.reserve_balance(params_kw.llmid, userorgid, luid, userid=userid)
|
|
if not reserved.get('ok'):
|
|
debug(f'{userid=} balance not enough: {reserved}')
|
|
return openai_429()
|
|
params_kw._luid = luid
|
|
params_kw._reserved = reserved
|
|
f = await checkCustomerBalance(params_kw.llmid, userid, userorgid)
|
|
if not f and reserved.get('no_redis'):
|
|
debug(f'{userid=} balance not enough (DB fallback)')
|
|
return openai_429()
|
|
return await env.stream_response(request, gen, content_type='application/json')
|