diff --git a/llmage/llmclient.py b/llmage/llmclient.py index 2b493ee..8dbea65 100644 --- a/llmage/llmclient.py +++ b/llmage/llmclient.py @@ -93,6 +93,18 @@ async def uapi_request(request, llm, callerid, callerorgid, params_kw=None): llmusage.accounting_status = 'created' await write_llmusage(llmusage) except Exception as e: + # Retry on duplicate key: append *1, *2, ... + retry = 0 + while 'Duplicate entry' in str(e) and retry < 3: + retry += 1 + llmusage.id = f'{luid}*{retry}' + try: + await write_llmusage(llmusage) + exception(f'write_llmusage retry {retry} succeeded with id={llmusage.id}') + e = None + break + except Exception as e2: + e = e2 # Refund balance reservation on failure try: from .balance import refund_balance diff --git a/wwwroot/v1/chat/completions/index.dspy b/wwwroot/v1/chat/completions/index.dspy index 6b726bf..87871bb 100644 --- a/wwwroot/v1/chat/completions/index.dspy +++ b/wwwroot/v1/chat/completions/index.dspy @@ -1,13 +1,24 @@ async def gen(): env = request._run_ns.copy() f = partial(inference_generator, request, params_kw=params_kw) - if params_kw.stream: - async for l in f(): - yield f'data: {l}\\n' - yield 'data: [DONE]\\n\\n' - else: - async for l in f(): - yield l + try: + if params_kw.stream: + async for l in f(): + yield f'data: {l}\\n' + yield 'data: [DONE]\\n\\n' + else: + async for l in f(): + yield l + except Exception as e: + luid = params_kw.get('_luid') + if luid: + try: + await env.refund_balance(luid) + except: + pass + import json as _json + err = _json.dumps({"error": str(e), "status": "FAILED"}, ensure_ascii=False) + yield f'data: {err}\\n' if params_kw.stream else err debug_params('params_kw', params_kw) catelogid = params_kw.catelogid or 't2t'