diff --git a/llmage/asyncinference.py b/llmage/asyncinference.py index f494414..34b06d0 100644 --- a/llmage/asyncinference.py +++ b/llmage/asyncinference.py @@ -112,7 +112,7 @@ async def async_uapi_request(request, llm, llmusage.accounting_status = 'created' b = json.dumps(d, ensure_ascii=False) yield b - await write_llmusage(llmusage) + # await write_llmusage(llmusage) # if llm.callbackurl: # return if d.status == 'FAILED': @@ -128,7 +128,29 @@ async def async_uapi_request(request, llm, s = ''.join(s.split('\n')) exception(s) yield f'{s}\n' + llmusage = DictObject() + llmusage.id = luid + llmusage.llmid = llm.id + llmusage.use_date = curDateString() + llmusage.use_time = timestampstr() + llmusage.userid = callerid + ioinfo = { + "input": params_kw, + 'output': [ed] + } + webpath = await write_llmio(llmusage.id, ioinfo) + llmusage.ioinfo = webpath + llmusage.taskid = d.taskid + llmusage.transno = params_kw.transno + llmusage.responsed_seconds = responsed_seconds + llmusage.finish_seconds = finish_seconds + llmusage.status = 'FAILED' + llmusage.userorgid = callerorgid + llmusage.tenantid = params_kw.get('tenantid', params_kw.get('tentantid')) + llmusage.ownerid = llm.ownerid return + finally: + await write_llmusage(llmusage) async def modify_llmusage(ns): env = ServerEnv() diff --git a/llmage/llmclient.py b/llmage/llmclient.py index 8dbea65..fe6a6c7 100644 --- a/llmage/llmclient.py +++ b/llmage/llmclient.py @@ -14,6 +14,7 @@ from ahserver.filestorage import FileStorage from .asyncinference import async_uapi_request from .syncinference import sync_uapi_request from .accounting import llm_accounting, llm_charging +from .balance import refund_balance from .utils import * async def uapi_request(request, llm, callerid, callerorgid, params_kw=None): @@ -91,28 +92,14 @@ async def uapi_request(request, llm, callerid, callerorgid, params_kw=None): llmusage.tenantid = params_kw.get('tenantid', params_kw.get('tentantid')) llmusage.ownerid = llm.ownerid llmusage.accounting_status = 'created' - await write_llmusage(llmusage) + # await write_llmusage(llmusage) except Exception as e: - # Retry on duplicate key: append *1, *2, ... - retry = 0 - while 'Duplicate entry' in str(e) and retry < 3: - retry += 1 - llmusage.id = f'{luid}*{retry}' - try: - await write_llmusage(llmusage) - exception(f'write_llmusage retry {retry} succeeded with id={llmusage.id}') - e = None - break - except Exception as e2: - e = e2 # Refund balance reservation on failure try: - from .balance import refund_balance - from ahserver.serverenv import ServerEnv - reserve_luid = params_kw.get('_luid') if params_kw else None - if reserve_luid: - await refund_balance(ServerEnv(), reserve_luid) + if luid: + await refund_balance(ServerEnv(), luid) except: + debug(f'refund_balance(ServerEnv(), {luid=}) errir') pass exception(f'{e=},{format_exc()}') estr = erase_apikey(e) @@ -121,15 +108,27 @@ async def uapi_request(request, llm, callerid, callerorgid, params_kw=None): s = ''.join(s.split('\\n')) outlines.append(ed) yield f'{s}\\n' - return + ## except happand at call llm server + llmusage = DictObject() + llmusage.id = luid + llmusage.llmid = llm.id + llmusage.use_date = curDateString() + llmusage.use_time = timestampstr() + llmusage.userid = callerid + ioinfo = { + "input": params_kw, + 'output': ed + } + webpath = await write_llmio(llmusage.id, ioinfo) + llmusage.ioinfo = webpath + llmusage.transno = params_kw.transno + llmusage.status = 'FAILED' + llmusage.userorgid = callerorgid + llmusage.tenantid = params_kw.get('tenantid', params_kw.get('tentantid')) + llmusage.ownerid = llm.ownerid finally: # GeneratorExit / client disconnect — flush partial usage - if llmusage and llmusage.get('id') == luid: - try: - llmusage.status = llmusage.status or 'UNKNOWN' - await write_llmusage(llmusage) - except: - pass + await write_llmusage(llmusage) async def inference_generator(request, *args, params_kw=None, **kw): env = request._run_ns.copy() diff --git a/llmage/syncinference.py b/llmage/syncinference.py index f41e6ef..d6c13c3 100644 --- a/llmage/syncinference.py +++ b/llmage/syncinference.py @@ -61,33 +61,13 @@ async def sync_uapi_request(request, llm, callerid, callerorgid, params_kw=None) llmusage.finish_seconds = finish_seconds llmusage.status = 'SUCCEEDED' llmusage.amount = 0.00 - """ 联机不记账 - if llm.ppid: - try: - charging = await llm_charging(llm.ppid, llmusage) - if charging: - llmusage.amount = charging.amount - llmusage.cost = charging.cost - else: - llmusage.amount = llmusage.cost = 0.0 - except Exception as e: - e = Exception(f'{llm.pid} charging error{e}') - exception(f'{e}') - else: - llmusage.amount = 0 - llmusage.cost = 0 - """ llmusage.userorgid = callerorgid llmusage.tenantid = params_kw.get('tenantid', params_kw.get('tentantid')) llmusage.ownerid = llm.ownerid llmusage.accounting_status = 'created' b = json.dumps(d, ensure_ascii=False) yield b - await write_llmusage(llmusage) - """联机不记账 - if llmusage.amount > 0.0001: - await llm_accounting(llmusage) - """ + # await write_llmusage(llmusage) except Exception as e: exception(f'{e=},{format_exc()}, {b=}') estr = erase_apikey(e) @@ -96,7 +76,29 @@ async def sync_uapi_request(request, llm, callerid, callerorgid, params_kw=None) s = ''.join(s.split('\n')) outlines.append(ed) yield f'{s}\n' - + llmusage = DictObject() + llmusage.id = luid + llmusage.llmid = llm.id + llmusage.use_date = curDateString() + llmusage.use_time = timestampstr() + llmusage.userid = callerid + llmusage.usages = None + ioinfo = { + "input": params_kw, + 'output': [ed] + } + webpath = await write_llmio(llmusage.id, ioinfo) + llmusage.ioinfo = webpath + llmusage.transno = params_kw.transno + llmusage.responsed_seconds = responsed_seconds + llmusage.finish_seconds = finish_seconds + llmusage.status = 'FAILED' + llmusage.amount = 0.00 + llmusage.userorgid = callerorgid + llmusage.tenantid = params_kw.get('tenantid', params_kw.get('tentantid')) + llmusage.ownerid = llm.ownerid + finally: + await write_llmusage(llmusage) async def sync_uapi_request_product(llm, api_userid, user_id, user_org_id, params_kw, luid): """Product interface version: no HTTP request dependency. Returns dict.""" diff --git a/llmage/utils.py b/llmage/utils.py index ecfc953..a1ba66c 100644 --- a/llmage/utils.py +++ b/llmage/utils.py @@ -556,6 +556,14 @@ async def get_llm(llmid, catelogid=None): async def write_llmusage(llmusage): env = ServerEnv() async with get_sor_context(env, 'llmage') as sor: + n = 0 + part0 = llmusage.id + while True: + recs = await sor.R('llmusage', {'id': llmusage.id}) + if len(recs) == 0: + break + n = n + 1 + llmusage.id = f'{part0}*{n}' await sor.C('llmusage', llmusage) async def llm_query_price(llmid, config_data):