From 3ac420984dc0368d24008f75fa03b9449e37e3ef Mon Sep 17 00:00:00 2001 From: yumoqing Date: Thu, 13 Nov 2025 13:16:33 +0800 Subject: [PATCH] bugfix --- llmage/llmclient.py | 114 ++++++++++++++++++++++++++++++++++++-------- models/llm.xlsx | Bin 17147 -> 17216 bytes 2 files changed, 94 insertions(+), 20 deletions(-) diff --git a/llmage/llmclient.py b/llmage/llmclient.py index 1fa142e..0bd1860 100644 --- a/llmage/llmclient.py +++ b/llmage/llmclient.py @@ -1,4 +1,5 @@ import json +import time import asyncio from random import randint from functools import partial @@ -7,7 +8,7 @@ from sqlor.dbpools import DBPools from appPublic.log import debug, exception from appPublic.uniqueID import getID from appPublic.dictObject import DictObject -from appPublic.timeUtils import curDateString +from appPublic.timeUtils import curDateString, timestampstr from appPublic.base64_to_file import base64_to_file, getFilenameFromBase64 from uapi.appapi import UAPI, sor_get_callerid, sor_get_uapi from ahserver.serverenv import get_serverenv @@ -98,16 +99,41 @@ where a.upappid=b.id i = randint(0, len(recs)-1) return recs[i].userid -async def uapi_request(request, llm, sor): +async def write_llmusage(llm, userid, usage, params_kw, outdata, sor): + d = { + "id": getID(), + "llmid": llm.id, + "use_date": curDateString(), + "use_time": timestampstr(), + "userid": userid, + "useages": usages, + "ioinfo": json.dumps({ + "input": params_kw, + "output": outdata + }) + } + await sor.C('llmusage', d) + +async def uapi_request(request, llm, sor, params_kw=None): env = request._run_ns.copy() + if not params_kw: + params_kw = env.params_kw caller_orgid = await env.get_userorgid() callerid = await env.get_user() uapi = UAPI(request, sor=sor) userid = await get_owner_userid(sor, llm) + outlines = [] txt = '' try: + t1 = time.time() + t2 = t1 + t3 = t1 + first = True async for l in uapi.stream_linify(llm.upappid, llm.apiname, userid, - params=env.params_kw): + params=params_kw): + if first: + first = False + t2 = time.time() if isinstance(l, bytes): l = l.decode('utf-8') if l[-1] == '\n': @@ -128,68 +154,106 @@ async def uapi_request(request, llm, sor): if d.get('content'): txt = txt + d['content'] yield_it = True + outlines.append(d) yield json.dumps(d) + '\n' + usage = outlines[-1].get('usage',{}) + t3 = time.time() + usage['response_time'] = t2 - t1 + usage['finish_time'] = t3 - t1 + if not usage.get('completions_tokens'): + usage['completions_tokens'] = len(txt) + if not usage.get('input_tokens'): + cnt = 0 + if params_kw.prompt: + cnt += len(params_kw.prompt) + if params_kw.negitive_prompt: + cnt += len(params_kw.negitive_promot) + usage['input_tokens'] = len + await write_llmusage(llm, callerid, usage, params_kw, outlines, sor) except Exception as e: exception(f'{e=},{format_exc()}') estr = erase_apikey(e) + outlines.append({"error": "ERROR:{estr}", "status": "FAILED" }) yield f'{{"error": "ERROR:{estr}", "status": "SUCCEEDED" }}\n' + await write_llmusage(llm, callerid, None, params_kw, outlines, sor) return debug(f'{txt=}') -async def sync_uapi_request(request, llm, sor): +async def sync_uapi_request(request, llm, sor, params_kw=None): env = request._run_ns.copy() + if not params_kw: + params_kw = env.params_kw caller_orgid = await env.get_userorgid() callerid = await env.get_user() uapi = UAPI(request, sor=sor) userid = await get_owner_userid(sor, llm) + outlines = [] b = None + d = None + t1 = t2 = t3 = time.time() try: - b = await uapi.call(llm.upappid, llm.apiname, userid, params=env.params_kw) + + b = await uapi.call(llm.upappid, llm.apiname, userid, params=params_kw) + if isinstance(b, bytes): + b = b.decode('utf-8') + d = json.loads(b) except Exception as e: exception(f'{e=},{format_exc()}') estr = erase_apikey(e) yield f'{{"error": "ERROR:{estr}", "status": "SUCCEEDED" }}\n' + outlines.append({"error": "ERROR:{estr}", "status": "FAILED" }) + await write_llmusage(llm, callerid, None, params_kw, outlines, sor) return - if isinstance(b, bytes): - b = b.decode('utf-8') + outlines.append(d) + t2 = t3 = time.time() + usage = d.get('usage', {}) + usage['response_time'] = t2 - t1 + usage['finish_time'] = t3 - t1 + await write_llmusage(llm, callerid, usage, params_kw, outlines, sor) debug(f'finished:{b}') yield b -async def async_uapi_request(request, llm, sor): +async def async_uapi_request(request, llm, sor, params_kw=None): env = request._run_ns.copy() + if not params_kw: + params_kw = env.params_kw caller_orgid = await env.get_userorgid() callerid = await env.get_user() uapi = UAPI(request, sor=sor) userid = await get_owner_userid(sor, llm) + outlines = [] b = None + t1 = t2 = t3 = time.time() try: - b = await uapi.call(llm.upappid, llm.apiname, userid, params=env.params_kw) + b = await uapi.call(llm.upappid, llm.apiname, userid, params=params_kw) except Exception as e: exception(f'{e=},{format_exc()}') estr = erase_apikey(e) yield f'{{"error": "ERROR:{estr}", "status": "SUCCEEDED" }}\n' + outlines.append({"error": "ERROR:{estr}", "status": "FAILED" }) + await write_llmusage(llm, callerid, None, params_kw, outlines, sor) return if isinstance(b, bytes): b = b.decode('utf-8') debug(f'task sumbited:{b}') d = DictObject(**json.loads(b)) - if not d.get('context'): - debug(f'{b} error') - yield '{"error":"server return no taskid", "status": "SUCCEEDED" }\n' - return + outlines.append(d) + t2 = time.time() uapi = UAPI(request, sor=sor) apinames = [ name.strip() for name in llm.query_apiname.split(',') ] for apiname in apinames: while True: b = None try: - b = await uapi.call(llm.upappid, apiname, userid, params=d.context) + b = await uapi.call(llm.upappid, apiname, userid, params=d) except Exception as e: exception(f'{e=},{format_exc()}') estr = erase_apikey(e) yield f'{{"error": "ERROR:{estr}", "status": "SUCCEEDED" }}\n' - break + outlines.append({"error": "ERROR:{estr}", "status": "FAILED" }) + await write_llmusage(llm, callerid, None, params_kw, outlines, sor) + return if isinstance(b, bytes): b = b.decode('utf-8') @@ -200,11 +264,19 @@ async def async_uapi_request(request, llm, sor): if not rzt.status or rzt.status == 'FAILED': debug(f'{b=} return error') yield f'{{"error": "ERROR:upapp return failed", "status": "SUCCEEDED" }}\n' + outlines.append({"error": "ERROR:{estr}", "status": "FAILED" }) + await write_llmusage(llm, callerid, None, params_kw, outlines, sor) return if rzt.status == 'SUCCEEDED': debug(f'{b=} return successed') await asyncio.sleep(1) d = rzt + outlines.append(d) + usage = d.get('usage', {}) + t3 = time.time() + usage['response_time'] = t2 - t1 + usage['finish_time'] = t3 -t1 + await write_llmusage(llm, callerid, usage, params_kw, outlines, sor) break period = llm.query_period or 30 await asyncio.sleep(period) @@ -228,20 +300,22 @@ def b64media2url(request, mediafile): url = entire_url('/idfile?path=') + env.quote(mediafile) return url -async def inference(request, *args, **kw): +async def inference(request, *args, params_kw=None, **kw): env = request._run_ns.copy() - llmid = env.params_kw.llmid + if not params_kw: + params_kw = env.params_kw + llmid = params_kw.llmid dbname = env.get_module_dbname('llmage') db = env.DBPools() async with db.sqlorContext(dbname) as sor: llm = await get_llm(llmid) if llm.stream == 'async': - f = partial(async_uapi_request, request, llm, sor) + f = partial(async_uapi_request, request, llm, sor, params_kw=params_kw) return await env.stream_response(request, f) if llm.stream == 'sync': - f = partial(sync_uapi_request, request, llm, sor) + f = partial(sync_uapi_request, request, llm, sor, params_kw=params_kw) return await env.stream_response(request, f) # env.update(llm) uapi = UAPI(request, sor=sor) - f = partial(uapi_request, request, llm, sor) + f = partial(uapi_request, request, llm, sor, params_kw=params_kw) return await env.stream_response(request, f) diff --git a/models/llm.xlsx b/models/llm.xlsx index 67bb365afc09a3741f1920cf2382058f3f28804a..8731c84e130c1dd526b76893ecea2e947f004187 100644 GIT binary patch delta 3547 zcmZ8kc{tSVyC1vBGQ;p%D`h8)HQ5?NmT5fte=A)%u|Cvd-Na> z1Ox)%Ffsw&z8+5A-X798PfQ`&&N~av@g%Q-^7L0OFAT|UX3~f%xHVJ!aD(jp^(pKX z?=Xe{L7#26iQwPfhuFoiKYt<<;kT{c(a}Ehpxa8mh&U{$Ep5A8qm~LDvtVoO%<`un zco+T+%qi!}HUr4UR_Wa9wDl?swAVY{1R<&flX*=;xi^$ucqX{&9Bx>qnz(%EGOMhS z<|v3VE>P1A3NucThPs>vtAzw9RCC+8^}Z zbr;43`yoO%yT{)S86-zn5-S~hZqr@9Sfn(akPo{peEwM!wUL`QPiC0{eExn$d?DbPti0X6q!H48X?<{ndz znIQ|`n>pa=2i6ZaB`YojW#5L@?!}ym6P;=T5J)KN3kB9S%;oFno}>iH^hilT*AXaaXZ_j9Mtj}@r8jKUg+zUkc9QOgRBFrXBB45*r2 zzkzv^aR+lG9*@?`wGR$Z;U}!s$dk5Ca%TB}DS)^;_fmef?d&^Ng?P#6v>N*2Lbv7n z_k|Hj)~k%i#+~xxkk(46UKagaez*3PhzjRzQ7zA)MylCw2VK)Tmt(w3Mh_ELqUI2c z_KPRbk5%@o1W<9-(60d);n^4*o7L$3qS|E9fS#sJ&$(xEGfvu}kABIjm@TjeTj#1A z22?mlTxa%)ahg!VyZ;^VC_YnO`+7iZ#2t&o@m&m&>it9bfpoEDQu zJIQogkyEYg#C;>yYa{mi8XwlZYY7_~armz5l!6TxEdILH+#XaiFAlr)X3}d(W%wn# zcTR44G_Hz2sN#og=$ZN}we>yseJ&7)Qg6omHz!aVi5fJg1%dKlAP~oYi@@)pqp!<- z(?`B;Uao%sD&dvom=T{YM~B5a#bN7833RrSL-B(@Yof%#N3Ec8358D=XA3q*F#%+b zz`7ZPo0e1IXKp-AKEBt4i509#)wT>oskMFk0OWg<`u|aN#`^osWn5qAI3gW&0LO`K z>aH1OR+}xM@7hyIiSJC(*M|ifk|#TkPh!f(8j|0zDJBexje6`&O?Yu>oQtirs?m9r zWlauc&yKG(RHW60&G8NQb3BM1u7ySzl_dLnG6?oaIS)(8pP@wjaKULC>nkfs$T%)BpbjfN!bE(ub1}}R`;OaN|H!cH7FWhE=rv-Dm!) z=E3_l(dEW02k?*iX$I(&LNi6W6cn>dlXj()yNByR^XKJ^DUYAP%?;9Ho<2_1_)+sC zQVH7I<7nY~ouxASL=~8LL&>BH&yF z+2Ykbci*sU1P^SN$M?PeXVqMI%NY)Vx+m8no+HA=0V~ym07C~7gBtj7omKcpH2w%xO z;U2-!40C^WP$1AdWczHd3z*ndwHZuExT9ltkpSCXsw!1yX$}{fjuJCB@Sci)BoydF zb5be6szmx0)%;k7r!9?ZPKdNJno6A@w56&MisA=7xjavwbeCPWQi4KwxYL26e>?!{D-et*2>oB#Cxp+3$j$~;_v_B#6N+zk3EKH3~wLm5Tul= zF*&9hhGKZ@8RSGob_nGV-dfblaF+`?%j}$q+DhZN)B+;Db2jZC?5dsNi9m+c{db`SK$|6ZD3qo zuqfYE9~scc>a-jb^PoI0G{c{k?w4D{*(aK$*g1GkxjdIKGl zD@m${dvJ5{xc!%SmfRJF_m9uG9C7{p81!71pv03b;Slwj_kx@2u{DFBn7lV5NCZyBW2Gg_1fkc*) ze>Htn(`P%1TX3*|9OXIdu6rr^@4lb;yH`CQ{<^oLueTRT-xwD3_UDP{B}(YM$~5i* zHYMd7u|XCP&lSMuzPQ?)&6z1@KG5B;TJ+Qe^GI3RFru;A|A^5YO&2U5gDRZR>rLe5 zq!?oiYRAOA#;Rxx`-78&3HzDs#NF4K8m6K*8S4!r{bZKE0Qw?1v@7;W=eoY*DC~FP z>05`&O?oz#%qnYA;VYcAhcUl?(@pu)3Y6f~PhQUu(6D<&OLPv15I{lC*K_BS zw)g3MIm))($Ks=^J7Z(Em%j;(vN(FdR{G%<{ls{1|{#LrM{?acfR)J2TS*QCd@imXWXtk|`{jIOl z53hO0^a)08yB)1+3YOz0+-E1uvSa%{^Jaseb-iTez596`?E^gc9BDKy7TGhzMk`su zCOm5Wg=E7!CH1@5j+KuS_*rT>o3K}#1}y3BV`=7-_AA8Pb+G9dOQ?>x!m#z(eMhEg zxL9b((RUMvx8{rmoI|b14vA14>FLTLZFwynxc&pCd-6F`&$ZHtJ4&I}OPN+?-2r$g zAJ@JOy-T^oCScg2Kazv-mXf}^XL2WHPt&PI~rl{%Z~OuJT+Aw1T_T=Jn51#HaxM9ymwX z0v$zbFet6Cz!W)&AIjywV92q6^lb}kMCtwW>73-&Hp1FR&hqUQv=^^ci{=>;^n2C?eu3yv9Z*4&)h#b?xc+DV57Q#WS9 z9eDcfG5Jd&>=#v{Q+!w;K92xC3~_3gW${G z_lF3E__ff_m%TxgKW}mcx-4VNl&7ZFxC4e$ouFgiNfxfovrPp58FJ*Yc*s5Vj(xOg zQJW1I;c|rz#)^Yi%{6-Bb^QX`GEZxwU8^_-+O&u!-8|mq32NvR_B#pffmph8&lhtY z{UC;nUI$F6kI8)T5oAMHLLRx7JZzF#xyDi;uQQ;V_Qk1_NkvV8tUzI}S1S@4S`l6U&UFL{Ar%*@b zSrtdh8OQPM(;vUj>-YTiem&3g{_lC-@8@~_5O|aXeCY{7cZs&&{y+x+kk0@B>;M2D z9HnqSI27p}9E_9?4@6bky9X~Qaz{UDIpsLKa+g&!>3LZ|*<8*mP#`TFXW^Ypw+V@9>Oo`>M766RtWW6GH07rA zN#(S?naEXZpJ3%6=e(z31+`$UOS?JMb-y~l(%}*rN_6_)bQf3iL#>Zwz5bDZenZ^e z37xZ$MfJa7?~@(MD7F{Vs0*HF-D5L3+aLGnnNrC=(ybkiw;qW^1&io(Jwk_c^y!&& zdOFo+fD&A(71iFS?2Oa1k^uclSMN{j%M1Wr2u!BPh1~bs_ZXr~E0P!8wkD$5)A@^Q^B%jj&^dfsys7mK$c4IG?MB59FjI6JAQ}5RB7|m}C zN!GnwGwhu~mn2eTOCdMA81SR__bbX8yE|jue?Rzwy1_Ph-!_;woNiJ7BSoDMrVxAJ z(7N=bbD7T%O{=`A`sj`>|ez%or5ikp&els zBF>+?%XvC#Bw8BIwfN0jZbT*2`s|;-GLpjbf~>P{;XBE{Pnzu7BbE4UOMI|V@ZPn0GQ&$eiRi%BZ37CxqHNS zgmG7PXJ^$@&c>64WJGT-1Ednmz3!v3JGGiOv-JB%;-TkM>&4G6UsN*H$G+#32wG4L z@WGzsziUQ5Ibi9cbUJ+Clk#gD8tbC{>fY|&CuD@dF3730#zOKH3+J0)$}Y^dC47l_ zzZM*?$Eg2q%nQ9-eny!OJs1%P!Y>f|kG5Cc@~gkN{#a{OUf*-mB>%X965T)3tdvwD z7ItZ1>1uWEy+9`MH*!8>aw;?gRHf(@N12P6Klt6!X^d)Jl}v7wV_%z*>__MNoG??- zAr@|Djr_GFH_Zbrx8=m5>2fj$%VqN^C&+D76NaJD*ZeH>nWRT4T0Gad;vB9n@7AE| zRGTuG{({Y5uJs*1=YmjJ+E%DkCxfyXl4TfYQk(!ED2w5|F>u~d0;E=H0oK+<)F35K zZm9~v%gu9mmoU_Mi$kC8jLJn{GdA_d5sY_!Vd8JB4I7F-y*Ifq^({7+0SJ*-13l7F zv-cYO#C)a7?7WW+ddsJy&74ii?zUob9)x(26J<7Kh!u5ab+A#ZE%SJ*IfkkEfyu3{ zeyWmMhRb{2Y8V)1jg}i1N`Le4bbgsM>Tl!Lc>2k6c{tLJa*zGwdqe7~tqnPCel(#w z>*lL|;)%!T=S~$t;Om_%k8nyg$K3TJQ`D1`$2a)GklgbcXbQ|FxYiOLV@q?#HKYXD zr|B#(Ygmanb&;=ZrUhBnnkycM>0b*oyZY-sdp8wi{g@f1x7l{Ji@;>&HtdA^oE-$T zRP(+_ur{4!Y+;^ZI;-~1;Hj5b1qM}mbmafD`*h_yul#fCdg6v)3Pr6-G9r`yMTe~E zB8uWcw6%MOMi#U2OHGC7&0!{dU!Y4$4p)jcK3YJjmMJUsI{E4zf`__2*K!fF?~nc0 zCyPju=N*}(p$>O1vCdt~x-^vT)={P+LuiY0CyKs$w;Px&$3ZW9sq%i0+_<_ywBZ`F zziRmk)9P`5=?{RqCGvTr`@=|}6!n5t?eN9Z#n>4)w1A6&N5v*jMa5`R-pRzF+}@Mj zPsucgIV3r?B+~ThCBH9|&%cH`2Pf#defwCT5k3WFQ(x<2(%;CgCp0= zw+~8)XkU|I=yFX~P3ry^_^eQpu|DR9fuJl@VDAc~$YU`vrXg(^@*-l5ylDDtYxNwA zu-U;<1&|myu(I;vr~9&9F%MUk)If2EFTKRB=#-2Y{ea1@FewdIe&$Q-j^`(WJ7j!+2@GqtT;dx#MLkl@0; zgz#ZY6?o89mfY6@OKSK03g1V&X{~YKJ`{W$H-=Gqw+rB$($hTPxM;<$i3oO`v8{VX z*mD{$P`1`ORu|)p7Ots{#>brQDRx#3Fiqv^eLBNTr7I;7KKB*{*1I0^i*QJ&_B_3N zmYl1kfK=?y4rDo}Q-f0x`%>?!{8^LZhLmz)l_>_TB5tR@pIC!W^r|y6J@8#!fOG9} zFu5@7ZX@TYaegb38sduq#H>m+IWZoyx}VbsX{AtV3kt`@l6iMK94hhgvbJqk!$Vaf z)frk**-Ucq%J6-)ALVh9XjzcMG#x>^;1&K>5M{UcT3jZ&NHQkcvxoPgIMkViMa zr~z&7JgIAn&Inn%ZDItk|L!9^4?+P z_XSw=OCJW-N92iZL3;ZdbQaNBkeZ{p4}&a7$MgG%{R~`3Y+PI{e6we^5X&3FhYY8Q z*GuwwE6(2w3FT*R5b7L1{W`1{-rh#7mO<-h-kLH8J$Bj%aWlBnyEhjkjF8)i4faz? zZQ)PV;H>y6oR%qlk8zOvaqo51*zZdtGf16^Dvj~ZU`?oPY1PRM!_FC4P`YcGF2Cd$DzC;`raHydOehKHji#7r2e z?MB>&KO9{pZ646Jf&}gj`5c&|nhU(%>ffhnJPDrJxwnZQou8Ol%`x4(L(_@zAijt$ zq7X+CCfK?mO?5};ZJ2w^HvV{b;+(Wpp@3~rU5E6l457Qyt-fPE7pc0R7h1nYvJHZT zbwq9*F0oL5vXuz;oPj0uuAp;TWieA`3;`P8vZfn-fsbeNj435M1Sdn5B%%6&aM=Xd z$AKAp!mHV0SAF?%Ga3 zz(mCYi7nk&qOtl-y|Jv)V4dTqhO#y7T0*M^mjPXZrc0{QJgzW|3vC?wuELW|)>y*n zvaHftlq%_{DvoFqCX_Uv#$_(STjSt8Mt<+Fr#|Zu+jpI{a?+N5N>zi$z(-J%;6n5D zVu|1t>?bCNNZ(X+NO`N7dp9}C{_MzuxG_k|^FAJl!YV(m9w9o`W={v$M-IOvS3)^g z4@=@eyC+F`>aAT;iAy4Wwqzv|dD{#083cS!;Z&ksRlYt3P_)@Zv|dRHidA||ve@cr znO5LVEpywV`JIeX0k+X&rd4Qz;>)^5`O~`Bm?(y07&~FlwdxdX{ciS1>BPix^>>Bl z=PCkB4C{a+bn}qrMIicWzan?iXUk)D^Uubv@JYW6gL79oEd{3z-nZPJeEG`U6>=Pj z2Xd9~?CcdEtri#g{Vr+1D1>-8tLWc>Thkdi)r>uO=rAbOZJ5fwXyV~auhPPly>>{k zhXz1#2^aj4A&9uEVk6)v)V_#{a^XZn zM(Bs+6ze0!G{vgEJ)%_CX~uH%%1nC358Z+9WW!=-P9~D9E{6qx_wH`q$%q`!`zo@x zRpQv)c&iYh^22-Jn7dcui3h&6MwwjC@KhTc4`R}G$`3ArzVQ!>#HSs-%XmOm(Eq(FSh%hg z&=H%hYe zBY=`v8H6UCg$y=BMiQHjkfF1Z{mUIgfCc~4asYt$KjI%`s_?goQ%?bCgw@l71LLs? ndg22APJ%rU0J!z%wL1U*2qD-$JsrAZZ7fj#@);4Ge{25(eOz)T