From d59f5c43a7cc586e90e87c6cbecda403260b3270 Mon Sep 17 00:00:00 2001 From: ymq Date: Wed, 16 Sep 2026 16:40:09 +0800 Subject: [PATCH] =?UTF-8?q?fix(timeout):=20QC/PM/=E5=A4=8D=E7=9B=98?= =?UTF-8?q?=E4=B8=89=E5=A4=84llm=5Fcall=5Fmsgs=E8=A1=A5timeout=3D=5FLLM=5F?= =?UTF-8?q?CLIENT=5FTIMEOUT=E2=80=94=E2=80=942026-09-16=20pbls=E5=AE=9E?= =?UTF-8?q?=E6=B5=8B:QC=E5=AE=A1=E6=A0=B8=E4=B8=8D=E4=BC=A0timeout?= =?UTF-8?q?=E2=86=92payload=E6=97=A0=5Ftimeout=E2=86=92inference=E6=8C=89?= =?UTF-8?q?=E4=BE=9B=E5=BA=94=E5=95=86=E7=AB=AF=E7=82=B9=E9=85=8D=E7=BD=AE?= =?UTF-8?q?(=E7=99=BE=E7=82=BC120s)=E6=8E=90=E6=96=AD=E4=B8=8A=E6=B8=B8,?= =?UTF-8?q?=E5=A4=A7=E4=B8=8A=E4=B8=8B=E6=96=87=E9=95=BF=E7=94=9F=E6=88=90?= =?UTF-8?q?(trace=E5=AE=9E=E6=B5=8B121s=E6=96=AD=E8=BF=9E19=E6=AC=A1/?= =?UTF-8?q?=E5=8D=95=E6=AC=A1=E6=9C=80=E9=95=BF510s)=E7=BB=93=E6=9E=84?= =?UTF-8?q?=E6=80=A7=E5=BF=85=E8=B6=85=E6=97=B6=E2=86=92=5Frequeue?= =?UTF-8?q?=E5=9B=9E=E7=BD=AE3=E6=AC=A1=E8=BE=BE=E4=B8=8A=E9=99=90?= =?UTF-8?q?=E2=86=92fault+pause(5d=5FnSJ=E4=B8=BB=E4=BB=BB=E5=8A=A116:11?= =?UTF-8?q?=E5=AE=9E=E9=94=A4)=E3=80=822026-09-14=E5=90=8C=E6=AC=BE?= =?UTF-8?q?=E9=A2=84=E7=AE=97=E9=94=99=E4=BD=8D=E5=BD=93=E6=97=B6=E5=8F=AA?= =?UTF-8?q?=E4=BF=AEdevelop=20native=E8=B7=AF=E5=BE=84(2986=E8=A1=8C),QC(4?= =?UTF-8?q?059)/PM(3681)/=E5=A4=8D=E7=9B=98(4314)=E4=B8=89=E5=A4=84?= =?UTF-8?q?=E6=96=87=E6=9C=AC=E8=B7=AF=E5=BE=84=E6=BC=8F=E6=8C=82=E3=80=82?= =?UTF-8?q?timeout=3D570=E5=90=8C=E6=97=B6=E9=80=8F=E4=BC=A0payload.=5Ftim?= =?UTF-8?q?eout=E2=86=92inference=E6=8C=89510s=E7=AD=89=E4=B8=8A=E6=B8=B8,?= =?UTF-8?q?=E5=AE=A2=E6=88=B7=E7=AB=AFaiohttp=E6=8C=89570s=E6=80=BB?= =?UTF-8?q?=E8=B6=85=E6=97=B6,=E5=A4=96=E5=B1=82wait=5Ffor=20600s=E5=85=9C?= =?UTF-8?q?=E5=BA=95,=E9=A2=84=E7=AE=97=E4=B8=89=E5=B1=82=E5=AF=B9?= =?UTF-8?q?=E9=BD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipeline_service/agent_loop.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/pipeline_service/agent_loop.py b/pipeline_service/agent_loop.py index 4a10219..c55f4e5 100644 --- a/pipeline_service/agent_loop.py +++ b/pipeline_service/agent_loop.py @@ -3678,7 +3678,13 @@ async def pm_review_run(project_id, agent_id=None, model_name=None): "随后必须立即输出 review_approve / review_reject / review_complete / review_rollback 之一," "禁止再调用其它工具。"}) try: - raw = await llm_call_msgs(msgs, model=model_name, temperature=0.3, org_id=org_id, project_id=project_id, session_id='task:%s' % task_id) + # timeout 必传(2026-09-16 pbls QC 超时事故):不传则 payload 无 + # _timeout → inference 用供应商端点配置(百炼 120s)掐断上游, + # QC/PM 大上下文长生成(170~370s 实测)必撞墙 → 3 次 attempt + # ≈370s 又超客户端缺省 330s → TimeoutError → 回置 3 次达上限 + # → fault+pause(2026-09-14 同款预算错位,当时只修了 develop + # native 路径,QC/PM/复盘三处漏挂)。 + raw = await llm_call_msgs(msgs, model=model_name, temperature=0.3, org_id=org_id, project_id=project_id, session_id='task:%s' % task_id, timeout=_LLM_CLIENT_TIMEOUT) except Exception as e: err_msg = f"{type(e).__name__}: {str(e)[:400]}" # PM 审核环节基础设施故障 → 回置 review 重审,不打 fail 让上游重做 @@ -4056,7 +4062,12 @@ async def qc_review_run(project_id, agent_id=None, model_name=None): msgs.append({"role": "user", "content": "已检查足够信息。现在必须立即输出 review_approve 或 review_reject,禁止再调用其它工具。"}) try: - raw = await llm_call_msgs(msgs, model=model_name, temperature=0.2, org_id=org_id, project_id=project_id, session_id='task:%s' % task_id) + # timeout 必传(2026-09-16 pbls QC 超时事故,与 PM 审核同款): + # 不传 → inference 按端点配置 120s 掐断 + 客户端缺省 330s,QC + # 大上下文审核(16 轮取证、170~370s 长生成)结构性必超时 → + # 回置 3 次达上限 → fault+pause。传 _LLM_CLIENT_TIMEOUT(570) + # 同时透传 payload._timeout 让 inference 侧按 510s 等上游。 + raw = await llm_call_msgs(msgs, model=model_name, temperature=0.2, org_id=org_id, project_id=project_id, session_id='task:%s' % task_id, timeout=_LLM_CLIENT_TIMEOUT) except Exception as e: err_msg = f"{type(e).__name__}: {str(e)[:400]}" # 审核方基础设施故障 ≠ develop 交付失败(2026-09-16 pbls M1a 事故根治): @@ -4311,7 +4322,8 @@ async def retrospective_run(project_id, agent_id=None, model_name=None): "已执行足够轮次。现在必须立即收尾:write_file 写复盘报告(若未写),然后输出 deliver 提交,禁止再调其它工具。"}) try: raw = await asyncio.wait_for( - llm_call_msgs(msgs, model=model_name, temperature=0.3, org_id=org_id, project_id=project_id, session_id='task:%s' % task_id), + # timeout 必传(2026-09-16 pbls 同款预算错位第三处:复盘) + llm_call_msgs(msgs, model=model_name, temperature=0.3, org_id=org_id, project_id=project_id, session_id='task:%s' % task_id, timeout=_LLM_CLIENT_TIMEOUT), timeout=_LLM_HARD_TIMEOUT) except Exception as e: err_msg = f"{type(e).__name__}: {str(e)[:400]}"