yumoqing/pipeline-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(timeout): QC/PM/复盘三处llm_call_msgs补timeout=_LLM_CLIENT_TIMEOUT——2026-09-16 pbls实测:QC审核不传timeout→payload无_timeout→inference按供应商端点配置(百炼120s)掐断上游,大上下文长生成(trace实测121s断连19次/单次最长510s)结构性必超时→_requeue回置3次达上限→fault+pause(5d_nSJ主任务16:11实锤)。2026-09-14同款预算错位当时只修develop native路径(2986行),QC(4059)/PM(3681)/复盘(4314)三处文本路径漏挂。timeout=570同时透传payload._timeout→inference按510s等上游,客户端aiohttp按570s总超时,外层wait_for 600s兜底,预算三层对齐

Browse Source
This commit is contained in:
ymq 2026-09-16 16:40:09 +08:00
parent c0cab536df
commit d59f5c43a7
1 changed files with 15 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
pipeline_service/agent_loop.py
View File

@ -3678,7 +3678,13 @@ async def pm_review_run(project_id, agent_id=None, model_name=None):
"随后必须立即输出 review_approve / review_reject / review_complete / review_rollback 之一，"
"禁止再调用其它工具。"})
try:
raw = await llm_call_msgs(msgs, model=model_name, temperature=0.3, org_id=org_id, project_id=project_id, session_id='task:%s' % task_id)
# timeout 必传（2026-09-16 pbls QC 超时事故）：不传则 payload 无
# _timeout → inference 用供应商端点配置（百炼 120s）掐断上游，
# QC/PM 大上下文长生成（170~370s 实测）必撞墙 → 3 次 attempt
# ≈370s 又超客户端缺省 330s → TimeoutError → 回置 3 次达上限
# → fault+pause（2026-09-14 同款预算错位，当时只修了 develop
# native 路径，QC/PM/复盘三处漏挂）。
raw = await llm_call_msgs(msgs, model=model_name, temperature=0.3, org_id=org_id, project_id=project_id, session_id='task:%s' % task_id, timeout=_LLM_CLIENT_TIMEOUT)
except Exception as e:
err_msg = f"{type(e).__name__}: {str(e)[:400]}"
# PM 审核环节基础设施故障 → 回置 review 重审，不打 fail 让上游重做
@ -4056,7 +4062,12 @@ async def qc_review_run(project_id, agent_id=None, model_name=None):
msgs.append({"role": "user", "content":
"已检查足够信息。现在必须立即输出 review_approve 或 review_reject，禁止再调用其它工具。"})
try:
raw = await llm_call_msgs(msgs, model=model_name, temperature=0.2, org_id=org_id, project_id=project_id, session_id='task:%s' % task_id)
# timeout 必传（2026-09-16 pbls QC 超时事故，与 PM 审核同款）：
# 不传 → inference 按端点配置 120s 掐断 + 客户端缺省 330s，QC
# 大上下文审核（16 轮取证、170~370s 长生成）结构性必超时 →
# 回置 3 次达上限 → fault+pause。传 _LLM_CLIENT_TIMEOUT(570)
# 同时透传 payload._timeout 让 inference 侧按 510s 等上游。
raw = await llm_call_msgs(msgs, model=model_name, temperature=0.2, org_id=org_id, project_id=project_id, session_id='task:%s' % task_id, timeout=_LLM_CLIENT_TIMEOUT)
except Exception as e:
err_msg = f"{type(e).__name__}: {str(e)[:400]}"
# 审核方基础设施故障 ≠ develop 交付失败（2026-09-16 pbls M1a 事故根治）：
@ -4311,7 +4322,8 @@ async def retrospective_run(project_id, agent_id=None, model_name=None):
"已执行足够轮次。现在必须立即收尾：write_file 写复盘报告（若未写），然后输出 deliver 提交，禁止再调其它工具。"})
try:
raw = await asyncio.wait_for(
llm_call_msgs(msgs, model=model_name, temperature=0.3, org_id=org_id, project_id=project_id, session_id='task:%s' % task_id),
# timeout 必传（2026-09-16 pbls 同款预算错位第三处：复盘）
llm_call_msgs(msgs, model=model_name, temperature=0.3, org_id=org_id, project_id=project_id, session_id='task:%s' % task_id, timeout=_LLM_CLIENT_TIMEOUT),
timeout=_LLM_HARD_TIMEOUT)
except Exception as e:
err_msg = f"{type(e).__name__}: {str(e)[:400]}"