fix(agent): 任务族工具被capability路由劫持FAIL + LLM超时误判永久错误致任务链暂停

pbls 项目事故(2026-09-14)四条根因一次修复:
- v2 _execute_tool 1.5步加 TOOL_SCHEMAS 双守卫(对齐 v1 _exec_agent_tool):
  task 技能 frontmatter 声明的任务状态机工具(reset_task_retry/claim_task/
  approve_task等)不在概念工具 schema 表,此前被劫持返回 FAIL:未注册的能力工具,
  能力包真实 handler(第2步)永远到不了——主 agent 无法恢复失败任务
- llm_bridge 如实区分超时/连接失败: TimeoutError(str为空)不再误报
  「端点不可达:。」(冒号后空白),超时文案明确瞬时故障可重试
- _classify_failure 补中文提示词(超时/不可达/瞬时等): 中文报错不再
  误判永久错误跳过重试直接 pause_project+fault_report
- role_agent_run 客户端单次调用超时缺省330→570(_LLM_CLIENT_TIMEOUT),
  与外层600硬超时衔接;实测368s长生成不再被提前掐断
- fault_report 文案区分「重试耗尽」与「判定永久错误(未触发自动重试)」,
  消除 retry_count=0 却报「重复0次已达上限(3)」的自相矛盾
This commit is contained in:
ymq 2026-09-14 20:20:12 +08:00
parent 1febcd3acb
commit 7f3ce26295
3 changed files with 45 additions and 6 deletions

View File

@ -2171,6 +2171,11 @@ _FORCE_PRODUCE_TURN = 5
# (420s 只够 ~1.4 轮,网关瞬时挂起时重试被提前掐断,可自愈的瞬时故障也 TimeoutError 超限暂停任务链)。 # (420s 只够 ~1.4 轮,网关瞬时挂起时重试被提前掐断,可自愈的瞬时故障也 TimeoutError 超限暂停任务链)。
_LLM_HARD_TIMEOUT = 600 _LLM_HARD_TIMEOUT = 600
# 单次 LLM 调用客户端超时(aiohttp total),略小于外层硬超时,让长生成用满外层
# 预算而不被缺省 330s 提前掐断(2026-09-14 pbls:需求分析生成 ~368s > 330s 缺省
# → 客户端 TimeoutError,旧代码误报「端点不可达」触发任务链暂停)。
_LLM_CLIENT_TIMEOUT = 570
_FORCE_PRODUCE_HINT = ( _FORCE_PRODUCE_HINT = (
"⚠️ 你已经探索了足够多轮(已超过 5 轮)。现在必须立即产出并交付:\n" "⚠️ 你已经探索了足够多轮(已超过 5 轮)。现在必须立即产出并交付:\n"
"1. 用 write_file 写出实际交付文件(代码/文档/契约/DDL),不要再 read_file / list_files / run_shell / git_status 等探索或检查类工具。\n" "1. 用 write_file 写出实际交付文件(代码/文档/契约/DDL),不要再 read_file / list_files / run_shell / git_status 等探索或检查类工具。\n"
@ -2477,7 +2482,7 @@ async def role_agent_run(project_id, role, agent_id=None, model_name=None):
msgs.append({"role": "user", "content": _FORCE_PRODUCE_HINT}) msgs.append({"role": "user", "content": _FORCE_PRODUCE_HINT})
try: try:
resp = await asyncio.wait_for( resp = await asyncio.wait_for(
llm_call_msgs_native(msgs, tools=tools_schema, model=model_name, temperature=0.4, org_id=org_id, project_id=project_id, session_id='task:%s' % task_id), llm_call_msgs_native(msgs, tools=tools_schema, model=model_name, temperature=0.4, org_id=org_id, project_id=project_id, session_id='task:%s' % task_id, timeout=_LLM_CLIENT_TIMEOUT),
timeout=_LLM_HARD_TIMEOUT) timeout=_LLM_HARD_TIMEOUT)
except Exception as e: except Exception as e:
err_msg = f"{type(e).__name__}: {str(e)[:400]}" err_msg = f"{type(e).__name__}: {str(e)[:400]}"
@ -3685,6 +3690,10 @@ _RETRY_HINTS = (
"timeout", "timed out", "connection", "connect", "network", "unreachable", "timeout", "timed out", "connection", "connect", "network", "unreachable",
"rate limit", "429", "500", "502", "503", "504", "reset", "broken pipe", "rate limit", "429", "500", "502", "503", "504", "reset", "broken pipe",
"eof", "temporar", "busy", "overloaded", "eof", "temporar", "busy", "overloaded",
# 中文提示词(llm_bridge 等模块的报错文案是中文,2026-09-14 pbls 事故:
# 「LLM 推理端点不可达/调用超时」不含任何英文 hint → 误判永久错误,
# 跳过自动重试直接 pause_project + fault_report)
"超时", "不可达", "瞬时", "重试", "连接", "网络", "限流", "繁忙",
) )
@ -3762,9 +3771,15 @@ async def handle_failed_task(task_id: str, project_id: str) -> dict:
logger.warning("failed task pause_project error: %s", e) logger.warning("failed task pause_project error: %s", e)
from .communication import raise_problem from .communication import raise_problem
reporter = "agent.main_agent" if role == "agent.pm" else "agent.pm" reporter = "agent.main_agent" if role == "agent.pm" else "agent.pm"
# 文案区分两种 fault 成因(2026-09-14 pbls:retry_count=0 却报「重复0次已达
# 上限(3)」自相矛盾——实为 _classify_failure 判永久错误跳过重试,非重试耗尽)
if retry_count >= max_retry:
_reason = f"已自动重试 {retry_count} 次仍失败(达上限 {max_retry})"
else:
_reason = f"判定为永久性错误(非瞬时故障),未触发自动重试(已重试 {retry_count}/{max_retry} 次)"
qid = await raise_problem( qid = await raise_problem(
"fault_report", "fault_report",
f"任务「{title}」重复 {retry_count} 次已达上限({max_retry}),任务链已暂停,请人工介入处理。失败原因:{last_error or '未知'}", f"任务「{title}」{_reason},任务链已暂停,请人工介入处理。失败原因:{last_error or '未知'}",
reporter, tenant_id=project_id, task_id=task_id, reporter, tenant_id=project_id, task_id=task_id,
first_handler_role="agent.main_agent", first_handler_role="agent.main_agent",
context={"fault": True, "last_error": last_error, "retry_count": retry_count, context={"fault": True, "last_error": last_error, "retry_count": retry_count,

View File

@ -1072,9 +1072,18 @@ class AgentExecutor:
# 路由到 capability_tools.exec_capability_tool(按 TOOL_SCHEMAS 分发到 # 路由到 capability_tools.exec_capability_tool(按 TOOL_SCHEMAS 分发到
# 对应 capability 模块,系统上下文 project_id/iteration_id/who/agent_id # 对应 capability 模块,系统上下文 project_id/iteration_id/who/agent_id
# 自动注入,不暴露给 LLM)。generic 会话不挂(作用域解析已排除)。 # 自动注入,不暴露给 LLM)。generic 会话不挂(作用域解析已排除)。
# ⚠️ 必须「名字被声明 且 TOOL_SCHEMAS 有 schema」双守卫(对齐 v1
# _exec_agent_tool 的 `if tool in TOOL_SCHEMAS`):技能 frontmatter 声明的
# capability 工具(task 技能的 claim_task/reset_task_retry 等)多数不在
# TOOL_SCHEMAS(那是概念工具 schema 表),真正实现是产线能力包 handler
# (第 2 步 _execute_ability_tool)。此前只查名字 → 整族任务状态机工具被
# 本步劫持,返回「FAIL: 未注册的能力工具」,能力包 handler 永远到不了
# (2026-09-14 pbls 项目实测:主 agent 反复调 reset_task_retry 全 FAIL)。
if tool_name in self._capability_tool_names: if tool_name in self._capability_tool_names:
from .capability_tools import exec_capability_tool from .capability_tools import TOOL_SCHEMAS, exec_capability_tool
return await exec_capability_tool(tool_name, params, self._build_ctx()) if tool_name in TOOL_SCHEMAS:
return await exec_capability_tool(tool_name, params, self._build_ctx())
# 声明了但无概念工具 schema → 落到第 2 步产线能力包分发(真实 handler 所在)
# 2. 产线能力包工具(从 PipelineAbility 注册表按 pipeline_id 取 handler) # 2. 产线能力包工具(从 PipelineAbility 注册表按 pipeline_id 取 handler)
result = await self._execute_ability_tool(tool_name, params) result = await self._execute_ability_tool(tool_name, params)

View File

@ -11,6 +11,7 @@ LLM bridge for pipeline handlers — 统一收敛到模型治理模块(pipelin
每次自调用带机构/用户上下文,用量流水可归属到真实用户。 每次自调用带机构/用户上下文,用量流水可归属到真实用户。
""" """
import asyncio
import json import json
import logging import logging
import time import time
@ -94,9 +95,23 @@ async def _http_chat(payload, org_id, user_id, model_name, timeout: int = 0,
text = await resp.text() text = await resp.text()
status = resp.status status = resp.status
except Exception as e: except Exception as e:
# 如实区分错误类型(2026-09-14 pbls 事故教训):TimeoutError/ServerTimeoutError
# 的 str() 为空 → 旧文案「端点不可达:。」冒号后空白且把超时误报成不可达,
# 下游 _classify_failure 按「不可达」的中文查不到英文 hint → 误判永久错误
# 直接暂停任务链。超时就报超时(瞬时故障,应重试);连接类错误才报不可达。
import aiohttp as _ah
if isinstance(e, (asyncio.TimeoutError, _ah.ServerTimeoutError)):
raise ValueError(
'LLM 推理端点调用超时(%s/chat/completions,客户端上限 %ss):'
'请求已发出但未在上限内收到响应,属瞬时故障可重试。'
% (base, _total))
if isinstance(e, (_ah.ClientConnectionError, ConnectionError, OSError)):
raise ValueError(
'LLM 推理端点不可达(%s/chat/completions):%s: %s。'
'请检查本进程服务是否正常' % (base, type(e).__name__, e))
raise ValueError( raise ValueError(
'LLM 推理端点不可达(%s/chat/completions):%s。' 'LLM 推理端点调用失败(%s/chat/completions):%s: %s'
'请检查本进程服务是否正常' % (base, e)) % (base, type(e).__name__, e))
try: try:
data = json.loads(text) data = json.loads(text)
except Exception: except Exception: