diff --git a/pipeline_service/agent_loop.py b/pipeline_service/agent_loop.py index 6341b88..e8a47b8 100644 --- a/pipeline_service/agent_loop.py +++ b/pipeline_service/agent_loop.py @@ -2171,6 +2171,11 @@ _FORCE_PRODUCE_TURN = 5 # (420s 只够 ~1.4 轮,网关瞬时挂起时重试被提前掐断,可自愈的瞬时故障也 TimeoutError 超限暂停任务链)。 _LLM_HARD_TIMEOUT = 600 +# 单次 LLM 调用客户端超时(aiohttp total),略小于外层硬超时,让长生成用满外层 +# 预算而不被缺省 330s 提前掐断(2026-09-14 pbls:需求分析生成 ~368s > 330s 缺省 +# → 客户端 TimeoutError,旧代码误报「端点不可达」触发任务链暂停)。 +_LLM_CLIENT_TIMEOUT = 570 + _FORCE_PRODUCE_HINT = ( "⚠️ 你已经探索了足够多轮(已超过 5 轮)。现在必须立即产出并交付:\n" "1. 用 write_file 写出实际交付文件(代码/文档/契约/DDL),不要再 read_file / list_files / run_shell / git_status 等探索或检查类工具。\n" @@ -2477,7 +2482,7 @@ async def role_agent_run(project_id, role, agent_id=None, model_name=None): msgs.append({"role": "user", "content": _FORCE_PRODUCE_HINT}) try: resp = await asyncio.wait_for( - llm_call_msgs_native(msgs, tools=tools_schema, model=model_name, temperature=0.4, org_id=org_id, project_id=project_id, session_id='task:%s' % task_id), + llm_call_msgs_native(msgs, tools=tools_schema, model=model_name, temperature=0.4, org_id=org_id, project_id=project_id, session_id='task:%s' % task_id, timeout=_LLM_CLIENT_TIMEOUT), timeout=_LLM_HARD_TIMEOUT) except Exception as e: err_msg = f"{type(e).__name__}: {str(e)[:400]}" @@ -3685,6 +3690,10 @@ _RETRY_HINTS = ( "timeout", "timed out", "connection", "connect", "network", "unreachable", "rate limit", "429", "500", "502", "503", "504", "reset", "broken pipe", "eof", "temporar", "busy", "overloaded", + # 中文提示词(llm_bridge 等模块的报错文案是中文,2026-09-14 pbls 事故: + # 「LLM 推理端点不可达/调用超时」不含任何英文 hint → 误判永久错误, + # 跳过自动重试直接 pause_project + fault_report) + "超时", "不可达", "瞬时", "重试", "连接", "网络", "限流", "繁忙", ) @@ -3762,9 +3771,15 @@ async def handle_failed_task(task_id: str, project_id: str) -> dict: logger.warning("failed task pause_project error: %s", e) from .communication import raise_problem reporter = "agent.main_agent" if role == "agent.pm" else "agent.pm" + # 文案区分两种 fault 成因(2026-09-14 pbls:retry_count=0 却报「重复0次已达 + # 上限(3)」自相矛盾——实为 _classify_failure 判永久错误跳过重试,非重试耗尽) + if retry_count >= max_retry: + _reason = f"已自动重试 {retry_count} 次仍失败(达上限 {max_retry})" + else: + _reason = f"判定为永久性错误(非瞬时故障),未触发自动重试(已重试 {retry_count}/{max_retry} 次)" qid = await raise_problem( "fault_report", - f"任务「{title}」重复 {retry_count} 次已达上限({max_retry}),任务链已暂停,请人工介入处理。失败原因:{last_error or '未知'}", + f"任务「{title}」{_reason},任务链已暂停,请人工介入处理。失败原因:{last_error or '未知'}", reporter, tenant_id=project_id, task_id=task_id, first_handler_role="agent.main_agent", context={"fault": True, "last_error": last_error, "retry_count": retry_count, diff --git a/pipeline_service/agent_loop_v2.py b/pipeline_service/agent_loop_v2.py index fae85c3..b9ce825 100644 --- a/pipeline_service/agent_loop_v2.py +++ b/pipeline_service/agent_loop_v2.py @@ -1072,9 +1072,18 @@ class AgentExecutor: # 路由到 capability_tools.exec_capability_tool(按 TOOL_SCHEMAS 分发到 # 对应 capability 模块,系统上下文 project_id/iteration_id/who/agent_id # 自动注入,不暴露给 LLM)。generic 会话不挂(作用域解析已排除)。 + # ⚠️ 必须「名字被声明 且 TOOL_SCHEMAS 有 schema」双守卫(对齐 v1 + # _exec_agent_tool 的 `if tool in TOOL_SCHEMAS`):技能 frontmatter 声明的 + # capability 工具(task 技能的 claim_task/reset_task_retry 等)多数不在 + # TOOL_SCHEMAS(那是概念工具 schema 表),真正实现是产线能力包 handler + # (第 2 步 _execute_ability_tool)。此前只查名字 → 整族任务状态机工具被 + # 本步劫持,返回「FAIL: 未注册的能力工具」,能力包 handler 永远到不了 + # (2026-09-14 pbls 项目实测:主 agent 反复调 reset_task_retry 全 FAIL)。 if tool_name in self._capability_tool_names: - from .capability_tools import exec_capability_tool - return await exec_capability_tool(tool_name, params, self._build_ctx()) + from .capability_tools import TOOL_SCHEMAS, exec_capability_tool + if tool_name in TOOL_SCHEMAS: + return await exec_capability_tool(tool_name, params, self._build_ctx()) + # 声明了但无概念工具 schema → 落到第 2 步产线能力包分发(真实 handler 所在) # 2. 产线能力包工具(从 PipelineAbility 注册表按 pipeline_id 取 handler) result = await self._execute_ability_tool(tool_name, params) diff --git a/pipeline_service/llm_bridge.py b/pipeline_service/llm_bridge.py index f1ba140..7f6695c 100644 --- a/pipeline_service/llm_bridge.py +++ b/pipeline_service/llm_bridge.py @@ -11,6 +11,7 @@ LLM bridge for pipeline handlers — 统一收敛到模型治理模块(pipelin 每次自调用带机构/用户上下文,用量流水可归属到真实用户。 """ +import asyncio import json import logging import time @@ -94,9 +95,23 @@ async def _http_chat(payload, org_id, user_id, model_name, timeout: int = 0, text = await resp.text() status = resp.status except Exception as e: + # 如实区分错误类型(2026-09-14 pbls 事故教训):TimeoutError/ServerTimeoutError + # 的 str() 为空 → 旧文案「端点不可达:。」冒号后空白且把超时误报成不可达, + # 下游 _classify_failure 按「不可达」的中文查不到英文 hint → 误判永久错误 + # 直接暂停任务链。超时就报超时(瞬时故障,应重试);连接类错误才报不可达。 + import aiohttp as _ah + if isinstance(e, (asyncio.TimeoutError, _ah.ServerTimeoutError)): + raise ValueError( + 'LLM 推理端点调用超时(%s/chat/completions,客户端上限 %ss):' + '请求已发出但未在上限内收到响应,属瞬时故障可重试。' + % (base, _total)) + if isinstance(e, (_ah.ClientConnectionError, ConnectionError, OSError)): + raise ValueError( + 'LLM 推理端点不可达(%s/chat/completions):%s: %s。' + '请检查本进程服务是否正常' % (base, type(e).__name__, e)) raise ValueError( - 'LLM 推理端点不可达(%s/chat/completions):%s。' - '请检查本进程服务是否正常' % (base, e)) + 'LLM 推理端点调用失败(%s/chat/completions):%s: %s' + % (base, type(e).__name__, e)) try: data = json.loads(text) except Exception: