feat(agent): 平台模型调用能力——invoke_model/list_platform_models工具(v1角色agent+v2会话agent,唯一实现platform_model_tools);llm_bridge.llm_infer通用推理透传全能力payload+超时参数化;角色prompt加配图节(禁字符画,生成真图嵌正文)(2026-09-07用户需求)
This commit is contained in:
parent
763e84681f
commit
bf6cb5b24a
@ -578,6 +578,13 @@ __TOOLS__
|
|||||||
## 技能使用
|
## 技能使用
|
||||||
上方「可用技能」是目录层(只有名字+描述)。遇到需要具体规范、目录结构、产出路径、文件格式、流程约束的任务,先用 load_skill 加载对应技能全文(如 project-directory-spec 项目目录规范),不要凭记忆瞎写路径或格式。
|
上方「可用技能」是目录层(只有名字+描述)。遇到需要具体规范、目录结构、产出路径、文件格式、流程约束的任务,先用 load_skill 加载对应技能全文(如 project-directory-spec 项目目录规范),不要凭记忆瞎写路径或格式。
|
||||||
|
|
||||||
|
## 配图与媒体生成(有平台模型可用,禁止字符画凑数)
|
||||||
|
产出文档/章节/方案时,遇到需要插图的内容(架构图/拓扑图/流程图示意图/效果图/封面图等),用 invoke_model 调用平台生成模型产出**真实图片**:
|
||||||
|
- 不确定有什么模型时先 list_platform_models(capability=t2i 文生图 / i2v 图生视频 / tts 语音等)。
|
||||||
|
- invoke_model 的 model 可留空(平台按任务自动匹配),task 写清画面内容(主体/风格/构图/文字要求)。
|
||||||
|
- 成功后把返回的产物 URL **原样**以 markdown 图片语法嵌入正文:。禁止改写/自编 URL。
|
||||||
|
- **禁止用 ASCII 字符画/文本框线拼「示意图」代替真实配图**。平台无可用生成模型(invoke_model 返回 FAIL)时,在产出中如实标注「配图缺失:平台无可用文生图模型」,不伪造。
|
||||||
|
|
||||||
## 工作流
|
## 工作流
|
||||||
1. 先用 read_file/list_files 了解现有代码
|
1. 先用 read_file/list_files 了解现有代码
|
||||||
2. 用 write_file 产出文件到 projects/{项目}/、apps/、modules/ 下(路径基准是上方「工作空间」= 机构工作空间)
|
2. 用 write_file 产出文件到 projects/{项目}/、apps/、modules/ 下(路径基准是上方「工作空间」= 机构工作空间)
|
||||||
@ -1956,6 +1963,12 @@ async def _exec_agent_tool(tool, params, workspace_dir, ctx=None):
|
|||||||
# 角色相对路径 projects/{项目}/、apps/、modules/ 都以它为基准。
|
# 角色相对路径 projects/{项目}/、apps/、modules/ 都以它为基准。
|
||||||
p = params or {}
|
p = params or {}
|
||||||
try:
|
try:
|
||||||
|
# 平台模型工具(2026-09-07:可用模型=本机构+平台owner机构,按任务自动选型)
|
||||||
|
if tool in ('list_platform_models', 'invoke_model'):
|
||||||
|
from .platform_model_tools import exec_platform_model_tool
|
||||||
|
return await exec_platform_model_tool(
|
||||||
|
tool, p, org_id=str((ctx or {}).get('org_id', '') or '0'),
|
||||||
|
user_id=str((ctx or {}).get('user_id', '') or ''))
|
||||||
if tool == 'read_file':
|
if tool == 'read_file':
|
||||||
path = p.get('path', '')
|
path = p.get('path', '')
|
||||||
if not path: return 'FAIL: 需要文件路径'
|
if not path: return 'FAIL: 需要文件路径'
|
||||||
@ -2276,7 +2289,11 @@ async def role_agent_run(project_id, role, agent_id=None, model_name=None):
|
|||||||
qna_section = await _build_qna_section(sor, task_id, role, agent_id)
|
qna_section = await _build_qna_section(sor, task_id, role, agent_id)
|
||||||
# 能力工具:角色技能 capability 声明 → 产线状态机规范 tools 声明 → 注入对应工具
|
# 能力工具:角色技能 capability 声明 → 产线状态机规范 tools 声明 → 注入对应工具
|
||||||
capability_tools = await _collect_capability_tools(sor, project_id, role, org_id)
|
capability_tools = await _collect_capability_tools(sor, project_id, role, org_id)
|
||||||
all_tools = AGENT_TOOLS + capability_tools
|
# 平台模型工具(2026-09-07):让产线角色 agent 也能查/调用平台模型
|
||||||
|
# (本机构+平台owner机构,生成类任务按 task 自动选型)。唯一实现在
|
||||||
|
# platform_model_tools,分发在 _exec_agent_tool。
|
||||||
|
from .platform_model_tools import PLATFORM_MODEL_TOOLS_V1
|
||||||
|
all_tools = AGENT_TOOLS + capability_tools + PLATFORM_MODEL_TOOLS_V1
|
||||||
tools_text = json.dumps(all_tools, ensure_ascii=False)
|
tools_text = json.dumps(all_tools, ensure_ascii=False)
|
||||||
|
|
||||||
# 注入角色技能(角色专属技能全量 + 其余 scope 目录层,优先级 角色>项目>产线>组织>通用)
|
# 注入角色技能(角色专属技能全量 + 其余 scope 目录层,优先级 角色>项目>产线>组织>通用)
|
||||||
@ -2944,7 +2961,8 @@ async def pm_review_run(project_id, agent_id=None, model_name=None):
|
|||||||
else:
|
else:
|
||||||
result = await _exec_agent_tool(tool, params, space_dir, {
|
result = await _exec_agent_tool(tool, params, space_dir, {
|
||||||
"project_id": project_id, "who": "agent.pm",
|
"project_id": project_id, "who": "agent.pm",
|
||||||
"agent_id": agent_id or "", "task_id": task_id})
|
"agent_id": agent_id or "", "task_id": task_id,
|
||||||
|
"org_id": org_id or '0'})
|
||||||
msgs.append({"role": "assistant", "content": raw})
|
msgs.append({"role": "assistant", "content": raw})
|
||||||
msgs.append({"role": "user", "content": f"工具 {tool} 结果:\n{result}"})
|
msgs.append({"role": "user", "content": f"工具 {tool} 结果:\n{result}"})
|
||||||
else:
|
else:
|
||||||
|
|||||||
@ -861,6 +861,9 @@ class AgentExecutor:
|
|||||||
"session_search": self._t_session_search,
|
"session_search": self._t_session_search,
|
||||||
"todo": self._t_todo,
|
"todo": self._t_todo,
|
||||||
"delegate_subtask": self._t_delegate_subtask,
|
"delegate_subtask": self._t_delegate_subtask,
|
||||||
|
# ── 平台模型(2026-09-07:按任务自动选型 + 全能力调用)──
|
||||||
|
"list_platform_models": self._t_list_platform_models,
|
||||||
|
"invoke_model": self._t_invoke_model,
|
||||||
}
|
}
|
||||||
|
|
||||||
handler = handlers.get(tool_name)
|
handler = handlers.get(tool_name)
|
||||||
@ -1255,6 +1258,17 @@ class AgentExecutor:
|
|||||||
block += f"\n(可用子文件,用 file_path 参数加载:{', '.join(linked)})\n"
|
block += f"\n(可用子文件,用 file_path 参数加载:{', '.join(linked)})\n"
|
||||||
return block
|
return block
|
||||||
|
|
||||||
|
async def _t_list_platform_models(self, sor, p, pid):
|
||||||
|
"""列出平台可用模型(薄壳委托 platform_model_tools 唯一实现)。"""
|
||||||
|
from .platform_model_tools import tool_list_platform_models
|
||||||
|
return await tool_list_platform_models(p, self.org_id or "0")
|
||||||
|
|
||||||
|
async def _t_invoke_model(self, sor, p, pid):
|
||||||
|
"""调用平台模型完成生成类任务(薄壳委托 platform_model_tools 唯一实现)。"""
|
||||||
|
from .platform_model_tools import tool_invoke_model
|
||||||
|
return await tool_invoke_model(
|
||||||
|
p, self.org_id or "0", user_id=self.user_id or "")
|
||||||
|
|
||||||
async def _t_list_packs(self, sor, p, pid):
|
async def _t_list_packs(self, sor, p, pid):
|
||||||
try:
|
try:
|
||||||
from pipeline_core.skill_pack import list_packs
|
from pipeline_core.skill_pack import list_packs
|
||||||
|
|||||||
@ -64,8 +64,12 @@ async def _get_internal_token(org_id, user_id, model_name):
|
|||||||
return token
|
return token
|
||||||
|
|
||||||
|
|
||||||
async def _http_chat(payload, org_id, user_id, model_name):
|
async def _http_chat(payload, org_id, user_id, model_name, timeout: int = 0):
|
||||||
"""POST 本进程推理端点。返回上游响应 dict;失败抛 ValueError(消息真实可行动)。"""
|
"""POST 本进程推理端点。返回上游响应 dict;失败抛 ValueError(消息真实可行动)。
|
||||||
|
|
||||||
|
timeout:客户端等待秒数(0=缺省 330)。异步生成模型(视频等)端点侧最长
|
||||||
|
等 900 秒,调用方须同步放大客户端超时,否则客户端先断连。
|
||||||
|
"""
|
||||||
import aiohttp
|
import aiohttp
|
||||||
|
|
||||||
token = await _get_internal_token(org_id, user_id, model_name)
|
token = await _get_internal_token(org_id, user_id, model_name)
|
||||||
@ -74,11 +78,12 @@ async def _http_chat(payload, org_id, user_id, model_name):
|
|||||||
# (2026-09-04 实测:Authorization 头变成 "***plk-..." 致端点校验失败)
|
# (2026-09-04 实测:Authorization 头变成 "***plk-..." 致端点校验失败)
|
||||||
_BEARER = 'Bea' + 'rer '
|
_BEARER = 'Bea' + 'rer '
|
||||||
headers = {"Authorization": _BEARER + token, "Content-Type": "application/json"}
|
headers = {"Authorization": _BEARER + token, "Content-Type": "application/json"}
|
||||||
|
_total = int(timeout) if timeout and int(timeout) > 0 else 330
|
||||||
try:
|
try:
|
||||||
async with aiohttp.ClientSession() as session:
|
async with aiohttp.ClientSession() as session:
|
||||||
async with session.post(
|
async with session.post(
|
||||||
base + "/chat/completions", headers=headers, json=payload,
|
base + "/chat/completions", headers=headers, json=payload,
|
||||||
timeout=aiohttp.ClientTimeout(total=330, connect=30),
|
timeout=aiohttp.ClientTimeout(total=_total, connect=30),
|
||||||
) as resp:
|
) as resp:
|
||||||
text = await resp.text()
|
text = await resp.text()
|
||||||
status = resp.status
|
status = resp.status
|
||||||
@ -196,3 +201,28 @@ async def llm_call_msgs_native(messages: list, tools: list = None, model: str =
|
|||||||
"content": msg.get("content") or "",
|
"content": msg.get("content") or "",
|
||||||
"tool_calls": msg.get("tool_calls") or [],
|
"tool_calls": msg.get("tool_calls") or [],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def llm_infer(payload: dict, model: str = None, org_id: str = None,
|
||||||
|
user_id: str = None, timeout: int = 0) -> dict:
|
||||||
|
"""通用推理(全能力,2026-09-07):透传任意 payload 到统一推理端点,
|
||||||
|
返回上游响应 dict(OpenAI 兼容 choices;生成类另带 media/output/task_id)。
|
||||||
|
|
||||||
|
与 llm_call* 的区别:不假设 messages 结构,调用方自己组包——供 agent 的
|
||||||
|
invoke_model 工具调用非对话能力(t2i/t2v/i2v/tts/asr 等)。payload 里的
|
||||||
|
messages 由调用方按能力构造;媒体参数按三数组契约(image_files/audio_files/
|
||||||
|
video_files),inference 层的 _normalize_media_aliases 会归一旧别名。
|
||||||
|
|
||||||
|
model 空 = 端点按机构策略缺省模型;owner/机构归属硬校验在治理层执行。
|
||||||
|
timeout:单次调用超时秒数(生成类慢,可传大值;上限 900,端点侧封顶)。
|
||||||
|
失败抛 ValueError(消息真实可行动,直接展示给 agent/用户)。
|
||||||
|
"""
|
||||||
|
body = dict(payload or {})
|
||||||
|
if model:
|
||||||
|
body["model"] = model
|
||||||
|
if timeout:
|
||||||
|
body["_timeout"] = int(timeout)
|
||||||
|
# 客户端等待须覆盖端点侧预算(_timeout 端点封顶 900)+ 余量,否则客户端先断连
|
||||||
|
client_timeout = min(int(timeout or 0), 900) + 60 if timeout else 0
|
||||||
|
return await _http_chat(body, org_id or '0', user_id or '', model or '',
|
||||||
|
timeout=client_timeout)
|
||||||
|
|||||||
179
pipeline_service/platform_model_tools.py
Normal file
179
pipeline_service/platform_model_tools.py
Normal file
@ -0,0 +1,179 @@
|
|||||||
|
"""pipeline_service.platform_model_tools — 平台模型工具(唯一实现,2026-09-07)。
|
||||||
|
|
||||||
|
用户需求:通用助手/产线 agent 可调用平台 owner 机构 + 本机构的全部
|
||||||
|
pipeline-llm 注册模型,按任务与用户输入自动匹配合适模型完成任务。
|
||||||
|
|
||||||
|
- 候选可见性/owner 硬校验/自动选型:pipeline_llm.selection
|
||||||
|
(models_catalog / auto_select_model,与推理治理链同一机构语义)
|
||||||
|
- 实际调用:llm_bridge.llm_infer → /pipeline-llm/api/v1/chat/completions
|
||||||
|
统一推理端点(门禁链 + 记账 + 同步/异步分流全在治理层,本层零旁路)
|
||||||
|
|
||||||
|
消费方(薄壳委托,禁止复制逻辑):
|
||||||
|
- agent_loop_v2.AgentExecutor._t_list_platform_models / _t_invoke_model
|
||||||
|
(会话 agent:驾驶舱 + 通用助手)
|
||||||
|
- agent_loop._exec_agent_tool(产线角色 agent v1)
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
|
||||||
|
logger = logging.getLogger("pipeline.platform_model_tools")
|
||||||
|
|
||||||
|
_CAP_DESC = {
|
||||||
|
"t2t": "文本对话", "i2t": "图像理解", "m2t": "多媒体理解",
|
||||||
|
"t2i": "文生图", "i2v": "图生视频", "t2v": "文生视频",
|
||||||
|
"r2v": "参考生视频", "tts": "语音合成", "asr": "语音识别",
|
||||||
|
"embedding": "向量化", "rerank": "重排序",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def tool_list_platform_models(params, org_id):
|
||||||
|
"""列出平台可用模型(本机构+平台owner机构,含能力类型与描述)。"""
|
||||||
|
cap = str((params or {}).get("capability") or "").strip().lower()
|
||||||
|
try:
|
||||||
|
from pipeline_llm.selection import models_catalog
|
||||||
|
except ImportError:
|
||||||
|
return "FAIL: 模型治理模块(pipeline-llm)未安装,无法列出平台模型。"
|
||||||
|
try:
|
||||||
|
caps = (cap,) if cap else ()
|
||||||
|
models = await models_catalog(org_id or "0", capabilities=caps)
|
||||||
|
except Exception as e:
|
||||||
|
return "ERROR: 列模型失败: " + str(e)[:300]
|
||||||
|
if not models:
|
||||||
|
scope = ("能力 " + cap) if cap else "全部能力"
|
||||||
|
return ("平台当前无可用模型(" + scope + ";范围=本机构+平台owner机构)。"
|
||||||
|
"请在模型治理→模型注册中添加。")
|
||||||
|
lines = ["平台可用模型(" + str(len(models)) + " 个,本机构+平台owner机构):"]
|
||||||
|
for m in models:
|
||||||
|
cap_txt = m.get("capability") or "t2t"
|
||||||
|
cap_cn = _CAP_DESC.get(cap_txt, cap_txt)
|
||||||
|
desc = (":" + m["description"][:80]) if m.get("description") else ""
|
||||||
|
vendor = (" [" + m["vendor"] + "]") if m.get("vendor") else ""
|
||||||
|
lines.append(" - " + m.get("name", "") + vendor
|
||||||
|
+ "(" + cap_cn + "/" + cap_txt + ")" + desc)
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
async def tool_invoke_model(params, org_id, user_id=""):
|
||||||
|
"""调用平台模型完成生成类任务(文生图/视频/语音等非对话能力)。
|
||||||
|
|
||||||
|
流程:解析 task/model/capability/params → 未指定 model 时按 task 自动
|
||||||
|
选型(auto_select_model,候选=本机构+平台owner,capability 空=全能力)
|
||||||
|
→ 组包 payload(messages=[user:task] + 业务参数平铺,媒体三数组契约由
|
||||||
|
inference 层归一)→ llm_infer 统一推理端点 → 提取生成物 URL 返回。
|
||||||
|
失败返回真实可行动错误(禁静默粉饰)。
|
||||||
|
"""
|
||||||
|
p = params or {}
|
||||||
|
task = str(p.get("task") or "").strip()
|
||||||
|
if not task:
|
||||||
|
return "FAIL: invoke_model 需要 task(任务描述/提示词)"
|
||||||
|
model = str(p.get("model") or "").strip()
|
||||||
|
cap = str(p.get("capability") or "").strip().lower()
|
||||||
|
|
||||||
|
# 业务参数(JSON 字符串 → dict)
|
||||||
|
biz = {}
|
||||||
|
raw_params = p.get("params")
|
||||||
|
if isinstance(raw_params, dict):
|
||||||
|
biz = dict(raw_params)
|
||||||
|
elif isinstance(raw_params, str) and raw_params.strip():
|
||||||
|
try:
|
||||||
|
parsed = json.loads(raw_params)
|
||||||
|
if isinstance(parsed, dict):
|
||||||
|
biz = parsed
|
||||||
|
except Exception:
|
||||||
|
return ("FAIL: params 不是合法 JSON 对象。媒体输入用三数组:"
|
||||||
|
"image_files/audio_files/video_files(URL或base64数组)")
|
||||||
|
|
||||||
|
# 自动选型:未显式指定 model 时按 task 匹配
|
||||||
|
if not model:
|
||||||
|
try:
|
||||||
|
from pipeline_llm.selection import auto_select_model
|
||||||
|
# capability 空 = 全能力候选,让匹配器按任务选最合适能力
|
||||||
|
model, reason = await auto_select_model(
|
||||||
|
org_id or "0", task, user_id=user_id or "",
|
||||||
|
capabilities=(cap if cap else ""))
|
||||||
|
if model:
|
||||||
|
logger.info("invoke_model 自动选型: %s(%s)org=%s",
|
||||||
|
model, reason, org_id or "0")
|
||||||
|
else:
|
||||||
|
return ("FAIL: 未能自动匹配到合适模型(" + str(reason) + ")。"
|
||||||
|
"可先用 list_platform_models 查看可用模型,"
|
||||||
|
"再用 model 参数指定。")
|
||||||
|
except ImportError:
|
||||||
|
return ("FAIL: 模型治理模块(pipeline-llm)未安装,"
|
||||||
|
"无法自动选型或调用平台模型。")
|
||||||
|
except Exception as e:
|
||||||
|
return "ERROR: 自动选型失败: " + str(e)[:300]
|
||||||
|
|
||||||
|
# 组包:生成类模型从 messages 末条取 prompt 文本(inference._last_prompt_text)
|
||||||
|
payload = {"messages": [{"role": "user", "content": task}]}
|
||||||
|
payload.update(biz)
|
||||||
|
try:
|
||||||
|
from .llm_bridge import llm_infer
|
||||||
|
data = await llm_infer(
|
||||||
|
payload, model=model, org_id=org_id or "0",
|
||||||
|
user_id=user_id or "", timeout=600)
|
||||||
|
except Exception as e:
|
||||||
|
return "FAIL: 模型「" + model + "」调用失败:" + str(e)[:400]
|
||||||
|
|
||||||
|
# 提取生成物:media(本地持久 URL,优先)> choices[0].message.content
|
||||||
|
result_url = ""
|
||||||
|
media = data.get("media") if isinstance(data, dict) else None
|
||||||
|
if isinstance(media, dict):
|
||||||
|
result_url = (media.get("video") or media.get("image")
|
||||||
|
or media.get("audio") or media.get("glb")
|
||||||
|
or media.get("3dmodel") or "")
|
||||||
|
if not result_url and isinstance(data, dict):
|
||||||
|
try:
|
||||||
|
result_url = ((data.get("choices") or [{}])[0]
|
||||||
|
.get("message") or {}).get("content") or ""
|
||||||
|
except Exception:
|
||||||
|
result_url = ""
|
||||||
|
result_url = str(result_url or "").strip()
|
||||||
|
if result_url:
|
||||||
|
return ("OK: 模型「" + model + "」生成完成。产物地址:" + result_url)
|
||||||
|
# 无产物 URL:回原始输出摘要(真实,不粉饰)
|
||||||
|
try:
|
||||||
|
summary = json.dumps(data, ensure_ascii=False, default=str)[:500]
|
||||||
|
except Exception:
|
||||||
|
summary = str(data)[:500]
|
||||||
|
return ("模型「" + model + "」已调用但未返回可识别的产物地址。原始输出:"
|
||||||
|
+ summary)
|
||||||
|
|
||||||
|
|
||||||
|
# 平台模型工具的 v1 形态定义(AGENT_TOOLS 同款 {name,description,params}),
|
||||||
|
# 供 agent_loop.role_agent_run 追加到角色 agent 工具清单。
|
||||||
|
PLATFORM_MODEL_TOOLS_V1 = [
|
||||||
|
{
|
||||||
|
"name": "list_platform_models",
|
||||||
|
"description": ("列出平台当前可用的模型(本机构+平台owner机构的模型,含能力类型:"
|
||||||
|
"t2t对话/i2t图像理解/t2i文生图/t2v文生视频/i2v图生视频/tts语音合成/"
|
||||||
|
"asr语音识别等)。需要调用非对话能力(生图/视频/语音)前先查模型时用。"
|
||||||
|
"capability 参数可按能力过滤(如 t2i)"),
|
||||||
|
"params": {"capability": "可选:能力类型过滤(如 t2i/t2v/tts),空=全部"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "invoke_model",
|
||||||
|
"description": ("调用平台模型完成生成类任务(文生图/图生视频/文生视频/语音合成等"
|
||||||
|
"非对话能力;写代码/写文档等对话类工作由你自己完成,不要用本工具)。"
|
||||||
|
"model 可空——空时平台根据 task 自动匹配最合适的可用模型。"
|
||||||
|
"生成产物返回本地持久 URL,产物需要落盘时用 write_file 记录 URL"),
|
||||||
|
"params": {
|
||||||
|
"task": "任务描述/提示词(必填,如「一只在月球上弹吉他的猫」)",
|
||||||
|
"model": "可选:模型注册名(空=按任务自动匹配)",
|
||||||
|
"capability": "可选:能力类型(t2i/t2v/i2v/tts/asr等,自动匹配时用于过滤候选)",
|
||||||
|
"params": ("可选:业务参数 JSON 字符串。媒体输入用三数组契约:"
|
||||||
|
"image_files/audio_files/video_files(值为公网URL或base64的数组);"
|
||||||
|
"生成参数如 resolution/duration/size 按模型文档"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
async def exec_platform_model_tool(tool, params, org_id, user_id=""):
|
||||||
|
"""v1/v2 统一分发入口(薄壳)。"""
|
||||||
|
if tool == "list_platform_models":
|
||||||
|
return await tool_list_platform_models(params, org_id)
|
||||||
|
if tool == "invoke_model":
|
||||||
|
return await tool_invoke_model(params, org_id, user_id=user_id)
|
||||||
|
return "未实现: " + str(tool)
|
||||||
Loading…
x
Reference in New Issue
Block a user