feat(agent): 五级作用域工具解析+patch_file+原生视觉上传构造
1. tool_sources.py(新): global/org/pipeline/role/project 五级工具作用域解析
- 策略表 pipeline_tool_policies allow/deny(配套 models+CRUD json)
- capability 两层语义: 映射手册(all/概念技能,含global层) + 声明注入
(仅org/pipeline/role/project/user层)——防 generic 会话经 global 概念
技能拿到~70个产线工具,击穿七层隔离(单测 29/29 含此回归)
- role 白名单是过滤器非添加器,且作用于 base+capability 全集
2. agent_config: GENERAL_TOOLS 加 patch_file(定点替换,唯一性校验);
write_file 描述引导改文件优先用 patch_file
3. upload_tools: is_image + build_image_parts(OpenAI多模态data URL,
8MB/张+4张/条上限,超限显式告知不静默丢)
4. agent_chat/agent_chat_generic: 图片分流不进文本上下文,
image_paths 传 gateway
5. load_path: 策略管理页注册
This commit is contained in:
parent
43f3ad48e5
commit
b399663863
47
json/pipeline_tool_policies_list.json
Normal file
47
json/pipeline_tool_policies_list.json
Normal file
@ -0,0 +1,47 @@
|
||||
{
|
||||
"tblname": "pipeline_tool_policies",
|
||||
"title": "工具作用域策略",
|
||||
"params": {
|
||||
"sortby": [
|
||||
"scope asc",
|
||||
"scope_id asc",
|
||||
"created_at desc"
|
||||
],
|
||||
"confidential_fields": [],
|
||||
"browserfields": {
|
||||
"exclouded": [],
|
||||
"alters": {
|
||||
"scope": {
|
||||
"uitype": "code",
|
||||
"data": [
|
||||
{"value": "org", "text": "机构"},
|
||||
{"value": "pipeline", "text": "产线"},
|
||||
{"value": "role", "text": "角色"},
|
||||
{"value": "project", "text": "项目"}
|
||||
]
|
||||
},
|
||||
"effect": {
|
||||
"uitype": "code",
|
||||
"data": [
|
||||
{"value": "allow", "text": "允许"},
|
||||
{"value": "deny", "text": "禁用"}
|
||||
]
|
||||
},
|
||||
"status": {
|
||||
"uitype": "code",
|
||||
"data": [
|
||||
{"value": "active", "text": "生效"},
|
||||
{"value": "invalid", "text": "停用"}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"editexclouded": [
|
||||
"id",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
"created_by"
|
||||
],
|
||||
"editable": {}
|
||||
}
|
||||
}
|
||||
106
models/pipeline_tool_policies.json
Normal file
106
models/pipeline_tool_policies.json
Normal file
@ -0,0 +1,106 @@
|
||||
{
|
||||
"summary": [
|
||||
{
|
||||
"name": "pipeline_tool_policies",
|
||||
"title": "工具作用域策略",
|
||||
"primary": [
|
||||
"id"
|
||||
],
|
||||
"catelog": "entity"
|
||||
}
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"name": "id",
|
||||
"title": "主键ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "no"
|
||||
},
|
||||
{
|
||||
"name": "scope",
|
||||
"title": "作用域(org/pipeline/role/project)",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "no",
|
||||
"default": "org"
|
||||
},
|
||||
{
|
||||
"name": "scope_id",
|
||||
"title": "作用域ID(机构ID/产线ID/产线:角色/项目ID)",
|
||||
"type": "str",
|
||||
"length": 128,
|
||||
"nullable": "no",
|
||||
"default": ""
|
||||
},
|
||||
{
|
||||
"name": "tool_name",
|
||||
"title": "工具名",
|
||||
"type": "str",
|
||||
"length": 64,
|
||||
"nullable": "no"
|
||||
},
|
||||
{
|
||||
"name": "effect",
|
||||
"title": "效果(allow允许/deny禁用)",
|
||||
"type": "str",
|
||||
"length": 16,
|
||||
"nullable": "no",
|
||||
"default": "allow"
|
||||
},
|
||||
{
|
||||
"name": "reason",
|
||||
"title": "配置原因(审计用)",
|
||||
"type": "str",
|
||||
"length": 500,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "status",
|
||||
"title": "状态(active/invalid)",
|
||||
"type": "str",
|
||||
"length": 16,
|
||||
"nullable": "no",
|
||||
"default": "active"
|
||||
},
|
||||
{
|
||||
"name": "created_by",
|
||||
"title": "创建人",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "created_at",
|
||||
"title": "创建时间",
|
||||
"type": "timestamp",
|
||||
"nullable": "no"
|
||||
},
|
||||
{
|
||||
"name": "updated_at",
|
||||
"title": "更新时间",
|
||||
"type": "timestamp",
|
||||
"nullable": "no"
|
||||
}
|
||||
],
|
||||
"indexes": [
|
||||
{
|
||||
"name": "idx_scope",
|
||||
"idxtype": "index",
|
||||
"idxfields": [
|
||||
"scope",
|
||||
"scope_id",
|
||||
"status"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "uk_scope_tool",
|
||||
"idxtype": "unique",
|
||||
"idxfields": [
|
||||
"scope",
|
||||
"scope_id",
|
||||
"tool_name"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -310,10 +310,17 @@ GENERAL_TOOLS = [
|
||||
),
|
||||
ToolDefinition(
|
||||
name="write_file",
|
||||
description="写入文件到工作空间（自动创建父目录）",
|
||||
description="写入文件到工作空间（自动创建父目录）。整文件覆盖——修改已有文件优先用 patch_file（定点替换），只有新建或全文重写才用本工具",
|
||||
parameters={"path": "相对路径", "content": "文件内容"},
|
||||
category="file",
|
||||
),
|
||||
ToolDefinition(
|
||||
name="patch_file",
|
||||
description="定点修改工作空间中的文本文件（找原文片段→替换新文本）。old_string 必须在文件中唯一出现（出现0次或多次都拒绝，多次时请带更多上下文）；old_string 须与文件内容完全一致（含空白/缩进，先用 read_file 核对）。新建文件用 write_file",
|
||||
parameters={"path": "相对路径", "old_string": "要替换的原文片段（须唯一）", "new_string": "替换后的新文本（空字符串=删除该片段）", "replace_all": "可选：true=替换全部出现（默认false须唯一）"},
|
||||
required=["path", "old_string", "new_string"],
|
||||
category="file",
|
||||
),
|
||||
ToolDefinition(
|
||||
name="list_files",
|
||||
description="列出工作空间目录内容",
|
||||
|
||||
218
pipeline_core/tool_sources.py
Normal file
218
pipeline_core/tool_sources.py
Normal file
@ -0,0 +1,218 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""tool_sources.py — 五级作用域工具解析（2026-09-10）
|
||||
|
||||
用户需求：全局/机构/产线/角色/项目 五个维度各自注册不同工具；产线特有工具
|
||||
按产线区分。此前只有两层（GENERAL_TOOLS 全局 + PipelineAbility 按产线），
|
||||
机构/角色/项目维度缺失，且 v2 会话根本没用 capability frontmatter 驱动的工具
|
||||
（resolve_capability_tools 只有 v1 agent_loop 在用）。
|
||||
|
||||
五级作用域（优先级低→高，同技能 SCOPE_PRIORITY 语义）：
|
||||
|
||||
| 级 | 工具来源 | 隔离机制 |
|
||||
|----|---------|---------|
|
||||
| 1 global | GENERAL_TOOLS（core 代码） | generic 会话剔除 project/data 类 |
|
||||
| 2 org | 机构装的技能包 capability 工具 + 策略表 allow/deny | 技能 get_merged 按 org 隔离（orgs/{org}/ 物理目录） |
|
||||
| 3 pipeline | PipelineAbility.tools + 产线技能 capability 工具 | 注册表按 pipeline_id；generic 不挂 |
|
||||
| 4 role | RoleSpec.tools（角色声明的工具名）+ 角色技能 capability 工具 | get_role_spec(pipeline_id, role) 按产线隔离 |
|
||||
| 5 project | 策略表 allow/deny（项目级微调） | 策略行 scope_id=project_id |
|
||||
|
||||
**关键设计：技能作用域即工具作用域。** 技能包里 frontmatter 声明
|
||||
`capability: xxx_capability` + `tools: [a, b]`，`get_merged` 已按六级可见性
|
||||
过滤（global→org→pipeline→role→project→user），所以机构 A 装的包只在 A 的
|
||||
merged 里出现 → 其 capability 工具只对 A 可见。复用已验证的技能隔离，不另造一套。
|
||||
|
||||
**执行层强制门禁**：resolve 返回 allowed 集合，AgentExecutor._execute_tool
|
||||
先查集合再分发——LLM 幻觉调用不属于本会话作用域的工具名（如 generic 会话喊
|
||||
create_task、A 产线喊 B 产线工具）时直接拒绝并给出可行动提示，而不是只有
|
||||
schema 里不给看（schema 防线挡不住幻觉，历史踩过：别名绕过、能力包同名工具
|
||||
绕过通用层安全门）。
|
||||
"""
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Optional, Set, Tuple
|
||||
|
||||
logger = logging.getLogger("pipeline.tool_sources")
|
||||
|
||||
# 策略表 effect 取值
|
||||
EFFECT_ALLOW = "allow"
|
||||
EFFECT_DENY = "deny"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolScopeResult:
|
||||
"""五级解析结果。"""
|
||||
tools: List = field(default_factory=list) # [ToolDefinition] 最终 schema 来源
|
||||
allowed: Set[str] = field(default_factory=set) # 执行层门禁允许集（工具名）
|
||||
capability_tools: List[str] = field(default_factory=list) # frontmatter 驱动的工具名
|
||||
trace: List[str] = field(default_factory=list) # 诊断：每级命中/剔除记录
|
||||
|
||||
|
||||
async def load_policies(sor, scope: str, scope_id: str) -> List[Tuple[str, str]]:
|
||||
"""读策略表某作用域的 (tool_name, effect) 列表。表缺失/查询失败 → 空（不阻断会话）。"""
|
||||
if not scope_id:
|
||||
return []
|
||||
try:
|
||||
recs = await sor.sqlExe(
|
||||
"SELECT tool_name, effect FROM pipeline_tool_policies "
|
||||
"WHERE scope=${s}$ AND scope_id=${sid}$ AND status='active' "
|
||||
"ORDER BY created_at ASC",
|
||||
{"s": scope, "sid": scope_id})
|
||||
return [(getattr(r, "tool_name", "") or "", getattr(r, "effect", "") or "")
|
||||
for r in (recs or []) if getattr(r, "tool_name", "")]
|
||||
except Exception as e:
|
||||
# 表不存在（未跑迁移）或查询失败：记 WARN 并按「无策略」继续，
|
||||
# 绝不因策略表问题让整个会话起不来（静默吞错的反面：有日志、有降级语义）
|
||||
logger.warning(f"tool policy load failed scope={scope} id={scope_id}: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def _apply_policies(names: Set[str], policies: List[Tuple[str, str]],
|
||||
trace: List[str], level: str) -> Set[str]:
|
||||
"""按序应用 allow/deny。deny 优先于后续 allow（同一级内后写的 deny 生效）。"""
|
||||
out = set(names)
|
||||
for tool_name, effect in policies:
|
||||
if effect == EFFECT_DENY:
|
||||
if tool_name in out:
|
||||
out.discard(tool_name)
|
||||
trace.append(f"{level}: deny {tool_name}")
|
||||
elif effect == EFFECT_ALLOW:
|
||||
if tool_name not in out:
|
||||
out.add(tool_name)
|
||||
trace.append(f"{level}: allow {tool_name}")
|
||||
return out
|
||||
|
||||
|
||||
# capability 声明只在「会话上下文层」生效的技能 scope（global 层排除，见下）
|
||||
_CAPABILITY_DECLARING_SCOPES = ("org", "pipeline", "role", "project", "user")
|
||||
|
||||
|
||||
def _collect_capability_tools(merged_skills) -> Tuple[List[str], Dict[str, str]]:
|
||||
"""从可见技能 frontmatter 收集 capability 工具（两层语义，与 v1 一致）。
|
||||
|
||||
merged_skills: SkillLoader.get_merged() 结果（已按六级可见性过滤）。
|
||||
纯读 Skill.capability / Skill.tools（core 自有数据结构），不依赖 service 层——
|
||||
避免 core→service 分层倒置。执行（exec_capability_tool）仍在 service。
|
||||
|
||||
⚠️ 两层语义（2026-09-10 隔离走查修复，别简化成一层）：
|
||||
1. 「capability 含哪些工具」映射：从全部可见技能收集（global 层的
|
||||
all/feature、all/bug 等概念规范技能 = 映射手册）；
|
||||
2. 「本会话需要哪些 capability」：只认 org/pipeline/role/project/user 层
|
||||
技能的声明——**global 层声明不作为注入依据**。
|
||||
为什么：all/ 的 10 个概念技能声明了 ~70 个产线工具（propose_feature/
|
||||
claim_task/create_project…），generic 会话的 get_merged 只加载 global 技能，
|
||||
若一层直取，通用助手会拿到全部产线概念工具——击穿 2026-09-05 七层隔离。
|
||||
v1 同款设计：角色 frontmatter 声明「我需要 capability」+ 规范技能声明
|
||||
「capability 含哪些工具」。
|
||||
|
||||
返回 (工具名列表, 工具名→capability 映射)
|
||||
"""
|
||||
cap_map: Dict[str, List[str]] = {} # capability → 工具清单（全部可见技能）
|
||||
declared: Set[str] = set() # 本会话声明需要的 capability（非 global 层）
|
||||
for skill in (merged_skills or {}).values():
|
||||
cap = getattr(skill, "capability", "") or ""
|
||||
if not cap:
|
||||
continue
|
||||
tools = getattr(skill, "tools", []) or []
|
||||
bucket = cap_map.setdefault(cap, [])
|
||||
for t in tools:
|
||||
if t not in bucket:
|
||||
bucket.append(t)
|
||||
scope = getattr(skill, "scope", "global") or "global"
|
||||
if scope in _CAPABILITY_DECLARING_SCOPES:
|
||||
declared.add(cap)
|
||||
names, owner = [], {}
|
||||
for cap in sorted(declared): # 只展开被声明的 capability
|
||||
for t in cap_map.get(cap, []):
|
||||
if t not in owner:
|
||||
names.append(t)
|
||||
owner[t] = cap
|
||||
return names, owner
|
||||
|
||||
|
||||
async def resolve_scoped_tools(
|
||||
sor,
|
||||
base_tools: List,
|
||||
merged_skills=None,
|
||||
generic: bool = False,
|
||||
org_id: str = "",
|
||||
user_id: str = "",
|
||||
pipeline_id: str = "",
|
||||
role: str = "",
|
||||
project_id: str = "",
|
||||
) -> ToolScopeResult:
|
||||
"""五级作用域解析主入口。
|
||||
|
||||
base_tools: 已按 generic 裁剪过的 GENERAL_TOOLS（load_agent_config 产物），
|
||||
即第 1 级 global 的结果。
|
||||
merged_skills: SkillLoader.get_merged(...) 结果，None 时跳过 capability 层。
|
||||
返回 ToolScopeResult（tools 含 schema 定义，allowed 是执行门禁集合）。
|
||||
"""
|
||||
res = ToolScopeResult()
|
||||
tools = list(base_tools or [])
|
||||
by_name = {t.name: t for t in tools}
|
||||
allowed = set(by_name.keys())
|
||||
trace = res.trace
|
||||
trace.append(f"global: {len(allowed)} 工具（generic={generic}）")
|
||||
|
||||
# ── 2. org：策略表 allow/deny（capability 工具在第 6 步统一收集）──
|
||||
org_policies = await load_policies(sor, "org", org_id)
|
||||
if org_policies:
|
||||
allowed = _apply_policies(allowed, org_policies, trace, f"org({org_id})")
|
||||
|
||||
# ── 3. pipeline：能力包工具（generic 不挂；已由 load_agent_config 合并进 base_tools，
|
||||
# 这里只补策略表级的产线 deny/allow）──
|
||||
pl_policies = await load_policies(sor, "pipeline", pipeline_id) if not generic else []
|
||||
if pl_policies:
|
||||
allowed = _apply_policies(allowed, pl_policies, trace, f"pipeline({pipeline_id})")
|
||||
if not generic and pipeline_id:
|
||||
try:
|
||||
from .ability import get_ability
|
||||
ab = get_ability(pipeline_id)
|
||||
if ab:
|
||||
trace.append(f"pipeline({pipeline_id}): 能力包 {len(ab.tools)} 工具")
|
||||
except Exception as e:
|
||||
logger.warning(f"ability lookup failed: {e}")
|
||||
|
||||
# ── 4. capability 工具（技能 frontmatter 驱动，作用域=技能可见性）。
|
||||
# 必须在 role 收窄**之前**并入——角色白名单要作用于全集(base+capability)，
|
||||
# 否则角色声明之外的技能工具会绕过白名单（2026-09-10 单测实抓）。──
|
||||
cap_names: List[str] = []
|
||||
if merged_skills:
|
||||
cap_names, cap_owner = _collect_capability_tools(merged_skills)
|
||||
if cap_names:
|
||||
res.capability_tools = cap_names
|
||||
allowed |= set(cap_names)
|
||||
trace.append(f"capability(技能声明): {len(cap_names)} 工具 "
|
||||
f"[{', '.join(cap_names[:8])}{'...' if len(cap_names) > 8 else ''}]")
|
||||
|
||||
# ── 5. role：RoleSpec.tools 是角色声明的工具名白名单（非空则收窄）──
|
||||
if role and pipeline_id and not generic:
|
||||
try:
|
||||
from .ability import get_role_spec
|
||||
spec = get_role_spec(pipeline_id, role)
|
||||
if spec and getattr(spec, "tools", None):
|
||||
declared = set(spec.tools)
|
||||
before = len(allowed)
|
||||
# 角色白名单只收窄「能力包/通用」里角色未声明的部分；
|
||||
# ask_user/todo/memory 等交互基础工具始终保留（否则角色 agent 无法提问）
|
||||
keep_always = {"ask_user", "todo", "memory", "load_skill",
|
||||
"read_file", "write_file", "list_files"}
|
||||
allowed = {n for n in allowed if n in declared or n in keep_always}
|
||||
trace.append(f"role({role}): 声明 {len(declared)} 工具，"
|
||||
f"{before}→{len(allowed)}")
|
||||
except Exception as e:
|
||||
logger.warning(f"role spec lookup failed: {e}")
|
||||
role_policies = await load_policies(sor, "role", f"{pipeline_id}:{role}") if role else []
|
||||
if role_policies:
|
||||
allowed = _apply_policies(allowed, role_policies, trace, f"role({role})")
|
||||
|
||||
# ── 6. project：项目级微调 ──
|
||||
pj_policies = await load_policies(sor, "project", project_id)
|
||||
if pj_policies:
|
||||
allowed = _apply_policies(allowed, pj_policies, trace, f"project({project_id})")
|
||||
|
||||
# ── 最终 schema：base_tools 里被 deny 的剔除；capability 工具无 ToolDefinition
|
||||
# （schema 由 capability_tools.TOOL_SCHEMAS 提供，执行走 exec_capability_tool）
|
||||
res.tools = [t for t in tools if t.name in allowed]
|
||||
res.allowed = allowed
|
||||
return res
|
||||
@ -94,6 +94,63 @@ def save_uploads(target_dir, uploads):
|
||||
return saved
|
||||
|
||||
|
||||
IMAGE_EXTS = ('.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp')
|
||||
IMAGE_MIME = {'.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg',
|
||||
'.gif': 'image/gif', '.webp': 'image/webp', '.bmp': 'image/bmp'}
|
||||
IMAGE_MAX_BYTES = 8 * 1024 * 1024 # 单图上限 8MB（base64 后约 10.7MB）
|
||||
IMAGE_MAX_COUNT = 4 # 单条消息最多 4 张
|
||||
|
||||
|
||||
def is_image(name: str) -> bool:
|
||||
"""按扩展名判断是否图片（原生视觉链路用）。"""
|
||||
return os.path.splitext((name or '').lower())[1] in IMAGE_EXTS
|
||||
|
||||
|
||||
def build_image_parts(paths):
|
||||
"""把上传图片构造为 OpenAI 多模态 content parts（data URL）。
|
||||
|
||||
paths: [(abs_path, filename)]
|
||||
返回 (parts, notes)：
|
||||
parts = [{"type":"image_url","image_url":{"url":"data:image/png;base64,..."}}]
|
||||
notes = 注入 prompt 的文本说明（图片清单/超限拒绝——绝不静默丢弃，
|
||||
沿用 extract_text「显式截断告知」的同一原则）。
|
||||
|
||||
上限策略：单图 >8MB 或超过 4 张 → 超限的图不进 parts，notes 显式告知
|
||||
文件名+原因+替代手段（invoke_model i2t 或压缩后重传），agent 可转达用户。
|
||||
"""
|
||||
import base64
|
||||
parts, notes, used = [], [], 0
|
||||
for src, name in (paths or []):
|
||||
ext = os.path.splitext((name or '').lower())[1]
|
||||
mime = IMAGE_MIME.get(ext)
|
||||
if not mime:
|
||||
notes.append(f"图片 {name}：不支持的格式（支持 {'/'.join(IMAGE_EXTS)}），未注入。")
|
||||
continue
|
||||
try:
|
||||
size = os.path.getsize(src)
|
||||
except OSError:
|
||||
notes.append(f"图片 {name}：文件不可读，未注入。")
|
||||
continue
|
||||
if size > IMAGE_MAX_BYTES:
|
||||
notes.append(f"图片 {name}：超过 {IMAGE_MAX_BYTES // 1024 // 1024}MB 上限"
|
||||
f"（实际 {size // 1024 // 1024}MB），未注入。请压缩后重传。")
|
||||
continue
|
||||
if used >= IMAGE_MAX_COUNT:
|
||||
notes.append(f"图片 {name}：超过单条消息 {IMAGE_MAX_COUNT} 张上限，未注入。")
|
||||
continue
|
||||
try:
|
||||
with open(src, 'rb') as f:
|
||||
b64 = base64.b64encode(f.read()).decode('ascii')
|
||||
except Exception:
|
||||
notes.append(f"图片 {name}：读取失败，未注入。")
|
||||
continue
|
||||
parts.append({"type": "image_url",
|
||||
"image_url": {"url": f"data:{mime};base64,{b64}"}})
|
||||
notes.append(f"图片 {name}（已注入，你可以直接看到内容）")
|
||||
used += 1
|
||||
return parts, notes
|
||||
|
||||
|
||||
def build_file_context(items):
|
||||
"""items: [(filename, relpath, preview, total, truncated, is_binary)]
|
||||
|
||||
|
||||
@ -11,6 +11,8 @@ MOD = "pipeline_core"
|
||||
|
||||
PATHS_LOGINED = [
|
||||
f"/{MOD}",
|
||||
# 工具作用域策略管理页（2026-09-10，五级作用域工具解析的 allow/deny 配置）
|
||||
f"/{MOD}/pipeline_tool_policies/*",
|
||||
]
|
||||
|
||||
|
||||
|
||||
@ -37,9 +37,10 @@ if action == 'send_message':
|
||||
|
||||
# 处理用户上传的文件（multipart file 字段 → web_path）：
|
||||
# 统一走 pipeline_core.upload_tools——截断显式告知、落盘位置=agent 可读位置。
|
||||
from pipeline_core.upload_tools import extract_text, resolve_upload_dir, save_uploads, build_file_context
|
||||
from pipeline_core.upload_tools import extract_text, resolve_upload_dir, save_uploads, build_file_context, is_image
|
||||
_items = [] # build_file_context 入参
|
||||
_uploads = [] # [(src_abs, filename)]
|
||||
_img_uploads = [] # 图片上传（原生视觉，2026-09-10）：不进文本上下文，走多模态注入
|
||||
_fval = (params_kw or {}).get('file')
|
||||
_fpaths = _fval if isinstance(_fval, list) else ([_fval] if _fval else [])
|
||||
for _fp in _fpaths:
|
||||
@ -115,6 +116,12 @@ if action == 'send_message':
|
||||
_saved = save_uploads(_udir, _uploads)
|
||||
_byname = {os.path.basename(src): n for (src, _n), (n, _p) in zip(_uploads, _saved)}
|
||||
for _src, _name in _uploads:
|
||||
if is_image(_name):
|
||||
# 图片走原生视觉（gateway.build_image_parts），不进文本上下文——
|
||||
# 避免注入「二进制无法读取」与「图片已注入」矛盾提示。
|
||||
# 仍随 save_uploads 落盘（agent 可再用 invoke_model i2t 处理）
|
||||
_img_uploads.append((_src, _name))
|
||||
continue
|
||||
_preview, _total, _trunc = extract_text(_src, _name)
|
||||
_isbin = (not _preview and _total == 0)
|
||||
_relp = (_rel + _byname.get(_name, _name)) if _name in _byname else ''
|
||||
@ -134,7 +141,7 @@ if action == 'send_message':
|
||||
|
||||
async def agent_stream():
|
||||
_base = entire_url("/")
|
||||
async for chunk in gateway.run_message("web", uid, prompt, base_url=_base, session_id=session_id, pipeline_id=pipeline_id, model_id=model_id):
|
||||
async for chunk in gateway.run_message("web", uid, prompt, base_url=_base, session_id=session_id, pipeline_id=pipeline_id, model_id=model_id, image_paths=_img_uploads or None):
|
||||
data = json.loads(chunk)
|
||||
t = data.get('type', '')
|
||||
|
||||
|
||||
@ -36,8 +36,9 @@ if action == 'send_message':
|
||||
import os
|
||||
from ahserver.filestorage import FileStorage
|
||||
from sqlor.dbpools import DBPools
|
||||
from pipeline_core.upload_tools import extract_text, resolve_upload_dir, save_uploads, build_file_context
|
||||
from pipeline_core.upload_tools import extract_text, resolve_upload_dir, save_uploads, build_file_context, is_image
|
||||
_uploads = []
|
||||
_img_uploads = [] # 图片上传（原生视觉，2026-09-10）：不进文本上下文，走多模态注入
|
||||
_fval = (params_kw or {}).get('file')
|
||||
_fpaths = _fval if isinstance(_fval, list) else ([_fval] if _fval else [])
|
||||
for _fp in _fpaths:
|
||||
@ -55,6 +56,12 @@ if action == 'send_message':
|
||||
_byname = {os.path.basename(src): n for (src, _n), (n, _p) in zip(_uploads, _saved)}
|
||||
_items = []
|
||||
for _src, _name in _uploads:
|
||||
if is_image(_name):
|
||||
# 图片走原生视觉（gateway.build_image_parts），不进文本上下文——
|
||||
# 否则会注入「二进制文件无法读取文本」与「图片已注入」矛盾提示。
|
||||
# 仍随 save_uploads 落盘工作空间（agent 可再用 invoke_model i2t 处理）
|
||||
_img_uploads.append((_src, _name))
|
||||
continue
|
||||
_preview, _total, _trunc = extract_text(_src, _name)
|
||||
_isbin = (not _preview and _total == 0)
|
||||
_relp = (_rel + _byname.get(_name, _name)) if _name in _byname else ''
|
||||
@ -66,7 +73,7 @@ if action == 'send_message':
|
||||
pass
|
||||
|
||||
async def agent_stream():
|
||||
async for chunk in gateway.run_message("web", uid, prompt, generic=True, model_id=model_id):
|
||||
async for chunk in gateway.run_message("web", uid, prompt, generic=True, model_id=model_id, image_paths=_img_uploads or None):
|
||||
data = json.loads(chunk)
|
||||
t = data.get('type', '')
|
||||
if t == 'tool_call':
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user