84bef76401
- 新增 file_read.py: v1/v2 共用统一读取(分页续读+docx/pdf解析+显式截断告知) 根治立项书76088字符只读到30000(v2)/12000硬截无提示(v1)的真实翻车 - 新增 web_tools.py: web_search(Bing RSS优先+HTML兜底)/fetch_url(超长落盘webcache+read_file续读) SSRF三层防护(公网域名/DNS私址拒绝/重定向逐跳)+不可信数据标注; 修复 aiohttp content.read(n) 对 chunked 响应提前返回半截的坑(改 resp.read()) - 新增 db_query.py: query_project_data 只读白名单查询(强制project_id参数化+where/order_by注入校验+审计) - agent_loop.py(v1角色agent): read_file 12000硬截→共享file_read分页; AGENT_TOOLS 加三工具(补required防全参必填); _run_shell 加 strict 档(通用会话:不挂平台目录+可写根收窄到用户专属目录); PM/QC/RETRO prompt 工具清单同步 - agent_loop_v2.py(v2会话agent): _t_read_file 改走共享模块; 新增三工具 handler; generic run_command 强制strict+无bwrap拒绝
141 lines
5.4 KiB
Python
141 lines
5.4 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""file_read.py - 统一文件读取与信息摄入（v1 角色 agent / v2 会话 agent 共用）
|
||
|
||
对齐 Hermes 的信息摄入能力，根治「读长文档只看到开头就以为读全了」
|
||
（立项书生成功能架构图只读到 30000 字符的实录；v1 角色 agent 旧实现更差：
|
||
硬截 12000 字符、无截断提示、无 offset 续读、不支持 docx）。
|
||
|
||
统一提供：
|
||
- 分页续读：offset + limit，返回 next_offset 与续读指引，agent 可逐段读完全文
|
||
- 文档解析：docx（zipfile 标准库，段落级）/ pdf（pypdf/PyPDF2/pdfminer 可选）
|
||
- 显式截断告知：返回全文总字符数 + 截断标志，绝不静默截断
|
||
|
||
一处实现，v1（agent_loop._exec_agent_tool）与 v2（agent_loop_v2._t_read_file）共用。
|
||
内部 agent（platform_ability）不走本模块，互不影响。
|
||
"""
|
||
import os
|
||
import re
|
||
import zipfile
|
||
|
||
DEFAULT_LIMIT = 30000
|
||
|
||
TEXT_EXTS = ('.txt', '.md', '.json', '.csv', '.py', '.log', '.yaml', '.yml',
|
||
'.xml', '.html', '.htm', '.ini', '.rst', '.conf', '.cfg', '.sh',
|
||
'.js', '.ts', '.sql', '.toml', '')
|
||
DOCX_EXTS = ('.docx',)
|
||
PDF_EXTS = ('.pdf',)
|
||
|
||
_HTML_ENTITIES = ((' ', ' '), ('<', '<'), ('>', '>'),
|
||
('"', '"'), (''', "'"), (''', "'"),
|
||
('&', '&'))
|
||
|
||
|
||
def _decode_entities(s):
|
||
for k, v in _HTML_ENTITIES:
|
||
s = s.replace(k, v)
|
||
return s
|
||
|
||
|
||
def extract_docx_text(path):
|
||
"""docx → 纯文本（段落级，保留换行）。失败返回 ''。"""
|
||
try:
|
||
with zipfile.ZipFile(path) as z:
|
||
xml = z.read('word/document.xml').decode('utf-8', errors='ignore')
|
||
except Exception:
|
||
return ''
|
||
lines = []
|
||
for para in re.split(r'</w:p>', xml):
|
||
ts = re.findall(r'<w:t[^>]*>(.*?)</w:t>', para, flags=re.S)
|
||
if ts:
|
||
lines.append(_decode_entities(''.join(ts)))
|
||
return '\n'.join(lines)
|
||
|
||
|
||
def extract_pdf_text(path):
|
||
"""pdf → 纯文本。无可用解析库或扫描件返回 ''（调用方给友好提示）。"""
|
||
for mod in ('pypdf', 'PyPDF2'):
|
||
try:
|
||
PdfReader = __import__(mod, fromlist=['PdfReader']).PdfReader
|
||
reader = PdfReader(path)
|
||
return '\n'.join((pg.extract_text() or '') for pg in reader.pages)
|
||
except Exception:
|
||
continue
|
||
try:
|
||
from pdfminer.high_level import extract_text as _ext
|
||
return _ext(path)
|
||
except Exception:
|
||
return ''
|
||
|
||
|
||
def read_text_file(path, offset=0, limit=DEFAULT_LIMIT):
|
||
"""统一读取文件文本：分页续读 + 文档解析 + 显式截断告知。
|
||
|
||
返回 dict：
|
||
kind: 'text'|'docx'|'pdf'|'binary'|'error'
|
||
content: 本段文本（truncated 时末尾附续读指引）
|
||
total: 全文总字符数（binary/error 为 0）
|
||
offset: 本段起始字符位
|
||
next_offset: 下一段起始（未截断为 -1）
|
||
truncated: 是否还有后文
|
||
message: 提示/错误信息（kind=binary/error 时有值）
|
||
"""
|
||
res = {'kind': 'text', 'content': '', 'total': 0, 'offset': 0,
|
||
'next_offset': -1, 'truncated': False, 'message': ''}
|
||
try:
|
||
if not os.path.isfile(path):
|
||
res['kind'] = 'error'
|
||
res['message'] = '文件不存在'
|
||
return res
|
||
ext = os.path.splitext(path)[1].lower()
|
||
if ext in DOCX_EXTS:
|
||
full = extract_docx_text(path)
|
||
res['kind'] = 'docx'
|
||
if not full:
|
||
res['kind'] = 'binary'
|
||
res['message'] = 'docx 无文本内容或解析失败'
|
||
return res
|
||
elif ext in PDF_EXTS:
|
||
full = extract_pdf_text(path)
|
||
res['kind'] = 'pdf'
|
||
if not full:
|
||
res['kind'] = 'binary'
|
||
res['message'] = ('PDF 文本提取失败（可能为扫描件，或服务器缺 pdf 解析库 '
|
||
'pypdf/PyPDF2/pdfminer）。可改用 run_command 配合 OCR，'
|
||
'或让用户提供文本版本。')
|
||
return res
|
||
elif ext in TEXT_EXTS:
|
||
with open(path, encoding='utf-8', errors='ignore') as f:
|
||
full = f.read()
|
||
else:
|
||
res['kind'] = 'binary'
|
||
res['message'] = ('该文件是二进制格式(%s)，无法直接读取文本。'
|
||
'可改用 run_command 处理，或让用户上传文本版本。'
|
||
% (ext or '无扩展名'))
|
||
return res
|
||
|
||
total = len(full)
|
||
res['total'] = total
|
||
try:
|
||
offset = max(0, int(offset or 0))
|
||
except (ValueError, TypeError):
|
||
offset = 0
|
||
try:
|
||
limit = max(1, int(limit or DEFAULT_LIMIT))
|
||
except (ValueError, TypeError):
|
||
limit = DEFAULT_LIMIT
|
||
end = min(offset + limit, total)
|
||
chunk = full[offset:end]
|
||
res['offset'] = offset
|
||
if end < total:
|
||
res['truncated'] = True
|
||
res['next_offset'] = end
|
||
chunk += ('\n\n[⚠️ 截断提示：以上为第 %d~%d 字符，全文共 %d 字符，尚未读完。'
|
||
'读后文请再次 read_file 并传 offset=%d，逐段读完全文。]'
|
||
% (offset, end, total, end))
|
||
res['content'] = chunk
|
||
return res
|
||
except Exception as e:
|
||
res['kind'] = 'error'
|
||
res['message'] = str(e)[:300]
|
||
return res
|