diff --git a/pipeline_platform/__pycache__/platform_ability.cpython-310.pyc b/pipeline_platform/__pycache__/platform_ability.cpython-310.pyc index 2bbe52a..132128b 100644 Binary files a/pipeline_platform/__pycache__/platform_ability.cpython-310.pyc and b/pipeline_platform/__pycache__/platform_ability.cpython-310.pyc differ diff --git a/pipeline_platform/platform_ability.py b/pipeline_platform/platform_ability.py index 7d6c7ae..0b6d83c 100644 --- a/pipeline_platform/platform_ability.py +++ b/pipeline_platform/platform_ability.py @@ -15,6 +15,7 @@ pipeline-service: platform_ability — 平台内部 agent 能力包(pipeline_i (llm_vendor.endpoints + llm_api_profile 模板 + llm_model 含四价) """ +import asyncio import json import logging import re @@ -105,6 +106,7 @@ async def _h_platform_llm_status(sor, params, ctx): # ────────────────────── 工具 2:抓取模型 API 文档 ────────────────────── _MAX_DOC_CHARS = 60000 +_MAX_REDIRECTS = 5 def _validate_doc_url(url: str) -> str: @@ -133,25 +135,69 @@ def _html_to_text(html: str) -> str: return txt.strip() +def _is_private_ip(ip: str) -> bool: + """DNS 解析后的二次防线:解析到私有/环回/链路本地地址一律拒绝(DNS rebinding 防护)。""" + import ipaddress + try: + a = ipaddress.ip_address(ip) + except ValueError: + return True # 解析不出就当私有(拒绝) + return (a.is_private or a.is_loopback or a.is_link_local + or a.is_reserved or a.is_multicast or a.is_unspecified) + + +async def _fetch_url_safe(url: str, max_redirects: int = _MAX_REDIRECTS): + """带重定向逐跳校验的抓取(每一跳都过域名+DNS双重校验)。 + + 返回 (text, content_type);失败抛 ValueError。 + """ + import aiohttp + current = url + for _ in range(max_redirects + 1): + verr = _validate_doc_url(current) + if verr: + raise ValueError(verr) + host = re.sub(r"^https?://", "", current).split("/")[0].split(":")[0].lower() + # DNS 解析校验(同步阻塞短调用,可接受;防 DNS rebinding 指向内网) + try: + infos = await asyncio.get_event_loop().getaddrinfo(host, None) + except Exception as e: + raise ValueError("文档域名无法解析:%s" % str(e)[:120]) + for info in infos: + ip = str(info[4][0]) + if _is_private_ip(ip): + raise ValueError("文档域名解析到内网地址(%s),拒绝访问" % ip) + timeout = aiohttp.ClientTimeout(total=30) + async with aiohttp.ClientSession(timeout=timeout) as sess: + async with sess.get(current, headers={"User-Agent": "Mozilla/5.0"}, + allow_redirects=False, ssl=False) as resp: + if resp.status in (301, 302, 303, 307, 308): + loc = resp.headers.get("Location", "") + if not loc: + raise ValueError("重定向缺少 Location") + if loc.startswith("/"): + scheme = "https" if current.startswith("https") else "http" + loc = "%s://%s%s" % (scheme, host, loc) + current = loc + continue + if resp.status != 200: + raise ValueError("抓取失败:HTTP %d" % resp.status) + ctype = resp.headers.get("Content-Type", "") + raw = await resp.text(errors="replace") + return raw, ctype + raise ValueError("重定向次数超限(>%d)" % max_redirects) + + async def _h_fetch_model_doc(sor, params, ctx): """抓取模型 API 文档页面 → 纯文本(供 LLM 提取配置规格)。""" err = await _require_owner(sor, ctx) if err: return err url = (params.get("url") or "").strip() - verr = _validate_doc_url(url) - if verr: - return verr - import aiohttp try: - timeout = aiohttp.ClientTimeout(total=30) - async with aiohttp.ClientSession(timeout=timeout) as sess: - async with sess.get(url, headers={"User-Agent": "Mozilla/5.0"}, - allow_redirects=True, ssl=False) as resp: - if resp.status != 200: - return "抓取失败:HTTP %d" % resp.status - ctype = resp.headers.get("Content-Type", "") - raw = await resp.text(errors="replace") + raw, ctype = await _fetch_url_safe(url) + except ValueError as e: + return str(e) except Exception as e: return "抓取失败:%s" % str(e)[:200] if "html" in ctype.lower() or raw.lstrip()[:15].lower().startswith(("