review修复: SSRF重定向绕过+无价歧义标注
- fetch逐跳校验重定向(每跳过域名+DNS私有IP双校验,防redirect到内网) - 文档未列价的模型落0时description显式标注(避免0=免费歧义) - 清理无用import
This commit is contained in:
parent
e95e618372
commit
dbfeb1498e
Binary file not shown.
@ -15,6 +15,7 @@ pipeline-service: platform_ability — 平台内部 agent 能力包（pipeline_i
|
||||
（llm_vendor.endpoints + llm_api_profile 模板 + llm_model 含四价）
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
@ -105,6 +106,7 @@ async def _h_platform_llm_status(sor, params, ctx):
|
||||
# ────────────────────── 工具 2：抓取模型 API 文档 ──────────────────────
|
||||
|
||||
_MAX_DOC_CHARS = 60000
|
||||
_MAX_REDIRECTS = 5
|
||||
|
||||
|
||||
def _validate_doc_url(url: str) -> str:
|
||||
@ -133,25 +135,69 @@ def _html_to_text(html: str) -> str:
|
||||
return txt.strip()
|
||||
|
||||
|
||||
def _is_private_ip(ip: str) -> bool:
|
||||
"""DNS 解析后的二次防线：解析到私有/环回/链路本地地址一律拒绝（DNS rebinding 防护）。"""
|
||||
import ipaddress
|
||||
try:
|
||||
a = ipaddress.ip_address(ip)
|
||||
except ValueError:
|
||||
return True # 解析不出就当私有（拒绝）
|
||||
return (a.is_private or a.is_loopback or a.is_link_local
|
||||
or a.is_reserved or a.is_multicast or a.is_unspecified)
|
||||
|
||||
|
||||
async def _fetch_url_safe(url: str, max_redirects: int = _MAX_REDIRECTS):
|
||||
"""带重定向逐跳校验的抓取（每一跳都过域名+DNS双重校验）。
|
||||
|
||||
返回 (text, content_type)；失败抛 ValueError。
|
||||
"""
|
||||
import aiohttp
|
||||
current = url
|
||||
for _ in range(max_redirects + 1):
|
||||
verr = _validate_doc_url(current)
|
||||
if verr:
|
||||
raise ValueError(verr)
|
||||
host = re.sub(r"^https?://", "", current).split("/")[0].split(":")[0].lower()
|
||||
# DNS 解析校验（同步阻塞短调用，可接受；防 DNS rebinding 指向内网）
|
||||
try:
|
||||
infos = await asyncio.get_event_loop().getaddrinfo(host, None)
|
||||
except Exception as e:
|
||||
raise ValueError("文档域名无法解析：%s" % str(e)[:120])
|
||||
for info in infos:
|
||||
ip = str(info[4][0])
|
||||
if _is_private_ip(ip):
|
||||
raise ValueError("文档域名解析到内网地址（%s），拒绝访问" % ip)
|
||||
timeout = aiohttp.ClientTimeout(total=30)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as sess:
|
||||
async with sess.get(current, headers={"User-Agent": "Mozilla/5.0"},
|
||||
allow_redirects=False, ssl=False) as resp:
|
||||
if resp.status in (301, 302, 303, 307, 308):
|
||||
loc = resp.headers.get("Location", "")
|
||||
if not loc:
|
||||
raise ValueError("重定向缺少 Location")
|
||||
if loc.startswith("/"):
|
||||
scheme = "https" if current.startswith("https") else "http"
|
||||
loc = "%s://%s%s" % (scheme, host, loc)
|
||||
current = loc
|
||||
continue
|
||||
if resp.status != 200:
|
||||
raise ValueError("抓取失败：HTTP %d" % resp.status)
|
||||
ctype = resp.headers.get("Content-Type", "")
|
||||
raw = await resp.text(errors="replace")
|
||||
return raw, ctype
|
||||
raise ValueError("重定向次数超限（>%d）" % max_redirects)
|
||||
|
||||
|
||||
async def _h_fetch_model_doc(sor, params, ctx):
|
||||
"""抓取模型 API 文档页面 → 纯文本（供 LLM 提取配置规格）。"""
|
||||
err = await _require_owner(sor, ctx)
|
||||
if err:
|
||||
return err
|
||||
url = (params.get("url") or "").strip()
|
||||
verr = _validate_doc_url(url)
|
||||
if verr:
|
||||
return verr
|
||||
import aiohttp
|
||||
try:
|
||||
timeout = aiohttp.ClientTimeout(total=30)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as sess:
|
||||
async with sess.get(url, headers={"User-Agent": "Mozilla/5.0"},
|
||||
allow_redirects=True, ssl=False) as resp:
|
||||
if resp.status != 200:
|
||||
return "抓取失败：HTTP %d" % resp.status
|
||||
ctype = resp.headers.get("Content-Type", "")
|
||||
raw = await resp.text(errors="replace")
|
||||
raw, ctype = await _fetch_url_safe(url)
|
||||
except ValueError as e:
|
||||
return str(e)
|
||||
except Exception as e:
|
||||
return "抓取失败：%s" % str(e)[:200]
|
||||
if "html" in ctype.lower() or raw.lstrip()[:15].lower().startswith(("<!doctype", "<html")):
|
||||
@ -332,9 +378,14 @@ async def _h_apply_llm_config(sor, params, ctx):
|
||||
skipped.append("(缺 vendor_model_id)")
|
||||
continue
|
||||
cap = m.get("capability") or "t2t"
|
||||
price_in, price_out = _f(m.get("price_input")), _f(m.get("price_output"))
|
||||
raw_pi, raw_po = m.get("price_input"), m.get("price_output")
|
||||
price_in, price_out = _f(raw_pi), _f(raw_po)
|
||||
cost_in = _f(m.get("cost_input"), price_in)
|
||||
cost_out = _f(m.get("cost_output"), price_out)
|
||||
desc = (m.get("description", "") or "").strip()
|
||||
# 文档未给价（null）→ 落 0 并显式标注，避免「0=免费」歧义
|
||||
if raw_pi is None or raw_po is None:
|
||||
desc = "⚠️文档未列定价（暂落0，需人工补录）。" + desc
|
||||
recs = await sor.sqlExe(
|
||||
"SELECT id FROM llm_model WHERE vendor_id=${v}$ AND vendor_model_id=${m}$",
|
||||
{"v": vendor_id, "m": vmid})
|
||||
@ -346,7 +397,7 @@ async def _h_apply_llm_config(sor, params, ctx):
|
||||
"cost_input=${ci}$, cost_output=${co}$, description=${d}$, updated_at=NOW() "
|
||||
"WHERE id=${i}$",
|
||||
{"pi": price_in, "po": price_out, "ci": cost_in, "co": cost_out,
|
||||
"d": m.get("description", "") or "", "i": mid})
|
||||
"d": desc, "i": mid})
|
||||
await sor.sqlExe("COMMIT", {})
|
||||
updated.append(vmid)
|
||||
continue
|
||||
@ -358,7 +409,7 @@ async def _h_apply_llm_config(sor, params, ctx):
|
||||
"ppid": "", "price_input": price_in, "price_output": price_out,
|
||||
"cost_input": cost_in, "cost_output": cost_out,
|
||||
"default_params": "{}", "status": "active",
|
||||
"description": m.get("description", "") or "", "org_id": "0"})
|
||||
"description": desc, "org_id": "0"})
|
||||
created.append(vmid)
|
||||
|
||||
rt_note = ""
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user