From dbfeb1498e6db7e4d589dbd775a5222520ba78ef Mon Sep 17 00:00:00 2001 From: yumoqing Date: Wed, 2 Sep 2026 16:20:17 +0800 Subject: [PATCH] =?UTF-8?q?review=E4=BF=AE=E5=A4=8D:=20SSRF=E9=87=8D?= =?UTF-8?q?=E5=AE=9A=E5=90=91=E7=BB=95=E8=BF=87+=E6=97=A0=E4=BB=B7?= =?UTF-8?q?=E6=AD=A7=E4=B9=89=E6=A0=87=E6=B3=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - fetch逐跳校验重定向(每跳过域名+DNS私有IP双校验,防redirect到内网) - 文档未列价的模型落0时description显式标注(避免0=免费歧义) - 清理无用import --- .../platform_ability.cpython-310.pyc | Bin 19467 -> 21270 bytes pipeline_platform/platform_ability.py | 81 ++++++++++++++---- 2 files changed, 66 insertions(+), 15 deletions(-) diff --git a/pipeline_platform/__pycache__/platform_ability.cpython-310.pyc b/pipeline_platform/__pycache__/platform_ability.cpython-310.pyc index 2bbe52a17911610ddf161cb82210d2e6cf2516e6..132128ba998b1775499eb196a26da3e8dbde718f 100644 GIT binary patch delta 4574 zcmZu!Yj9J?6~23QuOv&dY|FN6%f^&7*CG5E`YB5(w?IKQir~X8N4lX=mCZVNa)J+H^W;r&D^)l}+j6 z+V{KXoISgHcK7VJs~27&XWk&*cxkCif@kWNFF*N_cFNm8o;kIkB_sDMVTBTE>sJTt zVf%n1>=uHcSgWN!SXe(_yM#6JwJM92VBb^J)&4V|jH9^*rXT7%Z8wa|?dYPg+lqMM=94y|KEA`+E*rO=kt-)xfPO6Uo# zUmw&KB{epr=gyEz4zez{v&2Klf?U|uNyxq8dH3UFbM80pM`iL@Zk2a6A>mx2>_>z| z#m0&^$o*ofVykUhl|-lScjWImUnSJqB{o(qAPF&0RY#u4JzF(Os(N9$*`olanlzZ6 z2+1rg=J`7a7Z3R7WMWt%>xmcvVfKL#B`3U_;Y~gW0J$OeOKEb%!AX{IX_BPn$E87+ zA{p}GY67Dp(i0N3^-6~%E-LbE-)GQGj6uf1CM@>qWC zrJLi&Zd`e>aDH<7m3MAlxqj>F_)O7r7jJ&;^M#4$yKcVpcK+;_yQaklQJ2h&uRcii!F@q!u5jYDTUVd|-SAO{!`$^q zsKhMM6H!VtOiLzBm!61#Qj{Ny_NTNhjKvw)laU0|hoXjNx)o)D*E67 zu*9PMRvA==wbZETfwF;AzoEyXNyGGlmKsbZw3yzfYcynI6e3_6fJve(Q;nJhel&R_ z;6Kx4L{FRmTQUIPlx5;29%3i*ZJ(?F&b};;^-2e#PeqtUb*9CPWK0Iz?BfH$4e$5h zP40q{&y$(ul)02i+YG{C=hsLhq(>SfyM1ZJkhyJyq*bo6fMMh62sug&C96_}s@yhd z%i5`Z5*n$KKyu7Tj!7wPKT>Kqvn7*K*2V2qo|Lyi)^C;44#<^S$QAeDhl!L1UOD11 zY}9FZc*Z#bsYOeU%XN|;LO2Rq+WEBe5P?K;^hmp_(k0*qR8t2@SO*?ymr=@HBZTSD zgqypHwGKWC3FoCAS~^P7Zjj5kn|epda;X@?_;}jGJx`W%8!e;du#qd^&o`=M?l>6f zQeOIG1+5&C*oPqbvgNdjs*`fs%e~2S&{;Kx$GzD~96fu7+d%Sz^bP7CB(x@7Mr(N) zw{vfw%`aiCw5>#X9A-d~6lug(421lrB?Hq#q=Ut->kio1PoSd!22vc?Od$1EAmyM{ z(FQtal*~+Q1+U=ceXtef|I^L?+GOafS6VVNCu1b3W)6|r-TPmWS%D3D?sBfb+xK88 z1lq_+G0ad?>LjFB{=GM*pB>F#7|)NLnjRjTx|}OyC#T{qqsq0^aT_2ohXipgngR%Iap$!@l z<8VR?-MV@<|JpaF-hI7r;%xBW8y7aZRkOI;SviEp9zejuk0GJ4hX7J@q>LmL&OBR~ ze6<)x;pJ>0ce+SBz(4KD>8HSNT0!U40C)S|A*cK^$8TOcRU{%9V0#jRzjgJ+A{$Lz zxIX>zh3Q7993rM_#Eob_2o6IZ(BdgW{JOSId{WyI4ypPg;3mBjp$lOd!g2u9c|Wj> zp~nYPwHVFp4BxzZC4cc$aJJA?_ZvncX)5?RQqrF6?&@k!wkJiPZc_&Pzf%z?ws895 z)TQIoUmAme*SAxMI8<#qqIw)Fn<|SA?$g*4P>JnBxO0fG3RE8h0Pp1C!I&O5z56sH zq7C7qAVHdL(2-5`!M?azvgN6mmhfWqrVS1ti>|~k>zXn+7PbOm2w^_9so)xid~6q5 z5(vX+cEnPQLHjI(nlhv@#y*XrihU$aqL~UxMQ%-5PDvzo1z0&@+JL1^dos3P8_-NO zhM8m9GAw$iNWA}M-?B)bX2kYKQmj9cjP_}#p{3;K0OVn(st_M>5(VB4BFh1w<67bz zmTx=da(bomFP?Cx7uT{5Q9xrG+(5p{2<~;VRr;))R?~JM(j`Eo%HdT&qz)ibm!WdU2zgwx z;{i<4j;9^HGv7l5Jna_cqGvf(ioU@9s`zemn(;Z6z?`qPr4-7nPuzjHj{U1D$TqaD^l;zmCS#=xi-|>nx&SXVCzTC3Y3 zyNDutV67Fx#&b&j@11T8`iO1Mo;_@j_>W^J zcr%-QaJrD%td!ZRist5q+@6k*joc6?LfskOkhRh7Nn(B1OhV448`DjOmp8)td3nPgZk!{Odx5V&Df@Je3C1H7Jh&<7{%w21~s@sj_LNG$k-oEQ4SL*8$}0-1KE z?E?a2De=dJKUVPk5o1-=!d$e_jpZ}PlBnnm)LE%fjYbRnD6Iv!PxN#ywSODcl;E9| zjEdiMPITW@9gD2;OO_MDWrX(tLP~L8T|>hvgm=W{F3e7|Yf;Tt(Zj*6qTP#d1>s%M zxY$o#6Dt-UA#aI`i)%Ypa-;QT*y?3&B zUKS?a%s=-sTnlcze;IE8<432)t}TWma(ZI;#+57i%dbyOzMj8!x_Ib|6H6b44u7?D zLOi|9Pp*s8%XUK;A20h$=<67lMG)2w!+mG^qKU+VslEMrtTQ#JA53Z6df5A-d_`5j z8f-mQDUKMWdl6m{>sPGrp8J0(<2*X#5Z*iNG`s)N3F)mVnKvvgcgK0@!{4Vc|HVRIP2jO$aaa> zw*5q2B#R^4+m4-#VJQj(tqu4Z#&2;p~_w9pUgT<>%B@sW^-k!l5 z7sKC2tbkx0Gu9!r1l9Wyz61cj%64x2bmZYJJ=-4MviXtT;>9S2CCc7-yx+Rh6qD>U z))_~51K|P!5;wzTH5IDI47dcQ68##>OOVN|9z(!OAHz!@Ta2(&M0a$!o`Y(NU2n%K$*C_nZ+G0 delta 2906 zcmaKueQaCR6~NzletvfR6~}R$q;(P}Y0@SsrD@Z&q-h~(lcp_c^92D9NVA;R;E*Qi zwL>6WlR^qYpp6+*G|>KdnlzZi$G%|GCJok&>>Q`< z%7$j?{oeV!@4kEQInQS=!c*rUFzxqyH2SsM&rH00*XckM22MBCEI1~OW6rqqm>D;Z zx#BL+EIs2!BMb2YbjCes#=Yo@`_OIqkLp-}o|8sAfL`>S)Z&Hc#{k`fSQszDpykg; z8CZlNiV9)zHA@&v$Wn|EET#A`mc>i39M_!G7UB`yg;A_H3Gq^_#42)^VT=;1>8i2H zX~;E4-B^q3Xn?g?hxHT@wVb#fHzYJ{z{Vwmv?kn$LA)UW*DRaxM%;|e*IBmUR;&_S_={K7hvDDbFT=F7=9c7&>z$zT0%TEq7mF2f&@V7YbhoIHG2YF2X?-4fQx} z$E=uC9W5nlG4vyt&mAnL++H;wZdyA<`;!b3g1Q0q-~$6fbQvIqbFYS%Aly$rF-WkT z$&^UUH0a_EH5#dhdG$o3A~s5%F$dx5FF*qC`VZ(z{t{GXV^n3$=c^ySe7-J8C!!~IpfV)Zb59Vi!ZtQrY&KHX11Y

wz(ACY6|;vK@51#1Om8$^1zZjU3De%yvv#Q+8max8shH zy@&UV?CI$k7;FfLDh}Akh%tH?JbT-?-x5L`V;%E)(n=<6Lxip5jO`ptJvd|84lz1s zgC%CvUy6!Vb#-z76BL^4Cg>-)8n_bN`C4#;PL&bKS4Ut)2M^i9|0-JvIK$9*dBciv zBH)UF`-HwyuY17IeNe^~By;1{e{g&6r4i>HM~)m3_o%xYd(HgPq|~ONeQq_HV4+`X0q4c_VqoL7KC^BF>Qdsd`;U2Cn~ny>7@ozp>i^Z>o21 zY&(z&$Y{oaZCO}^EAmd3bv<2~uDW7r&w{L6)YGvm`YzI|NUxR}HTB)nD=YA3wfV;C zZMvvS6-W<;=;)QAx#&pO&9cX6tWqrnvrmVqE^}*)m1&%AiJ=8Qy zWsL&dE4{=nG>P4^g4mkp4p~C-KI|dxm&F-}EaA~eJCKiUla(azlvTub86)nJ)x?A7 zAl{1g#EwtR;wb0cg}u05K~}%oPGot;Co8g4{mXDXi)Ol3x>6*#IHcb4ht)vKW-mKei}H|KYFT6O8BiUqVJKIxwrnvkup8A!EeH>(ySH6vyDpwx zdH)ylL5%Z^FA*Ay)pE_SVLRiD+S*zJb?Wxk8#bNf2$y)C?cCSJvy5j5wim7B7<~@7 zbVa_ZA6J)J=gXhwh>~lFuvAKDR(rR<1X*=?d&+y=cxrx!1z%Hd?Wk7=ZW@56)w4HU zh>DAp&|up7OoFayTOXtQ*t88OzO4G%?uN(Ji*0`!ev-$|H=BIHjF4)FMrUUFWEl{h|Y?cXb&4y9OhNc5mNW;J_sT8jomfDw|UU~vfs?osobx3e+ZCJKf9$99#e%KQLxm;j+5}H zy3{cpd7oo`$KW2j+8Teasn*WEp-0*9XTsGj=k*d@41Vth^ZKW-^|u6SlGbtY0IUDC zr>~K1PT8HGgV%Dcdw!~$6C8A0xw>1-zq9yJzO4>*hv7Mu=&sW*YihasS8y>mzVAhV zbS|=={%!C^i%%FUTCV57UT4WaS)uPKeS8=?!%*47ulv499@|Ka(Ru4t>ApAhe}Ee7 zU$-$tAq^l-le?kXc8!eS^w`Kq{>^OHVrpFU^RR!_)Ytp#7W_PBhz&nx2uA+H z)y&eZj9JDoW0WBo{B;n^j5Cb43ATZWv6Ssh&CX0(f|o>GW(4>Ng^VDBXZWA>Qy&+3 zl^h6cK0%>|<}Pyc*F^W2hHf|vBj9wGxeJU~$Px0n14hgp_0+mu+QJ%-`u@PszX98e B$rb str: @@ -133,25 +135,69 @@ def _html_to_text(html: str) -> str: return txt.strip() +def _is_private_ip(ip: str) -> bool: + """DNS 解析后的二次防线:解析到私有/环回/链路本地地址一律拒绝(DNS rebinding 防护)。""" + import ipaddress + try: + a = ipaddress.ip_address(ip) + except ValueError: + return True # 解析不出就当私有(拒绝) + return (a.is_private or a.is_loopback or a.is_link_local + or a.is_reserved or a.is_multicast or a.is_unspecified) + + +async def _fetch_url_safe(url: str, max_redirects: int = _MAX_REDIRECTS): + """带重定向逐跳校验的抓取(每一跳都过域名+DNS双重校验)。 + + 返回 (text, content_type);失败抛 ValueError。 + """ + import aiohttp + current = url + for _ in range(max_redirects + 1): + verr = _validate_doc_url(current) + if verr: + raise ValueError(verr) + host = re.sub(r"^https?://", "", current).split("/")[0].split(":")[0].lower() + # DNS 解析校验(同步阻塞短调用,可接受;防 DNS rebinding 指向内网) + try: + infos = await asyncio.get_event_loop().getaddrinfo(host, None) + except Exception as e: + raise ValueError("文档域名无法解析:%s" % str(e)[:120]) + for info in infos: + ip = str(info[4][0]) + if _is_private_ip(ip): + raise ValueError("文档域名解析到内网地址(%s),拒绝访问" % ip) + timeout = aiohttp.ClientTimeout(total=30) + async with aiohttp.ClientSession(timeout=timeout) as sess: + async with sess.get(current, headers={"User-Agent": "Mozilla/5.0"}, + allow_redirects=False, ssl=False) as resp: + if resp.status in (301, 302, 303, 307, 308): + loc = resp.headers.get("Location", "") + if not loc: + raise ValueError("重定向缺少 Location") + if loc.startswith("/"): + scheme = "https" if current.startswith("https") else "http" + loc = "%s://%s%s" % (scheme, host, loc) + current = loc + continue + if resp.status != 200: + raise ValueError("抓取失败:HTTP %d" % resp.status) + ctype = resp.headers.get("Content-Type", "") + raw = await resp.text(errors="replace") + return raw, ctype + raise ValueError("重定向次数超限(>%d)" % max_redirects) + + async def _h_fetch_model_doc(sor, params, ctx): """抓取模型 API 文档页面 → 纯文本(供 LLM 提取配置规格)。""" err = await _require_owner(sor, ctx) if err: return err url = (params.get("url") or "").strip() - verr = _validate_doc_url(url) - if verr: - return verr - import aiohttp try: - timeout = aiohttp.ClientTimeout(total=30) - async with aiohttp.ClientSession(timeout=timeout) as sess: - async with sess.get(url, headers={"User-Agent": "Mozilla/5.0"}, - allow_redirects=True, ssl=False) as resp: - if resp.status != 200: - return "抓取失败:HTTP %d" % resp.status - ctype = resp.headers.get("Content-Type", "") - raw = await resp.text(errors="replace") + raw, ctype = await _fetch_url_safe(url) + except ValueError as e: + return str(e) except Exception as e: return "抓取失败:%s" % str(e)[:200] if "html" in ctype.lower() or raw.lstrip()[:15].lower().startswith(("