149dd5c811
缺陷1(新发现): WS_DIR=/../.. 只在机构工作空间成立；部署到 env.deploy.path=/d/pbl/pbls 后 WS_DIR=/d，ENV_FILE/WHEELHOUSE/modules 全不存在 → 第0步即 die，build.sh 在部署目录从未跑通 → venv未建/DDL未落库(表数0-37)/进程未起/9380无LISTEN/healthz=000/42条用例blocked。改为 _ws_probe() 三段式(环境变量覆盖 PBL_WS_DIR/WS_DIR_OVERRIDE/WS_DIR → 逐级向上探测含 projects/pbls/env/test.json 的目录 → 兜底绝对路径)，WHEELHOUSE 支持 PBL_WHEELHOUSE 覆盖。三场景实测全绿(含 /d/pbl/pbls)。 缺陷2: requirements.txt 幻影依赖 hashlib-additional>=1.0.1 ; python_version<3.0(hashlib 是标准库，marker 恒假；公共源真实名 hashlib_additional 对不上) → 删除。业主口径4项缺陷复扫确认已闭环: 基座7包==版本与 wheelhouse 实物逐字一致、pybricks/mysqlclient/10.20.30.41/相对find-links 生效行残留全0；pip download --no-deps 真实可满足性验证全部 Saved 零 No matching distribution。 缺陷3(机制根因): 门禁6.5 只 grep 声明存在不校验可满足 → 新增硬门禁6.6 scripts/check_requirements_satisfiable.py(349行) 7类校验(幻影包黑名单/标准库误当第三方/基座==版本与wheelhouse一致/wheelhouse覆盖完整/门禁6.5六项/失效内网源/相对find-links)，按任务要求同步扫描 40 个 modules/*/pyproject.toml 同类缺陷并输出本地模块依赖图(pbl_common <- pbl_analytics/pbl_appcodes/pbl_blueprint)与 build.sh R8 拓扑序交叉验证。负向测试构造8类缺陷抓出17项FAIL rc=1(证明非空壳)；正向 PASS 20项声明/基座7包一致/40模块零幻影。 附带加固: rsync 缺失(pbl 无 sudo 可能未装)时退化 tar 管道复制，避免部署目录建不出来。 自检: bash -n OK / py_compile OK / 门禁6.6 PASS / DDL 37表 / 门禁6.5六项全在 / 幻影包残留0 / 旧口径9300+/opt/pbls 0 / WS_DIR硬编码残留0
350 lines
16 KiB
Python
350 lines
16 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""依赖声明可满足性静态校验（build.sh 硬门禁 6.6）。
|
||
|
||
背景（Bug BWY68rZR_Uh2A41JlGT7D / elS7VxwO7ufbPiPl-ucGA）：
|
||
build.sh 原硬门禁 6.5 只用 `grep ^包名` 校验「声明存在」，不校验「声明可满足」。
|
||
结果 requirements.txt 里出现三类必然导致 `pip install -r` 失败（BUILD_RC=1）的缺陷，
|
||
门禁全绿放行，直到部署阶段才炸：
|
||
1) 版本下界凭空抬高、无对应发行物（ahserver>=1.4.0 而公共源最高 1.2.0）
|
||
→ pip "No matching distribution found"
|
||
2) 幻影包（pybricks：平台前端能力实际由 bricks_for_python 提供；
|
||
公共 pypi 同名包是 LEGO Mindstorms 机器人库，装上即污染 venv）
|
||
3) 不可编译包（mysqlclient：目标机无 mysql_config/mariadb_config，
|
||
且 pbl 用户无 sudo 装不了 mariadb-devel → 源码编译必失败）
|
||
本脚本把「可满足性」前移为静态门禁，并顺带扫描全部业务模块 pyproject.toml
|
||
的同类缺陷（任务要求：不能只修报出来的那一个）。
|
||
|
||
校验项：
|
||
A. 幻影包黑名单（应用 requirements*.txt + 全部模块 pyproject.toml）
|
||
B. 标准库名被误当第三方包声明（sys.stdlib_module_names 比对）
|
||
C. wheelhouse 固定版本一致性：`==` 声明的包若 wheelhouse 有 whl，版本必须逐字相同
|
||
D. wheelhouse 覆盖完整性：wheelhouse 里的基座包必须全部在 requirements 中固定
|
||
E. 硬门禁 6.5 六项基座声明在位（ahserver/sqlor/apppublic/appbase/rbac/PyMySQL）
|
||
F. 失效内网源残留（--index-url/--extra-index-url/--trusted-host 指向 10.20.30.41）
|
||
G. 相对路径 find-links（pip 不解析，实测 "Location '../../wheelhouse' is ignored"）
|
||
|
||
用法：
|
||
python3 scripts/check_requirements_satisfiable.py \
|
||
--requirements requirements.txt \
|
||
--wheelhouse ../../wheelhouse \
|
||
--modules-glob ../../modules/*/pyproject.toml
|
||
# rc=0 全部通过；rc=1 存在缺陷（逐条打印，build.sh die）
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import glob
|
||
import os
|
||
import re
|
||
import sys
|
||
|
||
# ---------------------------------------------------------------- 常量配置
|
||
|
||
# 基座包：必须由机构 wheelhouse 离线供给（内网 pypi 在目标机不可达）
|
||
BASE_PACKAGES = (
|
||
"ahserver",
|
||
"sqlor",
|
||
"apppublic",
|
||
"appbase",
|
||
"rbac",
|
||
"bricks_for_python",
|
||
"accounting",
|
||
)
|
||
|
||
# 硬门禁 6.5 口径：六项声明必须在位（与 build.sh 的 for dep in ... 一致）
|
||
GATE65_REQUIRED = ("ahserver", "sqlor", "apppublic", "appbase", "rbac", "pymysql")
|
||
|
||
# 幻影包黑名单：名字 -> 禁止理由（命中即 FAIL）
|
||
PHANTOM_BLACKLIST = {
|
||
"pybricks": (
|
||
"平台前端能力由 bricks_for_python(0.0.5) 提供（内含 load_pybricks）；"
|
||
"公共 pypi 的 pybricks 是 LEGO Mindstorms 机器人库，装上即污染 venv"
|
||
),
|
||
"mysqlclient": (
|
||
"需 mysql_config/mariadb_config，目标机实测 which mysql_config 无输出，"
|
||
"pbl 用户无 sudo 装不了 mariadb-devel → 源码编译必失败；"
|
||
"DB 驱动由 PyMySQL + aiomysql 承担（sqlor 走 aiomysql）"
|
||
),
|
||
"hashlib-additional": (
|
||
"Python 3 标准库已内置 hashlib；该声明带 `; python_version < \"3.0\"` 恒假 marker，"
|
||
"属无意义幻影依赖（且公共源真实包名为 hashlib_additional，声明名对不上）"
|
||
),
|
||
}
|
||
|
||
# 本地模块名：由 build.sh 以 `pip install -e modules/{模块}` 安装，任何索引都没有发行物。
|
||
# 作用域区分（关键）：
|
||
# - 写进应用 requirements*.txt → 致命（pip install -r 一次性解析，必报
|
||
# "No matching distribution found for pbl_common"，BUILD_RC=1）
|
||
# - 写进模块 pyproject.toml 的 dependencies → 合法的模块间依赖
|
||
# （module-development-spec 允许），由 build.sh 拓扑序先装 pbl_common 规避；
|
||
# 此时只校验「被依赖的本地模块目录真实存在」，不存在才是幻影。
|
||
LOCAL_MODULE_NAMES = ("pbl_common", "pbl_appcodes")
|
||
|
||
# 失效内网源（目标机 192.168.16.3 实测 connect timeout，5 次重试全失败）
|
||
DEAD_INDEX_HOSTS = ("10.20.30.41",)
|
||
|
||
# 依赖声明行：包名 + 可选版本约束
|
||
_REQ_LINE = re.compile(
|
||
r"^(?P<name>[A-Za-z0-9][A-Za-z0-9._-]*)\s*"
|
||
r"(?P<spec>(?:===|==|!=|~=|>=|<=|>|<)\s*[A-Za-z0-9._*+-]+\s*,?\s*)*"
|
||
)
|
||
|
||
_NAME_NORMALIZE = re.compile(r"[-_.]+")
|
||
|
||
|
||
def norm(name: str) -> str:
|
||
"""PEP 503 包名归一化：大小写无关，-/_/. 等价。"""
|
||
return _NAME_NORMALIZE.sub("-", name).lower()
|
||
|
||
|
||
def strip_comment(line: str) -> str:
|
||
"""剥掉行尾 `#` 注释（不处理引号内 #，requirements 语法里不存在该场景）。"""
|
||
idx = line.find("#")
|
||
return line if idx < 0 else line[:idx]
|
||
|
||
|
||
def parse_requirements(path: str):
|
||
"""解析 requirements.txt -> [(lineno, raw, name, spec, opts)]，跳过空行/注释/选项行。"""
|
||
out = []
|
||
with open(path, encoding="utf-8", errors="replace") as fh:
|
||
for lineno, raw in enumerate(fh.read().splitlines(), 1):
|
||
body = strip_comment(raw).strip()
|
||
if not body:
|
||
continue
|
||
if body.startswith("-"): # -r / --find-links / --index-url 等选项行
|
||
out.append((lineno, raw, None, None, body))
|
||
continue
|
||
m = _REQ_LINE.match(body)
|
||
if not m:
|
||
out.append((lineno, raw, None, None, body))
|
||
continue
|
||
out.append((lineno, raw, m.group("name"), (m.group("spec") or "").strip(), None))
|
||
return out
|
||
|
||
|
||
def parse_pyproject_deps(path: str):
|
||
"""从 pyproject.toml 抽 dependencies 字符串（不引 tomllib，py3.8 兼容）。"""
|
||
txt = open(path, encoding="utf-8", errors="replace").read()
|
||
deps = []
|
||
m = re.search(r"dependencies\s*=\s*\[(.*?)\]", txt, re.S)
|
||
if m:
|
||
for item in re.findall(r"""["']([^"']+)["']""", m.group(1)):
|
||
deps.append(item.strip())
|
||
return deps
|
||
|
||
|
||
def scan_wheelhouse(wh_dir: str):
|
||
"""wheelhouse 目录 -> {归一化包名: 版本}（取 *.whl 文件名前两段）。"""
|
||
found = {}
|
||
if not wh_dir or not os.path.isdir(wh_dir):
|
||
return found
|
||
for fn in sorted(os.listdir(wh_dir)):
|
||
if not fn.endswith(".whl"):
|
||
continue
|
||
parts = fn.split("-")
|
||
if len(parts) >= 2:
|
||
found[norm(parts[0])] = parts[1]
|
||
return found
|
||
|
||
|
||
def dep_name_of(spec_str: str):
|
||
"""从 'ahserver>=1.4.0' / 'pbl_common==1.0' 抽出归一化包名。"""
|
||
m = re.match(r"^([A-Za-z0-9][A-Za-z0-9._-]*)", spec_str.strip())
|
||
return norm(m.group(1)) if m else None
|
||
|
||
|
||
def pinned_version(spec: str):
|
||
"""从 '==1.2.0' / '== 1.2.0' 抽出精确固定版本；非 == 返回 None。"""
|
||
m = re.search(r"==\s*([A-Za-z0-9._*+-]+)", spec or "")
|
||
return m.group(1) if m else None
|
||
|
||
|
||
def main(argv=None) -> int:
|
||
ap = argparse.ArgumentParser(description="依赖声明可满足性静态校验（build.sh 门禁 6.6）")
|
||
ap.add_argument("--requirements", default="requirements.txt", help="应用 requirements.txt 路径")
|
||
ap.add_argument("--extra-requirements", action="append", default=[],
|
||
help="附加 requirements 文件（如 requirements-test.txt），可多次")
|
||
ap.add_argument("--wheelhouse", default="", help="机构 wheelhouse 目录（基座 whl 离线源）")
|
||
ap.add_argument("--modules-glob", default="",
|
||
help="业务模块 pyproject.toml 通配（如 ../../modules/*/pyproject.toml）")
|
||
ap.add_argument("--quiet", action="store_true", help="只打印结论行")
|
||
args = ap.parse_args(argv)
|
||
|
||
errors = [] # 致命：必然导致 pip install 失败 / 门禁不过
|
||
warns = [] # 提示：不阻断
|
||
|
||
def err(check, msg):
|
||
errors.append("[%s] %s" % (check, msg))
|
||
|
||
def warn(check, msg):
|
||
warns.append("[%s] %s" % (check, msg))
|
||
|
||
# ---------------- 0. 文件存在性 ----------------
|
||
if not os.path.isfile(args.requirements):
|
||
err("0", "requirements 文件不存在: %s" % args.requirements)
|
||
print("FAIL 依赖声明可满足性校验：requirements 文件缺失")
|
||
for e in errors:
|
||
print(" " + e)
|
||
return 1
|
||
|
||
req_files = [args.requirements] + [p for p in args.extra_requirements if os.path.isfile(p)]
|
||
wh = scan_wheelhouse(args.wheelhouse)
|
||
stdlib = set(getattr(sys, "stdlib_module_names", set()))
|
||
|
||
if not args.quiet:
|
||
print("[6.6] requirements 文件: %s" % ", ".join(req_files))
|
||
print("[6.6] wheelhouse: %s (%d 个 whl: %s)"
|
||
% (args.wheelhouse or "<未指定>", len(wh), ", ".join(sorted(wh)) or "-"))
|
||
|
||
# ---------------- A/B/F/G. 应用 requirements 逐行校验 ----------------
|
||
declared = {} # 归一化名 -> (lineno, spec, file)
|
||
for rf in req_files:
|
||
for lineno, raw, name, spec, opt in parse_requirements(rf):
|
||
if opt is not None:
|
||
# F. 失效内网源残留
|
||
for host in DEAD_INDEX_HOSTS:
|
||
if host in opt:
|
||
err("F", "%s:L%d 残留失效内网源 %s（目标机实测 connect timeout）: %s"
|
||
% (rf, lineno, host, opt.strip()[:100]))
|
||
# G. 相对路径 find-links（pip 不解析）
|
||
mfl = re.match(r"--find-links[=\s]+(\S+)", opt)
|
||
if mfl:
|
||
target = mfl.group(1)
|
||
if not os.path.isabs(target) and not target.startswith(("http://", "https://")):
|
||
err("G", "%s:L%d --find-links 用相对路径 %r，pip 不解析"
|
||
"（实测 WARNING: Location is ignored）；改绝对路径或由 build.sh 传 --find-links"
|
||
% (rf, lineno, target))
|
||
continue
|
||
if name is None:
|
||
warn("A", "%s:L%d 无法解析的声明行: %s" % (rf, lineno, raw.strip()[:100]))
|
||
continue
|
||
|
||
n = norm(name)
|
||
declared[n] = (lineno, spec or "", rf)
|
||
|
||
# A. 幻影包黑名单
|
||
if n in PHANTOM_BLACKLIST or n.replace("-", "_") in PHANTOM_BLACKLIST:
|
||
key = n if n in PHANTOM_BLACKLIST else n.replace("-", "_")
|
||
err("A", "%s:L%d 幻影依赖 %s —— %s" % (rf, lineno, name, PHANTOM_BLACKLIST[key]))
|
||
|
||
# A2. 本地模块写进 requirements（pip 索引无发行物 → 必失败）
|
||
if n in LOCAL_MODULE_NAMES or n.startswith("pbl-"):
|
||
err("A", "%s:L%d 本地模块 %s 不得写进 requirements —— 任何索引都无其发行物，"
|
||
"pip install -r 必报 No matching distribution；"
|
||
"由 build.sh 以 pip install -e modules/{模块} 安装"
|
||
% (rf, lineno, name))
|
||
|
||
# B. 标准库名误当第三方包
|
||
base = n.replace("-", "_")
|
||
if base in stdlib:
|
||
err("B", "%s:L%d %s 是 Python 标准库模块名，不得作为第三方依赖声明" % (rf, lineno, name))
|
||
|
||
# C. wheelhouse 固定版本一致性
|
||
pv = pinned_version(spec or "")
|
||
if n in wh:
|
||
if pv is None:
|
||
err("C", "%s:L%d 基座包 %s 未用 == 精确固定（实际 %r），"
|
||
"wheelhouse 实装 %s；非固定下界在公共源无对应发行物时 pip 必失败"
|
||
% (rf, lineno, name, (spec or "").strip(), wh[n]))
|
||
elif pv != wh[n]:
|
||
err("C", "%s:L%d %s==%s 与 wheelhouse 实装 %s 不一致（pip 找不到发行物）"
|
||
% (rf, lineno, name, pv, wh[n]))
|
||
elif pv is None and re.search(r">=\s*[0-9]", spec or ""):
|
||
# 非基座包用 >= 下界：提示核对下界是否有对应发行物（不阻断，公网包通常可满足）
|
||
warn("C", "%s:L%d %s 用下界约束 %r，请确认公共源存在满足该下界的发行物"
|
||
% (rf, lineno, name, (spec or "").strip()))
|
||
|
||
# ---------------- D. wheelhouse 覆盖完整性 ----------------
|
||
for pkg in BASE_PACKAGES:
|
||
npkg = norm(pkg)
|
||
if npkg in wh and npkg not in declared:
|
||
err("D", "wheelhouse 提供基座包 %s(%s) 但 requirements 未声明固定版本" % (pkg, wh[npkg]))
|
||
|
||
# ---------------- E. 硬门禁 6.5 六项声明 ----------------
|
||
for pkg in GATE65_REQUIRED:
|
||
if norm(pkg) not in declared:
|
||
err("E", "硬门禁 6.5 缺基座声明: %s" % pkg)
|
||
|
||
# ---------------- 模块 pyproject 同类缺陷扫描 ----------------
|
||
mod_files = sorted(glob.glob(args.modules_glob)) if args.modules_glob else []
|
||
if not args.quiet:
|
||
print("[6.6] 业务模块 pyproject.toml 扫描: %d 个" % len(mod_files))
|
||
mod_hits = 0
|
||
# modules 根目录：从 --modules-glob 反推（.../modules/*/pyproject.toml）
|
||
modules_root = ""
|
||
if args.modules_glob:
|
||
_g = args.modules_glob.replace("\\", "/")
|
||
if "/modules/" in _g:
|
||
modules_root = _g.split("/modules/")[0] + "/modules"
|
||
elif _g.startswith("modules/"):
|
||
modules_root = "modules"
|
||
local_deps = {} # 被依赖本地模块 -> [依赖它的 pyproject 列表]
|
||
for mf in mod_files:
|
||
for dep in parse_pyproject_deps(mf):
|
||
dn = dep_name_of(dep)
|
||
if not dn:
|
||
continue
|
||
if dn in PHANTOM_BLACKLIST or dn.replace("-", "_") in PHANTOM_BLACKLIST:
|
||
key = dn if dn in PHANTOM_BLACKLIST else dn.replace("-", "_")
|
||
err("A", "%s 声明幻影依赖 %s —— %s" % (mf, dep, PHANTOM_BLACKLIST[key]))
|
||
mod_hits += 1
|
||
# A2'. 模块间本地依赖：合法，但被依赖模块目录必须真实存在
|
||
# （否则 build.sh 拓扑序也装不上 → 入口 ImportError → 进程起不来）
|
||
if dn in LOCAL_MODULE_NAMES or dn.startswith("pbl-"):
|
||
local_name = dn.replace("-", "_")
|
||
if modules_root:
|
||
cand = os.path.join(modules_root, local_name)
|
||
if not os.path.isdir(cand):
|
||
err("A", "%s 依赖本地模块 %s，但 %s 不存在（幻影本地依赖，"
|
||
"pip install -e 阶段必失败）" % (mf, dep, cand))
|
||
mod_hits += 1
|
||
elif local_name != os.path.basename(mf).split(os.sep)[0]:
|
||
local_deps.setdefault(local_name, []).append(mf)
|
||
if dn.replace("-", "_") in stdlib:
|
||
err("B", "%s 把标准库 %s 当第三方依赖声明" % (mf, dep))
|
||
mod_hits += 1
|
||
pv = pinned_version(dep)
|
||
if dn in wh:
|
||
if pv is not None and pv != wh[dn]:
|
||
err("C", "%s 固定 %s 与 wheelhouse 实装 %s 不一致" % (mf, dep, wh[dn]))
|
||
mod_hits += 1
|
||
elif pv is None:
|
||
m = re.search(r">=\s*([0-9][0-9A-Za-z.]*)", dep)
|
||
if m and m.group(1) > wh[dn]:
|
||
err("C", "%s 下界 %s 高于 wheelhouse 实装 %s（无对应发行物）"
|
||
% (mf, dep, wh[dn]))
|
||
mod_hits += 1
|
||
if not args.quiet:
|
||
print("[6.6] 模块同类缺陷命中: %d" % mod_hits)
|
||
if local_deps:
|
||
print("[6.6] 本地模块间依赖（build.sh 须按拓扑序先装被依赖方）:")
|
||
for k in sorted(local_deps):
|
||
print(" %s <- %d 个模块依赖: %s"
|
||
% (k, len(local_deps[k]),
|
||
", ".join(sorted(os.path.basename(os.path.dirname(p))
|
||
for p in local_deps[k])[:6])))
|
||
if "pbl_common" in local_deps:
|
||
print(" 注: build.sh 第 8 步已显式把 pbl_common 置于 INSTALL_DIRS 首位"
|
||
"（R8 拓扑序修复），与上述依赖图一致")
|
||
|
||
# ---------------- 结论 ----------------
|
||
for w in warns:
|
||
print(" WARN " + w)
|
||
if errors:
|
||
print("FAIL 依赖声明可满足性校验未通过（%d 项）：" % len(errors))
|
||
for e in errors:
|
||
print(" " + e)
|
||
return 1
|
||
|
||
print("PASS 依赖声明可满足性校验通过：%d 个 requirements 文件 / %d 项声明 / "
|
||
"基座 %d 包与 wheelhouse 版本逐字一致 / %d 个模块 pyproject 零幻影依赖"
|
||
% (len(req_files), len(declared), len([p for p in BASE_PACKAGES if norm(p) in wh]),
|
||
len(mod_files)))
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|