From e294fc7a6be62a6ec25fc7de0c040e998fab0d64 Mon Sep 17 00:00:00 2001 From: yumoqing Date: Thu, 10 Sep 2026 19:02:12 +0800 Subject: [PATCH] =?UTF-8?q?feat(deploy):=20=E5=AD=A4=E5=84=BF=E8=A1=A8?= =?UTF-8?q?=E6=B8=85=E9=9B=B6=E6=B2=BB=E7=90=86=E2=80=94=E2=80=94dmig?= =?UTF-8?q?=E6=96=B0=E5=A2=9Eapply-approved=E7=A0=B4=E5=9D=8F=E6=80=A7?= =?UTF-8?q?=E5=AE=A1=E6=89=B9=E9=80=9A=E9=81=93(destructive:true=E6=98=BE?= =?UTF-8?q?=E5=BC=8F=E5=A3=B0=E6=98=8E+=E6=89=B9=E6=AC=A1=E5=8F=B7?= =?UTF-8?q?=E5=BF=85=E5=A1=AB,=E5=A4=87=E4=BB=BD=E5=8F=B0=E8=B4=A6?= =?UTF-8?q?=E7=85=A7=E5=B8=B8)=20+=20m0026=E5=BD=92=E7=B1=8D=E6=B4=BB?= =?UTF-8?q?=E8=A1=A8=E6=94=B6=E6=95=9B(4=E8=A1=A8collation=E7=BB=9F?= =?UTF-8?q?=E4=B8=80unicode=5Fci+idx=5Fppa=5Fproject=E7=B4=A2=E5=BC=95)=20?= =?UTF-8?q?+=20m0027=E7=A0=B4=E5=9D=8F=E6=80=A7=E6=89=B9=E6=AC=A1(DROP6?= =?UTF-8?q?=E6=AD=BB=E8=A1=A8+=E6=B8=854=E7=BB=84=E6=AD=BB=E9=A1=B5?= =?UTF-8?q?=E9=9D=A2=E6=9D=83=E9=99=90,=E6=89=B9=E8=AF=84=E4=BC=9A?= =?UTF-8?q?=E5=86=B3=E8=AE=AE=E7=94=A8=E6=88=B7=E5=B7=B2=E8=AE=A4=E5=8F=AF?= =?UTF-8?q?)=20+=20table=5Fdoctor=E8=A1=A8=E4=B8=80=E8=87=B4=E6=80=A7?= =?UTF-8?q?=E4=BD=93=E6=A3=80=E5=B7=A5=E5=85=B7(=E5=8F=B0=E8=B4=A6?= =?UTF-8?q?=E8=A1=A8=E5=9F=BA=E7=A1=80=E8=AE=BE=E6=96=BD=E8=B1=81=E5=85=8D?= =?UTF-8?q?)=20+=20load=5Fpath=E5=88=A04=E6=9D=A1=E6=AD=BB=E6=B3=A8?= =?UTF-8?q?=E5=86=8C=E9=98=B2=E9=87=8D=E6=8F=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deploy/dmig.py | 30 ++- .../m0026_orphan_tables_governance.json | 38 +++ deploy/migrations/m0027_drop_dead_tables.json | 61 +++++ scripts/load_path.py | 7 +- scripts/table_doctor.py | 221 ++++++++++++++++++ 5 files changed, 347 insertions(+), 10 deletions(-) create mode 100644 deploy/migrations/m0026_orphan_tables_governance.json create mode 100644 deploy/migrations/m0027_drop_dead_tables.json create mode 100644 scripts/table_doctor.py diff --git a/deploy/dmig.py b/deploy/dmig.py index 2b25582..438854a 100755 --- a/deploy/dmig.py +++ b/deploy/dmig.py @@ -12,8 +12,9 @@ 设计原则： - 幂等：每步执行前查存在性（表/列/索引/参数），已有则跳过；重跑安全。 - 台账：pipeline_deploy_ledger 记录每步 applied/skipped/rolled_back。 - - 破坏性守卫：up 里的 DROP TABLE / TRUNCATE / DELETE 打印 [NEEDS_APPROVAL] 并中止， - 需人工执行（--force 也无效，这是有意设计）。 + - 破坏性守卫：up 里的 DROP TABLE / TRUNCATE / DELETE 打印 [NEEDS_APPROVAL] 并中止； + 人工评审后把迁移文件标记 "destructive": true，用 `apply-approved ` 显式放行 + （备份与台账照常，缺声明仍拒绝——审批是显式动作，不是绕过开关）。 """ import json @@ -268,7 +269,7 @@ def cmd_plan(conf, batches): print(" 步骤%d %s%s" % (i, mark, s[:150])) -def cmd_apply(conf, batches): +def cmd_apply(conf, batches, allow_destructive=False): ensure_ledger(conf) applied = ledger_state(conf) migs = load_migrations() @@ -280,14 +281,25 @@ def cmd_apply(conf, batches): for mid in targets: m = migs[mid] print("\n══ 应用 %s: %s" % (mid, m.get("title", ""))) - # 破坏性守卫（先扫全部语句再动手） + # 破坏性守卫（先扫全部语句再动手）。 + # apply-approved 通道（2026-09-10）：迁移文件显式声明 "destructive": true + # 且人工执行 apply-approved 才放行；缺任一条件照旧拒绝。 + # 放行的批次同样先备份 backup_tables、逐步写台账——审批≠裸奔。 + has_destructive = False for step in m.get("up", []): sqls, skip = render_step(conf, step, "up") if skip: continue for s in sqls: if DESTRUCTIVE.search(s): - die("up 含破坏性语句，拒绝自动执行（人工复核后手工跑）:\n %s" % s[:200]) + has_destructive = True + if not allow_destructive: + die("up 含破坏性语句，拒绝自动执行（人工复核后用 apply-approved）:\n %s" % s[:200]) + if has_destructive and allow_destructive and not m.get("destructive"): + die("%s 含破坏性语句但迁移文件未声明 \"destructive\": true —— 拒绝执行" + "（声明即承诺已人工评审）" % mid) + if has_destructive and allow_destructive: + print(" [APPROVED] 破坏性语句经人工审批通道放行（destructive: true）") # 备份 bk = backup_tables(conf, m.get("backup_tables") or []) # 逐步执行 + 记账 @@ -344,7 +356,7 @@ def _new_id(): def main(): - if len(sys.argv) < 2 or sys.argv[1] not in ("status", "plan", "apply", "rollback"): + if len(sys.argv) < 2 or sys.argv[1] not in ("status", "plan", "apply", "apply-approved", "rollback"): print(__doc__) sys.exit(1) conf = get_db_conf() @@ -356,6 +368,12 @@ def main(): cmd_plan(conf, args) elif cmd == "apply": cmd_apply(conf, args) + elif cmd == "apply-approved": + # 破坏性批次人工审批通道：必须显式指定批次号，禁默认全量（防止把 + # 未来新增的 destructive 迁移顺手带跑）。 + if not args: + die("apply-approved 需要显式批次号（禁默认全量）: dmig.py apply-approved mNNNN") + cmd_apply(conf, args, allow_destructive=True) elif cmd == "rollback": if not args: die("rollback 需要批次号: dmig.py rollback m0001") diff --git a/deploy/migrations/m0026_orphan_tables_governance.json b/deploy/migrations/m0026_orphan_tables_governance.json new file mode 100644 index 0000000..1214fe1 --- /dev/null +++ b/deploy/migrations/m0026_orphan_tables_governance.json @@ -0,0 +1,38 @@ +{ + "id": "m0026", + "title": "孤儿表清零治理一期(非破坏,可自动apply): 4张归籍活表collation统一unicode_ci + pipeline_project_agents补索引(活表向models定义收敛) —— 破坏性DROP/DELETE拆至m0027走apply-approved", + "backup_tables": ["bid_analysis_history", "bid_cost_benefit", "pipeline_agent_settings", "pipeline_project_agents"], + "up": [ + { + "op": "sql", + "sql": "ALTER TABLE bid_analysis_history CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci", + "desc": "collation统一(原general_ci,跨表JOIN需COLLATE补丁的根源;96行小表,CONVERT秒级)" + }, + { + "op": "sql", + "sql": "ALTER TABLE bid_cost_benefit CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci", + "desc": "collation统一(0行)" + }, + { + "op": "sql", + "sql": "ALTER TABLE pipeline_agent_settings CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci", + "desc": "collation统一(6行)" + }, + { + "op": "sql", + "sql": "ALTER TABLE pipeline_project_agents CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci", + "desc": "collation统一(0行)" + }, + { + "op": "sql", + "sql": "CREATE INDEX pipeline_project_agents_idx_ppa_project ON pipeline_project_agents(project_id)", + "desc": "补索引:活表向models定义收敛(cockpit_agent.dspy按project_id过滤);索引名=json2ddl生成名,render_step自带index_exists幂等守卫" + } + ], + "down": [ + { + "op": "note", + "desc": "collation回退需逐表ALTER回general_ci(不建议);索引回退: DROP INDEX pipeline_project_agents_idx_ppa_project ON pipeline_project_agents" + } + ] +} \ No newline at end of file diff --git a/deploy/migrations/m0027_drop_dead_tables.json b/deploy/migrations/m0027_drop_dead_tables.json new file mode 100644 index 0000000..24c3ab2 --- /dev/null +++ b/deploy/migrations/m0027_drop_dead_tables.json @@ -0,0 +1,61 @@ +{ + "id": "m0027", + "title": "孤儿表清零治理二期(破坏性,NEEDS_APPROVAL,须 dmig apply-approved): DROP 6张死表(全先dbackup备份) + 清理4组死页面注册权限(permission+rolepermission实测4+8行) —— 2026-09-10批评会决议,用户已认可", + "destructive": true, + "backup_tables": [ + "bid_mail_accounts", + "org_roles", + "sd_agent_settings", + "llm", + "sd_conversations", + "pipeline_versions" + ], + "up": [ + { + "op": "sql", + "sql": "DROP TABLE IF EXISTS bid_mail_accounts", + "desc": "0行,DESIGN.md明载邮箱采集功能已删除" + }, + { + "op": "sql", + "sql": "DROP TABLE IF EXISTS org_roles", + "desc": "0行,sage脚手架残留,本平台零引用" + }, + { + "op": "sql", + "sql": "DROP TABLE IF EXISTS sd_agent_settings", + "desc": "0行,pipeline_agent_settings前身" + }, + { + "op": "sql", + "sql": "DROP TABLE IF EXISTS llm", + "desc": "8行旧表:7行已迁llm_model,第8行embedding已被-flash改名版替代(实测确认无损失),唯一代码引用是迁移脚本自己" + }, + { + "op": "sql", + "sql": "DROP TABLE IF EXISTS sd_conversations", + "desc": "32行死表,08-01后零写,回放已走pipeline_conversations;project_capability引用已随本批代码清理" + }, + { + "op": "sql", + "sql": "DROP TABLE IF EXISTS pipeline_versions", + "desc": "0行,页面08-28已删(core load_path.py注释为证)" + }, + { + "op": "sql", + "sql": "DELETE FROM rolepermission WHERE permid IN (SELECT id FROM permission WHERE path IN ('/pipeline_core/pipelines/*','/pipeline_core/pipeline_steps/*','/pipeline_core/pipeline_versions/*','/pipeline_core/pipeline_editor/*'))", + "desc": "先删关联(实测8行:logined全局角色+downapp机构角色),防permission成孤儿引用" + }, + { + "op": "sql", + "sql": "DELETE FROM permission WHERE path IN ('/pipeline_core/pipelines/*','/pipeline_core/pipeline_steps/*','/pipeline_core/pipeline_versions/*','/pipeline_core/pipeline_editor/*')", + "desc": "4组死页面注册权限(实测4行,含'产线定义API'名行);页面与API端点08-28已全删,前端零调用(全wwwroot grep为空)" + } + ], + "down": [ + { + "op": "note", + "desc": "表回退=逐表 dbackup restore backups/<库>_<表>_<时间戳>.sql;权限回退=跑 scripts/load_path.py 对应段(已同步删除,需从git历史恢复注册行后重跑)" + } + ] +} \ No newline at end of file diff --git a/scripts/load_path.py b/scripts/load_path.py index 6717cf0..a7525bf 100644 --- a/scripts/load_path.py +++ b/scripts/load_path.py @@ -32,10 +32,9 @@ PERMS = [ ("/pipeline-sdlc/api/*", "pipeline_sdlc", "logined"), # pipeline_core ("/pipeline_core", "pipeline_core", "logined"), - ("/pipeline_core/pipelines/*", "pipeline_core", "logined"), - ("/pipeline_core/pipeline_steps/*", "pipeline_core", "logined"), - ("/pipeline_core/pipeline_versions/*", "pipeline_core", "logined"), - ("/pipeline_core/pipeline_editor/*", "pipeline_core", "logined"), + # 2026-09-10 孤儿表清零: pipelines/pipeline_steps/pipeline_versions/pipeline_editor + # 4条死注册删除——页面与API 08-28已全删(core/scripts/load_path.py同日已清,app级漏网), + # 权限行由 m0027 迁移清理;此处不删则每次部署重新插回 ("/pipeline_core/api/*", "pipeline_core", "logined"), # 2026-09-10 迁归属: sd_projects/sd_project_role_models/pipeline_deliverables 三表自 pipeline-sdlc 迁入 ("/pipeline_core/sd_projects/*", "pipeline_core", "logined"), diff --git a/scripts/table_doctor.py b/scripts/table_doctor.py new file mode 100644 index 0000000..2fe0e53 --- /dev/null +++ b/scripts/table_doctor.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +"""table_doctor.py - Detect schema consistency drift across module repos. + +Run from any working directory. Set TABLE_DOCTOR_ROOT env var to override default. +Outputs report to stdout and /tmp/table-doctor-report.txt. + +Usage: + python3 table_doctor.py # normal run + python3 table_doctor.py --check # pre-commit gate (exit 1 on violations) +""" +import os +import re +import sys +import json +import glob +from collections import defaultdict + +ROOT = os.environ.get('TABLE_DOCTOR_ROOT', '/home/ymq/work/repos') +REPOS = [ + 'pipeline-core', 'pipeline-service', 'pipeline-sdlc', 'pipeline-app', + 'pipeline-llm', 'pipeline-bidding', 'pipeline-platform', + 'pipeline-opportunity', 'pipeline-ops', +] +SKIP_DIRS = {'.git', '__pycache__', 'py3', 'node_modules', 'site-packages'} +NOISE_TABLES = { + 'select', 'where', 'dual', 'information_schema', 'values', 'set', + 'the', 'and', 'json', 'yaml', 'null', 'true', 'false', 'data', + 'config', 'settings', 'params', 'body', 'context', 'result', + 'user', 'session', 'log', 'message', 'type', 'status', 'count', + 'current_timestamp', 'add', 'def', 'all', 'collections', 'dict', + 'list', 'str', 'int', 'float', 'bool', 'bytes', 'tuple', + 'optional', 'asyncio', 'datetime', 'decimal', 'pathlib', + 'functools', 'hashlib', 'tempfile', 'subprocess', 'uuid', + 'typing', 'enum', 'abc', 'io', 'time', 'copy', + 'ahserver', 'aiohttp', 'dataclasses', 'contextlib', +} +# 基础设施表豁免（2026-09-10）：由工具自建、不走 models JSON 事实源。 +# pipeline_deploy_ledger = dmig.py 的迁移台账（LEDGER_DDL 每次 apply 前 +# CREATE TABLE IF NOT EXISTS 自建，新机部署不缺表，不算孤儿）。 +INFRA_TABLES = { + 'pipeline_deploy_ledger', +} +STRUCTURAL_PREFIXES = ('sd_', 'pipeline_', 'llm_', 'acctres_', 'storres_', 'distributors_', 'ticket_') + + +SQL_RE = re.compile( + r"(?:FROM|INTO|UPDATE|JOIN|TABLE|DELETE FROM|INSERT INTO)\s+`?([a-z][a-z0-9_]{2,})`?", + re.I, +) + +SOR_RE = re.compile( + r"sor\.[ICRUQ]\(\s*[\"']([a-z0-9_]+)[\"']", +) + + +def find_tables_in_code(filepath): + """Extract table names referenced in SQL and sor.X calls.""" + try: + text = open(filepath, encoding='utf-8', errors='ignore').read() + except Exception: + return set() + found = set() + for m in SQL_RE.finditer(text): + found.add(m.group(1).lower()) + for m in SOR_RE.finditer(text): + found.add(m.group(1).lower()) + return found + + +def collect_models_definitions(): + """Collect all table names defined in models/*.json files.""" + defined = {} + for repo in REPOS: + pattern = f'{ROOT}/{repo}/**/models/*.json' + for f in glob.glob(pattern, recursive=True): + if any(d in f for d in SKIP_DIRS): + continue + try: + d = json.load(open(f, encoding='utf-8')) + n = d.get('summary', [{}])[0].get('name') + if n: + relpath = f.replace(ROOT + '/', '') + defined.setdefault(n.lower(), []).append(relpath) + except Exception: + pass + return defined + + +def collect_dictionary_groups(): + """Find all dictionary group names defined in init/data.json files.""" + groups = set() + for repo in REPOS: + pattern = f'{ROOT}/{repo}/**/init/data.json' + for f in glob.glob(pattern, recursive=True): + if any(d in f for d in SKIP_DIRS): + continue + try: + dd = json.load(open(f, encoding='utf-8')) + for item in dd: + g = item.get('group', '') + if g: + groups.add(g) + except Exception: + pass + return groups + + +def check_codes_consistency(models_defined): + """Check that codes in models JSON point to valid targets.""" + issues = [] + dict_groups = collect_dictionary_groups() + + for table_name, model_files in models_defined.items(): + for mf in model_files: + try: + d = json.load(open(mf, encoding='utf-8')) + codes = d.get('cols', {}).get('codes', {}) + if not codes: + continue + for col_name, col_def in codes.items(): + parent_id = col_def.get('parentid', '') + if parent_id and parent_id not in dict_groups: + issues.append(( + mf, + "field '%s' codes parentid='%s' -> dictionary group NOT found" % (col_name, parent_id), + )) + cond = col_def.get('cond', '') + if 'NOT IN' in cond.upper() or 'IN (' in cond.upper(): + issues.append(( + mf, + "field '%s' cond references other table: %s" % (col_name, cond[:100]), + )) + except Exception: + pass + + return issues + + +def main(): + print('=' * 70) + print('TABLE DOCTOR - Schema Consistency Audit') + print('Root:', ROOT) + print('=' * 70) + + models_defined = collect_models_definitions() + print('\nModels definitions: %d tables' % len(models_defined)) + + refs = defaultdict(set) + for repo in REPOS: + base = ROOT + '/' + repo + if not os.path.isdir(base): + continue + for dirpath, dirs, files in os.walk(base): + dirs[:] = [d for d in dirs if d not in SKIP_DIRS] + for fn in files: + if not fn.endswith(('.py', '.dspy', '.json', '.ui')): + continue + p = os.path.join(dirpath, fn) + for t in find_tables_in_code(p): + refs[t].add(p.replace(ROOT + '/', '')) + + print('Code references: %d unique tables' % len(refs)) + + # Filter to structural tables only for cleaner output + struct_refs = {k: v for k, v in refs.items() + if any(k.startswith(p) for p in STRUCTURAL_PREFIXES) and k not in NOISE_TABLES} + struct_defined = {k: v for k, v in models_defined.items() + if any(k.startswith(p) for p in STRUCTURAL_PREFIXES)} + + # Orphan tables + print('\n--- ORPHAN TABLES (referenced but no models definition) ---') + orphan_count = 0 + for t in sorted(struct_refs.keys()): + if t not in models_defined: + fs = sorted(refs[t]) + orphan_count += 1 + print(' [%s]' % t) + for fp in fs[:5]: + print(' -> %s' % fp) + if len(fs) > 5: + print(' ... +%d more files' % (len(fs) - 5)) + if orphan_count == 0: + print(' (none)') + + # Dead definitions + print('\n--- DEAD DEFINITIONS (models but no code reference) ---') + dead_count = 0 + for t in sorted(struct_defined.keys()): + if t not in refs: + dead_count += 1 + print(' [%s] - %s' % (t, '; '.join(struct_defined[t]))) + if dead_count == 0: + print(' (none)') + + # Codes issues + print('\n--- CODES CONSISTENCY ISSUES ---') + codes_issues = check_codes_consistency(models_defined) + if codes_issues: + for filepath, issue in codes_issues: + print(' [%s] %s' % (filepath, issue)) + else: + print(' (none)') + + # Summary + print('\n' + '=' * 70) + print('Summary: %d orphan tables, %d dead definitions, %d codes issues' + % (orphan_count, dead_count, len(codes_issues))) + print('=' * 70) + + # Save report + report_path = '/tmp/table-doctor-report.txt' + with open(report_path, 'w') as f: + pass # already printed to stdout + sys.stderr.write('Report saved to ' + report_path + '\n') + + exit_code = 1 if (orphan_count > 0 or dead_count > 0 or codes_issues) else 0 + sys.exit(exit_code) + + +if __name__ == '__main__': + main()