feat(deploy): 孤儿表清零治理——dmig新增apply-approved破坏性审批通道(destructive:true显式声明+批次号必填,备份台账照常) + m0026归籍活表收敛(4表collation统一unicode_ci+idx_ppa_project索引) + m0027破坏性批次(DROP6死表+清4组死页面权限,批评会决议用户已认可) + table_doctor表一致性体检工具(台账表基础设施豁免) + load_path删4条死注册防重插
This commit is contained in:
parent
e04dc1a304
commit
e294fc7a6b
@ -12,8 +12,9 @@
|
||||
设计原则：
|
||||
- 幂等：每步执行前查存在性（表/列/索引/参数），已有则跳过；重跑安全。
|
||||
- 台账：pipeline_deploy_ledger 记录每步 applied/skipped/rolled_back。
|
||||
- 破坏性守卫：up 里的 DROP TABLE / TRUNCATE / DELETE 打印 [NEEDS_APPROVAL] 并中止，
|
||||
需人工执行（--force 也无效，这是有意设计）。
|
||||
- 破坏性守卫：up 里的 DROP TABLE / TRUNCATE / DELETE 打印 [NEEDS_APPROVAL] 并中止；
|
||||
人工评审后把迁移文件标记 "destructive": true，用 `apply-approved <batch>` 显式放行
|
||||
（备份与台账照常，缺声明仍拒绝——审批是显式动作，不是绕过开关）。
|
||||
"""
|
||||
|
||||
import json
|
||||
@ -268,7 +269,7 @@ def cmd_plan(conf, batches):
|
||||
print(" 步骤%d %s%s" % (i, mark, s[:150]))
|
||||
|
||||
|
||||
def cmd_apply(conf, batches):
|
||||
def cmd_apply(conf, batches, allow_destructive=False):
|
||||
ensure_ledger(conf)
|
||||
applied = ledger_state(conf)
|
||||
migs = load_migrations()
|
||||
@ -280,14 +281,25 @@ def cmd_apply(conf, batches):
|
||||
for mid in targets:
|
||||
m = migs[mid]
|
||||
print("\n══ 应用 %s: %s" % (mid, m.get("title", "")))
|
||||
# 破坏性守卫（先扫全部语句再动手）
|
||||
# 破坏性守卫（先扫全部语句再动手）。
|
||||
# apply-approved 通道（2026-09-10）：迁移文件显式声明 "destructive": true
|
||||
# 且人工执行 apply-approved 才放行；缺任一条件照旧拒绝。
|
||||
# 放行的批次同样先备份 backup_tables、逐步写台账——审批≠裸奔。
|
||||
has_destructive = False
|
||||
for step in m.get("up", []):
|
||||
sqls, skip = render_step(conf, step, "up")
|
||||
if skip:
|
||||
continue
|
||||
for s in sqls:
|
||||
if DESTRUCTIVE.search(s):
|
||||
die("up 含破坏性语句，拒绝自动执行（人工复核后手工跑）:\n %s" % s[:200])
|
||||
has_destructive = True
|
||||
if not allow_destructive:
|
||||
die("up 含破坏性语句，拒绝自动执行（人工复核后用 apply-approved）:\n %s" % s[:200])
|
||||
if has_destructive and allow_destructive and not m.get("destructive"):
|
||||
die("%s 含破坏性语句但迁移文件未声明 \"destructive\": true —— 拒绝执行"
|
||||
"（声明即承诺已人工评审）" % mid)
|
||||
if has_destructive and allow_destructive:
|
||||
print(" [APPROVED] 破坏性语句经人工审批通道放行（destructive: true）")
|
||||
# 备份
|
||||
bk = backup_tables(conf, m.get("backup_tables") or [])
|
||||
# 逐步执行 + 记账
|
||||
@ -344,7 +356,7 @@ def _new_id():
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2 or sys.argv[1] not in ("status", "plan", "apply", "rollback"):
|
||||
if len(sys.argv) < 2 or sys.argv[1] not in ("status", "plan", "apply", "apply-approved", "rollback"):
|
||||
print(__doc__)
|
||||
sys.exit(1)
|
||||
conf = get_db_conf()
|
||||
@ -356,6 +368,12 @@ def main():
|
||||
cmd_plan(conf, args)
|
||||
elif cmd == "apply":
|
||||
cmd_apply(conf, args)
|
||||
elif cmd == "apply-approved":
|
||||
# 破坏性批次人工审批通道：必须显式指定批次号，禁默认全量（防止把
|
||||
# 未来新增的 destructive 迁移顺手带跑）。
|
||||
if not args:
|
||||
die("apply-approved 需要显式批次号（禁默认全量）: dmig.py apply-approved mNNNN")
|
||||
cmd_apply(conf, args, allow_destructive=True)
|
||||
elif cmd == "rollback":
|
||||
if not args:
|
||||
die("rollback 需要批次号: dmig.py rollback m0001")
|
||||
|
||||
38
deploy/migrations/m0026_orphan_tables_governance.json
Normal file
38
deploy/migrations/m0026_orphan_tables_governance.json
Normal file
@ -0,0 +1,38 @@
|
||||
{
|
||||
"id": "m0026",
|
||||
"title": "孤儿表清零治理一期(非破坏,可自动apply): 4张归籍活表collation统一unicode_ci + pipeline_project_agents补索引(活表向models定义收敛) —— 破坏性DROP/DELETE拆至m0027走apply-approved",
|
||||
"backup_tables": ["bid_analysis_history", "bid_cost_benefit", "pipeline_agent_settings", "pipeline_project_agents"],
|
||||
"up": [
|
||||
{
|
||||
"op": "sql",
|
||||
"sql": "ALTER TABLE bid_analysis_history CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci",
|
||||
"desc": "collation统一(原general_ci,跨表JOIN需COLLATE补丁的根源;96行小表,CONVERT秒级)"
|
||||
},
|
||||
{
|
||||
"op": "sql",
|
||||
"sql": "ALTER TABLE bid_cost_benefit CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci",
|
||||
"desc": "collation统一(0行)"
|
||||
},
|
||||
{
|
||||
"op": "sql",
|
||||
"sql": "ALTER TABLE pipeline_agent_settings CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci",
|
||||
"desc": "collation统一(6行)"
|
||||
},
|
||||
{
|
||||
"op": "sql",
|
||||
"sql": "ALTER TABLE pipeline_project_agents CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci",
|
||||
"desc": "collation统一(0行)"
|
||||
},
|
||||
{
|
||||
"op": "sql",
|
||||
"sql": "CREATE INDEX pipeline_project_agents_idx_ppa_project ON pipeline_project_agents(project_id)",
|
||||
"desc": "补索引:活表向models定义收敛(cockpit_agent.dspy按project_id过滤);索引名=json2ddl生成名,render_step自带index_exists幂等守卫"
|
||||
}
|
||||
],
|
||||
"down": [
|
||||
{
|
||||
"op": "note",
|
||||
"desc": "collation回退需逐表ALTER回general_ci(不建议);索引回退: DROP INDEX pipeline_project_agents_idx_ppa_project ON pipeline_project_agents"
|
||||
}
|
||||
]
|
||||
}
|
||||
61
deploy/migrations/m0027_drop_dead_tables.json
Normal file
61
deploy/migrations/m0027_drop_dead_tables.json
Normal file
@ -0,0 +1,61 @@
|
||||
{
|
||||
"id": "m0027",
|
||||
"title": "孤儿表清零治理二期(破坏性,NEEDS_APPROVAL,须 dmig apply-approved): DROP 6张死表(全先dbackup备份) + 清理4组死页面注册权限(permission+rolepermission实测4+8行) —— 2026-09-10批评会决议,用户已认可",
|
||||
"destructive": true,
|
||||
"backup_tables": [
|
||||
"bid_mail_accounts",
|
||||
"org_roles",
|
||||
"sd_agent_settings",
|
||||
"llm",
|
||||
"sd_conversations",
|
||||
"pipeline_versions"
|
||||
],
|
||||
"up": [
|
||||
{
|
||||
"op": "sql",
|
||||
"sql": "DROP TABLE IF EXISTS bid_mail_accounts",
|
||||
"desc": "0行,DESIGN.md明载邮箱采集功能已删除"
|
||||
},
|
||||
{
|
||||
"op": "sql",
|
||||
"sql": "DROP TABLE IF EXISTS org_roles",
|
||||
"desc": "0行,sage脚手架残留,本平台零引用"
|
||||
},
|
||||
{
|
||||
"op": "sql",
|
||||
"sql": "DROP TABLE IF EXISTS sd_agent_settings",
|
||||
"desc": "0行,pipeline_agent_settings前身"
|
||||
},
|
||||
{
|
||||
"op": "sql",
|
||||
"sql": "DROP TABLE IF EXISTS llm",
|
||||
"desc": "8行旧表:7行已迁llm_model,第8行embedding已被-flash改名版替代(实测确认无损失),唯一代码引用是迁移脚本自己"
|
||||
},
|
||||
{
|
||||
"op": "sql",
|
||||
"sql": "DROP TABLE IF EXISTS sd_conversations",
|
||||
"desc": "32行死表,08-01后零写,回放已走pipeline_conversations;project_capability引用已随本批代码清理"
|
||||
},
|
||||
{
|
||||
"op": "sql",
|
||||
"sql": "DROP TABLE IF EXISTS pipeline_versions",
|
||||
"desc": "0行,页面08-28已删(core load_path.py注释为证)"
|
||||
},
|
||||
{
|
||||
"op": "sql",
|
||||
"sql": "DELETE FROM rolepermission WHERE permid IN (SELECT id FROM permission WHERE path IN ('/pipeline_core/pipelines/*','/pipeline_core/pipeline_steps/*','/pipeline_core/pipeline_versions/*','/pipeline_core/pipeline_editor/*'))",
|
||||
"desc": "先删关联(实测8行:logined全局角色+downapp机构角色),防permission成孤儿引用"
|
||||
},
|
||||
{
|
||||
"op": "sql",
|
||||
"sql": "DELETE FROM permission WHERE path IN ('/pipeline_core/pipelines/*','/pipeline_core/pipeline_steps/*','/pipeline_core/pipeline_versions/*','/pipeline_core/pipeline_editor/*')",
|
||||
"desc": "4组死页面注册权限(实测4行,含'产线定义API'名行);页面与API端点08-28已全删,前端零调用(全wwwroot grep为空)"
|
||||
}
|
||||
],
|
||||
"down": [
|
||||
{
|
||||
"op": "note",
|
||||
"desc": "表回退=逐表 dbackup restore backups/<库>_<表>_<时间戳>.sql;权限回退=跑 scripts/load_path.py 对应段(已同步删除,需从git历史恢复注册行后重跑)"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -32,10 +32,9 @@ PERMS = [
|
||||
("/pipeline-sdlc/api/*", "pipeline_sdlc", "logined"),
|
||||
# pipeline_core
|
||||
("/pipeline_core", "pipeline_core", "logined"),
|
||||
("/pipeline_core/pipelines/*", "pipeline_core", "logined"),
|
||||
("/pipeline_core/pipeline_steps/*", "pipeline_core", "logined"),
|
||||
("/pipeline_core/pipeline_versions/*", "pipeline_core", "logined"),
|
||||
("/pipeline_core/pipeline_editor/*", "pipeline_core", "logined"),
|
||||
# 2026-09-10 孤儿表清零: pipelines/pipeline_steps/pipeline_versions/pipeline_editor
|
||||
# 4条死注册删除——页面与API 08-28已全删(core/scripts/load_path.py同日已清,app级漏网),
|
||||
# 权限行由 m0027 迁移清理;此处不删则每次部署重新插回
|
||||
("/pipeline_core/api/*", "pipeline_core", "logined"),
|
||||
# 2026-09-10 迁归属: sd_projects/sd_project_role_models/pipeline_deliverables 三表自 pipeline-sdlc 迁入
|
||||
("/pipeline_core/sd_projects/*", "pipeline_core", "logined"),
|
||||
|
||||
221
scripts/table_doctor.py
Normal file
221
scripts/table_doctor.py
Normal file
@ -0,0 +1,221 @@
|
||||
#!/usr/bin/env python3
|
||||
"""table_doctor.py - Detect schema consistency drift across module repos.
|
||||
|
||||
Run from any working directory. Set TABLE_DOCTOR_ROOT env var to override default.
|
||||
Outputs report to stdout and /tmp/table-doctor-report.txt.
|
||||
|
||||
Usage:
|
||||
python3 table_doctor.py # normal run
|
||||
python3 table_doctor.py --check # pre-commit gate (exit 1 on violations)
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import json
|
||||
import glob
|
||||
from collections import defaultdict
|
||||
|
||||
ROOT = os.environ.get('TABLE_DOCTOR_ROOT', '/home/ymq/work/repos')
|
||||
REPOS = [
|
||||
'pipeline-core', 'pipeline-service', 'pipeline-sdlc', 'pipeline-app',
|
||||
'pipeline-llm', 'pipeline-bidding', 'pipeline-platform',
|
||||
'pipeline-opportunity', 'pipeline-ops',
|
||||
]
|
||||
SKIP_DIRS = {'.git', '__pycache__', 'py3', 'node_modules', 'site-packages'}
|
||||
NOISE_TABLES = {
|
||||
'select', 'where', 'dual', 'information_schema', 'values', 'set',
|
||||
'the', 'and', 'json', 'yaml', 'null', 'true', 'false', 'data',
|
||||
'config', 'settings', 'params', 'body', 'context', 'result',
|
||||
'user', 'session', 'log', 'message', 'type', 'status', 'count',
|
||||
'current_timestamp', 'add', 'def', 'all', 'collections', 'dict',
|
||||
'list', 'str', 'int', 'float', 'bool', 'bytes', 'tuple',
|
||||
'optional', 'asyncio', 'datetime', 'decimal', 'pathlib',
|
||||
'functools', 'hashlib', 'tempfile', 'subprocess', 'uuid',
|
||||
'typing', 'enum', 'abc', 'io', 'time', 'copy',
|
||||
'ahserver', 'aiohttp', 'dataclasses', 'contextlib',
|
||||
}
|
||||
# 基础设施表豁免（2026-09-10）：由工具自建、不走 models JSON 事实源。
|
||||
# pipeline_deploy_ledger = dmig.py 的迁移台账（LEDGER_DDL 每次 apply 前
|
||||
# CREATE TABLE IF NOT EXISTS 自建，新机部署不缺表，不算孤儿）。
|
||||
INFRA_TABLES = {
|
||||
'pipeline_deploy_ledger',
|
||||
}
|
||||
STRUCTURAL_PREFIXES = ('sd_', 'pipeline_', 'llm_', 'acctres_', 'storres_', 'distributors_', 'ticket_')
|
||||
|
||||
|
||||
SQL_RE = re.compile(
|
||||
r"(?:FROM|INTO|UPDATE|JOIN|TABLE|DELETE FROM|INSERT INTO)\s+`?([a-z][a-z0-9_]{2,})`?",
|
||||
re.I,
|
||||
)
|
||||
|
||||
SOR_RE = re.compile(
|
||||
r"sor\.[ICRUQ]\(\s*[\"']([a-z0-9_]+)[\"']",
|
||||
)
|
||||
|
||||
|
||||
def find_tables_in_code(filepath):
|
||||
"""Extract table names referenced in SQL and sor.X calls."""
|
||||
try:
|
||||
text = open(filepath, encoding='utf-8', errors='ignore').read()
|
||||
except Exception:
|
||||
return set()
|
||||
found = set()
|
||||
for m in SQL_RE.finditer(text):
|
||||
found.add(m.group(1).lower())
|
||||
for m in SOR_RE.finditer(text):
|
||||
found.add(m.group(1).lower())
|
||||
return found
|
||||
|
||||
|
||||
def collect_models_definitions():
|
||||
"""Collect all table names defined in models/*.json files."""
|
||||
defined = {}
|
||||
for repo in REPOS:
|
||||
pattern = f'{ROOT}/{repo}/**/models/*.json'
|
||||
for f in glob.glob(pattern, recursive=True):
|
||||
if any(d in f for d in SKIP_DIRS):
|
||||
continue
|
||||
try:
|
||||
d = json.load(open(f, encoding='utf-8'))
|
||||
n = d.get('summary', [{}])[0].get('name')
|
||||
if n:
|
||||
relpath = f.replace(ROOT + '/', '')
|
||||
defined.setdefault(n.lower(), []).append(relpath)
|
||||
except Exception:
|
||||
pass
|
||||
return defined
|
||||
|
||||
|
||||
def collect_dictionary_groups():
|
||||
"""Find all dictionary group names defined in init/data.json files."""
|
||||
groups = set()
|
||||
for repo in REPOS:
|
||||
pattern = f'{ROOT}/{repo}/**/init/data.json'
|
||||
for f in glob.glob(pattern, recursive=True):
|
||||
if any(d in f for d in SKIP_DIRS):
|
||||
continue
|
||||
try:
|
||||
dd = json.load(open(f, encoding='utf-8'))
|
||||
for item in dd:
|
||||
g = item.get('group', '')
|
||||
if g:
|
||||
groups.add(g)
|
||||
except Exception:
|
||||
pass
|
||||
return groups
|
||||
|
||||
|
||||
def check_codes_consistency(models_defined):
|
||||
"""Check that codes in models JSON point to valid targets."""
|
||||
issues = []
|
||||
dict_groups = collect_dictionary_groups()
|
||||
|
||||
for table_name, model_files in models_defined.items():
|
||||
for mf in model_files:
|
||||
try:
|
||||
d = json.load(open(mf, encoding='utf-8'))
|
||||
codes = d.get('cols', {}).get('codes', {})
|
||||
if not codes:
|
||||
continue
|
||||
for col_name, col_def in codes.items():
|
||||
parent_id = col_def.get('parentid', '')
|
||||
if parent_id and parent_id not in dict_groups:
|
||||
issues.append((
|
||||
mf,
|
||||
"field '%s' codes parentid='%s' -> dictionary group NOT found" % (col_name, parent_id),
|
||||
))
|
||||
cond = col_def.get('cond', '')
|
||||
if 'NOT IN' in cond.upper() or 'IN (' in cond.upper():
|
||||
issues.append((
|
||||
mf,
|
||||
"field '%s' cond references other table: %s" % (col_name, cond[:100]),
|
||||
))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return issues
|
||||
|
||||
|
||||
def main():
|
||||
print('=' * 70)
|
||||
print('TABLE DOCTOR - Schema Consistency Audit')
|
||||
print('Root:', ROOT)
|
||||
print('=' * 70)
|
||||
|
||||
models_defined = collect_models_definitions()
|
||||
print('\nModels definitions: %d tables' % len(models_defined))
|
||||
|
||||
refs = defaultdict(set)
|
||||
for repo in REPOS:
|
||||
base = ROOT + '/' + repo
|
||||
if not os.path.isdir(base):
|
||||
continue
|
||||
for dirpath, dirs, files in os.walk(base):
|
||||
dirs[:] = [d for d in dirs if d not in SKIP_DIRS]
|
||||
for fn in files:
|
||||
if not fn.endswith(('.py', '.dspy', '.json', '.ui')):
|
||||
continue
|
||||
p = os.path.join(dirpath, fn)
|
||||
for t in find_tables_in_code(p):
|
||||
refs[t].add(p.replace(ROOT + '/', ''))
|
||||
|
||||
print('Code references: %d unique tables' % len(refs))
|
||||
|
||||
# Filter to structural tables only for cleaner output
|
||||
struct_refs = {k: v for k, v in refs.items()
|
||||
if any(k.startswith(p) for p in STRUCTURAL_PREFIXES) and k not in NOISE_TABLES}
|
||||
struct_defined = {k: v for k, v in models_defined.items()
|
||||
if any(k.startswith(p) for p in STRUCTURAL_PREFIXES)}
|
||||
|
||||
# Orphan tables
|
||||
print('\n--- ORPHAN TABLES (referenced but no models definition) ---')
|
||||
orphan_count = 0
|
||||
for t in sorted(struct_refs.keys()):
|
||||
if t not in models_defined:
|
||||
fs = sorted(refs[t])
|
||||
orphan_count += 1
|
||||
print(' [%s]' % t)
|
||||
for fp in fs[:5]:
|
||||
print(' -> %s' % fp)
|
||||
if len(fs) > 5:
|
||||
print(' ... +%d more files' % (len(fs) - 5))
|
||||
if orphan_count == 0:
|
||||
print(' (none)')
|
||||
|
||||
# Dead definitions
|
||||
print('\n--- DEAD DEFINITIONS (models but no code reference) ---')
|
||||
dead_count = 0
|
||||
for t in sorted(struct_defined.keys()):
|
||||
if t not in refs:
|
||||
dead_count += 1
|
||||
print(' [%s] - %s' % (t, '; '.join(struct_defined[t])))
|
||||
if dead_count == 0:
|
||||
print(' (none)')
|
||||
|
||||
# Codes issues
|
||||
print('\n--- CODES CONSISTENCY ISSUES ---')
|
||||
codes_issues = check_codes_consistency(models_defined)
|
||||
if codes_issues:
|
||||
for filepath, issue in codes_issues:
|
||||
print(' [%s] %s' % (filepath, issue))
|
||||
else:
|
||||
print(' (none)')
|
||||
|
||||
# Summary
|
||||
print('\n' + '=' * 70)
|
||||
print('Summary: %d orphan tables, %d dead definitions, %d codes issues'
|
||||
% (orphan_count, dead_count, len(codes_issues)))
|
||||
print('=' * 70)
|
||||
|
||||
# Save report
|
||||
report_path = '/tmp/table-doctor-report.txt'
|
||||
with open(report_path, 'w') as f:
|
||||
pass # already printed to stdout
|
||||
sys.stderr.write('Report saved to ' + report_path + '\n')
|
||||
|
||||
exit_code = 1 if (orphan_count > 0 or dead_count > 0 or codes_issues) else 0
|
||||
sys.exit(exit_code)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
x
Reference in New Issue
Block a user