commit f16b58568f8e45ad4f54bce5ec8fff368a135ea6 Author: yumoqing Date: Wed Aug 26 15:28:18 2026 +0800 feat: dbbackup 模块 — 从 FOMS 单体拆分出的独立模块 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d674a41 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +__pycache__/ +*.pyc +*.pyo +*.egg-info/ +dist/ +build/ +py3/ +models/mysql.ddl.sql +.DS_Store diff --git a/README.md b/README.md new file mode 100644 index 0000000..e28b91c --- /dev/null +++ b/README.md @@ -0,0 +1,31 @@ +# dbbackup — 数据库备份与日志同步管理模块 + +FOMS 平台拆出的独立模块之一。松耦合设计:表 `cluster_id` 仅存字符串 ID,不建外键, +不 import foms 核心代码,可独立部署测试。 + +## 职责 + +| 功能 | 表 | 说明 | +|---|---|---| +| 数据库复制(binlog 日志同步) | `dbbackup_replications` | MySQL 主从复制配置,原 `sync_databases` | +| 备份任务 | `dbbackup_jobs` | 全量/增量、cron、保留期、存储位置 | +| 备份记录 | `dbbackup_records` | 大小/耗时/校验/可恢复性 | + +## 端点 + +| 端点 | 说明 | +|---|---| +| `/dbbackup/api/agent_report_db_health.dspy` | Agent 上报复制状态 | +| `/dbbackup/api/run_backup.dspy` | 触发备份 | +| `/dbbackup/api/report_backup_result.dspy` | Agent 上报备份结果 | +| `/dbbackup/api/cleanup_expired_backups.dspy` | 清理过期备份 | + +## 加载 + +```python +from dbbackup.init import load_dbbackup +def init(): + load_dbbackup() +``` + +所有表共享宿主应用数据库(`get_module_dbname('dbbackup')`)。 diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..643ab37 --- /dev/null +++ b/__init__.py @@ -0,0 +1 @@ +from .init import load_dbbackup # noqa: F401 diff --git a/build.sh b/build.sh new file mode 100644 index 0000000..e170a94 --- /dev/null +++ b/build.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# dbbackup 模块 build:生成 DDL + CRUD 页面(xls2ui) +set -e +cdir=$(pwd) + +# 1. 生成 DDL +if [ -d models ]; then + json2ddl mysql models/ > "$cdir/models/mysql.ddl.sql" 2>/dev/null || echo " skip ddl" +fi + +# 2. 生成 CRUD UI(xls2ui 从 json/ 生成到 wwwroot/) +if [ -d json ]; then + cd "$cdir/json" + for f in *.json; do + echo " xls2ui $f" + xls2ui -m ../models -o ../wwwroot dbbackup "$f" 2>/dev/null || echo " skipped" + done +fi + +echo "=== dbbackup Build Complete ===" diff --git a/dbbackup/__init__.py b/dbbackup/__init__.py new file mode 100644 index 0000000..53a8877 --- /dev/null +++ b/dbbackup/__init__.py @@ -0,0 +1 @@ +# dbbackup 模块 — 数据库备份与日志同步管理 diff --git a/dbbackup/init.py b/dbbackup/init.py new file mode 100644 index 0000000..45511ab --- /dev/null +++ b/dbbackup/init.py @@ -0,0 +1,218 @@ +"""dbbackup 模块 — 数据库全量备份 + 日志同步(binlog 复制)管理。 + +独立模块,松耦合设计: + · 表 cluster_id 仅存字符串 ID,不建外键(引用 foms 核心的 clusters 表); + 引用断了仅表现为「集群不存在」,不会崩。 + · 不 import foms 核心代码,可独立部署、独立测试。 + · 备份执行通过 Agent 下发(Agent 部署在被管主机,有 mysqldump/xtrabackup 权限), + 管理中心只负责调度、记录、清理策略,不直接 SSH 连数据库。 + +表: + dbbackup_replications 数据库复制配置(binlog 日志同步,原 sync_databases) + dbbackup_jobs 备份任务(全量/增量、cron、保留期、存储位置) + dbbackup_records 备份记录(大小/耗时/校验/可恢复性) +""" +import json +from datetime import datetime +from appPublic.uniqueID import getID +from appPublic.timeUtils import curDateString, timestampstr +from sqlor.dbpools import DBPools, get_sor_context +from ahserver.serverenv import ServerEnv + +DBNAME = 'foms' + + +def _ns(params_kw): + ns = dict(params_kw) if hasattr(params_kw, 'items') else {} + for k in list(ns.keys()): + if k.endswith('_text'): + ns.pop(k) + return ns + + +def _get(params_kw, key, default=''): + return params_kw.get(key, default) if hasattr(params_kw, 'get') else default + + +# ═══════════════════════════════════════════ +# 复制配置 CRUD(binlog 日志同步) +# ═══════════════════════════════════════════ + +async def create_replication(request, params_kw): + env = ServerEnv() + async with get_sor_context(env, DBNAME) as sor: + ns = _ns(params_kw) + ns['id'] = getID() + ns['sync_status'] = 'not_configured' + ns['created_at'] = curDateString() + ns['updated_at'] = curDateString() + await sor.C('dbbackup_replications', ns) + return {'widgettype': 'Message', 'options': {'title': '成功', 'message': '复制配置创建成功', 'type': 'success'}} + + +async def update_replication(request, params_kw): + env = ServerEnv() + async with get_sor_context(env, DBNAME) as sor: + ns = _ns(params_kw) + ns['updated_at'] = curDateString() + await sor.U('dbbackup_replications', ns) + return {'widgettype': 'Message', 'options': {'title': '成功', 'message': '复制配置更新成功', 'type': 'success'}} + + +async def delete_replication(request, params_kw): + env = ServerEnv() + async with get_sor_context(env, DBNAME) as sor: + await sor.D('dbbackup_replications', {'id': _get(params_kw, 'id')}) + return {'widgettype': 'Message', 'options': {'title': '成功', 'message': '复制配置已删除', 'type': 'success'}} + + +# ═══════════════════════════════════════════ +# 备份任务 CRUD +# ═══════════════════════════════════════════ + +async def create_backup_job(request, params_kw): + env = ServerEnv() + async with get_sor_context(env, DBNAME) as sor: + ns = _ns(params_kw) + ns['id'] = getID() + ns['created_at'] = curDateString() + ns['updated_at'] = curDateString() + await sor.C('dbbackup_jobs', ns) + return {'widgettype': 'Message', 'options': {'title': '成功', 'message': '备份任务创建成功', 'type': 'success'}} + + +async def update_backup_job(request, params_kw): + env = ServerEnv() + async with get_sor_context(env, DBNAME) as sor: + ns = _ns(params_kw) + ns['updated_at'] = curDateString() + await sor.U('dbbackup_jobs', ns) + return {'widgettype': 'Message', 'options': {'title': '成功', 'message': '备份任务更新成功', 'type': 'success'}} + + +async def delete_backup_job(request, params_kw): + env = ServerEnv() + async with get_sor_context(env, DBNAME) as sor: + await sor.D('dbbackup_jobs', {'id': _get(params_kw, 'id')}) + return {'widgettype': 'Message', 'options': {'title': '成功', 'message': '备份任务已删除', 'type': 'success'}} + + +# ═══════════════════════════════════════════ +# 备份执行(调度 + 记录 + 清理) +# ═══════════════════════════════════════════ + +async def run_backup(request, params_kw): + """执行一次备份:创建 running 记录 → 下发 Agent → 等 Agent 回报结果。 + + 备份动作本身由 Agent 在被管主机执行(mysqldump / xtrabackup),管理中心只调度。 + 这里创建记录并返回命令描述,Agent 通过 poll_backup_commands 取任务。 + """ + env = ServerEnv() + job_id = _get(params_kw, 'job_id') + async with get_sor_context(env, DBNAME) as sor: + jobs = await sor.R('dbbackup_jobs', {'id': job_id}) + if not jobs: + return {'status': 'error', 'message': '备份任务不存在'} + job = jobs[0] + rec_id = getID() + now = curDateString() + await sor.C('dbbackup_records', { + 'id': rec_id, + 'job_id': job_id, + 'cluster_id': getattr(job, 'cluster_id', ''), + 'db_name': getattr(job, 'db_name', ''), + 'backup_type': getattr(job, 'backup_type', 'full'), + 'status': 'running', + 'started_at': now, + }) + await sor.U('dbbackup_jobs', {'id': job_id, 'last_run_at': now, 'updated_at': now}) + return {'status': 'ok', 'message': '备份已触发', 'record_id': rec_id, + 'command': f"backup db={getattr(job, 'db_name', '')} type={getattr(job, 'backup_type', 'full')}", + 'storage_path': getattr(job, 'storage_path', '')} + + +async def report_backup_result(request, params_kw): + """Agent 上报备份结果""" + env = ServerEnv() + rec_id = _get(params_kw, 'record_id') + async with get_sor_context(env, DBNAME) as sor: + await sor.U('dbbackup_records', { + 'id': rec_id, + 'file_path': _get(params_kw, 'file_path'), + 'file_size_mb': _get(params_kw, 'file_size_mb'), + 'duration_sec': _get(params_kw, 'duration_sec'), + 'checksum': _get(params_kw, 'checksum'), + 'status': _get(params_kw, 'status', 'failed'), + 'error': _get(params_kw, 'error'), + 'completed_at': curDateString(), + }) + return {'status': 'ok'} + + +async def cleanup_expired_backups(request, params_kw=None): + """按任务保留天数清理过期备份记录(保留元数据,标记清理,实际文件由 Agent 删)""" + env = ServerEnv() + async with get_sor_context(env, DBNAME) as sor: + jobs = await sor.R('dbbackup_jobs', {}) + cleaned = 0 + for job in (jobs or []): + retention = int(getattr(job, 'retention_days', '7') or 7) + old = await sor.sqlExe( + "SELECT id FROM dbbackup_records WHERE job_id=${jid}$ AND status='success' " + "AND completed_at < DATE_SUB(NOW(), INTERVAL ${d}$ DAY)", + {'jid': getattr(job, 'id', ''), 'd': retention}) + for r in (old or []): + await sor.U('dbbackup_records', {'id': getattr(r, 'id', ''), 'status': 'expired'}) + cleaned += 1 + return {'status': 'ok', 'cleaned': cleaned} + + +# ═══════════════════════════════════════════ +# Agent 上报:复制状态(原 agent_report_health 的 db 分支) +# ═══════════════════════════════════════════ + +async def agent_report_db_health(request, params_kw): + """Agent 上报 MySQL 复制状态""" + env = ServerEnv() + db_statuses = _get(params_kw, 'db_statuses', []) + if isinstance(db_statuses, str): + db_statuses = json.loads(db_statuses) + async with get_sor_context(env, DBNAME) as sor: + for db_stat in db_statuses: + await sor.U('dbbackup_replications', { + 'id': db_stat.get('id'), + 'sync_status': db_stat.get('status'), + 'seconds_behind': db_stat.get('seconds_behind'), + 'last_error': db_stat.get('error'), + 'updated_at': curDateString(), + }) + return {'status': 'ok'} + + +# ═══════════════════════════════════════════ +# 注册 +# ═══════════════════════════════════════════ + +def load_dbbackup(): + env = ServerEnv() + # 复制配置 CRUD + env.create_replication = create_replication + env.create_dbbackup_replications = create_replication + env.update_replication = update_replication + env.update_dbbackup_replications = update_replication + env.delete_replication = delete_replication + env.delete_dbbackup_replications = delete_replication + # 备份任务 CRUD + env.create_backup_job = create_backup_job + env.create_dbbackup_jobs = create_backup_job + env.update_backup_job = update_backup_job + env.update_dbbackup_jobs = update_backup_job + env.delete_backup_job = delete_backup_job + env.delete_dbbackup_jobs = delete_backup_job + # 备份执行 + env.run_backup = run_backup + env.report_backup_result = report_backup_result + env.cleanup_expired_backups = cleanup_expired_backups + # Agent + env.agent_report_db_health = agent_report_db_health + return True diff --git a/json/dbbackup_jobs_list.json b/json/dbbackup_jobs_list.json new file mode 100644 index 0000000..003f021 --- /dev/null +++ b/json/dbbackup_jobs_list.json @@ -0,0 +1,17 @@ +{ + "tblname": "dbbackup_jobs", + "alias": "dbbackup_jobs_list", + "title": "数据库备份任务", + "params": { + "sortby": ["updated_at desc"], + "browserfields": { + "exclouded": ["id"], + "alters": { + "backup_type": {"uitype": "code", "data": [{"value": "full", "text": "全量"}, {"value": "incremental", "text": "增量"}]}, + "compress": {"uitype": "code", "data": [{"value": "1", "text": "压缩"}, {"value": "0", "text": "不压缩"}]}, + "enabled": {"uitype": "code", "data": [{"value": "1", "text": "启用"}, {"value": "0", "text": "禁用"}]}, + "cluster_id": {"uitype": "code", "valueField": "id", "textField": "name"} + } + } + } +} diff --git a/json/dbbackup_records_list.json b/json/dbbackup_records_list.json new file mode 100644 index 0000000..80a3c79 --- /dev/null +++ b/json/dbbackup_records_list.json @@ -0,0 +1,16 @@ +{ + "tblname": "dbbackup_records", + "alias": "dbbackup_records_list", + "title": "数据库备份记录", + "params": { + "sortby": ["started_at desc"], + "browserfields": { + "exclouded": ["id"], + "alters": { + "backup_type": {"uitype": "code", "data": [{"value": "full", "text": "全量"}, {"value": "incremental", "text": "增量"}]}, + "status": {"uitype": "code", "data": [{"value": "running", "text": "进行中"}, {"value": "success", "text": "成功"}, {"value": "failed", "text": "失败"}, {"value": "expired", "text": "已过期"}]}, + "job_id": {"uitype": "code", "valueField": "id", "textField": "job_name"} + } + } + } +} diff --git a/json/dbbackup_replications_list.json b/json/dbbackup_replications_list.json new file mode 100644 index 0000000..6518ddc --- /dev/null +++ b/json/dbbackup_replications_list.json @@ -0,0 +1,16 @@ +{ + "tblname": "dbbackup_replications", + "alias": "dbbackup_replications_list", + "title": "数据库复制配置", + "params": { + "sortby": ["updated_at desc"], + "browserfields": { + "exclouded": ["id", "repl_password"], + "alters": { + "sync_status": {"uitype": "code"}, + "enabled": {"uitype": "code", "data": [{"value": "1", "text": "启用"}, {"value": "0", "text": "禁用"}]}, + "cluster_id": {"uitype": "code", "valueField": "id", "textField": "name"} + } + } + } +} diff --git a/models/dbbackup_jobs.json b/models/dbbackup_jobs.json new file mode 100644 index 0000000..196299f --- /dev/null +++ b/models/dbbackup_jobs.json @@ -0,0 +1,33 @@ +{ + "summary": [ + { + "name": "dbbackup_jobs", + "title": "数据库备份任务", + "primary": ["id"], + "catelog": "entity" + } + ], + "fields": [ + {"name": "id", "title": "任务ID", "type": "str", "length": 32, "nullable": "no"}, + {"name": "cluster_id", "title": "所属集群", "type": "str", "length": 32, "nullable": "no"}, + {"name": "job_name", "title": "任务名称", "type": "str", "length": 128, "nullable": "no"}, + {"name": "db_name", "title": "数据库名", "type": "str", "length": 128, "nullable": "no"}, + {"name": "backup_type", "title": "备份类型", "type": "str", "length": 16, "nullable": "no", "default": "full"}, + {"name": "schedule", "title": "调度(cron表达式)", "type": "str", "length": 64}, + {"name": "retention_days", "title": "保留天数", "type": "int", "nullable": "no", "default": "7"}, + {"name": "storage_path", "title": "备份存储路径", "type": "str", "length": 512, "nullable": "no"}, + {"name": "compress", "title": "压缩", "type": "str", "length": 1, "nullable": "no", "default": "1"}, + {"name": "enabled", "title": "启用", "type": "str", "length": 1, "nullable": "no", "default": "1"}, + {"name": "last_run_at", "title": "上次执行时间", "type": "datetime"}, + {"name": "created_at", "title": "创建时间", "type": "datetime"}, + {"name": "updated_at", "title": "更新时间", "type": "datetime"} + ], + "indexes": [ + {"name": "idx_dbbak_job_cluster", "idxtype": "index", "idxfields": ["cluster_id"]}, + {"name": "idx_dbbak_job_name", "idxtype": "index", "idxfields": ["db_name"]} + ], + "codes": [ + {"field": "backup_type", "table": "appcodes_kv", "valuefield": "k", "textfield": "v", "cond": "parentid='backup_type'"}, + {"field": "cluster_id", "table": "clusters", "valuefield": "id", "textfield": "name"} + ] +} diff --git a/models/dbbackup_records.json b/models/dbbackup_records.json new file mode 100644 index 0000000..bbbc5ef --- /dev/null +++ b/models/dbbackup_records.json @@ -0,0 +1,34 @@ +{ + "summary": [ + { + "name": "dbbackup_records", + "title": "数据库备份记录", + "primary": ["id"], + "catelog": "entity" + } + ], + "fields": [ + {"name": "id", "title": "记录ID", "type": "str", "length": 32, "nullable": "no"}, + {"name": "job_id", "title": "备份任务", "type": "str", "length": 32}, + {"name": "cluster_id", "title": "所属集群", "type": "str", "length": 32, "nullable": "no"}, + {"name": "db_name", "title": "数据库名", "type": "str", "length": 128, "nullable": "no"}, + {"name": "backup_type", "title": "备份类型", "type": "str", "length": 16}, + {"name": "file_path", "title": "备份文件路径", "type": "str", "length": 512}, + {"name": "file_size_mb", "title": "文件大小(MB)", "type": "int"}, + {"name": "duration_sec", "title": "耗时(秒)", "type": "int"}, + {"name": "checksum", "title": "校验和", "type": "str", "length": 128}, + {"name": "status", "title": "状态", "type": "str", "length": 16, "nullable": "no", "default": "running"}, + {"name": "error", "title": "错误信息", "type": "text"}, + {"name": "started_at", "title": "开始时间", "type": "datetime"}, + {"name": "completed_at", "title": "完成时间", "type": "datetime"} + ], + "indexes": [ + {"name": "idx_dbbak_rec_job", "idxtype": "index", "idxfields": ["job_id"]}, + {"name": "idx_dbbak_rec_cluster", "idxtype": "index", "idxfields": ["cluster_id"]} + ], + "codes": [ + {"field": "backup_type", "table": "appcodes_kv", "valuefield": "k", "textfield": "v", "cond": "parentid='backup_type'"}, + {"field": "status", "table": "appcodes_kv", "valuefield": "k", "textfield": "v", "cond": "parentid='backup_status'"}, + {"field": "job_id", "table": "dbbackup_jobs", "valuefield": "id", "textfield": "job_name"} + ] +} diff --git a/models/dbbackup_replications.json b/models/dbbackup_replications.json new file mode 100644 index 0000000..b190288 --- /dev/null +++ b/models/dbbackup_replications.json @@ -0,0 +1,35 @@ +{ + "summary": [ + { + "name": "dbbackup_replications", + "title": "数据库复制配置", + "primary": ["id"], + "catelog": "entity" + } + ], + "fields": [ + {"name": "id", "title": "配置ID", "type": "str", "length": 32, "nullable": "no"}, + {"name": "cluster_id", "title": "所属集群", "type": "str", "length": 32, "nullable": "no"}, + {"name": "db_name", "title": "数据库名", "type": "str", "length": 128, "nullable": "no"}, + {"name": "master_host", "title": "主库主机", "type": "str", "length": 255}, + {"name": "master_port", "title": "主库端口", "type": "int", "default": "3306"}, + {"name": "slave_host", "title": "从库主机", "type": "str", "length": 255}, + {"name": "slave_port", "title": "从库端口", "type": "int", "default": "3306"}, + {"name": "repl_user", "title": "复制用户", "type": "str", "length": 64}, + {"name": "repl_password", "title": "复制密码", "type": "str", "length": 255}, + {"name": "sync_status", "title": "同步状态", "type": "str", "length": 32, "default": "not_configured"}, + {"name": "seconds_behind", "title": "延迟秒数", "type": "int"}, + {"name": "last_error", "title": "最后错误", "type": "text"}, + {"name": "enabled", "title": "启用", "type": "str", "length": 1, "nullable": "no", "default": "1"}, + {"name": "created_at", "title": "创建时间", "type": "datetime"}, + {"name": "updated_at", "title": "更新时间", "type": "datetime"} + ], + "indexes": [ + {"name": "idx_dbbak_rep_cluster", "idxtype": "index", "idxfields": ["cluster_id"]}, + {"name": "idx_dbbak_rep_name", "idxtype": "index", "idxfields": ["db_name"]} + ], + "codes": [ + {"field": "sync_status", "table": "appcodes_kv", "valuefield": "k", "textfield": "v", "cond": "parentid='sync_status'"}, + {"field": "cluster_id", "table": "clusters", "valuefield": "id", "textfield": "name"} + ] +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..60d4847 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,16 @@ +[build-system] +requires = ["setuptools>=45", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "dbbackup" +version = "1.0.0" +description = "数据库备份与日志同步管理模块 — 全量/增量备份、binlog 复制配置、备份记录与清理" +requires-python = ">=3.8" +dependencies = [ + "sqlor", +] + +[tool.setuptools.packages.find] +where = ["."] +include = ["dbbackup*"] diff --git a/wwwroot/api/agent_report_db_health.dspy b/wwwroot/api/agent_report_db_health.dspy new file mode 100644 index 0000000..93d572b --- /dev/null +++ b/wwwroot/api/agent_report_db_health.dspy @@ -0,0 +1,3 @@ +# Agent 上报 MySQL 复制状态(binlog 日志同步) +result = await agent_report_db_health(request, params_kw) +return json.dumps(result, ensure_ascii=False) diff --git a/wwwroot/api/cleanup_expired_backups.dspy b/wwwroot/api/cleanup_expired_backups.dspy new file mode 100644 index 0000000..3a307f7 --- /dev/null +++ b/wwwroot/api/cleanup_expired_backups.dspy @@ -0,0 +1,3 @@ +# 按保留天数清理过期备份 +result = await cleanup_expired_backups(request, params_kw) +return json.dumps(result, ensure_ascii=False) diff --git a/wwwroot/api/report_backup_result.dspy b/wwwroot/api/report_backup_result.dspy new file mode 100644 index 0000000..4b18ac9 --- /dev/null +++ b/wwwroot/api/report_backup_result.dspy @@ -0,0 +1,3 @@ +# Agent 上报备份结果 +result = await report_backup_result(request, params_kw) +return json.dumps(result, ensure_ascii=False) diff --git a/wwwroot/api/run_backup.dspy b/wwwroot/api/run_backup.dspy new file mode 100644 index 0000000..6ca6ca6 --- /dev/null +++ b/wwwroot/api/run_backup.dspy @@ -0,0 +1,3 @@ +# 触发一次数据库备份(管理中心调度,Agent 执行) +result = await run_backup(request, params_kw) +return json.dumps(result, ensure_ascii=False)