refactor backup_api.sh: auto-read config.json, discover all modules, decrypt password via RC4
This commit is contained in:
parent
a8652c326f
commit
477076064e
@ -1,17 +1,17 @@
|
||||
#!/bin/bash
|
||||
# ============================================================
|
||||
# backup_api.sh
|
||||
# 导出 pricing, llmage, uapi 三个模块的表数据为 SQL 文件
|
||||
# 自动从 conf/config.json 读取数据库配置，扫描所有模块的
|
||||
# models 目录提取表名，导出为 SQL 文件。
|
||||
#
|
||||
# 运行目录: sage.py 所在目录 (即 repos/sage/)
|
||||
#
|
||||
# 用法:
|
||||
# ./backup_api.sh [options]
|
||||
# cd /path/to/sage && bash bin/backup_api.sh [options]
|
||||
#
|
||||
# 选项:
|
||||
# -h HOST MySQL 主机 (默认: localhost)
|
||||
# -P PORT MySQL 端口 (默认: 3306)
|
||||
# -u USER MySQL 用户名 (默认: root)
|
||||
# -p PASSWORD MySQL 密码
|
||||
# -d DATABASE 数据库名 (默认: sage)
|
||||
# -p PASSWORD 直接指定密码 (覆盖config.json中的加密密码)
|
||||
# -h HOST 覆盖数据库主机
|
||||
# -o OUTPUT_DIR 输出目录 (默认: ./sql_dumps)
|
||||
# --no-data 只导出表结构，不导出数据
|
||||
# --help 显示帮助
|
||||
@ -19,20 +19,22 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# --- 定位工作目录 (sage.py 所在目录) ---
|
||||
SAGE_DIR="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
REPOS_DIR="$(cd "$SAGE_DIR/.." && pwd)"
|
||||
CONFIG_FILE="$SAGE_DIR/conf/config.json"
|
||||
|
||||
if [[ ! -f "$CONFIG_FILE" ]]; then
|
||||
echo "错误: 找不到配置文件 $CONFIG_FILE"
|
||||
echo "请在 sage.py 所在目录下运行此脚本"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 默认参数
|
||||
HOST="localhost"
|
||||
PORT="3306"
|
||||
USER="root"
|
||||
PASSWORD=""
|
||||
DATABASE="sage"
|
||||
OUTPUT_DIR="./sql_dumps"
|
||||
NO_DATA=""
|
||||
REPOS_DIR="$(cd "$(dirname "$0")/../.." 2>/dev/null && pwd || echo "/home/hermesai/repos")"
|
||||
|
||||
# 各模块排除的表
|
||||
declare -A MODULE_EXCLUDES
|
||||
MODULE_EXCLUDES["llmage"]="llmusage llmusage_accounting_failed llmusage_history"
|
||||
MODULE_EXCLUDES["uapi"]="uapiset uptask"
|
||||
PASSWORD_OVERRIDE=""
|
||||
HOST_OVERRIDE=""
|
||||
|
||||
usage() {
|
||||
sed -n '2,/^# ===/p' "$0" | grep '^#' | sed 's/^# \?//'
|
||||
@ -42,11 +44,8 @@ usage() {
|
||||
# 解析参数
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
-h) HOST="$2"; shift 2 ;;
|
||||
-P) PORT="$2"; shift 2 ;;
|
||||
-u) USER="$2"; shift 2 ;;
|
||||
-p) PASSWORD="$2"; shift 2 ;;
|
||||
-d) DATABASE="$2"; shift 2 ;;
|
||||
-p) PASSWORD_OVERRIDE="$2"; shift 2 ;;
|
||||
-h) HOST_OVERRIDE="$2"; shift 2 ;;
|
||||
-o) OUTPUT_DIR="$2"; shift 2 ;;
|
||||
--no-data) NO_DATA="--no-data"; shift ;;
|
||||
--help) usage ;;
|
||||
@ -54,13 +53,108 @@ while [[ $# -gt 0 ]]; do
|
||||
esac
|
||||
done
|
||||
|
||||
# 构建 mysqldump 基础命令
|
||||
MYSQLDUMP_BASE="mysqldump -h${HOST} -P${PORT} -u${USER}"
|
||||
if [[ -n "$PASSWORD" ]]; then
|
||||
MYSQLDUMP_BASE="${MYSQLDUMP_BASE} -p${PASSWORD}"
|
||||
# --- 从 config.json 提取数据库配置并解密密码 ---
|
||||
read_config() {
|
||||
# Use the sage venv if available, otherwise system python
|
||||
local PYTHON="python3"
|
||||
if [[ -x "$SAGE_DIR/py3/bin/python3" ]]; then
|
||||
PYTHON="$SAGE_DIR/py3/bin/python3"
|
||||
fi
|
||||
|
||||
$PYTHON -c "
|
||||
import re, json, sys, os
|
||||
|
||||
# 读取并修复 config.json (可能有非标准 JSON)
|
||||
text = open('$CONFIG_FILE').read()
|
||||
# 移除无 key 的裸对象 (如 hot_reload)
|
||||
text = re.sub(r',\s*\{[^{}]*\"hot_reload\"[^{}]*\{[^{}]*\}[^{}]*\}', '', text)
|
||||
config = json.loads(text)
|
||||
|
||||
# 获取数据库配置 (取第一个数据库)
|
||||
db_name = list(config['databases'].keys())[0]
|
||||
db_cfg = config['databases'][db_name]['kwargs']
|
||||
password_key = config.get('password_key', 'QRIVSRHrthhwyjy176556332')
|
||||
|
||||
# 尝试 RC4 解密
|
||||
try:
|
||||
from appPublic.rc4 import unpassword
|
||||
decrypted = unpassword(db_cfg['password'], key=password_key)
|
||||
except ImportError:
|
||||
# Fallback: inline RC4 implementation
|
||||
import base64
|
||||
from hashlib import sha1
|
||||
class RC4:
|
||||
def __init__(self):
|
||||
self.bcoding = 'iso-8859-1'
|
||||
self.dcoding = 'utf8'
|
||||
self.salt = b'AFUqx9WZuI32lnHk'
|
||||
def _crypt(self, data, key):
|
||||
x = 0; box = list(range(256))
|
||||
for i in range(256):
|
||||
x = (x + box[i] + key[i % len(key)]) % 256
|
||||
box[i], box[x] = box[x], box[i]
|
||||
x = y = 0; out = []
|
||||
for char in data:
|
||||
x = (x + 1) % 256; y = (y + box[x]) % 256
|
||||
box[x], box[y] = box[y], box[x]
|
||||
out.append(chr(char ^ box[(box[x] + box[y]) % 256]))
|
||||
return ''.join(out).encode(self.bcoding)
|
||||
def decode(self, data, key):
|
||||
if isinstance(data, str): data = data.encode(self.dcoding)
|
||||
key = key.encode(self.bcoding)
|
||||
data = base64.b64decode(data)
|
||||
a = sha1(key + self.salt); k = a.digest()
|
||||
return self._crypt(data[16:], k).decode(self.dcoding)
|
||||
rc4 = RC4()
|
||||
decrypted = rc4.decode(db_cfg['password'], password_key)
|
||||
except Exception as e:
|
||||
decrypted = ''
|
||||
|
||||
# 如果解密失败或为空，使用原始值
|
||||
if not decrypted:
|
||||
decrypted = db_cfg['password']
|
||||
|
||||
host = '$HOST_OVERRIDE' if '$HOST_OVERRIDE' else db_cfg.get('host', 'localhost')
|
||||
port = str(db_cfg.get('port', '3306'))
|
||||
user = db_cfg.get('user', 'root')
|
||||
database = db_cfg.get('db', 'sage')
|
||||
|
||||
# Shell-safe output
|
||||
print(f'HOST={host}')
|
||||
print(f'PORT={port}')
|
||||
print(f'USER={user}')
|
||||
print(f'DATABASE={database}')
|
||||
# Password needs quoting for special chars
|
||||
print(f\"PASSWORD='{decrypted}'\")
|
||||
"
|
||||
}
|
||||
|
||||
echo "正在读取配置..."
|
||||
eval "$(read_config)"
|
||||
|
||||
# 命令行密码覆盖
|
||||
if [[ -n "$PASSWORD_OVERRIDE" ]]; then
|
||||
PASSWORD="$PASSWORD_OVERRIDE"
|
||||
fi
|
||||
|
||||
# 从 models/*.json 提取表名
|
||||
if [[ -z "$PASSWORD" ]]; then
|
||||
echo "错误: 密码为空。请使用 -p 参数指定密码，或检查 config.json 中的密码配置"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 构建 mysqldump 基础命令 (密码通过环境变量传递，避免命令行暴露)
|
||||
export MYSQL_PWD="$PASSWORD"
|
||||
MYSQLDUMP_CMD="mysqldump -h${HOST} -P${PORT} -u${USER}"
|
||||
|
||||
# --- 各模块排除的表 (大数据量/日志表) ---
|
||||
declare -A MODULE_EXCLUDES
|
||||
MODULE_EXCLUDES["llmage"]="llmusage llmusage_accounting_failed llmusage_history"
|
||||
MODULE_EXCLUDES["uapi"]="uapiset uptask"
|
||||
MODULE_EXCLUDES["accounting"]="accountingdetail accountinghistory"
|
||||
MODULE_EXCLUDES["harnessed_agent"]="harnessed_agent_log"
|
||||
MODULE_EXCLUDES["harnessed_reasoning"]="harnessed_reasoning_log"
|
||||
|
||||
# --- 从 models/*.json 提取表名 ---
|
||||
get_tables() {
|
||||
local module_dir="$1"
|
||||
shift
|
||||
@ -68,13 +162,11 @@ get_tables() {
|
||||
local tables=()
|
||||
|
||||
if [[ ! -d "$module_dir/models" ]]; then
|
||||
echo "警告: 目录不存在 $module_dir/models" >&2
|
||||
return
|
||||
fi
|
||||
|
||||
for f in "$module_dir/models"/*.json; do
|
||||
[[ -f "$f" ]] || continue
|
||||
# 从 summary[0].name 提取表名
|
||||
local tbl
|
||||
tbl=$(python3 -c "
|
||||
import json, sys
|
||||
@ -91,9 +183,9 @@ except:
|
||||
if [[ -z "$tbl" ]]; then
|
||||
continue
|
||||
fi
|
||||
# 检查是否在排除列表中
|
||||
# 检查排除
|
||||
local excluded=false
|
||||
for ex in "${excludes[@]}"; do
|
||||
for ex in "${excludes[@]+"${excludes[@]}"}"; do
|
||||
if [[ "$tbl" == "$ex" ]]; then
|
||||
excluded=true
|
||||
break
|
||||
@ -106,45 +198,54 @@ except:
|
||||
echo "${tables[@]}"
|
||||
}
|
||||
|
||||
# 创建输出目录
|
||||
# --- 自动发现所有模块 ---
|
||||
discover_modules() {
|
||||
for dir in "$REPOS_DIR"/*/; do
|
||||
[[ -d "$dir/models" ]] || continue
|
||||
local name
|
||||
name=$(basename "$dir")
|
||||
# 跳过无模型文件的模块
|
||||
if ls "$dir/models/"*.json &>/dev/null; then
|
||||
echo "$name"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# --- 创建输出目录 ---
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
||||
|
||||
echo "============================================================"
|
||||
echo " Sage 模块表数据导出"
|
||||
echo " Sage 全模块表数据导出"
|
||||
echo " 时间: $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo " 数据库: ${DATABASE}@${HOST}:${PORT}"
|
||||
echo " 用户: ${USER}"
|
||||
echo " 模块目录: ${REPOS_DIR}"
|
||||
echo " 输出目录: ${OUTPUT_DIR}"
|
||||
echo "============================================================"
|
||||
|
||||
# 定义模块
|
||||
declare -A MODULES
|
||||
MODULES=(
|
||||
["pricing"]="$REPOS_DIR/pricing"
|
||||
["llmage"]="$REPOS_DIR/llmage"
|
||||
["uapi"]="$REPOS_DIR/uapi"
|
||||
)
|
||||
# 发现所有模块
|
||||
MODULES=($(discover_modules))
|
||||
echo ""
|
||||
echo "发现 ${#MODULES[@]} 个模块: ${MODULES[*]}"
|
||||
|
||||
TOTAL_TABLES=0
|
||||
TOTAL_FILES=0
|
||||
FAILED_MODULES=()
|
||||
|
||||
for module in pricing llmage uapi; do
|
||||
module_dir="${MODULES[$module]}"
|
||||
for module in "${MODULES[@]}"; do
|
||||
module_dir="$REPOS_DIR/$module"
|
||||
echo ""
|
||||
echo "--- 模块: $module ---"
|
||||
|
||||
if [[ ! -d "$module_dir" ]]; then
|
||||
echo " 跳过: 模块目录不存在 $module_dir"
|
||||
continue
|
||||
fi
|
||||
|
||||
# 获取表名（按模块排除指定表）
|
||||
# 获取排除列表
|
||||
excludes_str="${MODULE_EXCLUDES[$module]:-}"
|
||||
excludes_arr=()
|
||||
if [[ -n "$excludes_str" ]]; then
|
||||
read -ra excludes_arr <<< "$excludes_str"
|
||||
fi
|
||||
|
||||
tables=$(get_tables "$module_dir" "${excludes_arr[@]+"${excludes_arr[@]}"}")
|
||||
|
||||
if [[ -z "$tables" ]]; then
|
||||
@ -160,7 +261,7 @@ for module in pricing llmage uapi; do
|
||||
|
||||
for tbl in $tables; do
|
||||
echo -n " 导出 $tbl ... "
|
||||
if ${MYSQLDUMP_BASE} \
|
||||
if ${MYSQLDUMP_CMD} \
|
||||
--single-transaction \
|
||||
--routines \
|
||||
--triggers \
|
||||
@ -172,7 +273,6 @@ for module in pricing llmage uapi; do
|
||||
echo "OK"
|
||||
((table_count++))
|
||||
else
|
||||
# 如果表不存在，跳过
|
||||
echo "跳过(表不存在或无权限)"
|
||||
fi
|
||||
done
|
||||
@ -184,13 +284,21 @@ for module in pricing llmage uapi; do
|
||||
else
|
||||
rm -f "$outfile"
|
||||
echo " => 无有效表数据"
|
||||
FAILED_MODULES+=("$module")
|
||||
fi
|
||||
done
|
||||
|
||||
# 清理密码环境变量
|
||||
unset MYSQL_PWD
|
||||
|
||||
echo ""
|
||||
echo "============================================================"
|
||||
echo " 导出完成"
|
||||
echo " 模块数: ${#MODULES[@]}"
|
||||
echo " 文件数: $TOTAL_FILES"
|
||||
echo " 表总数: $TOTAL_TABLES"
|
||||
if [[ ${#FAILED_MODULES[@]} -gt 0 ]]; then
|
||||
echo " 无数据模块: ${FAILED_MODULES[*]}"
|
||||
fi
|
||||
echo " 输出目录: $(cd "$OUTPUT_DIR" && pwd)"
|
||||
echo "============================================================"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user