From 17f6855b5a4323751b6ff1d4c3a1bae858e62e6e Mon Sep 17 00:00:00 2001 From: yumoqing Date: Thu, 21 May 2026 10:59:15 +0800 Subject: [PATCH] feat: add deployment scripts for llm_api_map migration - scripts/migrate_llm_api_map_db.py: Direct DB migration (create table + migrate data) Supports --dry-run and --drop-old flags - scripts/deploy_llmage.sh: One-click deployment script Orchestrates: git pull -> build -> migrate -> perms -> restart --- scripts/deploy_llmage.sh | 153 +++++++++++++++++ scripts/migrate_llm_api_map_db.py | 264 ++++++++++++++++++++++++++++++ 2 files changed, 417 insertions(+) create mode 100644 scripts/deploy_llmage.sh create mode 100644 scripts/migrate_llm_api_map_db.py diff --git a/scripts/deploy_llmage.sh b/scripts/deploy_llmage.sh new file mode 100644 index 0000000..2f4f997 --- /dev/null +++ b/scripts/deploy_llmage.sh @@ -0,0 +1,153 @@ +#!/bin/bash +# deploy_llmage.sh +# 一键部署 llmage 模块的 llm_api_map 变更 +# +# 包含：代码拉取 -> 构建 -> 数据库迁移 -> 权限设置 -> 重启 +# +# 用法： +# bash deploy_llmage.sh # 完整部署 +# bash deploy_llmage.sh --dry-run # 预览，不执行变更 +# bash deploy_llmage.sh --skip-db # 跳过数据库迁移 +# bash deploy_llmage.sh --skip-perm # 跳过权限设置 + +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +LLMAGE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +SAGE_DIR="$(cd "$LLMAGE_DIR/../sage" && pwd 2>/dev/null || echo "")" + +# 参数解析 +DRY_RUN=false +SKIP_DB=false +SKIP_PERM=false + +for arg in "$@"; do + case $arg in + --dry-run) DRY_RUN=true ;; + --skip-db) SKIP_DB=true ;; + --skip-perm) SKIP_PERM=true ;; + -h|--help) + echo "用法: bash deploy_llmage.sh [options]" + echo " --dry-run 预览模式，不执行变更" + echo " --skip-db 跳过数据库迁移" + echo " --skip-perm 跳过权限设置" + echo " -h, --help 显示帮助" + exit 0 + ;; + esac +done + +echo "============================================" +echo " llmage 模块部署: llm_api_map" +echo "============================================" +echo "LLMAGE_DIR: $LLMAGE_DIR" +echo "SAGE_DIR: $SAGE_DIR" +echo "Dry run: $DRY_RUN" +echo "Skip DB: $SKIP_DB" +echo "Skip Perm: $SKIP_PERM" +echo "" + +# ============================================= +# Step 1: 拉取最新代码 +# ============================================= +echo ">>> [1/5] 拉取最新代码..." +cd "$LLMAGE_DIR" + +if $DRY_RUN; then + echo " [DRY] git pull origin main" +else + git pull origin main + echo " Done" +fi + +# ============================================= +# Step 2: 构建模块（生成 UI/DDL） +# ============================================= +echo "" +echo ">>> [2/5] 构建模块..." +cd "$LLMAGE_DIR/json" + +if $DRY_RUN; then + echo " [DRY] bash build.sh" +else + bash build.sh + echo " Done" +fi + +# ============================================= +# Step 3: 数据库迁移 +# ============================================= +if ! $SKIP_DB; then + echo "" + echo ">>> [3/5] 数据库迁移..." + + # 先在 Sage 虚拟环境中 dry-run 验证 + cd "$SAGE_DIR" + if $DRY_RUN; then + echo " [DRY] python $LLMAGE_DIR/scripts/migrate_llm_api_map_db.py --dry-run" + python "$LLMAGE_DIR/scripts/migrate_llm_api_map_db.py" --dry-run + else + echo " 执行 dry-run 预览..." + python "$LLMAGE_DIR/scripts/migrate_llm_api_map_db.py" --dry-run + echo "" + echo " 确认以上预览无误后，执行实际迁移..." + python "$LLMAGE_DIR/scripts/migrate_llm_api_map_db.py" + fi + echo " Done" +else + echo "" + echo ">>> [3/5] 跳过数据库迁移" +fi + +# ============================================= +# Step 4: 权限设置 +# ============================================= +if ! $SKIP_PERM; then + echo "" + echo ">>> [4/5] 设置 RBAC 权限..." + + cd "$SAGE_DIR" + if $DRY_RUN; then + echo " [DRY] bash $LLMAGE_DIR/scripts/setup_llmage_perms.sh" + else + bash "$LLMAGE_DIR/scripts/setup_llmage_perms.sh" + fi + echo " Done" +else + echo "" + echo ">>> [4/5] 跳过权限设置" +fi + +# ============================================= +# Step 5: 重启服务 +# ============================================= +echo "" +echo ">>> [5/5] 重启服务..." + +if $DRY_RUN; then + echo " [DRY] 请手动重启 Sage 服务以生效新代码" + echo " 示例: systemctl restart sage 或 supervisorctl restart sage" +else + echo " 请手动重启 Sage 服务以使变更生效：" + echo " systemctl restart sage" + echo " 或：" + echo " supervisorctl restart sage" +fi + +# ============================================= +# 完成 +# ============================================= +echo "" +echo "============================================" +echo " 部署完成" +echo "============================================" +echo "" +echo "验证清单：" +echo " 1. 检查 llm_api_map 表是否创建成功" +echo " 2. 访问 /llmage/llm_api_map_manage.ui 确认页面可访问" +echo " 3. 确认有权限的用户可以正常操作" +echo " 4. 确认无权限的用户返回 401" +echo "" +echo "如需回滚旧 llm 表字段（已删除的话）：" +echo " 请从 git 历史恢复或手动添加 apiname/query_apiname/query_period/ppid 字段" +echo "" diff --git a/scripts/migrate_llm_api_map_db.py b/scripts/migrate_llm_api_map_db.py new file mode 100644 index 0000000..190a919 --- /dev/null +++ b/scripts/migrate_llm_api_map_db.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python3 +""" +llm_api_map 数据库迁移脚本 +直接操作数据库，完成以下任务： + 1. 创建 llm_api_map 表（如不存在） + 2. 从 llm 表迁移 apiname/query_apiname/query_period/ppid 到 llm_api_map + 3. 关联 llm_catalog_rel 获取 llmcatelogid + 4. 可选：删除 llm 表中的旧字段（需用户确认） + +运行位置：Sage 虚拟环境 +用法：python scripts/migrate_llm_api_map_db.py [--dry-run] [--drop-old] +""" +import sys +import os +import asyncio +import argparse + +# 确保 Sage 虚拟环境的包可用 +sage_root = os.path.expanduser('~/repos/sage') +sys.path.insert(0, sage_root) +sys.path.insert(0, os.path.join(sage_root, 'py3/lib/python3.10/site-packages')) + +from appPublic.jsonConfig import getConfig +from appPublic.uniqueID import getID +from sqlor.dbpools import DBPools + + +def print_sql(sql, label=""): + if label: + print(f" [{label}] {sql}") + else: + print(f" {sql}") + + +async def migrate(dry_run=False, drop_old=False): + """Execute migration.""" + config = getConfig(sage_root) + db = DBPools(config.databases) + dbname = list(config.databases.keys())[0] + + print(f"Using database: {dbname}") + print(f"Dry run: {dry_run}") + print(f"Drop old columns: {drop_old}") + print() + + async with db.sqlorContext(dbname) as sor: + # ============================================= + # Step 1: Check if llm_api_map already has data + # ============================================= + existing = await sor.sqlExe("SELECT COUNT(*) as cnt FROM llm_api_map", {}) + existing_count = existing[0]['cnt'] if existing else 0 + + if existing_count > 0: + print(f"[WARNING] llm_api_map already has {existing_count} records.") + resp = input("Continue anyway? (y/N): ") + if resp.lower() != 'y': + print("Aborted.") + return False + + # ============================================= + # Step 2: Create llm_api_map table + # ============================================= + print("[Step 1] Creating llm_api_map table...") + + create_sql = """ +CREATE TABLE IF NOT EXISTS llm_api_map ( + id VARCHAR(32) NOT NULL PRIMARY KEY, + llmid VARCHAR(32) NOT NULL, + llmcatelogid VARCHAR(32) NOT NULL, + apiname VARCHAR(100) NOT NULL, + query_apiname VARCHAR(100), + query_period INT, + ppid VARCHAR(32) +) + """ + if dry_run: + print_sql(create_sql.strip(), "SQL") + else: + try: + await sor.sqlExe(create_sql, {}) + print(" Table created (or already exists)") + except Exception as e: + if 'already exists' in str(e).lower() or 'Duplicate' in str(e): + print(" Table already exists, continuing") + else: + print(f" ERROR: {e}") + return False + + # Create indexes + indexes = [ + ("CREATE INDEX idx_api_map_llm ON llm_api_map (llmid)", "index llmid"), + ("CREATE INDEX idx_api_map_catelog ON llm_api_map (llmcatelogid)", "index catelog"), + ("CREATE INDEX idx_api_map_apiname ON llm_api_map (apiname)", "index apiname"), + ("CREATE UNIQUE INDEX uk_llmid_apiname ON llm_api_map (llmid, apiname)", "unique (llmid, apiname)"), + ] + + for sql, label in indexes: + if dry_run: + print_sql(sql, label) + else: + try: + await sor.sqlExe(sql, {}) + print(f" Index created: {label}") + except Exception as e: + if 'already exists' in str(e).lower() or 'Duplicate' in str(e): + print(f" Index already exists: {label}") + else: + print(f" WARNING creating {label}: {e}") + + # ============================================= + # Step 3: Migrate data from llm -> llm_api_map + # ============================================= + print("\n[Step 2] Migrating data...") + + # Get all llm records that have apiname + llms = await sor.sqlExe(""" + SELECT id, name, apiname, query_apiname, query_period, ppid + FROM llm + WHERE apiname IS NOT NULL AND apiname != '' + """, {}) + + print(f" Found {len(llms or [])} llm records with apiname") + + if not llms: + print(" No data to migrate. Done.") + return True + + # Build catalog_rel lookup + rels = await sor.sqlExe("SELECT llmid, llmcatelogid FROM llm_catalog_rel", {}) + catelog_map = {} + for r in (rels or []): + catelog_map.setdefault(r['llmid'], []).append(r['llmcatelogid']) + + migrated = 0 + skipped = 0 + errors = [] + + for llm in llms: + llmid = llm['id'] + catelog_ids = catelog_map.get(llmid) + + if not catelog_ids: + print(f" [SKIP] llm '{llm.get('name', llmid)}' has no catalog_rel entry") + skipped += 1 + continue + + for catelogid in catelog_ids: + if dry_run: + print(f" [DRY] Would insert: llm={llm.get('name', llmid)}, catelog={catelogid}, api={llm['apiname']}") + migrated += 1 + continue + + # Check if already exists + exists = await sor.sqlExe( + "SELECT id FROM llm_api_map WHERE llmid=${llmid}$ AND apiname=${apiname}$", + {'llmid': llmid, 'apiname': llm['apiname']} + ) + if exists: + print(f" [SKIP] Already exists: {llm.get('name', llmid)} / {llm['apiname']}") + skipped += 1 + continue + + # Insert + data = { + 'id': getID(), + 'llmid': llmid, + 'llmcatelogid': catelogid, + 'apiname': llm['apiname'], + } + if llm.get('query_apiname'): + data['query_apiname'] = llm['query_apiname'] + if llm.get('query_period') is not None: + data['query_period'] = int(llm['query_period']) if llm['query_period'] != '' else None + if llm.get('ppid'): + data['ppid'] = llm['ppid'] + + try: + await sor.C('llm_api_map', data) + migrated += 1 + except Exception as e: + errors.append(f"{llm.get('name', llmid)}: {e}") + + print(f"\n Migrated: {migrated}, Skipped: {skipped}") + if errors: + print(f" Errors: {len(errors)}") + for e in errors[:5]: + print(f" - {e}") + + if dry_run: + print("\n (Dry run, no changes made)") + return True + + # ============================================= + # Step 4: Verify migration + # ============================================= + print("\n[Step 3] Verifying migration...") + + llm_api_count = await sor.sqlExe("SELECT COUNT(*) as cnt FROM llm_api_map", {}) + api_count = llm_api_count[0]['cnt'] if llm_api_count else 0 + print(f" llm_api_map total records: {api_count}") + + # Check for llm records without corresponding llm_api_map + orphan_check = await sor.sqlExe(""" + SELECT l.id, l.name + FROM llm l + WHERE l.apiname IS NOT NULL AND l.apiname != '' + AND NOT EXISTS ( + SELECT 1 FROM llm_api_map m WHERE m.llmid = l.id + ) + """, {}) + + if orphan_check: + print(f" [WARNING] {len(orphan_check)} llm records have no llm_api_map entry:") + for o in orphan_check[:5]: + print(f" - {o.get('name', o['id'])}") + if len(orphan_check) > 5: + print(f" ... and {len(orphan_check) - 5} more") + else: + print(" All llm records with apiname have corresponding llm_api_map entries") + + # ============================================= + # Step 5: Optional - drop old columns from llm + # ============================================= + if drop_old: + print("\n[Step 4] Dropping old columns from llm table...") + + old_columns = ['apiname', 'query_apiname', 'query_period', 'ppid'] + for col in old_columns: + try: + await sor.sqlExe(f"ALTER TABLE llm DROP COLUMN {col}", {}) + print(f" Dropped column: {col}") + except Exception as e: + if 'column' in str(e).lower() and ('not exist' in str(e).lower() or 'doesn\'t exist' in str(e).lower()): + print(f" Column already dropped: {col}") + else: + print(f" WARNING dropping {col}: {e}") + + print(" Old columns removed from llm table") + print(" NOTE: Update code to use llm_api_map instead of llm columns") + + print("\n[Done] Migration complete") + return True + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Migrate llm data to llm_api_map table') + parser.add_argument('--dry-run', action='store_true', help='Preview changes without executing') + parser.add_argument('--drop-old', action='store_true', help='Drop old columns from llm table after migration') + args = parser.parse_args() + + if not args.dry_run: + print("=" * 60) + print(" llm_api_map Database Migration") + print("=" * 60) + resp = input("\nThis will modify the database. Continue? (y/N): ") + if resp.lower() != 'y': + print("Aborted.") + sys.exit(0) + print() + + success = asyncio.get_event_loop().run_until_complete( + migrate(dry_run=args.dry_run, drop_old=args.drop_old) + ) + sys.exit(0 if success else 1)