refactor: 废弃llm_catalog_rel表, 分类关系改用llm_api_map

- 删除 llm_catalog_rel 表定义(models/json/xlsx)、CRUD文件、管理页面、迁移脚本
- utils.py: get_llms_by_catelog/get_llms_by_catelog_to_customer 的SQL从 llm_catalog_rel 改为 llm_api_map (加distinct去重)
- init.py: 缓存清除事件从 llm_catalog_rel 改为 llm_api_map
- menu.ui/index.ui: 移除类型关联菜单项
- dspy文件: v1/chat/completions, t2t, get_type_llms, list_catelog_models, list_paging_catelog_llms, llmcatelog_delete 全部改为 join llm_api_map
- 迁移脚本: 添加try/except兼容旧表不存在的情况
This commit is contained in:
yumoqing 2026-05-21 16:22:59 +08:00
parent f1498178cc
commit 5b3c7d4d02
21 changed files with 46 additions and 469 deletions

View File

@ -1,15 +0,0 @@
{
"tblname": "llm_catelog_rel",
"title":"模型类型",
"params": {
"browserfields": {
"exclouded": ["id", "llmid"],
"alters": {}
},
"editexclouded": [
"id",
"llmid"
]
}
}

View File

@ -51,10 +51,10 @@ def _bind_llmage_events(dbpools, dbname):
(f'{dbname}.llmcatelog:c:after', BufferedLLMs.clear_cache),
(f'{dbname}.llmcatelog:u:after', BufferedLLMs.clear_cache),
(f'{dbname}.llmcatelog:d:after', BufferedLLMs.clear_cache),
# llm_catalog_rel 关联表变更:清除缓存
(f'{dbname}.llm_catalog_rel:c:after', BufferedLLMs.clear_cache),
(f'{dbname}.llm_catalog_rel:u:after', BufferedLLMs.clear_cache),
(f'{dbname}.llm_catalog_rel:d:after', BufferedLLMs.clear_cache),
# llm_api_map 关联表变更:清除缓存
(f'{dbname}.llm_api_map:c:after', BufferedLLMs.clear_cache),
(f'{dbname}.llm_api_map:u:after', BufferedLLMs.clear_cache),
(f'{dbname}.llm_api_map:d:after', BufferedLLMs.clear_cache),
]
for event_name, handler in bindings:
dbpools.bind(event_name, handler)

View File

@ -207,11 +207,11 @@ async def get_llms_by_catelog_to_customer(catelogid=None, orderby='providerid'):
env = ServerEnv()
async with get_sor_context(env, 'llmage') as sor:
today = curDateString()
# Join with llm_catalog_rel to support multiple catalogs per LLM
sql = """select a.*, b.name as catelogname, rel.llmcatelogid as catelog_id
# Join with llm_api_map to get catalog relationship
sql = """select distinct a.*, b.name as catelogname, m.llmcatelogid as catelog_id
from llm a
join llm_catalog_rel rel on a.id = rel.llmid
join llmcatelog b on rel.llmcatelogid = b.id
join llm_api_map m on a.id = m.llmid
join llmcatelog b on m.llmcatelogid = b.id
where a.enabled_date <= ${today}$
and a.ppid is not null
and a.expired_date > ${today}$
@ -219,7 +219,7 @@ async def get_llms_by_catelog_to_customer(catelogid=None, orderby='providerid'):
sortstr='catelog_id, ' + orderby
params = {'today': today, 'sort': sortstr}
if catelogid:
sql += " and rel.llmcatelogid = ${catelogid}$"
sql += " and m.llmcatelogid = ${catelogid}$"
params['catelogid'] = catelogid
debug(f'{sql=}')
@ -246,19 +246,19 @@ async def get_llms_by_catelog(catelogid=None, orderby='providerid'):
env = ServerEnv()
async with get_sor_context(env, 'llmage') as sor:
today = curDateString()
# Join with llm_catalog_rel to support multiple catalogs per LLM
sql = """select a.*, b.name as catelogname, rel.llmcatelogid as catelog_id
# Join with llm_api_map to get catalog relationship
sql = """select distinct a.*, b.name as catelogname, m.llmcatelogid as catelog_id
from llm a
join llm_catalog_rel rel on a.id = rel.llmid
join llmcatelog b on rel.llmcatelogid = b.id
join llm_api_map m on a.id = m.llmid
join llmcatelog b on m.llmcatelogid = b.id
where a.enabled_date <= ${today}$
and a.expired_date > ${today}$"""
params = {'today': today, 'sort': orderby}
if catelogid:
sql += " and rel.llmcatelogid = ${catelogid}$"
sql += " and m.llmcatelogid = ${catelogid}$"
params['catelogid'] = catelogid
sql += " order by rel.llmcatelogid, a.id"
sql += " order by m.llmcatelogid, a.id"
recs = await sor.sqlExe(sql, params)
d = []

View File

@ -1,56 +0,0 @@
#!/usr/bin/env python3
"""
Migration script: Move llm.llmcatelogid to llm_catalog_rel.
Run this AFTER creating the llm_catalog_rel table via build.sh.
"""
import asyncio
from sqlor.dbpools import DBPools
from appPublic.jsonConfig import getConfig
from appPublic.log import info, error
from ahserver.serverenv import ServerEnv
async def migrate():
env = ServerEnv()
try:
dbname = env.get_module_dbname('llmage')
except:
dbname = 'default'
config = getConfig()
db = DBPools()
db.databases = config.databases
async with db.sqlorContext(dbname) as sor:
# 1. Migrate data
print("Migrating data...")
# Get all llms with a llmcatelogid
# Note: llmcatelogid still exists in DB until we drop it, or we assume it's there.
# Assuming it's there.
sql = "select id, llmcatelogid from llm where llmcatelogid is not null and llmcatelogid != ''"
rows = await sor.sqlExe(sql, {})
if not rows:
print("No data to migrate.")
return
print(f"Found {len(rows)} records to migrate.")
for r in rows:
# Insert into llm_catalog_rel
# Use getID() logic or simple uuid, here assuming we can use a function or simple generation
# but sqlor insert C() is better
data = {
'llmid': r['id'],
'llmcatelogid': r['llmcatelogid']
}
await sor.C('llm_catalog_rel', data)
print("Migration complete.")
# 2. Drop column (Optional but recommended)
# print("Dropping column...")
# await sor.sqlExe("alter table llm drop column llmcatelogid", {})
# print("Column dropped.")
if __name__ == '__main__':
asyncio.run(migrate())

View File

@ -1,30 +0,0 @@
#!/usr/bin/env python3
"""
Execute migration via sage environment.
Run: cd /home/hermesai/repos/sage && ./py3/bin/python migrate_rel.py
"""
import asyncio
import sys
import os
# Add sage to path
sys.path.insert(0, '/home/hermesai/repos/sage/py3/lib/python3.10/site-packages')
sys.path.insert(0, '/home/hermesai/repos/sage')
from ahserver.serverenv import ServerEnv
from sqlor.dbpools import DBPools
from appPublic.jsonConfig import getConfig
async def migrate():
# Initialize env to get config
# Note: This script expects sage to be configured.
# We can't easily import sage's init here, but we can try to load config if available.
# Alternatively, just use raw sql.
pass
if __name__ == '__main__':
# Use a simpler approach: connect directly
import pymysql
# Assuming config is in sage's config dir
# We can read the config file
pass

View File

@ -1,56 +0,0 @@
{
"summary": [
{
"name": "llm_catelog_rel",
"title": "模型分类对照表",
"primary": [
"id"
],
"catelog": "relation"
}
],
"fields": [
{
"name": "id",
"title": "id",
"type": "str",
"length": 32
},
{
"name": "llmid",
"title": "模型id",
"type": "str",
"length": 32
},
{
"name": "llmcatelogid",
"title": "模型分类id",
"type": "str",
"length": 32
}
],
"indexes": [
{
"name": "idx_uniue_llm_catelogid",
"idxtype": "unique",
"idxfields": [
"llmid",
"llmcatelogid"
]
}
],
"codes": [
{
"field": "llmid",
"table": "llm",
"valuefield": "id",
"textfield": "name"
},
{
"field": "llmcatelogid",
"table": "llmcatelog",
"valuefield": "id",
"textfield": "name"
}
]
}

Binary file not shown.

View File

@ -20,8 +20,11 @@ def generate_migration_sql(llm_records, catalog_rel_records=None):
Generate INSERT statements for llm_api_map from existing llm data.
For each llm record:
- If llm_catalog_rel exists: create one llm_api_map per (llmid, llmcatelogid)
- If no catalog_rel: create one llm_api_map with the llm's default catalog
- If catalog info provided: create one llm_api_map per (llmid, llmcatelogid)
- If no catalog info: use llm's llmcatelogid field (legacy)
NOTE: llm_catalog_rel has been deprecated. Catalog relationship is now
maintained directly in llm_api_map table.
"""
inserts = []
@ -97,7 +100,7 @@ def main():
parser.add_argument('--input', '-i',
help='Input JSON file with llm records (for offline mode)')
parser.add_argument('--catalog-rel', '-c',
help='Input JSON file with llm_catalog_rel records')
help='Input JSON file with catalog records (deprecated, use llm_api_map instead)')
parser.add_argument('--output', '-o', default='-',
help='Output file for SQL statements (default: stdout)')
parser.add_argument('--dry-run', action='store_true',

View File

@ -4,7 +4,7 @@ llm_api_map 数据库迁移脚本
直接操作数据库完成以下任务
1. 创建 llm_api_map 如不存在
2. llm 表迁移 apiname/query_apiname/query_period/ppid llm_api_map
3. 关联 llm_catalog_rel 获取 llmcatelogid
3. 分类关系已从 llm 表迁移llmcatelogid 字段已移除
4. 可选删除 llm 表中的旧字段需用户确认
运行位置Sage 虚拟环境
@ -126,7 +126,12 @@ CREATE TABLE IF NOT EXISTS llm_api_map (
return True
# Build catalog_rel lookup
rels = await sor.sqlExe("SELECT llmid, llmcatelogid FROM llm_catalog_rel", {})
# NOTE: llm_catalog_rel has been deprecated; catalog relationship is now in llm_api_map.
# This lookup is kept for backward compatibility with old migrations.
try:
rels = await sor.sqlExe("SELECT llmid, llmcatelogid FROM llm_catalog_rel", {})
except Exception:
rels = []
catelog_map = {}
for r in (rels or []):
catelog_map.setdefault(r['llmid'], []).append(r['llmcatelogid'])
@ -140,7 +145,7 @@ CREATE TABLE IF NOT EXISTS llm_api_map (
catelog_ids = catelog_map.get(llmid)
if not catelog_ids:
print(f" [SKIP] llm '{llm.get('name', llmid)}' has no catalog_rel entry")
print(f" [SKIP] llm '{llm.get('name', llmid)}' has no catalog entry (llm_catalog_rel deprecated)")
skipped += 1
continue

View File

@ -1,32 +0,0 @@
#!/usr/bin/env python3
import json
result = {'widgettype': 'Message', 'options': {'title': 'Error', 'message': 'Invalid', 'type': 'error'}}
try:
dbname = get_module_dbname('llmage')
llmid = params_kw.get('llmid', '')
catelogid = params_kw.get('llmcatelogid', '')
if not llmid or not catelogid:
result['options'] = {'title': 'Error', 'message': '请选择模型和目录', 'type': 'error'}
else:
from appPublic.uniqueID import getID
new_id = getID()
async with DBPools().sqlorContext(dbname) as sor:
# 检查是否已存在
check_sql = "select id from llm_catalog_rel where llmid = ${llmid}$ and llmcatelogid = ${catelogid}$"
exists = await sor.sqlExe(check_sql, {'llmid': llmid, 'catelogid': catelogid})
if exists:
result['options'] = {'title': '提示', 'message': '该关联已存在', 'type': 'warning'}
else:
data = {'id': new_id, 'llmid': llmid, 'llmcatelogid': catelogid}
await sor.C('llm_catalog_rel', data)
result['options'] = {'title': 'Success', 'message': '添加成功', 'type': 'success'}
except Exception as e:
result['options'] = {'title': 'Error', 'message': f'添加失败: {str(e)}', 'type': 'error'}
return json.dumps(result, ensure_ascii=False)

View File

@ -1,20 +0,0 @@
#!/usr/bin/env python3
import json
result = {'widgettype': 'Message', 'options': {'title': 'Error', 'message': 'Invalid', 'type': 'error'}}
try:
dbname = get_module_dbname('llmage')
rel_id = params_kw.get('id', '')
if not rel_id:
result['options'] = {'title': 'Error', 'message': '缺少ID参数', 'type': 'error'}
else:
async with DBPools().sqlorContext(dbname) as sor:
await sor.sqlExe("delete from llm_catalog_rel where id = ${id}$", {'id': rel_id})
result['options'] = {'title': 'Success', 'message': '删除成功', 'type': 'success'}
except Exception as e:
result['options'] = {'title': 'Error', 'message': f'删除失败: {str(e)}', 'type': 'error'}
return json.dumps(result, ensure_ascii=False)

View File

@ -1,25 +0,0 @@
#!/usr/bin/env python3
import json
result = {'success': False, 'rows': [], 'total': 0}
try:
dbname = get_module_dbname('llmage')
sql = """
select r.id, r.llmid, l.name as llm_name, r.llmcatelogid, c.name as catelog_name
from llm_catalog_rel r
join llm l on r.llmid = l.id
join llmcatelog c on r.llmcatelogid = c.id
order by l.name, c.name
"""
async with DBPools().sqlorContext(dbname) as sor:
rows = await sor.sqlExe(sql, {})
result['rows'] = [dict(r) for r in (rows or [])]
result['total'] = len(result['rows'])
result['success'] = True
except Exception as e:
result['error'] = str(e)
return json.dumps(result, ensure_ascii=False, default=str)

View File

@ -12,7 +12,7 @@ try:
else:
async with DBPools().sqlorContext(dbname) as sor:
# 检查是否有模型关联此类型
check_sql = "select count(*) as cnt from llm_catalog_rel where llmcatelogid=${id}$"
check_sql = "select count(*) as cnt from llm_api_map where llmcatelogid=${id}$"
rel_count = await sor.sqlExe(check_sql, {'id': id})
cnt = rel_count[0]['cnt'] if rel_count else 0
if cnt > 0:

View File

@ -3,9 +3,9 @@ lt = '文生视频'
if params_kw.type in ['文生视频', '参考生视频', '图生视频']:
lt = params_kw.type
async with get_sor_context(request._run_ns, 'llmage') as sor:
sql = '''select a.*, e.input_fields from llm a
join llm_catalog_rel rel on a.id = rel.llmid
join llmcatelog b on rel.llmcatelogid = b.id
sql = '''select distinct a.*, e.input_fields from llm a
join llm_api_map m on a.id = m.llmid
join llmcatelog b on m.llmcatelogid = b.id
join upapp c on a.upappid = c.id
join uapi d on c.apisetid = d.apisetid and a.apiname = d.name
join uapiio e on d.ioid = e.id

View File

@ -68,53 +68,6 @@
}
]
},
{
"widgettype": "VBox",
"options": {
"backgroundColor": "#1e3a5f",
"padding": "24px",
"cursor": "pointer",
"borderRadius": "8px"
},
"binds": [
{
"wid": "self",
"event": "click",
"actiontype": "urlwidget",
"target": "app.llmage_content",
"options": {
"url": "{{entire_url('/llmage/llm_catalog_rel_manage.ui')}}"
},
"mode": "replace"
}
],
"subwidgets": [
{
"widgettype": "Svg",
"options": {
"svg": "<svg width=\"40\" height=\"40\" viewBox=\"0 0 24 24\" fill=\"none\" stroke=\"#ce93d8\" stroke-width=\"2\"><path d=\"M13.5 6H5.25A2.25 2.25 0 003 8.25v10.5A2.25 2.25 0 005.25 21h10.5A2.25 2.25 0 0018 18.75V10.5m-10.5 6L21 3m0 0h-5.25M21 3v5.25\"/></svg>",
"width": "40px",
"height": "40px"
}
},
{
"widgettype": "Title4",
"options": {
"text": "模型-类型关联",
"color": "#ffffff",
"marginTop": "12px"
}
},
{
"widgettype": "Text",
"options": {
"text": "管理模型与类型的多对多关系",
"color": "#ce93d8",
"fontSize": "14px"
}
}
]
},
{
"widgettype": "VBox",
"options": {

View File

@ -1,9 +1,9 @@
dbname = get_module_dbname('llmage')
db = DBPools()
async with db.sqlorContext(dbname) as sor:
sql = """select * from llm a
join llm_catalog_rel rel on a.id = rel.llmid
where rel.llmcatelogid = ${llmcatelogid}$ and a.id != ${llmid}$"""
sql = """select distinct a.* from llm a
join llm_api_map m on a.id = m.llmid
where m.llmcatelogid = ${llmcatelogid}$ and a.id != ${llmid}$"""
ns = params_kw.copy()
recs = await sor.sqlExe(sql, ns)
for r in recs.get('rows', []):

View File

@ -12,15 +12,15 @@ y.system_message,
y.user_message,
y.assisant_message
from (
select a.*, b.hfid, e.ioid, e.stream
select distinct a.*, b.hfid, e.ioid, e.stream
from llm a
join llm_catalog_rel rel on a.id = rel.llmid
join llmcatelog b on rel.llmcatelogid = b.id
join llm_api_map m on a.id = m.llmid
join llmcatelog b on m.llmcatelogid = b.id
join upapp c on a.upappid = c.id
join uapi e on c.apisetid = e.apisetid and a.apiname = e.name
) x left join historyformat y on x.hfid = y.id
left join uapiio z on x.ioid = z.id
where rel.llmcatelogid = ${llmcatelogid}$
where m.llmcatelogid = ${llmcatelogid}$
and x.id != ${llmid}$
"""
ns = params_kw.copy()

View File

@ -1,145 +0,0 @@
{
"widgettype": "VBox",
"options": {
"width": "100%",
"height": "100%",
"spacing": 16
},
"subwidgets": [
{
"widgettype": "Title2",
"options": {
"text": "LLM 目录关联管理",
"halign": "left"
}
},
{
"widgettype": "VBox",
"options": {
"width": "calc(100% - 40px)",
"margin": "0 20px",
"padding": "16px",
"bgcolor": "#f5f5f5",
"spacing": 12
},
"subwidgets": [
{
"widgettype": "Text",
"options": {
"text": "添加新关联",
"fontSize": "16px",
"fontWeight": "bold"
}
},
{
"widgettype": "Form",
"id": "add_form",
"options": {
"layout": "horizontal",
"cols": 3,
"fields": [
{
"name": "llmid",
"label": "选择模型",
"uitype": "select",
"dataurl": "{{entire_url('./api/llm_catelog_options.dspy')}}",
"data_field": "llms",
"placeholder": "请选择模型"
},
{
"name": "llmcatelogid",
"label": "选择目录",
"uitype": "select",
"dataurl": "{{entire_url('./api/llm_catelog_options.dspy')}}",
"data_field": "catelogs",
"placeholder": "请选择目录"
}
],
"buttons": [
{
"name": "add_btn",
"label": "添加关联",
"variant": "primary"
}
]
},
"binds": [
{
"wid": "add_btn",
"event": "click",
"actiontype": "urlwidget",
"target": "PopupWindow",
"popup_options": {"archor": "cc", "width": "30%", "height": "20%"},
"options": {
"url": "{{entire_url('./api/llm_catalog_rel_create.dspy')}}",
"params": {
"llmid": "$[add_form.llmid]$",
"llmcatelogid": "$[add_form.llmcatelogid]$"
}
}
}
]
}
]
},
{
"widgettype": "VBox",
"options": {
"width": "calc(100% - 40px)",
"margin": "0 20px",
"spacing": 12
},
"subwidgets": [
{
"widgettype": "Text",
"options": {
"text": "当前关联列表",
"fontSize": "16px",
"fontWeight": "bold"
}
},
{
"widgettype": "Tabular",
"id": "rel_table",
"options": {
"width": "100%",
"height": "400px",
"data_url": "{{entire_url('./api/llm_catalog_rel_list.dspy')}}",
"data_method": "GET",
"page_rows": 20,
"row_options": {
"fields": [
{"name": "llm_name", "title": "模型名称", "width": 200},
{"name": "catelog_name", "title": "目录名称", "width": 150},
{
"name": "actions",
"title": "操作",
"width": 100,
"uitype": "button",
"data": [
{"text": "删除", "event": "delete_rel"}
]
}
]
}
},
"binds": [
{
"wid": "self",
"event": "delete_rel",
"actiontype": "urlwidget",
"target": "PopupWindow",
"popup_options": {"archor": "cc", "width": "30%", "height": "20%"},
"options": {
"url": "{{entire_url('./api/llm_catalog_rel_delete.dspy')}}",
"params": {
"id": "$[event.params.id]$"
}
}
}
]
}
]
}
]
}

View File

@ -31,11 +31,6 @@
"name":"llm",
"label":"模型",
"url":"{{entire_url('/llmage/llm')}}"
},
{
"name":"llmcatelog_rel",
"label":"类型关联",
"url":"{{entire_url('/llmage/llm_catalog_rel_manage.ui')}}"
}
]
}

View File

@ -17,9 +17,9 @@ if not params_kw.prompt:
return json_response(d, status=400)
env = request._run_ns
async with get_sor_context(env, 'llmage') as sor:
sql = """select a.* from llm a
join llm_catalog_rel rel on a.id = rel.llmid
join llmcatelog b on rel.llmcatelogid = b.id
sql = """select distinct a.* from llm a
join llm_api_map m on a.id = m.llmid
join llmcatelog b on m.llmcatelogid = b.id
where b.name = ${lctype}$
and a.model=${model}$"""
recs = await sor.sqlExe(sql, {

View File

@ -30,9 +30,9 @@ if not params_kw.prompt and not params_kw.messages:
return json_response(d, status=400)
env = request._run_ns
async with get_sor_context(env, 'llmage') as sor:
sql = """select a.* from llm a
join llm_catalog_rel rel on a.id = rel.llmid
join llmcatelog b on rel.llmcatelogid = b.id
sql = """select distinct a.* from llm a
join llm_api_map m on a.id = m.llmid
join llmcatelog b on m.llmcatelogid = b.id
where b.name = ${lctype}$
and a.model=${model}$"""
recs = await sor.sqlExe(sql, {