perf: optimize get_inference_history query and add indexes

Query optimization (dspy):
- Replace UNION ALL + sort with two parallel queries (asyncio.gather)
  that each use (userid, use_time) composite index
- Python-side merge-sort of two pre-sorted sequences O(n)
- Concurrent FileStorage reads for ioinfo (asyncio.gather)

Indexes (models/*.json + /tmp/llmage_history_indexes.sql):
- llmusage: add idx_llmusage_userid_usetime (userid, use_time)
- llmusage_history: add idx_lh_userid_usetime (userid, use_time)
  (was missing userid index entirely - main bottleneck)
This commit is contained in:
yumoqing 2026-06-05 17:42:15 +08:00
parent 2ebe811c34
commit d4e455ba9a
3 changed files with 59 additions and 37 deletions

View File

@ -135,6 +135,14 @@
"accounting_status",
"use_date"
]
},
{
"name": "idx_llmusage_userid_usetime",
"idxtype": "index",
"idxfields": [
"userid",
"use_time"
]
}
]
}

View File

@ -136,6 +136,11 @@
"name": "idx_lh_backup_time",
"idxtype": "index",
"idxfields": ["backup_time"]
},
{
"name": "idx_lh_userid_usetime",
"idxtype": "index",
"idxfields": ["userid", "use_time"]
}
]
}

View File

@ -1,5 +1,6 @@
#!/usr/bin/env python3
import json
import asyncio
result = {'success': False, 'rows': [], 'total': 0, 'page': 1, 'page_size': 10}
@ -22,63 +23,71 @@ try:
where_clause = " and ".join(conditions)
# Count total from both tables
count_sql = f"""
select count(*) as cnt from (
select id from llmusage where {where_clause}
union all
select id from llmusage_history where {where_clause}
) t
"""
count_recs = await sor.sqlExe(count_sql, ns)
total = count_recs[0].cnt if count_recs else 0
# Count total from both tables (并行两个 count 查询)
count_sql = f"select count(*) as cnt from {{table}} where {where_clause}"
cnt1_recs, cnt2_recs = await asyncio.gather(
sor.sqlExe(count_sql.format(table='llmusage'), ns),
sor.sqlExe(count_sql.format(table='llmusage_history'), ns),
)
total = (cnt1_recs[0].cnt if cnt1_recs else 0) + (cnt2_recs[0].cnt if cnt2_recs else 0)
# UNION ALL query with pagination, time descending
# 优化点 1: 分别查询两张表, 让各自走 (userid, use_time) 复合索引
# 每表取前 offset+page_size 条 (已按 use_time desc 排好)
offset = (page - 1) * page_size
query_sql = f"""
select id, llmid, use_date, use_time, userid, usages, ioinfo,
status, taskid, amount, cost, userorgid, accounting_status
from (
select id, llmid, use_date, use_time, userid, usages, ioinfo,
status, taskid, amount, cost, userorgid, accounting_status
from llmusage where {where_clause}
union all
select id, llmid, use_date, use_time, userid, usages, ioinfo,
status, taskid, amount, cost, userorgid, accounting_status
from llmusage_history where {where_clause}
) t
order by use_time desc
limit {page_size} offset {offset}
"""
recs = await sor.sqlExe(query_sql, ns)
fetch = offset + page_size
select_cols = ("id, llmid, use_date, use_time, userid, usages, ioinfo, "
"status, taskid, amount, cost, userorgid, accounting_status")
rows = []
for r in (recs or []):
row = dict(r)
# Read ioinfo content from FileStorage
q1 = f"select {select_cols} from llmusage where {where_clause} order by use_time desc limit {fetch}"
q2 = f"select {select_cols} from llmusage_history where {where_clause} order by use_time desc limit {fetch}"
recs1, recs2 = await asyncio.gather(
sor.sqlExe(q1, ns),
sor.sqlExe(q2, ns),
)
# 优化点 2: Python 归并两个已排序序列 (O(n) 比 SQL UNION+sort 快)
merged = []
i = j = 0
rows1 = [dict(r) for r in (recs1 or [])]
rows2 = [dict(r) for r in (recs2 or [])]
while i < len(rows1) and j < len(rows2):
if (rows1[i].get('use_time') or '') >= (rows2[j].get('use_time') or ''):
merged.append(rows1[i]); i += 1
else:
merged.append(rows2[j]); j += 1
merged.extend(rows1[i:])
merged.extend(rows2[j:])
# 应用分页
page_rows = merged[offset:offset + page_size]
# 优化点 3: 并发读取 ioinfo 文件 (不再串行 await)
import aiofiles
from ahserver.filestorage import FileStorage
fs = FileStorage()
async def _load_io(row):
webpath = row.get('ioinfo')
io_content = None
if webpath:
try:
from ahserver.filestorage import FileStorage
fs = FileStorage()
real_path = fs.realPath(webpath)
import aiofiles
async with aiofiles.open(real_path, 'rb') as f:
bin_data = await f.read()
io_content = json.loads(bin_data.decode('utf-8'))
except Exception:
io_content = None
row['io_content'] = io_content
# Parse usages if it's a JSON string
if isinstance(row.get('usages'), str):
try:
row['usages'] = json.loads(row['usages'])
except Exception:
pass
rows.append(row)
return row
result['rows'] = rows
rows = await asyncio.gather(*[_load_io(r) for r in page_rows])
result['rows'] = list(rows)
result['total'] = total
result['page'] = page
result['page_size'] = page_size