feat: data_mart module — model perf monitoring + provider ROI dashboard

This commit is contained in:
yumoqing 2026-07-17 16:56:52 +08:00
commit 4d24094048
18 changed files with 582 additions and 0 deletions

3
.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
__pycache__/
*.pyc
*.egg-info/

4
build.sh Normal file
View File

@ -0,0 +1,4 @@
#!/bin/bash
set -e
cd "$(dirname "$0")"
echo "[data_mart] build done"

2
data_mart/__init__.py Normal file
View File

@ -0,0 +1,2 @@
"""data_mart — Sage 数据集市模块"""
from .init import load_data_mart

286
data_mart/etl.py Normal file
View File

@ -0,0 +1,286 @@
"""
数据集市 ETL 增量同步 llmusage 集市表
"""
import json
import asyncio
from datetime import datetime, timedelta
from appPublic.uniqueID import getID
from appPublic.dictObject import DictObject
from appPublic.jsonConfig import getConfig
from appPublic.log import debug, info, exception
from sqlor.dbpools import DBPools
from ahserver.serverenv import ServerEnv
MODULE_NAME = "data_mart"
async def ensure_tables(sor):
"""创建集市表(如不存在)"""
tables = [
"""CREATE TABLE IF NOT EXISTS dm_model_call_fact (
id VARCHAR(32) PRIMARY KEY,
luid VARCHAR(32) NOT NULL,
llmid VARCHAR(32) NOT NULL,
model VARCHAR(100),
catelogid VARCHAR(10),
userid VARCHAR(32) NOT NULL,
userorgid VARCHAR(32) NOT NULL,
ownerid VARCHAR(32),
providerid VARCHAR(32),
call_date DATE NOT NULL,
call_hour TINYINT NOT NULL,
call_time DATETIME NOT NULL,
ttft_ms INT,
ttot_ms INT,
prompt_tokens INT DEFAULT 0,
completion_tokens INT DEFAULT 0,
status VARCHAR(20),
fail_reason VARCHAR(200),
amount DECIMAL(10,4) DEFAULT 0,
currency VARCHAR(5) DEFAULT 'CNY',
distributor_orgid VARCHAR(32),
sale_userid VARCHAR(32),
created_at DATETIME DEFAULT NOW(),
INDEX idx_call_date (call_date),
INDEX idx_llmid (llmid),
INDEX idx_userorgid (userorgid),
INDEX idx_ownerid (ownerid),
INDEX idx_status (status),
INDEX idx_model (model)
)""",
"""CREATE TABLE IF NOT EXISTS dm_model_perf_daily (
id VARCHAR(32) PRIMARY KEY,
stat_date DATE NOT NULL,
llmid VARCHAR(32) NOT NULL,
model VARCHAR(100),
catelogid VARCHAR(10),
providerid VARCHAR(32),
ownerid VARCHAR(32),
total_calls INT DEFAULT 0,
success_calls INT DEFAULT 0,
fail_calls INT DEFAULT 0,
success_rate DECIMAL(5,2),
avg_ttft_ms DECIMAL(10,1),
p50_ttft_ms DECIMAL(10,1),
p95_ttft_ms DECIMAL(10,1),
p99_ttft_ms DECIMAL(10,1),
avg_ttot_ms DECIMAL(10,1),
p50_ttot_ms DECIMAL(10,1),
p95_ttot_ms DECIMAL(10,1),
max_concurrent INT,
avg_concurrent DECIMAL(5,1),
total_prompt_tokens BIGINT DEFAULT 0,
total_completion_tokens BIGINT DEFAULT 0,
total_amount DECIMAL(12,4) DEFAULT 0,
userorgid VARCHAR(32),
distributor_orgid VARCHAR(32),
UNIQUE KEY uk_date_model_org (stat_date, llmid, userorgid),
INDEX idx_date (stat_date),
INDEX idx_model (model)
)""",
"""CREATE TABLE IF NOT EXISTS dm_provider_cost_daily (
id VARCHAR(32) PRIMARY KEY,
stat_date DATE NOT NULL,
model VARCHAR(100) NOT NULL,
catelogid VARCHAR(10),
providerid VARCHAR(32) NOT NULL,
provider_name VARCHAR(100),
total_calls INT DEFAULT 0,
total_tokens BIGINT DEFAULT 0,
total_amount DECIMAL(12,4) DEFAULT 0,
unit_price DECIMAL(10,6),
avg_ttft_ms DECIMAL(10,1),
UNIQUE KEY uk_date_model_provider (stat_date, model, providerid),
INDEX idx_model (model),
INDEX idx_date (stat_date)
)""",
]
for sql in tables:
await sor.sqlExe(sql, {})
def parse_usages(usages_str):
"""从 llmusage.usages JSON 中提取 token 数"""
try:
u = json.loads(usages_str) if isinstance(usages_str, str) else usages_str
if isinstance(u, dict):
return (
u.get('prompt_tokens', 0) or 0,
u.get('completion_tokens', 0) or 0,
u.get('total_tokens', 0) or 0
)
except:
pass
return 0, 0, 0
async def sync_call_fact(sor, last_sync=None):
"""增量同步 llmusage → dm_model_call_fact"""
if last_sync is None:
last_sync = (datetime.now() - timedelta(minutes=10)).strftime('%Y-%m-%d %H:%M:%S')
sql = """
SELECT lu.*, l.model as llm_model, l.catelogid, l.providerid, l.ownerid
FROM llmusage lu
JOIN llm l ON l.id = lu.llmid
WHERE lu.use_time >= ${last_sync}$
ORDER BY lu.use_time
"""
rows = await sor.sqlExe(sql, {'last_sync': last_sync})
info('data_mart ETL: syncing ' + str(len(rows)) + ' rows from ' + last_sync)
count = 0
for r in rows:
r = DictObject(r)
# Skip existing
existing = await sor.sqlExe(
'SELECT id FROM dm_model_call_fact WHERE luid=${luid}$',
{'luid': r.id})
if existing:
continue
prompt_tokens, completion_tokens, _ = parse_usages(r.usages)
call_time = datetime.strptime(r.use_time[:19], '%Y-%m-%d %H:%M:%S')
ns = {
'id': getID(),
'luid': r.id,
'llmid': r.llmid,
'model': r.llm_model or r.model,
'catelogid': r.catelogid,
'userid': r.userid,
'userorgid': r.userorgid,
'ownerid': r.ownerid,
'providerid': r.providerid,
'call_date': r.use_date or call_time.strftime('%Y-%m-%d'),
'call_hour': call_time.hour,
'call_time': r.use_time,
'ttft_ms': int(r.responsed_seconds * 1000) if r.responsed_seconds else None,
'ttot_ms': int(r.finish_seconds * 1000) if r.finish_seconds else None,
'prompt_tokens': prompt_tokens,
'completion_tokens': completion_tokens,
'status': r.status or 'SUCCEEDED',
'amount': r.amount or 0,
'currency': getattr(r, 'currency', 'CNY') or 'CNY',
}
await sor.C('dm_model_call_fact', ns)
count += 1
info('data_mart ETL: inserted ' + str(count) + ' new rows')
return count
async def aggregate_daily_perf(sor, stat_date=None):
"""聚合当天 dm_model_call_fact → dm_model_perf_daily"""
if stat_date is None:
stat_date = datetime.now().strftime('%Y-%m-%d')
# 按 llmid + userorgid 聚合
sql = """
SELECT
call_date, llmid, model, catelogid, providerid, ownerid,
COALESCE(userorgid, '') as userorgid,
COUNT(*) as total_calls,
SUM(CASE WHEN status='SUCCEEDED' THEN 1 ELSE 0 END) as success_calls,
SUM(CASE WHEN status='FAILED' THEN 1 ELSE 0 END) as fail_calls,
AVG(ttft_ms) as avg_ttft,
AVG(ttot_ms) as avg_ttot,
SUM(prompt_tokens) as total_prompt,
SUM(completion_tokens) as total_completion,
SUM(amount) as total_amount
FROM dm_model_call_fact
WHERE call_date = ${stat_date}$
GROUP BY call_date, llmid, model, catelogid, providerid, ownerid, userorgid
"""
rows = await sor.sqlExe(sql, {'stat_date': stat_date})
info('data_mart AGG: aggregating ' + str(len(rows)) + ' groups for ' + stat_date)
for r in rows:
r = DictObject(r)
# Delete old row, insert new
await sor.sqlExe(
'DELETE FROM dm_model_perf_daily WHERE stat_date=${d}$ AND llmid=${l}$ AND COALESCE(userorgid,\"\")=${u}$',
{'d': stat_date, 'l': r.llmid, 'u': r.userorgid or ''})
ns = {
'id': getID(),
'stat_date': stat_date,
'llmid': r.llmid,
'model': r.model,
'catelogid': r.catelogid,
'providerid': r.providerid,
'ownerid': r.ownerid,
'total_calls': r.total_calls,
'success_calls': r.success_calls,
'fail_calls': r.fail_calls,
'success_rate': round(r.success_calls * 100.0 / r.total_calls, 2) if r.total_calls else 0,
'avg_ttft_ms': round(r.avg_ttft, 1) if r.avg_ttft else None,
'avg_ttot_ms': round(r.avg_ttot, 1) if r.avg_ttot else None,
'total_prompt_tokens': r.total_prompt or 0,
'total_completion_tokens': r.total_completion or 0,
'total_amount': r.total_amount or 0,
'userorgid': r.userorgid or None,
}
await sor.C('dm_model_perf_daily', ns)
return len(rows)
async def aggregate_provider_cost(sor, stat_date=None):
"""聚合同模型供应商性价比 — dm_provider_cost_daily"""
if stat_date is None:
stat_date = (datetime.now() - timedelta(days=1)).strftime('%Y-%m-%d')
sql = """
SELECT
model, providerid,
COUNT(*) as total_calls,
SUM(prompt_tokens + completion_tokens) as total_tokens,
SUM(amount) as total_amount,
AVG(ttft_ms) as avg_ttft
FROM dm_model_call_fact
WHERE call_date = ${stat_date}$
GROUP BY model, providerid
"""
rows = await sor.sqlExe(sql, {'stat_date': stat_date})
for r in rows:
r = DictObject(r)
await sor.sqlExe(
'DELETE FROM dm_provider_cost_daily WHERE stat_date=${d}$ AND model=${m}$ AND providerid=${p}$',
{'d': stat_date, 'm': r.model, 'p': r.providerid})
ns = {
'id': getID(),
'stat_date': stat_date,
'model': r.model,
'providerid': r.providerid,
'total_calls': r.total_calls,
'total_tokens': r.total_tokens or 0,
'total_amount': r.total_amount or 0,
'unit_price': round(float(r.total_amount) / r.total_tokens, 8) if r.total_tokens else None,
'avg_ttft_ms': round(r.avg_ttft, 1) if r.avg_ttft else None,
}
await sor.C('dm_provider_cost_daily', ns)
return len(rows)
async def run_etl_sync(sor):
"""每5分钟运行: 确保表存在 + 增量同步"""
await ensure_tables(sor)
await sync_call_fact(sor)
async def run_etl_aggregate(sor):
"""每小时运行: 聚合天级性能 + 并发统计"""
await ensure_tables(sor)
await aggregate_daily_perf(sor)
async def run_etl_provider_cost(sor):
"""每天运行: 供应商性价比"""
await ensure_tables(sor)
await aggregate_provider_cost(sor)

78
data_mart/init.py Normal file
View File

@ -0,0 +1,78 @@
"""data_mart 模块初始化"""
from ahserver.serverenv import ServerEnv
from appPublic.jsonConfig import getConfig
from appPublic.log import debug, exception
from sqlor.dbpools import DBPools
from .etl import sync_call_fact, aggregate_daily_perf, aggregate_provider_cost, ensure_tables
MODULE_NAME = "data_mart"
def get_dbname():
env = ServerEnv()
return env.get_module_dbname(MODULE_NAME)
async def api_model_perf(sor, params_kw=None):
"""模型性能统计 API — POST /data_mart/api/model_perf.dspy"""
model = params_kw.get('model') if params_kw else None
stat_date = params_kw.get('date') if params_kw else None
sql = "SELECT * FROM dm_model_perf_daily WHERE 1=1"
ns = {}
if stat_date:
sql += " AND stat_date=${stat_date}$"
ns['stat_date'] = stat_date
if model:
sql += " AND model=${model}$"
ns['model'] = model
sql += " ORDER BY total_calls DESC LIMIT 50"
rows = await sor.sqlExe(sql, ns)
return [dict(r) for r in rows]
async def api_provider_roi(sor, params_kw=None):
"""供应商性价比 API — POST /data_mart/api/provider_roi.dspy"""
model = params_kw.get('model') if params_kw else None
stat_date = params_kw.get('date') if params_kw else None
sql = "SELECT * FROM dm_provider_cost_daily WHERE 1=1"
ns = {}
if stat_date:
sql += " AND stat_date=${stat_date}$"
ns['stat_date'] = stat_date
if model:
sql += " AND model=${model}$"
ns['model'] = model
sql += " ORDER BY total_amount DESC LIMIT 50"
rows = await sor.sqlExe(sql, ns)
return [dict(r) for r in rows]
async def api_call_detail(sor, params_kw=None):
"""调用明细 API — POST /data_mart/api/call_detail.dspy"""
stat_date = (params_kw or {}).get('date')
userorgid = (params_kw or {}).get('userorgid')
sql = "SELECT * FROM dm_model_call_fact WHERE 1=1"
ns = {}
if stat_date:
sql += " AND call_date=${stat_date}$"
ns['stat_date'] = stat_date
if userorgid:
sql += " AND userorgid=${userorgid}$"
ns['userorgid'] = userorgid
sql += " ORDER BY call_time DESC LIMIT 200"
rows = await sor.sqlExe(sql, ns)
return [dict(r) for r in rows]
def load_data_mart():
env = ServerEnv()
env.api_model_perf = api_model_perf
env.api_provider_roi = api_provider_roi
env.api_call_detail = api_call_detail
debug("[data_mart] registered API endpoints")

1
i18n/zh/msg.txt Normal file
View File

@ -0,0 +1 @@
{"模型性能监控":"Model Performance","供应商性价比对比":"Provider ROI","平均 TTFT":"Avg TTFT","成功率":"Success Rate","今日调用":"Today Calls","失败数":"Failures","模型性能":"Model Perf","供应商性价比":"Provider ROI","刷新":"Refresh"}

3
json/build.sh Normal file
View File

@ -0,0 +1,3 @@
#!/bin/bash
# data_mart has no CRUD tables - skip xls2ui
echo "[data_mart] json build skipped (no CRUD)"

12
pyproject.toml Normal file
View File

@ -0,0 +1,12 @@
[build-system]
requires = ["setuptools>=61", "wheel"]
build-backend = "setuptools.build_meta"
[project]
name = "data_mart"
version = "0.1.0"
description = "Sage 数据集市 — 模型性能指标、供应商性价比、多维度分析"
requires-python = ">=3.10"
[tool.setuptools.packages.find]
include = ["data_mart*"]

27
scripts/load_path.py Normal file
View File

@ -0,0 +1,27 @@
#!/usr/bin/env python3
"""data_mart 模块 load_path"""
import os, sys, subprocess
MOD = "data_mart"
SAGE_ROOT = os.environ.get("SAGE_ROOT", os.path.expanduser("~/sage"))
PYTHON = os.path.join(SAGE_ROOT, "py3", "bin", "python")
SET_PERM = os.path.join(SAGE_ROOT, "set_role_perm.py")
PATHS_LOGINED = [
f"/{MOD}",
f"/{MOD}/index.ui",
f"/{MOD}/provider_roi.ui",
f"/{MOD}/stat_avg_ttft.ui",
f"/{MOD}/stat_success_rate.ui",
f"/{MOD}/stat_total_calls.ui",
f"/{MOD}/stat_fail_calls.ui",
f"/{MOD}/api/%",
]
def run(role, path):
subprocess.run([PYTHON, SET_PERM, role, path], capture_output=True)
for p in PATHS_LOGINED:
run("logined", p)
print(f"[{MOD}] {len(PATHS_LOGINED)} paths registered")

View File

@ -0,0 +1,5 @@
# 模型性能统计 API
env = request._run_ns
async with get_sor_context(env, 'data_mart') as sor:
rows = await env.api_model_perf(sor, params_kw)
return json.dumps(rows, ensure_ascii=False)

View File

@ -0,0 +1,5 @@
# 供应商性价比 API
env = request._run_ns
async with get_sor_context(env, 'data_mart') as sor:
rows = await env.api_provider_roi(sor, params_kw)
return json.dumps(rows, ensure_ascii=False)

58
wwwroot/index.ui Normal file
View File

@ -0,0 +1,58 @@
{
"widgettype": "VBox",
"id": "datamart_root",
"options": {"width": "100%", "height": "100%", "bgcolor": "var(--sage-bg-primary, transparent)"},
"subwidgets": [
{
"widgettype": "VScrollPanel",
"options": {"css": "filler"},
"subwidgets": [
{
"widgettype": "HBox",
"options": {"width": "100%", "alignItems": "center", "marginBottom": "12px"},
"subwidgets": [
{"widgettype": "Title2", "options": {"fontWeight": "700", "otext": "模型性能监控", "i18n": true}},
{"widgettype": "Filler"},
{
"widgettype": "Button",
"options": {"label": "刷新", "fontSize": "12px"},
"binds": [{"wid": "self", "event": "click", "actiontype": "method", "target": "-@Cols", "method": "render_urldata", "params": {}}]
}
]
},
{
"widgettype": "HBox",
"options": {"width": "100%", "gap": "16px", "marginBottom": "16px"},
"subwidgets": [
{"widgettype": "RefreshWidget", "id": "stat_avg_ttft", "options": {"period_seconds": 60, "url": "{{entire_url('stat_avg_ttft.ui')}}", "width": "25%"}},
{"widgettype": "RefreshWidget", "id": "stat_success_rate", "options": {"period_seconds": 60, "url": "{{entire_url('stat_success_rate.ui')}}", "width": "25%"}},
{"widgettype": "RefreshWidget", "id": "stat_total_calls", "options": {"period_seconds": 60, "url": "{{entire_url('stat_total_calls.ui')}}", "width": "25%"}},
{"widgettype": "RefreshWidget", "id": "stat_fail_calls", "options": {"period_seconds": 60, "url": "{{entire_url('stat_fail_calls.ui')}}", "width": "25%"}}
]
},
{
"widgettype": "Cols",
"options": {
"css": "card",
"col_cwidth": 28,
"title": "模型性能排行",
"data_url": "{{entire_url('api/model_perf.dspy')}}",
"data_params": {},
"record_view": {
"widgettype": "HBox",
"options": {"cheight": 4, "width": "100%"},
"subwidgets": [
{"widgettype": "Text", "options": {"text": "${model}", "width": "30%"}},
{"widgettype": "Text", "options": {"text": "${avg_ttft_ms}ms", "width": "15%"}},
{"widgettype": "Text", "options": {"text": "${avg_ttot_ms}ms", "width": "15%"}},
{"widgettype": "Text", "options": {"text": "${success_rate}%", "width": "15%"}},
{"widgettype": "Text", "options": {"text": "${total_calls}", "width": "15%"}},
{"widgettype": "Text", "options": {"text": "¥${total_amount}", "width": "10%"}}
]
}
}
}
]
}
]
}

22
wwwroot/menu.ui Normal file
View File

@ -0,0 +1,22 @@
{
"widgettype": "VBox",
"options": {"width": "100%"},
"subwidgets": [
{
"widgettype": "MenuItem",
"options": {
"label": "模型性能",
"url": "{{entire_url('/data_mart/index.ui')}}",
"icon": "{{entire_url('/data_mart/imgs/perf.svg')}}"
}
},
{
"widgettype": "MenuItem",
"options": {
"label": "供应商性价比",
"url": "{{entire_url('/data_mart/provider_roi.ui')}}",
"icon": "{{entire_url('/data_mart/imgs/roi.svg')}}"
}
}
]
}

48
wwwroot/provider_roi.ui Normal file
View File

@ -0,0 +1,48 @@
{
"widgettype": "VBox",
"id": "provider_roi_root",
"options": {"width": "100%", "height": "100%"},
"subwidgets": [
{
"widgettype": "VScrollPanel",
"options": {"css": "filler"},
"subwidgets": [
{
"widgettype": "HBox",
"options": {"width": "100%", "alignItems": "center", "marginBottom": "12px"},
"subwidgets": [
{"widgettype": "Title2", "options": {"fontWeight": "700", "otext": "供应商性价比对比", "i18n": true}},
{"widgettype": "Filler"},
{
"widgettype": "Button",
"options": {"label": "刷新", "fontSize": "12px"},
"binds": [{"wid": "self", "event": "click", "actiontype": "method", "target": "-@Cols", "method": "render_urldata", "params": {}}]
}
]
},
{
"widgettype": "Cols",
"options": {
"css": "card",
"col_cwidth": 28,
"title": "同模型供应商价格对比",
"data_url": "{{entire_url('api/provider_roi.dspy')}}",
"data_params": {},
"record_view": {
"widgettype": "HBox",
"options": {"cheight": 4, "width": "100%"},
"subwidgets": [
{"widgettype": "Text", "options": {"text": "${model}", "width": "30%"}},
{"widgettype": "Text", "options": {"text": "${providerid}", "width": "20%"}},
{"widgettype": "Text", "options": {"text": "${total_calls}次", "width": "12%"}},
{"widgettype": "Text", "options": {"text": "¥${unit_price}/t", "width": "15%"}},
{"widgettype": "Text", "options": {"text": "${avg_ttft_ms}ms", "width": "13%"}},
{"widgettype": "Text", "options": {"text": "¥${total_amount}", "width": "10%"}}
]
}
}
}
]
}
]
}

12
wwwroot/stat_avg_ttft.ui Normal file
View File

@ -0,0 +1,12 @@
{% set rows = api_model_perf(request) %}
{% set total = [] %}{% set avg = 0 %}{% set cnt = 0 %}
{% for r in rows %}{% if r.avg_ttft_ms %}{% set avg = avg + r.avg_ttft_ms %}{% set cnt = cnt + 1 %}{% endif %}{% endfor %}
{% if cnt > 0 %}{% set avg = (avg / cnt)|round(0) %}{% endif %}
{
"widgettype": "VBox",
"options": {"css": "card", "padding": "16px", "borderRadius": "12px", "width": "100%"},
"subwidgets": [
{"widgettype": "Text", "options": {"text": "平均 TTFT", "fontSize": "12px", "color": "var(--sage-text-secondary)"}},
{"widgettype": "Title2", "options": {"text": "{{avg}}ms", "fontWeight": "700", "marginTop": "4px"}}
]
}

View File

@ -0,0 +1,2 @@
{% set rows = api_model_perf(request) %}{% set t = 0 %}{% for r in rows %}{% set t = t + r.fail_calls %}{% endfor %}
{"widgettype":"VBox","options":{"css":"card","padding":"16px","borderRadius":"12px","width":"100%"},"subwidgets":[{"widgettype":"Text","options":{"text":"失败数","fontSize":"12px","color":"var(--sage-text-secondary)"}},{"widgettype":"Title2","options":{"text":"{{t}}","fontWeight":"700","marginTop":"4px"}}]}

View File

@ -0,0 +1,12 @@
{% set rows = api_model_perf(request) %}
{% set total = 0 %}{% set succ = 0 %}
{% for r in rows %}{% set total = total + r.total_calls %}{% set succ = succ + r.success_calls %}{% endfor %}
{% set rate = (succ * 100 / total)|round(1) if total > 0 else 0 %}
{
"widgettype": "VBox",
"options": {"css": "card", "padding": "16px", "borderRadius": "12px", "width": "100%"},
"subwidgets": [
{"widgettype": "Text", "options": {"text": "成功率", "fontSize": "12px", "color": "var(--sage-text-secondary)"}},
{"widgettype": "Title2", "options": {"text": "{{rate}}%", "fontWeight": "700", "marginTop": "4px"}}
]
}

View File

@ -0,0 +1,2 @@
{% set rows = api_model_perf(request) %}{% set t = 0 %}{% for r in rows %}{% set t = t + r.total_calls %}{% endfor %}
{"widgettype":"VBox","options":{"css":"card","padding":"16px","borderRadius":"12px","width":"100%"},"subwidgets":[{"widgettype":"Text","options":{"text":"今日调用","fontSize":"12px","color":"var(--sage-text-secondary)"}},{"widgettype":"Title2","options":{"text":"{{t}}","fontWeight":"700","marginTop":"4px"}}]}