diff --git a/.gitignore b/.gitignore index 9f7983e..93504ce 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ __pycache__/ *.pyc *.egg-info/ +models/mysql.ddl.sql diff --git a/build.sh b/build.sh index 6cbcedf..df970b9 100644 --- a/build.sh +++ b/build.sh @@ -1,4 +1,40 @@ #!/bin/bash +# Sage_datamart module build script +# Generates DDL from models and links wwwroot to Sage + set -e -cd "$(dirname "$0")" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Find Sage root +SAGE_ROOT="" +for candidate in "$SCRIPT_DIR/../.." "$HOME/repos/sage" "$HOME/sage"; do + if [ -d "$candidate/wwwroot" ] && [ -d "$candidate/py3/bin" ]; then + SAGE_ROOT="$(cd "$candidate" && pwd)" + break + fi +done + +if [ -z "$SAGE_ROOT" ]; then + echo "ERROR: Sage root not found" + exit 1 +fi + +echo "Sage root: $SAGE_ROOT" + +# Generate DDL from models if present +if [ -d "$SCRIPT_DIR/models" ]; then + echo "Generating DDL from models..." + cd "$SCRIPT_DIR/models" + if ls *.json 1>/dev/null 2>&1; then + "$SAGE_ROOT/py3/bin/json2ddl" mysql . > mysql.ddl.sql + echo "DDL generated: models/mysql.ddl.sql" + fi +fi + +# Link wwwroot to Sage +echo "Linking wwwroot to Sage..." +rm -f "$SAGE_ROOT/wwwroot/sage_datamart" +ln -sf "$SCRIPT_DIR/wwwroot" "$SAGE_ROOT/wwwroot/sage_datamart" + echo "[sage_datamart] build done" diff --git a/models/dm_model_call_fact.json b/models/dm_model_call_fact.json new file mode 100644 index 0000000..45c973e --- /dev/null +++ b/models/dm_model_call_fact.json @@ -0,0 +1,220 @@ +{ + "summary": [ + { + "name": "dm_model_call_fact", + "title": "LLM调用明细事实表", + "primary": ["id"], + "catelog": "fact" + } + ], + "fields": [ + { + "name": "id", + "title": "主键ID", + "type": "str", + "length": 32, + "nullable": "no" + }, + { + "name": "luid", + "title": "LLM使用记录ID", + "type": "str", + "length": 32, + "nullable": "no" + }, + { + "name": "llmid", + "title": "LLM模型ID", + "type": "str", + "length": 32, + "nullable": "no" + }, + { + "name": "model", + "title": "模型名称", + "type": "str", + "length": 100 + }, + { + "name": "catelogid", + "title": "目录ID", + "type": "str", + "length": 10 + }, + { + "name": "userid", + "title": "用户ID", + "type": "str", + "length": 32, + "nullable": "no" + }, + { + "name": "userorgid", + "title": "用户机构ID", + "type": "str", + "length": 32, + "nullable": "no" + }, + { + "name": "ownerid", + "title": "所有者ID", + "type": "str", + "length": 32 + }, + { + "name": "providerid", + "title": "供应商ID", + "type": "str", + "length": 32 + }, + { + "name": "call_date", + "title": "调用日期", + "type": "date", + "nullable": "no" + }, + { + "name": "call_hour", + "title": "调用小时", + "type": "int", + "nullable": "no" + }, + { + "name": "call_time", + "title": "调用时间", + "type": "datetime", + "nullable": "no" + }, + { + "name": "ttft_ms", + "title": "首Token响应时间(毫秒)", + "type": "int" + }, + { + "name": "ttot_ms", + "title": "总响应时间(毫秒)", + "type": "int" + }, + { + "name": "prompt_tokens", + "title": "提示Token数", + "type": "int", + "default": "0" + }, + { + "name": "completion_tokens", + "title": "完成Token数", + "type": "int", + "default": "0" + }, + { + "name": "status", + "title": "状态", + "type": "str", + "length": 20 + }, + { + "name": "fail_reason", + "title": "失败原因", + "type": "str", + "length": 200 + }, + { + "name": "amount", + "title": "金额", + "type": "double", + "length": 10, + "dec": 4, + "default": "0" + }, + { + "name": "currency", + "title": "货币", + "type": "str", + "length": 5, + "default": "CNY" + }, + { + "name": "distributor_orgid", + "title": "分销商机构ID", + "type": "str", + "length": 32 + }, + { + "name": "sale_userid", + "title": "销售用户ID", + "type": "str", + "length": 32 + }, + { + "name": "created_at", + "title": "创建时间", + "type": "datetime", + "default": "NOW()" + } + ], + "indexes": [ + { + "name": "idx_call_date", + "idxtype": "index", + "idxfields": ["call_date"] + }, + { + "name": "idx_llmid", + "idxtype": "index", + "idxfields": ["llmid"] + }, + { + "name": "idx_userorgid", + "idxtype": "index", + "idxfields": ["userorgid"] + }, + { + "name": "idx_ownerid", + "idxtype": "index", + "idxfields": ["ownerid"] + }, + { + "name": "idx_status", + "idxtype": "index", + "idxfields": ["status"] + }, + { + "name": "idx_model", + "idxtype": "index", + "idxfields": ["model"] + } + ], + "codes": [ + { + "field": "llmid", + "table": "llm", + "valuefield": "id", + "textfield": "model" + }, + { + "field": "userid", + "table": "users", + "valuefield": "id", + "textfield": "username" + }, + { + "field": "userorgid", + "table": "organization", + "valuefield": "id", + "textfield": "orgname" + }, + { + "field": "ownerid", + "table": "organization", + "valuefield": "id", + "textfield": "orgname" + }, + { + "field": "providerid", + "table": "llmprovider", + "valuefield": "id", + "textfield": "name" + } + ] +} diff --git a/models/dm_model_perf_daily.json b/models/dm_model_perf_daily.json new file mode 100644 index 0000000..151a543 --- /dev/null +++ b/models/dm_model_perf_daily.json @@ -0,0 +1,223 @@ +{ + "summary": [ + { + "name": "dm_model_perf_daily", + "title": "模型性能日汇总表", + "primary": ["id"], + "catelog": "summary" + } + ], + "fields": [ + { + "name": "id", + "title": "主键ID", + "type": "str", + "length": 32, + "nullable": "no" + }, + { + "name": "stat_date", + "title": "统计日期", + "type": "date", + "nullable": "no" + }, + { + "name": "llmid", + "title": "LLM模型ID", + "type": "str", + "length": 32, + "nullable": "no" + }, + { + "name": "model", + "title": "模型名称", + "type": "str", + "length": 100 + }, + { + "name": "catelogid", + "title": "目录ID", + "type": "str", + "length": 10 + }, + { + "name": "providerid", + "title": "供应商ID", + "type": "str", + "length": 32 + }, + { + "name": "ownerid", + "title": "所有者ID", + "type": "str", + "length": 32 + }, + { + "name": "total_calls", + "title": "总调用次数", + "type": "int", + "default": "0" + }, + { + "name": "success_calls", + "title": "成功调用次数", + "type": "int", + "default": "0" + }, + { + "name": "fail_calls", + "title": "失败调用次数", + "type": "int", + "default": "0" + }, + { + "name": "success_rate", + "title": "成功率(%)", + "type": "double", + "length": 5, + "dec": 2 + }, + { + "name": "avg_ttft_ms", + "title": "平均首Token响应时间(毫秒)", + "type": "double", + "length": 10, + "dec": 1 + }, + { + "name": "p50_ttft_ms", + "title": "P50首Token响应时间(毫秒)", + "type": "double", + "length": 10, + "dec": 1 + }, + { + "name": "p95_ttft_ms", + "title": "P95首Token响应时间(毫秒)", + "type": "double", + "length": 10, + "dec": 1 + }, + { + "name": "p99_ttft_ms", + "title": "P99首Token响应时间(毫秒)", + "type": "double", + "length": 10, + "dec": 1 + }, + { + "name": "avg_ttot_ms", + "title": "平均总响应时间(毫秒)", + "type": "double", + "length": 10, + "dec": 1 + }, + { + "name": "p50_ttot_ms", + "title": "P50总响应时间(毫秒)", + "type": "double", + "length": 10, + "dec": 1 + }, + { + "name": "p95_ttot_ms", + "title": "P95总响应时间(毫秒)", + "type": "double", + "length": 10, + "dec": 1 + }, + { + "name": "max_concurrent", + "title": "最大并发数", + "type": "int" + }, + { + "name": "avg_concurrent", + "title": "平均并发数", + "type": "double", + "length": 5, + "dec": 1 + }, + { + "name": "total_prompt_tokens", + "title": "总提示Token数", + "type": "bigint", + "default": "0" + }, + { + "name": "total_completion_tokens", + "title": "总完成Token数", + "type": "bigint", + "default": "0" + }, + { + "name": "total_amount", + "title": "总金额", + "type": "double", + "length": 12, + "dec": 4, + "default": "0" + }, + { + "name": "userorgid", + "title": "用户机构ID", + "type": "str", + "length": 32 + }, + { + "name": "distributor_orgid", + "title": "分销商机构ID", + "type": "str", + "length": 32 + } + ], + "indexes": [ + { + "name": "uk_date_model_org", + "idxtype": "unique", + "idxfields": ["stat_date", "llmid", "userorgid"] + }, + { + "name": "idx_date", + "idxtype": "index", + "idxfields": ["stat_date"] + }, + { + "name": "idx_model", + "idxtype": "index", + "idxfields": ["model"] + } + ], + "codes": [ + { + "field": "llmid", + "table": "llm", + "valuefield": "id", + "textfield": "model" + }, + { + "field": "providerid", + "table": "llmprovider", + "valuefield": "id", + "textfield": "name" + }, + { + "field": "ownerid", + "table": "organization", + "valuefield": "id", + "textfield": "orgname" + }, + { + "field": "userorgid", + "table": "organization", + "valuefield": "id", + "textfield": "orgname" + }, + { + "field": "distributor_orgid", + "table": "organization", + "valuefield": "id", + "textfield": "orgname" + } + ] +} diff --git a/models/dm_provider_cost_daily.json b/models/dm_provider_cost_daily.json new file mode 100644 index 0000000..35cde16 --- /dev/null +++ b/models/dm_provider_cost_daily.json @@ -0,0 +1,110 @@ +{ + "summary": [ + { + "name": "dm_provider_cost_daily", + "title": "供应商成本日汇总表", + "primary": ["id"], + "catelog": "summary" + } + ], + "fields": [ + { + "name": "id", + "title": "主键ID", + "type": "str", + "length": 32, + "nullable": "no" + }, + { + "name": "stat_date", + "title": "统计日期", + "type": "date", + "nullable": "no" + }, + { + "name": "model", + "title": "模型名称", + "type": "str", + "length": 100, + "nullable": "no" + }, + { + "name": "catelogid", + "title": "目录ID", + "type": "str", + "length": 10 + }, + { + "name": "providerid", + "title": "供应商ID", + "type": "str", + "length": 32, + "nullable": "no" + }, + { + "name": "provider_name", + "title": "供应商名称", + "type": "str", + "length": 100 + }, + { + "name": "total_calls", + "title": "总调用次数", + "type": "int", + "default": "0" + }, + { + "name": "total_tokens", + "title": "总Token数", + "type": "bigint", + "default": "0" + }, + { + "name": "total_amount", + "title": "总金额", + "type": "double", + "length": 12, + "dec": 4, + "default": "0" + }, + { + "name": "unit_price", + "title": "单价(每Token)", + "type": "double", + "length": 10, + "dec": 6 + }, + { + "name": "avg_ttft_ms", + "title": "平均首Token响应时间(毫秒)", + "type": "double", + "length": 10, + "dec": 1 + } + ], + "indexes": [ + { + "name": "uk_date_model_provider", + "idxtype": "unique", + "idxfields": ["stat_date", "model", "providerid"] + }, + { + "name": "idx_model", + "idxtype": "index", + "idxfields": ["model"] + }, + { + "name": "idx_date", + "idxtype": "index", + "idxfields": ["stat_date"] + } + ], + "codes": [ + { + "field": "providerid", + "table": "llmprovider", + "valuefield": "id", + "textfield": "name" + } + ] +}