diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f68195f --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +# CRUD definition directories (auto-generated by Sage platform) +wwwroot/uapi/ +wwwroot/uapiio/ +wwwroot/upapp/ +wwwroot/upappkey/ +__pycache__/ diff --git a/README.md b/README.md index bcf3c4d..a61abff 100644 --- a/README.md +++ b/README.md @@ -397,6 +397,37 @@ uapi --- +## 在 Sage 系统中的角色 + +uapi 是 Sage 平台的**配置化 API 网关层**，llmage（大模型管理模块）是其主要消费者。两者的协同关系： + +``` +llmage (模型管理) uapi (API 网关) + │ │ + │ llm 表 │ + │ upappid ──────────────→│ upapp 表 (baseurl, myappid, ownerid) + │ apiname ──────────────→│ uapi 表 (httpmethod, path, headers, ...) + │ │ uapiset 表 (auth_apiname) + │ │ upappkey 表 (apikey, secretkey) + │ │ + │ UpAppApi(request) │ + │ .stream_linify() ─────→│ StreamHttpClient → 外部 LLM API + │ .call() ──────────────→│ 同步/流式 HTTP 调用 + │ │ +``` + +**新增一个 LLM 的完整流程**： +1. 在 uapi 模块的 `uapiset` 中创建 API 集合（配置认证方式） +2. 在 `upapp` 中注册上位系统（baseurl、appkey 等） +3. 在 `uapi` 中定义具体的 API 端点（path、method、headers 模板、response 模板） +4. 在 `upappkey` 中分配 API 密钥给调用方 +5. 在 llmage 模块的 `llm` 表中注册模型，关联 `upappid` + `apiname` +6. 用户在 llmage 前端页面点击模型卡片 → 推理 → 通过 uapi 网关调用外部 API + +**优势**：新增模型无需修改 Python 代码，只需在数据库/CRUD 页面中配置 API 定义。 + +--- + ## 开发注意事项 1. **dbname 获取**：必须通过 `get_serverenv('get_module_dbname')('uapi')` 动态获取，禁止硬编码 diff --git a/i18n/en/msg.txt b/i18n/en/msg.txt new file mode 100644 index 0000000..eb38b58 --- /dev/null +++ b/i18n/en/msg.txt @@ -0,0 +1,74 @@ +API: API +APIKEY: APIKEY +API接口: API Interface +API标题: API Title +API输入输出: API Input/Output +API集: API Collection +API集名称: API Collection Name +Add Error: Add Error +Add Success: Add Success +Authorization Error: Authorization Error +Cancel: Cancel +Conform: Confirm +Delete Error: Delete Error +Delete Success: Delete Success +Discard: Discard +Please login: Please login +Record no exist or with wrong ownership: Record no exist or with wrong ownership +Reset: Reset +Submit: Submit +Update Error: Update Error +Update Success: Update Success +api名称: API Name +api密码: API Password +api密钥: API Secret +api测试: API Test +api用户: API User +api集id: API Collection ID +failed: failed +headers模版: Headers Template +http方法: HTTP Method +id: id +ok: ok +path: path +上位应用名: Parent Application Name +上位系统: Parent System +上位系统ID: Parent System ID +上位系统密码: Parent System Password +上位系统密码表: Parent System Password Table +上游任务: Upstream Task +加密密钥: Encryption Key +动态headers函数: Dynamic Headers Function +参数模版: Parameter Template +响应模版: Response Template +响应转换函数名: Response Transform Function Name +回调url: Callback URL +外部系统: External System +属主id: Owner ID +属主机构id: Owner Organization ID +应用id: Application ID +开始时间: Start Time +我的appid: My App ID +所属机构: Organization +执行方任务id: Executor Task ID +授权api名: Authorized API Name +接口描述: Interface Description +描述: Description +数据模版: Data Template +是否第一用户: Is First User +本地业务id: Local Business ID +流式匹配串: Stream Match String +流式输出: Stream Output +状态: Status +用户id: User ID +相应数据: Response Data +类型名: Type Name +类型说明: Type Description +系统url: System URL +结束时间: End Time +认证API名: Auth API Name +跳转到: Jump to +输入字段: Input Field +输入输出: Input/Output +输入输出id: Input/Output ID +需要鉴权: Requires Auth diff --git a/i18n/jp/msg.txt b/i18n/jp/msg.txt new file mode 100644 index 0000000..c21db94 --- /dev/null +++ b/i18n/jp/msg.txt @@ -0,0 +1,74 @@ +API: API +APIKEY: APIKEY +API接口: APIインターフェース +API标题: APIタイトル +API输入输出: API入出力 +API集: APIコレクション +API集名称: APIコレクション名 +Add Error: 追加エラー +Add Success: 追加成功 +Authorization Error: 認証エラー +Cancel: キャンセル +Conform: 確認 +Delete Error: 削除エラー +Delete Success: 削除成功 +Discard: 破棄 +Please login: ログインしてください +Record no exist or with wrong ownership: レコードが存在しないか、所有権が不正です +Reset: リセット +Submit: 送信 +Update Error: 更新エラー +Update Success: 更新成功 +api名称: API名称 +api密码: APIパスワード +api密钥: APIシークレット +api测试: APIテスト +api用户: APIユーザー +api集id: APIコレクションID +failed: 失敗 +headers模版: ヘッダーテンプレート +http方法: HTTPメソッド +id: id +ok: ok +path: path +上位应用名: 上位アプリケーション名 +上位系统: 上位システム +上位系统ID: 上位システムID +上位系统密码: 上位システムパスワード +上位系统密码表: 上位システムパスワード表 +上游任务: 上流タスク +加密密钥: 暗号化キー +动态headers函数: 動的ヘッダー関数 +参数模版: パラメーターテンプレート +响应模版: レスポンステンプレート +响应转换函数名: レスポンス変換関数名 +回调url: コールバックURL +外部系统: 外部システム +属主id: 所有者ID +属主机构id: 所有者組織ID +应用id: アプリケーションID +开始时间: 開始時間 +我的appid: マイAppID +所属机构: 所属組織 +执行方任务id: 実行タスクID +授权api名: 認可API名 +接口描述: インターフェース説明 +描述: 説明 +数据模版: データテンプレート +是否第一用户: 最初のユーザーか +本地业务id: ローカル業務ID +流式匹配串: ストリームマッチ文字列 +流式输出: ストリーム出力 +状态: ステータス +用户id: ユーザーID +相应数据: 対応データ +类型名: 型名 +类型说明: 型説明 +系统url: システムURL +结束时间: 終了時間 +认证API名: 認証API名 +跳转到: ジャンプ先 +输入字段: 入力フィールド +输入输出: 入出力 +输入输出id: 入出力ID +需要鉴权: 認証が必要 diff --git a/i18n/ko/msg.txt b/i18n/ko/msg.txt new file mode 100644 index 0000000..3c6913d --- /dev/null +++ b/i18n/ko/msg.txt @@ -0,0 +1,74 @@ +API: API +APIKEY: APIKEY +API接口: API 인터페이스 +API标题: API 제목 +API输入输出: API 입출력 +API集: API 컬렉션 +API集名称: API 컬렉션 이름 +Add Error: 추가 오류 +Add Success: 추가 성공 +Authorization Error: 인증 오류 +Cancel: 취소 +Conform: 확인 +Delete Error: 삭제 오류 +Delete Success: 삭제 성공 +Discard: 폐기 +Please login: 로그인해주세요 +Record no exist or with wrong ownership: 레코드가 존재하지 않거나 소유권이 잘못되었습니다 +Reset: 초기화 +Submit: 제출 +Update Error: 업데이트 오류 +Update Success: 업데이트 성공 +api名称: API 이름 +api密码: API 비밀번호 +api密钥: API 비밀키 +api测试: API 테스트 +api用户: API 사용자 +api集id: API 컬렉션 ID +failed: 실패 +headers模版: 헤더 템플릿 +http方法: HTTP 메서드 +id: id +ok: ok +path: path +上位应用名: 상위 애플리케이션 이름 +上位系统: 상위 시스템 +上位系统ID: 상위 시스템 ID +上位系统密码: 상위 시스템 비밀번호 +上位系统密码表: 상위 시스템 비밀번호 테이블 +上游任务: 업스트림 작업 +加密密钥: 암호화 키 +动态headers函数: 동적 헤더 함수 +参数模版: 파라미터 템플릿 +响应模版: 응답 템플릿 +响应转换函数名: 응답 변환 함수 이름 +回调url: 콜백 URL +外部系统: 외부 시스템 +属主id: 소유자 ID +属主机构id: 소유자 조직 ID +应用id: 애플리케이션 ID +开始时间: 시작 시간 +我的appid: 내 App ID +所属机构: 소속 조직 +执行方任务id: 실행자 작업 ID +授权api名: 인증된 API 이름 +接口描述: 인터페이스 설명 +描述: 설명 +数据模版: 데이터 템플릿 +是否第一用户: 첫 번째 사용자 여부 +本地业务id: 로컬 비즈니스 ID +流式匹配串: 스트림 매칭 문자열 +流式输出: 스트림 출력 +状态: 상태 +用户id: 사용자 ID +相应数据: 응답 데이터 +类型名: 타입 이름 +类型说明: 타입 설명 +系统url: 시스템 URL +结束时间: 종료 시간 +认证API名: 인증 API 이름 +跳转到: 이동 +输入字段: 입력 필드 +输入输出: 입출력 +输入输出id: 입출력 ID +需要鉴权: 인증 필요 diff --git a/i18n/zh/msg.txt b/i18n/zh/msg.txt new file mode 100644 index 0000000..c93ae7c --- /dev/null +++ b/i18n/zh/msg.txt @@ -0,0 +1,74 @@ +API: API +APIKEY: APIKEY +API接口: API接口 +API标题: API标题 +API输入输出: API输入输出 +API集: API集 +API集名称: API集名称 +Add Error: Add Error +Add Success: Add Success +Authorization Error: Authorization Error +Cancel: Cancel +Conform: Conform +Delete Error: Delete Error +Delete Success: Delete Success +Discard: Discard +Please login: Please login +Record no exist or with wrong ownership: Record no exist or with wrong ownership +Reset: Reset +Submit: Submit +Update Error: Update Error +Update Success: Update Success +api名称: api名称 +api密码: api密码 +api密钥: api密钥 +api测试: api测试 +api用户: api用户 +api集id: api集id +failed: failed +headers模版: headers模版 +http方法: http方法 +id: id +ok: ok +path: path +上位应用名: 上位应用名 +上位系统: 上位系统 +上位系统ID: 上位系统ID +上位系统密码: 上位系统密码 +上位系统密码表: 上位系统密码表 +上游任务: 上游任务 +加密密钥: 加密密钥 +动态headers函数: 动态headers函数 +参数模版: 参数模版 +响应模版: 响应模版 +响应转换函数名: 响应转换函数名 +回调url: 回调url +外部系统: 外部系统 +属主id: 属主id +属主机构id: 属主机构id +应用id: 应用id +开始时间: 开始时间 +我的appid: 我的appid +所属机构: 所属机构 +执行方任务id: 执行方任务id +授权api名: 授权api名 +接口描述: 接口描述 +描述: 描述 +数据模版: 数据模版 +是否第一用户: 是否第一用户 +本地业务id: 本地业务id +流式匹配串: 流式匹配串 +流式输出: 流式输出 +状态: 状态 +用户id: 用户id +相应数据: 相应数据 +类型名: 类型名 +类型说明: 类型说明 +系统url: 系统url +结束时间: 结束时间 +认证API名: 认证API名 +跳转到: 跳转到 +输入字段: 输入字段 +输入输出: 输入输出 +输入输出id: 输入输出id +需要鉴权: 需要鉴权 diff --git a/json/uapi.json b/json/uapi.json index e28048f..106708b 100644 --- a/json/uapi.json +++ b/json/uapi.json @@ -27,11 +27,11 @@ "title":"API", "description":"API定义", - "sortby":["apisetid", "name"], + "sortby":"name", "browserfields":{ - "exclouded":["id", "apisetid"], + "exclouded":["id"], "alters":{} }, - "editexclouded":["id", "apisetid"] + "editexclouded":["id","upappid"] } } diff --git a/json/uapiio.json b/json/uapiio.json index 07d5129..314348c 100644 --- a/json/uapiio.json +++ b/json/uapiio.json @@ -4,6 +4,12 @@ "title":"API输入输出", "description":"API的输入输出定义", "sortby": "name", + "data_filter":{ + "fields":[ + {"field":"name","title":"类型名","uitype":"str"}, + {"field":"description","title":"类型说明","uitype":"str"} + ] + }, "browserfields":{ "exclouded":["id"], "alters":{} diff --git a/json/uapiset.json b/json/uapiset.json deleted file mode 100644 index 0889ed8..0000000 --- a/json/uapiset.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "tblname":"uapiset", - "params":{ - "title":"API集", - "sortby":"name", - "browserfields":{ - "exclouded":["id" ], - "alters":{} - }, - "editexclouded":["id"], - "subtables":[ - { - "field":"apisetid", - "subtable":"uapi", - "title":"定义api" - } - ] - } -} diff --git a/json/upapp.json b/json/upapp.json index 8e8680e..e234cd1 100644 --- a/json/upapp.json +++ b/json/upapp.json @@ -16,27 +16,12 @@ "field":"upappid", "subtable": "upappkey", "title": "APIKEY" + }, + { + "field":"upappid", + "subtable": "uapi", + "title": "API" } - ], - "toolbar":{ - "tools":[ - { - "name":"jumpin", - "label":"跳转到", - "selected_data": true - } - ] - }, - "binds":[ - { - "wid":"self", - "event":"jumpin", - "actiontype":"urlwidget", - "target":"self", - "options":{ - "url":"{{entire_url('/uapi/jump_in.dspy')}}" - } - } - ] + ] } } diff --git a/models/uapi.json b/models/uapi.json new file mode 100644 index 0000000..1e2ebd7 --- /dev/null +++ b/models/uapi.json @@ -0,0 +1,153 @@ +{ + "summary": [ + { + "name": "uapi", + "title": "API接口", + "primary": [ + "id" + ], + "catelog": "entity" + } + ], + "fields": [ + { + "name": "id", + "title": "id", + "type": "str", + "length": 32 + }, + { + "name": "upappid", + "title": "上位系统ID", + "type": "str", + "length": 32 + }, + { + "name": "name", + "title": "api名称", + "type": "str", + "length": 200 + }, + { + "name": "title", + "title": "API标题", + "type": "str", + "length": 100 + }, + { + "name": "description", + "title": "接口描述", + "type": "text" + }, + { + "name": "need_auth", + "title": "需要鉴权", + "type": "str", + "length": 1, + "default": "0" + }, + { + "name": "stream", + "title": "流式输出", + "type": "str", + "length": 20 + }, + { + "name": "path", + "title": "path", + "type": "str", + "length": 4000 + }, + { + "name": "httpmethod", + "title": "http方法", + "type": "str", + "length": 20, + "nullable": "yes", + "default": "GET" + }, + { + "name": "chunk_match", + "title": "流式匹配串", + "type": "str", + "length": 100 + }, + { + "name": "headers", + "title": "headers模版", + "type": "text", + "nullable": "yes" + }, + { + "name": "params", + "title": "参数模版", + "type": "text", + "nullable": "yes" + }, + { + "name": "data", + "title": "数据模版", + "type": "text", + "nullable": "yes" + }, + { + "name": "response", + "title": "响应模版", + "type": "text", + "nullable": "yes" + }, + { + "name": "ioid", + "title": "输入输出id", + "type": "str", + "length": 32, + "nullable": "yes" + }, + { + "name": "callbackurl", + "title": "回调url", + "type": "str", + "length": 1000, + "nullable": "yes" + } + ], + "indexes": [ + { + "name": "idx1", + "idxtype": "unique", + "idxfields": [ + "upappid", + "name" + ] + } + ], + "codes": [ + { + "field": "httpmethod", + "table": "appcodes_kv", + "valuefield": "k", + "textfield": "v", + "cond": "parentid='httpmethod'" + }, + { + "field": "need_auth", + "table": "appcodes_kv", + "valuefield": "k", + "textfield": "v", + "cond": "parentid='yesno'" + }, + { + "field": "stream", + "table": "appcodes_kv", + "valuefield": "k", + "textfield": "v", + "cond": "parentid='resp_mode'" + }, + { + "field": "ioid", + "table": "uapiio", + "valuefield": "id", + "textfield": "name" + } + ] +} \ No newline at end of file diff --git a/models/uapiio.json b/models/uapiio.json new file mode 100644 index 0000000..e60446c --- /dev/null +++ b/models/uapiio.json @@ -0,0 +1,36 @@ +{ + "summary": [ + { + "name": "uapiio", + "title": "输入输出", + "primary": [ + "id" + ], + "catelog": "entity" + } + ], + "fields": [ + { + "name": "id", + "title": "id", + "type": "str", + "length": 32 + }, + { + "name": "name", + "title": "类型名", + "type": "str", + "length": 100 + }, + { + "name": "description", + "title": "类型说明", + "type": "text" + }, + { + "name": "input_fields", + "title": "输入字段", + "type": "text" + } + ] +} \ No newline at end of file diff --git a/models/uapiset.json b/models/uapiset.json new file mode 100644 index 0000000..0426ea7 --- /dev/null +++ b/models/uapiset.json @@ -0,0 +1,39 @@ +{ + "summary": [ + { + "name": "uapiset", + "title": "API集", + "primary": [ + "id" + ], + "catelog": "entity" + } + ], + "fields": [ + { + "name": "id", + "title": "id", + "type": "str", + "length": 32 + }, + { + "name": "name", + "title": "API集名称", + "type": "str", + "length": 200 + }, + { + "name": "description", + "title": "描述", + "type": "text", + "default": "0" + }, + { + "name": "auth_apiname", + "title": "授权api名", + "type": "str", + "length": 200, + "nullable": "yes" + } + ] +} \ No newline at end of file diff --git a/models/upapp.json b/models/upapp.json new file mode 100644 index 0000000..cf87b9c --- /dev/null +++ b/models/upapp.json @@ -0,0 +1,83 @@ +{ + "summary": [ + { + "name": "upapp", + "title": "外部系统", + "primary": [ + "id" + ] + } + ], + "fields": [ + { + "name": "id", + "title": "id", + "type": "str", + "length": 32 + }, + { + "name": "name", + "title": "上位应用名", + "type": "str", + "length": 200 + }, + { + "name": "description", + "title": "描述", + "type": "text", + "default": "0" + }, + { + "name": "ownerid", + "title": "所属机构", + "type": "str", + "length": 32, + "nullable": "yes" + }, + { + "name": "apisetid", + "title": "api集id", + "type": "str", + "length": 32 + }, + { + "name": "secretkey", + "title": "加密密钥", + "type": "str", + "length": 100 + }, + { + "name": "baseurl", + "title": "系统url", + "type": "str", + "length": 500 + }, + { + "name": "myappid", + "title": "我的appid", + "type": "str", + "length": 100 + }, + { + "name": "dynamic_func", + "title": "动态headers函数", + "type": "str", + "length": 255 + }, + { + "name": "auth_apiname", + "title": "认证API名", + "type": "str", + "length": 200, + "nullable": "yes" + } + ], + "codes": [ + { + "field": "ownerid", + "table": "organization", + "valuefield": "id", + "textfield": "orgname" + } + ] +} \ No newline at end of file diff --git a/models/upappkey.json b/models/upappkey.json new file mode 100644 index 0000000..58f1969 --- /dev/null +++ b/models/upappkey.json @@ -0,0 +1,90 @@ +{ + "summary": [ + { + "name": "upappkey", + "title": "上位系统密码表", + "primary": [ + "id" + ] + } + ], + "fields": [ + { + "name": "id", + "title": "id", + "type": "str", + "length": 32 + }, + { + "name": "upappid", + "title": "应用id", + "type": "str", + "length": 32 + }, + { + "name": "ownerid", + "title": "属主id", + "type": "str", + "length": 32, + "default": "0" + }, + { + "name": "apikey", + "title": "api密钥", + "type": "str", + "length": 4000, + "default": "0" + }, + { + "name": "apiuser", + "title": "api用户", + "type": "str", + "length": 100 + }, + { + "name": "apipasswd", + "title": "api密码", + "type": "str", + "length": 100 + }, + { + "name": "orgid", + "title": "属主机构id", + "type": "str", + "length": 32 + }, + { + "name": "is_first", + "title": "是否第一用户", + "type": "str", + "length": 1 + } + ], + "codes": [ + { + "field": "ownerid", + "table": "users", + "valuefield": "id", + "textfield": "username" + }, + { + "field": "upappid", + "table": "upapp", + "valuefield": "id", + "textfield": "name" + }, + { + "field": "orgid", + "table": "organization", + "valuefield": "id", + "textfield": "orgname" + }, + { + "field": "is_first", + "table": "appcodes_kv", + "valuefield": "k", + "textfield": "v", + "cond": "parentid='yesno'" + } + ] +} \ No newline at end of file diff --git a/models/uptask.json b/models/uptask.json new file mode 100644 index 0000000..eebd3e8 --- /dev/null +++ b/models/uptask.json @@ -0,0 +1,74 @@ +{ + "summary": [ + { + "name": "uptask", + "title": "上游任务", + "primary": [ + "id" + ], + "catelog": "entity" + } + ], + "fields": [ + { + "name": "id", + "title": "id", + "type": "str", + "length": 32 + }, + { + "name": "userid", + "title": "用户id", + "type": "str", + "length": 32 + }, + { + "name": "executor_taskid", + "title": "执行方任务id", + "type": "str", + "length": 64 + }, + { + "name": "convert_func_name", + "title": "响应转换函数名", + "type": "str", + "length": 128 + }, + { + "name": "local_bizid", + "title": "本地业务id", + "type": "str", + "length": 32 + }, + { + "name": "status", + "title": "状态", + "type": "str", + "length": 24 + }, + { + "name": "response_data", + "title": "相应数据", + "type": "text" + }, + { + "name": "start_timestamp", + "title": "开始时间", + "type": "time" + }, + { + "name": "end_timestamp", + "title": "结束时间", + "type": "time" + } + ], + "indexes": [ + { + "name": "idx1", + "idxtype": "unique", + "idxfields": [ + "executor_taskid" + ] + } + ] +} \ No newline at end of file diff --git a/scripts/load_path.py b/scripts/load_path.py new file mode 100644 index 0000000..8a758ce --- /dev/null +++ b/scripts/load_path.py @@ -0,0 +1,56 @@ +"""Generate RBAC permissions for uapi module paths. + +Run from Sage root with Sage venv: + cd ~/repos/sage && ./py3/bin/python ../uapi/scripts/load_path.py +""" +import os +import sys +import asyncio + +sage_root = os.environ.get('SAGE_ROOT') +if sage_root and sage_root not in sys.path: + sys.path.insert(0, sage_root) + +from sqlor.dbpools import DBPools +from appPublic.jsonConfig import getConfig +from appPublic.dictObject import DictObject +from appPublic.uniqueID import getID + + +paths = [ + ("/uapi", "logined"), + ("/uapi/jump_in.dspy", "logined"), + ("/uapi/minimax_callback.dspy", "any"), + ("/uapi/uptask_callback.dspy", "any"), + ("/uapi/viducallback", "any"), +] + + +async def main(): + config = getConfig('.') + DBPools(config.databases) + dbname = 'sage' + async with DBPools().sqlorContext(dbname) as sor: + cnt = 0 + for path, role in paths: + r = await sor.sqlExe( + 'select * from permission where permcode = ${permcode}$', + {'permcode': path} + ) + if len(r) == 0: + await sor.sqlExe( + '''insert into permission (id, permcode, permname, permtype) + values (${id}$, ${permcode}$, ${permname}$, ${permtype}$)''', + { + 'id': getID(), + 'permcode': path, + 'permname': path, + 'permtype': role, + } + ) + cnt += 1 + print(f'{cnt} path(s) inserted for uapi') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/scripts/migrate_uapi_upappid.py b/scripts/migrate_uapi_upappid.py new file mode 100644 index 0000000..039588b --- /dev/null +++ b/scripts/migrate_uapi_upappid.py @@ -0,0 +1,133 @@ +""" +Migration: uapi table apisetid -> upappid + +Removes uapiset intermediate layer. Each upapp now owns its own uapi records. +For shared apisetid: copies uapi records so each upapp has its own copy. + +Usage (in Sage virtual env): + ./py3/bin/python3 ~/repos/uapi/scripts/migrate_uapi_upappid.py --output /tmp/migrate_uapi.sql + +Review the output SQL, then execute on your database. +""" +from appPublic.jsonConfig import getConfig +import asyncio, json, sys, argparse +from sqlor.dbpools import DBPools +from appPublic.uniqueID import getID + +config = getConfig('.') +db = DBPools(config.databases) +dbname = list(config.databases.keys())[0] + + +async def generate_migration_sql(): + """Generate SQL to migrate uapi.apisetid -> uapi.upappid.""" + lines = [ + "-- Migration: uapi table apisetid -> upappid", + "-- Removes uapiset intermediate layer.", + "", + "-- Step 1: Add upappid column to uapi", + "ALTER TABLE uapi ADD COLUMN upappid VARCHAR(32) DEFAULT NULL COMMENT '上位系统ID' AFTER id;", + "" + ] + + # Load data + try: + async with db.sqlorContext(dbname) as sor: + # Get all upapps + upapps = await sor.sqlExe('select id, name, apisetid from upapp', {}) + # Get all uapis (removed non-existent auth_apiname field) + uapis = await sor.sqlExe('select id, apisetid, name, httpmethod, path, headers, ioid, response, params, data, chunk_match from uapi', {}) + except Exception as e: + print(f"Error querying database: {e}", file=sys.stderr) + sys.exit(1) + + # Build mapping: apisetid -> [upapp1, upapp2, ...] + apiset_to_upapps = {} + for u in upapps: + aid = u.get('apisetid') + if aid: + apiset_to_upapps.setdefault(aid, []).append(u) + + # Build mapping: apisetid -> [uapi_records] + apiset_to_uapis = {} + for a in uapis: + aid = a.get('apisetid') + if aid: + apiset_to_uapis.setdefault(aid, []).append(a) + + inserts = [] + for apisetid, upapp_list in apiset_to_upapps.items(): + uapi_records = apiset_to_uapis.get(apisetid, []) + + if len(upapp_list) == 1: + # Single upapp owns this apisetid -> just update + upapp = upapp_list[0] + for uapi in uapi_records: + inserts.append( + f"UPDATE uapi SET upappid = '{upapp['id']}' WHERE id = '{uapi['id']}';" + ) + else: + # Multiple upapps share this apisetid -> pick first as owner, copy for rest + owner = upapp_list[0] + + # Update existing records to point to owner + for uapi in uapi_records: + inserts.append( + f"UPDATE uapi SET upappid = '{owner['id']}' WHERE id = '{uapi['id']}';" + ) + + # Copy for other upapps + for other_upapp in upapp_list[1:]: + for uapi in uapi_records: + new_id = getID() + old_id = uapi['id'] + fields = ['id', 'upappid', 'name', 'httpmethod', 'path', 'headers', 'ioid', 'response', 'params', 'data', 'chunk_match'] + vals = [f"'{new_id}'", f"'{other_upapp['id']}'"] + for f in fields[2:]: + v = uapi.get(f) + if v is None: + vals.append('NULL') + elif isinstance(v, str): + escaped = v.replace("'", "\\'") + vals.append(f"'{escaped}'") + else: + vals.append(str(v)) + col_str = ', '.join(fields) + val_str = ', '.join(vals) + inserts.append(f"INSERT INTO uapi ({col_str}) VALUES ({val_str});") + + if inserts: + lines.append("-- Step 2: Migrate data (updates + copies for shared apisetid)") + lines.extend(inserts) + lines.append("") + + lines.extend([ + "-- Step 3: Drop apisetid column (verify upappid has no NULLs first)", + "-- UPDATE uapi SET upappid = (SELECT ownerid FROM organization LIMIT 1) WHERE upappid IS NULL;", + "-- ALTER TABLE uapi DROP COLUMN apisetid;", + "", + "-- Step 4: Add index", + "CREATE INDEX idx_uapi_upappid ON uapi (upappid);", + "" + ]) + + return '\n'.join(lines) + + +async def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--output', '-o', default='-') + args = parser.parse_args() + + sql = await generate_migration_sql() + + if args.output == '-': + print(sql) + else: + with open(args.output, 'w', encoding='utf-8') as f: + f.write(sql) + print(f"Generated migration SQL -> {args.output}") + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/uapi/apidata.py b/uapi/apidata.py index 91a5469..f4e43a5 100644 --- a/uapi/apidata.py +++ b/uapi/apidata.py @@ -8,10 +8,23 @@ from appPublic.streamhttpclient import StreamHttpClient, liner from appPublic.dictObject import DictObject from appPublic.log import debug, exception, error from appPublic.aes import aes_encode_b64 +from appPublic.jsonConfig import getConfig from ahserver.globalEnv import password_decode from ahserver.serverenv import get_serverenv, ServerEnv from random import randint + +def _cache_enabled(): + """Check if cache is enabled for uapi module in config.json""" + try: + config = getConfig() + module_cache = config.module_cache + if module_cache is None: + return True + return getattr(module_cache, 'uapi', True) + except Exception: + return True + async def get_deerer(upappid, callerid): db = DBPools() dbname = get_dbname() @@ -25,10 +38,9 @@ async def get_deerer(upappid, callerid): async def sor_get_uapi(sor, upappid, apiname): sql = """select a.*, -c.auth_apiname -from uapi a, upapp b, uapiset c -where a.apisetid = b.apisetid - and b.apisetid = c.id +b.auth_apiname +from uapi a, upapp b +where a.upappid = b.id and a.name = ${apiname}$ and b.id = ${upappid}$""" recs = await sor.sqlExe(sql, {'upappid': upappid, 'apiname': apiname}) @@ -52,7 +64,15 @@ class UAPIData: self.apidata = {} self.apikeys = {} self.org_users = {} - + + def on_hot_reload(self, data=None): + """Event handler for hot_reload event. Clears all caches.""" + from appPublic.log import debug + debug(f'[uapi] on_hot_reload called, clearing caches (data={data})') + self.apidata.clear() + self.apikeys.clear() + self.org_users.clear() + async def get_userapikey(self, appid, callerid): users = await self.get_apiusers(appid) for u in users: @@ -69,10 +89,11 @@ class UAPIData: return None async def get_apiusers(self, appid, orgid=None): - key = appid - d = self.org_users.get(key) - if d: - return d + key = f'{appid}.{orgid}' if orgid else str(appid) + if _cache_enabled(): + d = self.org_users.get(key) + if d: + return d env = ServerEnv() async with get_sor_context(env, 'uapi') as sor: sql = """select @@ -105,7 +126,8 @@ where b.orgid = c.ownerid e = Exception(f'{appid=} {orgid=} get none user') exception(f'{e}') raise e - self.apidata[key] = d + if _cache_enabled(): + self.org_users[key] = d return d e = Exception(f'{appid=} {orgid=} get none user') exception(f'{e}') @@ -123,6 +145,10 @@ where b.orgid = c.ownerid async def get_api_from_db(self, appid, apiname): key = f'{appid}.{apiname}' + if _cache_enabled(): + api = self.apidata.get(key) + if api: + return api env = ServerEnv() async with get_sor_context(env, 'uapi') as sor: d = await sor_get_uapi(sor, appid, apiname) @@ -130,7 +156,8 @@ where b.orgid = c.ownerid e = Exception(f'{appid=}, {apiname=} get none api') exception(f'{e}') raise e - self.apidata[key] = d + if _cache_enabled(): + self.apidata[key] = d return d e = Exception(f'{appid=}, {apiname=} get none api') exception(f'{e}') diff --git a/uapi/appapi.py b/uapi/appapi.py index 6d205cf..57d671d 100644 --- a/uapi/appapi.py +++ b/uapi/appapi.py @@ -20,7 +20,7 @@ async def get_callerid(orgid): async def sor_get_uapi_by_appname_apiname(sor, appname, apiname): sql = """select a.* from uapi a, upapp b -where a.apisetid = b.apisetid +where a.upappid = b.id and b.name = ${appname}$ and a.name = ${apiname}$""" recs = await sor.sqlExe(sql, {'apiname': apiname, 'appname': appname}) @@ -90,10 +90,9 @@ async def get_userapikey(sor, upappid, callerid): async def sor_get_uapi(sor, upappid, apiname): sql = """select a.*, -c.auth_apiname -from uapi a, upapp b, uapiset c -where a.apisetid = b.apisetid - and b.apisetid = c.id +b.auth_apiname +from uapi a, upapp b +where a.upappid = b.id and a.name = ${apiname}$ and b.id = ${upappid}$""" recs = await sor.sqlExe(sql, {'upappid': upappid, 'apiname': apiname}) diff --git a/uapi/init.py b/uapi/init.py index 04978ff..8e49e78 100644 --- a/uapi/init.py +++ b/uapi/init.py @@ -32,6 +32,9 @@ def load_uapi(): g.get_my_uptasks = get_my_uptasks g.uptask_feedback = uptask_feedback g.uptask_started = uptask_started + # Bind hot_reload event — instance method, WeakMethod safe (stored on g) + if hasattr(g, 'event_dispatcher'): + g.event_dispatcher.bind('hot_reload', g.uapi_data.on_hot_reload) # USAGE in dspy diff --git a/wwwroot/uptask_callback.dspy b/wwwroot/uptask_callback.dspy index 219a9ad..e06b2e6 100644 --- a/wwwroot/uptask_callback.dspy +++ b/wwwroot/uptask_callback.dspy @@ -1,4 +1,4 @@ -debug(f'{params_kw=}') +debug_params('params_kw', params_kw) if parmas_kw.task_id is None: e = Exception(f'need a task_id') raise e diff --git a/wwwroot/viducallback/index.dspy b/wwwroot/viducallback/index.dspy index 62b85c3..6ab62cd 100644 --- a/wwwroot/viducallback/index.dspy +++ b/wwwroot/viducallback/index.dspy @@ -1,4 +1,4 @@ -debug(f'{params_kw=} +debug_params('params_kw', params_kw) taskid = params_kw.id async with get_sor_context(request._run_ns, 'uapi') as sor: llmusage = await get_llmlage_by_taskid(taskid)