refactor(schema): rag 模块全部数据表加 rag_ 前缀（模块跨应用复用防冲突）
- models/ 重建为正确表定义格式(summary+fields+indexes)，覆盖全部 11 个表 修正原 models/ 误放 CRUD 格式副本(缺 summary)导致 json2ddl 100% 失败被静默忽略 - json/ 6 个 CRUD 定义 tblname 加前缀 + 文件改名 - 129 处 SQL/sor 表名加 rag_ 前缀（URL 路径 knowledge_bases_list 保持不变） - 新增 init/migrate_rag_prefix.sql 幂等 RENAME TABLE 迁移
This commit is contained in:
ymq
parent
fb2cf3e703
commit
789960a1cf
54
init/migrate_rag_prefix.sql
Normal file
54
init/migrate_rag_prefix.sql
Normal file
@ -0,0 +1,54 @@
|
||||
-- rag 模块表名加 rag_ 前缀（模块跨应用复用，避免与业务表冲突）
|
||||
-- 幂等：仅当旧表存在且新表不存在时才 RENAME。
|
||||
-- 用法: mysql -h <host> -u<user> -p<pwd> <db> < migrate_rag_prefix.sql
|
||||
|
||||
DELIMITER $$
|
||||
DROP PROCEDURE IF EXISTS rag_rename_tables$$
|
||||
CREATE PROCEDURE rag_rename_tables()
|
||||
BEGIN
|
||||
DECLARE done INT DEFAULT 0;
|
||||
DECLARE oldname VARCHAR(64);
|
||||
DECLARE cur CURSOR FOR
|
||||
SELECT t FROM (
|
||||
SELECT 'knowledge_bases' AS t UNION ALL
|
||||
SELECT 'documents' UNION ALL
|
||||
SELECT 'document_chunks' UNION ALL
|
||||
SELECT 'tags' UNION ALL
|
||||
SELECT 'media_tags' UNION ALL
|
||||
SELECT 'entities' UNION ALL
|
||||
SELECT 'entity_relations' UNION ALL
|
||||
SELECT 'engine_configs' UNION ALL
|
||||
SELECT 'subscriptions' UNION ALL
|
||||
SELECT 'org_storage_limits' UNION ALL
|
||||
SELECT 'usage_logs'
|
||||
) x;
|
||||
DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = 1;
|
||||
|
||||
OPEN cur;
|
||||
read_loop: LOOP
|
||||
FETCH cur INTO oldname;
|
||||
IF done THEN LEAVE read_loop; END IF;
|
||||
|
||||
SET @has_old = (SELECT COUNT(*) FROM information_schema.TABLES
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = oldname);
|
||||
SET @has_new = (SELECT COUNT(*) FROM information_schema.TABLES
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = CONCAT('rag_', oldname));
|
||||
|
||||
IF @has_old = 1 AND @has_new = 0 THEN
|
||||
SET @sql = CONCAT('RENAME TABLE `', oldname, '` TO `rag_', oldname, '`');
|
||||
PREPARE st FROM @sql;
|
||||
EXECUTE st;
|
||||
DEALLOCATE PREPARE st;
|
||||
SELECT CONCAT('RENAMED: ', oldname, ' -> rag_', oldname) AS result;
|
||||
ELSEIF @has_new = 1 THEN
|
||||
SELECT CONCAT('SKIP (already migrated): rag_', oldname) AS result;
|
||||
ELSE
|
||||
SELECT CONCAT('SKIP (no such table): ', oldname) AS result;
|
||||
END IF;
|
||||
END LOOP;
|
||||
CLOSE cur;
|
||||
END$$
|
||||
DELIMITER ;
|
||||
|
||||
CALL rag_rename_tables();
|
||||
DROP PROCEDURE rag_rename_tables;
|
||||
@ -1,25 +0,0 @@
|
||||
{
|
||||
"tblname": "documents",
|
||||
"params": {
|
||||
"title": "文档管理",
|
||||
"browserfields": [
|
||||
{"field": "file_name", "title": "文件名", "width": "25%"},
|
||||
{"field": "file_type", "title": "类型", "width": "10%"},
|
||||
{"field": "file_size", "title": "大小", "width": "10%"},
|
||||
{"field": "chunk_count", "title": "分片数", "width": "8%"},
|
||||
{"field": "status", "title": "状态", "width": "10%"},
|
||||
{"field": "kb_id", "title": "知识库", "width": "17%"},
|
||||
{"field": "created_at", "title": "上传时间", "width": "20%"}
|
||||
],
|
||||
"editfields": [
|
||||
{"field": "file_name", "uitype": "Text", "required": true},
|
||||
{"field": "kb_id", "uitype": "Text", "required": true},
|
||||
{"field": "file_type", "uitype": "Text"}
|
||||
],
|
||||
"searchfields": ["file_name"],
|
||||
"sort": "created_at desc",
|
||||
"data_filter": {
|
||||
"org_id": "{{userorgid}}"
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,26 +0,0 @@
|
||||
{
|
||||
"tblname": "engine_configs",
|
||||
"params": {
|
||||
"title": "引擎配置",
|
||||
"browserfields": [
|
||||
{"field": "engine_type", "title": "引擎类型", "width": "15%"},
|
||||
{"field": "engine_name", "title": "名称", "width": "15%"},
|
||||
{"field": "endpoint_url", "title": "服务地址", "width": "25%"},
|
||||
{"field": "model_name", "title": "模型", "width": "15%"},
|
||||
{"field": "is_default", "title": "默认", "width": "8%"},
|
||||
{"field": "status", "title": "状态", "width": "10%"},
|
||||
{"field": "org_id", "title": "机构", "width": "12%"}
|
||||
],
|
||||
"editfields": [
|
||||
{"field": "engine_type", "uitype": "Text", "required": true},
|
||||
{"field": "engine_name", "uitype": "Text", "required": true},
|
||||
{"field": "endpoint_url", "uitype": "Text"},
|
||||
{"field": "api_key", "uitype": "Text"},
|
||||
{"field": "model_name", "uitype": "Text"},
|
||||
{"field": "is_default", "uitype": "Text"},
|
||||
{"field": "config_json", "uitype": "Text"}
|
||||
],
|
||||
"searchfields": ["engine_type", "engine_name", "endpoint_url"],
|
||||
"sort": "engine_type, priority desc"
|
||||
}
|
||||
}
|
||||
@ -1,26 +0,0 @@
|
||||
{
|
||||
"tblname": "knowledge_bases",
|
||||
"params": {
|
||||
"title": "知识库管理",
|
||||
"browserfields": [
|
||||
{"field": "name", "title": "名称", "width": "20%"},
|
||||
{"field": "description", "title": "描述", "width": "30%"},
|
||||
{"field": "doc_count", "title": "文档数", "width": "10%"},
|
||||
{"field": "total_size", "title": "大小", "width": "10%"},
|
||||
{"field": "embedding_engine", "title": "向量引擎", "width": "15%"},
|
||||
{"field": "created_at", "title": "创建时间", "width": "15%"}
|
||||
],
|
||||
"editfields": [
|
||||
{"field": "name", "uitype": "Text", "required": true},
|
||||
{"field": "description", "uitype": "Text"},
|
||||
{"field": "embedding_engine", "uitype": "Text", "default": "clip-vith14"},
|
||||
{"field": "vdb_collection", "uitype": "Text"},
|
||||
{"field": "graph_name", "uitype": "Text"}
|
||||
],
|
||||
"searchfields": ["name", "description"],
|
||||
"sort": "created_at desc",
|
||||
"data_filter": {
|
||||
"org_id": "{{userorgid}}"
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,17 +0,0 @@
|
||||
{
|
||||
"tblname": "org_storage_limits",
|
||||
"params": {
|
||||
"title": "机构存储限额",
|
||||
"browserfields": [
|
||||
{"field": "org_id", "title": "机构ID", "width": "35%"},
|
||||
{"field": "limit_bytes", "title": "限额(字节)", "width": "35%"},
|
||||
{"field": "updated_at", "title": "更新时间", "width": "30%"}
|
||||
],
|
||||
"editfields": [
|
||||
{"field": "org_id", "uitype": "Text", "required": true},
|
||||
{"field": "limit_bytes", "uitype": "Text", "required": true, "default": "104857600"}
|
||||
],
|
||||
"searchfields": ["org_id"],
|
||||
"sort": "updated_at desc"
|
||||
}
|
||||
}
|
||||
66
json/rag_documents.json
Normal file
66
json/rag_documents.json
Normal file
@ -0,0 +1,66 @@
|
||||
{
|
||||
"tblname": "rag_documents",
|
||||
"params": {
|
||||
"title": "文档管理",
|
||||
"browserfields": [
|
||||
{
|
||||
"field": "file_name",
|
||||
"title": "文件名",
|
||||
"width": "25%"
|
||||
},
|
||||
{
|
||||
"field": "file_type",
|
||||
"title": "类型",
|
||||
"width": "10%"
|
||||
},
|
||||
{
|
||||
"field": "file_size",
|
||||
"title": "大小",
|
||||
"width": "10%"
|
||||
},
|
||||
{
|
||||
"field": "chunk_count",
|
||||
"title": "分片数",
|
||||
"width": "8%"
|
||||
},
|
||||
{
|
||||
"field": "status",
|
||||
"title": "状态",
|
||||
"width": "10%"
|
||||
},
|
||||
{
|
||||
"field": "kb_id",
|
||||
"title": "知识库",
|
||||
"width": "17%"
|
||||
},
|
||||
{
|
||||
"field": "created_at",
|
||||
"title": "上传时间",
|
||||
"width": "20%"
|
||||
}
|
||||
],
|
||||
"editfields": [
|
||||
{
|
||||
"field": "file_name",
|
||||
"uitype": "Text",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"field": "kb_id",
|
||||
"uitype": "Text",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"field": "file_type",
|
||||
"uitype": "Text"
|
||||
}
|
||||
],
|
||||
"searchfields": [
|
||||
"file_name"
|
||||
],
|
||||
"sort": "created_at desc",
|
||||
"data_filter": {
|
||||
"org_id": "{{userorgid}}"
|
||||
}
|
||||
}
|
||||
}
|
||||
81
json/rag_engine_configs.json
Normal file
81
json/rag_engine_configs.json
Normal file
@ -0,0 +1,81 @@
|
||||
{
|
||||
"tblname": "rag_engine_configs",
|
||||
"params": {
|
||||
"title": "引擎配置",
|
||||
"browserfields": [
|
||||
{
|
||||
"field": "engine_type",
|
||||
"title": "引擎类型",
|
||||
"width": "15%"
|
||||
},
|
||||
{
|
||||
"field": "engine_name",
|
||||
"title": "名称",
|
||||
"width": "15%"
|
||||
},
|
||||
{
|
||||
"field": "endpoint_url",
|
||||
"title": "服务地址",
|
||||
"width": "25%"
|
||||
},
|
||||
{
|
||||
"field": "model_name",
|
||||
"title": "模型",
|
||||
"width": "15%"
|
||||
},
|
||||
{
|
||||
"field": "is_default",
|
||||
"title": "默认",
|
||||
"width": "8%"
|
||||
},
|
||||
{
|
||||
"field": "status",
|
||||
"title": "状态",
|
||||
"width": "10%"
|
||||
},
|
||||
{
|
||||
"field": "org_id",
|
||||
"title": "机构",
|
||||
"width": "12%"
|
||||
}
|
||||
],
|
||||
"editfields": [
|
||||
{
|
||||
"field": "engine_type",
|
||||
"uitype": "Text",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"field": "engine_name",
|
||||
"uitype": "Text",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"field": "endpoint_url",
|
||||
"uitype": "Text"
|
||||
},
|
||||
{
|
||||
"field": "api_key",
|
||||
"uitype": "Text"
|
||||
},
|
||||
{
|
||||
"field": "model_name",
|
||||
"uitype": "Text"
|
||||
},
|
||||
{
|
||||
"field": "is_default",
|
||||
"uitype": "Text"
|
||||
},
|
||||
{
|
||||
"field": "config_json",
|
||||
"uitype": "Text"
|
||||
}
|
||||
],
|
||||
"searchfields": [
|
||||
"engine_type",
|
||||
"engine_name",
|
||||
"endpoint_url"
|
||||
],
|
||||
"sort": "engine_type, priority desc"
|
||||
}
|
||||
}
|
||||
70
json/rag_knowledge_bases.json
Normal file
70
json/rag_knowledge_bases.json
Normal file
@ -0,0 +1,70 @@
|
||||
{
|
||||
"tblname": "rag_knowledge_bases",
|
||||
"params": {
|
||||
"title": "知识库管理",
|
||||
"browserfields": [
|
||||
{
|
||||
"field": "name",
|
||||
"title": "名称",
|
||||
"width": "20%"
|
||||
},
|
||||
{
|
||||
"field": "description",
|
||||
"title": "描述",
|
||||
"width": "30%"
|
||||
},
|
||||
{
|
||||
"field": "doc_count",
|
||||
"title": "文档数",
|
||||
"width": "10%"
|
||||
},
|
||||
{
|
||||
"field": "total_size",
|
||||
"title": "大小",
|
||||
"width": "10%"
|
||||
},
|
||||
{
|
||||
"field": "embedding_engine",
|
||||
"title": "向量引擎",
|
||||
"width": "15%"
|
||||
},
|
||||
{
|
||||
"field": "created_at",
|
||||
"title": "创建时间",
|
||||
"width": "15%"
|
||||
}
|
||||
],
|
||||
"editfields": [
|
||||
{
|
||||
"field": "name",
|
||||
"uitype": "Text",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"field": "description",
|
||||
"uitype": "Text"
|
||||
},
|
||||
{
|
||||
"field": "embedding_engine",
|
||||
"uitype": "Text",
|
||||
"default": "clip-vith14"
|
||||
},
|
||||
{
|
||||
"field": "vdb_collection",
|
||||
"uitype": "Text"
|
||||
},
|
||||
{
|
||||
"field": "graph_name",
|
||||
"uitype": "Text"
|
||||
}
|
||||
],
|
||||
"searchfields": [
|
||||
"name",
|
||||
"description"
|
||||
],
|
||||
"sort": "created_at desc",
|
||||
"data_filter": {
|
||||
"org_id": "{{userorgid}}"
|
||||
}
|
||||
}
|
||||
}
|
||||
40
json/rag_org_storage_limits.json
Normal file
40
json/rag_org_storage_limits.json
Normal file
@ -0,0 +1,40 @@
|
||||
{
|
||||
"tblname": "rag_org_storage_limits",
|
||||
"params": {
|
||||
"title": "机构存储限额",
|
||||
"browserfields": [
|
||||
{
|
||||
"field": "org_id",
|
||||
"title": "机构ID",
|
||||
"width": "35%"
|
||||
},
|
||||
{
|
||||
"field": "limit_bytes",
|
||||
"title": "限额(字节)",
|
||||
"width": "35%"
|
||||
},
|
||||
{
|
||||
"field": "updated_at",
|
||||
"title": "更新时间",
|
||||
"width": "30%"
|
||||
}
|
||||
],
|
||||
"editfields": [
|
||||
{
|
||||
"field": "org_id",
|
||||
"uitype": "Text",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"field": "limit_bytes",
|
||||
"uitype": "Text",
|
||||
"required": true,
|
||||
"default": "104857600"
|
||||
}
|
||||
],
|
||||
"searchfields": [
|
||||
"org_id"
|
||||
],
|
||||
"sort": "updated_at desc"
|
||||
}
|
||||
}
|
||||
90
json/rag_subscriptions.json
Normal file
90
json/rag_subscriptions.json
Normal file
@ -0,0 +1,90 @@
|
||||
{
|
||||
"tblname": "rag_subscriptions",
|
||||
"params": {
|
||||
"title": "订阅管理",
|
||||
"browserfields": [
|
||||
{
|
||||
"field": "org_id",
|
||||
"title": "机构",
|
||||
"width": "15%"
|
||||
},
|
||||
{
|
||||
"field": "plan_name",
|
||||
"title": "套餐",
|
||||
"width": "15%"
|
||||
},
|
||||
{
|
||||
"field": "disk_quota_bytes",
|
||||
"title": "磁盘配额",
|
||||
"width": "12%"
|
||||
},
|
||||
{
|
||||
"field": "disk_used_bytes",
|
||||
"title": "已用磁盘",
|
||||
"width": "12%"
|
||||
},
|
||||
{
|
||||
"field": "doc_used",
|
||||
"title": "文档数",
|
||||
"width": "10%"
|
||||
},
|
||||
{
|
||||
"field": "start_date",
|
||||
"title": "开始",
|
||||
"width": "12%"
|
||||
},
|
||||
{
|
||||
"field": "end_date",
|
||||
"title": "结束",
|
||||
"width": "12%"
|
||||
},
|
||||
{
|
||||
"field": "status",
|
||||
"title": "状态",
|
||||
"width": "12%"
|
||||
}
|
||||
],
|
||||
"editfields": [
|
||||
{
|
||||
"field": "org_id",
|
||||
"uitype": "Text",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"field": "plan_name",
|
||||
"uitype": "Text",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"field": "disk_quota_bytes",
|
||||
"uitype": "Text",
|
||||
"default": "1073741824"
|
||||
},
|
||||
{
|
||||
"field": "doc_quota",
|
||||
"uitype": "Text"
|
||||
},
|
||||
{
|
||||
"field": "kb_quota",
|
||||
"uitype": "Text"
|
||||
},
|
||||
{
|
||||
"field": "api_call_quota",
|
||||
"uitype": "Text"
|
||||
},
|
||||
{
|
||||
"field": "start_date",
|
||||
"uitype": "Text"
|
||||
},
|
||||
{
|
||||
"field": "end_date",
|
||||
"uitype": "Text"
|
||||
}
|
||||
],
|
||||
"searchfields": [
|
||||
"org_id",
|
||||
"plan_name"
|
||||
],
|
||||
"sort": "created_at desc"
|
||||
}
|
||||
}
|
||||
55
json/rag_tags.json
Normal file
55
json/rag_tags.json
Normal file
@ -0,0 +1,55 @@
|
||||
{
|
||||
"tblname": "rag_tags",
|
||||
"params": {
|
||||
"title": "标签管理",
|
||||
"browserfields": [
|
||||
{
|
||||
"field": "name",
|
||||
"title": "标签名",
|
||||
"width": "25%"
|
||||
},
|
||||
{
|
||||
"field": "color",
|
||||
"title": "颜色",
|
||||
"width": "15%"
|
||||
},
|
||||
{
|
||||
"field": "kb_id",
|
||||
"title": "知识库",
|
||||
"width": "30%"
|
||||
},
|
||||
{
|
||||
"field": "created_at",
|
||||
"title": "创建时间",
|
||||
"width": "30%"
|
||||
}
|
||||
],
|
||||
"editfields": [
|
||||
{
|
||||
"field": "name",
|
||||
"uitype": "Text",
|
||||
"required": true,
|
||||
"label": "标签名"
|
||||
},
|
||||
{
|
||||
"field": "kb_id",
|
||||
"uitype": "Text",
|
||||
"required": true,
|
||||
"label": "知识库ID"
|
||||
},
|
||||
{
|
||||
"field": "color",
|
||||
"uitype": "Text",
|
||||
"default": "#3b82f6",
|
||||
"label": "颜色(#hex)"
|
||||
}
|
||||
],
|
||||
"searchfields": [
|
||||
"name"
|
||||
],
|
||||
"sort": "created_at desc",
|
||||
"data_filter": {
|
||||
"org_id": "{{userorgid}}"
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,28 +0,0 @@
|
||||
{
|
||||
"tblname": "subscriptions",
|
||||
"params": {
|
||||
"title": "订阅管理",
|
||||
"browserfields": [
|
||||
{"field": "org_id", "title": "机构", "width": "15%"},
|
||||
{"field": "plan_name", "title": "套餐", "width": "15%"},
|
||||
{"field": "disk_quota_bytes", "title": "磁盘配额", "width": "12%"},
|
||||
{"field": "disk_used_bytes", "title": "已用磁盘", "width": "12%"},
|
||||
{"field": "doc_used", "title": "文档数", "width": "10%"},
|
||||
{"field": "start_date", "title": "开始", "width": "12%"},
|
||||
{"field": "end_date", "title": "结束", "width": "12%"},
|
||||
{"field": "status", "title": "状态", "width": "12%"}
|
||||
],
|
||||
"editfields": [
|
||||
{"field": "org_id", "uitype": "Text", "required": true},
|
||||
{"field": "plan_name", "uitype": "Text", "required": true},
|
||||
{"field": "disk_quota_bytes", "uitype": "Text", "default": "1073741824"},
|
||||
{"field": "doc_quota", "uitype": "Text"},
|
||||
{"field": "kb_quota", "uitype": "Text"},
|
||||
{"field": "api_call_quota", "uitype": "Text"},
|
||||
{"field": "start_date", "uitype": "Text"},
|
||||
{"field": "end_date", "uitype": "Text"}
|
||||
],
|
||||
"searchfields": ["org_id", "plan_name"],
|
||||
"sort": "created_at desc"
|
||||
}
|
||||
}
|
||||
@ -1,22 +0,0 @@
|
||||
{
|
||||
"tblname": "tags",
|
||||
"params": {
|
||||
"title": "标签管理",
|
||||
"browserfields": [
|
||||
{"field": "name", "title": "标签名", "width": "25%"},
|
||||
{"field": "color", "title": "颜色", "width": "15%"},
|
||||
{"field": "kb_id", "title": "知识库", "width": "30%"},
|
||||
{"field": "created_at", "title": "创建时间", "width": "30%"}
|
||||
],
|
||||
"editfields": [
|
||||
{"field": "name", "uitype": "Text", "required": true, "label": "标签名"},
|
||||
{"field": "kb_id", "uitype": "Text", "required": true, "label": "知识库ID"},
|
||||
{"field": "color", "uitype": "Text", "default": "#3b82f6", "label": "颜色(#hex)"}
|
||||
],
|
||||
"searchfields": ["name"],
|
||||
"sort": "created_at desc",
|
||||
"data_filter": {
|
||||
"org_id": "{{userorgid}}"
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,25 +0,0 @@
|
||||
{
|
||||
"tblname": "documents",
|
||||
"params": {
|
||||
"title": "文档管理",
|
||||
"browserfields": [
|
||||
{"field": "file_name", "title": "文件名", "width": "25%"},
|
||||
{"field": "file_type", "title": "类型", "width": "10%"},
|
||||
{"field": "file_size", "title": "大小", "width": "10%"},
|
||||
{"field": "chunk_count", "title": "分片数", "width": "8%"},
|
||||
{"field": "status", "title": "状态", "width": "10%"},
|
||||
{"field": "kb_id", "title": "知识库", "width": "17%"},
|
||||
{"field": "created_at", "title": "上传时间", "width": "20%"}
|
||||
],
|
||||
"editfields": [
|
||||
{"field": "file_name", "uitype": "Text", "required": true},
|
||||
{"field": "kb_id", "uitype": "Text", "required": true},
|
||||
{"field": "file_type", "uitype": "Text"}
|
||||
],
|
||||
"searchfields": ["file_name"],
|
||||
"sort": "created_at desc",
|
||||
"data_filter": {
|
||||
"org_id": "{{userorgid}}"
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,26 +0,0 @@
|
||||
{
|
||||
"tblname": "engine_configs",
|
||||
"params": {
|
||||
"title": "引擎配置",
|
||||
"browserfields": [
|
||||
{"field": "engine_type", "title": "引擎类型", "width": "15%"},
|
||||
{"field": "engine_name", "title": "名称", "width": "15%"},
|
||||
{"field": "endpoint_url", "title": "服务地址", "width": "25%"},
|
||||
{"field": "model_name", "title": "模型", "width": "15%"},
|
||||
{"field": "is_default", "title": "默认", "width": "8%"},
|
||||
{"field": "status", "title": "状态", "width": "10%"},
|
||||
{"field": "org_id", "title": "机构", "width": "12%"}
|
||||
],
|
||||
"editfields": [
|
||||
{"field": "engine_type", "uitype": "Text", "required": true},
|
||||
{"field": "engine_name", "uitype": "Text", "required": true},
|
||||
{"field": "endpoint_url", "uitype": "Text"},
|
||||
{"field": "api_key", "uitype": "Text"},
|
||||
{"field": "model_name", "uitype": "Text"},
|
||||
{"field": "is_default", "uitype": "Text"},
|
||||
{"field": "config_json", "uitype": "Text"}
|
||||
],
|
||||
"searchfields": ["engine_type", "engine_name", "endpoint_url"],
|
||||
"sort": "engine_type, priority desc"
|
||||
}
|
||||
}
|
||||
@ -1,26 +0,0 @@
|
||||
{
|
||||
"tblname": "knowledge_bases",
|
||||
"params": {
|
||||
"title": "知识库管理",
|
||||
"browserfields": [
|
||||
{"field": "name", "title": "名称", "width": "20%"},
|
||||
{"field": "description", "title": "描述", "width": "30%"},
|
||||
{"field": "doc_count", "title": "文档数", "width": "10%"},
|
||||
{"field": "total_size", "title": "大小", "width": "10%"},
|
||||
{"field": "embedding_engine", "title": "向量引擎", "width": "15%"},
|
||||
{"field": "created_at", "title": "创建时间", "width": "15%"}
|
||||
],
|
||||
"editfields": [
|
||||
{"field": "name", "uitype": "Text", "required": true},
|
||||
{"field": "description", "uitype": "Text"},
|
||||
{"field": "embedding_engine", "uitype": "Text", "default": "clip-vith14"},
|
||||
{"field": "vdb_collection", "uitype": "Text"},
|
||||
{"field": "graph_name", "uitype": "Text"}
|
||||
],
|
||||
"searchfields": ["name", "description"],
|
||||
"sort": "created_at desc",
|
||||
"data_filter": {
|
||||
"org_id": "{{userorgid}}"
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,17 +0,0 @@
|
||||
{
|
||||
"tblname": "org_storage_limits",
|
||||
"params": {
|
||||
"title": "机构存储限额",
|
||||
"browserfields": [
|
||||
{"field": "org_id", "title": "机构ID", "width": "35%"},
|
||||
{"field": "limit_bytes", "title": "限额(字节)", "width": "35%"},
|
||||
{"field": "updated_at", "title": "更新时间", "width": "30%"}
|
||||
],
|
||||
"editfields": [
|
||||
{"field": "org_id", "uitype": "Text", "required": true},
|
||||
{"field": "limit_bytes", "uitype": "Text", "required": true, "default": "104857600"}
|
||||
],
|
||||
"searchfields": ["org_id"],
|
||||
"sort": "updated_at desc"
|
||||
}
|
||||
}
|
||||
129
models/rag_document_chunks.json
Normal file
129
models/rag_document_chunks.json
Normal file
@ -0,0 +1,129 @@
|
||||
{
|
||||
"summary": [
|
||||
{
|
||||
"name": "rag_document_chunks",
|
||||
"title": "文档分片",
|
||||
"primary": [
|
||||
"id"
|
||||
]
|
||||
}
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"name": "id",
|
||||
"title": "ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "no"
|
||||
},
|
||||
{
|
||||
"name": "doc_id",
|
||||
"title": "文档ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "kb_id",
|
||||
"title": "知识库ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "chunk_index",
|
||||
"title": "分片序号",
|
||||
"type": "int",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "chunk_type",
|
||||
"title": "分片类型",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "content",
|
||||
"title": "文本内容",
|
||||
"type": "text",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "description",
|
||||
"title": "描述",
|
||||
"type": "text",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "vector_id",
|
||||
"title": "向量库ID",
|
||||
"type": "str",
|
||||
"length": 128,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "start_offset",
|
||||
"title": "起始偏移",
|
||||
"type": "int",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "end_offset",
|
||||
"title": "结束偏移",
|
||||
"type": "int",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "time_start",
|
||||
"title": "时间起始(秒)",
|
||||
"type": "double",
|
||||
"length": 10,
|
||||
"dec": 3,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "time_end",
|
||||
"title": "时间结束(秒)",
|
||||
"type": "double",
|
||||
"length": 10,
|
||||
"dec": 3,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "metadata",
|
||||
"title": "元数据JSON",
|
||||
"type": "text",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "created_at",
|
||||
"title": "创建时间",
|
||||
"type": "datetime",
|
||||
"nullable": "yes"
|
||||
}
|
||||
],
|
||||
"indexes": [
|
||||
{
|
||||
"name": "idx_chunk_doc",
|
||||
"idxtype": "index",
|
||||
"idxfields": [
|
||||
"doc_id"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "idx_chunk_kb",
|
||||
"idxtype": "index",
|
||||
"idxfields": [
|
||||
"kb_id"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "idx_chunk_vector",
|
||||
"idxtype": "index",
|
||||
"idxfields": [
|
||||
"vector_id"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
151
models/rag_documents.json
Normal file
151
models/rag_documents.json
Normal file
@ -0,0 +1,151 @@
|
||||
{
|
||||
"summary": [
|
||||
{
|
||||
"name": "rag_documents",
|
||||
"title": "文档",
|
||||
"primary": [
|
||||
"id"
|
||||
]
|
||||
}
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"name": "id",
|
||||
"title": "ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "no"
|
||||
},
|
||||
{
|
||||
"name": "kb_id",
|
||||
"title": "知识库ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "folder_id",
|
||||
"title": "目录ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "yes",
|
||||
"default": ""
|
||||
},
|
||||
{
|
||||
"name": "file_name",
|
||||
"title": "文件名",
|
||||
"type": "str",
|
||||
"length": 255,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "file_type",
|
||||
"title": "文件类型",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "file_size",
|
||||
"title": "文件大小",
|
||||
"type": "int",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "file_path",
|
||||
"title": "文件路径",
|
||||
"type": "str",
|
||||
"length": 500,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "mime_type",
|
||||
"title": "MIME类型",
|
||||
"type": "str",
|
||||
"length": 128,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "chunk_count",
|
||||
"title": "分片数",
|
||||
"type": "int",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "embed_dim",
|
||||
"title": "向量维度",
|
||||
"type": "int",
|
||||
"nullable": "yes",
|
||||
"default": "1024"
|
||||
},
|
||||
{
|
||||
"name": "metadata",
|
||||
"title": "元数据JSON",
|
||||
"type": "text",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "status",
|
||||
"title": "状态",
|
||||
"type": "str",
|
||||
"length": 16,
|
||||
"nullable": "yes",
|
||||
"default": "pending"
|
||||
},
|
||||
{
|
||||
"name": "error_msg",
|
||||
"title": "错误信息",
|
||||
"type": "text",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "org_id",
|
||||
"title": "所属机构",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "created_by",
|
||||
"title": "创建人",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "created_at",
|
||||
"title": "创建时间",
|
||||
"type": "datetime",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "updated_at",
|
||||
"title": "更新时间",
|
||||
"type": "datetime",
|
||||
"nullable": "yes"
|
||||
}
|
||||
],
|
||||
"indexes": [
|
||||
{
|
||||
"name": "idx_doc_kb",
|
||||
"idxtype": "index",
|
||||
"idxfields": [
|
||||
"kb_id"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "idx_doc_org",
|
||||
"idxtype": "index",
|
||||
"idxfields": [
|
||||
"org_id"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "idx_doc_status",
|
||||
"idxtype": "index",
|
||||
"idxfields": [
|
||||
"status"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
124
models/rag_engine_configs.json
Normal file
124
models/rag_engine_configs.json
Normal file
@ -0,0 +1,124 @@
|
||||
{
|
||||
"summary": [
|
||||
{
|
||||
"name": "rag_engine_configs",
|
||||
"title": "引擎配置",
|
||||
"primary": [
|
||||
"id"
|
||||
]
|
||||
}
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"name": "id",
|
||||
"title": "ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "no"
|
||||
},
|
||||
{
|
||||
"name": "org_id",
|
||||
"title": "机构ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "engine_type",
|
||||
"title": "引擎类型",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "engine_name",
|
||||
"title": "引擎名称",
|
||||
"type": "str",
|
||||
"length": 64,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "endpoint_url",
|
||||
"title": "服务地址",
|
||||
"type": "str",
|
||||
"length": 255,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "api_key",
|
||||
"title": "API密钥",
|
||||
"type": "str",
|
||||
"length": 255,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "model_name",
|
||||
"title": "模型名称",
|
||||
"type": "str",
|
||||
"length": 128,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "config_json",
|
||||
"title": "扩展配置JSON",
|
||||
"type": "text",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "is_default",
|
||||
"title": "是否默认",
|
||||
"type": "int",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "priority",
|
||||
"title": "优先级",
|
||||
"type": "int",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "status",
|
||||
"title": "状态",
|
||||
"type": "str",
|
||||
"length": 16,
|
||||
"nullable": "yes",
|
||||
"default": "active"
|
||||
},
|
||||
{
|
||||
"name": "health_check_at",
|
||||
"title": "最近健康检查",
|
||||
"type": "datetime",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "created_at",
|
||||
"title": "创建时间",
|
||||
"type": "datetime",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "updated_at",
|
||||
"title": "更新时间",
|
||||
"type": "datetime",
|
||||
"nullable": "yes"
|
||||
}
|
||||
],
|
||||
"indexes": [
|
||||
{
|
||||
"name": "idx_eng_type",
|
||||
"idxtype": "index",
|
||||
"idxfields": [
|
||||
"engine_type",
|
||||
"is_default"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "idx_eng_org",
|
||||
"idxtype": "index",
|
||||
"idxfields": [
|
||||
"org_id",
|
||||
"engine_type"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
138
models/rag_entities.json
Normal file
138
models/rag_entities.json
Normal file
@ -0,0 +1,138 @@
|
||||
{
|
||||
"summary": [
|
||||
{
|
||||
"name": "rag_entities",
|
||||
"title": "实体",
|
||||
"primary": [
|
||||
"id"
|
||||
]
|
||||
}
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"name": "id",
|
||||
"title": "ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "no"
|
||||
},
|
||||
{
|
||||
"name": "kb_id",
|
||||
"title": "知识库ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "name",
|
||||
"title": "实体名",
|
||||
"type": "str",
|
||||
"length": 255,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "entity_type",
|
||||
"title": "实体类型",
|
||||
"type": "str",
|
||||
"length": 64,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "description",
|
||||
"title": "描述",
|
||||
"type": "text",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "graph_node_id",
|
||||
"title": "图节点ID",
|
||||
"type": "str",
|
||||
"length": 128,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "source_doc_ids",
|
||||
"title": "来源文档ID列表",
|
||||
"type": "text",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "face_embedding_id",
|
||||
"title": "人脸特征ID",
|
||||
"type": "str",
|
||||
"length": 128,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "voice_embedding_id",
|
||||
"title": "声纹特征ID",
|
||||
"type": "str",
|
||||
"length": 128,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "metadata",
|
||||
"title": "元数据JSON",
|
||||
"type": "text",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "org_id",
|
||||
"title": "所属机构",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "created_at",
|
||||
"title": "创建时间",
|
||||
"type": "datetime",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "updated_at",
|
||||
"title": "更新时间",
|
||||
"type": "datetime",
|
||||
"nullable": "yes"
|
||||
}
|
||||
],
|
||||
"indexes": [
|
||||
{
|
||||
"name": "idx_entity_kb",
|
||||
"idxtype": "index",
|
||||
"idxfields": [
|
||||
"kb_id"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "idx_entity_name",
|
||||
"idxtype": "index",
|
||||
"idxfields": [
|
||||
"kb_id",
|
||||
"name"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "idx_entity_type",
|
||||
"idxtype": "index",
|
||||
"idxfields": [
|
||||
"kb_id",
|
||||
"entity_type"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "idx_entity_face",
|
||||
"idxtype": "index",
|
||||
"idxfields": [
|
||||
"face_embedding_id"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "idx_entity_voice",
|
||||
"idxtype": "index",
|
||||
"idxfields": [
|
||||
"voice_embedding_id"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
103
models/rag_entity_relations.json
Normal file
103
models/rag_entity_relations.json
Normal file
@ -0,0 +1,103 @@
|
||||
{
|
||||
"summary": [
|
||||
{
|
||||
"name": "rag_entity_relations",
|
||||
"title": "实体关系",
|
||||
"primary": [
|
||||
"id"
|
||||
]
|
||||
}
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"name": "id",
|
||||
"title": "ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "no"
|
||||
},
|
||||
{
|
||||
"name": "kb_id",
|
||||
"title": "知识库ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "subject_id",
|
||||
"title": "主体实体ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "object_id",
|
||||
"title": "客体实体ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "relation_type",
|
||||
"title": "关系类型",
|
||||
"type": "str",
|
||||
"length": 64,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "description",
|
||||
"title": "关系描述",
|
||||
"type": "text",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "graph_edge_id",
|
||||
"title": "图边ID",
|
||||
"type": "str",
|
||||
"length": 128,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "source_chunk_ids",
|
||||
"title": "来源分片ID列表",
|
||||
"type": "text",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "org_id",
|
||||
"title": "所属机构",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "created_at",
|
||||
"title": "创建时间",
|
||||
"type": "datetime",
|
||||
"nullable": "yes"
|
||||
}
|
||||
],
|
||||
"indexes": [
|
||||
{
|
||||
"name": "idx_rel_kb",
|
||||
"idxtype": "index",
|
||||
"idxfields": [
|
||||
"kb_id"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "idx_rel_subject",
|
||||
"idxtype": "index",
|
||||
"idxfields": [
|
||||
"subject_id"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "idx_rel_object",
|
||||
"idxtype": "index",
|
||||
"idxfields": [
|
||||
"object_id"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
124
models/rag_knowledge_bases.json
Normal file
124
models/rag_knowledge_bases.json
Normal file
@ -0,0 +1,124 @@
|
||||
{
|
||||
"summary": [
|
||||
{
|
||||
"name": "rag_knowledge_bases",
|
||||
"title": "知识库",
|
||||
"primary": [
|
||||
"id"
|
||||
]
|
||||
}
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"name": "id",
|
||||
"title": "ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "no"
|
||||
},
|
||||
{
|
||||
"name": "name",
|
||||
"title": "名称",
|
||||
"type": "str",
|
||||
"length": 100,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "description",
|
||||
"title": "描述",
|
||||
"type": "text",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "org_id",
|
||||
"title": "所属机构",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "embedding_engine",
|
||||
"title": "向量化引擎",
|
||||
"type": "str",
|
||||
"length": 64,
|
||||
"nullable": "yes",
|
||||
"default": "clip-vith14"
|
||||
},
|
||||
{
|
||||
"name": "vdb_collection",
|
||||
"title": "向量库集合名",
|
||||
"type": "str",
|
||||
"length": 128,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "graph_name",
|
||||
"title": "图名称",
|
||||
"type": "str",
|
||||
"length": 128,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "doc_count",
|
||||
"title": "文档数",
|
||||
"type": "int",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "chunk_count",
|
||||
"title": "分片数",
|
||||
"type": "int",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "total_size",
|
||||
"title": "总大小(字节)",
|
||||
"type": "int",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "status",
|
||||
"title": "状态",
|
||||
"type": "str",
|
||||
"length": 16,
|
||||
"nullable": "yes",
|
||||
"default": "active"
|
||||
},
|
||||
{
|
||||
"name": "created_by",
|
||||
"title": "创建人",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "created_at",
|
||||
"title": "创建时间",
|
||||
"type": "datetime",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "updated_at",
|
||||
"title": "更新时间",
|
||||
"type": "datetime",
|
||||
"nullable": "yes"
|
||||
}
|
||||
],
|
||||
"indexes": [
|
||||
{
|
||||
"name": "idx_kb_name",
|
||||
"idxtype": "unique",
|
||||
"idxfields": [
|
||||
"org_id",
|
||||
"name"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "idx_kb_org",
|
||||
"idxtype": "index",
|
||||
"idxfields": [
|
||||
"org_id"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
87
models/rag_media_tags.json
Normal file
87
models/rag_media_tags.json
Normal file
@ -0,0 +1,87 @@
|
||||
{
|
||||
"summary": [
|
||||
{
|
||||
"name": "rag_media_tags",
|
||||
"title": "媒体标签关联",
|
||||
"primary": [
|
||||
"id"
|
||||
]
|
||||
}
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"name": "id",
|
||||
"title": "ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "no"
|
||||
},
|
||||
{
|
||||
"name": "kb_id",
|
||||
"title": "知识库ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "no"
|
||||
},
|
||||
{
|
||||
"name": "media_type",
|
||||
"title": "媒体类型",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "no"
|
||||
},
|
||||
{
|
||||
"name": "media_id",
|
||||
"title": "媒体ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "no"
|
||||
},
|
||||
{
|
||||
"name": "tag_id",
|
||||
"title": "标签ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "no"
|
||||
},
|
||||
{
|
||||
"name": "created_at",
|
||||
"title": "创建时间",
|
||||
"type": "datetime",
|
||||
"nullable": "yes"
|
||||
}
|
||||
],
|
||||
"indexes": [
|
||||
{
|
||||
"name": "idx_mt_unique",
|
||||
"idxtype": "unique",
|
||||
"idxfields": [
|
||||
"media_type",
|
||||
"media_id",
|
||||
"tag_id"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "idx_mt_media",
|
||||
"idxtype": "index",
|
||||
"idxfields": [
|
||||
"media_type",
|
||||
"media_id"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "idx_mt_tag",
|
||||
"idxtype": "index",
|
||||
"idxfields": [
|
||||
"tag_id"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "idx_mt_kb",
|
||||
"idxtype": "index",
|
||||
"idxfields": [
|
||||
"kb_id"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
39
models/rag_org_storage_limits.json
Normal file
39
models/rag_org_storage_limits.json
Normal file
@ -0,0 +1,39 @@
|
||||
{
|
||||
"summary": [
|
||||
{
|
||||
"name": "rag_org_storage_limits",
|
||||
"title": "机构存储限额",
|
||||
"primary": [
|
||||
"org_id"
|
||||
]
|
||||
}
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"name": "org_id",
|
||||
"title": "机构ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "no"
|
||||
},
|
||||
{
|
||||
"name": "limit_bytes",
|
||||
"title": "限额(字节)",
|
||||
"type": "bigint",
|
||||
"nullable": "no",
|
||||
"default": "104857600"
|
||||
},
|
||||
{
|
||||
"name": "created_at",
|
||||
"title": "创建时间",
|
||||
"type": "datetime",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "updated_at",
|
||||
"title": "更新时间",
|
||||
"type": "datetime",
|
||||
"nullable": "yes"
|
||||
}
|
||||
]
|
||||
}
|
||||
134
models/rag_subscriptions.json
Normal file
134
models/rag_subscriptions.json
Normal file
@ -0,0 +1,134 @@
|
||||
{
|
||||
"summary": [
|
||||
{
|
||||
"name": "rag_subscriptions",
|
||||
"title": "租户订阅",
|
||||
"primary": [
|
||||
"id"
|
||||
]
|
||||
}
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"name": "id",
|
||||
"title": "ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "no"
|
||||
},
|
||||
{
|
||||
"name": "org_id",
|
||||
"title": "机构ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "plan_name",
|
||||
"title": "套餐名称",
|
||||
"type": "str",
|
||||
"length": 64,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "disk_quota_bytes",
|
||||
"title": "磁盘配额(字节)",
|
||||
"type": "int",
|
||||
"nullable": "yes",
|
||||
"default": "1073741824"
|
||||
},
|
||||
{
|
||||
"name": "disk_used_bytes",
|
||||
"title": "已用磁盘(字节)",
|
||||
"type": "int",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "doc_quota",
|
||||
"title": "文档数配额",
|
||||
"type": "int",
|
||||
"nullable": "yes",
|
||||
"default": "1000"
|
||||
},
|
||||
{
|
||||
"name": "doc_used",
|
||||
"title": "已用文档数",
|
||||
"type": "int",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "kb_quota",
|
||||
"title": "知识库数配额",
|
||||
"type": "int",
|
||||
"nullable": "yes",
|
||||
"default": "10"
|
||||
},
|
||||
{
|
||||
"name": "kb_used",
|
||||
"title": "已用知识库数",
|
||||
"type": "int",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "api_call_quota",
|
||||
"title": "API调用配额(月)",
|
||||
"type": "int",
|
||||
"nullable": "yes",
|
||||
"default": "10000"
|
||||
},
|
||||
{
|
||||
"name": "api_call_used",
|
||||
"title": "已用API调用(月)",
|
||||
"type": "int",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "start_date",
|
||||
"title": "开始日期",
|
||||
"type": "date",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "end_date",
|
||||
"title": "结束日期",
|
||||
"type": "date",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "status",
|
||||
"title": "状态",
|
||||
"type": "str",
|
||||
"length": 16,
|
||||
"nullable": "yes",
|
||||
"default": "active"
|
||||
},
|
||||
{
|
||||
"name": "created_at",
|
||||
"title": "创建时间",
|
||||
"type": "datetime",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "updated_at",
|
||||
"title": "更新时间",
|
||||
"type": "datetime",
|
||||
"nullable": "yes"
|
||||
}
|
||||
],
|
||||
"indexes": [
|
||||
{
|
||||
"name": "idx_sub_org",
|
||||
"idxtype": "unique",
|
||||
"idxfields": [
|
||||
"org_id"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "idx_sub_status",
|
||||
"idxtype": "index",
|
||||
"idxfields": [
|
||||
"status"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
73
models/rag_tags.json
Normal file
73
models/rag_tags.json
Normal file
@ -0,0 +1,73 @@
|
||||
{
|
||||
"summary": [
|
||||
{
|
||||
"name": "rag_tags",
|
||||
"title": "标签",
|
||||
"primary": [
|
||||
"id"
|
||||
]
|
||||
}
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"name": "id",
|
||||
"title": "ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "no"
|
||||
},
|
||||
{
|
||||
"name": "kb_id",
|
||||
"title": "知识库ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "no"
|
||||
},
|
||||
{
|
||||
"name": "name",
|
||||
"title": "标签名",
|
||||
"type": "str",
|
||||
"length": 64,
|
||||
"nullable": "no"
|
||||
},
|
||||
{
|
||||
"name": "color",
|
||||
"title": "颜色",
|
||||
"type": "str",
|
||||
"length": 16,
|
||||
"nullable": "no",
|
||||
"default": "#3b82f6"
|
||||
},
|
||||
{
|
||||
"name": "org_id",
|
||||
"title": "所属机构",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "no",
|
||||
"default": ""
|
||||
},
|
||||
{
|
||||
"name": "created_at",
|
||||
"title": "创建时间",
|
||||
"type": "datetime",
|
||||
"nullable": "yes"
|
||||
}
|
||||
],
|
||||
"indexes": [
|
||||
{
|
||||
"name": "idx_tag_name",
|
||||
"idxtype": "unique",
|
||||
"idxfields": [
|
||||
"kb_id",
|
||||
"name"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "idx_tag_kb",
|
||||
"idxtype": "index",
|
||||
"idxfields": [
|
||||
"kb_id"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
110
models/rag_usage_logs.json
Normal file
110
models/rag_usage_logs.json
Normal file
@ -0,0 +1,110 @@
|
||||
{
|
||||
"summary": [
|
||||
{
|
||||
"name": "rag_usage_logs",
|
||||
"title": "用量日志",
|
||||
"primary": [
|
||||
"id"
|
||||
]
|
||||
}
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"name": "id",
|
||||
"title": "ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "no"
|
||||
},
|
||||
{
|
||||
"name": "org_id",
|
||||
"title": "机构ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "kb_id",
|
||||
"title": "知识库ID",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "operation",
|
||||
"title": "操作类型",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "bytes_delta",
|
||||
"title": "磁盘变化(字节)",
|
||||
"type": "int",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "bytes_total",
|
||||
"title": "累计磁盘",
|
||||
"type": "int",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "api_calls",
|
||||
"title": "API调用数",
|
||||
"type": "int",
|
||||
"nullable": "yes",
|
||||
"default": "1"
|
||||
},
|
||||
{
|
||||
"name": "engine_type",
|
||||
"title": "引擎类型",
|
||||
"type": "str",
|
||||
"length": 32,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "tokens_used",
|
||||
"title": "Token消耗",
|
||||
"type": "int",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "cost_estimate",
|
||||
"title": "预估费用",
|
||||
"type": "double",
|
||||
"length": 20,
|
||||
"dec": 4,
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "detail",
|
||||
"title": "详情JSON",
|
||||
"type": "text",
|
||||
"nullable": "yes"
|
||||
},
|
||||
{
|
||||
"name": "created_at",
|
||||
"title": "创建时间",
|
||||
"type": "datetime",
|
||||
"nullable": "yes"
|
||||
}
|
||||
],
|
||||
"indexes": [
|
||||
{
|
||||
"name": "idx_usage_org",
|
||||
"idxtype": "index",
|
||||
"idxfields": [
|
||||
"org_id",
|
||||
"created_at"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "idx_usage_kb",
|
||||
"idxtype": "index",
|
||||
"idxfields": [
|
||||
"kb_id"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -1,28 +0,0 @@
|
||||
{
|
||||
"tblname": "subscriptions",
|
||||
"params": {
|
||||
"title": "订阅管理",
|
||||
"browserfields": [
|
||||
{"field": "org_id", "title": "机构", "width": "15%"},
|
||||
{"field": "plan_name", "title": "套餐", "width": "15%"},
|
||||
{"field": "disk_quota_bytes", "title": "磁盘配额", "width": "12%"},
|
||||
{"field": "disk_used_bytes", "title": "已用磁盘", "width": "12%"},
|
||||
{"field": "doc_used", "title": "文档数", "width": "10%"},
|
||||
{"field": "start_date", "title": "开始", "width": "12%"},
|
||||
{"field": "end_date", "title": "结束", "width": "12%"},
|
||||
{"field": "status", "title": "状态", "width": "12%"}
|
||||
],
|
||||
"editfields": [
|
||||
{"field": "org_id", "uitype": "Text", "required": true},
|
||||
{"field": "plan_name", "uitype": "Text", "required": true},
|
||||
{"field": "disk_quota_bytes", "uitype": "Text", "default": "1073741824"},
|
||||
{"field": "doc_quota", "uitype": "Text"},
|
||||
{"field": "kb_quota", "uitype": "Text"},
|
||||
{"field": "api_call_quota", "uitype": "Text"},
|
||||
{"field": "start_date", "uitype": "Text"},
|
||||
{"field": "end_date", "uitype": "Text"}
|
||||
],
|
||||
"searchfields": ["org_id", "plan_name"],
|
||||
"sort": "created_at desc"
|
||||
}
|
||||
}
|
||||
@ -1,22 +0,0 @@
|
||||
{
|
||||
"tblname": "tags",
|
||||
"params": {
|
||||
"title": "标签管理",
|
||||
"browserfields": [
|
||||
{"field": "name", "title": "标签名", "width": "25%"},
|
||||
{"field": "color", "title": "颜色", "width": "15%"},
|
||||
{"field": "kb_id", "title": "知识库", "width": "30%"},
|
||||
{"field": "created_at", "title": "创建时间", "width": "30%"}
|
||||
],
|
||||
"editfields": [
|
||||
{"field": "name", "uitype": "Text", "required": true, "label": "标签名"},
|
||||
{"field": "kb_id", "uitype": "Text", "required": true, "label": "知识库ID"},
|
||||
{"field": "color", "uitype": "Text", "default": "#3b82f6", "label": "颜色(#hex)"}
|
||||
],
|
||||
"searchfields": ["name"],
|
||||
"sort": "created_at desc",
|
||||
"data_filter": {
|
||||
"org_id": "{{userorgid}}"
|
||||
}
|
||||
}
|
||||
}
|
||||
88
rag/init.py
88
rag/init.py
@ -27,10 +27,10 @@ async def kb_list_handler(request, params_kw, *args, **kwargs):
|
||||
try:
|
||||
userorgid = await env.get_userorgid()
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
recs = await sor.R("knowledge_bases", {})
|
||||
recs = await sor.R("rag_knowledge_bases", {})
|
||||
cards = []
|
||||
for r in recs:
|
||||
doc_recs = await sor.R("documents", {"kb_id": r.id})
|
||||
doc_recs = await sor.R("rag_documents", {"kb_id": r.id})
|
||||
doc_count = len(doc_recs)
|
||||
total_size = r.total_size or 0
|
||||
if total_size >= 1073741824:
|
||||
@ -64,7 +64,7 @@ async def engines_handler(request, params_kw, *args, **kwargs):
|
||||
try:
|
||||
userorgid = await env.get_userorgid()
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
sql = "SELECT * FROM engine_configs WHERE status='active' AND (org_id IS NULL OR org_id=${org_id}$) ORDER BY engine_type, priority DESC"
|
||||
sql = "SELECT * FROM rag_engine_configs WHERE status='active' AND (org_id IS NULL OR org_id=${org_id}$) ORDER BY engine_type, priority DESC"
|
||||
recs = await sor.sqlExe(sql, {"org_id": userorgid})
|
||||
rows = [dict(r) for r in recs]
|
||||
return json.dumps({"status": "SUCCEEDED", "data": {"rows": rows, "total": len(rows)}}, ensure_ascii=False, default=str)
|
||||
@ -173,10 +173,10 @@ async def _resolve_search_kbs(env, userorgid, kb_id):
|
||||
"""Resolve KB IDs: specific or all org KBs"""
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
if kb_id:
|
||||
recs = await sor.R("knowledge_bases", {"id": kb_id})
|
||||
recs = await sor.R("rag_knowledge_bases", {"id": kb_id})
|
||||
return [r.id for r in recs]
|
||||
# All org KBs (global + org-specific)
|
||||
sql = "SELECT id FROM knowledge_bases WHERE org_id IS NULL OR org_id=${org_id}$"
|
||||
sql = "SELECT id FROM rag_knowledge_bases WHERE org_id IS NULL OR org_id=${org_id}$"
|
||||
recs = await sor.sqlExe(sql, {"org_id": userorgid})
|
||||
return [r.id for r in recs]
|
||||
|
||||
@ -219,7 +219,7 @@ async def _build_search_vector(query, file_data, file_name, env=None, kb_id=''):
|
||||
emb_engine = 'clip-vith14'
|
||||
if env is not None and kb_id:
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
krecs = await sor.sqlExe("SELECT embedding_engine FROM knowledge_bases WHERE id=${kb_id}$", {"kb_id": kb_id})
|
||||
krecs = await sor.sqlExe("SELECT embedding_engine FROM rag_knowledge_bases WHERE id=${kb_id}$", {"kb_id": kb_id})
|
||||
if krecs:
|
||||
emb_engine = (getattr(krecs[0], 'embedding_engine', '') or 'clip-vith14').strip()
|
||||
if emb_engine == 'bge-m3':
|
||||
@ -287,7 +287,7 @@ async def _enrich_search_results(env, hits):
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
recs = await sor.sqlExe(
|
||||
"SELECT id, file_name, file_type, file_size, status, kb_id, created_at "
|
||||
"FROM documents WHERE id IN (" + ",".join(repr(d) for d in doc_ids) + ")", {})
|
||||
"FROM rag_documents WHERE id IN (" + ",".join(repr(d) for d in doc_ids) + ")", {})
|
||||
doc_map = {r.id: dict(r) for r in recs}
|
||||
|
||||
for h in hits:
|
||||
@ -325,9 +325,9 @@ async def doc_upload_handler(request, params_kw, *args, **kwargs):
|
||||
quota_limit = 104857600
|
||||
used = 0
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
rec = await sor.sqlExe("SELECT COALESCE(SUM(file_size),0) AS used FROM documents WHERE org_id=${org_id}$", {"org_id": userorgid})
|
||||
rec = await sor.sqlExe("SELECT COALESCE(SUM(file_size),0) AS used FROM rag_documents WHERE org_id=${org_id}$", {"org_id": userorgid})
|
||||
if rec: used = int(rec[0].used)
|
||||
lim = await sor.sqlExe("SELECT limit_bytes FROM org_storage_limits WHERE org_id=${org_id}$", {"org_id": userorgid})
|
||||
lim = await sor.sqlExe("SELECT limit_bytes FROM rag_org_storage_limits WHERE org_id=${org_id}$", {"org_id": userorgid})
|
||||
if lim: quota_limit = int(lim[0].limit_bytes)
|
||||
if used + file_size > quota_limit:
|
||||
return json.dumps({"error": "storage_quota_exceeded",
|
||||
@ -342,7 +342,7 @@ async def doc_upload_handler(request, params_kw, *args, **kwargs):
|
||||
# Create document record
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
await sor.sqlExe(
|
||||
"INSERT INTO documents (id, kb_id, file_name, file_type, file_size, file_path, mime_type, status, org_id, created_at, updated_at) "
|
||||
"INSERT INTO rag_documents (id, kb_id, file_name, file_type, file_size, file_path, mime_type, status, org_id, created_at, updated_at) "
|
||||
"VALUES (${id}$, ${kb_id}$, ${file_name}$, ${file_type}$, ${file_size}$, ${file_path}$, ${mime_type}$, 'pending', ${org_id}$, NOW(), NOW())",
|
||||
{"id": doc_id, "kb_id": kb_id, "file_name": file_name, "file_type": file_type,
|
||||
"file_size": file_size, "file_path": web_path,
|
||||
@ -350,7 +350,7 @@ async def doc_upload_handler(request, params_kw, *args, **kwargs):
|
||||
|
||||
# Update KB stats
|
||||
await sor.sqlExe(
|
||||
"UPDATE knowledge_bases SET doc_count=doc_count+1, total_size=total_size+${size}$ WHERE id=${kb_id}$",
|
||||
"UPDATE rag_knowledge_bases SET doc_count=doc_count+1, total_size=total_size+${size}$ WHERE id=${kb_id}$",
|
||||
{"size": file_size, "kb_id": kb_id})
|
||||
|
||||
# Trigger async ingest for text-based files via uapi
|
||||
@ -363,17 +363,17 @@ async def doc_upload_handler(request, params_kw, *args, **kwargs):
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
chunks_n = ingest_result.get("chunks", 0) if ingest_result else 0
|
||||
await sor.sqlExe(
|
||||
"UPDATE documents SET status='done', chunk_count=${chunks}$ WHERE id=${id}$",
|
||||
"UPDATE rag_documents SET status='done', chunk_count=${chunks}$ WHERE id=${id}$",
|
||||
{"chunks": chunks_n, "id": doc_id})
|
||||
if chunks_n:
|
||||
await sor.sqlExe(
|
||||
"UPDATE knowledge_bases SET chunk_count=chunk_count+${n}$ WHERE id=${kb_id}$",
|
||||
"UPDATE rag_knowledge_bases SET chunk_count=chunk_count+${n}$ WHERE id=${kb_id}$",
|
||||
{"n": chunks_n, "kb_id": kb_id})
|
||||
except Exception as e:
|
||||
exception(f"uapi ingest failed: {e}")
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
await sor.sqlExe(
|
||||
"UPDATE documents SET status='error' WHERE id=${id}$", {"id": doc_id})
|
||||
"UPDATE rag_documents SET status='error' WHERE id=${id}$", {"id": doc_id})
|
||||
|
||||
return json.dumps({
|
||||
"status": "SUCCEEDED",
|
||||
@ -399,13 +399,13 @@ async def doc_delete_handler(request, params_kw, *args, **kwargs):
|
||||
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
# Get document info
|
||||
recs = await sor.R("documents", {"id": doc_id})
|
||||
recs = await sor.R("rag_documents", {"id": doc_id})
|
||||
if not recs:
|
||||
return json.dumps({"error": "document not found"})
|
||||
doc = recs[0]
|
||||
|
||||
# Get chunks to clean VDB
|
||||
chunks = await sor.R("document_chunks", {"doc_id": doc_id})
|
||||
chunks = await sor.R("rag_document_chunks", {"doc_id": doc_id})
|
||||
|
||||
# Delete from VDB
|
||||
if chunks:
|
||||
@ -424,14 +424,14 @@ async def doc_delete_handler(request, params_kw, *args, **kwargs):
|
||||
exception(f"graph delete failed: {e}")
|
||||
|
||||
# Delete DB records
|
||||
await sor.sqlExe("DELETE FROM document_chunks WHERE doc_id=${id}$", {"id": doc_id})
|
||||
await sor.sqlExe("DELETE FROM entities WHERE kb_id=${kb_id}$", {"kb_id": doc.kb_id})
|
||||
await sor.sqlExe("DELETE FROM entity_relations WHERE kb_id=${kb_id}$", {"kb_id": doc.kb_id})
|
||||
await sor.sqlExe("DELETE FROM documents WHERE id=${id}$", {"id": doc_id})
|
||||
await sor.sqlExe("DELETE FROM rag_document_chunks WHERE doc_id=${id}$", {"id": doc_id})
|
||||
await sor.sqlExe("DELETE FROM rag_entities WHERE kb_id=${kb_id}$", {"kb_id": doc.kb_id})
|
||||
await sor.sqlExe("DELETE FROM rag_entity_relations WHERE kb_id=${kb_id}$", {"kb_id": doc.kb_id})
|
||||
await sor.sqlExe("DELETE FROM rag_documents WHERE id=${id}$", {"id": doc_id})
|
||||
|
||||
# Update KB stats
|
||||
await sor.sqlExe(
|
||||
"UPDATE knowledge_bases SET doc_count=GREATEST(doc_count-1,0), total_size=GREATEST(total_size-${size}$,0), chunk_count=GREATEST(chunk_count-${n}$,0) WHERE id=${kb_id}$",
|
||||
"UPDATE rag_knowledge_bases SET doc_count=GREATEST(doc_count-1,0), total_size=GREATEST(total_size-${size}$,0), chunk_count=GREATEST(chunk_count-${n}$,0) WHERE id=${kb_id}$",
|
||||
{"size": doc.file_size, "n": len(chunks), "kb_id": doc.kb_id})
|
||||
|
||||
# Delete file from disk
|
||||
@ -496,7 +496,7 @@ async def _rag_ingest_async(env, text, kb_id, doc_id):
|
||||
try:
|
||||
emb_engine = 'clip-vith14'
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
krecs = await sor.sqlExe("SELECT embedding_engine FROM knowledge_bases WHERE id=${kb_id}$", {"kb_id": kb_id})
|
||||
krecs = await sor.sqlExe("SELECT embedding_engine FROM rag_knowledge_bases WHERE id=${kb_id}$", {"kb_id": kb_id})
|
||||
if krecs:
|
||||
emb_engine = (getattr(krecs[0], 'embedding_engine', '') or 'clip-vith14').strip()
|
||||
if emb_engine == 'bge-m3':
|
||||
@ -552,7 +552,7 @@ async def _rag_ingest_async(env, text, kb_id, doc_id):
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
for i, (chunk_text, vid) in enumerate(zip(chunks, vector_ids)):
|
||||
await sor.sqlExe(
|
||||
"INSERT INTO document_chunks (id, doc_id, kb_id, chunk_index, content, vector_id, metadata, created_at) "
|
||||
"INSERT INTO rag_document_chunks (id, doc_id, kb_id, chunk_index, content, vector_id, metadata, created_at) "
|
||||
"VALUES (${id}$, ${doc_id}$, ${kb_id}$, ${idx}$, ${content}$, ${vid}$, NOW())",
|
||||
{"id": f"{doc_id}_c{i}", "doc_id": doc_id, "kb_id": kb_id,
|
||||
"idx": i, "content": chunk_text[:2000], "vid": vid})
|
||||
@ -642,7 +642,7 @@ async def dir_create_handler(request, params_kw, *args, **kwargs):
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
dir_id = uuid.uuid4().hex
|
||||
await sor.sqlExe(
|
||||
"INSERT INTO document_chunks (id, doc_id, kb_id, chunk_index, chunk_type, content, description, created_at) "
|
||||
"INSERT INTO rag_document_chunks (id, doc_id, kb_id, chunk_index, chunk_type, content, description, created_at) "
|
||||
"VALUES (${id}$, '', ${kb_id}$, 0, 'directory', ${name}$, ${parent}$, NOW())",
|
||||
{"id": dir_id, "kb_id": kb_id, "name": dir_name, "parent": parent_id})
|
||||
return json.dumps({"status": "SUCCEEDED", "dir_id": dir_id})
|
||||
@ -659,7 +659,7 @@ async def dir_delete_handler(request, params_kw, *args, **kwargs):
|
||||
if not item_id:
|
||||
return json.dumps({"error": "item_id required"})
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
await sor.sqlExe("DELETE FROM document_chunks WHERE id=${id}$ OR doc_id=${id}$", {"id": item_id})
|
||||
await sor.sqlExe("DELETE FROM rag_document_chunks WHERE id=${id}$ OR doc_id=${id}$", {"id": item_id})
|
||||
return json.dumps({"status": "SUCCEEDED"})
|
||||
except Exception as e:
|
||||
exception(f"dir_delete: {e}")
|
||||
@ -676,11 +676,11 @@ async def dir_list_handler(request, params_kw, *args, **kwargs):
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
# Get directories (chunk_type='directory')
|
||||
dirs = await sor.sqlExe(
|
||||
"SELECT id, content as label, description as parent_id FROM document_chunks WHERE kb_id=${kb_id}$ AND chunk_type='directory'",
|
||||
"SELECT id, content as label, description as parent_id FROM rag_document_chunks WHERE kb_id=${kb_id}$ AND chunk_type='directory'",
|
||||
{"kb_id": kb_id})
|
||||
# Get files (documents table)
|
||||
docs = await sor.sqlExe(
|
||||
"SELECT id, file_name as label, '' as parent_id FROM documents WHERE kb_id=${kb_id}$",
|
||||
"SELECT id, file_name as label, '' as parent_id FROM rag_documents WHERE kb_id=${kb_id}$",
|
||||
{"kb_id": kb_id})
|
||||
items = []
|
||||
for d in dirs:
|
||||
@ -705,14 +705,14 @@ async def tag_create_handler(request, params_kw, *args, **kwargs):
|
||||
userorgid = await env.get_userorgid()
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
existing = await sor.sqlExe(
|
||||
"SELECT id, color FROM tags WHERE kb_id=${kb_id}$ AND name=${name}$ AND org_id=${org_id}$",
|
||||
"SELECT id, color FROM rag_tags WHERE kb_id=${kb_id}$ AND name=${name}$ AND org_id=${org_id}$",
|
||||
{"kb_id": kb_id, "name": name, "org_id": userorgid})
|
||||
if existing:
|
||||
return json.dumps({"status": "SUCCEEDED", "tag_id": existing[0].id, "name": name,
|
||||
"color": existing[0].color, "duplicate": True}, ensure_ascii=False)
|
||||
tag_id = uuid.uuid4().hex
|
||||
await sor.sqlExe(
|
||||
"INSERT INTO tags (id, kb_id, name, color, org_id, created_at) "
|
||||
"INSERT INTO rag_tags (id, kb_id, name, color, org_id, created_at) "
|
||||
"VALUES (${id}$, ${kb_id}$, ${name}$, ${color}$, ${org_id}$, NOW())",
|
||||
{"id": tag_id, "kb_id": kb_id, "name": name, "color": color, "org_id": userorgid})
|
||||
return json.dumps({"status": "SUCCEEDED", "tag_id": tag_id, "name": name, "color": color})
|
||||
@ -730,7 +730,7 @@ async def tag_list_handler(request, params_kw, *args, **kwargs):
|
||||
return json.dumps({"error": "kb_id required"})
|
||||
userorgid = await env.get_userorgid()
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
recs = await sor.R("tags", {"kb_id": kb_id, "org_id": userorgid})
|
||||
recs = await sor.R("rag_tags", {"kb_id": kb_id, "org_id": userorgid})
|
||||
tags = [{"id": r.id, "name": r.name, "color": r.color, "created_at": str(r.created_at)} for r in recs]
|
||||
return json.dumps({"status": "SUCCEEDED", "tags": tags}, ensure_ascii=False, default=str)
|
||||
except Exception as e:
|
||||
@ -746,8 +746,8 @@ async def tag_delete_handler(request, params_kw, *args, **kwargs):
|
||||
if not tag_id:
|
||||
return json.dumps({"error": "tag_id required"})
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
await sor.sqlExe("DELETE FROM media_tags WHERE tag_id=${id}$", {"id": tag_id})
|
||||
await sor.sqlExe("DELETE FROM tags WHERE id=${id}$", {"id": tag_id})
|
||||
await sor.sqlExe("DELETE FROM rag_media_tags WHERE tag_id=${id}$", {"id": tag_id})
|
||||
await sor.sqlExe("DELETE FROM rag_tags WHERE id=${id}$", {"id": tag_id})
|
||||
return json.dumps({"status": "SUCCEEDED", "tag_id": tag_id})
|
||||
except Exception as e:
|
||||
exception(f"tag_delete: {e}, {format_exc()}")
|
||||
@ -769,7 +769,7 @@ async def tag_assign_handler(request, params_kw, *args, **kwargs):
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
mt_id = uuid.uuid4().hex
|
||||
await sor.sqlExe(
|
||||
"INSERT INTO media_tags (id, kb_id, media_type, media_id, tag_id, created_at) "
|
||||
"INSERT INTO rag_media_tags (id, kb_id, media_type, media_id, tag_id, created_at) "
|
||||
"VALUES (${id}$, ${kb_id}$, ${type}$, ${mid}$, ${tid}$, NOW())",
|
||||
{"id": mt_id, "kb_id": kb_id, "type": media_type, "mid": media_id, "tid": tag_id})
|
||||
return json.dumps({"status": "SUCCEEDED", "media_tag_id": mt_id})
|
||||
@ -789,7 +789,7 @@ async def tag_unassign_handler(request, params_kw, *args, **kwargs):
|
||||
return json.dumps({"error": "media_type, media_id, tag_id required"})
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
await sor.sqlExe(
|
||||
"DELETE FROM media_tags WHERE media_type=${type}$ AND media_id=${mid}$ AND tag_id=${tid}$",
|
||||
"DELETE FROM rag_media_tags WHERE media_type=${type}$ AND media_id=${mid}$ AND tag_id=${tid}$",
|
||||
{"type": media_type, "mid": media_id, "tid": tag_id})
|
||||
return json.dumps({"status": "SUCCEEDED"})
|
||||
except Exception as e:
|
||||
@ -807,8 +807,8 @@ async def tag_media_tags_handler(request, params_kw, *args, **kwargs):
|
||||
return json.dumps({"error": "media_type and media_id required"})
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
recs = await sor.sqlExe(
|
||||
"SELECT t.id, t.name, t.color FROM media_tags mt "
|
||||
"JOIN tags t ON mt.tag_id=t.id "
|
||||
"SELECT t.id, t.name, t.color FROM rag_media_tags mt "
|
||||
"JOIN rag_tags t ON mt.tag_id=t.id "
|
||||
"WHERE mt.media_type=${type}$ AND mt.media_id=${mid}$",
|
||||
{"type": media_type, "mid": media_id})
|
||||
tags = [{"id": r.id, "name": r.name, "color": r.color} for r in recs]
|
||||
@ -835,7 +835,7 @@ async def tag_search_handler(request, params_kw, *args, **kwargs):
|
||||
media_ids_by_tag = []
|
||||
for tid in tag_ids:
|
||||
recs = await sor.sqlExe(
|
||||
"SELECT media_type, media_id FROM media_tags WHERE kb_id=${kb_id}$ AND tag_id=${tid}$",
|
||||
"SELECT media_type, media_id FROM rag_media_tags WHERE kb_id=${kb_id}$ AND tag_id=${tid}$",
|
||||
{"kb_id": kb_id, "tid": tid})
|
||||
mids = {(r.media_type, r.media_id) for r in recs}
|
||||
media_ids_by_tag.append(mids)
|
||||
@ -855,19 +855,19 @@ async def tag_search_handler(request, params_kw, *args, **kwargs):
|
||||
results = []
|
||||
if doc_ids:
|
||||
docs = await sor.sqlExe(
|
||||
"SELECT id, file_name, file_type, file_size, status, created_at FROM documents WHERE id IN (${ids}$)",
|
||||
"SELECT id, file_name, file_type, file_size, status, created_at FROM rag_documents WHERE id IN (${ids}$)",
|
||||
{"ids": doc_ids})
|
||||
for d in docs:
|
||||
results.append({"type": "document", "id": d.id, "name": d.file_name, "file_type": d.file_type, "size": d.file_size, "status": d.status, "created_at": str(d.created_at)})
|
||||
if face_ids:
|
||||
faces = await sor.sqlExe(
|
||||
"SELECT id, name, description, face_embedding_id, created_at FROM entities WHERE id IN (${ids}$) AND entity_type='person'",
|
||||
"SELECT id, name, description, face_embedding_id, created_at FROM rag_entities WHERE id IN (${ids}$) AND entity_type='person'",
|
||||
{"ids": face_ids})
|
||||
for f in faces:
|
||||
results.append({"type": "face", "id": f.id, "name": f.name, "description": f.description, "created_at": str(f.created_at)})
|
||||
if voice_ids:
|
||||
voices = await sor.sqlExe(
|
||||
"SELECT id, name, description, voice_embedding_id, created_at FROM entities WHERE id IN (${ids}$) AND entity_type='voice'",
|
||||
"SELECT id, name, description, voice_embedding_id, created_at FROM rag_entities WHERE id IN (${ids}$) AND entity_type='voice'",
|
||||
{"ids": voice_ids})
|
||||
for v in voices:
|
||||
results.append({"type": "voice", "id": v.id, "name": v.name, "description": v.description, "created_at": str(v.created_at)})
|
||||
@ -877,7 +877,7 @@ async def tag_search_handler(request, params_kw, *args, **kwargs):
|
||||
vdb_docs = []
|
||||
for doc in tag_filtered_docs:
|
||||
chunks = await sor.sqlExe(
|
||||
"SELECT content FROM document_chunks WHERE doc_id=${id}$ LIMIT 3",
|
||||
"SELECT content FROM rag_document_chunks WHERE doc_id=${id}$ LIMIT 3",
|
||||
{"id": doc["id"]})
|
||||
for c in chunks:
|
||||
vdb_docs.append({"doc_id": doc["id"], "content": c.content})
|
||||
@ -911,20 +911,20 @@ async def tag_sync_handler(request, params_kw, *args, **kwargs):
|
||||
wanted_ids = [t.strip() for t in tag_ids_str.split(",") if t.strip()]
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
recs = await sor.sqlExe(
|
||||
"SELECT id, tag_id FROM media_tags WHERE media_type=${type}$ AND media_id=${mid}$",
|
||||
"SELECT id, tag_id FROM rag_media_tags WHERE media_type=${type}$ AND media_id=${mid}$",
|
||||
{"type": media_type, "mid": media_id})
|
||||
current = {r.tag_id: r.id for r in recs}
|
||||
removed = 0
|
||||
for tid, mt_id in current.items():
|
||||
if tid not in wanted_ids:
|
||||
await sor.sqlExe("DELETE FROM media_tags WHERE id=${id}$", {"id": mt_id})
|
||||
await sor.sqlExe("DELETE FROM rag_media_tags WHERE id=${id}$", {"id": mt_id})
|
||||
removed += 1
|
||||
added = 0
|
||||
for tid in wanted_ids:
|
||||
if tid not in current:
|
||||
mt_id = uuid.uuid4().hex
|
||||
await sor.sqlExe(
|
||||
"INSERT INTO media_tags (id, kb_id, media_type, media_id, tag_id, created_at) "
|
||||
"INSERT INTO rag_media_tags (id, kb_id, media_type, media_id, tag_id, created_at) "
|
||||
"VALUES (${id}$, ${kb_id}$, ${type}$, ${mid}$, ${tid}$, NOW())",
|
||||
{"id": mt_id, "kb_id": kb_id, "type": media_type, "mid": media_id, "tid": tid})
|
||||
added += 1
|
||||
|
||||
@ -185,19 +185,19 @@ async def process_upload(env, file_data, kb_id, folder_id, file_name):
|
||||
from sqlor.dbpools import get_sor_context
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
await sor.sqlExe(
|
||||
"INSERT INTO documents (id, kb_id, folder_id, file_name, file_type, file_size, file_path, mime_type, status, metadata, org_id, created_at, updated_at) "
|
||||
"INSERT INTO rag_documents (id, kb_id, folder_id, file_name, file_type, file_size, file_path, mime_type, status, metadata, org_id, created_at, updated_at) "
|
||||
"VALUES (" + "${id}$, ${kb_id}$, ${folder_id}$, ${file_name}$, 'other', ${file_size}$, ${file_path}$, 'application/octet-stream', ${status}$, ${meta}$, ${org_id}$, NOW(), NOW())",
|
||||
{"id": doc_id, "kb_id": kb_id, "folder_id": folder_id, "file_name": file_name,
|
||||
"file_size": file_size, "file_path": "/idfile/files/" + saved_name,
|
||||
"status": status, "meta": meta, "org_id": userorgid})
|
||||
await sor.sqlExe(
|
||||
"UPDATE knowledge_bases SET doc_count=doc_count+1, total_size=total_size+" + "${size}$ WHERE id=${kb_id}$",
|
||||
"UPDATE rag_knowledge_bases SET doc_count=doc_count+1, total_size=total_size+" + "${size}$ WHERE id=${kb_id}$",
|
||||
{"size": file_size, "kb_id": kb_id})
|
||||
if ingest_result and status == 'done':
|
||||
chunks_n = ingest_result.get('chunks', 0) if isinstance(ingest_result, dict) else 0
|
||||
if chunks_n:
|
||||
await sor.sqlExe("UPDATE documents SET chunk_count=${chunks}$ WHERE id=${id}$", {"chunks": chunks_n, "id": doc_id})
|
||||
await sor.sqlExe("UPDATE knowledge_bases SET chunk_count=chunk_count+${n}$ WHERE id=${kb_id}$", {"n": chunks_n, "kb_id": kb_id})
|
||||
await sor.sqlExe("UPDATE rag_documents SET chunk_count=${chunks}$ WHERE id=${id}$", {"chunks": chunks_n, "id": doc_id})
|
||||
await sor.sqlExe("UPDATE rag_knowledge_bases SET chunk_count=chunk_count+${n}$ WHERE id=${kb_id}$", {"n": chunks_n, "kb_id": kb_id})
|
||||
|
||||
faces_n = face_result.get('faces', 0) if face_result and isinstance(face_result, dict) else 0
|
||||
speakers_n = voice_result.get('speakers', 0) if voice_result and isinstance(voice_result, dict) else 0
|
||||
|
||||
@ -9,7 +9,7 @@ if not doc_id or not tag:
|
||||
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
# Get existing tags
|
||||
recs = await sor.sqlExe("SELECT metadata FROM documents WHERE id=${id}$ AND kb_id=${kb_id}$",
|
||||
recs = await sor.sqlExe("SELECT metadata FROM rag_documents WHERE id=${id}$ AND kb_id=${kb_id}$",
|
||||
{"id": doc_id, "kb_id": kb_id})
|
||||
if not recs:
|
||||
return json.dumps({"status": "error", "error": "document not found"}, ensure_ascii=False)
|
||||
@ -29,7 +29,7 @@ async with get_sor_context(env, 'rag') as sor:
|
||||
tags.append(tag)
|
||||
meta['tags'] = tags
|
||||
await sor.sqlExe(
|
||||
"UPDATE documents SET metadata=${meta}$ WHERE id=${id}$",
|
||||
"UPDATE rag_documents SET metadata=${meta}$ WHERE id=${id}$",
|
||||
{"meta": _json.dumps(meta, ensure_ascii=False), "id": doc_id})
|
||||
return _json.dumps({"status": "SUCCEEDED", "tags": tags, "added": tag}, ensure_ascii=False)
|
||||
else:
|
||||
|
||||
@ -6,7 +6,7 @@ rows = []
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
recs = await sor.sqlExe(
|
||||
"SELECT id, file_name, file_size, file_type, file_path, metadata, status, chunk_count "
|
||||
"FROM documents WHERE kb_id=${kb_id}$ AND metadata IS NOT NULL AND metadata != '' "
|
||||
"FROM rag_documents WHERE kb_id=${kb_id}$ AND metadata IS NOT NULL AND metadata != '' "
|
||||
"ORDER BY created_at DESC LIMIT 50",
|
||||
{"kb_id": kb_id})
|
||||
rows = [dict(r) for r in recs]
|
||||
|
||||
@ -6,10 +6,10 @@ env = request._run_ns
|
||||
rows = []
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
recs = await sor.sqlExe(
|
||||
"SELECT id, file_name, file_size, file_path FROM documents WHERE kb_id=${kb_id}$ AND (chunk_count IS NULL OR chunk_count=0 OR status='pending') LIMIT 20",
|
||||
"SELECT id, file_name, file_size, file_path FROM rag_documents WHERE kb_id=${kb_id}$ AND (chunk_count IS NULL OR chunk_count=0 OR status='pending') LIMIT 20",
|
||||
{"kb_id": kb_id})
|
||||
rows = [dict(r) for r in recs]
|
||||
krecs = await sor.sqlExe("SELECT embedding_engine FROM knowledge_bases WHERE id=${kb_id}$", {"kb_id": kb_id})
|
||||
krecs = await sor.sqlExe("SELECT embedding_engine FROM rag_knowledge_bases WHERE id=${kb_id}$", {"kb_id": kb_id})
|
||||
emb_engine = (getattr(krecs[0], 'embedding_engine', '') or 'clip-vith14').strip() if krecs else 'clip-vith14'
|
||||
if emb_engine == 'bge-m3':
|
||||
emb_url = 'https://embedding.opencomputing.net:10443/txte/api/embed'
|
||||
@ -156,7 +156,7 @@ async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=60)) as ses
|
||||
meta = j2.dumps({"face": face_count, "speakers": speakers}, ensure_ascii=False)
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
await sor.sqlExe(
|
||||
"UPDATE documents SET status='done',chunk_count=0,metadata=${meta}$ WHERE id=${id}$",
|
||||
"UPDATE rag_documents SET status='done',chunk_count=0,metadata=${meta}$ WHERE id=${id}$",
|
||||
{"meta": meta, "id": fid})
|
||||
results.append(f"🎬 {fn}: faces={face_count} speakers={speakers}")
|
||||
continue
|
||||
@ -201,10 +201,10 @@ async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=60)) as ses
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
for i, ct in enumerate(chunks):
|
||||
await sor.sqlExe(
|
||||
"INSERT INTO document_chunks (id,doc_id,kb_id,chunk_index,content,vector_id,created_at) VALUES (${id}$,${doc_id}$,${kb_id}$,${idx}$,${content}$,${vid}$,NOW())",
|
||||
"INSERT INTO rag_document_chunks (id,doc_id,kb_id,chunk_index,content,vector_id,created_at) VALUES (${id}$,${doc_id}$,${kb_id}$,${idx}$,${content}$,${vid}$,NOW())",
|
||||
{"id": f"{fid}_c{i}", "doc_id": fid, "kb_id": kb_id, "idx": i, "content": ct[:2000], "vid": ""})
|
||||
await sor.sqlExe("UPDATE documents SET status='done',chunk_count=${n}$ WHERE id=${id}$", {"n": len(chunks), "id": fid})
|
||||
await sor.sqlExe("UPDATE knowledge_bases SET chunk_count=chunk_count+${n}$ WHERE id=${kb_id}$", {"n": len(chunks), "kb_id": kb_id})
|
||||
await sor.sqlExe("UPDATE rag_documents SET status='done',chunk_count=${n}$ WHERE id=${id}$", {"n": len(chunks), "id": fid})
|
||||
await sor.sqlExe("UPDATE rag_knowledge_bases SET chunk_count=chunk_count+${n}$ WHERE id=${kb_id}$", {"n": len(chunks), "kb_id": kb_id})
|
||||
|
||||
results.append(f"✓ {fn}: {len(text)} chars → {len(chunks)} chunks")
|
||||
|
||||
|
||||
@ -12,7 +12,7 @@ async with db.sqlorContext(dbname) as sor:
|
||||
if emb_type not in ("bge-m3", "clip-vith14"):
|
||||
emb_type = "clip-vith14"
|
||||
await sor.sqlExe(
|
||||
"INSERT INTO knowledge_bases (id, name, description, org_id, embedding_engine, vdb_collection, doc_count, total_size, chunk_count, status, created_at) "
|
||||
"INSERT INTO rag_knowledge_bases (id, name, description, org_id, embedding_engine, vdb_collection, doc_count, total_size, chunk_count, status, created_at) "
|
||||
"VALUES (${id}$, ${name}$, ${desc}$, ${org_id}$, ${emb}$, 'rag_collection', 0, 0, 0, 'active', NOW())",
|
||||
{"id": kb_id, "name": name, "desc": desc, "org_id": userorgid, "emb": emb_type})
|
||||
return {
|
||||
|
||||
@ -12,7 +12,7 @@ userorgid = await env.get_userorgid()
|
||||
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
recs = await sor.sqlExe(
|
||||
"SELECT file_path, file_size, kb_id FROM documents WHERE id=${id}$",
|
||||
"SELECT file_path, file_size, kb_id FROM rag_documents WHERE id=${id}$",
|
||||
{"id": doc_id})
|
||||
if not recs:
|
||||
return json.dumps({"status": "error", "error": "file not found"}, ensure_ascii=False)
|
||||
@ -21,12 +21,12 @@ async with get_sor_context(env, 'rag') as sor:
|
||||
file_size = doc.file_size
|
||||
|
||||
# Delete document + chunks
|
||||
await sor.sqlExe("DELETE FROM document_chunks WHERE doc_id=${doc_id}$", {"doc_id": doc_id})
|
||||
await sor.sqlExe("DELETE FROM documents WHERE id=${id}$", {"id": doc_id})
|
||||
await sor.sqlExe("DELETE FROM rag_document_chunks WHERE doc_id=${doc_id}$", {"doc_id": doc_id})
|
||||
await sor.sqlExe("DELETE FROM rag_documents WHERE id=${id}$", {"id": doc_id})
|
||||
|
||||
# Update KB stats
|
||||
await sor.sqlExe(
|
||||
"UPDATE knowledge_bases SET doc_count=GREATEST(doc_count-1, 0), total_size=GREATEST(total_size-${size}$, 0) WHERE id=${kb_id}$",
|
||||
"UPDATE rag_knowledge_bases SET doc_count=GREATEST(doc_count-1, 0), total_size=GREATEST(total_size-${size}$, 0) WHERE id=${kb_id}$",
|
||||
{"size": file_size, "kb_id": kb_id})
|
||||
|
||||
# Delete physical file via env.realpath
|
||||
@ -45,11 +45,11 @@ rows = []
|
||||
async with db.sqlorContext(dbname) as sor:
|
||||
if folder_id and folder_id != '__root__':
|
||||
recs = await sor.sqlExe(
|
||||
"SELECT id, file_name, file_type, file_size, file_path, status, chunk_count, created_at FROM documents WHERE kb_id=${kb_id}$ AND folder_id=${folder_id}$ ORDER BY created_at DESC",
|
||||
"SELECT id, file_name, file_type, file_size, file_path, status, chunk_count, created_at FROM rag_documents WHERE kb_id=${kb_id}$ AND folder_id=${folder_id}$ ORDER BY created_at DESC",
|
||||
{"kb_id": kb_id, "folder_id": folder_id})
|
||||
else:
|
||||
recs = await sor.sqlExe(
|
||||
"SELECT id, file_name, file_type, file_size, file_path, status, chunk_count, created_at FROM documents WHERE kb_id=${kb_id}$ AND (folder_id IS NULL OR folder_id='') ORDER BY created_at DESC",
|
||||
"SELECT id, file_name, file_type, file_size, file_path, status, chunk_count, created_at FROM rag_documents WHERE kb_id=${kb_id}$ AND (folder_id IS NULL OR folder_id='') ORDER BY created_at DESC",
|
||||
{"kb_id": kb_id})
|
||||
for r in recs:
|
||||
rows.append({
|
||||
|
||||
@ -14,21 +14,21 @@ deleted = {"documents": 0, "chunks": 0, "tags": 0, "media_tags": 0,
|
||||
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
kb = await sor.sqlExe(
|
||||
"SELECT id, name FROM knowledge_bases WHERE id=${id}$ AND (org_id=${org_id}$ OR org_id IS NULL)",
|
||||
"SELECT id, name FROM rag_knowledge_bases WHERE id=${id}$ AND (org_id=${org_id}$ OR org_id IS NULL)",
|
||||
{"id": kb_id, "org_id": userorgid})
|
||||
if not kb:
|
||||
return {"widgettype": "urlwidget",
|
||||
"options": {"url": entire_url('/rag/knowledge_bases_list/index.ui')}}
|
||||
|
||||
docs = await sor.sqlExe(
|
||||
"SELECT id, file_path FROM documents WHERE kb_id=${kb_id}$",
|
||||
"SELECT id, file_path FROM rag_documents WHERE kb_id=${kb_id}$",
|
||||
{"kb_id": kb_id})
|
||||
doc_ids = [d.id for d in docs]
|
||||
|
||||
# 1. VDB cleanup
|
||||
if doc_ids:
|
||||
chunks = await sor.sqlExe(
|
||||
"SELECT vector_id FROM document_chunks WHERE doc_id IN (${ids}$) AND vector_id IS NOT NULL AND vector_id != ''",
|
||||
"SELECT vector_id FROM rag_document_chunks WHERE doc_id IN (${ids}$) AND vector_id IS NOT NULL AND vector_id != ''",
|
||||
{"ids": doc_ids})
|
||||
vector_ids = [c.vector_id for c in chunks]
|
||||
if vector_ids:
|
||||
@ -53,40 +53,40 @@ async with get_sor_context(env, 'rag') as sor:
|
||||
|
||||
# 3. Entities & relations
|
||||
er = await sor.sqlExe(
|
||||
"SELECT COUNT(*) AS cnt FROM entity_relations WHERE kb_id=${kb_id}$",
|
||||
"SELECT COUNT(*) AS cnt FROM rag_entity_relations WHERE kb_id=${kb_id}$",
|
||||
{"kb_id": kb_id})
|
||||
await sor.sqlExe("DELETE FROM entity_relations WHERE kb_id=${kb_id}$", {"kb_id": kb_id})
|
||||
await sor.sqlExe("DELETE FROM rag_entity_relations WHERE kb_id=${kb_id}$", {"kb_id": kb_id})
|
||||
deleted["entities"] += er[0].cnt if er else 0
|
||||
|
||||
ent = await sor.sqlExe(
|
||||
"SELECT COUNT(*) AS cnt FROM entities WHERE kb_id=${kb_id}$",
|
||||
"SELECT COUNT(*) AS cnt FROM rag_entities WHERE kb_id=${kb_id}$",
|
||||
{"kb_id": kb_id})
|
||||
await sor.sqlExe("DELETE FROM entities WHERE kb_id=${kb_id}$", {"kb_id": kb_id})
|
||||
await sor.sqlExe("DELETE FROM rag_entities WHERE kb_id=${kb_id}$", {"kb_id": kb_id})
|
||||
deleted["entities"] += ent[0].cnt if ent else 0
|
||||
|
||||
# 4. Chunks
|
||||
ch = await sor.sqlExe(
|
||||
"SELECT COUNT(*) AS cnt FROM document_chunks WHERE doc_id IN (${ids}$)",
|
||||
"SELECT COUNT(*) AS cnt FROM rag_document_chunks WHERE doc_id IN (${ids}$)",
|
||||
{"ids": doc_ids})
|
||||
await sor.sqlExe("DELETE FROM document_chunks WHERE doc_id IN (${ids}$)", {"ids": doc_ids})
|
||||
await sor.sqlExe("DELETE FROM rag_document_chunks WHERE doc_id IN (${ids}$)", {"ids": doc_ids})
|
||||
deleted["chunks"] = ch[0].cnt if ch else 0
|
||||
|
||||
# Media tags
|
||||
mt = await sor.sqlExe("SELECT COUNT(*) AS cnt FROM media_tags WHERE kb_id=${kb_id}$", {"kb_id": kb_id})
|
||||
await sor.sqlExe("DELETE FROM media_tags WHERE kb_id=${kb_id}$", {"kb_id": kb_id})
|
||||
mt = await sor.sqlExe("SELECT COUNT(*) AS cnt FROM rag_media_tags WHERE kb_id=${kb_id}$", {"kb_id": kb_id})
|
||||
await sor.sqlExe("DELETE FROM rag_media_tags WHERE kb_id=${kb_id}$", {"kb_id": kb_id})
|
||||
deleted["media_tags"] = mt[0].cnt if mt else 0
|
||||
|
||||
# Tags
|
||||
t = await sor.sqlExe("SELECT COUNT(*) AS cnt FROM tags WHERE kb_id=${kb_id}$", {"kb_id": kb_id})
|
||||
await sor.sqlExe("DELETE FROM tags WHERE kb_id=${kb_id}$", {"kb_id": kb_id})
|
||||
t = await sor.sqlExe("SELECT COUNT(*) AS cnt FROM rag_tags WHERE kb_id=${kb_id}$", {"kb_id": kb_id})
|
||||
await sor.sqlExe("DELETE FROM rag_tags WHERE kb_id=${kb_id}$", {"kb_id": kb_id})
|
||||
deleted["tags"] = t[0].cnt if t else 0
|
||||
|
||||
# Documents
|
||||
await sor.sqlExe("DELETE FROM documents WHERE kb_id=${kb_id}$", {"kb_id": kb_id})
|
||||
await sor.sqlExe("DELETE FROM rag_documents WHERE kb_id=${kb_id}$", {"kb_id": kb_id})
|
||||
deleted["documents"] = len(doc_ids)
|
||||
|
||||
# KB itself
|
||||
await sor.sqlExe("DELETE FROM knowledge_bases WHERE id=${id}$", {"id": kb_id})
|
||||
await sor.sqlExe("DELETE FROM rag_knowledge_bases WHERE id=${id}$", {"id": kb_id})
|
||||
|
||||
# Physical files — paths are like /44/126/174/61/file.mp4
|
||||
for d in docs:
|
||||
|
||||
@ -2,6 +2,6 @@ ns = params_kw.copy()
|
||||
db = DBPools()
|
||||
dbname = get_module_dbname('rag')
|
||||
async with db.sqlorContext(dbname) as sor:
|
||||
await sor.sqlExe("DELETE FROM document_chunks WHERE id=${id}$", {"id": ns.get("id","")})
|
||||
await sor.sqlExe("DELETE FROM rag_document_chunks WHERE id=${id}$", {"id": ns.get("id","")})
|
||||
return {"widgettype": "Message", "options": {"user_data": {"id": ns.get("id","")}}}
|
||||
return {"error": "failed"}
|
||||
|
||||
@ -14,11 +14,11 @@ try:
|
||||
async with db.sqlorContext(dbname) as sor:
|
||||
if folder_id:
|
||||
recs = await sor.sqlExe(
|
||||
"SELECT id, file_name, file_type, file_size, file_path, status, chunk_count, created_at FROM documents WHERE kb_id=${kb_id}$ AND folder_id=${folder_id}$ ORDER BY created_at DESC",
|
||||
"SELECT id, file_name, file_type, file_size, file_path, status, chunk_count, created_at FROM rag_documents WHERE kb_id=${kb_id}$ AND folder_id=${folder_id}$ ORDER BY created_at DESC",
|
||||
{"kb_id": kb_id, "folder_id": folder_id})
|
||||
else:
|
||||
recs = await sor.sqlExe(
|
||||
"SELECT id, file_name, file_type, file_size, file_path, status, chunk_count, created_at FROM documents WHERE kb_id=${kb_id}$ AND (folder_id IS NULL OR folder_id IN ('','__root__')) ORDER BY created_at DESC",
|
||||
"SELECT id, file_name, file_type, file_size, file_path, status, chunk_count, created_at FROM rag_documents WHERE kb_id=${kb_id}$ AND (folder_id IS NULL OR folder_id IN ('','__root__')) ORDER BY created_at DESC",
|
||||
{"kb_id": kb_id})
|
||||
for r in recs:
|
||||
rows.append({
|
||||
|
||||
@ -7,7 +7,7 @@ try:
|
||||
async with db.sqlorContext(dbname) as sor:
|
||||
if not id:
|
||||
recs = await sor.sqlExe(
|
||||
"SELECT id, '' as parentid, content as label FROM document_chunks WHERE kb_id=${kb_id}$ AND chunk_type='directory' AND (description IS NULL OR description='') ORDER BY content",
|
||||
"SELECT id, '' as parentid, content as label FROM rag_document_chunks WHERE kb_id=${kb_id}$ AND chunk_type='directory' AND (description IS NULL OR description='') ORDER BY content",
|
||||
{"kb_id": kb_id})
|
||||
result = [{"id": "__root__", "parentid": "", "label": "📁 根目录"}]
|
||||
for r in recs:
|
||||
@ -15,7 +15,7 @@ try:
|
||||
return result
|
||||
else:
|
||||
recs = await sor.sqlExe(
|
||||
"SELECT id, description as parentid, content as label FROM document_chunks WHERE kb_id=${kb_id}$ AND chunk_type='directory' AND description=${id}$ ORDER BY content",
|
||||
"SELECT id, description as parentid, content as label FROM rag_document_chunks WHERE kb_id=${kb_id}$ AND chunk_type='directory' AND description=${id}$ ORDER BY content",
|
||||
{"kb_id": kb_id, "id": id})
|
||||
return [dict(r) for r in recs]
|
||||
except:
|
||||
|
||||
@ -3,7 +3,7 @@ userorgid = await env.get_userorgid()
|
||||
cards = []
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
recs = await sor.sqlExe(
|
||||
"SELECT * FROM knowledge_bases WHERE org_id=${org_id}$ ORDER BY created_at DESC",
|
||||
"SELECT * FROM rag_knowledge_bases WHERE org_id=${org_id}$ ORDER BY created_at DESC",
|
||||
{"org_id": userorgid})
|
||||
for r in recs:
|
||||
doc_count = r.doc_count or 0
|
||||
|
||||
@ -3,7 +3,7 @@ env = request._run_ns
|
||||
userorgid = await env.get_userorgid()
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
recs = await sor.sqlExe(
|
||||
"SELECT id, name FROM knowledge_bases WHERE org_id=${org_id}$ ORDER BY created_at DESC",
|
||||
"SELECT id, name FROM rag_knowledge_bases WHERE org_id=${org_id}$ ORDER BY created_at DESC",
|
||||
{"org_id": userorgid})
|
||||
items = [{"value": r.id, "text": r.name} for r in recs]
|
||||
return json.dumps(items, ensure_ascii=False)
|
||||
|
||||
@ -12,14 +12,14 @@ try:
|
||||
async with db.sqlorContext(dbname) as sor:
|
||||
if kind == 'voice':
|
||||
recs = await sor.sqlExe(
|
||||
"SELECT id, file_name, file_path, metadata, created_at FROM documents "
|
||||
"SELECT id, file_name, file_path, metadata, created_at FROM rag_documents "
|
||||
"WHERE kb_id=${kb_id}$ AND (metadata LIKE '%%voiceprint_status%%done%%' OR LOWER(file_name) LIKE '%%.mp3' OR LOWER(file_name) LIKE '%%.wav' "
|
||||
"OR LOWER(file_name) LIKE '%%.m4a' OR LOWER(file_name) LIKE '%%.aac' OR LOWER(file_name) LIKE '%%.ogg' "
|
||||
"OR LOWER(file_name) LIKE '%%.flac') ORDER BY created_at DESC",
|
||||
{"kb_id": kb_id})
|
||||
else:
|
||||
recs = await sor.sqlExe(
|
||||
"SELECT id, file_name, file_path, metadata, created_at FROM documents "
|
||||
"SELECT id, file_name, file_path, metadata, created_at FROM rag_documents "
|
||||
"WHERE kb_id=${kb_id}$ AND (LOWER(file_name) LIKE '%%.jpg' OR LOWER(file_name) LIKE '%%.jpeg' "
|
||||
"OR LOWER(file_name) LIKE '%%.png' OR LOWER(file_name) LIKE '%%.gif' OR LOWER(file_name) LIKE '%%.webp' "
|
||||
"OR LOWER(file_name) LIKE '%%.bmp') ORDER BY created_at DESC",
|
||||
@ -41,8 +41,8 @@ try:
|
||||
try:
|
||||
id_list = ','.join(["'" + str(x) + "'" for x in doc_ids])
|
||||
mt_recs = await sor.sqlExe(
|
||||
"SELECT mt.media_id, t.name, t.color FROM media_tags mt " +
|
||||
"JOIN tags t ON mt.tag_id=t.id " +
|
||||
"SELECT mt.media_id, t.name, t.color FROM rag_media_tags mt " +
|
||||
"JOIN rag_tags t ON mt.tag_id=t.id " +
|
||||
"WHERE mt.media_type='document' AND mt.media_id IN (" + id_list + ")",
|
||||
ns={})
|
||||
for mt in mt_recs:
|
||||
|
||||
@ -7,7 +7,7 @@ async with db.sqlorContext(dbname) as sor:
|
||||
parent_id = ns.get("parentid", "")
|
||||
kb_id = ns.get("kb_id", "")
|
||||
await sor.sqlExe(
|
||||
"INSERT INTO document_chunks (id, doc_id, kb_id, chunk_index, chunk_type, content, description, created_at) "
|
||||
"INSERT INTO rag_document_chunks (id, doc_id, kb_id, chunk_index, chunk_type, content, description, created_at) "
|
||||
"VALUES (${id}$, '', ${kb_id}$, 0, 'directory', ${name}$, ${parentid}$, NOW())",
|
||||
{"id": dir_id, "kb_id": kb_id, "name": name, "parentid": parent_id})
|
||||
return {"widgettype": "Message", "options": {"user_data": {"id": dir_id, "label": name, "parentid": parent_id}}}
|
||||
|
||||
@ -11,7 +11,7 @@ userorgid = await env.get_userorgid()
|
||||
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
await sor.sqlExe(
|
||||
"UPDATE knowledge_bases SET name=${name}$, description=${desc}$, updated_at=NOW() "
|
||||
"UPDATE rag_knowledge_bases SET name=${name}$, description=${desc}$, updated_at=NOW() "
|
||||
"WHERE id=${id}$ AND (org_id=${org_id}$ OR org_id IS NULL)",
|
||||
{"name": name, "desc": desc, "id": kb_id, "org_id": userorgid})
|
||||
|
||||
|
||||
@ -11,7 +11,7 @@ env = request._run_ns
|
||||
if not name:
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
recs = await sor.sqlExe(
|
||||
"SELECT name, description FROM knowledge_bases WHERE id=${id}$",
|
||||
"SELECT name, description FROM rag_knowledge_bases WHERE id=${id}$",
|
||||
{"id": kb_id})
|
||||
if recs:
|
||||
name = recs[0].name or ''
|
||||
|
||||
@ -22,18 +22,18 @@ else:
|
||||
added = 0; removed = 0
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
recs = await sor.sqlExe(
|
||||
"SELECT id, tag_id FROM media_tags WHERE media_type='document' AND media_id=${mid}$",
|
||||
"SELECT id, tag_id FROM rag_media_tags WHERE media_type='document' AND media_id=${mid}$",
|
||||
{"mid": doc_id})
|
||||
current = {r.tag_id: r.id for r in recs}
|
||||
for tid, mt_id in list(current.items()):
|
||||
if tid not in wanted_ids:
|
||||
await sor.sqlExe("DELETE FROM media_tags WHERE id=${id}$", {"id": mt_id})
|
||||
await sor.sqlExe("DELETE FROM rag_media_tags WHERE id=${id}$", {"id": mt_id})
|
||||
removed += 1
|
||||
for tid in wanted_ids:
|
||||
if tid not in current:
|
||||
mt_id = uuid()
|
||||
await sor.sqlExe(
|
||||
"INSERT INTO media_tags (id, kb_id, media_type, media_id, tag_id, created_at) "
|
||||
"INSERT INTO rag_media_tags (id, kb_id, media_type, media_id, tag_id, created_at) "
|
||||
"VALUES (${id}$, ${kb_id}$, 'document', ${mid}$, ${tid}$, NOW())",
|
||||
{"id": mt_id, "kb_id": kb_id, "mid": doc_id, "tid": tid})
|
||||
added += 1
|
||||
|
||||
@ -51,7 +51,7 @@ if tag_ids_str:
|
||||
for i, tid in enumerate(wanted_tags):
|
||||
placeholders.append("${tid_" + str(i) + "}$")
|
||||
nsq["tid_" + str(i)] = tid
|
||||
sql = ("SELECT media_id FROM media_tags "
|
||||
sql = ("SELECT media_id FROM rag_media_tags "
|
||||
"WHERE " + kb_filter + "media_type='document' AND tag_id IN (" + ",".join(placeholders) + ") "
|
||||
"GROUP BY media_id HAVING COUNT(DISTINCT tag_id)=" + str(len(wanted_tags)))
|
||||
recs = await sor.sqlExe(sql, nsq)
|
||||
@ -79,7 +79,7 @@ kw_rows = []
|
||||
emb_engine = 'clip-vith14'
|
||||
try:
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
krecs = await sor.sqlExe("SELECT embedding_engine FROM knowledge_bases WHERE id=${kb_id}$", {"kb_id": kb_id})
|
||||
krecs = await sor.sqlExe("SELECT embedding_engine FROM rag_knowledge_bases WHERE id=${kb_id}$", {"kb_id": kb_id})
|
||||
if krecs:
|
||||
emb_engine = (getattr(krecs[0], 'embedding_engine', '') or 'clip-vith14').strip()
|
||||
except: pass
|
||||
@ -126,7 +126,7 @@ if query:
|
||||
for i, t in enumerate(tokens):
|
||||
conds.append("content LIKE ${kw_" + str(i) + "}$")
|
||||
nsq["kw_" + str(i)] = "%" + t + "%"
|
||||
ksql = "SELECT id, doc_id, content FROM document_chunks WHERE kb_id=${kb_id}$ AND (" + " OR ".join(conds) + ") LIMIT 20"
|
||||
ksql = "SELECT id, doc_id, content FROM rag_document_chunks WHERE kb_id=${kb_id}$ AND (" + " OR ".join(conds) + ") LIMIT 20"
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
krecs = await sor.sqlExe(ksql, nsq)
|
||||
for r in krecs:
|
||||
@ -145,7 +145,7 @@ if query:
|
||||
chunk_text = ''
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
recs = await sor.sqlExe(
|
||||
"SELECT content, doc_id, metadata FROM document_chunks WHERE id=${id}$",
|
||||
"SELECT content, doc_id, metadata FROM rag_document_chunks WHERE id=${id}$",
|
||||
{"id": rid})
|
||||
chunk_meta = {}
|
||||
if recs:
|
||||
@ -190,7 +190,7 @@ else:
|
||||
placeholders2.append("${did_" + str(i) + "}$")
|
||||
nsq2["did_" + str(i)] = did
|
||||
# Get documents (exclude face/voice generated derivatives)
|
||||
sql2 = ("SELECT id, file_name, kb_id, file_path FROM documents WHERE " + kb_cond +
|
||||
sql2 = ("SELECT id, file_name, kb_id, file_path FROM rag_documents WHERE " + kb_cond +
|
||||
"id IN (" + ",".join(placeholders2) + ") AND (metadata IS NULL OR metadata NOT LIKE '%%\"face\"%%') ORDER BY created_at DESC LIMIT " + str(top_k * 2))
|
||||
docs = await sor.sqlExe(sql2, nsq2)
|
||||
doc_list = [(r.id, r.file_name or '', r.kb_id or '', r.file_path or '') for r in docs]
|
||||
@ -203,7 +203,7 @@ else:
|
||||
for i, did in enumerate(doc_ids):
|
||||
chunk_placeholders.append("${cdid_" + str(i) + "}$")
|
||||
nsq_c["cdid_" + str(i)] = did
|
||||
chunks_sql = ("SELECT id, doc_id, content FROM document_chunks WHERE doc_id IN (" +
|
||||
chunks_sql = ("SELECT id, doc_id, content FROM rag_document_chunks WHERE doc_id IN (" +
|
||||
",".join(chunk_placeholders) + ") ORDER BY created_at DESC LIMIT " + str(top_k))
|
||||
chunk_recs = await sor.sqlExe(chunks_sql, nsq_c)
|
||||
chunks_by_doc = {}
|
||||
|
||||
@ -2,7 +2,7 @@ ns = params_kw.copy()
|
||||
env = request._run_ns
|
||||
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
rec = await sor.sqlExe("SELECT COALESCE(SUM(total_size),0) used FROM knowledge_bases", {})
|
||||
rec = await sor.sqlExe("SELECT COALESCE(SUM(total_size),0) used FROM rag_knowledge_bases", {})
|
||||
used_bytes = int(rec[0].used) if rec else 0
|
||||
|
||||
used_mb = round(used_bytes / 1048576, 1)
|
||||
|
||||
@ -9,13 +9,13 @@ docs = 0
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
rec = await sor.sqlExe(
|
||||
"SELECT COALESCE(SUM(d.file_size),0) AS used, COUNT(*) AS docs "
|
||||
"FROM documents d JOIN knowledge_bases k ON d.kb_id=k.id "
|
||||
"FROM rag_documents d JOIN rag_knowledge_bases k ON d.kb_id=k.id "
|
||||
"WHERE k.org_id=${org_id}$",
|
||||
{"org_id": userorgid})
|
||||
if rec:
|
||||
used = int(rec[0].used)
|
||||
docs = int(rec[0].docs)
|
||||
lim = await sor.sqlExe("SELECT limit_bytes FROM org_storage_limits WHERE org_id=${org_id}$", {"org_id": userorgid})
|
||||
lim = await sor.sqlExe("SELECT limit_bytes FROM rag_org_storage_limits WHERE org_id=${org_id}$", {"org_id": userorgid})
|
||||
if lim:
|
||||
limit = int(lim[0].limit_bytes)
|
||||
|
||||
|
||||
@ -17,14 +17,14 @@ all_tags = [] # all tags for kb: list of (id, name, color)
|
||||
try:
|
||||
async with db.sqlorContext(dbname) as sor:
|
||||
recs = await sor.sqlExe(
|
||||
"SELECT file_name FROM documents WHERE id=${id}$ AND kb_id=${kb_id}$",
|
||||
"SELECT file_name FROM rag_documents WHERE id=${id}$ AND kb_id=${kb_id}$",
|
||||
{"id": doc_id, "kb_id": kb_id})
|
||||
if recs:
|
||||
fname = recs[0].file_name or ''
|
||||
|
||||
# 有哪些标签
|
||||
tag_recs = await sor.sqlExe(
|
||||
"SELECT id, name, color FROM tags WHERE kb_id=${kb_id}$ ORDER BY created_at",
|
||||
"SELECT id, name, color FROM rag_tags WHERE kb_id=${kb_id}$ ORDER BY created_at",
|
||||
{"kb_id": kb_id})
|
||||
for t in tag_recs:
|
||||
all_tags.append((t.id, t.name, t.color or '#3b82f6'))
|
||||
@ -32,7 +32,7 @@ try:
|
||||
|
||||
# 当前已关联的
|
||||
mt_recs = await sor.sqlExe(
|
||||
"SELECT tag_id FROM media_tags WHERE media_type=${type}$ AND media_id=${mid}$",
|
||||
"SELECT tag_id FROM rag_media_tags WHERE media_type=${type}$ AND media_id=${mid}$",
|
||||
{"type": media_type, "mid": doc_id})
|
||||
cur_tag_ids = [r.tag_id for r in mt_recs]
|
||||
except Exception:
|
||||
|
||||
@ -3,7 +3,7 @@ env = request._run_ns
|
||||
userorgid = await env.get_userorgid()
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
recs = await sor.sqlExe(
|
||||
"SELECT id, name, color FROM tags WHERE org_id=${org_id}$ ORDER BY created_at",
|
||||
"SELECT id, name, color FROM rag_tags WHERE org_id=${org_id}$ ORDER BY created_at",
|
||||
{"org_id": userorgid})
|
||||
items = [{"value": r.id, "text": r.name, "color": r.color or '#3b82f6'} for r in recs]
|
||||
return json.dumps(items, ensure_ascii=False)
|
||||
|
||||
@ -4,7 +4,7 @@ dbname = get_module_dbname('rag')
|
||||
async with db.sqlorContext(dbname) as sor:
|
||||
id = ns.get("id", "")
|
||||
name = ns.get("name", "")
|
||||
await sor.sqlExe("UPDATE document_chunks SET content=${name}$ WHERE id=${id}$",
|
||||
await sor.sqlExe("UPDATE rag_document_chunks SET content=${name}$ WHERE id=${id}$",
|
||||
{"name": name, "id": id})
|
||||
return {"widgettype": "Message", "options": {"user_data": {"id": id, "label": name}}}
|
||||
return {"error": "failed"}
|
||||
|
||||
@ -23,9 +23,9 @@ def fmt_bytes(n):
|
||||
quota_limit = 104857600
|
||||
used = 0
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
rec = await sor.sqlExe("SELECT COALESCE(SUM(file_size),0) AS used FROM documents WHERE org_id=${org_id}$", {"org_id": userorgid})
|
||||
rec = await sor.sqlExe("SELECT COALESCE(SUM(file_size),0) AS used FROM rag_documents WHERE org_id=${org_id}$", {"org_id": userorgid})
|
||||
if rec: used = int(rec[0].used)
|
||||
lim = await sor.sqlExe("SELECT limit_bytes FROM org_storage_limits WHERE org_id=${org_id}$", {"org_id": userorgid})
|
||||
lim = await sor.sqlExe("SELECT limit_bytes FROM rag_org_storage_limits WHERE org_id=${org_id}$", {"org_id": userorgid})
|
||||
if lim: quota_limit = int(lim[0].limit_bytes)
|
||||
if used + file_size > quota_limit:
|
||||
return json.dumps({"status": "error", "error": "storage_quota_exceeded",
|
||||
@ -52,7 +52,7 @@ async def ingest_doc(doc_id, kb_id, file_name, ext_l, real_path):
|
||||
emb_engine = 'clip-vith14'
|
||||
try:
|
||||
async with db.sqlorContext('rag') as sor:
|
||||
krecs = await sor.sqlExe("SELECT embedding_engine FROM knowledge_bases WHERE id=${kb_id}$", {"kb_id": kb_id})
|
||||
krecs = await sor.sqlExe("SELECT embedding_engine FROM rag_knowledge_bases WHERE id=${kb_id}$", {"kb_id": kb_id})
|
||||
if krecs:
|
||||
emb_engine = (getattr(krecs[0], 'embedding_engine', '') or 'clip-vith14').strip()
|
||||
except: pass
|
||||
@ -118,7 +118,7 @@ async def ingest_doc(doc_id, kb_id, file_name, ext_l, real_path):
|
||||
try:
|
||||
async with db.sqlorContext('rag') as sor:
|
||||
await sor.sqlExe(
|
||||
"UPDATE documents SET status='failed', metadata=${meta}$, updated_at=NOW() WHERE id=${id}$",
|
||||
"UPDATE rag_documents SET status='failed', metadata=${meta}$, updated_at=NOW() WHERE id=${id}$",
|
||||
{"id": doc_id, "meta": json.dumps({"error": "文本知识库不支持媒体文件，请上传文本类文件(txt/md/pdf/docx等)或改用多媒体知识库"}, ensure_ascii=False)})
|
||||
except: pass
|
||||
return
|
||||
@ -195,7 +195,7 @@ async def ingest_doc(doc_id, kb_id, file_name, ext_l, real_path):
|
||||
chunk_meta["bboxes"] = frame_bboxes
|
||||
async with db.sqlorContext('rag') as sor:
|
||||
await sor.sqlExe(
|
||||
"INSERT INTO document_chunks (id, doc_id, kb_id, chunk_index, content, vector_id, metadata, created_at) "
|
||||
"INSERT INTO rag_document_chunks (id, doc_id, kb_id, chunk_index, content, vector_id, metadata, created_at) "
|
||||
"VALUES (${id}$, ${doc_id}$, ${kb_id}$, 0, ${content}$, ${vid}$, ${meta}$, NOW())",
|
||||
{"id": doc_id + "_c0", "doc_id": doc_id, "kb_id": kb_id,
|
||||
"content": file_name, "vid": doc_id + "_c0",
|
||||
@ -277,7 +277,7 @@ async def ingest_doc(doc_id, kb_id, file_name, ext_l, real_path):
|
||||
for i, chunk_text in enumerate(chunks):
|
||||
vid = vector_ids[i] if i < len(vector_ids) else ''
|
||||
await sor.sqlExe(
|
||||
"INSERT INTO document_chunks (id, doc_id, kb_id, chunk_index, content, vector_id, created_at) "
|
||||
"INSERT INTO rag_document_chunks (id, doc_id, kb_id, chunk_index, content, vector_id, created_at) "
|
||||
"VALUES (${id}$, ${doc_id}$, ${kb_id}$, ${idx}$, ${content}$, ${vid}$, NOW())",
|
||||
{"id": doc_id + "_c" + str(i), "doc_id": doc_id, "kb_id": kb_id,
|
||||
"idx": i, "content": chunk_text[:2000], "vid": vid})
|
||||
@ -286,7 +286,7 @@ async def ingest_doc(doc_id, kb_id, file_name, ext_l, real_path):
|
||||
try:
|
||||
async with db.sqlorContext('rag') as sor:
|
||||
await sor.sqlExe(
|
||||
"UPDATE documents SET status='failed', metadata=${meta}$, updated_at=NOW() WHERE id=${id}$",
|
||||
"UPDATE rag_documents SET status='failed', metadata=${meta}$, updated_at=NOW() WHERE id=${id}$",
|
||||
{"id": doc_id, "meta": json.dumps({"error": str(e)[:300]}, ensure_ascii=False)})
|
||||
except: pass
|
||||
return
|
||||
@ -296,11 +296,11 @@ async def ingest_doc(doc_id, kb_id, file_name, ext_l, real_path):
|
||||
try:
|
||||
async with db.sqlorContext('rag') as sor:
|
||||
await sor.sqlExe(
|
||||
"UPDATE documents SET status='done', chunk_count=${chunks}$, metadata=${meta}$, updated_at=NOW() WHERE id=${id}$",
|
||||
"UPDATE rag_documents SET status='done', chunk_count=${chunks}$, metadata=${meta}$, updated_at=NOW() WHERE id=${id}$",
|
||||
{"id": doc_id, "chunks": chunks_n, "meta": meta_json})
|
||||
if chunks_n:
|
||||
await sor.sqlExe(
|
||||
"UPDATE knowledge_bases SET chunk_count=chunk_count+${n}$ WHERE id=${kb_id}$",
|
||||
"UPDATE rag_knowledge_bases SET chunk_count=chunk_count+${n}$ WHERE id=${kb_id}$",
|
||||
{"n": chunks_n, "kb_id": kb_id})
|
||||
except: pass
|
||||
|
||||
@ -310,12 +310,12 @@ async def ingest_doc(doc_id, kb_id, file_name, ext_l, real_path):
|
||||
# ============================================================
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
await sor.sqlExe(
|
||||
"INSERT INTO documents (id, kb_id, folder_id, file_name, file_type, file_size, file_path, mime_type, status, chunk_count, metadata, org_id, created_at, updated_at) "
|
||||
"INSERT INTO rag_documents (id, kb_id, folder_id, file_name, file_type, file_size, file_path, mime_type, status, chunk_count, metadata, org_id, created_at, updated_at) "
|
||||
"VALUES (${id}$, ${kb_id}$, ${folder_id}$, ${file_name}$, 'other', ${file_size}$, ${file_path}$, 'application/octet-stream', 'pending', 0, '{}', ${org_id}$, NOW(), NOW())",
|
||||
{"id": doc_id, "kb_id": kb_id, "folder_id": folder_id, "file_name": file_name,
|
||||
"file_size": file_size, "file_path": web_path, "org_id": userorgid})
|
||||
await sor.sqlExe(
|
||||
"UPDATE knowledge_bases SET doc_count=doc_count+1, total_size=total_size+${size}$ WHERE id=${kb_id}$",
|
||||
"UPDATE rag_knowledge_bases SET doc_count=doc_count+1, total_size=total_size+${size}$ WHERE id=${kb_id}$",
|
||||
{"size": file_size, "kb_id": kb_id})
|
||||
|
||||
# Fire background ingestion — pass primitives only (no env/request/proxy objects)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user