diff --git a/build.sh b/build.sh index d2e7df3..8ea91b2 100755 --- a/build.sh +++ b/build.sh @@ -2,36 +2,34 @@ set -e cdir=$(cd "$(dirname "$0")"; pwd) -# Environment export SAGE_RBAC_DB=rag -# Virtualenv if [ ! -d py3 ]; then python3 -m venv py3 fi source py3/bin/activate -# Install build tools pip install -q xls2ddl -# pkgs directory mkdir -p pkgs logs files -# Clone and install foundation modules +# Foundation modules for repo in apppublic sqlor ahserver rbac appbase bricks-for-python; do [ -d pkgs/$repo ] || git clone git@git.opencomputing.cn:yumoqing/$repo.git pkgs/$repo pip install -q -e pkgs/$repo done -# Clone and install rag-pipeline (core engine - add to PYTHONPATH, not pip install) -[ -d pkgs/rag-pipeline ] || git clone git@git.opencomputing.cn:yumoqing/rag-pipeline.git pkgs/rag-pipeline -# rag-pipeline is imported directly, not via pip +# Clone and install rag business module +[ -d pkgs/rag ] || git clone git@git.opencomputing.cn:yumoqing/rag.git pkgs/rag +pip install -q -e pkgs/rag -# Clone service modules (for reference, not imported) +# rag-pipeline (core engine - PYTHONPATH only) +[ -d pkgs/rag-pipeline ] || git clone git@git.opencomputing.cn:yumoqing/rag-pipeline.git pkgs/rag-pipeline + +# Service modules (reference only) for svc in clip_embedding vdb face-service graph-service ner-service; do - [ -d pkgs/$svc ] || git clone git@git.opencomputing.cn:yumoqing/$svc.git pkgs/$svc 2>/dev/null || echo "Skip $svc (repo may not exist yet)" + [ -d pkgs/$svc ] || git clone git@git.opencomputing.cn:yumoqing/$svc.git pkgs/$svc 2>/dev/null || echo "Skip $svc" done -# bge-reranker and speaker-id repos may not exist yet [ -d pkgs/bge-reranker ] || git clone git@git.opencomputing.cn:yumoqing/bge-reranker.git pkgs/bge-reranker 2>/dev/null || echo "Skip bge-reranker" [ -d pkgs/speaker-id ] || git clone git@git.opencomputing.cn:yumoqing/speaker-id.git pkgs/speaker-id 2>/dev/null || echo "Skip speaker-id" @@ -44,39 +42,20 @@ cd pkgs/bricks bash build.sh cd $cdir -# Symlink bricks +# Symlink bricks + module wwwroots ln -sf ../pkgs/bricks/dist wwwroot/bricks 2>/dev/null || true +ln -sf ../pkgs/rbac/wwwroot wwwroot/rbac 2>/dev/null || true +ln -sf ../pkgs/rag/wwwroot rag_wwwroot 2>/dev/null || true -# Install appbase and rbac -pip install -q -e pkgs/appbase -pip install -q -e pkgs/rbac - -# Generate DDL from models -if ls models/*.json >/dev/null 2>&1; then - cd models - json2ddl mysql . > mysql.ddl.sql - mysql -h db -utest -ptest123 rag < mysql.ddl.sql 2>/dev/null || echo "DDL already applied or mysql not available" +# Generate DDL from rag module models +if ls pkgs/rag/models/*.json >/dev/null 2>&1; then + cd pkgs/rag/models + json2ddl mysql . > /tmp/rag_ddl.sql + mysql -h db -utest -ptest123 rag < /tmp/rag_ddl.sql 2>/dev/null || echo "DDL apply skipped" cd $cdir fi -# Generate CRUD from json/ -if ls json/*.json >/dev/null 2>&1; then - xls2ui -m models -o wwwroot ragserver json/*.json 2>/dev/null || true -fi - -# Symlink module wwwroots -for m in rag-pipeline; do - [ -d pkgs/$m/wwwroot ] && ln -sf ../pkgs/$m/wwwroot wwwroot/$m 2>/dev/null || true -done -# Always symlink rbac wwwroot for login pages -ln -sf ../pkgs/rbac/wwwroot wwwroot/rbac 2>/dev/null || true - -# Fix created_by in CRUD dspy files -if [ -d wwwroot/knowledge_bases_list ]; then - find wwwroot -name '*.dspy' -exec sed -i "s/ns\['created_by'\] = ''/ns['created_by'] = userorgid/" {} \; 2>/dev/null || true -fi - -# Sync password_key from Sage if available (only update key, preserve rest) +# Sync password_key from Sage if [ -f /d/apitest/sage/conf/config.json ]; then sage_key=$(python3 -c "import json; print(json.load(open('/d/apitest/sage/conf/config.json'))['password_key'])") python3 -c " @@ -85,7 +64,7 @@ c = json.load(open('$cdir/conf/config.json')) c['password_key'] = '$sage_key' json.dump(c, open('$cdir/conf/config.json', 'w'), indent=4, ensure_ascii=False) " 2>/dev/null - echo "password_key synced from Sage" + echo "password_key synced" fi echo "=== RAG Server build complete ===" diff --git a/init.py b/init.py index e7f3fda..b294728 100644 --- a/init.py +++ b/init.py @@ -1,551 +1,25 @@ # -*- coding:utf-8 -*- """ -RagServer DSPY Handlers - RAG 核心业务逻辑 + 文件上传/删除 +RagServer Shell - 前端壳应用，业务逻辑由 rag 模块提供 """ -from traceback import format_exc from ahserver.serverenv import ServerEnv from appPublic.registerfunction import RegisterFunction -from appPublic.log import debug, exception -from sqlor.dbpools import get_sor_context -import json, os, uuid, time +import json async def status_handler(request, params_kw, *args, **kwargs): return json.dumps({ - "service": "ragserver", "version": "0.1.0", - "endpoints": ["/api/status", "/api/kb/list", "/api/doc/upload", - "/api/doc/delete", "/api/search", "/api/engines", - "/api/dir/create", "/api/dir/delete", "/api/dir/list", - "/api/tag/create", "/api/tag/list", "/api/tag/delete", - "/api/tag/assign", "/api/tag/unassign", "/api/tag/media_tags", - "/api/tag/search"] + "service": "ragserver", + "version": "0.1.0", + "module": "rag" }, indent=2, ensure_ascii=False) -async def kb_list_handler(request, params_kw, *args, **kwargs): - env = request._run_ns - try: - userorgid = await env.get_userorgid() - async with get_sor_context(env, 'rag') as sor: - recs = await sor.R("knowledge_bases", {}) - cards = [] - for r in recs: - doc_recs = await sor.R("documents", {"kb_id": r.id}) - doc_count = len(doc_recs) - total_size = r.total_size or 0 - if total_size >= 1073741824: - size_str = f"{total_size/1073741824:.1f}GB" - elif total_size >= 1048576: - size_str = f"{total_size/1048576:.0f}MB" - elif total_size >= 1024: - size_str = f"{total_size/1024:.0f}KB" - else: - size_str = f"{total_size}B" - engine = r.embedding_engine or "CLIP" - card = {"widgettype":"VBox","options":{"cwidth":16,"cheight":12,"bgcolor":"#f0f7ff","padding":"16px","css":"card clickable","border":"1px solid #d0e4f7"},"subwidgets":[ - {"widgettype":"Text","options":{"text":"📚 " + str(r.name),"cfontsize":16,"fontWeight":"bold"}}, - {"widgettype":"Text","options":{"text":str(engine)+" · 1024维","cfontsize":12,"color":"#888"}}, - {"widgettype":"HBox","options":{"spacing":"12px"},"subwidgets":[ - {"widgettype":"Text","options":{"text":"📄 "+str(doc_count)+"文档","cfontsize":12,"color":"#666"}}, - {"widgettype":"Text","options":{"text":"💾 "+size_str,"cfontsize":12,"color":"#666"}}]}], - "binds":[{"wid":"self","event":"click","actiontype":"urlwidget","target":"app.rag_main_content","mode":"replace","options":{"url":"/knowledge_bases_list/detail.ui","params":{"kb_id":str(r.id),"kb_name":str(r.name)}}}]} - cards.append(card) - if not cards: - cards.append({"widgettype":"Text","options":{"text":"暂无知识库","color":"#aaa","cfontsize":14,"halign":"center"}}) - result = {"widgettype":"HBox","options":{"width":"100%","spacing":"12px","wrap":True},"subwidgets":cards} - return json.dumps(result, ensure_ascii=False) - except Exception as e: - exception(f"kb_list: {e}, {format_exc()}") - return json.dumps({"error": str(e)}) - - -async def engines_handler(request, params_kw, *args, **kwargs): - env = request._run_ns - try: - userorgid = await env.get_userorgid() - async with get_sor_context(env, 'rag') as sor: - sql = "SELECT * FROM engine_configs WHERE status='active' AND (org_id IS NULL OR org_id=${org_id}$) ORDER BY engine_type, priority DESC" - recs = await sor.sqlExe(sql, {"org_id": userorgid}) - rows = [dict(r) for r in recs] - return json.dumps({"status": "SUCCEEDED", "data": {"rows": rows, "total": len(rows)}}, ensure_ascii=False, default=str) - except Exception as e: - exception(f"engines: {e}, {format_exc()}") - return json.dumps({"error": str(e)}) - - -async def search_handler(request, params_kw, *args, **kwargs): - env = request._run_ns - try: - query = params_kw.get("query", "") - kb_id = params_kw.get("kb_id", "") - top_k = int(params_kw.get("top_k", 5)) - if not query: - return json.dumps({"error": "query required"}) - try: - from pipeline import search as pipeline_search - result = pipeline_search(query, pipeline_name="kg-rag-standard", - collection=kb_id or "knowledge", - graph_name=kb_id or "knowledge", - top_k=top_k, llm_func=None) - return json.dumps(result, ensure_ascii=False) - except ImportError: - return json.dumps({"status": "FALLBACK", "message": "pipeline not available"}) - except Exception as e: - exception(f"search: {e}, {format_exc()}") - return json.dumps({"error": str(e)}) - - -async def doc_upload_handler(request, params_kw, *args, **kwargs): - """文件上传 → 保存 → DB记录 → 触发RAG入库""" - env = request._run_ns - try: - userorgid = await env.get_userorgid() - kb_id = params_kw.get("kb_id", "") - file_name = params_kw.get("file_name", "upload.bin") - if not kb_id: - return json.dumps({"error": "kb_id required"}) - - # Read raw file data from request body - file_data = await request.read() - - if not file_data: - return json.dumps({"error": "no file data"}) - - # Save file - doc_id = uuid.uuid4().hex[:16] - ext = os.path.splitext(file_name)[1] or ".bin" - saved_name = f"{doc_id}{ext}" - files_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "files") - os.makedirs(files_dir, exist_ok=True) - file_path = os.path.join(files_dir, saved_name) - with open(file_path, "wb") as f: - f.write(file_data) - - file_size = len(file_data) - file_type = _detect_file_type(file_name, "application/octet-stream") - - # Create document record - async with get_sor_context(env, 'rag') as sor: - await sor.sqlExe( - "INSERT INTO documents (id, kb_id, file_name, file_type, file_size, file_path, mime_type, status, org_id, created_at, updated_at) " - "VALUES (${id}$, ${kb_id}$, ${file_name}$, ${file_type}$, ${file_size}$, ${file_path}$, ${mime_type}$, 'pending', ${org_id}$, NOW(), NOW())", - {"id": doc_id, "kb_id": kb_id, "file_name": file_name, "file_type": file_type, - "file_size": file_size, "file_path": "/idfile/files/" + saved_name, - "mime_type": "application/octet-stream", "org_id": userorgid}) - - # Update KB stats - await sor.sqlExe( - "UPDATE knowledge_bases SET doc_count=doc_count+1, total_size=total_size+${size}$ WHERE id=${kb_id}$", - {"size": file_size, "kb_id": kb_id}) - - # Trigger async ingest for text-based files - ingest_result = None - if file_type == "text": - try: - text = file_data.decode("utf-8", errors="replace") - from pipeline import ingest as pipeline_ingest - ingest_result = pipeline_ingest( - text, pipeline_name="kg-rag-standard", - collection=kb_id, graph_name=kb_id, llm_func=None) - # Update status to done - async with get_sor_context(env, 'rag') as sor: - await sor.sqlExe( - "UPDATE documents SET status='done', chunk_count=${chunks}$ WHERE id=${id}$", - {"chunks": ingest_result.get("chunks", 0) if ingest_result else 0, "id": doc_id}) - if ingest_result: - await sor.sqlExe( - "UPDATE knowledge_bases SET chunk_count=chunk_count+${n}$ WHERE id=${kb_id}$", - {"n": ingest_result.get("chunks", 0), "kb_id": kb_id}) - except Exception as e: - exception(f"async ingest failed: {e}") - - return json.dumps({ - "status": "SUCCEEDED", - "doc_id": doc_id, - "file_name": file_name, - "file_size": file_size, - "file_type": file_type, - "ingest": ingest_result - }, ensure_ascii=False, default=str) - - except Exception as e: - exception(f"doc_upload: {e}, {format_exc()}") - return json.dumps({"error": str(e)}) - - -async def doc_delete_handler(request, params_kw, *args, **kwargs): - """文件删除 → 清理VDB → 清理图 → 清理DB → 删文件""" - env = request._run_ns - try: - doc_id = params_kw.get("doc_id", "") - if not doc_id: - return json.dumps({"error": "doc_id required"}) - - async with get_sor_context(env, 'rag') as sor: - # Get document info - recs = await sor.R("documents", {"id": doc_id}) - if not recs: - return json.dumps({"error": "document not found"}) - doc = recs[0] - - # Get chunks to clean VDB - chunks = await sor.R("document_chunks", {"doc_id": doc_id}) - - # Delete from VDB - if chunks: - vector_ids = [c.vector_id for c in chunks if c.vector_id] - if vector_ids: - try: - _call_vdb("/v1/delete", {"colname": doc.kb_id, "ids": vector_ids}) - except Exception as e: - exception(f"vdb delete failed: {e}") - - # Delete from graph - entities = await sor.R("entities", {"kb_id": doc.kb_id}) - if entities: - try: - _call_graph("/api/graph/save", {"graph": doc.kb_id}) - except Exception as e: - exception(f"graph cleanup failed: {e}") - - # Delete DB records - await sor.sqlExe("DELETE FROM document_chunks WHERE doc_id=${id}$", {"id": doc_id}) - await sor.sqlExe("DELETE FROM entities WHERE kb_id=${kb_id}$", {"kb_id": doc.kb_id}) - await sor.sqlExe("DELETE FROM entity_relations WHERE kb_id=${kb_id}$", {"kb_id": doc.kb_id}) - await sor.sqlExe("DELETE FROM documents WHERE id=${id}$", {"id": doc_id}) - - # Update KB stats - await sor.sqlExe( - "UPDATE knowledge_bases SET doc_count=GREATEST(doc_count-1,0), total_size=GREATEST(total_size-${size}$,0), chunk_count=GREATEST(chunk_count-${n}$,0) WHERE id=${kb_id}$", - {"size": doc.file_size, "n": len(chunks), "kb_id": doc.kb_id}) - - # Delete file from disk - file_path = doc.file_path - if file_path and file_path.startswith("/idfile/"): - real_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "files", - os.path.basename(file_path)) - if os.path.exists(real_path): - os.remove(real_path) - - return json.dumps({"status": "SUCCEEDED", "doc_id": doc_id, "chunks_deleted": len(chunks)}) - - except Exception as e: - exception(f"doc_delete: {e}, {format_exc()}") - return json.dumps({"error": str(e)}) - - -def _detect_file_type(name, mime): - """Detect file type from name and MIME""" - ext = os.path.splitext(name)[1].lower() - if ext in ('.txt', '.md', '.json', '.csv', '.xml', '.html', '.py', '.js', '.css', '.yaml', '.yml'): - return "text" - if ext in ('.jpg', '.jpeg', '.png', '.gif', '.bmp', '.webp', '.svg'): - return "image" - if ext in ('.mp3', '.wav', '.flac', '.ogg', '.m4a', '.aac'): - return "audio" - if ext in ('.mp4', '.avi', '.mov', '.mkv', '.webm'): - return "video" - if ext == '.pdf': - return "text" - return "other" - - -def _call_vdb(path, data, timeout=10): - """Call VDB service""" - import urllib.request - url = f"http://localhost:8886{path}" - req = urllib.request.Request(url, data=json.dumps(data).encode(), - headers={"Content-Type": "application/json"}) - with urllib.request.urlopen(req, timeout=timeout) as resp: - return json.loads(resp.read()) - - -def _call_graph(path, data, timeout=10): - """Call Graph service""" - import urllib.request - url = f"http://localhost:9092{path}" - req = urllib.request.Request(url, data=json.dumps(data).encode(), - headers={"Content-Type": "application/json"}) - with urllib.request.urlopen(req, timeout=timeout) as resp: - return json.loads(resp.read()) - - -async def dir_create_handler(request, params_kw, *args, **kwargs): - """创建目录""" - env = request._run_ns - try: - kb_id = params_kw.get("kb_id", "") - parent_id = params_kw.get("parent_id", "") - dir_name = params_kw.get("dir_name", "") - if not kb_id or not dir_name: - return json.dumps({"error": "kb_id and dir_name required"}) - async with get_sor_context(env, 'rag') as sor: - dir_id = uuid.uuid4().hex[:16] - await sor.sqlExe( - "INSERT INTO document_chunks (id, doc_id, kb_id, chunk_index, chunk_type, content, description, created_at) " - "VALUES (${id}$, '', ${kb_id}$, 0, 'directory', ${name}$, ${parent}$, NOW())", - {"id": dir_id, "kb_id": kb_id, "name": dir_name, "parent": parent_id}) - return json.dumps({"status": "SUCCEEDED", "dir_id": dir_id}) - except Exception as e: - exception(f"dir_create: {e}") - return json.dumps({"error": str(e)}) - - -async def dir_delete_handler(request, params_kw, *args, **kwargs): - """删除目录/文件""" - env = request._run_ns - try: - item_id = params_kw.get("item_id", "") - if not item_id: - return json.dumps({"error": "item_id required"}) - async with get_sor_context(env, 'rag') as sor: - await sor.sqlExe("DELETE FROM document_chunks WHERE id=${id}$ OR doc_id=${id}$", {"id": item_id}) - return json.dumps({"status": "SUCCEEDED"}) - except Exception as e: - exception(f"dir_delete: {e}") - return json.dumps({"error": str(e)}) - - -async def dir_list_handler(request, params_kw, *args, **kwargs): - """列出知识库的目录树""" - env = request._run_ns - try: - kb_id = params_kw.get("kb_id", "") - if not kb_id: - return json.dumps({"error": "kb_id required"}) - async with get_sor_context(env, 'rag') as sor: - # Get directories (chunk_type='directory') - dirs = await sor.sqlExe( - "SELECT id, content as label, description as parent_id FROM document_chunks WHERE kb_id=${kb_id}$ AND chunk_type='directory'", - {"kb_id": kb_id}) - # Get files (documents table) - docs = await sor.sqlExe( - "SELECT id, file_name as label, '' as parent_id FROM documents WHERE kb_id=${kb_id}$", - {"kb_id": kb_id}) - items = [] - for d in dirs: - items.append({"id": d.id, "label": d.label, "parent_id": d.parent_id or "", "type": "dir"}) - for d in docs: - items.append({"id": d.id, "label": d.label, "parent_id": "", "type": "file"}) - return json.dumps({"status": "SUCCEEDED", "items": items}) - except Exception as e: - exception(f"dir_list: {e}") - return json.dumps({"error": str(e)}) - - -async def tag_create_handler(request, params_kw, *args, **kwargs): - """创建标签""" - env = request._run_ns - try: - kb_id = params_kw.get("kb_id", "") - name = params_kw.get("name", "").strip() - color = params_kw.get("color", "#3b82f6") - if not kb_id or not name: - return json.dumps({"error": "kb_id and name required"}) - userorgid = await env.get_userorgid() - async with get_sor_context(env, 'rag') as sor: - tag_id = uuid.uuid4().hex[:16] - await sor.sqlExe( - "INSERT INTO tags (id, kb_id, name, color, org_id, created_at) " - "VALUES (${id}$, ${kb_id}$, ${name}$, ${color}$, ${org_id}$, NOW())", - {"id": tag_id, "kb_id": kb_id, "name": name, "color": color, "org_id": userorgid}) - return json.dumps({"status": "SUCCEEDED", "tag_id": tag_id, "name": name, "color": color}) - except Exception as e: - exception(f"tag_create: {e}, {format_exc()}") - return json.dumps({"error": str(e)}) - - -async def tag_list_handler(request, params_kw, *args, **kwargs): - """列出知识库的所有标签""" - env = request._run_ns - try: - kb_id = params_kw.get("kb_id", "") - if not kb_id: - return json.dumps({"error": "kb_id required"}) - userorgid = await env.get_userorgid() - async with get_sor_context(env, 'rag') as sor: - recs = await sor.R("tags", {"kb_id": kb_id, "org_id": userorgid}) - tags = [{"id": r.id, "name": r.name, "color": r.color, "created_at": str(r.created_at)} for r in recs] - return json.dumps({"status": "SUCCEEDED", "tags": tags}, ensure_ascii=False, default=str) - except Exception as e: - exception(f"tag_list: {e}, {format_exc()}") - return json.dumps({"error": str(e)}) - - -async def tag_delete_handler(request, params_kw, *args, **kwargs): - """删除标签（级联删除关联）""" - env = request._run_ns - try: - tag_id = params_kw.get("tag_id", "") - if not tag_id: - return json.dumps({"error": "tag_id required"}) - async with get_sor_context(env, 'rag') as sor: - await sor.sqlExe("DELETE FROM media_tags WHERE tag_id=${id}$", {"id": tag_id}) - await sor.sqlExe("DELETE FROM tags WHERE id=${id}$", {"id": tag_id}) - return json.dumps({"status": "SUCCEEDED", "tag_id": tag_id}) - except Exception as e: - exception(f"tag_delete: {e}, {format_exc()}") - return json.dumps({"error": str(e)}) - - -async def tag_assign_handler(request, params_kw, *args, **kwargs): - """给媒体/人脸/声纹打标签""" - env = request._run_ns - try: - kb_id = params_kw.get("kb_id", "") - media_type = params_kw.get("media_type", "") # document / face / voice - media_id = params_kw.get("media_id", "") - tag_id = params_kw.get("tag_id", "") - if not all([kb_id, media_type, media_id, tag_id]): - return json.dumps({"error": "kb_id, media_type, media_id, tag_id required"}) - if media_type not in ("document", "face", "voice"): - return json.dumps({"error": "media_type must be document/face/voice"}) - async with get_sor_context(env, 'rag') as sor: - mt_id = uuid.uuid4().hex[:16] - await sor.sqlExe( - "INSERT INTO media_tags (id, kb_id, media_type, media_id, tag_id, created_at) " - "VALUES (${id}$, ${kb_id}$, ${type}$, ${mid}$, ${tid}$, NOW())", - {"id": mt_id, "kb_id": kb_id, "type": media_type, "mid": media_id, "tid": tag_id}) - return json.dumps({"status": "SUCCEEDED", "media_tag_id": mt_id}) - except Exception as e: - exception(f"tag_assign: {e}, {format_exc()}") - return json.dumps({"error": str(e)}) - - -async def tag_unassign_handler(request, params_kw, *args, **kwargs): - """取消标签关联""" - env = request._run_ns - try: - media_type = params_kw.get("media_type", "") - media_id = params_kw.get("media_id", "") - tag_id = params_kw.get("tag_id", "") - if not all([media_type, media_id, tag_id]): - return json.dumps({"error": "media_type, media_id, tag_id required"}) - async with get_sor_context(env, 'rag') as sor: - await sor.sqlExe( - "DELETE FROM media_tags WHERE media_type=${type}$ AND media_id=${mid}$ AND tag_id=${tid}$", - {"type": media_type, "mid": media_id, "tid": tag_id}) - return json.dumps({"status": "SUCCEEDED"}) - except Exception as e: - exception(f"tag_unassign: {e}, {format_exc()}") - return json.dumps({"error": str(e)}) - - -async def tag_media_tags_handler(request, params_kw, *args, **kwargs): - """查询某媒体的所有标签""" - env = request._run_ns - try: - media_type = params_kw.get("media_type", "") - media_id = params_kw.get("media_id", "") - if not all([media_type, media_id]): - return json.dumps({"error": "media_type and media_id required"}) - async with get_sor_context(env, 'rag') as sor: - recs = await sor.sqlExe( - "SELECT t.id, t.name, t.color FROM media_tags mt " - "JOIN tags t ON mt.tag_id=t.id " - "WHERE mt.media_type=${type}$ AND mt.media_id=${mid}$", - {"type": media_type, "mid": media_id}) - tags = [{"id": r.id, "name": r.name, "color": r.color} for r in recs] - return json.dumps({"status": "SUCCEEDED", "tags": tags}) - except Exception as e: - exception(f"tag_media_tags: {e}, {format_exc()}") - return json.dumps({"error": str(e)}) - - -async def tag_search_handler(request, params_kw, *args, **kwargs): - """组合标签检索：按 tag_ids 过滤，再语义检索""" - env = request._run_ns - try: - query = params_kw.get("query", "") - kb_id = params_kw.get("kb_id", "") - tag_ids_str = params_kw.get("tag_ids", "") # comma-separated tag IDs - top_k = int(params_kw.get("top_k", 5)) - match_mode = params_kw.get("match_mode", "any") # any / all - if not kb_id: - return json.dumps({"error": "kb_id required"}) - async with get_sor_context(env, 'rag') as sor: - if tag_ids_str: - tag_ids = [t.strip() for t in tag_ids_str.split(",") if t.strip()] - media_ids_by_tag = [] - for tid in tag_ids: - recs = await sor.sqlExe( - "SELECT media_type, media_id FROM media_tags WHERE kb_id=${kb_id}$ AND tag_id=${tid}$", - {"kb_id": kb_id, "tid": tid}) - mids = {(r.media_type, r.media_id) for r in recs} - media_ids_by_tag.append(mids) - if match_mode == "all": - matched = media_ids_by_tag[0] - for s in media_ids_by_tag[1:]: - matched = matched & s - else: - matched = set() - for s in media_ids_by_tag: - matched |= s - if not matched: - return json.dumps({"status": "SUCCEEDED", "results": [], "message": "no media match tags"}) - doc_ids = [mid for mt, mid in matched if mt == "document"] - face_ids = [mid for mt, mid in matched if mt == "face"] - voice_ids = [mid for mt, mid in matched if mt == "voice"] - results = [] - if doc_ids: - docs = await sor.sqlExe( - "SELECT id, file_name, file_type, file_size, status, created_at FROM documents WHERE id IN (${ids}$)", - {"ids": doc_ids}) - for d in docs: - results.append({"type": "document", "id": d.id, "name": d.file_name, "file_type": d.file_type, "size": d.file_size, "status": d.status, "created_at": str(d.created_at)}) - if face_ids: - faces = await sor.sqlExe( - "SELECT id, name, description, face_embedding_id, created_at FROM entities WHERE id IN (${ids}$) AND entity_type='person'", - {"ids": face_ids}) - for f in faces: - results.append({"type": "face", "id": f.id, "name": f.name, "description": f.description, "created_at": str(f.created_at)}) - if voice_ids: - voices = await sor.sqlExe( - "SELECT id, name, description, voice_embedding_id, created_at FROM entities WHERE id IN (${ids}$) AND entity_type='voice'", - {"ids": voice_ids}) - for v in voices: - results.append({"type": "voice", "id": v.id, "name": v.name, "description": v.description, "created_at": str(v.created_at)}) - if query: - import urllib.request - tag_filtered_docs = [r for r in results if r["type"] == "document"] - vdb_docs = [] - for doc in tag_filtered_docs: - chunks = await sor.sqlExe( - "SELECT content FROM document_chunks WHERE doc_id=${id}$ LIMIT 3", - {"id": doc["id"]}) - for c in chunks: - vdb_docs.append({"doc_id": doc["id"], "content": c.content}) - results = {"documents": [{"id": r["id"], "name": r["name"], "file_type": r["file_type"]} for r in tag_filtered_docs], - "faces": [r for r in results if r["type"] == "face"], - "voices": [r for r in results if r["type"] == "voice"], - "chunks": vdb_docs, - "query": query, - "tag_ids": tag_ids, - "match_mode": match_mode} - return json.dumps({"status": "SUCCEEDED", "results": results, "tag_ids": tag_ids, "match_mode": match_mode}, ensure_ascii=False, default=str) - else: - return json.dumps({"status": "SUCCEEDED", "results": [], "message": "no tag_ids provided"}) - except Exception as e: - exception(f"tag_search: {e}, {format_exc()}") - return json.dumps({"error": str(e)}) - - def init_ragserver(): env = ServerEnv() rf = RegisterFunction() rf.register("status", status_handler) - rf.register("kb_list", kb_list_handler) - rf.register("engines", engines_handler) - rf.register("search", search_handler) - rf.register("doc_upload", doc_upload_handler) - rf.register("doc_delete", doc_delete_handler) - rf.register("dir_create", dir_create_handler) - rf.register("dir_delete", dir_delete_handler) - rf.register("dir_list", dir_list_handler) - rf.register("tag_create", tag_create_handler) - rf.register("tag_list", tag_list_handler) - rf.register("tag_delete", tag_delete_handler) - rf.register("tag_assign", tag_assign_handler) - rf.register("tag_unassign", tag_unassign_handler) - rf.register("tag_media_tags", tag_media_tags_handler) - rf.register("tag_search", tag_search_handler) + + # 加载 rag 业务模块 + from rag.init import init_rag_module + init_rag_module() diff --git a/json/documents.json b/json/documents.json deleted file mode 100644 index 937f7d1..0000000 --- a/json/documents.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "tblname": "documents", - "params": { - "title": "文档管理", - "browserfields": [ - {"field": "file_name", "title": "文件名", "width": "25%"}, - {"field": "file_type", "title": "类型", "width": "10%"}, - {"field": "file_size", "title": "大小", "width": "10%"}, - {"field": "chunk_count", "title": "分片数", "width": "8%"}, - {"field": "status", "title": "状态", "width": "10%"}, - {"field": "kb_id", "title": "知识库", "width": "17%"}, - {"field": "created_at", "title": "上传时间", "width": "20%"} - ], - "editfields": [ - {"field": "file_name", "uitype": "Text", "required": true}, - {"field": "kb_id", "uitype": "Text", "required": true}, - {"field": "file_type", "uitype": "Text"} - ], - "searchfields": ["file_name"], - "sort": "created_at desc", - "data_filter": { - "org_id": "{{userorgid}}" - } - } -} diff --git a/json/engine_configs.json b/json/engine_configs.json deleted file mode 100644 index c338fb2..0000000 --- a/json/engine_configs.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "tblname": "engine_configs", - "params": { - "title": "引擎配置", - "browserfields": [ - {"field": "engine_type", "title": "引擎类型", "width": "15%"}, - {"field": "engine_name", "title": "名称", "width": "15%"}, - {"field": "endpoint_url", "title": "服务地址", "width": "25%"}, - {"field": "model_name", "title": "模型", "width": "15%"}, - {"field": "is_default", "title": "默认", "width": "8%"}, - {"field": "status", "title": "状态", "width": "10%"}, - {"field": "org_id", "title": "机构", "width": "12%"} - ], - "editfields": [ - {"field": "engine_type", "uitype": "Text", "required": true}, - {"field": "engine_name", "uitype": "Text", "required": true}, - {"field": "endpoint_url", "uitype": "Text"}, - {"field": "api_key", "uitype": "Text"}, - {"field": "model_name", "uitype": "Text"}, - {"field": "is_default", "uitype": "Text"}, - {"field": "config_json", "uitype": "Text"} - ], - "searchfields": ["engine_type", "engine_name", "endpoint_url"], - "sort": "engine_type, priority desc" - } -} diff --git a/json/knowledge_bases.json b/json/knowledge_bases.json deleted file mode 100644 index 99482d5..0000000 --- a/json/knowledge_bases.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "tblname": "knowledge_bases", - "params": { - "title": "知识库管理", - "browserfields": [ - {"field": "name", "title": "名称", "width": "20%"}, - {"field": "description", "title": "描述", "width": "30%"}, - {"field": "doc_count", "title": "文档数", "width": "10%"}, - {"field": "total_size", "title": "大小", "width": "10%"}, - {"field": "embedding_engine", "title": "向量引擎", "width": "15%"}, - {"field": "created_at", "title": "创建时间", "width": "15%"} - ], - "editfields": [ - {"field": "name", "uitype": "Text", "required": true}, - {"field": "description", "uitype": "Text"}, - {"field": "embedding_engine", "uitype": "Text", "default": "clip-vith14"}, - {"field": "vdb_collection", "uitype": "Text"}, - {"field": "graph_name", "uitype": "Text"} - ], - "searchfields": ["name", "description"], - "sort": "created_at desc", - "data_filter": { - "org_id": "{{userorgid}}" - } - } -} diff --git a/json/subscriptions.json b/json/subscriptions.json deleted file mode 100644 index 53f7ae5..0000000 --- a/json/subscriptions.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "tblname": "subscriptions", - "params": { - "title": "订阅管理", - "browserfields": [ - {"field": "org_id", "title": "机构", "width": "15%"}, - {"field": "plan_name", "title": "套餐", "width": "15%"}, - {"field": "disk_quota_bytes", "title": "磁盘配额", "width": "12%"}, - {"field": "disk_used_bytes", "title": "已用磁盘", "width": "12%"}, - {"field": "doc_used", "title": "文档数", "width": "10%"}, - {"field": "start_date", "title": "开始", "width": "12%"}, - {"field": "end_date", "title": "结束", "width": "12%"}, - {"field": "status", "title": "状态", "width": "12%"} - ], - "editfields": [ - {"field": "org_id", "uitype": "Text", "required": true}, - {"field": "plan_name", "uitype": "Text", "required": true}, - {"field": "disk_quota_bytes", "uitype": "Text", "default": "1073741824"}, - {"field": "doc_quota", "uitype": "Text"}, - {"field": "kb_quota", "uitype": "Text"}, - {"field": "api_call_quota", "uitype": "Text"}, - {"field": "start_date", "uitype": "Text"}, - {"field": "end_date", "uitype": "Text"} - ], - "searchfields": ["org_id", "plan_name"], - "sort": "created_at desc" - } -} diff --git a/json/tags.json b/json/tags.json deleted file mode 100644 index acdf5dd..0000000 --- a/json/tags.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "tblname": "tags", - "params": { - "title": "标签管理", - "browserfields": [ - {"field": "name", "title": "标签名", "width": "25%"}, - {"field": "color", "title": "颜色", "width": "15%"}, - {"field": "kb_id", "title": "知识库", "width": "30%"}, - {"field": "created_at", "title": "创建时间", "width": "30%"} - ], - "editfields": [ - {"field": "name", "uitype": "Text", "required": true, "label": "标签名"}, - {"field": "kb_id", "uitype": "Text", "required": true, "label": "知识库ID"}, - {"field": "color", "uitype": "Text", "default": "#3b82f6", "label": "颜色(#hex)"} - ], - "searchfields": ["name"], - "sort": "created_at desc", - "data_filter": { - "org_id": "{{userorgid}}" - } - } -} diff --git a/models/document_chunks.json b/models/document_chunks.json deleted file mode 100644 index c545831..0000000 --- a/models/document_chunks.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "summary": [ - { - "name": "document_chunks", - "title": "文档分片", - "primary": ["id"] - } - ], - "fields": [ - {"name": "id", "title": "ID", "type": "str", "length": 32}, - {"name": "doc_id", "title": "文档ID", "type": "str", "length": 32}, - {"name": "kb_id", "title": "知识库ID", "type": "str", "length": 32}, - {"name": "chunk_index", "title": "分片序号", "type": "int", "default": 0}, - {"name": "chunk_type", "title": "分片类型", "type": "str", "length": 32}, - {"name": "content", "title": "文本内容", "type": "text", "length": 65535}, - {"name": "description", "title": "描述", "type": "text", "length": 65535}, - {"name": "vector_id", "title": "向量库ID", "type": "str", "length": 128}, - {"name": "start_offset", "title": "起始偏移", "type": "int", "default": 0}, - {"name": "end_offset", "title": "结束偏移", "type": "int", "default": 0}, - {"name": "time_start", "title": "时间起始(秒)", "type": "float", "length": 10, "dec": 3}, - {"name": "time_end", "title": "时间结束(秒)", "type": "float", "length": 10, "dec": 3}, - {"name": "metadata", "title": "元数据JSON", "type": "text", "length": 2000}, - {"name": "created_at", "title": "创建时间", "type": "datetime"} - ], - "indexes": [ - {"name": "idx_chunk_doc", "idxtype": "index", "idxfields": ["doc_id"]}, - {"name": "idx_chunk_kb", "idxtype": "index", "idxfields": ["kb_id"]}, - {"name": "idx_chunk_vector", "idxtype": "index", "idxfields": ["vector_id"]} - ] -} diff --git a/models/documents.json b/models/documents.json deleted file mode 100644 index ff04880..0000000 --- a/models/documents.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "summary": [ - { - "name": "documents", - "title": "文档", - "primary": ["id"] - } - ], - "fields": [ - {"name": "id", "title": "ID", "type": "str", "length": 32}, - {"name": "kb_id", "title": "知识库ID", "type": "str", "length": 32}, - {"name": "file_name", "title": "文件名", "type": "str", "length": 255}, - {"name": "file_type", "title": "文件类型", "type": "str", "length": 32}, - {"name": "file_size", "title": "文件大小", "type": "int", "default": 0}, - {"name": "file_path", "title": "文件路径", "type": "str", "length": 500}, - {"name": "mime_type", "title": "MIME类型", "type": "str", "length": 128}, - {"name": "chunk_count", "title": "分片数", "type": "int", "default": 0}, - {"name": "embed_dim", "title": "向量维度", "type": "int", "default": 1024}, - {"name": "metadata", "title": "元数据JSON", "type": "text", "length": 4000}, - {"name": "status", "title": "状态", "type": "str", "length": 16, "default": "pending"}, - {"name": "error_msg", "title": "错误信息", "type": "text", "length": 2000}, - {"name": "org_id", "title": "所属机构", "type": "str", "length": 32}, - {"name": "created_by", "title": "创建人", "type": "str", "length": 32}, - {"name": "created_at", "title": "创建时间", "type": "datetime"}, - {"name": "updated_at", "title": "更新时间", "type": "datetime"} - ], - "codes": [ - {"field": "file_type", "table": "appcodes_kv", "valuefield": "k", "textfield": "v", "cond": "parentid='file_type'"}, - {"field": "status", "table": "appcodes_kv", "valuefield": "k", "textfield": "v", "cond": "parentid='doc_status'"}, - {"field": "kb_id", "table": "knowledge_bases", "valuefield": "id", "textfield": "name"} - ], - "indexes": [ - {"name": "idx_doc_kb", "idxtype": "index", "idxfields": ["kb_id"]}, - {"name": "idx_doc_org", "idxtype": "index", "idxfields": ["org_id"]}, - {"name": "idx_doc_status", "idxtype": "index", "idxfields": ["status"]} - ] -} diff --git a/models/engine_configs.json b/models/engine_configs.json deleted file mode 100644 index be5dfe7..0000000 --- a/models/engine_configs.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "summary": [ - { - "name": "engine_configs", - "title": "引擎配置", - "primary": ["id"] - } - ], - "fields": [ - {"name": "id", "title": "ID", "type": "str", "length": 32}, - {"name": "org_id", "title": "机构ID", "type": "str", "length": 32, "nullable": "yes"}, - {"name": "engine_type", "title": "引擎类型", "type": "str", "length": 32}, - {"name": "engine_name", "title": "引擎名称", "type": "str", "length": 64}, - {"name": "endpoint_url", "title": "服务地址", "type": "str", "length": 255}, - {"name": "api_key", "title": "API密钥", "type": "str", "length": 255}, - {"name": "model_name", "title": "模型名称", "type": "str", "length": 128}, - {"name": "config_json", "title": "扩展配置JSON", "type": "text", "length": 4000}, - {"name": "is_default", "title": "是否默认", "type": "int", "default": 0}, - {"name": "priority", "title": "优先级", "type": "int", "default": 0}, - {"name": "status", "title": "状态", "type": "str", "length": 16, "default": "active"}, - {"name": "health_check_at", "title": "最近健康检查", "type": "datetime"}, - {"name": "created_at", "title": "创建时间", "type": "datetime"}, - {"name": "updated_at", "title": "更新时间", "type": "datetime"} - ], - "codes": [ - {"field": "engine_type", "table": "appcodes_kv", "valuefield": "k", "textfield": "v", "cond": "parentid='engine_type'"}, - {"field": "status", "table": "appcodes_kv", "valuefield": "k", "textfield": "v", "cond": "parentid='engine_status'"} - ], - "indexes": [ - {"name": "idx_eng_type", "idxtype": "index", "idxfields": ["engine_type", "is_default"]}, - {"name": "idx_eng_org", "idxtype": "index", "idxfields": ["org_id", "engine_type"]} - ] -} diff --git a/models/entities.json b/models/entities.json deleted file mode 100644 index 68b896a..0000000 --- a/models/entities.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "summary": [ - { - "name": "entities", - "title": "实体", - "primary": ["id"] - } - ], - "fields": [ - {"name": "id", "title": "ID", "type": "str", "length": 32}, - {"name": "kb_id", "title": "知识库ID", "type": "str", "length": 32}, - {"name": "name", "title": "实体名", "type": "str", "length": 255}, - {"name": "entity_type", "title": "实体类型", "type": "str", "length": 64}, - {"name": "description", "title": "描述", "type": "text", "length": 2000}, - {"name": "graph_node_id", "title": "图节点ID", "type": "str", "length": 128}, - {"name": "source_doc_ids", "title": "来源文档ID列表", "type": "text", "length": 2000}, - {"name": "face_embedding_id", "title": "人脸特征ID", "type": "str", "length": 128}, - {"name": "voice_embedding_id", "title": "声纹特征ID", "type": "str", "length": 128}, - {"name": "metadata", "title": "元数据JSON", "type": "text", "length": 2000}, - {"name": "org_id", "title": "所属机构", "type": "str", "length": 32}, - {"name": "created_at", "title": "创建时间", "type": "datetime"}, - {"name": "updated_at", "title": "更新时间", "type": "datetime"} - ], - "codes": [ - {"field": "entity_type", "table": "appcodes_kv", "valuefield": "k", "textfield": "v", "cond": "parentid='entity_type'"}, - {"field": "kb_id", "table": "knowledge_bases", "valuefield": "id", "textfield": "name"} - ], - "indexes": [ - {"name": "idx_entity_kb", "idxtype": "index", "idxfields": ["kb_id"]}, - {"name": "idx_entity_name", "idxtype": "index", "idxfields": ["kb_id", "name"]}, - {"name": "idx_entity_type", "idxtype": "index", "idxfields": ["kb_id", "entity_type"]}, - {"name": "idx_entity_face", "idxtype": "index", "idxfields": ["face_embedding_id"]}, - {"name": "idx_entity_voice", "idxtype": "index", "idxfields": ["voice_embedding_id"]} - ] -} diff --git a/models/entity_relations.json b/models/entity_relations.json deleted file mode 100644 index 243b31e..0000000 --- a/models/entity_relations.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "summary": [ - { - "name": "entity_relations", - "title": "实体关系", - "primary": ["id"] - } - ], - "fields": [ - {"name": "id", "title": "ID", "type": "str", "length": 32}, - {"name": "kb_id", "title": "知识库ID", "type": "str", "length": 32}, - {"name": "subject_id", "title": "主体实体ID", "type": "str", "length": 32}, - {"name": "object_id", "title": "客体实体ID", "type": "str", "length": 32}, - {"name": "relation_type", "title": "关系类型", "type": "str", "length": 64}, - {"name": "description", "title": "关系描述", "type": "text", "length": 2000}, - {"name": "graph_edge_id", "title": "图边ID", "type": "str", "length": 128}, - {"name": "source_chunk_ids", "title": "来源分片ID列表", "type": "text", "length": 2000}, - {"name": "org_id", "title": "所属机构", "type": "str", "length": 32}, - {"name": "created_at", "title": "创建时间", "type": "datetime"} - ], - "codes": [ - {"field": "kb_id", "table": "knowledge_bases", "valuefield": "id", "textfield": "name"} - ], - "indexes": [ - {"name": "idx_rel_kb", "idxtype": "index", "idxfields": ["kb_id"]}, - {"name": "idx_rel_subject", "idxtype": "index", "idxfields": ["subject_id"]}, - {"name": "idx_rel_object", "idxtype": "index", "idxfields": ["object_id"]} - ] -} diff --git a/models/knowledge_bases.json b/models/knowledge_bases.json deleted file mode 100644 index 0ea0967..0000000 --- a/models/knowledge_bases.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "summary": [ - { - "name": "knowledge_bases", - "title": "知识库", - "primary": ["id"] - } - ], - "fields": [ - {"name": "id", "title": "ID", "type": "str", "length": 32}, - {"name": "name", "title": "名称", "type": "str", "length": 100}, - {"name": "description", "title": "描述", "type": "text", "length": 2000}, - {"name": "org_id", "title": "所属机构", "type": "str", "length": 32}, - {"name": "embedding_engine", "title": "向量化引擎", "type": "str", "length": 64, "default": "clip-vith14"}, - {"name": "vdb_collection", "title": "向量库集合名", "type": "str", "length": 128}, - {"name": "graph_name", "title": "图名称", "type": "str", "length": 128}, - {"name": "doc_count", "title": "文档数", "type": "int", "default": 0}, - {"name": "chunk_count", "title": "分片数", "type": "int", "default": 0}, - {"name": "total_size", "title": "总大小(字节)", "type": "int", "default": 0}, - {"name": "status", "title": "状态", "type": "str", "length": 16, "default": "active"}, - {"name": "created_by", "title": "创建人", "type": "str", "length": 32}, - {"name": "created_at", "title": "创建时间", "type": "datetime"}, - {"name": "updated_at", "title": "更新时间", "type": "datetime"} - ], - "codes": [ - {"field": "status", "table": "appcodes_kv", "valuefield": "k", "textfield": "v", "cond": "parentid='kb_status'"} - ], - "indexes": [ - {"name": "idx_kb_org", "idxtype": "index", "idxfields": ["org_id"]}, - {"name": "idx_kb_name", "idxtype": "unique", "idxfields": ["org_id", "name"]} - ] -} diff --git a/models/media_tags.json b/models/media_tags.json deleted file mode 100644 index 1e34090..0000000 --- a/models/media_tags.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "summary": [ - { - "name": "media_tags", - "title": "媒体标签关联", - "primary": ["id"] - } - ], - "fields": [ - {"name": "id", "title": "ID", "type": "str", "length": 32}, - {"name": "kb_id", "title": "知识库ID", "type": "str", "length": 32}, - {"name": "media_type", "title": "媒体类型", "type": "str", "length": 32}, - {"name": "media_id", "title": "媒体ID", "type": "str", "length": 32}, - {"name": "tag_id", "title": "标签ID", "type": "str", "length": 32}, - {"name": "created_at", "title": "创建时间", "type": "datetime"} - ], - "codes": [ - {"field": "media_type", "table": "appcodes_kv", "valuefield": "k", "textfield": "v", "cond": "parentid='media_type'"}, - {"field": "tag_id", "table": "tags", "valuefield": "id", "textfield": "name"}, - {"field": "kb_id", "table": "knowledge_bases", "valuefield": "id", "textfield": "name"} - ], - "indexes": [ - {"name": "idx_mt_media", "idxtype": "index", "idxfields": ["media_type", "media_id"]}, - {"name": "idx_mt_tag", "idxtype": "index", "idxfields": ["tag_id"]}, - {"name": "idx_mt_kb", "idxtype": "index", "idxfields": ["kb_id"]}, - {"name": "idx_mt_unique", "idxtype": "unique", "idxfields": ["media_type", "media_id", "tag_id"]} - ] -} diff --git a/models/subscriptions.json b/models/subscriptions.json deleted file mode 100644 index bec6ac9..0000000 --- a/models/subscriptions.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "summary": [ - { - "name": "subscriptions", - "title": "租户订阅", - "primary": ["id"] - } - ], - "fields": [ - {"name": "id", "title": "ID", "type": "str", "length": 32}, - {"name": "org_id", "title": "机构ID", "type": "str", "length": 32}, - {"name": "plan_name", "title": "套餐名称", "type": "str", "length": 64}, - {"name": "disk_quota_bytes", "title": "磁盘配额(字节)", "type": "int", "default": 1073741824}, - {"name": "disk_used_bytes", "title": "已用磁盘(字节)", "type": "int", "default": 0}, - {"name": "doc_quota", "title": "文档数配额", "type": "int", "default": 1000}, - {"name": "doc_used", "title": "已用文档数", "type": "int", "default": 0}, - {"name": "kb_quota", "title": "知识库数配额", "type": "int", "default": 10}, - {"name": "kb_used", "title": "已用知识库数", "type": "int", "default": 0}, - {"name": "api_call_quota", "title": "API调用配额(月)", "type": "int", "default": 10000}, - {"name": "api_call_used", "title": "已用API调用(月)", "type": "int", "default": 0}, - {"name": "start_date", "title": "开始日期", "type": "date"}, - {"name": "end_date", "title": "结束日期", "type": "date"}, - {"name": "status", "title": "状态", "type": "str", "length": 16, "default": "active"}, - {"name": "created_at", "title": "创建时间", "type": "datetime"}, - {"name": "updated_at", "title": "更新时间", "type": "datetime"} - ], - "codes": [ - {"field": "status", "table": "appcodes_kv", "valuefield": "k", "textfield": "v", "cond": "parentid='sub_status'"} - ], - "indexes": [ - {"name": "idx_sub_org", "idxtype": "unique", "idxfields": ["org_id"]}, - {"name": "idx_sub_status", "idxtype": "index", "idxfields": ["status"]} - ] -} diff --git a/models/tags.json b/models/tags.json deleted file mode 100644 index 3953f1b..0000000 --- a/models/tags.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "summary": [ - { - "name": "tags", - "title": "标签", - "primary": ["id"] - } - ], - "fields": [ - {"name": "id", "title": "ID", "type": "str", "length": 32}, - {"name": "kb_id", "title": "知识库ID", "type": "str", "length": 32}, - {"name": "name", "title": "标签名", "type": "str", "length": 64}, - {"name": "color", "title": "颜色", "type": "str", "length": 16, "default": "#3b82f6"}, - {"name": "org_id", "title": "所属机构", "type": "str", "length": 32}, - {"name": "created_at", "title": "创建时间", "type": "datetime"} - ], - "codes": [ - {"field": "kb_id", "table": "knowledge_bases", "valuefield": "id", "textfield": "name"} - ], - "indexes": [ - {"name": "idx_tag_kb", "idxtype": "index", "idxfields": ["kb_id"]}, - {"name": "idx_tag_name", "idxtype": "unique", "idxfields": ["kb_id", "name"]} - ] -} diff --git a/models/usage_logs.json b/models/usage_logs.json deleted file mode 100644 index 4b28e7d..0000000 --- a/models/usage_logs.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "summary": [ - { - "name": "usage_logs", - "title": "用量日志", - "primary": ["id"] - } - ], - "fields": [ - {"name": "id", "title": "ID", "type": "str", "length": 32}, - {"name": "org_id", "title": "机构ID", "type": "str", "length": 32}, - {"name": "kb_id", "title": "知识库ID", "type": "str", "length": 32}, - {"name": "operation", "title": "操作类型", "type": "str", "length": 32}, - {"name": "bytes_delta", "title": "磁盘变化(字节)", "type": "int", "default": 0}, - {"name": "bytes_total", "title": "累计磁盘", "type": "int", "default": 0}, - {"name": "api_calls", "title": "API调用数", "type": "int", "default": 1}, - {"name": "engine_type", "title": "引擎类型", "type": "str", "length": 32}, - {"name": "tokens_used", "title": "Token消耗", "type": "int", "default": 0}, - {"name": "cost_estimate", "title": "预估费用", "type": "float", "length": 20, "dec": 4}, - {"name": "detail", "title": "详情JSON", "type": "text", "length": 2000}, - {"name": "created_at", "title": "创建时间", "type": "datetime"} - ], - "codes": [ - {"field": "operation", "table": "appcodes_kv", "valuefield": "k", "textfield": "v", "cond": "parentid='usage_op'"} - ], - "indexes": [ - {"name": "idx_usage_org", "idxtype": "index", "idxfields": ["org_id", "created_at"]}, - {"name": "idx_usage_kb", "idxtype": "index", "idxfields": ["kb_id"]} - ] -} diff --git a/wwwroot/documents_list/index.ui b/wwwroot/documents_list/index.ui deleted file mode 100644 index daf666a..0000000 --- a/wwwroot/documents_list/index.ui +++ /dev/null @@ -1,65 +0,0 @@ -{ - "widgettype": "VBox", - "options": { - "cheight": 40, - "width": "100%", - "padding": "16px", - "spacing": "12px" - }, - "subwidgets": [ - { - "widgettype": "HBox", - "options": { - "width": "100%", - "justifyContent": "space-between", - "alignItems": "center" - }, - "subwidgets": [ - { - "widgettype": "VBox", - "options": {"spacing": "4px"}, - "subwidgets": [ - {"widgettype": "Text", "options": {"text": "文档管理", "cfontsize": 22, "fontWeight": "bold", "color": "#333"}}, - {"widgettype": "Text", "options": {"text": "上传文本、图片、音频、视频文件", "cfontsize": 14, "color": "#888"}} - ] - }, - { - "widgettype": "Button", - "options": { - "label": "📁 上传文件", - "bgcolor": "#50b86c", - "color": "#fff" - } - } - ] - }, - { - "widgettype": "VBox", - "options": { - "width": "100%", - "cheight": 8, - "bgcolor": "#f8f9fa", - "padding": "24px", - "css": "card", - "border": "2px dashed #d0d0d0", - "alignItems": "center", - "justifyContent": "center" - }, - "subwidgets": [ - {"widgettype": "Text", "options": {"text": "📂", "cfontsize": 32}}, - {"widgettype": "Text", "options": {"text": "拖拽文件到此处上传", "cfontsize": 16, "color": "#888"}}, - {"widgettype": "Text", "options": {"text": "支持: TXT, PDF, JPG, PNG, MP3, WAV, MP4, AVI", "cfontsize": 12, "color": "#aaa"}}, - {"widgettype": "Text", "options": {"text": "单个文件最大 500MB · 自动向量化入库", "cfontsize": 12, "color": "#aaa"}} - ] - }, - { - "widgettype": "Text", - "options": { - "text": "暂无文档", - "color": "#aaa", - "cfontsize": 14, - "halign": "center" - } - } - ] -} diff --git a/wwwroot/engine_configs_list/index.ui b/wwwroot/engine_configs_list/index.ui deleted file mode 100644 index f976585..0000000 --- a/wwwroot/engine_configs_list/index.ui +++ /dev/null @@ -1,148 +0,0 @@ -{ - "widgettype": "VBox", - "options": { - "cheight": 40, - "width": "100%", - "padding": "16px", - "spacing": "16px" - }, - "subwidgets": [ - { - "widgettype": "Text", - "options": { - "text": "引擎配置", - "cfontsize": 22, - "fontWeight": "bold", - "color": "#333" - } - }, - { - "widgettype": "Text", - "options": { - "text": "管理多媒体RAG的各层引擎，支持租户级自定义覆盖", - "color": "#888", - "cfontsize": 14 - } - }, - { - "widgettype": "HBox", - "options": { - "width": "100%", - "spacing": "12px", - "wrap": true - }, - "subwidgets": [ - { - "widgettype": "VBox", - "options": { - "cwidth": 20, - "cheight": 10, - "bgcolor": "#f8f9fa", - "padding": "12px", - "css": "card" - }, - "subwidgets": [ - {"widgettype": "Text", "options": {"text": "🔤 向量化引擎", "cfontsize": 16, "fontWeight": "bold"}}, - {"widgettype": "Text", "options": {"text": "CLIP ViT-H-14", "cfontsize": 14, "color": "#4a90d9"}}, - {"widgettype": "Text", "options": {"text": "1024维 · 图文跨模态", "cfontsize": 12, "color": "#888"}}, - {"widgettype": "Text", "options": {"text": "状态: ✅ 运行中", "cfontsize": 12, "color": "#50b86c"}}, - {"widgettype": "Text", "options": {"text": "端口: 9086", "cfontsize": 12, "color": "#aaa"}} - ] - }, - { - "widgettype": "VBox", - "options": { - "cwidth": 20, - "cheight": 10, - "bgcolor": "#f8f9fa", - "padding": "12px", - "css": "card" - }, - "subwidgets": [ - {"widgettype": "Text", "options": {"text": "🗄️ 向量数据库", "cfontsize": 16, "fontWeight": "bold"}}, - {"widgettype": "Text", "options": {"text": "Milvus Lite", "cfontsize": 14, "color": "#4a90d9"}}, - {"widgettype": "Text", "options": {"text": "COSINE度量 · HNSW索引", "cfontsize": 12, "color": "#888"}}, - {"widgettype": "Text", "options": {"text": "状态: ✅ 运行中", "cfontsize": 12, "color": "#50b86c"}}, - {"widgettype": "Text", "options": {"text": "端口: 8886", "cfontsize": 12, "color": "#aaa"}} - ] - }, - { - "widgettype": "VBox", - "options": { - "cwidth": 20, - "cheight": 10, - "bgcolor": "#f8f9fa", - "padding": "12px", - "css": "card" - }, - "subwidgets": [ - {"widgettype": "Text", "options": {"text": "📊 重排引擎", "cfontsize": 16, "fontWeight": "bold"}}, - {"widgettype": "Text", "options": {"text": "BGE Reranker v2-m3", "cfontsize": 14, "color": "#4a90d9"}}, - {"widgettype": "Text", "options": {"text": "Cross-encoder精排", "cfontsize": 12, "color": "#888"}}, - {"widgettype": "Text", "options": {"text": "状态: ✅ 运行中", "cfontsize": 12, "color": "#50b86c"}}, - {"widgettype": "Text", "options": {"text": "端口: 9090", "cfontsize": 12, "color": "#aaa"}} - ] - }, - { - "widgettype": "VBox", - "options": { - "cwidth": 20, - "cheight": 10, - "bgcolor": "#f8f9fa", - "padding": "12px", - "css": "card" - }, - "subwidgets": [ - {"widgettype": "Text", "options": {"text": "🕸️ 知识图谱", "cfontsize": 16, "fontWeight": "bold"}}, - {"widgettype": "Text", "options": {"text": "NetworkX", "cfontsize": 14, "color": "#4a90d9"}}, - {"widgettype": "Text", "options": {"text": "实体关系 · 邻居查询", "cfontsize": 12, "color": "#888"}}, - {"widgettype": "Text", "options": {"text": "状态: ✅ 运行中", "cfontsize": 12, "color": "#50b86c"}}, - {"widgettype": "Text", "options": {"text": "端口: 9092", "cfontsize": 12, "color": "#aaa"}} - ] - }, - { - "widgettype": "VBox", - "options": { - "cwidth": 20, - "cheight": 10, - "bgcolor": "#f8f9fa", - "padding": "12px", - "css": "card" - }, - "subwidgets": [ - {"widgettype": "Text", "options": {"text": "🏷️ 实体识别", "cfontsize": 16, "fontWeight": "bold"}}, - {"widgettype": "Text", "options": {"text": "GLiNER Multitask", "cfontsize": 14, "color": "#4a90d9"}}, - {"widgettype": "Text", "options": {"text": "零样本NER · 中文支持", "cfontsize": 12, "color": "#888"}}, - {"widgettype": "Text", "options": {"text": "状态: ✅ 运行中", "cfontsize": 12, "color": "#50b86c"}}, - {"widgettype": "Text", "options": {"text": "端口: 9093", "cfontsize": 12, "color": "#aaa"}} - ] - }, - { - "widgettype": "VBox", - "options": { - "cwidth": 20, - "cheight": 10, - "bgcolor": "#f8f9fa", - "padding": "12px", - "css": "card" - }, - "subwidgets": [ - {"widgettype": "Text", "options": {"text": "👤 人脸识别", "cfontsize": 16, "fontWeight": "bold"}}, - {"widgettype": "Text", "options": {"text": "InsightFace buffalo_l", "cfontsize": 14, "color": "#4a90d9"}}, - {"widgettype": "Text", "options": {"text": "检测+识别 · 512维", "cfontsize": 12, "color": "#888"}}, - {"widgettype": "Text", "options": {"text": "状态: ✅ 运行中", "cfontsize": 12, "color": "#50b86c"}}, - {"widgettype": "Text", "options": {"text": "端口: 9091", "cfontsize": 12, "color": "#aaa"}} - ] - } - ] - }, - { - "widgettype": "Text", - "options": { - "text": "💡 引擎默认全局配置，租户可在订阅设置中覆盖为私有实例", - "color": "#aaa", - "cfontsize": 12 - } - } - ] -} diff --git a/wwwroot/knowledge_bases_list/create_kb.dspy b/wwwroot/knowledge_bases_list/create_kb.dspy deleted file mode 100644 index 8b70633..0000000 --- a/wwwroot/knowledge_bases_list/create_kb.dspy +++ /dev/null @@ -1,19 +0,0 @@ -ns = params_kw.copy() -db = DBPools() -dbname = get_module_dbname('rag') -async with db.sqlorContext(dbname) as sor: - import uuid - env = request._run_ns - userorgid = await env.get_userorgid() - kb_id = uuid.uuid4().hex[:12] - name = ns.get("name", "") - desc = ns.get("description", "") - await sor.sqlExe( - "INSERT INTO knowledge_bases (id, name, description, org_id, embedding_engine, vdb_collection, doc_count, total_size, chunk_count, status, created_at) " - "VALUES (${id}$, ${name}$, ${desc}$, ${org_id}$, 'CLIP ViT-H-14', 'rag_collection', 0, 0, 0, 'active', NOW())", - {"id": kb_id, "name": name, "desc": desc, "org_id": userorgid}) - return { - "widgettype": "urlwidget", - "options": {"url": entire_url('/knowledge_bases_list/index.ui')} - } -return {"error": "failed"} diff --git a/wwwroot/knowledge_bases_list/delete_tree_item.dspy b/wwwroot/knowledge_bases_list/delete_tree_item.dspy deleted file mode 100644 index 6319d8f..0000000 --- a/wwwroot/knowledge_bases_list/delete_tree_item.dspy +++ /dev/null @@ -1,7 +0,0 @@ -ns = params_kw.copy() -db = DBPools() -dbname = get_module_dbname('rag') -async with db.sqlorContext(dbname) as sor: - await sor.sqlExe("DELETE FROM document_chunks WHERE id=${id}$", {"id": ns.get("id","")}) - return {"widgettype": "Message", "options": {"user_data": {"id": ns.get("id","")}}} -return {"error": "failed"} diff --git a/wwwroot/knowledge_bases_list/detail.ui b/wwwroot/knowledge_bases_list/detail.ui deleted file mode 100644 index fdb81d2..0000000 --- a/wwwroot/knowledge_bases_list/detail.ui +++ /dev/null @@ -1,92 +0,0 @@ -{ - "widgettype": "VBox", - "options": {"cheight": 40, "width": "100%", "padding": "0", "spacing": 0}, - "subwidgets": [ - { - "widgettype": "HBox", - "options": {"width": "100%", "bgcolor": "#f8f9fa", "padding": "10px 16px", "alignItems": "center", "borderBottom": "1px solid #e0e0e0"}, - "subwidgets": [ - {"widgettype": "Text", "options": {"text": "← 知识库列表", "css": "clickable", "cfontsize": 14, "color": "#4a90d9"}, "binds": [ - {"wid": "self", "event": "click", "actiontype": "urlwidget", "target": "app.rag_main_content", "mode": "replace", - "options": {"url": "{{entire_url('/knowledge_bases_list/index.ui')}}"}} - ]}, - {"widgettype": "Text", "options": {"text": " / ", "cfontsize": 14, "color": "#ccc"}}, - {"widgettype": "Text", "options": {"text": "📚 {{params_kw.kb_name or '知识库'}}", "cfontsize": 16, "fontWeight": "bold", "color": "#333"}}, - {"widgettype": "Text", "options": {"text": "", "css": "filler"}} - ] - }, - { - "widgettype": "HBox", - "options": {"css": "filler", "width": "100%", "spacing": 0}, - "subwidgets": [ - { - "widgettype": "VBox", - "options": {"cwidth": 16, "bgcolor": "#fafafa", "borderRight": "1px solid #e0e0e0", "spacing": 0}, - "subwidgets": [ - { - "widgettype": "HBox", - "options": {"padding": "10px 12px", "bgcolor": "#f0f0f0", "borderBottom": "1px solid #e0e0e0"}, - "subwidgets": [ - {"widgettype": "Text", "options": {"text": "📁 目录结构", "cfontsize": 13, "fontWeight": "bold", "color": "#666"}} - ] - }, - { - "widgettype": "Tree", - "options": { - "title": "目录", - "id": "dir_tree", - "parentField": "parentid", - "idField": "id", - "textField": "label", - "dataurl": "{{entire_url('./get_tree_data.dspy')}}?kb_id={{params_kw.kb_id}}", - "newdata_params": {"kb_id": "{{params_kw.kb_id}}"}, - "editable": { - "fields": [ - {"name": "name", "title": "名称", "type": "str", "length": 255, "uitype": "str", "label": "名称"} - ], - "add_url": "{{entire_url('./new_tree_item.dspy')}}", - "update_url": "{{entire_url('./update_tree_item.dspy')}}", - "delete_url": "{{entire_url('./delete_tree_item.dspy')}}" - } - }, - "binds": [ - { - "wid": "self", - "event": "selected", - "actiontype": "script", - "script": "window._rag_current_folder = event.params.id ? event.params.id : ''" - } - ] - } - ] - }, - { - "widgettype": "VBox", - "options": {"id": "detail_content", "css": "filler", "padding": "16px", "spacing": "12px"}, - "subwidgets": [ - { - "widgettype": "VBox", - "options": {"id": "drop_zone", "width": "100%", "cheight": 8, "bgcolor": "#f8f9fa", "padding": "16px", "css": "card", "border": "2px dashed #3b82f6", "alignItems": "center", "justifyContent": "center"}, - "subwidgets": [ - {"widgettype": "Text", "options": {"text": "📂 点击选择文件上传", "cfontsize": 15, "color": "#888"}}, - {"widgettype": "Text", "options": {"text": "支持 TXT/PDF/JPG/PNG/MP3/WAV/MP4", "cfontsize": 12, "color": "#aaa"}}, - { - "widgettype": "Button", - "options": {"label": "选择文件", "bgcolor": "#3b82f6", "color": "#fff", "marginTop": "8px"}, - "binds": [{ - "wid": "self", - "event": "click", - "actiontype": "script", - "script": "var i=document.createElement('input');i.type='file';i.multiple=true;i.onchange=function(){var u=new URL(window.location.href);var kb=u.searchParams.get('kb_id')||'';var fd=window._rag_current_folder||'';for(var j=0;j