sync: add init_rbac_v2.py, test_rag.sh; remove stale search files moved to rag module
This commit is contained in:
parent
318b69f4a5
commit
b6da2a9f35
65
scripts/init_rbac_v2.py
Normal file
65
scripts/init_rbac_v2.py
Normal file
@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env python3
|
||||
"""RAG Server RBAC 权限初始化 — 自动扫描 wwwroot, 幂等"""
|
||||
import sys, os, asyncio, hashlib
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
|
||||
|
||||
from sqlor.dbpools import DBPools
|
||||
from appPublic.jsonConfig import getConfig
|
||||
|
||||
config = getConfig('.')
|
||||
DBPools(config.databases)
|
||||
|
||||
BASE = os.path.dirname(os.path.abspath(__file__))
|
||||
WWWROOT = os.path.join(BASE, '..', 'pkgs', 'rag', 'wwwroot')
|
||||
|
||||
PUBLIC = [
|
||||
'/', '/index.ui', '/top.ui', '/user_menu.ui', '/shell_theme.css',
|
||||
'/i18n_getmsgs', '/bricks/**', '/i18n/**', '/uapi/**',
|
||||
'/rbac/user/login.ui', '/rbac/user/register.ui',
|
||||
'/rbac/user/login.dspy', '/rbac/user/register.dspy', '/rbac/user/logout.dspy',
|
||||
'/rbac/**',
|
||||
'/api/status', '/api/engines', '/api/kb/list',
|
||||
]
|
||||
LOGINED = [
|
||||
'/api/search', '/api/doc/upload', '/api/doc/delete',
|
||||
'/api/dir/create', '/api/dir/delete', '/api/dir/list',
|
||||
'/api/tag/create', '/api/tag/list', '/api/tag/delete',
|
||||
'/api/tag/assign', '/api/tag/unassign', '/api/tag/media_tags', '/api/tag/search',
|
||||
]
|
||||
|
||||
def pid(path):
|
||||
return 'p_' + hashlib.md5(path.encode()).hexdigest()[:12]
|
||||
|
||||
def discover_paths(root, prefix='/rag'):
|
||||
paths = set()
|
||||
for dirpath, _, filenames in os.walk(root):
|
||||
rel = dirpath[len(root):] + '/'
|
||||
paths.add(prefix + rel)
|
||||
for f in filenames:
|
||||
if f.endswith(('.dspy', '.ui')):
|
||||
paths.add(prefix + rel + f)
|
||||
return sorted(paths)
|
||||
|
||||
async def main():
|
||||
db = DBPools()
|
||||
async with db.sqlorContext('rag') as sor:
|
||||
for rid, rname in [('any', 'any'), ('logined', 'logined')]:
|
||||
await sor.sqlExe(
|
||||
f"INSERT IGNORE INTO role (id, orgtypeid, name) VALUES ('{rid}', '', '{rname}')", {})
|
||||
|
||||
for path in PUBLIC:
|
||||
p = pid(path)
|
||||
await sor.sqlExe(f"INSERT IGNORE INTO permission (id, path, name) VALUES ('{p}', '{path}', 'RAG')", {})
|
||||
await sor.sqlExe(f"INSERT IGNORE INTO rolepermission (id, roleid, permid) VALUES ('rp_{p}', 'any', '{p}')", {})
|
||||
|
||||
all_pages = discover_paths(WWWROOT)
|
||||
for path in LOGINED + all_pages:
|
||||
p = pid(path)
|
||||
await sor.sqlExe(f"INSERT IGNORE INTO permission (id, path, name) VALUES ('{p}', '{path}', 'RAG')", {})
|
||||
await sor.sqlExe(f"INSERT IGNORE INTO rolepermission (id, roleid, permid) VALUES ('rp_{p}', 'logined', '{p}')", {})
|
||||
|
||||
print(f"OK: {len(PUBLIC)} public + {len(LOGINED) + len(all_pages)} logined")
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
48
test_rag.sh
Executable file
48
test_rag.sh
Executable file
@ -0,0 +1,48 @@
|
||||
#!/bin/bash
|
||||
# RAG Server smoke test — run after any change
|
||||
# Usage: bash test_rag.sh [host]
|
||||
HOST="${1:-http://localhost:9181}"
|
||||
ADMIN="${2:-admin}"
|
||||
PASS="${3:-admin123}"
|
||||
|
||||
RED='\033[0;31m'; GREEN='\033[0;32m'; NC='\033[0m'
|
||||
errors=0
|
||||
|
||||
check() { local desc="$1" code="$2" expect="$3"
|
||||
if [ "$code" = "$expect" ]; then echo -e " ${GREEN}OK${NC} $desc"; else echo -e " ${RED}FAIL${NC} $desc (got $code, want $expect)"; errors=$((errors+1)); fi
|
||||
}
|
||||
|
||||
echo "=== RAG Server Smoke Test ==="
|
||||
|
||||
# 1. Public endpoints (no login)
|
||||
echo "-- Public --"
|
||||
check "health" "$(curl -s -o /dev/null -w '%{http_code}' "$HOST/api/status")" 200
|
||||
check "no login kb" "$(curl -s -o /dev/null -w '%{http_code}' "$HOST/rag/knowledge_bases_list/index.ui")" 401
|
||||
|
||||
# 2. Login
|
||||
S=$(curl -s -D - -X POST "$HOST/rbac/user/up_login.dspy" -d "username=$ADMIN&password=$PASS" | grep -oP 'AIOHTTP_SESSION=\K[^;]+' | head -1)
|
||||
if [ -z "$S" ]; then echo -e " ${RED}FAIL${NC} login"; exit 1; fi
|
||||
echo -e " ${GREEN}OK${NC} login"
|
||||
|
||||
# 3. Logged-in endpoints
|
||||
echo "-- Logined --"
|
||||
|
||||
check "index.ui" "$(curl -s -o /dev/null -w '%{http_code}' -b "AIOHTTP_SESSION=$S" "$HOST/rag/knowledge_bases_list/index.ui")" 200
|
||||
check "detail.ui" "$(curl -s -o /dev/null -w '%{http_code}' -b "AIOHTTP_SESSION=$S" "$HOST/rag/knowledge_bases_list/detail.ui?kb_id=41c9f1cd45e0")" 200
|
||||
check "get_tree_data" "$(curl -s -o /dev/null -w '%{http_code}' -b "AIOHTTP_SESSION=$S" "$HOST/rag/knowledge_bases_list/get_tree_data.dspy?kb_id=41c9f1cd45e0")" 200
|
||||
check "file_list" "$(curl -s -o /dev/null -w '%{http_code}' -b "AIOHTTP_SESSION=$S" "$HOST/rag/knowledge_bases_list/file_list.dspy?kb_id=41c9f1cd45e0&id=__root__")" 200
|
||||
check "storage_card" "$(curl -s -o /dev/null -w '%{http_code}' -b "AIOHTTP_SESSION=$S" "$HOST/rag/knowledge_bases_list/storage_card.dspy?_webbricks_=1")" 200
|
||||
check "new_kb_form" "$(curl -s -o /dev/null -w '%{http_code}' -b "AIOHTTP_SESSION=$S" "$HOST/rag/knowledge_bases_list/new_kb_form.ui")" 200
|
||||
check "kb_list" "$(curl -s -o /dev/null -w '%{http_code}' -b "AIOHTTP_SESSION=$S" "$HOST/api/kb/list")" 200
|
||||
|
||||
# 4. Verify storage shows real data
|
||||
used=$(curl -s -b "AIOHTTP_SESSION=$S" "$HOST/rag/knowledge_bases_list/storage_card.dspy?_webbricks_=1" | grep -oP '已用 \K[\d.]+')
|
||||
if [ "$used" != "0.0" ] && [ -n "$used" ]; then
|
||||
echo -e " ${GREEN}OK${NC} storage shows ${used}MB"
|
||||
else
|
||||
echo -e " ${RED}FAIL${NC} storage shows $used (want >0)"
|
||||
errors=$((errors+1))
|
||||
fi
|
||||
|
||||
echo "=== $errors errors ==="
|
||||
exit $errors
|
||||
@ -1,18 +0,0 @@
|
||||
"""返回知识库选择按钮列表"""
|
||||
env = request._run_ns
|
||||
userorgid = await env.get_userorgid()
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
sql = "SELECT id, name FROM knowledge_bases WHERE org_id IS NULL OR org_id=${org_id}$ ORDER BY name"
|
||||
recs = await sor.sqlExe(sql, {"org_id": userorgid})
|
||||
|
||||
buttons = []
|
||||
for r in recs:
|
||||
buttons.append({
|
||||
"widgettype": "Text",
|
||||
"options": {"text": str(r.name), "cfontsize": 12, "padding": "4px 12px",
|
||||
"bgcolor": "#eee", "color": "#333", "borderRadius": "4px", "css": "clickable kb-btn"},
|
||||
"binds": [{"wid": "self", "event": "click", "actiontype": "script", "target": "self",
|
||||
"script": f"var f=document.querySelector('[name=kb_id]');if(f)f.value='{r.id}';var btns=document.querySelectorAll('.kb-btn');for(var i=0;i<btns.length;i++){{btns[i].style.background='#eee';btns[i].style.color='#333'}}this.dom_element.style.background='#3b82f6';this.dom_element.style.color='#fff';var all=document.getElementById('kb_btn_all');if(all){{all.style.background='#eee';all.style.color='#333'}}"}]
|
||||
})
|
||||
|
||||
{"widgettype": "HBox", "options": {"spacing": "8px", "width": "100%", "wrap": true}, "subwidgets": buttons}
|
||||
@ -1,75 +0,0 @@
|
||||
{
|
||||
"widgettype": "VBox",
|
||||
"id": "rag_search_page",
|
||||
"options": {"width": "100%", "padding": "24px", "spacing": "16px"},
|
||||
"subwidgets": [
|
||||
{
|
||||
"widgettype": "Text",
|
||||
"options": {"text": "知识库检索", "cfontsize": 24, "fontWeight": "bold"}
|
||||
},
|
||||
{
|
||||
"widgettype": "Text",
|
||||
"options": {"text": "文本搜索，自动向量化 + 重排序，支持全库或指定知识库", "cfontsize": 13, "color": "#888"}
|
||||
},
|
||||
{
|
||||
"widgettype": "Form",
|
||||
"id": "search_form",
|
||||
"options": {
|
||||
"submit_url": "/rag/search_result.dspy",
|
||||
"fields": [
|
||||
{"name": "query", "label": "搜索", "uitype": "Text", "required": false, "placeholder": "输入关键词检索知识库..."},
|
||||
{"name": "kb_id", "label": "kb_id", "uitype": "Text", "required": false, "hidden": true, "value": ""},
|
||||
{"name": "top_k", "label": "top_k", "uitype": "Text", "required": false, "hidden": true, "value": "10"},
|
||||
{"name": "recall_k", "label": "recall_k", "uitype": "Text", "required": false, "hidden": true, "value": "30"}
|
||||
]
|
||||
},
|
||||
"subwidgets": [
|
||||
{
|
||||
"widgettype": "Text",
|
||||
"options": {"text": "知识库：", "cfontsize": 13, "color": "#666", "padding": "4px 0 8px 0"}
|
||||
},
|
||||
{
|
||||
"widgettype": "HBox",
|
||||
"id": "kb_selector_row",
|
||||
"options": {"spacing": "8px", "width": "100%", "wrap": true},
|
||||
"subwidgets": [
|
||||
{
|
||||
"widgettype": "Text",
|
||||
"id": "kb_btn_all",
|
||||
"options": {"text": "全部知识库", "cfontsize": 12, "padding": "4px 12px", "bgcolor": "#3b82f6", "color": "#fff", "borderRadius": "4px", "css": "clickable"},
|
||||
"binds": [{"wid": "self", "event": "click", "actiontype": "script", "target": "self",
|
||||
"script": "var f=document.querySelector('[name=kb_id]');if(f)f.value='';var btns=document.querySelectorAll('.kb-btn');for(var i=0;i<btns.length;i++){btns[i].style.background='#eee';btns[i].style.color='#333'}this.dom_element.style.background='#3b82f6';this.dom_element.style.color='#fff'"}]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"widgettype": "HBox",
|
||||
"id": "kb_list_area",
|
||||
"options": {"spacing": "8px", "width": "100%", "wrap": true},
|
||||
"binds": [{"wid": "self", "event": "rendered", "actiontype": "urlwidget", "target": "self", "mode": "replace",
|
||||
"options": {"url": "/rag/kb_buttons.dspy"}}]
|
||||
},
|
||||
{"widgettype": "Text", "options": {"text": " ", "cheight": 0.3}},
|
||||
{
|
||||
"widgettype": "HBox",
|
||||
"options": {"spacing": "12px"},
|
||||
"subwidgets": [
|
||||
{"widgettype": "Button", "id": "search_submit_btn", "options": {"label": "搜索", "bgcolor": "#3b82f6", "color": "#fff", "borderRadius": "6px", "padding": "8px 24px", "cfontsize": 14}}
|
||||
]
|
||||
}
|
||||
],
|
||||
"binds": [
|
||||
{"wid": "self", "event": "submited", "actiontype": "urlwidget", "target": "root.search_results_container", "mode": "replace",
|
||||
"options": {"url": "/rag/search_result.dspy"}}
|
||||
]
|
||||
},
|
||||
{
|
||||
"widgettype": "VBox",
|
||||
"id": "search_results_container",
|
||||
"options": {"width": "100%", "spacing": "12px"},
|
||||
"subwidgets": [
|
||||
{"widgettype": "Text", "options": {"text": "输入关键词搜索知识库内容", "cfontsize": 14, "color": "#aaa", "halign": "center", "padding": "40px"}}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -1,106 +0,0 @@
|
||||
"""搜索 DSPY — 接收 Form 参数，返回搜索结果 Widget"""
|
||||
query = (params_kw.get("query") or "").strip()
|
||||
kb_id = (params_kw.get("kb_id") or "").strip()
|
||||
top_k = int(params_kw.get("top_k") or 10)
|
||||
recall_k = int(params_kw.get("recall_k") or top_k * 3)
|
||||
env = request._run_ns
|
||||
userorgid = await env.get_userorgid()
|
||||
|
||||
# Resolve KBs
|
||||
async with get_sor_context(env, 'rag') as sor:
|
||||
if kb_id:
|
||||
recs = await sor.R("knowledge_bases", {"id": kb_id})
|
||||
kb_ids = [r.id for r in recs]
|
||||
else:
|
||||
sql = "SELECT id FROM knowledge_bases WHERE org_id IS NULL OR org_id=${org_id}$"
|
||||
recs = await sor.sqlExe(sql, {"org_id": userorgid})
|
||||
kb_ids = [r.id for r in recs]
|
||||
|
||||
if not kb_ids:
|
||||
result = {"widgettype": "Text", "options": {"text": "没有可搜索的知识库", "color": "#aaa", "halign": "center", "padding": "40px", "cfontsize": 14}}
|
||||
|
||||
elif not query:
|
||||
result = {"widgettype": "Text", "options": {"text": "请输入搜索关键词", "color": "#aaa", "halign": "center", "padding": "40px", "cfontsize": 14}}
|
||||
|
||||
else:
|
||||
# Build embedding
|
||||
from rag.init import _call_uapi
|
||||
try:
|
||||
emb_resp = await _call_uapi("rag-embedding", "embed", {"texts": [query], "model": "CLIP-ViT-H-14"})
|
||||
query_vec = (emb_resp.get("embeddings", []) or [None])[0] if isinstance(emb_resp, dict) else None
|
||||
except Exception:
|
||||
query_vec = None
|
||||
|
||||
if not query_vec:
|
||||
result = {"widgettype": "Text", "options": {"text": "向量化失败，请稍后重试", "color": "#e53e3e", "halign": "center", "padding": "40px", "cfontsize": 14}}
|
||||
else:
|
||||
# Multi-KB VDB search
|
||||
all_hits = []
|
||||
for kid in kb_ids:
|
||||
try:
|
||||
vdb_resp = await _call_uapi("rag-vdb", "search", {"collection": kid, "vector": query_vec, "topK": recall_k})
|
||||
data = vdb_resp
|
||||
if isinstance(data, dict):
|
||||
for key in ("results", "data", "hits", "rows"):
|
||||
if isinstance(data.get(key), list):
|
||||
data = data[key]
|
||||
break
|
||||
if isinstance(data, list):
|
||||
for item in data:
|
||||
if isinstance(item, dict):
|
||||
all_hits.append({"id": item.get("id", ""), "text": item.get("text", item.get("content", "")),
|
||||
"score": item.get("score", item.get("distance", 0)), "kb_id": kid})
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Deduplicate + sort
|
||||
seen = set()
|
||||
unique_hits = []
|
||||
for h in sorted(all_hits, key=lambda x: x.get("score", 0), reverse=True):
|
||||
hid = h.get("id", h.get("text", ""))
|
||||
if hid not in seen:
|
||||
seen.add(hid)
|
||||
unique_hits.append(h)
|
||||
|
||||
# Rerank
|
||||
if query and unique_hits:
|
||||
docs = [h.get("text", "")[:500] for h in unique_hits[:recall_k]]
|
||||
try:
|
||||
rr = await _call_uapi("rag-reranker", "rerank", {"query": query, "documents": docs})
|
||||
scores = rr.get("scores", rr.get("results", [])) if isinstance(rr, dict) else []
|
||||
if isinstance(scores, list) and len(scores) == len(docs):
|
||||
for i, s in enumerate(scores):
|
||||
unique_hits[i]["rerank_score"] = s.get("score", 0) if isinstance(s, dict) else float(s or 0)
|
||||
unique_hits.sort(key=lambda x: x.get("rerank_score", 0), reverse=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
final = unique_hits[:top_k]
|
||||
|
||||
if not final:
|
||||
result = {"widgettype": "Text", "options": {"text": "未找到相关结果", "color": "#aaa", "halign": "center", "padding": "40px", "cfontsize": 14}}
|
||||
else:
|
||||
cards = []
|
||||
for i, h in enumerate(final):
|
||||
text = h.get("text", "")[:300]
|
||||
score = h.get("rerank_score", h.get("score", 0))
|
||||
score_pct = f"{min(abs(float(score)) * 100, 100):.0f}%" if score else ""
|
||||
cards.append({
|
||||
"widgettype": "VBox",
|
||||
"options": {"cwidth": 100, "padding": "16px", "bgcolor": "#f8fafc", "border": "1px solid #e2e8f0", "borderRadius": "8px", "spacing": "8px"},
|
||||
"subwidgets": [
|
||||
{"widgettype": "HBox", "options": {"spacing": "8px"}, "subwidgets": [
|
||||
{"widgettype": "Text", "options": {"text": f"#{i+1}", "cfontsize": 12, "color": "#3b82f6", "fontWeight": "bold", "padding": "2px 8px", "bgcolor": "#eff6ff", "borderRadius": "4px"}},
|
||||
{"widgettype": "Text", "options": {"text": f"相关度: {score_pct}", "cfontsize": 11, "color": "#888"}},
|
||||
{"widgettype": "Text", "options": {"text": f"KB: {h.get('kb_id','')[:12]}", "cfontsize": 11, "color": "#aaa"}}
|
||||
]},
|
||||
{"widgettype": "Text", "options": {"text": text, "cfontsize": 13, "color": "#333", "lineHeight": 1.6}},
|
||||
{"widgettype": "Text", "options": {"text": h.get("id", "")[:40], "cfontsize": 10, "color": "#bbb"}}
|
||||
]
|
||||
})
|
||||
result = {"widgettype": "VBox", "options": {"width": "100%", "spacing": "12px"}, "subwidgets": [
|
||||
{"widgettype": "Text", "options": {"text": f"找到 {len(final)} 条结果 (共召回 {len(all_hits)} 条)", "cfontsize": 13, "color": "#666", "padding": "0 0 8px 0"}},
|
||||
*cards
|
||||
]}
|
||||
|
||||
result
|
||||
Loading…
x
Reference in New Issue
Block a user