fix: KB删除时清理VDB/Graph/physical文件
- 修复物理文件删除:路径不以/idfile/开头,改为检查/开头 - 新增VDB向量删除:调用vectordb API删除该KB的所有向量 - 新增Graph图数据库清理 - 新增entities/entity_relations表清理
This commit is contained in:
parent
3b1cda8d2f
commit
239b82c85d
@ -9,10 +9,10 @@ if not kb_id:
|
|||||||
|
|
||||||
env = request._run_ns
|
env = request._run_ns
|
||||||
userorgid = await env.get_userorgid()
|
userorgid = await env.get_userorgid()
|
||||||
deleted = {"documents": 0, "chunks": 0, "tags": 0, "media_tags": 0, "files": 0}
|
deleted = {"documents": 0, "chunks": 0, "tags": 0, "media_tags": 0,
|
||||||
|
"files": 0, "vdb": 0, "graph": 0, "entities": 0}
|
||||||
|
|
||||||
async with get_sor_context(env, 'rag') as sor:
|
async with get_sor_context(env, 'rag') as sor:
|
||||||
# Check owner
|
|
||||||
kb = await sor.sqlExe(
|
kb = await sor.sqlExe(
|
||||||
"SELECT id, name FROM knowledge_bases WHERE id=${id}$ AND (org_id=${org_id}$ OR org_id IS NULL)",
|
"SELECT id, name FROM knowledge_bases WHERE id=${id}$ AND (org_id=${org_id}$ OR org_id IS NULL)",
|
||||||
{"id": kb_id, "org_id": userorgid})
|
{"id": kb_id, "org_id": userorgid})
|
||||||
@ -20,52 +20,78 @@ async with get_sor_context(env, 'rag') as sor:
|
|||||||
return {"widgettype": "urlwidget",
|
return {"widgettype": "urlwidget",
|
||||||
"options": {"url": entire_url('/rag/knowledge_bases_list/index.ui')}}
|
"options": {"url": entire_url('/rag/knowledge_bases_list/index.ui')}}
|
||||||
|
|
||||||
# Get all doc IDs and file paths for physical cleanup
|
|
||||||
docs = await sor.sqlExe(
|
docs = await sor.sqlExe(
|
||||||
"SELECT id, file_path FROM documents WHERE kb_id=${kb_id}$",
|
"SELECT id, file_path FROM documents WHERE kb_id=${kb_id}$",
|
||||||
{"kb_id": kb_id})
|
{"kb_id": kb_id})
|
||||||
doc_ids = [d.id for d in docs]
|
doc_ids = [d.id for d in docs]
|
||||||
|
|
||||||
|
# 1. VDB cleanup
|
||||||
if doc_ids:
|
if doc_ids:
|
||||||
# Delete chunks
|
chunks = await sor.sqlExe(
|
||||||
await sor.sqlExe(
|
"SELECT vector_id FROM document_chunks WHERE doc_id IN (${ids}$) AND vector_id IS NOT NULL AND vector_id != ''",
|
||||||
"DELETE FROM document_chunks WHERE doc_id IN (${ids}$)",
|
|
||||||
{"ids": doc_ids})
|
{"ids": doc_ids})
|
||||||
deleted["chunks"] = len(doc_ids)
|
vector_ids = [c.vector_id for c in chunks]
|
||||||
|
if vector_ids:
|
||||||
|
try:
|
||||||
|
client = StreamHttpClient()
|
||||||
|
await client.request('POST',
|
||||||
|
'https://vectordb.opencomputing.net:10443/v1/delete',
|
||||||
|
json={"colname": kb_id, "ids": vector_ids})
|
||||||
|
deleted["vdb"] = len(vector_ids)
|
||||||
|
except Exception as e:
|
||||||
|
info(f'delete_kb vdb error: {e}')
|
||||||
|
|
||||||
# Delete media_tags
|
# 2. Graph DB cleanup
|
||||||
mt = await sor.sqlExe(
|
try:
|
||||||
"SELECT COUNT(*) AS cnt FROM media_tags WHERE kb_id=${kb_id}$",
|
client2 = StreamHttpClient()
|
||||||
{"kb_id": kb_id})
|
await client2.request('POST',
|
||||||
await sor.sqlExe(
|
'https://graph.opencomputing.net:10443/api/graph/delete',
|
||||||
"DELETE FROM media_tags WHERE kb_id=${kb_id}$",
|
json={"graph": kb_id})
|
||||||
{"kb_id": kb_id})
|
deleted["graph"] = 1
|
||||||
|
except Exception as e:
|
||||||
|
info(f'delete_kb graph error: {e}')
|
||||||
|
|
||||||
|
# 3. Entities & relations
|
||||||
|
er = await sor.sqlExe(
|
||||||
|
"SELECT COUNT(*) AS cnt FROM entity_relations WHERE kb_id=${kb_id}$",
|
||||||
|
{"kb_id": kb_id})
|
||||||
|
await sor.sqlExe("DELETE FROM entity_relations WHERE kb_id=${kb_id}$", {"kb_id": kb_id})
|
||||||
|
deleted["entities"] += er[0].cnt if er else 0
|
||||||
|
|
||||||
|
ent = await sor.sqlExe(
|
||||||
|
"SELECT COUNT(*) AS cnt FROM entities WHERE kb_id=${kb_id}$",
|
||||||
|
{"kb_id": kb_id})
|
||||||
|
await sor.sqlExe("DELETE FROM entities WHERE kb_id=${kb_id}$", {"kb_id": kb_id})
|
||||||
|
deleted["entities"] += ent[0].cnt if ent else 0
|
||||||
|
|
||||||
|
# 4. Chunks
|
||||||
|
ch = await sor.sqlExe(
|
||||||
|
"SELECT COUNT(*) AS cnt FROM document_chunks WHERE doc_id IN (${ids}$)",
|
||||||
|
{"ids": doc_ids})
|
||||||
|
await sor.sqlExe("DELETE FROM document_chunks WHERE doc_id IN (${ids}$)", {"ids": doc_ids})
|
||||||
|
deleted["chunks"] = ch[0].cnt if ch else 0
|
||||||
|
|
||||||
|
# Media tags
|
||||||
|
mt = await sor.sqlExe("SELECT COUNT(*) AS cnt FROM media_tags WHERE kb_id=${kb_id}$", {"kb_id": kb_id})
|
||||||
|
await sor.sqlExe("DELETE FROM media_tags WHERE kb_id=${kb_id}$", {"kb_id": kb_id})
|
||||||
deleted["media_tags"] = mt[0].cnt if mt else 0
|
deleted["media_tags"] = mt[0].cnt if mt else 0
|
||||||
|
|
||||||
# Delete tags
|
# Tags
|
||||||
t = await sor.sqlExe(
|
t = await sor.sqlExe("SELECT COUNT(*) AS cnt FROM tags WHERE kb_id=${kb_id}$", {"kb_id": kb_id})
|
||||||
"SELECT COUNT(*) AS cnt FROM tags WHERE kb_id=${kb_id}$",
|
await sor.sqlExe("DELETE FROM tags WHERE kb_id=${kb_id}$", {"kb_id": kb_id})
|
||||||
{"kb_id": kb_id})
|
|
||||||
await sor.sqlExe(
|
|
||||||
"DELETE FROM tags WHERE kb_id=${kb_id}$",
|
|
||||||
{"kb_id": kb_id})
|
|
||||||
deleted["tags"] = t[0].cnt if t else 0
|
deleted["tags"] = t[0].cnt if t else 0
|
||||||
|
|
||||||
# Delete documents
|
# Documents
|
||||||
await sor.sqlExe(
|
await sor.sqlExe("DELETE FROM documents WHERE kb_id=${kb_id}$", {"kb_id": kb_id})
|
||||||
"DELETE FROM documents WHERE kb_id=${kb_id}$",
|
|
||||||
{"kb_id": kb_id})
|
|
||||||
deleted["documents"] = len(doc_ids)
|
deleted["documents"] = len(doc_ids)
|
||||||
|
|
||||||
# Delete the KB itself
|
# KB itself
|
||||||
await sor.sqlExe(
|
await sor.sqlExe("DELETE FROM knowledge_bases WHERE id=${id}$", {"id": kb_id})
|
||||||
"DELETE FROM knowledge_bases WHERE id=${id}$",
|
|
||||||
{"id": kb_id})
|
|
||||||
|
|
||||||
# Delete physical files
|
# Physical files — paths are like /44/126/174/61/file.mp4
|
||||||
for d in docs:
|
for d in docs:
|
||||||
fp = d.file_path or ''
|
fp = d.file_path or ''
|
||||||
if fp and fp.startswith("/idfile/"):
|
if fp and fp.startswith("/"):
|
||||||
try:
|
try:
|
||||||
real_path = env.realpath(fp)
|
real_path = env.realpath(fp)
|
||||||
if os.path.exists(real_path):
|
if os.path.exists(real_path):
|
||||||
@ -74,5 +100,6 @@ for d in docs:
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
info(f'delete_kb {kb_id}: {deleted}')
|
||||||
return {"widgettype": "urlwidget",
|
return {"widgettype": "urlwidget",
|
||||||
"options": {"url": entire_url('/rag/knowledge_bases_list/index.ui')}}
|
"options": {"url": entire_url('/rag/knowledge_bases_list/index.ui')}}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user