yumoqing/rag
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: GPU URLs need :10443, create VDB collection before upsert, align VDB id with chunk id

Browse Source
This commit is contained in:
yumoqing 2026-08-03 17:02:06 +08:00
parent 598f5c5c23
commit 84448259e2
2 changed files with 22 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
wwwroot/knowledge_bases_list/batch_ingest.dspy
View File

@ -183,8 +183,10 @@ async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=60)) as ses
# VDB # VDB
if embeddings: if embeddings:
try: try:
await session.post('https://vectordb.opencomputing.net:10443/v1/createcollection',
json={"colname": kb_id, "fields": [{"name": "id", "type": "str", "is_primary": True, "max_length": 64}, {"name": "vector", "type": "fvector", "dim": 1024}, {"name": "text", "type": "str", "max_length": 65535}], "description": "RAG kb", "metric": "COSINE"})
await session.post('https://vectordb.opencomputing.net:10443/v1/upsert', await session.post('https://vectordb.opencomputing.net:10443/v1/upsert',
json={"colname": kb_id, "data": [{"id": f"{fid}_{i}", "vector": e, "text": chunks[i]} for i, e in enumerate(embeddings)]}) json={"colname": kb_id, "data": [{"id": fid + "_c" + str(i), "vector": e, "text": chunks[i]} for i, e in enumerate(embeddings)]})
except: pass except: pass
# DB # DB

30
wwwroot/knowledge_bases_list/upload_file.dspy
View File

@ -30,6 +30,11 @@ ext_l = ext.lower()
# ============================================================ # ============================================================
async def ingest_doc(doc_id, kb_id, file_name, ext_l, real_path): async def ingest_doc(doc_id, kb_id, file_name, ext_l, real_path):
db = DBPools() db = DBPools()
async def ensure_vdb_collection(client, kb_id):
payload = {"colname": kb_id, "fields": [{"name": "id", "type": "str", "is_primary": True, "max_length": 64}, {"name": "vector", "type": "fvector", "dim": 1024}, {"name": "text", "type": "str", "max_length": 65535}], "description": "RAG kb", "metric": "COSINE"}
await client.request('POST', 'https://vectordb.opencomputing.net:10443/v1/createcollection', json=payload)
text = '' text = ''
chunks_n = 0 chunks_n = 0
face_count = 0 face_count = 0
@ -80,7 +85,7 @@ async def ingest_doc(doc_id, kb_id, file_name, ext_l, real_path):
img_b64 = base64.b64encode(file_data).decode() img_b64 = base64.b64encode(file_data).decode()
try: try:
client = StreamHttpClient() client = StreamHttpClient()
resp = await client.request('POST', 'https://media.opencomputing.net/face/api/detect', json={"images": [img_b64]}) resp = await client.request('POST', 'https://media.opencomputing.net:10443/face/api/detect', json={"images": [img_b64]})
fd = json.loads(resp) fd = json.loads(resp)
results = fd.get("results", []) results = fd.get("results", [])
if results and isinstance(results[0], dict): if results and isinstance(results[0], dict):
@ -92,7 +97,7 @@ async def ingest_doc(doc_id, kb_id, file_name, ext_l, real_path):
if ext_l in audio_exts: if ext_l in audio_exts:
try: try:
client = StreamHttpClient() client = StreamHttpClient()
resp = await client.request('POST', 'https://media.opencomputing.net/voiceprint/extract/submit', resp = await client.request('POST', 'https://media.opencomputing.net:10443/voiceprint/extract/submit',
files={'file': (file_name, file_data)}) files={'file': (file_name, file_data)})
vd = json.loads(resp) vd = json.loads(resp)
voice_speakers = vd.get('speakers', 1) if vd.get('status') == 'SUCCEEDED' else (1 if vd.get('embedding') else 0) voice_speakers = vd.get('speakers', 1) if vd.get('status') == 'SUCCEEDED' else (1 if vd.get('embedding') else 0)
@ -112,7 +117,7 @@ async def ingest_doc(doc_id, kb_id, file_name, ext_l, real_path):
img_b64 = base64.b64encode(frame_data).decode() img_b64 = base64.b64encode(frame_data).decode()
try: try:
client = StreamHttpClient() client = StreamHttpClient()
resp = await client.request('POST', 'https://media.opencomputing.net/face/api/detect', json={"images": [img_b64]}) resp = await client.request('POST', 'https://media.opencomputing.net:10443/face/api/detect', json={"images": [img_b64]})
fd = json.loads(resp) fd = json.loads(resp)
results = fd.get("results", []) results = fd.get("results", [])
if results and isinstance(results[0], dict): if results and isinstance(results[0], dict):
@ -121,7 +126,7 @@ async def ingest_doc(doc_id, kb_id, file_name, ext_l, real_path):
# --- CLIP image embedding for video frame --- # --- CLIP image embedding for video frame ---
try: try:
client2 = StreamHttpClient() client2 = StreamHttpClient()
resp2 = await client2.request('POST', 'https://embedding.opencomputing.net/api/embed', resp2 = await client2.request('POST', 'https://embedding.opencomputing.net:10443/api/embed',
json={"images": [img_b64], "model": "CLIP-ViT-H-14"}) json={"images": [img_b64], "model": "CLIP-ViT-H-14"})
emb_data = json.loads(resp2) emb_data = json.loads(resp2)
img_embeddings = emb_data.get("image_embeddings", emb_data.get("embeddings", [])) img_embeddings = emb_data.get("image_embeddings", emb_data.get("embeddings", []))
@ -130,16 +135,17 @@ async def ingest_doc(doc_id, kb_id, file_name, ext_l, real_path):
if img_embeddings: if img_embeddings:
try: try:
client3 = StreamHttpClient() client3 = StreamHttpClient()
await ensure_vdb_collection(client3, kb_id)
vdb_data = {"colname": kb_id, "data": [ vdb_data = {"colname": kb_id, "data": [
{"id": doc_id + "_frame0", "vector": img_embeddings[0], "text": file_name} {"id": doc_id + "_c0", "vector": img_embeddings[0], "text": file_name}
]} ]}
await client3.request('POST', 'https://vectordb.opencomputing.net/v1/upsert', json=vdb_data) await client3.request('POST', 'https://vectordb.opencomputing.net:10443/v1/upsert', json=vdb_data)
async with db.sqlorContext('rag') as sor: async with db.sqlorContext('rag') as sor:
await sor.sqlExe( await sor.sqlExe(
"INSERT INTO document_chunks (id, doc_id, kb_id, chunk_index, content, vector_id, created_at) " "INSERT INTO document_chunks (id, doc_id, kb_id, chunk_index, content, vector_id, created_at) "
"VALUES (${id}$, ${doc_id}$, ${kb_id}$, 0, ${content}$, ${vid}$, NOW())", "VALUES (${id}$, ${doc_id}$, ${kb_id}$, 0, ${content}$, ${vid}$, NOW())",
{"id": doc_id + "_c0", "doc_id": doc_id, "kb_id": kb_id, {"id": doc_id + "_c0", "doc_id": doc_id, "kb_id": kb_id,
"content": file_name, "vid": doc_id + "_frame0"}) "content": file_name, "vid": doc_id + "_c0"})
except: except:
pass pass
os.remove(tmp_img) os.remove(tmp_img)
@ -165,7 +171,7 @@ async def ingest_doc(doc_id, kb_id, file_name, ext_l, real_path):
if chunks: if chunks:
try: try:
client = StreamHttpClient() client = StreamHttpClient()
resp = await client.request('POST', 'https://embedding.opencomputing.net/api/embed', resp = await client.request('POST', 'https://embedding.opencomputing.net:10443/api/embed',
json={"texts": chunks, "model": "CLIP-ViT-H-14"}) json={"texts": chunks, "model": "CLIP-ViT-H-14"})
emb_data = json.loads(resp) emb_data = json.loads(resp)
embeddings = emb_data.get("text_embeddings", emb_data.get("embeddings", [])) embeddings = emb_data.get("text_embeddings", emb_data.get("embeddings", []))
@ -176,11 +182,13 @@ async def ingest_doc(doc_id, kb_id, file_name, ext_l, real_path):
if embeddings: if embeddings:
try: try:
client2 = StreamHttpClient() client2 = StreamHttpClient()
await ensure_vdb_collection(client2, kb_id)
vdb_data = {"colname": kb_id, "data": [ vdb_data = {"colname": kb_id, "data": [
{"id": doc_id + "_" + str(i), "vector": emb, "text": chunks[i]} {"id": doc_id + "_c" + str(i), "vector": emb, "text": chunks[i]}
for i, emb in enumerate(embeddings)]} for i, emb in enumerate(embeddings)]}
await client2.request('POST', 'https://vectordb.opencomputing.net/v1/upsert', json=vdb_data) resp3 = await client2.request('POST', 'https://vectordb.opencomputing.net:10443/v1/upsert', json=vdb_data)
vector_ids = [doc_id + "_" + str(i) for i in range(len(embeddings))] if json.loads(resp3).get('status') == 'SUCCEEDED':
vector_ids = [doc_id + "_c" + str(i) for i in range(len(embeddings))]
except: except:
pass pass