fix: Sage-compliant DSPY — no imports, env delegation, init.py, scripts/load_path.py

- Add hotspot/init.py: load_hotspot() registers run_fetch/run_analysis/get_stats on ServerEnv
- Rewrite .dspy files: only import json, use get_sor_context + env.*() delegation
- Update db.py/engine.py/analysis.py: accept optional sor parameter for DSPY context
- Add scripts/load_path.py: per-module RBAC permission registration
- Follows sage-dspy-development + rbac-permission-initialization-pattern skills
This commit is contained in:
yumoqing 2026-08-01 16:49:49 +08:00
parent b9bd705b43
commit 7bb9bfc97e
9 changed files with 213 additions and 75 deletions

View File

@ -1,14 +1,15 @@
""" """
热点雷达 (Hotspot Radar) Sage 框架模块 热点雷达 (Hotspot Radar) Sage 框架模块
用法: 集成方式:
from hotspot.engine import run_fetch # sage/app/sage.py
from hotspot.analysis import run_analysis from hotspot.init import load_hotspot
from hotspot.db import get_stats # 在 init() 中添加: load_hotspot()
""" """
from .init import load_hotspot
from .engine import run_fetch from .engine import run_fetch
from .analysis import run_analysis from .analysis import run_analysis
from .db import get_stats from .db import get_stats
from .parsers import BUILTIN as builtin_parsers from .parsers import BUILTIN as builtin_parsers
__all__ = ['run_fetch', 'run_analysis', 'get_stats', 'builtin_parsers'] __all__ = ['load_hotspot', 'run_fetch', 'run_analysis', 'get_stats', 'builtin_parsers']

View File

@ -94,13 +94,12 @@ async def analyze_item(item):
return new_status, dims, detail return new_status, dims, detail
async def save_analysis(item_id, dims, detail, now=None): async def save_analysis(item_id, dims, detail, now=None, sor=None):
"""保存/更新五维分析记录""" """保存/更新五维分析记录"""
if now is None: if now is None:
now = now_str() now = now_str()
db = DBPools()
async with db.sqlorContext('sage') as sor: async def _do(s):
for dim, score in dims.items(): for dim, score in dims.items():
analysis_data = json.dumps({ analysis_data = json.dumps({
'dimension': dim, 'dimension': dim,
@ -108,51 +107,71 @@ async def save_analysis(item_id, dims, detail, now=None):
'score': score, 'score': score,
'detail': detail, 'detail': detail,
}, ensure_ascii=False) }, ensure_ascii=False)
old = await s.sqlExe(
old = await sor.sqlExe(
"select id from hotspot_analysis where item_id=${iid}$ and dimension=${dim}$", "select id from hotspot_analysis where item_id=${iid}$ and dimension=${dim}$",
{'iid': item_id, 'dim': dim}) {'iid': item_id, 'dim': dim})
if old: if old:
await sor.U('hotspot_analysis', { await s.U('hotspot_analysis', {
'id': old[0]['id'], 'score': score, 'id': old[0]['id'], 'score': score,
'analysis_data': analysis_data, 'analyzed_at': now, 'analysis_data': analysis_data, 'analyzed_at': now,
}) })
else: else:
await sor.C('hotspot_analysis', { await s.C('hotspot_analysis', {
'id': getID(), 'item_id': item_id, 'id': getID(), 'item_id': item_id,
'dimension': dim, 'score': score, 'dimension': dim, 'score': score,
'analysis_data': analysis_data, 'analyzed_at': now, 'analysis_data': analysis_data, 'analyzed_at': now,
}) })
if sor:
await _do(sor)
else:
db = DBPools()
async with db.sqlorContext('sage') as s:
await _do(s)
async def run_analysis(limit=500):
"""批量分析最近的热点 (默认500条)""" async def run_analysis(limit=500, sor=None):
db = DBPools() """批量分析最近的热点 (默认500条)。sor 可选,由 DSPY 提供。"""
now = now_str() now = now_str()
results = {'analyzed': 0, 'status_changes': 0} results = {'analyzed': 0, 'status_changes': 0}
async with db.sqlorContext('sage') as sor: async def _get_items(s):
items = await sor.sqlExe( return await s.sqlExe(
"select * from hotspot_item order by first_seen desc limit ${n}$", "select * from hotspot_item order by first_seen desc limit ${n}$",
{'n': limit}) {'n': limit})
async def _update_item(s, item_id, adj_heat, velocity, new_status, now):
await s.U('hotspot_item', {
'id': item_id,
'heat_score': round(adj_heat, 2),
'heat_velocity': velocity,
'status': new_status,
'last_updated': now,
})
if sor:
items = await _get_items(sor)
else:
db = DBPools()
async with db.sqlorContext('sage') as s:
items = await _get_items(s)
for item in items: for item in items:
current_status = item.get('status', 'emerging') current_status = item.get('status', 'emerging')
new_status, dims, detail = await analyze_item(item) new_status, dims, detail = await analyze_item(item)
await save_analysis(item['id'], dims, detail, now, sor=sor)
await save_analysis(item['id'], dims, detail, now)
if new_status != current_status: if new_status != current_status:
results['status_changes'] += 1 results['status_changes'] += 1
async with db.sqlorContext('sage') as sor: adj_heat = detail['adjusted_heat']
await sor.U('hotspot_item', { velocity = detail['heat_velocity']
'id': item['id'], if sor:
'heat_score': round(detail['adjusted_heat'], 2), await _update_item(sor, item['id'], adj_heat, velocity, new_status, now)
'heat_velocity': detail['heat_velocity'], else:
'status': new_status, db = DBPools()
'last_updated': now, async with db.sqlorContext('sage') as s:
}) await _update_item(s, item['id'], adj_heat, velocity, new_status, now)
results['analyzed'] += 1 results['analyzed'] += 1

View File

@ -1,5 +1,6 @@
""" """
数据库操作: 保存热点条目写抓取日志统计查询 数据库操作: 保存热点条目写抓取日志统计查询
所有函数接受可选 sor 参数 DSPY get_sor_context 提供时不另开连接
""" """
from datetime import datetime from datetime import datetime
from appPublic.uniqueID import getID from appPublic.uniqueID import getID
@ -8,32 +9,38 @@ from sqlor.dbpools import DBPools
def now_str(): def now_str():
return datetime.now().strftime('%Y-%m-%d %H:%M:%S') return datetime.now().strftime('%Y-%m-%d %H:%M:%S')
async def get_enabled_sources(): async def _get_sor():
"""获取所有启用的来源""" """standalone 模式:创建自己的数据库连接"""
db = DBPools() db = DBPools()
async with db.sqlorContext('sage') as sor: return db.sqlorContext('sage')
async def get_enabled_sources(sor=None):
"""获取所有启用的来源"""
if sor:
sources = await sor.R('hotspot_source', {}) sources = await sor.R('hotspot_source', {})
return [s for s in sources if s.get('enabled') == '1']
async with await _get_sor() as s:
sources = await s.R('hotspot_source', {})
return [s for s in sources if s.get('enabled') == '1'] return [s for s in sources if s.get('enabled') == '1']
async def save_items(items, source_id, fetch_time=None): async def save_items(items, source_id, fetch_time=None, sor=None):
"""批量保存热点条目 (去重)""" """批量保存热点条目 (去重)"""
if fetch_time is None: if fetch_time is None:
fetch_time = now_str() fetch_time = now_str()
new_count = 0 new_count = 0
dup_count = 0 dup_count = 0
db = DBPools()
async with db.sqlorContext('sage') as sor: async def _do(s):
nonlocal new_count, dup_count
for item in items: for item in items:
url = item.get('url', '') url = item.get('url', '')
if url: if url:
existing = await sor.sqlExe( existing = await s.sqlExe(
"select id from hotspot_item where url=${u}$ limit 1", {'u': url}) "select id from hotspot_item where url=${u}$ limit 1", {'u': url})
if existing: if existing:
dup_count += 1 dup_count += 1
continue continue
await s.C('hotspot_item', {
await sor.C('hotspot_item', {
'id': getID(), 'source_id': source_id, 'id': getID(), 'source_id': source_id,
'title': item.get('title', '')[:500], 'title': item.get('title', '')[:500],
'url': (url or '')[:2000], 'url': (url or '')[:2000],
@ -53,46 +60,58 @@ async def save_items(items, source_id, fetch_time=None):
}) })
new_count += 1 new_count += 1
if sor:
await _do(sor)
else:
async with await _get_sor() as s:
await _do(s)
return new_count, dup_count return new_count, dup_count
async def update_source_last_fetch(source_id): async def update_source_last_fetch(source_id, sor=None):
db = DBPools() if sor:
async with db.sqlorContext('sage') as sor:
await sor.U('hotspot_source', {'id': source_id, 'last_fetch': now_str()}) await sor.U('hotspot_source', {'id': source_id, 'last_fetch': now_str()})
else:
async with await _get_sor() as s:
await s.U('hotspot_source', {'id': source_id, 'last_fetch': now_str()})
async def write_fetch_log(source_id, start_time, elapsed_ms, async def write_fetch_log(source_id, start_time, elapsed_ms,
status, items_total, items_new, items_duplicate, status, items_total, items_new, items_duplicate,
error_msg='', response_code=0, response_size_bytes=0): error_msg='', response_code=0, response_size_bytes=0,
db = DBPools() sor=None):
async with db.sqlorContext('sage') as sor: data = {
await sor.C('hotspot_fetch_log', { 'id': getID(), 'source_id': source_id,
'id': getID(), 'start_time': start_time, 'end_time': now_str(),
'source_id': source_id, 'duration_ms': elapsed_ms, 'status': status,
'start_time': start_time, 'items_total': items_total, 'items_new': items_new,
'end_time': now_str(), 'items_duplicate': items_duplicate,
'duration_ms': elapsed_ms, 'error_msg': error_msg[:1000] if error_msg else '',
'status': status, 'response_code': response_code, 'response_size_bytes': response_size_bytes,
'items_total': items_total, }
'items_new': items_new, if sor:
'items_duplicate': items_duplicate, await sor.C('hotspot_fetch_log', data)
'error_msg': error_msg[:1000] if error_msg else '', else:
'response_code': response_code, async with await _get_sor() as s:
'response_size_bytes': response_size_bytes, await s.C('hotspot_fetch_log', data)
})
async def get_stats(): async def get_stats(sor=None):
"""全维度统计""" """全维度统计"""
from datetime import timedelta from datetime import timedelta
now = datetime.now() now = datetime.now()
day_ago = (now - timedelta(hours=24)).strftime('%Y-%m-%d %H:%M:%S') day_ago = (now - timedelta(hours=24)).strftime('%Y-%m-%d %H:%M:%S')
db = DBPools() async def _do(s):
async with db.sqlorContext('sage') as sor: items = await s.R('hotspot_item', {'order': 'heat_score desc'})
items = await sor.R('hotspot_item', {'order': 'heat_score desc'}) sources = await s.R('hotspot_source', {})
sources = await sor.R('hotspot_source', {}) logs = await s.sqlExe(
logs = await sor.sqlExe(
"select * from hotspot_fetch_log where start_time >= ${t}$", "select * from hotspot_fetch_log where start_time >= ${t}$",
{'t': day_ago}) {'t': day_ago})
return items, sources, logs
if sor:
items, sources, logs = await _do(sor)
else:
async with await _get_sor() as s:
items, sources, logs = await _do(s)
status_count = {'emerging': 0, 'rising': 0, 'hot': 0, 'cooling': 0, 'expired': 0} status_count = {'emerging': 0, 'rising': 0, 'hot': 0, 'cooling': 0, 'expired': 0}
for item in items: for item in items:

View File

@ -108,9 +108,9 @@ async def fetch_and_extract(session, src):
return [], 0, 0 return [], 0, 0
async def run_fetch(): async def run_fetch(sor=None):
"""遍历所有启用的来源,执行抓取""" """遍历所有启用的来源,执行抓取。sor 可选,由 DSPY 的 get_sor_context 提供。"""
sources = await get_enabled_sources() sources = await get_enabled_sources(sor)
now = now_str() now = now_str()
results = {'total': len(sources), 'new_items': 0, 'errors': 0, 'details': []} results = {'total': len(sources), 'new_items': 0, 'errors': 0, 'details': []}
@ -124,18 +124,18 @@ async def run_fetch():
print(f'[hotspot] run_fetch error for {src.get("name",src["id"])}: {e}') print(f'[hotspot] run_fetch error for {src.get("name",src["id"])}: {e}')
traceback.print_exc() traceback.print_exc()
await write_fetch_log(src['id'], now, int((time.time() - t0) * 1000), await write_fetch_log(src['id'], now, int((time.time() - t0) * 1000),
'failed', 0, 0, 0, str(e)[:1000]) 'failed', 0, 0, 0, str(e)[:1000], sor=sor)
results['errors'] += 1 results['errors'] += 1
continue continue
elapsed = int((time.time() - t0) * 1000) elapsed = int((time.time() - t0) * 1000)
status = 'success' if code in (0, 200) else ('partial' if items else 'failed') status = 'success' if code in (0, 200) else ('partial' if items else 'failed')
new_count, dup_count = await save_items(items, src['id'], now) new_count, dup_count = await save_items(items, src['id'], now, sor=sor)
await update_source_last_fetch(src['id']) await update_source_last_fetch(src['id'], sor=sor)
await write_fetch_log(src['id'], now, elapsed, status, await write_fetch_log(src['id'], now, elapsed, status,
len(items), new_count, dup_count, len(items), new_count, dup_count,
response_code=code, response_size_bytes=size) response_code=code, response_size_bytes=size, sor=sor)
results['new_items'] += new_count results['new_items'] += new_count
results['details'].append({ results['details'].append({

20
hotspot/init.py Normal file
View File

@ -0,0 +1,20 @@
"""
热点雷达模块 Sage 框架集成入口
sage/app/sage.py 中添加:
from hotspot.init import load_hotspot
并在 init() 中调用: load_hotspot()
"""
from ahserver.serverenv import ServerEnv
from .engine import run_fetch
from .analysis import run_analysis
from .db import get_stats, get_enabled_sources
def load_hotspot():
"""注册热点雷达函数到 ServerEnv供 DSPY 和 Jinja2 模板调用"""
env = ServerEnv()
env.run_fetch = run_fetch
env.run_analysis = run_analysis
env.get_stats = get_stats
env.get_enabled_sources = get_enabled_sources

70
scripts/load_path.py Normal file
View File

@ -0,0 +1,70 @@
"""
热点雷达模块 独立权限注册脚本
Sage 环境中执行: python scripts/load_path.py
"""
import os, sys, subprocess
# 查找 Sage 根目录 (从当前模块位置推断)
def find_sage_root():
for candidate in [
os.path.expanduser('~/repos/sage'),
os.path.expanduser('~/sage'),
os.path.expanduser('~/py/sage'),
]:
if os.path.isdir(os.path.join(candidate, 'py3', 'bin')):
return candidate
# Fallback: 通过导入路径查找
try:
import load_path as _ # Sage 自己的 load_path.py
sage_root = os.path.dirname(os.path.abspath(_.__file__))
return sage_root
except ImportError:
pass
print('ERROR: Sage root not found. Set SAGE_ROOT env var.')
sys.exit(1)
SAGE_ROOT = os.environ.get('SAGE_ROOT', find_sage_root())
PY = os.path.join(SAGE_ROOT, 'py3', 'bin', 'python')
SET_ROLE_PERM = os.path.join(SAGE_ROOT, 'set_role_perm.py')
MOD = 'hotspot'
PATHS_LOGINED = [
f'/{MOD}',
f'/{MOD}/index.ui',
f'/{MOD}/stats.dspy',
f'/{MOD}/fetch_now.dspy',
f'/{MOD}/analyze.dspy',
# 6 张表的 CRUD
f'/{MOD}_source',
f'/{MOD}_source/%',
f'/{MOD}_schedule',
f'/{MOD}_schedule/%',
f'/{MOD}_fetch_log',
f'/{MOD}_fetch_log/%',
f'/{MOD}_item',
f'/{MOD}_item/%',
f'/{MOD}_analysis',
f'/{MOD}_analysis/%',
f'/{MOD}_alert',
f'/{MOD}_alert/%',
]
def set_perm(role, path):
env = os.environ.copy()
env['SAGE_RBAC_DB'] = 'sage'
subprocess.run(
[PY, SET_ROLE_PERM, role, path],
cwd=SAGE_ROOT, capture_output=True, env=env
)
def main():
total = 0
for p in PATHS_LOGINED:
set_perm('logined', p)
total += 1
print(f'Registered {total} hotspot paths')
print('Note: restart Sage for permissions to take effect')
if __name__ == '__main__':
main()

View File

@ -1,2 +1,5 @@
from hotspot.analysis import run_analysis import json
return await run_analysis() env = request._run_ns
async with get_sor_context(env, 'sage') as sor:
result = await env.run_analysis(sor)
return json.dumps(result, ensure_ascii=False)

View File

@ -1,2 +1,5 @@
from hotspot.engine import run_fetch import json
return await run_fetch() env = request._run_ns
async with get_sor_context(env, 'sage') as sor:
result = await env.run_fetch(sor)
return json.dumps(result, ensure_ascii=False)

View File

@ -1,2 +1,5 @@
from hotspot.db import get_stats import json
return await get_stats() env = request._run_ns
async with get_sor_context(env, 'sage') as sor:
result = await env.get_stats(sor)
return json.dumps(result, ensure_ascii=False)