From 71ecb8fea1848fe1950acd2a9c7ce061f2811925 Mon Sep 17 00:00:00 2001 From: yumoqing Date: Sat, 1 Aug 2026 10:39:52 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20add=20hotspot=20radar=20module=20(?= =?UTF-8?q?=E7=83=AD=E7=82=B9=E9=9B=B7=E8=BE=BE=E7=B3=BB=E7=BB=9F)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add hotspot_source/hotspot_item/hotspot_analysis table definitions - Add MySQL DDL for all three tables with indexes - Create hotspot dashboard UI (index.ui) with stats, tabs for CRUD - Add DSPY endpoints: stats, fetch_now (RSS/API), analyze (auto-classify) - Register paths in load_path.py with logined access - Add '热点雷达' menu entry --- json/hotspot_analysis.json | 19 ++++ json/hotspot_item.json | 22 +++++ json/hotspot_source.json | 20 ++++ load_path.py | 23 +++++ models/mysql.ddl.sql | 66 +++++++++++++ wwwroot/hotspot/analyze.dspy | 55 +++++++++++ wwwroot/hotspot/fetch_now.dspy | 165 +++++++++++++++++++++++++++++++++ wwwroot/hotspot/index.ui | 161 ++++++++++++++++++++++++++++++++ wwwroot/hotspot/stats.dspy | 24 +++++ wwwroot/menu.ui | 7 +- 10 files changed, 561 insertions(+), 1 deletion(-) create mode 100644 json/hotspot_analysis.json create mode 100644 json/hotspot_item.json create mode 100644 json/hotspot_source.json create mode 100644 wwwroot/hotspot/analyze.dspy create mode 100644 wwwroot/hotspot/fetch_now.dspy create mode 100644 wwwroot/hotspot/index.ui create mode 100644 wwwroot/hotspot/stats.dspy diff --git a/json/hotspot_analysis.json b/json/hotspot_analysis.json new file mode 100644 index 00000000..67c97f2c --- /dev/null +++ b/json/hotspot_analysis.json @@ -0,0 +1,19 @@ +{ + "models_dir": "${HOME}$/py/sage/models", + "output_dir": "${HOME}$/py/sage/wwwroot/_a/hotspot_analysis", + "dbname": "sage", + "tblname": "hotspot_analysis", + "title": "热点分析", + "params": { + "sortby": "analyzed_at desc", + "browserfields": { + "exclouded": ["id"], + "cwidth": {} + }, + "editexclouded": [ + "id", + "analyzed_at" + ], + "record_toolbar": null + } +} diff --git a/json/hotspot_item.json b/json/hotspot_item.json new file mode 100644 index 00000000..6a43eca4 --- /dev/null +++ b/json/hotspot_item.json @@ -0,0 +1,22 @@ +{ + "models_dir": "${HOME}$/py/sage/models", + "output_dir": "${HOME}$/py/sage/wwwroot/_a/hotspot_item", + "dbname": "sage", + "tblname": "hotspot_item", + "title": "热点条目", + "params": { + "sortby": "heat_score desc", + "browserfields": { + "exclouded": ["id", "content"], + "cwidth": { + "title": 300, + "summary": 400 + } + }, + "editexclouded": [ + "id", + "fetch_time" + ], + "record_toolbar": null + } +} diff --git a/json/hotspot_source.json b/json/hotspot_source.json new file mode 100644 index 00000000..e9b124fa --- /dev/null +++ b/json/hotspot_source.json @@ -0,0 +1,20 @@ +{ + "models_dir": "${HOME}$/py/sage/models", + "output_dir": "${HOME}$/py/sage/wwwroot/_a/hotspot_source", + "dbname": "sage", + "tblname": "hotspot_source", + "title": "热点来源", + "params": { + "sortby": "name", + "browserfields": { + "exclouded": ["id"], + "cwidth": {} + }, + "editexclouded": [ + "id", + "last_fetch", + "created_at" + ], + "record_toolbar": null + } +} diff --git a/load_path.py b/load_path.py index 4ab67663..b846951d 100644 --- a/load_path.py +++ b/load_path.py @@ -1079,6 +1079,29 @@ paths="""/ any /unipay/imgs/payment_log.svg any /unipay/usermenu.ui logined /unipay/refund.ui customer.* +/hotspot logined +/hotspot/index.ui logined +/hotspot/stats.dspy logined +/hotspot/fetch_now.dspy logined +/hotspot/analyze.dspy logined +/hotspot_source logined +/hotspot_source/index.ui logined +/hotspot_source/get_hotspot_source.dspy logined +/hotspot_source/add_hotspot_source.dspy logined +/hotspot_source/update_hotspot_source.dspy logined +/hotspot_source/delete_hotspot_source.dspy logined +/hotspot_item logined +/hotspot_item/index.ui logined +/hotspot_item/get_hotspot_item.dspy logined +/hotspot_item/add_hotspot_item.dspy logined +/hotspot_item/update_hotspot_item.dspy logined +/hotspot_item/delete_hotspot_item.dspy logined +/hotspot_analysis logined +/hotspot_analysis/index.ui logined +/hotspot_analysis/get_hotspot_analysis.dspy logined +/hotspot_analysis/add_hotspot_analysis.dspy logined +/hotspot_analysis/update_hotspot_analysis.dspy logined +/hotspot_analysis/delete_hotspot_analysis.dspy logined """ diff --git a/models/mysql.ddl.sql b/models/mysql.ddl.sql index ba5f3b38..6992ec0c 100644 --- a/models/mysql.ddl.sql +++ b/models/mysql.ddl.sql @@ -6578,4 +6578,70 @@ default charset=utf8 comment '模型供应商' ; +-- ./hotspot_source +drop table if exists hotspot_source; +CREATE TABLE hotspot_source +( + `id` VARCHAR(32) comment 'id', + `name` VARCHAR(200) comment '来源名称', + `type` VARCHAR(20) comment '类型: rss/api/web', + `url` VARCHAR(2000) comment '来源URL', + `fetch_method` VARCHAR(20) DEFAULT 'GET' comment '请求方法', + `headers` TEXT comment '自定义请求头 JSON', + `enabled` VARCHAR(1) DEFAULT '1' comment '是否启用', + `fetch_interval_min` INT DEFAULT 60 comment '抓取间隔(分钟)', + `last_fetch` VARCHAR(50) comment '上次抓取时间', + `created_at` VARCHAR(50) comment '创建时间' +,primary key(id) +) +engine=innodb +default charset=utf8 +comment '热点来源' +; + +-- ./hotspot_item +drop table if exists hotspot_item; +CREATE TABLE hotspot_item +( + `id` VARCHAR(32) comment 'id', + `source_id` VARCHAR(32) comment '来源ID', + `title` VARCHAR(500) comment '标题', + `url` VARCHAR(2000) comment '原文链接', + `summary` TEXT comment '摘要', + `content` TEXT comment '正文', + `heat_score` INT DEFAULT 0 comment '热度分数', + `publish_time` VARCHAR(50) comment '发布时间', + `fetch_time` VARCHAR(50) comment '抓取时间', + `status` VARCHAR(20) DEFAULT 'rising' comment '状态: rising/hot/cooling/expired', + `category` VARCHAR(100) comment '分类', + `tags` VARCHAR(500) comment '标签逗号分隔' +,primary key(id) +) +engine=innodb +default charset=utf8 +comment '热点条目' +; + +CREATE INDEX hotspot_item_source_idx ON hotspot_item(source_id); +CREATE INDEX hotspot_item_status_idx ON hotspot_item(status); + +-- ./hotspot_analysis +drop table if exists hotspot_analysis; +CREATE TABLE hotspot_analysis +( + `id` VARCHAR(32) comment 'id', + `item_id` VARCHAR(32) comment '关联热点ID', + `dimension` VARCHAR(50) comment '分析维度', + `score` DOUBLE(10,2) DEFAULT 0 comment '维度得分', + `analysis_data` TEXT comment '分析数据JSON', + `analyzed_at` VARCHAR(50) comment '分析时间' +,primary key(id) +) +engine=innodb +default charset=utf8 +comment '热点分析' +; + +CREATE INDEX hotspot_analysis_item_idx ON hotspot_analysis(item_id); + diff --git a/wwwroot/hotspot/analyze.dspy b/wwwroot/hotspot/analyze.dspy new file mode 100644 index 00000000..058737c2 --- /dev/null +++ b/wwwroot/hotspot/analyze.dspy @@ -0,0 +1,55 @@ +# 自动分析热点状态：根据时间和热度自动分类 +# rising -> hot -> cooling -> expired +from datetime import datetime, timedelta +from sqlor.dbpools import DBPools + +async def main(request): + db = DBPools() + now = datetime.now() + analyzed = 0 + + async with db.sqlorContext('sage') as sor: + items = await sor.sqlExe( + "select * from hotspot_item order by fetch_time desc", + {} + ) + + for item in items: + try: + fetch_time = datetime.strptime(str(item.get('fetch_time', '')), '%Y-%m-%d %H:%M:%S') + except: + fetch_time = now + + hours_ago = (now - fetch_time).total_seconds() / 3600 + heat = int(item.get('heat_score', 0)) + current_status = item.get('status', 'rising') + + # 热度衰减计算 + decay_factor = max(0, 1 - hours_ago / 168) # 7天衰减到0 + adjusted_heat = heat * decay_factor + + # 状态分类 + if hours_ago > 168: # 超过7天 + new_status = 'expired' + elif hours_ago > 72: # 超过3天 + new_status = 'cooling' + elif adjusted_heat > 500: + new_status = 'hot' + elif adjusted_heat > 100: + new_status = 'rising' + else: + new_status = 'cooling' if hours_ago > 24 else 'rising' + + if new_status != current_status: + async with db.sqlorContext('sage') as sor: + await sor.U('hotspot_item', { + 'id': item['id'], + 'status': new_status, + 'heat_score': int(adjusted_heat), + }) + analyzed += 1 + + return { + 'analyzed': analyzed, + 'time': now.strftime('%Y-%m-%d %H:%M:%S'), + } diff --git a/wwwroot/hotspot/fetch_now.dspy b/wwwroot/hotspot/fetch_now.dspy new file mode 100644 index 00000000..075618a3 --- /dev/null +++ b/wwwroot/hotspot/fetch_now.dspy @@ -0,0 +1,165 @@ +# 从所有启用的来源抓取热点数据 +import json +import xml.etree.ElementTree as ET +from datetime import datetime +from appPublic.uniqueID import getID +from sqlor.dbpools import DBPools + +async def fetch_rss(url): + """解析 RSS/Atom feed""" + import aiohttp + items = [] + try: + async with aiohttp.ClientSession() as session: + async with session.get(url, timeout=aiohttp.ClientTimeout(total=30)) as resp: + if resp.status != 200: + return items + text = await resp.text() + + root = ET.fromstring(text) + ns = {'atom': 'http://www.w3.org/2005/Atom'} + + # RSS 2.0 + for item in root.findall('.//item'): + title = item.findtext('title', '') + link = item.findtext('link', '') + desc = item.findtext('description', '')[:500] + pub_date = item.findtext('pubDate', '') + items.append({ + 'title': title, + 'url': link, + 'summary': desc, + 'publish_time': pub_date, + }) + + # Atom + if not items: + for entry in root.findall('.//atom:entry', ns): + title = entry.findtext('atom:title', '', ns) + link_el = entry.find('atom:link', ns) + link = link_el.get('href', '') if link_el is not None else '' + summary = entry.findtext('atom:summary', '', ns) or '' + updated = entry.findtext('atom:updated', '', ns) or '' + items.append({ + 'title': title, + 'url': link, + 'summary': summary[:500], + 'publish_time': updated, + }) + except Exception as e: + print(f'RSS fetch error for {url}: {e}') + return items + +async def fetch_api(url, method='GET', headers=None, json_path=None): + """调用 API 端点获取热点数据""" + import aiohttp + items = [] + try: + hdrs = {} + if headers: + try: + hdrs = json.loads(headers) + except: + pass + + async with aiohttp.ClientSession() as session: + if method.upper() == 'GET': + async with session.get(url, headers=hdrs, timeout=aiohttp.ClientTimeout(total=30)) as resp: + if resp.status == 200: + data = await resp.json() + else: + async with session.post(url, headers=hdrs, timeout=aiohttp.ClientTimeout(total=30)) as resp: + if resp.status == 200: + data = await resp.json() + + # 尝试从 JSON 中提取 items + if isinstance(data, list): + entries = data + elif isinstance(data, dict): + # 常见热点 API 格式 + entries = data.get('data', data.get('items', data.get('list', []))) + else: + entries = [] + + for entry in entries[:50]: # limit + if isinstance(entry, dict): + title = entry.get('title', entry.get('name', '')) + url_val = entry.get('url', entry.get('link', '')) + summary = entry.get('summary', entry.get('description', entry.get('desc', ''))) + heat = entry.get('heat', entry.get('hot', entry.get('score', 0))) + items.append({ + 'title': str(title), + 'url': str(url_val), + 'summary': str(summary)[:500], + 'heat_score': int(heat) if heat else 0, + }) + except Exception as e: + print(f'API fetch error for {url}: {e}') + return items + +async def main(request): + db = DBPools() + now = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + new_items = 0 + total_sources = 0 + + async with db.sqlorContext('sage') as sor: + sources = await sor.R('hotspot_source', {}) + + for src in sources: + if src.get('enabled') != '1': + continue + + total_sources += 1 + src_type = src.get('type', 'rss') + src_url = src.get('url', '') + + if not src_url: + continue + + if src_type == 'rss': + items = await fetch_rss(src_url) + elif src_type == 'api': + fetch_method = src.get('fetch_method', 'GET') + headers = src.get('headers', '') + items = await fetch_api(src_url, fetch_method, headers) + else: + continue + + # 存入数据库 + async with db.sqlorContext('sage') as sor: + # 更新 last_fetch + await sor.U('hotspot_source', { + 'id': src['id'], + 'last_fetch': now, + }) + + for item in items: + # 去重：检查 URL 是否已存在 + existing = await sor.sqlExe( + "select id from hotspot_item where url=${url}$", + {'url': item.get('url', '')} + ) + if existing: + continue + + new_id = getID() + status = 'rising' # 新抓取的默认 rising + await sor.C('hotspot_item', { + 'id': new_id, + 'source_id': src['id'], + 'title': item.get('title', ''), + 'url': item.get('url', ''), + 'summary': item.get('summary', '')[:500], + 'heat_score': item.get('heat_score', 100), + 'publish_time': item.get('publish_time', now), + 'fetch_time': now, + 'status': status, + }) + new_items += 1 + + return { + 'total': total_sources, + 'new_items': new_items, + 'time': now, + } diff --git a/wwwroot/hotspot/index.ui b/wwwroot/hotspot/index.ui new file mode 100644 index 00000000..e20fd95b --- /dev/null +++ b/wwwroot/hotspot/index.ui @@ -0,0 +1,161 @@ +{% set roles = get_user_roles(get_user()) %} +{ + "widgettype": "VBox", + "options": { + "children": [ + { + "widgettype": "Text", + "options": { + "text": "热点雷达系统", + "style": "font-size:24px;font-weight:bold;padding:16px 16px 0 16px;" + } + }, + { + "widgettype": "Text", + "options": { + "text": "指定来源的热点聚合分析平台", + "style": "font-size:14px;color:#999;padding:0 16px 16px 16px;" + } + }, + { + "widgettype": "HBox", + "options": { + "children": [ + { + "widgettype": "VBox", + "options": { + "id": "stat_total", + "children": [ + {"widgettype": "Text", "options": {"text": "热点总数", "style": "color:#666;font-size:12px;"}}, + {"widgettype": "Text", "options": {"text": "0", "style": "font-size:28px;font-weight:bold;color:#1890ff;"}} + ], + "style": "padding:16px;background:#f0f5ff;border-radius:8px;margin:8px;min-width:100px;text-align:center;" + } + }, + { + "widgettype": "VBox", + "options": { + "id": "stat_rising", + "children": [ + {"widgettype": "Text", "options": {"text": "上升中", "style": "color:#666;font-size:12px;"}}, + {"widgettype": "Text", "options": {"text": "0", "style": "font-size:28px;font-weight:bold;color:#52c41a;"}} + ], + "style": "padding:16px;background:#f6ffed;border-radius:8px;margin:8px;min-width:100px;text-align:center;" + } + }, + { + "widgettype": "VBox", + "options": { + "id": "stat_hot", + "children": [ + {"widgettype": "Text", "options": {"text": "正热点", "style": "color:#666;font-size:12px;"}}, + {"widgettype": "Text", "options": {"text": "0", "style": "font-size:28px;font-weight:bold;color:#ff4d4f;"}} + ], + "style": "padding:16px;background:#fff2f0;border-radius:8px;margin:8px;min-width:100px;text-align:center;" + } + }, + { + "widgettype": "VBox", + "options": { + "id": "stat_cooling", + "children": [ + {"widgettype": "Text", "options": {"text": "降温中", "style": "color:#666;font-size:12px;"}}, + {"widgettype": "Text", "options": {"text": "0", "style": "font-size:28px;font-weight:bold;color:#faad14;"}} + ], + "style": "padding:16px;background:#fffbe6;border-radius:8px;margin:8px;min-width:100px;text-align:center;" + } + }, + { + "widgettype": "VBox", + "options": { + "id": "stat_expired", + "children": [ + {"widgettype": "Text", "options": {"text": "已过期", "style": "color:#666;font-size:12px;"}}, + {"widgettype": "Text", "options": {"text": "0", "style": "font-size:28px;font-weight:bold;color:#999;"}} + ], + "style": "padding:16px;background:#f5f5f5;border-radius:8px;margin:8px;min-width:100px;text-align:center;" + } + }, + { + "widgettype": "VBox", + "options": { + "id": "stat_sources", + "children": [ + {"widgettype": "Text", "options": {"text": "数据来源", "style": "color:#666;font-size:12px;"}}, + {"widgettype": "Text", "options": {"text": "0", "style": "font-size:28px;font-weight:bold;color:#722ed1;"}} + ], + "style": "padding:16px;background:#f9f0ff;border-radius:8px;margin:8px;min-width:100px;text-align:center;" + } + } + ] + } + }, + { + "widgettype": "HBox", + "options": { + "children": [ + { + "widgettype": "Button", + "options": { + "label": "立即抓取全部来源", + "actiontype": "script", + "script": "fetch('/hotspot/fetch_now.dspy').then(r=>r.json()).then(d=>{alert('抓取完成: 处理'+d.total+'个来源，新增'+d.new_items+'条');setTimeout(()=>location.reload(),1000);}).catch(e=>alert('抓取出错: '+e))", + "style": "margin:8px;" + } + }, + { + "widgettype": "Button", + "options": { + "label": "刷新统计", + "actiontype": "script", + "script": "fetch('/hotspot/stats.dspy').then(r=>r.json()).then(d=>{document.querySelectorAll('[id^=stat_]').forEach(el=>{const num=el.querySelectorAll('div')[1];if(num){const k=el.id.replace('stat_','');num.textContent=d[k]||0;}});}).catch(e=>console.error(e))", + "style": "margin:8px;" + } + } + ] + } + }, + { + "widgettype": "Tab", + "options": { + "tabs": [ + { + "title": "热点条目", + "content": { + "widgettype": "urlwidget", + "options": { + "url": "{{ entire_url('hotspot_item') }}" + } + } + }, + { + "title": "来源管理", + "content": { + "widgettype": "urlwidget", + "options": { + "url": "{{ entire_url('hotspot_source') }}" + } + } + }, + { + "title": "分析记录", + "content": { + "widgettype": "urlwidget", + "options": { + "url": "{{ entire_url('hotspot_analysis') }}" + } + } + } + ] + } + }, + { + "widgettype": "Text", + "options": { + "text": "提示：在「来源管理」中添加 RSS/API 来源后，点击「立即抓取」获取热点数据。", + "style": "font-size:12px;color:#bbb;padding:16px;" + } + } + ] + } +} diff --git a/wwwroot/hotspot/stats.dspy b/wwwroot/hotspot/stats.dspy new file mode 100644 index 00000000..175cc6ee --- /dev/null +++ b/wwwroot/hotspot/stats.dspy @@ -0,0 +1,24 @@ +from appPublic.uniqueID import getID +from sqlor.dbpools import DBPools + +async def main(request): + db = DBPools() + async with db.sqlorContext('sage') as sor: + items = await sor.R('hotspot_item', {'order': 'fetch_time desc'}) + sources = await sor.R('hotspot_source', {}) + + total = len(items) + rising = sum(1 for i in items if i.get('status') == 'rising') + hot = sum(1 for i in items if i.get('status') == 'hot') + cooling = sum(1 for i in items if i.get('status') == 'cooling') + expired = sum(1 for i in items if i.get('status') == 'expired') + sources_count = len([s for s in sources if s.get('enabled') == '1']) + + return { + 'total': total, + 'rising': rising, + 'hot': hot, + 'cooling': cooling, + 'expired': expired, + 'sources': sources_count + } diff --git a/wwwroot/menu.ui b/wwwroot/menu.ui index 7ad8acd5..9a9fc99d 100644 --- a/wwwroot/menu.ui +++ b/wwwroot/menu.ui @@ -172,10 +172,15 @@ } {% endif %} ,{ + "name": "hotspot", + "label": "热点雷达", + "url": "{{entire_url('hotspot')}}" + }, + { "name": "skillmgr", "label": "技能管理", "submenu": "{{entire_url('/skillagent/menu.ui')}}" - } + } {% if 'reseller.sale' in roles %} ,{ "name": "reseller",