From 26721a1b28401167afa9cf7210dbc38fa5aa2fb3 Mon Sep 17 00:00:00 2001 From: yumoqing Date: Sat, 1 Aug 2026 12:01:16 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20initial=20commit=20=E2=80=94=20hotspot?= =?UTF-8?q?=20radar=20module=20for=20Sage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 6 tables (source/schedule/fetch_log/item/analysis/alert) 3 fetch modes (API + Browser + Crawler) 5-dimension analysis + auto status classification 7 stat cards + 6 tab dashboard --- .gitignore | 3 + README.md | 92 ++++++++ ddl/mysql.sql | 145 ++++++++++++ install/load_path_append.txt | 41 ++++ install/menu_append.json | 5 + json/hotspot_alert.json | 16 ++ json/hotspot_analysis.json | 16 ++ json/hotspot_fetch_log.json | 16 ++ json/hotspot_item.json | 16 ++ json/hotspot_schedule.json | 16 ++ json/hotspot_source.json | 16 ++ wwwroot/hotspot/analyze.dspy | 205 ++++++++++++++++ wwwroot/hotspot/fetch_now.dspy | 419 +++++++++++++++++++++++++++++++++ wwwroot/hotspot/index.ui | 182 ++++++++++++++ wwwroot/hotspot/stats.dspy | 63 +++++ 15 files changed, 1251 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 ddl/mysql.sql create mode 100644 install/load_path_append.txt create mode 100644 install/menu_append.json create mode 100644 json/hotspot_alert.json create mode 100644 json/hotspot_analysis.json create mode 100644 json/hotspot_fetch_log.json create mode 100644 json/hotspot_item.json create mode 100644 json/hotspot_schedule.json create mode 100644 json/hotspot_source.json create mode 100644 wwwroot/hotspot/analyze.dspy create mode 100644 wwwroot/hotspot/fetch_now.dspy create mode 100644 wwwroot/hotspot/index.ui create mode 100644 wwwroot/hotspot/stats.dspy diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2348c91 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.DS_Store +__pycache__/ +*.pyc diff --git a/README.md b/README.md new file mode 100644 index 0000000..0231b76 --- /dev/null +++ b/README.md @@ -0,0 +1,92 @@ +# 热点雷达 (Hotspot Radar) + +Sage 框架模块 — 指定来源的热点聚合分析平台，支持动态添加来源。 + +## 功能 + +- **多类型来源**：API / RSS / Browser (HTML抓取) / Crawler (递归爬虫) +- **调度管理**：cron 定时 + 间隔模式，retry 策略 +- **抓取日志**：每次执行一条审计记录（耗时/成功数/错误/HTTP状态码） +- **热点分类**：5级状态 — emerging → rising → hot → cooling → expired +- **五维分析**：时效 / 热度 / 内容 / 传播 / 受众，每维度 0-100 分 +- **预警规则**：热度阈值 / 加速度 / 情感触发，email/webhook/sms 通知 + +## 目录结构 + +``` +hotspot/ +├── json/ # Sage 表定义（放入 Sage 的 json/ 目录） +│ ├── hotspot_source.json +│ ├── hotspot_schedule.json +│ ├── hotspot_fetch_log.json +│ ├── hotspot_item.json +│ ├── hotspot_analysis.json +│ └── hotspot_alert.json +├── wwwroot/hotspot/ # Web UI + 后端端点（放入 Sage 的 wwwroot/hotspot/） +│ ├── index.ui # 仪表盘（7 统计卡片 + 6 标签页） +│ ├── stats.dspy # 统计 API +│ ├── fetch_now.dspy # 抓取引擎（3种模式） +│ └── analyze.dspy # 分析引擎（5维 + 状态分类） +├── ddl/mysql.sql # 建表 SQL +├── install/ # 安装片段 +│ ├── load_path_append.txt # 追加到 load_path.py +│ └── menu_append.json # 追加到 menu.ui +└── README.md +``` + +## 安装 + +### 1. 复制文件到 Sage 部署目录 + +```bash +SAGE_ROOT=/path/to/sage +cp json/hotspot_*.json $SAGE_ROOT/json/ +cp -r wwwroot/hotspot $SAGE_ROOT/wwwroot/ +``` + +### 2. 建表 + +```bash +mysql -u user -p sage < ddl/mysql.sql +``` + +### 3. 注册权限 + +在 `load_path.py` 的 `"""` 结束符前追加 `install/load_path_append.txt` 的内容，然后运行： + +```bash +cd $SAGE_ROOT && python load_path.py +``` + +### 4. 添加菜单 + +在 `wwwroot/menu.ui` 的 `items` 数组中追加 `install/menu_append.json` 的内容。 + +### 5. 重启 Sage + +访问 `/hotspot` 即可看到仪表盘。 + +## 使用流程 + +1. **来源管理** 标签页 → 添加来源（选择 type: api/rss/browser/crawler） +2. **调度配置** 标签页 → 为来源配置抓取频率 +3. 仪表盘点击「立即抓取全部来源」 +4. **抓取日志** 标签页 → 检查执行结果 +5. 点击「执行分析」→ 自动计算五维评分和状态分类 +6. **热点条目** 标签页 → 按状态筛选浏览 +7. **预警规则** 标签页 → 设置自动告警 + +## 来源类型说明 + +| type | 适用场景 | 配置要点 | +|------|---------|---------| +| `rss` | RSS/Atom feed | url 指向 feed 地址 | +| `api` | REST API | url + api_key + headers + fetch_method | +| `browser` | 网页热点列表 | url + browser_wait_selector | +| `crawler` | 递归抓取 | url + crawler_depth + crawler_rules | + +## 依赖 + +- Sage >= 0.0.1 +- aiohttp (Python) +- MySQL / MariaDB diff --git a/ddl/mysql.sql b/ddl/mysql.sql new file mode 100644 index 0000000..51fab1e --- /dev/null +++ b/ddl/mysql.sql @@ -0,0 +1,145 @@ +-- ============================================ +-- 热点雷达系统 (Hotspot Radar) — 6 tables +-- ============================================ + +-- ./hotspot_source (来源配置: API / Browser / Crawler) +drop table if exists hotspot_source; +CREATE TABLE hotspot_source +( + `id` VARCHAR(32) comment 'id', + `name` VARCHAR(200) comment '来源名称', + `type` VARCHAR(20) comment '类型: rss/api/browser/crawler', + `url` VARCHAR(2000) comment '目标URL', + -- API 类型专用 + `api_key` VARCHAR(500) comment 'API Key', + `api_secret` VARCHAR(500) comment 'API Secret(加密)', + `headers` TEXT comment '自定义请求头 JSON', + `fetch_method` VARCHAR(10) DEFAULT 'GET' comment 'HTTP方法', + `request_body` TEXT comment 'POST请求体', + -- Browser 类型专用 + `browser_headless` VARCHAR(1) DEFAULT '1' comment '无头模式', + `browser_wait_selector` VARCHAR(500) comment '等待元素选择器', + `browser_scroll` VARCHAR(1) DEFAULT '0' comment '是否自动滚动', + -- Crawler 类型专用 + `crawler_depth` INT DEFAULT 1 comment '爬取深度', + `crawler_rules` TEXT comment '爬取规则 JSON', + `crawler_start_urls` TEXT comment '起始URL列表 JSON', + -- 通用 + `proxy_enabled` VARCHAR(1) DEFAULT '0' comment '启用代理', + `proxy_url` VARCHAR(500) comment '代理地址', + `enabled` VARCHAR(1) DEFAULT '1' comment '是否启用', + `priority` INT DEFAULT 0 comment '优先级(越大越高)', + `last_fetch` VARCHAR(50) comment '上次抓取时间', + `created_at` VARCHAR(50) comment '创建时间' +,primary key(id) +) +engine=innodb default charset=utf8 comment '热点来源配置' +; + +-- ./hotspot_schedule (调度配置) +drop table if exists hotspot_schedule; +CREATE TABLE hotspot_schedule +( + `id` VARCHAR(32) comment 'id', + `source_id` VARCHAR(32) comment '关联来源ID', + `cron_expr` VARCHAR(100) comment 'cron表达式', + `interval_min` INT DEFAULT 60 comment '间隔(分钟)', + `enabled` VARCHAR(1) DEFAULT '1' comment '是否启用', + `last_run` VARCHAR(50) comment '上次运行', + `next_run` VARCHAR(50) comment '下次运行', + `retry_max` INT DEFAULT 3 comment '最大重试', + `retry_count` INT DEFAULT 0 comment '已重试次数' +,primary key(id) +) +engine=innodb default charset=utf8 comment '调度配置' +; +CREATE INDEX hotspot_schedule_source_idx ON hotspot_schedule(source_id); + +-- ./hotspot_fetch_log (抓取日志) +drop table if exists hotspot_fetch_log; +CREATE TABLE hotspot_fetch_log +( + `id` VARCHAR(32) comment 'id', + `source_id` VARCHAR(32) comment '关联来源ID', + `start_time` VARCHAR(50) comment '开始时间', + `end_time` VARCHAR(50) comment '结束时间', + `duration_ms` INT DEFAULT 0 comment '耗时(毫秒)', + `status` VARCHAR(20) comment '状态: success/failed/partial', + `items_total` INT DEFAULT 0 comment '抓取总数', + `items_new` INT DEFAULT 0 comment '新增数量', + `items_duplicate` INT DEFAULT 0 comment '重复数量', + `error_msg` TEXT comment '错误信息', + `response_code` INT comment 'HTTP状态码', + `response_size_bytes` INT DEFAULT 0 comment '响应大小' +,primary key(id) +) +engine=innodb default charset=utf8 comment '抓取日志' +; +CREATE INDEX hotspot_fetch_log_source_idx ON hotspot_fetch_log(source_id); +CREATE INDEX hotspot_fetch_log_status_idx ON hotspot_fetch_log(status); + +-- ./hotspot_item (热点条目) +drop table if exists hotspot_item; +CREATE TABLE hotspot_item +( + `id` VARCHAR(32) comment 'id', + `source_id` VARCHAR(32) comment '来源ID', + `title` VARCHAR(500) comment '标题', + `url` VARCHAR(2000) comment '原文链接', + `summary` TEXT comment '摘要', + `content` TEXT comment '正文', + `heat_score` DOUBLE(10,2) DEFAULT 0 comment '热度分数', + `heat_velocity` DOUBLE(10,2) DEFAULT 0 comment '热度加速度', + `publish_time` VARCHAR(50) comment '发布时间', + `first_seen` VARCHAR(50) comment '首次发现', + `last_updated` VARCHAR(50) comment '最后更新', + `status` VARCHAR(20) DEFAULT 'emerging' comment '状态: emerging/rising/hot/cooling/expired', + `category` VARCHAR(100) comment '分类', + `tags` VARCHAR(500) comment '标签', + `engagement_count` INT DEFAULT 0 comment '互动数', + `comment_count` INT DEFAULT 0 comment '评论数', + `share_count` INT DEFAULT 0 comment '转发数', + `sentiment` VARCHAR(20) DEFAULT 'neutral' comment '情感: positive/negative/neutral', + `is_verified` VARCHAR(1) DEFAULT '0' comment '是否已验证', + `duplicate_of` VARCHAR(32) comment '重复于(另一热点ID)' +,primary key(id) +) +engine=innodb default charset=utf8 comment '热点条目' +; +CREATE INDEX hotspot_item_source_idx ON hotspot_item(source_id); +CREATE INDEX hotspot_item_status_idx ON hotspot_item(status); +CREATE INDEX hotspot_item_score_idx ON hotspot_item(heat_score); + +-- ./hotspot_analysis (五维分析) +drop table if exists hotspot_analysis; +CREATE TABLE hotspot_analysis +( + `id` VARCHAR(32) comment 'id', + `item_id` VARCHAR(32) comment '关联热点ID', + `dimension` VARCHAR(50) comment '维度: time/heat/content/propagation/audience', + `score` DOUBLE(10,2) DEFAULT 0 comment '维度得分(0-100)', + `analysis_data` TEXT comment '分析数据JSON', + `analyzed_at` VARCHAR(50) comment '分析时间' +,primary key(id) +) +engine=innodb default charset=utf8 comment '五维分析' +; +CREATE INDEX hotspot_analysis_item_idx ON hotspot_analysis(item_id); + +-- ./hotspot_alert (预警规则) +drop table if exists hotspot_alert; +CREATE TABLE hotspot_alert +( + `id` VARCHAR(32) comment 'id', + `name` VARCHAR(200) comment '规则名称', + `source_id` VARCHAR(32) comment '来源ID(空=全局)', + `condition_type` VARCHAR(50) comment '条件: heat_threshold/velocity/sentiment/status_change', + `condition_value` VARCHAR(500) comment '条件值 JSON', + `enabled` VARCHAR(1) DEFAULT '1' comment '是否启用', + `notify_method` VARCHAR(50) comment '通知方式: email/webhook/sms', + `notify_target` VARCHAR(500) comment '通知目标', + `last_triggered` VARCHAR(50) comment '上次触发时间' +,primary key(id) +) +engine=innodb default charset=utf8 comment '预警规则' +; diff --git a/install/load_path_append.txt b/install/load_path_append.txt new file mode 100644 index 0000000..8cebca3 --- /dev/null +++ b/install/load_path_append.txt @@ -0,0 +1,41 @@ +/hotspot logined +/hotspot/index.ui logined +/hotspot/stats.dspy logined +/hotspot/fetch_now.dspy logined +/hotspot/analyze.dspy logined +/hotspot_source logined +/hotspot_source/index.ui logined +/hotspot_source/get_hotspot_source.dspy logined +/hotspot_source/add_hotspot_source.dspy logined +/hotspot_source/update_hotspot_source.dspy logined +/hotspot_source/delete_hotspot_source.dspy logined +/hotspot_schedule logined +/hotspot_schedule/index.ui logined +/hotspot_schedule/get_hotspot_schedule.dspy logined +/hotspot_schedule/add_hotspot_schedule.dspy logined +/hotspot_schedule/update_hotspot_schedule.dspy logined +/hotspot_schedule/delete_hotspot_schedule.dspy logined +/hotspot_fetch_log logined +/hotspot_fetch_log/index.ui logined +/hotspot_fetch_log/get_hotspot_fetch_log.dspy logined +/hotspot_fetch_log/add_hotspot_fetch_log.dspy logined +/hotspot_fetch_log/update_hotspot_fetch_log.dspy logined +/hotspot_fetch_log/delete_hotspot_fetch_log.dspy logined +/hotspot_item logined +/hotspot_item/index.ui logined +/hotspot_item/get_hotspot_item.dspy logined +/hotspot_item/add_hotspot_item.dspy logined +/hotspot_item/update_hotspot_item.dspy logined +/hotspot_item/delete_hotspot_item.dspy logined +/hotspot_analysis logined +/hotspot_analysis/index.ui logined +/hotspot_analysis/get_hotspot_analysis.dspy logined +/hotspot_analysis/add_hotspot_analysis.dspy logined +/hotspot_analysis/update_hotspot_analysis.dspy logined +/hotspot_analysis/delete_hotspot_analysis.dspy logined +/hotspot_alert logined +/hotspot_alert/index.ui logined +/hotspot_alert/get_hotspot_alert.dspy logined +/hotspot_alert/add_hotspot_alert.dspy logined +/hotspot_alert/update_hotspot_alert.dspy logined +/hotspot_alert/delete_hotspot_alert.dspy logined diff --git a/install/menu_append.json b/install/menu_append.json new file mode 100644 index 0000000..010db2f --- /dev/null +++ b/install/menu_append.json @@ -0,0 +1,5 @@ +{ + "name": "hotspot", + "label": "热点雷达", + "url": "{{entire_url('hotspot')}}" +} diff --git a/json/hotspot_alert.json b/json/hotspot_alert.json new file mode 100644 index 0000000..13575a0 --- /dev/null +++ b/json/hotspot_alert.json @@ -0,0 +1,16 @@ +{ + "models_dir": "${HOME}$/py/sage/models", + "output_dir": "${HOME}$/py/sage/wwwroot/_a/hotspot_alert", + "dbname": "sage", + "tblname": "hotspot_alert", + "title": "预警规则", + "params": { + "sortby": "name", + "browserfields": { + "exclouded": ["id"], + "cwidth": {} + }, + "editexclouded": ["id", "last_triggered"], + "record_toolbar": null + } +} diff --git a/json/hotspot_analysis.json b/json/hotspot_analysis.json new file mode 100644 index 0000000..901ed07 --- /dev/null +++ b/json/hotspot_analysis.json @@ -0,0 +1,16 @@ +{ + "models_dir": "${HOME}$/py/sage/models", + "output_dir": "${HOME}$/py/sage/wwwroot/_a/hotspot_analysis", + "dbname": "sage", + "tblname": "hotspot_analysis", + "title": "热点分析", + "params": { + "sortby": "analyzed_at desc", + "browserfields": { + "exclouded": ["id"], + "cwidth": {"analysis_data": 400} + }, + "editexclouded": ["id", "analyzed_at"], + "record_toolbar": null + } +} diff --git a/json/hotspot_fetch_log.json b/json/hotspot_fetch_log.json new file mode 100644 index 0000000..8a92fb5 --- /dev/null +++ b/json/hotspot_fetch_log.json @@ -0,0 +1,16 @@ +{ + "models_dir": "${HOME}$/py/sage/models", + "output_dir": "${HOME}$/py/sage/wwwroot/_a/hotspot_fetch_log", + "dbname": "sage", + "tblname": "hotspot_fetch_log", + "title": "抓取日志", + "params": { + "sortby": "start_time desc", + "browserfields": { + "exclouded": ["id", "error_msg"], + "cwidth": {"error_msg": 400} + }, + "editexclouded": ["id"], + "record_toolbar": null + } +} diff --git a/json/hotspot_item.json b/json/hotspot_item.json new file mode 100644 index 0000000..5a584ba --- /dev/null +++ b/json/hotspot_item.json @@ -0,0 +1,16 @@ +{ + "models_dir": "${HOME}$/py/sage/models", + "output_dir": "${HOME}$/py/sage/wwwroot/_a/hotspot_item", + "dbname": "sage", + "tblname": "hotspot_item", + "title": "热点条目", + "params": { + "sortby": "heat_score desc", + "browserfields": { + "exclouded": ["id", "content"], + "cwidth": {"title": 300, "summary": 400, "url": 200} + }, + "editexclouded": ["id", "first_seen", "last_updated"], + "record_toolbar": null + } +} diff --git a/json/hotspot_schedule.json b/json/hotspot_schedule.json new file mode 100644 index 0000000..3072296 --- /dev/null +++ b/json/hotspot_schedule.json @@ -0,0 +1,16 @@ +{ + "models_dir": "${HOME}$/py/sage/models", + "output_dir": "${HOME}$/py/sage/wwwroot/_a/hotspot_schedule", + "dbname": "sage", + "tblname": "hotspot_schedule", + "title": "调度配置", + "params": { + "sortby": "source_id", + "browserfields": { + "exclouded": ["id"], + "cwidth": {} + }, + "editexclouded": ["id", "last_run", "next_run"], + "record_toolbar": null + } +} diff --git a/json/hotspot_source.json b/json/hotspot_source.json new file mode 100644 index 0000000..2e43c6e --- /dev/null +++ b/json/hotspot_source.json @@ -0,0 +1,16 @@ +{ + "models_dir": "${HOME}$/py/sage/models", + "output_dir": "${HOME}$/py/sage/wwwroot/_a/hotspot_source", + "dbname": "sage", + "tblname": "hotspot_source", + "title": "热点来源", + "params": { + "sortby": "priority asc", + "browserfields": { + "exclouded": ["id", "api_secret", "headers", "request_body", "crawler_rules"], + "cwidth": {"name": 150, "url": 300} + }, + "editexclouded": ["id", "created_at", "last_fetch"], + "record_toolbar": null + } +} diff --git a/wwwroot/hotspot/analyze.dspy b/wwwroot/hotspot/analyze.dspy new file mode 100644 index 0000000..87f45ae --- /dev/null +++ b/wwwroot/hotspot/analyze.dspy @@ -0,0 +1,205 @@ +# 五维 + 状态自动分析 +# emerging → rising → hot → cooling → expired +import json +from datetime import datetime, timedelta +from sqlor.dbpools import DBPools +from appPublic.uniqueID import getID + +def now_str(): + return datetime.now().strftime('%Y-%m-%d %H:%M:%S') + +async def main(request): + db = DBPools() + now = datetime.now() + results = {'analyzed': 0, 'status_changes': 0, 'dimensions': {}} + + async with db.sqlorContext('sage') as sor: + items = await sor.sqlExe( + "select * from hotspot_item order by first_seen desc limit 500", + {} + ) + + for item in items: + try: + first_seen = datetime.strptime(str(item.get('first_seen', '')), '%Y-%m-%d %H:%M:%S') + except: + first_seen = now + + hours_alive = max(0.1, (now - first_seen).total_seconds() / 3600) + heat = float(item.get('heat_score', 0)) + velocity = float(item.get('heat_velocity', 0)) + engagement = int(item.get('engagement_count', 0)) + comments = int(item.get('comment_count', 0)) + shares = int(item.get('share_count', 0)) + current_status = item.get('status', 'emerging') + item_id = item['id'] + + # === 五维分析 === + dimensions = {} + + # 1. 时效维度 (0-100) — 越新越高 + if hours_alive < 1: + dim_time = 95 + elif hours_alive < 6: + dim_time = 85 + elif hours_alive < 24: + dim_time = 70 + elif hours_alive < 72: + dim_time = 50 + elif hours_alive < 168: + dim_time = 30 + else: + dim_time = 10 + dimensions['time'] = dim_time + + # 2. 热度维度 (0-100) + if heat > 10000: + dim_heat = 95 + elif heat > 5000: + dim_heat = 85 + elif heat > 1000: + dim_heat = 70 + elif heat > 500: + dim_heat = 55 + elif heat > 100: + dim_heat = 35 + else: + dim_heat = 15 + dimensions['heat'] = dim_heat + + # 3. 内容维度 (0-100) — 基于标题长度+摘要丰富度 + title_len = len(item.get('title', '')) + summary_len = len(item.get('summary', '')) + has_tags = bool(item.get('tags')) + has_category = bool(item.get('category')) + + dim_content = min(100, + (20 if title_len > 15 else 10) + + (30 if summary_len > 100 else 15) + + (25 if has_tags else 0) + + (25 if has_category else 0) + ) + dimensions['content'] = dim_content + + # 4. 传播维度 (0-100) — 互动量 + total_engagement = engagement + comments * 2 + shares * 3 + if total_engagement > 10000: + dim_propagation = 95 + elif total_engagement > 5000: + dim_propagation = 80 + elif total_engagement > 1000: + dim_propagation = 60 + elif total_engagement > 100: + dim_propagation = 35 + else: + dim_propagation = 10 + dimensions['propagation'] = dim_propagation + + # 5. 受众维度 (0-100) — 基于互动率 + if heat > 0: + engagement_rate = total_engagement / heat + else: + engagement_rate = 0 + + if engagement_rate > 0.5: + dim_audience = 90 + elif engagement_rate > 0.2: + dim_audience = 70 + elif engagement_rate > 0.05: + dim_audience = 45 + elif total_engagement > 0: + dim_audience = 25 + else: + dim_audience = 5 + dimensions['audience'] = dim_audience + + # === 状态分类 (基于热度 + 时间) === + # 热度衰减: heat * e^(-hours/168) ~ 7天半衰期 + import math + decay = math.exp(-hours_alive / 168) + adjusted_heat = heat * decay + + # 热度加速度 (简化：基于当前热度/time) + new_velocity = round(heat / max(hours_alive, 0.1), 2) + + if hours_alive > 336: # 超过14天 + new_status = 'expired' + elif hours_alive > 168: # 7-14天 + new_status = 'cooling' + elif adjusted_heat > 5000: + new_status = 'hot' + elif adjusted_heat > 500: + new_status = 'rising' if new_velocity > 50 else 'emerging' + elif adjusted_heat > 100: + new_status = 'rising' if new_velocity > 100 else 'emerging' + else: + new_status = 'emerging' + + # 保存 + async with db.sqlorContext('sage') as sor: + # 更新条目 + await sor.U('hotspot_item', { + 'id': item_id, + 'heat_score': round(adjusted_heat, 2), + 'heat_velocity': new_velocity, + 'status': new_status, + 'last_updated': now_str(), + }) + + results['analyzed'] += 1 + if new_status != current_status: + results['status_changes'] += 1 + + # 保存五维分析 + for dim, score in dimensions.items(): + dim_names = { + 'time': '时效维度', + 'heat': '热度指标', + 'content': '内容属性', + 'propagation': '传播路径', + 'audience': '受众画像', + } + analysis_data = json.dumps({ + 'dimension': dim, + 'dimension_cn': dim_names.get(dim, dim), + 'score': score, + 'detail': { + 'hours_alive': round(hours_alive, 1), + 'adjusted_heat': round(adjusted_heat, 2), + 'heat_velocity': new_velocity, + 'total_engagement': total_engagement, + 'decay_factor': round(decay, 4), + } + }, ensure_ascii=False) + + # Upsert: delete old analysis for this item+dimension, insert new + old = await sor.sqlExe( + "select id from hotspot_analysis where item_id=${iid}$ and dimension=${dim}$", + {'iid': item_id, 'dim': dim} + ) + if old: + await sor.U('hotspot_analysis', { + 'id': old[0]['id'], + 'score': score, + 'analysis_data': analysis_data, + 'analyzed_at': now_str(), + }) + else: + await sor.C('hotspot_analysis', { + 'id': getID(), + 'item_id': item_id, + 'dimension': dim, + 'score': score, + 'analysis_data': analysis_data, + 'analyzed_at': now_str(), + }) + + results['dimensions'] = { + 'time': dim_names['time'], + 'heat': dim_names['heat'], + 'content': dim_names['content'], + 'propagation': dim_names['propagation'], + 'audience': dim_names['audience'], + } + + return results diff --git a/wwwroot/hotspot/fetch_now.dspy b/wwwroot/hotspot/fetch_now.dspy new file mode 100644 index 0000000..4f1ea8e --- /dev/null +++ b/wwwroot/hotspot/fetch_now.dspy @@ -0,0 +1,419 @@ +# 抓取引擎 — 支持 API / Browser / Crawler 三种模式 +import json, time, hashlib +import xml.etree.ElementTree as ET +from datetime import datetime +from appPublic.uniqueID import getID +from sqlor.dbpools import DBPools + +def now_str(): + return datetime.now().strftime('%Y-%m-%d %H:%M:%S') + +async def fetch_rss(session, url): + """RSS/Atom 解析""" + import aiohttp + items = [] + try: + async with session.get(url, timeout=aiohttp.ClientTimeout(total=30)) as resp: + if resp.status != 200: + return items, resp.status, 0 + text = await resp.text() + size = len(text.encode()) + + root = ET.fromstring(text) + ns_atom = 'http://www.w3.org/2005/Atom' + + # RSS 2.0 + for item in root.findall('.//item'): + items.append({ + 'title': item.findtext('title', ''), + 'url': item.findtext('link', ''), + 'summary': (item.findtext('description', '') or '')[:500], + 'publish_time': item.findtext('pubDate', ''), + }) + + # Atom fallback + if not items: + for entry in root.findall(f'.//{{{ns_atom}}}entry'): + link_el = entry.find(f'{{{ns_atom}}}link') + items.append({ + 'title': entry.findtext(f'{{{ns_atom}}}title', ''), + 'url': link_el.get('href', '') if link_el is not None else '', + 'summary': (entry.findtext(f'{{{ns_atom}}}summary', '') or '')[:500], + 'publish_time': entry.findtext(f'{{{ns_atom}}}updated', '') or '', + }) + + return items, resp.status, size + except Exception as e: + return [], 0, 0 + +async def fetch_api(session, src): + """API 模式：支持 GET/POST + Headers + Auth""" + import aiohttp + url = src.get('url', '') + method = src.get('fetch_method', 'GET').upper() + + headers = {} + if src.get('headers'): + try: + headers = json.loads(src.get('headers', '{}')) + except: + pass + + if src.get('api_key'): + headers['Authorization'] = f'Bearer {src["api_key"]}' + + body = None + if src.get('request_body') and method == 'POST': + try: + body = json.dumps(json.loads(src['request_body'])) + except: + body = src['request_body'] + + items = [] + try: + if method == 'POST': + async with session.post(url, headers=headers, data=body or '', timeout=aiohttp.ClientTimeout(total=30)) as resp: + status = resp.status + text = await resp.text() + size = len(text.encode()) + if status == 200: + try: + data = json.loads(text) + except: + data = {} + else: + async with session.get(url, headers=headers, timeout=aiohttp.ClientTimeout(total=30)) as resp: + status = resp.status + text = await resp.text() + size = len(text.encode()) + if status == 200: + try: + data = json.loads(text) + except: + # Maybe it's XML/RSS — try RSS parser + data = {} + + # Extract items from JSON response + if isinstance(data, list): + entries = data + elif isinstance(data, dict): + entries = data.get('data', data.get('items', data.get('list', data.get('result', [])))) + else: + entries = [] + + for entry in entries[:100]: + if not isinstance(entry, dict): + continue + items.append({ + 'title': str(entry.get('title', entry.get('name', ''))), + 'url': str(entry.get('url', entry.get('link', ''))), + 'summary': str(entry.get('summary', entry.get('description', entry.get('desc', ''))))[:500], + 'heat_score': float(entry.get('heat', entry.get('hot', entry.get('score', 0))) or 0), + 'category': str(entry.get('category', entry.get('type', ''))), + 'tags': str(entry.get('tags', '')), + 'engagement_count': int(entry.get('engagement', entry.get('interactions', 0)) or 0), + 'comment_count': int(entry.get('comments', entry.get('replies', 0)) or 0), + 'share_count': int(entry.get('shares', entry.get('forwards', 0)) or 0), + }) + + return items, status, size + except Exception as e: + return [], 0, 0 + +async def fetch_browser(session, src): + """ + Browser 模式：使用 HTTP 请求 + 从 HTML 提取结构化数据 + (完整 Headless Browser 需要 playwright/selenium — 这里用 HTTP+解析做轻量版) + """ + import aiohttp + from html.parser import HTMLParser + + url = src.get('url', '') + items = [] + + try: + headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'} + if src.get('headers'): + try: + headers.update(json.loads(src.get('headers', '{}'))) + except: + pass + + async with session.get(url, headers=headers, timeout=aiohttp.ClientTimeout(total=30)) as resp: + status = resp.status + html = await resp.text() + size = len(html.encode()) + + if status != 200: + return items, status, size + + # 通用 HTML 提取：找常见的热点列表结构 + # 策略：提取所有 标签中带标题属性的，或
/
  • 块 + from html.parser import HTMLParser as HP + + class HotExtractor(HP): + def __init__(self): + super().__init__() + self.items = [] + self.current = {} + self.in_item = False + self.tag_stack = [] + self.text_buf = '' + + def handle_starttag(self, tag, attrs): + attrs_d = dict(attrs) + self.tag_stack.append(tag) + # 检测列表项容器 + if tag in ('article', 'li') and 'class' in attrs_d: + cls = attrs_d['class'].lower() + if any(k in cls for k in ('post', 'item', 'entry', 'card', 'topic', 'hot', 'trend')): + self.in_item = True + self.current = {'url': attrs_d.get('href', '')} + + if tag == 'a' and ('title' in attrs_d or self._is_hot_class(attrs_d.get('class', ''))): + href = attrs_d.get('href', '') + title = attrs_d.get('title', '') + if href and title: + self.items.append({'title': title, 'url': href, 'summary': ''}) + + def handle_endtag(self, tag): + if self.tag_stack: + self.tag_stack.pop() + if self.in_item and tag in ('article', 'li', 'div'): + if self.current.get('title'): + self.items.append(self.current) + self.in_item = False + self.current = {} + + def handle_data(self, data): + if self.in_item and not self.current.get('title'): + d = data.strip() + if len(d) > 3: + self.current['title'] = d[:200] + + def _is_hot_class(self, cls): + if not cls: + return False + cl = cls.lower() + return any(k in cl for k in ('title', 'hot', 'trending', 'headline')) + + extractor = HotExtractor() + try: + extractor.feed(html) + except: + pass + + for it in extractor.items: + if it.get('title') and len(it['title']) > 2: + items.append({ + 'title': it['title'][:200], + 'url': it.get('url', ''), + 'summary': it.get('summary', '')[:500], + 'heat_score': 80, + }) + + # 如果没提取到，尝试 JSON-LD 或 meta 标签 + if not items: + import re + # 尝试提取 og:title / twitter:title + og_titles = re.findall(r']+property="og:title"[^>]+content="([^"]+)"', html) + for t in og_titles[:20]: + items.append({'title': t, 'url': url, 'summary': '', 'heat_score': 60}) + + return items, status, size + except Exception as e: + return [], 0, 0 + +async def fetch_crawler(session, src): + """ + Crawler 模式：从起始URL递归爬取，遵守深度和规则 + """ + import aiohttp + from urllib.parse import urljoin, urlparse + + start_urls = [src.get('url', '')] + if src.get('crawler_start_urls'): + try: + start_urls = json.loads(src.get('crawler_start_urls', '[]')) + except: + pass + + depth = int(src.get('crawler_depth', 1)) + rules = {} + if src.get('crawler_rules'): + try: + rules = json.loads(src.get('crawler_rules', '{}')) + except: + pass + + # 规则：allowed_domains, link_selector, item_selector, title_selector + allowed_domains = rules.get('allowed_domains', []) + + visited = set() + to_visit = list(start_urls) + items = [] + headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'} + + for d in range(depth): + next_urls = [] + for url in to_visit: + if url in visited or len(items) >= 200: + continue + visited.add(url) + + # 域名白名单检查 + if allowed_domains: + domain = urlparse(url).netloc + if not any(ad in domain for ad in allowed_domains): + continue + + try: + async with session.get(url, headers=headers, timeout=aiohttp.ClientTimeout(total=20)) as resp: + if resp.status != 200: + continue + html = await resp.text() + + import re + # 提取页面内链接 + links = re.findall(r'href=["\']([^"\']+)["\']', html) + for link in links: + full = urljoin(url, link) + if full not in visited and full.startswith(('http://', 'https://')): + next_urls.append(full) + + # 提取标题 + titles = re.findall(r'([^<]+)', html) + if titles: + items.append({ + 'title': titles[0][:200], + 'url': url, + 'summary': '', + 'heat_score': 50 + (3 - d) * 15, + }) + + # 尝试提取 h1/h2 标题 + h_tags = re.findall(r']*>([^<]+)', html) + for h in h_tags[:10]: + h = h.strip() + if len(h) > 5: + items.append({ + 'title': h[:200], + 'url': url, + 'summary': '', + 'heat_score': 40 + (3 - d) * 10, + }) + + except: + continue + + to_visit = next_urls[:50] # 每层限制50个链接 + + return items, 200, 0 + +async def main(request): + db = DBPools() + now = now_str() + results = {'total': 0, 'new_items': 0, 'errors': 0, 'details': []} + + async with db.sqlorContext('sage') as sor: + sources = await sor.R('hotspot_source', {}) + + import aiohttp + async with aiohttp.ClientSession() as session: + for src in sources: + if src.get('enabled') != '1': + continue + + results['total'] += 1 + src_id = src['id'] + src_type = src.get('type', 'rss') + t0 = time.time() + + # 执行抓取 + try: + if src_type == 'rss': + items, code, size = await fetch_rss(session, src.get('url', '')) + elif src_type == 'api': + items, code, size = await fetch_api(session, src) + elif src_type == 'browser': + items, code, size = await fetch_browser(session, src) + elif src_type == 'crawler': + items, code, size = await fetch_crawler(session, src) + else: + items, code, size = [], 0, 0 + except Exception as e: + items, code, size = [], 0, 0 + results['errors'] += 1 + async with db.sqlorContext('sage') as sor: + await sor.C('hotspot_fetch_log', { + 'id': getID(), 'source_id': src_id, + 'start_time': now, 'end_time': now_str(), + 'duration_ms': int((time.time() - t0) * 1000), + 'status': 'failed', 'items_total': 0, 'items_new': 0, + 'items_duplicate': 0, 'error_msg': str(e)[:1000], + 'response_code': 0, 'response_size_bytes': 0, + }) + continue + + elapsed = int((time.time() - t0) * 1000) + status = 'success' if code in (0, 200) else 'partial' + new_count = 0 + dup_count = 0 + + # 存入数据库 + async with db.sqlorContext('sage') as sor: + # 更新来源 last_fetch + await sor.U('hotspot_source', {'id': src_id, 'last_fetch': now}) + + for item in items: + url = item.get('url', '') + if url: + existing = await sor.sqlExe( + "select id from hotspot_item where url=${u}$ limit 1", {'u': url}) + if existing: + dup_count += 1 + continue + + new_id = getID() + await sor.C('hotspot_item', { + 'id': new_id, 'source_id': src_id, + 'title': item.get('title', '')[:500], + 'url': url[:2000] if url else '', + 'summary': item.get('summary', '')[:1000], + 'heat_score': item.get('heat_score', 100), + 'heat_velocity': 0, + 'publish_time': item.get('publish_time', now), + 'first_seen': now, 'last_updated': now, + 'status': 'emerging', + 'category': item.get('category', ''), + 'tags': item.get('tags', ''), + 'engagement_count': item.get('engagement_count', 0), + 'comment_count': item.get('comment_count', 0), + 'share_count': item.get('share_count', 0), + 'sentiment': 'neutral', + }) + new_count += 1 + + # 写抓取日志 + await sor.C('hotspot_fetch_log', { + 'id': getID(), 'source_id': src_id, + 'start_time': now, 'end_time': now_str(), + 'duration_ms': elapsed, + 'status': status, 'items_total': len(items), + 'items_new': new_count, 'items_duplicate': dup_count, + 'error_msg': '', + 'response_code': code, 'response_size_bytes': size, + }) + + results['new_items'] += new_count + results['details'].append({ + 'source': src.get('name', src_id), + 'type': src_type, + 'total': len(items), + 'new': new_count, + 'dup': dup_count, + 'elapsed_ms': elapsed, + 'code': code, + }) + + return results diff --git a/wwwroot/hotspot/index.ui b/wwwroot/hotspot/index.ui new file mode 100644 index 0000000..c69350f --- /dev/null +++ b/wwwroot/hotspot/index.ui @@ -0,0 +1,182 @@ +{% set roles = get_user_roles(get_user()) %} +{ + "widgettype": "VBox", + "options": { + "children": [ + { + "widgettype": "HBox", + "options": { + "children": [ + { + "widgettype": "VBox", + "options": { + "id": "stat_emerging", + "children": [ + {"widgettype": "Text", "options": {"text": "潜在热点", "style": "color:#666;font-size:12px;"}}, + {"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#52c41a;"}} + ], + "style": "padding:14px;background:#f6ffed;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #b7eb8f;" + } + }, + { + "widgettype": "VBox", + "options": { + "id": "stat_rising", + "children": [ + {"widgettype": "Text", "options": {"text": "上升中", "style": "color:#666;font-size:12px;"}}, + {"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#1890ff;"}} + ], + "style": "padding:14px;background:#e6f7ff;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #91d5ff;" + } + }, + { + "widgettype": "VBox", + "options": { + "id": "stat_hot", + "children": [ + {"widgettype": "Text", "options": {"text": "正热点", "style": "color:#666;font-size:12px;"}}, + {"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#ff4d4f;"}} + ], + "style": "padding:14px;background:#fff1f0;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #ffa39e;" + } + }, + { + "widgettype": "VBox", + "options": { + "id": "stat_cooling", + "children": [ + {"widgettype": "Text", "options": {"text": "降温中", "style": "color:#666;font-size:12px;"}}, + {"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#faad14;"}} + ], + "style": "padding:14px;background:#fffbe6;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #ffe58f;" + } + }, + { + "widgettype": "VBox", + "options": { + "id": "stat_expired", + "children": [ + {"widgettype": "Text", "options": {"text": "已过期", "style": "color:#666;font-size:12px;"}}, + {"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#bfbfbf;"}} + ], + "style": "padding:14px;background:#fafafa;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #d9d9d9;" + } + }, + { + "widgettype": "VBox", + "options": { + "id": "stat_sources", + "children": [ + {"widgettype": "Text", "options": {"text": "活跃来源", "style": "color:#666;font-size:12px;"}}, + {"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#722ed1;"}} + ], + "style": "padding:14px;background:#f9f0ff;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #d3adf7;" + } + }, + { + "widgettype": "VBox", + "options": { + "id": "stat_failures", + "children": [ + {"widgettype": "Text", "options": {"text": "24h 抓取失败", "style": "color:#666;font-size:12px;"}}, + {"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#ff7875;"}} + ], + "style": "padding:14px;background:#fff2f0;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #ffccc7;" + } + } + ] + } + }, + { + "widgettype": "HBox", + "options": { + "children": [ + { + "widgettype": "Button", + "options": { + "label": "立即抓取全部来源", + "actiontype": "script", + "script": "fetch('/hotspot/fetch_now.dspy').then(r=>r.json()).then(d=>{var m='完成: '+d.total+'来源, 新增'+d.new_items+'条';if(d.errors)m+='\\\\n失败: '+d.errors;alert(m);setTimeout(()=>location.reload(),1000);}).catch(e=>alert('异常: '+e))", + "style": "margin:6px;" + } + }, + { + "widgettype": "Button", + "options": { + "label": "执行分析", + "actiontype": "script", + "script": "fetch('/hotspot/analyze.dspy').then(r=>r.json()).then(d=>{alert('已分析: '+d.analyzed+'条热点');setTimeout(()=>location.reload(),1000);}).catch(e=>alert('分析异常: '+e))", + "style": "margin:6px;" + } + }, + { + "widgettype": "Button", + "options": { + "label": "刷新统计", + "actiontype": "script", + "script": "fetch('/hotspot/stats.dspy').then(r=>r.json()).then(d=>{['emerging','rising','hot','cooling','expired','sources','failures'].forEach(k=>{var el=document.querySelectorAll('[id=stat_'+k+'] div')[1];if(el)el.textContent=d[k]||0;});}).catch(e=>console.error('stats err',e))", + "style": "margin:6px;" + } + } + ] + } + }, + { + "widgettype": "Tab", + "options": { + "tabs": [ + { + "title": "来源管理", + "content": { + "widgettype": "urlwidget", + "options": {"url": "{{ entire_url('hotspot_source') }}"} + } + }, + { + "title": "调度配置", + "content": { + "widgettype": "urlwidget", + "options": {"url": "{{ entire_url('hotspot_schedule') }}"} + } + }, + { + "title": "抓取日志", + "content": { + "widgettype": "urlwidget", + "options": {"url": "{{ entire_url('hotspot_fetch_log') }}"} + } + }, + { + "title": "热点条目", + "content": { + "widgettype": "urlwidget", + "options": {"url": "{{ entire_url('hotspot_item') }}"} + } + }, + { + "title": "五维分析", + "content": { + "widgettype": "urlwidget", + "options": {"url": "{{ entire_url('hotspot_analysis') }}"} + } + }, + { + "title": "预警规则", + "content": { + "widgettype": "urlwidget", + "options": {"url": "{{ entire_url('hotspot_alert') }}"} + } + } + ] + } + }, + { + "widgettype": "Text", + "options": { + "text": "工作流: ① 添加来源(type=api/browser/crawler) → ② 配置调度(cron/interval) → ③ 查看抓取日志(成功率/错误) → ④ 浏览热点(emerging/rising/hot/cooling/expired) → ⑤ 五维分析 → ⑥ 设置预警", + "style": "font-size:12px;color:#bbb;padding:16px;border-top:1px solid #eee;margin-top:8px;" + } + } + ] + } +} diff --git a/wwwroot/hotspot/stats.dspy b/wwwroot/hotspot/stats.dspy new file mode 100644 index 0000000..497f863 --- /dev/null +++ b/wwwroot/hotspot/stats.dspy @@ -0,0 +1,63 @@ +# 热点统计：全维度 +from datetime import datetime, timedelta +from sqlor.dbpools import DBPools + +async def main(request): + db = DBPools() + now = datetime.now() + day_ago = (now - timedelta(hours=24)).strftime('%Y-%m-%d %H:%M:%S') + + async with db.sqlorContext('sage') as sor: + items = await sor.R('hotspot_item', {'order': 'heat_score desc'}) + sources = await sor.R('hotspot_source', {}) + logs = await sor.sqlExe( + "select * from hotspot_fetch_log where start_time >= ${t}$", + {'t': day_ago} + ) + + status_count = {'emerging': 0, 'rising': 0, 'hot': 0, 'cooling': 0, 'expired': 0} + for item in items: + s = item.get('status', 'emerging') + status_count[s] = status_count.get(s, 0) + 1 + + sources_active = sum(1 for s in sources if s.get('enabled') == '1') + failures = sum(1 for log in logs if log.get('status') == 'failed') + + # Top sources by item count + src_count = {} + for item in items: + sid = item.get('source_id', '') + src_count[sid] = src_count.get(sid, 0) + 1 + + top_src = sorted(src_count.items(), key=lambda x: x[1], reverse=True)[:5] + src_names = {s['id']: s.get('name', s['id']) for s in sources} + + # Avg heat by category + cat_heat = {} + for item in items: + cat = item.get('category', '未分类') or '未分类' + h = float(item.get('heat_score', 0)) + if cat not in cat_heat: + cat_heat[cat] = {'sum': 0, 'cnt': 0} + cat_heat[cat]['sum'] += h + cat_heat[cat]['cnt'] += 1 + + top_categories = sorted( + [{'name': k, 'avg': round(v['sum']/v['cnt'], 1), 'cnt': v['cnt']} + for k, v in cat_heat.items()], + key=lambda x: x['cnt'], reverse=True + )[:10] + + return { + 'total': len(items), + 'emerging': status_count['emerging'], + 'rising': status_count['rising'], + 'hot': status_count['hot'], + 'cooling': status_count['cooling'], + 'expired': status_count['expired'], + 'sources': sources_active, + 'failures': failures, + 'top_sources': [{'name': src_names.get(k, k), 'count': v} for k, v in top_src], + 'top_categories': top_categories, + 'log_count_24h': len(logs), + }