feat: initial commit — hotspot radar module for Sage
6 tables (source/schedule/fetch_log/item/analysis/alert) 3 fetch modes (API + Browser + Crawler) 5-dimension analysis + auto status classification 7 stat cards + 6 tab dashboard
This commit is contained in:
commit
26721a1b28
3
.gitignore
vendored
Normal file
3
.gitignore
vendored
Normal file
@ -0,0 +1,3 @@
|
||||
.DS_Store
|
||||
__pycache__/
|
||||
*.pyc
|
||||
92
README.md
Normal file
92
README.md
Normal file
@ -0,0 +1,92 @@
|
||||
# 热点雷达 (Hotspot Radar)
|
||||
|
||||
Sage 框架模块 — 指定来源的热点聚合分析平台，支持动态添加来源。
|
||||
|
||||
## 功能
|
||||
|
||||
- **多类型来源**：API / RSS / Browser (HTML抓取) / Crawler (递归爬虫)
|
||||
- **调度管理**：cron 定时 + 间隔模式，retry 策略
|
||||
- **抓取日志**：每次执行一条审计记录（耗时/成功数/错误/HTTP状态码）
|
||||
- **热点分类**：5级状态 — emerging → rising → hot → cooling → expired
|
||||
- **五维分析**：时效 / 热度 / 内容 / 传播 / 受众，每维度 0-100 分
|
||||
- **预警规则**：热度阈值 / 加速度 / 情感触发，email/webhook/sms 通知
|
||||
|
||||
## 目录结构
|
||||
|
||||
```
|
||||
hotspot/
|
||||
├── json/ # Sage 表定义（放入 Sage 的 json/ 目录）
|
||||
│ ├── hotspot_source.json
|
||||
│ ├── hotspot_schedule.json
|
||||
│ ├── hotspot_fetch_log.json
|
||||
│ ├── hotspot_item.json
|
||||
│ ├── hotspot_analysis.json
|
||||
│ └── hotspot_alert.json
|
||||
├── wwwroot/hotspot/ # Web UI + 后端端点（放入 Sage 的 wwwroot/hotspot/）
|
||||
│ ├── index.ui # 仪表盘（7 统计卡片 + 6 标签页）
|
||||
│ ├── stats.dspy # 统计 API
|
||||
│ ├── fetch_now.dspy # 抓取引擎（3种模式）
|
||||
│ └── analyze.dspy # 分析引擎（5维 + 状态分类）
|
||||
├── ddl/mysql.sql # 建表 SQL
|
||||
├── install/ # 安装片段
|
||||
│ ├── load_path_append.txt # 追加到 load_path.py
|
||||
│ └── menu_append.json # 追加到 menu.ui
|
||||
└── README.md
|
||||
```
|
||||
|
||||
## 安装
|
||||
|
||||
### 1. 复制文件到 Sage 部署目录
|
||||
|
||||
```bash
|
||||
SAGE_ROOT=/path/to/sage
|
||||
cp json/hotspot_*.json $SAGE_ROOT/json/
|
||||
cp -r wwwroot/hotspot $SAGE_ROOT/wwwroot/
|
||||
```
|
||||
|
||||
### 2. 建表
|
||||
|
||||
```bash
|
||||
mysql -u user -p sage < ddl/mysql.sql
|
||||
```
|
||||
|
||||
### 3. 注册权限
|
||||
|
||||
在 `load_path.py` 的 `"""` 结束符前追加 `install/load_path_append.txt` 的内容，然后运行：
|
||||
|
||||
```bash
|
||||
cd $SAGE_ROOT && python load_path.py
|
||||
```
|
||||
|
||||
### 4. 添加菜单
|
||||
|
||||
在 `wwwroot/menu.ui` 的 `items` 数组中追加 `install/menu_append.json` 的内容。
|
||||
|
||||
### 5. 重启 Sage
|
||||
|
||||
访问 `/hotspot` 即可看到仪表盘。
|
||||
|
||||
## 使用流程
|
||||
|
||||
1. **来源管理** 标签页 → 添加来源（选择 type: api/rss/browser/crawler）
|
||||
2. **调度配置** 标签页 → 为来源配置抓取频率
|
||||
3. 仪表盘点击「立即抓取全部来源」
|
||||
4. **抓取日志** 标签页 → 检查执行结果
|
||||
5. 点击「执行分析」→ 自动计算五维评分和状态分类
|
||||
6. **热点条目** 标签页 → 按状态筛选浏览
|
||||
7. **预警规则** 标签页 → 设置自动告警
|
||||
|
||||
## 来源类型说明
|
||||
|
||||
| type | 适用场景 | 配置要点 |
|
||||
|------|---------|---------|
|
||||
| `rss` | RSS/Atom feed | url 指向 feed 地址 |
|
||||
| `api` | REST API | url + api_key + headers + fetch_method |
|
||||
| `browser` | 网页热点列表 | url + browser_wait_selector |
|
||||
| `crawler` | 递归抓取 | url + crawler_depth + crawler_rules |
|
||||
|
||||
## 依赖
|
||||
|
||||
- Sage >= 0.0.1
|
||||
- aiohttp (Python)
|
||||
- MySQL / MariaDB
|
||||
145
ddl/mysql.sql
Normal file
145
ddl/mysql.sql
Normal file
@ -0,0 +1,145 @@
|
||||
-- ============================================
|
||||
-- 热点雷达系统 (Hotspot Radar) — 6 tables
|
||||
-- ============================================
|
||||
|
||||
-- ./hotspot_source (来源配置: API / Browser / Crawler)
|
||||
drop table if exists hotspot_source;
|
||||
CREATE TABLE hotspot_source
|
||||
(
|
||||
`id` VARCHAR(32) comment 'id',
|
||||
`name` VARCHAR(200) comment '来源名称',
|
||||
`type` VARCHAR(20) comment '类型: rss/api/browser/crawler',
|
||||
`url` VARCHAR(2000) comment '目标URL',
|
||||
-- API 类型专用
|
||||
`api_key` VARCHAR(500) comment 'API Key',
|
||||
`api_secret` VARCHAR(500) comment 'API Secret(加密)',
|
||||
`headers` TEXT comment '自定义请求头 JSON',
|
||||
`fetch_method` VARCHAR(10) DEFAULT 'GET' comment 'HTTP方法',
|
||||
`request_body` TEXT comment 'POST请求体',
|
||||
-- Browser 类型专用
|
||||
`browser_headless` VARCHAR(1) DEFAULT '1' comment '无头模式',
|
||||
`browser_wait_selector` VARCHAR(500) comment '等待元素选择器',
|
||||
`browser_scroll` VARCHAR(1) DEFAULT '0' comment '是否自动滚动',
|
||||
-- Crawler 类型专用
|
||||
`crawler_depth` INT DEFAULT 1 comment '爬取深度',
|
||||
`crawler_rules` TEXT comment '爬取规则 JSON',
|
||||
`crawler_start_urls` TEXT comment '起始URL列表 JSON',
|
||||
-- 通用
|
||||
`proxy_enabled` VARCHAR(1) DEFAULT '0' comment '启用代理',
|
||||
`proxy_url` VARCHAR(500) comment '代理地址',
|
||||
`enabled` VARCHAR(1) DEFAULT '1' comment '是否启用',
|
||||
`priority` INT DEFAULT 0 comment '优先级(越大越高)',
|
||||
`last_fetch` VARCHAR(50) comment '上次抓取时间',
|
||||
`created_at` VARCHAR(50) comment '创建时间'
|
||||
,primary key(id)
|
||||
)
|
||||
engine=innodb default charset=utf8 comment '热点来源配置'
|
||||
;
|
||||
|
||||
-- ./hotspot_schedule (调度配置)
|
||||
drop table if exists hotspot_schedule;
|
||||
CREATE TABLE hotspot_schedule
|
||||
(
|
||||
`id` VARCHAR(32) comment 'id',
|
||||
`source_id` VARCHAR(32) comment '关联来源ID',
|
||||
`cron_expr` VARCHAR(100) comment 'cron表达式',
|
||||
`interval_min` INT DEFAULT 60 comment '间隔(分钟)',
|
||||
`enabled` VARCHAR(1) DEFAULT '1' comment '是否启用',
|
||||
`last_run` VARCHAR(50) comment '上次运行',
|
||||
`next_run` VARCHAR(50) comment '下次运行',
|
||||
`retry_max` INT DEFAULT 3 comment '最大重试',
|
||||
`retry_count` INT DEFAULT 0 comment '已重试次数'
|
||||
,primary key(id)
|
||||
)
|
||||
engine=innodb default charset=utf8 comment '调度配置'
|
||||
;
|
||||
CREATE INDEX hotspot_schedule_source_idx ON hotspot_schedule(source_id);
|
||||
|
||||
-- ./hotspot_fetch_log (抓取日志)
|
||||
drop table if exists hotspot_fetch_log;
|
||||
CREATE TABLE hotspot_fetch_log
|
||||
(
|
||||
`id` VARCHAR(32) comment 'id',
|
||||
`source_id` VARCHAR(32) comment '关联来源ID',
|
||||
`start_time` VARCHAR(50) comment '开始时间',
|
||||
`end_time` VARCHAR(50) comment '结束时间',
|
||||
`duration_ms` INT DEFAULT 0 comment '耗时(毫秒)',
|
||||
`status` VARCHAR(20) comment '状态: success/failed/partial',
|
||||
`items_total` INT DEFAULT 0 comment '抓取总数',
|
||||
`items_new` INT DEFAULT 0 comment '新增数量',
|
||||
`items_duplicate` INT DEFAULT 0 comment '重复数量',
|
||||
`error_msg` TEXT comment '错误信息',
|
||||
`response_code` INT comment 'HTTP状态码',
|
||||
`response_size_bytes` INT DEFAULT 0 comment '响应大小'
|
||||
,primary key(id)
|
||||
)
|
||||
engine=innodb default charset=utf8 comment '抓取日志'
|
||||
;
|
||||
CREATE INDEX hotspot_fetch_log_source_idx ON hotspot_fetch_log(source_id);
|
||||
CREATE INDEX hotspot_fetch_log_status_idx ON hotspot_fetch_log(status);
|
||||
|
||||
-- ./hotspot_item (热点条目)
|
||||
drop table if exists hotspot_item;
|
||||
CREATE TABLE hotspot_item
|
||||
(
|
||||
`id` VARCHAR(32) comment 'id',
|
||||
`source_id` VARCHAR(32) comment '来源ID',
|
||||
`title` VARCHAR(500) comment '标题',
|
||||
`url` VARCHAR(2000) comment '原文链接',
|
||||
`summary` TEXT comment '摘要',
|
||||
`content` TEXT comment '正文',
|
||||
`heat_score` DOUBLE(10,2) DEFAULT 0 comment '热度分数',
|
||||
`heat_velocity` DOUBLE(10,2) DEFAULT 0 comment '热度加速度',
|
||||
`publish_time` VARCHAR(50) comment '发布时间',
|
||||
`first_seen` VARCHAR(50) comment '首次发现',
|
||||
`last_updated` VARCHAR(50) comment '最后更新',
|
||||
`status` VARCHAR(20) DEFAULT 'emerging' comment '状态: emerging/rising/hot/cooling/expired',
|
||||
`category` VARCHAR(100) comment '分类',
|
||||
`tags` VARCHAR(500) comment '标签',
|
||||
`engagement_count` INT DEFAULT 0 comment '互动数',
|
||||
`comment_count` INT DEFAULT 0 comment '评论数',
|
||||
`share_count` INT DEFAULT 0 comment '转发数',
|
||||
`sentiment` VARCHAR(20) DEFAULT 'neutral' comment '情感: positive/negative/neutral',
|
||||
`is_verified` VARCHAR(1) DEFAULT '0' comment '是否已验证',
|
||||
`duplicate_of` VARCHAR(32) comment '重复于(另一热点ID)'
|
||||
,primary key(id)
|
||||
)
|
||||
engine=innodb default charset=utf8 comment '热点条目'
|
||||
;
|
||||
CREATE INDEX hotspot_item_source_idx ON hotspot_item(source_id);
|
||||
CREATE INDEX hotspot_item_status_idx ON hotspot_item(status);
|
||||
CREATE INDEX hotspot_item_score_idx ON hotspot_item(heat_score);
|
||||
|
||||
-- ./hotspot_analysis (五维分析)
|
||||
drop table if exists hotspot_analysis;
|
||||
CREATE TABLE hotspot_analysis
|
||||
(
|
||||
`id` VARCHAR(32) comment 'id',
|
||||
`item_id` VARCHAR(32) comment '关联热点ID',
|
||||
`dimension` VARCHAR(50) comment '维度: time/heat/content/propagation/audience',
|
||||
`score` DOUBLE(10,2) DEFAULT 0 comment '维度得分(0-100)',
|
||||
`analysis_data` TEXT comment '分析数据JSON',
|
||||
`analyzed_at` VARCHAR(50) comment '分析时间'
|
||||
,primary key(id)
|
||||
)
|
||||
engine=innodb default charset=utf8 comment '五维分析'
|
||||
;
|
||||
CREATE INDEX hotspot_analysis_item_idx ON hotspot_analysis(item_id);
|
||||
|
||||
-- ./hotspot_alert (预警规则)
|
||||
drop table if exists hotspot_alert;
|
||||
CREATE TABLE hotspot_alert
|
||||
(
|
||||
`id` VARCHAR(32) comment 'id',
|
||||
`name` VARCHAR(200) comment '规则名称',
|
||||
`source_id` VARCHAR(32) comment '来源ID(空=全局)',
|
||||
`condition_type` VARCHAR(50) comment '条件: heat_threshold/velocity/sentiment/status_change',
|
||||
`condition_value` VARCHAR(500) comment '条件值 JSON',
|
||||
`enabled` VARCHAR(1) DEFAULT '1' comment '是否启用',
|
||||
`notify_method` VARCHAR(50) comment '通知方式: email/webhook/sms',
|
||||
`notify_target` VARCHAR(500) comment '通知目标',
|
||||
`last_triggered` VARCHAR(50) comment '上次触发时间'
|
||||
,primary key(id)
|
||||
)
|
||||
engine=innodb default charset=utf8 comment '预警规则'
|
||||
;
|
||||
41
install/load_path_append.txt
Normal file
41
install/load_path_append.txt
Normal file
@ -0,0 +1,41 @@
|
||||
/hotspot logined
|
||||
/hotspot/index.ui logined
|
||||
/hotspot/stats.dspy logined
|
||||
/hotspot/fetch_now.dspy logined
|
||||
/hotspot/analyze.dspy logined
|
||||
/hotspot_source logined
|
||||
/hotspot_source/index.ui logined
|
||||
/hotspot_source/get_hotspot_source.dspy logined
|
||||
/hotspot_source/add_hotspot_source.dspy logined
|
||||
/hotspot_source/update_hotspot_source.dspy logined
|
||||
/hotspot_source/delete_hotspot_source.dspy logined
|
||||
/hotspot_schedule logined
|
||||
/hotspot_schedule/index.ui logined
|
||||
/hotspot_schedule/get_hotspot_schedule.dspy logined
|
||||
/hotspot_schedule/add_hotspot_schedule.dspy logined
|
||||
/hotspot_schedule/update_hotspot_schedule.dspy logined
|
||||
/hotspot_schedule/delete_hotspot_schedule.dspy logined
|
||||
/hotspot_fetch_log logined
|
||||
/hotspot_fetch_log/index.ui logined
|
||||
/hotspot_fetch_log/get_hotspot_fetch_log.dspy logined
|
||||
/hotspot_fetch_log/add_hotspot_fetch_log.dspy logined
|
||||
/hotspot_fetch_log/update_hotspot_fetch_log.dspy logined
|
||||
/hotspot_fetch_log/delete_hotspot_fetch_log.dspy logined
|
||||
/hotspot_item logined
|
||||
/hotspot_item/index.ui logined
|
||||
/hotspot_item/get_hotspot_item.dspy logined
|
||||
/hotspot_item/add_hotspot_item.dspy logined
|
||||
/hotspot_item/update_hotspot_item.dspy logined
|
||||
/hotspot_item/delete_hotspot_item.dspy logined
|
||||
/hotspot_analysis logined
|
||||
/hotspot_analysis/index.ui logined
|
||||
/hotspot_analysis/get_hotspot_analysis.dspy logined
|
||||
/hotspot_analysis/add_hotspot_analysis.dspy logined
|
||||
/hotspot_analysis/update_hotspot_analysis.dspy logined
|
||||
/hotspot_analysis/delete_hotspot_analysis.dspy logined
|
||||
/hotspot_alert logined
|
||||
/hotspot_alert/index.ui logined
|
||||
/hotspot_alert/get_hotspot_alert.dspy logined
|
||||
/hotspot_alert/add_hotspot_alert.dspy logined
|
||||
/hotspot_alert/update_hotspot_alert.dspy logined
|
||||
/hotspot_alert/delete_hotspot_alert.dspy logined
|
||||
5
install/menu_append.json
Normal file
5
install/menu_append.json
Normal file
@ -0,0 +1,5 @@
|
||||
{
|
||||
"name": "hotspot",
|
||||
"label": "热点雷达",
|
||||
"url": "{{entire_url('hotspot')}}"
|
||||
}
|
||||
16
json/hotspot_alert.json
Normal file
16
json/hotspot_alert.json
Normal file
@ -0,0 +1,16 @@
|
||||
{
|
||||
"models_dir": "${HOME}$/py/sage/models",
|
||||
"output_dir": "${HOME}$/py/sage/wwwroot/_a/hotspot_alert",
|
||||
"dbname": "sage",
|
||||
"tblname": "hotspot_alert",
|
||||
"title": "预警规则",
|
||||
"params": {
|
||||
"sortby": "name",
|
||||
"browserfields": {
|
||||
"exclouded": ["id"],
|
||||
"cwidth": {}
|
||||
},
|
||||
"editexclouded": ["id", "last_triggered"],
|
||||
"record_toolbar": null
|
||||
}
|
||||
}
|
||||
16
json/hotspot_analysis.json
Normal file
16
json/hotspot_analysis.json
Normal file
@ -0,0 +1,16 @@
|
||||
{
|
||||
"models_dir": "${HOME}$/py/sage/models",
|
||||
"output_dir": "${HOME}$/py/sage/wwwroot/_a/hotspot_analysis",
|
||||
"dbname": "sage",
|
||||
"tblname": "hotspot_analysis",
|
||||
"title": "热点分析",
|
||||
"params": {
|
||||
"sortby": "analyzed_at desc",
|
||||
"browserfields": {
|
||||
"exclouded": ["id"],
|
||||
"cwidth": {"analysis_data": 400}
|
||||
},
|
||||
"editexclouded": ["id", "analyzed_at"],
|
||||
"record_toolbar": null
|
||||
}
|
||||
}
|
||||
16
json/hotspot_fetch_log.json
Normal file
16
json/hotspot_fetch_log.json
Normal file
@ -0,0 +1,16 @@
|
||||
{
|
||||
"models_dir": "${HOME}$/py/sage/models",
|
||||
"output_dir": "${HOME}$/py/sage/wwwroot/_a/hotspot_fetch_log",
|
||||
"dbname": "sage",
|
||||
"tblname": "hotspot_fetch_log",
|
||||
"title": "抓取日志",
|
||||
"params": {
|
||||
"sortby": "start_time desc",
|
||||
"browserfields": {
|
||||
"exclouded": ["id", "error_msg"],
|
||||
"cwidth": {"error_msg": 400}
|
||||
},
|
||||
"editexclouded": ["id"],
|
||||
"record_toolbar": null
|
||||
}
|
||||
}
|
||||
16
json/hotspot_item.json
Normal file
16
json/hotspot_item.json
Normal file
@ -0,0 +1,16 @@
|
||||
{
|
||||
"models_dir": "${HOME}$/py/sage/models",
|
||||
"output_dir": "${HOME}$/py/sage/wwwroot/_a/hotspot_item",
|
||||
"dbname": "sage",
|
||||
"tblname": "hotspot_item",
|
||||
"title": "热点条目",
|
||||
"params": {
|
||||
"sortby": "heat_score desc",
|
||||
"browserfields": {
|
||||
"exclouded": ["id", "content"],
|
||||
"cwidth": {"title": 300, "summary": 400, "url": 200}
|
||||
},
|
||||
"editexclouded": ["id", "first_seen", "last_updated"],
|
||||
"record_toolbar": null
|
||||
}
|
||||
}
|
||||
16
json/hotspot_schedule.json
Normal file
16
json/hotspot_schedule.json
Normal file
@ -0,0 +1,16 @@
|
||||
{
|
||||
"models_dir": "${HOME}$/py/sage/models",
|
||||
"output_dir": "${HOME}$/py/sage/wwwroot/_a/hotspot_schedule",
|
||||
"dbname": "sage",
|
||||
"tblname": "hotspot_schedule",
|
||||
"title": "调度配置",
|
||||
"params": {
|
||||
"sortby": "source_id",
|
||||
"browserfields": {
|
||||
"exclouded": ["id"],
|
||||
"cwidth": {}
|
||||
},
|
||||
"editexclouded": ["id", "last_run", "next_run"],
|
||||
"record_toolbar": null
|
||||
}
|
||||
}
|
||||
16
json/hotspot_source.json
Normal file
16
json/hotspot_source.json
Normal file
@ -0,0 +1,16 @@
|
||||
{
|
||||
"models_dir": "${HOME}$/py/sage/models",
|
||||
"output_dir": "${HOME}$/py/sage/wwwroot/_a/hotspot_source",
|
||||
"dbname": "sage",
|
||||
"tblname": "hotspot_source",
|
||||
"title": "热点来源",
|
||||
"params": {
|
||||
"sortby": "priority asc",
|
||||
"browserfields": {
|
||||
"exclouded": ["id", "api_secret", "headers", "request_body", "crawler_rules"],
|
||||
"cwidth": {"name": 150, "url": 300}
|
||||
},
|
||||
"editexclouded": ["id", "created_at", "last_fetch"],
|
||||
"record_toolbar": null
|
||||
}
|
||||
}
|
||||
205
wwwroot/hotspot/analyze.dspy
Normal file
205
wwwroot/hotspot/analyze.dspy
Normal file
@ -0,0 +1,205 @@
|
||||
# 五维 + 状态自动分析
|
||||
# emerging → rising → hot → cooling → expired
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
from sqlor.dbpools import DBPools
|
||||
from appPublic.uniqueID import getID
|
||||
|
||||
def now_str():
|
||||
return datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
async def main(request):
|
||||
db = DBPools()
|
||||
now = datetime.now()
|
||||
results = {'analyzed': 0, 'status_changes': 0, 'dimensions': {}}
|
||||
|
||||
async with db.sqlorContext('sage') as sor:
|
||||
items = await sor.sqlExe(
|
||||
"select * from hotspot_item order by first_seen desc limit 500",
|
||||
{}
|
||||
)
|
||||
|
||||
for item in items:
|
||||
try:
|
||||
first_seen = datetime.strptime(str(item.get('first_seen', '')), '%Y-%m-%d %H:%M:%S')
|
||||
except:
|
||||
first_seen = now
|
||||
|
||||
hours_alive = max(0.1, (now - first_seen).total_seconds() / 3600)
|
||||
heat = float(item.get('heat_score', 0))
|
||||
velocity = float(item.get('heat_velocity', 0))
|
||||
engagement = int(item.get('engagement_count', 0))
|
||||
comments = int(item.get('comment_count', 0))
|
||||
shares = int(item.get('share_count', 0))
|
||||
current_status = item.get('status', 'emerging')
|
||||
item_id = item['id']
|
||||
|
||||
# === 五维分析 ===
|
||||
dimensions = {}
|
||||
|
||||
# 1. 时效维度 (0-100) — 越新越高
|
||||
if hours_alive < 1:
|
||||
dim_time = 95
|
||||
elif hours_alive < 6:
|
||||
dim_time = 85
|
||||
elif hours_alive < 24:
|
||||
dim_time = 70
|
||||
elif hours_alive < 72:
|
||||
dim_time = 50
|
||||
elif hours_alive < 168:
|
||||
dim_time = 30
|
||||
else:
|
||||
dim_time = 10
|
||||
dimensions['time'] = dim_time
|
||||
|
||||
# 2. 热度维度 (0-100)
|
||||
if heat > 10000:
|
||||
dim_heat = 95
|
||||
elif heat > 5000:
|
||||
dim_heat = 85
|
||||
elif heat > 1000:
|
||||
dim_heat = 70
|
||||
elif heat > 500:
|
||||
dim_heat = 55
|
||||
elif heat > 100:
|
||||
dim_heat = 35
|
||||
else:
|
||||
dim_heat = 15
|
||||
dimensions['heat'] = dim_heat
|
||||
|
||||
# 3. 内容维度 (0-100) — 基于标题长度+摘要丰富度
|
||||
title_len = len(item.get('title', ''))
|
||||
summary_len = len(item.get('summary', ''))
|
||||
has_tags = bool(item.get('tags'))
|
||||
has_category = bool(item.get('category'))
|
||||
|
||||
dim_content = min(100,
|
||||
(20 if title_len > 15 else 10) +
|
||||
(30 if summary_len > 100 else 15) +
|
||||
(25 if has_tags else 0) +
|
||||
(25 if has_category else 0)
|
||||
)
|
||||
dimensions['content'] = dim_content
|
||||
|
||||
# 4. 传播维度 (0-100) — 互动量
|
||||
total_engagement = engagement + comments * 2 + shares * 3
|
||||
if total_engagement > 10000:
|
||||
dim_propagation = 95
|
||||
elif total_engagement > 5000:
|
||||
dim_propagation = 80
|
||||
elif total_engagement > 1000:
|
||||
dim_propagation = 60
|
||||
elif total_engagement > 100:
|
||||
dim_propagation = 35
|
||||
else:
|
||||
dim_propagation = 10
|
||||
dimensions['propagation'] = dim_propagation
|
||||
|
||||
# 5. 受众维度 (0-100) — 基于互动率
|
||||
if heat > 0:
|
||||
engagement_rate = total_engagement / heat
|
||||
else:
|
||||
engagement_rate = 0
|
||||
|
||||
if engagement_rate > 0.5:
|
||||
dim_audience = 90
|
||||
elif engagement_rate > 0.2:
|
||||
dim_audience = 70
|
||||
elif engagement_rate > 0.05:
|
||||
dim_audience = 45
|
||||
elif total_engagement > 0:
|
||||
dim_audience = 25
|
||||
else:
|
||||
dim_audience = 5
|
||||
dimensions['audience'] = dim_audience
|
||||
|
||||
# === 状态分类 (基于热度 + 时间) ===
|
||||
# 热度衰减: heat * e^(-hours/168) ~ 7天半衰期
|
||||
import math
|
||||
decay = math.exp(-hours_alive / 168)
|
||||
adjusted_heat = heat * decay
|
||||
|
||||
# 热度加速度 (简化：基于当前热度/time)
|
||||
new_velocity = round(heat / max(hours_alive, 0.1), 2)
|
||||
|
||||
if hours_alive > 336: # 超过14天
|
||||
new_status = 'expired'
|
||||
elif hours_alive > 168: # 7-14天
|
||||
new_status = 'cooling'
|
||||
elif adjusted_heat > 5000:
|
||||
new_status = 'hot'
|
||||
elif adjusted_heat > 500:
|
||||
new_status = 'rising' if new_velocity > 50 else 'emerging'
|
||||
elif adjusted_heat > 100:
|
||||
new_status = 'rising' if new_velocity > 100 else 'emerging'
|
||||
else:
|
||||
new_status = 'emerging'
|
||||
|
||||
# 保存
|
||||
async with db.sqlorContext('sage') as sor:
|
||||
# 更新条目
|
||||
await sor.U('hotspot_item', {
|
||||
'id': item_id,
|
||||
'heat_score': round(adjusted_heat, 2),
|
||||
'heat_velocity': new_velocity,
|
||||
'status': new_status,
|
||||
'last_updated': now_str(),
|
||||
})
|
||||
|
||||
results['analyzed'] += 1
|
||||
if new_status != current_status:
|
||||
results['status_changes'] += 1
|
||||
|
||||
# 保存五维分析
|
||||
for dim, score in dimensions.items():
|
||||
dim_names = {
|
||||
'time': '时效维度',
|
||||
'heat': '热度指标',
|
||||
'content': '内容属性',
|
||||
'propagation': '传播路径',
|
||||
'audience': '受众画像',
|
||||
}
|
||||
analysis_data = json.dumps({
|
||||
'dimension': dim,
|
||||
'dimension_cn': dim_names.get(dim, dim),
|
||||
'score': score,
|
||||
'detail': {
|
||||
'hours_alive': round(hours_alive, 1),
|
||||
'adjusted_heat': round(adjusted_heat, 2),
|
||||
'heat_velocity': new_velocity,
|
||||
'total_engagement': total_engagement,
|
||||
'decay_factor': round(decay, 4),
|
||||
}
|
||||
}, ensure_ascii=False)
|
||||
|
||||
# Upsert: delete old analysis for this item+dimension, insert new
|
||||
old = await sor.sqlExe(
|
||||
"select id from hotspot_analysis where item_id=${iid}$ and dimension=${dim}$",
|
||||
{'iid': item_id, 'dim': dim}
|
||||
)
|
||||
if old:
|
||||
await sor.U('hotspot_analysis', {
|
||||
'id': old[0]['id'],
|
||||
'score': score,
|
||||
'analysis_data': analysis_data,
|
||||
'analyzed_at': now_str(),
|
||||
})
|
||||
else:
|
||||
await sor.C('hotspot_analysis', {
|
||||
'id': getID(),
|
||||
'item_id': item_id,
|
||||
'dimension': dim,
|
||||
'score': score,
|
||||
'analysis_data': analysis_data,
|
||||
'analyzed_at': now_str(),
|
||||
})
|
||||
|
||||
results['dimensions'] = {
|
||||
'time': dim_names['time'],
|
||||
'heat': dim_names['heat'],
|
||||
'content': dim_names['content'],
|
||||
'propagation': dim_names['propagation'],
|
||||
'audience': dim_names['audience'],
|
||||
}
|
||||
|
||||
return results
|
||||
419
wwwroot/hotspot/fetch_now.dspy
Normal file
419
wwwroot/hotspot/fetch_now.dspy
Normal file
@ -0,0 +1,419 @@
|
||||
# 抓取引擎 — 支持 API / Browser / Crawler 三种模式
|
||||
import json, time, hashlib
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import datetime
|
||||
from appPublic.uniqueID import getID
|
||||
from sqlor.dbpools import DBPools
|
||||
|
||||
def now_str():
|
||||
return datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
async def fetch_rss(session, url):
|
||||
"""RSS/Atom 解析"""
|
||||
import aiohttp
|
||||
items = []
|
||||
try:
|
||||
async with session.get(url, timeout=aiohttp.ClientTimeout(total=30)) as resp:
|
||||
if resp.status != 200:
|
||||
return items, resp.status, 0
|
||||
text = await resp.text()
|
||||
size = len(text.encode())
|
||||
|
||||
root = ET.fromstring(text)
|
||||
ns_atom = 'http://www.w3.org/2005/Atom'
|
||||
|
||||
# RSS 2.0
|
||||
for item in root.findall('.//item'):
|
||||
items.append({
|
||||
'title': item.findtext('title', ''),
|
||||
'url': item.findtext('link', ''),
|
||||
'summary': (item.findtext('description', '') or '')[:500],
|
||||
'publish_time': item.findtext('pubDate', ''),
|
||||
})
|
||||
|
||||
# Atom fallback
|
||||
if not items:
|
||||
for entry in root.findall(f'.//{{{ns_atom}}}entry'):
|
||||
link_el = entry.find(f'{{{ns_atom}}}link')
|
||||
items.append({
|
||||
'title': entry.findtext(f'{{{ns_atom}}}title', ''),
|
||||
'url': link_el.get('href', '') if link_el is not None else '',
|
||||
'summary': (entry.findtext(f'{{{ns_atom}}}summary', '') or '')[:500],
|
||||
'publish_time': entry.findtext(f'{{{ns_atom}}}updated', '') or '',
|
||||
})
|
||||
|
||||
return items, resp.status, size
|
||||
except Exception as e:
|
||||
return [], 0, 0
|
||||
|
||||
async def fetch_api(session, src):
|
||||
"""API 模式：支持 GET/POST + Headers + Auth"""
|
||||
import aiohttp
|
||||
url = src.get('url', '')
|
||||
method = src.get('fetch_method', 'GET').upper()
|
||||
|
||||
headers = {}
|
||||
if src.get('headers'):
|
||||
try:
|
||||
headers = json.loads(src.get('headers', '{}'))
|
||||
except:
|
||||
pass
|
||||
|
||||
if src.get('api_key'):
|
||||
headers['Authorization'] = f'Bearer {src["api_key"]}'
|
||||
|
||||
body = None
|
||||
if src.get('request_body') and method == 'POST':
|
||||
try:
|
||||
body = json.dumps(json.loads(src['request_body']))
|
||||
except:
|
||||
body = src['request_body']
|
||||
|
||||
items = []
|
||||
try:
|
||||
if method == 'POST':
|
||||
async with session.post(url, headers=headers, data=body or '', timeout=aiohttp.ClientTimeout(total=30)) as resp:
|
||||
status = resp.status
|
||||
text = await resp.text()
|
||||
size = len(text.encode())
|
||||
if status == 200:
|
||||
try:
|
||||
data = json.loads(text)
|
||||
except:
|
||||
data = {}
|
||||
else:
|
||||
async with session.get(url, headers=headers, timeout=aiohttp.ClientTimeout(total=30)) as resp:
|
||||
status = resp.status
|
||||
text = await resp.text()
|
||||
size = len(text.encode())
|
||||
if status == 200:
|
||||
try:
|
||||
data = json.loads(text)
|
||||
except:
|
||||
# Maybe it's XML/RSS — try RSS parser
|
||||
data = {}
|
||||
|
||||
# Extract items from JSON response
|
||||
if isinstance(data, list):
|
||||
entries = data
|
||||
elif isinstance(data, dict):
|
||||
entries = data.get('data', data.get('items', data.get('list', data.get('result', []))))
|
||||
else:
|
||||
entries = []
|
||||
|
||||
for entry in entries[:100]:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
items.append({
|
||||
'title': str(entry.get('title', entry.get('name', ''))),
|
||||
'url': str(entry.get('url', entry.get('link', ''))),
|
||||
'summary': str(entry.get('summary', entry.get('description', entry.get('desc', ''))))[:500],
|
||||
'heat_score': float(entry.get('heat', entry.get('hot', entry.get('score', 0))) or 0),
|
||||
'category': str(entry.get('category', entry.get('type', ''))),
|
||||
'tags': str(entry.get('tags', '')),
|
||||
'engagement_count': int(entry.get('engagement', entry.get('interactions', 0)) or 0),
|
||||
'comment_count': int(entry.get('comments', entry.get('replies', 0)) or 0),
|
||||
'share_count': int(entry.get('shares', entry.get('forwards', 0)) or 0),
|
||||
})
|
||||
|
||||
return items, status, size
|
||||
except Exception as e:
|
||||
return [], 0, 0
|
||||
|
||||
async def fetch_browser(session, src):
|
||||
"""
|
||||
Browser 模式：使用 HTTP 请求 + 从 HTML 提取结构化数据
|
||||
(完整 Headless Browser 需要 playwright/selenium — 这里用 HTTP+解析做轻量版)
|
||||
"""
|
||||
import aiohttp
|
||||
from html.parser import HTMLParser
|
||||
|
||||
url = src.get('url', '')
|
||||
items = []
|
||||
|
||||
try:
|
||||
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'}
|
||||
if src.get('headers'):
|
||||
try:
|
||||
headers.update(json.loads(src.get('headers', '{}')))
|
||||
except:
|
||||
pass
|
||||
|
||||
async with session.get(url, headers=headers, timeout=aiohttp.ClientTimeout(total=30)) as resp:
|
||||
status = resp.status
|
||||
html = await resp.text()
|
||||
size = len(html.encode())
|
||||
|
||||
if status != 200:
|
||||
return items, status, size
|
||||
|
||||
# 通用 HTML 提取：找常见的热点列表结构
|
||||
# 策略：提取所有 <a> 标签中带标题属性的，或 <article>/<li> 块
|
||||
from html.parser import HTMLParser as HP
|
||||
|
||||
class HotExtractor(HP):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.items = []
|
||||
self.current = {}
|
||||
self.in_item = False
|
||||
self.tag_stack = []
|
||||
self.text_buf = ''
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
attrs_d = dict(attrs)
|
||||
self.tag_stack.append(tag)
|
||||
# 检测列表项容器
|
||||
if tag in ('article', 'li') and 'class' in attrs_d:
|
||||
cls = attrs_d['class'].lower()
|
||||
if any(k in cls for k in ('post', 'item', 'entry', 'card', 'topic', 'hot', 'trend')):
|
||||
self.in_item = True
|
||||
self.current = {'url': attrs_d.get('href', '')}
|
||||
|
||||
if tag == 'a' and ('title' in attrs_d or self._is_hot_class(attrs_d.get('class', ''))):
|
||||
href = attrs_d.get('href', '')
|
||||
title = attrs_d.get('title', '')
|
||||
if href and title:
|
||||
self.items.append({'title': title, 'url': href, 'summary': ''})
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if self.tag_stack:
|
||||
self.tag_stack.pop()
|
||||
if self.in_item and tag in ('article', 'li', 'div'):
|
||||
if self.current.get('title'):
|
||||
self.items.append(self.current)
|
||||
self.in_item = False
|
||||
self.current = {}
|
||||
|
||||
def handle_data(self, data):
|
||||
if self.in_item and not self.current.get('title'):
|
||||
d = data.strip()
|
||||
if len(d) > 3:
|
||||
self.current['title'] = d[:200]
|
||||
|
||||
def _is_hot_class(self, cls):
|
||||
if not cls:
|
||||
return False
|
||||
cl = cls.lower()
|
||||
return any(k in cl for k in ('title', 'hot', 'trending', 'headline'))
|
||||
|
||||
extractor = HotExtractor()
|
||||
try:
|
||||
extractor.feed(html)
|
||||
except:
|
||||
pass
|
||||
|
||||
for it in extractor.items:
|
||||
if it.get('title') and len(it['title']) > 2:
|
||||
items.append({
|
||||
'title': it['title'][:200],
|
||||
'url': it.get('url', ''),
|
||||
'summary': it.get('summary', '')[:500],
|
||||
'heat_score': 80,
|
||||
})
|
||||
|
||||
# 如果没提取到，尝试 JSON-LD 或 meta 标签
|
||||
if not items:
|
||||
import re
|
||||
# 尝试提取 og:title / twitter:title
|
||||
og_titles = re.findall(r'<meta[^>]+property="og:title"[^>]+content="([^"]+)"', html)
|
||||
for t in og_titles[:20]:
|
||||
items.append({'title': t, 'url': url, 'summary': '', 'heat_score': 60})
|
||||
|
||||
return items, status, size
|
||||
except Exception as e:
|
||||
return [], 0, 0
|
||||
|
||||
async def fetch_crawler(session, src):
|
||||
"""
|
||||
Crawler 模式：从起始URL递归爬取，遵守深度和规则
|
||||
"""
|
||||
import aiohttp
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
start_urls = [src.get('url', '')]
|
||||
if src.get('crawler_start_urls'):
|
||||
try:
|
||||
start_urls = json.loads(src.get('crawler_start_urls', '[]'))
|
||||
except:
|
||||
pass
|
||||
|
||||
depth = int(src.get('crawler_depth', 1))
|
||||
rules = {}
|
||||
if src.get('crawler_rules'):
|
||||
try:
|
||||
rules = json.loads(src.get('crawler_rules', '{}'))
|
||||
except:
|
||||
pass
|
||||
|
||||
# 规则：allowed_domains, link_selector, item_selector, title_selector
|
||||
allowed_domains = rules.get('allowed_domains', [])
|
||||
|
||||
visited = set()
|
||||
to_visit = list(start_urls)
|
||||
items = []
|
||||
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'}
|
||||
|
||||
for d in range(depth):
|
||||
next_urls = []
|
||||
for url in to_visit:
|
||||
if url in visited or len(items) >= 200:
|
||||
continue
|
||||
visited.add(url)
|
||||
|
||||
# 域名白名单检查
|
||||
if allowed_domains:
|
||||
domain = urlparse(url).netloc
|
||||
if not any(ad in domain for ad in allowed_domains):
|
||||
continue
|
||||
|
||||
try:
|
||||
async with session.get(url, headers=headers, timeout=aiohttp.ClientTimeout(total=20)) as resp:
|
||||
if resp.status != 200:
|
||||
continue
|
||||
html = await resp.text()
|
||||
|
||||
import re
|
||||
# 提取页面内链接
|
||||
links = re.findall(r'href=["\']([^"\']+)["\']', html)
|
||||
for link in links:
|
||||
full = urljoin(url, link)
|
||||
if full not in visited and full.startswith(('http://', 'https://')):
|
||||
next_urls.append(full)
|
||||
|
||||
# 提取标题
|
||||
titles = re.findall(r'<title>([^<]+)</title>', html)
|
||||
if titles:
|
||||
items.append({
|
||||
'title': titles[0][:200],
|
||||
'url': url,
|
||||
'summary': '',
|
||||
'heat_score': 50 + (3 - d) * 15,
|
||||
})
|
||||
|
||||
# 尝试提取 h1/h2 标题
|
||||
h_tags = re.findall(r'<h[12][^>]*>([^<]+)</h[12]>', html)
|
||||
for h in h_tags[:10]:
|
||||
h = h.strip()
|
||||
if len(h) > 5:
|
||||
items.append({
|
||||
'title': h[:200],
|
||||
'url': url,
|
||||
'summary': '',
|
||||
'heat_score': 40 + (3 - d) * 10,
|
||||
})
|
||||
|
||||
except:
|
||||
continue
|
||||
|
||||
to_visit = next_urls[:50] # 每层限制50个链接
|
||||
|
||||
return items, 200, 0
|
||||
|
||||
async def main(request):
|
||||
db = DBPools()
|
||||
now = now_str()
|
||||
results = {'total': 0, 'new_items': 0, 'errors': 0, 'details': []}
|
||||
|
||||
async with db.sqlorContext('sage') as sor:
|
||||
sources = await sor.R('hotspot_source', {})
|
||||
|
||||
import aiohttp
|
||||
async with aiohttp.ClientSession() as session:
|
||||
for src in sources:
|
||||
if src.get('enabled') != '1':
|
||||
continue
|
||||
|
||||
results['total'] += 1
|
||||
src_id = src['id']
|
||||
src_type = src.get('type', 'rss')
|
||||
t0 = time.time()
|
||||
|
||||
# 执行抓取
|
||||
try:
|
||||
if src_type == 'rss':
|
||||
items, code, size = await fetch_rss(session, src.get('url', ''))
|
||||
elif src_type == 'api':
|
||||
items, code, size = await fetch_api(session, src)
|
||||
elif src_type == 'browser':
|
||||
items, code, size = await fetch_browser(session, src)
|
||||
elif src_type == 'crawler':
|
||||
items, code, size = await fetch_crawler(session, src)
|
||||
else:
|
||||
items, code, size = [], 0, 0
|
||||
except Exception as e:
|
||||
items, code, size = [], 0, 0
|
||||
results['errors'] += 1
|
||||
async with db.sqlorContext('sage') as sor:
|
||||
await sor.C('hotspot_fetch_log', {
|
||||
'id': getID(), 'source_id': src_id,
|
||||
'start_time': now, 'end_time': now_str(),
|
||||
'duration_ms': int((time.time() - t0) * 1000),
|
||||
'status': 'failed', 'items_total': 0, 'items_new': 0,
|
||||
'items_duplicate': 0, 'error_msg': str(e)[:1000],
|
||||
'response_code': 0, 'response_size_bytes': 0,
|
||||
})
|
||||
continue
|
||||
|
||||
elapsed = int((time.time() - t0) * 1000)
|
||||
status = 'success' if code in (0, 200) else 'partial'
|
||||
new_count = 0
|
||||
dup_count = 0
|
||||
|
||||
# 存入数据库
|
||||
async with db.sqlorContext('sage') as sor:
|
||||
# 更新来源 last_fetch
|
||||
await sor.U('hotspot_source', {'id': src_id, 'last_fetch': now})
|
||||
|
||||
for item in items:
|
||||
url = item.get('url', '')
|
||||
if url:
|
||||
existing = await sor.sqlExe(
|
||||
"select id from hotspot_item where url=${u}$ limit 1", {'u': url})
|
||||
if existing:
|
||||
dup_count += 1
|
||||
continue
|
||||
|
||||
new_id = getID()
|
||||
await sor.C('hotspot_item', {
|
||||
'id': new_id, 'source_id': src_id,
|
||||
'title': item.get('title', '')[:500],
|
||||
'url': url[:2000] if url else '',
|
||||
'summary': item.get('summary', '')[:1000],
|
||||
'heat_score': item.get('heat_score', 100),
|
||||
'heat_velocity': 0,
|
||||
'publish_time': item.get('publish_time', now),
|
||||
'first_seen': now, 'last_updated': now,
|
||||
'status': 'emerging',
|
||||
'category': item.get('category', ''),
|
||||
'tags': item.get('tags', ''),
|
||||
'engagement_count': item.get('engagement_count', 0),
|
||||
'comment_count': item.get('comment_count', 0),
|
||||
'share_count': item.get('share_count', 0),
|
||||
'sentiment': 'neutral',
|
||||
})
|
||||
new_count += 1
|
||||
|
||||
# 写抓取日志
|
||||
await sor.C('hotspot_fetch_log', {
|
||||
'id': getID(), 'source_id': src_id,
|
||||
'start_time': now, 'end_time': now_str(),
|
||||
'duration_ms': elapsed,
|
||||
'status': status, 'items_total': len(items),
|
||||
'items_new': new_count, 'items_duplicate': dup_count,
|
||||
'error_msg': '',
|
||||
'response_code': code, 'response_size_bytes': size,
|
||||
})
|
||||
|
||||
results['new_items'] += new_count
|
||||
results['details'].append({
|
||||
'source': src.get('name', src_id),
|
||||
'type': src_type,
|
||||
'total': len(items),
|
||||
'new': new_count,
|
||||
'dup': dup_count,
|
||||
'elapsed_ms': elapsed,
|
||||
'code': code,
|
||||
})
|
||||
|
||||
return results
|
||||
182
wwwroot/hotspot/index.ui
Normal file
182
wwwroot/hotspot/index.ui
Normal file
@ -0,0 +1,182 @@
|
||||
{% set roles = get_user_roles(get_user()) %}
|
||||
{
|
||||
"widgettype": "VBox",
|
||||
"options": {
|
||||
"children": [
|
||||
{
|
||||
"widgettype": "HBox",
|
||||
"options": {
|
||||
"children": [
|
||||
{
|
||||
"widgettype": "VBox",
|
||||
"options": {
|
||||
"id": "stat_emerging",
|
||||
"children": [
|
||||
{"widgettype": "Text", "options": {"text": "潜在热点", "style": "color:#666;font-size:12px;"}},
|
||||
{"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#52c41a;"}}
|
||||
],
|
||||
"style": "padding:14px;background:#f6ffed;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #b7eb8f;"
|
||||
}
|
||||
},
|
||||
{
|
||||
"widgettype": "VBox",
|
||||
"options": {
|
||||
"id": "stat_rising",
|
||||
"children": [
|
||||
{"widgettype": "Text", "options": {"text": "上升中", "style": "color:#666;font-size:12px;"}},
|
||||
{"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#1890ff;"}}
|
||||
],
|
||||
"style": "padding:14px;background:#e6f7ff;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #91d5ff;"
|
||||
}
|
||||
},
|
||||
{
|
||||
"widgettype": "VBox",
|
||||
"options": {
|
||||
"id": "stat_hot",
|
||||
"children": [
|
||||
{"widgettype": "Text", "options": {"text": "正热点", "style": "color:#666;font-size:12px;"}},
|
||||
{"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#ff4d4f;"}}
|
||||
],
|
||||
"style": "padding:14px;background:#fff1f0;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #ffa39e;"
|
||||
}
|
||||
},
|
||||
{
|
||||
"widgettype": "VBox",
|
||||
"options": {
|
||||
"id": "stat_cooling",
|
||||
"children": [
|
||||
{"widgettype": "Text", "options": {"text": "降温中", "style": "color:#666;font-size:12px;"}},
|
||||
{"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#faad14;"}}
|
||||
],
|
||||
"style": "padding:14px;background:#fffbe6;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #ffe58f;"
|
||||
}
|
||||
},
|
||||
{
|
||||
"widgettype": "VBox",
|
||||
"options": {
|
||||
"id": "stat_expired",
|
||||
"children": [
|
||||
{"widgettype": "Text", "options": {"text": "已过期", "style": "color:#666;font-size:12px;"}},
|
||||
{"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#bfbfbf;"}}
|
||||
],
|
||||
"style": "padding:14px;background:#fafafa;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #d9d9d9;"
|
||||
}
|
||||
},
|
||||
{
|
||||
"widgettype": "VBox",
|
||||
"options": {
|
||||
"id": "stat_sources",
|
||||
"children": [
|
||||
{"widgettype": "Text", "options": {"text": "活跃来源", "style": "color:#666;font-size:12px;"}},
|
||||
{"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#722ed1;"}}
|
||||
],
|
||||
"style": "padding:14px;background:#f9f0ff;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #d3adf7;"
|
||||
}
|
||||
},
|
||||
{
|
||||
"widgettype": "VBox",
|
||||
"options": {
|
||||
"id": "stat_failures",
|
||||
"children": [
|
||||
{"widgettype": "Text", "options": {"text": "24h 抓取失败", "style": "color:#666;font-size:12px;"}},
|
||||
{"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#ff7875;"}}
|
||||
],
|
||||
"style": "padding:14px;background:#fff2f0;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #ffccc7;"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"widgettype": "HBox",
|
||||
"options": {
|
||||
"children": [
|
||||
{
|
||||
"widgettype": "Button",
|
||||
"options": {
|
||||
"label": "立即抓取全部来源",
|
||||
"actiontype": "script",
|
||||
"script": "fetch('/hotspot/fetch_now.dspy').then(r=>r.json()).then(d=>{var m='完成: '+d.total+'来源, 新增'+d.new_items+'条';if(d.errors)m+='\\\\n失败: '+d.errors;alert(m);setTimeout(()=>location.reload(),1000);}).catch(e=>alert('异常: '+e))",
|
||||
"style": "margin:6px;"
|
||||
}
|
||||
},
|
||||
{
|
||||
"widgettype": "Button",
|
||||
"options": {
|
||||
"label": "执行分析",
|
||||
"actiontype": "script",
|
||||
"script": "fetch('/hotspot/analyze.dspy').then(r=>r.json()).then(d=>{alert('已分析: '+d.analyzed+'条热点');setTimeout(()=>location.reload(),1000);}).catch(e=>alert('分析异常: '+e))",
|
||||
"style": "margin:6px;"
|
||||
}
|
||||
},
|
||||
{
|
||||
"widgettype": "Button",
|
||||
"options": {
|
||||
"label": "刷新统计",
|
||||
"actiontype": "script",
|
||||
"script": "fetch('/hotspot/stats.dspy').then(r=>r.json()).then(d=>{['emerging','rising','hot','cooling','expired','sources','failures'].forEach(k=>{var el=document.querySelectorAll('[id=stat_'+k+'] div')[1];if(el)el.textContent=d[k]||0;});}).catch(e=>console.error('stats err',e))",
|
||||
"style": "margin:6px;"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"widgettype": "Tab",
|
||||
"options": {
|
||||
"tabs": [
|
||||
{
|
||||
"title": "来源管理",
|
||||
"content": {
|
||||
"widgettype": "urlwidget",
|
||||
"options": {"url": "{{ entire_url('hotspot_source') }}"}
|
||||
}
|
||||
},
|
||||
{
|
||||
"title": "调度配置",
|
||||
"content": {
|
||||
"widgettype": "urlwidget",
|
||||
"options": {"url": "{{ entire_url('hotspot_schedule') }}"}
|
||||
}
|
||||
},
|
||||
{
|
||||
"title": "抓取日志",
|
||||
"content": {
|
||||
"widgettype": "urlwidget",
|
||||
"options": {"url": "{{ entire_url('hotspot_fetch_log') }}"}
|
||||
}
|
||||
},
|
||||
{
|
||||
"title": "热点条目",
|
||||
"content": {
|
||||
"widgettype": "urlwidget",
|
||||
"options": {"url": "{{ entire_url('hotspot_item') }}"}
|
||||
}
|
||||
},
|
||||
{
|
||||
"title": "五维分析",
|
||||
"content": {
|
||||
"widgettype": "urlwidget",
|
||||
"options": {"url": "{{ entire_url('hotspot_analysis') }}"}
|
||||
}
|
||||
},
|
||||
{
|
||||
"title": "预警规则",
|
||||
"content": {
|
||||
"widgettype": "urlwidget",
|
||||
"options": {"url": "{{ entire_url('hotspot_alert') }}"}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"widgettype": "Text",
|
||||
"options": {
|
||||
"text": "工作流: ① 添加来源(type=api/browser/crawler) → ② 配置调度(cron/interval) → ③ 查看抓取日志(成功率/错误) → ④ 浏览热点(emerging/rising/hot/cooling/expired) → ⑤ 五维分析 → ⑥ 设置预警",
|
||||
"style": "font-size:12px;color:#bbb;padding:16px;border-top:1px solid #eee;margin-top:8px;"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
63
wwwroot/hotspot/stats.dspy
Normal file
63
wwwroot/hotspot/stats.dspy
Normal file
@ -0,0 +1,63 @@
|
||||
# 热点统计：全维度
|
||||
from datetime import datetime, timedelta
|
||||
from sqlor.dbpools import DBPools
|
||||
|
||||
async def main(request):
|
||||
db = DBPools()
|
||||
now = datetime.now()
|
||||
day_ago = (now - timedelta(hours=24)).strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
async with db.sqlorContext('sage') as sor:
|
||||
items = await sor.R('hotspot_item', {'order': 'heat_score desc'})
|
||||
sources = await sor.R('hotspot_source', {})
|
||||
logs = await sor.sqlExe(
|
||||
"select * from hotspot_fetch_log where start_time >= ${t}$",
|
||||
{'t': day_ago}
|
||||
)
|
||||
|
||||
status_count = {'emerging': 0, 'rising': 0, 'hot': 0, 'cooling': 0, 'expired': 0}
|
||||
for item in items:
|
||||
s = item.get('status', 'emerging')
|
||||
status_count[s] = status_count.get(s, 0) + 1
|
||||
|
||||
sources_active = sum(1 for s in sources if s.get('enabled') == '1')
|
||||
failures = sum(1 for log in logs if log.get('status') == 'failed')
|
||||
|
||||
# Top sources by item count
|
||||
src_count = {}
|
||||
for item in items:
|
||||
sid = item.get('source_id', '')
|
||||
src_count[sid] = src_count.get(sid, 0) + 1
|
||||
|
||||
top_src = sorted(src_count.items(), key=lambda x: x[1], reverse=True)[:5]
|
||||
src_names = {s['id']: s.get('name', s['id']) for s in sources}
|
||||
|
||||
# Avg heat by category
|
||||
cat_heat = {}
|
||||
for item in items:
|
||||
cat = item.get('category', '未分类') or '未分类'
|
||||
h = float(item.get('heat_score', 0))
|
||||
if cat not in cat_heat:
|
||||
cat_heat[cat] = {'sum': 0, 'cnt': 0}
|
||||
cat_heat[cat]['sum'] += h
|
||||
cat_heat[cat]['cnt'] += 1
|
||||
|
||||
top_categories = sorted(
|
||||
[{'name': k, 'avg': round(v['sum']/v['cnt'], 1), 'cnt': v['cnt']}
|
||||
for k, v in cat_heat.items()],
|
||||
key=lambda x: x['cnt'], reverse=True
|
||||
)[:10]
|
||||
|
||||
return {
|
||||
'total': len(items),
|
||||
'emerging': status_count['emerging'],
|
||||
'rising': status_count['rising'],
|
||||
'hot': status_count['hot'],
|
||||
'cooling': status_count['cooling'],
|
||||
'expired': status_count['expired'],
|
||||
'sources': sources_active,
|
||||
'failures': failures,
|
||||
'top_sources': [{'name': src_names.get(k, k), 'count': v} for k, v in top_src],
|
||||
'top_categories': top_categories,
|
||||
'log_count_24h': len(logs),
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user