From 64cc51cb6e200faed5e063081d6c96a674d23c87 Mon Sep 17 00:00:00 2001 From: yumoqing Date: Sat, 1 Aug 2026 11:14:58 +0800 Subject: [PATCH] =?UTF-8?q?feat(hotspot):=20comprehensive=20revamp=20?= =?UTF-8?q?=E2=80=94=206=20tables,=203=20fetch=20modes,=205-dim=20analysis?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Database (3→6 tables): - hotspot_source: support api/browser/crawler types with type-specific fields - hotspot_schedule: cron/interval daemon config per source - hotspot_fetch_log: per-run audit trail (status/duration/items/errors) - hotspot_item: add heat_velocity, engagement/comment/share counts, sentiment - hotspot_analysis: 5-dimension scoring (time/heat/content/propagation/audience) - hotspot_alert: alert rules (heat threshold/velocity/sentiment triggers) Fetch engine (3 modes): - API: GET/POST + headers + Bearer auth + JSON extraction - Browser: HTML title/link extraction + og:title fallback - Crawler: recursive with depth/domain allowlist/link following Analysis engine: - Exponential decay model (7-day half-life) - Auto-classify: emerging→rising→hot→cooling→expired - 5-dimension scoring with detail JSON per dimension Dashboard: 6 tabs + 7 stat cards + workflow guide Permission paths: 42 paths registered (6 tables × 7 CRUD ops) --- json/hotspot_alert.json | 16 ++ json/hotspot_analysis.json | 7 +- json/hotspot_fetch_log.json | 16 ++ json/hotspot_item.json | 10 +- json/hotspot_schedule.json | 16 ++ json/hotspot_source.json | 12 +- load_path.py | 18 ++ models/mysql.ddl.sql | 128 +++++++-- wwwroot/hotspot/analyze.dspy | 218 ++++++++++++--- wwwroot/hotspot/fetch_now.dspy | 498 +++++++++++++++++++++++++-------- wwwroot/hotspot/index.ui | 123 ++++---- wwwroot/hotspot/stats.dspy | 69 ++++- 12 files changed, 864 insertions(+), 267 deletions(-) create mode 100644 json/hotspot_alert.json create mode 100644 json/hotspot_fetch_log.json create mode 100644 json/hotspot_schedule.json diff --git a/json/hotspot_alert.json b/json/hotspot_alert.json new file mode 100644 index 00000000..13575a03 --- /dev/null +++ b/json/hotspot_alert.json @@ -0,0 +1,16 @@ +{ + "models_dir": "${HOME}$/py/sage/models", + "output_dir": "${HOME}$/py/sage/wwwroot/_a/hotspot_alert", + "dbname": "sage", + "tblname": "hotspot_alert", + "title": "预警规则", + "params": { + "sortby": "name", + "browserfields": { + "exclouded": ["id"], + "cwidth": {} + }, + "editexclouded": ["id", "last_triggered"], + "record_toolbar": null + } +} diff --git a/json/hotspot_analysis.json b/json/hotspot_analysis.json index 67c97f2c..901ed07d 100644 --- a/json/hotspot_analysis.json +++ b/json/hotspot_analysis.json @@ -8,12 +8,9 @@ "sortby": "analyzed_at desc", "browserfields": { "exclouded": ["id"], - "cwidth": {} + "cwidth": {"analysis_data": 400} }, - "editexclouded": [ - "id", - "analyzed_at" - ], + "editexclouded": ["id", "analyzed_at"], "record_toolbar": null } } diff --git a/json/hotspot_fetch_log.json b/json/hotspot_fetch_log.json new file mode 100644 index 00000000..8a92fb58 --- /dev/null +++ b/json/hotspot_fetch_log.json @@ -0,0 +1,16 @@ +{ + "models_dir": "${HOME}$/py/sage/models", + "output_dir": "${HOME}$/py/sage/wwwroot/_a/hotspot_fetch_log", + "dbname": "sage", + "tblname": "hotspot_fetch_log", + "title": "抓取日志", + "params": { + "sortby": "start_time desc", + "browserfields": { + "exclouded": ["id", "error_msg"], + "cwidth": {"error_msg": 400} + }, + "editexclouded": ["id"], + "record_toolbar": null + } +} diff --git a/json/hotspot_item.json b/json/hotspot_item.json index 6a43eca4..5a584ba4 100644 --- a/json/hotspot_item.json +++ b/json/hotspot_item.json @@ -8,15 +8,9 @@ "sortby": "heat_score desc", "browserfields": { "exclouded": ["id", "content"], - "cwidth": { - "title": 300, - "summary": 400 - } + "cwidth": {"title": 300, "summary": 400, "url": 200} }, - "editexclouded": [ - "id", - "fetch_time" - ], + "editexclouded": ["id", "first_seen", "last_updated"], "record_toolbar": null } } diff --git a/json/hotspot_schedule.json b/json/hotspot_schedule.json new file mode 100644 index 00000000..30722963 --- /dev/null +++ b/json/hotspot_schedule.json @@ -0,0 +1,16 @@ +{ + "models_dir": "${HOME}$/py/sage/models", + "output_dir": "${HOME}$/py/sage/wwwroot/_a/hotspot_schedule", + "dbname": "sage", + "tblname": "hotspot_schedule", + "title": "调度配置", + "params": { + "sortby": "source_id", + "browserfields": { + "exclouded": ["id"], + "cwidth": {} + }, + "editexclouded": ["id", "last_run", "next_run"], + "record_toolbar": null + } +} diff --git a/json/hotspot_source.json b/json/hotspot_source.json index e9b124fa..2e43c6e1 100644 --- a/json/hotspot_source.json +++ b/json/hotspot_source.json @@ -5,16 +5,12 @@ "tblname": "hotspot_source", "title": "热点来源", "params": { - "sortby": "name", + "sortby": "priority asc", "browserfields": { - "exclouded": ["id"], - "cwidth": {} + "exclouded": ["id", "api_secret", "headers", "request_body", "crawler_rules"], + "cwidth": {"name": 150, "url": 300} }, - "editexclouded": [ - "id", - "last_fetch", - "created_at" - ], + "editexclouded": ["id", "created_at", "last_fetch"], "record_toolbar": null } } diff --git a/load_path.py b/load_path.py index ea027855..44f70da4 100644 --- a/load_path.py +++ b/load_path.py @@ -1508,6 +1508,24 @@ paths="""/ any /hotspot_analysis/add_hotspot_analysis.dspy logined /hotspot_analysis/update_hotspot_analysis.dspy logined /hotspot_analysis/delete_hotspot_analysis.dspy logined +/hotspot_schedule logined +/hotspot_schedule/index.ui logined +/hotspot_schedule/get_hotspot_schedule.dspy logined +/hotspot_schedule/add_hotspot_schedule.dspy logined +/hotspot_schedule/update_hotspot_schedule.dspy logined +/hotspot_schedule/delete_hotspot_schedule.dspy logined +/hotspot_fetch_log logined +/hotspot_fetch_log/index.ui logined +/hotspot_fetch_log/get_hotspot_fetch_log.dspy logined +/hotspot_fetch_log/add_hotspot_fetch_log.dspy logined +/hotspot_fetch_log/update_hotspot_fetch_log.dspy logined +/hotspot_fetch_log/delete_hotspot_fetch_log.dspy logined +/hotspot_alert logined +/hotspot_alert/index.ui logined +/hotspot_alert/get_hotspot_alert.dspy logined +/hotspot_alert/add_hotspot_alert.dspy logined +/hotspot_alert/update_hotspot_alert.dspy logined +/hotspot_alert/delete_hotspot_alert.dspy logined """ diff --git a/models/mysql.ddl.sql b/models/mysql.ddl.sql index 6992ec0c..f1eed647 100644 --- a/models/mysql.ddl.sql +++ b/models/mysql.ddl.sql @@ -6578,28 +6578,87 @@ default charset=utf8 comment '模型供应商' ; --- ./hotspot_source +-- ============================================ +-- 热点雷达系统 - 6 tables +-- ============================================ + +-- ./hotspot_source (来源配置: API / Browser / Crawler) drop table if exists hotspot_source; CREATE TABLE hotspot_source ( `id` VARCHAR(32) comment 'id', `name` VARCHAR(200) comment '来源名称', - `type` VARCHAR(20) comment '类型: rss/api/web', - `url` VARCHAR(2000) comment '来源URL', - `fetch_method` VARCHAR(20) DEFAULT 'GET' comment '请求方法', + `type` VARCHAR(20) comment '类型: rss/api/browser/crawler', + `url` VARCHAR(2000) comment '目标URL', + -- API 类型专用 + `api_key` VARCHAR(500) comment 'API Key', + `api_secret` VARCHAR(500) comment 'API Secret(加密)', `headers` TEXT comment '自定义请求头 JSON', + `fetch_method` VARCHAR(10) DEFAULT 'GET' comment 'HTTP方法', + `request_body` TEXT comment 'POST请求体', + -- Browser 类型专用 + `browser_headless` VARCHAR(1) DEFAULT '1' comment '无头模式', + `browser_wait_selector` VARCHAR(500) comment '等待元素选择器', + `browser_scroll` VARCHAR(1) DEFAULT '0' comment '是否自动滚动', + -- Crawler 类型专用 + `crawler_depth` INT DEFAULT 1 comment '爬取深度', + `crawler_rules` TEXT comment '爬取规则 JSON', + `crawler_start_urls` TEXT comment '起始URL列表 JSON', + -- 通用 + `proxy_enabled` VARCHAR(1) DEFAULT '0' comment '启用代理', + `proxy_url` VARCHAR(500) comment '代理地址', `enabled` VARCHAR(1) DEFAULT '1' comment '是否启用', - `fetch_interval_min` INT DEFAULT 60 comment '抓取间隔(分钟)', + `priority` INT DEFAULT 0 comment '优先级(越大越高)', `last_fetch` VARCHAR(50) comment '上次抓取时间', `created_at` VARCHAR(50) comment '创建时间' ,primary key(id) ) -engine=innodb -default charset=utf8 -comment '热点来源' +engine=innodb default charset=utf8 comment '热点来源配置' ; --- ./hotspot_item +-- ./hotspot_schedule (调度配置) +drop table if exists hotspot_schedule; +CREATE TABLE hotspot_schedule +( + `id` VARCHAR(32) comment 'id', + `source_id` VARCHAR(32) comment '关联来源ID', + `cron_expr` VARCHAR(100) comment 'cron表达式', + `interval_min` INT DEFAULT 60 comment '间隔(分钟)', + `enabled` VARCHAR(1) DEFAULT '1' comment '是否启用', + `last_run` VARCHAR(50) comment '上次运行', + `next_run` VARCHAR(50) comment '下次运行', + `retry_max` INT DEFAULT 3 comment '最大重试', + `retry_count` INT DEFAULT 0 comment '已重试次数' +,primary key(id) +) +engine=innodb default charset=utf8 comment '调度配置' +; +CREATE INDEX hotspot_schedule_source_idx ON hotspot_schedule(source_id); + +-- ./hotspot_fetch_log (抓取日志) +drop table if exists hotspot_fetch_log; +CREATE TABLE hotspot_fetch_log +( + `id` VARCHAR(32) comment 'id', + `source_id` VARCHAR(32) comment '关联来源ID', + `start_time` VARCHAR(50) comment '开始时间', + `end_time` VARCHAR(50) comment '结束时间', + `duration_ms` INT DEFAULT 0 comment '耗时(毫秒)', + `status` VARCHAR(20) comment '状态: success/failed/partial', + `items_total` INT DEFAULT 0 comment '抓取总数', + `items_new` INT DEFAULT 0 comment '新增数量', + `items_duplicate` INT DEFAULT 0 comment '重复数量', + `error_msg` TEXT comment '错误信息', + `response_code` INT comment 'HTTP状态码', + `response_size_bytes` INT DEFAULT 0 comment '响应大小' +,primary key(id) +) +engine=innodb default charset=utf8 comment '抓取日志' +; +CREATE INDEX hotspot_fetch_log_source_idx ON hotspot_fetch_log(source_id); +CREATE INDEX hotspot_fetch_log_status_idx ON hotspot_fetch_log(status); + +-- ./hotspot_item (热点条目) drop table if exists hotspot_item; CREATE TABLE hotspot_item ( @@ -6609,39 +6668,60 @@ CREATE TABLE hotspot_item `url` VARCHAR(2000) comment '原文链接', `summary` TEXT comment '摘要', `content` TEXT comment '正文', - `heat_score` INT DEFAULT 0 comment '热度分数', + `heat_score` DOUBLE(10,2) DEFAULT 0 comment '热度分数', + `heat_velocity` DOUBLE(10,2) DEFAULT 0 comment '热度加速度', `publish_time` VARCHAR(50) comment '发布时间', - `fetch_time` VARCHAR(50) comment '抓取时间', - `status` VARCHAR(20) DEFAULT 'rising' comment '状态: rising/hot/cooling/expired', + `first_seen` VARCHAR(50) comment '首次发现', + `last_updated` VARCHAR(50) comment '最后更新', + `status` VARCHAR(20) DEFAULT 'emerging' comment '状态: emerging/rising/hot/cooling/expired', `category` VARCHAR(100) comment '分类', - `tags` VARCHAR(500) comment '标签逗号分隔' + `tags` VARCHAR(500) comment '标签', + `engagement_count` INT DEFAULT 0 comment '互动数', + `comment_count` INT DEFAULT 0 comment '评论数', + `share_count` INT DEFAULT 0 comment '转发数', + `sentiment` VARCHAR(20) DEFAULT 'neutral' comment '情感: positive/negative/neutral', + `is_verified` VARCHAR(1) DEFAULT '0' comment '是否已验证', + `duplicate_of` VARCHAR(32) comment '重复于(另一热点ID)' ,primary key(id) ) -engine=innodb -default charset=utf8 -comment '热点条目' +engine=innodb default charset=utf8 comment '热点条目' ; - CREATE INDEX hotspot_item_source_idx ON hotspot_item(source_id); CREATE INDEX hotspot_item_status_idx ON hotspot_item(status); +CREATE INDEX hotspot_item_score_idx ON hotspot_item(heat_score); --- ./hotspot_analysis +-- ./hotspot_analysis (五维分析) drop table if exists hotspot_analysis; CREATE TABLE hotspot_analysis ( `id` VARCHAR(32) comment 'id', `item_id` VARCHAR(32) comment '关联热点ID', - `dimension` VARCHAR(50) comment '分析维度', - `score` DOUBLE(10,2) DEFAULT 0 comment '维度得分', + `dimension` VARCHAR(50) comment '维度: time/heat/content/propagation/audience', + `score` DOUBLE(10,2) DEFAULT 0 comment '维度得分(0-100)', `analysis_data` TEXT comment '分析数据JSON', `analyzed_at` VARCHAR(50) comment '分析时间' ,primary key(id) ) -engine=innodb -default charset=utf8 -comment '热点分析' +engine=innodb default charset=utf8 comment '五维分析' ; - CREATE INDEX hotspot_analysis_item_idx ON hotspot_analysis(item_id); +-- ./hotspot_alert (预警规则) +drop table if exists hotspot_alert; +CREATE TABLE hotspot_alert +( + `id` VARCHAR(32) comment 'id', + `name` VARCHAR(200) comment '规则名称', + `source_id` VARCHAR(32) comment '来源ID(空=全局)', + `condition_type` VARCHAR(50) comment '条件: heat_threshold/velocity/sentiment/status_change', + `condition_value` VARCHAR(500) comment '条件值 JSON', + `enabled` VARCHAR(1) DEFAULT '1' comment '是否启用', + `notify_method` VARCHAR(50) comment '通知方式: email/webhook/sms', + `notify_target` VARCHAR(500) comment '通知目标', + `last_triggered` VARCHAR(50) comment '上次触发时间' +,primary key(id) +) +engine=innodb default charset=utf8 comment '预警规则' +; + diff --git a/wwwroot/hotspot/analyze.dspy b/wwwroot/hotspot/analyze.dspy index 058737c2..87f45aed 100644 --- a/wwwroot/hotspot/analyze.dspy +++ b/wwwroot/hotspot/analyze.dspy @@ -1,55 +1,205 @@ -# 自动分析热点状态：根据时间和热度自动分类 -# rising -> hot -> cooling -> expired +# 五维 + 状态自动分析 +# emerging → rising → hot → cooling → expired +import json from datetime import datetime, timedelta from sqlor.dbpools import DBPools +from appPublic.uniqueID import getID + +def now_str(): + return datetime.now().strftime('%Y-%m-%d %H:%M:%S') async def main(request): db = DBPools() now = datetime.now() - analyzed = 0 + results = {'analyzed': 0, 'status_changes': 0, 'dimensions': {}} async with db.sqlorContext('sage') as sor: items = await sor.sqlExe( - "select * from hotspot_item order by fetch_time desc", + "select * from hotspot_item order by first_seen desc limit 500", {} ) for item in items: try: - fetch_time = datetime.strptime(str(item.get('fetch_time', '')), '%Y-%m-%d %H:%M:%S') + first_seen = datetime.strptime(str(item.get('first_seen', '')), '%Y-%m-%d %H:%M:%S') except: - fetch_time = now + first_seen = now - hours_ago = (now - fetch_time).total_seconds() / 3600 - heat = int(item.get('heat_score', 0)) - current_status = item.get('status', 'rising') + hours_alive = max(0.1, (now - first_seen).total_seconds() / 3600) + heat = float(item.get('heat_score', 0)) + velocity = float(item.get('heat_velocity', 0)) + engagement = int(item.get('engagement_count', 0)) + comments = int(item.get('comment_count', 0)) + shares = int(item.get('share_count', 0)) + current_status = item.get('status', 'emerging') + item_id = item['id'] - # 热度衰减计算 - decay_factor = max(0, 1 - hours_ago / 168) # 7天衰减到0 - adjusted_heat = heat * decay_factor + # === 五维分析 === + dimensions = {} - # 状态分类 - if hours_ago > 168: # 超过7天 - new_status = 'expired' - elif hours_ago > 72: # 超过3天 - new_status = 'cooling' - elif adjusted_heat > 500: - new_status = 'hot' - elif adjusted_heat > 100: - new_status = 'rising' + # 1. 时效维度 (0-100) — 越新越高 + if hours_alive < 1: + dim_time = 95 + elif hours_alive < 6: + dim_time = 85 + elif hours_alive < 24: + dim_time = 70 + elif hours_alive < 72: + dim_time = 50 + elif hours_alive < 168: + dim_time = 30 else: - new_status = 'cooling' if hours_ago > 24 else 'rising' + dim_time = 10 + dimensions['time'] = dim_time - if new_status != current_status: - async with db.sqlorContext('sage') as sor: - await sor.U('hotspot_item', { - 'id': item['id'], - 'status': new_status, - 'heat_score': int(adjusted_heat), - }) - analyzed += 1 + # 2. 热度维度 (0-100) + if heat > 10000: + dim_heat = 95 + elif heat > 5000: + dim_heat = 85 + elif heat > 1000: + dim_heat = 70 + elif heat > 500: + dim_heat = 55 + elif heat > 100: + dim_heat = 35 + else: + dim_heat = 15 + dimensions['heat'] = dim_heat + + # 3. 内容维度 (0-100) — 基于标题长度+摘要丰富度 + title_len = len(item.get('title', '')) + summary_len = len(item.get('summary', '')) + has_tags = bool(item.get('tags')) + has_category = bool(item.get('category')) + + dim_content = min(100, + (20 if title_len > 15 else 10) + + (30 if summary_len > 100 else 15) + + (25 if has_tags else 0) + + (25 if has_category else 0) + ) + dimensions['content'] = dim_content + + # 4. 传播维度 (0-100) — 互动量 + total_engagement = engagement + comments * 2 + shares * 3 + if total_engagement > 10000: + dim_propagation = 95 + elif total_engagement > 5000: + dim_propagation = 80 + elif total_engagement > 1000: + dim_propagation = 60 + elif total_engagement > 100: + dim_propagation = 35 + else: + dim_propagation = 10 + dimensions['propagation'] = dim_propagation + + # 5. 受众维度 (0-100) — 基于互动率 + if heat > 0: + engagement_rate = total_engagement / heat + else: + engagement_rate = 0 + + if engagement_rate > 0.5: + dim_audience = 90 + elif engagement_rate > 0.2: + dim_audience = 70 + elif engagement_rate > 0.05: + dim_audience = 45 + elif total_engagement > 0: + dim_audience = 25 + else: + dim_audience = 5 + dimensions['audience'] = dim_audience + + # === 状态分类 (基于热度 + 时间) === + # 热度衰减: heat * e^(-hours/168) ~ 7天半衰期 + import math + decay = math.exp(-hours_alive / 168) + adjusted_heat = heat * decay + + # 热度加速度 (简化：基于当前热度/time) + new_velocity = round(heat / max(hours_alive, 0.1), 2) + + if hours_alive > 336: # 超过14天 + new_status = 'expired' + elif hours_alive > 168: # 7-14天 + new_status = 'cooling' + elif adjusted_heat > 5000: + new_status = 'hot' + elif adjusted_heat > 500: + new_status = 'rising' if new_velocity > 50 else 'emerging' + elif adjusted_heat > 100: + new_status = 'rising' if new_velocity > 100 else 'emerging' + else: + new_status = 'emerging' + + # 保存 + async with db.sqlorContext('sage') as sor: + # 更新条目 + await sor.U('hotspot_item', { + 'id': item_id, + 'heat_score': round(adjusted_heat, 2), + 'heat_velocity': new_velocity, + 'status': new_status, + 'last_updated': now_str(), + }) + + results['analyzed'] += 1 + if new_status != current_status: + results['status_changes'] += 1 + + # 保存五维分析 + for dim, score in dimensions.items(): + dim_names = { + 'time': '时效维度', + 'heat': '热度指标', + 'content': '内容属性', + 'propagation': '传播路径', + 'audience': '受众画像', + } + analysis_data = json.dumps({ + 'dimension': dim, + 'dimension_cn': dim_names.get(dim, dim), + 'score': score, + 'detail': { + 'hours_alive': round(hours_alive, 1), + 'adjusted_heat': round(adjusted_heat, 2), + 'heat_velocity': new_velocity, + 'total_engagement': total_engagement, + 'decay_factor': round(decay, 4), + } + }, ensure_ascii=False) + + # Upsert: delete old analysis for this item+dimension, insert new + old = await sor.sqlExe( + "select id from hotspot_analysis where item_id=${iid}$ and dimension=${dim}$", + {'iid': item_id, 'dim': dim} + ) + if old: + await sor.U('hotspot_analysis', { + 'id': old[0]['id'], + 'score': score, + 'analysis_data': analysis_data, + 'analyzed_at': now_str(), + }) + else: + await sor.C('hotspot_analysis', { + 'id': getID(), + 'item_id': item_id, + 'dimension': dim, + 'score': score, + 'analysis_data': analysis_data, + 'analyzed_at': now_str(), + }) + + results['dimensions'] = { + 'time': dim_names['time'], + 'heat': dim_names['heat'], + 'content': dim_names['content'], + 'propagation': dim_names['propagation'], + 'audience': dim_names['audience'], + } - return { - 'analyzed': analyzed, - 'time': now.strftime('%Y-%m-%d %H:%M:%S'), - } + return results diff --git a/wwwroot/hotspot/fetch_now.dspy b/wwwroot/hotspot/fetch_now.dspy index 075618a3..4f1ea8e2 100644 --- a/wwwroot/hotspot/fetch_now.dspy +++ b/wwwroot/hotspot/fetch_now.dspy @@ -1,165 +1,419 @@ -# 从所有启用的来源抓取热点数据 -import json +# 抓取引擎 — 支持 API / Browser / Crawler 三种模式 +import json, time, hashlib import xml.etree.ElementTree as ET from datetime import datetime from appPublic.uniqueID import getID from sqlor.dbpools import DBPools -async def fetch_rss(url): - """解析 RSS/Atom feed""" +def now_str(): + return datetime.now().strftime('%Y-%m-%d %H:%M:%S') + +async def fetch_rss(session, url): + """RSS/Atom 解析""" import aiohttp items = [] try: - async with aiohttp.ClientSession() as session: - async with session.get(url, timeout=aiohttp.ClientTimeout(total=30)) as resp: - if resp.status != 200: - return items - text = await resp.text() + async with session.get(url, timeout=aiohttp.ClientTimeout(total=30)) as resp: + if resp.status != 200: + return items, resp.status, 0 + text = await resp.text() + size = len(text.encode()) root = ET.fromstring(text) - ns = {'atom': 'http://www.w3.org/2005/Atom'} + ns_atom = 'http://www.w3.org/2005/Atom' # RSS 2.0 for item in root.findall('.//item'): - title = item.findtext('title', '') - link = item.findtext('link', '') - desc = item.findtext('description', '')[:500] - pub_date = item.findtext('pubDate', '') items.append({ - 'title': title, - 'url': link, - 'summary': desc, - 'publish_time': pub_date, + 'title': item.findtext('title', ''), + 'url': item.findtext('link', ''), + 'summary': (item.findtext('description', '') or '')[:500], + 'publish_time': item.findtext('pubDate', ''), }) - # Atom + # Atom fallback if not items: - for entry in root.findall('.//atom:entry', ns): - title = entry.findtext('atom:title', '', ns) - link_el = entry.find('atom:link', ns) - link = link_el.get('href', '') if link_el is not None else '' - summary = entry.findtext('atom:summary', '', ns) or '' - updated = entry.findtext('atom:updated', '', ns) or '' + for entry in root.findall(f'.//{{{ns_atom}}}entry'): + link_el = entry.find(f'{{{ns_atom}}}link') items.append({ - 'title': title, - 'url': link, - 'summary': summary[:500], - 'publish_time': updated, + 'title': entry.findtext(f'{{{ns_atom}}}title', ''), + 'url': link_el.get('href', '') if link_el is not None else '', + 'summary': (entry.findtext(f'{{{ns_atom}}}summary', '') or '')[:500], + 'publish_time': entry.findtext(f'{{{ns_atom}}}updated', '') or '', }) + + return items, resp.status, size except Exception as e: - print(f'RSS fetch error for {url}: {e}') - return items + return [], 0, 0 -async def fetch_api(url, method='GET', headers=None, json_path=None): - """调用 API 端点获取热点数据""" +async def fetch_api(session, src): + """API 模式：支持 GET/POST + Headers + Auth""" import aiohttp + url = src.get('url', '') + method = src.get('fetch_method', 'GET').upper() + + headers = {} + if src.get('headers'): + try: + headers = json.loads(src.get('headers', '{}')) + except: + pass + + if src.get('api_key'): + headers['Authorization'] = f'Bearer {src["api_key"]}' + + body = None + if src.get('request_body') and method == 'POST': + try: + body = json.dumps(json.loads(src['request_body'])) + except: + body = src['request_body'] + items = [] try: - hdrs = {} - if headers: - try: - hdrs = json.loads(headers) - except: - pass + if method == 'POST': + async with session.post(url, headers=headers, data=body or '', timeout=aiohttp.ClientTimeout(total=30)) as resp: + status = resp.status + text = await resp.text() + size = len(text.encode()) + if status == 200: + try: + data = json.loads(text) + except: + data = {} + else: + async with session.get(url, headers=headers, timeout=aiohttp.ClientTimeout(total=30)) as resp: + status = resp.status + text = await resp.text() + size = len(text.encode()) + if status == 200: + try: + data = json.loads(text) + except: + # Maybe it's XML/RSS — try RSS parser + data = {} - async with aiohttp.ClientSession() as session: - if method.upper() == 'GET': - async with session.get(url, headers=hdrs, timeout=aiohttp.ClientTimeout(total=30)) as resp: - if resp.status == 200: - data = await resp.json() - else: - async with session.post(url, headers=hdrs, timeout=aiohttp.ClientTimeout(total=30)) as resp: - if resp.status == 200: - data = await resp.json() - - # 尝试从 JSON 中提取 items + # Extract items from JSON response if isinstance(data, list): entries = data elif isinstance(data, dict): - # 常见热点 API 格式 - entries = data.get('data', data.get('items', data.get('list', []))) + entries = data.get('data', data.get('items', data.get('list', data.get('result', [])))) else: entries = [] - for entry in entries[:50]: # limit - if isinstance(entry, dict): - title = entry.get('title', entry.get('name', '')) - url_val = entry.get('url', entry.get('link', '')) - summary = entry.get('summary', entry.get('description', entry.get('desc', ''))) - heat = entry.get('heat', entry.get('hot', entry.get('score', 0))) - items.append({ - 'title': str(title), - 'url': str(url_val), - 'summary': str(summary)[:500], - 'heat_score': int(heat) if heat else 0, - }) + for entry in entries[:100]: + if not isinstance(entry, dict): + continue + items.append({ + 'title': str(entry.get('title', entry.get('name', ''))), + 'url': str(entry.get('url', entry.get('link', ''))), + 'summary': str(entry.get('summary', entry.get('description', entry.get('desc', ''))))[:500], + 'heat_score': float(entry.get('heat', entry.get('hot', entry.get('score', 0))) or 0), + 'category': str(entry.get('category', entry.get('type', ''))), + 'tags': str(entry.get('tags', '')), + 'engagement_count': int(entry.get('engagement', entry.get('interactions', 0)) or 0), + 'comment_count': int(entry.get('comments', entry.get('replies', 0)) or 0), + 'share_count': int(entry.get('shares', entry.get('forwards', 0)) or 0), + }) + + return items, status, size except Exception as e: - print(f'API fetch error for {url}: {e}') - return items + return [], 0, 0 + +async def fetch_browser(session, src): + """ + Browser 模式：使用 HTTP 请求 + 从 HTML 提取结构化数据 + (完整 Headless Browser 需要 playwright/selenium — 这里用 HTTP+解析做轻量版) + """ + import aiohttp + from html.parser import HTMLParser + + url = src.get('url', '') + items = [] + + try: + headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'} + if src.get('headers'): + try: + headers.update(json.loads(src.get('headers', '{}'))) + except: + pass + + async with session.get(url, headers=headers, timeout=aiohttp.ClientTimeout(total=30)) as resp: + status = resp.status + html = await resp.text() + size = len(html.encode()) + + if status != 200: + return items, status, size + + # 通用 HTML 提取：找常见的热点列表结构 + # 策略：提取所有 标签中带标题属性的，或
/
  • 块 + from html.parser import HTMLParser as HP + + class HotExtractor(HP): + def __init__(self): + super().__init__() + self.items = [] + self.current = {} + self.in_item = False + self.tag_stack = [] + self.text_buf = '' + + def handle_starttag(self, tag, attrs): + attrs_d = dict(attrs) + self.tag_stack.append(tag) + # 检测列表项容器 + if tag in ('article', 'li') and 'class' in attrs_d: + cls = attrs_d['class'].lower() + if any(k in cls for k in ('post', 'item', 'entry', 'card', 'topic', 'hot', 'trend')): + self.in_item = True + self.current = {'url': attrs_d.get('href', '')} + + if tag == 'a' and ('title' in attrs_d or self._is_hot_class(attrs_d.get('class', ''))): + href = attrs_d.get('href', '') + title = attrs_d.get('title', '') + if href and title: + self.items.append({'title': title, 'url': href, 'summary': ''}) + + def handle_endtag(self, tag): + if self.tag_stack: + self.tag_stack.pop() + if self.in_item and tag in ('article', 'li', 'div'): + if self.current.get('title'): + self.items.append(self.current) + self.in_item = False + self.current = {} + + def handle_data(self, data): + if self.in_item and not self.current.get('title'): + d = data.strip() + if len(d) > 3: + self.current['title'] = d[:200] + + def _is_hot_class(self, cls): + if not cls: + return False + cl = cls.lower() + return any(k in cl for k in ('title', 'hot', 'trending', 'headline')) + + extractor = HotExtractor() + try: + extractor.feed(html) + except: + pass + + for it in extractor.items: + if it.get('title') and len(it['title']) > 2: + items.append({ + 'title': it['title'][:200], + 'url': it.get('url', ''), + 'summary': it.get('summary', '')[:500], + 'heat_score': 80, + }) + + # 如果没提取到，尝试 JSON-LD 或 meta 标签 + if not items: + import re + # 尝试提取 og:title / twitter:title + og_titles = re.findall(r']+property="og:title"[^>]+content="([^"]+)"', html) + for t in og_titles[:20]: + items.append({'title': t, 'url': url, 'summary': '', 'heat_score': 60}) + + return items, status, size + except Exception as e: + return [], 0, 0 + +async def fetch_crawler(session, src): + """ + Crawler 模式：从起始URL递归爬取，遵守深度和规则 + """ + import aiohttp + from urllib.parse import urljoin, urlparse + + start_urls = [src.get('url', '')] + if src.get('crawler_start_urls'): + try: + start_urls = json.loads(src.get('crawler_start_urls', '[]')) + except: + pass + + depth = int(src.get('crawler_depth', 1)) + rules = {} + if src.get('crawler_rules'): + try: + rules = json.loads(src.get('crawler_rules', '{}')) + except: + pass + + # 规则：allowed_domains, link_selector, item_selector, title_selector + allowed_domains = rules.get('allowed_domains', []) + + visited = set() + to_visit = list(start_urls) + items = [] + headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'} + + for d in range(depth): + next_urls = [] + for url in to_visit: + if url in visited or len(items) >= 200: + continue + visited.add(url) + + # 域名白名单检查 + if allowed_domains: + domain = urlparse(url).netloc + if not any(ad in domain for ad in allowed_domains): + continue + + try: + async with session.get(url, headers=headers, timeout=aiohttp.ClientTimeout(total=20)) as resp: + if resp.status != 200: + continue + html = await resp.text() + + import re + # 提取页面内链接 + links = re.findall(r'href=["\']([^"\']+)["\']', html) + for link in links: + full = urljoin(url, link) + if full not in visited and full.startswith(('http://', 'https://')): + next_urls.append(full) + + # 提取标题 + titles = re.findall(r'([^<]+)', html) + if titles: + items.append({ + 'title': titles[0][:200], + 'url': url, + 'summary': '', + 'heat_score': 50 + (3 - d) * 15, + }) + + # 尝试提取 h1/h2 标题 + h_tags = re.findall(r']*>([^<]+)', html) + for h in h_tags[:10]: + h = h.strip() + if len(h) > 5: + items.append({ + 'title': h[:200], + 'url': url, + 'summary': '', + 'heat_score': 40 + (3 - d) * 10, + }) + + except: + continue + + to_visit = next_urls[:50] # 每层限制50个链接 + + return items, 200, 0 async def main(request): db = DBPools() - now = datetime.now().strftime('%Y-%m-%d %H:%M:%S') - new_items = 0 - total_sources = 0 + now = now_str() + results = {'total': 0, 'new_items': 0, 'errors': 0, 'details': []} async with db.sqlorContext('sage') as sor: sources = await sor.R('hotspot_source', {}) - for src in sources: - if src.get('enabled') != '1': - continue - - total_sources += 1 - src_type = src.get('type', 'rss') - src_url = src.get('url', '') - - if not src_url: - continue - - if src_type == 'rss': - items = await fetch_rss(src_url) - elif src_type == 'api': - fetch_method = src.get('fetch_method', 'GET') - headers = src.get('headers', '') - items = await fetch_api(src_url, fetch_method, headers) - else: - continue - - # 存入数据库 - async with db.sqlorContext('sage') as sor: - # 更新 last_fetch - await sor.U('hotspot_source', { - 'id': src['id'], - 'last_fetch': now, - }) + import aiohttp + async with aiohttp.ClientSession() as session: + for src in sources: + if src.get('enabled') != '1': + continue - for item in items: - # 去重：检查 URL 是否已存在 - existing = await sor.sqlExe( - "select id from hotspot_item where url=${url}$", - {'url': item.get('url', '')} - ) - if existing: - continue + results['total'] += 1 + src_id = src['id'] + src_type = src.get('type', 'rss') + t0 = time.time() + + # 执行抓取 + try: + if src_type == 'rss': + items, code, size = await fetch_rss(session, src.get('url', '')) + elif src_type == 'api': + items, code, size = await fetch_api(session, src) + elif src_type == 'browser': + items, code, size = await fetch_browser(session, src) + elif src_type == 'crawler': + items, code, size = await fetch_crawler(session, src) + else: + items, code, size = [], 0, 0 + except Exception as e: + items, code, size = [], 0, 0 + results['errors'] += 1 + async with db.sqlorContext('sage') as sor: + await sor.C('hotspot_fetch_log', { + 'id': getID(), 'source_id': src_id, + 'start_time': now, 'end_time': now_str(), + 'duration_ms': int((time.time() - t0) * 1000), + 'status': 'failed', 'items_total': 0, 'items_new': 0, + 'items_duplicate': 0, 'error_msg': str(e)[:1000], + 'response_code': 0, 'response_size_bytes': 0, + }) + continue + + elapsed = int((time.time() - t0) * 1000) + status = 'success' if code in (0, 200) else 'partial' + new_count = 0 + dup_count = 0 + + # 存入数据库 + async with db.sqlorContext('sage') as sor: + # 更新来源 last_fetch + await sor.U('hotspot_source', {'id': src_id, 'last_fetch': now}) - new_id = getID() - status = 'rising' # 新抓取的默认 rising - await sor.C('hotspot_item', { - 'id': new_id, - 'source_id': src['id'], - 'title': item.get('title', ''), - 'url': item.get('url', ''), - 'summary': item.get('summary', '')[:500], - 'heat_score': item.get('heat_score', 100), - 'publish_time': item.get('publish_time', now), - 'fetch_time': now, - 'status': status, + for item in items: + url = item.get('url', '') + if url: + existing = await sor.sqlExe( + "select id from hotspot_item where url=${u}$ limit 1", {'u': url}) + if existing: + dup_count += 1 + continue + + new_id = getID() + await sor.C('hotspot_item', { + 'id': new_id, 'source_id': src_id, + 'title': item.get('title', '')[:500], + 'url': url[:2000] if url else '', + 'summary': item.get('summary', '')[:1000], + 'heat_score': item.get('heat_score', 100), + 'heat_velocity': 0, + 'publish_time': item.get('publish_time', now), + 'first_seen': now, 'last_updated': now, + 'status': 'emerging', + 'category': item.get('category', ''), + 'tags': item.get('tags', ''), + 'engagement_count': item.get('engagement_count', 0), + 'comment_count': item.get('comment_count', 0), + 'share_count': item.get('share_count', 0), + 'sentiment': 'neutral', + }) + new_count += 1 + + # 写抓取日志 + await sor.C('hotspot_fetch_log', { + 'id': getID(), 'source_id': src_id, + 'start_time': now, 'end_time': now_str(), + 'duration_ms': elapsed, + 'status': status, 'items_total': len(items), + 'items_new': new_count, 'items_duplicate': dup_count, + 'error_msg': '', + 'response_code': code, 'response_size_bytes': size, }) - new_items += 1 + + results['new_items'] += new_count + results['details'].append({ + 'source': src.get('name', src_id), + 'type': src_type, + 'total': len(items), + 'new': new_count, + 'dup': dup_count, + 'elapsed_ms': elapsed, + 'code': code, + }) - return { - 'total': total_sources, - 'new_items': new_items, - 'time': now, - } + return results diff --git a/wwwroot/hotspot/index.ui b/wwwroot/hotspot/index.ui index e20fd95b..c69350f3 100644 --- a/wwwroot/hotspot/index.ui +++ b/wwwroot/hotspot/index.ui @@ -3,20 +3,6 @@ "widgettype": "VBox", "options": { "children": [ - { - "widgettype": "Text", - "options": { - "text": "热点雷达系统", - "style": "font-size:24px;font-weight:bold;padding:16px 16px 0 16px;" - } - }, - { - "widgettype": "Text", - "options": { - "text": "指定来源的热点聚合分析平台", - "style": "font-size:14px;color:#999;padding:0 16px 16px 16px;" - } - }, { "widgettype": "HBox", "options": { @@ -24,12 +10,12 @@ { "widgettype": "VBox", "options": { - "id": "stat_total", + "id": "stat_emerging", "children": [ - {"widgettype": "Text", "options": {"text": "热点总数", "style": "color:#666;font-size:12px;"}}, - {"widgettype": "Text", "options": {"text": "0", "style": "font-size:28px;font-weight:bold;color:#1890ff;"}} + {"widgettype": "Text", "options": {"text": "潜在热点", "style": "color:#666;font-size:12px;"}}, + {"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#52c41a;"}} ], - "style": "padding:16px;background:#f0f5ff;border-radius:8px;margin:8px;min-width:100px;text-align:center;" + "style": "padding:14px;background:#f6ffed;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #b7eb8f;" } }, { @@ -38,9 +24,9 @@ "id": "stat_rising", "children": [ {"widgettype": "Text", "options": {"text": "上升中", "style": "color:#666;font-size:12px;"}}, - {"widgettype": "Text", "options": {"text": "0", "style": "font-size:28px;font-weight:bold;color:#52c41a;"}} + {"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#1890ff;"}} ], - "style": "padding:16px;background:#f6ffed;border-radius:8px;margin:8px;min-width:100px;text-align:center;" + "style": "padding:14px;background:#e6f7ff;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #91d5ff;" } }, { @@ -49,9 +35,9 @@ "id": "stat_hot", "children": [ {"widgettype": "Text", "options": {"text": "正热点", "style": "color:#666;font-size:12px;"}}, - {"widgettype": "Text", "options": {"text": "0", "style": "font-size:28px;font-weight:bold;color:#ff4d4f;"}} + {"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#ff4d4f;"}} ], - "style": "padding:16px;background:#fff2f0;border-radius:8px;margin:8px;min-width:100px;text-align:center;" + "style": "padding:14px;background:#fff1f0;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #ffa39e;" } }, { @@ -60,9 +46,9 @@ "id": "stat_cooling", "children": [ {"widgettype": "Text", "options": {"text": "降温中", "style": "color:#666;font-size:12px;"}}, - {"widgettype": "Text", "options": {"text": "0", "style": "font-size:28px;font-weight:bold;color:#faad14;"}} + {"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#faad14;"}} ], - "style": "padding:16px;background:#fffbe6;border-radius:8px;margin:8px;min-width:100px;text-align:center;" + "style": "padding:14px;background:#fffbe6;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #ffe58f;" } }, { @@ -71,9 +57,9 @@ "id": "stat_expired", "children": [ {"widgettype": "Text", "options": {"text": "已过期", "style": "color:#666;font-size:12px;"}}, - {"widgettype": "Text", "options": {"text": "0", "style": "font-size:28px;font-weight:bold;color:#999;"}} + {"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#bfbfbf;"}} ], - "style": "padding:16px;background:#f5f5f5;border-radius:8px;margin:8px;min-width:100px;text-align:center;" + "style": "padding:14px;background:#fafafa;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #d9d9d9;" } }, { @@ -81,10 +67,21 @@ "options": { "id": "stat_sources", "children": [ - {"widgettype": "Text", "options": {"text": "数据来源", "style": "color:#666;font-size:12px;"}}, - {"widgettype": "Text", "options": {"text": "0", "style": "font-size:28px;font-weight:bold;color:#722ed1;"}} + {"widgettype": "Text", "options": {"text": "活跃来源", "style": "color:#666;font-size:12px;"}}, + {"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#722ed1;"}} ], - "style": "padding:16px;background:#f9f0ff;border-radius:8px;margin:8px;min-width:100px;text-align:center;" + "style": "padding:14px;background:#f9f0ff;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #d3adf7;" + } + }, + { + "widgettype": "VBox", + "options": { + "id": "stat_failures", + "children": [ + {"widgettype": "Text", "options": {"text": "24h 抓取失败", "style": "color:#666;font-size:12px;"}}, + {"widgettype": "Text", "options": {"text": "0", "style": "font-size:32px;font-weight:bold;color:#ff7875;"}} + ], + "style": "padding:14px;background:#fff2f0;border-radius:8px;margin:6px;min-width:90px;text-align:center;border:1px solid #ffccc7;" } } ] @@ -99,8 +96,17 @@ "options": { "label": "立即抓取全部来源", "actiontype": "script", - "script": "fetch('/hotspot/fetch_now.dspy').then(r=>r.json()).then(d=>{alert('抓取完成: 处理'+d.total+'个来源，新增'+d.new_items+'条');setTimeout(()=>location.reload(),1000);}).catch(e=>alert('抓取出错: '+e))", - "style": "margin:8px;" + "script": "fetch('/hotspot/fetch_now.dspy').then(r=>r.json()).then(d=>{var m='完成: '+d.total+'来源, 新增'+d.new_items+'条';if(d.errors)m+='\\\\n失败: '+d.errors;alert(m);setTimeout(()=>location.reload(),1000);}).catch(e=>alert('异常: '+e))", + "style": "margin:6px;" + } + }, + { + "widgettype": "Button", + "options": { + "label": "执行分析", + "actiontype": "script", + "script": "fetch('/hotspot/analyze.dspy').then(r=>r.json()).then(d=>{alert('已分析: '+d.analyzed+'条热点');setTimeout(()=>location.reload(),1000);}).catch(e=>alert('分析异常: '+e))", + "style": "margin:6px;" } }, { @@ -108,8 +114,8 @@ "options": { "label": "刷新统计", "actiontype": "script", - "script": "fetch('/hotspot/stats.dspy').then(r=>r.json()).then(d=>{document.querySelectorAll('[id^=stat_]').forEach(el=>{const num=el.querySelectorAll('div')[1];if(num){const k=el.id.replace('stat_','');num.textContent=d[k]||0;}});}).catch(e=>console.error(e))", - "style": "margin:8px;" + "script": "fetch('/hotspot/stats.dspy').then(r=>r.json()).then(d=>{['emerging','rising','hot','cooling','expired','sources','failures'].forEach(k=>{var el=document.querySelectorAll('[id=stat_'+k+'] div')[1];if(el)el.textContent=d[k]||0;});}).catch(e=>console.error('stats err',e))", + "style": "margin:6px;" } } ] @@ -119,31 +125,46 @@ "widgettype": "Tab", "options": { "tabs": [ - { - "title": "热点条目", - "content": { - "widgettype": "urlwidget", - "options": { - "url": "{{ entire_url('hotspot_item') }}" - } - } - }, { "title": "来源管理", "content": { "widgettype": "urlwidget", - "options": { - "url": "{{ entire_url('hotspot_source') }}" - } + "options": {"url": "{{ entire_url('hotspot_source') }}"} } }, { - "title": "分析记录", + "title": "调度配置", "content": { "widgettype": "urlwidget", - "options": { - "url": "{{ entire_url('hotspot_analysis') }}" - } + "options": {"url": "{{ entire_url('hotspot_schedule') }}"} + } + }, + { + "title": "抓取日志", + "content": { + "widgettype": "urlwidget", + "options": {"url": "{{ entire_url('hotspot_fetch_log') }}"} + } + }, + { + "title": "热点条目", + "content": { + "widgettype": "urlwidget", + "options": {"url": "{{ entire_url('hotspot_item') }}"} + } + }, + { + "title": "五维分析", + "content": { + "widgettype": "urlwidget", + "options": {"url": "{{ entire_url('hotspot_analysis') }}"} + } + }, + { + "title": "预警规则", + "content": { + "widgettype": "urlwidget", + "options": {"url": "{{ entire_url('hotspot_alert') }}"} } } ] @@ -152,8 +173,8 @@ { "widgettype": "Text", "options": { - "text": "提示：在「来源管理」中添加 RSS/API 来源后，点击「立即抓取」获取热点数据。", - "style": "font-size:12px;color:#bbb;padding:16px;" + "text": "工作流: ① 添加来源(type=api/browser/crawler) → ② 配置调度(cron/interval) → ③ 查看抓取日志(成功率/错误) → ④ 浏览热点(emerging/rising/hot/cooling/expired) → ⑤ 五维分析 → ⑥ 设置预警", + "style": "font-size:12px;color:#bbb;padding:16px;border-top:1px solid #eee;margin-top:8px;" } } ] diff --git a/wwwroot/hotspot/stats.dspy b/wwwroot/hotspot/stats.dspy index 175cc6ee..497f8633 100644 --- a/wwwroot/hotspot/stats.dspy +++ b/wwwroot/hotspot/stats.dspy @@ -1,24 +1,63 @@ -from appPublic.uniqueID import getID +# 热点统计：全维度 +from datetime import datetime, timedelta from sqlor.dbpools import DBPools async def main(request): db = DBPools() + now = datetime.now() + day_ago = (now - timedelta(hours=24)).strftime('%Y-%m-%d %H:%M:%S') + async with db.sqlorContext('sage') as sor: - items = await sor.R('hotspot_item', {'order': 'fetch_time desc'}) + items = await sor.R('hotspot_item', {'order': 'heat_score desc'}) sources = await sor.R('hotspot_source', {}) - - total = len(items) - rising = sum(1 for i in items if i.get('status') == 'rising') - hot = sum(1 for i in items if i.get('status') == 'hot') - cooling = sum(1 for i in items if i.get('status') == 'cooling') - expired = sum(1 for i in items if i.get('status') == 'expired') - sources_count = len([s for s in sources if s.get('enabled') == '1']) + logs = await sor.sqlExe( + "select * from hotspot_fetch_log where start_time >= ${t}$", + {'t': day_ago} + ) + + status_count = {'emerging': 0, 'rising': 0, 'hot': 0, 'cooling': 0, 'expired': 0} + for item in items: + s = item.get('status', 'emerging') + status_count[s] = status_count.get(s, 0) + 1 + + sources_active = sum(1 for s in sources if s.get('enabled') == '1') + failures = sum(1 for log in logs if log.get('status') == 'failed') + + # Top sources by item count + src_count = {} + for item in items: + sid = item.get('source_id', '') + src_count[sid] = src_count.get(sid, 0) + 1 + + top_src = sorted(src_count.items(), key=lambda x: x[1], reverse=True)[:5] + src_names = {s['id']: s.get('name', s['id']) for s in sources} + + # Avg heat by category + cat_heat = {} + for item in items: + cat = item.get('category', '未分类') or '未分类' + h = float(item.get('heat_score', 0)) + if cat not in cat_heat: + cat_heat[cat] = {'sum': 0, 'cnt': 0} + cat_heat[cat]['sum'] += h + cat_heat[cat]['cnt'] += 1 + + top_categories = sorted( + [{'name': k, 'avg': round(v['sum']/v['cnt'], 1), 'cnt': v['cnt']} + for k, v in cat_heat.items()], + key=lambda x: x['cnt'], reverse=True + )[:10] return { - 'total': total, - 'rising': rising, - 'hot': hot, - 'cooling': cooling, - 'expired': expired, - 'sources': sources_count + 'total': len(items), + 'emerging': status_count['emerging'], + 'rising': status_count['rising'], + 'hot': status_count['hot'], + 'cooling': status_count['cooling'], + 'expired': status_count['expired'], + 'sources': sources_active, + 'failures': failures, + 'top_sources': [{'name': src_names.get(k, k), 'count': v} for k, v in top_src], + 'top_categories': top_categories, + 'log_count_24h': len(logs), }