From fcee64ba7ec74a8cb03129b803a24bbedb253e26 Mon Sep 17 00:00:00 2001 From: yumoqing Date: Sat, 1 Aug 2026 15:35:34 +0800 Subject: [PATCH] revert: remove hotspot module (moved to standalone repo) --- .DS_Store | Bin 0 -> 8196 bytes hotspot/__init__.py | 14 - hotspot/analysis.py | 159 ---------- hotspot/db.py | 107 ------- hotspot/engine.py | 141 --------- hotspot/parsers.py | 297 ------------------ init/data.xlsx | Bin 0 -> 10555 bytes json/hotspot_alert.json | 16 - json/hotspot_analysis.json | 16 - json/hotspot_fetch_log.json | 16 - json/hotspot_item.json | 16 - json/hotspot_schedule.json | 16 - json/hotspot_source.json | 16 - skills/kboss-promote/SKILL.md | 13 + .../scripts/get_kboss_products.sh | 3 + skills/media-convert/SKILL.md | 35 +++ skills/media-convert/ktvmake.sh | 21 ++ skills/media-convert/scripts/audiosplitor.sh | 4 + skills/media-convert/scripts/videosplitor.sh | 16 + skills/media-convert/videosplitor.sh | 0 wwwroot/hotspot/analyze.dspy | 2 - wwwroot/hotspot/fetch_now.dspy | 2 - wwwroot/hotspot/index.ui | 182 ----------- wwwroot/hotspot/stats.dspy | 2 - wwwroot/imgs/ocai1.svg | 168 ++++++++++ 25 files changed, 260 insertions(+), 1002 deletions(-) create mode 100644 .DS_Store delete mode 100644 hotspot/__init__.py delete mode 100644 hotspot/analysis.py delete mode 100644 hotspot/db.py delete mode 100644 hotspot/engine.py delete mode 100644 hotspot/parsers.py create mode 100644 init/data.xlsx delete mode 100644 json/hotspot_alert.json delete mode 100644 json/hotspot_analysis.json delete mode 100644 json/hotspot_fetch_log.json delete mode 100644 json/hotspot_item.json delete mode 100644 json/hotspot_schedule.json delete mode 100644 json/hotspot_source.json create mode 100644 skills/kboss-promote/SKILL.md create mode 100644 skills/kboss-promote/scripts/get_kboss_products.sh create mode 100644 skills/media-convert/SKILL.md create mode 100644 skills/media-convert/ktvmake.sh create mode 100644 skills/media-convert/scripts/audiosplitor.sh create mode 100644 skills/media-convert/scripts/videosplitor.sh create mode 100644 skills/media-convert/videosplitor.sh delete mode 100644 wwwroot/hotspot/analyze.dspy delete mode 100644 wwwroot/hotspot/fetch_now.dspy delete mode 100644 wwwroot/hotspot/index.ui delete mode 100644 wwwroot/hotspot/stats.dspy create mode 100644 wwwroot/imgs/ocai1.svg diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..9cf22fdbf0cb862901f09d518ad33c6d1cc725e8 GIT binary patch literal 8196 zcmeHMTWl0n7(U6tRd&{@!18EasS$#ik5)>8K`jqCX}qe+3JiYUmRF^7I$w&adxtr^y#Ak09R zfiMGM2Eq)48MqxXKzlZC@+|wl(1v}OfiMIA%M9@MLyRs?69JtN)PHqQ=N|!xR+89n zbWi61-zOH(L_jA5^<8O9ae6@Dir|O=;ZFJ(Hz%41=!Brcok6%W1T!Nzp}?P={9?X2 zLqgE74>J&EU}gsR*}aC9{9pd&^ShHJ&AZuA%XGb#maiaHR902j$Tf1Ud?-2Wl#*V? zE4ZDR`~lAOI*yes-)FM@rcoMF=XW`_moaT4?>m^fK_&-NrmZ`rPN!h&uJ4-^3OOQ2 zhSWvlKM> zW@cQvo3paCPHs6ZWXmmfWi1|ZMRz&mZ_6P^?y9|8nWOb7sq{cJ87OUhSWNj zBHu~uzPwSSh*LSoDva3fkXoP1Sh<3g@rAT|;!~ ziU(TOZrI%Q@a~>#YW>`K^A{+}AZ6V4%p<0e?<<+QH$0GWbB3*(_R&<)bj_?~^xByb zqaIC>i0`#p^`b27<4ZK{0M~To4eIs4 zVP&JH^tzM}H4_*%H7iQ*F)9hw4W?>sQ?x!SlQk?=kBE8JJ*a4@qKHo29SA!XC|Ww_ zjEqnyZXnx|(6q<+w0knH*E4F^w4}?y+})%bbcZu}qo+hA8Ax}K)PIsB=T6&n({swo zJruiF zz-ny7W^6$MJJE-Jq;MDpj$#-l@+jdrPGB6T@C;7l44%b{cnRn63SPwpT*TXW2Or{d ze1WU@65rtle!|cAL#mXjq-v>7S|}}&G-;W%Tv{nLN*&ThX`8fN>XFjYA+dU?488+X zqmpdmXCjO~3UY)=mu5j&bceEQ_nuvQ?<9m>uM?QM=iY|eYPDdS zqpM}YsVe!b(iUxx5E|7A!8S)9ipVMi)1hpN5H8h9!7f&|Mj|TnnP_)(yG+h@uH=u?bz+izIrf%+nY|4myrdvGY=QQ9u!6cmhvSxj%)c@f^%yP+Pvha zm3A?iw=Ks$N}4htraIlOT!mOz13&+7oBjL$?YjJMU10{o4BUbYpt3vJ-AMz@_WF`Y z7o+PiUA%F-2|;}q>bMD)`^Ryjlm9TJ`&gh$1av}B(op-)KLp&;zp%soKivQC%>Dl# DaifN9 literal 0 HcmV?d00001 diff --git a/hotspot/__init__.py b/hotspot/__init__.py deleted file mode 100644 index 8a0dfa42..00000000 --- a/hotspot/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -热点雷达 (Hotspot Radar) — Sage 框架模块 - -用法: - from hotspot.engine import run_fetch - from hotspot.analysis import run_analysis - from hotspot.db import get_stats -""" -from .engine import run_fetch -from .analysis import run_analysis -from .db import get_stats -from .parsers import BUILTIN as builtin_parsers - -__all__ = ['run_fetch', 'run_analysis', 'get_stats', 'builtin_parsers'] diff --git a/hotspot/analysis.py b/hotspot/analysis.py deleted file mode 100644 index aecae172..00000000 --- a/hotspot/analysis.py +++ /dev/null @@ -1,159 +0,0 @@ -""" -五维分析引擎 + 自动状态分类 - -维度: time(时效) / heat(热度) / content(内容) / propagation(传播) / audience(受众) -状态: emerging → rising → hot → cooling → expired -""" -import json, math -from datetime import datetime, timedelta -from appPublic.uniqueID import getID -from sqlor.dbpools import DBPools - -def now_str(): - return datetime.now().strftime('%Y-%m-%d %H:%M:%S') - -DIM_NAMES = { - 'time': '时效维度', - 'heat': '热度指标', - 'content': '内容属性', - 'propagation': '传播路径', - 'audience': '受众画像', -} - -async def analyze_item(item): - """分析单条热点，返回 (new_status, dimensions_dict, detail)""" - now = datetime.now() - try: - first_seen = datetime.strptime(str(item.get('first_seen', '')), '%Y-%m-%d %H:%M:%S') - except Exception: - first_seen = now - - hours_alive = max(0.1, (now - first_seen).total_seconds() / 3600) - heat = float(item.get('heat_score', 0)) - engagement = int(item.get('engagement_count', 0)) - comments = int(item.get('comment_count', 0)) - shares = int(item.get('share_count', 0)) - - # ---- 五维评分 (0-100) ---- - dims = {} - - # 时效: 越新越高 - dims['time'] = 95 if hours_alive < 1 else 85 if hours_alive < 6 else \ - 70 if hours_alive < 24 else 50 if hours_alive < 72 else \ - 30 if hours_alive < 168 else 10 - - # 热度: 绝对值 - dims['heat'] = 95 if heat > 1e4 else 85 if heat > 5e3 else \ - 70 if heat > 1e3 else 55 if heat > 500 else \ - 35 if heat > 100 else 15 - - # 内容: 标题长度 + 摘要丰富度 + 标签 - title_len = len(item.get('title', '')) - summary_len = len(item.get('summary', '')) - dims['content'] = min(100, (20 if title_len > 15 else 10) + - (30 if summary_len > 100 else 15) + - (25 if item.get('tags') else 0) + - (25 if item.get('category') else 0)) - - # 传播: 互动总量 - total_eng = engagement + comments * 2 + shares * 3 - dims['propagation'] = 95 if total_eng > 1e4 else 80 if total_eng > 5e3 else \ - 60 if total_eng > 1e3 else 35 if total_eng > 100 else 10 - - # 受众: 互动率 - rate = total_eng / heat if heat > 0 else 0 - dims['audience'] = 90 if rate > 0.5 else 70 if rate > 0.2 else \ - 45 if rate > 0.05 else 25 if total_eng > 0 else 5 - - # ---- 状态分类 ---- - decay = math.exp(-hours_alive / 168) # 7天半衰期 - adjusted_heat = heat * decay - velocity = round(heat / hours_alive, 2) - - if hours_alive > 336: # >14天 - new_status = 'expired' - elif hours_alive > 168: # 7-14天 - new_status = 'cooling' - elif adjusted_heat > 5000: - new_status = 'hot' - elif adjusted_heat > 500: - new_status = 'rising' if velocity > 50 else 'emerging' - elif adjusted_heat > 100: - new_status = 'rising' if velocity > 100 else 'emerging' - else: - new_status = 'emerging' - - detail = { - 'hours_alive': round(hours_alive, 1), - 'adjusted_heat': round(adjusted_heat, 2), - 'heat_velocity': velocity, - 'total_engagement': total_eng, - 'decay_factor': round(decay, 4), - } - - return new_status, dims, detail - - -async def save_analysis(item_id, dims, detail, now=None): - """保存/更新五维分析记录""" - if now is None: - now = now_str() - db = DBPools() - - async with db.sqlorContext('sage') as sor: - for dim, score in dims.items(): - analysis_data = json.dumps({ - 'dimension': dim, - 'dimension_cn': DIM_NAMES.get(dim, dim), - 'score': score, - 'detail': detail, - }, ensure_ascii=False) - - old = await sor.sqlExe( - "select id from hotspot_analysis where item_id=${iid}$ and dimension=${dim}$", - {'iid': item_id, 'dim': dim}) - if old: - await sor.U('hotspot_analysis', { - 'id': old[0]['id'], 'score': score, - 'analysis_data': analysis_data, 'analyzed_at': now, - }) - else: - await sor.C('hotspot_analysis', { - 'id': getID(), 'item_id': item_id, - 'dimension': dim, 'score': score, - 'analysis_data': analysis_data, 'analyzed_at': now, - }) - - -async def run_analysis(limit=500): - """批量分析最近的热点 (默认500条)""" - db = DBPools() - now = now_str() - results = {'analyzed': 0, 'status_changes': 0} - - async with db.sqlorContext('sage') as sor: - items = await sor.sqlExe( - "select * from hotspot_item order by first_seen desc limit ${n}$", - {'n': limit}) - - for item in items: - current_status = item.get('status', 'emerging') - new_status, dims, detail = await analyze_item(item) - - await save_analysis(item['id'], dims, detail, now) - - if new_status != current_status: - results['status_changes'] += 1 - - async with db.sqlorContext('sage') as sor: - await sor.U('hotspot_item', { - 'id': item['id'], - 'heat_score': round(detail['adjusted_heat'], 2), - 'heat_velocity': detail['heat_velocity'], - 'status': new_status, - 'last_updated': now, - }) - - results['analyzed'] += 1 - - return results diff --git a/hotspot/db.py b/hotspot/db.py deleted file mode 100644 index 24fd6355..00000000 --- a/hotspot/db.py +++ /dev/null @@ -1,107 +0,0 @@ -""" -数据库操作: 保存热点条目、写抓取日志、统计查询 -""" -from datetime import datetime -from appPublic.uniqueID import getID -from sqlor.dbpools import DBPools - -def now_str(): - return datetime.now().strftime('%Y-%m-%d %H:%M:%S') - -async def get_enabled_sources(): - """获取所有启用的来源""" - db = DBPools() - async with db.sqlorContext('sage') as sor: - sources = await sor.R('hotspot_source', {}) - return [s for s in sources if s.get('enabled') == '1'] - -async def save_items(items, source_id, fetch_time=None): - """批量保存热点条目 (去重)""" - if fetch_time is None: - fetch_time = now_str() - new_count = 0 - dup_count = 0 - db = DBPools() - - async with db.sqlorContext('sage') as sor: - for item in items: - url = item.get('url', '') - if url: - existing = await sor.sqlExe( - "select id from hotspot_item where url=${u}$ limit 1", {'u': url}) - if existing: - dup_count += 1 - continue - - await sor.C('hotspot_item', { - 'id': getID(), 'source_id': source_id, - 'title': item.get('title', '')[:500], - 'url': (url or '')[:2000], - 'summary': item.get('summary', '')[:1000], - 'heat_score': item.get('heat_score', 100), - 'heat_velocity': 0, - 'publish_time': fetch_time, - 'first_seen': fetch_time, - 'last_updated': fetch_time, - 'status': 'emerging', - 'category': item.get('category', ''), - 'tags': item.get('tags', ''), - 'engagement_count': item.get('engagement_count', 0), - 'comment_count': item.get('comment_count', 0), - 'share_count': item.get('share_count', 0), - 'sentiment': 'neutral', - }) - new_count += 1 - - return new_count, dup_count - -async def update_source_last_fetch(source_id): - db = DBPools() - async with db.sqlorContext('sage') as sor: - await sor.U('hotspot_source', {'id': source_id, 'last_fetch': now_str()}) - -async def write_fetch_log(source_id, start_time, elapsed_ms, - status, items_total, items_new, items_duplicate, - error_msg='', response_code=0, response_size_bytes=0): - db = DBPools() - async with db.sqlorContext('sage') as sor: - await sor.C('hotspot_fetch_log', { - 'id': getID(), - 'source_id': source_id, - 'start_time': start_time, - 'end_time': now_str(), - 'duration_ms': elapsed_ms, - 'status': status, - 'items_total': items_total, - 'items_new': items_new, - 'items_duplicate': items_duplicate, - 'error_msg': error_msg[:1000] if error_msg else '', - 'response_code': response_code, - 'response_size_bytes': response_size_bytes, - }) - -async def get_stats(): - """全维度统计""" - from datetime import timedelta - now = datetime.now() - day_ago = (now - timedelta(hours=24)).strftime('%Y-%m-%d %H:%M:%S') - - db = DBPools() - async with db.sqlorContext('sage') as sor: - items = await sor.R('hotspot_item', {'order': 'heat_score desc'}) - sources = await sor.R('hotspot_source', {}) - logs = await sor.sqlExe( - "select * from hotspot_fetch_log where start_time >= ${t}$", - {'t': day_ago}) - - status_count = {'emerging': 0, 'rising': 0, 'hot': 0, 'cooling': 0, 'expired': 0} - for item in items: - s = item.get('status', 'emerging') - status_count[s] = status_count.get(s, 0) + 1 - - return { - 'total': len(items), - **status_count, - 'sources': sum(1 for s in sources if s.get('enabled') == '1'), - 'failures': sum(1 for log in logs if log.get('status') == 'failed'), - } diff --git a/hotspot/engine.py b/hotspot/engine.py deleted file mode 100644 index bdd7d1c0..00000000 --- a/hotspot/engine.py +++ /dev/null @@ -1,141 +0,0 @@ -""" -抓取引擎 — 调度所有来源，fetch + extract + save -""" -import json, time -import aiohttp -from datetime import datetime - -from .parsers import (BUILTIN, extract_json_path, extract_css, - extract_regex, extract_rss, fetch_hn) -from .db import (save_items, update_source_last_fetch, - write_fetch_log, get_enabled_sources) - -def now_str(): - return datetime.now().strftime('%Y-%m-%d %H:%M:%S') - -async def fetch_and_extract(session, src): - """根据 source 的 parser_config 抓取并提取条目""" - url = src.get('url', '') - headers = {} - if src.get('headers'): - try: - headers = json.loads(src.get('headers', '{}')) - except Exception: - pass - - parser_config = {} - if src.get('parser_config'): - try: - parser_config = json.loads(src.get('parser_config', '{}')) - except Exception: - pass - - parser_type = parser_config.get('parser', 'json_path') - builtin_name = parser_config.get('builtin_name', '') - - # === 内置解析器 === - if parser_type == 'builtin' and builtin_name in BUILTIN: - bp = BUILTIN[builtin_name] - fetch_url = bp['url'] or url - fetch_headers = {**headers, **bp.get('headers', {})} - - if builtin_name == 'hackernews': - items = await fetch_hn(session) - return items, 200, 0 - - if builtin_name == 'rsshub': - try: - async with session.get(url, headers=fetch_headers, - timeout=aiohttp.ClientTimeout(total=30)) as resp: - text = await resp.text() - items = extract_rss(text) - return items, resp.status, len(text.encode()) - except Exception: - return [], 0, 0 - - # 标准内置: GET → JSON → parser_fn - try: - async with session.get(fetch_url, headers=fetch_headers, - timeout=aiohttp.ClientTimeout(total=30)) as resp: - if resp.status != 200: - return [], resp.status, 0 - text = await resp.text() - size = len(text.encode()) - try: - data = json.loads(text) - except json.JSONDecodeError: - return [], resp.status, size - items = bp['parser'](data) - return items, resp.status, size - except Exception: - return [], 0, 0 - - # === 通用提取器 === - try: - async with session.get(url, headers=headers, - timeout=aiohttp.ClientTimeout(total=30)) as resp: - if resp.status != 200: - return [], resp.status, 0 - text = await resp.text() - size = len(text.encode()) - - if parser_type == 'json_path': - try: - data = json.loads(text) - except json.JSONDecodeError: - return [], 0, size - items = extract_json_path(data, parser_config) - return items, resp.status, size - - elif parser_type == 'css': - items = extract_css(text, parser_config) - return items, resp.status, size - - elif parser_type == 'regex': - items = extract_regex(text, parser_config) - return items, resp.status, size - - elif parser_type == 'rss': - items = extract_rss(text) - return items, resp.status, size - - return [], resp.status, size - - except Exception: - return [], 0, 0 - - -async def run_fetch(): - """遍历所有启用的来源，执行抓取""" - sources = await get_enabled_sources() - now = now_str() - results = {'total': len(sources), 'new_items': 0, 'errors': 0, 'details': []} - - async with aiohttp.ClientSession() as session: - for src in sources: - t0 = time.time() - try: - items, code, size = await fetch_and_extract(session, src) - except Exception as e: - await write_fetch_log(src['id'], now, int((time.time() - t0) * 1000), - 'failed', 0, 0, 0, str(e)[:1000]) - results['errors'] += 1 - continue - - elapsed = int((time.time() - t0) * 1000) - status = 'success' if code in (0, 200) else ('partial' if items else 'failed') - - new_count, dup_count = await save_items(items, src['id'], now) - await update_source_last_fetch(src['id']) - await write_fetch_log(src['id'], now, elapsed, status, - len(items), new_count, dup_count, - response_code=code, response_size_bytes=size) - - results['new_items'] += new_count - results['details'].append({ - 'source': src.get('name', src['id']), - 'items': len(items), 'new': new_count, 'dup': dup_count, - 'elapsed_ms': elapsed, 'code': code, - }) - - return results diff --git a/hotspot/parsers.py b/hotspot/parsers.py deleted file mode 100644 index c746a549..00000000 --- a/hotspot/parsers.py +++ /dev/null @@ -1,297 +0,0 @@ -""" -内置平台解析器 — 每个返回 (fetch_url, headers_override, extract_fn) - -添加新平台: 在模块底部调用 _reg() 注册即可。 -""" -import json -import re -import xml.etree.ElementTree as ET -import aiohttp - -BUILTIN = {} - -def _reg(name, url, parser_fn, headers=None): - BUILTIN[name] = {'url': url, 'parser': parser_fn, 'headers': headers or {}} - -# ---- 国内平台 ---- - -_reg('weibo_hot', - 'https://weibo.com/ajax/side/hotSearch', - lambda data: [ - {'title': item.get('word', ''), - 'url': f'https://s.weibo.com/weibo?q={item.get("word","")}', - 'heat_score': item.get('raw_hot', item.get('num', 0))} - for item in data.get('data', {}).get('realtime', [])[:50] - if isinstance(item, dict) and item.get('word') - ], - {'User-Agent': 'Mozilla/5.0', 'X-Requested-With': 'XMLHttpRequest'}) - -_reg('zhihu_hot', - 'https://www.zhihu.com/api/v3/feed/topstory/hot-lists/total?limit=50', - lambda data: [ - {'title': item.get('target', {}).get('title', ''), - 'url': f'https://www.zhihu.com/question/{item.get("target",{}).get("id","")}', - 'heat_score': int(str(item.get('detail_text', '0')) - .replace('万','0000').replace('亿','00000000') - .replace('热度','').strip() or 0), - 'summary': (item.get('target', {}).get('excerpt', '') or '')[:500]} - for item in data.get('data', []) if isinstance(item, dict) - ], - {'User-Agent': 'Mozilla/5.0'}) - -_reg('baidu_hot', - 'https://top.baidu.com/board?tab=realtime', - lambda data: [ - {'title': item.get('word', ''), - 'url': item.get('url', ''), - 'heat_score': int(item.get('hotScore', 0)), - 'summary': item.get('desc', '')} - for item in (data.get('data', {}).get('cards', [{}])[0].get('content', [])) - if isinstance(item, dict) - ], - {'User-Agent': 'Mozilla/5.0'}) - -_reg('toutiao_hot', - 'https://www.toutiao.com/hot-event/hot-board/?origin=toutiao_pc', - lambda data: [ - {'title': item.get('Title', ''), - 'url': item.get('Url', ''), - 'heat_score': int(float(item.get('HotValue', 0)))} - for item in (data.get('data', []) if isinstance(data.get('data'), list) else [])[:50] - if isinstance(item, dict) and item.get('Title') - ], - {'User-Agent': 'Mozilla/5.0', 'Referer': 'https://www.toutiao.com/'}) - -_reg('bilibili_hot', - 'https://api.bilibili.com/x/web-interface/popular?ps=50', - lambda data: [ - {'title': item.get('title', ''), - 'url': f'https://www.bilibili.com/video/{item.get("bvid","")}', - 'heat_score': item.get('stat', {}).get('view', 0), - 'summary': item.get('desc', '')[:500], - 'engagement_count': item.get('stat', {}).get('like', 0), - 'comment_count': item.get('stat', {}).get('reply', 0), - 'share_count': item.get('stat', {}).get('share', 0), - 'category': item.get('tname', '')} - for item in data.get('data', {}).get('list', []) - if isinstance(item, dict) and item.get('title') - ], - {'User-Agent': 'Mozilla/5.0', 'Referer': 'https://www.bilibili.com/'}) - -_reg('douyin_hot', - 'https://dy.feigua.cn/api/hot/board', - lambda data: [ - {'title': item.get('title', item.get('word', '')), - 'url': item.get('url', ''), - 'heat_score': int(item.get('hot_value', item.get('heat', 0)))} - for item in (data.get('data', []) if isinstance(data.get('data'), list) else [])[:50] - if isinstance(item, dict) - ], - {'User-Agent': 'Mozilla/5.0'}) - -_reg('36kr_hot', - 'https://www.36kr.com/api/search/list/info-flow/hot?pageSize=30', - lambda data: [ - {'title': item.get('title', ''), - 'url': f'https://www.36kr.com/p/{item.get("id","")}', - 'heat_score': item.get('total_count', 0), - 'summary': (item.get('summary', '') or '')[:500]} - for item in data.get('data', {}).get('items', []) - if isinstance(item, dict) and item.get('title') - ], - {'User-Agent': 'Mozilla/5.0'}) - -# ---- 海外平台 ---- - -_reg('github_trending', - 'https://api.github.com/search/repositories?q=stars:>1&sort=stars&order=desc&per_page=20', - lambda data: [ - {'title': item.get('full_name', ''), - 'url': item.get('html_url', ''), - 'heat_score': item.get('stargazers_count', 0), - 'summary': (item.get('description', '') or '')[:500], - 'category': item.get('language', '')} - for item in data.get('items', []) if isinstance(item, dict) - ], - {'Accept': 'application/vnd.github.v3+json', 'User-Agent': 'HotspotRadar/1.0'}) - -_reg('hackernews', - 'https://hacker-news.firebaseio.com/v0/topstories.json', - None, # 特殊: 两步抓取, 见 engine.py - {'User-Agent': 'Mozilla/5.0'}) - -_reg('v2ex_hot', - 'https://www.v2ex.com/api/topics/hot.json', - lambda data: [ - {'title': item.get('title', ''), - 'url': item.get('url', ''), - 'heat_score': item.get('replies', 0) * 10, - 'summary': (item.get('content', '') or '')[:500], - 'comment_count': item.get('replies', 0), - 'category': item.get('node', {}).get('title', '') - if isinstance(item.get('node'), dict) else ''} - for item in data if isinstance(item, dict) and item.get('title') - ], - {'User-Agent': 'Mozilla/5.0'}) - -_reg('rsshub', None, None, {}) # url 来自 source 配置 - -# ---- 通用提取器 ---- - -def extract_json_path(data, config): - """简易 JSONPath: $.data.list[*] → {title, url, heat_score, ...}""" - path = config.get('item_path', '$[*]') - fm = config.get('field_map', {}) - - if path.startswith('$.'): - path = path[2:] - elif path.startswith('$'): - path = path[1:] - - parts = [] - for p in path.split('.'): - p = p.strip() - if not p: continue - if p.endswith('[*]'): - parts.append(p[:-3]); parts.append('*') - else: - parts.append(p) - - items = data - for part in parts: - if part == '*': - continue - if isinstance(items, dict): - items = items.get(part) - elif isinstance(items, list): - items = [i.get(part) for i in items if isinstance(i, dict)] - items = [x for sub in items for x in (sub if isinstance(sub, list) else [sub])] - else: - return [] - if not isinstance(items, list): - items = [items] if items else [] - - result = [] - for item in items: - if not isinstance(item, dict): - continue - entry = { - 'title': _field(item, fm.get('title', 'title')), - 'url': _field(item, fm.get('url', 'url')), - 'heat_score': float(_field(item, fm.get('heat_score', '0')) or 0), - 'summary': str(_field(item, fm.get('summary', '')) or '')[:500], - 'category': str(_field(item, fm.get('category', '')) or ''), - 'tags': str(_field(item, fm.get('tags', '')) or ''), - 'engagement_count': int(_field(item, fm.get('engagement_count', '0')) or 0), - 'comment_count': int(_field(item, fm.get('comment_count', '0')) or 0), - 'share_count': int(_field(item, fm.get('share_count', '0')) or 0), - } - if entry['title']: - result.append(entry) - return result - -def _field(obj, path): - if not path or not isinstance(obj, dict): - return obj if not isinstance(obj, dict) else '' - val = obj - for p in path.split('.'): - if isinstance(val, dict): - val = val.get(p, '') - else: - return '' - return val - -def extract_css(html, config): - """CSS 选择器提取 (轻量正则版)""" - results = [] - for m in re.finditer( - r']+href=["\']([^"\']+)["\'][^>]*>([^<]*)', - html, re.IGNORECASE | re.DOTALL - ): - url, title = m.group(1), m.group(2).strip() - if len(title) >= 3: - results.append({ - 'title': title[:200], 'url': url[:2000], - 'heat_score': 60, 'summary': '', - 'category': '', 'tags': '', - 'engagement_count': 0, 'comment_count': 0, 'share_count': 0, - }) - return results[:100] - -def extract_regex(text, config): - pattern = config.get('item_regex', '') - fm = config.get('field_map', {}) - if not pattern: return [] - results = [] - for m in re.finditer(pattern, text, re.DOTALL): - gd = m.groupdict() - entry = { - 'title': gd.get(fm.get('title', 'title'), '')[:200], - 'url': gd.get(fm.get('url', 'url'), ''), - 'heat_score': float(gd.get(fm.get('heat_score', '0'), 0) or 0), - 'summary': str(gd.get(fm.get('summary', ''), ''))[:500], - 'category': '', 'tags': '', - 'engagement_count': 0, 'comment_count': 0, 'share_count': 0, - } - if entry['title']: - results.append(entry) - return results - -def extract_rss(xml_text): - items = [] - try: - root = ET.fromstring(xml_text) - ns_a = 'http://www.w3.org/2005/Atom' - for item in root.findall('.//item'): - items.append({ - 'title': item.findtext('title', '')[:200], - 'url': item.findtext('link', ''), - 'summary': (item.findtext('description', '') or '')[:500], - 'heat_score': 80, 'category': item.findtext('category', ''), - 'tags': '', 'engagement_count': 0, 'comment_count': 0, 'share_count': 0, - }) - if not items: - for e in root.findall(f'.//{{{ns_a}}}entry'): - link = e.find(f'{{{ns_a}}}link') - items.append({ - 'title': e.findtext(f'{{{ns_a}}}title', '')[:200], - 'url': link.get('href', '') if link is not None else '', - 'summary': (e.findtext(f'{{{ns_a}}}summary', '') or '')[:500], - 'heat_score': 70, 'category': '', - 'tags': '', 'engagement_count': 0, 'comment_count': 0, 'share_count': 0, - }) - except ET.ParseError: - pass - return items - -async def fetch_hn(session): - """HackerNews 两步抓取""" - items = [] - try: - async with session.get( - 'https://hacker-news.firebaseio.com/v0/topstories.json', - timeout=aiohttp.ClientTimeout(total=15) - ) as resp: - ids = await resp.json() - for sid in ids[:30]: - try: - async with session.get( - f'https://hacker-news.firebaseio.com/v0/item/{sid}.json', - timeout=aiohttp.ClientTimeout(total=10) - ) as r2: - s = await r2.json() - if s and s.get('title'): - items.append({ - 'title': s.get('title', '')[:200], - 'url': s.get('url', f'https://news.ycombinator.com/item?id={sid}'), - 'heat_score': s.get('score', 0), - 'engagement_count': s.get('score', 0), - 'comment_count': s.get('descendants', 0), - 'summary': '', 'category': s.get('type', ''), - 'tags': '', 'share_count': 0, - }) - except Exception: - continue - except Exception: - pass - return items diff --git a/init/data.xlsx b/init/data.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..82098619ab0e529a9200c60d8ad5524766f2ba67 GIT binary patch literal 10555 zcmeHtbyQUC-uBSlEe%5>ol=qlLw8E6G|Uhp2uKXwp|o^^G?LO14&5cCC@o0GJNlgS z>BHl@zJI@OowL`PJ+b!hTC?|kUBA3FKuE~M08{`v005u|d}M1dg&+U`)TjUe5da<0 zSk~Fe&Bn>iOvlT`24c$f)X{-12N{v|H2@KQ{eO@D;3qJYXlmcdLk!(RyTD^=$c?&3 zZe`Fq)3?Dqbmd_Mtrk7~U(y3ym(bnH)|Ys_g#l-s^WRf-z;Gj?xA_Yv%>MtHqVMnB^y0irmYmpL6;m6= z&&CF5xz47tUk}uxQf?fC^N{bkJ$dXaJTuj*VIL45t11^d+=-8BW_!KTdAa!3tI8}d zaBQ-!eT*doHSVCu70m$)Q?+_*Aozx!Wh}izt;4QS}wcb7TPE<^~C%@i(m!J#4c< zgllyfE=4T3R?WaR4iIjxALsvS^gmdbzukISyebHZ7azJ8a6>=TDmKr~Fy_E5zo}|{ zg}|F~WiFf&o7Z%8q9E;-T~G5Yz0xz&E9Lv6>fEQf_}(6b$?_Nh>5s;n^*TXUjg4po z31Ee&ZN<(P3=Sn*rQi9Z*f+9gzDtIet>y%AG2Qr+yFbwD36OYeVk=^h1SU5+QpPJ` zGD+H?hA!XH_#n^&Q$!boAmvD&ww$#H39DAx4nA}4&u8R_Es%r78lJ|^%6vT(%+brv z2X?&i9rzMDaKT`OM8jfqm9@ENKdPmrz!aQ{l=10oTLh)HB~KngH5g?~zB3FC_Ohh!#@0DwCHbcCl4+<&RY(-~}U>FjL( zBf$MzK?v|D2fy||`)ElRw(H`-?%fM)4IF*$8t-J2c&{~5y3+Urz^u7C(k+yH%;GGS zAlr1mpDUXz5P11qYNYULrEssC4Am|}H{q*YQpkZ!7eETg8#{C_xP)|)MF}j4DMYkwm#3f zkn}~%1FSzW2&C20C}px?QZr#G**#)JRd%=U)7MV57Dt}4aVvYjJ_?)E(qTjlIhU9ys?=)U8Fl$13kb(o&lqr0zkOo$N}9^h6R z5JUu#sUtL{DI5a~mB$&fQ&OU%?@^CskjM=mv@%zK<6=^wqG}kOD&mSZ@QxE2QI+`mTGl>46zGACJS2%9DaBv7}b-kK3vT6u0Jr zX+YWCCbS3wPKtYH>QpN4oAJwztz5{p@Idt+nL~{A#;$-Io)H^uF9<#lK0(gYweJb} z_?)o3ePc?xkOh)ib)nEidUw!1HBH6l{Xo&!T9AD*@+LXdvVRHtxpjXDHb-U$C%xKZ z|NSp|Fd_`3_g7v8DV#-*WHq0}&Oy)hZFH^hv=JEUoo4pJL8zW+K;+%~Age6cM9|lX z5^GXApTc)iEik_|a8yShg)a7CfPD+2R8!^m8+^K`s2g%0r>BC$v{z`)#+>DaM#lm^ zM~pHdpC@zCZO1NHLpZoqOqyflIiG(!CBzAn4JDfkWTl#?|3V;kBxBi3L*bPvc8|Gh zg+qkVREo5A>b?6CipxtAB&K!z-3J_*Eu&9eg=puGXj46QCq*3WrX?POz9)P0 z*XrT5D0UO#x_EOvt#j%N!XA2Rs4lfHSBUV{!}PKJMaQW&W_rK5d8ELUPe*|pD(RYB z9}mum7&I4ZflNCuut&a19qvAfT;lFbQa2`zt5K9qV_Y{GL>*pt^nvXg1AXt{;&-M5 zQGM08nO&X+aC82lE6BTh?a=ZwvGZe41F1TQyTDL9F8PZc$y1kB@tT10qx>;n z&^J=(85Pr2-UMOWGzU~-*o&EqwDODByc9TnSMN7FTTCyLq|^PXpJ2PRrfHgXIP#N~ z(_9op2~Cx|E|Ck-`+7!Qrp3A1)LfWhIj`)bR3VW)I6>p0dT6FviEH95nkgwMhsGt@ z{#Nuxkov}<;bS{0x-4$CP;E;04z2&V6kJLF^w!Y*ruWk*0DuI=k7DGf*4&=hINEUk zJoEnW)!T;ZF_ePD?TiQ13Xno2kHbPkLF1iI>SP`bCO>CZKU%MogkIT|YkeqEOE6BAk%VNcB!RMwG>xpcyNC(aPLQ5p~ zOM?N2TrC0wY)02;8z-{1VmWV`nN@q0X19?(3Z_FR%UG@o8D zmjzLcZ}vcH0N)`}Yb&Bps^#hBi1)#1$3ez}K2KlCHt&;DnEG2TkPBLV9NXz-^YG32 zInmy5$g+6*Nj@dF%H^Jv@A-NI_rm>|2l?{q?a%i6a?#ZH{EoLTym3!SA21S00x?LJ zY<$*DtuV?|#d_|-u<{ik;AnhRG;A)h!ouD|FWQg(-_ZE zGO3&Q&?(*E-Sgf}p*1zM?UJ|HczA`j>zV25stbZaxTX+KO}pTFsSo~QVIUjf-sBX0 zHf5%8m+<_;r7Dcx=*dN3_qQ!c3=tv+&epWA< zuK7n3bzf;DYvbO~LppW%)cPym;1Lgl%DA#6|PiJe2B| zX|%3zF{mUsGJTEbIdk%bvntLcJ`D7k#&SsRcqWfB7YLbC8Jp?$Wo3@ddEY#cht)Q` zO`VfTkIQOGu}g_;yds%_UV#KjWlqA#SyFk+nEaBb=4350nqZ0(soUwmU1m(X|AN|) zyJVJ>1@@r^5>TeUavey#C_8?{4mGVZ$ywq(5`!M!oM5-wac4mbPm4quh7KXY9q}WDfjlbtsd@d*(#gV2}wx^xnE6HDHQIyNp2nOR6n3rqEYER zuISlm6@P z)`Y)$3FA2u<%sCG4p3gmyXm^X1jb6!l|En>buRXbBMm4vB$ac#EYy2hI0urO34Mf2w;8%l4rfp>95QW9zRE$a7pZaR95rwzoN_M{Vzl%xReNGFfeB zJtfynDNgUViCi`3euaLlFEzd zCd1f^gUm4VQFz}>f*RNbnY?VcF?LZig~2EmgbnE0aT2I<_*WLD_fiO%cpW-K_~%iv z7~@pAleKKCKgQba;3+}v{PK`=wy2eeTKgCc3Uu=^A&89??3l@;7hlvC!a&tXgmkna z>me0n3!@LW7~`N71(c6=x|DQSZP^ox>jZdd++F}nBy(_9%qwFSfdj<)cPLyi!YAZ_ z1ho>b7i(LUS<9)~962034Xq;Na|Tw9o?1MtwKz@5wfU0K#dHA&jv#@jlHa>!#Pc?1 z#iPbB^Y~yVy0{KW^s#ay(NMdTPE_};Ek_K&d0oZvCh zmWhmZNuGvm;A$-oopH^^fG?$P=VuuTq{VbOQAQ9GMuXFilN(lSf+nnT+uqk~p`!u| zIL$!vi6Mq}2xaTg2}7pICSuwtKGdZ?F+h2fvIJ+)VcO>}%*=U~fq9&H(`%s7*DS@c4`PLvGe>l5@3`la2fJwpKpgki&}$GIG8QV66ea z(<=uQBdil)0DmQ3fmi+GNNV4x3~$nXu0j+;2&DCfw>RWFWok_w`f=e>brc_tTqNEW zXx{kxJ$Pi!L~sAo3T3%q#6jWVY-#_cI8J(%Fr`=*0#1gKNSfHnzJpL!ArbdJO!7ow zpC8rbFa|+&6|ayagtuI*Dx8NLC11+Bl)d? zXU56T)2;3~2C8bspIXlDwb{y>?lZ_B_7ANK^QZWUON3(X+^4Lt4hW2nAy6#f4<5J4K_N z;}I%Y*(g!35s6G$iCHXQs$a1ui?0dAohu*N*%ei_Z|?_+j1Lqocu=LJ+qF6R$teU? zV|}T=Y37lZ6WD&r|Bi#*r!4Qar$w;7!HKlN4WfM;D5gOiv_EW#pBLfu(Ywn=>RW)J zz|kk}!IaP3V-N4(Djbe-UiBo1tT}b6>MP1-el+y&2Rii}_EG=@(E$JT-#-g{=)i=p3G#>;#sz{N@t6EBJqc*p?fCPKgcBKW42f@+*w=-eartGH z@ax6Zzv_jWlB%PDkJ+}BT%aum_D~IBTqe0T)F{Ie?|r+>Fk5~OcAt;;<`-b4VYXn+74S9v*+3^s?f#<-@S;%Bx*(utYb3Jo{XZ)fvzgb8%)c3_kU&(B#Y zPvBOqk~H*@hR2!rgYPC4ln#7cP5%KLHk3)4qkt( zh6nxPu^$<8XegB08TF}ra7+^M5m(th=9VP-mYMkn&o9=4hr~DCfiuH7yAfol<3m`@ zaeFw`vz-xvX9ASn679kj&Onz|K)`s zPb|PT*4l1hTc^i=#q4-#H6&ha`4hzj`fr(!HWBfa$K;vvb`9Pk0bq48v`83imE&lxnb%YdYr(&dRTj@_GD z{`7Q$mpP-WDj65D#*I+NJ!HrX1NcI1AtpLmXbo{x5Es@26W%Sawc>yX9q; z9l!g&EWP&)0%b~yl&0EL#^)S|d#O6Yqf&M*H%qSe#1s?s20%I(d508!ZOez$Bns`v`O1e0IF_3SR(KzW#BYd@GRyFQlJks?Q>^QJtNKh& z55+5`FTzdGLi1I=D?rI&a$?PzbE z#2;xRU;jyuwKZ%5-1@B5x_mlXtXzsi9H`Z0LrL2X@zMd&beeBW!pgOHHEM@o?-bgH zUM1X`Qanp}{xald{~)WYe$Ngu z!sX!|(+&Z8eDn_==)M;RRE9_l(C=qY%Sut)WpBj9Yxur{wayPZe;N z_xO@iXD@+#>s4fLv1IpX#v6(U{qm1b%YtxU5rSjRwgWZaU%T$lbG+O)1M^&t!D#VH zqulL}11rsSq;W|uH>>Nn_~PlCIE)hGYo&WktE7P}K|-~QWO&W$=-W0aGOxW!)ct(C z!nb)2M~1+6qC4fEVQg40`?v@)QOMh-cGEJSLA9Wj6kY8kcF@YK()ORcy*<8+o0h87 zf%C=|?i&gI#vNy{&3~EmL+pPYKJl8ME*|3GWt1D)e($sb8*Q2S7Xz78n0my^GT`Rm zIwhr?#p_nv=1JV1zBI^gpXf{LQdw;@GxM?u_6NXlO+dIk{iM)?%8h$1@-N&~F!bzE zMDGR2m#&_(uif34dsj1hmz<>oH8YQM!!gd7e-PwD8s5l~nXPZgk##oW~%l_@qszCwpc8yPJK8!&!$qfwz5*KJMEI~WUO zdYf&nw04?1>{_*qMopWlo$43X-?`QaWLrz*(o&A6IKXCwF1*ajJ%#UU`DfazM6q>N z2B)GTJR`^dor)GNF8^iXzj+AvVF?{h6F<^^#TMCh?`c~y-6tE3GIxv6*!AWdE8mMG zL``0%6^~NwJm*f>{LrNzb5@^~dzE8aSzr*?sZ2Ty9QUFj z+s#)ybpCsD!Zd>{Uz>pHzD6kZ>&6jovmzTz0z%Puk;YFsy}SBa)ho`pS8nZzdCOZ>stIJEPXHD%UCe zEUjgcG(>24HKkQXp}+gFhkdpt*iL)sE#gHi)s9=mx_~Z+{hh{jv&{hkbY)4>3Bgys zKA<&1yUEcUBoMvLrsN@DSnjx=?#MiNQw7{hgmhfbOR>t#z;t!d*I*{qetC9osx;L% zhp$#XKGQmO*zvBTs>+!M{KiG4(PIMOTa?T^%Kz?=ZT}t5YP1Nm!V>8_wv`u2WO?p0 zl~Xm}uIFP;788(g8LcM#J}!y*#Wc85S73Me9{0)Dm8b^9S8llohXgczswymS1{+eg zzX0!fu^)*+ge9?`Ka(8%EXv)HSFpZDH1Z=9|HW_w#Jlhe{LeePey!7VO9S+G zfWL1y`5mAPE}P#rpWHV5`_TP&!)3S{|MwC6ZJgVq*I!8V82=c=-Zs8HOZsITh5g6L z(rtj-O~_vWNrXQEezhcTo8E5B{4#AJ`9FXBA8nf3D7S0#Unnr.json()).then(d=>{var m='完成: '+d.total+'来源, 新增'+d.new_items+'条';if(d.errors)m+='\\\\n失败: '+d.errors;alert(m);setTimeout(()=>location.reload(),1000);}).catch(e=>alert('异常: '+e))", - "style": "margin:6px;" - } - }, - { - "widgettype": "Button", - "options": { - "label": "执行分析", - "actiontype": "script", - "script": "fetch('/hotspot/analyze.dspy').then(r=>r.json()).then(d=>{alert('已分析: '+d.analyzed+'条热点');setTimeout(()=>location.reload(),1000);}).catch(e=>alert('分析异常: '+e))", - "style": "margin:6px;" - } - }, - { - "widgettype": "Button", - "options": { - "label": "刷新统计", - "actiontype": "script", - "script": "fetch('/hotspot/stats.dspy').then(r=>r.json()).then(d=>{['emerging','rising','hot','cooling','expired','sources','failures'].forEach(k=>{var el=document.querySelectorAll('[id=stat_'+k+'] div')[1];if(el)el.textContent=d[k]||0;});}).catch(e=>console.error('stats err',e))", - "style": "margin:6px;" - } - } - ] - } - }, - { - "widgettype": "Tab", - "options": { - "tabs": [ - { - "title": "来源管理", - "content": { - "widgettype": "urlwidget", - "options": {"url": "{{ entire_url('hotspot_source') }}"} - } - }, - { - "title": "调度配置", - "content": { - "widgettype": "urlwidget", - "options": {"url": "{{ entire_url('hotspot_schedule') }}"} - } - }, - { - "title": "抓取日志", - "content": { - "widgettype": "urlwidget", - "options": {"url": "{{ entire_url('hotspot_fetch_log') }}"} - } - }, - { - "title": "热点条目", - "content": { - "widgettype": "urlwidget", - "options": {"url": "{{ entire_url('hotspot_item') }}"} - } - }, - { - "title": "五维分析", - "content": { - "widgettype": "urlwidget", - "options": {"url": "{{ entire_url('hotspot_analysis') }}"} - } - }, - { - "title": "预警规则", - "content": { - "widgettype": "urlwidget", - "options": {"url": "{{ entire_url('hotspot_alert') }}"} - } - } - ] - } - }, - { - "widgettype": "Text", - "options": { - "text": "工作流: ① 添加来源 → parser_config选parser=builtin,builtin_name=weibo/zhihu/baidu/bilibili/... 或自定义json_path/css/regex → ② 调度配置 → ③ 抓取 → ④ 分析 → ⑤ 浏览 | 内置解析器: weibo_hot zhihu_hot baidu_hot toutiao_hot bilibili_hot douyin_hot 36kr_hot github_trending hackernews v2ex_hot rsshub", - "style": "font-size:12px;color:#bbb;padding:16px;border-top:1px solid #eee;margin-top:8px;" - } - } - ] - } -} diff --git a/wwwroot/hotspot/stats.dspy b/wwwroot/hotspot/stats.dspy deleted file mode 100644 index cd064e5e..00000000 --- a/wwwroot/hotspot/stats.dspy +++ /dev/null @@ -1,2 +0,0 @@ -from hotspot.db import get_stats -return await get_stats() diff --git a/wwwroot/imgs/ocai1.svg b/wwwroot/imgs/ocai1.svg new file mode 100644 index 00000000..a08000df --- /dev/null +++ b/wwwroot/imgs/ocai1.svg @@ -0,0 +1,168 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +