142 lines
4.9 KiB
Python
142 lines
4.9 KiB
Python
"""
|
||
抓取引擎 — 调度所有来源，fetch + extract + save
|
||
"""
|
||
import json, time
|
||
import aiohttp
|
||
from datetime import datetime
|
||
|
||
from .parsers import (BUILTIN, extract_json_path, extract_css,
|
||
extract_regex, extract_rss, fetch_hn)
|
||
from .db import (save_items, update_source_last_fetch,
|
||
write_fetch_log, get_enabled_sources)
|
||
|
||
def now_str():
|
||
return datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||
|
||
async def fetch_and_extract(session, src):
|
||
"""根据 source 的 parser_config 抓取并提取条目"""
|
||
url = src.get('url', '')
|
||
headers = {}
|
||
if src.get('headers'):
|
||
try:
|
||
headers = json.loads(src.get('headers', '{}'))
|
||
except Exception:
|
||
pass
|
||
|
||
parser_config = {}
|
||
if src.get('parser_config'):
|
||
try:
|
||
parser_config = json.loads(src.get('parser_config', '{}'))
|
||
except Exception:
|
||
pass
|
||
|
||
parser_type = parser_config.get('parser', 'json_path')
|
||
builtin_name = parser_config.get('builtin_name', '')
|
||
|
||
# === 内置解析器 ===
|
||
if parser_type == 'builtin' and builtin_name in BUILTIN:
|
||
bp = BUILTIN[builtin_name]
|
||
fetch_url = bp['url'] or url
|
||
fetch_headers = {**headers, **bp.get('headers', {})}
|
||
|
||
if builtin_name == 'hackernews':
|
||
items = await fetch_hn(session)
|
||
return items, 200, 0
|
||
|
||
if builtin_name == 'rsshub':
|
||
try:
|
||
async with session.get(url, headers=fetch_headers,
|
||
timeout=aiohttp.ClientTimeout(total=30)) as resp:
|
||
text = await resp.text()
|
||
items = extract_rss(text)
|
||
return items, resp.status, len(text.encode())
|
||
except Exception:
|
||
return [], 0, 0
|
||
|
||
# 标准内置: GET → JSON → parser_fn
|
||
try:
|
||
async with session.get(fetch_url, headers=fetch_headers,
|
||
timeout=aiohttp.ClientTimeout(total=30)) as resp:
|
||
if resp.status != 200:
|
||
return [], resp.status, 0
|
||
text = await resp.text()
|
||
size = len(text.encode())
|
||
try:
|
||
data = json.loads(text)
|
||
except json.JSONDecodeError:
|
||
return [], resp.status, size
|
||
items = bp['parser'](data)
|
||
return items, resp.status, size
|
||
except Exception:
|
||
return [], 0, 0
|
||
|
||
# === 通用提取器 ===
|
||
try:
|
||
async with session.get(url, headers=headers,
|
||
timeout=aiohttp.ClientTimeout(total=30)) as resp:
|
||
if resp.status != 200:
|
||
return [], resp.status, 0
|
||
text = await resp.text()
|
||
size = len(text.encode())
|
||
|
||
if parser_type == 'json_path':
|
||
try:
|
||
data = json.loads(text)
|
||
except json.JSONDecodeError:
|
||
return [], 0, size
|
||
items = extract_json_path(data, parser_config)
|
||
return items, resp.status, size
|
||
|
||
elif parser_type == 'css':
|
||
items = extract_css(text, parser_config)
|
||
return items, resp.status, size
|
||
|
||
elif parser_type == 'regex':
|
||
items = extract_regex(text, parser_config)
|
||
return items, resp.status, size
|
||
|
||
elif parser_type == 'rss':
|
||
items = extract_rss(text)
|
||
return items, resp.status, size
|
||
|
||
return [], resp.status, size
|
||
|
||
except Exception:
|
||
return [], 0, 0
|
||
|
||
|
||
async def run_fetch():
|
||
"""遍历所有启用的来源，执行抓取"""
|
||
sources = await get_enabled_sources()
|
||
now = now_str()
|
||
results = {'total': len(sources), 'new_items': 0, 'errors': 0, 'details': []}
|
||
|
||
async with aiohttp.ClientSession() as session:
|
||
for src in sources:
|
||
t0 = time.time()
|
||
try:
|
||
items, code, size = await fetch_and_extract(session, src)
|
||
except Exception as e:
|
||
await write_fetch_log(src['id'], now, int((time.time() - t0) * 1000),
|
||
'failed', 0, 0, 0, str(e)[:1000])
|
||
results['errors'] += 1
|
||
continue
|
||
|
||
elapsed = int((time.time() - t0) * 1000)
|
||
status = 'success' if code in (0, 200) else ('partial' if items else 'failed')
|
||
|
||
new_count, dup_count = await save_items(items, src['id'], now)
|
||
await update_source_last_fetch(src['id'])
|
||
await write_fetch_log(src['id'], now, elapsed, status,
|
||
len(items), new_count, dup_count,
|
||
response_code=code, response_size_bytes=size)
|
||
|
||
results['new_items'] += new_count
|
||
results['details'].append({
|
||
'source': src.get('name', src['id']),
|
||
'items': len(items), 'new': new_count, 'dup': dup_count,
|
||
'elapsed_ms': elapsed, 'code': code,
|
||
})
|
||
|
||
return results
|