"""
抓取引擎 — 调度所有来源，fetch + extract + save
"""
import json, time
import aiohttp
from datetime import datetime

from .parsers import (BUILTIN, extract_json_path, extract_css,
                       extract_regex, extract_rss, fetch_hn)
from .db import (save_items, update_source_last_fetch,
                  write_fetch_log, get_enabled_sources)

def now_str():
    return datetime.now().strftime('%Y-%m-%d %H:%M:%S')

async def fetch_and_extract(session, src):
    """根据 source 的 parser_config 抓取并提取条目"""
    url = src.get('url', '')
    headers = {}
    if src.get('headers'):
        try:
            headers = json.loads(src.get('headers', '{}'))
        except Exception:
            pass

    parser_config = {}
    if src.get('parser_config'):
        try:
            parser_config = json.loads(src.get('parser_config', '{}'))
        except Exception:
            pass

    parser_type = parser_config.get('parser', 'json_path')
    builtin_name = parser_config.get('builtin_name', '')

    # === 内置解析器 ===
    if parser_type == 'builtin' and builtin_name in BUILTIN:
        bp = BUILTIN[builtin_name]
        fetch_url = bp['url'] or url
        fetch_headers = {**headers, **bp.get('headers', {})}

        if builtin_name == 'hackernews':
            items = await fetch_hn(session)
            return items, 200, 0

        if builtin_name == 'rsshub':
            try:
                async with session.get(url, headers=fetch_headers,
                                       timeout=aiohttp.ClientTimeout(total=30)) as resp:
                    text = await resp.text()
                    items = extract_rss(text)
                return items, resp.status, len(text.encode())
            except Exception:
                return [], 0, 0

        # 标准内置: GET → JSON → parser_fn
        try:
            async with session.get(fetch_url, headers=fetch_headers,
                                   timeout=aiohttp.ClientTimeout(total=30)) as resp:
                if resp.status != 200:
                    return [], resp.status, 0
                text = await resp.text()
                size = len(text.encode())
                try:
                    data = json.loads(text)
                except json.JSONDecodeError:
                    return [], resp.status, size
            items = bp['parser'](data)
            return items, resp.status, size
        except Exception:
            return [], 0, 0

    # === 通用提取器 ===
    try:
        async with session.get(url, headers=headers,
                               timeout=aiohttp.ClientTimeout(total=30)) as resp:
            if resp.status != 200:
                return [], resp.status, 0
            text = await resp.text()
            size = len(text.encode())

        if parser_type == 'json_path':
            try:
                data = json.loads(text)
            except json.JSONDecodeError:
                return [], 0, size
            items = extract_json_path(data, parser_config)
            return items, resp.status, size

        elif parser_type == 'css':
            items = extract_css(text, parser_config)
            return items, resp.status, size

        elif parser_type == 'regex':
            items = extract_regex(text, parser_config)
            return items, resp.status, size

        elif parser_type == 'rss':
            items = extract_rss(text)
            return items, resp.status, size

        return [], resp.status, size

    except Exception:
        return [], 0, 0


async def run_fetch():
    """遍历所有启用的来源，执行抓取"""
    sources = await get_enabled_sources()
    now = now_str()
    results = {'total': len(sources), 'new_items': 0, 'errors': 0, 'details': []}

    async with aiohttp.ClientSession() as session:
        for src in sources:
            t0 = time.time()
            try:
                items, code, size = await fetch_and_extract(session, src)
            except Exception as e:
                await write_fetch_log(src['id'], now, int((time.time() - t0) * 1000),
                                      'failed', 0, 0, 0, str(e)[:1000])
                results['errors'] += 1
                continue

            elapsed = int((time.time() - t0) * 1000)
            status = 'success' if code in (0, 200) else ('partial' if items else 'failed')

            new_count, dup_count = await save_items(items, src['id'], now)
            await update_source_last_fetch(src['id'])
            await write_fetch_log(src['id'], now, elapsed, status,
                                  len(items), new_count, dup_count,
                                  response_code=code, response_size_bytes=size)

            results['new_items'] += new_count
            results['details'].append({
                'source': src.get('name', src['id']),
                'items': len(items), 'new': new_count, 'dup': dup_count,
                'elapsed_ms': elapsed, 'code': code,
            })

    return results