#!/usr/bin/env python3
"""Feeds on the top 1,000 front pages. Question, method and kill rule: README.md.

Usage: python probe.py [--summarise-only]
Raw: ~/data/feeds/results.jsonl (resumable). Out: summary.md, domains.csv here.
"""
import csv, json, os, re, sys, xml.etree.ElementTree as ET
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timedelta, timezone
from email.utils import parsedate_to_datetime
from urllib.parse import urljoin
import requests

HERE = os.path.dirname(os.path.abspath(__file__))
DATA = os.path.expanduser("~/data/feeds")
RAW = os.path.join(DATA, "results.jsonl")
DOMAINS = os.path.join(HERE, "..", "llms-txt", "domains.csv")
UA = "Mozilla/5.0 (compatible; Vesper/1.0; +https://untilnextsession.com/agents)"
FEED_TYPES = ("application/rss+xml", "application/atom+xml", "application/feed+json")  # not application/json: WordPress advertises its REST API that way
GUESSES = ("/feed", "/rss.xml", "/atom.xml")
NOW = datetime.now(timezone.utc)
FRESH = timedelta(days=30)
LINK_RE = re.compile(r"<link\b[^>]*>", re.I)
ATTR_RE = re.compile(r"""([a-zA-Z:-]+)\s*=\s*("([^"]*)"|'([^']*)'|([^\s>]+))""")
DATE_TAGS = {"pubdate", "published", "updated", "date", "date_published", "lastbuilddate"}


def get(url, timeout=15, limit=400_000):
    try:
        r = requests.get(url, headers={"User-Agent": UA}, timeout=timeout, allow_redirects=True, stream=True)
        body = b""
        for chunk in r.iter_content(65536):
            body += chunk
            if len(body) > limit:
                break
        r.close()
        return r.status_code, r.headers.get("content-type", ""), body, r.url, None
    except Exception as e:  # noqa: BLE001
        return None, "", b"", url, type(e).__name__


def attrs(tag):
    out = {}
    for m in ATTR_RE.finditer(tag):
        out[m.group(1).lower()] = (m.group(3) or m.group(4) or m.group(5) or "")
    return out


def advertised(html, base):
    found = []
    for tag in LINK_RE.findall(html):
        a = attrs(tag)
        rel = a.get("rel", "").lower().split()
        typ = a.get("type", "").lower().split(";")[0].strip()
        if "alternate" in rel and typ in FEED_TYPES and a.get("href"):
            found.append({"url": urljoin(base, a["href"].strip()), "type": typ, "title": a.get("title", "")[:80]})
    seen, uniq = set(), []
    for f in found:
        if f["url"] not in seen:
            seen.add(f["url"]); uniq.append(f)
    return uniq


def parse_date(text):
    text = (text or "").strip()
    if not text:
        return None
    try:
        d = parsedate_to_datetime(text)
    except Exception:  # noqa: BLE001
        try:
            d = datetime.fromisoformat(text.replace("Z", "+00:00"))
        except Exception:  # noqa: BLE001
            return None
    if d.tzinfo is None:
        d = d.replace(tzinfo=timezone.utc)
    return d


def judge(body, ctype):
    """(kind, items, newest_iso) — kind in rss/atom/json/none."""
    text = body.lstrip()[:200]
    if text.startswith(b"{"):
        try:
            j = json.loads(body.decode("utf-8", "replace"))
        except Exception:  # noqa: BLE001
            return "none", 0, None
        items = j.get("items") if isinstance(j, dict) else None
        if not isinstance(items, list):
            return "none", 0, None
        dates = [parse_date(i.get("date_published") or i.get("date_modified")) for i in items if isinstance(i, dict)]
        dates = [d for d in dates if d]
        return "json", len(items), (max(dates).isoformat() if dates else None)
    try:
        root = ET.fromstring(body)
    except Exception:  # noqa: BLE001
        return "none", 0, None
    local = root.tag.split("}")[-1].lower()
    if local == "rss" or local == "rdf":
        kind, item_tag = "rss", "item"
    elif local == "feed":
        kind, item_tag = "atom", "entry"
    else:
        return "none", 0, None
    items = [e for e in root.iter() if e.tag.split("}")[-1].lower() == item_tag]
    dates = []
    for it in items:
        for e in it:
            if e.tag.split("}")[-1].lower() in DATE_TAGS:
                d = parse_date(e.text)
                if d:
                    dates.append(d)
    return kind, len(items), (max(dates).isoformat() if dates else None)


def check_feed(url):
    status, ctype, body, final, err = get(url, timeout=15, limit=4_000_000)  # release feeds run to 2 MB
    if err or status != 200:
        return {"url": url, "status": status, "error": err, "kind": "none", "items": 0, "newest": None, "fresh": False}
    kind, items, newest = judge(body, ctype)
    fresh = bool(newest) and (NOW - datetime.fromisoformat(newest)) <= FRESH
    return {"url": url, "status": status, "error": None, "kind": kind, "items": items, "newest": newest, "fresh": fresh, "final": final}


def probe(rank, domain):
    status, ctype, body, final, err = get(f"https://{domain}/")
    row = {"rank": rank, "domain": domain, "status": status, "error": err, "final": final, "html": False,
           "advertised": [], "checked": [], "hidden": []}
    if err or status != 200:
        return row
    html = body.decode("utf-8", "replace")
    head = html.lstrip()[:300].lower()
    row["html"] = "<html" in head or "<!doctype" in head or "text/html" in ctype.lower()
    if not row["html"]:
        return row
    row["advertised"] = advertised(html, final)
    for f in row["advertised"][:3]:
        row["checked"].append(check_feed(f["url"]))
    if not row["advertised"]:
        for path in GUESSES:
            c = check_feed(urljoin(final, path))
            if c["kind"] != "none":
                row["hidden"].append(c)
    return row


def summarise():
    rows = [json.loads(l) for l in open(RAW)] if os.path.exists(RAW) else []
    rows.sort(key=lambda r: r["rank"])
    readable = [r for r in rows if r["status"] == 200 and r["html"]]
    adv = [r for r in readable if r["advertised"]]
    valid = [r for r in adv if any(c["kind"] != "none" for c in r["checked"])]
    fresh = [r for r in adv if any(c["fresh"] for c in r["checked"])]
    hidden = [r for r in readable if not r["advertised"] and r["hidden"]]
    hidden_fresh = [r for r in hidden if any(c["fresh"] for c in r["hidden"])]
    kinds = {}
    for r in adv:
        for c in r["checked"]:
            if c["kind"] != "none":
                kinds[c["kind"]] = kinds.get(c["kind"], 0) + 1
    n = len(readable) or 1
    share = 100 * len(adv) / n
    verdict = "dead" if share >= 40 else ("alive" if share <= 15 else "partly")
    by_band = []
    for lo in range(0, 1000, 100):
        band = [r for r in readable if lo < r["rank"] <= lo + 100]
        ba = [r for r in band if r["advertised"]]
        bf = [r for r in ba if any(c["fresh"] for c in r["checked"])]
        by_band.append((lo + 1, lo + 100, len(band), len(ba), len(bf)))
    with open(os.path.join(HERE, "summary.md"), "w") as f:
        f.write(f"# Feeds on the top 1,000 — run {NOW.date().isoformat()}\n\n")
        f.write(f"- Domains probed: {len(rows)}\n- Front page 200 and HTML (the denominator): {len(readable)}\n")
        f.write(f"- Refused, timed out or not HTML: {len(rows) - len(readable)}\n")
        f.write(f"- **Advertise a feed by autodiscovery: {len(adv)} = {share:.1f} % of readable**\n")
        f.write(f"- Of those, at least one advertised feed parses: {len(valid)} ({100*len(valid)/max(1,len(adv)):.0f} % of advertisers)\n")
        f.write(f"- Of those, at least one advertised feed has an item from the last 30 days: {len(fresh)} ({100*len(fresh)/n:.1f} % of readable)\n")
        f.write(f"- Feed formats among valid advertised feeds (feeds, not domains): {kinds}\n")
        f.write(f"- No advertised feed but one of /feed, /rss.xml, /atom.xml parses (hidden): {len(hidden)}, fresh: {len(hidden_fresh)}\n")
        f.write(f"- **Verdict by the pre-registered rule (dead ≥ 40 %, alive ≤ 15 %): the claim 'RSS is dead' is {verdict}.**\n\n")
        f.write("| rank band | readable | advertise | fresh |\n|---|---|---|---|\n")
        for lo, hi, nb, na, nf in by_band:
            f.write(f"| {lo}–{hi} | {nb} | {na} | {nf} |\n")
        f.write("\nFresh advertised feeds, by domain (rank, domain, newest item):\n\n")
        for r in fresh:
            best = max((c for c in r["checked"] if c["fresh"]), key=lambda c: c["newest"])
            f.write(f"- {r['rank']} {r['domain']} — {best['newest'][:10]} ({best['kind']}, {best['items']} items)\n")
    with open(os.path.join(HERE, "domains.csv"), "w", newline="") as f:
        w = csv.writer(f)
        w.writerow(["rank", "domain", "status", "html", "advertised", "valid", "fresh", "newest", "hidden", "first_feed_url"])
        for r in rows:
            ch = r["checked"]
            newest = max((c["newest"] for c in ch if c["newest"]), default="")
            w.writerow([r["rank"], r["domain"], r["status"] or r["error"], int(r["html"]), len(r["advertised"]),
                        int(any(c["kind"] != "none" for c in ch)), int(any(c["fresh"] for c in ch)), newest,
                        len(r["hidden"]), (r["advertised"][0]["url"] if r["advertised"] else "")])
    print(open(os.path.join(HERE, "summary.md")).read()[:1500])


def main():
    os.makedirs(DATA, exist_ok=True)
    if "--summarise-only" not in sys.argv:
        done = set()
        if os.path.exists(RAW):
            done = {json.loads(l)["domain"] for l in open(RAW)}
        todo = [(int(r["rank"]), r["domain"]) for r in csv.DictReader(open(DOMAINS)) if r["domain"] not in done]
        print(f"{len(todo)} domains to probe", flush=True)
        with open(RAW, "a") as fo, ThreadPoolExecutor(max_workers=32) as ex:
            futs = {ex.submit(probe, rk, d): d for rk, d in todo}
            for i, fut in enumerate(as_completed(futs), 1):
                try:
                    fo.write(json.dumps(fut.result()) + "\n"); fo.flush()
                except Exception as e:  # noqa: BLE001
                    print("fail", futs[fut], type(e).__name__, flush=True)
                if i % 100 == 0:
                    print(i, flush=True)
    summarise()


if __name__ == "__main__":
    main()