"""Why the dead links are dead: a block, a rule, or rot?

55 of the 99 dead links in the first pass answered **403**, not 404, which is a
different fact about a different thing. A 404 is rot: the page moved and the
list was not updated. A 403 is a refusal, and there are three quite different
reasons a site might refuse, only one of which is the publisher's fault in the
way "broken link" implies:

  1. the URL is gone for everybody, and 403 is how this site says so;
  2. the site refuses **declared bots** while serving browsers — so its own
     llms.txt, written for machines, names pages machines may not have;
  3. the site's **robots.txt** already forbade the path, so the llms.txt and the
     robots.txt contradict each other and the 403 is the site being consistent
     with the rule it published rather than with the list it published.

So each refused URL is asked twice more: once with an ordinary browser
User-Agent, and once against the domain's own robots.txt. The browser fetch is a
single request per URL, nothing is scraped from it, and the point of it is to
report *why*, not to get past anything.

    python research/llms-txt-links/control.py

Reads `links.jsonl` from the probe, writes `control.jsonl` beside it.
"""

import argparse
import json
import time
import urllib.error
import urllib.parse
import urllib.request
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

BROWSER = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
           "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
DECLARED = ("Mozilla/5.0 (compatible; VesperBot/1.0; "
            "+https://untilnextsession.com/agents/)")
NAMED_AGENTS = ("vesperbot", "gptbot", "claudebot", "ccbot", "anthropic-ai",
                "google-extended", "perplexitybot")


def get(url, agent, timeout=12):
    request = urllib.request.Request(url, headers={"User-Agent": agent})
    try:
        with urllib.request.urlopen(request, timeout=timeout) as answer:
            answer.read(60_000)
            return answer.status, None
    except urllib.error.HTTPError as err:
        try:
            err.read(2_000)
        except Exception:
            pass
        return err.code, None
    except Exception as exc:
        return None, f"{type(exc).__name__}"


def robots_for(domain, cache):
    if domain not in cache:
        status, text = None, ""
        try:
            request = urllib.request.Request(f"https://{domain}/robots.txt",
                                             headers={"User-Agent": DECLARED})
            with urllib.request.urlopen(request, timeout=12) as answer:
                status = answer.status
                text = answer.read(200_000).decode("utf-8", "replace")
        except Exception:
            pass
        cache[domain] = (status, text)
    return cache[domain]


def groups(text):
    """`{user-agent: [disallow paths]}`, lowercased, the crude reading."""
    out, current = {}, []
    for raw in text.splitlines():
        line = raw.split("#", 1)[0].strip()
        if not line or ":" not in line:
            continue
        field, _, value = line.partition(":")
        field, value = field.strip().lower(), value.strip()
        if field == "user-agent":
            current = out.setdefault(value.lower(), [])
        elif field == "disallow" and value:
            for agent in ([current] if isinstance(current, list) else []):
                agent.append(value)
    return out


def forbidden(path, rules):
    """Does any group that would apply to a declared bot disallow this path?

    Deliberately generous to the site: `*` counts, and so does any of the named
    AI agents, because a site that names GPTBot is plainly making a rule about
    machines and my crawler is one.
    """
    for agent, paths in rules.items():
        if agent != "*" and agent not in NAMED_AGENTS:
            continue
        for rule in paths:
            if rule == "/" or path.startswith(rule.rstrip("*")):
                return f"{agent}: Disallow: {rule}"
    return None


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--out", default=str(Path.home() / "data" / "llms-txt-links"))
    args = ap.parse_args()
    out = Path(args.out).expanduser()
    rows = [json.loads(l) for l in open(out / "links.jsonl") if l.strip()]
    dead = [r for r in rows if r["dead"]]
    print(f"{len(dead)} dead links to explain")

    cache = {}
    for row in dead:                       # sequential: one robots.txt a domain
        robots_for(row["domain"], cache)

    def explain(row):
        status, error = get(row["url"], BROWSER)
        time.sleep(0.2)
        _, text = cache.get(row["domain"], (None, ""))
        path = urllib.parse.urlsplit(row["url"]).path or "/"
        rule = forbidden(path, groups(text)) if text else None
        return {**row, "browser_status": status, "browser_error": error,
                "robots_rule": rule}

    with ThreadPoolExecutor(max_workers=8) as pool:
        explained = list(pool.map(explain, dead))

    with open(out / "control.jsonl", "w") as handle:
        for row in explained:
            handle.write(json.dumps(row) + "\n")

    def bucket(row):
        if row["status"] == 404:
            return "gone (404)"
        if row["robots_rule"]:
            return "the site's own robots.txt forbids the path"
        browser_ok = row["browser_status"] is not None and row["browser_status"] < 400
        if browser_ok:
            return "served to a browser, refused to a declared bot"
        return "refused to everyone"

    counts = {}
    for row in explained:
        counts[bucket(row)] = counts.get(bucket(row), 0) + 1
    print()
    for name, n in sorted(counts.items(), key=lambda kv: -kv[1]):
        print(f"  {n:>4}  {name}")
    print()
    per_domain = {}
    for row in explained:
        per_domain.setdefault(row["domain"], set()).add(bucket(row))
    for domain in sorted(per_domain):
        print(f"  {domain:<22} {', '.join(sorted(per_domain[domain]))}")


if __name__ == "__main__":
    main()