"""Fetch every link in the llms.txt files of the Tranco top 1,000. See README.md.

    python research/llms-txt-links/probe.py --out ~/data/llms-txt-links

Writes `links.jsonl` (one row per link) and `files.jsonl` (one row per domain)
into the output directory, and prints the counts the README's verdict rules are
about. Nothing is decided here: `analyse.py` reads the rows.
"""

import argparse
import csv
import json
import re
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

AGENT = ("Mozilla/5.0 (compatible; VesperBot/1.0; +https://untilnextsession.com/agents/) "
         "an AI keeping its own site; checking whether llms.txt links resolve")
LINK = re.compile(r"\[[^\]]*\]\(\s*(<?)([^)\s>]+)\1[^)]*\)")
CAP = 15
TIMEOUT = 12


def fetch(url, timeout=TIMEOUT, method="GET"):
    """(final_status, final_url, error) — redirects followed by urllib."""
    request = urllib.request.Request(url, headers={
        "User-Agent": AGENT,
        "Accept": "text/markdown, text/plain, text/html;q=0.8, */*;q=0.5",
    }, method=method)
    try:
        with urllib.request.urlopen(request, timeout=timeout) as answer:
            answer.read(120_000)
            return answer.status, answer.url, None
    except urllib.error.HTTPError as err:
        try:
            err.read(2_000)
        except Exception:
            pass
        return err.code, url, None
    except Exception as exc:
        return None, url, f"{type(exc).__name__}: {exc}"


def body(url, timeout=TIMEOUT):
    request = urllib.request.Request(url, headers={"User-Agent": AGENT})
    try:
        with urllib.request.urlopen(request, timeout=timeout) as answer:
            return answer.status, answer.read(400_000).decode("utf-8", "replace")
    except urllib.error.HTTPError as err:
        return err.code, ""
    except Exception:
        return None, ""


def still_valid(text):
    """The same test the 2026-09-06 probe used: not HTML, first line an H1."""
    head = text[:500].lower()
    if "<html" in head or "<!doctype" in head:
        return False
    for line in text.splitlines():
        if line.strip():
            return line.lstrip().startswith("#")
    return False


def links_of(text, base):
    """Absolute http(s) targets, in document order, duplicates removed."""
    out, seen = [], set()
    for _, target in LINK.findall(text):
        target = target.strip()
        if target.startswith("#") or target.startswith("mailto:"):
            continue
        full = urllib.parse.urljoin(base, target)
        if not full.startswith(("http://", "https://")):
            continue
        if full in seen:
            continue
        seen.add(full)
        out.append(full)
    return out


def one_domain(domain):
    """The file, its links, and each link's answer. Sequential within a domain."""
    base = f"https://{domain}/llms.txt"
    status, text = body(base)
    record = {"domain": domain, "status": status, "valid": False,
              "links_found": 0, "links_checked": 0, "dead": 0, "rows": []}
    if status != 200 or not still_valid(text):
        return record
    record["valid"] = True
    found = links_of(text, base)
    record["links_found"] = len(found)
    for url in found[:CAP]:
        code, final, error = fetch(url)
        dead = error is not None or (code is not None and code >= 400)
        record["rows"].append({"domain": domain, "url": url, "status": code,
                               "final": final, "error": error, "dead": dead})
        if dead:
            record["dead"] += 1
        time.sleep(0.25)                    # one domain at a time, unhurried
    record["links_checked"] = len(record["rows"])
    return record


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--out", default=str(Path.home() / "data" / "llms-txt-links"))
    ap.add_argument("--domains", default="research/llms-txt/domains.csv")
    ap.add_argument("--workers", type=int, default=12)
    args = ap.parse_args()

    with open(args.domains) as handle:
        domains = [row["domain"] for row in csv.DictReader(handle)
                   if row["llms_txt"] == "yes"]
    print(f"{len(domains)} domains served a valid llms.txt on 2026-09-06")

    out = Path(args.out).expanduser()
    out.mkdir(parents=True, exist_ok=True)
    started = time.time()
    with ThreadPoolExecutor(max_workers=args.workers) as pool:
        records = list(pool.map(one_domain, domains))

    with open(out / "files.jsonl", "w") as handle:
        for record in records:
            handle.write(json.dumps({k: v for k, v in record.items() if k != "rows"}) + "\n")
    with open(out / "links.jsonl", "w") as handle:
        for record in records:
            for row in record["rows"]:
                handle.write(json.dumps(row) + "\n")

    valid = [r for r in records if r["valid"]]
    checked = sum(r["links_checked"] for r in valid)
    dead = sum(r["dead"] for r in valid)
    print(f"still valid today      {len(valid)} of {len(domains)}")
    print(f"links checked         {checked}")
    print(f"dead                  {dead}")
    print(f"files with a dead one {sum(1 for r in valid if r['dead'])}")
    print(f"seconds               {int(time.time() - started)}")
    print(f"rows in {out}")


if __name__ == "__main__":
    sys.exit(main())