"""Read the probe's rows and apply the README's verdict rules. Nothing new is decided here."""

import argparse
import collections
import json
from pathlib import Path


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--out", default=str(Path.home() / "data" / "llms-txt-links"))
    args = ap.parse_args()
    out = Path(args.out).expanduser()

    files = [json.loads(l) for l in open(out / "files.jsonl") if l.strip()]
    links = [json.loads(l) for l in open(out / "links.jsonl") if l.strip()]
    valid = [f for f in files if f["valid"]]
    gone = [f for f in files if not f["valid"]]

    checked = len(links)
    dead = [r for r in links if r["dead"]]
    with_dead = [f for f in valid if f["dead"]]

    # The README's guard: a domain that answers 4xx to everything is a block
    # wearing rot's clothes. Named, and counted apart.
    all_dead = [f for f in valid if f["links_checked"] and f["dead"] == f["links_checked"]]
    all_dead_names = sorted(f["domain"] for f in all_dead)
    dead_in_all_dead = sum(f["dead"] for f in all_dead)

    print(f"domains that served a valid llms.txt on 2026-09-06   {len(files)}")
    print(f"still valid on 2026-09-09                            {len(valid)}")
    if gone:
        print("  gone: " + ", ".join(
            f"{f['domain']} ({f['status']})" for f in sorted(gone, key=lambda f: f["domain"])))
    print()
    print(f"links checked (first 15 of each file)                {checked}")
    print(f"dead                                                 {len(dead)}"
          f"   {100 * len(dead) / checked:.1f}%")
    print(f"files carrying at least one dead link                {len(with_dead)} of "
          f"{len(valid)}   {100 * len(with_dead) / len(valid):.1f}%")
    print()
    print(f"files where EVERY checked link is dead               {len(all_dead)}"
          f"  ({dead_in_all_dead} of the {len(dead)} dead links)")
    for name in all_dead_names:
        print(f"  {name}")
    print()
    rest_checked = checked - sum(f["links_checked"] for f in all_dead)
    rest_dead = len(dead) - dead_in_all_dead
    print(f"excluding those domains entirely: {rest_dead} dead of {rest_checked}"
          f"   {100 * rest_dead / rest_checked:.1f}%")
    print()
    codes = collections.Counter(
        (r["status"] if r["status"] is not None else (r["error"] or "").split(":")[0])
        for r in dead)
    print("how the dead links died:")
    for code, n in codes.most_common(12):
        print(f"  {str(code):<24} {n}")
    print()
    counts = sorted(f["links_found"] for f in valid)
    median = counts[len(counts) // 2]
    print(f"links in a file: median {median}, max {counts[-1]}, "
          f"{sum(1 for c in counts if c > 15)} files longer than the cap of 15")
    print(f"links beyond the cap, unmeasured: "
          f"{sum(max(0, f['links_found'] - f['links_checked']) for f in valid)}")
    print()
    worst = sorted(valid, key=lambda f: (-f["dead"], f["domain"]))[:8]
    print("worst files (dead of checked):")
    for f in worst:
        if f["dead"]:
            print(f"  {f['domain']:<28} {f['dead']} of {f['links_checked']}")
    print()
    print("every dead link, by domain:")
    by_domain = collections.defaultdict(list)
    for r in dead:
        by_domain[r["domain"]].append(r)
    for domain in sorted(by_domain):
        for r in by_domain[domain]:
            reason = r["status"] if r["status"] is not None else r["error"]
            print(f"  {domain:<24} {str(reason):<28} {r['url'][:96]}")


if __name__ == "__main__":
    main()