#!/usr/bin/env python3
"""Run the sealed detector over a corpus and print the verdict.

    python analyse.py <dir-of-robots-txt> [--label fresh] [--json out.json]

Refuses to run unless `controls.py` passes first, because the pre-registration
says both controls must pass before the corpus is touched and a rule nobody
enforces is exactly what this study is about.
"""
import json
import math
import os
import subprocess
import sys

from dead import score

HERE = os.path.dirname(os.path.abspath(__file__))
K1 = 0.05      # >= 5 % of all rules dead  -> claim killed
K2 = 0.20      # >= 20 % of sites with >=1 -> claim killed


def controls_pass():
    run = subprocess.run([sys.executable, os.path.join(HERE, "controls.py")],
                         capture_output=True, text=True, cwd=HERE)
    return run.returncode == 0, run.stdout


def floor_fraction(rules_total, sites_total):
    """The smallest dead fraction distinguishable from zero on this corpus.

    One dead rule is the smallest non-zero observation there is, so the floor
    for K1 is 1/rules_total and for K2 is 1/sites_total. If an observed value
    sits at or below its floor the result is VOID: UNFAILABLE rather than
    "survived" — the corpus could not have shown less.
    """
    return (1.0 / rules_total if rules_total else float("inf"),
            1.0 / sites_total if sites_total else float("inf"))


def wilson(k, n, z=1.96):
    """A 95 % interval for a proportion; 0/n must not come back as a point."""
    if not n:
        return (0.0, 0.0)
    p = k / n
    d = 1 + z * z / n
    centre = (p + z * z / (2 * n)) / d
    half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d
    return (max(0.0, centre - half), min(1.0, centre + half))


def looks_html(text):
    head = text[:400].lower()
    return ("<html" in head or "<!doctype html" in head
            or head.lstrip().startswith("<?xml"))


def load(path):
    """-> (body, ok). Understands both corpus formats on this machine.

    The fresh corpus writes the body and nothing else, with every attempt's
    status in a separate meta.jsonl. The 2026-09-06 corpus was written by
    `research/llms-txt/probe.py`, which prepends a JSON status line to every
    file INCLUDING the failures — so `{"status": null, "err": "ConnectTimeout"}`
    is a file on disk that is not a robots.txt at all.

    Reading that header and refusing anything that is not a 200 is a change to
    how a file is loaded, not to what counts as dead: the detector in dead.py
    is untouched and was frozen and committed before either corpus was run.
    """
    with open(path, encoding="utf-8", errors="replace") as fh:
        text = fh.read()
    first, newline, rest = text.partition("\n")
    if first.startswith('{"status":'):
        try:
            status = json.loads(first).get("status")
        except ValueError:
            return text, True
        return rest, status == 200
    return text, True


def main():
    directory = os.path.expanduser(sys.argv[1])
    label = sys.argv[sys.argv.index("--label") + 1] if "--label" in sys.argv else \
        os.path.basename(directory.rstrip("/"))

    ok, out = controls_pass()
    print(out)
    if not ok:
        print("controls failed; refusing to touch the corpus")
        return 1

    sites, skipped = [], {"not_200": 0, "html": 0, "empty": 0, "no_rules": 0}
    for name in sorted(os.listdir(directory)):
        if not name.endswith(".txt"):
            continue
        text, fetched = load(os.path.join(directory, name))
        if not fetched:
            skipped["not_200"] += 1
            continue
        if looks_html(text):
            skipped["html"] += 1
            continue
        if not text.strip():
            skipped["empty"] += 1
            continue
        row = score(text)
        row["domain"] = name[:-4]
        if row["rules"] == 0:
            # A robots.txt with only Sitemap: lines is real and legal, but it
            # has no rule to be dead, so it is out of both numerator and
            # denominator rather than counted as a clean site.
            skipped["no_rules"] += 1
            continue
        sites.append(row)

    rules_total = sum(r["rules"] for r in sites)
    opaque_total = sum(r["opaque"] for r in sites)
    dead_total = sum(r["dead"] for r in sites)
    dirty_sites = sum(1 for r in sites if r["dead"])
    classes = {}
    for row in sites:
        for name, count in row["classes"].items():
            classes[name] = classes.get(name, 0) + count

    f1 = dead_total / rules_total if rules_total else 0.0
    f2 = dirty_sites / len(sites) if sites else 0.0
    floor1, floor2 = floor_fraction(rules_total, len(sites))
    k1_fired, k2_fired = f1 >= K1, f2 >= K2

    if k1_fired and k2_fired:
        verdict = "KILLED (both thresholds)"
    elif k1_fired or k2_fired:
        verdict = "INCONCLUSIVE (one threshold of two)"
    elif f1 <= floor1 or f2 <= floor2:
        verdict = "VOID: UNFAILABLE (observed at or below the corpus floor)"
    else:
        verdict = "SURVIVED"

    print(f"corpus            {label}  ({directory})")
    print(f"sites analysed    {len(sites)}")
    print(f"  skipped         {skipped}")
    print(f"rules             {rules_total}")
    print(f"  of those opaque {opaque_total} "
          f"({opaque_total / rules_total:.1%}) — in a group holding a wildcard,"
          f" where the detector mostly cannot see")
    print(f"dead rules        {dead_total}")
    for name in sorted(classes):
        print(f"    {name:24} {classes[name]}")
    lo1, hi1 = wilson(dead_total, rules_total)
    lo2, hi2 = wilson(dirty_sites, len(sites))
    print(f"\nK1 dead / all rules   {f1:.4f}  ({lo1:.4f}-{hi1:.4f})"
          f"   threshold {K1}  fired={k1_fired}   floor {floor1:.5f}")
    print(f"K2 sites with >=1     {f2:.4f}  ({lo2:.4f}-{hi2:.4f})"
          f"   threshold {K2}  fired={k2_fired}   floor {floor2:.5f}")
    print(f"\nVERDICT  {verdict}")

    worst = sorted(sites, key=lambda r: -r["dead"])[:10]
    print("\nmost dead rules (domain, dead/rules):")
    for row in worst:
        print(f"   {row['domain']:28} {row['dead']:4} / {row['rules']}")

    if "--json" in sys.argv:
        path = sys.argv[sys.argv.index("--json") + 1]
        with open(path, "w", encoding="utf-8") as fh:
            json.dump({"label": label, "sites": len(sites), "skipped": skipped,
                       "rules": rules_total, "dead": dead_total,
                       "opaque": opaque_total,
                       "classes": classes, "k1": f1, "k2": f2,
                       "k1_fired": k1_fired, "k2_fired": k2_fired,
                       "floor_k1": floor1, "floor_k2": floor2,
                       "verdict": verdict,
                       "per_site": [{k: r[k] for k in
                                     ("domain", "rules", "dead", "classes")}
                                    for r in sites]}, fh, indent=1)
        print(f"\nwrote {path}")
    return 0


if __name__ == "__main__":
    sys.exit(main())