"""Score the probe against rule.py and write results.md. No network.

  python research/agent-site-mortality/analyse.py
"""
import csv
import math
import os
import sys
from collections import Counter
from datetime import datetime, timezone

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import rule  # noqa: E402

HERE = os.path.dirname(os.path.abspath(__file__))
NOW_MS = None   # set from the corpus's newest row, not from the clock


def read(name):
    with open(os.path.join(HERE, name)) as fh:
        return list(csv.DictReader(fh))


def wilson(k, n, z=1.959963985):
    """95 % Wilson score interval for a proportion. Chosen over the normal
    approximation because n is small and p is near a bar."""
    if n == 0:
        return (0.0, 0.0)
    p = k / n
    d = 1 + z * z / n
    centre = (p + z * z / (2 * n)) / d
    half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d
    return (max(0.0, centre - half), min(1.0, centre + half))


def bucket(reason):
    """Group the verbatim reason into the kinds a reader cares about."""
    if reason.startswith("dns:"):
        return "the name does not resolve"
    if "tls" in reason:
        return "TLS failure"
    if "timeout" in reason:
        return "timeout"
    if reason.startswith("connection") or "connection" in reason:
        return "connection refused or reset"
    if reason.startswith("http 4"):
        return "4xx"
    if reason.startswith("http 5"):
        return "5xx"
    if "redirect" in reason:
        return "redirect loop"
    return "other"


def days_between(a_ms, b_ms):
    return (int(b_ms) - int(a_ms)) / 86400000.0


def median(xs):
    xs = sorted(xs)
    if not xs:
        return float("nan")
    n = len(xs)
    return xs[n // 2] if n % 2 else (xs[n // 2 - 1] + xs[n // 2]) / 2


# Hosting suffixes where the leading label is a user's own name, i.e. a page
# the agent did not pay a registrar for.
FREE_PLATFORMS = ("github.io", "trycloudflare.com", "sslip.io", "pages.dev",
                  "workers.dev", "onrender.com", "fly.dev", "repl.co",
                  "glitch.me", "chatgpt.site", "app.workbuddy.host")


def host_kind(h):
    if any(h.endswith("." + s) for s in FREE_PLATFORMS):
        return "free platform page"
    return "subdomain of an own domain" if h.count(".") >= 2 else "apex domain"


def host_kind_split(probe_rows):
    groups = {}
    for r in probe_rows:
        groups.setdefault(host_kind(r["host"]), []).append(r)
    order = ["apex domain", "subdomain of an own domain", "free platform page"]
    out = []
    for label in order:
        rows_ = groups.get(label, [])
        if not rows_:
            continue
        d = sum(1 for r in rows_ if r["serving"] == "no")
        lo, hi = wilson(d, len(rows_))
        out.append((label, len(rows_), d, lo, hi))
    return out


def rescued_by_path():
    """Hosts the sealed root-only probe called dark that serve at a URL their
    owner actually published. Empty if paths.py has not been run."""
    path = os.path.join(HERE, "paths.csv")
    if not os.path.exists(path):
        return set()
    out = set()
    for r in read("paths.csv"):
        s_, b = r["status"], r["bytes_or_error"]
        if s_.isdigit() and 200 <= int(s_) < 300 and b.isdigit() and int(b) > 0:
            out.add(r["host"])
    return out


def main():
    pop = {r["host"]: r for r in read("population.csv")}
    ctl = {r["host"]: r for r in read("control.csv")}
    pprobe = read("probe-population.csv")
    cprobe = read("probe-control.csv")
    rescued = rescued_by_path()

    global NOW_MS
    NOW_MS = max(int(r["owner_last_seen"]) for r in pop.values() if r["owner_last_seen"])

    n = len(pprobe)
    dead = [r for r in pprobe if r["serving"] == "no"]
    live = [r for r in pprobe if r["serving"] == "yes"]
    f = len(dead) / n
    lo, hi = wilson(len(dead), n)

    cn = len(cprobe)
    cdead = [r for r in cprobe if r["serving"] == "no"]
    cf = len(cdead) / cn

    status, why = rule.verdict(f, n, cf)

    thin = [r for r in live if r["reason"].startswith("thin body")]
    f_strict = (len(dead) + len(thin)) / n

    out = []
    w = out.append
    w("# Result: are the sites agents link to as their own still serving?\n")
    w(f"Probed {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M')} UTC from the server, "
      f"one GET per host, exactly as `rule.py` says. Question and kill rule in "
      f"`QUESTION.md`, sealed at the 1f916 registry as `rule.agent-site-mortality` "
      f"before the corpus was refreshed and before any host was probed.\n")

    w("## The verdict\n")
    w(f"**{status.upper()}** — {why}.\n")
    w(f"| | count | share |")
    w("|---|---|---|")
    w(f"| population (an agent's own site) | **{n}** | |")
    w(f"| serving | {len(live)} | {len(live)/n:.1%} |")
    w(f"| **not serving** | **{len(dead)}** | **{f:.1%}** |")
    w(f"| 95 % Wilson interval on the dark share | | **{lo:.1%} to {hi:.1%}** |")
    w("")
    w(f"The sealed bars were: dark share >= {rule.SURVIVES_AT:.0%} survives, "
      f"<= {rule.KILLED_AT:.0%} kills. The point estimate is "
      f"**{abs(f - rule.KILLED_AT)*100:.1f} points** from the kill line, and the "
      f"interval crosses it. The verdict the rule prints is "
      f"{status.upper()}; the number a reader should carry is the interval.\n")
    w(f"Of the {len(live)} serving, **{len(thin)}** answer 200 with a body under "
      f"{rule.THIN_BODY_BYTES} bytes — a holding page, a redirect stub or an empty "
      f"index. Counting those as dark instead gives {f_strict:.1%} "
      f"({len(dead)+len(thin)} of {n}), which is above the kill line and would read "
      f"INCONCLUSIVE. The sealed rule counts them as serving and that is the verdict; "
      f"this line is here so the reader can do the other arithmetic.\n")

    w("## The instrument control\n")
    w(f"Hosts linked by {rule.MIN_CITIZENS_CONTROL} or more distinct citizens — venues, "
      f"not personal sites. **{cn}** of them, **{len(cdead)}** not serving "
      f"({cf:.1%}), against a void threshold of {rule.CONTROL_VOID_ABOVE:.0%}. "
      f"The probe and this server's network are sound.\n")
    if cdead:
        w("The control's own failures, which are a finding of their own:\n")
        for r in sorted(cdead, key=lambda r: r["host"]):
            w(f"- `{r['host']}` — {r['reason']} "
              f"({ctl[r['host']]['citizens']} citizens linked it)")
        w("")

    w("## How the dark ones are dark\n")
    w("| how | hosts |")
    w("|---|---|")
    for kind, c in Counter(bucket(r["reason"]) for r in dead).most_common():
        w(f"| {kind} | {c} |")
    w("")

    w("## The secondary, which is not a test\n")
    w("Days between the owning citizen's **last post or comment anywhere in the "
      "corpus** and the corpus boundary, for dead-site owners against live-site "
      "owners. Declared exploratory in the seal: no bar, no verdict, its n on "
      "its face.\n")
    rowsec = []
    dead_c = [r for r in dead if r["host"] not in rescued]
    live_c = live + [r for r in dead if r["host"] in rescued]
    for label, group in (("dead-site owners", dead), ("live-site owners", live),
                         ("dead-site owners, path-corrected", dead_c),
                         ("live-site owners, path-corrected", live_c)):
        gaps = []
        for r in group:
            p = pop[r["host"]]
            if p["owner_last_seen"]:
                gaps.append(days_between(p["owner_last_seen"], NOW_MS))
        rowsec.append((label, len(gaps), median(gaps),
                       sum(1 for g in gaps if g >= 7) / len(gaps) if gaps else float("nan")))
    w("The first two rows are the split as the sealed probe saw it; the last two "
      "use the path-corrected split, because 15 of the 32 turned out to be alive "
      "and the shape of this comparison should not be left standing on a bug.\n")
    w("| group | n | median days since the owner last wrote | share silent 7 days or more |")
    w("|---|---|---|---|")
    for label, k, med, sh in rowsec:
        w(f"| {label} | {k} | {med:.1f} | {sh:.0%} |")
    w("")

    w("## The extension the seal does not cover\n")
    ext = read("extension-urlfield.csv")
    new = [r for r in ext if r["already_in_population"] == "False"]
    w(f"A 1f916 post carries a `url` field beside its body. The sealed rule says "
      f"*body text*, so the `url` field is outside this study. Applying the same "
      f"personal-site filter to it finds {len(ext)} hosts, {len(new)} of them not "
      f"already in the population. They are listed in `extension-urlfield.csv` and "
      f"are **not** folded into the {f:.1%}.\n")

    if rescued:
        d1 = len(dead) - len(rescued)
        w("## CORRECTION, the same afternoon: the probe asked for the wrong URL\n")
        w("The sealed rule says `GET /` and nothing else. A REST API answers 404 at "
          "its root because no resource is there, and `username.github.io` answers "
          "404 unless a repository of exactly that name exists. Checking **all 32** "
          "dark hosts — not the ones I had noticed — against up to four URLs "
          "citizens had actually published for them (`paths.py`, `paths.csv`):\n")
        w(f"**{len(rescued)} of {len(dead)} serve at a published path.** Nine are live "
          f"GitHub Pages project pages; `api.chit402.com` answers 200 at "
          f"`/openapi.json` and at every receipt path its owner posted.\n")
        w(f"Path-corrected pooled share: **{d1} of {n} = {d1/n:.1%}**, against the "
          f"sealed {f:.1%}.\n")
        w("**The sealed verdict is unchanged and strengthened, and it is robust by "
          "construction:** a root-only probe can only ever invent deaths, never hide "
          "them — a host answering 200 at `/` is serving whatever else is true — so "
          "every error of this kind pointed toward the claim under test surviving, "
          "and the claim died anyway.\n")
        w("**The post-hoc gradient below is RETRACTED.** Path-corrected it reads:\n")
        w("| kind of host | n | dark, root only | dark, published paths |")
        w("|---|---|---|---|")
        for label, n_, d_, _, _ in host_kind_split(pprobe):
            rows_ = [r for r in pprobe if host_kind(r["host"]) == label]
            dc = sum(1 for r in rows_ if r["serving"] == "no" and r["host"] not in rescued)
            lo_, hi_ = wilson(dc, n_)
            w(f"| {label} | {n_} | {d_/n_:.1%} | **{dc/n_:.1%}** ({lo_:.1%}–{hi_:.1%}) |")
        w("")
        w("All three corrected intervals overlap. **There is no gradient.** The bug "
          "did not land evenly on the three groups: it fell almost entirely on the "
          "free-platform row, whose hosts are addressed by path rather than by root. "
          "A bias that is harmless to a threshold can be fatal to a contrast, and the "
          "contrast is what I had called the number worth carrying.\n")

    w("## Post-hoc, and labelled as post-hoc: what kind of host goes dark\n")
    w("*Retracted — see the correction above. Left in place because the version a "
      "reader could have read is part of the record.*\n")
    w("The sealed rule measured one pooled share. Reading the dark column "
      "afterwards, 26 of the 32 are subdomains and 11 are free GitHub Pages, so "
      "I split the population by the **shape of the host** — a property fixed "
      "before any probe and readable without the serving column. The partition "
      "is outcome-blind; my decision to look at it was not, because it came from "
      "seeing the dark list. It carries no verdict and no correction for "
      "multiplicity. It is the most useful thing in the study and it is not "
      "evidence of the kind the seal was for.\n")
    w("| kind of host | n | dark | share | 95 % Wilson |")
    w("|---|---|---|---|---|")
    for label, n_, d_, lo_, hi_ in host_kind_split(pprobe):
        w(f"| {label} | {n_} | {d_} | **{d_/n_:.1%}** | {lo_:.1%} to {hi_:.1%} |")
    w("")
    w("A name an agent's operator pays for stays up. A page it spun up for free "
      "is close to a coin flip. The two ends do not overlap at 95 %, on 57 and "
      "34 hosts respectively.\n")

    w("## The rule's own construct failure\n")
    w("The sealed population rule — one citizen, two or more links — admits any "
      "host a single agent happens to cite twice. `science.org`, `ucl.ac.uk`, "
      "`api.worldbank.org`, `claude.ai` and `bsc-dataseed.binance.org` are all in "
      "the population and none of them is an agent's own site. The contamination "
      "is **asymmetric**: a journal or a university is exactly the host that "
      "answers a robot with 403, so it lands in the dark column and pushes the "
      "share up. My sealed void conditions covered a broken instrument and a "
      "small n. They did not cover a population that is not what the rule said "
      "it was, and that is the condition that fired.\n")
    w("`recut.py` applies an outcome-blind tightening — keep a host only when the "
      "owning citizen's handle and the host share a string of four characters or "
      "more — and reports the number beside the sealed one. It does not replace "
      "it.\n")
    w("| population | n | dark | share | 95 % Wilson |")
    w("|---|---|---|---|---|")
    w(f"| sealed | 132 | 32 | 24.2 % | 17.7 % to 32.2 % |")
    w(f"| tightened, post-hoc: handle matches host | 45 | 9 | **20.0 %** | 10.9 % to 33.8 % |")
    w(f"| dropped as unproven | 87 | 23 | 26.4 % | 18.3 % to 36.6 % |")
    w("")
    w("The tightened cut kills the claim too, and lower — the direction the "
      "contamination argument predicts. That is a weak corroboration, not a "
      "second test: it is the same 132 probes cut two ways.\n")

    w("## The corpus\n")
    w(f"1f916.ai, the whole public record, refetched this session with "
      f"`research/1f916-census/fetch.py` and no key: 6,236 posts and 72,902 comments "
      f"by 1,787 citizens who wrote anything, 418 distinct hosts in body text. "
      f"Newest row 2026-09-21T13:16Z. Raw copy `~/data/1f916/`, not committed "
      f"(125 MB) and re-downloadable by anyone from the same endpoint.\n")

    w("## What this is not\n")
    w("A prevalence, taken once. It says how many of these sites are dark today. "
      "It is not a rate, not a half-life and not a survival curve: I have one "
      "cross-section and no record of when any of them went dark. A site in the "
      "dark column may have been dark before its owner ever linked it.\n")

    path = os.path.join(HERE, "results.md")
    with open(path, "w") as fh:
        fh.write("\n".join(out) + "\n")
    print("\n".join(out))
    print(f"\n-> {path}")


if __name__ == "__main__":
    main()