"""POST-HOC. A tightened population, and the number it gives.

This is NOT the sealed study. The sealed rule is rule.py, the sealed verdict is
in results.md, and nothing here replaces either. This file exists because the
sealed population rule -- "linked by exactly one citizen, at least twice" --
turned out to admit hosts that are plainly not an agent's own site: science.org,
ucl.ac.uk, api.worldbank.org, claude.ai, bsc-dataseed.binance.org. A single
agent citing one journal or one API twice satisfies the rule. That is a
construct failure, and it is ASYMMETRIC: a university or a journal is exactly
the kind of host that answers a robot with 403, so the contamination lands
disproportionately in the DARK column and pushes the measured share up.

The tightened rule below is applied to hosts I have already probed, so it cannot
be a pre-registration. What it CAN be is outcome-blind: it is a string
comparison between the host and the owning citizen's handle, and it never reads
the serving column. I could not have tuned it toward a number without rewriting
it to name hosts, which it does not do.

    a host is the owner's own site if, after lowercasing and deleting every
    character that is not a-z or 0-9:
      the handle is a substring of the host, or
      the host's most specific distinguishing label is a substring of the handle

It is strict. An agent whose site carries a name unlike its handle is dropped,
and that is a known and stated cost: the tightened population is a SUBSET of
sites I can prove belong to their linker, not all of them.

  python research/agent-site-mortality/recut.py
"""
import csv
import os
import re
import sys

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import rule  # noqa: E402
from analyse import wilson  # noqa: E402

HERE = os.path.dirname(os.path.abspath(__file__))

# Hosting suffixes whose leading label is the user's own name. For these the
# distinguishing label is the FIRST one; for everything else it is the
# registrable domain's own name.
USER_LABEL_SUFFIXES = ("github.io", "trycloudflare.com", "sslip.io", "1f916.de",
                       "pages.dev", "workers.dev", "onrender.com", "fly.dev",
                       "repl.co", "glitch.me", "neocities.org", "bearblog.dev",
                       "substack.com", "wordpress.com", "blogspot.com")


def norm(s):
    return re.sub(r"[^a-z0-9]", "", s.lower())


def labels(host):
    """Every label of the host that could carry the owner's name, plus the
    registrable domain's own name. Order does not matter; any match counts."""
    parts = host.split(".")
    out = set()
    for suf in USER_LABEL_SUFFIXES:
        if host.endswith("." + suf):
            out.add(parts[0])
    if len(parts) >= 2:
        out.add(parts[-2])          # the registrable domain's name
    out.add(parts[0])               # the leading label, whatever it is
    return {p for p in out if p}


def owns(handle, host):
    h = norm(handle)
    if not h:
        return False
    hostn = norm(host)
    if len(h) >= 4 and h in hostn:
        return True
    for lab in labels(host):
        ln = norm(lab)
        if len(ln) >= 4 and (ln in h or h in ln):
            return True
    return False


def main():
    pop = {r["host"]: r for r in csv.DictReader(open(os.path.join(HERE, "population.csv")))}
    probe = list(csv.DictReader(open(os.path.join(HERE, "probe-population.csv"))))

    kept, dropped = [], []
    for r in probe:
        p = pop[r["host"]]
        (kept if owns(p["owner"], r["host"]) else dropped).append((p, r))

    def score(rows, name):
        n = len(rows)
        dead = [x for x in rows if x[1]["serving"] == "no"]
        f = len(dead) / n if n else float("nan")
        lo, hi = wilson(len(dead), n)
        print(f"{name}: n={n}  dark={len(dead)}  f={f:.1%}  95% CI {lo:.1%}-{hi:.1%}")
        return n, len(dead), f, lo, hi

    print("POST-HOC re-cut. Not the sealed verdict.\n")
    a = score(kept, "handle matches host (an agent's own site, provably)")
    b = score(dropped, "handle does not match (dropped as unproven)")
    c = score(kept + dropped, "sealed population, for comparison")

    with open(os.path.join(HERE, "recut.csv"), "w", newline="") as fh:
        w = csv.writer(fh)
        w.writerow(["host", "owner", "handle_matches", "serving", "reason", "links"])
        for p, r in sorted(kept + dropped, key=lambda x: x[0]["host"]):
            w.writerow([p["host"], p["owner"], "yes" if (p, r) in kept else "no",
                        r["serving"], r["reason"], p["links"]])

    verdict_kept = rule.verdict(a[2], a[0], 0.129)
    print(f"\nthe sealed kill rule, applied to the tightened population: "
          f"{verdict_kept[0].upper()} — {verdict_kept[1]}")
    print("This is reported BESIDE the sealed verdict and does not replace it.")
    print(f"\n-> {os.path.join(HERE, 'recut.csv')}")


if __name__ == "__main__":
    main()