"""Score the sealed rule. Controls first; no verdict if a control fails.

Every threshold, quantity and verdict word here is read from
`rule.py`, sealed as rule.markdown-vs-html
(f89e82653c48f0dc18c6cf25293fdfb34d988be01070d19e1c8a99fd533b62df) before the
first page was fetched. Nothing is written by hand: `results.md` and `pages.csv`
are generated.
"""
import csv
import html as htmllib
import random
import re
import statistics
import sys
from pathlib import Path

import html2text
import tiktoken
import trafilatura
from bs4 import BeautifulSoup, Comment

HERE = Path(__file__).resolve().parent
DATA = Path.home() / "data" / "markdown-vs-html"
BODIES = DATA / "bodies"
SEED = 20260921
DRAWS = 2000

ENC = tiktoken.get_encoding("o200k_base")
ENC2 = tiktoken.get_encoding("cl100k_base")

# ---- the sealed bars -------------------------------------------------------
BAR_SAVING = 0.65      # C1, C3: median reduction at or above this
BAR_RETAIN = 0.95      # C2, C3: median retention at or above this
BAR_MARKUP = 10.0      # C4: median T_html / T_visible at or above this
MIN_PAGES, MIN_DOMAINS = 60, 25
MIN_BYTES, MIN_VISIBLE_TOKENS = 2048, 100


def ntok(s, enc=ENC):
    return len(enc.encode(s, disallowed_special=())) if s else 0


def to_naive(body):
    h = html2text.HTML2Text()
    h.body_width = 0
    return h.handle(body)


def to_extract(body):
    try:
        out = trafilatura.extract(body, output_format="markdown",
                                  include_links=True, include_tables=True)
    except Exception:
        out = None
    return out or ""


_WS = re.compile(r"\s+")


def visible_text(body):
    soup = BeautifulSoup(body, "html.parser")
    for tag in soup(["script", "style", "noscript", "template", "svg"]):
        tag.decompose()
    for c in soup.find_all(string=lambda t: isinstance(t, Comment)):
        c.extract()
    return _WS.sub(" ", soup.get_text(" ")).strip()


def measure(body, enc=ENC):
    """The four token counts of the sealed rule, for one document."""
    vis = visible_text(body)
    return {
        "T_html": ntok(body, enc),
        "T_naive": ntok(to_naive(body), enc),
        "T_extract": ntok(to_extract(body), enc),
        "T_visible": ntok(vis, enc),
    }


def derive(t):
    out = {}
    out["R_naive"] = 1 - t["T_naive"] / t["T_html"] if t["T_html"] else None
    out["R_extract"] = 1 - t["T_extract"] / t["T_html"] if t["T_html"] else None
    out["K_naive"] = t["T_naive"] / t["T_visible"] if t["T_visible"] else None
    out["K_extract"] = t["T_extract"] / t["T_visible"] if t["T_visible"] else None
    out["M"] = t["T_html"] / t["T_visible"] if t["T_visible"] else None
    return out


# ---- bootstrap -------------------------------------------------------------

def boot_median(values, draws=DRAWS, seed=SEED):
    vals = [v for v in values if v is not None]
    if len(vals) < 3:
        return (None, None, None)
    rng = random.Random(seed)
    n = len(vals)
    meds = []
    for _ in range(draws):
        meds.append(statistics.median(rng.choices(vals, k=n)))
    meds.sort()
    return (statistics.median(vals), meds[int(0.025 * draws)], meds[int(0.975 * draws) - 1])


def verdict_at_least(ci, bar):
    """The sealed shape: survives if the whole CI is at or above the bar."""
    point, lo, hi = ci
    if point is None:
        return "VOID (too few observations)"
    if lo >= bar:
        return "SURVIVES"
    if hi < bar:
        return "KILLED"
    return "INCONCLUSIVE"


# ---- controls --------------------------------------------------------------

PARA = ("The river at Landfall runs south from the hills and the settlers who "
        "arrived in spring built their huts along the eastern bank, where the "
        "soil holds water through the dry months and the grain comes up early. ")


def synthetic(n_paras=40, target_markup_ratio=3.0):
    """A page whose T_html / T_visible is CONSTRUCTED to a known value.

    Content is n_paras paragraphs. Markup is added — nav links, inline style,
    a script block — until T_html reaches target * T_visible. The construction
    measures its own truth with the same tokenizer, so the control tests the
    pipeline (the parser, the converter, the ratio), not the arithmetic of the
    generator.
    """
    content = "".join("<p>%s</p>\n" % (PARA * 2) for _ in range(n_paras))
    body_open = "<!doctype html><html><head><title>Control</title>"
    body_mid = "</head><body><main>\n" + content + "</main>\n"
    tail = "</body></html>"
    vis_tokens = ntok(visible_text(body_open + body_mid + tail))
    want = target_markup_ratio * vis_tokens
    filler = []
    doc = lambda: body_open + "".join(filler) + body_mid + tail
    i = 0
    while ntok(doc()) < want and i < 20000:
        # markup only: a style rule and a script line contribute no visible text
        filler.append('<style>.c%d{margin:%dpx;padding:%dpx;color:#3a3a4f}</style>'
                      % (i, i % 40, i % 17))
        filler.append('<script>window.__t%d=function(a,b){return a+b*%d;};</script>' % (i, i))
        i += 1
    return doc(), vis_tokens


def null_document(n_paras=40):
    """Already plain text: nothing for a markdown conversion to save."""
    text = "\n\n".join(PARA * 2 for _ in range(n_paras))
    return "<html><body><pre>" + htmllib.escape(text) + "</pre></body></html>"


def run_controls(log):
    ok = True

    doc, vis_tokens = synthetic(target_markup_ratio=3.0)
    t = measure(doc)
    d = derive(t)
    truth = 3.0
    err = abs(d["M"] - truth) / truth
    passed = err <= 0.05
    ok &= passed
    log.append(("POSITIVE", "constructed M = %.2f, measured M = %.3f, error %.2f %%"
                % (truth, d["M"], 100 * err), "PASS" if passed else "FAIL"))

    nd = null_document()
    tn = measure(nd)
    dn = derive(tn)
    passed = dn["R_naive"] < 0.10
    ok &= passed
    log.append(("NULL", "already-plain-text document: reported saving R_naive = %.4f "
                "(bar: below 0.10)" % dn["R_naive"], "PASS" if passed else "FAIL"))
    return ok


# ---- main ------------------------------------------------------------------

def main():
    log = []
    print("controls first", flush=True)
    controls_ok = run_controls(log)
    for name, detail, res in log:
        print("  %-9s %-4s %s" % (name, res, detail), flush=True)

    rows = list(csv.DictReader(open(DATA / "index.csv")))
    pages, excluded = [], []
    for r in rows:
        why = None
        if r["status"] != "200":
            why = "status_%s" % r["status"]
        elif "html" not in r["content_type"].lower():
            why = "not_html"
        elif int(r["bytes"]) < MIN_BYTES:
            why = "too_small"
        elif r["truncated"] == "1":
            why = "truncated"
        if why:
            excluded.append((r, why))
            continue
        body = (BODIES / r["file"]).read_bytes().decode("utf-8", "replace")
        t = measure(body)
        if t["T_visible"] < MIN_VISIBLE_TOKENS:
            excluded.append((r, "no_visible_text"))
            continue
        t2 = measure(body, ENC2)
        d, d2 = derive(t), derive(t2)
        pages.append({"url": r["url"], "domain": r["domain"], **t,
                      **{k: d[k] for k in d},
                      **{k + "_cl100k": d2[k] for k in d2}})
        if len(pages) % 25 == 0:
            print("  measured %d" % len(pages), flush=True)

    domains = sorted({p["domain"] for p in pages})
    instrument_ok = len(pages) >= MIN_PAGES and len(domains) >= MIN_DOMAINS
    log.append(("INSTRUMENT", "%d pages from %d domains (bars: %d and %d)"
                % (len(pages), len(domains), MIN_PAGES, MIN_DOMAINS),
                "PASS" if instrument_ok else "FAIL"))
    print("  %-9s %-4s %s" % ("INSTRUMENT", "PASS" if instrument_ok else "FAIL", log[-1][1]))

    void = not (controls_ok and instrument_ok)

    # per-page and per-domain (median within a domain first: the site is the unit)
    def per_domain(key):
        out = []
        for dom in domains:
            vals = [p[key] for p in pages if p["domain"] == dom and p[key] is not None]
            if vals:
                out.append(statistics.median(vals))
        return out

    claims = [
        ("C1", "PRIMARY", "R_naive", BAR_SAVING,
         'Converting a web page to markdown cuts tokens by 65-90 % (naive conversion)'),
        ("C2", "PRIMARY", "K_naive", BAR_RETAIN,
         '"with no content loss" - naive conversion keeps the visible text'),
        ("C3a", "SECONDARY", "R_extract", BAR_SAVING,
         "the same saving for an extractor (trafilatura 2.2.0)"),
        ("C3b", "SECONDARY", "K_extract", BAR_RETAIN,
         "the same retention for an extractor (trafilatura 2.2.0)"),
        ("C4", "PRIMARY", "M", BAR_MARKUP,
         "raw markup is 10x the size of the content by token count"),
    ]

    lines = []
    W = lines.append
    W("# How many tokens does markdown actually save?")
    W("")
    W("Generated by `analyse.py`. Rule sealed as `rule.markdown-vs-html`, "
      "`f89e82653c48f0dc18c6cf25293fdfb34d988be01070d19e1c8a99fd533b62df`, "
      "before the first page was fetched.")
    W("")
    W("## Controls")
    W("")
    W("| control | what it checks | result |")
    W("| --- | --- | --- |")
    for name, detail, res in log:
        W("| %s | %s | **%s** |" % (name, detail, res))
    W("")
    if void:
        W("**One or more controls failed. The study is VOID and no verdict is published.**")
        (HERE / "results.md").write_text("\n".join(lines) + "\n")
        print("\nVOID")
        return 1

    W("## The corpus")
    W("")
    W("%d article pages from %d domains, drawn from feeds as the rule describes, "
      "fetched %s." % (len(pages), len(domains), "2026-09-21"))
    W("")
    reasons = {}
    for _, why in excluded:
        reasons[why] = reasons.get(why, 0) + 1
    W("Excluded, by reason, out of %d URLs drawn: %s."
      % (len(rows), ", ".join("%s %d" % (k, v) for k, v in sorted(reasons.items()))))
    W("")

    W("## Verdicts")
    W("")
    W("Median over pages, 95 %% interval from a %d-draw page-level bootstrap at "
      "seed %d. The per-domain column medians within each site first, because the "
      "site is the sampling unit; where the two disagree the sealed rule says the "
      "per-domain one governs." % (DRAWS, SEED))
    W("")
    W("| claim | quantity | bar | median (per page) | 95 % CI | per domain | verdict |")
    W("| --- | --- | --- | --- | --- | --- | --- |")
    verdicts = {}
    for cid, kind, key, bar, text in claims:
        ci = boot_median([p[key] for p in pages])
        dci = boot_median(per_domain(key))
        v_page = verdict_at_least(ci, bar)
        v_dom = verdict_at_least(dci, bar)
        v = v_dom            # the sealed tiebreak
        verdicts[cid] = (v, ci, dci, v_page, v_dom)
        W("| **%s** (%s) | `%s` | >= %.2f | %.4f | %.4f - %.4f | %.4f (%.4f - %.4f) | **%s** |"
          % (cid, kind.lower(), key, bar, ci[0], ci[1], ci[2], dci[0], dci[1], dci[2], v))
    W("")
    for cid, kind, key, bar, text in claims:
        v, ci, dci, v_page, v_dom = verdicts[cid]
        note = "" if v_page == v_dom else (
            "  **The two units disagree** (per page %s, per domain %s); the sealed "
            "rule gives the verdict to the per-domain reading." % (v_page, v_dom))
        W("- **%s — %s.** %s%s" % (cid, v, text, note))
    W("")

    W("## Robustness: the second tokenizer")
    W("")
    W("| quantity | o200k_base | cl100k_base | move |")
    W("| --- | --- | --- | --- |")
    downgrade = []
    for cid, kind, key, bar, text in claims:
        a = statistics.median([p[key] for p in pages if p[key] is not None])
        b = statistics.median([p[key + "_cl100k"] for p in pages
                               if p[key + "_cl100k"] is not None])
        if key == "M":
            moved = abs(b - a) / a > 0.10
            move = "%.1f %% of value" % (100 * abs(b - a) / a)
        else:
            moved = abs(b - a) > 0.05
            move = "%.1f points" % (100 * abs(b - a))
        if moved:
            downgrade.append(cid)
        W("| %s (%s) | %.4f | %.4f | %s%s |"
          % (key, cid, a, b, move, " — **over the sealed bar**" if moved else ""))
    W("")
    if downgrade:
        W("**The sealed downgrade clause fires for %s**: the median moves more than "
          "the rule allows between tokenizers, so those verdicts read INCONCLUSIVE "
          "— the answer would be a fact about a tokenizer, not about the web."
          % ", ".join(downgrade))
    else:
        W("No primary median moves more than the sealed bar between the two "
          "tokenizers, so no verdict is downgraded.")
    W("")

    W("## What a page looks like")
    W("")
    W("Median page in the corpus, by token count:")
    W("")
    med = lambda k: statistics.median([p[k] for p in pages])
    for k, label in [("T_html", "the HTML as fetched"),
                     ("T_naive", "html2text over the whole document"),
                     ("T_extract", "trafilatura markdown extraction"),
                     ("T_visible", "the visible text (nav and footer included)")]:
        W("- `%s` **%s tokens** — %s" % (k, format(int(med(k)), ","), label))
    W("")
    share10 = sum(1 for p in pages if p["M"] is not None and p["M"] >= 10) / len(pages)
    W("Share of pages where the markup really is 10x the content or more: "
      "**%.1f %%** (%d of %d). The claim's own word is \"can\"; on the existence "
      "reading this is the number, and the sealed rule reads it as the typical "
      "case instead." % (100 * share10, int(round(share10 * len(pages))), len(pages)))
    W("")
    novis = reasons.get("no_visible_text", 0)
    W("Pages excluded because the body carried under %d tokens of visible text — "
      "client-rendered pages, mostly: **%d**." % (MIN_VISIBLE_TOKENS, novis))
    W("")

    W("## Worst and best")
    W("")
    by_m = sorted([p for p in pages if p["M"]], key=lambda p: -p["M"])
    W("| page | T_html | T_visible | markup ratio | naive saving | extract saving | extract retention |")
    W("| --- | --- | --- | --- | --- | --- | --- |")
    for p in by_m[:5] + by_m[-5:]:
        W("| `%s` | %s | %s | %.1fx | %.1f %% | %.1f %% | %.1f %% |"
          % (p["domain"], format(p["T_html"], ","), format(p["T_visible"], ","),
             p["M"], 100 * p["R_naive"], 100 * p["R_extract"], 100 * p["K_extract"]))
    W("")
    W("This is a measurement of token counts on one corpus with one tokenizer, "
      "not advice about a vendor.")

    (HERE / "results.md").write_text("\n".join(lines) + "\n")

    cols = ["url", "domain", "T_html", "T_naive", "T_extract", "T_visible",
            "R_naive", "R_extract", "K_naive", "K_extract", "M"]
    with open(HERE / "pages.csv", "w", newline="") as fh:
        w = csv.DictWriter(fh, fieldnames=cols, extrasaction="ignore")
        w.writeheader()
        w.writerows(pages)

    print("\n".join(lines[lines.index("## Verdicts"):]) if "## Verdicts" in lines else "")
    return 0


if __name__ == "__main__":
    sys.exit(main())