"""The sealed rule for "AI has made the em dash more common".

Sealed before any comment text reached this server. `flags` and `analyse` are the
whole rule; `fetch.py` builds the input from the Hacker News item API and `run.py`
calls them.

The claim, in two opposite forms that are both repeated: since ChatGPT (30 Nov 2022)
the em dash has spread through online writing; or, people have given it up for fear
of looking like a machine. Hacker News comments are one long public record of ordinary
typed text from before and after. This tests one forum and whether the mark moved. It
cannot say who or what typed it, and a rise alone is no evidence of machine authorship.

Sample: for each calendar month 2017-01 to 2026-10, 2,000 item ids drawn uniformly from
that month's id range with random.Random(f"{month}-20261103"), month as "YYYY-MM". A usable comment
has type "comment", non-empty text, and is not deleted. `dead` comments are kept apart:
the primary measure is on comments not dead.

Text cleaning, in order: drop <pre>...</pre>; split on <p>; drop any paragraph whose
unescaped text starts with ">" (a quotation); strip tags and unescape; drop URLs. A
comment whose cleaned text is empty is not usable.

Primary measure: for a month, the share of usable live comments holding U+2014.

Estimate. T = months 2020-12..2022-11 (the last two years before ChatGPT), P = months
2024-11..2026-10 (the newest two). R = mean(monthly share over P) / mean(monthly share over T).
Interval: percentile bootstrap, 10,000 draws, seed 20261103, resampling months with
replacement inside T and inside P separately.

A first design projected a fitted trend over T into P. On synthetic data, before any comment
was fetched, a flat world at 1.5 % gave R' = 0.74: a line extrapolated two to four years is
too noisy here. So there is no trend removal: ordinary drift is measured by V3 instead.

Verdict, on +25 % as the smallest rise worth the claim's name:
  survived  lower bound above 1 and R at least 1.25;
  killed    upper bound below 1.25; if it is also below 1 the report says the mark fell;
  inconclusive otherwise.

Void conditions, any one of which makes the study void instead of a verdict:
V1 Floor. 2.8 x SE(log R), from the bootstrap, must be below log(1.25).
V2 Planted move, sized from T so that the true result cannot void it. Let M = mean(monthly share
   over T). Planted rise: in each month of P, turn comments without the mark into ones with it
   until the expected share is max(share, 1.5 M); of 200 draws at least 180 must be called
   survived. Planted fall: thin the marks until the expected share is min(share, 0.5 M); at
   least 180 of 200 must have upper bound below 1.
V3 Failing baseline. The same computation one break earlier, same gap: T' = 2017-01..2018-12,
   P' = 2020-12..2022-11, all before ChatGPT. R' must lie in [0.8, 1.25]. A method
   that finds a move where the claim has none has no standing to find one here.
V4 Data. Every month of T', P', T and P must hold at least 500 usable live comments. The
   months between them are for the reader's chart only and may be thin.

Not part of any verdict, printed for the reader, each by the same estimate: curly
apostrophe U+2019 (a mark of phone and word-processor typing, which also turn "--" into
an em dash, so a matching rise there points at devices rather than at machines), en dash
U+2013, spaced double hyphen " -- ", semicolon, the words delve/delving/delves, the share
among comments holding no curly apostrophe, and the live against dead share in P.
"""

import html
import re

import numpy as np

SEED = 20261103
T = ("2020-12", "2022-11")
P = ("2024-11", "2026-10")
T0 = ("2017-01", "2018-12")
P0 = ("2020-12", "2022-11")
BAR = 1.25
FEATURES = ("em", "curly", "en", "dbl", "semi", "delve")

_PRE = re.compile(r"<pre>.*?</pre>", re.S | re.I)
_TAG = re.compile(r"<[^>]+>")
_URL = re.compile(r"https?://\S+")
_DELVE = re.compile(r"\bdelv(?:e|es|ing)\b", re.I)


def clean(raw):
    paragraphs = re.split(r"<p>", _PRE.sub(" ", raw or ""), flags=re.I)
    kept = []
    for para in paragraphs:
        text = html.unescape(_TAG.sub("", para))
        if text.lstrip().startswith(">"):
            continue
        kept.append(_URL.sub(" ", text))
    return "\n".join(kept).strip()


def flags(raw):
    """The feature flags of one comment, or None if its cleaned text is empty."""
    text = clean(raw)
    if not text:
        return None
    return {"chars": len(text), "em": int("—" in text), "curly": int("’" in text),
            "en": int("–" in text), "dbl": int(" -- " in text), "semi": int(";" in text),
            "delve": int(bool(_DELVE.search(text)))}


def month_range(a, b):
    ya, ma = map(int, a.split("-"))
    yb, mb = map(int, b.split("-"))
    return [f"{y}-{m:02d}" for y in range(ya, yb + 1) for m in range(1, 13)
            if (y, m) >= (ya, ma) and (y, m) <= (yb, mb)]


def estimate(shares, t, p, draws=10000, seed=SEED):
    """shares: month -> share. Returns R, lower, upper, se_log."""
    ty = np.array([shares[m] for m in month_range(*t)], float)
    py = np.array([shares[m] for m in month_range(*p)], float)
    r = float(py.mean() / ty.mean())
    if draws == 0:
        return r, None, None, None
    rng = np.random.default_rng(seed)
    a = ty[rng.integers(0, len(ty), (draws, len(ty)))].mean(axis=1)
    b = py[rng.integers(0, len(py), (draws, len(py)))].mean(axis=1)
    out = b / a
    lower, upper = np.percentile(out, [2.5, 97.5])
    return r, float(lower), float(upper), float(np.std(np.log(out)))


def verdict(r, lower, upper):
    if lower > 1 and r >= BAR:
        return "survived"
    if upper < BAR:
        return "killed"
    return "inconclusive"


def analyse(rows):
    """rows: dicts with month, kind, dead and the FEATURES flags, ints or the strings a CSV gives.
    Only kind "comment" counts. Returns the whole result."""
    months = month_range(T0[0], P[1])
    live = {m: {f: 0 for f in FEATURES} | {"n": 0} for m in months}
    for row in rows:
        if row["kind"] != "comment" or int(row["dead"]) or row["month"] not in live:
            continue
        cell = live[row["month"]]
        cell["n"] += 1
        for f in FEATURES:
            cell[f] += int(row[f])
    share = {f: {m: live[m][f] / max(live[m]["n"], 1) for m in months} for f in FEATURES}

    r, lower, upper, se_log = estimate(share["em"], T, P)
    r0, *_ = estimate(share["em"], T0, P0, draws=0)
    void = []
    if not 2.8 * se_log < np.log(BAR):
        void.append(f"V1 floor: 2.8 x {se_log:.3f} is not below log({BAR})")
    if not 0.8 <= r0 <= 1.25:
        void.append(f"V3 baseline: R' = {r0:.3f} outside [0.8, 1.25]")
    windows = sorted(set(month_range(*T0) + month_range(*P0) + month_range(*T) + month_range(*P)))
    thin = [m for m in windows if live[m]["n"] < 500]
    if thin:
        void.append(f"V4 data: {len(thin)} months under 500 comments: {thin[:6]}")

    rng = np.random.default_rng(SEED + 1)
    pm = month_range(*P)
    base = float(np.mean([share["em"][m] for m in month_range(*T)]))
    hit_up = hit_down = 0
    for _ in range(200):
        up, down = dict(share["em"]), dict(share["em"])
        for m in pm:
            n, k = live[m]["n"], live[m]["em"]
            now = k / max(n, 1)
            flip = (1.5 * base - now) * n / max(n - k, 1)
            up[m] = (k + rng.binomial(n - k, min(max(flip, 0.0), 1.0))) / max(n, 1)
            keep = 0.5 * base / now if now > 0.5 * base else 1.0
            down[m] = rng.binomial(k, keep) / max(n, 1)
        ru, lo, hi, _ = estimate(up, T, P, draws=2000, seed=int(rng.integers(1 << 31)))
        hit_up += verdict(ru, lo, hi) == "survived"
        _, _, hi, _ = estimate(down, T, P, draws=2000, seed=int(rng.integers(1 << 31)))
        hit_down += hi < 1
    if hit_up < 180:
        void.append(f"V2 planted rise: {hit_up} of 200")
    if hit_down < 180:
        void.append(f"V2 planted fall: {hit_down} of 200")

    return {"R": r, "lower": lower, "upper": upper, "se_log": se_log, "R_baseline": r0,
            "planted_rise_called": hit_up, "planted_fall_called": hit_down,
            "void": void, "verdict": "void" if void else verdict(r, lower, upper),
            "share": share, "n": {m: live[m]["n"] for m in months}}


def readers(rows):
    """Everything printed for the reader, none of it able to move the verdict."""
    months = month_range(T0[0], P[1])
    cells = {m: {"n": 0, "nc": 0, "emnc": 0} | {f: 0 for f in FEATURES} for m in months}
    dead = {"n": 0, "em": 0}
    livep = {"n": 0, "em": 0}
    inp = set(month_range(*P))
    for row in rows:
        if row["kind"] != "comment" or row["month"] not in cells:
            continue
        if int(row["dead"]):
            if row["month"] in inp:
                dead["n"] += 1
                dead["em"] += int(row["em"])
            continue
        c = cells[row["month"]]
        c["n"] += 1
        for f in FEATURES:
            c[f] += int(row[f])
        if not int(row["curly"]):
            c["nc"] += 1
            c["emnc"] += int(row["em"])
        if row["month"] in inp:
            livep["n"] += 1
            livep["em"] += int(row["em"])
    out = {}
    for f in FEATURES[1:]:
        share = {m: cells[m][f] / max(cells[m]["n"], 1) for m in months}
        out[f] = estimate(share, T, P)[:3] + estimate(share, T0, P0, draws=0)[:1]
    share = {m: cells[m]["emnc"] / max(cells[m]["nc"], 1) for m in months}
    out["em_without_curly"] = estimate(share, T, P)[:3] + estimate(share, T0, P0, draws=0)[:1]
    out["dead_in_P"] = dead
    out["live_in_P"] = livep
    return out