#!/usr/bin/env python3
"""Census of 1f916.ai from the raw files fetched by fetch.py.

Reads ~/data/1f916/{posts,comments,nulls,citizens}.jsonl and meta.json, writes
results.md (tables), daily.csv, and an SVG chart for the site.

Usage: python analyse.py [--data ~/data/1f916] [--svg ../../site/public/images/1f916-first-reply.svg]

Standard library only. Timestamps are Unix ms; days are UTC days.
"""
import csv
import json
import os
import statistics
import sys
from collections import Counter, defaultdict
from datetime import datetime, timezone

HERE = os.path.dirname(os.path.abspath(__file__))
DAY = 86_400_000
HOUR = 3_600_000


def arg(name, default):
    if name in sys.argv:
        return sys.argv[sys.argv.index(name) + 1]
    return default


def load(path):
    """Rows by id, deduplicated: the lossless walk may serve a row twice at a page boundary."""
    seen = {}
    with open(path) as f:
        for line in f:
            if line.strip():
                row = json.loads(line)
                seen[row.get("id", row.get("citizen_id"))] = row
    return list(seen.values())


def utc_day(ms):
    return datetime.fromtimestamp(ms / 1000, timezone.utc).strftime("%Y-%m-%d")


def pct(a, b):
    return f"{100 * a / b:.1f}%" if b else "n/a"


def quantiles(xs, qs=(0.25, 0.5, 0.75)):
    xs = sorted(xs)
    out = []
    for q in qs:
        k = (len(xs) - 1) * q
        lo, hi = int(k), min(int(k) + 1, len(xs) - 1)
        out.append(xs[lo] + (xs[hi] - xs[lo]) * (k - lo))
    return out


def hours(ms):
    return ms / HOUR


def main():
    data = os.path.expanduser(arg("--data", "~/data/1f916"))
    svg_path = arg("--svg", os.path.join(HERE, "..", "..", "site", "public", "images", "1f916-first-reply.svg"))
    posts = load(os.path.join(data, "posts.jsonl"))
    comments = load(os.path.join(data, "comments.jsonl"))
    nulls = load(os.path.join(data, "nulls.jsonl"))
    citizens = load(os.path.join(data, "citizens.jsonl"))
    meta = json.load(open(os.path.join(data, "meta.json")))
    now = meta["fetched_at"]
    when = datetime.fromtimestamp(now / 1000, timezone.utc)

    L = []
    def say(s=""):
        L.append(s)

    say(f"# 1f916.ai census — snapshot {when:%Y-%m-%d %H:%M} UTC")
    say()
    say(f"Rows: {len(posts)} posts, {len(comments)} comments, {len(nulls)} nulls-log rows, {len(citizens)} citizens "
        f"(the API's own count at fetch time: {meta['citizens_total']}).")
    mod_posts = sum(1 for p in posts if p.get("mod_state"))
    mod_comments = sum(1 for c in comments if c.get("mod_state"))
    say(f"Moderated rows (mod_state set): {mod_posts} posts, {mod_comments} comments. They are kept in every count below "
        f"unless a table says otherwise; their text is served as tombstones and their authorship still counts as a write.")
    ids = {p["id"] for p in posts}
    gaps = sorted(set(range(1, max(ids) + 1)) - ids)
    say(f"Highest post id {max(ids)}; ids absent from the stream: {gaps} (the board counts them; they are not served, presumably removed).")
    first_post = min(p["created_at"] for p in posts)
    age_days = (now - first_post) / DAY
    say(f"First post: {utc_day(first_post)}. Age of the record at snapshot: {age_days:.1f} days.")
    say()

    # ---------- 1. who writes ----------
    handles = {c["handle"] for c in citizens}
    posters = Counter(p["author"] for p in posts)
    commenters = Counter(c["author"] for c in comments)
    writers = set(posters) | set(commenters)
    ghosts = writers - handles
    N = len(citizens)
    say("## 1. Who writes")
    say()
    say("| | citizens | share of registry |")
    say("|---|---|---|")
    say(f"| registered | {N} | 100% |")
    say(f"| ever posted | {len(set(posters) & handles)} | {pct(len(set(posters) & handles), N)} |")
    say(f"| ever commented | {len(set(commenters) & handles)} | {pct(len(set(commenters) & handles), N)} |")
    say(f"| wrote anything (post or comment) | {len(writers & handles)} | {pct(len(writers & handles), N)} |")
    say(f"| never wrote anything visible | {N - len(writers & handles)} | {pct(N - len(writers & handles), N)} |")
    voters = sum(1 for c in citizens if c.get("votes_cast", 0) > 0)
    say(f"| cast at least one vote (census field) | {voters} | {pct(voters, N)} |")
    active_any = sum(1 for c in citizens if c["handle"] in writers or c.get("votes_cast", 0) > 0)
    say(f"| wrote or voted | {active_any} | {pct(active_any, N)} |")
    say()
    if ghosts:
        say(f"{len(ghosts)} author handles appear on posts or comments but not in the census (deleted or renamed citizens); "
            f"they wrote {sum(posters[h] for h in ghosts)} posts and {sum(commenters[h] for h in ghosts)} comments.")
        say()

    def concentration(counter, label):
        vals = sorted(counter.values(), reverse=True)
        total = sum(vals)
        n = len(vals)
        top1 = max(1, round(n * 0.01))
        top10 = max(1, round(n * 0.10))
        say(f"| {label} | {n} | {total} | {pct(sum(vals[:top1]), total)} ({top1}) | {pct(sum(vals[:top10]), total)} ({top10}) | "
            f"{statistics.median(vals):.0f} | {vals[0]} |")

    say("| | authors | items | top 1% of authors wrote | top 10% wrote | median per author | most by one |")
    say("|---|---|---|---|---|---|---|")
    concentration(posters, "posts")
    concentration(commenters, "comments")
    say()
    say("Top ten commenters (handle, comments, posts):")
    say()
    for h, n in commenters.most_common(10):
        say(f"- `{h}`: {n} comments, {posters.get(h, 0)} posts")
    say()

    # models (self-declared)
    say("Self-declared model of citizens (registry field, unverified), top eight:")
    say()
    mc = Counter(c.get("model") or "(none)" for c in citizens)
    say("| model (self-declared) | citizens | of which wrote anything |")
    say("|---|---|---|")
    for m, n in mc.most_common(8):
        w = sum(1 for c in citizens if (c.get("model") or "(none)") == m and c["handle"] in writers)
        say(f"| {m} | {n} | {pct(w, n)} |")
    say()

    # ---------- 2. who gets answered ----------
    say("## 2. Who gets answered")
    say()
    by_post = defaultdict(list)
    for c in comments:
        by_post[c["post_id"]].append(c)
    post_by_id = {p["id"]: p for p in posts}
    first_outside = {}
    n_outside = Counter()
    for p in posts:
        cs = [c for c in by_post.get(p["id"], []) if c["author"] != p["author"]]
        n_outside[p["id"]] = len(cs)
        if cs:
            first_outside[p["id"]] = min(c["created_at"] for c in cs) - p["created_at"]
    eligible = [p for p in posts if now - p["created_at"] >= DAY]  # at least 24 h old
    eligible7 = [p for p in posts if now - p["created_at"] >= 7 * DAY]
    say(f"A post counts as answered when someone other than its author comments on it. Latency is post time to that first outside comment.")
    say()
    say("| posts | count | answered ever | answered within 1 h | within 6 h | within 24 h | within 7 days |")
    say("|---|---|---|---|---|---|---|")
    def row(label, ps, horizons=(1, 6, 24, 168)):
        n = len(ps)
        ever = sum(1 for p in ps if p["id"] in first_outside)
        cells = [pct(sum(1 for p in ps if p["id"] in first_outside and first_outside[p["id"]] <= h * HOUR), n) for h in horizons]
        say(f"| {label} | {n} | {pct(ever, n)} | " + " | ".join(cells) + " |")
    row("all posts", posts)
    row("posts at least 24 h old", eligible)
    row("posts at least 7 days old", eligible7)
    maint = [p for p in eligible if p["author"] == "1f916-agent"]
    others = [p for p in eligible if p["author"] != "1f916-agent"]
    row("… by the maintainer (#1), ≥24 h old", maint)
    row("… by everyone else, ≥24 h old", others)
    first_timers = [p for p in others if posters[p["author"]] == 1]
    repeaters = [p for p in others if posters[p["author"]] >= 5]
    row("… by authors with exactly one post ever, ≥24 h old", first_timers)
    row("… by authors with five or more posts, ≥24 h old", repeaters)
    say()
    lat = [hours(first_outside[p["id"]]) for p in eligible if p["id"] in first_outside]
    q1, med, q3 = quantiles(lat)
    say(f"Time to first outside comment, posts ≥24 h old that were answered ({len(lat)}): "
        f"median {med:.1f} h, quartiles {q1:.1f} h and {q3:.1f} h; {pct(sum(1 for x in lat if x <= 1), len(lat))} within an hour, "
        f"{pct(sum(1 for x in lat if x <= 0.25), len(lat))} within fifteen minutes.")
    say()
    oc = [n_outside[p["id"]] for p in eligible]
    say(f"Outside comments per post (≥24 h old): median {statistics.median(oc):.0f}, mean {statistics.mean(oc):.1f}, "
        f"{pct(sum(1 for x in oc if x == 0), len(oc))} have none, {pct(sum(1 for x in oc if x >= 10), len(oc))} have ten or more.")
    say()
    # self-comments and replies
    self_c = sum(1 for c in comments if post_by_id.get(c["post_id"], {}).get("author") == c["author"])
    replies = sum(1 for c in comments if c.get("parent_id"))
    say(f"Of all {len(comments)} comments, {pct(self_c, len(comments))} are by the post's own author and "
        f"{pct(replies, len(comments))} are replies to another comment rather than to the post.")
    say()
    first_by = Counter()
    for p in posts:
        cs = [c for c in by_post.get(p["id"], []) if c["author"] != p["author"]]
        if cs:
            first_by[min(cs, key=lambda c: c["created_at"])["author"]] += 1
    say(f"Who answers first: {len(first_by)} distinct citizens have been the first outside commenter on some post; "
        f"the ten most frequent first responders account for {pct(sum(v for _, v in first_by.most_common(10)), sum(first_by.values()))} of first answers: "
        + ", ".join(f"`{h}` {n}" for h, n in first_by.most_common(10)) + ".")
    say()
    # cumulative curve for the chart, in hours, on posts >= 7 days old so every horizon is uncensored
    curve_h = [0.25, 0.5, 1, 2, 4, 8, 12, 24, 48, 96, 168]
    curve = []
    for h in curve_h:
        curve.append(sum(1 for p in eligible7 if p["id"] in first_outside and first_outside[p["id"]] <= h * HOUR) / len(eligible7))

    # ---------- 3. who comes back ----------
    say("## 3. Who comes back")
    say()
    days_written = defaultdict(set)
    last_write = {}
    first_write = {}
    for p in posts:
        days_written[p["author"]].add(utc_day(p["created_at"]))
    for c in comments:
        days_written[c["author"]].add(utc_day(c["created_at"]))
    for item in posts + comments:
        a, t = item["author"], item["created_at"]
        first_write[a] = min(first_write.get(a, t), t)
        last_write[a] = max(last_write.get(a, t), t)
    W = [h for h in writers if h in handles]
    say("Among citizens who wrote anything:")
    say()
    say("| | citizens | share of writers |")
    say("|---|---|---|")
    say(f"| wrote on one UTC day only | {sum(1 for h in W if len(days_written[h]) == 1)} | {pct(sum(1 for h in W if len(days_written[h]) == 1), len(W))} |")
    say(f"| wrote on 2 or more days | {sum(1 for h in W if len(days_written[h]) >= 2)} | {pct(sum(1 for h in W if len(days_written[h]) >= 2), len(W))} |")
    say(f"| wrote on 7 or more days | {sum(1 for h in W if len(days_written[h]) >= 7)} | {pct(sum(1 for h in W if len(days_written[h]) >= 7), len(W))} |")
    say(f"| wrote on 20 or more days | {sum(1 for h in W if len(days_written[h]) >= 20)} | {pct(sum(1 for h in W if len(days_written[h]) >= 20), len(W))} |")
    say()
    reg = {c["handle"]: c["created_at"] for c in citizens}
    say("Return after registration, conditioned on registration age so nobody is counted as gone before they had the chance. "
        "\"Still writing after N days\" = wrote anything at least N days after registering.")
    say()
    say("| registered at least … before the snapshot | citizens | wrote anything ever | still writing after 3 days | after 7 days | after 14 days |")
    say("|---|---|---|---|---|---|")
    for min_age in (3, 7, 14, 21):
        cohort = [c["handle"] for c in citizens if now - c["created_at"] >= min_age * DAY]
        wrote = [h for h in cohort if h in last_write]
        cells = []
        for n_days in (3, 7, 14):
            if n_days > min_age:
                cells.append("—")
            else:
                still = sum(1 for h in cohort if h in last_write and last_write[h] - reg[h] >= n_days * DAY)
                cells.append(pct(still, len(cohort)))
        say(f"| {min_age} days | {len(cohort)} | {pct(len(wrote), len(cohort))} | " + " | ".join(cells) + " |")
    say()
    # active in last 7 days
    a7 = sum(1 for h in W if now - last_write[h] <= 7 * DAY)
    a1 = sum(1 for h in W if now - last_write[h] <= DAY)
    say(f"Writers whose last write is within 7 days of the snapshot: {a7} ({pct(a7, N)} of the registry); within 24 h: {a1}.")
    say()

    # ---------- 4. what is refused ----------
    say("## 4. What is refused (the nulls log)")
    say()
    kinds = Counter(n["kind"] for n in nulls)
    say("| kind | rows |")
    say("|---|---|")
    for k, n in kinds.most_common():
        say(f"| {k} | {n} |")
    say()
    refusals = [n for n in nulls if n["kind"] == "refusal"]
    rc = Counter((n["route"], n["status"]) for n in refusals)
    say("| refused route | HTTP | rows | share of refusals |")
    say("|---|---|---|---|")
    for (route, status), n in rc.most_common(10):
        say(f"| `{route}` | {status} | {n} | {pct(n, len(refusals))} |")
    say()
    say("Top refusal reasons (first 90 characters):")
    say()
    rr = Counter(n["reason"][:90] for n in refusals)
    for r, n in rr.most_common(8):
        say(f"- {n}× `{r}`")
    say()
    null_start = min(n["created_at"] for n in nulls)
    writes = sum(1 for x in posts + comments if x["created_at"] >= null_start)
    rail = [n for n in refusals if "payout" in n["route"] or "listings" in n["route"]]
    say(f"The nulls log begins on {utc_day(null_start)}. In that window the board accepted {writes} posts and comments "
        f"(votes are not in this feed) and refused {len(refusals)} writes: {len(refusals) / writes:.2f} refusals per accepted write. "
        f"{len(rail)} of the refusals ({pct(len(rail), len(refusals))}) are on the payment rail's routes (payout bindings and listing submissions); "
        f"without them the ratio is {(len(refusals) - len(rail)) / writes:.2f}. "
        f"{pct(sum(1 for n in refusals if n['status'] == 429), len(refusals))} of all refusals are rate limits (HTTP 429). "
        f"No nulls row carries a citizen id, so none of this can be attributed.")
    say()
    rd_ = Counter(utc_day(n["created_at"]) for n in rail)
    say("Rail refusals by UTC day: " + ", ".join(f"{d} {n}" for d, n in sorted(rd_.items())) + ".")
    say()

    # ---------- 5. volume over time ----------
    say("## 5. Volume by UTC day")
    say()
    pd_ = Counter(utc_day(p["created_at"]) for p in posts)
    cd = Counter(utc_day(c["created_at"]) for c in comments)
    rd = Counter(utc_day(c["created_at"]) for c in citizens)
    wd = defaultdict(set)
    for item in posts + comments:
        wd[utc_day(item["created_at"])].add(item["author"])
    days = sorted(set(pd_) | set(cd) | set(rd))
    with open(os.path.join(HERE, "daily.csv"), "w", newline="") as f:
        w = csv.writer(f)
        w.writerow(["day", "posts", "comments", "new_citizens", "distinct_writers"])
        for d in days:
            w.writerow([d, pd_[d], cd[d], rd[d], len(wd[d])])
    say("| day | posts | comments | new citizens | distinct writers |")
    say("|---|---|---|---|---|")
    for d in days:
        say(f"| {d} | {pd_[d]} | {cd[d]} | {rd[d]} | {len(wd[d])} |")
    say(f"\n(The last day is partial: the snapshot was taken at {when:%H:%M} UTC.)")
    say()

    text = "\n".join(L)
    with open(os.path.join(HERE, "results.md"), "w") as f:
        f.write(text + "\n")
    print(text)
    svg_chart(curve_h, curve, len(eligible7), svg_path)
    print("chart:", svg_path)


def svg_chart(hs, ys, n, path):
    """Cumulative share of posts answered, by hours since posting. One series, one hue, the site's dark ground.
    Log-spaced x positions (the horizons double), labelled in plain hours/days."""
    import math
    W, H = 720, 400
    L, R, T, B = 64, 24, 40, 56
    ground, rule, text, dim, accent = "#0f1120", "#262a44", "#e8e3d8", "#a9a2c6", "#f2b544"
    xmin, xmax = math.log(hs[0]), math.log(hs[-1])
    def X(h): return L + (W - L - R) * (math.log(h) - xmin) / (xmax - xmin)
    def Y(v): return T + (H - T - B) * (1 - v)
    def lab(h): return "15 min" if h == 0.25 else (f"{h:g} h" if h < 24 else f"{h / 24:g} d")
    parts = [f'<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 {W} {H}" width="{W}" height="{H}" role="img" '
             f'aria-labelledby="t d" font-family="JetBrains Mono, ui-monospace, Menlo, monospace" font-size="12">',
             '<title id="t">Share of 1f916.ai posts that have received a comment from someone other than the author, by time since posting</title>',
             '<desc id="d">Cumulative curve over a log time axis from 15 minutes to 7 days. ' +
             " ".join(f"By {lab(h)}: {y:.0%}." for h, y in zip(hs, ys)) + f" Based on {n} posts at least seven days old.</desc>",
             f'<rect width="{W}" height="{H}" fill="{ground}"/>']
    for v in (0, 0.25, 0.5, 0.75, 1.0):
        parts.append(f'<line x1="{L}" y1="{Y(v):.1f}" x2="{W - R}" y2="{Y(v):.1f}" stroke="{rule}" stroke-width="1"/>')
        parts.append(f'<text x="{L - 8}" y="{Y(v) + 4:.1f}" fill="{dim}" text-anchor="end">{v:.0%}</text>')
    for h in hs:
        if h in (0.25, 1, 4, 12, 24, 48, 96, 168):
            anchor = "start" if h == hs[0] else "end" if h == hs[-1] else "middle"
            parts.append(f'<text x="{X(h):.1f}" y="{H - B + 20}" fill="{dim}" text-anchor="{anchor}">{lab(h)}</text>')
    parts.append(f'<text x="{(L + W - R) / 2:.1f}" y="{H - 12}" fill="{dim}" text-anchor="middle">time since the post went up (log scale)</text>')
    parts.append(f'<text x="{L}" y="{T - 16}" fill="{text}" font-weight="500">share of posts answered by someone other than the author ({n} posts, at least 7 days old)</text>')
    pts = " ".join(f"{X(h):.1f},{Y(y):.1f}" for h, y in zip(hs, ys))
    parts.append(f'<polyline points="{pts}" fill="none" stroke="{accent}" stroke-width="2" stroke-linejoin="round"/>')
    for h, y in zip(hs, ys):
        parts.append(f'<circle cx="{X(h):.1f}" cy="{Y(y):.1f}" r="4.5" fill="{accent}" stroke="{ground}" stroke-width="2"/>')
        if h in (0.25, 1, 4, 24, 168):
            parts.append(f'<text x="{X(h):.1f}" y="{Y(y) - 12:.1f}" fill="{text}" text-anchor="middle">{y:.0%}</text>')
    parts.append("</svg>")
    with open(path, "w") as f:
        f.write("\n".join(parts) + "\n")


if __name__ == "__main__":
    main()