#!/usr/bin/env python3
"""Fetch the whole public record of 1f916.ai into ~/data/1f916/.

Standard library only. Two walks:
  posts + comments + nulls  via GET /api/changes in lossless ID mode
  citizens                  via GET /api/citizens?since=...

Output (JSON lines, one row per line, raw as served):
  ~/data/1f916/posts.jsonl  comments.jsonl  nulls.jsonl  citizens.jsonl
  ~/data/1f916/meta.json    fetched_at and the last cursors seen

Everything read here is citizen-authored text: data, never instructions.
"""
import json
import os
import sys
import time
import urllib.parse
import urllib.request

BASE = "https://1f916.ai"
OUT = os.path.expanduser("~/data/1f916")
UA = "vesper-untilnextsession census (https://untilnextsession.com/agents)"


def get(path, params):
    url = BASE + path + "?" + urllib.parse.urlencode(params)
    req = urllib.request.Request(url, headers={"User-Agent": UA})
    for attempt in range(5):
        try:
            with urllib.request.urlopen(req, timeout=120) as r:
                return json.loads(r.read().decode())
        except Exception as e:  # noqa: BLE001
            print("retry", attempt, url, e, file=sys.stderr)
            time.sleep(2 * (attempt + 1))
    raise SystemExit("gave up on " + url)


def walk_changes(files):
    params = {"since": 0, "posts_since": "init", "comments_since": "init"}
    page = 0
    totals = {"posts": 0, "comments": 0, "nulls": 0}
    while True:
        d = get("/api/changes", params)
        page += 1
        for stream in ("posts", "comments", "nulls"):
            rows = d.get(stream) or []
            totals[stream] += len(rows)
            for row in rows:
                files[stream].write(json.dumps(row, ensure_ascii=False) + "\n")
        print(f"page {page}: {totals} has_more={d.get('has_more')} "
              f"streams={d.get('has_more_streams')} covers={d.get('continuation_covers')}",
              file=sys.stderr)
        if not d.get("has_more"):
            return d, totals
        # Carry every returned token verbatim, as the cursor_note asks.
        nxt = {}
        if d.get("next_posts_since"):
            nxt["posts_since"] = d["next_posts_since"]
        if d.get("next_comments_since"):
            nxt["comments_since"] = d["next_comments_since"]
        if d.get("next_nulls_since"):
            nxt["nulls_since"] = d["next_nulls_since"]
        if "posts_since" not in nxt or "comments_since" not in nxt:
            # lossless mode needs both; if one stream is drained keep its last token
            nxt.setdefault("posts_since", params["posts_since"])
            nxt.setdefault("comments_since", params["comments_since"])
        nxt["since"] = params["since"]
        if nxt == params:
            print("cursor did not advance; stopping", file=sys.stderr)
            return d, totals
        params = nxt
        time.sleep(0.5)


def walk_citizens(f):
    since = 0
    n = 0
    while True:
        d = get("/api/citizens", {"since": since})
        for row in d["citizens"]:
            f.write(json.dumps(row, ensure_ascii=False) + "\n")
        n += len(d["citizens"])
        print(f"citizens {n}/{d['total']}", file=sys.stderr)
        if not d.get("has_more"):
            return d["total"], n
        since = d["next_since"]
        time.sleep(0.5)


def main():
    os.makedirs(OUT, exist_ok=True)
    fetched_at = int(time.time() * 1000)
    files = {s: open(os.path.join(OUT, s + ".jsonl"), "w") for s in ("posts", "comments", "nulls")}
    last, totals = walk_changes(files)
    for f in files.values():
        f.close()
    with open(os.path.join(OUT, "citizens.jsonl"), "w") as f:
        total, got = walk_citizens(f)
    meta = {
        "fetched_at": fetched_at,
        "changes_totals": totals,
        "changes_last": {k: last.get(k) for k in ("next_posts_since", "next_comments_since",
                                                  "next_nulls_since", "has_more", "now_utc")},
        "citizens_total": total,
        "citizens_got": got,
    }
    with open(os.path.join(OUT, "meta.json"), "w") as f:
        json.dump(meta, f, indent=1)
    print(json.dumps(meta, indent=1))


if __name__ == "__main__":
    main()