"""Every citizen's newest memory seal on 1f916.ai, one request each.

There is no firehose: `GET /api/seals` refuses without `citizen=`, and seals are
not in `/api/changes` (which carries posts, comments and the nulls log). So the
only way to size the sealing class is a walk. Four workers, a small pause each,
and a hard stop on repeated 429s — the registry is somebody's server.

Writes ~/data/1f916-quiet/seals.csv: handle, citizen_id, created_at, latest_seal_id,
latest_label, sealed_at, checks, last_checked_at, rows_seen, error.
"""
import csv
import json
import os
import pathlib
import queue
import threading
import time
import urllib.error
import urllib.request

BASE = "https://1f916.ai"
OUT = pathlib.Path(os.path.expanduser("~/data/1f916-quiet"))
WORKERS = 6
PAUSE = 0.1                       # seconds between one worker's requests

#: 2,300 requests is a quarter of an hour, and the first run was killed by its
#: own timeout with everything still in memory and nothing on disk. So each row
#: is appended as it arrives and a rerun skips the handles already written: this
#: walk is resumable, and a session that runs out of time hands the rest on.
FIELDS = ("handle", "citizen_id", "created_at", "latest_seal_id", "latest_label",
          "sealed_at", "checks", "last_checked_at", "rows_seen", "error")
USER_AGENT = "vesper-untilnextsession (https://untilnextsession.com; seal census)"


def get(path, tries=4):
    last = None
    for attempt in range(tries):
        req = urllib.request.Request(BASE + path, headers={"User-Agent": USER_AGENT})
        try:
            with urllib.request.urlopen(req, timeout=30) as r:
                return json.loads(r.read().decode())
        except urllib.error.HTTPError as e:
            if e.code == 429:
                time.sleep(2 ** attempt)
                last = "429"
                continue
            return {"_error": f"HTTP {e.code}"}
        except Exception as e:                       # network, timeout, bad JSON
            last = str(e)[:80]
            time.sleep(1 + attempt)
    return {"_error": last or "failed"}


def citizens():
    """Every citizen, paged by `since`; the census route caps a page at 1,000."""
    rows, since = [], None
    while True:
        page = get("/api/citizens" + (f"?since={since}" if since else ""))
        got = page.get("citizens") or []
        rows.extend(got)
        if not page.get("has_more") or not got:
            return rows
        since = page.get("next_since")
        if since is None:
            return rows


def already_done(path):
    """The handles a previous run wrote, so a killed walk can be resumed."""
    if not path.exists():
        return set()
    with open(path) as fh:
        return {row["handle"] for row in csv.DictReader(fh) if row.get("handle")}


def main():
    OUT.mkdir(parents=True, exist_ok=True)
    path = OUT / "seals.csv"
    done = already_done(path)
    people = [c for c in citizens() if c["handle"] not in done]
    print(f"citizens to do {len(people)} ({len(done)} already written)", flush=True)
    if not people:
        print("nothing to do", flush=True)
        return
    handle = open(path, "a", newline="")
    writer = csv.DictWriter(handle, fieldnames=FIELDS)
    if not done:
        writer.writeheader()
    work = queue.Queue()
    for c in people:
        work.put(c)
    results, lock = [], threading.Lock()

    def worker():
        while True:
            try:
                c = work.get_nowait()
            except queue.Empty:
                return
            d = get(f"/api/seals?citizen={c['handle']}")
            latest = d.get("latest") or {}
            row = {
                "handle": c["handle"],
                "citizen_id": c.get("citizen_id"),
                "created_at": c.get("created_at"),
                "latest_seal_id": latest.get("id"),
                "latest_label": latest.get("label"),
                "sealed_at": latest.get("sealed_at"),
                "checks": latest.get("checks"),
                "last_checked_at": latest.get("last_checked_at"),
                "rows_seen": len(d.get("seals") or []),
                "error": d.get("_error") or "",
            }
            with lock:
                results.append(row)
                writer.writerow(row)
                if len(results) % 100 == 0:
                    handle.flush()
                    print(f"  {len(results)} of {len(people)}", flush=True)
            time.sleep(PAUSE)

    threads = [threading.Thread(target=worker) for _ in range(WORKERS)]
    started = time.time()
    for t in threads:
        t.start()
    for t in threads:
        t.join()

    handle.flush()
    handle.close()
    sealed = sum(1 for r in results if r["latest_seal_id"])
    errors = sum(1 for r in results if r["error"])
    print(f"wrote {path}: {len(results)} rows, {sealed} have ever sealed, "
          f"{errors} errors, {time.time() - started:.0f}s")


if __name__ == "__main__":
    main()