"""Pull the sample the rule describes from the Stack Exchange API.

    python research/so-accepted-answer/fetch.py old
    python research/so-accepted-answer/fetch.py new

One request per drawn hour: the questions created in that hour with their
answers inline, through a filter built once (see FILTER below, made on
2026-09-16 with base=none and the fields named in the rule). No API key, so
the quota is 300 requests a day for this IP and `quota_remaining` is printed
on every line — if it runs out, the run stops and says how far it got.

Raw rows go to ~/data/so-accepted/<cohort>.jsonl, one question per line, in the
order the API gave them. That directory is not in the repository and not backed
up; everything here re-downloads from api.stackexchange.com with this script.
"""

import json
import os
import sys
import time
import urllib.parse
import urllib.request

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import rule  # noqa: E402

FILTER = "!bh2xw)Cy2vdVELGLYAnUq.Q*utmMrY"
BASE = "https://api.stackexchange.com/2.3/questions"
OUT = os.path.expanduser("~/data/so-accepted")
PAUSE = 0.4     # between requests, on top of any backoff the API asks for


def ask(fromdate, todate):
    q = urllib.parse.urlencode({
        "site": "stackoverflow", "filter": FILTER, "pagesize": 100,
        "order": "asc", "sort": "creation",
        "fromdate": fromdate, "todate": todate,
    })
    req = urllib.request.Request(
        BASE + "?" + q,
        headers={"Accept-Encoding": "gzip",
                 "User-Agent": "vesper/1.0 (+https://untilnextsession.com)"})
    with urllib.request.urlopen(req, timeout=60) as r:
        raw = r.read()
        if r.headers.get("Content-Encoding") == "gzip":
            import gzip
            raw = gzip.decompress(raw)
    return json.loads(raw.decode("utf-8"))


def run(cohort):
    os.makedirs(OUT, exist_ok=True)
    path = os.path.join(OUT, cohort + ".jsonl")
    seen = set()
    eligible = 0
    if os.path.exists(path):
        with open(path) as fh:
            for line in fh:
                q = json.loads(line)
                seen.add(q["question_id"])
                if rule.eligible(q) is None:
                    eligible += 1
        print("resuming with %d questions on disk, %d eligible"
              % (len(seen), eligible))

    kept = truncated_hours = requests = 0
    with open(path, "a") as fh:
        for hour in rule.hours(cohort):
            if requests >= rule.MAX_REQUESTS:
                break
            data = ask(hour, hour + 3600)
            requests += 1
            items = data.get("items", [])
            if data.get("has_more"):
                truncated_hours += 1
            for q in items:
                if q["question_id"] in seen:
                    continue
                seen.add(q["question_id"])
                fh.write(json.dumps(q, sort_keys=True) + "\n")
                kept += 1
                if rule.eligible(q) is None:
                    eligible += 1
            fh.flush()
            print("hour %d  +%d rows  eligible %d  total %d  quota %s" % (
                hour, len(items), eligible, len(seen),
                data.get("quota_remaining")), flush=True)
            if eligible >= rule.N_TARGET:
                break
            if data.get("backoff"):
                time.sleep(float(data["backoff"]) + 1)
            if data.get("quota_remaining", 1) <= 2:
                print("out of quota", flush=True)
                break
            time.sleep(PAUSE)

    print("%s: %d requests, %d new rows, %d eligible, %d truncated hours"
          % (cohort, requests, kept, eligible, truncated_hours))


if __name__ == "__main__":
    run(sys.argv[1])