"""Sample Hacker News comments for the sealed rule. Keeps flags only, never comment text.
Usage: python fetch.py  (resumable). Writes ~/data/hn-em-dash/items.csv and bounds.json.
Source: https://hacker-news.firebaseio.com/v0/item/<id>.json, no key."""
import calendar, concurrent.futures as cf, csv, json, os, random, sys, threading, time
import requests
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import rule

OUT = os.path.expanduser("~/data/hn-em-dash")
API = "https://hacker-news.firebaseio.com/v0"
HEAD = ["id", "month", "kind", "dead", "chars"] + list(rule.FEATURES)
local = threading.local()

def get(i):
    s = getattr(local, "s", None) or requests.Session()
    local.s = s
    for attempt in range(6):
        try:
            r = s.get(f"{API}/item/{i}.json", timeout=30)
            if r.status_code == 200:
                return r.json()
        except requests.RequestException:
            pass
        time.sleep(2 ** attempt)
    return "failed"

def time_of(i, top):
    """The time of item i, walking forward past the empty ids."""
    for j in range(i, min(i + 60, top)):
        item = get(j)
        if isinstance(item, dict) and item.get("time"):
            return item["time"]
    return None

def first_id_at(ts, top):
    lo, hi = 1, top
    while hi - lo > 50:
        mid = (lo + hi) // 2
        t = time_of(mid, top)
        if t is None or t >= ts:
            hi = mid
        else:
            lo = mid
    return hi

def bounds(months):
    path = f"{OUT}/bounds.json"
    if os.path.exists(path):
        return json.load(open(path))
    top = requests.get(f"{API}/maxitem.json", timeout=30).json()
    edges = {}
    for m in months:
        y, mo = map(int, m.split("-"))
        edges[m] = first_id_at(calendar.timegm((y, mo, 1, 0, 0, 0)), top)
    y, mo = map(int, months[-1].split("-"))
    y, mo = (y + 1, 1) if mo == 12 else (y, mo + 1)
    edges["end"] = first_id_at(calendar.timegm((y, mo, 1, 0, 0, 0)), top)
    json.dump(edges, open(path, "w"))
    return edges

def row(i, m):
    item = get(i)
    if item == "failed":
        return [i, m, "failed", 0, 0] + [0] * 6
    if not item:
        return [i, m, "none", 0, 0] + [0] * 6
    kind = item.get("type", "none")
    if kind != "comment" or item.get("deleted") or not item.get("text"):
        return [i, m, "other", 0, 0] + [0] * 6
    f = rule.flags(item["text"])
    if f is None:
        return [i, m, "empty", 0, 0] + [0] * 6
    return [i, m, "comment", int(bool(item.get("dead"))), f["chars"]] + [f[k] for k in rule.FEATURES]

def main():
    os.makedirs(OUT, exist_ok=True)
    months = rule.month_range(rule.T0[0], rule.P[1])
    edges = bounds(months)
    path = f"{OUT}/items.csv"
    done = set()
    if os.path.exists(path):
        with open(path) as f:
            done = {int(r["id"]) for r in csv.DictReader(f)}
    jobs = []
    for k, m in enumerate(months):
        lo, hi = edges[m], edges[months[k + 1]] if k + 1 < len(months) else edges["end"]
        ids = random.Random(f"{m}-20261103").sample(range(lo, hi), 2000)
        jobs += [(i, m) for i in ids if i not in done]
    # The months that carry the verdict first; the months between are the reader's chart.
    windows = set(sum((rule.month_range(*w) for w in (rule.T0, rule.P0, rule.T, rule.P)), []))
    jobs.sort(key=lambda j: j[1] not in windows)
    new = not done
    with open(path, "a", newline="") as f, cf.ThreadPoolExecutor(8) as pool:
        w = csv.writer(f)
        if new:
            w.writerow(HEAD)
        for n, res in enumerate(pool.map(lambda j: row(*j), jobs)):
            w.writerow(res)
            if n % 500 == 0:
                f.flush()
                print(n, len(jobs), time.strftime("%H:%M:%S"), flush=True)
    print("done", len(jobs), time.strftime("%H:%M:%S"), flush=True)

if __name__ == "__main__":
    main()