"""Draw the sample and follow every chain under rule.py.

    python fetch.py probe <title>...   first link of named pages, no chain
    python fetch.py run                draw 500 starts, follow the chains

Writes, in this directory: `starts.csv` (the sample, in draw order) and
`steps.csv` (every page fetched: asked title, resolved title, revision id,
next title under the rule). analyse.py reads only those two files, so the
tables reproduce from the repository without the network. The raw parser HTML
is cached in ~/data/getting-to-philosophy/ and is not needed to reproduce.
"""

import csv
import gzip
import hashlib
import json
import os
import sys
import time
from concurrent.futures import ThreadPoolExecutor
from urllib.parse import urlencode

import requests

import rule

HERE = os.path.dirname(os.path.abspath(__file__))
CACHE = os.path.expanduser("~/data/getting-to-philosophy/html")
API = "https://en.wikipedia.org/w/api.php"
UA = "VesperResearch/1.0 (https://untilnextsession.com; vesper@untilnextsession.com) getting-to-philosophy"
S = requests.Session()
S.headers["User-Agent"] = UA


def api(params):
    params = dict(params, format="json", formatversion="2")
    for attempt in range(6):
        try:
            r = S.get(API, params=params, timeout=60)
            if r.status_code == 200:
                return r.json()
        except requests.RequestException:
            pass
        time.sleep(2 ** attempt)
    raise RuntimeError(f"api failed: {params}")


def page(title):
    """(resolved title or None, revid, next title or None) for one asked title."""
    j = api({"action": "parse", "page": title, "redirects": "1",
             "prop": "text|revid", "disabletoc": "1"})
    if "error" in j:
        return None, "", None
    p = j["parse"]
    resolved = p["title"]
    key = hashlib.sha1(resolved.encode()).hexdigest()
    os.makedirs(CACHE, exist_ok=True)
    with gzip.open(os.path.join(CACHE, key + ".html.gz"), "wt") as f:
        f.write(p["text"])
    return resolved, p.get("revid", ""), rule.first_link(p["text"], resolved)


def probe(titles):
    for t in titles:
        print(t, "->", page(t))


def draw(n):
    starts, seen = [], set()
    while len(starts) < n:
        j = api({"action": "query", "list": "random", "rnnamespace": "0",
                 "rnfilterredir": "nonredirects", "rnlimit": "50"})
        for r in j["query"]["random"]:
            if r["title"] not in seen and len(starts) < n:
                seen.add(r["title"])
                starts.append(r["title"])
    return starts


def run():
    starts = draw(rule.N)
    with open(os.path.join(HERE, "starts.csv"), "w", newline="") as f:
        w = csv.writer(f)
        w.writerow(["start"])
        w.writerows([s] for s in starts)
    steps = {}  # asked title -> (resolved, revid, next)

    def ensure(titles):
        todo = [t for t in set(titles) if t and t not in steps]
        with ThreadPoolExecutor(3) as ex:
            for t, res in zip(todo, ex.map(page, todo)):
                steps[t] = res

    # Advance all chains a layer at a time, fetching each new title once.
    frontier = list(starts)
    rounds = 0
    while frontier and rounds < rule.CAP + 1:
        ensure(frontier)
        nxt = set()
        for t in frontier:
            resolved, _, link = steps[t]
            if resolved and resolved != "Philosophy" and link:
                nxt.add(link)
        frontier = [t for t in nxt if t not in steps]
        rounds += 1
        print(f"round {rounds}: {len(steps)} pages, {len(frontier)} new", flush=True)
    with open(os.path.join(HERE, "steps.csv"), "w", newline="") as f:
        w = csv.writer(f)
        w.writerow(["asked", "resolved", "revid", "next"])
        for t in sorted(steps):
            r, rev, nx = steps[t]
            w.writerow([t, r or "", rev, nx or ""])


if __name__ == "__main__":
    if sys.argv[1] == "probe":
        probe(sys.argv[2:])
    elif sys.argv[1] == "run":
        run()