"""Fetch the sealed sample of PyPI projects and write one row per release.

    python fetch.py            # resumes: projects already in the csv are skipped

Reads `sample.txt` (the project names the sealed rule drew) and asks
https://pypi.org/pypi/<project>/json for each. Writes `~/data/friday-releases-replication/
releases.csv` with one row per distinct version:

    project,version,first_upload_iso,n_files,yanked

`first_upload_iso` is the earliest `upload_time_iso_8601` among the version's
files, which is the moment the release began (a wheel matrix takes minutes).
`yanked` is true when every file of the version is yanked.

Pre-releases are NOT filtered here — the filter belongs to the analysis, so
that the count dropped at each step can be published. Versions with no files
are written with n_files = 0 and no date; they are projects' deleted releases.
"""

import csv
import urllib.parse
import json
import os
import pathlib
import sys
import threading
import time
import urllib.error
import urllib.request
from concurrent.futures import ThreadPoolExecutor

HERE = pathlib.Path(__file__).resolve().parent
OUT = pathlib.Path(os.path.expanduser("~/data/friday-releases-replication"))
OUT.mkdir(parents=True, exist_ok=True)
CSV = OUT / "releases.csv"
LOG = OUT / "fetch-log.csv"
UA = "vesper (https://untilnextsession.com; research on release weekdays)"
WORKERS = 8

lock = threading.Lock()


def get(project):
    url = f"https://pypi.org/pypi/{urllib.parse.quote(project)}/json"
    req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "application/json"})
    for attempt in range(3):
        try:
            with urllib.request.urlopen(req, timeout=30) as r:
                return r.status, json.load(r)
        except urllib.error.HTTPError as e:
            if e.code in (403, 404, 410):
                return e.code, None
            time.sleep(2 + 3 * attempt)
        except Exception:
            time.sleep(2 + 3 * attempt)
    return 0, None


def rows_for(project, doc):
    out = []
    for version, files in (doc.get("releases") or {}).items():
        if not files:
            out.append((project, version, "", 0, ""))
            continue
        times = sorted(f["upload_time_iso_8601"] for f in files if f.get("upload_time_iso_8601"))
        yanked = all(bool(f.get("yanked")) for f in files)
        out.append((project, version, times[0] if times else "", len(files), "1" if yanked else "0"))
    return out


def main():
    names = [n.strip() for n in (HERE / "sample.txt").read_text().split() if n.strip()]
    done = set()
    if CSV.exists():
        with CSV.open() as fh:
            for row in csv.reader(fh):
                if row and row[0] != "project":
                    done.add(row[0])
    todo = [n for n in names if n not in done]
    print(f"{len(names)} sampled, {len(done)} already fetched, {len(todo)} to go", flush=True)

    new_csv = not CSV.exists()
    fh = CSV.open("a", newline="")
    lf = LOG.open("a", newline="")
    w, wl = csv.writer(fh), csv.writer(lf)
    if new_csv:
        w.writerow(["project", "version", "first_upload_iso", "n_files", "yanked"])
        wl.writerow(["project", "status", "n_versions"])

    n_ok = n_bad = 0

    def work(project):
        nonlocal n_ok, n_bad
        status, doc = get(project)
        rows = rows_for(project, doc) if doc else []
        with lock:
            if doc is None:
                n_bad += 1
            else:
                n_ok += 1
            wl.writerow([project, status, len(rows)])
            w.writerows(rows)
            if (n_ok + n_bad) % 100 == 0:
                print(f"  {n_ok + n_bad}/{len(todo)}  ok={n_ok} failed={n_bad}", flush=True)
                fh.flush()
                lf.flush()

    with ThreadPoolExecutor(max_workers=WORKERS) as pool:
        list(pool.map(work, todo))

    fh.close()
    lf.close()
    print(f"done: ok={n_ok} failed={n_bad}  -> {CSV}", flush=True)


if __name__ == "__main__":
    import urllib.parse  # noqa: E402  (used inside get)

    sys.exit(main())