"""The measurement for research/paper-code-links/. The rule is rule.py, sealed first.

    python research/paper-code-links/measure.py [--target 200] [--out .]

Writes repos.csv (one row per distinct repository) and results.txt (the verdict
arithmetic). Fetches the arXiv API and then github.com, serialised, politely.
Nothing here may change the rule: if something in the frame turns out to be
unworkable, that is a finding to write down, not a threshold to move.
"""

import argparse
import csv
import os
import re
import sys
import time
import urllib.error
import urllib.parse
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import rule  # noqa: E402  the sealed pre-registration; thresholds come from it

AGENT = "vesper-paper-code-links/1.0 (+https://untilnextsession.com)"
ARXIV = "http://export.arxiv.org/api/query"
CATS = "(cat:cs.LG OR cat:cs.CL OR cat:cs.CV)"
COHORTS = {
    "old": ("201801010000", "201812312359"),
    "recent": ("202506010000", "202512312359"),
}
# A github URL naming an owner and a repository, anywhere in the abstract.
REPO = re.compile(r"https?://(?:www\.)?github\.com/([A-Za-z0-9][\w.-]*)/([\w.-]+)", re.I)
# Words that are never a repository name, only a GitHub route.
NOT_A_REPO = {"about", "features", "pricing", "topics", "collections", "orgs",
              "sponsors", "settings", "explore", "search", "login", "join"}


def get(url, timeout=30, retries=2):
    """(final status, final url). A network failure after the retries is None."""
    for attempt in range(retries + 1):
        try:
            req = urllib.request.Request(url, headers={"User-Agent": AGENT})
            with urllib.request.urlopen(req, timeout=timeout) as answer:
                return answer.status, answer.geturl()
        except urllib.error.HTTPError as err:
            return err.code, url
        except Exception:
            if attempt == retries:
                return None, url
            time.sleep(2.0)
    return None, url


def normalise(owner, repo):
    """https://github.com/<owner>/<repo>, trailing junk removed, lowercased key."""
    repo = repo.rstrip(".,;:)]}\"'")
    if repo.lower().endswith(".git"):
        repo = repo[:-4]
    if not repo or owner.lower() in NOT_A_REPO:
        return None
    return f"{owner}/{repo}", f"https://github.com/{owner}/{repo}"


def abstracts(cohort, target, pause=3.0, page=100, cap=60):
    """Distinct repositories named in a cohort's abstracts, newest first."""
    lo, hi = COHORTS[cohort]
    query = f"{CATS} AND submittedDate:[{lo} TO {hi}]"
    found, papers = {}, 0
    for start in range(0, cap * page, page):
        url = (f"{ARXIV}?search_query={urllib.parse.quote(query)}"
               f"&start={start}&max_results={page}"
               f"&sortBy=submittedDate&sortOrder=descending")
        try:
            req = urllib.request.Request(url, headers={"User-Agent": AGENT})
            with urllib.request.urlopen(req, timeout=60) as answer:
                body = answer.read().decode("utf-8", "replace")
        except Exception as exc:
            print(f"  arXiv page at {start} failed: {exc}")
            time.sleep(pause)
            continue
        entries = body.split("<entry>")[1:]
        if not entries:
            print(f"  arXiv returned no entries at start={start}; window exhausted")
            break
        papers += len(entries)
        for entry in entries:
            summary = entry.split("<summary>")[-1].split("</summary>")[0]
            ident = entry.split("<id>")[-1].split("</id>")[0].strip()
            for owner, repo in REPO.findall(summary):
                pair = normalise(owner, repo)
                if pair and pair[0].lower() not in found:
                    found[pair[0].lower()] = (pair[1], ident)
        print(f"  {cohort}: {papers} abstracts, {len(found)} distinct repos")
        if len(found) >= target:
            break
        time.sleep(pause)
    return list(found.items())[:target], papers


def classify(status):
    if status == 200:
        return "ALIVE"
    if status == 404:
        return "GONE"
    if status in (403, 429, 451) or status is None:
        return "REFUSED"
    return "OTHER"


def run(cohort, target, pause):
    print(f"\n== {cohort} cohort ==")
    repos, papers = abstracts(cohort, target, pause=3.0)
    print(f"  collected {len(repos)} distinct repos from {papers} abstracts")
    rows = []
    for key, (url, paper) in repos:
        status, final = get(url)
        rows.append({"cohort": cohort, "repo": key, "url": url, "paper": paper,
                     "status": "" if status is None else status,
                     "final": final, "class": classify(status)})
        time.sleep(pause)
    counts = {}
    for row in rows:
        counts[row["class"]] = counts.get(row["class"], 0) + 1
    n = sum(v for k, v in counts.items() if k != "REFUSED")
    k = counts.get("ALIVE", 0)
    print(f"  {counts}  ->  n={n} k={k} " + (f"rate={k/n:.4f}" if n else "rate=n/a"))
    return rows, counts, n, k, papers


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--target", type=int, default=200)
    ap.add_argument("--pause", type=float, default=0.4)
    ap.add_argument("--out", default=HERE)
    args = ap.parse_args()

    all_rows, report = [], []
    stats = {}
    for cohort in ("recent", "old"):   # the gate first, deliberately
        rows, counts, n, k, papers = run(cohort, args.target, args.pause)
        all_rows += rows
        stats[cohort] = (counts, n, k, papers)

    with open(os.path.join(args.out, "repos.csv"), "w", newline="") as f:
        writer = csv.DictWriter(f, fieldnames=["cohort", "repo", "url", "paper",
                                               "status", "final", "class"])
        writer.writeheader()
        writer.writerows(all_rows)

    rc, rn, rk, rpapers = stats["recent"]
    oc, on, ok, opapers = stats["old"]
    report.append("research/paper-code-links — results")
    report.append(f"rule.py sha256 is sealed as rule.paper-code-links at 1f916.ai\n")
    for name, (counts, n, k, papers) in (("recent (gate)", stats["recent"]),
                                         ("old (under test)", stats["old"])):
        rate = f"{k/n:.4f}" if n else "n/a"
        report.append(f"{name}: {papers} abstracts, {sum(counts.values())} distinct repos")
        report.append(f"  {counts}")
        report.append(f"  n={n} (REFUSED excluded) k={k} alive-rate={rate}")

    gate_ok = rn >= rule.GATE_N and rn and (rk / rn) >= rule.GATE_RATE
    report.append(f"\nGATE: recent n>={rule.GATE_N} and rate>={rule.GATE_RATE} -> "
                  + ("PASS" if gate_ok else "FAIL"))
    if not gate_ok:
        report.append("VOID: the gate failed, so no verdict is published for the")
        report.append("old cohort. The instrument could not show a thing known to")
        report.append("be true, so it cannot be trusted about a thing that is not.")
    else:
        report.append(f"VERDICT: {rule.verdict(ok, on).upper()} "
                      f"(k/n = {ok}/{on} = {ok/on:.4f} if on else n/a)"
                      .replace(" if on else n/a", ""))
        report.append(f"  survived at >= {rule.SURVIVE_AT}, killed under "
                      f"{rule.KILL_UNDER}, inconclusive between or under n={rule.MIN_N}")
        if on and rn:
            report.append(f"  descriptive only: old {ok/on:.4f} vs recent "
                          f"{rk/rn:.4f}, difference {ok/on - rk/rn:+.4f}")
    text = "\n".join(report)
    print("\n" + text)
    with open(os.path.join(args.out, "results.txt"), "w") as f:
        f.write(text + "\n")


if __name__ == "__main__":
    main()