"""The same study, off two different APIs, because I spent the first one.

WHY THIS FILE EXISTS, and it is not a good reason: the sealed rule names
OpenAlex. The first run of `fetch.py` used eight threads with no pause between
requests, and OpenAlex's anonymous quota is 1,000 requests a day. It answered
429 with `retry-after: 15953` — four and a half hours, past the end of this
session. The rule was sealed before any data; the data source had to change
after it, and that deviation is declared in the entry and here rather than
quietly absorbed.

What changes, and what it costs:

  * Citation dates come from OpenCitations (`/index/api/v1/citations/<doi>`),
    which returns every citing DOI with a `creation` date. That is finer than
    OpenAlex's year histogram, not coarser, and it carries `author_sc` — an
    author self-citation flag, which the sealed rule listed as a bias it could
    not see. It is reported, still not used in the verdict, because the rule
    did not say it would be.
  * Controls are drawn from Crossref by SAME JOURNAL (ISSN) and SAME
    PUBLICATION YEAR, instead of OpenAlex's same-field-and-year. This is a
    tighter match, not a looser one: a journal fixes the discipline, the
    audience and the citation culture at once.
  * The `cited_by_count:>4` enrichment is GONE. The sealed rule declared it as
    a bias that conditioned partly on the outcome window; drawing uniformly
    from a journal-year removes it. The cost is that more controls will have an
    empty pre-window and be dropped, which the results report.
  * Target sample cut from 400 to 150, for the clock and nothing else. No
    citation number had been computed when this was decided.

Everything else — the population filters, the windows, the strata, the two
controls, the kill rule — is unchanged from the seal.
"""

import csv, json, os, random, sys, threading, time
import urllib.parse, urllib.request
from concurrent.futures import ThreadPoolExecutor

MAIL = "vesper@untilnextsession.com"
OUT = os.path.expanduser("~/data/retraction-citations")
SEED = 20260920
TARGET = 150

import importlib.util as _ilu
_spec = _ilu.spec_from_file_location(
    "fetch1", os.path.join(os.path.dirname(os.path.abspath(__file__)), "fetch.py"))
_f1 = _ilu.module_from_spec(_spec)
_spec.loader.exec_module(_f1)
population = _f1.population          # the sealed filters, unchanged

_throttle = threading.Semaphore(4)
_pace, _last = threading.Lock(), [0.0]


def _turn():
    with _pace:
        gap = time.time() - _last[0]
        if gap < 0.12:
            time.sleep(0.12 - gap)
        _last[0] = time.time()


def get(url, tries=3):
    for attempt in range(tries):
        with _throttle:
            _turn()
            try:
                req = urllib.request.Request(
                    url, headers={"User-Agent": "vesper/1.0 (mailto:%s)" % MAIL})
                with urllib.request.urlopen(req, timeout=45) as r:
                    return json.loads(r.read().decode())
            except urllib.error.HTTPError as exc:
                if exc.code == 404:
                    return None
                if attempt == tries - 1:
                    return None
                time.sleep(2.0 * (attempt + 1))
            except Exception:
                if attempt == tries - 1:
                    return None
                time.sleep(2.0 * (attempt + 1))
    return None


def crossref_record(doi):
    """ISSN and published year for one DOI, or None if Crossref has neither."""
    data = get("https://api.crossref.org/works/%s?mailto=%s"
               % (urllib.parse.quote(doi, safe="/."), MAIL))
    msg = (data or {}).get("message")
    if not msg:
        return None
    issns = msg.get("ISSN") or []
    parts = ((msg.get("published") or {}).get("date-parts")
             or (msg.get("issued") or {}).get("date-parts") or [[None]])
    year = parts[0][0] if parts and parts[0] else None
    return {"issn": issns[0] if issns else None, "cr_year": year,
            "journal": (msg.get("container-title") or [None])[0]}


def controls_for(paper):
    """Three random articles from the same journal in the same year.

    Crossref's `sample` is a true random draw over the filtered set, so this
    conditions on nothing but venue and vintage.
    """
    if not paper.get("issn") or not paper.get("cr_year"):
        return []
    year = int(paper["cr_year"])
    url = ("https://api.crossref.org/works?filter=issn:%s,type:journal-article,"
           "from-pub-date:%d-01-01,until-pub-date:%d-12-31&sample=3"
           "&select=DOI,issued&mailto=%s" % (paper["issn"], year, year, MAIL))
    data = get(url)
    items = ((data or {}).get("message") or {}).get("items") or []
    out = []
    for it in items:
        doi = (it.get("DOI") or "").lower()
        if not doi or doi == paper["doi"]:
            continue
        out.append({"doi": doi, "for_record": paper["record_id"],
                    "oa_year": year, "retraction_year": paper["retraction_year"],
                    "paper_year": paper["paper_year"]})
    return out


def citations(doi):
    """[(citing doi, creation date, author self-citation flag)] or None."""
    data = get("https://opencitations.net/index/api/v1/citations/%s"
               % urllib.parse.quote(doi, safe="/."))
    if data is None:
        return None
    return [(c.get("citing"), c.get("creation") or "", c.get("author_sc") or "")
            for c in data]


def main():
    os.makedirs(OUT, exist_ok=True)
    pool = population()
    print("population passing the sealed filters:", len(pool), flush=True)
    random.Random(SEED).shuffle(pool)

    sample, cursor = [], 0
    with ThreadPoolExecutor(max_workers=4) as ex:
        while len(sample) < TARGET and cursor < len(pool):
            batch = pool[cursor:cursor + 60]
            cursor += 60
            for row, rec in zip(batch, ex.map(
                    lambda r: crossref_record(r["doi"]), batch)):
                if rec and rec["issn"] and rec["cr_year"]:
                    row.update(rec)
                    row["oa_year"] = row["paper_year"]
                    sample.append(row)
            print("  crossref: %d/%d resolved from %d asked"
                  % (len(sample), TARGET, cursor), flush=True)
    sample = sample[:TARGET]
    print("sample:", len(sample), flush=True)

    with ThreadPoolExecutor(max_workers=4) as ex:
        controls = [c for lst in ex.map(controls_for, sample) for c in lst]
    print("controls drawn:", len(controls), flush=True)

    works = ([(p["doi"], "retracted") for p in sample]
             + [(c["doi"], "control") for c in controls])
    with ThreadPoolExecutor(max_workers=4) as ex:
        cites = list(ex.map(lambda w: citations(w[0]), works))

    missing = 0
    with open(os.path.join(OUT, "cites.csv"), "w", newline="") as fh:
        w = csv.writer(fh)
        w.writerow(["doi", "arm", "cite_year", "count", "self_citations"])
        for (doi, arm), rows in zip(works, cites):
            if rows is None:
                missing += 1
                continue
            per_year, selfs = {}, {}
            for _citing, created, sc in rows:
                year = created[:4]
                if not year.isdigit():
                    continue
                per_year[int(year)] = per_year.get(int(year), 0) + 1
                if sc == "yes":
                    selfs[int(year)] = selfs.get(int(year), 0) + 1
            # A paper with no citations at all writes one zero row, so that a
            # real zero and a failed request stay distinguishable downstream.
            if not per_year:
                w.writerow([doi, arm, 0, 0, 0])
            for year, count in sorted(per_year.items()):
                w.writerow([doi, arm, year, count, selfs.get(year, 0)])
    print("citation lookups that failed:", missing, flush=True)

    with open(os.path.join(OUT, "sample.csv"), "w", newline="") as fh:
        w = csv.DictWriter(fh, fieldnames=list(sample[0].keys()))
        w.writeheader(); w.writerows(sample)
    with open(os.path.join(OUT, "controls.csv"), "w", newline="") as fh:
        w = csv.DictWriter(fh, fieldnames=list(controls[0].keys()))
        w.writeheader(); w.writerows(controls)
    print("written to", OUT, flush=True)


if __name__ == "__main__":
    main()