"""Freeze the third claim token: one GET of each host's recorded best_url, title only.

Run ONCE, on 2026-09-22, AFTER rule.py was sealed (seal 16:18:31Z,
b93ce4f1b3dc1fc5c8325723bcb04583e4b5e65042d7bb917fe50b6b25fe0cf3). Writes
titles-2026-09-22.csv and must never be re-run against a later web.
"""
import csv, os, re, sys, urllib.request, urllib.error
from concurrent.futures import ThreadPoolExecutor

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import importlib.util
spec = importlib.util.spec_from_file_location("rule", os.path.join(HERE, "rule.py"))
rule = importlib.util.module_from_spec(spec); spec.loader.exec_module(rule)

OUT = os.path.join(HERE, "titles-2026-09-22.csv")
TITLE_RE = re.compile(rb"<title[^>]*>(.*?)</title>", re.I | re.S)


def fetch(row):
    host, url = row["host"], row["best_url"] or "https://%s/" % row["host"]
    req = urllib.request.Request(url, headers={"User-Agent": rule.UA})
    try:
        with urllib.request.urlopen(req, timeout=rule.TIMEOUT) as r:
            raw = r.read(rule.MAX_BYTES)
            status = r.status
    except urllib.error.HTTPError as e:
        return {"host": host, "title": "", "status": e.code, "bytes": 0}
    except Exception as e:
        return {"host": host, "title": "", "status": "", "bytes": 0,
                "error": type(e).__name__}
    m = TITLE_RE.search(raw)
    title = ""
    if m:
        title = re.sub(r"\s+", " ", m.group(1).decode("utf-8", "replace")).strip()[:300]
    return {"host": host, "title": title, "status": status, "bytes": len(raw)}


def main():
    if os.path.exists(OUT):
        print("%s exists; this file is written once and never edited." % OUT)
        return 1
    with open(os.path.join(os.path.dirname(HERE), "agent-site-change",
                           "before-2026-09-21.csv")) as f:
        rows = list(csv.DictReader(f))
    with ThreadPoolExecutor(max_workers=8) as ex:
        out = list(ex.map(fetch, rows))
    with open(OUT, "w", newline="") as f:
        w = csv.DictWriter(f, fieldnames=["host", "title", "status", "bytes"],
                           extrasaction="ignore")
        w.writeheader()
        for r in sorted(out, key=lambda r: r["host"]):
            w.writerow(r)
    got = sum(1 for r in out if r["title"])
    print("%d hosts, %d with a title (rule.MIN_TITLES = %d)"
          % (len(out), got, rule.MIN_TITLES))
    return 0


if __name__ == "__main__":
    sys.exit(main())