#!/usr/bin/env python3
"""Fetch /robots.txt for the top 1,000 Tranco domains into a dated directory.

    python fetch.py [N=1000] [--out ~/data/robots-dead-rules/<date>]

The domain list is the same one `research/llms-txt/` used, so the population is
identical and the 2026-09-06 copy at ~/data/llms-txt/robots/ is a second draw
of it rather than a different study. One file per domain, `<domain>.txt`, plus
`meta.jsonl` with the status of every attempt — including the failures, which
have to be counted and not quietly dropped.
"""
import json
import os
import sys
from concurrent.futures import ThreadPoolExecutor, as_completed

import requests

LIST = os.path.expanduser("~/data/llms-txt/top-1m.csv")
UA = "Mozilla/5.0 (compatible; Vesper/1.0; +https://untilnextsession.com/agents)"
N = int(next((a for a in sys.argv[1:] if a.isdigit()), 1000))
OUT = os.path.expanduser(
    sys.argv[sys.argv.index("--out") + 1] if "--out" in sys.argv
    else "~/data/robots-dead-rules/fresh")


def domains(n):
    out = []
    with open(LIST, encoding="utf-8") as fh:
        for line in fh:
            rank, _, domain = line.strip().partition(",")
            if domain:
                out.append((int(rank), domain))
            if len(out) >= n:
                break
    return out


def one(rank, domain):
    row = {"rank": rank, "domain": domain}
    for scheme in ("https", "http"):
        try:
            r = requests.get(f"{scheme}://{domain}/robots.txt",
                             headers={"User-Agent": UA}, timeout=15,
                             allow_redirects=True)
            # requests defaults text/* to ISO-8859-1, which garbles UTF-8.
            if "charset" not in r.headers.get("content-type", "").lower():
                r.encoding = "utf-8"
            row.update(status=r.status_code, scheme=scheme,
                       final_url=r.url, bytes=len(r.content))
            if r.status_code == 200:
                # A robots.txt served as HTML is a soft 404, not a robots.txt.
                head = r.text[:400].lower()
                row["looks_html"] = "<html" in head or "<!doctype html" in head
                with open(os.path.join(OUT, domain + ".txt"), "w",
                          encoding="utf-8") as fh:
                    fh.write(r.text)
            return row
        except Exception as error:      # noqa: BLE001 - the failure is the datum
            row["error"] = f"{type(error).__name__}: {error}"[:200]
    return row


def main():
    os.makedirs(OUT, exist_ok=True)
    rows = []
    with ThreadPoolExecutor(max_workers=24) as pool:
        futures = [pool.submit(one, r, d) for r, d in domains(N)]
        for i, future in enumerate(as_completed(futures), 1):
            rows.append(future.result())
            if i % 100 == 0:
                print(f"  {i} done", flush=True)
    with open(os.path.join(OUT, "meta.jsonl"), "w", encoding="utf-8") as fh:
        for row in sorted(rows, key=lambda r: r["rank"]):
            fh.write(json.dumps(row) + "\n")
    ok = sum(1 for r in rows if r.get("status") == 200 and not r.get("looks_html"))
    print(f"{len(rows)} attempted, {ok} robots.txt bodies written to {OUT}")


if __name__ == "__main__":
    main()