#!/usr/bin/env python3
"""Follow-up to probe.py: every AI-bot token in the agentswelcome.dev crawler registry,
grouped by what the bot is for, on the robots.txt of the Tranco top N.

Question (written 2026-09-07 before the run): do sites that block AI *training*
crawlers also block the *user-triggered* fetchers (ChatGPT-User, Claude-User,
Perplexity-User, ...) that fetch a page because a person asked a question about it?
Claim under test, heard often: "sites block training bots but let the answer engines
through". Kill rule: among domains that fully block at least one training token, if
fewer than half also fully block at least one user-fetch token, the claim stands;
if more than half do, it is dead; in between, "partly".

Usage: python robots_purpose.py [N=1000] [--summarise-only]
Bodies: ~/data/llms-txt/robots/<domain>.txt (fetched once, resumable; re-downloadable).
Registry: crawler-registry.json next to this script (trimmed copy of
https://agentswelcome.dev/api/crawlers, updated 2026-07-06). Output: purpose.md here.
"""
import json
import os
import sys
from collections import Counter
from concurrent.futures import ThreadPoolExecutor, as_completed

from probe import DATA, HERE, get, parse_robots, blocked  # same definitions as the first probe

ROBOTS = os.path.join(DATA, "robots")
N = int(next((a for a in sys.argv[1:] if a.isdigit()), 1000))
SUMMARISE_ONLY = "--summarise-only" in sys.argv


def registry():
    recs = json.load(open(os.path.join(HERE, "crawler-registry.json")))["records"]
    return [r for r in recs if r["token"] and not r["token"].startswith("(")]


def fetch(rank, domain):
    path = os.path.join(ROBOTS, domain + ".txt")
    if os.path.exists(path):
        return rank, domain, "cached"
    status, _, body, err = get(f"https://{domain}/robots.txt")
    with open(path, "w") as f:
        f.write(json.dumps({"status": status, "err": err}) + "\n" + body)
    return rank, domain, status or err


def load(domain):
    path = os.path.join(ROBOTS, domain + ".txt")
    if not os.path.exists(path):
        return None, ""
    head, _, body = open(path).read().partition("\n")
    meta = json.loads(head)
    if meta["status"] != 200 or "<html" in body[:500].lower():
        return meta["status"] or meta["err"], ""
    return 200, body


def main():
    os.makedirs(ROBOTS, exist_ok=True)
    domains = []
    with open(os.path.join(DATA, "top-1m.csv")) as f:
        for ln in f:
            rank, dom = ln.strip().split(",", 1)
            if int(rank) > N:
                break
            domains.append((int(rank), dom))
    if not SUMMARISE_ONLY:
        with ThreadPoolExecutor(max_workers=32) as ex:
            futs = [ex.submit(fetch, r, d) for r, d in domains]
            for i, fut in enumerate(as_completed(futs), 1):
                fut.result()
                if i % 100 == 0:
                    print(f"{i}/{len(domains)} fetched", flush=True)
    summarise(domains)


def summarise(domains):
    bots = registry()
    by_purpose = {}
    for b in bots:
        by_purpose.setdefault(b["purpose"], []).append(b["token"])
    rows, n_robots = [], 0
    for rank, dom in domains:
        status, body = load(dom)
        if status != 200:
            rows.append({"rank": rank, "domain": dom, "robots": status, "rules": {}})
            continue
        n_robots += 1
        groups = parse_robots(body)
        rules = {b["token"]: blocked(groups, b["token"]) for b in bots}
        rules["*"] = blocked(groups, "*")
        rows.append({"rank": rank, "domain": dom, "robots": 200, "rules": rules})
    with_robots = [r for r in rows if r["robots"] == 200]

    def blocks_any(r, tokens):
        return any(r["rules"].get(t) == "blocked" for t in tokens)

    lines = [f"# AI-bot tokens by purpose on the Tranco top {N}", "",
             f"Run {os.popen('date -u +%Y-%m-%d').read().strip()} from the server. {len(rows)} domains, "
             f"{n_robots} with a parseable robots.txt. Tokens and purposes from crawler-registry.json "
             f"({len(bots)} tokens). Definitions of blocked/restricted as in README.md.", "",
             "## Per token", "", "| token | operator | purpose | named | fully blocked | restricted |", "|---|---|---|---|---|---|"]
    named = Counter()
    for b in bots:
        t = b["token"]
        nm = sum(r["rules"].get(t, "absent") != "absent" for r in with_robots)
        bl = sum(r["rules"].get(t) == "blocked" for r in with_robots)
        rs = sum(r["rules"].get(t) == "restricted" for r in with_robots)
        named[t] = nm
        lines.append(f"| {t} | {b['operator']} | {b['purpose']} | {nm} | {bl} ({bl / max(n_robots, 1):.1%}) | {rs} |")
    lines += ["", "## Per purpose (a domain counts once if it fully blocks at least one token of that purpose)", "",
              "| purpose | tokens | domains blocking ≥1 | share of robots.txt |", "|---|---|---|---|"]
    for p, toks in sorted(by_purpose.items()):
        c = sum(blocks_any(r, toks) for r in with_robots)
        lines.append(f"| {p} | {len(toks)} | {c} | {c / max(n_robots, 1):.1%} |")
    training = by_purpose.get("training", [])
    user = by_purpose.get("inference", [])  # the registry's name for user-triggered fetchers
    tb = [r for r in with_robots if blocks_any(r, training)]
    both = [r for r in tb if blocks_any(r, user)]
    ub = [r for r in with_robots if blocks_any(r, user)]
    lines += ["", "## The question", "",
              f"- Training tokens: {', '.join(training)}.",
              f"- User-fetch tokens (registry purpose 'inference'): {', '.join(user)}.",
              f"- Domains fully blocking at least one training token: **{len(tb)}** ({len(tb) / max(n_robots, 1):.1%} of robots.txt).",
              f"- Of those, also fully blocking at least one user-fetch token: **{len(both)}** ({len(both) / max(len(tb), 1):.1%}).",
              f"- Domains blocking a user-fetch token without blocking any training token: {len(ub) - len(both)}.",
              "- Kill rule (README of this script): under 50% → claim stands; over 50% → dead; else partly.", "",
              "## Domains that block a training token and a user-fetch token", "", "| rank | domain | blocked tokens |", "|---|---|---|"]
    for r in both:
        bl = [t for t, v in r["rules"].items() if v == "blocked" and t != "*"]
        lines.append(f"| {r['rank']} | {r['domain']} | {', '.join(bl)} |")
    open(os.path.join(HERE, "purpose.md"), "w").write("\n".join(lines) + "\n")
    print("\n".join(lines[:12]))
    print("...")
    print("\n".join(lines[lines.index("## The question"):lines.index("## The question") + 8]))


if __name__ == "__main__":
    main()