"""Do these sites' llms.txt files link to paths their robots.txt forbids machines?

**Not pre-registered.** It came out of the control run, where two of the dead
links turned out to be forbidden by their own site's robots.txt, and the
question is obviously bigger than the dead ones: a list written for machines
that points at a path machines are told to stay out of is a contradiction
whether or not the link answers. Reported as a description, not as a tested
claim.

    python research/llms-txt-links/contradiction.py
"""

import json
import urllib.parse
from collections import defaultdict
from pathlib import Path

from control import DECLARED, NAMED_AGENTS, forbidden, groups   # noqa: E402
import urllib.request

OUT = Path.home() / "data" / "llms-txt-links"


def robots(domain):
    try:
        request = urllib.request.Request(f"https://{domain}/robots.txt",
                                         headers={"User-Agent": DECLARED})
        with urllib.request.urlopen(request, timeout=12) as answer:
            if answer.status != 200:
                return ""
            return answer.read(300_000).decode("utf-8", "replace")
    except Exception:
        return ""


def main():
    rows = [json.loads(l) for l in open(OUT / "links.jsonl") if l.strip()]
    by_domain = defaultdict(list)
    for row in rows:
        by_domain[row["domain"]].append(row)

    hits, domains_hit = 0, {}
    for domain, links in sorted(by_domain.items()):
        rules = groups(robots(domain))
        if not rules:
            continue
        for row in links:
            path = urllib.parse.urlsplit(row["url"]).path or "/"
            rule = forbidden(path, rules)
            if rule:
                hits += 1
                domains_hit.setdefault(domain, [0, rule])
                domains_hit[domain][0] += 1

    print(f"{len(rows)} links checked across {len(by_domain)} domains")
    print(f"{hits} of them are paths the site's own robots.txt forbids to a "
          f"named machine, across {len(domains_hit)} domains:")
    for domain, (n, rule) in sorted(domains_hit.items(), key=lambda kv: -kv[1][0]):
        print(f"  {domain:<22} {n:>3} links   {rule}")


if __name__ == "__main__":
    main()