#!/usr/bin/env python3
"""Sitemaps on the same readable domains as probe.py: the machine surface written for
search engines, beside the feed written for readers. Uses the robots.txt bodies cached by
research/llms-txt/probe.py (~/data/llms-txt/robots/<domain>.txt, first line is JSON meta)
for `Sitemap:` lines, and one GET of /sitemap.xml for the rest. Writes sitemaps.csv here."""
import csv, json, os, re, sys, xml.etree.ElementTree as ET
from concurrent.futures import ThreadPoolExecutor
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import probe

ROBOTS = os.path.expanduser("~/data/llms-txt/robots")
rows = [json.loads(l) for l in open(probe.RAW)]
readable = [r for r in rows if r["status"] == 200 and r["html"]]

def robots_sitemaps(domain):
    p = os.path.join(ROBOTS, f"{domain}.txt")
    if not os.path.exists(p):
        return None
    lines = open(p, encoding="utf-8", errors="replace").read().split("\n", 1)
    meta = json.loads(lines[0]) if lines[0].startswith("{") else {}
    if meta.get("status") != 200:
        return None
    body = lines[1] if len(lines) > 1 else ""
    return [l.split(":", 1)[1].strip() for l in body.splitlines() if l.lower().startswith("sitemap:")]

def sitemap_xml(domain, final):
    from urllib.parse import urljoin
    status, ctype, body, url, err = probe.get(urljoin(final, "/sitemap.xml"), timeout=15, limit=2_000_000)
    if err or status != 200:
        return "no", status or err
    try:
        root = ET.fromstring(body)
    except Exception:
        return "no", "not xml"
    local = root.tag.split("}")[-1].lower()
    return ("index" if local == "sitemapindex" else "urlset" if local == "urlset" else "no"), local

def one(r):
    rs = robots_sitemaps(r["domain"])
    kind, note = sitemap_xml(r["domain"], r["final"])
    return {"rank": r["rank"], "domain": r["domain"], "robots_sitemap_lines": len(rs) if rs else 0,
            "robots_has_sitemap": int(bool(rs)), "sitemap_xml": kind,
            "has_sitemap": int(bool(rs) or kind != "no"),
            "advertises_feed": int(bool(r["advertised"])),
            "feed_valid": int(any(c["kind"] != "none" for c in r["checked"])),
            "hidden_feed": int(bool(r["hidden"]))}

with ThreadPoolExecutor(max_workers=32) as ex:
    out = sorted(ex.map(one, readable), key=lambda x: x["rank"])
with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "sitemaps.csv"), "w", newline="") as f:
    w = csv.DictWriter(f, fieldnames=list(out[0].keys())); w.writeheader(); w.writerows(out)
n = len(out)
s = sum(o["has_sitemap"] for o in out); rb = sum(o["robots_has_sitemap"] for o in out)
sx = sum(o["sitemap_xml"] != "no" for o in out)
both = sum(o["has_sitemap"] and (o["feed_valid"] or o["hidden_feed"]) for o in out)
feed_any = sum(o["feed_valid"] or o["hidden_feed"] for o in out)
print(f"readable {n}; has a sitemap {s} ({100*s/n:.1f} %): robots Sitemap line {rb}, /sitemap.xml parses {sx}")
print(f"feed (advertised valid or hidden) {feed_any} ({100*feed_any/n:.1f} %); both {both}; feed without sitemap {feed_any-both}; sitemap without feed {s-both}")