"""Screen the frame for agent storefronts. Sealed before it was run.

    python research/agent-storefronts/screen.py > research/agent-storefronts/screen.log

Frame: every host in research/agent-sites/hosts-from-1f916.csv (committed
2026-09-07), plus every https host named in memory/agents-met.md as of commit
c0d014f, minus the platform list below. For each host, GET https://<host>/ and
https://<host>/llms.txt (15 s, 400 KB cap, redirects followed) and flag:

  ai_self  the text says the seller is an AI agent (AI_SELF below)
  price    the text states a price (PRICE below)

A host flagged on both is a *candidate*. Candidates are confirmed by hand in
storefronts.csv against the tests written in README.md; the screen decides
nothing on its own. Output: screen.csv, one row per host.
"""
import csv
import re
import subprocess
import sys
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

import requests

HERE = Path(__file__).resolve().parent
ROOT = HERE.parent.parent
UA = "Vesper-storefront-census (https://untilnextsession.com; one GET per page)"
CAP = 400_000

PLATFORMS = ("github", "githubusercontent", "1f916", "x.com", "twitter", "arxiv",
             "wikipedia", "google", "youtube", "npmjs", "pypi", "w3.org", "example",
             "basescan", "etherscan", "huggingface", "openai.com", "anthropic.com",
             "reddit", "medium.com", "doi.org", "nature.com", "bbc.", "stripe.com",
             "base.org", "clinicaltrials.gov", "weather.gov", "cloudflare", "vercel.com",
             "localhost", "127.0.0.1")

AI_SELF = re.compile(
    r"\b(i am|i'm|i’m)\s+an?\s+(autonomous\s+)?(ai|artificial intelligence|llm|language model)\b"
    r"|\b(is|am)\s+an?\s+autonomous\s+(ai\s+)?agent\b"
    r"|\bautonomous\s+ai\s+agent\b"
    r"|\brun by an ai\b|\bwritten by an ai\b|\boperated by an ai\b",
    re.I)
PRICE = re.compile(
    r"[$£€]\s?\d+(?:[.,]\d+)?"
    r"|\b\d+(?:\.\d+)?\s?(?:usdc|usd|sol|eth|sats|dollars)\b",
    re.I)


def frame():
    hosts = set()
    with open(ROOT / "research/agent-sites/hosts-from-1f916.csv") as f:
        for row in csv.DictReader(f):
            hosts.add(row["host"].strip().lower())
    met = subprocess.run(["git", "-C", str(ROOT), "show", "c0d014f:memory/agents-met.md"],
                         capture_output=True, text=True, check=True).stdout
    for m in re.finditer(r"https?://([a-z0-9.-]+\.[a-z]{2,})", met, re.I):
        hosts.add(m.group(1).lower())
    for m in re.finditer(r"\*\*([a-z0-9-]+\.(?:com|org|net|uk|me|dev|xyz|to|ai|town|city|rest|io|app))\b", met, re.I):
        hosts.add(m.group(1).lower())
    keep = []
    for h in sorted(hosts):
        h = h.strip().strip(".").removeprefix("www.")
        if not h or re.match(r"^\d+\.\d+", h) or any(p in h for p in PLATFORMS):
            continue
        keep.append(h)
    return sorted(set(keep))


def get(url):
    try:
        r = requests.get(url, headers={"User-Agent": UA}, timeout=15, stream=True, allow_redirects=True)
        body = r.raw.read(CAP, decode_content=True).decode("utf-8", "replace")
        return r.status_code, body
    except Exception as e:  # noqa: BLE001 — a screen records the failure, it does not stop
        return None, type(e).__name__


def text_of(html):
    html = re.sub(r"<script.*?</script>|<style.*?</style>", " ", html, flags=re.S | re.I)
    return re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", html))


def screen(host):
    s1, home = get(f"https://{host}/")
    s2, llms = get(f"https://{host}/llms.txt")
    if s2 != 200 or "<html" in (llms or "")[:400].lower():
        llms = ""
    text = text_of(home if s1 == 200 else "") + " " + llms
    ai = AI_SELF.search(text)
    price = PRICE.search(text)
    return {"host": host, "home_status": s1, "llms_status": s2,
            "ai_self": int(bool(ai)), "price": int(bool(price)),
            "candidate": int(bool(ai and price)),
            "ai_evidence": text[max(0, ai.start() - 60): ai.end() + 60].strip() if ai else "",
            "price_evidence": text[max(0, price.start() - 60): price.end() + 40].strip() if price else ""}


def main():
    hosts = frame()
    print(f"frame: {len(hosts)} hosts", file=sys.stderr)
    with ThreadPoolExecutor(max_workers=8) as pool:
        rows = list(pool.map(screen, hosts))
    with open(HERE / "screen.csv", "w", newline="") as f:
        w = csv.DictWriter(f, fieldnames=list(rows[0].keys()))
        w.writeheader()
        w.writerows(rows)
    reach = sum(1 for r in rows if r["home_status"] == 200)
    print(f"hosts {len(rows)}, home 200: {reach}, ai_self {sum(r['ai_self'] for r in rows)}, "
          f"price {sum(r['price'] for r in rows)}, candidates {sum(r['candidate'] for r in rows)}")
    for r in rows:
        if r["candidate"]:
            print(r["host"], "|", r["ai_evidence"][:120], "|", r["price_evidence"][:80])


if __name__ == "__main__":
    main()