"""CORRECTION PASS. The sealed probe asked every host for `/`, and an API that
correctly 404s at its root was counted dark.

Found by hand on api.chit402.com: `/` is 404, `/openapi.json` is 200 (60 KB) and
every `/receipt/<id>` its owner posted is 200. The host is alive. The sealed rule
says root-only and the sealed verdict stands on the sealed rule; this pass
measures how big the error is and which way it points, which is the only thing
that can honestly be done about it afterwards.

It is applied to EVERY dark host, not to the ones I noticed, so it cannot be a
search for a number. For each, it takes up to MAX_PATHS distinct URLs that
citizens actually published on 1f916 for that host and asks for them.

  python research/agent-site-mortality/paths.py
"""
import csv
import json
import os
import re
import sys
from collections import OrderedDict
from concurrent.futures import ThreadPoolExecutor

import requests

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import rule  # noqa: E402
from extract import URL_RE, clean, host_of  # noqa: E402

HERE = os.path.dirname(os.path.abspath(__file__))
RAW = os.path.expanduser("~/data/1f916")
MAX_PATHS = 4


def published_urls(hosts):
    want = set(hosts)
    found = {h: OrderedDict() for h in hosts}
    for kind in ("posts", "comments"):
        with open(os.path.join(RAW, kind + ".jsonl")) as fh:
            for line in fh:
                if not line.strip():
                    continue
                body = json.loads(line).get("body") or ""
                if "http" not in body:
                    continue
                for raw in URL_RE.findall(body):
                    u = clean(raw)
                    h = host_of(u)
                    if h in want and re.sub(r"https?://[^/]+", "", u).strip("/"):
                        found[h][u] = None          # dedupe, keep order
    return found


def ask(url):
    try:
        r = requests.get(url, timeout=rule.TIMEOUT_S, allow_redirects=True,
                         headers={"User-Agent": rule.USER_AGENT})
        return r.status_code, len(r.content)
    except Exception as e:                               # noqa: BLE001
        return None, " ".join(str(e).split())[:80]


def main():
    probe = list(csv.DictReader(open(os.path.join(HERE, "probe-population.csv"))))
    dark = [r for r in probe if r["serving"] == "no"]
    urls = published_urls([r["host"] for r in dark])

    jobs = []
    for r in dark:
        for u in list(urls[r["host"]])[:MAX_PATHS]:
            jobs.append((r["host"], u))

    print(f"{len(dark)} dark hosts, {len(jobs)} published paths to try "
          f"(hosts with no published path beyond `/` cannot be rescued this way)\n")

    with ThreadPoolExecutor(max_workers=8) as pool:
        results = list(pool.map(lambda j: (j[0], j[1]) + ask(j[1]), jobs))

    alive = {}
    for host, url, status, extra in results:
        if status and 200 <= status < 300 and isinstance(extra, int) and extra > 0:
            alive.setdefault(host, []).append((url, status, extra))

    with open(os.path.join(HERE, "paths.csv"), "w", newline="") as fh:
        w = csv.writer(fh)
        w.writerow(["host", "url", "status", "bytes_or_error"])
        for row in results:
            w.writerow(row)

    print(f"dark hosts with at least one published path that serves: "
          f"**{len(alive)}** of {len(dark)}\n")
    for h, v in sorted(alive.items()):
        u, s, n = v[0]
        print(f"  {h:<42} {len(v)} of {min(MAX_PATHS, len(urls[h]))} paths serve"
              f"   e.g. {u[:64]} -> {s} {n}b")

    n = len(probe)
    d0 = len(dark)
    d1 = d0 - len(alive)
    print(f"\nsealed:            {d0} of {n} dark = {d0/n:.1%}")
    print(f"root-404 rescued:  {d1} of {n} dark = {d1/n:.1%}  "
          f"(NOT the sealed verdict; the sealed rule is root-only)")
    print(f"\n-> {os.path.join(HERE, 'paths.csv')}")


if __name__ == "__main__":
    main()