"""Probe every host in population.csv and control.csv, exactly as rule.py says.

One GET per host, 15 s, up to 5 redirects, https first and http only on a
transport-layer failure. Writes probe-<set>.csv with the verbatim reason for
every host that is not serving.

  python research/agent-site-mortality/probe.py population
  python research/agent-site-mortality/probe.py control
"""
import csv
import os
import socket
import sys
from concurrent.futures import ThreadPoolExecutor

import requests
from requests.adapters import HTTPAdapter

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import rule  # noqa: E402

OUT = os.path.dirname(os.path.abspath(__file__))
WORKERS = 12


def resolve(host):
    try:
        socket.getaddrinfo(host, None)
        return True, ""
    except socket.gaierror as e:
        return False, f"dns: {e.strerror or e}"
    except Exception as e:                                   # noqa: BLE001
        return False, f"dns: {type(e).__name__}: {e}"


def fetch(session, url):
    """(status, body_len, final_url, error). error is '' on an HTTP answer of
    any status; a non-empty error means the transport never delivered one."""
    try:
        r = session.get(url, timeout=rule.TIMEOUT_S, allow_redirects=True,
                        headers={"User-Agent": rule.USER_AGENT})
        return r.status_code, len(r.content), r.url, ""
    except requests.exceptions.SSLError as e:
        return None, 0, "", f"tls: {short(e)}"
    except requests.exceptions.ConnectTimeout:
        return None, 0, "", "timeout: connect"
    except requests.exceptions.ReadTimeout:
        return None, 0, "", "timeout: read"
    except requests.exceptions.TooManyRedirects:
        return None, 0, "", "redirect loop"
    except requests.exceptions.ConnectionError as e:
        return None, 0, "", f"connection: {short(e)}"
    except Exception as e:                                   # noqa: BLE001
        return None, 0, "", f"{type(e).__name__}: {short(e)}"


def short(e):
    s = " ".join(str(e).split())
    return s[:160]


def probe(host):
    row = {"host": host, "resolves": "", "scheme": "", "status": "",
           "body_bytes": "", "final_url": "", "serving": "", "reason": ""}
    ok, why = resolve(host)
    row["resolves"] = "yes" if ok else "no"
    if not ok:
        row["serving"] = "no"
        row["reason"] = why
        return row

    session = requests.Session()
    session.max_redirects = rule.MAX_REDIRECTS
    session.mount("https://", HTTPAdapter(max_retries=0))
    session.mount("http://", HTTPAdapter(max_retries=0))

    status, n, final, err = fetch(session, f"https://{host}/")
    scheme = "https"
    if err:                       # transport failure -> the rule allows one http try
        status2, n2, final2, err2 = fetch(session, f"http://{host}/")
        if not err2:
            status, n, final, err, scheme = status2, n2, final2, "", "http"
        else:
            err = f"https {err} / http {err2}"

    row["scheme"] = scheme
    row["final_url"] = final
    if err:
        row["serving"] = "no"
        row["reason"] = err
        return row
    row["status"] = status
    row["body_bytes"] = n
    if 200 <= status < 300 and n > 0:
        row["serving"] = "yes"
        if n < rule.THIN_BODY_BYTES:
            row["reason"] = f"thin body ({n} bytes)"
    else:
        row["serving"] = "no"
        row["reason"] = f"http {status}" + (" empty body" if n == 0 else "")
    return row


def main():
    which = sys.argv[1] if len(sys.argv) > 1 else "population"
    src = os.path.join(OUT, which + ".csv")
    with open(src) as fh:
        hosts = [r["host"] for r in csv.DictReader(fh)]
    print(f"{which}: probing {len(hosts)} hosts with {WORKERS} workers")

    with ThreadPoolExecutor(max_workers=WORKERS) as pool:
        results = list(pool.map(probe, hosts))

    dst = os.path.join(OUT, f"probe-{which}.csv")
    with open(dst, "w", newline="") as fh:
        w = csv.DictWriter(fh, fieldnames=list(results[0].keys()))
        w.writeheader()
        w.writerows(results)

    dead = [r for r in results if r["serving"] == "no"]
    print(f"  serving {len(results) - len(dead)}  not serving {len(dead)}  "
          f"({len(dead) / len(results):.1%}) -> {os.path.basename(dst)}")


if __name__ == "__main__":
    main()