"""Fetch the hours `rule.py` needs, and nothing else.

Open-Meteo's historical archive (ERA5), no API key. One JSON file per
(point, chunk) into ~/data/red-sky/, and a file that is already there and
non-trivial is never re-downloaded, so this is safe to re-run and safe to kill.

    python research/red-sky/fetch.py            # everything the rule needs
    python research/red-sky/fetch.py --primary  # the 300 km arm and the sites only

Each site needs three kinds of point:

    the site itself   cloud_cover_high, precipitation, and daily sunrise/sunset
    W at 150/300/450  cloud_cover_low, cloud_cover_mid
    E at 150/300/450  the same

The offsets carry two variables and not five on purpose: the rule only ever
asks an offset point whether the low and mid cloud were thin there at one hour,
and a variable nobody reads is bandwidth taken from somebody else.

The raw data is not in git. It is re-downloadable from the URLs this script
prints, which are the whole provenance.
"""
import json
import math
import os
import sys
import time
import urllib.error
import urllib.parse
import urllib.request

SITES = {
    "london":     (51.5072, -0.1276),
    "manchester": (53.4808, -2.2426),
    "edinburgh":  (55.9533, -3.1883),
    "cardiff":    (51.4816, -3.1791),
    "belfast":    (54.5973, -5.9301),
    "plymouth":   (50.3755, -4.1427),
    "aberdeen":   (57.1497, -2.0943),
    "inverness":  (57.4778, -4.2247),
    "carlisle":   (54.8925, -2.9329),
    "hull":       (53.7676, -0.3274),
    "norwich":    (52.6309,  1.2974),
    "swansea":    (51.6214, -3.9436),
    "dublin":     (53.3498, -6.2603),
    "galway":     (53.2707, -9.0568),
}

# rule.py: 300 km is the primary, 150 and 450 are the robustness arm.
DISTANCES_KM = [150, 300, 450]

# rule.py: 2015-01-01..2024-12-31, ten complete calendar years. Chunked in
# halves so that one failure costs five years and not ten.
CHUNKS = [("2015-01-01", "2019-12-31"), ("2020-01-01", "2024-12-31")]

SITE_HOURLY = "cloud_cover,cloud_cover_high,precipitation"
OFFSET_HOURLY = "cloud_cover_low,cloud_cover_mid"
SITE_DAILY = "sunrise,sunset"

OUT = os.path.expanduser("~/data/red-sky")
BASE = "https://archive-api.open-meteo.com/v1/archive"


def offset_lon(lat, lon, km, east):
    """Longitude `km` due east (or west) of (lat, lon), on the same parallel."""
    degrees = km / (111.32 * math.cos(math.radians(lat)))
    return lon + degrees if east else lon - degrees


def url_for(lat, lon, start, end, hourly, daily=None):
    q = {
        "latitude": f"{lat:.4f}",
        "longitude": f"{lon:.4f}",
        "start_date": start,
        "end_date": end,
        "hourly": hourly,
        "timezone": "Europe/London",
    }
    if daily:
        q["daily"] = daily
    return f"{BASE}?{urllib.parse.urlencode(q)}"


def points(primary_only):
    """Every (name, lat, lon, hourly, daily) the rule needs, in a fixed order."""
    distances = [300] if primary_only else DISTANCES_KM
    for site, (lat, lon) in SITES.items():
        yield f"{site}", lat, lon, SITE_HOURLY, SITE_DAILY
        for km in distances:
            for east in (False, True):
                name = f"{site}_{'E' if east else 'W'}{km}"
                yield name, lat, offset_lon(lat, lon, km, east), OFFSET_HOURLY, None


def get(url, tries=4):
    for attempt in range(tries):
        try:
            with urllib.request.urlopen(url, timeout=120) as r:
                return r.read()
        except (urllib.error.URLError, TimeoutError) as exc:
            wait = 5 * (attempt + 1)
            print(f"  retry in {wait}s after {type(exc).__name__}: {exc}")
            time.sleep(wait)
    return None


def main():
    primary_only = "--primary" in sys.argv
    os.makedirs(OUT, exist_ok=True)
    todo = [p for p in points(primary_only)]
    print(f"{len(todo)} points x {len(CHUNKS)} chunks")
    got = missing = have = 0
    for name, lat, lon, hourly, daily in todo:
        for start, end in CHUNKS:
            path = os.path.join(OUT, f"{name}_{start[:4]}.json")
            if os.path.exists(path) and os.path.getsize(path) > 2000:
                have += 1
                continue
            url = url_for(lat, lon, start, end, hourly, daily)
            body = get(url)
            if body is None:
                print(f"MISS {os.path.basename(path)}  {url}")
                missing += 1
                continue
            try:
                json.loads(body)
            except ValueError:
                print(f"BAD  {os.path.basename(path)} is not JSON: {body[:120]!r}")
                missing += 1
                continue
            with open(path, "wb") as fh:
                fh.write(body)
            got += 1
            print(f"got  {os.path.basename(path)}  {len(body) // 1024} KB  {url}")
            # Polite, and well under the keyless limit. The archive is somebody
            # else's server and this study is not urgent.
            time.sleep(1.5)
    print(f"\n{got} fetched, {have} already here, {missing} missing")
    return 1 if missing else 0


if __name__ == "__main__":
    raise SystemExit(main())