#!/usr/bin/env python3
"""Fetch Dukascopy historical tick files into a local cache. Standard library only. Resumable.

Usage:
    python3 fetch.py XAUUSD 2026-01-01 2026-08-31 [--workers 2]

Writes ~/data/dukascopy/<INST>/YYYY/MM/DD/HHh_ticks.bi5 (MM zero-based, as on the server).
Existing files are skipped, so re-running continues where the last run stopped. An empty file
means the server had no data for that hour (weekends, holidays) and is not fetched again.

Format notes, verified 2026-09-05 against live files:
- URL: https://datafeed.dukascopy.com/datafeed/{INST}/{YYYY}/{MM}/{DD}/{HH}h_ticks.bi5
  where MM is ZERO-BASED (July is 06). Hours without trading return HTTP 200 with an empty body.
- Body is LZMA (xz "alone" format) of 20-byte big-endian records:
  uint32 ms offset within the hour, uint32 ask, uint32 bid, float32 ask volume, float32 bid volume.
- Prices are integers scaled by a per-instrument point: 1000 for XAUUSD, 100000 for most FX pairs.
- Throttling: the first request answered instantly; after ~40 requests from 12 threads every
  response took 10-20 s and some timed out. Use 2-3 workers and let it run for hours.
"""
import lzma
import os
import struct
import sys
import time
import urllib.error
import urllib.request
from concurrent.futures import ThreadPoolExecutor
from datetime import date, datetime, timedelta, timezone

POINT = {"XAUUSD": 1000, "XAGUSD": 1000, "USDJPY": 1000}
DEFAULT_POINT = 100000
BASE = "https://datafeed.dukascopy.com/datafeed"
DATA = os.path.expanduser("~/data")
REC = struct.Struct(">IIIff")


def hour_path(inst, day, hour):
    return os.path.join(DATA, "dukascopy", inst, f"{day.year:04d}", f"{day.month - 1:02d}",
                        f"{day.day:02d}", f"{hour:02d}h_ticks.bi5")


def fetch_hour(inst, day, hour, retries=4):
    path = hour_path(inst, day, hour)
    if os.path.exists(path):
        return path
    if day.weekday() == 5:  # Saturday: never any data
        body = b""
    else:
        url = f"{BASE}/{inst}/{day.year:04d}/{day.month - 1:02d}/{day.day:02d}/{hour:02d}h_ticks.bi5"
        for attempt in range(retries):
            try:
                with urllib.request.urlopen(url, timeout=90) as r:
                    body = r.read()
                break
            except urllib.error.HTTPError as e:
                if e.code == 404:
                    body = b""
                    break
                time.sleep(5 * (attempt + 1))
            except Exception:
                time.sleep(5 * (attempt + 1))
        else:
            return None
    os.makedirs(os.path.dirname(path), exist_ok=True)
    with open(path, "wb") as f:
        f.write(body)
    return path


def decode_hour(path, inst, day, hour):
    """Yield (utc_ms, bid, ask) for one cached hourly file."""
    with open(path, "rb") as f:
        data = f.read()
    if not data:
        return
    raw = lzma.decompress(data)
    point = POINT.get(inst, DEFAULT_POINT)
    base_ms = int(datetime(day.year, day.month, day.day, hour, tzinfo=timezone.utc).timestamp() * 1000)
    for i in range(0, len(raw) - len(raw) % REC.size, REC.size):
        ms, ask, bid, _, _ = REC.unpack_from(raw, i)
        yield base_ms + ms, bid / point, ask / point


def days(start, end):
    d = start
    while d <= end:
        yield d
        d += timedelta(days=1)


def main():
    inst, start, end = sys.argv[1], date.fromisoformat(sys.argv[2]), date.fromisoformat(sys.argv[3])
    workers = int(sys.argv[sys.argv.index("--workers") + 1]) if "--workers" in sys.argv else 2
    jobs = [(d, h) for d in days(start, end) for h in range(24)]
    todo = [j for j in jobs if not os.path.exists(hour_path(inst, *j))]
    print(f"{inst}: {len(jobs)} hours {start}..{end}, {len(todo)} still to fetch, {workers} workers", flush=True)
    t0 = time.time()
    done = failed = 0
    with ThreadPoolExecutor(max_workers=workers) as ex:
        for p in ex.map(lambda j: fetch_hour(inst, *j), todo):
            done += 1
            failed += p is None
            if done % 50 == 0:
                print(f"  {done}/{len(todo)} after {time.time() - t0:.0f}s, {failed} failed", flush=True)
    print(f"finished: {done} fetched, {failed} failed, {time.time() - t0:.0f}s", flush=True)


if __name__ == "__main__":
    main()