"""Fetch monthly en.wikipedia pageviews from Wikimedia's public REST API.

No key. One call per agent class. Writes raw JSON into ~/data/wikipedia-traffic/
so a rerun does not hit the API again; delete the files to refresh.

    python research/wikipedia-traffic/fetch.py
"""

import json
import os
import sys
import urllib.request

BASE = ("https://wikimedia.org/api/rest_v1/metrics/pageviews/aggregate"
        "/{project}/{access}/{agent}/monthly/{start}/{end}")
UA = "Vesper/1.0 (https://untilnextsession.com; vesper@untilnextsession.com)"
OUT = os.path.expanduser("~/data/wikipedia-traffic")

START = "2015070100"          # the series begins here
END = "2026090100"            # the API returns complete months only

AGENTS = ("user", "automated", "spider")
PROJECTS = ("en.wikipedia", "all-projects")
ACCESSES = ("all-access", "desktop", "mobile-web", "mobile-app")


def get(url):
    req = urllib.request.Request(url, headers={"User-Agent": UA,
                                               "Accept": "application/json"})
    with urllib.request.urlopen(req, timeout=60) as fh:
        return json.loads(fh.read().decode("utf-8"))


def pull(project, access, agent):
    name = "%s__%s__%s.json" % (project, access, agent)
    path = os.path.join(OUT, name)
    if os.path.exists(path):
        return path, "cached"
    url = BASE.format(project=project, access=access, agent=agent,
                      start=START, end=END)
    data = get(url)
    with open(path, "w") as fh:
        json.dump(data, fh)
    return path, "%d months" % len(data.get("items", []))


def main():
    os.makedirs(OUT, exist_ok=True)
    # The three agent classes on en.wikipedia, all access methods: the main series.
    for agent in AGENTS:
        path, how = pull("en.wikipedia", "all-access", agent)
        print("en.wikipedia all-access", agent, "->", how)
    # Human traffic split by access method, and the whole of Wikimedia, as checks.
    for access in ACCESSES[1:]:
        path, how = pull("en.wikipedia", access, "user")
        print("en.wikipedia", access, "user ->", how)
    path, how = pull("all-projects", "all-access", "user")
    print("all-projects all-access user ->", how)
    return 0


if __name__ == "__main__":
    sys.exit(main())