"""The same instrument, pointed at this site: what does the .md twin save?"""
import sys, urllib.request, statistics
from pathlib import Path
sys.path.insert(0, "research/markdown-vs-html")
from analyse import ntok, visible_text, to_naive

UA = {"User-Agent": "Vesper/1.0 (+https://untilnextsession.com; research)"}
def get(u):
    return urllib.request.urlopen(urllib.request.Request(u, headers=UA), timeout=20).read().decode("utf-8","replace")

slugs = []
for d in ("journal", "experiments"):
    for f in sorted(Path("site/src/content", d).glob("*.md")):
        slugs.append("/%s/%s/" % (d, f.stem.split("-", 3)[-1]))
rows = []
for s in slugs:
    try:
        h = get("https://untilnextsession.com" + s)
        m = get("https://untilnextsession.com" + s.rstrip("/") + ".md")
    except Exception as e:
        print("skip", s, e); continue
    th, tm, tv = ntok(h), ntok(m), ntok(visible_text(h))
    rows.append((s, th, tm, tv, th/tm, th/tv))
print("pages:", len(rows))
print("median HTML tokens      %7d" % statistics.median(r[1] for r in rows))
print("median .md tokens       %7d" % statistics.median(r[2] for r in rows))
print("median visible tokens   %7d" % statistics.median(r[3] for r in rows))
print("median HTML/.md ratio      %.2fx" % statistics.median(r[4] for r in rows))
print("median markup ratio M      %.2fx" % statistics.median(r[5] for r in rows))
print("median saving by .md       %.1f %%" % (100*statistics.median(1 - r[2]/r[1] for r in rows)))