"""POST-HOC, NOT SEALED. What did the extractor actually delete? The sealed retention statistic K = T_extract / T_visible is a ratio of token counts, and it is confounded in two directions at once: markdown ADDS tokens (a link becomes `[text](https://a/very/long/url)`), while extraction REMOVES text. On one page in this corpus it reads 128 % for an extraction that is plainly shorter than the page. A number that can go up when text is deleted cannot be read as retention, and that is a defect in the sealed rule, not in the result it produced. This file replaces it with two quantities that additions cannot move, and it is written AFTER the sealed verdicts are known, so it decides nothing and is published as description: RECALL of the word 5-grams in the page's visible text, the share that survive into the extraction. Adding a URL cannot raise it. and the split that answers the real question — is the deleted half the navigation, or the article? SHARED a 5-gram of this page that also appears in the visible text of another page from the same domain. That is what boilerplate IS: the text a site repeats on every page. Requires a domain with at least 2 pages in the corpus. UNIQUE a 5-gram of this page that appears on no other page of its site. As close to "the article" as a measurement with no human in it can get. If extraction is doing what it claims — removing the chrome and keeping the article — then unique recall is near 1 and shared recall is low. If unique recall is also low, the missing half is the page. """ import csv import re import statistics import sys from collections import defaultdict from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) from analyse import (BODIES, DATA, MIN_BYTES, MIN_VISIBLE_TOKENS, boot_median, ntok, to_extract, visible_text) HERE = Path(__file__).resolve().parent N = 5 _WORD = re.compile(r"[^\W\d_]+", re.UNICODE) def shingles(text, n=N): w = [t.lower() for t in _WORD.findall(text)] return {tuple(w[i:i + n]) for i in range(len(w) - n + 1)} def main(): rows = list(csv.DictReader(open(DATA / "index.csv"))) pages = [] for r in rows: if r["status"] != "200" or "html" not in r["content_type"].lower(): continue if int(r["bytes"]) < MIN_BYTES or r["truncated"] == "1": continue body = (BODIES / r["file"]).read_bytes().decode("utf-8", "replace") vis = visible_text(body) if ntok(vis) < MIN_VISIBLE_TOKENS: continue pages.append({"url": r["url"], "domain": r["domain"], "vis": shingles(vis), "ext": shingles(to_extract(body))}) print("pages: %d" % len(pages), flush=True) by_dom = defaultdict(list) for p in pages: by_dom[p["domain"]].append(p) out = [] for dom, ps in by_dom.items(): if len(ps) < 2: continue for p in ps: others = set() for q in ps: if q is not p: others |= q["vis"] V = p["vis"] if not V: continue shared = V & others unique = V - others rec = lambda S: (len(S & p["ext"]) / len(S)) if S else None out.append({ "url": p["url"], "domain": dom, "n_shingles": len(V), "share_shared": len(shared) / len(V), "recall_all": rec(V), "recall_shared": rec(shared), "recall_unique": rec(unique), }) doms = sorted({o["domain"] for o in out}) print("pages on multi-page domains: %d from %d domains" % (len(out), len(doms))) def per_domain(key): vals = [] for d in doms: v = [o[key] for o in out if o["domain"] == d and o[key] is not None] if v: vals.append(statistics.median(v)) return vals L = [] W = L.append W("# What the extractor deleted (post-hoc, not sealed)") W("") W("Written after the sealed verdicts were known. It decides nothing; it " "describes. See the docstring of `recall.py` for why the sealed retention " "statistic had to be replaced: markdown adds link URLs, so a token ratio " "can rise while text is deleted.") W("") W("Unit: word 5-grams of the page's visible text. **Recall** is the share of " "them that survive into trafilatura's markdown. **Shared** 5-grams also " "appear on another page of the same site — that is what boilerplate is. " "**Unique** ones appear on no other page of the site, which is as close to " "\"the article\" as a measurement with no human in it can get.") W("") W("%d pages from %d domains that contributed more than one page." % (len(out), len(doms))) W("") W("| quantity | median (per page) | 95 % CI | median (per domain) |") W("| --- | --- | --- | --- |") for key, label in [("share_shared", "share of the page that is site boilerplate"), ("recall_all", "recall, all visible text"), ("recall_shared", "recall of the boilerplate"), ("recall_unique", "recall of what is unique to the page")]: ci = boot_median([o[key] for o in out]) dci = boot_median(per_domain(key)) W("| %s | %.3f | %.3f - %.3f | %.3f |" % (label, ci[0], ci[1], ci[2], dci[0])) W("") ru = [o["recall_unique"] for o in out if o["recall_unique"] is not None] rs = [o["recall_shared"] for o in out if o["recall_shared"] is not None] W("Pages where the extractor kept at least 90 %% of what is unique to them: " "**%.1f %%** (%d of %d)." % (100 * sum(1 for v in ru if v >= 0.9) / len(ru), sum(1 for v in ru if v >= 0.9), len(ru))) W("") W("Pages where it kept at least half the site boilerplate too: **%.1f %%**." % (100 * sum(1 for v in rs if v >= 0.5) / len(rs))) W("") worst = sorted([o for o in out if o["recall_unique"] is not None], key=lambda o: o["recall_unique"])[:8] W("The eight pages the extractor treated worst, by unique recall:") W("") W("| domain | boilerplate share | recall (unique) | recall (shared) |") W("| --- | --- | --- | --- |") for o in worst: W("| `%s` | %.2f | **%.2f** | %.2f |" % (o["domain"], o["share_shared"], o["recall_unique"], o["recall_shared"])) (HERE / "recall.md").write_text("\n".join(L) + "\n") with open(HERE / "recall.csv", "w", newline="") as fh: w = csv.DictWriter(fh, fieldnames=list(out[0].keys())) w.writeheader() w.writerows(out) print("\n".join(L)) if __name__ == "__main__": main()