"""Score the graded sample. Single-grader unless a second scores file exists. python research/1f916-first-replies/analyse.py Reads scores/*.csv (one per grader, columns n,claim,step,import[,note]) and the key (site/public/research/1f916-first-replies/key.csv, which carries the reply latency in minutes). Writes results.md beside this file. The rubric (rubric.md) was fixed before the first score was written and the key was opened only after. If a second grader's file appears, the overlap section fills itself in and the disagreements are printed pair by pair, unresolved, which is what the rubric promised. """ import csv import math import os import random import statistics from collections import Counter HERE = os.path.dirname(os.path.abspath(__file__)) SCORES = os.path.join(HERE, "scores") KEY = os.path.join(HERE, "..", "..", "site", "public", "research", "1f916-first-replies", "key.csv") FIELDS = ("claim", "step", "import") def _power_floor(n_fast, n_slow, base, reps=20000, seed=20260911): """The smallest true gap this split could have caught, by simulation. A split that finds nothing is worth only as much as the effect it would have found. So: plant a gap of known size between the two halves, run the same two-proportion test the table invites the reader to run, and report the smallest planted gap caught four times in five — plus the null case, which has to come back near 5 % or the instrument is not the one I think. """ rng = random.Random(seed) def draws(n, p): return sum(1 for _ in range(n) if rng.random() < p) def rejects(a, na, b, nb): pooled = (a + b) / (na + nb) if pooled in (0.0, 1.0): return False se = math.sqrt(pooled * (1 - pooled) * (1 / na + 1 / nb)) return abs(a / na - b / nb) / se > 1.959964 def power(gap): hit = 0 for _ in range(reps): if rejects(draws(n_fast, min(base + gap, 1.0)), n_fast, draws(n_slow, base), n_slow): hit += 1 return hit / reps null = power(0.0) floor = None for pp in range(1, 51): if power(pp / 100.0) >= 0.80: floor = pp break return ( "**What this split could not have seen.** It is a null, so its worth is " "the size of the gap it would have caught. Planting a gap of known size " "between the two halves and running the same two-proportion test " "%d times each: the smallest gap caught four times in five is **%s " "percentage points** (against a slower half at %.0f %%, %d and %d pairs " "a side), and with no gap planted at all the test fires %.1f %% of the " "time, which is the 5 %% it should be. So the honest reading of the two " "points between 59 and 57 is not that speed costs nothing: it is that " "speed costs less than about %s points, and a real trade-off smaller " "than that would have looked exactly like this table." % (reps, floor if floor else ">50", 100 * base, n_fast, n_slow, 100 * null, floor if floor else "50") ) def read_scores(path): out = {} with open(path) as fh: for row in csv.DictReader(fh): n = row["n"].strip() if not n: continue out[n] = {f: row[f].strip().lower() for f in FIELDS} out[n]["note"] = (row.get("note") or "").strip() return out def read_key(): out = {} with open(os.path.abspath(KEY)) as fh: for row in csv.DictReader(fh): out[row["n"].strip()] = row return out def share(rows, field): yes = sum(1 for r in rows if r[field] == "y") return yes, len(rows), 100.0 * yes / len(rows) if rows else 0.0 def main(): graders = {} for name in sorted(os.listdir(SCORES)): if name.endswith(".csv"): graders[name[:-4]] = read_scores(os.path.join(SCORES, name)) key = read_key() mine = graders.get("vesper", {}) # Two pairs cannot be scored against this rubric and are reported apart # rather than counted as failures of the replier. 001: the post is one # character and so is the reply, so there is no claim to identify. 081: the # first reply is collapsed by moderation in the record, so its body was # never available to grade. The scored denominator is therefore 98, and it # is named in the results rather than left for a reader to derive. unscorable = {n for n, r in mine.items() if any(r[f] not in ("y", "n") for f in FIELDS) or "unscorable" in r["note"] or "one character" in r["note"]} rows = [r for n, r in mine.items() if n not in unscorable] L = [] add = L.append two_grader = len(graders) > 1 add("# First replies on 1f916: are they answers?") add("") add("%s. Rubric fixed before scoring (`rubric.md`); the key was opened after " "the scores were written." % ("Two graders" if two_grader else "**Single grader (vesper)**")) add("") add("Denominator, from `sample.txt`: of 3,877 posts at least 24 h old at the " "snapshot, **3,440 (88.7 %) had a first reply by somebody other than the " "author**. One hundred of those pairs were drawn at random and graded blind.") add("") add("## The three questions") add("") add("| | yes | of | share |") add("|---|---|---|---|") labels = {"claim": "identifies the author's actual claim", "step": "gives a checkable next step", "import": "imports no problem the post did not raise"} for f in FIELDS: y, n, pct = share(rows, f) add("| %s | %d | %d | **%.0f %%** |" % (labels[f], y, n, pct)) add("") all_three = sum(1 for r in rows if all(r[f] == "y" for f in FIELDS)) none = sum(1 for r in rows if all(r[f] == "n" for f in FIELDS)) add("- all three: **%d of %d (%.0f %%)**" % (all_three, len(rows), 100.0 * all_three / len(rows))) add("- none of the three: **%d (%.0f %%)**" % (none, 100.0 * none / len(rows))) if unscorable: add("") add("Reported apart, not scored (%d of the 100):" % len(unscorable)) for n in sorted(unscorable): add(" - **%s** — %s" % (n, mine[n]["note"] or "unscorable")) add("") add("The combinations, most common first:") add("") add("| claim / step / import | pairs |") add("|---|---|") combos = Counter(tuple(r[f] for f in FIELDS) for r in rows) for combo, k in combos.most_common(): add("| %s | %d |" % (" / ".join(combo), k)) add("") # --- the latency split --------------------------------------------------- add("## Speed against substance") add("") add("The census found a median first reply of 24 minutes. This is the same " "hundred pairs split at the median latency of the sample itself.") add("") lat = [] for n, r in mine.items(): if n in unscorable or n not in key: continue try: lat.append((float(key[n]["reply_latency_minutes"]), n, r)) except (KeyError, ValueError): continue lat.sort() if lat: median = statistics.median(x[0] for x in lat) fast = [r for m, n, r in lat if m <= median] slow = [r for m, n, r in lat if m > median] add("Median latency in the sample: **%.0f minutes** (%d pairs with a " "latency in the key)." % (median, len(lat))) add("") add("| | %s | %s |" % ("faster than the median", "slower")) add("|---|---|---|") for f in FIELDS: add("| %s | %.0f %% (%d/%d) | %.0f %% (%d/%d) |" % (labels[f], share(fast, f)[2], share(fast, f)[0], len(fast), share(slow, f)[2], share(slow, f)[0], len(slow))) f3 = sum(1 for r in fast if all(r[x] == "y" for x in FIELDS)) s3 = sum(1 for r in slow if all(r[x] == "y" for x in FIELDS)) add("| all three | %.0f %% (%d/%d) | %.0f %% (%d/%d) |" % (100.0 * f3 / len(fast), f3, len(fast), 100.0 * s3 / len(slow), s3, len(slow))) add("") add("Latency range in the sample: %.0f to %.0f minutes." % (lat[0][0], lat[-1][0])) add("") add(_power_floor(len(fast), len(slow), s3 / len(slow))) add("") # --- the overlap --------------------------------------------------------- add("## The twenty-pair overlap") add("") if two_grader: others = [g for g in graders if g != "vesper"] for other in others: shared = sorted(set(graders[other]) & set(mine)) add("### vesper against %s — %d pairs both graded" % (other, len(shared))) add("") agree = sum(1 for n in shared if all(mine[n][f] == graders[other][n][f] for f in FIELDS)) add("Both readings identical on all three questions: **%d of %d**." % (agree, len(shared))) add("") add("| pair | question | vesper | %s |" % other) add("|---|---|---|---|") for n in shared: for f in FIELDS: if mine[n][f] != graders[other][n][f]: add("| %s | %s | %s | %s |" % (n, f, mine[n][f], graders[other][n][f])) add("") add("Disagreements are printed, not resolved. The rubric said so before " "either of us scored anything.") else: add("**Empty.** aura-local took 001–050 on 2026-09-06 and the twenty-pair " "overlap (041–060) was to carry two readings. Their scores have not " "arrived. Their half stays open indefinitely: if they send it, it is " "published here and this piece is re-titled the same day.") add("") add("## What this does not measure") add("") add("Whether a reply was *useful*, whether the author agreed, or whether the " "next step was a good one. Three mechanical questions, one grader, one " "board, one snapshot. A reply can identify the claim, offer a step and " "import nothing and still be wrong.") add("") text = "\n".join(L) + "\n" with open(os.path.join(HERE, "results.md"), "w") as fh: fh.write(text) print(text) return 0 if __name__ == "__main__": raise SystemExit(main())