Replace clustering with a catalogue, and hand-write the references to judge it against

The pipeline's embed-and-cluster step is dead, and this commit holds both the evidence for that and the step proposed to replace it. Predicaments. Scenes are re-described as "what the person is up against", with no names, jobs or places, then embedded and clustered (redescribe.py, topic_words.py, topic_share.py). The pilot chose qwen3:14b over 3b by reading both side by side. Two defects the pilot exposed are fixed: split.py missed titles in quotes and a contents subtitle after a dash, so three stories had been merged into their neighbours, and strip_names.py read New York place names as people. The corrected corpus is probe/v2 (97 stories, 839 scenes); carry_summaries.py reuses the 829 unchanged v1 summaries. Topic share fell from 20% to 13% at k=60, short of the pre-registered 10%. Hand references. Three corpora were read scene by scene and written up by hand, under the same prompt rules the local models get, as a baseline to judge them against: O. Henry (probe/v2/claude, 839 scenes, 20 situations), Wharton's Descent of Man (probe/wharton, 262 scenes, 16 groups) and Jacobs's The Lady of the Barge (probe/jacobs, 157 scenes, 19 groups). Each has its own README and a readable page. No inference was used for any of them. Catalogue. probe/catalogue maps every hand group in the three references onto 36 situation entries, with an answer key per corpus and one recurrence rule applied to all three. classify.py assigns a scene one entry or none, leave-one-corpus- out; score.py checks it against the key, with a self-test on random labels. Why clustering is out: hand-written predicaments, embedded and clustered exactly as the model's were, agree with the hand grouping at ARI 0.05 — no better than the 14B text's 0.07. Better rewriting cannot rescue it. Embeddings cannot even shortlist: the hand label is the nearest entry 13% of the time and in the top 8 half the time. The classification runs are not here. The dev and test runs are pre-registered in probe/catalogue/README.md with the bar set beforehand, and are blocked on the inference host, whose GPU has fallen off the PCIe bus three times. The 30-scene partial output in out/ is not a result. Review page. The situation review is now a browser page rather than JSON edited by hand (review_page.py, review_page_logic.cjs with Node tests, format schema v2). It has never been rendered in a real browser. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014BygvsUXV9eU6oHkTCkKZ1
2026-09-20 17:19:42 -04:00
parent fa3769d0fe
commit e5617b86ba
135 changed files with 82027 additions and 1778 deletions
@@ -0,0 +1,132 @@
+// Tests for the rules behind REVIEW.html.
+//
+//   node --test test_review_page.mjs
+//
+// The page's drawing code needs a browser and is not tested here; these cover every decision
+// the page makes about a group, and saving and loading a review file.
+
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+const L = require('./review_page_logic.cjs');
+
+const RULES = { minMembers: 5, minStories: 4, maxDominant: 0.4, maxFieldChars: 80 };
+
+function group(id, kind, stories) {
+  return {
+    kind, stats: { stories: new Set(stories).size }, gate_failures: [], role_hints: [],
+    members: stories.map((story, n) => ({
+      member: `${id}.${String(n + 1).padStart(2, '0')}`, chunk: `${id}-${n}`, story,
+      title: `Story ${story}`, summary: `${id} scene ${n + 1}`,
+    })),
+  };
+}
+
+function doc() {
+  return {
+    run: { id: 'k60-s99', fingerprint: 'abc123', scenes: 21, stories: 13 },
+    rules: RULES,
+    candidates: {
+      C01: group('C01', 'accepted', [1, 2, 3, 4, 5, 6, 7, 8]),
+      C02: group('C02', 'accepted', [9, 10, 11, 12, 13, 14]),
+      B01: group('B01', 'borderline', [1, 3, 5, 7, 9, 11, 13]),
+    },
+  };
+}
+
+const find = (entries, id) => entries.find((e) => e.id === id);
+
+test('groups are named the way a reviewer reads them', () => {
+  const d = doc();
+  assert.equal(L.displayName(d, 'C02'), 'Group 2');
+  assert.equal(L.displayName(d, 'B01'), 'Extra group 1');
+  assert.equal(L.shortName(d, 'B01'), '+1');
+});
+
+test('a group goes from not decided, to kept but unnamed, to done', () => {
+  const d = doc();
+  const entries = L.initialEntries(d);
+  assert.equal(L.status(d, entries, 'C01').state, 'todo');
+  assert.equal(L.status(d, entries, 'B01').state, 'skipped', 'extras start left out');
+
+  find(entries, 'C01').decision = 'keep';
+  assert.equal(L.status(d, entries, 'C01').state, 'problem');
+  assert.match(L.status(d, entries, 'C01').message, /name is not finished/);
+
+  Object.assign(find(entries, 'C01'), { actor: 'lodger', wants: 'more time', counterpart: 'a landlady' });
+  assert.equal(L.status(d, entries, 'C01').state, 'done');
+  assert.equal(L.sentence(find(entries, 'C01')), 'A lodger wants more time from a landlady.');
+});
+
+test('unticking too much is flagged as it happens, with what a situation needs', () => {
+  const d = doc();
+  const entries = L.initialEntries(d);
+  Object.assign(find(entries, 'C02'), {
+    decision: 'keep', actor: 'clerk', wants: 'a raise', counterpart: 'an employer',
+    excluded: ['C02.01'],
+  });
+  assert.equal(L.status(d, entries, 'C02').state, 'done', 'five scenes from five stories is enough');
+  find(entries, 'C02').excluded.push('C02.02', 'C02.03');
+  const s = L.status(d, entries, 'C02');
+  assert.equal(s.state, 'problem');
+  assert.match(s.message, /3 scenes from 3 stories\. A situation needs at least 5 scenes/);
+});
+
+test('same-as offers only kept groups, merges scenes, and breaks if the target is dropped', () => {
+  const d = doc();
+  const entries = L.initialEntries(d);
+  assert.deepEqual(L.keptTargets(d, entries, 'C02'), [], 'nothing to merge into yet');
+
+  Object.assign(find(entries, 'C01'), { decision: 'keep', actor: 'lodger', wants: 'time', counterpart: 'a landlady' });
+  assert.deepEqual(L.keptTargets(d, entries, 'C02'), ['C01']);
+  assert.deepEqual(L.keptTargets(d, entries, 'C01'), [], 'never itself');
+
+  Object.assign(find(entries, 'C02'), { decision: 'same', same_as: 'C01' });
+  assert.equal(L.groupMembers(d, entries, 'C01').length, 14);
+  assert.equal(L.status(d, entries, 'C02').state, 'done');
+
+  find(entries, 'C01').decision = 'drop';
+  assert.equal(L.status(d, entries, 'C02').state, 'problem');
+});
+
+test('progress counts required groups only', () => {
+  const d = doc();
+  const entries = L.initialEntries(d);
+  find(entries, 'C01').decision = 'drop';
+  assert.deepEqual(L.progress(d, entries), { total: 2, decided: 1, problems: 0 });
+});
+
+test('a saved review loads back exactly', () => {
+  const d = doc();
+  const entries = L.initialEntries(d);
+  Object.assign(find(entries, 'C01'), { decision: 'keep', actor: 'lodger', wants: 'time', counterpart: 'a landlady', excluded: ['C01.03'] });
+  const file = JSON.parse(JSON.stringify(L.toReviewFile(d, entries, '2026-09-15T12:00:00Z')));
+  assert.equal(file.schema_version, 2);
+  assert.equal(file.fingerprint, 'abc123');
+  assert.deepEqual(L.fromReviewFile(d, file).entries, entries);
+});
+
+test('loading repairs what does not fit, and refuses the wrong run', () => {
+  const d = doc();
+  const repaired = L.fromReviewFile(d, {
+    schema_version: 2, run: 'k60-s99', fingerprint: 'abc123',
+    entries: [
+      { id: 'C01', decision: 'maybe', excluded: ['C01.02', 'C99.01', 'C01.02'], actor: 'x'.repeat(200) },
+      { id: 'B01', decision: 'drop' },
+      { id: 'ZZZ', decision: 'keep' },
+    ],
+  });
+  const c01 = find(repaired.entries, 'C01');
+  assert.equal(c01.decision, null, 'an unknown decision becomes undecided');
+  assert.deepEqual(c01.excluded, ['C01.02'], 'unknown and repeated scenes are dropped');
+  assert.equal(c01.actor.length, 80);
+  assert.equal(find(repaired.entries, 'B01').decision, 'skip', 'an extra cannot be dropped');
+  assert.equal(repaired.entries.length, 3);
+
+  assert.match(L.fromReviewFile(d, { schema_version: 2, run: 'k60-s24', entries: [] }).error, /for run k60-s24/);
+  assert.match(L.fromReviewFile(d, { schema_version: 1, run: 'k60-s99' }).error, /different version/);
+  assert.match(L.fromReviewFile(d, { schema_version: 2, run: 'k60-s99', fingerprint: 'nope', entries: [] }).error,
+    /different set of groups/);
+});