Replace clustering with a catalogue, and hand-write the references to judge it against
The pipeline's embed-and-cluster step is dead, and this commit holds both the evidence for that and the step proposed to replace it. Predicaments. Scenes are re-described as "what the person is up against", with no names, jobs or places, then embedded and clustered (redescribe.py, topic_words.py, topic_share.py). The pilot chose qwen3:14b over 3b by reading both side by side. Two defects the pilot exposed are fixed: split.py missed titles in quotes and a contents subtitle after a dash, so three stories had been merged into their neighbours, and strip_names.py read New York place names as people. The corrected corpus is probe/v2 (97 stories, 839 scenes); carry_summaries.py reuses the 829 unchanged v1 summaries. Topic share fell from 20% to 13% at k=60, short of the pre-registered 10%. Hand references. Three corpora were read scene by scene and written up by hand, under the same prompt rules the local models get, as a baseline to judge them against: O. Henry (probe/v2/claude, 839 scenes, 20 situations), Wharton's Descent of Man (probe/wharton, 262 scenes, 16 groups) and Jacobs's The Lady of the Barge (probe/jacobs, 157 scenes, 19 groups). Each has its own README and a readable page. No inference was used for any of them. Catalogue. probe/catalogue maps every hand group in the three references onto 36 situation entries, with an answer key per corpus and one recurrence rule applied to all three. classify.py assigns a scene one entry or none, leave-one-corpus- out; score.py checks it against the key, with a self-test on random labels. Why clustering is out: hand-written predicaments, embedded and clustered exactly as the model's were, agree with the hand grouping at ARI 0.05 — no better than the 14B text's 0.07. Better rewriting cannot rescue it. Embeddings cannot even shortlist: the hand label is the nearest entry 13% of the time and in the top 8 half the time. The classification runs are not here. The dev and test runs are pre-registered in probe/catalogue/README.md with the bar set beforehand, and are blocked on the inference host, whose GPU has fallen off the PCIe bus three times. The 30-scene partial output in out/ is not a result. Review page. The situation review is now a browser page rather than JSON edited by hand (review_page.py, review_page_logic.cjs with Node tests, format schema v2). It has never been rendered in a real browser. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014BygvsUXV9eU6oHkTCkKZ1
This commit is contained in:
JesseMarkowitz
co-authored by
Claude Opus 5
Claude Opus 5
parent
fa3769d0fe
commit
e5617b86ba
@@ -0,0 +1,164 @@
|
||||
// The rules behind REVIEW.html, with no page code in them.
|
||||
//
|
||||
// review_page.py inlines this file into the page; test_review_page.mjs loads it under Node.
|
||||
// It mirrors review_format.py, which stays the authority: apply_review.py checks the saved
|
||||
// file again. The page applies the rules as the reviewer goes, so a problem shows where it
|
||||
// happens instead of as an error message at the end.
|
||||
|
||||
const ReviewLogic = (() => {
|
||||
const SCHEMA_VERSION = 2;
|
||||
const ALLOWED = {
|
||||
accepted: ['keep', 'drop', 'same'],
|
||||
borderline: ['skip', 'keep', 'same'],
|
||||
};
|
||||
|
||||
const orderedIds = (doc) => Object.keys(doc.candidates);
|
||||
const idsOfKind = (doc, kind) => orderedIds(doc).filter((id) => doc.candidates[id].kind === kind);
|
||||
const allowedDecisions = (kind) => ALLOWED[kind] ?? [];
|
||||
|
||||
function displayName(doc, id) {
|
||||
const kind = doc.candidates[id].kind;
|
||||
const n = idsOfKind(doc, kind).indexOf(id) + 1;
|
||||
return kind === 'accepted' ? `Group ${n}` : `Extra group ${n}`;
|
||||
}
|
||||
|
||||
function shortName(doc, id) {
|
||||
const kind = doc.candidates[id].kind;
|
||||
const n = idsOfKind(doc, kind).indexOf(id) + 1;
|
||||
return kind === 'accepted' ? String(n) : `+${n}`;
|
||||
}
|
||||
|
||||
function blankEntry(id, kind) {
|
||||
return {
|
||||
id, decision: kind === 'accepted' ? null : 'skip', same_as: null,
|
||||
actor: '', wants: '', counterpart: '', excluded: [],
|
||||
};
|
||||
}
|
||||
|
||||
const initialEntries = (doc) => orderedIds(doc).map((id) => blankEntry(id, doc.candidates[id].kind));
|
||||
const entryOf = (entries, id) => entries.find((e) => e.id === id);
|
||||
|
||||
/** Groups this one could be marked the same as: every other group currently kept. */
|
||||
const keptTargets = (doc, entries, id) =>
|
||||
orderedIds(doc).filter((other) => other !== id && entryOf(entries, other)?.decision === 'keep');
|
||||
|
||||
/** The ticked scenes of a group, plus the ticked scenes of every group marked the same as it. */
|
||||
function groupMembers(doc, entries, id) {
|
||||
const sources = [id, ...entries.filter((e) => e.decision === 'same' && e.same_as === id).map((e) => e.id)];
|
||||
const seen = new Set();
|
||||
const members = [];
|
||||
for (const source of sources) {
|
||||
const entry = entryOf(entries, source);
|
||||
const excluded = new Set(entry ? entry.excluded : []);
|
||||
for (const m of doc.candidates[source].members) {
|
||||
if (excluded.has(m.member) || seen.has(m.chunk)) continue;
|
||||
seen.add(m.chunk);
|
||||
members.push(m);
|
||||
}
|
||||
}
|
||||
return members;
|
||||
}
|
||||
|
||||
function shapeCheck(members, rules) {
|
||||
const counts = new Map();
|
||||
for (const m of members) counts.set(m.story, (counts.get(m.story) ?? 0) + 1);
|
||||
const size = members.length;
|
||||
const stories = counts.size;
|
||||
const dominant = size ? Math.max(...counts.values()) / size : 0;
|
||||
const pct = (x) => Math.round(x * 100);
|
||||
const ok = size >= rules.minMembers && stories >= rules.minStories && dominant <= rules.maxDominant;
|
||||
const text = ok
|
||||
? `${size} scenes from ${stories} stories: enough for a situation.`
|
||||
: `${size} scenes from ${stories} stories${dominant > rules.maxDominant ? `, ${pct(dominant)}% from one story` : ''}. ` +
|
||||
`A situation needs at least ${rules.minMembers} scenes from at least ${rules.minStories} stories, ` +
|
||||
`with no story over ${pct(rules.maxDominant)}%.`;
|
||||
return { size, stories, dominant, ok, text };
|
||||
}
|
||||
|
||||
const blank = (s) => (typeof s === 'string' && s.trim() ? s.trim() : '___');
|
||||
const sentence = (entry) => `A ${blank(entry.actor)} wants ${blank(entry.wants)} from ${blank(entry.counterpart)}.`;
|
||||
const named = (entry) => ['actor', 'wants', 'counterpart'].every((k) => typeof entry[k] === 'string' && entry[k].trim());
|
||||
|
||||
/** Where a group stands: todo, done, skipped or problem, with a sentence saying why. */
|
||||
function status(doc, entries, id) {
|
||||
const c = doc.candidates[id];
|
||||
const e = entryOf(entries, id);
|
||||
if (!e || e.decision === null) return { state: 'todo', message: 'Not decided yet.' };
|
||||
if (e.decision === 'skip') return { state: 'skipped', message: 'Optional: left out.' };
|
||||
if (e.decision === 'drop') return { state: 'done', message: 'Dropped.' };
|
||||
if (e.decision === 'same') {
|
||||
const target = entryOf(entries, e.same_as);
|
||||
if (!target || target.decision !== 'keep') {
|
||||
return { state: 'problem', message: `Marked the same as ${e.same_as ? displayName(doc, e.same_as) : 'another group'}, which is not kept.` };
|
||||
}
|
||||
return { state: 'done', message: `Same as ${displayName(doc, e.same_as)}.` };
|
||||
}
|
||||
if (e.decision === 'keep') {
|
||||
if (!named(e)) return { state: 'problem', message: 'Kept, but the name is not finished.' };
|
||||
const shape = shapeCheck(groupMembers(doc, entries, id), doc.rules);
|
||||
if (!shape.ok) return { state: 'problem', message: shape.text };
|
||||
return { state: 'done', message: `Kept: ${sentence(e)}` };
|
||||
}
|
||||
return { state: 'problem', message: `Unknown decision for a ${c.kind} group.` };
|
||||
}
|
||||
|
||||
function progress(doc, entries) {
|
||||
const accepted = idsOfKind(doc, 'accepted');
|
||||
return {
|
||||
total: accepted.length,
|
||||
decided: accepted.filter((id) => entryOf(entries, id)?.decision !== null).length,
|
||||
problems: orderedIds(doc).filter((id) => status(doc, entries, id).state === 'problem').length,
|
||||
};
|
||||
}
|
||||
|
||||
function toReviewFile(doc, entries, savedAt) {
|
||||
return {
|
||||
schema_version: SCHEMA_VERSION,
|
||||
run: doc.run.id,
|
||||
fingerprint: doc.run.fingerprint ?? null,
|
||||
saved_at: savedAt,
|
||||
entries: entries.map((e) => ({ ...e, excluded: [...e.excluded] })),
|
||||
};
|
||||
}
|
||||
|
||||
/** Read a saved review back, repairing anything that does not fit this run rather than failing. */
|
||||
function fromReviewFile(doc, data) {
|
||||
if (!data || typeof data !== 'object') return { error: 'That is not a review file.' };
|
||||
if (data.schema_version !== SCHEMA_VERSION) {
|
||||
return { error: 'That review file was made by a different version of the review tool.' };
|
||||
}
|
||||
if (data.run !== doc.run.id) return { error: `That review file is for run ${data.run}, not ${doc.run.id}.` };
|
||||
if (doc.run.fingerprint && data.fingerprint && data.fingerprint !== doc.run.fingerprint) {
|
||||
return { error: 'That review file was made for a different set of groups with the same run name.' };
|
||||
}
|
||||
const stored = new Map((Array.isArray(data.entries) ? data.entries : [])
|
||||
.filter((x) => x && typeof x === 'object').map((x) => [x.id, x]));
|
||||
const max = doc.rules.maxFieldChars;
|
||||
const text = (v) => (typeof v === 'string' ? v.slice(0, max) : '');
|
||||
const entries = orderedIds(doc).map((id) => {
|
||||
const kind = doc.candidates[id].kind;
|
||||
const base = blankEntry(id, kind);
|
||||
const s = stored.get(id);
|
||||
if (!s) return base;
|
||||
const valid = new Set(doc.candidates[id].members.map((m) => m.member));
|
||||
const decisionOk = allowedDecisions(kind).includes(s.decision) || (kind === 'accepted' && s.decision === null);
|
||||
return {
|
||||
id,
|
||||
decision: decisionOk ? s.decision : base.decision,
|
||||
same_as: typeof s.same_as === 'string' && s.same_as in doc.candidates && s.same_as !== id ? s.same_as : null,
|
||||
actor: text(s.actor),
|
||||
wants: text(s.wants),
|
||||
counterpart: text(s.counterpart),
|
||||
excluded: [...new Set(Array.isArray(s.excluded) ? s.excluded.filter((x) => valid.has(x)) : [])],
|
||||
};
|
||||
});
|
||||
return { entries };
|
||||
}
|
||||
|
||||
return {
|
||||
SCHEMA_VERSION, orderedIds, allowedDecisions, displayName, shortName, blankEntry, initialEntries,
|
||||
keptTargets, groupMembers, shapeCheck, sentence, status, progress, toReviewFile, fromReviewFile,
|
||||
};
|
||||
})();
|
||||
|
||||
if (typeof module !== 'undefined') module.exports = ReviewLogic;
|
||||
Reference in New Issue
Block a user