JesseMarkowitzandClaude Opus 5
fa3769d0fe
Research toward building a content pack from a story corpus, kept on its own branch and independent of the game. Records the selection experiments against blind labels, and settles selection as gate G2 followed by a human review: review.py writes REVIEW.md and a review.json form, apply_review.py checks the filled form and writes situations.json for the next stage. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01C6UDQ9o6L6Ey173U7XVou6
203 lines
9.0 KiB
Python
203 lines
9.0 KiB
Python
"""The human review form: its schema, member ids, checks, and conversion to situations.json.
|
|
|
|
Kept free of clustering code so apply_review.py and the tests load instantly.
|
|
|
|
A review run has three files in one directory:
|
|
candidates.json written by review.py: what the program found (read-only)
|
|
review.json written blank by review.py, filled in by the reviewer
|
|
situations.json written by apply_review.py once review.json passes every check
|
|
"""
|
|
import collections
|
|
|
|
SCHEMA_VERSION = 1
|
|
MIN_MEMBERS, MIN_STORIES, MAX_DOMINANT = 5, 4, 0.40
|
|
MAX_SITUATION_CHARS = 200
|
|
DECISIONS = {
|
|
'accepted': ('accept', 'reject', 'merge'), # passed the gate: the reviewer must decide
|
|
'borderline': ('skip', 'accept', 'merge'), # narrowly failed: optional, "skip" by default
|
|
}
|
|
|
|
|
|
def member_id(cid, n):
|
|
"""C04.07 — the 7th scene of candidate C04."""
|
|
return f'{cid}.{n:02d}'
|
|
|
|
|
|
def blank_entry(cid, kind):
|
|
return {
|
|
'id': cid,
|
|
'kind': kind,
|
|
'decision': None if kind == 'accepted' else 'skip',
|
|
'situation': '',
|
|
'actor': '',
|
|
'counterpart': '',
|
|
'stakes': '',
|
|
'exclude': [],
|
|
'merge_into': None,
|
|
'notes': '',
|
|
}
|
|
|
|
|
|
def blank_form(candidates):
|
|
return {
|
|
'schema_version': SCHEMA_VERSION,
|
|
'run': candidates['run']['id'],
|
|
'instructions': 'Read REVIEW.md in this directory. Fill in each entry below, then run: '
|
|
f'python3 apply_review.py review/{candidates["run"]["id"]}',
|
|
'entries': [blank_entry(cid, c['kind']) for cid, c in candidates['candidates'].items()],
|
|
}
|
|
|
|
|
|
def shape(members):
|
|
"""(scene count, story count, share held by the largest story)."""
|
|
counts = collections.Counter(m['story'] for m in members)
|
|
return len(members), len(counts), (max(counts.values()) / len(members) if members else 0.0)
|
|
|
|
|
|
def _text(value):
|
|
return value.strip() if isinstance(value, str) else ''
|
|
|
|
|
|
def validate(candidates, form):
|
|
"""Return (errors, warnings, situations). situations is None whenever there is any error.
|
|
|
|
Every problem is collected, so the reviewer can fix them all in one pass.
|
|
"""
|
|
errors, warnings = [], []
|
|
cands = candidates['candidates']
|
|
|
|
if form.get('schema_version') != SCHEMA_VERSION:
|
|
errors.append(f'"schema_version" is {form.get("schema_version")!r}; this checker reads {SCHEMA_VERSION}.')
|
|
if form.get('run') != candidates['run']['id']:
|
|
errors.append(f'This form is for run {form.get("run")!r}, but candidates.json is run '
|
|
f'{candidates["run"]["id"]!r}. Use the review.json from the same directory.')
|
|
entries = form.get('entries')
|
|
if not isinstance(entries, list):
|
|
errors.append('"entries" must be a list of candidate entries.')
|
|
return errors, warnings, None
|
|
|
|
by_id = {}
|
|
for e in entries:
|
|
cid = e.get('id') if isinstance(e, dict) else None
|
|
if cid not in cands:
|
|
errors.append(f'An entry has id {cid!r}, which is not a candidate in this run.')
|
|
elif cid in by_id:
|
|
errors.append(f'{cid}: appears more than once; keep one entry.')
|
|
else:
|
|
by_id[cid] = e
|
|
for cid in cands:
|
|
if cid not in by_id:
|
|
errors.append(f'{cid}: its entry is missing from the form. Restore it (the blank entry is '
|
|
f'{{"id": "{cid}", ...}}) and set "decision".')
|
|
|
|
for cid, e in by_id.items():
|
|
kind = cands[cid]['kind']
|
|
allowed = DECISIONS[kind]
|
|
decision = e.get('decision')
|
|
if decision is None:
|
|
errors.append(f'{cid}: no decision yet. Set "decision" to one of: {", ".join(allowed)}.')
|
|
continue
|
|
if decision not in allowed:
|
|
errors.append(f'{cid}: "decision" is {decision!r}; for a {kind} candidate use one of: '
|
|
f'{", ".join(allowed)}.')
|
|
continue
|
|
|
|
members = cands[cid]['members']
|
|
valid = {m['member'] for m in members}
|
|
exclude = e.get('exclude', [])
|
|
if not isinstance(exclude, list):
|
|
errors.append(f'{cid}: "exclude" must be a list of member ids, like ["{cid}.03"].')
|
|
exclude = []
|
|
unknown = [x for x in exclude if x not in valid]
|
|
if unknown:
|
|
errors.append(f'{cid}: "exclude" lists {", ".join(map(str, unknown))}, which are not scenes of '
|
|
f'{cid}; its scenes are {member_id(cid, 1)} to {member_id(cid, len(members))}.')
|
|
|
|
if decision == 'accept':
|
|
for field in ('situation', 'actor', 'counterpart'):
|
|
if not _text(e.get(field)):
|
|
errors.append(f'{cid}: accepted, so "{field}" must be filled in.')
|
|
situation = _text(e.get('situation'))
|
|
if len(situation) > MAX_SITUATION_CHARS:
|
|
errors.append(f'{cid}: "situation" is {len(situation)} characters; keep it to one sentence '
|
|
f'of at most {MAX_SITUATION_CHARS}.')
|
|
if situation and 'want' not in situation.lower():
|
|
warnings.append(f'{cid}: "situation" does not say who wants what ("… wants … from …"). '
|
|
f'Accepted as written.')
|
|
if not _text(e.get('stakes')):
|
|
warnings.append(f'{cid}: "stakes" is empty.')
|
|
|
|
if decision == 'merge':
|
|
target = e.get('merge_into')
|
|
if target == cid:
|
|
errors.append(f'{cid}: cannot merge into itself.')
|
|
elif target not in by_id:
|
|
errors.append(f'{cid}: "merge_into" is {target!r}, which is not a candidate id in this form.')
|
|
elif by_id[target].get('decision') != 'accept':
|
|
errors.append(f'{cid}: merges into {target}, but {target} has decision '
|
|
f'{by_id[target].get("decision")!r}. Merge only into an accepted candidate, '
|
|
f'or accept {cid} instead.')
|
|
elif e.get('merge_into') not in (None, ''):
|
|
warnings.append(f'{cid}: "merge_into" is set but the decision is {decision!r}; it is ignored.')
|
|
|
|
if decision in ('reject', 'skip') and exclude:
|
|
warnings.append(f'{cid}: "exclude" is ignored because the decision is {decision!r}.')
|
|
if decision in ('accept', 'merge') and len(exclude) > len(members) / 2:
|
|
warnings.append(f'{cid}: more than half of its scenes are excluded. Check that the situation '
|
|
f'named is what this candidate is mostly about.')
|
|
|
|
if errors:
|
|
return errors, warnings, None
|
|
|
|
groups = collections.OrderedDict((cid, [cid]) for cid, e in by_id.items() if e['decision'] == 'accept')
|
|
for cid, e in by_id.items():
|
|
if e['decision'] == 'merge':
|
|
groups[e['merge_into']].append(cid)
|
|
|
|
situations = []
|
|
for cid, sources in groups.items():
|
|
members, seen = [], set()
|
|
for src in sources:
|
|
excluded = set(by_id[src].get('exclude', []))
|
|
for m in cands[src]['members']:
|
|
if m['member'] in excluded or m['chunk'] in seen:
|
|
continue
|
|
seen.add(m['chunk'])
|
|
members.append(m)
|
|
size, stories, dominant = shape(members)
|
|
if size < MIN_MEMBERS or stories < MIN_STORIES or dominant > MAX_DOMINANT:
|
|
merged = f' (with {", ".join(sources[1:])} merged in)' if len(sources) > 1 else ''
|
|
errors.append(f'{cid}{merged}: after exclusions {size} scenes remain from {stories} stories, the '
|
|
f'largest story holding {dominant:.0%}. A situation needs at least {MIN_MEMBERS} '
|
|
f'scenes from at least {MIN_STORIES} stories, with no story above '
|
|
f'{MAX_DOMINANT:.0%}. Exclude fewer scenes, merge another candidate into it, '
|
|
f'or reject it.')
|
|
continue
|
|
e = by_id[cid]
|
|
situations.append({
|
|
'id': f'S{len(situations) + 1:02d}',
|
|
'from': sources,
|
|
'situation': _text(e['situation']),
|
|
'actor': _text(e['actor']),
|
|
'counterpart': _text(e['counterpart']),
|
|
'stakes': _text(e.get('stakes')),
|
|
'notes': [_text(by_id[s].get('notes')) for s in sources if _text(by_id[s].get('notes'))],
|
|
'size': size,
|
|
'stories': stories,
|
|
'members': [{k: m[k] for k in ('member', 'chunk', 'story', 'title', 'summary')} for m in members],
|
|
})
|
|
if errors:
|
|
return errors, warnings, None
|
|
|
|
decided = collections.Counter(e['decision'] for e in by_id.values())
|
|
result = {
|
|
'schema_version': SCHEMA_VERSION,
|
|
'run': candidates['run'],
|
|
'counts': {'candidates': len(cands), 'situations': len(situations), 'accepted': decided['accept'],
|
|
'merged': decided['merge'], 'rejected': decided['reject'], 'skipped': decided['skip']},
|
|
'situations': situations,
|
|
'rejected': [cid for cid, e in by_id.items() if e['decision'] == 'reject'],
|
|
'skipped': [cid for cid, e in by_id.items() if e['decision'] == 'skip'],
|
|
}
|
|
return errors, warnings, result
|