Add story-to-pack research and a structured situation review
Research toward building a content pack from a story corpus, kept on its own branch and independent of the game. Records the selection experiments against blind labels, and settles selection as gate G2 followed by a human review: review.py writes REVIEW.md and a review.json form, apply_review.py checks the filled form and writes situations.json for the next stage. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01C6UDQ9o6L6Ey173U7XVou6
This commit is contained in:
JesseMarkowitz
co-authored by
Claude Opus 5
Claude Opus 5
parent
d84ae495f4
commit
fa3769d0fe
@@ -0,0 +1,202 @@
|
||||
"""The human review form: its schema, member ids, checks, and conversion to situations.json.
|
||||
|
||||
Kept free of clustering code so apply_review.py and the tests load instantly.
|
||||
|
||||
A review run has three files in one directory:
|
||||
candidates.json written by review.py: what the program found (read-only)
|
||||
review.json written blank by review.py, filled in by the reviewer
|
||||
situations.json written by apply_review.py once review.json passes every check
|
||||
"""
|
||||
import collections
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
MIN_MEMBERS, MIN_STORIES, MAX_DOMINANT = 5, 4, 0.40
|
||||
MAX_SITUATION_CHARS = 200
|
||||
DECISIONS = {
|
||||
'accepted': ('accept', 'reject', 'merge'), # passed the gate: the reviewer must decide
|
||||
'borderline': ('skip', 'accept', 'merge'), # narrowly failed: optional, "skip" by default
|
||||
}
|
||||
|
||||
|
||||
def member_id(cid, n):
|
||||
"""C04.07 — the 7th scene of candidate C04."""
|
||||
return f'{cid}.{n:02d}'
|
||||
|
||||
|
||||
def blank_entry(cid, kind):
|
||||
return {
|
||||
'id': cid,
|
||||
'kind': kind,
|
||||
'decision': None if kind == 'accepted' else 'skip',
|
||||
'situation': '',
|
||||
'actor': '',
|
||||
'counterpart': '',
|
||||
'stakes': '',
|
||||
'exclude': [],
|
||||
'merge_into': None,
|
||||
'notes': '',
|
||||
}
|
||||
|
||||
|
||||
def blank_form(candidates):
|
||||
return {
|
||||
'schema_version': SCHEMA_VERSION,
|
||||
'run': candidates['run']['id'],
|
||||
'instructions': 'Read REVIEW.md in this directory. Fill in each entry below, then run: '
|
||||
f'python3 apply_review.py review/{candidates["run"]["id"]}',
|
||||
'entries': [blank_entry(cid, c['kind']) for cid, c in candidates['candidates'].items()],
|
||||
}
|
||||
|
||||
|
||||
def shape(members):
|
||||
"""(scene count, story count, share held by the largest story)."""
|
||||
counts = collections.Counter(m['story'] for m in members)
|
||||
return len(members), len(counts), (max(counts.values()) / len(members) if members else 0.0)
|
||||
|
||||
|
||||
def _text(value):
|
||||
return value.strip() if isinstance(value, str) else ''
|
||||
|
||||
|
||||
def validate(candidates, form):
|
||||
"""Return (errors, warnings, situations). situations is None whenever there is any error.
|
||||
|
||||
Every problem is collected, so the reviewer can fix them all in one pass.
|
||||
"""
|
||||
errors, warnings = [], []
|
||||
cands = candidates['candidates']
|
||||
|
||||
if form.get('schema_version') != SCHEMA_VERSION:
|
||||
errors.append(f'"schema_version" is {form.get("schema_version")!r}; this checker reads {SCHEMA_VERSION}.')
|
||||
if form.get('run') != candidates['run']['id']:
|
||||
errors.append(f'This form is for run {form.get("run")!r}, but candidates.json is run '
|
||||
f'{candidates["run"]["id"]!r}. Use the review.json from the same directory.')
|
||||
entries = form.get('entries')
|
||||
if not isinstance(entries, list):
|
||||
errors.append('"entries" must be a list of candidate entries.')
|
||||
return errors, warnings, None
|
||||
|
||||
by_id = {}
|
||||
for e in entries:
|
||||
cid = e.get('id') if isinstance(e, dict) else None
|
||||
if cid not in cands:
|
||||
errors.append(f'An entry has id {cid!r}, which is not a candidate in this run.')
|
||||
elif cid in by_id:
|
||||
errors.append(f'{cid}: appears more than once; keep one entry.')
|
||||
else:
|
||||
by_id[cid] = e
|
||||
for cid in cands:
|
||||
if cid not in by_id:
|
||||
errors.append(f'{cid}: its entry is missing from the form. Restore it (the blank entry is '
|
||||
f'{{"id": "{cid}", ...}}) and set "decision".')
|
||||
|
||||
for cid, e in by_id.items():
|
||||
kind = cands[cid]['kind']
|
||||
allowed = DECISIONS[kind]
|
||||
decision = e.get('decision')
|
||||
if decision is None:
|
||||
errors.append(f'{cid}: no decision yet. Set "decision" to one of: {", ".join(allowed)}.')
|
||||
continue
|
||||
if decision not in allowed:
|
||||
errors.append(f'{cid}: "decision" is {decision!r}; for a {kind} candidate use one of: '
|
||||
f'{", ".join(allowed)}.')
|
||||
continue
|
||||
|
||||
members = cands[cid]['members']
|
||||
valid = {m['member'] for m in members}
|
||||
exclude = e.get('exclude', [])
|
||||
if not isinstance(exclude, list):
|
||||
errors.append(f'{cid}: "exclude" must be a list of member ids, like ["{cid}.03"].')
|
||||
exclude = []
|
||||
unknown = [x for x in exclude if x not in valid]
|
||||
if unknown:
|
||||
errors.append(f'{cid}: "exclude" lists {", ".join(map(str, unknown))}, which are not scenes of '
|
||||
f'{cid}; its scenes are {member_id(cid, 1)} to {member_id(cid, len(members))}.')
|
||||
|
||||
if decision == 'accept':
|
||||
for field in ('situation', 'actor', 'counterpart'):
|
||||
if not _text(e.get(field)):
|
||||
errors.append(f'{cid}: accepted, so "{field}" must be filled in.')
|
||||
situation = _text(e.get('situation'))
|
||||
if len(situation) > MAX_SITUATION_CHARS:
|
||||
errors.append(f'{cid}: "situation" is {len(situation)} characters; keep it to one sentence '
|
||||
f'of at most {MAX_SITUATION_CHARS}.')
|
||||
if situation and 'want' not in situation.lower():
|
||||
warnings.append(f'{cid}: "situation" does not say who wants what ("… wants … from …"). '
|
||||
f'Accepted as written.')
|
||||
if not _text(e.get('stakes')):
|
||||
warnings.append(f'{cid}: "stakes" is empty.')
|
||||
|
||||
if decision == 'merge':
|
||||
target = e.get('merge_into')
|
||||
if target == cid:
|
||||
errors.append(f'{cid}: cannot merge into itself.')
|
||||
elif target not in by_id:
|
||||
errors.append(f'{cid}: "merge_into" is {target!r}, which is not a candidate id in this form.')
|
||||
elif by_id[target].get('decision') != 'accept':
|
||||
errors.append(f'{cid}: merges into {target}, but {target} has decision '
|
||||
f'{by_id[target].get("decision")!r}. Merge only into an accepted candidate, '
|
||||
f'or accept {cid} instead.')
|
||||
elif e.get('merge_into') not in (None, ''):
|
||||
warnings.append(f'{cid}: "merge_into" is set but the decision is {decision!r}; it is ignored.')
|
||||
|
||||
if decision in ('reject', 'skip') and exclude:
|
||||
warnings.append(f'{cid}: "exclude" is ignored because the decision is {decision!r}.')
|
||||
if decision in ('accept', 'merge') and len(exclude) > len(members) / 2:
|
||||
warnings.append(f'{cid}: more than half of its scenes are excluded. Check that the situation '
|
||||
f'named is what this candidate is mostly about.')
|
||||
|
||||
if errors:
|
||||
return errors, warnings, None
|
||||
|
||||
groups = collections.OrderedDict((cid, [cid]) for cid, e in by_id.items() if e['decision'] == 'accept')
|
||||
for cid, e in by_id.items():
|
||||
if e['decision'] == 'merge':
|
||||
groups[e['merge_into']].append(cid)
|
||||
|
||||
situations = []
|
||||
for cid, sources in groups.items():
|
||||
members, seen = [], set()
|
||||
for src in sources:
|
||||
excluded = set(by_id[src].get('exclude', []))
|
||||
for m in cands[src]['members']:
|
||||
if m['member'] in excluded or m['chunk'] in seen:
|
||||
continue
|
||||
seen.add(m['chunk'])
|
||||
members.append(m)
|
||||
size, stories, dominant = shape(members)
|
||||
if size < MIN_MEMBERS or stories < MIN_STORIES or dominant > MAX_DOMINANT:
|
||||
merged = f' (with {", ".join(sources[1:])} merged in)' if len(sources) > 1 else ''
|
||||
errors.append(f'{cid}{merged}: after exclusions {size} scenes remain from {stories} stories, the '
|
||||
f'largest story holding {dominant:.0%}. A situation needs at least {MIN_MEMBERS} '
|
||||
f'scenes from at least {MIN_STORIES} stories, with no story above '
|
||||
f'{MAX_DOMINANT:.0%}. Exclude fewer scenes, merge another candidate into it, '
|
||||
f'or reject it.')
|
||||
continue
|
||||
e = by_id[cid]
|
||||
situations.append({
|
||||
'id': f'S{len(situations) + 1:02d}',
|
||||
'from': sources,
|
||||
'situation': _text(e['situation']),
|
||||
'actor': _text(e['actor']),
|
||||
'counterpart': _text(e['counterpart']),
|
||||
'stakes': _text(e.get('stakes')),
|
||||
'notes': [_text(by_id[s].get('notes')) for s in sources if _text(by_id[s].get('notes'))],
|
||||
'size': size,
|
||||
'stories': stories,
|
||||
'members': [{k: m[k] for k in ('member', 'chunk', 'story', 'title', 'summary')} for m in members],
|
||||
})
|
||||
if errors:
|
||||
return errors, warnings, None
|
||||
|
||||
decided = collections.Counter(e['decision'] for e in by_id.values())
|
||||
result = {
|
||||
'schema_version': SCHEMA_VERSION,
|
||||
'run': candidates['run'],
|
||||
'counts': {'candidates': len(cands), 'situations': len(situations), 'accepted': decided['accept'],
|
||||
'merged': decided['merge'], 'rejected': decided['reject'], 'skipped': decided['skip']},
|
||||
'situations': situations,
|
||||
'rejected': [cid for cid, e in by_id.items() if e['decision'] == 'reject'],
|
||||
'skipped': [cid for cid, e in by_id.items() if e['decision'] == 'skip'],
|
||||
}
|
||||
return errors, warnings, result
|
||||
Reference in New Issue
Block a user