Add story-to-pack research and a structured situation review
Research toward building a content pack from a story corpus, kept on its own branch and independent of the game. Records the selection experiments against blind labels, and settles selection as gate G2 followed by a human review: review.py writes REVIEW.md and a review.json form, apply_review.py checks the filled form and writes situations.json for the next stage. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01C6UDQ9o6L6Ey173U7XVou6
This commit is contained in:
JesseMarkowitz
co-authored by
Claude Opus 5
Claude Opus 5
parent
d84ae495f4
commit
fa3769d0fe
@@ -0,0 +1,120 @@
|
||||
"""Validate G3 on fresh partitions (SELECTION.md, "G3").
|
||||
|
||||
python3 validate_gate3.py build # V0-V3 -> sheet-validate3.txt, validate3-map.json
|
||||
python3 validate_gate3.py score # after labels-validate3.json is filled in
|
||||
"""
|
||||
import json, pathlib, random, sys
|
||||
import gate2 as g
|
||||
import gate3 as g3
|
||||
|
||||
mode = sys.argv[1] if len(sys.argv) > 1 else 'build'
|
||||
G3 = json.loads(pathlib.Path('gate3-frozen.json').read_text())
|
||||
G2 = json.loads(pathlib.Path('gate2-frozen.json').read_text())
|
||||
GROW_CAP = 45
|
||||
|
||||
if mode == 'build':
|
||||
entries = {}
|
||||
|
||||
def add(members, tag, stop=None):
|
||||
e = entries.setdefault(tuple(sorted(members)), {'sets': [], 'stop': None})
|
||||
e['sets'].append(tag)
|
||||
if stop: e['stop'] = stop
|
||||
|
||||
for tag, k, seed, count in (('V0', 150, 20, 25), ('V1', 100, 17, 20), ('V2', 40, 18, 20)):
|
||||
cands = [c for c in g.kmeans(k, seed) if g.eligible(c)]
|
||||
sample = random.Random(seed).sample(cands, min(count, len(cands)))
|
||||
for c in sample: add(c, tag)
|
||||
print(f'{tag}: k={k} seed={seed}: {len(cands)} candidates, sampled {len(sample)}', flush=True)
|
||||
|
||||
cands = [c for c in g.kmeans(60, 19) if g.eligible(c)]
|
||||
accepted = [c for c in cands if g3.passes(g3.signals(c), G3)]
|
||||
stops = {'gate': 0, 'cap': 0}
|
||||
for c in accepted:
|
||||
m, stop = list(c), 'cap'
|
||||
while len(m) < GROW_CAP:
|
||||
cen, members = g.centroid(m), set(m)
|
||||
j = max((i for i in range(g.n) if i not in members), key=lambda i: g.dot(g.V[i], cen))
|
||||
if not g3.passes(g3.signals(m + [j]), G3):
|
||||
stop = 'gate'
|
||||
break
|
||||
m.append(j)
|
||||
stops[stop] += 1
|
||||
add(m, 'V3', stop)
|
||||
print(f'V3: k=60 seed=19: {len(cands)} candidates, G3 accepts {len(accepted)}; '
|
||||
f'growth stopped by gate {stops["gate"]}, by cap {stops["cap"]}', flush=True)
|
||||
|
||||
keys = list(entries)
|
||||
rnd = random.Random(1919)
|
||||
rnd.shuffle(keys)
|
||||
mapping = {f'Y{i + 1:03d}': {'members': list(k), **entries[k]} for i, k in enumerate(keys)}
|
||||
pathlib.Path('validate3-map.json').write_text(json.dumps(mapping, indent=1))
|
||||
lines = []
|
||||
for yid, e in mapping.items():
|
||||
shown = e['members'][:]
|
||||
rnd.shuffle(shown)
|
||||
lines.append(f'=== {yid}')
|
||||
lines.extend(f' - {g.text[i]}' for i in shown)
|
||||
lines.append('')
|
||||
pathlib.Path('sheet-validate3.txt').write_text('\n'.join(lines), encoding='utf-8')
|
||||
lab = pathlib.Path('labels-validate3.json')
|
||||
if not lab.exists():
|
||||
lab.write_text(json.dumps({y: {'label': None, 'name': ''} for y in mapping}, indent=1))
|
||||
print(f'blind sheet: {len(mapping)} clusters, {sum(len(e["members"]) for e in mapping.values())} lines')
|
||||
|
||||
elif mode == 'score':
|
||||
mapping = json.loads(pathlib.Path('validate3-map.json').read_text())
|
||||
labels = json.loads(pathlib.Path('labels-validate3.json').read_text())
|
||||
assert all(labels[y]['label'] in (0, 1, 2) for y in mapping), 'labels-validate3.json is incomplete'
|
||||
rows = {y: dict(g3.signals(e['members']), label=labels[y]['label'], name=labels[y]['name'],
|
||||
sets=e['sets'], stop=e['stop']) for y, e in mapping.items()}
|
||||
rules = {'old': g.old_passes, 'G2': lambda r: g.g2_passes(r, G2), 'G3': lambda r: g3.passes(r, G3)}
|
||||
|
||||
def score(ids, accept):
|
||||
acc = [y for y in ids if accept(rows[y])]
|
||||
n1 = sum(rows[y]['label'] >= 1 for y in acc)
|
||||
return len(acc), n1, sum(rows[y]['label'] == 2 for y in acc), (n1 / len(acc) if acc else 0.0)
|
||||
|
||||
sets = {t: [y for y, r in rows.items() if t in r['sets']] for t in ('V0', 'V1', 'V2', 'V3')}
|
||||
for t in ('V0', 'V1', 'V2'):
|
||||
base = sum(rows[y]['label'] >= 1 for y in sets[t])
|
||||
print(f'{t}: {len(sets[t])} sampled, {base} labelled >=1')
|
||||
for name, accept in rules.items():
|
||||
a, n1, n2, p = score(sets[t], accept)
|
||||
print(f' {name:4} accepted {a:2} label>=1 {n1:2} label2 {n2} P1+ {p:.2f}')
|
||||
|
||||
small = [y for y in set(sets['V0']) | set(sets['V1']) if rows[y]['size'] <= 9]
|
||||
print(f'\n<=9 members in V0+V1: {len(small)} clusters, {sum(rows[y]["label"] >= 1 for y in small)} labelled >=1')
|
||||
for name, accept in rules.items():
|
||||
a, n1, n2, p = score(small, accept)
|
||||
print(f' {name:4} accepted {a:2} label>=1 {n1:2} P1+ {p:.2f}')
|
||||
|
||||
v3 = sets['V3']
|
||||
by_gate = sum(rows[y]['stop'] == 'gate' for y in v3)
|
||||
n1_3 = sum(rows[y]['label'] >= 1 for y in v3)
|
||||
p3 = n1_3 / len(v3) if v3 else 0.0
|
||||
print(f'\nV3: {len(v3)} grown, stopped by gate {by_gate}, by cap {len(v3) - by_gate}, label>=1 {n1_3}, P1+ {p3:.2f}')
|
||||
|
||||
v1, v2 = score(sets['V1'], rules['G3']), score(sets['V2'], rules['G3'])
|
||||
s3, s2 = score(small, rules['G3']), score(small, rules['G2'])
|
||||
pool = list(set(sets['V0']) | set(sets['V1']) | set(sets['V2']))
|
||||
all3, all2 = score(pool, rules['G3']), score(pool, rules['G2'])
|
||||
c1 = v1[0] >= 3 and v1[3] >= 0.80 and v2[0] >= 3 and v2[3] >= 0.80
|
||||
c2 = s3[0] >= 2 and s3[3] >= 0.67 and s3[0] > s2[0]
|
||||
c3 = bool(v3) and by_gate > len(v3) / 2 and p3 >= 0.80
|
||||
c4 = all3[1] >= all2[1] and all3[3] >= all2[3]
|
||||
print(f'\nV0-V2 pooled: G3 {all3[0]} accepted, {all3[1]} >=1, P1+ {all3[3]:.2f}; G2 {all2[0]}, {all2[1]}, {all2[3]:.2f}')
|
||||
for i, (c, text) in enumerate(((c1, 'P1+ >= 0.80 and >= 3 accepted on V1 and V2'),
|
||||
(c2, '<=9 members: >= 2 accepted, P1+ >= 0.67, more than G2'),
|
||||
(c3, 'V3 mostly gate-stopped, P1+ >= 0.80'),
|
||||
(c4, 'V0-V2: at least G2\'s label>=1 count at no lower precision')), 1):
|
||||
print(f'criterion {i} ({text}): {"PASS" if c else "fail"}')
|
||||
print(f'OVERALL: {"PASS" if all((c1, c2, c3, c4)) else "fail"}')
|
||||
|
||||
print('\nper cluster:')
|
||||
for y, r in sorted(rows.items()):
|
||||
print(f" {y} {','.join(r['sets']):6} size {r['size']:2} label {r['label']} "
|
||||
+ ' '.join(f"{n} {'Y' if a(r) else '-'}" for n, a in rules.items())
|
||||
+ f" PCT {r['PCT']:.3f} MMIN {r['MMIN']:.3f} stop {r['stop'] or '-':4} {r['name'] or ''}")
|
||||
pathlib.Path('validate3-result.json').write_text(json.dumps(
|
||||
{'criteria': [c1, c2, c3, c4], 'V1': v1, 'V2': v2, 'small_G3': s3, 'small_G2': s2,
|
||||
'V3': [len(v3), by_gate, n1_3, p3], 'pooled_G3': all3, 'pooled_G2': all2}, indent=1))
|
||||
Reference in New Issue
Block a user