JesseMarkowitzandClaude Opus 5
fa3769d0fe
Research toward building a content pack from a story corpus, kept on its own branch and independent of the game. Records the selection experiments against blind labels, and settles selection as gate G2 followed by a human review: review.py writes REVIEW.md and a review.json form, apply_review.py checks the filled form and writes situations.json for the next stage. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01C6UDQ9o6L6Ey173U7XVou6
60 lines
2.8 KiB
Python
60 lines
2.8 KiB
Python
"""Choose G3's thresholds on all 175 labelled clusters (SELECTION.md, "G3").
|
|
|
|
python3 tune_gate3.py # writes gate3-tuning.json and gate3-frozen.json
|
|
"""
|
|
import itertools, json, pathlib
|
|
import gate2 as g
|
|
import gate3 as g3
|
|
|
|
G2 = json.loads(pathlib.Path('gate2-frozen.json').read_text())
|
|
|
|
rows = []
|
|
for name in ('dev', 'test'):
|
|
cand = json.loads(pathlib.Path(f'candidates-{name}.json').read_text())
|
|
lab = json.loads(pathlib.Path(f'labels-{name}.json').read_text())
|
|
rows += [dict(g3.signals(m), id=c, label=lab[c]['label']) for c, m in cand.items()]
|
|
for mapfile, labfile in (('recover-map.json', 'labels-recover.json'), ('validate-map.json', 'labels-validate.json')):
|
|
mp = json.loads(pathlib.Path(mapfile).read_text())
|
|
lab = json.loads(pathlib.Path(labfile).read_text())
|
|
rows += [dict(g3.signals(e['members']), id=c, label=lab[c]['label']) for c, e in mp.items()]
|
|
pathlib.Path('gate3-tuning.json').write_text(json.dumps(rows, indent=1))
|
|
|
|
def deciles(key):
|
|
v = sorted(r[key] for r in rows)
|
|
return [v[int(len(v) * q / 10)] for q in range(1, 10)]
|
|
|
|
best = None
|
|
for s1, pct, mmin, w in itertools.product(deciles('S1'), [0.90, 0.95, 0.975, 0.99, 0.999],
|
|
deciles('MMIN'), [None, 0.30, 0.40, 0.50, 0.60]):
|
|
gate = {'S1': s1, 'PCT': pct, 'MMIN': mmin, 'W': w}
|
|
acc = [r for r in rows if g3.passes(r, gate)]
|
|
if not acc: continue
|
|
n1 = sum(r['label'] >= 1 for r in acc)
|
|
if n1 / len(acc) < 0.85: continue
|
|
key = (n1, sum(r['label'] == 2 for r in acc), -(3 + (w is not None)))
|
|
if best is None or key > best[0]:
|
|
best = (key, gate)
|
|
gate = best[1]
|
|
pathlib.Path('gate3-frozen.json').write_text(json.dumps(gate, indent=1))
|
|
|
|
def band(size):
|
|
return '<=9' if size <= 9 else '10-12' if size <= 12 else '13-25' if size <= 25 else '>=26'
|
|
|
|
rules = (('old gate', g.old_passes), ('G2', lambda r: g.g2_passes(r, G2)), ('G3', lambda r: g3.passes(r, gate)))
|
|
print(f'tuning set: {len(rows)} clusters, {sum(r["label"] >= 1 for r in rows)} labelled >=1, '
|
|
f'{sum(r["label"] == 2 for r in rows)} labelled 2')
|
|
print('CHOSEN G3:', gate)
|
|
for name, accept in rules:
|
|
acc = [r for r in rows if accept(r)]
|
|
n1 = sum(r['label'] >= 1 for r in acc)
|
|
print(f' {name:9} accepted {len(acc):3} label>=1 {n1:3} P1+ {n1 / len(acc) if acc else 0:.2f} '
|
|
f'label2 {sum(r["label"] == 2 for r in acc)}')
|
|
print('\nby size band (accepted / label>=1):')
|
|
for b in ('<=9', '10-12', '13-25', '>=26'):
|
|
sub = [r for r in rows if band(r['size']) == b]
|
|
cells = []
|
|
for name, accept in rules:
|
|
acc = [r for r in sub if accept(r)]
|
|
cells.append(f'{name} {len(acc)}/{sum(r["label"] >= 1 for r in acc)}')
|
|
print(f' {b:6} {len(sub):3} clusters ({sum(r["label"] >= 1 for r in sub)} >=1) ' + ' '.join(cells))
|