JesseMarkowitzandClaude Opus 5
fa3769d0fe
Research toward building a content pack from a story corpus, kept on its own branch and independent of the game. Records the selection experiments against blind labels, and settles selection as gate G2 followed by a human review: review.py writes REVIEW.md and a review.json form, apply_review.py checks the filled form and writes situations.json for the next stage. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01C6UDQ9o6L6Ey173U7XVou6
171 lines
7.0 KiB
Python
171 lines
7.0 KiB
Python
"""Recall recovery: baseline gate and three methods on the seed-13 partition (SELECTION.md).
|
||
|
||
python3 recover.py
|
||
|
||
Writes:
|
||
recover-map.json sheet id -> member chunks and which methods produced it (do not read before labelling)
|
||
sheet-recover.txt blind sheet: member summaries only, shuffled
|
||
labels-recover.json empty label template (only if absent)
|
||
Prints counts per method only.
|
||
"""
|
||
import collections, json, math, pathlib, random, re, sys
|
||
|
||
SEED, K = 13, 60
|
||
MIN_GATE_SIZE, GROW_CAP = 13, 30
|
||
|
||
sys.argv = ['measure.py', 'summary_embeddings.json']
|
||
exec(pathlib.Path('measure.py').read_text(encoding='utf-8').split("print(f'{path.name}")[0])
|
||
# defined: chunks, V, n, dim, story, dot, norm, kmeans (on V), describe, MIN_SIZE, MIN_STORIES, MAX_DOMINANT
|
||
|
||
frozen = json.loads(pathlib.Path('rule-frozen.json').read_text())
|
||
CUT = {s: t for s, _, t in frozen['terms']}
|
||
S1_CUT, S6_CUT = CUT['S1'], CUT['S6']
|
||
|
||
summaries = json.loads(pathlib.Path('summaries_clean.json').read_text(encoding='utf-8'))['summaries']
|
||
text = {x['chunk']: x['summary'] for x in summaries}
|
||
|
||
# Identical to signals.py.
|
||
STOP = set("""a an the and or but of to in on at for with by from as into over under about after before
|
||
while during than then that this these those it its his her hers him he she they them their theirs we our
|
||
you your i me my is are was were be been being has have had do does did not no nor so such who whom whose
|
||
which what when where why how all any each both either neither one two some other others another more most
|
||
less least very just only also even yet still now up down out off away back again once own same too can
|
||
could will would shall should may might must upon despite toward towards between among amid amidst
|
||
someone someone's someone’s""".split())
|
||
docs = [set(re.findall(r"[a-z][a-z'’]+", text[i].lower())) - STOP for i in range(n)]
|
||
df = collections.Counter(w for d in docs for w in d)
|
||
|
||
def s1(m):
|
||
tot, cnt = 0.0, 0
|
||
for x in range(len(m)):
|
||
for y in range(x + 1, len(m)):
|
||
if story[m[x]] != story[m[y]]:
|
||
tot += dot(V[m[x]], V[m[y]]); cnt += 1
|
||
return tot / cnt if cnt else 0.0
|
||
|
||
def s6(m):
|
||
size = len(m)
|
||
cw = collections.Counter(w for i in m for w in docs[i])
|
||
best = None
|
||
for w, c in cw.items():
|
||
if c < 2: continue
|
||
score = math.log((c + .5) / (size - c + .5)) - math.log((df[w] + .5) / (n - df[w] + .5))
|
||
if best is None or score > best[0]: best = (score, c)
|
||
return best[1] / size if best else 0.0
|
||
|
||
def eligible(m):
|
||
size, nst, dom, _ = describe(m)
|
||
return size >= MIN_SIZE and nst >= MIN_STORIES and dom <= MAX_DOMINANT
|
||
|
||
def passes(m):
|
||
return eligible(m) and s1(m) >= S1_CUT and s6(m) <= S6_CUT
|
||
|
||
# Guard against drift from signals.py: recompute S1/S6 for the dev candidates.
|
||
dev_sig = json.loads(pathlib.Path('signals-dev.json').read_text())
|
||
dev_cand = json.loads(pathlib.Path('candidates-dev.json').read_text())
|
||
for cid, m in dev_cand.items():
|
||
assert abs(s1(m) - dev_sig[cid]['S1']) < 1e-3 and abs(s6(m) - dev_sig[cid]['S6']) < 1e-3, cid
|
||
|
||
def kmeans_on(vecs, k, seed, iters=15):
|
||
"""Same algorithm as measure.kmeans, over arbitrary unit vectors; returns index groups."""
|
||
m, d = len(vecs), len(vecs[0])
|
||
rnd = random.Random(seed)
|
||
cent = [vecs[i] for i in rnd.sample(range(m), k)]
|
||
assign = [-1] * m
|
||
for _ in range(iters):
|
||
moved = 0
|
||
for i, v in enumerate(vecs):
|
||
best, bs = 0, -2.0
|
||
for j, c in enumerate(cent):
|
||
s = sum(a * b for a, b in zip(v, c))
|
||
if s > bs: bs, best = s, j
|
||
if assign[i] != best: assign[i] = best; moved += 1
|
||
if moved == 0: break
|
||
groups = collections.defaultdict(list)
|
||
for i, j in enumerate(assign): groups[j].append(i)
|
||
for j, mem in groups.items():
|
||
cent[j] = norm([sum(vecs[i][x] for i in mem) / len(mem) for x in range(d)])
|
||
groups = collections.defaultdict(list)
|
||
for i, j in enumerate(assign): groups[j].append(i)
|
||
return list(groups.values())
|
||
|
||
def centroid(m):
|
||
return norm([sum(V[i][x] for i in m) / len(m) for x in range(dim)])
|
||
|
||
# Baseline
|
||
partition = [sorted(c) for c in kmeans(K, SEED)]
|
||
candidates = [c for c in partition if eligible(c)]
|
||
B = [c for c in candidates if passes(c)]
|
||
print(f'seed {SEED}: {len(candidates)} candidates; B accepts {len(B)}', flush=True)
|
||
|
||
# M1: re-cluster everything outside B's accepted clusters
|
||
in_B = {i for c in B for i in c}
|
||
pool = [i for i in range(n) if i not in in_B]
|
||
k1 = max(2, round(len(pool) / 20))
|
||
M1 = [list(c) for c in B]
|
||
for g in kmeans_on([V[i] for i in pool], k1, SEED):
|
||
m = sorted(pool[x] for x in g)
|
||
if passes(m): M1.append(m)
|
||
print(f'M1: pool {len(pool)}, k={k1}; accepts {len(M1)} ({len(M1) - len(B)} new)', flush=True)
|
||
|
||
# M2: consensus clustering over co-association rows
|
||
co = json.loads(pathlib.Path('coassoc.json').read_text())
|
||
rows = [norm([float(x) for x in co[i]]) for i in range(n)]
|
||
M2 = [sorted(g) for g in kmeans_on(rows, K, SEED) if passes(sorted(g))]
|
||
print(f'M2: accepts {len(M2)}', flush=True)
|
||
|
||
# M3: prune failing candidates to a passing core, then grow every passing cluster
|
||
def prune(m):
|
||
m = list(m)
|
||
while not passes(m):
|
||
if len(m) <= MIN_GATE_SIZE: return None
|
||
def cross_mean(x):
|
||
others = [y for y in m if story[y] != story[x]]
|
||
return sum(dot(V[x], V[y]) for y in others) / len(others) if others else -1.0
|
||
m.remove(min(m, key=cross_mean))
|
||
return m
|
||
|
||
def grow(m):
|
||
m = list(m)
|
||
while len(m) < GROW_CAP:
|
||
cen = centroid(m)
|
||
members = set(m)
|
||
j = max((i for i in range(n) if i not in members), key=lambda i: dot(V[i], cen))
|
||
if not passes(m + [j]): break
|
||
m.append(j)
|
||
return sorted(m)
|
||
|
||
M3, pruned_ok = [], 0
|
||
for c in candidates:
|
||
core = c if passes(c) else prune(c)
|
||
if core is None: continue
|
||
if core is not c: pruned_ok += 1
|
||
M3.append(grow(core))
|
||
cover3 = collections.Counter(i for c in M3 for i in c)
|
||
print(f'M3: accepts {len(M3)} ({pruned_ok} recovered by pruning); scenes in >1 cluster: {sum(v > 1 for v in cover3.values())}', flush=True)
|
||
|
||
# Pool for blind labelling
|
||
methods = {'B': B, 'M1': M1, 'M2': M2, 'M3': M3}
|
||
union = {}
|
||
for name, clusters in methods.items():
|
||
for c in clusters:
|
||
union.setdefault(tuple(sorted(c)), []).append(name)
|
||
keys = list(union)
|
||
rnd = random.Random(1307)
|
||
rnd.shuffle(keys)
|
||
mapping = {f'R{i + 1:03d}': {'members': list(k), 'methods': union[k]} for i, k in enumerate(keys)}
|
||
pathlib.Path('recover-map.json').write_text(json.dumps(mapping, indent=1))
|
||
|
||
lines = []
|
||
for rid, entry in mapping.items():
|
||
shown = entry['members'][:]
|
||
rnd.shuffle(shown)
|
||
lines.append(f'=== {rid}')
|
||
lines.extend(f' - {text[i]}' for i in shown)
|
||
lines.append('')
|
||
pathlib.Path('sheet-recover.txt').write_text('\n'.join(lines), encoding='utf-8')
|
||
lab = pathlib.Path('labels-recover.json')
|
||
if not lab.exists():
|
||
lab.write_text(json.dumps({rid: {'label': None, 'name': ''} for rid in mapping}, indent=1))
|
||
print(f'blind sheet: {len(mapping)} distinct clusters, {sum(len(e["members"]) for e in mapping.values())} lines')
|