"""Recall recovery: baseline gate and three methods on the seed-13 partition (SELECTION.md). python3 recover.py Writes: recover-map.json sheet id -> member chunks and which methods produced it (do not read before labelling) sheet-recover.txt blind sheet: member summaries only, shuffled labels-recover.json empty label template (only if absent) Prints counts per method only. """ import collections, json, math, pathlib, random, re, sys SEED, K = 13, 60 MIN_GATE_SIZE, GROW_CAP = 13, 30 sys.argv = ['measure.py', 'summary_embeddings.json'] exec(pathlib.Path('measure.py').read_text(encoding='utf-8').split("print(f'{path.name}")[0]) # defined: chunks, V, n, dim, story, dot, norm, kmeans (on V), describe, MIN_SIZE, MIN_STORIES, MAX_DOMINANT frozen = json.loads(pathlib.Path('rule-frozen.json').read_text()) CUT = {s: t for s, _, t in frozen['terms']} S1_CUT, S6_CUT = CUT['S1'], CUT['S6'] summaries = json.loads(pathlib.Path('summaries_clean.json').read_text(encoding='utf-8'))['summaries'] text = {x['chunk']: x['summary'] for x in summaries} # Identical to signals.py. STOP = set("""a an the and or but of to in on at for with by from as into over under about after before while during than then that this these those it its his her hers him he she they them their theirs we our you your i me my is are was were be been being has have had do does did not no nor so such who whom whose which what when where why how all any each both either neither one two some other others another more most less least very just only also even yet still now up down out off away back again once own same too can could will would shall should may might must upon despite toward towards between among amid amidst someone someone's someone’s""".split()) docs = [set(re.findall(r"[a-z][a-z'’]+", text[i].lower())) - STOP for i in range(n)] df = collections.Counter(w for d in docs for w in d) def s1(m): tot, cnt = 0.0, 0 for x in range(len(m)): for y in range(x + 1, len(m)): if story[m[x]] != story[m[y]]: tot += dot(V[m[x]], V[m[y]]); cnt += 1 return tot / cnt if cnt else 0.0 def s6(m): size = len(m) cw = collections.Counter(w for i in m for w in docs[i]) best = None for w, c in cw.items(): if c < 2: continue score = math.log((c + .5) / (size - c + .5)) - math.log((df[w] + .5) / (n - df[w] + .5)) if best is None or score > best[0]: best = (score, c) return best[1] / size if best else 0.0 def eligible(m): size, nst, dom, _ = describe(m) return size >= MIN_SIZE and nst >= MIN_STORIES and dom <= MAX_DOMINANT def passes(m): return eligible(m) and s1(m) >= S1_CUT and s6(m) <= S6_CUT # Guard against drift from signals.py: recompute S1/S6 for the dev candidates. dev_sig = json.loads(pathlib.Path('signals-dev.json').read_text()) dev_cand = json.loads(pathlib.Path('candidates-dev.json').read_text()) for cid, m in dev_cand.items(): assert abs(s1(m) - dev_sig[cid]['S1']) < 1e-3 and abs(s6(m) - dev_sig[cid]['S6']) < 1e-3, cid def kmeans_on(vecs, k, seed, iters=15): """Same algorithm as measure.kmeans, over arbitrary unit vectors; returns index groups.""" m, d = len(vecs), len(vecs[0]) rnd = random.Random(seed) cent = [vecs[i] for i in rnd.sample(range(m), k)] assign = [-1] * m for _ in range(iters): moved = 0 for i, v in enumerate(vecs): best, bs = 0, -2.0 for j, c in enumerate(cent): s = sum(a * b for a, b in zip(v, c)) if s > bs: bs, best = s, j if assign[i] != best: assign[i] = best; moved += 1 if moved == 0: break groups = collections.defaultdict(list) for i, j in enumerate(assign): groups[j].append(i) for j, mem in groups.items(): cent[j] = norm([sum(vecs[i][x] for i in mem) / len(mem) for x in range(d)]) groups = collections.defaultdict(list) for i, j in enumerate(assign): groups[j].append(i) return list(groups.values()) def centroid(m): return norm([sum(V[i][x] for i in m) / len(m) for x in range(dim)]) # Baseline partition = [sorted(c) for c in kmeans(K, SEED)] candidates = [c for c in partition if eligible(c)] B = [c for c in candidates if passes(c)] print(f'seed {SEED}: {len(candidates)} candidates; B accepts {len(B)}', flush=True) # M1: re-cluster everything outside B's accepted clusters in_B = {i for c in B for i in c} pool = [i for i in range(n) if i not in in_B] k1 = max(2, round(len(pool) / 20)) M1 = [list(c) for c in B] for g in kmeans_on([V[i] for i in pool], k1, SEED): m = sorted(pool[x] for x in g) if passes(m): M1.append(m) print(f'M1: pool {len(pool)}, k={k1}; accepts {len(M1)} ({len(M1) - len(B)} new)', flush=True) # M2: consensus clustering over co-association rows co = json.loads(pathlib.Path('coassoc.json').read_text()) rows = [norm([float(x) for x in co[i]]) for i in range(n)] M2 = [sorted(g) for g in kmeans_on(rows, K, SEED) if passes(sorted(g))] print(f'M2: accepts {len(M2)}', flush=True) # M3: prune failing candidates to a passing core, then grow every passing cluster def prune(m): m = list(m) while not passes(m): if len(m) <= MIN_GATE_SIZE: return None def cross_mean(x): others = [y for y in m if story[y] != story[x]] return sum(dot(V[x], V[y]) for y in others) / len(others) if others else -1.0 m.remove(min(m, key=cross_mean)) return m def grow(m): m = list(m) while len(m) < GROW_CAP: cen = centroid(m) members = set(m) j = max((i for i in range(n) if i not in members), key=lambda i: dot(V[i], cen)) if not passes(m + [j]): break m.append(j) return sorted(m) M3, pruned_ok = [], 0 for c in candidates: core = c if passes(c) else prune(c) if core is None: continue if core is not c: pruned_ok += 1 M3.append(grow(core)) cover3 = collections.Counter(i for c in M3 for i in c) print(f'M3: accepts {len(M3)} ({pruned_ok} recovered by pruning); scenes in >1 cluster: {sum(v > 1 for v in cover3.values())}', flush=True) # Pool for blind labelling methods = {'B': B, 'M1': M1, 'M2': M2, 'M3': M3} union = {} for name, clusters in methods.items(): for c in clusters: union.setdefault(tuple(sorted(c)), []).append(name) keys = list(union) rnd = random.Random(1307) rnd.shuffle(keys) mapping = {f'R{i + 1:03d}': {'members': list(k), 'methods': union[k]} for i, k in enumerate(keys)} pathlib.Path('recover-map.json').write_text(json.dumps(mapping, indent=1)) lines = [] for rid, entry in mapping.items(): shown = entry['members'][:] rnd.shuffle(shown) lines.append(f'=== {rid}') lines.extend(f' - {text[i]}' for i in shown) lines.append('') pathlib.Path('sheet-recover.txt').write_text('\n'.join(lines), encoding='utf-8') lab = pathlib.Path('labels-recover.json') if not lab.exists(): lab.write_text(json.dumps({rid: {'label': None, 'name': ''} for rid in mapping}, indent=1)) print(f'blind sheet: {len(mapping)} distinct clusters, {sum(len(e["members"]) for e in mapping.values())} lines')