"""Validate G3 on fresh partitions (SELECTION.md, "G3"). python3 validate_gate3.py build # V0-V3 -> sheet-validate3.txt, validate3-map.json python3 validate_gate3.py score # after labels-validate3.json is filled in """ import json, pathlib, random, sys import gate2 as g import gate3 as g3 mode = sys.argv[1] if len(sys.argv) > 1 else 'build' G3 = json.loads(pathlib.Path('gate3-frozen.json').read_text()) G2 = json.loads(pathlib.Path('gate2-frozen.json').read_text()) GROW_CAP = 45 if mode == 'build': entries = {} def add(members, tag, stop=None): e = entries.setdefault(tuple(sorted(members)), {'sets': [], 'stop': None}) e['sets'].append(tag) if stop: e['stop'] = stop for tag, k, seed, count in (('V0', 150, 20, 25), ('V1', 100, 17, 20), ('V2', 40, 18, 20)): cands = [c for c in g.kmeans(k, seed) if g.eligible(c)] sample = random.Random(seed).sample(cands, min(count, len(cands))) for c in sample: add(c, tag) print(f'{tag}: k={k} seed={seed}: {len(cands)} candidates, sampled {len(sample)}', flush=True) cands = [c for c in g.kmeans(60, 19) if g.eligible(c)] accepted = [c for c in cands if g3.passes(g3.signals(c), G3)] stops = {'gate': 0, 'cap': 0} for c in accepted: m, stop = list(c), 'cap' while len(m) < GROW_CAP: cen, members = g.centroid(m), set(m) j = max((i for i in range(g.n) if i not in members), key=lambda i: g.dot(g.V[i], cen)) if not g3.passes(g3.signals(m + [j]), G3): stop = 'gate' break m.append(j) stops[stop] += 1 add(m, 'V3', stop) print(f'V3: k=60 seed=19: {len(cands)} candidates, G3 accepts {len(accepted)}; ' f'growth stopped by gate {stops["gate"]}, by cap {stops["cap"]}', flush=True) keys = list(entries) rnd = random.Random(1919) rnd.shuffle(keys) mapping = {f'Y{i + 1:03d}': {'members': list(k), **entries[k]} for i, k in enumerate(keys)} pathlib.Path('validate3-map.json').write_text(json.dumps(mapping, indent=1)) lines = [] for yid, e in mapping.items(): shown = e['members'][:] rnd.shuffle(shown) lines.append(f'=== {yid}') lines.extend(f' - {g.text[i]}' for i in shown) lines.append('') pathlib.Path('sheet-validate3.txt').write_text('\n'.join(lines), encoding='utf-8') lab = pathlib.Path('labels-validate3.json') if not lab.exists(): lab.write_text(json.dumps({y: {'label': None, 'name': ''} for y in mapping}, indent=1)) print(f'blind sheet: {len(mapping)} clusters, {sum(len(e["members"]) for e in mapping.values())} lines') elif mode == 'score': mapping = json.loads(pathlib.Path('validate3-map.json').read_text()) labels = json.loads(pathlib.Path('labels-validate3.json').read_text()) assert all(labels[y]['label'] in (0, 1, 2) for y in mapping), 'labels-validate3.json is incomplete' rows = {y: dict(g3.signals(e['members']), label=labels[y]['label'], name=labels[y]['name'], sets=e['sets'], stop=e['stop']) for y, e in mapping.items()} rules = {'old': g.old_passes, 'G2': lambda r: g.g2_passes(r, G2), 'G3': lambda r: g3.passes(r, G3)} def score(ids, accept): acc = [y for y in ids if accept(rows[y])] n1 = sum(rows[y]['label'] >= 1 for y in acc) return len(acc), n1, sum(rows[y]['label'] == 2 for y in acc), (n1 / len(acc) if acc else 0.0) sets = {t: [y for y, r in rows.items() if t in r['sets']] for t in ('V0', 'V1', 'V2', 'V3')} for t in ('V0', 'V1', 'V2'): base = sum(rows[y]['label'] >= 1 for y in sets[t]) print(f'{t}: {len(sets[t])} sampled, {base} labelled >=1') for name, accept in rules.items(): a, n1, n2, p = score(sets[t], accept) print(f' {name:4} accepted {a:2} label>=1 {n1:2} label2 {n2} P1+ {p:.2f}') small = [y for y in set(sets['V0']) | set(sets['V1']) if rows[y]['size'] <= 9] print(f'\n<=9 members in V0+V1: {len(small)} clusters, {sum(rows[y]["label"] >= 1 for y in small)} labelled >=1') for name, accept in rules.items(): a, n1, n2, p = score(small, accept) print(f' {name:4} accepted {a:2} label>=1 {n1:2} P1+ {p:.2f}') v3 = sets['V3'] by_gate = sum(rows[y]['stop'] == 'gate' for y in v3) n1_3 = sum(rows[y]['label'] >= 1 for y in v3) p3 = n1_3 / len(v3) if v3 else 0.0 print(f'\nV3: {len(v3)} grown, stopped by gate {by_gate}, by cap {len(v3) - by_gate}, label>=1 {n1_3}, P1+ {p3:.2f}') v1, v2 = score(sets['V1'], rules['G3']), score(sets['V2'], rules['G3']) s3, s2 = score(small, rules['G3']), score(small, rules['G2']) pool = list(set(sets['V0']) | set(sets['V1']) | set(sets['V2'])) all3, all2 = score(pool, rules['G3']), score(pool, rules['G2']) c1 = v1[0] >= 3 and v1[3] >= 0.80 and v2[0] >= 3 and v2[3] >= 0.80 c2 = s3[0] >= 2 and s3[3] >= 0.67 and s3[0] > s2[0] c3 = bool(v3) and by_gate > len(v3) / 2 and p3 >= 0.80 c4 = all3[1] >= all2[1] and all3[3] >= all2[3] print(f'\nV0-V2 pooled: G3 {all3[0]} accepted, {all3[1]} >=1, P1+ {all3[3]:.2f}; G2 {all2[0]}, {all2[1]}, {all2[3]:.2f}') for i, (c, text) in enumerate(((c1, 'P1+ >= 0.80 and >= 3 accepted on V1 and V2'), (c2, '<=9 members: >= 2 accepted, P1+ >= 0.67, more than G2'), (c3, 'V3 mostly gate-stopped, P1+ >= 0.80'), (c4, 'V0-V2: at least G2\'s label>=1 count at no lower precision')), 1): print(f'criterion {i} ({text}): {"PASS" if c else "fail"}') print(f'OVERALL: {"PASS" if all((c1, c2, c3, c4)) else "fail"}') print('\nper cluster:') for y, r in sorted(rows.items()): print(f" {y} {','.join(r['sets']):6} size {r['size']:2} label {r['label']} " + ' '.join(f"{n} {'Y' if a(r) else '-'}" for n, a in rules.items()) + f" PCT {r['PCT']:.3f} MMIN {r['MMIN']:.3f} stop {r['stop'] or '-':4} {r['name'] or ''}") pathlib.Path('validate3-result.json').write_text(json.dumps( {'criteria': [c1, c2, c3, c4], 'V1': v1, 'V2': v2, 'small_G3': s3, 'small_G2': s2, 'V3': [len(v3), by_gate, n1_3, p3], 'pooled_G3': all3, 'pooled_G2': all2}, indent=1))