"""Near-duplicate situations among accepted clusters (SELECTION.md protocol step 6).

  python3 merge_dupes.py dev            # list every accepted pair's centroid cosine, with label names
  python3 merge_dupes.py test 0.93      # merge pairs at or above a threshold chosen on dev

Centroids are the mean unit summary vector of each cluster. On dev the threshold is chosen by
reading: the lowest cosine that still separates pairs whose names say "same situation" from pairs
that do not. On test it is applied as given, and each merge is reported with both names so it can
be checked.
"""
import json, pathlib, sys

set_name = sys.argv[1]
threshold = float(sys.argv[2]) if len(sys.argv) > 2 else None

sys.argv = ['measure.py', 'summary_embeddings.json']
exec(pathlib.Path('measure.py').read_text(encoding='utf-8').split("print(f'{path.name}")[0])
sys.argv = ['select_rule.py']
exec(pathlib.Path('select_rule.py').read_text(encoding='utf-8').split("mode = sys.argv")[0])

candidates = json.loads(pathlib.Path(f'candidates-{set_name}.json').read_text())
frozen = json.loads(pathlib.Path('rule-frozen.json').read_text())
terms = [tuple(t) for t in frozen['terms']]
rows = {r['id']: r for r in load(set_name)}
accepted = [cid for cid, r in rows.items() if all(passes(r, t) for t in terms)]

cent = {cid: norm([sum(V[i][d] for i in candidates[cid]) / len(candidates[cid]) for d in range(dim)])
        for cid in accepted}
pairs = sorted(((dot(cent[a], cent[b]), a, b) for x, a in enumerate(accepted) for b in accepted[x + 1:]),
               reverse=True)

print(f'{set_name}: {len(accepted)} accepted clusters, {len(pairs)} pairs\n')
for cos, a, b in pairs[:25]:
    mark = '  MERGE' if threshold is not None and cos >= threshold else ''
    print(f"{cos:.4f}  {a}({rows[a]['label']}) {rows[a]['name'] or '-'}\n        {b}({rows[b]['label']}) {rows[b]['name'] or '-'}{mark}")
if threshold is not None:
    merged = [(a, b) for cos, a, b in pairs if cos >= threshold]
    print(f'\nthreshold {threshold}: {len(merged)} merges')