"""Choose G3's thresholds on all 175 labelled clusters (SELECTION.md, "G3").

  python3 tune_gate3.py        # writes gate3-tuning.json and gate3-frozen.json
"""
import itertools, json, pathlib
import gate2 as g
import gate3 as g3

G2 = json.loads(pathlib.Path('gate2-frozen.json').read_text())

rows = []
for name in ('dev', 'test'):
    cand = json.loads(pathlib.Path(f'candidates-{name}.json').read_text())
    lab = json.loads(pathlib.Path(f'labels-{name}.json').read_text())
    rows += [dict(g3.signals(m), id=c, label=lab[c]['label']) for c, m in cand.items()]
for mapfile, labfile in (('recover-map.json', 'labels-recover.json'), ('validate-map.json', 'labels-validate.json')):
    mp = json.loads(pathlib.Path(mapfile).read_text())
    lab = json.loads(pathlib.Path(labfile).read_text())
    rows += [dict(g3.signals(e['members']), id=c, label=lab[c]['label']) for c, e in mp.items()]
pathlib.Path('gate3-tuning.json').write_text(json.dumps(rows, indent=1))

def deciles(key):
    v = sorted(r[key] for r in rows)
    return [v[int(len(v) * q / 10)] for q in range(1, 10)]

best = None
for s1, pct, mmin, w in itertools.product(deciles('S1'), [0.90, 0.95, 0.975, 0.99, 0.999],
                                          deciles('MMIN'), [None, 0.30, 0.40, 0.50, 0.60]):
    gate = {'S1': s1, 'PCT': pct, 'MMIN': mmin, 'W': w}
    acc = [r for r in rows if g3.passes(r, gate)]
    if not acc: continue
    n1 = sum(r['label'] >= 1 for r in acc)
    if n1 / len(acc) < 0.85: continue
    key = (n1, sum(r['label'] == 2 for r in acc), -(3 + (w is not None)))
    if best is None or key > best[0]:
        best = (key, gate)
gate = best[1]
pathlib.Path('gate3-frozen.json').write_text(json.dumps(gate, indent=1))

def band(size):
    return '<=9' if size <= 9 else '10-12' if size <= 12 else '13-25' if size <= 25 else '>=26'

rules = (('old gate', g.old_passes), ('G2', lambda r: g.g2_passes(r, G2)), ('G3', lambda r: g3.passes(r, gate)))
print(f'tuning set: {len(rows)} clusters, {sum(r["label"] >= 1 for r in rows)} labelled >=1, '
      f'{sum(r["label"] == 2 for r in rows)} labelled 2')
print('CHOSEN G3:', gate)
for name, accept in rules:
    acc = [r for r in rows if accept(r)]
    n1 = sum(r['label'] >= 1 for r in acc)
    print(f'  {name:9} accepted {len(acc):3}  label>=1 {n1:3}  P1+ {n1 / len(acc) if acc else 0:.2f}  '
          f'label2 {sum(r["label"] == 2 for r in acc)}')
print('\nby size band (accepted / label>=1):')
for b in ('<=9', '10-12', '13-25', '>=26'):
    sub = [r for r in rows if band(r['size']) == b]
    cells = []
    for name, accept in rules:
        acc = [r for r in sub if accept(r)]
        cells.append(f'{name} {len(acc)}/{sum(r["label"] >= 1 for r in acc)}')
    print(f'  {b:6} {len(sub):3} clusters ({sum(r["label"] >= 1 for r in sub)} >=1)   ' + '   '.join(cells))