"""Assemble the hand-written reference (claude/batches/*.json) into files the probe's scripts read,
and measure the hand groups with the probe's own topic-share rule.

  cd probe/wharton && python3 claude_export.py

Writes claude/summaries.json and claude/predicaments.json in the {"summaries": [{chunk, summary}]}
shape that embed.py, topic_share.py and groups_page.py take, so a local-inference run on this corpus
can be set beside it scene for scene. Then reports, for each group in claude/groups.json, the share
of its scenes whose summary contains the group's most common job, relationship or place word
(topic_share.py's measure, applied to hand groups instead of k-means groups), and how many
predicament sentences leak such a word. No inference.
"""
import collections, glob, json, pathlib, re, statistics, sys
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
from topic_words import TOPIC_WORDS

rows = []
for f in sorted(glob.glob('claude/batches/s*.json')):
    rows += json.loads(pathlib.Path(f).read_text(encoding='utf-8'))
chunks = json.loads(pathlib.Path('chunks.json').read_text(encoding='utf-8'))
assert [r['chunk'] for r in rows] == list(range(len(chunks)))
for key, out in (('summary', 'summaries.json'), ('predicament', 'predicaments.json')):
    pathlib.Path('claude', out).write_text(json.dumps(
        {'source': 'hand-written by Claude, reading each scene', 'field': key,
         'summaries': [{'chunk': r['chunk'], 'summary': r[key]} for r in rows]},
        indent=1, ensure_ascii=False), encoding='utf-8')

words = lambda t: set(re.findall(r"[a-zé]+", t.lower())) & TOPIC_WORDS
summ = {r['chunk']: words(r['summary']) for r in rows}
groups = json.loads(pathlib.Path('claude/groups.json').read_text(encoding='utf-8'))['groups']
shares = []
for g in groups:
    m = g['members']
    top = collections.Counter(w for i in m for w in summ[i]).most_common(1)
    word, hits = top[0] if top else ('-', 0)
    shares.append(hits / len(m))
    print(f"{g['id']:18} {len(m):2} scenes  top topic word {word!r:14} in {hits / len(m):4.0%}")
print(f'\nmedian top topic-word share over {len(groups)} hand groups: {statistics.median(shares):.0%}')
leak = [r for r in rows if words(r['predicament'])]
print(f'predicaments containing a topic word: {len(leak)} of {len(rows)} '
      f'({collections.Counter(w for r in leak for w in words(r["predicament"])).most_common(8)})')