TheLadder/tools/story-to-pack/probe/groups_page.py

"""A reading page for regrouped predicaments: does each group read as one predicament?

  cd v2 && python3 ../groups_page.py predicament_embeddings.json predicaments_clean.json > groups-k60-s24.html

Clusters the embeddings at k = 60, seed 24 (the seed the first review used, kept apart from the
measurement seeds 11-13), keeps the candidate groups (at least 5 scenes, 4 stories, no story over 40%),
and lists them most cohesive first. Each scene shows its predicament sentence, the scene's original
summary underneath, and its story. The reviewer marks each group: one predicament, a few mixed, or not
a predicament, with an optional name and note. Marks are kept in the browser and saved as a JSON file.

Naming a kept group ("A ___ wants ___ from ___") is left for the full review page, once groups read well.
No inference.
"""
import collections, html, json, pathlib, re, sys
from topic_words import TOPIC_WORDS

emb_path, text_path = sys.argv[1], sys.argv[2]
K, SEED = 60, 24
sys.argv = ['measure.py', emb_path]
exec(pathlib.Path(__file__).with_name('measure.py').read_text(encoding='utf-8').split("print(f'{path.name}")[0])
# now defined: chunks, V, n, story, kmeans, describe, MIN_SIZE, MIN_STORIES, MAX_DOMINANT

pred = {x['chunk']: x['summary'] for x in json.loads(pathlib.Path(text_path).read_text(encoding='utf-8'))['summaries']}
orig = {x['chunk']: x['summary'] for x in
        json.loads(pathlib.Path('summaries_clean.json').read_text(encoding='utf-8'))['summaries']}


def cross_story_cohesion(group):
    tot = cnt = 0
    for a in group:
        for b in group:
            if a < b and story[a] != story[b]:
                tot += sum(x * y for x, y in zip(V[a], V[b])); cnt += 1
    return tot / cnt if cnt else 0.0


groups = []
for g in kmeans(K, SEED):
    size, nst, dom, _ = describe(g)
    if size < MIN_SIZE or nst < MIN_STORIES or dom > MAX_DOMINANT:
        continue
    topics = collections.Counter(w for i in g for w in set(re.findall(r"[a-zé]+", orig.get(i, '').lower())) & TOPIC_WORDS)
    word, hits = topics.most_common(1)[0] if topics else ('-', 0)
    groups.append({'members': g, 'stories': nst, 'cohesion': cross_story_cohesion(g), 'topic': (word, hits / size)})
groups.sort(key=lambda x: -x['cohesion'])
run = f'{pathlib.Path(emb_path).stem}-k{K}-s{SEED}'

sections = []
for gi, grp in enumerate(groups, 1):
    rows = ''.join(
        f'<li><span class="p">{html.escape(pred.get(i, ""))}</span>'
        f'<span class="o">{html.escape(orig.get(i, ""))}</span>'
        f'<span class="t">{html.escape(chunks[i]["title"].title())}</span></li>'
        for i in grp['members'])
    word, share = grp['topic']
    sections.append(f'''<section id="g{gi}" data-g="{gi}">
<h2>Group {gi}</h2>
<p class="meta">{len(grp["members"])} scenes from {grp["stories"]} stories · cohesion {grp["cohesion"]:.3f} ·
most common job, relationship or place word in the original summaries: <b>{html.escape(word)}</b>, {share:.0%} of scenes</p>
<ol>{rows}</ol>
<div class="verdict">
<label><input type="radio" name="v{gi}" value="one"> One predicament</label>
<label><input type="radio" name="v{gi}" value="mixed"> Two or three mixed together</label>
<label><input type="radio" name="v{gi}" value="none"> Not a predicament</label>
<input type="text" class="name" data-g="{gi}" placeholder="Short name, if it is one (optional)" maxlength="80">
<input type="text" class="note" data-g="{gi}" placeholder="Note (optional)" maxlength="300">
</div></section>''')

index = ''.join(f'<a href="#g{i}" data-g="{i}">{i}</a>' for i in range(1, len(groups) + 1))
data = json.dumps({'run': run, 'groups': [{'group': i, 'chunks': grp['members']} for i, grp in enumerate(groups, 1)]})

print(f'''<!doctype html><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>Predicament groups</title>
<style>
:root{{--bg:#f7f6f2;--card:#fff;--ink:#222;--dim:#666;--line:#ddd;--one:#dff0e3;--mixed:#fdf1d6;--none:#f6dede;--accent:#2f6f4f}}
@media (prefers-color-scheme:dark){{:root{{--bg:#1b1b1d;--card:#252528;--ink:#e6e6e6;--dim:#9a9a9a;--line:#3a3a3e;--one:#233d2a;--mixed:#443a1c;--none:#472828;--accent:#7cc39c}}}}
body{{background:var(--bg);color:var(--ink);font:15px/1.5 system-ui,sans-serif;margin:0;padding:24px 16px}}
main{{max-width:900px;margin:auto}} h1{{margin:0 0 6px}} .dim,.meta{{color:var(--dim)}} .meta{{font-size:13px;margin:0 0 8px}}
#bar{{position:sticky;top:0;background:var(--bg);border-bottom:1px solid var(--line);padding:8px 0;z-index:1}}
#index{{display:flex;flex-wrap:wrap;gap:4px;margin-top:6px}}
#index a{{min-width:26px;text-align:center;font-size:12px;border:1px solid var(--line);border-radius:4px;color:var(--ink);text-decoration:none;padding:1px 3px}}
#index a.one{{background:var(--one)}} #index a.mixed{{background:var(--mixed)}} #index a.none{{background:var(--none)}}
button{{font:inherit;padding:4px 12px;border-radius:6px;border:1px solid var(--accent);background:var(--accent);color:var(--bg);cursor:pointer}}
section{{background:var(--card);border:1px solid var(--line);border-radius:8px;padding:12px 16px;margin:16px 0}}
section.one{{border-left:6px solid #3a9a63}} section.mixed{{border-left:6px solid #c9962b}} section.none{{border-left:6px solid #c45050}}
h2{{margin:0 0 2px;font-size:18px}} ol{{padding-left:22px;margin:6px 0}} li{{margin:0 0 8px}}
.p{{display:block}} .o{{display:block;font-size:13px;color:var(--dim)}} .t{{display:block;font-size:12px;color:var(--dim);font-style:italic}}
.verdict{{display:flex;flex-wrap:wrap;gap:6px 14px;align-items:center;border-top:1px solid var(--line);padding-top:8px}}
.verdict input[type=text]{{flex:1 1 220px;min-width:0;font:inherit;padding:4px 6px;border:1px solid var(--line);border-radius:4px;background:var(--bg);color:var(--ink)}}
</style>
<main>
<h1>Predicament groups</h1>
<p class="dim">Every scene was re-described as its main person's predicament and the scenes were regrouped by
those sentences. Each line below is one scene: its predicament, the scene's original summary under it, and
its story. Read a group and mark whether most of its scenes are <b>one predicament</b> (the same kind of
trouble, and wanting the same kind of way out), <b>two or three mixed together</b>, or <b>not a
predicament</b> (held together by wording, a topic, or nothing). Most cohesive groups first; the first ten
or so are enough to judge. Marks save in this browser as you go; "Save marks" downloads them for the program.</p>
<div id="bar"><span id="tally"></span> <button id="save">Save marks</button><div id="index">{index}</div></div>
{"".join(sections)}
</main>
<script>
const DATA={data.replace("</", "<\\\\/")};
const KEY='stp-groups:'+DATA.run;
let marks={{}}; try{{marks=JSON.parse(localStorage.getItem(KEY)||'{{}}')}}catch(e){{}}
function mark(g){{return marks[g]||(marks[g]={{verdict:'',name:'',note:''}})}}
function paint(){{
  const c={{one:0,mixed:0,none:0}};
  for(const {{group:g}} of DATA.groups){{
    const m=marks[g]||{{}}, v=m.verdict||'';
    if(v) c[v]++;
    document.getElementById('g'+g).className=v;
    document.querySelector(`#index a[data-g="${{g}}"]`).className=v;
  }}
  const done=c.one+c.mixed+c.none;
  document.getElementById('tally').textContent=`Marked ${{done}} of ${{DATA.groups.length}}: one predicament ${{c.one}}, mixed ${{c.mixed}}, not ${{c.none}}`;
}}
function store(){{try{{localStorage.setItem(KEY,JSON.stringify(marks))}}catch(e){{}} paint()}}
for(const [g,m] of Object.entries(marks)){{
  const r=document.querySelector(`input[name="v${{g}}"][value="${{m.verdict}}"]`); if(r) r.checked=true;
  const n=document.querySelector(`.name[data-g="${{g}}"]`); if(n) n.value=m.name||'';
  const t=document.querySelector(`.note[data-g="${{g}}"]`); if(t) t.value=m.note||'';
}}
document.addEventListener('change',e=>{{
  if(e.target.type==='radio'){{mark(e.target.name.slice(1)).verdict=e.target.value; store()}}
}});
document.addEventListener('input',e=>{{
  const g=e.target.dataset.g; if(!g) return;
  mark(g)[e.target.classList.contains('name')?'name':'note']=e.target.value; store();
}});
document.getElementById('save').onclick=()=>{{
  const out={{run:DATA.run,saved:new Date().toISOString(),groups:DATA.groups.map(x=>({{...x,...(marks[x.group]||{{verdict:'',name:'',note:''}})}}))}};
  const a=document.createElement('a');
  a.href=URL.createObjectURL(new Blob([JSON.stringify(out,null,1)],{{type:'application/json'}}));
  a.download=`groups-marks-${{DATA.run}}.json`; a.click(); URL.revokeObjectURL(a.href);
}};
paint();
</script>''')