#!/usr/bin/env python3 """Assemble records.json, groups.json and options_by_entry.json from batches/, groups_src.py and options_src.py, and check them. Jacobs' build.py plus the checks this corpus needs: one scene per fable, predicament length and name leaks, verbatim morals and evidence, the 13-fable recurrence rule, and the catalogue ids the options measurement uses. No inference.""" import collections, glob, json, pathlib, re, sys sys.dont_write_bytecode = True # the catalogue import below must not leave caches outside probe/aesop/ from groups_src import G, STANDING from options_src import O, NEW_ENTRIES here = pathlib.Path(__file__).parent sys.path.insert(0, str(here.parent.parent / 'catalogue' / 'v2')) from catalogue_v2_src import CATALOGUE chunks = json.loads((here.parent / 'chunks.json').read_text()) stories = json.loads((here.parent / 'stories.json').read_text()) recs = [r for f in sorted(glob.glob(str(here / 'batches/s*.json'))) for r in json.loads(open(f).read())] assert [r['chunk'] for r in recs] == list(range(len(chunks))), 'every fable, once, in order' assert all(chunks[r['chunk']]['story'] == r['story'] for r in recs) assert all(set(r) == {'chunk', 'story', 'summary', 'predicament', 'moral'} for r in recs) # ---- records ------------------------------------------------------------------------------------- NAMES = ('amaranth ant ape apple ass bat bear bee beetle bird boar bramble bull calf camel cat cock crab crane ' 'crow dog doe dolphin dove eagle elephant fawn fig filbert fir fish flea fly flies frog gnat goat goose ' 'geese grape grasshopper hare hart hawk hedgehog heifer hen horse hound jackdaw kid kite lamb lark ' 'leopard lion lioness locust mole monkey mouse mice mule nettle nightingale oak olive ox oxen oyster ' 'panther partridge peacock pig pigeon pomegranate porker raven reed rose satyr seagull serpent sheep ' 'snake sow sparrow sprat stag stork swallow swan thrush tortoise vine viper walnut wasp weasel whale ' 'whelp wolf wolves jupiter juno venus mercury minerva neptune momus hercules apollo pallas hermes ' 'axe net sling fleece wool manger pitcher honey acorn').split() NOPLURAL = {'doe'} # 'does' is a verb leak = re.compile(r'\b(' + '|'.join(n + ('' if n in NOPLURAL else 's?') for n in NAMES) + r')\b', re.I) def norm(s): return ' '.join(s.split()) problems = [] for r in recs: p, n = r['predicament'], len(r['predicament'].split()) if not 15 <= n <= 24: problems.append(f"chunk {r['chunk']}: predicament has {n} words") if len(re.findall(r'[.!?](?:\s|$)', p)) != 1 or not p.endswith('.'): problems.append(f"chunk {r['chunk']}: not one sentence") if leak.search(p): problems.append(f"chunk {r['chunk']}: name or story object in predicament: {leak.search(p).group(0)}") if r['moral'] is not None and norm(r['moral']) not in norm(chunks[r['chunk']]['text']): problems.append(f"chunk {r['chunk']}: moral is not verbatim") # ---- groups -------------------------------------------------------------------------------------- MIN_MEMBERS = max(3, round(0.04 * len(stories))) # 13 for 313 fables SKILLS = {'wit', 'prudence'} seen = collections.Counter(m for g in G for m in g['members']) dup = [m for m, k in seen.items() if k > 1] if dup: problems.append(f'fables in two groups: {dup}') assert set(STANDING) == set(range(len(recs))), 'STANDING covers every fable' by = {r['chunk']: r for r in recs} out = [] for g in G: small = g.get('small', False) k = len(g['members']) if small and not 5 <= k <= 12: problems.append(f"{g['id']}: marked small but has {k} fables") if not small and k < MIN_MEMBERS: problems.append(f"{g['id']}: {k} fables, below the {MIN_MEMBERS}-fable rule") if not 3 <= len(g['options']) <= 5: problems.append(f"{g['id']}: {len(g['options'])} options") used = {p for _, _, mv in g['options'] for p in mv if '.' not in p and p not in ('money', 'standing')} if not used <= SKILLS: problems.append(f"{g['id']}: skill labels outside {SKILLS}: {used - SKILLS}") for _, s, mv in g['options']: if not all(v in ('+', '-', 'risk', 'set') for v in mv.values()): problems.append(f"{g['id']}: bad direction in {mv}") if s.startswith('scene:') and not 0 <= int(s.split(':')[1]) < len(recs): problems.append(f"{g['id']}: bad source {s}") st = sorted({by[m]['story'] for m in g['members']}) stand = collections.Counter(STANDING[by[m]['story']] for m in g['members']) out.append({ 'id': g['id'], 'name': g['name'], 'predicament': g['predicament'], 'small': small, 'sentence': f"A {g['actor']} wants {g['wants']} from {g['counterpart']}.", 'actor': g['actor'], 'wants': g['wants'], 'counterpart': g['counterpart'], 'stories': st, 'story_titles': [stories[s]['title'] for s in st], 'standing': dict(stand), 'stage': stand.most_common(1)[0][0], 'members': [{'chunk': m, 'story': by[m]['story'], 'predicament': by[m]['predicament'], 'moral': by[m]['moral']} for m in g['members']], 'options': [{'text': t, 'source': s, 'moves': mv, 'in_group': (int(s.split(':')[1]) in g['members']) if s.startswith('scene:') else None} for t, s, mv in g['options']], }) unassigned = [r['chunk'] for r in recs if r['chunk'] not in seen] # ---- options measurement ------------------------------------------------------------------------- v2 = {e['id'] for e in CATALOGUE} obe, new_used = collections.defaultdict(list), set() for entry, c, text, mv, ev, stated in O: if entry.startswith('new:'): if entry[4:] not in NEW_ENTRIES: problems.append(f'{entry}: not defined in NEW_ENTRIES') if entry[4:] in v2: problems.append(f'{entry}: already a v2 id') new_used.add(entry[4:]) elif entry not in v2: problems.append(f'{entry}: not a v2 catalogue id') ev = by[c]['moral'] if ev == 'M' else ev if ev is None: problems.append(f'chunk {c}: evidence M but the fable has no moral') elif norm(ev) not in norm(chunks[c]['text']): problems.append(f'chunk {c}: evidence not verbatim: {ev[:50]}') elif len(ev.split()) > 30: problems.append(f'chunk {c}: evidence over 30 words') bad = [p for p in mv if not (p in ('money', 'standing') or p.startswith('relationships.'))] if bad: problems.append(f'chunk {c}: measurement moves use {bad}; money, standing, relationships only') if not all(v in ('+', '-', 'risk') for v in mv.values()): problems.append(f'chunk {c}: bad direction in {mv}') obe[entry].append({'text': text, 'moves': mv, 'source': f'chunk:{c}', 'evidence': ev, 'stated': stated}) unused = sorted(set(NEW_ENTRIES) - new_used) if unused: problems.append(f'NEW_ENTRIES never used: {unused}') if problems: print('\n'.join(problems)); sys.exit(1) obe_out = {k: obe[k] for k in sorted(obe, key=lambda k: (k.startswith('new:'), k))} obe_out['_new_entries'] = {f'new:{k}': {'name': n, 'definition': d, 'differs_from': x} for k, (n, d, x) in NEW_ENTRIES.items()} (here / 'records.json').write_text(json.dumps(recs, indent=1, ensure_ascii=False)) (here / 'groups.json').write_text(json.dumps({'groups': out, 'unassigned': unassigned}, indent=1, ensure_ascii=False)) (here / 'options_by_entry.json').write_text(json.dumps(obe_out, indent=1, ensure_ascii=False)) # ---- report -------------------------------------------------------------------------------------- big = [g for g in out if not g['small']] print(f'{len(recs)} fables, {sum(r["moral"] is not None for r in recs)} with a stated moral; ' f'{len(big)} groups of {MIN_MEMBERS}+ fables, {len(out) - len(big)} small groups (5-12); ' f'{sum(seen.values())} assigned, {len(unassigned)} unassigned') for g in out: print(f"{g['id']} {len(g['members']):2} fables stage {g['stage']:6} {'SMALL ' if g['small'] else ''}{g['name']}") src = collections.Counter(o['source'].split(':')[0] for g in out for o in g['options']) print('group option sources:', dict(src), '| scene options citing a fable outside their group:', sum(o['in_group'] is False for g in out for o in g['options'])) opts = [o for k, v in obe.items() for o in v] cov_v2 = sorted(k for k in obe if not k.startswith('new:')) print(f'options measurement: {len(opts)} options, {sum(o["stated"] for o in opts)} stated ' f'({sum(o["stated"] for o in opts) / len(opts):.0%}); {len(cov_v2)} of {len(v2)} v2 entries covered, ' f'{len(new_used)} new entries; from {len({o["source"] for o in opts})} distinct fables')