story-to-pack: add the corpora and the catalogue v2/v3
Fourteen corpora split and classified: aesop, bierce, chekhov, holmes, keefe, lawson, lorimer, maupassant, nobody, plaintales, poe, torchy, wallingford and winesburg, each with its splitter and the hand-written groups and pages; catalogue v2 and v3; and the shared splitters gutenberg_chunks.py, se_split.py and se_build.py. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019rwKTmug58sEsJ72AuWsEi
This commit is contained in:
JesseMarkowitz
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
e5617b86ba
commit
3a4758172d
@@ -0,0 +1,120 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Assemble records.json, groups.json and options_by_entry.json from batches/, groups_src.py and
|
||||
options_src.py, and check them. Jacobs' build.py plus the checks this corpus needs: one scene per
|
||||
fable, predicament length and name leaks, verbatim morals and evidence, the 13-fable recurrence rule,
|
||||
and the catalogue ids the options measurement uses. No inference."""
|
||||
import collections, glob, json, pathlib, re, sys
|
||||
sys.dont_write_bytecode = True # the catalogue import below must not leave caches outside probe/aesop/
|
||||
from groups_src import G, STANDING
|
||||
from options_src import O, NEW_ENTRIES
|
||||
here = pathlib.Path(__file__).parent
|
||||
sys.path.insert(0, str(here.parent.parent / 'catalogue' / 'v2'))
|
||||
from catalogue_v2_src import CATALOGUE
|
||||
chunks = json.loads((here.parent / 'chunks.json').read_text())
|
||||
stories = json.loads((here.parent / 'stories.json').read_text())
|
||||
recs = [r for f in sorted(glob.glob(str(here / 'batches/s*.json'))) for r in json.loads(open(f).read())]
|
||||
assert [r['chunk'] for r in recs] == list(range(len(chunks))), 'every fable, once, in order'
|
||||
assert all(chunks[r['chunk']]['story'] == r['story'] for r in recs)
|
||||
assert all(set(r) == {'chunk', 'story', 'summary', 'predicament', 'moral'} for r in recs)
|
||||
|
||||
# ---- records -------------------------------------------------------------------------------------
|
||||
NAMES = ('amaranth ant ape apple ass bat bear bee beetle bird boar bramble bull calf camel cat cock crab crane '
|
||||
'crow dog doe dolphin dove eagle elephant fawn fig filbert fir fish flea fly flies frog gnat goat goose '
|
||||
'geese grape grasshopper hare hart hawk hedgehog heifer hen horse hound jackdaw kid kite lamb lark '
|
||||
'leopard lion lioness locust mole monkey mouse mice mule nettle nightingale oak olive ox oxen oyster '
|
||||
'panther partridge peacock pig pigeon pomegranate porker raven reed rose satyr seagull serpent sheep '
|
||||
'snake sow sparrow sprat stag stork swallow swan thrush tortoise vine viper walnut wasp weasel whale '
|
||||
'whelp wolf wolves jupiter juno venus mercury minerva neptune momus hercules apollo pallas hermes '
|
||||
'axe net sling fleece wool manger pitcher honey acorn').split()
|
||||
NOPLURAL = {'doe'} # 'does' is a verb
|
||||
leak = re.compile(r'\b(' + '|'.join(n + ('' if n in NOPLURAL else 's?') for n in NAMES) + r')\b', re.I)
|
||||
def norm(s): return ' '.join(s.split())
|
||||
problems = []
|
||||
for r in recs:
|
||||
p, n = r['predicament'], len(r['predicament'].split())
|
||||
if not 15 <= n <= 24: problems.append(f"chunk {r['chunk']}: predicament has {n} words")
|
||||
if len(re.findall(r'[.!?](?:\s|$)', p)) != 1 or not p.endswith('.'): problems.append(f"chunk {r['chunk']}: not one sentence")
|
||||
if leak.search(p): problems.append(f"chunk {r['chunk']}: name or story object in predicament: {leak.search(p).group(0)}")
|
||||
if r['moral'] is not None and norm(r['moral']) not in norm(chunks[r['chunk']]['text']):
|
||||
problems.append(f"chunk {r['chunk']}: moral is not verbatim")
|
||||
|
||||
# ---- groups --------------------------------------------------------------------------------------
|
||||
MIN_MEMBERS = max(3, round(0.04 * len(stories))) # 13 for 313 fables
|
||||
SKILLS = {'wit', 'prudence'}
|
||||
seen = collections.Counter(m for g in G for m in g['members'])
|
||||
dup = [m for m, k in seen.items() if k > 1]
|
||||
if dup: problems.append(f'fables in two groups: {dup}')
|
||||
assert set(STANDING) == set(range(len(recs))), 'STANDING covers every fable'
|
||||
by = {r['chunk']: r for r in recs}
|
||||
out = []
|
||||
for g in G:
|
||||
small = g.get('small', False)
|
||||
k = len(g['members'])
|
||||
if small and not 5 <= k <= 12: problems.append(f"{g['id']}: marked small but has {k} fables")
|
||||
if not small and k < MIN_MEMBERS: problems.append(f"{g['id']}: {k} fables, below the {MIN_MEMBERS}-fable rule")
|
||||
if not 3 <= len(g['options']) <= 5: problems.append(f"{g['id']}: {len(g['options'])} options")
|
||||
used = {p for _, _, mv in g['options'] for p in mv if '.' not in p and p not in ('money', 'standing')}
|
||||
if not used <= SKILLS: problems.append(f"{g['id']}: skill labels outside {SKILLS}: {used - SKILLS}")
|
||||
for _, s, mv in g['options']:
|
||||
if not all(v in ('+', '-', 'risk', 'set') for v in mv.values()): problems.append(f"{g['id']}: bad direction in {mv}")
|
||||
if s.startswith('scene:') and not 0 <= int(s.split(':')[1]) < len(recs): problems.append(f"{g['id']}: bad source {s}")
|
||||
st = sorted({by[m]['story'] for m in g['members']})
|
||||
stand = collections.Counter(STANDING[by[m]['story']] for m in g['members'])
|
||||
out.append({
|
||||
'id': g['id'], 'name': g['name'], 'predicament': g['predicament'], 'small': small,
|
||||
'sentence': f"A {g['actor']} wants {g['wants']} from {g['counterpart']}.",
|
||||
'actor': g['actor'], 'wants': g['wants'], 'counterpart': g['counterpart'],
|
||||
'stories': st, 'story_titles': [stories[s]['title'] for s in st],
|
||||
'standing': dict(stand), 'stage': stand.most_common(1)[0][0],
|
||||
'members': [{'chunk': m, 'story': by[m]['story'], 'predicament': by[m]['predicament'],
|
||||
'moral': by[m]['moral']} for m in g['members']],
|
||||
'options': [{'text': t, 'source': s, 'moves': mv,
|
||||
'in_group': (int(s.split(':')[1]) in g['members']) if s.startswith('scene:') else None}
|
||||
for t, s, mv in g['options']],
|
||||
})
|
||||
unassigned = [r['chunk'] for r in recs if r['chunk'] not in seen]
|
||||
|
||||
# ---- options measurement -------------------------------------------------------------------------
|
||||
v2 = {e['id'] for e in CATALOGUE}
|
||||
obe, new_used = collections.defaultdict(list), set()
|
||||
for entry, c, text, mv, ev, stated in O:
|
||||
if entry.startswith('new:'):
|
||||
if entry[4:] not in NEW_ENTRIES: problems.append(f'{entry}: not defined in NEW_ENTRIES')
|
||||
if entry[4:] in v2: problems.append(f'{entry}: already a v2 id')
|
||||
new_used.add(entry[4:])
|
||||
elif entry not in v2: problems.append(f'{entry}: not a v2 catalogue id')
|
||||
ev = by[c]['moral'] if ev == 'M' else ev
|
||||
if ev is None: problems.append(f'chunk {c}: evidence M but the fable has no moral')
|
||||
elif norm(ev) not in norm(chunks[c]['text']): problems.append(f'chunk {c}: evidence not verbatim: {ev[:50]}')
|
||||
elif len(ev.split()) > 30: problems.append(f'chunk {c}: evidence over 30 words')
|
||||
bad = [p for p in mv if not (p in ('money', 'standing') or p.startswith('relationships.'))]
|
||||
if bad: problems.append(f'chunk {c}: measurement moves use {bad}; money, standing, relationships only')
|
||||
if not all(v in ('+', '-', 'risk') for v in mv.values()): problems.append(f'chunk {c}: bad direction in {mv}')
|
||||
obe[entry].append({'text': text, 'moves': mv, 'source': f'chunk:{c}', 'evidence': ev, 'stated': stated})
|
||||
unused = sorted(set(NEW_ENTRIES) - new_used)
|
||||
if unused: problems.append(f'NEW_ENTRIES never used: {unused}')
|
||||
if problems:
|
||||
print('\n'.join(problems)); sys.exit(1)
|
||||
|
||||
obe_out = {k: obe[k] for k in sorted(obe, key=lambda k: (k.startswith('new:'), k))}
|
||||
obe_out['_new_entries'] = {f'new:{k}': {'name': n, 'definition': d, 'differs_from': x}
|
||||
for k, (n, d, x) in NEW_ENTRIES.items()}
|
||||
(here / 'records.json').write_text(json.dumps(recs, indent=1, ensure_ascii=False))
|
||||
(here / 'groups.json').write_text(json.dumps({'groups': out, 'unassigned': unassigned}, indent=1, ensure_ascii=False))
|
||||
(here / 'options_by_entry.json').write_text(json.dumps(obe_out, indent=1, ensure_ascii=False))
|
||||
|
||||
# ---- report --------------------------------------------------------------------------------------
|
||||
big = [g for g in out if not g['small']]
|
||||
print(f'{len(recs)} fables, {sum(r["moral"] is not None for r in recs)} with a stated moral; '
|
||||
f'{len(big)} groups of {MIN_MEMBERS}+ fables, {len(out) - len(big)} small groups (5-12); '
|
||||
f'{sum(seen.values())} assigned, {len(unassigned)} unassigned')
|
||||
for g in out:
|
||||
print(f"{g['id']} {len(g['members']):2} fables stage {g['stage']:6} {'SMALL ' if g['small'] else ''}{g['name']}")
|
||||
src = collections.Counter(o['source'].split(':')[0] for g in out for o in g['options'])
|
||||
print('group option sources:', dict(src), '| scene options citing a fable outside their group:',
|
||||
sum(o['in_group'] is False for g in out for o in g['options']))
|
||||
opts = [o for k, v in obe.items() for o in v]
|
||||
cov_v2 = sorted(k for k in obe if not k.startswith('new:'))
|
||||
print(f'options measurement: {len(opts)} options, {sum(o["stated"] for o in opts)} stated '
|
||||
f'({sum(o["stated"] for o in opts) / len(opts):.0%}); {len(cov_v2)} of {len(v2)} v2 entries covered, '
|
||||
f'{len(new_used)} new entries; from {len({o["source"] for o in opts})} distinct fables')
|
||||
Reference in New Issue
Block a user