JesseMarkowitzandClaude Opus 5.5
3a4758172d
Fourteen corpora split and classified: aesop, bierce, chekhov, holmes, keefe, lawson, lorimer, maupassant, nobody, plaintales, poe, torchy, wallingford and winesburg, each with its splitter and the hand-written groups and pages; catalogue v2 and v3; and the shared splitters gutenberg_chunks.py, se_split.py and se_build.py. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019rwKTmug58sEsJ72AuWsEi
74 lines
3.8 KiB
Python
74 lines
3.8 KiB
Python
"""Split the Project Gutenberg plain text of Ambrose Bierce, *Tales of Soldiers and Civilians*
|
|
(1891/1898; here from "The Collected Works of Ambrose Bierce, Volume II", eBook #13334, which
|
|
contains this book and nothing else), into stories, then cut scenes with the same rule as
|
|
../chunk.py, ../jacobs/split_se.py and ../wharton/split_gutenberg.py (paragraph boundaries,
|
|
~320 words, tail >= 80).
|
|
Deterministic: titles are the 26 contents entries, matched as whole upper-case lines after the
|
|
contents block; roman-numeral section headings within a story are dropped; the Gutenberg preface,
|
|
contents and licence are dropped."""
|
|
import json, pathlib, re
|
|
|
|
TITLES = [
|
|
'A Horseman in the Sky', 'An Occurrence at Owl Creek Bridge', 'Chickamauga', 'A Son of the Gods',
|
|
'One of the Missing', 'Killed at Resaca', "The Affair at Coulter's Notch", 'The Coup de Grâce',
|
|
'Parker Adderson, Philosopher', 'An Affair of Outposts', 'The Story of a Conscience',
|
|
'One Kind of Officer', 'One Officer, One Man', 'George Thurston', 'The Mocking-Bird',
|
|
'The Man Out of the Nose', 'An Adventure at Brownville', 'The Famous Gilson Bequest',
|
|
'The Applicant', 'A Watcher by the Dead', 'The Man and the Snake', 'A Holy Terror',
|
|
'The Suitable Surroundings', 'The Boarded Window', 'A Lady from Redhorse', 'The Eyes of the Panther',
|
|
]
|
|
TARGET = 320
|
|
VOLUME = 'Tales of Soldiers and Civilians (Ambrose Bierce, 1891)'
|
|
|
|
raw = pathlib.Path('corpus/tales-of-soldiers-and-civilians.txt').read_text(encoding='utf-8').replace('\r', '')
|
|
raw = raw[:raw.index('*** END OF THE PROJECT GUTENBERG')]
|
|
lines = raw.split('\n')
|
|
start = next(i for i, l in enumerate(lines) if l.strip() == 'CONTENTS')
|
|
heads = []
|
|
for t in TITLES:
|
|
want = t.upper()
|
|
hits = [i for i, l in enumerate(lines)
|
|
if i > start + 40 and re.sub(r'\[\d+\]$', '', l.strip()) == want]
|
|
assert len(hits) == 1, (t, hits)
|
|
heads.append(hits[0])
|
|
assert heads == sorted(heads)
|
|
|
|
stories = []
|
|
for n, (t, a) in enumerate(zip(TITLES, heads)):
|
|
b = heads[n + 1] if n + 1 < len(heads) else len(lines)
|
|
body = '\n'.join(lines[a + 1:b])
|
|
paras = []
|
|
for p in re.split(r'\n\s*\n', body):
|
|
p = re.sub(r'\s+', ' ', p).strip()
|
|
if not p or re.fullmatch(r'[IVX]+\.?', p): continue
|
|
if re.match(r'^\[\d+\]', p): continue # footnote (e.g. Brownville's collaboration credit)
|
|
paras.append(p)
|
|
stories.append({'volume': VOLUME, 'title': t, 'words': sum(len(p.split()) for p in paras),
|
|
'text': '\n\n'.join(paras)})
|
|
|
|
chunks = []
|
|
for si, s in enumerate(stories):
|
|
buf, n = [], 0
|
|
for p in s['text'].split('\n\n'):
|
|
w = len(p.split())
|
|
if n + w > TARGET and buf:
|
|
chunks.append({'story': si, 'title': s['title'], 'volume': s['volume'], 'text': ' '.join(buf), 'words': n})
|
|
buf, n = [], 0
|
|
buf.append(p); n += w
|
|
if buf and n >= 80:
|
|
chunks.append({'story': si, 'title': s['title'], 'volume': s['volume'], 'text': ' '.join(buf), 'words': n})
|
|
elif buf:
|
|
# A short remainder (e.g. a story's final one- or two-line gut-punch, as in
|
|
# Chickamauga's "The child was a deaf mute.") would otherwise be silently
|
|
# dropped by the tail rule. Fold it into the previous chunk instead of losing it.
|
|
last = next(c for c in reversed(chunks) if c['story'] == si)
|
|
last['text'] += ' ' + ' '.join(buf)
|
|
last['words'] += n
|
|
|
|
pathlib.Path('stories.json').write_text(json.dumps(stories, indent=1, ensure_ascii=False), encoding='utf-8')
|
|
pathlib.Path('chunks.json').write_text(json.dumps(chunks, ensure_ascii=False), encoding='utf-8')
|
|
for si, s in enumerate(stories):
|
|
print(f"{si:2} {s['title']:34} {s['words']:6} words {sum(c['story']==si for c in chunks):3} scenes")
|
|
ws = sorted(c['words'] for c in chunks)
|
|
print(f'stories {len(stories)} scenes {len(chunks)} words/scene min {ws[0]} median {ws[len(ws)//2]} max {ws[-1]} total {sum(ws):,}')
|