JesseMarkowitzandClaude Opus 5.5
3a4758172d
Fourteen corpora split and classified: aesop, bierce, chekhov, holmes, keefe, lawson, lorimer, maupassant, nobody, plaintales, poe, torchy, wallingford and winesburg, each with its splitter and the hand-written groups and pages; catalogue v2 and v3; and the shared splitters gutenberg_chunks.py, se_split.py and se_build.py. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019rwKTmug58sEsJ72AuWsEi
73 lines
4.0 KiB
Python
73 lines
4.0 KiB
Python
"""Split the Project Gutenberg plain text of *The Works of Edgar Allan Poe, Volume 2* (Raven Edition,
|
||
eBook #2148) into its 22 tales, then cut scenes with the same rule as ../chunk.py,
|
||
../jacobs/split_se.py and ../wharton/split_gutenberg.py (paragraph boundaries, ~320 words, tail >= 80).
|
||
Deterministic: titles are the 22 contents entries, matched as whole upper-case lines after the
|
||
contents list. Header lines differ slightly from the titles (trailing periods, a curly apostrophe,
|
||
"MAELSTROM" without the ring, "SILENCE—A FABLE" with no spaces), so both sides are normalised before
|
||
matching. Front matter, the contents, the redactor's note, the "Notes to the Second Volume" and the
|
||
Gutenberg licence are dropped. Poe's footnote markers ("(*1)") are left in the text as split."""
|
||
import json, pathlib, re, unicodedata
|
||
|
||
TITLES = ['The Purloined Letter', 'The Thousand-and-Second Tale of Scheherazade',
|
||
'A Descent into the Maelström', 'Von Kempelen and His Discovery', 'Mesmeric Revelation',
|
||
'The Facts in the Case of M. Valdemar', 'The Black Cat', 'The Fall of the House of Usher',
|
||
'Silence — A Fable', 'The Masque of the Red Death', 'The Cask of Amontillado',
|
||
'The Imp of the Perverse', 'The Island of the Fay', 'The Assignation',
|
||
'The Pit and the Pendulum', 'The Premature Burial', 'The Domain of Arnheim',
|
||
"Landor's Cottage", 'William Wilson', 'The Tell-Tale Heart', 'Berenice', 'Eleonora']
|
||
END_MATTER = 'NOTES TO THE SECOND VOLUME'
|
||
TARGET = 320
|
||
VOLUME = 'The Works of Edgar Allan Poe, Volume 2 (Raven Edition; tales)'
|
||
|
||
def norm(s):
|
||
s = re.sub(r'\s*[-—]+\s*', '-', s.replace('’', "'")) # before the ascii fold drops the dash
|
||
s = unicodedata.normalize('NFKD', s).encode('ascii', 'ignore').decode() # Maelström -> Maelstrom
|
||
return re.sub(r'\s+', ' ', s.upper().strip().rstrip('.'))
|
||
|
||
raw = pathlib.Path('corpus/tales-vol2.txt').read_text(encoding='utf-8').replace('\r', '')
|
||
raw = raw[:raw.index('*** END OF THE PROJECT GUTENBERG')]
|
||
lines = [l.replace('’', "'") for l in raw.split('\n')]
|
||
start = next(i for i, l in enumerate(lines) if l.strip() == 'Contents')
|
||
end = [i for i, l in enumerate(lines) if l.strip() == END_MATTER]
|
||
assert len(end) == 2, end # once in the contents, once as the heading
|
||
end = end[1]
|
||
heads = []
|
||
for t in TITLES:
|
||
# headings sit flush left; the contents entries are indented by one space
|
||
hits = [i for i, l in enumerate(lines) if start + 30 < i < end and l == l.lstrip() and l.strip()
|
||
and norm(l) == norm(t)]
|
||
assert len(hits) == 1, (t, hits)
|
||
heads.append(hits[0])
|
||
assert heads == sorted(heads)
|
||
|
||
stories = []
|
||
for n, (t, a) in enumerate(zip(TITLES, heads)):
|
||
b = heads[n + 1] if n + 1 < len(heads) else end
|
||
body = '\n'.join(lines[a + 1:b])
|
||
paras = []
|
||
for p in re.split(r'\n\s*\n', body):
|
||
p = re.sub(r'\s+', ' ', p).strip()
|
||
if not p or re.fullmatch(r'[IVX]+\.?', p): continue
|
||
paras.append(p)
|
||
stories.append({'volume': VOLUME, 'title': t, 'words': sum(len(p.split()) for p in paras),
|
||
'text': '\n\n'.join(paras)})
|
||
|
||
chunks = []
|
||
for si, s in enumerate(stories):
|
||
buf, n = [], 0
|
||
for p in s['text'].split('\n\n'):
|
||
w = len(p.split())
|
||
if n + w > TARGET and buf:
|
||
chunks.append({'story': si, 'title': s['title'], 'volume': s['volume'], 'text': ' '.join(buf), 'words': n})
|
||
buf, n = [], 0
|
||
buf.append(p); n += w
|
||
if buf and n >= 80:
|
||
chunks.append({'story': si, 'title': s['title'], 'volume': s['volume'], 'text': ' '.join(buf), 'words': n})
|
||
|
||
pathlib.Path('stories.json').write_text(json.dumps(stories, indent=1, ensure_ascii=False), encoding='utf-8')
|
||
pathlib.Path('chunks.json').write_text(json.dumps(chunks, ensure_ascii=False), encoding='utf-8')
|
||
for si, s in enumerate(stories):
|
||
print(f"{si:2} {s['title']:46} {s['words']:6} words {sum(c['story']==si for c in chunks):3} scenes")
|
||
ws = sorted(c['words'] for c in chunks)
|
||
print(f'stories {len(stories)} scenes {len(chunks)} words/scene min {ws[0]} median {ws[len(ws)//2]} max {ws[-1]} total {sum(ws):,}')
|