story-to-pack: add the corpora and the catalogue v2/v3

Fourteen corpora split and classified: aesop, bierce, chekhov, holmes, keefe, lawson, lorimer, maupassant, nobody, plaintales, poe, torchy, wallingford and winesburg, each with its splitter and the hand-written groups and pages; catalogue v2 and v3; and the shared splitters gutenberg_chunks.py, se_split.py and se_build.py. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019rwKTmug58sEsJ72AuWsEi
2026-10-07 06:05:51 -04:00
parent e5617b86ba
commit 3a4758172d
566 changed files with 303901 additions and 0 deletions
@@ -0,0 +1,59 @@
+"""Split the Project Gutenberg plain text of Arthur Conan Doyle, *The Adventures of Sherlock
+Holmes* (eBook #1661), into stories, then cut scenes with the same rule as
+../jacobs/split_se.py and ../wharton/split_gutenberg.py (paragraph boundaries, ~320 words,
+tail >= 80). Deterministic: titles are matched as the roman-numeral story headers in the body
+(not the contents list), and the Gutenberg licence after END is dropped."""
+import json, pathlib, re
+
+TITLES = ['A Scandal in Bohemia', 'The Red-Headed League', 'A Case of Identity',
+          'The Boscombe Valley Mystery', 'The Five Orange Pips', 'The Man with the Twisted Lip',
+          'The Adventure of the Blue Carbuncle', 'The Adventure of the Speckled Band',
+          "The Adventure of the Engineer's Thumb", 'The Adventure of the Noble Bachelor',
+          'The Adventure of the Beryl Coronet', 'The Adventure of the Copper Beeches']
+TARGET = 320
+VOLUME = 'The Adventures of Sherlock Holmes (Arthur Conan Doyle, 1892)'
+
+raw = pathlib.Path('corpus/adventures-of-sherlock-holmes.txt').read_text(encoding='utf-8').replace('\r', '')
+raw = raw[:raw.index('*** END OF THE PROJECT GUTENBERG')]
+lines = raw.split('\n')
+
+ROMAN = ['I', 'II', 'III', 'IV', 'V', 'VI', 'VII', 'VIII', 'IX', 'X', 'XI', 'XII']
+heads = []
+for n, t in enumerate(TITLES):
+    norm_t = t.upper().replace("'", "’")
+    pat = re.compile(r'^' + re.escape(ROMAN[n]) + r'\. ' + re.escape(norm_t) + r'$')
+    hits = [i for i, l in enumerate(lines) if pat.match(l.strip())]
+    assert len(hits) == 1, (t, hits)
+    heads.append(hits[0])
+assert heads == sorted(heads)
+
+stories = []
+for n, (t, a) in enumerate(zip(TITLES, heads)):
+    b = heads[n + 1] if n + 1 < len(heads) else len(lines)
+    body = '\n'.join(lines[a + 1:b])
+    paras = []
+    for p in re.split(r'\n\s*\n', body):
+        p = re.sub(r'\s+', ' ', p).strip()
+        if not p: continue
+        paras.append(p)
+    stories.append({'volume': VOLUME, 'title': t, 'words': sum(len(p.split()) for p in paras),
+                    'text': '\n\n'.join(paras)})
+
+chunks = []
+for si, s in enumerate(stories):
+    buf, n = [], 0
+    for p in s['text'].split('\n\n'):
+        w = len(p.split())
+        if n + w > TARGET and buf:
+            chunks.append({'story': si, 'title': s['title'], 'volume': s['volume'], 'text': ' '.join(buf), 'words': n})
+            buf, n = [], 0
+        buf.append(p); n += w
+    if buf and n >= 80:
+        chunks.append({'story': si, 'title': s['title'], 'volume': s['volume'], 'text': ' '.join(buf), 'words': n})
+
+pathlib.Path('stories.json').write_text(json.dumps(stories, indent=1, ensure_ascii=False), encoding='utf-8')
+pathlib.Path('chunks.json').write_text(json.dumps(chunks, ensure_ascii=False), encoding='utf-8')
+for si, s in enumerate(stories):
+    print(f"{si:2} {s['title']:40} {s['words']:6} words  {sum(c['story']==si for c in chunks):3} scenes")
+ws = sorted(c['words'] for c in chunks)
+print(f'stories {len(stories)}  scenes {len(chunks)}  words/scene min {ws[0]} median {ws[len(ws)//2]} max {ws[-1]}  total {sum(ws):,}')