"""Split the Project Gutenberg plain text of Edith Wharton, *The Descent of Man and Other Stories*
(eBook #4519), into stories, then cut scenes with the same rule as ../chunk.py and
../jacobs/split_se.py (paragraph boundaries, ~320 words, tail >= 80).
Deterministic: titles are the ten contents entries, matched as whole upper-case lines after the
contents; roman-numeral section headings are dropped; the Gutenberg licence after END is dropped."""
import json, pathlib, re

TITLES = ['The Descent of Man', 'The Other Two', 'Expiation', "The Lady's Maid's Bell",
          'The Mission of Jane', 'The Reckoning', 'The Letter', 'The Dilettante', 'The Quicksand',
          "A Venetian Night's Entertainment"]
TARGET = 320
VOLUME = 'The Descent of Man and Other Stories (Edith Wharton, 1904)'

raw = pathlib.Path('corpus/descent-of-man.txt').read_text(encoding='utf-8').replace('\r', '')
raw = raw[:raw.index('*** END OF THE PROJECT GUTENBERG')]
lines = raw.split('\n')
start = next(i for i, l in enumerate(lines) if l.strip() == 'TABLE OF CONTENTS')
heads = []
for t in TITLES:
    hits = [i for i, l in enumerate(lines) if i > start + 20 and l.strip() == t.upper()]
    assert len(hits) == 1, (t, hits)
    heads.append(hits[0])
assert heads == sorted(heads)

stories = []
for n, (t, a) in enumerate(zip(TITLES, heads)):
    b = heads[n + 1] if n + 1 < len(heads) else len(lines)
    body = '\n'.join(lines[a + 1:b])
    paras = []
    for p in re.split(r'\n\s*\n', body):
        p = re.sub(r'\s+', ' ', p).strip()
        if not p or re.fullmatch(r'[IVX]+\.?', p): continue
        paras.append(p)
    stories.append({'volume': VOLUME, 'title': t, 'words': sum(len(p.split()) for p in paras),
                    'text': '\n\n'.join(paras)})

chunks = []
for si, s in enumerate(stories):
    buf, n = [], 0
    for p in s['text'].split('\n\n'):
        w = len(p.split())
        if n + w > TARGET and buf:
            chunks.append({'story': si, 'title': s['title'], 'volume': s['volume'], 'text': ' '.join(buf), 'words': n})
            buf, n = [], 0
        buf.append(p); n += w
    if buf and n >= 80:
        chunks.append({'story': si, 'title': s['title'], 'volume': s['volume'], 'text': ' '.join(buf), 'words': n})

pathlib.Path('stories.json').write_text(json.dumps(stories, indent=1, ensure_ascii=False), encoding='utf-8')
pathlib.Path('chunks.json').write_text(json.dumps(chunks, ensure_ascii=False), encoding='utf-8')
for si, s in enumerate(stories):
    print(f"{si:2} {s['title']:34} {s['words']:6} words  {sum(c['story']==si for c in chunks):3} scenes")
ws = sorted(c['words'] for c in chunks)
print(f'stories {len(stories)}  scenes {len(chunks)}  words/scene min {ws[0]} median {ws[len(ws)//2]} max {ws[-1]}  total {sum(ws):,}')