"""Split the Project Gutenberg plain text of *The Works of Edgar Allan Poe, Volume 2* (Raven Edition,
eBook #2148) into its 22 tales, then cut scenes with the same rule as ../chunk.py,
../jacobs/split_se.py and ../wharton/split_gutenberg.py (paragraph boundaries, ~320 words, tail >= 80).
Deterministic: titles are the 22 contents entries, matched as whole upper-case lines after the
contents list. Header lines differ slightly from the titles (trailing periods, a curly apostrophe,
"MAELSTROM" without the ring, "SILENCE—A FABLE" with no spaces), so both sides are normalised before
matching. Front matter, the contents, the redactor's note, the "Notes to the Second Volume" and the
Gutenberg licence are dropped. Poe's footnote markers ("(*1)") are left in the text as split."""
import json, pathlib, re, unicodedata

TITLES = ['The Purloined Letter', 'The Thousand-and-Second Tale of Scheherazade',
          'A Descent into the Maelström', 'Von Kempelen and His Discovery', 'Mesmeric Revelation',
          'The Facts in the Case of M. Valdemar', 'The Black Cat', 'The Fall of the House of Usher',
          'Silence — A Fable', 'The Masque of the Red Death', 'The Cask of Amontillado',
          'The Imp of the Perverse', 'The Island of the Fay', 'The Assignation',
          'The Pit and the Pendulum', 'The Premature Burial', 'The Domain of Arnheim',
          "Landor's Cottage", 'William Wilson', 'The Tell-Tale Heart', 'Berenice', 'Eleonora']
END_MATTER = 'NOTES TO THE SECOND VOLUME'
TARGET = 320
VOLUME = 'The Works of Edgar Allan Poe, Volume 2 (Raven Edition; tales)'

def norm(s):
    s = re.sub(r'\s*[-—]+\s*', '-', s.replace('’', "'"))   # before the ascii fold drops the dash
    s = unicodedata.normalize('NFKD', s).encode('ascii', 'ignore').decode()  # Maelström -> Maelstrom
    return re.sub(r'\s+', ' ', s.upper().strip().rstrip('.'))

raw = pathlib.Path('corpus/tales-vol2.txt').read_text(encoding='utf-8').replace('\r', '')
raw = raw[:raw.index('*** END OF THE PROJECT GUTENBERG')]
lines = [l.replace('’', "'") for l in raw.split('\n')]
start = next(i for i, l in enumerate(lines) if l.strip() == 'Contents')
end = [i for i, l in enumerate(lines) if l.strip() == END_MATTER]
assert len(end) == 2, end            # once in the contents, once as the heading
end = end[1]
heads = []
for t in TITLES:
    # headings sit flush left; the contents entries are indented by one space
    hits = [i for i, l in enumerate(lines) if start + 30 < i < end and l == l.lstrip() and l.strip()
            and norm(l) == norm(t)]
    assert len(hits) == 1, (t, hits)
    heads.append(hits[0])
assert heads == sorted(heads)

stories = []
for n, (t, a) in enumerate(zip(TITLES, heads)):
    b = heads[n + 1] if n + 1 < len(heads) else end
    body = '\n'.join(lines[a + 1:b])
    paras = []
    for p in re.split(r'\n\s*\n', body):
        p = re.sub(r'\s+', ' ', p).strip()
        if not p or re.fullmatch(r'[IVX]+\.?', p): continue
        paras.append(p)
    stories.append({'volume': VOLUME, 'title': t, 'words': sum(len(p.split()) for p in paras),
                    'text': '\n\n'.join(paras)})

chunks = []
for si, s in enumerate(stories):
    buf, n = [], 0
    for p in s['text'].split('\n\n'):
        w = len(p.split())
        if n + w > TARGET and buf:
            chunks.append({'story': si, 'title': s['title'], 'volume': s['volume'], 'text': ' '.join(buf), 'words': n})
            buf, n = [], 0
        buf.append(p); n += w
    if buf and n >= 80:
        chunks.append({'story': si, 'title': s['title'], 'volume': s['volume'], 'text': ' '.join(buf), 'words': n})

pathlib.Path('stories.json').write_text(json.dumps(stories, indent=1, ensure_ascii=False), encoding='utf-8')
pathlib.Path('chunks.json').write_text(json.dumps(chunks, ensure_ascii=False), encoding='utf-8')
for si, s in enumerate(stories):
    print(f"{si:2} {s['title']:46} {s['words']:6} words  {sum(c['story']==si for c in chunks):3} scenes")
ws = sorted(c['words'] for c in chunks)
print(f'stories {len(stories)}  scenes {len(chunks)}  words/scene min {ws[0]} median {ws[len(ws)//2]} max {ws[-1]}  total {sum(ws):,}')