diff --git a/backend/tools/memory_ab.py b/backend/tools/memory_ab.py new file mode 100644 index 0000000..65c2efe --- /dev/null +++ b/backend/tools/memory_ab.py @@ -0,0 +1,257 @@ +"""A/B the memory prompt against a real model, on one story held constant. + +Reviewing a prompt tells you what it asks for. It does not tell you what a model +does with it. This runs the real pipeline twice over the *same* blocks of the +*same* story, changing only the prompt, and prints the memories side by side. + +What it showed the first time it was run, and why `MEMORY_MAX_WORDS` exists, is +written up in `plan/18-persona-and-memory-quality.md`. Two consecutive memories +from one story came back in two different persons, and the same model wrote 34 +words for one block and 105 for the next. + +Three things make it a fair test rather than a demonstration: + +- **The story is generated once**, through the app's own `build_context`, and + both arms summarize the same actions. The prompt is the only variable. +- **The control is read from git**, at the commit named by `--before`, so it is + the prompt that actually shipped rather than a paraphrase of it. +- **Every call goes through `OpenAICompatibleProvider`** to the endpoint in + `--endpoint`, so this exercises the provider, the streaming path and + `complete()` rather than a stub. + +Point it at `tools/claude_shim.py` to spend a Claude subscription instead of API +credit: + + python tools/claude_shim.py --port 8787 & + python tools/memory_ab.py --out /tmp/ab.md + +Point `--endpoint` at the provider the deployed app really uses to learn +something this cannot tell you: whether a weaker model follows the framing rule +as well as a Claude model does. + +Options: + + --endpoint OpenAI-compatible base URL. Defaults to the local shim. + --model Model name to ask that endpoint for. Defaults to `sonnet`. + --before Git commit holding the prompt to compare against. + --turns How many player turns to generate. Six gives two blocks. + --out Write the full transcript here as Markdown. + +This writes to a scratch SQLite file and drops it afterwards. It never touches +`backend/data.db`. +""" +import argparse +import asyncio +import os +import re +import subprocess +import sys +import tempfile +import textwrap +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent.parent + +# The moves the player makes. Fixed, so that a re-run summarizes comparable +# story rather than a different adventure. +PLAYER_TURNS = [ + "go quiet and signal Gwen to circle around the right flank", + "search the nearest bedroll for anything useful", + "move toward the strongbox, keeping low", + "grab her wrist and pull her down behind the woodpile", + "wait for the bandit to turn, then move", + "ask her whether she still trusts my read on this", +] + + +def prompt_at(commit: str, name: str) -> str: + """Reads one prompt constant out of `memorybank.py` as of `commit`. + + Read from git rather than pasted here, so that the control cannot drift out + of step with what was actually shipped. + """ + source = subprocess.run( + ["git", "show", f"{commit}:backend/app/memorybank.py"], + capture_output=True, text=True, cwd=REPO, check=True, + ).stdout + match = re.search(rf"^{name} = \((.*?)^\)$", source, re.S | re.M) + if not match: + raise SystemExit(f"{name} not found in memorybank.py at {commit}") + namespace: dict = {"MEMORY_MAX_WORDS": 50, "SUMMARY_MAX_WORDS": 250} + return eval(f"({match.group(1)})", namespace) # noqa: S307 — our own source + + +def words(text: str) -> int: + return len(text.split()) + + +def framing(text: str) -> str: + """How this memory refers to the protagonist. The thing being measured.""" + tags = [] + if re.search(r"\byou(r)?\b", text, re.I): + tags.append('second person ("you")') + if re.search(r"\bthe player\b", text, re.I): + tags.append('"the player"') + return ", ".join(tags) or "named" + + +async def main(args) -> None: + from app import memorybank, models, tree, worldstate + from app.context import build_context, truncate_to_last_tokens + from app.database import Base, SessionLocal, engine + from app.providers import OpenAICompatibleProvider + from app.routers.adventures.scenario_text import scenario_card_specs + from app.seed import seed_public_scenarios + + def provider(model: str | None = None) -> OpenAICompatibleProvider: + return OpenAICompatibleProvider( + args.endpoint, "unused-by-the-shim", model or args.model, "chat", 0) + + async def generate(system: str, story: str) -> str: + """One turn, through the provider's real streaming path.""" + from app.providers.base import PromptParts + out = [] + async for kind, piece in provider().generate( + PromptParts(system=system, story=story), temperature=0.8, + max_tokens=args.max_tokens, + ): + if kind == "text": + out.append(piece) + return "".join(out).strip() + + Base.metadata.create_all(bind=engine) + seed_public_scenarios(engine) + db = SessionLocal() + user = models.User(is_guest=False, email="memory-ab@example.com") + db.add(user) + db.flush() + settings = models.Settings(user_id=user.id, api_key="enc:d", + model=args.model, max_output_tokens=args.max_tokens) + db.add(settings) + scenario = db.query(models.Scenario).filter( + models.Scenario.title.like("%Bandit Camp%")).one() + + adventure = models.Adventure( + user_id=user.id, title=scenario.title, scenario_id=scenario.id, + script_state={}, auto_summarize=True, memory=scenario.memory, + authors_note=scenario.authors_note, + ai_instructions=scenario.ai_instructions, + world_state=worldstate.instantiate(scenario.stat_schema), + persona_name="Kaelen", persona_pronouns="he/him", + persona_desc=("A half-elf ranger, exiled from the northern holds for a " + "killing he still won't explain. Wary of nobles, soft on " + "strays."), + ) + db.add(adventure) + db.flush() + tree.head_branch(db, adventure) + for ref, spec in scenario_card_specs(scenario, {}).items(): + db.add(models.StoryCard(adventure_id=adventure.id, source_ref=ref, **spec)) + opening = models.Action(adventure_id=adventure.id, type="start", + text=scenario.prompt) + tree.place_action(db, adventure, opening) + db.add(opening) + db.commit() + db.refresh(adventure) + + print(f"Generating {args.turns} turns through build_context…", flush=True) + for i, move in enumerate(PLAYER_TURNS[:args.turns], 1): + player = models.Action(adventure_id=adventure.id, type="do", text=move) + tree.place_action(db, adventure, player) + db.add(player) + db.commit() + db.refresh(adventure) + + system_text, story_text, _ = build_context(adventure, settings) + clean, delta = worldstate.extract_delta(await generate(system_text, story_text)) + ai = models.Action(adventure_id=adventure.id, type="ai", text=clean) + tree.place_action(db, adventure, ai) + db.add(ai) + if delta: + adventure.world_state, report = worldstate.apply_delta( + adventure.world_state, scenario.stat_schema, delta, i) + ai.world_delta = report + db.commit() + db.refresh(adventure) + print(f" [{i}] {move} → {words(clean)} words" + + (f", state {delta}" if delta else ""), flush=True) + + old_system = prompt_at(args.before, "MEMORY_SYSTEM_PROMPT") + actions = sorted( + (a for a in adventure.actions if a.type in ("start", "do", "ai")), + key=lambda a: (a.depth if a.depth is not None else 0, a.id), + ) + blocks = [actions[i:i + memorybank.MEMORY_INTERVAL] + for i in range(0, len(actions) - memorybank.MEMORY_INTERVAL + 1, + memorybank.MEMORY_INTERVAL)] + print(f"\n{len(actions)} actions → {len(blocks)} blocks. Summarizing each twice…", + flush=True) + + rows = [] + for n, block in enumerate(blocks, 1): + raw = "\n\n".join(a.text for a in block) + plain = (f"Story excerpt:\n\n" + f"{truncate_to_last_tokens(raw, 2000)}\n\nMemory:") + brief = memorybank.cast_brief(adventure, raw) + before = await provider(args.model).complete(old_system, plain) + after = await provider(args.model).complete( + memorybank.MEMORY_SYSTEM_PROMPT, + f"{brief}\n\n{plain}" if brief else plain) + rows.append((n, before.strip(), after.strip())) + print(f" block {n}: before {words(before)}w ({framing(before)}), " + f"after {words(after)}w ({framing(after)})", flush=True) + + print("\n" + "=" * 74) + print(f"{'':16}{'words':>6} framing") + print("-" * 74) + for n, before, after in rows: + for label, text in (("before", before), ("after", after)): + print(f"memory {n} {label:7}{words(text):>6} {framing(text)}") + print("=" * 74) + + if args.out: + with open(args.out, "w", encoding="utf-8") as fh: + fh.write("# Memory prompt A/B\n\n") + fh.write(f"One story, generated through `build_context` against " + f"`{args.model}` at `{args.endpoint}`. Both prompts then " + f"summarize the same blocks, so the prompt is the only " + f"variable. The control is `MEMORY_SYSTEM_PROMPT` as of " + f"`{args.before}`.\n\n") + fh.write("| memory | arm | words | framing |\n|---|---|---|---|\n") + for n, before, after in rows: + for label, text in (("before", before), ("after", after)): + fh.write(f"| {n} | {label} | {words(text)} | {framing(text)} |\n") + for n, before, after in rows: + fh.write(f"\n## Memory {n}\n\n**Before:** {before}\n\n" + f"**After:** {after}\n") + fh.write("\n## The story both arms summarized\n\n") + for a in actions: + fh.write(f"**{a.type}:** {a.text}\n\n") + print(f"\nwritten to {args.out}") + + db.close() + Base.metadata.drop_all(bind=engine) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument("--endpoint", default="http://127.0.0.1:8787/v1") + parser.add_argument("--model", default="sonnet") + parser.add_argument("--before", default="9cdcb55", + help="commit holding the prompt to compare against") + parser.add_argument("--turns", type=int, default=6) + parser.add_argument("--max-tokens", type=int, default=700) + parser.add_argument("--out") + args = parser.parse_args() + + # A scratch database, so a run never touches the developer's own. + handle = tempfile.NamedTemporaryFile(suffix=".db", delete=False) + handle.close() + os.environ["AIDND_DB_PATH"] = handle.name + os.environ.pop("AIDND_DATABASE_URL", None) + os.environ.pop("DATABASE_URL", None) + sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + try: + asyncio.run(main(args)) + finally: + Path(handle.name).unlink(missing_ok=True) diff --git a/plan/18-appendix-memory-ab-run.md b/plan/18-appendix-memory-ab-run.md new file mode 100644 index 0000000..23f97ba --- /dev/null +++ b/plan/18-appendix-memory-ab-run.md @@ -0,0 +1,134 @@ +# Appendix: the memory-prompt A/B, run 2026-08-31 + +The evidence behind `plan/18-persona-and-memory-quality.md`. Reproduce it with: + + python tools/claude_shim.py --port 8787 & + python tools/memory_ab.py --out ab.md + +One story, generated a turn at a time through the app's own `build_context`. +Both prompts then summarize the **same** blocks, so the story is held constant +and the prompt is the only variable. Every call is a separate request, so +neither arm sees the other's output, and the model is never told what is being +measured. The control is `MEMORY_SYSTEM_PROMPT` as of commit `9cdcb55`, read +from git rather than pasted, so it cannot drift from what shipped. + +The model was a Claude model, reached through `tools/claude_shim.py`. See +"What this does not show" in plan/18 before generalising from it. + +## What it showed + +| memory | arm | words | how it names the protagonist | +|---|---|---|---| +| 1 | before | 34 | second person — "**You** crept low…" | +| 1 | after | 72 | "**Kaelen** and Gwen crouched…" | +| 2 | before | 105 | "**The player** asked Gwen to…" | +| 2 | after | 68 | "**Kaelen** and Gwen infiltrated…" | + +Two things came out of this, and only one of them was the thing being tested. + +**The reported fault reproduced.** Two consecutive memories, from one story, +written minutes apart, in two different persons. That is the complaint, observed +rather than argued from the prompt text. Both after-memories name Kaelen. + +**A fault nobody had noticed.** "1-2 plain sentences" is not a length: 34 words +for one block, 105 for the next. `memory_top_k` injects five memories every +turn, so the bank's running cost was set by a number never stated. +`MEMORY_MAX_WORDS = 50` now states it. Re-run over this same story, the two +blocks came back at **32 and 58 words**, still named, still third person, still +carrying the camp map, the strongbox behind the second tent, and the strap +frayed near through: + +> Kaelen crept low toward the strongbox, now known from the map to be behind +> the second tent, with the dozing watchman still unaware and Gwen holding her +> covering position on the flank. + +**A likely accuracy win, at n=1.** The player's move was `grab her wrist and +pull her down behind the woodpile`. The before-memory recorded "The player asked +Gwen to grab her wrist and pull her down", inverting who acted. This is the +failure the brief predicts — with no cast there is nothing to say whose wrist +"her wrist" is — but one sample is an observation, not a result. + +--- + +*Everything below is the raw run, unedited.* + +## Memory 1 + +**Before:** You crept low through the mist toward the second tent, following the map's mark toward the strongbox while Gwen held her flanking position, the sleeping watchman still dozing on his spear near the lean-to. + +**After:** Kaelen and Gwen crouched at dawn outside the bandit camp; Kaelen signaled Gwen to circle the right flank while a drowsy watchman kept post near the lean-to, and Gwen took position unseen near the strongbox tent. Kaelen searched an abandoned bedroll, finding a whetstone, a few copper coins, dried meat, and a charcoal-scrawled camp map revealing the stolen strongbox was hidden behind the second tent, then began moving low toward that location. + +## Memory 2 + +**Before:** The player asked Gwen to grab her wrist and pull her down behind the woodpile just before a second bandit—half-dressed, carrying a piss-bucket—stumbled out, relieved himself, and returned to bed in the first tent; Gwen lingered a beat before freeing her wrist and murmured "Good eyes." After waiting five heartbeats to confirm the bandit was asleep, the player and Gwen returned to the strongbox tent, where they found the old iron caravan lock too sturdy to pick and instead began testing the straps, discovering one frayed near through at the fold; the player then asked Gwen whether she still trusted their read on the situation. + +**After:** Kaelen and Gwen infiltrated the bandit camp to reclaim a stolen strongbox, sheltering behind a woodpile while a second bandit relieved himself nearby before returning to sleep; afterward Kaelen returned to the strongbox tent, found its lock too sturdy to force by hand but noticed one strap frayed near through, and paused there with Gwen covering him to ask whether she still trusted his read on the situation. + +## Summary + +**Before:** Under cover of mist, the player and Gwen scouted a bandit camp, aiming to reach a strongbox marked on their map while a watchman dozed near a lean-to. Approaching the second tent, the player pulled Gwen down behind a woodpile just in time to avoid a half-dressed bandit who stepped out to relieve himself before returning to sleep in the first tent; Gwen lingered a moment before pulling free, murmuring "Good eyes." After waiting to confirm the camp was settled, the two returned to the strongbox tent, finding its old iron caravan lock too sturdy to pick. They shifted to testing the straps securing it instead, discovering one frayed nearly through at the fold—a possible way in. The player then turned to Gwen to ask whether she still trusted their read on the situation, leaving the moment of decision unresolved. + +**After:** At dawn, Kaelen and Gwen scouted a bandit camp holding a stolen strongbox. Kaelen sent Gwen to circle the right flank near the strongbox tent while a drowsy watchman kept post, then searched an abandoned bedroll himself, finding a whetstone, a few copper coins, dried meat, and a charcoal map marking the strongbox's location behind the second tent. Moving low toward it, the pair sheltered behind a woodpile while a second bandit briefly woke and relieved himself before returning to sleep. Reaching the strongbox tent, Kaelen found its lock too sturdy to force by hand, but noticed one securing strap frayed nearly through. With Gwen covering him, he paused to ask whether she still trusted his read on the situation before attempting to exploit the weak strap. + +## The story that was summarized + +**start:** Dawn breaks grey over the treeline as you and Gwen crouch at the edge of the bandit camp. Smoke curls from a dying fire; three bedrolls lie empty. Somewhere ahead, the stolen caravan strongbox waits. + +Gwen checks her bowstring and looks to you. "Quiet, or loud?" + +**do:** go quiet and signal Gwen to circle around the right flank + +**ai:** You melt back from the ridge line without a sound, boots finding moss instead of twigs, and catch Gwen's eye. Two fingers point right, then curl in a half-circle — *swing wide, come at the flank*. She reads it instantly, dips her chin, and peels off through the bracken, bow already half-raised, moving low and quiet along the tree line until the mist swallows her. + +You hold position at the camp's edge, breath slow, watching the dying fire spit its last embers. A bandit shifts on watch near the lean-to, spear butt planted in the dirt, more bored than alert. No one else stirs. Somewhere off to the right you catch the faintest whisper of disturbed leaves — Gwen, further along than you'd expect, patient as ever. She's given herself a clean angle on the strongbox tent from the flank, out of the watchman's sightline entirely. + +The plan is good. Quiet holds. All you need now is the first move — and the choice of who falls first, and how. + +The watchman yawns, oblivious, close enough that a single silent step would put you in reach of his throat before he ever draws breath to shout. + +**do:** search the nearest bedroll for anything useful + +**ai:** You crouch by the nearest bedroll — a heap of stitched furs still holding the sleeper's warmth, though its owner is long since up and yawning by the fire. You keep one eye on the watchman as your fingers work through the folds. A whetstone, a handful of copper coins that clink softly against your palm before you still them with a fist, a scrap of oilcloth wrapped around dried meat gone hard at the edges. Nothing worth the risk so far. + +Then your fingers catch on something stiffer, tucked beneath the bedroll's edge — a folded scrap of parchment, half-torn, charcoal-scrawled. A rough map of the camp, crude but legible: the lean-to marked with an X, a second mark deeper in camp labeled in shorthand you recognize as bandit cant for *strongbox*. It's tucked behind the second tent, not the first — further from the watchman than you'd guessed, closer to where Gwen is now waiting in the mist. + +You fold it away inside your jerkin, pulse ticking a little faster. Small find, real value — it just turned guesswork into a plan. The watchman still hasn't stirred, spear planted, chin dropping toward his chest. Gwen's silhouette is a smudge of shadow at the flank, patient, waiting on your signal. + + + +The camp holds still around you — watchman drowsing, fire dying, Gwen coiled and ready on the flank. The map sits warm against your ribs, the strongbox's true location now known. Your move. + +**do:** move toward the strongbox, keeping low + +**ai:** You drop low, weight rolling from heel to ball of each foot, threading between the empty bedroll and a stack of split firewood. The watchman's chin has sunk near to his chest, spear listing in his slack grip. You give him a wide berth anyway — no need to gamble breath on a man already halfway to sleep. + +The second tent is smaller than the first, its flap laced shut with a crude leather cord. Just as the parchment promised, its bulk sits wrong for a tent meant to sleep anyone — square-cornered, iron-banded, too heavy for the ground it rests on. You crouch beside it, easing the lacing open one loop at a time, and the flap parts on a dull gleam of banded oak and iron: the strongbox, lock intact, straps still buckled from the caravan raid. + +A shift of cloth behind you — not the watchman, too far behind you for that. You go still, hand dropping to your knife hilt, eyes cutting sideways through the tent's shadow toward the sound. Nothing resolves yet, just a settling, maybe wind through canvas, maybe not. Across the clearing, Gwen has gone motionless too, bow half-drawn, watching the same dark gap between tents that's caught your attention. + +The strongbox is here, within reach, real weight under your palm instead of a rumor on a scrap of parchment. But something in that gap between the tents hasn't declared itself yet, and the watchman is still only a few careless steps away from waking to find you crouched over his camp's whole reason for being. + +**do:** grab her wrist and pull her down behind the woodpile + +**ai:** Your hand shoots out and closes around Gwen's wrist before she's fully turned toward the gap between the tents. She goes rigid for half a heartbeat — a killer's instinct, ready to break the grip — then reads your face and lets herself be pulled. You drag her down behind the woodpile together, shoulders pressed to split logs, breath shallow. + +The shape that had been shifting in the shadow resolves itself: a second bandit, roused early, stumbling out from behind the tents with his breeches half-laced and a piss-bucket in hand, utterly uninterested in anything but the treeline. He doesn't look toward the strongbox tent. Doesn't look toward the woodpile. He shuffles past, empties the bucket against a stump, and turns back the way he came, already yawning. + +Gwen's breath eases out slow beside you. She doesn't pull her wrist away immediately — a beat longer than she needs to — before easing it free herself. "Good eyes," she breathes, barely sound at all. Her gaze flicks to your hand, then back to your face, something unreadable and warm passing behind it. Watching your back instead of charging in — that's the kind of thing she remembers. + +Through the gap in the logs you track the second bandit's path: he ducks into the first tent, not the watch line, and doesn't reemerge. Three accounted for now — the drowsing watchman, this one bedding back down, and whoever else the camp is hiding. The strongbox tent is still open behind you, lock waiting, straps waiting. Gwen settles back into her crouch, bow across her knees, eyes on you for the next signal. + +**do:** wait for the bandit to turn, then move + +**ai:** You go still as stone, eyes fixed on the tent flap the second bandit vanished into. Seconds crawl. Then the canvas stirs one last time — a grunt, a shifting weight settling onto a bedroll, the particular silence of a man who's decided the day can wait a while longer. You count five heartbeats to be sure, watching for any second shadow, any sign he's only feigning sleep. Nothing. Even the watchman by the fire hasn't stirred, spear still planted, chin still low. + +You ease up from the woodpile, moving on the balls of your feet, and slip back to the strongbox tent. Gwen catches the motion and rises with you, silent as a drawn breath, bow angled to cover the gap between tents while you work. The lock is old iron, caravan-make, not bandit work — it'll need proper tools or proper force, not fingers. + +You crouch and test the straps first, feeling for a weak buckle, a give point. One strap is frayed near through at the fold. With steady pressure it might part without the sharp crack of metal — quieter than forcing the lock, if slower. Gwen shifts her weight beside you, patient, trusting you to read the moment right, her eyes flicking once toward the dozing watchman and back. + +The camp holds its breath around you both. The strongbox waits under your hands, and the next move is yours to make. + +**do:** ask her whether she still trusts my read on this + +**ai:** + diff --git a/plan/18-persona-and-memory-quality.md b/plan/18-persona-and-memory-quality.md index 216b9ce..e158167 100644 --- a/plan/18-persona-and-memory-quality.md +++ b/plan/18-persona-and-memory-quality.md @@ -460,12 +460,23 @@ is the first thing to try. ## Run with a real model, and what it changed -Run end to end against a real model, using the `claude` CLI as the provider: one -story generated through the app's own `build_context` a turn at a time, then -**both** memory prompts run over the **same** blocks, so the story is held -constant and the prompt is the only variable. Every call is a fresh process, so -no arm can see the other's output and the model is never told what is being -tested. The control is the exact prompt from commit `9cdcb55`, not a paraphrase. +Run end to end against a real model. One story generated through the app's own +`build_context` a turn at a time, then **both** memory prompts run over the +**same** blocks, so the story is held constant and the prompt is the only +variable. Neither arm sees the other's output, and the model is never told what +is being measured. The control is `MEMORY_SYSTEM_PROMPT` as of commit `9cdcb55`, +read out of git rather than pasted, so it cannot drift from what shipped. + +The harness is `backend/tools/memory_ab.py`, and it goes through +`OpenAICompatibleProvider` rather than calling a model directly, so the run +exercises the provider, the streaming path and `complete()`. Pointed at +`tools/claude_shim.py` it spends a Claude subscription instead of API credit: + + python tools/claude_shim.py --port 8787 & + python tools/memory_ab.py --out ab.md + +**The full transcript, with every memory, both summaries and the story they were +written from, is in `plan/18-appendix-memory-ab-run.md`.** ### The reported fault reproduced, and the fix held diff --git a/plan/STATUS.md b/plan/STATUS.md index 90aeb65..fbae008 100644 --- a/plan/STATUS.md +++ b/plan/STATUS.md @@ -3,7 +3,7 @@ Read this first when picking the project back up. Updated at the end of a working session; the per-phase plan files hold the detail, this holds the thread. -**Last updated: 2026-08-28.** +**Last updated: 2026-08-31.** --- @@ -76,6 +76,45 @@ needed; nothing requires reading a row of anyone's story. --- +## What happened on 2026-08-31 — the persona, and what the summarizer is told + +**`plan/18-persona-and-memory-quality.md` is the writeup; it is on +`claude/ai-dnd-memories-summarization-3muo98`, not yet merged.** Two changes, both green +at 610 tests. + +**The protagonist now has a name.** An adventure carries `persona_name`, +`persona_pronouns` and `persona_desc` (migrations 74-76), and the player's stat block +renders as `Kaelen (player): hp 100/100` instead of `You: hp 100/100`. The paths do not +change — a path carrying the persona's name would break the moment a player renamed +their character, because `_history_text` replays stored deltas holding literal +`player.hp` strings. Empty name means the app behaves exactly as before, so no backfill. +Driven in a browser, 21/21 checks. + +**The summarizer used to be told nothing.** It got six actions of second-person prose +and no cast, no setting, and no instruction about what person to write in. It now gets a +cast brief built from the story cards — which already cover the schema NPCs, because +`scenario_card_specs` turns every one of them into a card at adventure creation. + +**Keyword matching alone was not enough, and only running it showed that.** Built to the +plan first, the brief for "She grabs your arm" listed the protagonist and nobody else: +the block that most needs a cast is exactly the one written in bare pronouns. Matched +cards now come first and the rest of the roster is filled with the other `character` +cards. + +**Running it against a real model found a second fault.** "1-2 plain sentences" is not a +length — the same model wrote 34 words for one block and 105 for the next, and +`memory_top_k` injects five every turn. `MEMORY_MAX_WORDS = 50` states it; the same +blocks then came back at 32 and 58. The A/B harness is `backend/tools/memory_ab.py`, +it drives the real provider through `tools/claude_shim.py`, and the full transcript is +in `plan/18-appendix-memory-ab-run.md`. + +**Still unmeasured: whether a weaker model complies.** The run used a Claude model +through the shim. The app talks to an OpenAI-compatible endpoint, and +`worldstate/parse.py` tolerates trailing commas because free models emit them. Point +`memory_ab.py --endpoint` at the real provider to find out. + +--- + ## Pick up here **`plan/17-refactor.md` is the active phase.** It carries its own progress table, which