Jesse.Markowitz/interactive-story
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v1.1 WP-B.1: diagnose independent long-term memory retention

Browse Source 
Diagnostic only; no memory behaviour changes.

- tools/memory_diagnostic.py: planted-fact isolation checks, the four-stage
  diagnosis (created / retained / ranked / injected) with a verdict, a
  production-ranking replica, deterministic summariser/embedder/narrator
  stubs and seven scenarios (default, past capacity, pinned, low top_k,
  long-block early/late, lineage control)
- tools/v11_b1_memory.py: CLI for the scenarios and for diagnosing a copy of
  a finished real campaign
- tools/m11_long_run.py: opt-in --independent-fact mode with per-turn
  isolation tracking and the recovered_through_memory_independent verdict;
  M04 verdicts unchanged
- tests: diagnostic stages, eviction, creation window, ranking, lineage and
  authority controls; two strict xfails record the diagnosed retention and
  creation defects for WP-B.2 to flip
- planning/reports/v1.1/V1.1-WP-B1-REPORT.md

First failing stage: ranking (real model); retention past capacity and
creation for early facts in long blocks (deterministic, same on v1.0.0).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VvegagkhuCZoFPdv4M1egY
This commit is contained in:
JesseMarkowitz
2026-09-14 20:50:05 -04:00
co-authored by Claude Opus 5
parent d63804f22e
commit beb17ada10
6 changed files with 2184 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tools/m11_long_run.py
+206
View File
@@ -190,6 +190,31 @@ CLUE_FACT = {
"fact_id": "silver-key-opens-crypt",
}
#: v1.1 WP-B.1: a second planted fact, established in the **story only**.
#:
#: The M04 clue above is planted as accepted state, and memories are written
#: from story text, so no memory could ever carry it on its own. That is why
#: every M04 recovery so far ran through state. This fact is told to the reader
#: in narration and never corrected into state, so memory is the only layer that
#: is meant to carry it. `--independent-fact` plants it and reports
#: `recovered_through_memory_independent` only when every other layer is proven
#: not to carry it. The words are copied from `tools/memory_diagnostic.FACT_F`,
#: for the reason `HISTORY_LABELS` is copied.
INDEPENDENT_FACT_TEXT = ("I watch Mara slip the amber sundial inside the cracked teapot on "
"the tavern's top shelf, and she makes me promise to tell no one.")
INDEPENDENT_FACT_TERMS = ("sundial", "teapot")
INDEPENDENT_RECALL_TEXT = "I ask Mara quietly where she hid the amber sundial."
#: How far past the planting turn its memory block can reach. Narration inside
#: that block may repeat the fact; narration after it may not.
INDEPENDENT_BLOCK_SLACK = 6
INDEPENDENT_PRECONDITIONS = (
"planted_turn_outside_history",
"absent_from_state",
"absent_from_summary",
"absent_from_knowledge",
"absent_from_later_narration",
)
CANON = [
"The dead do not return. No rite, relic or bargain has ever returned anyone.",
"The abbey crypt has been sealed since the founding.",
@@ -365,6 +390,14 @@ class Run:
#: The depth of the player turn that planted the clue. M04's
#: precondition is that this turn has left the history window.
self.planted_depth: int | None = None
#: v1.1 WP-B.1, with --independent-fact: where the story-only fact was
#: planted, and the accepted-turn count at which each isolation
#: precondition first failed.
self.independent_fact = False
self.independent_depth: int | None = None
self.independent_violations: dict[str, int] = {}
self.last_done: dict = {}
self.last_report: dict = {}
# ------------------------------------------------------------ recording
@@ -393,6 +426,8 @@ class Run:
"turns_target": self.turns_target,
"log_offset": self.log_offset,
"planted_depth": self.planted_depth,
"independent_depth": self.independent_depth,
"independent_violations": self.independent_violations,
"written": datetime.now().isoformat(timespec="seconds"),
}
tmp = self.out / (RESUME_FILE + ".tmp")
@@ -414,6 +449,8 @@ class Run:
self.elapsed_before = prior.get("elapsed_seconds", 0)
self.log_offset = prior.get("log_offset", 0)
self.planted_depth = prior.get("planted_depth")
self.independent_depth = prior.get("independent_depth")
self.independent_violations = dict(prior.get("independent_violations") or {})
self.resumed = True
def reattach(self) -> None:
@@ -570,15 +607,44 @@ class Run:
"observed_margin": accounting.get("observed_margin"),
"safety_reserve": accounting.get("safety_reserve"),
})
self.last_done = done
if self.independent_fact and self.independent_depth is not None:
self._check_independent_isolation(done, sample)
self.note("turn", text=text, seconds=round(seconds, 1), **sample)
return {"accepted": True, "seconds": seconds, **sample}
def _check_independent_isolation(self, done: dict, sample: dict) -> None:
"""v1.1 WP-B.1: does anything but memory carry the story-only fact yet?
Checked on every accepted turn, so a run knows the first turn at which
the experiment stopped being about memory, instead of finding out at
recall. Each precondition records only its first failure.
"""
depth = sample.get("total_actions", 0) - 1
text = (done.get("action") or {}).get("text") or ""
found = {}
if depth > self.independent_depth + INDEPENDENT_BLOCK_SLACK and _mentions_fact(text):
found["absent_from_later_narration"] = f"narration at depth {depth}"
document = self.state().get("document") or {}
if _mentions_fact(json.dumps(document)):
found["absent_from_state"] = "the narrative state names the fact"
summary = next((sec.get("text", "") for sec in (self.last_report.get("sections") or [])
if sec.get("label") == SUMMARY_LABEL), "")
if _mentions_fact(summary):
found["absent_from_summary"] = "the active summary names the fact"
for name, detail in found.items():
if name not in self.independent_violations:
self.independent_violations[name] = self.accepted
self.note("independent_precondition_failed", precondition=name, detail=detail)
sample["independent_violations"] = dict(self.independent_violations)
def count_actions(self) -> int:
return self.server.call("GET", f"/adventures/{self.adv}/actions?limit=1")["total"]
def measure(self) -> dict:
"""M03's numbers, read from the prompt the app would send right now."""
report = self.server.call("GET", f"/adventures/{self.adv}/context")
self.last_report = report
tokens = report["tokens"]
sections = {s["label"]: s["tokens"] for s in report["sections"]}
window = report.get("window") or {}
@@ -714,6 +780,10 @@ def main() -> int:
"--max-consecutive-failures", type=int,
default=DEFAULT_MAX_CONSECUTIVE_FAILURES,
help="stop and write the evidence after this many unaccepted turns")
parser.add_argument(
"--independent-fact", action="store_true",
help=("v1.1 WP-B.1: also plant a story-only fact at depth 3 and report "
"whether memory alone recovers it"))
args = parser.parse_args()
if not (ENDPOINT and MODEL and EMBED_MODEL):
@@ -746,6 +816,7 @@ def main() -> int:
server.start()
run = Run(server, out, turns_target=args.turns,
turn_timeout=args.turn_timeout)
run.independent_fact = args.independent_fact
if prior:
run.adopt(prior)
@@ -782,6 +853,18 @@ def main() -> int:
"the planted clue is not in accepted state, so M04 cannot "
"be measured from this run. Stopping before the campaign "
"starts rather than reporting a recall failure later.")
if args.independent_fact:
# v1.1 WP-B.1: the story-only fact, told in the next turn and
# never corrected into state. Depth 3: the opening, the clue turn
# and its reply come first.
if any(_mentions_fact(md) for md in (CANON_MD, REFERENCE_MD, INSPIRATION_MD)):
raise SystemExit("the imported knowledge names the independent fact")
planting_f = run.turn(INDEPENDENT_FACT_TEXT)
if not planting_f.get("accepted"):
raise SystemExit("the turn that plants the independent fact was not accepted")
run.independent_depth = planting_f["total_actions"] - 2
run.note("independent_fact_planted", depth=run.independent_depth,
terms=list(INDEPENDENT_FACT_TERMS))
# The first checkpoint, and the point from which --resume works: the
# campaign exists and its clue is planted.
run.save_resume()
@@ -846,10 +929,15 @@ def main() -> int:
# Skipped on an aborted run: it asks the narrator a question, and the
# reason the run stopped is that the narrator does not answer.
recall = None
independent = None
if aborted is None:
run.note("recall_begin")
recall = _recall(run)
(out / "recall.json").write_text(json.dumps(recall, indent=2))
if args.independent_fact and run.independent_depth is not None:
independent = _independent_recall(run, out / "campaign.db")
(out / "recall-independent.json").write_text(json.dumps(independent, indent=2))
run.note("independent_recall", verdict=independent["verdict"])
# ---- Export whatever exists, for the recovery evidence. ----
# Attempted even for an aborted run: the recovery check and the storage
@@ -897,6 +985,7 @@ def main() -> int:
"elapsed_seconds": run.elapsed(),
"turn_timeout_seconds": args.turn_timeout,
"recall": recall,
"independent_recall": independent,
"final_state": _or_none(lambda: run.state()["document"]),
"final_measurement": _or_none(run.measure),
"db_bytes": db_path.stat().st_size,
@@ -1224,6 +1313,123 @@ def _m04_verdict(recall: dict) -> str:
return "not_recovered"
def _mentions_fact(text: str | None) -> bool:
"""v1.1 WP-B.1: whether `text` names the independent fact, as a whole word."""
low = (text or "").lower()
return any(re.search(rf"(?<![a-z]){term}(?![a-z])", low) for term in INDEPENDENT_FACT_TERMS)
def _independent_memory_verdict(check: dict) -> str:
"""v1.1 WP-B.1: whether memory alone recovered the story-only fact.
`recovered_through_memory_independent` requires every precondition, so no
other layer could have carried the fact. It also requires that a memory
covering the planting turn carries the fact and was injected into the recall
turn. A failed precondition is named and is never a recovery, and the M04
verdicts above are untouched.
"""
if check.get("independent_planted_depth") is None:
return "precondition_unknown:planted_depth"
for name in INDEPENDENT_PRECONDITIONS:
value = check.get(name)
if value is None:
return f"precondition_unknown:{name}"
if not value:
return f"precondition_failed:{name}"
if not check.get("memory_covering_planting_carries_fact"):
return "not_recovered:not_created"
if check.get("memory_forgotten"):
return "not_recovered:evicted"
if not check.get("memory_injected"):
return "not_recovered:not_injected"
return "recovered_through_memory_independent"
def _independent_recall(run: "Run", db_path: Path) -> dict:
"""v1.1 WP-B.1: ask for the story-only fact, and find out which layer answered.
The prompt-level facts come from the recall turn's own stored context. The
memory rows come from the campaign database, read-only. Ranking is not
recomputed here, because that needs the embedding model;
`tools/v11_b1_memory.py diagnose` does it afterwards against a copy of the
database.
"""
import sqlite3
import zlib
result = run.turn(INDEPENDENT_RECALL_TEXT)
action_id = (run.last_done.get("action") or {}).get("id")
snapshot = (run.server.call("GET", f"/adventures/{run.adv}/actions/{action_id}/context")
if result.get("accepted") and action_id else {}) or {}
sections = {}
for sec in snapshot.get("sections") or []:
sections.setdefault(sec.get("label"), []).append(sec.get("text", ""))
text_of = {label: "\n".join(parts) for label, parts in sections.items()}
floor = (snapshot.get("history") or {}).get("floor_depth")
depth = run.independent_depth
used = [m.get("id") for m in (snapshot.get("memories") or {}).get("used") or []]
covering = []
connection = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
try:
rows = connection.execute(
"SELECT id, text, source_start, source_end, forgotten, pinned, use_count, "
"branch_id, depth FROM memories WHERE adventure_id = ? AND source_start <= ? "
"AND source_end >= ? ORDER BY id", (run.adv, depth, depth)).fetchall()
blob = connection.execute(
"SELECT context_snapshot FROM actions WHERE id = ?", (action_id or -1,)).fetchone()
finally:
connection.close()
for row in rows:
memory_id, text, start, end, forgotten, pinned, use_count, branch_id, node_depth = row
covering.append({
"memory_id": memory_id, "text": text, "source_start": start, "source_end": end,
"forgotten": bool(forgotten), "pinned": bool(pinned), "use_count": use_count,
"branch_id": branch_id, "depth": node_depth,
"carries_fact": all(re.search(rf"(?<![a-z]){t}(?![a-z])", (text or "").lower())
for t in INDEPENDENT_FACT_TERMS),
"injected": memory_id in used,
})
carrying = [c for c in covering if c["carries_fact"]]
best = next((c for c in carrying if c["injected"]), carrying[0] if carrying else None)
stored_snapshot_readable = blob is not None and blob[0] is not None
if stored_snapshot_readable:
try:
json.loads(zlib.decompress(blob[0]))
except Exception: # noqa: BLE001
stored_snapshot_readable = False
document = run.state().get("document") or {}
violations = dict(run.independent_violations)
check = {
"independent_planted_depth": depth,
"recall_accepted": bool(result.get("accepted")),
"history_floor_depth": floor,
"planted_turn_outside_history": (None if not snapshot else
floor is not None and depth < floor),
"absent_from_state": ("absent_from_state" not in violations
and not _mentions_fact(json.dumps(document))
and not _mentions_fact(text_of.get(STATE_LABEL))),
"absent_from_summary": ("absent_from_summary" not in violations
and not _mentions_fact(text_of.get(SUMMARY_LABEL))),
"absent_from_knowledge": not any(
_mentions_fact(text_of.get(label)) for label in IMPORTED_KNOWLEDGE_LABELS),
"absent_from_later_narration": "absent_from_later_narration" not in violations,
"violations_first_turn": violations,
"covering_memories": covering,
"memory_covering_planting_carries_fact": bool(carrying),
"memory_forgotten": bool(best and best["forgotten"]),
"memory_injected": bool(best and best["injected"]),
"memory_text_in_memories_section": bool(
best and best["text"] and best["text"] in (text_of.get(MEMORIES_LABEL) or "")),
"memory_ids_used": used,
"recall_action_id": action_id,
"stored_snapshot_readable": stored_snapshot_readable,
}
check["verdict"] = _independent_memory_verdict(check)
return check
#: Signs the application stored protocol as story. The first is a state-section
#: heading with an indented entry under it, in any markdown, because
#: `## Established:` got past a plain substring match and the count read 1