From a408c7b6f7fe2f185de823e85984f42990a3c192 Mon Sep 17 00:00:00 2001 From: parththakkar106 Date: Sun, 23 Aug 2026 06:31:30 +0530 Subject: [PATCH] Lay the prompt out so the endpoint can cache most of it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prompt caching bills on a shared prefix: the endpoint reuses the request up to the first byte that differs from last time and no further. The live world-state block sat third from the top of the system message, so every turn re-priced the instructions, the plot essentials and the whole story history underneath it. The retrieved memories and the rewritten summary did it again. Everything fixed is emitted first now, and everything that moves goes after the history, ordered least-volatile first — which is also where recency serves it best, the reasoning that already put the emit reminder last. The three tail sections that are last for their own reasons stay last. The moved sections are still charged to the token budget; only their position changed. Two smaller halves of the same problem. OpenRouter serves a model from whichever upstream is free and each upstream holds its own cache, so a deepseek model now names deepseek as its preferred upstream — a preference, not a restriction, so a turn still runs if that upstream is down. And the endpoint's usage block is read back off the response and kept per attempt, so the hit rate shows up in Insights and the debug log instead of being assumed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01DfMCsN1KBLsTqMkj5hSgrY --- backend/app/attempts.py | 6 +- backend/app/context/builder.py | 79 ++++-- backend/app/debuglog.py | 14 +- backend/app/providers/base.py | 7 + backend/app/providers/openai_compatible.py | 66 ++++- backend/app/routers/adventures.py | 4 + backend/tests/test_attempt_siblings.py | 1 + backend/tests/test_branch_forking.py | 1 + backend/tests/test_branch_management.py | 1 + backend/tests/test_bundle_v2.py | 1 + backend/tests/test_chat.py | 1 + backend/tests/test_prompt_caching.py | 241 +++++++++++++++++++ backend/tests/test_retry_variants.py | 1 + backend/tests/test_story_tree_baseline.py | 1 + backend/tests/test_take_edit.py | 1 + backend/tests/test_take_parentage.py | 1 + backend/tests/test_take_state.py | 1 + backend/tests/test_turn_flow_integration.py | 1 + backend/tests/test_worldstate_integration.py | 1 + frontend/src/pages/Play.jsx | 21 ++ 20 files changed, 425 insertions(+), 25 deletions(-) create mode 100644 backend/tests/test_prompt_caching.py diff --git a/backend/app/attempts.py b/backend/app/attempts.py index a4b13ef..37e50a1 100644 --- a/backend/app/attempts.py +++ b/backend/app/attempts.py @@ -40,9 +40,11 @@ from .context import lineage # The slices of a context snapshot that belong to one attempt rather than to # the turn: the world-state delta it proposed and what the referee did with it, -# the script report, and the model's literal reply. Everything else in a +# the script report, the model's literal reply, and the endpoint's token +# accounting — each attempt is its own API call, and a retry is precisely the +# call expected to read the prompt back out of cache. Everything else in a # snapshot is the prompt, which is assembled once per turn. -ATTEMPT_KEYS = ("world_state", "script", "raw_output") +ATTEMPT_KEYS = ("world_state", "script", "raw_output", "usage") # ------------------------------------------------------------------ reading diff --git a/backend/app/context/builder.py b/backend/app/context/builder.py index 75fb1eb..6e71850 100644 --- a/backend/app/context/builder.py +++ b/backend/app/context/builder.py @@ -9,6 +9,12 @@ [Story history] newest actions that fit the remaining token budget [Author's Note] injected AUTHORS_NOTE_DEPTH actions before the end of history [Latest player action] (+ script frontMemory right after it, Phase 4) + +Which components are present is AI Dungeon's design, above. The *order* they +are laid out in is not: everything fixed is emitted first and everything that +moves after the history, because prompt caching bills on a shared prefix and +one mutable section high up re-prices the whole prompt under it. See the two +"static block" / "live sections" comments in `build_context`. """ import functools @@ -190,24 +196,26 @@ def build_context( `exclude_action_id` omits one action from the story (see history.py).""" script_mem = _script_memory(adventure) - # ----- Always-included components ----- + # ----- The static block: byte-for-byte the same prompt every turn ----- + # The ordering here is a billing decision, not a stylistic one. Prompt + # caching matches a *prefix*: an endpoint reuses the prompt up to the first + # byte that differs from last time and no further. So one mutable section + # near the top re-prices everything below it, and what is below it is the + # story history, which is the bulk of the prompt. Anything that changes + # turn to turn therefore goes *after* the history, in the live sections — + # which is also where recency serves it best, the same reasoning that + # already puts EMIT_REMINDER last. system_sections: list[Section] = [Section("narrator", settings.narrator_prompt.strip())] - # RPG world state (Phase 12): current stats/milestones + how to report changes. + # RPG world state (Phase 12): how to report changes. The live values are a + # live section below; the schema-derived guide and the emit rule are fixed + # for as long as the scenario is. stat_schema = adventure.scenario.stat_schema if adventure.scenario else None has_ws = worldstate.has_schema(stat_schema) if has_ws: guide = worldstate.render_reference(stat_schema) if guide: system_sections.append(Section("world_state_guide", guide)) - block = worldstate.render_state_section( - adventure.world_state, stat_schema, - _visible_npcs( - history.tail(adventure, NPC_WINDOW, exclude_action_id), stat_schema - ), - ) - if block: - system_sections.append(Section("world_state", block)) system_sections.append(Section("world_state_rule", worldstate.EMIT_RULE)) if isinstance(script_mem.get("context"), str) and script_mem["context"].strip(): @@ -218,13 +226,33 @@ def build_context( system_sections.append( Section("plot_essentials", f"Plot essentials:\n{adventure.memory.strip()}") ) - if adventure.story_summary.strip(): - system_sections.append( - Section("story_summary", f"Story summary:\n{adventure.story_summary.strip()}") - ) + + # ----- Live sections: everything that moves, built here, placed after the + # history further down. Ordered least-volatile first, so a turn that + # changes only the fastest-moving one keeps the others cached too: the + # summary is rewritten every few turns, lore turns over with the scene, the + # retrieved memories change on most turns, the stat values on nearly all. + # `world_lore` joins them below — it is the history window that triggers + # the cards, so it cannot be known yet. + summary_section = ( + Section("story_summary", f"Story summary:\n{adventure.story_summary.strip()}") + if adventure.story_summary.strip() + else None + ) + memories_section = None if memory_bank and memory_bank.get("used"): - lines = "\n".join(f"- {m['text']}" for m in memory_bank["used"]) - system_sections.append(Section("used_memories", f"Memories:\n{lines}")) + lines_text = "\n".join(f"- {m['text']}" for m in memory_bank["used"]) + memories_section = Section("used_memories", f"Memories:\n{lines_text}") + world_state_section = None + if has_ws: + block = worldstate.render_state_section( + adventure.world_state, stat_schema, + _visible_npcs( + history.tail(adventure, NPC_WINDOW, exclude_action_id), stat_schema + ), + ) + if block: + world_state_section = Section("world_state", block) authors_note_text = adventure.authors_note.strip() if isinstance(script_mem.get("authorsNote"), str) and script_mem["authorsNote"].strip(): @@ -237,8 +265,16 @@ def build_context( length_note = length_hint(settings.max_output_tokens, has_ws=has_ws) + # The live sections moved below the history but they are still in the + # prompt, so they are still reserved against the budget. (`world_lore` is + # not: it is budgeted out of `available` further down, as it always was.) reserved = ( sum(s.tokens for s in system_sections) + + sum( + s.tokens + for s in (summary_section, memories_section, world_state_section) + if s is not None + ) + count_tokens(authors_note) + count_tokens(front_memory) + count_tokens(length_note) @@ -273,8 +309,9 @@ def build_context( {"id": match["id"], "name": match["name"], "keyword": match["keyword"], "included": included} ) - if lore_lines: - system_sections.append(Section("world_lore", "\n".join(lore_lines))) + lore_section = ( + Section("world_lore", "\n".join(lore_lines)) if lore_lines else None + ) # ----- Story history: newest first until the remaining budget is spent ----- history_budget = available - used @@ -316,6 +353,12 @@ def build_context( note_sections.append(Section("recent_history", SEPARATOR.join(after))) else: note_sections.append(Section("history", SEPARATOR.join(texts))) + # The live sections, least volatile first (see where they are built). They + # sit below the history so the history caches, and above the tail so the + # three sections that are last for a reason stay last. + for live in (summary_section, lore_section, memories_section, world_state_section): + if live is not None: + note_sections.append(live) if front_memory: note_sections.append(Section("front_memory", front_memory)) # Sits just above the emit reminder, which keeps the strongest recency slot: diff --git a/backend/app/debuglog.py b/backend/app/debuglog.py index dcd1cd7..146eeb7 100644 --- a/backend/app/debuglog.py +++ b/backend/app/debuglog.py @@ -40,14 +40,26 @@ def start_entry(url: str, model: str, body: dict) -> dict: "request": _clip_obj(body), "status": "pending", "response": "", + "usage": None, "error": None, } _entries.appendleft(entry) return entry -def finish_entry(entry: dict, *, response: str = "", error: str | None = None) -> None: +def finish_entry( + entry: dict, + *, + response: str = "", + error: str | None = None, + usage: dict | None = None, +) -> None: + """`usage` is the endpoint's own token accounting when it reported any. + On OpenRouter it carries `prompt_tokens_details.cached_tokens`, which is + the only direct read on whether the prompt prefix is actually being + cached — a number worth seeing beside the request that produced it.""" entry["response"] = _clip(response) + entry["usage"] = usage entry["error"] = error entry["status"] = "error" if error else "ok" diff --git a/backend/app/providers/base.py b/backend/app/providers/base.py index 7a9f2f9..0c905af 100644 --- a/backend/app/providers/base.py +++ b/backend/app/providers/base.py @@ -16,6 +16,13 @@ class ProviderError(Exception): class Provider(ABC): + # The endpoint's own token accounting for the most recent call, when it + # reported any — notably `prompt_tokens_details.cached_tokens`, which is + # the only direct read on whether the prompt prefix is being cached. + # Callers read it after the call they made; one provider is built per + # request, so there is nothing to race. + last_usage: dict | None = None + @abstractmethod def generate( self, diff --git a/backend/app/providers/openai_compatible.py b/backend/app/providers/openai_compatible.py index 2fd746e..67fbc8e 100644 --- a/backend/app/providers/openai_compatible.py +++ b/backend/app/providers/openai_compatible.py @@ -10,6 +10,26 @@ from .base import PromptParts, Provider, ProviderError # continuing prose instead of replying conversationally. CHAT_CONTINUE_HINT = "\n\n[Continue the story directly. Output only story text.]" +# OpenRouter serves one model from whichever upstream is available, and every +# upstream holds its own prompt cache — so a request that lands somewhere new +# starts cold however stable the prompt is. Naming a preferred upstream makes +# routing deterministic, which is what lets a cache be hit at all. +# +# `allow_fallbacks` is deliberately left at its default of true: this is a +# preference, not a restriction. If the named upstream is down the request +# still goes through somewhere else and merely misses the cache, which is the +# behaviour we had anyway. +# +# A whitelist rather than a derivation from the model slug. The vendor half of +# a slug is *usually* the provider slug ("deepseek/..." -> "deepseek", verified +# against /api/v1/providers) but not reliably: Google's models are served by +# "google-ai-studio" and "google-vertex", and there is no "google". Look a +# vendor up on the model's Providers tab before adding it here — a slug that +# does not exist is a routing preference that silently does nothing at best. +_OPENROUTER_HOST = "openrouter.ai" +_PREFERRED_UPSTREAM = {"deepseek": "deepseek"} + + # Completion endpoints have no roles, so a plain chat has to be flattened into # one labelled transcript that trails off on "Assistant:" for the model to continue. _ROLE_LABELS = {"system": "System", "user": "User", "assistant": "Assistant"} @@ -42,6 +62,12 @@ class OpenAICompatibleProvider(Provider): # reject unknown fields); negative = explicitly ask the endpoint to # turn reasoning off. self.reasoning_max_tokens = reasoning_max_tokens + # Token accounting from the last call, when the endpoint reported any: + # prompt/completion counts plus, on OpenRouter, `prompt_tokens_details. + # cached_tokens` — the number of prompt tokens read from cache instead + # of billed in full. Written by every request method, so a caller reads + # it after the call it made; one provider is built per request. + self.last_usage: dict | None = None def _headers(self) -> dict: headers = {"Content-Type": "application/json"} @@ -66,6 +92,29 @@ class OpenAICompatibleProvider(Provider): body["reasoning"] = {"max_tokens": self.reasoning_max_tokens} body["max_tokens"] += self.reasoning_max_tokens + def _apply_provider_routing(self, body: dict) -> None: + """Prefer one upstream on OpenRouter, so the prompt cache is warm. + + Silent no-op everywhere else: `provider` is an OpenRouter extension and + Ollama and friends reject fields they do not know — the same trap the + `reasoning` param above is written around.""" + if _OPENROUTER_HOST not in self.base_url: + return + upstream = _PREFERRED_UPSTREAM.get(self.model.split("/", 1)[0].lower()) + if upstream: + body["provider"] = {"order": [upstream]} + + def _record_usage(self, payload: dict) -> None: + """Record the endpoint's own token accounting, if it reported any. + + OpenRouter always reports usage now (`usage: {include: true}` and + `stream_options` are deprecated no-ops), and in a stream it rides on a + final chunk that carries no choices — which is why this is read + separately from the text extraction rather than beside it.""" + usage = payload.get("usage") + if isinstance(usage, dict) and usage: + self.last_usage = usage + def _request(self, parts: PromptParts, temperature: float, max_tokens: int) -> tuple[str, dict]: if self.api_mode == "completion": url = f"{self.base_url}/completions" @@ -89,6 +138,7 @@ class OpenAICompatibleProvider(Provider): "stream": True, } self._apply_reasoning_budget(body) + self._apply_provider_routing(body) return url, body @staticmethod @@ -162,6 +212,7 @@ class OpenAICompatibleProvider(Provider): "stream": True, } self._apply_reasoning_budget(body) + self._apply_provider_routing(body) async for event in self._stream(url, body): yield event @@ -193,12 +244,15 @@ class OpenAICompatibleProvider(Provider): saw_sse = True data = line[5:].strip() if data == "[DONE]": - debuglog.finish_entry(log, response="".join(received)) + debuglog.finish_entry( + log, response="".join(received), usage=self.last_usage + ) return try: payload = json.loads(data) except ValueError: continue + self._record_usage(payload) reasoning = self._extract_reasoning(payload) if reasoning: yield "reasoning", reasoning @@ -215,6 +269,7 @@ class OpenAICompatibleProvider(Provider): "AI endpoint returned neither an SSE stream nor JSON: " + body_text[:200] ) + self._record_usage(payload) reasoning = self._extract_reasoning(payload) if reasoning: yield "reasoning", reasoning @@ -227,7 +282,7 @@ class OpenAICompatibleProvider(Provider): "AI endpoint returned a response with no text: " + body_text[:200] ) - debuglog.finish_entry(log, response="".join(received)) + debuglog.finish_entry(log, response="".join(received), usage=self.last_usage) except httpx.ConnectError as exc: error = f"Could not connect to {self.base_url} — is the AI server running?" debuglog.finish_entry(log, response="".join(received), error=error) @@ -272,6 +327,7 @@ class OpenAICompatibleProvider(Provider): "stream": False, } self._apply_reasoning_budget(body) + self._apply_provider_routing(body) log = debuglog.start_entry(url, self.model, body) try: @@ -285,11 +341,13 @@ class OpenAICompatibleProvider(Provider): debuglog.finish_entry(log, error=error) raise ProviderError(error) try: - text = self._extract_chunk(resp.json()) + payload = resp.json() except ValueError as exc: debuglog.finish_entry(log, error="Invalid JSON response") raise ProviderError("AI endpoint returned invalid JSON.") from exc - debuglog.finish_entry(log, response=text) + self._record_usage(payload) + text = self._extract_chunk(payload) + debuglog.finish_entry(log, response=text, usage=self.last_usage) return text.strip() async def embed(self, texts: list[str]) -> list[list[float]]: diff --git a/backend/app/routers/adventures.py b/backend/app/routers/adventures.py index b2ca1bf..467550f 100644 --- a/backend/app/routers/adventures.py +++ b/backend/app/routers/adventures.py @@ -809,6 +809,10 @@ async def _generate_turn( snapshot["world_state"] = {"delta": delta, "report": ws_report, "state": new_world_state} snapshot["raw_output"] = raw_output + # What the endpoint says the call cost, including how much of the prompt it + # read from cache rather than billed in full. Recorded per attempt, beside + # the prompt it priced. + snapshot["usage"] = provider.last_usage reasoning = "".join(reasoning_chunks).strip() or None ai_action = models.Action( diff --git a/backend/tests/test_attempt_siblings.py b/backend/tests/test_attempt_siblings.py index 35f1c66..063f1c9 100644 --- a/backend/tests/test_attempt_siblings.py +++ b/backend/tests/test_attempt_siblings.py @@ -41,6 +41,7 @@ modifier(text); class ScriptedProvider: + last_usage = None replies: list = [] calls = 0 prompts: list = [] diff --git a/backend/tests/test_branch_forking.py b/backend/tests/test_branch_forking.py index ea7a3c6..4b40cbb 100644 --- a/backend/tests/test_branch_forking.py +++ b/backend/tests/test_branch_forking.py @@ -51,6 +51,7 @@ modifier(text); class ScriptedProvider: + last_usage = None replies: list = [] calls = 0 prompts: list = [] diff --git a/backend/tests/test_branch_management.py b/backend/tests/test_branch_management.py index cd428fd..8381339 100644 --- a/backend/tests/test_branch_management.py +++ b/backend/tests/test_branch_management.py @@ -42,6 +42,7 @@ from app.routers import adventures class ScriptedProvider: + last_usage = None replies: list = [] calls = 0 diff --git a/backend/tests/test_bundle_v2.py b/backend/tests/test_bundle_v2.py index d151cdd..502bf84 100644 --- a/backend/tests/test_bundle_v2.py +++ b/backend/tests/test_bundle_v2.py @@ -58,6 +58,7 @@ OPENING = "You enter a cave." class ScriptedProvider: + last_usage = None replies: list = [] calls = 0 diff --git a/backend/tests/test_chat.py b/backend/tests/test_chat.py index 755160a..ac50437 100644 --- a/backend/tests/test_chat.py +++ b/backend/tests/test_chat.py @@ -28,6 +28,7 @@ class FakeProvider: """Records what it was constructed with, then streams a fixed reply. Stands in for the real egress point, so asserting on last_key/last_model is asserting on exactly what would have gone over the wire.""" + last_usage = None last_model = None last_key = None last_endpoint = None diff --git a/backend/tests/test_prompt_caching.py b/backend/tests/test_prompt_caching.py new file mode 100644 index 0000000..73b9e44 --- /dev/null +++ b/backend/tests/test_prompt_caching.py @@ -0,0 +1,241 @@ +"""Prompt caching: the prompt has to start with the same bytes every turn. + +Every endpoint that caches prompts caches a *prefix* — it reuses the request up +to the first byte that differs from last time and no further. So the cost of a +turn is decided by layout: one section that changes each turn, placed near the +top, re-prices everything underneath it, and underneath it is the story +history, which is most of the prompt. + +Three things have to hold, and each is easy to undo by accident: + +* the static block is byte-identical across turns — adding a section that moves + (live stats, retrieved memories, a rewritten summary) to `system_sections` is + the mistake this file exists to catch; +* the sections that move sit *after* the history, but still *before* the tail + that is last for its own reasons (front memory, the length hint, and + EMIT_REMINDER, which is what keeps the state block emitted at all); +* moving a section out of the system block does not drop it from the token + budget — it is still in the prompt. + +Plus the two request-level halves: preferring one OpenRouter upstream, since +each upstream holds its own cache, and reading back the usage the endpoint +reports so the hit rate is measurable rather than assumed. + + python -m pytest tests/test_prompt_caching.py -v +""" +import os +import tempfile + +_tmp = tempfile.NamedTemporaryFile(suffix=".db", delete=False) +_tmp.close() +os.environ["AIDND_DB_PATH"] = _tmp.name +os.environ.pop("AIDND_DATABASE_URL", None) +os.environ.pop("DATABASE_URL", None) + +import pytest + +from app import models, worldstate +from app.context import builder +from app.database import Base, SessionLocal, engine +from app.providers.openai_compatible import OpenAICompatibleProvider + +SCHEMA = { + "player": {"hp": {"min": 0, "max": 100, "initial": 100, "desc": "Health"}}, + "world": {"alarm": {"min": 0, "max": 10, "initial": 0, "desc": "Alarm level"}}, +} + + +# ------------------------------------------------- preferring one upstream + +def _routed(endpoint, model): + provider = OpenAICompatibleProvider(endpoint, "k", model, "chat", 0) + body = {"max_tokens": 100} + provider._apply_provider_routing(body) + return body + + +def test_openrouter_deepseek_pins_the_upstream(): + """Each upstream has its own cache, so routing has to be deterministic.""" + body = _routed("https://openrouter.ai/api/v1", "deepseek/deepseek-v4-flash-0731") + assert body["provider"] == {"order": ["deepseek"]} + + +def test_fallbacks_stay_on(): + """A preference, not a restriction: if deepseek is down the turn still runs + somewhere else and merely misses the cache.""" + body = _routed("https://openrouter.ai/api/v1", "deepseek/deepseek-v4-flash-0731") + assert "allow_fallbacks" not in body["provider"] + + +def test_non_openrouter_endpoints_get_no_provider_field(): + """Ollama and friends reject fields they do not know — the same trap the + `reasoning` param is written around.""" + body = _routed("http://localhost:11434/v1", "deepseek/deepseek-v4-flash-0731") + assert "provider" not in body + + +def test_unknown_vendors_are_left_alone(): + """The vendor half of a slug is not reliably a provider slug: Google's + models are served by "google-ai-studio", and there is no "google". A guess + would be a routing preference naming an upstream that does not exist.""" + body = _routed("https://openrouter.ai/api/v1", "google/gemma-4-26b-a4b-it:free") + assert "provider" not in body + + +# ------------------------------------------------------ reading usage back + +def test_usage_is_recorded_from_a_final_chunk(): + """In a stream the usage block rides on a last chunk carrying no choices, + which is why it is read separately from the text extraction.""" + provider = OpenAICompatibleProvider("https://openrouter.ai/api/v1", "k", "m") + assert provider.last_usage is None + provider._record_usage({"choices": [{"delta": {"content": "hi"}}]}) + assert provider.last_usage is None + provider._record_usage( + {"choices": [], "usage": {"prompt_tokens": 900, + "prompt_tokens_details": {"cached_tokens": 768}}} + ) + assert provider.last_usage["prompt_tokens_details"]["cached_tokens"] == 768 + + +def test_a_later_chunk_without_usage_does_not_erase_it(): + provider = OpenAICompatibleProvider("https://openrouter.ai/api/v1", "k", "m") + provider._record_usage({"usage": {"prompt_tokens": 5}}) + provider._record_usage({"choices": [{"delta": {"content": "x"}}]}) + provider._record_usage({"usage": {}}) + assert provider.last_usage == {"prompt_tokens": 5} + + +# ------------------------------------------------------------ prompt layout + +def _with_hp(world_state, hp): + """`world_state` is nested by group, and the JSON column only notices a + whole new object — so build one rather than mutating in place.""" + return {**world_state, "player": {**world_state["player"], "hp": hp}} + + +@pytest.fixture() +def story(): + Base.metadata.create_all(bind=engine) + db = SessionLocal() + user = models.User(is_guest=False, email="cache@example.com") + db.add(user) + db.flush() + settings = models.Settings(user_id=user.id, api_key="enc:dummy", model="m") + db.add(settings) + scenario = models.Scenario( + user_id=user.id, title="S", prompt="A road.", stat_schema=SCHEMA + ) + db.add(scenario) + db.flush() + adventure = models.Adventure( + user_id=user.id, title="A", scenario_id=scenario.id, script_state={}, + memory="The hero hunts bandits.", + ai_instructions="Write in second person.", + story_summary="The hero left the village.", + world_state=worldstate.instantiate(SCHEMA), + ) + db.add(adventure) + db.flush() + for i in range(6): + db.add(models.Action(adventure_id=adventure.id, index=i, + type="ai" if i % 2 else "do", + text=f"[{i}] The road bends onward past the treeline.")) + db.commit() + db.expire_all() + adventure = db.get(models.Adventure, adventure.id) + settings = db.get(models.Settings, settings.id) + try: + yield db, adventure, settings + finally: + db.close() + Base.metadata.drop_all(bind=engine) + + +def test_changing_a_stat_leaves_the_static_block_untouched(story): + """The whole point. Live values used to sit third from the top, so a single + point of damage re-priced the instructions, the plot and the history.""" + db, adventure, settings = story + before, _, _ = builder.build_context(adventure, settings) + adventure.world_state = _with_hp(adventure.world_state, 40) + db.commit() + after, story_text, _ = builder.build_context(adventure, settings) + assert before == after + assert "hp 40/100" in story_text, "the new value still has to reach the model" + + +def test_the_static_block_holds_the_things_that_do_not_move(story): + db, adventure, settings = story + system_text, story_text, _ = builder.build_context(adventure, settings) + for fixed in ("Write in second person.", "The hero hunts bandits."): + assert fixed in system_text + # The stat *guide* is derived from the schema and so is fixed; the live + # values it describes are not, and belong to the story text. + assert "Stat guide" in system_text + for moves in ("The hero left the village.", "hp 100/100"): + assert moves not in system_text + assert moves in story_text + + +def test_volatile_sections_sit_after_the_history(story): + db, adventure, settings = story + _, story_text, _ = builder.build_context(adventure, settings) + history_at = story_text.index("[5] The road bends") + for label in ("Story summary:", "World state"): + assert story_text.index(label) > history_at, label + + +def test_the_tail_stays_the_tail(story): + """front memory, the length hint and EMIT_REMINDER are last for reasons of + their own, and the live sections must not have displaced them.""" + db, adventure, settings = story + _, story_text, report = builder.build_context(adventure, settings) + labels = [s["label"] for s in report["sections"]] + assert labels[-1] == "world_state_reminder" + assert labels[-2] == "length_hint" + assert labels.index("world_state") < labels.index("length_hint") + assert story_text.rstrip().endswith(worldstate.EMIT_REMINDER.rstrip()) + + +def test_live_sections_are_still_charged_to_the_budget(story): + """They moved out of `system_sections`, so it would be easy to stop + counting them in `reserved` — and then the history, which is budgeted with + what is left over, would quietly overrun.""" + db, adventure, settings = story + for i in range(6, 90): + db.add(models.Action( + adventure_id=adventure.id, index=i, type="do", + text=f"[{i}] " + "The road bends onward past the treeline. " * 6, + )) + settings.context_token_budget = 4000 + db.commit() + db.expire_all() + adventure = db.get(models.Adventure, adventure.id) + settings = db.get(models.Settings, settings.id) + + _, _, lean = builder.build_context(adventure, settings) + adventure.story_summary = "The hero left the village. " * 150 + db.commit() + _, _, fat = builder.build_context(adventure, settings) + + assert fat["history"]["included"] < lean["history"]["included"], ( + "a bigger summary has to leave less room for history" + ) + + +def test_a_new_turn_only_appends_to_the_cached_prefix(story): + """Playing on must extend the previous prompt, not rewrite it: the shared + prefix has to still contain the whole static block and the older history.""" + db, adventure, settings = story + system_a, story_a, _ = builder.build_context(adventure, settings) + db.add(models.Action(adventure_id=adventure.id, index=6, type="do", + text="[6] You step into the clearing.")) + db.commit() + db.expire_all() + adventure = db.get(models.Adventure, adventure.id) + system_b, story_b, _ = builder.build_context(adventure, settings) + + assert system_a == system_b + shared = os.path.commonprefix([story_a, story_b]) + assert "[0] The road bends" in shared + assert "[5] The road bends" in shared diff --git a/backend/tests/test_retry_variants.py b/backend/tests/test_retry_variants.py index 8e65278..f7ee03b 100644 --- a/backend/tests/test_retry_variants.py +++ b/backend/tests/test_retry_variants.py @@ -37,6 +37,7 @@ modifier(text); class ScriptedProvider: """Streams the next canned reply each call, so successive retries differ.""" + last_usage = None replies: list = [] calls = 0 prompts: list = [] # every assembled (system, story) pair, for context assertions diff --git a/backend/tests/test_story_tree_baseline.py b/backend/tests/test_story_tree_baseline.py index 795270c..f1e65dc 100644 --- a/backend/tests/test_story_tree_baseline.py +++ b/backend/tests/test_story_tree_baseline.py @@ -53,6 +53,7 @@ OPENING = "You enter a cave." class ScriptedProvider: """Streams the next canned reply each call, so successive turns differ.""" + last_usage = None replies: list = [] calls = 0 diff --git a/backend/tests/test_take_edit.py b/backend/tests/test_take_edit.py index b25ccfa..545eea0 100644 --- a/backend/tests/test_take_edit.py +++ b/backend/tests/test_take_edit.py @@ -39,6 +39,7 @@ from app.routers import adventures class ScriptedProvider: + last_usage = None replies: list = [] calls = 0 diff --git a/backend/tests/test_take_parentage.py b/backend/tests/test_take_parentage.py index 764f022..e285acb 100644 --- a/backend/tests/test_take_parentage.py +++ b/backend/tests/test_take_parentage.py @@ -38,6 +38,7 @@ from app.routers import adventures class ScriptedProvider: + last_usage = None replies: list = [] calls = 0 diff --git a/backend/tests/test_take_state.py b/backend/tests/test_take_state.py index eee415c..d43f3c1 100644 --- a/backend/tests/test_take_state.py +++ b/backend/tests/test_take_state.py @@ -47,6 +47,7 @@ modifier(text); class ScriptedProvider: + last_usage = None replies: list = [] calls = 0 diff --git a/backend/tests/test_turn_flow_integration.py b/backend/tests/test_turn_flow_integration.py index c199db2..5cf83f9 100644 --- a/backend/tests/test_turn_flow_integration.py +++ b/backend/tests/test_turn_flow_integration.py @@ -36,6 +36,7 @@ modifier(text); class FakeProvider: """Stand-in for OpenAICompatibleProvider: streams one fixed line, no network.""" + last_usage = None def __init__(self, *a, **k): pass diff --git a/backend/tests/test_worldstate_integration.py b/backend/tests/test_worldstate_integration.py index 92a6225..9c47211 100644 --- a/backend/tests/test_worldstate_integration.py +++ b/backend/tests/test_worldstate_integration.py @@ -45,6 +45,7 @@ AI_REPLY = ( class FakeProvider: + last_usage = None def __init__(self, *a, **k): pass diff --git a/frontend/src/pages/Play.jsx b/frontend/src/pages/Play.jsx index 0a680d9..09c094f 100644 --- a/frontend/src/pages/Play.jsx +++ b/frontend/src/pages/Play.jsx @@ -1298,6 +1298,26 @@ function ScriptReport({ script }) { ) } +// What the endpoint charged for the turn, and how much of the prompt it read +// back out of its cache instead of billing in full. Only shown on a past turn: +// the "next turn" view has not been sent anywhere yet, so it has no usage. A +// cached read costs a tenth of a fresh one, which is the whole reason the +// prompt is laid out static-first — so this is the number that says whether +// that layout is working. +function CacheReport({ usage }) { + if (!usage) return null + const prompt = usage.prompt_tokens || 0 + const cached = usage.prompt_tokens_details?.cached_tokens || 0 + if (!prompt) return null + const pct = Math.round((cached / prompt) * 100) + return ( +
+ Prompt cache: {cached} of {prompt} prompt tokens read from cache ({pct}%) + {usage.cost != null && ` · cost $${Number(usage.cost).toFixed(5)}`} +
+ ) +} + const LEGEND_VISIBLE = 6 // enough to cover what actually moves the budget // Where the prompt's tokens actually went: one stacked bar scaled to the @@ -1435,6 +1455,7 @@ function InsightsPanel({ advId, inspectActionId, onClearInspect, refreshKey }) { {history.total > history.included && ' (older history trimmed)'} {history.oldest_truncated && ' — oldest entry cut mid-text'} + {cards.length > 0 && (
{cards.map((c, i) => (