diff --git a/backend/app/attempts.py b/backend/app/attempts.py index a4b13ef..37e50a1 100644 --- a/backend/app/attempts.py +++ b/backend/app/attempts.py @@ -40,9 +40,11 @@ from .context import lineage # The slices of a context snapshot that belong to one attempt rather than to # the turn: the world-state delta it proposed and what the referee did with it, -# the script report, and the model's literal reply. Everything else in a +# the script report, the model's literal reply, and the endpoint's token +# accounting — each attempt is its own API call, and a retry is precisely the +# call expected to read the prompt back out of cache. Everything else in a # snapshot is the prompt, which is assembled once per turn. -ATTEMPT_KEYS = ("world_state", "script", "raw_output") +ATTEMPT_KEYS = ("world_state", "script", "raw_output", "usage") # ------------------------------------------------------------------ reading diff --git a/backend/app/context/builder.py b/backend/app/context/builder.py index 75fb1eb..6e71850 100644 --- a/backend/app/context/builder.py +++ b/backend/app/context/builder.py @@ -9,6 +9,12 @@ [Story history] newest actions that fit the remaining token budget [Author's Note] injected AUTHORS_NOTE_DEPTH actions before the end of history [Latest player action] (+ script frontMemory right after it, Phase 4) + +Which components are present is AI Dungeon's design, above. The *order* they +are laid out in is not: everything fixed is emitted first and everything that +moves after the history, because prompt caching bills on a shared prefix and +one mutable section high up re-prices the whole prompt under it. See the two +"static block" / "live sections" comments in `build_context`. """ import functools @@ -190,24 +196,26 @@ def build_context( `exclude_action_id` omits one action from the story (see history.py).""" script_mem = _script_memory(adventure) - # ----- Always-included components ----- + # ----- The static block: byte-for-byte the same prompt every turn ----- + # The ordering here is a billing decision, not a stylistic one. Prompt + # caching matches a *prefix*: an endpoint reuses the prompt up to the first + # byte that differs from last time and no further. So one mutable section + # near the top re-prices everything below it, and what is below it is the + # story history, which is the bulk of the prompt. Anything that changes + # turn to turn therefore goes *after* the history, in the live sections — + # which is also where recency serves it best, the same reasoning that + # already puts EMIT_REMINDER last. system_sections: list[Section] = [Section("narrator", settings.narrator_prompt.strip())] - # RPG world state (Phase 12): current stats/milestones + how to report changes. + # RPG world state (Phase 12): how to report changes. The live values are a + # live section below; the schema-derived guide and the emit rule are fixed + # for as long as the scenario is. stat_schema = adventure.scenario.stat_schema if adventure.scenario else None has_ws = worldstate.has_schema(stat_schema) if has_ws: guide = worldstate.render_reference(stat_schema) if guide: system_sections.append(Section("world_state_guide", guide)) - block = worldstate.render_state_section( - adventure.world_state, stat_schema, - _visible_npcs( - history.tail(adventure, NPC_WINDOW, exclude_action_id), stat_schema - ), - ) - if block: - system_sections.append(Section("world_state", block)) system_sections.append(Section("world_state_rule", worldstate.EMIT_RULE)) if isinstance(script_mem.get("context"), str) and script_mem["context"].strip(): @@ -218,13 +226,33 @@ def build_context( system_sections.append( Section("plot_essentials", f"Plot essentials:\n{adventure.memory.strip()}") ) - if adventure.story_summary.strip(): - system_sections.append( - Section("story_summary", f"Story summary:\n{adventure.story_summary.strip()}") - ) + + # ----- Live sections: everything that moves, built here, placed after the + # history further down. Ordered least-volatile first, so a turn that + # changes only the fastest-moving one keeps the others cached too: the + # summary is rewritten every few turns, lore turns over with the scene, the + # retrieved memories change on most turns, the stat values on nearly all. + # `world_lore` joins them below — it is the history window that triggers + # the cards, so it cannot be known yet. + summary_section = ( + Section("story_summary", f"Story summary:\n{adventure.story_summary.strip()}") + if adventure.story_summary.strip() + else None + ) + memories_section = None if memory_bank and memory_bank.get("used"): - lines = "\n".join(f"- {m['text']}" for m in memory_bank["used"]) - system_sections.append(Section("used_memories", f"Memories:\n{lines}")) + lines_text = "\n".join(f"- {m['text']}" for m in memory_bank["used"]) + memories_section = Section("used_memories", f"Memories:\n{lines_text}") + world_state_section = None + if has_ws: + block = worldstate.render_state_section( + adventure.world_state, stat_schema, + _visible_npcs( + history.tail(adventure, NPC_WINDOW, exclude_action_id), stat_schema + ), + ) + if block: + world_state_section = Section("world_state", block) authors_note_text = adventure.authors_note.strip() if isinstance(script_mem.get("authorsNote"), str) and script_mem["authorsNote"].strip(): @@ -237,8 +265,16 @@ def build_context( length_note = length_hint(settings.max_output_tokens, has_ws=has_ws) + # The live sections moved below the history but they are still in the + # prompt, so they are still reserved against the budget. (`world_lore` is + # not: it is budgeted out of `available` further down, as it always was.) reserved = ( sum(s.tokens for s in system_sections) + + sum( + s.tokens + for s in (summary_section, memories_section, world_state_section) + if s is not None + ) + count_tokens(authors_note) + count_tokens(front_memory) + count_tokens(length_note) @@ -273,8 +309,9 @@ def build_context( {"id": match["id"], "name": match["name"], "keyword": match["keyword"], "included": included} ) - if lore_lines: - system_sections.append(Section("world_lore", "\n".join(lore_lines))) + lore_section = ( + Section("world_lore", "\n".join(lore_lines)) if lore_lines else None + ) # ----- Story history: newest first until the remaining budget is spent ----- history_budget = available - used @@ -316,6 +353,12 @@ def build_context( note_sections.append(Section("recent_history", SEPARATOR.join(after))) else: note_sections.append(Section("history", SEPARATOR.join(texts))) + # The live sections, least volatile first (see where they are built). They + # sit below the history so the history caches, and above the tail so the + # three sections that are last for a reason stay last. + for live in (summary_section, lore_section, memories_section, world_state_section): + if live is not None: + note_sections.append(live) if front_memory: note_sections.append(Section("front_memory", front_memory)) # Sits just above the emit reminder, which keeps the strongest recency slot: diff --git a/backend/app/debuglog.py b/backend/app/debuglog.py index dcd1cd7..146eeb7 100644 --- a/backend/app/debuglog.py +++ b/backend/app/debuglog.py @@ -40,14 +40,26 @@ def start_entry(url: str, model: str, body: dict) -> dict: "request": _clip_obj(body), "status": "pending", "response": "", + "usage": None, "error": None, } _entries.appendleft(entry) return entry -def finish_entry(entry: dict, *, response: str = "", error: str | None = None) -> None: +def finish_entry( + entry: dict, + *, + response: str = "", + error: str | None = None, + usage: dict | None = None, +) -> None: + """`usage` is the endpoint's own token accounting when it reported any. + On OpenRouter it carries `prompt_tokens_details.cached_tokens`, which is + the only direct read on whether the prompt prefix is actually being + cached — a number worth seeing beside the request that produced it.""" entry["response"] = _clip(response) + entry["usage"] = usage entry["error"] = error entry["status"] = "error" if error else "ok" diff --git a/backend/app/providers/base.py b/backend/app/providers/base.py index 7a9f2f9..0c905af 100644 --- a/backend/app/providers/base.py +++ b/backend/app/providers/base.py @@ -16,6 +16,13 @@ class ProviderError(Exception): class Provider(ABC): + # The endpoint's own token accounting for the most recent call, when it + # reported any — notably `prompt_tokens_details.cached_tokens`, which is + # the only direct read on whether the prompt prefix is being cached. + # Callers read it after the call they made; one provider is built per + # request, so there is nothing to race. + last_usage: dict | None = None + @abstractmethod def generate( self, diff --git a/backend/app/providers/openai_compatible.py b/backend/app/providers/openai_compatible.py index 2fd746e..67fbc8e 100644 --- a/backend/app/providers/openai_compatible.py +++ b/backend/app/providers/openai_compatible.py @@ -10,6 +10,26 @@ from .base import PromptParts, Provider, ProviderError # continuing prose instead of replying conversationally. CHAT_CONTINUE_HINT = "\n\n[Continue the story directly. Output only story text.]" +# OpenRouter serves one model from whichever upstream is available, and every +# upstream holds its own prompt cache — so a request that lands somewhere new +# starts cold however stable the prompt is. Naming a preferred upstream makes +# routing deterministic, which is what lets a cache be hit at all. +# +# `allow_fallbacks` is deliberately left at its default of true: this is a +# preference, not a restriction. If the named upstream is down the request +# still goes through somewhere else and merely misses the cache, which is the +# behaviour we had anyway. +# +# A whitelist rather than a derivation from the model slug. The vendor half of +# a slug is *usually* the provider slug ("deepseek/..." -> "deepseek", verified +# against /api/v1/providers) but not reliably: Google's models are served by +# "google-ai-studio" and "google-vertex", and there is no "google". Look a +# vendor up on the model's Providers tab before adding it here — a slug that +# does not exist is a routing preference that silently does nothing at best. +_OPENROUTER_HOST = "openrouter.ai" +_PREFERRED_UPSTREAM = {"deepseek": "deepseek"} + + # Completion endpoints have no roles, so a plain chat has to be flattened into # one labelled transcript that trails off on "Assistant:" for the model to continue. _ROLE_LABELS = {"system": "System", "user": "User", "assistant": "Assistant"} @@ -42,6 +62,12 @@ class OpenAICompatibleProvider(Provider): # reject unknown fields); negative = explicitly ask the endpoint to # turn reasoning off. self.reasoning_max_tokens = reasoning_max_tokens + # Token accounting from the last call, when the endpoint reported any: + # prompt/completion counts plus, on OpenRouter, `prompt_tokens_details. + # cached_tokens` — the number of prompt tokens read from cache instead + # of billed in full. Written by every request method, so a caller reads + # it after the call it made; one provider is built per request. + self.last_usage: dict | None = None def _headers(self) -> dict: headers = {"Content-Type": "application/json"} @@ -66,6 +92,29 @@ class OpenAICompatibleProvider(Provider): body["reasoning"] = {"max_tokens": self.reasoning_max_tokens} body["max_tokens"] += self.reasoning_max_tokens + def _apply_provider_routing(self, body: dict) -> None: + """Prefer one upstream on OpenRouter, so the prompt cache is warm. + + Silent no-op everywhere else: `provider` is an OpenRouter extension and + Ollama and friends reject fields they do not know — the same trap the + `reasoning` param above is written around.""" + if _OPENROUTER_HOST not in self.base_url: + return + upstream = _PREFERRED_UPSTREAM.get(self.model.split("/", 1)[0].lower()) + if upstream: + body["provider"] = {"order": [upstream]} + + def _record_usage(self, payload: dict) -> None: + """Record the endpoint's own token accounting, if it reported any. + + OpenRouter always reports usage now (`usage: {include: true}` and + `stream_options` are deprecated no-ops), and in a stream it rides on a + final chunk that carries no choices — which is why this is read + separately from the text extraction rather than beside it.""" + usage = payload.get("usage") + if isinstance(usage, dict) and usage: + self.last_usage = usage + def _request(self, parts: PromptParts, temperature: float, max_tokens: int) -> tuple[str, dict]: if self.api_mode == "completion": url = f"{self.base_url}/completions" @@ -89,6 +138,7 @@ class OpenAICompatibleProvider(Provider): "stream": True, } self._apply_reasoning_budget(body) + self._apply_provider_routing(body) return url, body @staticmethod @@ -162,6 +212,7 @@ class OpenAICompatibleProvider(Provider): "stream": True, } self._apply_reasoning_budget(body) + self._apply_provider_routing(body) async for event in self._stream(url, body): yield event @@ -193,12 +244,15 @@ class OpenAICompatibleProvider(Provider): saw_sse = True data = line[5:].strip() if data == "[DONE]": - debuglog.finish_entry(log, response="".join(received)) + debuglog.finish_entry( + log, response="".join(received), usage=self.last_usage + ) return try: payload = json.loads(data) except ValueError: continue + self._record_usage(payload) reasoning = self._extract_reasoning(payload) if reasoning: yield "reasoning", reasoning @@ -215,6 +269,7 @@ class OpenAICompatibleProvider(Provider): "AI endpoint returned neither an SSE stream nor JSON: " + body_text[:200] ) + self._record_usage(payload) reasoning = self._extract_reasoning(payload) if reasoning: yield "reasoning", reasoning @@ -227,7 +282,7 @@ class OpenAICompatibleProvider(Provider): "AI endpoint returned a response with no text: " + body_text[:200] ) - debuglog.finish_entry(log, response="".join(received)) + debuglog.finish_entry(log, response="".join(received), usage=self.last_usage) except httpx.ConnectError as exc: error = f"Could not connect to {self.base_url} — is the AI server running?" debuglog.finish_entry(log, response="".join(received), error=error) @@ -272,6 +327,7 @@ class OpenAICompatibleProvider(Provider): "stream": False, } self._apply_reasoning_budget(body) + self._apply_provider_routing(body) log = debuglog.start_entry(url, self.model, body) try: @@ -285,11 +341,13 @@ class OpenAICompatibleProvider(Provider): debuglog.finish_entry(log, error=error) raise ProviderError(error) try: - text = self._extract_chunk(resp.json()) + payload = resp.json() except ValueError as exc: debuglog.finish_entry(log, error="Invalid JSON response") raise ProviderError("AI endpoint returned invalid JSON.") from exc - debuglog.finish_entry(log, response=text) + self._record_usage(payload) + text = self._extract_chunk(payload) + debuglog.finish_entry(log, response=text, usage=self.last_usage) return text.strip() async def embed(self, texts: list[str]) -> list[list[float]]: diff --git a/backend/app/routers/adventures.py b/backend/app/routers/adventures.py index b2ca1bf..467550f 100644 --- a/backend/app/routers/adventures.py +++ b/backend/app/routers/adventures.py @@ -809,6 +809,10 @@ async def _generate_turn( snapshot["world_state"] = {"delta": delta, "report": ws_report, "state": new_world_state} snapshot["raw_output"] = raw_output + # What the endpoint says the call cost, including how much of the prompt it + # read from cache rather than billed in full. Recorded per attempt, beside + # the prompt it priced. + snapshot["usage"] = provider.last_usage reasoning = "".join(reasoning_chunks).strip() or None ai_action = models.Action( diff --git a/backend/tests/test_attempt_siblings.py b/backend/tests/test_attempt_siblings.py index 35f1c66..063f1c9 100644 --- a/backend/tests/test_attempt_siblings.py +++ b/backend/tests/test_attempt_siblings.py @@ -41,6 +41,7 @@ modifier(text); class ScriptedProvider: + last_usage = None replies: list = [] calls = 0 prompts: list = [] diff --git a/backend/tests/test_branch_forking.py b/backend/tests/test_branch_forking.py index ea7a3c6..4b40cbb 100644 --- a/backend/tests/test_branch_forking.py +++ b/backend/tests/test_branch_forking.py @@ -51,6 +51,7 @@ modifier(text); class ScriptedProvider: + last_usage = None replies: list = [] calls = 0 prompts: list = [] diff --git a/backend/tests/test_branch_management.py b/backend/tests/test_branch_management.py index cd428fd..8381339 100644 --- a/backend/tests/test_branch_management.py +++ b/backend/tests/test_branch_management.py @@ -42,6 +42,7 @@ from app.routers import adventures class ScriptedProvider: + last_usage = None replies: list = [] calls = 0 diff --git a/backend/tests/test_bundle_v2.py b/backend/tests/test_bundle_v2.py index d151cdd..502bf84 100644 --- a/backend/tests/test_bundle_v2.py +++ b/backend/tests/test_bundle_v2.py @@ -58,6 +58,7 @@ OPENING = "You enter a cave." class ScriptedProvider: + last_usage = None replies: list = [] calls = 0 diff --git a/backend/tests/test_chat.py b/backend/tests/test_chat.py index 755160a..ac50437 100644 --- a/backend/tests/test_chat.py +++ b/backend/tests/test_chat.py @@ -28,6 +28,7 @@ class FakeProvider: """Records what it was constructed with, then streams a fixed reply. Stands in for the real egress point, so asserting on last_key/last_model is asserting on exactly what would have gone over the wire.""" + last_usage = None last_model = None last_key = None last_endpoint = None diff --git a/backend/tests/test_prompt_caching.py b/backend/tests/test_prompt_caching.py new file mode 100644 index 0000000..73b9e44 --- /dev/null +++ b/backend/tests/test_prompt_caching.py @@ -0,0 +1,241 @@ +"""Prompt caching: the prompt has to start with the same bytes every turn. + +Every endpoint that caches prompts caches a *prefix* — it reuses the request up +to the first byte that differs from last time and no further. So the cost of a +turn is decided by layout: one section that changes each turn, placed near the +top, re-prices everything underneath it, and underneath it is the story +history, which is most of the prompt. + +Three things have to hold, and each is easy to undo by accident: + +* the static block is byte-identical across turns — adding a section that moves + (live stats, retrieved memories, a rewritten summary) to `system_sections` is + the mistake this file exists to catch; +* the sections that move sit *after* the history, but still *before* the tail + that is last for its own reasons (front memory, the length hint, and + EMIT_REMINDER, which is what keeps the state block emitted at all); +* moving a section out of the system block does not drop it from the token + budget — it is still in the prompt. + +Plus the two request-level halves: preferring one OpenRouter upstream, since +each upstream holds its own cache, and reading back the usage the endpoint +reports so the hit rate is measurable rather than assumed. + + python -m pytest tests/test_prompt_caching.py -v +""" +import os +import tempfile + +_tmp = tempfile.NamedTemporaryFile(suffix=".db", delete=False) +_tmp.close() +os.environ["AIDND_DB_PATH"] = _tmp.name +os.environ.pop("AIDND_DATABASE_URL", None) +os.environ.pop("DATABASE_URL", None) + +import pytest + +from app import models, worldstate +from app.context import builder +from app.database import Base, SessionLocal, engine +from app.providers.openai_compatible import OpenAICompatibleProvider + +SCHEMA = { + "player": {"hp": {"min": 0, "max": 100, "initial": 100, "desc": "Health"}}, + "world": {"alarm": {"min": 0, "max": 10, "initial": 0, "desc": "Alarm level"}}, +} + + +# ------------------------------------------------- preferring one upstream + +def _routed(endpoint, model): + provider = OpenAICompatibleProvider(endpoint, "k", model, "chat", 0) + body = {"max_tokens": 100} + provider._apply_provider_routing(body) + return body + + +def test_openrouter_deepseek_pins_the_upstream(): + """Each upstream has its own cache, so routing has to be deterministic.""" + body = _routed("https://openrouter.ai/api/v1", "deepseek/deepseek-v4-flash-0731") + assert body["provider"] == {"order": ["deepseek"]} + + +def test_fallbacks_stay_on(): + """A preference, not a restriction: if deepseek is down the turn still runs + somewhere else and merely misses the cache.""" + body = _routed("https://openrouter.ai/api/v1", "deepseek/deepseek-v4-flash-0731") + assert "allow_fallbacks" not in body["provider"] + + +def test_non_openrouter_endpoints_get_no_provider_field(): + """Ollama and friends reject fields they do not know — the same trap the + `reasoning` param is written around.""" + body = _routed("http://localhost:11434/v1", "deepseek/deepseek-v4-flash-0731") + assert "provider" not in body + + +def test_unknown_vendors_are_left_alone(): + """The vendor half of a slug is not reliably a provider slug: Google's + models are served by "google-ai-studio", and there is no "google". A guess + would be a routing preference naming an upstream that does not exist.""" + body = _routed("https://openrouter.ai/api/v1", "google/gemma-4-26b-a4b-it:free") + assert "provider" not in body + + +# ------------------------------------------------------ reading usage back + +def test_usage_is_recorded_from_a_final_chunk(): + """In a stream the usage block rides on a last chunk carrying no choices, + which is why it is read separately from the text extraction.""" + provider = OpenAICompatibleProvider("https://openrouter.ai/api/v1", "k", "m") + assert provider.last_usage is None + provider._record_usage({"choices": [{"delta": {"content": "hi"}}]}) + assert provider.last_usage is None + provider._record_usage( + {"choices": [], "usage": {"prompt_tokens": 900, + "prompt_tokens_details": {"cached_tokens": 768}}} + ) + assert provider.last_usage["prompt_tokens_details"]["cached_tokens"] == 768 + + +def test_a_later_chunk_without_usage_does_not_erase_it(): + provider = OpenAICompatibleProvider("https://openrouter.ai/api/v1", "k", "m") + provider._record_usage({"usage": {"prompt_tokens": 5}}) + provider._record_usage({"choices": [{"delta": {"content": "x"}}]}) + provider._record_usage({"usage": {}}) + assert provider.last_usage == {"prompt_tokens": 5} + + +# ------------------------------------------------------------ prompt layout + +def _with_hp(world_state, hp): + """`world_state` is nested by group, and the JSON column only notices a + whole new object — so build one rather than mutating in place.""" + return {**world_state, "player": {**world_state["player"], "hp": hp}} + + +@pytest.fixture() +def story(): + Base.metadata.create_all(bind=engine) + db = SessionLocal() + user = models.User(is_guest=False, email="cache@example.com") + db.add(user) + db.flush() + settings = models.Settings(user_id=user.id, api_key="enc:dummy", model="m") + db.add(settings) + scenario = models.Scenario( + user_id=user.id, title="S", prompt="A road.", stat_schema=SCHEMA + ) + db.add(scenario) + db.flush() + adventure = models.Adventure( + user_id=user.id, title="A", scenario_id=scenario.id, script_state={}, + memory="The hero hunts bandits.", + ai_instructions="Write in second person.", + story_summary="The hero left the village.", + world_state=worldstate.instantiate(SCHEMA), + ) + db.add(adventure) + db.flush() + for i in range(6): + db.add(models.Action(adventure_id=adventure.id, index=i, + type="ai" if i % 2 else "do", + text=f"[{i}] The road bends onward past the treeline.")) + db.commit() + db.expire_all() + adventure = db.get(models.Adventure, adventure.id) + settings = db.get(models.Settings, settings.id) + try: + yield db, adventure, settings + finally: + db.close() + Base.metadata.drop_all(bind=engine) + + +def test_changing_a_stat_leaves_the_static_block_untouched(story): + """The whole point. Live values used to sit third from the top, so a single + point of damage re-priced the instructions, the plot and the history.""" + db, adventure, settings = story + before, _, _ = builder.build_context(adventure, settings) + adventure.world_state = _with_hp(adventure.world_state, 40) + db.commit() + after, story_text, _ = builder.build_context(adventure, settings) + assert before == after + assert "hp 40/100" in story_text, "the new value still has to reach the model" + + +def test_the_static_block_holds_the_things_that_do_not_move(story): + db, adventure, settings = story + system_text, story_text, _ = builder.build_context(adventure, settings) + for fixed in ("Write in second person.", "The hero hunts bandits."): + assert fixed in system_text + # The stat *guide* is derived from the schema and so is fixed; the live + # values it describes are not, and belong to the story text. + assert "Stat guide" in system_text + for moves in ("The hero left the village.", "hp 100/100"): + assert moves not in system_text + assert moves in story_text + + +def test_volatile_sections_sit_after_the_history(story): + db, adventure, settings = story + _, story_text, _ = builder.build_context(adventure, settings) + history_at = story_text.index("[5] The road bends") + for label in ("Story summary:", "World state"): + assert story_text.index(label) > history_at, label + + +def test_the_tail_stays_the_tail(story): + """front memory, the length hint and EMIT_REMINDER are last for reasons of + their own, and the live sections must not have displaced them.""" + db, adventure, settings = story + _, story_text, report = builder.build_context(adventure, settings) + labels = [s["label"] for s in report["sections"]] + assert labels[-1] == "world_state_reminder" + assert labels[-2] == "length_hint" + assert labels.index("world_state") < labels.index("length_hint") + assert story_text.rstrip().endswith(worldstate.EMIT_REMINDER.rstrip()) + + +def test_live_sections_are_still_charged_to_the_budget(story): + """They moved out of `system_sections`, so it would be easy to stop + counting them in `reserved` — and then the history, which is budgeted with + what is left over, would quietly overrun.""" + db, adventure, settings = story + for i in range(6, 90): + db.add(models.Action( + adventure_id=adventure.id, index=i, type="do", + text=f"[{i}] " + "The road bends onward past the treeline. " * 6, + )) + settings.context_token_budget = 4000 + db.commit() + db.expire_all() + adventure = db.get(models.Adventure, adventure.id) + settings = db.get(models.Settings, settings.id) + + _, _, lean = builder.build_context(adventure, settings) + adventure.story_summary = "The hero left the village. " * 150 + db.commit() + _, _, fat = builder.build_context(adventure, settings) + + assert fat["history"]["included"] < lean["history"]["included"], ( + "a bigger summary has to leave less room for history" + ) + + +def test_a_new_turn_only_appends_to_the_cached_prefix(story): + """Playing on must extend the previous prompt, not rewrite it: the shared + prefix has to still contain the whole static block and the older history.""" + db, adventure, settings = story + system_a, story_a, _ = builder.build_context(adventure, settings) + db.add(models.Action(adventure_id=adventure.id, index=6, type="do", + text="[6] You step into the clearing.")) + db.commit() + db.expire_all() + adventure = db.get(models.Adventure, adventure.id) + system_b, story_b, _ = builder.build_context(adventure, settings) + + assert system_a == system_b + shared = os.path.commonprefix([story_a, story_b]) + assert "[0] The road bends" in shared + assert "[5] The road bends" in shared diff --git a/backend/tests/test_retry_variants.py b/backend/tests/test_retry_variants.py index 8e65278..f7ee03b 100644 --- a/backend/tests/test_retry_variants.py +++ b/backend/tests/test_retry_variants.py @@ -37,6 +37,7 @@ modifier(text); class ScriptedProvider: """Streams the next canned reply each call, so successive retries differ.""" + last_usage = None replies: list = [] calls = 0 prompts: list = [] # every assembled (system, story) pair, for context assertions diff --git a/backend/tests/test_story_tree_baseline.py b/backend/tests/test_story_tree_baseline.py index 795270c..f1e65dc 100644 --- a/backend/tests/test_story_tree_baseline.py +++ b/backend/tests/test_story_tree_baseline.py @@ -53,6 +53,7 @@ OPENING = "You enter a cave." class ScriptedProvider: """Streams the next canned reply each call, so successive turns differ.""" + last_usage = None replies: list = [] calls = 0 diff --git a/backend/tests/test_take_edit.py b/backend/tests/test_take_edit.py index b25ccfa..545eea0 100644 --- a/backend/tests/test_take_edit.py +++ b/backend/tests/test_take_edit.py @@ -39,6 +39,7 @@ from app.routers import adventures class ScriptedProvider: + last_usage = None replies: list = [] calls = 0 diff --git a/backend/tests/test_take_parentage.py b/backend/tests/test_take_parentage.py index 764f022..e285acb 100644 --- a/backend/tests/test_take_parentage.py +++ b/backend/tests/test_take_parentage.py @@ -38,6 +38,7 @@ from app.routers import adventures class ScriptedProvider: + last_usage = None replies: list = [] calls = 0 diff --git a/backend/tests/test_take_state.py b/backend/tests/test_take_state.py index eee415c..d43f3c1 100644 --- a/backend/tests/test_take_state.py +++ b/backend/tests/test_take_state.py @@ -47,6 +47,7 @@ modifier(text); class ScriptedProvider: + last_usage = None replies: list = [] calls = 0 diff --git a/backend/tests/test_turn_flow_integration.py b/backend/tests/test_turn_flow_integration.py index c199db2..5cf83f9 100644 --- a/backend/tests/test_turn_flow_integration.py +++ b/backend/tests/test_turn_flow_integration.py @@ -36,6 +36,7 @@ modifier(text); class FakeProvider: """Stand-in for OpenAICompatibleProvider: streams one fixed line, no network.""" + last_usage = None def __init__(self, *a, **k): pass diff --git a/backend/tests/test_worldstate_integration.py b/backend/tests/test_worldstate_integration.py index 92a6225..9c47211 100644 --- a/backend/tests/test_worldstate_integration.py +++ b/backend/tests/test_worldstate_integration.py @@ -45,6 +45,7 @@ AI_REPLY = ( class FakeProvider: + last_usage = None def __init__(self, *a, **k): pass diff --git a/frontend/src/pages/Play.jsx b/frontend/src/pages/Play.jsx index 0a680d9..09c094f 100644 --- a/frontend/src/pages/Play.jsx +++ b/frontend/src/pages/Play.jsx @@ -1298,6 +1298,26 @@ function ScriptReport({ script }) { ) } +// What the endpoint charged for the turn, and how much of the prompt it read +// back out of its cache instead of billing in full. Only shown on a past turn: +// the "next turn" view has not been sent anywhere yet, so it has no usage. A +// cached read costs a tenth of a fresh one, which is the whole reason the +// prompt is laid out static-first — so this is the number that says whether +// that layout is working. +function CacheReport({ usage }) { + if (!usage) return null + const prompt = usage.prompt_tokens || 0 + const cached = usage.prompt_tokens_details?.cached_tokens || 0 + if (!prompt) return null + const pct = Math.round((cached / prompt) * 100) + return ( +
+ Prompt cache: {cached} of {prompt} prompt tokens read from cache ({pct}%) + {usage.cost != null && ` · cost $${Number(usage.cost).toFixed(5)}`} +
+ ) +} + const LEGEND_VISIBLE = 6 // enough to cover what actually moves the budget // Where the prompt's tokens actually went: one stacked bar scaled to the @@ -1435,6 +1455,7 @@ function InsightsPanel({ advId, inspectActionId, onClearInspect, refreshKey }) { {history.total > history.included && ' (older history trimmed)'} {history.oldest_truncated && ' — oldest entry cut mid-text'} + {cards.length > 0 && (
{cards.map((c, i) => (