diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..3b11fe3 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,19 @@ +.git +.claude +plan +*.md +LICENSE +start.ps1 +start.sh + +# Secrets & local data — never in the image +*.env +backend/openrouter_key.env +*.db + +# Build/runtime artifacts rebuilt inside the image +backend/.venv +**/__pycache__ +**/*.pyc +frontend/node_modules +frontend/dist diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..50300e0 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +# Shell scripts must stay LF or the shebang/exec breaks on Linux/macOS, +# including when a Windows checkout is COPY'd into a Docker build. +*.sh text eol=lf +Dockerfile text eol=lf +*.ps1 text eol=crlf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..5cb74dd --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,74 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + backend: + name: Backend tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" # matches the Dockerfile + cache: pip + cache-dependency-path: | + backend/requirements.txt + backend/requirements-dev.txt + + - name: Install dependencies + working-directory: backend + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt -r requirements-dev.txt + + - name: Run tests + working-directory: backend + run: python -m pytest tests/ -q + + frontend: + name: Frontend lint + build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "24" # matches the Dockerfile + cache: npm + cache-dependency-path: frontend/package-lock.json + + - name: Install dependencies + working-directory: frontend + run: npm ci + + - name: Lint + working-directory: frontend + run: npm run lint + + - name: Build + working-directory: frontend + run: npm run build + + docker: + name: Docker image builds + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + - name: Build image + uses: docker/build-push-action@v6 + with: + context: . + push: false + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8712e3d --- /dev/null +++ b/.gitignore @@ -0,0 +1,21 @@ +# Secrets +*.env +backend/openrouter_key.env + +# Python +backend/.venv/ +__pycache__/ +*.pyc + +# Database +*.db + +# Node +node_modules/ +frontend/dist/ + +# Misc +.claude/ + +# Phase 8: auto-generated session/encryption secret (lives next to the DB) +secret.key diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0446512 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,46 @@ +# Stage 1 — build the React SPA. Node 24: package-lock.json is written by +# npm 11, which older bundled npms (node 22's 10.x) refuse as out-of-sync. +FROM node:24-alpine AS frontend-build +WORKDIR /build +COPY frontend/package.json frontend/package-lock.json ./ +RUN npm ci +COPY frontend/ ./ +RUN npm run build + +# Stage 2 — build Python wheels (quickjs compiles from source if no wheel +# matches, so keep the toolchain out of the final image) +FROM python:3.12-slim AS python-build +RUN apt-get update && apt-get install -y --no-install-recommends gcc make \ + && rm -rf /var/lib/apt/lists/* +COPY backend/requirements.txt /tmp/requirements.txt +RUN pip wheel --no-cache-dir -r /tmp/requirements.txt -w /wheels + +# Stage 3 — runtime +FROM python:3.12-slim +WORKDIR /app + +COPY --from=python-build /wheels /wheels +RUN pip install --no-cache-dir /wheels/* && rm -rf /wheels + +# Layout mirrors the repo: main.py finds the SPA at ../../frontend/dist +# relative to backend/app/main.py. +COPY backend/app /app/backend/app +COPY --from=frontend-build /build/dist /app/frontend/dist + +# Database lives on a volume; parent dir is created by the app if missing. +ENV AIDND_DB_PATH=/data/data.db +VOLUME /data + +EXPOSE 8000 +WORKDIR /app/backend +# --proxy-headers lets uvicorn fix up the request scheme (https) behind the +# platform's edge. We deliberately do NOT pass --forwarded-allow-ips "*": that +# made uvicorn trust the LEFTMOST X-Forwarded-For value, which the client fully +# controls, so anyone could rotate the header to dodge the per-IP rate limits. +# The client IP used for rate limiting is derived in limits._client_ip from the +# hop the edge appends (rightmost), which a client cannot spoof past; tune with +# AIDND_TRUSTED_PROXY_HOPS if the platform adds more proxy hops. +# Single worker on purpose: the turn lock, rate limiter, and debug log are +# in-process state. +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", \ + "--proxy-headers"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..9464f62 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Parth Thakkar + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..210f393 --- /dev/null +++ b/README.md @@ -0,0 +1,292 @@ +# AI D&D + +[![CI](https://github.com/parththakkar106/AI-DnD/actions/workflows/ci.yml/badge.svg)](https://github.com/parththakkar106/AI-DnD/actions/workflows/ci.yml) +[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) + +An AI Dungeon-style interactive storytelling app that runs entirely on your own machine, with +your own AI model. Create scenarios, play open-ended adventures where an LLM narrates the +world, and extend the engine with **JavaScript scripts compatible with real AI Dungeon +scripting**. + +> ### ▶️ Try it live: **[parththakkar106.github.io/AI-DnD](https://parththakkar106.github.io/AI-DnD/)** +> The project page loads instantly and launches the hosted demo in one tap. Play a scenario as +> a guest: no sign-up and no API key needed. The demo runs on a free tier that sleeps, so the +> first load after it's been idle takes about 30 to 60 seconds to wake up. +> +> For the internals, read the **[design notes](https://parththakkar106.github.io/AI-DnD/guide.html)**. +> They walk through the context budgeting, the world-state referee, and the memory bank, and +> state the reasoning behind each one ([Markdown version](docs/GUIDE.md)). + +Built with FastAPI and SQLAlchemy on the backend and React (Vite) on the frontend, running on +SQLite locally and Postgres in the cloud. It works with **any OpenAI-compatible endpoint**: +Ollama and LM Studio locally, or OpenRouter, OpenAI, Groq, or vLLM in the cloud. Endpoint, key, +and model are all runtime settings, and OpenRouter's free-tier models make the whole experience +cost nothing. + +![The play screen, with the world-state rail open](docs/images/play-world-state.jpg) + +*The play screen. The left rail shows live world state. The AI proposes changes each turn, and +a Python engine decides what actually sticks. The chip under the narration reports what +changed. The `‹ 2/2 ›` under a turn steps between the takes it has. Writing below a take that +isn't the live one starts a new branch.* + +## Features + +- **The full play loop.** Do / Say / Story / Continue actions, streamed AI responses (SSE), + retry, undo, and edit. Reasoning models are supported: "thinking" streams into a collapsible + 💭 panel with its own token budget. +- **A branching story tree.** The story is a tree, not a list. Any turn can hold more than one + **take**, and `‹ 2/4 ›` steps between them. Stepping is free: the story below simply empties, + and the server is told nothing. Writing below a take that isn't the live one is what makes a + branch. Branches borrow their ancestors' turns instead of copying them, so a fork costs about + 100 bytes, and a 20-fork story loads within 1% of the same story flat. Switching restores that + line's world state, script state, and cooldown clocks. A branch panel switches, renames, and + deletes; **⌗ See the tree** draws every line against the story's own clock + (`backend/app/tree.py`, `backend/app/context/lineage.py`). +- **An RPG world-state engine.** A scenario can declare stats, flags, milestones, and a named + cast; the adventure carries their live values. The AI proposes deltas, and a Python engine + referees them: it clamps values to range, enforces per-turn caps and cooldowns, keeps counters + monotonic and milestones sticky, then strips the machine-readable block out of the prose + (`backend/app/worldstate/engine.py`). Word-labeled bands (`40–60: minor damage`) make the + model reliable at it. No dice and no scripting are required. +- **AI Dungeon-compatible context engine.** Memory, author's note, and story cards (world + info) are triggered by keywords in recent story text, then assembled under a token budget + (`backend/app/context/builder.py`). +- **Insights: total prompt transparency.** Every turn stores the exact prompt sent to the + model. Open 🔍 on any AI action to see each context component, its token cost, and why it was + included. +- **JavaScript scripting, AI Dungeon-compatible.** `onInput` / `onModelContext` / `onOutput` + modifiers share `state` and a `worldEntries` API, and run in an embedded quickjs sandbox + (`backend/app/scripting/`). Real AI Dungeon scripts import and run as is. An in-app CodeMirror + editor is included. +- **Auto-summarization and Memory Bank.** The modern AI Dungeon memory system: AI-generated + memories every few actions, a running story summary, and embedding-based retrieval that + pulls old-but-relevant facts back into context, with similarity scores visible in Insights + (`backend/app/memorybank.py`). +- **Undo and retry that actually roll back state.** Undo and retry roll back the world state + and script state to a per-node snapshot, not just the text, and prune the memories that + covered the removed turns. Nothing a retry replaces is discarded: the old attempt stays as + another take of that turn, one keystroke and one click from becoming a branch of its own. +- **Import and export.** AI Dungeon-compatible formats for scripts and scenarios; JSON for + everything else. An adventure exports as `ai-dnd-adventure-v2`, which carries the whole tree: + every branch, every take, and the fork points, since those were chosen rather than computed. + Files saved in the old single-line format still import. +- **Optional accounts for hosted deployments.** By default the app is single-user with zero + auth friction. Set `AIDND_MULTI_USER=1` and visitors play instantly as guests (signed + session cookie), can register (email and password) at any point to keep their data, and each + user gets isolated data plus their own encrypted-at-rest API key. A server-funded **shared + demo key** with a daily turn cap lets people try it without bringing a key + (`backend/app/auth.py`). Each new guest is also given a copy of a short pre-played + adventure, so the first screen shows real turns and their world-state changes without + spending a demo turn (`backend/app/starter.py`). + +## Screenshots + +| | | +|---|---| +| ![Insights panel](docs/images/insights.jpg) | ![Scenario editor](docs/images/scenario-editor-npcs.jpg) | +| **Insights**: the exact prompt for the next turn, broken into components with token counts and the trigger word that pulled each story card in. | **Authoring**: stats with ranges, per-turn caps, cooldowns, and word-labeled bands; NPCs the AI addresses by id. | +| ![Script editor](docs/images/script-editor.jpg) | ![Home](docs/images/home.jpg) | +| **Scripting**: the three AI Dungeon hooks with shared persistent `state`, run in a quickjs sandbox. | **Home**: continue a story in progress or start from a scenario. | +| ![The branch map](docs/images/branch-map.jpg) | ![The branches panel](docs/images/branches-panel.jpg) | +| **The tree**: one lane per line, from the moment it left its parent to the moment it ends. The horizontal axis is the story's own clock, so a short branch reads as short. | **Branches**: every line the story has taken, and the three things you can do to one. A line the one you're reading was forked from can't be deleted, and says so. | + +## Quick start + +### Docker (any OS) + +```sh +docker compose up --build +``` + +Open http://localhost:8000. Your data persists in a named volume across restarts. + +### Windows + +```powershell +cd backend; python -m venv .venv; .\.venv\Scripts\pip.exe install -r requirements.txt; cd .. +cd frontend; npm install; cd .. +.\start.ps1 +``` + +Open http://localhost:5173 (dev servers; API docs at http://localhost:8000/docs). + +### macOS / Linux + +```sh +./start.sh # creates the venv and installs dependencies on first run +``` + +Open http://localhost:5173. + +## Connect a model + +Open **Settings** in the app and point it at any OpenAI-compatible endpoint: + +| Provider | Endpoint URL | Notes | +|---|---|---| +| Ollama (local) | `http://localhost:11434/v1` | free, private; also serves embedding models for the Memory Bank (e.g. `nomic-embed-text`) | +| LM Studio (local) | `http://localhost:1234/v1` | free, private | +| OpenRouter | `https://openrouter.ai/api/v1` | `:free` models cost nothing (no embeddings on the free tier) | +| OpenAI / Groq / vLLM / … | provider's `/v1` URL | anything speaking `/v1/chat/completions` | +| Claude Code CLI (local) | `http://127.0.0.1:8787/v1` | your Claude subscription instead of an API key; see [Playing against Claude locally](#playing-against-claude-locally) | + +Model name, API key, generation parameters, and (optionally) summary and embedding models for +the Memory Bank are all configured there too. No config files and no rebuild are needed. + +### Playing against Claude locally + +`backend/tools/claude_shim.py` serves an OpenAI-compatible endpoint backed by the +`claude` command line tool, so you can play the demos against a real model without an +API key. Each request spawns one `claude --print` process, which suits the turn engine: +the app assembles the whole prompt every turn and expects a stateless endpoint. + +```sh +cd backend +.venv/Scripts/python.exe tools/claude_shim.py # listens on 127.0.0.1:8787 +``` + +In Settings, choose the OpenAI-compatible provider, set the base URL to +`http://127.0.0.1:8787/v1`, put any non-empty string in the API key field, and pick +`sonnet`. The shim ignores the key and authenticates as you, through the CLI. Set the +reasoning budget to `0` or `-1`: a positive budget sends a `reasoning.max_tokens` field +that Claude 5 models reject. + +Embeddings are not served. Leave the embedding model blank, or point the Memory Bank at +a real endpoint. + +Run it against a local backend only. The endpoint has no authentication, and anything +reaching it spends your Claude quota. `app/netguard.py` blocks localhost endpoints when +`AIDND_MULTI_USER` is set, so a deployed instance cannot be pointed at it. + +## How a turn works + +``` +player input + → onInput script modifier + → assemble context: [narrator prompt] + [world state + stat guide] + [AI instructions] + + [plot essentials] + [story summary] + [retrieved memories] + + [triggered story cards] + [history along this branch, token-budgeted] + + [author's note] + [player action] + → onModelContext script modifier + → snapshot context (Insights) + → provider adapter → AI (streamed) + → extract + referee the world-state delta block, strip it from the prose + → onOutput script modifier + → store & render +``` + +## Architecture + +``` +frontend/ React + Vite SPA ──HTTP/SSE──► backend/ FastAPI + ├─ routers/ auth, scenarios, adventures, story cards, scripts, chat, settings, analytics, debug + ├─ models.py SQLAlchemy: User, Scenario, Adventure, Branch, Action, StoryCard, Script, Settings, Memory + ├─ migrations.py hand-rolled, versioned via PRAGMA user_version (64 and counting) + ├─ auth.py guest/registered users, sessions, shared demo key + ├─ security.py password hashing, cookie signing, API-key encryption + ├─ tree.py forking, promotion, and where a node is placed + ├─ attempts.py the takes of one turn, grouped by parent + ├─ context/ prompt assembly under a token budget + lineage/history windowing + ├─ worldstate/ the stat engine: clamps, cooldowns, bands, milestones + ├─ scripting/ quickjs sandbox + AI Dungeon API surface + ├─ memorybank.py auto-summarization + embedding retrieval + ├─ analytics.py buffered visit counters + the owner's dashboard query + ├─ bundle.py the export/import formats, v2 (tree) and a v1 reader + ├─ providers/ OpenAI-compatible adapter, streaming + └─ data.db SQLite (path overridable via AIDND_DB_PATH) +``` + +In production the backend serves the built SPA from one port (see `Dockerfile`). In +development, Vite proxies `/api` to FastAPI. + +## Tests + +549 backend tests: unit tests plus full HTTP integration through the real quickjs scripting +engine, with the LLM provider mocked. CI runs them on every push, alongside the frontend +lint/build and a Docker image build. + +```sh +cd backend && pip install -r requirements.txt -r requirements-dev.txt +python -m pytest tests/ +``` + +## Notes on performance + +Two of the tests exist because of bugs that were measured rather than guessed at. They are the +most interesting engineering in the repo. + +- **Database egress, cut about 189x.** Every adventure load pulled `Action.context_snapshot`, + the entire assembled prompt at about 74 KB per turn, to read two small fields off it. Moving + those fields into their own columns and marking the heavy ones `deferred` took one adventure + load from 38.5 MB to 0.20 MB. The backfill runs server-side with dialect-specific SQL, so the + old data never crosses the wire. `tests/test_egress.py` hooks into SQLAlchemy's cursor events + and fails if a bulk load ever names those columns again. +- **Turn cost, made flat.** Assembling a turn walked the whole story, so cost scaled with story + length: 839 KB of reads at turn 200. `backend/app/context/history.py` now serves tails and + slices from SQL and measures what it fetched. The same turn now costs 129 KB and stops + growing at around turn 50. +- **Branching that costs nothing to read.** A branch stores no turns. It stores where it left + its parent, and borrows everything above that. A 40-turn story forked twenty times loads in + 31,652 bytes against 31,433 bytes for the same story flat: a 1.007x ratio, or about 103 bytes + per branch. Reads stay cheap because the lineage is windowed the same way the history is, so + the number of SQL clauses is bounded by the context window rather than by the number of + forks. + +## Visit analytics + +The hosted demo keeps its own analytics: an owner-only dashboard at `/analytics` shows +traffic, which shared scenarios get played, turns and demo-key spend, errors, and a funnel +from *visited* to *played a turn* to *signed up*. It is visible only to the emails listed in +`AIDND_ANALYTICS_EMAILS`, and the route returns 404 for everyone else. + +This is built into the app rather than added with a third-party script, for reasons specific +to this project: the CSP allows only `script-src 'self'`, ad blockers block the popular +trackers, and none of those trackers can see the measurement that matters here, a turn. Counts +are aggregated in memory and flushed as UPSERTs, so a visit is a write and never a read, and +every dashboard query is a `GROUP BY` that returns tens of rows regardless of traffic volume. +That matters: see the egress note above for what reading rows per request costs on this stack. + +## Deploy (Render) + +The repo ships a [`render.yaml`](render.yaml) blueprint: one Docker web service that serves +the SPA and API same-origin, backed by external [Neon](https://neon.tech) Postgres. The free +Render tier has no persistent disk, so the database lives off-box. + +1. Create a **Neon** project and copy its pooled connection string. +2. In Render, choose **New → Blueprint** and point it at this repo. Render reads + `render.yaml`. +3. Fill in the secrets it prompts for (`sync: false` vars): `AIDND_DATABASE_URL` (the Neon + string); `AIDND_DEMO_API_KEY` and `AIDND_DEMO_MODELS` to offer a no-signup demo; and + `AIDND_ANALYTICS_EMAILS` (your own account's email) to see the Visitors dashboard. + `AIDND_SECRET_KEY` is generated automatically and stays stable across deploys. +4. Deploy. Pushes to `main` auto-deploy after this. The health check is `/api/health`. + +On the free tier the service sleeps after about 15 minutes idle, and the first request after +that takes about 30 to 60 seconds to wake it. Point any keep-warm pinger at `/api/health`, +which deliberately doesn't touch the database: waking the database around the clock costs far +more than the cold start saves. + +If you put another proxy or CDN in front of Render, set `AIDND_TRUSTED_PROXY_HOPS` to the +number of proxies in the chain. It defaults to 1. The rate limiter reads the client IP that +many entries from the right of `X-Forwarded-For`, because the trusted edge appends the real +one last. Leave it at 1 behind two proxies and the limiter reads an entry the caller +supplied, so anyone can rotate the header for a fresh rate-limit bucket per request. + +## Repo notes + +- `plan/` holds the phased implementation plan this project was built from, kept as a build + log. All fourteen phases are complete. The later files (11, 12, 14) also serve as design + notes for the state-revert, world-state, and story-tree work. + [`plan/STATUS.md`](plan/STATUS.md) is the running thread: what shipped, what was measured, + and what is owed next. +- [`docs/GUIDE.md`](docs/GUIDE.md) holds design notes: how each subsystem works and why it was + built that way, with the measurements behind the decisions. It is also rendered as a + [reading page](https://parththakkar106.github.io/AI-DnD/guide.html). +- `backend/.env.example` lists the few environment variables the backend reads. +- [`docs/self-review.md`](docs/self-review.md) records a full-codebase self-review pass and + what came out of it. All correctness findings are resolved. + +## License + +[MIT](LICENSE) diff --git a/backend/.env.example b/backend/.env.example new file mode 100644 index 0000000..97e7e5c --- /dev/null +++ b/backend/.env.example @@ -0,0 +1,110 @@ +# Environment variables read by the backend. +# +# NOTE: the app reads real environment variables — it does NOT auto-load this +# file. Set them in your shell, in docker-compose.yml, or in your host's +# dashboard. This file is documentation (and a template for deploy configs). + +# Absolute path for the SQLite database file. Parent directory is created if +# missing. Default when unset: backend/data.db +# Docker compose sets this to /data/data.db (a named volume). +AIDND_DB_PATH= + +# --------------------------------------------------------------------------- +# Phase 9 — production hardening +# --------------------------------------------------------------------------- + +# Switch from SQLite to a server database (hosted deploys use Neon Postgres). +# Any SQLAlchemy URL; postgres:// and postgresql:// schemes are rewritten to +# the psycopg3 driver automatically. The platform-conventional DATABASE_URL +# is honored too (AIDND_DATABASE_URL wins if both are set). Unset = SQLite. +AIDND_DATABASE_URL= + +# Comma-separated list of allowed CORS origins. Only needed when the frontend +# is served from a different origin than the API; the production build is +# served same-origin by FastAPI, so hosted deploys can leave this unset. +# Default: http://localhost:5173,http://127.0.0.1:5173 (the Vite dev server). +AIDND_CORS_ORIGINS= + +# How many proxy hops the rate limiter trusts in `X-Forwarded-For`. It reads +# the entry that many places from the right, because the trusted edge appends +# the real client IP last. Set this to the number of proxies in front of the +# app. Default: 1, which is correct for a single edge such as Render. +# +# Get it wrong in either direction and the rate limits weaken. Too low reads an +# entry the caller supplied, so anyone can rotate the header for a fresh +# rate-limit bucket per request and walk past the auth and guest limits. Too +# high reads past the real client. Only multi-user mode rate-limits at all, so +# local installs can ignore this. +AIDND_TRUSTED_PROXY_HOPS= + +# --------------------------------------------------------------------------- +# Phase 8 — optional accounts & multi-user (all optional; defaults keep the +# app in frictionless single-user "local mode") +# --------------------------------------------------------------------------- + +# "1"/"true" turns on multi-user mode: guest sessions via signed cookies, +# register/login UI, per-user data. Leave unset for local installs. +AIDND_MULTI_USER= + +# Secret for signing session cookies and encrypting stored API keys at rest. +# If unset in local mode, one is auto-generated into `secret.key` next to the +# database (fine for local/docker-volume runs). REQUIRED when +# AIDND_MULTI_USER is on — the app refuses to start without it, because a +# regenerated secret on an ephemeral hosted filesystem would log out every +# user on each deploy. Generate one: +# python -c "import secrets; print(secrets.token_urlsafe(48))" +AIDND_SECRET_KEY= + +# Session cookie Secure flag (HTTPS-only). Defaults to on when +# AIDND_MULTI_USER is on, off otherwise — set 0/1 only to override (e.g. 0 +# when testing multi-user mode over plain http on a LAN address). +AIDND_COOKIE_SECURE= + +# --- Shared demo key (BYOK fallback; only active when AIDND_MULTI_USER=1) --- +# Users with no API key of their own get this server-funded endpoint with a +# model whitelist and a per-day turn cap. Unset = no demo, users must bring +# their own key. Memory bank/auto-summarization are disabled on demo turns. +AIDND_DEMO_API_KEY= +# Default endpoint if unset: https://openrouter.ai/api/v1 +AIDND_DEMO_ENDPOINT_URL= +# Comma-separated model whitelist. Default: google/gemma-4-26b-a4b-it:free +AIDND_DEMO_MODELS= +# Successful AI turns per user per day on the demo key. Default: 20 +AIDND_DEMO_TURNS_PER_DAY= +# Comma-separated emails of "power users" (trusted testers) who bypass the daily +# demo cap entirely — unmetered turns on the shared demo key — and get the AI Chat +# page (a plain scratchpad for talking to a model, hidden from everyone else). +# Registered accounts only (guests have no email). Matched case-insensitively. +# Local (single-user) installs are always treated as power users. +AIDND_POWER_USERS= + +# --- Visit analytics --- +# Comma-separated emails allowed to see the Visitors dashboard (/analytics) and +# its nav link. Deliberately separate from AIDND_POWER_USERS: a trusted tester +# gets unmetered turns, which is no reason to hand them the traffic numbers. +# Unset = nobody sees it in a hosted deploy. Local installs always can, and are +# the only mode where the viewer's own visits are still counted (excluding them +# would leave the page permanently empty on the machine it's developed on). +# Collection itself is always on; only the dashboard is gated. +AIDND_ANALYTICS_EMAILS= +# Days to keep the one-row-per-visitor-per-day table that makes the funnel +# count people rather than clicks. The daily counters are aggregate and kept +# forever. Default: 400. Set 0 to keep visitor-days forever. +AIDND_ANALYTICS_RETENTION_DAYS= + +# --- Guest retention (only active when AIDND_MULTI_USER=1) --- +# Every first visit mints a guest account, so a public demo collects one row +# per visitor. A guest with no activity for this many days is deleted along +# with its scenarios, adventures and actions. Registered accounts are never +# touched. Default: 5. Set 0 to keep guests forever. +AIDND_GUEST_RETENTION_DAYS= +# How often a running process re-checks. The sweep also runs once at startup, +# which is what actually fires on hosts that sleep. Default: 6 +AIDND_CLEANUP_INTERVAL_HOURS= + +# The AI endpoint/API key/model are NOT env vars — they are configured at +# runtime in the app's Settings page and stored (encrypted) in the database. +# +# Rate limits, request size limits, and per-user row caps are hardcoded with +# generous values (see backend/app/limits.py) and active only in multi-user +# mode — local installs are never throttled. diff --git a/backend/app/__init__.py b/backend/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/accesslog.py b/backend/app/accesslog.py new file mode 100644 index 0000000..361e1b8 --- /dev/null +++ b/backend/app/accesslog.py @@ -0,0 +1,168 @@ +"""The access log: who arrived, when, and from where. + +The deliberate opposite of analytics.py. That module counts and stores nothing +that points at a person; this one records addresses, email addresses and +devices, because an access log that cannot identify the access is not an access +log. The two live in separate modules and separate tables on purpose, so that +the anonymity of the counters is a property of the code rather than a convention +someone has to remember. + +Owner-only, and never shown to the people it records. + +Four kinds of row: + +- `session` A browser that has a session made a request. For a guest, this + is their first visit. +- `login` An existing account signed in. +- `register` A guest upgraded to an account. +- `login_failed` A password attempt that did not match, with the address tried. + +Session rows are the only ones that need thinning. `/auth/me` runs on every page +load, and one row per load would be noise rather than a log. A row is written +when the day or the address changes for that user. That is the granularity a log +is read at, such as seen on the 3rd from 1.2.3.4, and it still records someone +moving networks during a day. +""" + +import logging +import threading + +from sqlalchemy import desc, or_, select +from sqlalchemy.orm import Session + +from . import analytics, models + +logger = logging.getLogger(__name__) + +SESSION = "session" +LOGIN = "login" +REGISTER = "register" +LOGIN_FAILED = "login_failed" + +MAX_UA = 200 + +# user id -> (day, ip) of the last session row written for them. Process-local +# like the rate limiter's windows, and for the same reason: this is a single +# process, and the worst case after a restart is one redundant row per user. +_last_session: dict[int, tuple[str, str]] = {} +_guard = threading.Lock() +_MAX_TRACKED = 10_000 + + +def _client_ip(request) -> str: + # This import is deferred. `limits` imports `auth`, which the routers that + # call this function import, so a module-level import here would create a + # cycle. The spoof resistance lives in `limits` and must not be + # reimplemented. A second, looser answer to which address belongs to the + # client is how one of them ends up trusting a header it should not. + from . import limits + + return limits.client_ip(request) + + +def describe(user: models.User) -> str: + """Returns how a user is named in the log. + + A guest has no email, and their id is the only handle anyone has for them. + The third case is a local install's implicit single user, who also has no + email but is the operator rather than a visitor. Naming that user "Guest #1" + would be wrong in the one row they are certain to read. + """ + if user.email: + return user.email + return f"Guest #{user.id}" if user.is_guest else f"Local user #{user.id}" + + +def _country(request) -> str: + """Returns the edge's country header, or "" when there is none. + + The blank differs from the counters' "(unknown)" label. A table column reads + better as a dash than as a word, and an empty string is the correct value for + a country that is not known. + """ + country = analytics.country_of(request.headers) + return "" if country == analytics.UNKNOWN else country + + +def record( + db: Session, + kind: str, + request, + *, + user: models.User | None = None, + who: str | None = None, +) -> None: + """Writes one row. + + This function never raises. The log observes sign-in rather than guarding it, + and a logging failure must not lock anyone out. + """ + try: + event = models.AccessEvent( + kind=kind, + user_id=user.id if user is not None else None, + who=(who if who is not None else describe(user) if user else "")[:320], + is_guest=bool(user.is_guest) if user is not None else False, + ip=_client_ip(request)[:45], + country=_country(request), + device=analytics.device_of(request.headers.get("user-agent", "")), + user_agent=(request.headers.get("user-agent") or "")[:MAX_UA], + ) + db.add(event) + db.commit() + except Exception: # pragma: no cover - defensive + db.rollback() + logger.exception("Access log write failed; continuing.") + + +def note_session(db: Session, user: models.User, request) -> None: + """Records that a session made a request, at most one row per day per address.""" + try: + today = analytics._today() + ip = _client_ip(request) + with _guard: + if _last_session.get(user.id) == (today, ip): + return + _last_session[user.id] = (today, ip) + if len(_last_session) > _MAX_TRACKED: + # Nothing here needs to persist. Clearing the map costs at most + # one extra row per active user. + _last_session.clear() + _last_session[user.id] = (today, ip) + except Exception: # pragma: no cover - defensive + logger.exception("Access log session check failed; continuing.") + return + record(db, SESSION, request, user=user) + + +def recent( + db: Session, + *, + limit: int = 50, + before_id: int | None = None, + kind: str | None = None, + query: str | None = None, +) -> dict: + """Returns a page of the log, newest first. + + The page is anchored on a row id rather than an offset, as the story pager + is. Rows keep arriving while the log is read, and an offset would shift the + page under whoever is reading it. + """ + statement = select(models.AccessEvent).order_by(desc(models.AccessEvent.id)) + if before_id is not None: + statement = statement.where(models.AccessEvent.id < before_id) + if kind: + statement = statement.where(models.AccessEvent.kind == kind) + if query: + like = f"%{query.strip()}%" + statement = statement.where(or_( + models.AccessEvent.who.ilike(like), + models.AccessEvent.ip.ilike(like), + models.AccessEvent.country.ilike(like), + )) + # Requesting one extra row reports whether more rows exist, without a + # second COUNT over the whole table. + rows = list(db.scalars(statement.limit(limit + 1))) + has_more = len(rows) > limit + return {"events": rows[:limit], "has_more": has_more} diff --git a/backend/app/analytics.py b/backend/app/analytics.py new file mode 100644 index 0000000..86c1489 --- /dev/null +++ b/backend/app/analytics.py @@ -0,0 +1,583 @@ +"""Visit analytics for the hosted demo. + +This is a small self-hosted counter that answers whether anyone visited and +whether they played. It is built into the app rather than added with a +third-party script, because the CSP in `main.py` allows scripts from 'self' +only, ad blockers block the popular trackers, and none of those trackers can see +what is worth knowing here: turns taken, demo-key spend, and which seeded +scenario people pick. + +Three rules shape the design: + +1. It stores nothing personal. It records no IP addresses, no user agents, no + user ids, and no title of anything a player wrote. A visitor appears only as + an HMAC of their user id, which is one-way and salted with the app's secret + key, so these tables cannot be joined back to an account even by someone + holding the database. Story content never reaches this module. What one + specific person did is unanswerable by design, and only totals are + available. +2. Egress is the budget. Neon bills for bytes leaving the database, and this + project has already paid for forgetting that once. Counts are therefore + aggregated in memory and flushed as UPSERTs, so a visit is a write and never + a read, and every dashboard query is a GROUP BY that returns tens of rows + rather than per-visit rows. A month of traffic costs a few kilobytes to read + back. +3. The numbers come from the server, not from the browser. The client reports + one thing, which is the page that was viewed. Everything with meaning, such + as a turn happening or an account being created, is recorded by the code that + performs it, where a stranger cannot fake it and an extension cannot block + it. + +Storage is two tables, both bounded. `analytics_daily` holds one counter row per +day, metric, and label, which is a few dozen rows a day. +`analytics_visitor_days` holds one row per visitor per day carrying the funnel +flags, which is what makes the funnel count people rather than clicks. It is the +only table that grows with traffic, and cleanup ages it out. +""" + +import hmac +import logging +import os +import re +import threading +from datetime import timedelta +from hashlib import sha256 +from urllib.parse import urlsplit + +from sqlalchemy import case, func, or_, select +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.dialects.sqlite import insert as sqlite_insert +from sqlalchemy.orm import Session + +from . import models, security + +logger = logging.getLogger(__name__) + +# ---------- Metrics ---------- +# `metric` is the family, and `label` is the bucket within it. One generic +# counter table is better than a column per measurement, because adding a new +# question later costs nothing rather than a migration. + +M_PAGE = "pageview" +M_EVENT = "event" +M_REFERRER = "referrer" +M_DEVICE = "device" +M_COUNTRY = "country" +M_SCENARIO = "scenario" # Which seeded or public scenario was played. +M_ERROR = "error" # " " for a 4xx or 5xx on /api. + +EV_SCENARIO_OPEN = "scenario_opened" +EV_ADVENTURE = "adventure_created" +EV_IMPORT = "adventure_imported" +EV_TURN = "turn" +EV_DEMO_TURN = "demo_turn" # A turn billed to the shared demo key. +EV_TURN_ERROR = "turn_error" +EV_SIGNUP = "signup" +EV_LOGIN = "login" + +# Events that are also funnel steps. Recording one sets a flag on the visitor's +# row for the day, so the funnel counts distinct visitor-days rather than repeat +# clicks. This name-to-column map is the whole definition of the funnel, and the +# dashboard reads it back in this order. +FUNNEL_FLAGS = { + EV_SCENARIO_OPEN: "opened", + EV_ADVENTURE: "created", + EV_TURN: "played", + EV_SIGNUP: "signed_up", +} + +OTHER = "(other)" +NONE_LABEL = "(direct)" +UNKNOWN = "(unknown)" + +# ---------- Bounds ---------- +# These bounds exist so that a hostile visitor can add rows to these tables no +# faster than an honest one. The only label a client can influence is the +# referrer, and together these caps mean the worst it can do is fill one day's +# referrer list and then be folded into "(other)". + +MAX_LABEL_LEN = 80 +MAX_LABELS_PER_METRIC = 200 # Distinct labels per metric per day, then OTHER. +MAX_PENDING = 4000 # Buffered entries before an inline flush. +FLUSH_INTERVAL_SECONDS = 60 + +# How long the per-visitor-day rows are kept. The daily counters are small and +# are kept indefinitely. These rows are the ones that scale with traffic. A +# visitor whose last visit ages out counts as new again, which is an acceptable +# trade at this horizon and keeps the table from being a permanent record of +# anyone. +RETENTION_DAYS = int(os.environ.get("AIDND_ANALYTICS_RETENTION_DAYS", "400") or 400) + +_HOST_OK = re.compile(r"^[a-z0-9.-]+$") +_COUNTRY_OK = re.compile(r"^[A-Z]{2}$") +_NUMERIC_SEGMENT = re.compile(r"^\d+$") + +# SPA routes, in the form the dashboard shows them. Any other path a client +# reports becomes OTHER, so the page list cannot be filled with junk and cannot +# record which adventure someone is reading. +KNOWN_ROUTES = { + "/", "/adventures", "/scenarios", "/scenarios/:id", "/play/:id", + "/scripts", "/scripts/:id", "/settings", "/chat", "/analytics", +} + +# ---------- In-process buffer ---------- +# The deployment is a single process, which is the same assumption `limits.py` +# makes, so a plain dict under a lock is the whole design. Losing up to a minute +# of counts to a hard restart is acceptable for traffic numbers, and the flusher +# also runs on shutdown. On Render's free tier the service is idle when it +# sleeps, so the buffer it sleeps on is empty. + +_counts: dict[tuple[str, str, str], int] = {} +_visits: dict[tuple[str, str], set[str]] = {} # (day, visitor) -> flags. +_labels_seen: dict[tuple[str, str], set[str]] = {} # (day, metric) -> labels. +_guard = threading.Lock() + + +def _today() -> str: + return models.utcnow().date().isoformat() + + +def record(metric: str, label: str = "", *, n: int = 1) -> None: + """Adds `n` to one counter. + + This function never raises. Analytics must not fail a request that it is + only observing. + """ + try: + day = _today() + label = (label or "").strip()[:MAX_LABEL_LEN] + with _guard: + seen = _labels_seen.setdefault((day, metric), set()) + if label not in seen: + if len(seen) >= MAX_LABELS_PER_METRIC: + label = OTHER + else: + seen.add(label) + key = (day, metric, label) + _counts[key] = _counts.get(key, 0) + n + pending = len(_counts) + len(_visits) + except Exception: # pragma: no cover - defensive + logger.exception("Analytics counter failed; continuing.") + return + if pending >= MAX_PENDING: + flush() + + +def visitor_id(user: models.User) -> str: + """Returns a stable, one-way handle for one visitor. + + The handle is an HMAC of the user id under the app's secret key. It is + stable, so a returning visitor can be distinguished from a new one. It is + one-way, so nothing in the analytics tables points back at an account. It is + keyed, so a client cannot compute one and claim to be someone else. One + consequence follows: rotating `AIDND_SECRET_KEY` makes every returning + visitor look new. + """ + digest = hmac.new(security.SECRET_KEY, f"visitor:{user.id}".encode(), sha256) + return digest.hexdigest()[:32] + + +def record_visit(user: models.User | None, *, flag: str | None = None) -> None: + """Records that this visitor was here today, and optionally sets one funnel + flag. + + Without a user the call does nothing. A page loaded before a session exists + still counts as a pageview, but not as a person. + """ + if user is None: + return + try: + with _guard: + flags = _visits.setdefault((_today(), visitor_id(user)), set()) + if flag: + flags.add(flag) + except Exception: # pragma: no cover - defensive + logger.exception("Analytics visit failed; continuing.") + + +def record_event(name: str, user: models.User | None = None) -> None: + """Records one event, and credits the visitor's day if it is a funnel step. + + This is the whole interface the call sites use. + """ + record(M_EVENT, name) + record_visit(user, flag=FUNNEL_FLAGS.get(name)) + + +# ---------- Normalizing what the browser reports ---------- + +def normalize_route(path: str) -> str: + """Reduces a client-reported path to one of `KNOWN_ROUTES`. + + Numeric segments become ":id". That bounds the label count, and it keeps + which adventure someone opened out of the statistics. + """ + path = (path or "/").split("?")[0].split("#")[0] + if not path.startswith("/"): + path = "/" + path + if len(path) > 1: + path = path.rstrip("/") + parts = [":id" if _NUMERIC_SEGMENT.match(p) else p for p in path.split("/")] + route = "/".join(parts) or "/" + return route if route in KNOWN_ROUTES else OTHER + + +def normalize_referrer(referrer: str, own_host: str = "") -> str: + """Returns the sending site as a bare host. + + This app's own host means an internal navigation, which is not a referral. + In that case the function returns "", which tells the caller to skip it. + """ + if not referrer: + return NONE_LABEL + host = (urlsplit(referrer).hostname or "").lower().lstrip(".") + if not host or not _HOST_OK.match(host) or len(host) > MAX_LABEL_LEN: + return OTHER + if host == (own_host or "").lower() or host in ("localhost", "127.0.0.1"): + return "" + return host[4:] if host.startswith("www.") else host + + +def api_route_label(scope: dict, status: int) -> str: + """Returns an error bucket such as "500 /api/adventures/{adventure_id}". + + The label uses the route template, never the request path. That keeps one + bucket per endpoint rather than one per adventure id. It also bounds the + table: an unmatched path is chosen entirely by the caller, so labeling by it + would let anyone create rows by requesting arbitrary paths. + """ + template = getattr(scope.get("route"), "path", None) + return f"{status} {template}" if template else f"{status} (unmatched)" + + +def device_of(user_agent: str) -> str: + """Returns "mobile", "tablet", or "desktop", and nothing more specific. + + The user-agent string itself is never stored, because it is a fingerprint + and the useful answer is one word. + """ + ua = (user_agent or "").lower() + if not ua: + return UNKNOWN + if any(bot in ua for bot in ("bot", "crawler", "spider", "headless", "preview")): + return "bot" + if "ipad" in ua or "tablet" in ua or ("android" in ua and "mobile" not in ua): + return "tablet" + if any(m in ua for m in ("mobi", "iphone", "ipod", "android", "phone")): + return "mobile" + return "desktop" + + +# Geo headers an edge network may add. Render fronts services with a CDN that +# can set `cf-ipcountry`, and the others cost nothing to check. A value is +# trusted only if it looks like an ISO code, because a client can send any +# header, so the worst case is a wrong country rather than an unbounded label. +_GEO_HEADERS = ("cf-ipcountry", "x-vercel-ip-country", "x-geo-country", "x-country-code") + + +def country_of(headers) -> str: + for name in _GEO_HEADERS: + value = (headers.get(name) or "").strip().upper() + if _COUNTRY_OK.match(value) and value != "XX": + return value + return UNKNOWN + + +# ---------- Flushing ---------- + +def _insert(db: Session): + return sqlite_insert if db.get_bind().dialect.name == "sqlite" else pg_insert + + +def _drain() -> tuple[dict, dict]: + with _guard: + counts, visits = _counts.copy(), _visits.copy() + _counts.clear() + _visits.clear() + # The label sets bound cardinality within one day, so drop the + # previous day's rather than grow a map that never shrinks. + today = _today() + for key in [k for k in _labels_seen if k[0] != today]: + del _labels_seen[key] + return counts, visits + + +def _restore(counts: dict, visits: dict) -> None: + """Returns a failed flush's work to the buffer, so the next flush retries it.""" + with _guard: + for key, n in counts.items(): + _counts[key] = _counts.get(key, 0) + n + for key, flags in visits.items(): + _visits.setdefault(key, set()).update(flags) + + +def flush(db: Session | None = None) -> None: + """Writes the buffer out. This is safe to call from anywhere and never raises.""" + counts, visits = _drain() + if not counts and not visits: + return + own_session = db is None + if own_session: + from .database import SessionLocal + db = SessionLocal() + try: + _write_counts(db, counts) + _write_visits(db, visits) + db.commit() + except Exception: + db.rollback() + _restore(counts, visits) + logger.exception("Analytics flush failed; counts held for the next one.") + finally: + if own_session: + db.close() + + +def _write_counts(db: Session, counts: dict) -> None: + if not counts: + return + table = models.AnalyticsDaily.__table__ + rows = [ + {"day": day, "metric": metric, "label": label, "hits": hits} + for (day, metric, label), hits in counts.items() + ] + stmt = _insert(db)(table).values(rows) + db.execute(stmt.on_conflict_do_update( + index_elements=["day", "metric", "label"], + set_={"hits": table.c.hits + stmt.excluded.hits}, + )) + + +def _write_visits(db: Session, visits: dict) -> None: + if not visits: + return + table = models.AnalyticsVisitorDay.__table__ + ids = {visitor for _, visitor in visits} + # One indexed lookup decides new against returning for the whole batch. It + # is the only read this module makes outside the dashboard, and it returns + # short hashes for the visitors active right now, so the batch bounds it. + known = set(db.scalars( + select(models.AnalyticsVisitorDay.visitor) + .where(models.AnalyticsVisitorDay.visitor.in_(ids)) + .distinct() + )) + rows = [ + { + "day": day, + "visitor": visitor, + "is_new": visitor not in known, + **{column: column in flags for column in FUNNEL_FLAGS.values()}, + } + for (day, visitor), flags in visits.items() + ] + stmt = _insert(db)(table).values(rows) + db.execute(stmt.on_conflict_do_update( + index_elements=["day", "visitor"], + # Flags only turn on, and `is_new` is absent on purpose. The first + # write of a visitor's first day is what decided it. + set_={ + column: or_(table.c[column], stmt.excluded[column]) + for column in FUNNEL_FLAGS.values() + }, + )) + + +def purge_old_visitor_days(db: Session) -> int: + """Deletes visitor-day rows past the retention horizon. + + The cleanup sweeper calls this. The daily counters are never purged, because + they are aggregates, they are small, and this project keeps its history. + """ + if RETENTION_DAYS <= 0: + return 0 + cutoff = (models.utcnow().date() - timedelta(days=RETENTION_DAYS)).isoformat() + removed = db.query(models.AnalyticsVisitorDay).filter( + models.AnalyticsVisitorDay.day < cutoff + ).delete(synchronize_session=False) + db.commit() + return removed or 0 + + +# ---------- Reading it back ---------- +# Every query below is an aggregate. The database does the counting and returns +# tens of rows, however much traffic is behind them. No query here can return a +# row that belongs to one visitor. + +TOP_N = 12 + + +def _top(rows: list[dict], limit: int = TOP_N) -> list[dict]: + return rows[:limit] + + +def summary(db: Session, days: int = 30) -> dict: + """Returns everything the dashboard shows for the last `days` days, including + today. + + The function flushes first, so the numbers include the last minute. + """ + flush(db) + today = models.utcnow().date() + since = (today - timedelta(days=days - 1)).isoformat() + daily = models.AnalyticsDaily + visitor = models.AnalyticsVisitorDay + + # 1. Every counter in the window, reduced to (metric, label) totals. The + # page, referrer, country, device, scenario, and error tables all come + # from this one pass rather than from a query each. + by_metric: dict[str, list[dict]] = {} + for metric, label, hits in db.execute( + select(daily.metric, daily.label, func.sum(daily.hits)) + .where(daily.day >= since) + .group_by(daily.metric, daily.label) + ): + by_metric.setdefault(metric, []).append({"label": label, "hits": int(hits)}) + for rows in by_metric.values(): + rows.sort(key=lambda row: -row["hits"]) + events = {row["label"]: row["hits"] for row in by_metric.get(M_EVENT, [])} + + # 2. The two per-day series the dashboard draws. + pageviews_by_day = { + day: int(hits) + for day, hits in db.execute( + select(daily.day, func.sum(daily.hits)) + .where(daily.day >= since, daily.metric == M_PAGE) + .group_by(daily.day) + ) + } + turns_by_day = { + day: int(hits) + for day, hits in db.execute( + select(daily.day, func.sum(daily.hits)) + .where(daily.day >= since, daily.metric == M_EVENT, daily.label == EV_TURN) + .group_by(daily.day) + ) + } + + # 3. People, per day. There is one row per visitor per day, so COUNT(*) is + # already the day's unique visitors and no DISTINCT is needed. + visitors_by_day: dict[str, dict] = {} + for day, total, fresh in db.execute( + select( + visitor.day, + func.count(), + func.sum(case((visitor.is_new, 1), else_=0)), + ) + .where(visitor.day >= since) + .group_by(visitor.day) + ): + visitors_by_day[day] = {"visitors": int(total), "new": int(fresh or 0)} + + # 4. The funnel over the whole window, counting each person once. + # COUNT(DISTINCT CASE WHEN flag THEN visitor END) ignores the NULLs the + # CASE leaves for everyone who did not reach that step. + unique, unique_new, *reached = db.execute( + select( + func.count(func.distinct(visitor.visitor)), + func.count(func.distinct(case((visitor.is_new, visitor.visitor)))), + *[ + func.count(func.distinct(case((visitor.__table__.c[column], visitor.visitor)))) + for column in FUNNEL_FLAGS.values() + ], + ).where(visitor.day >= since) + ).one() + + series = [] + for offset in range(days): + day = (today - timedelta(days=days - 1 - offset)).isoformat() + counted = visitors_by_day.get(day, {}) + series.append({ + "day": day, + "visitors": counted.get("visitors", 0), + "new": counted.get("new", 0), + "pageviews": pageviews_by_day.get(day, 0), + "turns": turns_by_day.get(day, 0), + }) + + visits = sum(row["visitors"] for row in series) + pageviews = sum(pageviews_by_day.values()) + turns = events.get(EV_TURN, 0) + errors = by_metric.get(M_ERROR, []) + return { + "days": days, + "since": since, + "until": today.isoformat(), + "generated_at": models.utcnow().isoformat(), + "totals": { + # `visitors` counts each person once for the window. `visits` + # counts them once per day they returned, which is the closest + # measure to "sessions" that does not track sessions. + "visitors": int(unique), + "new_visitors": int(unique_new), + "visits": visits, + "pageviews": pageviews, + "turns": turns, + "demo_turns": events.get(EV_DEMO_TURN, 0), + "adventures": events.get(EV_ADVENTURE, 0), + "signups": events.get(EV_SIGNUP, 0), + "logins": events.get(EV_LOGIN, 0), + "turn_errors": events.get(EV_TURN_ERROR, 0), + "errors": sum(row["hits"] for row in errors), + "turns_per_visit": round(turns / visits, 1) if visits else 0, + "pages_per_visit": round(pageviews / visits, 1) if visits else 0, + }, + "series": series, + # Step 0 is everyone who arrived, so the drop-off between it and + # "Opened a scenario" appears as a step like any other. + "funnel": [{"step": "Visited", "count": int(unique)}] + [ + {"step": step, "count": int(count)} + for step, count in zip( + ["Opened a scenario", "Started an adventure", "Played a turn", "Signed up"], + reached, + ) + ], + "pages": _top(by_metric.get(M_PAGE, [])), + "referrers": _top(by_metric.get(M_REFERRER, [])), + "countries": _top(by_metric.get(M_COUNTRY, [])), + "devices": by_metric.get(M_DEVICE, []), + "scenarios": _top(by_metric.get(M_SCENARIO, [])), + "errors": _top(errors), + "events": by_metric.get(M_EVENT, []), + } + + +# ---------- Background flusher ---------- +# This matches the start and stop pair in `cleanup`, so the lifespan in +# `main.py` reads the same way for both. The interval bounds how much a hard +# restart can lose. + +async def _flush_loop() -> None: + import asyncio + + from starlette.concurrency import run_in_threadpool + + while True: + await asyncio.sleep(FLUSH_INTERVAL_SECONDS) + # This is blocking database work, so keep it off the event loop, which + # is also serving SSE turn streams. + await run_in_threadpool(flush) + + +def start_flusher(): + import asyncio + + return asyncio.create_task(_flush_loop()) + + +async def stop_flusher(task) -> None: + """Cancels the loop and writes out whatever it was holding. + + A deploy is the one restart that is both frequent and predictable, so it + should not be what loses a minute of counts. + """ + import asyncio + + from starlette.concurrency import run_in_threadpool + + if task is not None: + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + await run_in_threadpool(flush) diff --git a/backend/app/attempts.py b/backend/app/attempts.py new file mode 100644 index 0000000..aeb4154 --- /dev/null +++ b/backend/app/attempts.py @@ -0,0 +1,261 @@ +"""Phase 14, SP4: the attempts at one turn. + +A retry used to rewrite the AI action in place and append the discarded attempt +to a JSON list on the same row. Seven separate bugs came from that arrangement. +The row's `text` duplicated one entry of a repeating group, a second column +duplicated its length, and every reader that touched the story during a retry +had to be told to ignore the row. + +Now an attempt is a node. A retry writes a sibling at the same `(branch_id, +depth)` and marks it live. The previous attempt stays as it was written, at the +same coordinate, with `live = False`. Nothing is duplicated, so nothing can +diverge. + +Two invariants hold the arrangement together, and this module is the only place +that maintains either one: + +* Exactly one sibling in a group is live. `lineage.Path.clause` selects on it, so + the other attempts are invisible to every read of the story, and none of those + reads has to know that attempts exist. +* The assembled prompt is stored once per turn, on the live sibling. A + `context_snapshot` is about 163 kB of prompt that every attempt at a turn + shares, plus a few hundred bytes that differ, listed in `ATTEMPT_KEYS`. Giving + each sibling its own copy would make a retry a permanent multiplier on the + largest column in the database, which is what the JSON list was invented to + avoid. The prompt therefore moves with the live flag, and a superseded sibling + keeps only its own slices. + +Ordering inside a group comes from `id`, not from `created_at`. Two attempts made +in the same second still have to page in the order they were made, and `id` +increases with every insert. SP8 dropped `variant_index`, an explicit ordinal +that carried the same order, once a run of the suite confirmed that the two +agreed in every group. +""" + +import copy + +from sqlalchemy.orm import Session, undefer + +from . import models +from .context import lineage + +# The slices of a context snapshot that belong to one attempt rather than to the +# turn. They are the world-state delta the attempt proposed and what the engine +# did with it, the script report, the model's literal reply, and the endpoint's +# token accounting. Each attempt is its own API call, and a retry is the call +# most likely to read the prompt back out of cache. Everything else in a snapshot +# is the prompt, which is assembled once per turn. +ATTEMPT_KEYS = ("world_state", "script", "raw_output", "usage") + + +# ------------------------------------------------------------------ reading + +def group(db: Session, action: models.Action) -> list[models.Action]: + """Returns every attempt at `action`'s turn, oldest first. + + The query keys on the parent rather than on the coordinate (SP9). The two + agree until an attempt is forked onto its own branch. That attempt keeps its + parent but leaves the `(branch, depth)` its siblings are still at, so a + coordinate would report it as the only attempt at its turn, showing `1/1` + where the player should see `1/3`. + + The parent also nests groups correctly without extra work. Attempts under C1 + and attempts under C2 share a depth, and until one of them forks they share a + branch. Only the parent separates them, which is what makes a pager under C2 + read `2/2` rather than count C1's three as well. + + There are two fallbacks, and both mean the row predates the key being asked + about. A node with no branch is a pre-tree row that no path contains, and a + node with no parent is a pre-SP9 row the backfill could not place. Under the + rule each was written with, both are the only attempt at their turn. + """ + if action.branch_id is None or action.depth is None: + return [action] + if action.parent_id is None: + # This row is pre-SP9, and the coordinate is the key those rows were + # written under. A root node also reaches this branch and is genuinely + # alone, because nothing is an attempt at the opening of a story. + return ( + db.query(models.Action) + .filter( + models.Action.adventure_id == action.adventure_id, + models.Action.branch_id == action.branch_id, + models.Action.depth == action.depth, + models.Action.parent_id.is_(None), + ) + .order_by(models.Action.id) + .all() + ) + return ( + db.query(models.Action) + .filter( + models.Action.adventure_id == action.adventure_id, + models.Action.parent_id == action.parent_id, + ) + .order_by(models.Action.id) + .all() + ) + + +def on_branch(rows: list[models.Action], node: models.Action) -> list[models.Action]: + """Returns the attempts in `rows` that are on `node`'s own branch. + + `group` reports which attempts belong to this turn, and since SP9 that spans + branches. An attempt forked onto its own line is still an attempt at the same + turn, which is the reason for keying on the parent. + + Deletion is the one caller that must not follow a group across branches. An + attempt on another branch is reachable through that branch and belongs to the + story someone is telling there. Removing it because a turn was undone here + would delete a line nobody asked about. The same parent and the same branch + together are the coordinate, which is what every attempt at this turn meant + before a fork could move one out of it. + """ + return [row for row in rows if row.branch_id == node.branch_id] + + +def live_in(rows: list[models.Action]) -> models.Action | None: + for row in rows: + if row.live: + return row + return None + + +def preceding( + db: Session, adventure: models.Adventure, node: models.Action +) -> models.Action | None: + """Returns the node the story tells immediately before `node`. + + This reads "before this turn" as a fact about the path rather than as a + snapshot taken from inside the turn, which is what makes the after-snapshots + sufficient on their own. The query undefers both of them, because the only + reason to fetch this row is to restore what it left behind. + """ + if node.depth is None: + return None + return ( + db.query(models.Action) + .filter( + models.Action.adventure_id == adventure.id, + lineage.path_of(db, adventure).clause(models.Action), + models.Action.depth < node.depth, + ) + .options( + undefer(models.Action.state_after), + undefer(models.Action.world_state_after), + ) + .order_by(models.Action.depth.desc(), models.Action.id.desc()) + .first() + ) + + +# ------------------------------------------------------------------ writing + +def restore_state(adventure: models.Adventure, node: models.Action | None) -> None: + """Restores the script state and world state that `node` left behind. + + A NULL snapshot means leave the live state as it is, never reset it. Rows + written before SP4 that the migration could not derive an outcome for carry + NULLs, and overwriting a running adventure's state with an empty dict would + be worse than doing nothing. + """ + if node is None: + return + if isinstance(node.state_after, dict): + adventure.script_state = copy.deepcopy(node.state_after) + if isinstance(node.world_state_after, dict): + adventure.world_state = copy.deepcopy(node.world_state_after) + + +def snapshot_outcome(adventure: models.Adventure, node: models.Action) -> None: + """Records on `node` the state of the adventure now that the node has played.""" + state = adventure.script_state if isinstance(adventure.script_state, dict) else {} + world = adventure.world_state if isinstance(adventure.world_state, dict) else {} + node.state_after = copy.deepcopy(state) + node.world_state_after = copy.deepcopy(world) + + +def roll_back_before( + db: Session, adventure: models.Adventure, node: models.Action +) -> None: + """Rewinds the shared state to what it was before `node` was played.""" + restore_state(adventure, preceding(db, adventure, node)) + + +def add_attempt( + db: Session, + adventure: models.Adventure, + previous: models.Action, + replacement: models.Action, +) -> None: + """Places `replacement` next to `previous` as the newer attempt at that turn. + + The placement is done here rather than through `tree.place_action`, which + moves the head. A sibling is not a new turn. It is another attempt at the + turn the head is already on. + """ + replacement.branch_id = previous.branch_id + replacement.depth = previous.depth + # Copy the parent rather than resolve it from the path. An attempt belongs + # to the turn it is an attempt at, and that is what `group` keys on. + # Resolving it here would ask which node is live one depth back. That is the + # same node right now, and it stops being the same node once the story forks + # away from this turn. + replacement.parent_id = previous.parent_id + replacement.live = True + # The replacement takes its place at the end of the group, because `group` + # orders by `id` and this row has no id yet. Switching a three-attempt turn + # back to attempt 1 and retrying therefore still pages 1, 2, 3, 4, which is + # the order the attempts were made in. + previous.live = False + # The replacement was assembled with a fresh snapshot, so the prompt for + # this turn is now the one it carries. The superseded attempt keeps only the + # slices that were its own. + keep_own_slices(previous) + + +def make_live( + db: Session, adventure: models.Adventure, node: models.Action +) -> list[models.Action]: + """Makes `node` the attempt the story tells, and restores its outcome. + + Returns the group, so that a caller reporting on it does not read it twice. + """ + rows = group(db, node) + previous = live_in(rows) + if previous is not None and previous is not node: + hand_over_the_prompt(previous, node) + for row in rows: + row.live = row is node + restore_state(adventure, node) + return rows + + +# ------------------------------------------------- the prompt, stored once + +def keep_own_slices(node: models.Action) -> None: + """Reduces `node`'s snapshot to the slices that are only its own.""" + snapshot = node.context_snapshot + if not isinstance(snapshot, dict): + return + node.context_snapshot = { + key: snapshot[key] for key in ATTEMPT_KEYS if key in snapshot + } or None + + +def hand_over_the_prompt(giver: models.Action, taker: models.Action) -> None: + """Moves the turn's assembled prompt from one attempt to another. + + The caller runs this when the live flag moves, so that the row in the story + is always the row the Insights viewer can explain. Nothing is copied. The + prompt exists once before and once after, on whichever sibling is being read. + """ + held = giver.context_snapshot if isinstance(giver.context_snapshot, dict) else {} + shared = {k: v for k, v in held.items() if k not in ATTEMPT_KEYS} + if not shared: + return + keep_own_slices(giver) + own = taker.context_snapshot if isinstance(taker.context_snapshot, dict) else {} + taker.context_snapshot = shared | { + k: v for k, v in own.items() if k in ATTEMPT_KEYS + } diff --git a/backend/app/auth.py b/backend/app/auth.py new file mode 100644 index 0000000..20da441 --- /dev/null +++ b/backend/app/auth.py @@ -0,0 +1,263 @@ +"""Phase 8: user resolution, sessions, and the shared demo key. + +The `AIDND_MULTI_USER` environment variable selects one of two modes: + +* Local mode, the default. Every request resolves to one automatically created + local user. There are no cookies and no login UI, so a clone or a + docker-compose run behaves like the single-user app from before Phase 8. +* Multi-user mode, used for hosted deployments. Requests carry a signed session + cookie. `GET /api/auth/me` creates a guest user on the first visit, and + registering upgrades that guest in place so their data survives. A request + without a valid session gets a 401, and the frontend re-establishes the + session through `/me`. + +The shared demo key, which is the fallback when a user brings no key of their +own, is also configured here. A user whose settings hold no API key is routed to +a server-funded endpoint with a model allowlist and a per-day turn cap. +""" + +import os +from dataclasses import dataclass +from datetime import timezone + +from fastapi import Depends, HTTPException, Request +from sqlalchemy.orm import Session + +from . import models, security +from .database import get_db + + +def _env_flag(name: str) -> bool: + return os.environ.get(name, "").strip().lower() in ("1", "true", "yes", "on") + + +MULTI_USER = _env_flag("AIDND_MULTI_USER") + +SESSION_COOKIE = "aidnd_session" +# Secure cookies are on by default in multi-user mode, because a hosted +# deployment serves HTTPS and browsers also accept Secure on http://localhost. +# `AIDND_COOKIE_SECURE` overrides the default with 0 or 1. Use 0 when testing +# multi-user mode over plain HTTP on a LAN address. +_cookie_secure_env = os.environ.get("AIDND_COOKIE_SECURE", "").strip().lower() +COOKIE_SECURE = ( + _cookie_secure_env in ("1", "true", "yes", "on") + if _cookie_secure_env + else MULTI_USER +) +COOKIE_MAX_AGE = 60 * 60 * 24 * 365 + +# ---------- Shared demo key (BYOK fallback) ---------- + +DEMO_API_KEY = os.environ.get("AIDND_DEMO_API_KEY", "").strip() +DEMO_ENDPOINT_URL = ( + os.environ.get("AIDND_DEMO_ENDPOINT_URL", "").strip() + or "https://openrouter.ai/api/v1" +) +DEMO_MODELS = [ + m.strip() + for m in os.environ.get("AIDND_DEMO_MODELS", "").split(",") + if m.strip() +] or ["google/gemma-4-26b-a4b-it:free"] +DEMO_TURNS_PER_DAY = int(os.environ.get("AIDND_DEMO_TURNS_PER_DAY", "20") or 20) + +# Trusted testers, listed by email, who bypass the daily demo cap and take +# unmetered turns on the shared demo key. The list is comma-separated, and the +# match ignores case. +POWER_USERS = { + e.strip().lower() + for e in os.environ.get("AIDND_POWER_USERS", "").split(",") + if e.strip() +} + +# Who can see the visit analytics. This is a separate list from `POWER_USERS` on +# purpose. A trusted tester gets unmetered turns and the AI Chat page, which is +# not a reason to give them the site's traffic numbers. An empty list, which is +# the default, means nobody sees the dashboard in a hosted deployment. +ANALYTICS_EMAILS = { + e.strip().lower() + for e in os.environ.get("AIDND_ANALYTICS_EMAILS", "").split(",") + if e.strip() +} + +DEMO_CAP_MESSAGE = ( + f"You've used all {DEMO_TURNS_PER_DAY} free demo turns for today. " + "Add your own API key in Settings to keep playing (it resets tomorrow)." +) + + +def demo_enabled() -> bool: + # The demo key is a hosted-deployment feature. A local install talks to + # whatever endpoint Settings points at, even with no API key, such as + # Ollama. + return MULTI_USER and bool(DEMO_API_KEY) + + +@dataclass +class ProviderConfig: + """What the turn engine connects with, after the decision between a + user-supplied key and the demo key. + + Build one of these with `resolve_provider_config()`. + """ + + endpoint_url: str + api_key: str + model: str + using_demo: bool + + def __post_init__(self) -> None: + # A second guard around server-funded turns. `resolve_provider_config()` + # already pins the model, and this makes the pin a property of the config + # object too, so a later caller cannot construct an unpinned one. This + # raise is unreachable by design. Reaching it means a new code path + # bypassed the pinning, which is worth failing on rather than billing + # for. + # + # The test is `using_demo`, not `api_key == DEMO_API_KEY`. Keying on the + # key value looks stricter and is wrong. The demo key is an ordinary + # OpenRouter key, so a user can legitimately paste that same key into + # their own Settings. Every resolution then raised, which returned a 500 + # even from `GET /auth/me` and took the whole SPA down. `using_demo` is + # what means the server is paying, and only the demo branch below sets + # it. + if self.using_demo and self.model not in DEMO_MODELS: + raise ValueError( + f"Refusing to use the shared demo key with non-whitelisted model {self.model!r}" + ) + + +def resolve_provider_config( + settings: models.Settings, *, model_override: str | None = None +) -> ProviderConfig: + """Returns the user's own key when they have one, and the shared demo key + otherwise. + + The demo branch is the security-relevant one, and it is the only place the + allowlist rule lives. Every caller has to come through this function rather + than build a `ProviderConfig` itself. On the demo key: + + * The model is pinned to `DEMO_MODELS`, so a caller-supplied override from + the AI Chat page, or a hand-edited Settings row, cannot point a + server-funded key at a paid model. An unrecognized model falls back to + `DEMO_MODELS[0]`. + * The endpoint is pinned to `DEMO_ENDPOINT_URL`, so the key cannot be + redirected to a URL the user controls and captured there. + + `model_override` is a per-request preference and never a grant. It is used + verbatim with the user's own key, and on the demo key only when the model is + on the allowlist. + """ + key = settings.api_key_plain + requested = (model_override or "").strip() or settings.model + if key or not demo_enabled(): + return ProviderConfig(settings.endpoint_url, key, requested, False) + model = requested if requested in DEMO_MODELS else DEMO_MODELS[0] + return ProviderConfig(DEMO_ENDPOINT_URL, DEMO_API_KEY, model, True) + + +def _today() -> str: + return models.utcnow().date().isoformat() + + +def is_power_user(user: models.User) -> bool: + """Returns whether this user is a trusted tester. + + A trusted tester gets unmetered demo turns, plus tooling that is not part of + the game, such as the AI Chat scratchpad. A local install is always trusted, + because it runs on the operator's own machine with their own API key. The + provider debug log is local-only for the same reason. + """ + if not MULTI_USER: + return True + return bool(user.email) and user.email.lower() in POWER_USERS + + +def is_owner(user: models.User) -> bool: + """Returns whether this user may see the visit analytics. + + A local install always may, because it runs on the operator's own machine and + shows their own visits. The provider debug log follows the same reasoning. A + hosted deployment checks `AIDND_ANALYTICS_EMAILS`. + """ + if not MULTI_USER: + return True + return bool(user.email) and user.email.lower() in ANALYTICS_EMAILS + + +def demo_turns_left(user: models.User) -> int: + # A power user is never capped, so report the full cap and let the banner + # read "N of N" rather than count down. + if is_power_user(user): + return DEMO_TURNS_PER_DAY + used = user.demo_turns_used if user.demo_turns_date == _today() else 0 + return max(0, DEMO_TURNS_PER_DAY - used) + + +def count_demo_turn(user: models.User) -> None: + """Records one demo turn. The caller's commit stores it.""" + if is_power_user(user): + return # A power user's turns do not count against the cap. + today = _today() + if user.demo_turns_date != today: + user.demo_turns_date = today + user.demo_turns_used = 0 + user.demo_turns_used += 1 + + +# ---------- User resolution ---------- + +def local_user(db: Session) -> models.User: + """Returns the single implicit user used in local mode. + + A migration gives this user ownership of data written before Phase 8. On a + fresh database the user is created on first use. + """ + user = ( + db.query(models.User) + .filter(models.User.email.is_(None), models.User.is_guest.is_(False)) + .order_by(models.User.id) + .first() + ) + if user is None: + user = models.User(is_guest=False) + db.add(user) + db.commit() + return user + + +def _touch(user: models.User, db: Session) -> None: + now = models.utcnow() + last = user.last_seen_at + if last is not None and last.tzinfo is None: + # SQLite returns DateTime columns without a timezone. They were stored + # as UTC. + last = last.replace(tzinfo=timezone.utc) + if last is None or (now - last).total_seconds() > 3600: + user.last_seen_at = now + db.commit() + + +def resolve_session_user(request: Request, db: Session) -> models.User | None: + token = request.cookies.get(SESSION_COOKIE) + if not token: + return None + user_id = security.verify_session(token) + if user_id is None: + return None + return db.get(models.User, user_id) + + +def get_current_user(request: Request, db: Session = Depends(get_db)) -> models.User: + """The dependency every router uses to resolve the current user. + + In multi-user mode a 401 means the frontend has to establish a session again + through `GET /api/auth/me`. + """ + if not MULTI_USER: + user = local_user(db) + else: + user = resolve_session_user(request, db) + if user is None: + raise HTTPException(401, "No session. Call GET /api/auth/me first.") + _touch(user, db) + return user diff --git a/backend/app/bundle.py b/backend/app/bundle.py new file mode 100644 index 0000000..9381244 --- /dev/null +++ b/backend/app/bundle.py @@ -0,0 +1,735 @@ +"""Phase 14, SP6: the export bundle, as a tree. + +A bundle is the one place a story leaves the database, and the only part of the +tree no migration can reach. A file downloaded today has to still import into a +build shipped next year. The format is therefore versioned, both versions are +defined here, and nothing else in the app knows either of them. + +Version 1 is a flat list of turns, each with an optional `variants` array, which +is the repeating group SP4 unpacked into rows. Nothing writes that shape now. +The reader stays, because bundles already on people's disks still use it, and a +backup that stops importing is not a backup. + +Version 2 carries the tree. It holds three things version 1 could not, and each +one is required: + +* The branches, because a forked adventure is two stories and a flat list holds + one. A version 1 export interleaved them by turn number, which read as a + garbled story rather than as lost data. +* `live`, because a coordinate can hold several attempts at one turn and exactly + one of them is the story. +* Both after-snapshots, because they are what a branch switch and an undo + restore. A bundle carrying the actions but not the outcomes would import a + tree nobody could switch inside. + +## The rule about what a bundle carries + +A bundle carries what was chosen, never what is derived. The head branch, the +fork points, the live flags, and the anchors are decisions somebody made, so +they are in the file. `lineage` and the head depth are computed from those, and +the import recomputes them: + +* `lineage` is a cache of `parent` plus `fork_depth`. Shipping it as well would + put a second source of truth for one fact into a file anyone can hand-edit, + and the two could then disagree without any read reporting it. +* The head depth is the tip of the head branch, which is a fact about the nodes + that arrived with it. + +Every hand-editable coordinate is therefore checked before a row is written, in +`plan`, rather than repaired afterwards. An import that fails partway leaves an +adventure holding half a tree, and a tree missing a branch is a story that stops +without reporting anything. +""" + +from datetime import datetime + +from fastapi import HTTPException +from sqlalchemy import insert, update +from sqlalchemy.orm import Session, undefer + +from . import attempts, models, schemas +from .context import cursors, lineage + +FORMAT = "ai-dnd-adventure-v2" +LEGACY_FORMAT = "ai-dnd-adventure-v1" + +# `actions.type` is VARCHAR(20), and a raw-dict import bypasses the schemas. +TYPE_MAX = 20 + + +# ---------------------------------------------------------------- exporting + +def export(db: Session, adventure: models.Adventure) -> dict: + """Returns the whole adventure as a version 2 bundle. + + The export is not scoped to a path. A backup holds the entire tree, not the + branch its owner is currently reading. Both after-snapshots are undeferred in + the one query, because they are per-node columns nothing else reads in bulk + and requesting them a row at a time would cost one query per turn. + + The bundle carries no context snapshots. It has never carried the assembled + prompts, and it still does not. They run about 163 kB per turn, they explain + a generation rather than form part of the story, and the Insights viewer they + feed reads the adventure they came from. + """ + branches = ( + db.query(models.Branch) + .filter(models.Branch.adventure_id == adventure.id) + .order_by(models.Branch.id) + .all() + ) + # Branch ids are local to the file and are positions in this list, because + # the database ids they hold here are already in use on the importing + # side. + local = {branch.id: i for i, branch in enumerate(branches)} + nodes = ( + db.query(models.Action) + .filter(models.Action.adventure_id == adventure.id) + .options( + undefer(models.Action.state_after), + undefer(models.Action.world_state_after), + ) + .order_by( + models.Action.branch_id, models.Action.depth, models.Action.id, + ) + .all() + ) + return { + "format": FORMAT, + "title": adventure.title, + "memory": adventure.memory, + "authorsNote": adventure.authors_note, + "aiInstructions": adventure.ai_instructions, + "storySummary": adventure.story_summary, + # Phase 18. A bundle written before personas existed has no key here, + # and the import below reads it with `.get`, so it lands with an empty + # persona — which is the same as having none. No FORMAT bump needed. + "persona": { + "name": adventure.persona_name, + "pronouns": adventure.persona_pronouns, + "desc": adventure.persona_desc, + }, + "scriptState": adventure.script_state, + "worldState": adventure.world_state, + "autoSummarize": adventure.auto_summarize, + "memoryBankEnabled": adventure.memory_bank_enabled, + # Write a root entry even for an adventure whose branch row was never + # created. A story with no branch is a pre-tree story, and the tree it + # belongs to is the root. `_local` places its nodes there. + "branches": [_exported_branch(b, local) for b in branches] or [_ROOT], + "headBranch": local.get(adventure.head_branch_id, 0), + "memoryCursor": _exported_anchor(adventure, cursors.MEMORY, local), + "summaryCursor": _exported_anchor(adventure, cursors.SUMMARY, local), + "memories": [_exported_memory(m, local) for m in adventure.memories], + "storyCards": [ + {"type": c.type, "name": c.name, "keys": c.keys, + "entry": c.entry, "notes": c.notes} + for c in adventure.story_cards + ], + "scripts": [ + { + "position": s.position, "enabled": s.enabled, + "name": s.name, "description": s.description, + "library": s.library_js, "input": s.input_js, + "context": s.context_js, "output": s.output_js, + } + for s in adventure.scripts + ], + "actions": [_exported_node(a, local) for a in nodes], + } + + +_ROOT = {"parent": None, "forkDepth": None} + + +def _local(branch_id: int | None, local: dict[int, int]) -> int: + return local.get(branch_id, 0) if branch_id is not None else 0 + + +def _exported_branch(branch: models.Branch, local: dict[int, int]) -> dict: + parent = ( + local.get(branch.parent_branch_id) + if branch.parent_branch_id is not None else None + ) + out = ( + dict(_ROOT) if parent is None + else {"parent": parent, "forkDepth": branch.fork_depth} + ) + # A player chose the name, so it goes into the file. That is the same rule + # that puts the fork points in the file and leaves `lineage` out. An unnamed + # branch omits the key rather than carry a null, which keeps the file for an + # unnamed tree byte-identical to the one SP6 wrote. + if branch.name: + out["name"] = branch.name + return out + + +def _exported_node(action: models.Action, local: dict[int, int]) -> dict: + node = { + "branch": _local(action.branch_id, local), + "depth": action.depth, + "live": bool(action.live), + "type": action.type, + "text": action.text, + "createdAt": action.created_at.isoformat() if action.created_at else None, + } + if action.reasoning: + node["reasoning"] = action.reasoning + # `{}` and an absent key mean different things. `{}` means the node left an + # empty state behind, and an absent key means the state is unknown and the + # live state stays as it is. An empty snapshot is therefore written rather + # than omitted. It costs about eighteen bytes per row, and it decides + # whether an undo clears a score or leaves it in place. + if action.state_after is not None: + node["stateAfter"] = action.state_after + if action.world_state_after is not None: + node["worldStateAfter"] = action.world_state_after + if action.world_delta: + node["worldDelta"] = action.world_delta + return node + + +def _exported_memory(memory: models.Memory, local: dict[int, int]) -> dict: + return { + "text": memory.text, "pinned": memory.pinned, "forgotten": memory.forgotten, + "sourceStart": memory.source_start, "sourceEnd": memory.source_end, + "useCount": memory.use_count, + # The node this memory is attached to. A hand-written memory summarizes + # no node, so it has a branch and no depth, and it keeps that shape + # here. + "branch": _local(memory.branch_id, local), + "depth": memory.depth, + } + + +def _imported_persona(persona) -> dict: + """Reads a bundle's `persona` block into `Adventure` keyword arguments. + + A raw-dict import bypasses the schemas, so the strings are truncated to the + widths the columns declare, in the same way the rest of `_import` does. A + bundle written before Phase 18 has no block at all, and an empty persona is + the same as having none. + """ + if not isinstance(persona, dict): + return {} + return { + "persona_name": str(persona.get("name") or "")[:schemas.PERSONA_NAME_MAX], + "persona_pronouns": + str(persona.get("pronouns") or "")[:schemas.PERSONA_PRONOUNS_MAX], + "persona_desc": str(persona.get("desc") or ""), + } + + +def _exported_anchor( + adventure: models.Adventure, cursor: cursors.Cursor, local: dict[int, int] +) -> dict: + branch_id, depth = cursor.stored(adventure) + return { + "branch": local.get(branch_id) if branch_id is not None else None, + "depth": depth, + } + + +# ---------------------------------------------------------------- importing + +def check_format(bundle: dict) -> str: + """Returns the bundle's version, or raises a 400.""" + fmt = bundle.get("format") + if fmt in (FORMAT, LEGACY_FORMAT): + return fmt + raise HTTPException( + 400, + f"Not an adventure export file (expected format {FORMAT} or {LEGACY_FORMAT}).", + ) + + +def plan(bundle: dict, version: str) -> dict: + """Returns the bundle's tree, checked and normalized, before a row is written. + + The function has no side effects. It opens no session, needs no adventure, + and creates nothing. It catches everything a hand-edited file can get wrong + about the shape of a tree, because the alternative is an import that fails + partway and leaves an adventure holding a story with a gap in it. + + Both versions produce the same shape, so `write` never learns that there are + two formats. A version 1 bundle is a linear story, which is a tree with one + branch, and its `variants` array is a sibling group written the old way. + """ + branches = ( + _planned_branches(bundle) if version == FORMAT else [dict(_ROOT)] + ) + nodes = ( + _planned_nodes(bundle, len(branches)) if version == FORMAT + else _planned_v1_nodes(bundle) + ) + return { + "branches": branches, + "nodes": nodes, + "memories": _planned_memories(bundle, len(branches)), + "head": _as_index(bundle.get("headBranch"), len(branches), default=0), + # Version 2 records where the derived work reached. Version 1 counted + # it, and a count cannot become a node until the nodes exist. See + # `settle`. + "anchors": _planned_anchors(bundle, len(branches)) if version == FORMAT else None, + "positions": None if version == FORMAT else { + "memory": _as_int(bundle.get("memoryCursor"), 0), + "summary": _as_int(bundle.get("summaryCursor"), 0), + }, + } + + +def _planned_branches(bundle: dict) -> list[dict]: + raw = bundle.get("branches") + entries = [b for b in raw if isinstance(b, dict)] if isinstance(raw, list) else [] + if not entries: + return [dict(_ROOT)] + specs: list[dict] = [] + for i, entry in enumerate(entries): + parent = entry.get("parent") + name = _planned_branch_name(entry, i) + if parent is None: + specs.append(dict(_ROOT, **({"name": name} if name else {}))) + continue + # A branch may fork only from a branch listed before it. The export + # writes them that way, because branches are numbered in creation order + # and a parent always exists first. Requiring it here guarantees the + # graph is acyclic for the cost of one comparison. A lineage is computed + # by walking to the parent, so a cycle would be an import that never + # returns. + if not _is_int(parent) or not 0 <= parent < i: + raise HTTPException( + 400, + f"Branch {i} forks from branch {parent!r}, which is not one of " + f"the {i} branches listed before it.", + ) + fork_depth = entry.get("forkDepth") + if not _is_int(fork_depth): + raise HTTPException( + 400, + f"Branch {i} forks from branch {parent} but does not say at " + f"what depth.", + ) + specs.append({ + "parent": parent, "forkDepth": fork_depth, + **({"name": name} if name else {}), + }) + return specs + + +def _planned_branch_name(entry: dict, i: int) -> str | None: + """Returns the name a branch entry carries, or `None` if nobody named it. + + The check runs before the row is created rather than being left to the + column, for the reason the planner exists. A 400 from a function with no side + effects is better than a half-written adventure and a database error three + branches in. + """ + raw = entry.get("name") + if raw is None: + return None + if not isinstance(raw, str): + raise HTTPException(400, f"Branch {i} has a name that is not text.") + name = raw.strip() + if len(name) > schemas.BRANCH_NAME_MAX: + raise HTTPException( + 400, + f"Branch {i}'s name is longer than {schemas.BRANCH_NAME_MAX} " + f"characters.", + ) + return name or None + + +def _planned_nodes(bundle: dict, branches: int) -> list[dict]: + raw = bundle.get("actions") + nodes: list[dict] = [] + for entry in raw if isinstance(raw, list) else []: + if not isinstance(entry, dict) or not str(entry.get("text") or ""): + continue + branch = entry.get("branch", 0) + if not _is_int(branch) or not 0 <= branch < branches: + raise HTTPException( + 400, + f"An action names branch {branch!r}, but the file lists " + f"{branches}.", + ) + depth = entry.get("depth") + if not _is_int(depth) or depth < 0: + raise HTTPException( + 400, f"An action on branch {branch} has no depth to sit at." + ) + nodes.append({ + "branch": branch, + "depth": depth, + "live": bool(entry.get("live", True)), + "type": str(entry.get("type") or "story")[:TYPE_MAX], + "text": str(entry.get("text") or ""), + "reasoning": _as_text(entry.get("reasoning")), + "stateAfter": _as_dict(entry.get("stateAfter")), + "worldStateAfter": _as_dict(entry.get("worldStateAfter")), + "worldDelta": _as_dict(entry.get("worldDelta")), + "createdAt": _as_time(entry.get("createdAt")), + }) + return nodes + + +def _planned_v1_nodes(bundle: dict) -> list[dict]: + """Returns a version 1 bundle's turns as nodes, one node per attempt. + + The `variants` array is the repeating group SP4 unpacked, so reading one + performs the same split that migration 60 does. Every attempt becomes a row + at the turn's coordinate, and `variantIndex` selects the live one. The index + is clamped, because a hand-edited bundle can name an attempt its own list + does not contain, and a turn with no live node is a turn no read can see. + + The depth is the bundle's `index`. A version 1 bundle has one branch, where + the two numbers agree. + """ + raw = bundle.get("actions") + nodes: list[dict] = [] + for i, entry in enumerate(raw if isinstance(raw, list) else []): + if not isinstance(entry, dict) or not str(entry.get("text") or ""): + continue + depth = _as_int(entry.get("index"), i) + kind = str(entry.get("type") or "story")[:TYPE_MAX] + variants = [v for v in (entry.get("variants") or []) if isinstance(v, dict)] + if not variants: + variants = [{"text": entry["text"], "reasoning": entry.get("reasoning")}] + live = min(max(_as_int(entry.get("variantIndex"), 0), 0), len(variants) - 1) + for n, variant in enumerate(variants): + text = str(variant.get("text") or "") + if not text: + continue + nodes.append({ + "branch": 0, + "depth": max(depth, 0), + "live": n == live, + "type": kind, + "text": text, + "reasoning": _as_text(variant.get("reasoning")), + "stateAfter": None, + "worldStateAfter": None, + "worldDelta": None, + "createdAt": _as_time(variant.get("createdAt")), + }) + return nodes + + +def _planned_memories(bundle: dict, branches: int) -> list[dict]: + raw = bundle.get("memories") + out: list[dict] = [] + for entry in raw if isinstance(raw, list) else []: + if not isinstance(entry, dict) or not str(entry.get("text") or "").strip(): + continue + out.append({ + "text": str(entry["text"]), + "pinned": bool(entry.get("pinned", False)), + "forgotten": bool(entry.get("forgotten", False)), + "sourceStart": entry.get("sourceStart"), + "sourceEnd": entry.get("sourceEnd"), + "useCount": _as_int(entry.get("useCount"), 0), + # A value that is out of range, rather than absent, means the file + # disagrees with itself. The root is the safe reading, because a + # memory on a branch nothing can see never reaches a prompt + # again. + "branch": _as_index(entry.get("branch"), branches, default=0), + "depth": entry.get("depth") if _is_int(entry.get("depth")) else None, + }) + return out + + +def _planned_anchors(bundle: dict, branches: int) -> dict: + anchors = {} + for name in ("memory", "summary"): + raw = bundle.get(f"{name}Cursor") + raw = raw if isinstance(raw, dict) else {} + branch = raw.get("branch") + anchors[name] = ( + _as_index(branch, branches) if branch is not None else None, + _as_int(raw.get("depth"), lineage.NO_DEPTH), + ) + return anchors + + +# ------------------------------------------------------------------ writing + +def write(db: Session, adventure: models.Adventure, story: dict) -> None: + """Writes a planned tree onto a newly created adventure. + + The order is fixed. Branches come first, because a node needs a branch id. + The nodes come next, because the head and the anchors name a node. + """ + ids = _write_branches(db, adventure, story["branches"]) + _write_nodes(db, adventure, story["nodes"], ids) + _write_memories(db, adventure, story["memories"], ids) + _point_the_head(adventure, story, ids) + _write_anchors(adventure, story, ids) + + +def _write_branches( + db: Session, adventure: models.Adventure, specs: list[dict] +) -> list[int]: + """Writes one row per branch, computing the lineage rather than reading it. + + The rows are inserted through Core and the lineage is written second, for the + reason `tree.root_branch` gives: the lineage names the row's own id. The + parent's cached ancestry is capped at this fork, which is the arithmetic + `tree.fork` performs. A fork made now and a fork made a year ago and then + exported have to produce the same rows. + """ + ids: list[int] = [] + lineages: list[list[list]] = [] + for spec in specs: + parent = spec.get("parent") + fork_depth = spec.get("forkDepth") + new_id = db.execute( + insert(models.Branch).values( + adventure_id=adventure.id, + parent_branch_id=ids[parent] if parent is not None else None, + fork_depth=fork_depth if parent is not None else None, + lineage=[], + name=spec.get("name"), + created_at=models.utcnow(), + ) + ).inserted_primary_key[0] + entries = [[new_id, None]] + if parent is not None: + entries += [ + [branch_id, fork_depth if cap is None else min(cap, fork_depth)] + for branch_id, cap in lineages[parent] + ] + db.execute( + update(models.Branch) + .where(models.Branch.id == new_id) + .values(lineage=entries) + ) + ids.append(new_id) + lineages.append(entries) + return ids + + +def _write_nodes( + db: Session, adventure: models.Adventure, specs: list[dict], ids: list[int] +) -> None: + """Writes the nodes, grouped into the turns they are attempts at. + + One value is decided here rather than read from the file. Exactly one + attempt in each group is made live, because a file can name none or several, + and a turn with no live node disappears from the story. + """ + groups: dict[tuple[int, int], list[models.Action]] = {} + for spec in specs: + key = (spec["branch"], spec["depth"]) + action = models.Action( + adventure_id=adventure.id, + branch_id=ids[spec["branch"]], + depth=spec["depth"], + type=spec["type"], + text=spec["text"], + reasoning=spec["reasoning"], + live=spec["live"], + state_after=spec["stateAfter"], + world_state_after=spec["worldStateAfter"], + world_delta=spec["worldDelta"], + ) + if spec["createdAt"] is not None: + action.created_at = spec["createdAt"] + db.add(action) + groups.setdefault(key, []).append(action) + + for rows in groups.values(): + live = next((row for row in rows if row.live), rows[0]) + for row in rows: + row.live = row is live + + +def _write_memories( + db: Session, adventure: models.Adventure, specs: list[dict], ids: list[int] +) -> None: + for spec in specs: + memory = models.Memory( + adventure_id=adventure.id, + text=spec["text"], + pinned=spec["pinned"], + forgotten=spec["forgotten"], + source_start=spec["sourceStart"], + source_end=spec["sourceEnd"], + use_count=spec["useCount"], + branch_id=ids[spec["branch"]], + depth=spec["depth"], + ) + # A version 1 memory has no depth of its own. `source_end` is the index + # of the last action it summarizes, which on one branch is that node's + # depth. + if memory.depth is None and memory.source_end is not None: + memory.depth = memory.source_end + # A version 1 memory that summarizes nothing, which means one the player + # typed, has no depth to derive. Leaving it NULL here would recreate on + # import the state migration 62 exists to end. `Path._entry_clause` + # compares `depth <= max_depth`, which a NULL fails, so the memory would + # disappear from every branch as soon as the imported adventure was + # forked. The root is the answer the migration gives, for the same + # reason: 0 is at or before every fork point, so the memory is visible + # from every path this adventure can grow. + if memory.depth is None: + memory.depth = lineage.ROOT_DEPTH + db.add(memory) + + +def _point_the_head( + adventure: models.Adventure, story: dict, ids: list[int] +) -> None: + """Sets which branch the story is played on, and how deep it goes. + + The branch comes from the file and the depth does not. The tip of a branch is + whatever arrived on it, and a branch with no nodes of its own sits at its + fork point, which is the last node its story contains. That node is borrowed + but it is still the tip. This is the rule `tree.refresh_head` applies, run + here before a flush. + """ + head = story["head"] + adventure.head_branch_id = ids[head] + depths = [n["depth"] for n in story["nodes"] if n["branch"] == head] + if depths: + adventure.head_depth = max(depths) + return + fork_depth = story["branches"][head].get("forkDepth") + adventure.head_depth = fork_depth if fork_depth is not None else lineage.NO_DEPTH + + +def _write_anchors( + adventure: models.Adventure, story: dict, ids: list[int] +) -> None: + """Records how far the memories and the summary have read. Version 2 only. + + A version 1 bundle stores a count instead, and a count cannot be resolved to + a node until the nodes are in the database. `settle` does that part + afterwards. + """ + anchors = story["anchors"] + if anchors is None: + return + for cursor in cursors.ALL: + branch, depth = anchors[cursor.name] + cursor.anchor(adventure, ids[branch] if branch is not None else None, depth) + + +def settle(adventure: models.Adventure, story: dict) -> None: + """Resolves a version 1 bundle's counts into anchors, once the nodes exist. + + A version 1 file records how far the memories and the summary have read as + a count, so the anchor is found by counting that far along the story. A + version 2 file carries the anchor itself, which `_write_anchors` has + already stored, so there is nothing left to do. + + The caller runs this after the flush, because counting needs the actions to + be queryable. + """ + positions = story["positions"] + if positions is None: + return + for cursor in cursors.ALL: + cursors.anchor_at_position(adventure, cursor, positions[cursor.name]) + + +def materialize( + db: Session, payload: dict, story: dict, user_id: int | None +) -> models.Adventure: + """Writes a planned bundle into a new adventure owned by `user_id`. + + Call `check_format` and `plan` first, and apply any rate or size limits + before calling this. The split exists because those checks belong to the + import endpoint alone: the guest starter writes a file the server ships, so + it has nothing to rate-limit and no untrusted list to cap. + + The caller commits. Two flushes happen here, because the anchors and the + legacy counts describe one boundary in two coordinate systems, and aligning + them needs the actions to be queryable. + """ + # A raw-dict import bypasses the schemas, so truncate strings bound for + # VARCHAR columns. Postgres enforces the widths. See `schemas.py`. + adventure = models.Adventure( + user_id=user_id, + title=str(payload.get("title") or "Imported Adventure")[:schemas.NAME_MAX], + memory=str(payload.get("memory") or ""), + authors_note=str(payload.get("authorsNote") or ""), + ai_instructions=str(payload.get("aiInstructions") or ""), + story_summary=str(payload.get("storySummary") or ""), + script_state=payload.get("scriptState") or {}, + world_state=payload.get("worldState") or {}, + auto_summarize=bool(payload.get("autoSummarize", False)), + memory_bank_enabled=bool(payload.get("memoryBankEnabled", False)), + **_imported_persona(payload.get("persona")), + ) + db.add(adventure) + db.flush() + + for card in payload.get("storyCards") or []: + if isinstance(card, dict): + db.add(models.StoryCard( + adventure_id=adventure.id, + type=str(card.get("type") or "")[:schemas.CARD_TYPE_MAX], + name=str(card.get("name") or "")[:schemas.NAME_MAX], + keys=str(card.get("keys") or ""), + entry=str(card.get("entry") or ""), + notes=str(card.get("notes") or ""), + )) + + for i, item in enumerate(payload.get("scripts") or []): + if isinstance(item, dict): + db.add(models.AdventureScript( + adventure_id=adventure.id, + position=int(item.get("position", i)), + enabled=bool(item.get("enabled", True)), + name=str(item.get("name") or "Imported Script")[:schemas.NAME_MAX], + description=str(item.get("description") or ""), + library_js=str(item.get("library") or ""), + input_js=str(item.get("input") or ""), + context_js=str(item.get("context") or ""), + output_js=str(item.get("output") or ""), + )) + + write(db, adventure, story) + db.flush() + db.expire(adventure, ["actions"]) + settle(adventure, story) + return adventure + +# ------------------------------------------------------------------ reading +# Small coercions. A raw-dict import bypasses the schemas, so every value from a +# bundle is whatever the JSON held. + +def _is_int(value) -> bool: + """Returns whether `value` is an integer. Python counts `True` as an `int`, + and a coordinate does not.""" + return isinstance(value, int) and not isinstance(value, bool) + + +def _as_int(value, default: int) -> int: + return value if _is_int(value) else default + + +def _as_index(value, count: int, default: int | None = None) -> int | None: + """Returns a local branch number, or `default` when the file names a branch + that is not present. + + An out-of-range value means the file disagrees with itself, and it is not a + value any read can be given. + """ + return value if _is_int(value) and 0 <= value < count else default + + +def _as_text(value) -> str | None: + return str(value) if value else None + + +def _as_dict(value) -> dict | None: + return value if isinstance(value, dict) else None + + +def _as_time(value) -> datetime | None: + if not isinstance(value, str): + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None diff --git a/backend/app/cleanup.py b/backend/app/cleanup.py new file mode 100644 index 0000000..b727fd8 --- /dev/null +++ b/backend/app/cleanup.py @@ -0,0 +1,178 @@ +"""Retention policy for throwaway guest accounts. + +In multi-user mode every first visit creates a `users` row through +`GET /api/auth/me`, so a public demo accumulates one account per visitor. Most +of those visitors never return, and each one leaves behind whatever scenarios, +adventures, actions, and memories they generated. This module deletes guests +that have been inactive for `AIDND_GUEST_RETENTION_DAYS`, which defaults to 5, +along with everything they made. + +Why this is safe to run unattended: + +- Only rows with `is_guest` AND `email IS NULL` are ever touched, and both + clauses are checked rather than either alone. Registering upgrades the row + in place (is_guest -> False), so a guest who signs up keeps everything; + local mode's implicit single user is also is_guest=False. +- Idle time is `COALESCE(last_seen_at, created_at)`. `auth._touch` writes + `last_seen_at` at most once an hour, and a guest created by `/auth/me` has + NULL there until its second request, so `created_at` is the correct floor for + a new visitor. Without the coalesce, those rows look arbitrarily old. +- Nothing a guest owns is reachable by anyone else. `is_public` is an + output-only field, as `schemas.ScenarioBase` shows, so the only shared + scenarios are the seeded ones, which have a NULL `user_id` and are outside + this filter. Deleting a guest cannot remove content from another user. + +The sweep uses one Core DELETE rather than an ORM cascade. `db.delete(user)` +would SELECT every adventure, action, memory, and story card into Python only to +delete them, which on Neon is the egress pattern that has already cost this +project once. Every foreign key from `users` downward is ON DELETE CASCADE, from +users to scenarios, adventures, scripts, and settings, and from those to actions, +memories, and cards, so the database deletes the whole graph in one statement and +returns a row count. + +The scan gets no index. The sweep runs a few times a day against a table holding +at most a few thousand rows, which does not justify a migration and the schema +surface it adds. +""" + +import asyncio +import logging +import os +from datetime import datetime, timedelta + +from sqlalchemy import delete, func +from sqlalchemy.orm import Session +from starlette.concurrency import run_in_threadpool + +from . import analytics, auth, models +from .database import SessionLocal + +logger = logging.getLogger(__name__) + + +def _int_env(name: str, default: int) -> int: + try: + return int(os.environ.get(name, "").strip() or default) + except ValueError: + logger.warning("%s is not an integer; using %d.", name, default) + return default + + +# Days of inactivity before a guest account is deleted. A value of 0 or less +# disables the policy, for a deployment that keeps everything. +RETENTION_DAYS = _int_env("AIDND_GUEST_RETENTION_DAYS", 5) + +# How often a long-lived process re-checks. Hours, not minutes: nothing here is +# time-critical, and on Render's free tier the service sleeps and cold-starts +# often enough that the startup sweep does most of the work by itself. +SWEEP_INTERVAL_SECONDS = _int_env("AIDND_CLEANUP_INTERVAL_HOURS", 6) * 3600 + + +def enabled() -> bool: + """Guests only exist in multi-user mode, so local runs skip the sweep + rather than pointing a DELETE at a database that has nothing to collect.""" + return auth.MULTI_USER and RETENTION_DAYS > 0 + + +def anything_to_sweep() -> bool: + """Whether the periodic task is worth starting at all. The two jobs it runs + are independent: a deployment can keep every guest forever and still want + its analytics rows aged out, and vice versa.""" + return enabled() or analytics.RETENTION_DAYS > 0 + + +def delete_stale_guests(db: Session, *, now: datetime | None = None) -> int: + """Delete guests idle for RETENTION_DAYS or more. Returns the row count. + + The caller owns error handling; `sweep` is the safe wrapper. + """ + if RETENTION_DAYS <= 0: + return 0 + # Stored timestamps are UTC without a timezone on both backends. SQLite + # drops the timezone, and the Postgres columns are TIMESTAMP WITHOUT TIME + # ZONE with the session pinned to UTC in `database.py`. Match that, so the + # comparison does not depend on how a dialect renders a value that carries a + # timezone. + reference = now or models.utcnow() + cutoff = reference.replace(tzinfo=None) - timedelta(days=RETENTION_DAYS) + + stmt = ( + delete(models.User) + .where( + models.User.is_guest.is_(True), + models.User.email.is_(None), + func.coalesce(models.User.last_seen_at, models.User.created_at) < cutoff, + ) + # Without this option, the "auto" strategy cannot evaluate coalesce in + # Python and falls back to fetching every matching primary key first. + # That is a second round trip for no benefit, because this session holds + # no User objects to synchronize. + .execution_options(synchronize_session=False) + ) + removed = db.execute(stmt).rowcount or 0 + db.commit() + return removed + + +def sweep() -> int: + """One pass, with its own session. Never raises: a failed cleanup must not + be able to take the app down (same rule as seeding). Returns the guest + count, which is the number worth logging about.""" + if not anything_to_sweep(): + return 0 + db = SessionLocal() + try: + # Ages out the per-visitor analytics rows, on its own terms: it is not + # about guests, and it must still happen on a deployment that has + # chosen to keep every account it ever minted. + aged = analytics.purge_old_visitor_days(db) + if aged: + logger.info("Aged out %d analytics visitor-day row(s).", aged) + removed = delete_stale_guests(db) if enabled() else 0 + if removed: + logger.info( + "Cleaned up %d guest account(s) idle for %d+ days.", + removed, + RETENTION_DAYS, + ) + return removed + except Exception: + db.rollback() + logger.exception("Guest cleanup failed; continuing without it.") + return 0 + finally: + db.close() + + +async def _sweep_loop() -> None: + while True: + # Blocking DB work: keep it off the event loop, which is also serving + # SSE turn streams. + await run_in_threadpool(sweep) + await asyncio.sleep(SWEEP_INTERVAL_SECONDS) + + +def start_sweeper() -> asyncio.Task | None: + """Kick off the periodic sweep; None when there is nothing to sweep.""" + if not enabled(): + logger.info("Guest cleanup disabled (multi_user=%s, retention_days=%d).", + auth.MULTI_USER, RETENTION_DAYS) + else: + logger.info( + "Guest cleanup on: deleting guests idle %d+ days, every %d hour(s).", + RETENTION_DAYS, + SWEEP_INTERVAL_SECONDS // 3600, + ) + if not anything_to_sweep(): + return None + return asyncio.create_task(_sweep_loop()) + + +async def stop_sweeper(task: asyncio.Task | None) -> None: + if task is None: + return + task.cancel() + try: + await task + except asyncio.CancelledError: + pass diff --git a/backend/app/compression.py b/backend/app/compression.py new file mode 100644 index 0000000..e33c71d --- /dev/null +++ b/backend/app/compression.py @@ -0,0 +1,70 @@ +"""Storing a JSON column compressed. + +`actions.context_snapshot` holds the entire assembled prompt for a turn. It is +89% of the database, which is 150.8 MB of JSON across 944 actions in production +and 232 kB per row on the longest adventure, and the free tier this deploys to +allows 512 MB. Reads are not the problem. The column is deferred, so a page load +never touches it and exactly one endpoint fetches one row of it at a time. +Storage is the problem, and storage has a hard limit. + +Postgres already compresses the column. TOAST brings 150.8 MB down to about +89 MB, a factor of 1.7. Postgres chooses pglz for decompression speed on data a +query might filter on, and no query filters on this column. A prompt is written +once and read whole, occasionally, by one screen. zlib at the application layer +reaches three to four times on the same text, and the cost is one decompression +on a request that already makes an LLM call. + +Doing it as a TypeDecorator rather than a second column keeps every call site +writing `action.context_snapshot = {...}` and reading a dict back, and keeps +`deferred=True`, `undefer()` and `load_only()` naming the same attribute they +named before. The storage format changes; nothing else does. + +Level 6 is zlib's default and the knee of the curve here: 9 spends noticeably +more CPU on prompt text for about a percent more space. +""" +from __future__ import annotations + +import json +import zlib + +from sqlalchemy import LargeBinary +from sqlalchemy.types import TypeDecorator + +LEVEL = 6 + + +def pack(value) -> bytes: + """A JSON-able value as compressed UTF-8.""" + raw = json.dumps(value, separators=(",", ":"), default=str).encode("utf-8") + return zlib.compress(raw, LEVEL) + + +def unpack(blob: bytes) -> object: + """The value `pack` was given.""" + return json.loads(zlib.decompress(bytes(blob)).decode("utf-8")) + + +class CompressedJSON(TypeDecorator): + """A JSON column stored as zlib-compressed UTF-8 in a BLOB/BYTEA. + + `cache_ok = True`: the type carries no per-instance configuration, so + SQLAlchemy may reuse a compiled statement across instances of it. + """ + + impl = LargeBinary + cache_ok = True + + def process_bind_param(self, value, dialect): + return None if value is None else pack(value) + + def process_result_value(self, value, dialect): + # Tolerate a row the backfill has not reached yet, or one written + # before the conversion: a snapshot that cannot be read back is worth + # less than the screen that shows it, and never worth a 500 on the + # turn that happens to load it. + if value is None: + return None + try: + return unpack(value) + except (zlib.error, UnicodeDecodeError, ValueError): + return None diff --git a/backend/app/context/__init__.py b/backend/app/context/__init__.py new file mode 100644 index 0000000..cd3be11 --- /dev/null +++ b/backend/app/context/__init__.py @@ -0,0 +1,19 @@ +from . import history +from .builder import ( + build_context, + count_tokens, + match_cards, + render_persona, + truncate_to_last_tokens, +) +from .history import story_actions + +__all__ = [ + "build_context", + "count_tokens", + "history", + "match_cards", + "render_persona", + "story_actions", + "truncate_to_last_tokens", +] diff --git a/backend/app/context/builder.py b/backend/app/context/builder.py new file mode 100644 index 0000000..98afc63 --- /dev/null +++ b/backend/app/context/builder.py @@ -0,0 +1,473 @@ +"""Context assembly per AI Dungeon's memory system +(help.aidungeon.com/faq/the-memory-system): + + [AI Instructions] always included + [Player Character] always included when the adventure has a persona + [Plot Essentials] always included (classic "Memory") + [Story Summary] always included (manual in Phase 3, auto in Phase 6) + [Used Memories] top-K memory-bank retrievals (Phase 6, when enabled) + [Triggered Story Cards] "World Lore: ", conditional; first dropped when over budget + [Story history] newest actions that fit the remaining token budget + [Author's Note] injected AUTHORS_NOTE_DEPTH actions before the end of history + [Latest player action] (+ script frontMemory right after it, Phase 4) + +The list above comes from AI Dungeon's design. The order does not. This module +emits every fixed section first and every changing section after the history, +because prompt caching bills on a shared prefix. A section that changes near the +top of the prompt re-prices everything below it. See the comments on the static +block and the live sections in `build_context`. +""" + +import functools +from dataclasses import dataclass + +import tiktoken + +from .. import models, worldstate +from . import history + +AUTHORS_NOTE_DEPTH = 3 # actions from the end of history +CARD_BUDGET_SHARE = 0.4 # max share of non-reserved budget that story cards may take +NPC_WINDOW = 6 # actions of story searched for NPC trigger words ("in scene") +SEPARATOR = "\n\n" + +# Output-length guidance. The endpoint enforces `max_output_tokens` as a hard +# limit, and it truncates the reply mid-sentence when the model reaches it. The +# state block is emitted last, so truncation removes it. Asking the model to +# finish inside the limit prevents the truncation. +LENGTH_HEADROOM = 50 # Tokens reserved from the cap for the state block. +# Models cannot count their own tokens, but they do follow a word budget, so the +# hint states a number of words. English prose averages 0.75 words per token. +WORDS_PER_TOKEN = 0.75 +# Models regularly exceed a word budget, and the cap it protects is a hard +# limit. Aiming 10% below the real ceiling leaves room for that overshoot, so it +# does not consume the state block. +LENGTH_BUFFER = 0.90 +MIN_LENGTH_HINT_WORDS = 40 # Below this, the hint adds nothing useful. +# A ceiling on its own gives one-sided guidance, and models respond to it +# differently. A verbose model treats it as a limit. A terse model has only the +# instruction to write as much as the moment needs, and it produces two +# paragraphs. Adding a floor turns the guidance into a range, so the same prompt +# produces a similar length from either model. The floor is a share of the +# ceiling so that it can never approach the ceiling. +LENGTH_FLOOR_SHARE = 0.35 +# Below this word count, a floor means nothing, because a short turn is the +# correct turn at a tight cap. The wording used at a tight cap is also the +# wording that was measured to preserve the state block, so it is unchanged. +MIN_LENGTH_FLOOR_WORDS = 60 +# The floor prevents a collapse to two paragraphs. It does not ask for an essay. +# At a 2400-token cap, the share alone would request a minimum of 555 words. A +# reader who wants longer turns can ask for them in the author's note. +MAX_LENGTH_FLOOR_WORDS = 300 + + +@functools.lru_cache(maxsize=1) +def _encoding() -> tiktoken.Encoding: + return tiktoken.get_encoding("cl100k_base") + + +def count_tokens(text: str) -> int: + return len(_encoding().encode(text)) + + +def truncate_to_last_tokens(text: str, budget: int) -> str: + tokens = _encoding().encode(text) + if len(tokens) <= budget: + return text + return _encoding().decode(tokens[-budget:]) + + +@dataclass +class Section: + label: str + text: str + + @property + def tokens(self) -> int: + return count_tokens(self.text) + + +def length_hint(max_output_tokens: int, *, has_ws: bool) -> str: + """Ask for a turn that fits inside the output cap, stated as a word budget. + + Returns an empty string when the cap is too small to state usefully. The + model can exceed the hint, so the hint earns its tokens only when there is + enough room for that overshoot to stay inside the cap. + """ + words = int((max_output_tokens - LENGTH_HEADROOM) * WORDS_PER_TOKEN * LENGTH_BUFFER) + if words < MIN_LENGTH_HINT_WORDS: + return "" + tail = ( + " Finish the narration and append the state block well inside the limit." + if has_ws + else " Bring the turn to a close well inside the limit rather than " + "stopping mid-sentence." + ) + # State the number as a ceiling, never as a budget. In measurements, the + # wording "keep this turn under about N words" read to the model as a target + # to fill. It raised the average from 174 words to 246 across five runs, and + # every hinted run was longer than every unhinted run. The hint therefore + # pushed turns toward the limit it exists to avoid. Naming the number as a + # limit, and adding that a typical turn is much shorter, held the average at + # 170 while still preserving the state block at tight caps. + floor = min(int(words * LENGTH_FLOOR_SHARE), MAX_LENGTH_FLOOR_WORDS) + if floor < MIN_LENGTH_FLOOR_WORDS: + return ( + f"[Hard limit: this turn must not exceed {words} words. Write only as " + f"much as the moment needs — a typical turn is much shorter.{tail}]" + ) + # Both numbers are bounds, and the wording is deliberately asymmetric. The + # ceiling uses "must not exceed", because the endpoint enforces it. The floor + # uses "should not stop short of". Neither reads as a target, which the + # measurement above shows is what matters. The clause that asks the model to + # prefer the lower end does the job the earlier wording did, which was to + # keep a verbose model away from the ceiling. It now has a number beneath it, + # so a terse model reading the same clause stops at the floor rather than at + # forty words. + return ( + f"[Hard limit: this turn must not exceed {words} words, and it should not " + f"stop short of about {floor}. Prefer the lower end of that range unless " + f"the scene genuinely needs more.{tail}]" + ) + + +def render_persona(adventure: models.Adventure) -> str: + """Returns the Player Character section, or "" when there is no persona. + + The three fields are independent. A name alone is enough, a description + alone is enough, and the wording holds together for either. Pronouns are + stated because the summarizer in `memorybank` writes about the protagonist + in the third person, and a model that has to infer a pronoun from a name + will sometimes infer wrongly and then repeat that error in every memory it + writes. + """ + name = adventure.persona_name.strip() + pronouns = adventure.persona_pronouns.strip() + desc = adventure.persona_desc.strip() + if not (name or desc): + return "" + head = f"You are {name}" if name else "" + if head and pronouns: + head += f" ({pronouns})" + # Joined with a space, not `SEPARATOR`: this is one short paragraph about + # one character, and a blank line inside it reads as two unrelated notes. + body = " ".join(part for part in (f"{head}." if head else "", desc) if part) + return f"Player character:\n{body}" + + +def _script_memory(adventure: models.Adventure) -> dict: + """Script-provided memory overrides (populated by Phase 4 scripting).""" + state = adventure.script_state if isinstance(adventure.script_state, dict) else {} + memory = state.get("memory") + return memory if isinstance(memory, dict) else {} + + +def _history_text(action: models.Action) -> str: + """Returns an AI turn as the model should see it in replayed history. + + The result is the narration with its state block appended again, + reconstructed from the stored delta. The app strips that block before + storing and displaying the turn. Without this function, every past AI turn + would appear to have emitted no state, and the model would copy that pattern + and stop emitting state itself. Player turns and turns with no block pass + through unchanged. + + The block replays the changes the engine ACCEPTED, not the ones the model + sent. Replaying what was sent showed the model a refused change standing as + though it had been applied, while the live values in the same prompt + disagreed with it. Nothing marked which of the two was true, so the model + read its own refused change as correct and sent it again. + + This function reads `world_delta` rather than `context_snapshot`. It runs + for every action in the replayed history, and `context_snapshot` is deferred + so that a turn never loads the prompt archive from the database. + """ + text = action.text + wd = action.world_delta if isinstance(action.world_delta, dict) else None + if wd: + block = worldstate.render_delta_block(worldstate.applied_delta(wd)) + if block: + text = f"{text}\n{block}" + return text + + +def _visible_npcs(actions: list[models.Action], stat_schema: dict) -> dict[str, str]: + """Returns the NPCs whose trigger words appear in the recent story. + + These are the NPCs in scene, and the prompt includes stats for them only. + The result maps an NPC id to its display name. + + `actions` holds only the most recent actions. See `NPC_WINDOW`. + """ + recent = SEPARATOR.join(a.text for a in actions).lower() + visible: dict[str, str] = {} + for npc_key, ndef in (stat_schema.get("npcs") or {}).items(): + if not isinstance(ndef, dict): + continue + if any(trigger in recent for trigger in worldstate.npc_triggers(ndef, npc_key)): + visible[npc_key] = worldstate.npc_name(ndef, npc_key) + return visible + + +def match_cards(cards: list[models.StoryCard], window_text: str) -> list[dict]: + """Returns one record per matched story card, naming the keyword that matched. + + Matching follows AI Dungeon's rules. It ignores case, respects spaces, and + matches partial words, so "boat" matches "boats". + + Public since Phase 18b: `memorybank.cast_brief` runs the same rule over the + block it is about to summarize, so that the summarizer is told who the + characters in that stretch of story are. One rule, one implementation. + """ + haystack = window_text.lower() + matched = [] + for card in cards: + for key in (k.strip().lower() for k in card.keys.split(",")): + if key and key in haystack: + matched.append( + {"id": card.id, "name": card.name, "keyword": key, "entry": card.entry} + ) + break + return matched + + +def build_context( + adventure: models.Adventure, + settings: models.Settings, + memory_bank: dict | None = None, + exclude_action_id: int | None = None, +) -> tuple[str, str, dict]: + """Returns (system_text, story_text, context_report). `memory_bank` is the + result of memorybank.retrieve_memories (None when the bank is off); + `exclude_action_id` omits one action from the story (see history.py).""" + script_mem = _script_memory(adventure) + + # ----- The static block, which is identical on every turn ----- + # This ordering exists to reduce cost. Prompt caching matches a prefix. The + # endpoint reuses the prompt up to the first byte that differs from the + # previous request, and no further. A section that changes near the top + # therefore re-prices everything below it, and what sits below it is the + # story history, which is most of the prompt. Sections that change from turn + # to turn go after the history, among the live sections. Placing them there + # also gives them the most recency, which is why `EMIT_REMINDER` goes last. + system_sections: list[Section] = [Section("narrator", settings.narrator_prompt.strip())] + + # RPG world state (Phase 12): the instructions for reporting changes. The + # live values go into a live section below. The guide derived from the + # schema and the emit rule do not change while the scenario is unchanged. + stat_schema = adventure.scenario.stat_schema if adventure.scenario else None + has_ws = worldstate.has_schema(stat_schema) + persona_name = adventure.persona_name.strip() + if has_ws: + guide = worldstate.render_reference(stat_schema, persona_name) + if guide: + system_sections.append(Section("world_state_guide", guide)) + system_sections.append(Section("world_state_rule", worldstate.EMIT_RULE)) + + if isinstance(script_mem.get("context"), str) and script_mem["context"].strip(): + system_sections.append(Section("script_context", script_mem["context"].strip())) + if adventure.ai_instructions.strip(): + system_sections.append(Section("ai_instructions", adventure.ai_instructions.strip())) + # Phase 18. This sits in the static block because only the user can edit it, + # so it never changes mid-story and stays inside the cached prefix. It is + # emitted whether or not the adventure has an RPG layer: an adventure with + # no stats still has a protagonist, and that is the case the persona was + # added for. + persona_text = render_persona(adventure) + if persona_text: + system_sections.append(Section("persona", persona_text)) + if adventure.memory.strip(): + system_sections.append( + Section("plot_essentials", f"Plot essentials:\n{adventure.memory.strip()}") + ) + + # ----- Live sections, which hold everything that changes ----- + # This code builds them here and places them after the history further down. + # They are ordered from least to most volatile, so a turn that changes only + # the fastest-moving section leaves the others cached. The summary is + # rewritten every few turns. Lore changes with the scene. The retrieved + # memories change on most turns, and the stat values change on nearly every + # turn. `world_lore` is added below, because the history window determines + # which cards trigger and that window is not known yet. + summary_section = ( + Section("story_summary", f"Story summary:\n{adventure.story_summary.strip()}") + if adventure.story_summary.strip() + else None + ) + memories_section = None + if memory_bank and memory_bank.get("used"): + lines_text = "\n".join(f"- {m['text']}" for m in memory_bank["used"]) + memories_section = Section("used_memories", f"Memories:\n{lines_text}") + world_state_section = None + refusal_note = "" + if has_ws: + # One read serves both the in-scene NPCs and the refusal note below. + recent = history.tail(adventure, NPC_WINDOW, exclude_action_id) + block = worldstate.render_state_section( + adventure.world_state, stat_schema, _visible_npcs(recent, stat_schema), + persona_name, + ) + if block: + world_state_section = Section("world_state", block) + # Corrections for the previous AI turn only. A refusal the model has + # already had one chance to fix is stale, and repeating it every turn + # would price a correction into the whole rest of the adventure. + last_ai = next((a for a in reversed(recent) if a.type == "ai"), None) + if last_ai is not None: + refusal_note = worldstate.render_refusals(last_ai.world_delta) + + authors_note_text = adventure.authors_note.strip() + if isinstance(script_mem.get("authorsNote"), str) and script_mem["authorsNote"].strip(): + authors_note_text = script_mem["authorsNote"].strip() + authors_note = f"[Author's note: {authors_note_text}]" if authors_note_text else "" + + front_memory = "" + if isinstance(script_mem.get("frontMemory"), str): + front_memory = script_mem["frontMemory"].strip() + + length_note = length_hint(settings.max_output_tokens, has_ws=has_ws) + + # The live sections sit below the history, but they are still part of the + # prompt, so they still count against the budget. `world_lore` is the + # exception, because the code below budgets it out of `available`. + reserved = ( + sum(s.tokens for s in system_sections) + + sum( + s.tokens + for s in (summary_section, memories_section, world_state_section) + if s is not None + ) + + count_tokens(authors_note) + + count_tokens(front_memory) + + count_tokens(length_note) + + (count_tokens(worldstate.EMIT_REMINDER) if has_ws else 0) + + count_tokens(refusal_note) + ) + available = max(256, settings.context_token_budget - reserved) + + # Only the newest actions can reach the prompt, because the code below + # either truncates the text to `available` tokens or stops at the budget. + # Fetch a window that is provably larger than that and no larger. Otherwise + # a long adventure reads its whole history on every turn and uses only the + # end of it. + actions = history.window_covering( + adventure, available, count_tokens, exclude_action_id + ) + + # ----- Story cards: triggered by recent story text (the window history could fill) ----- + trigger_window = truncate_to_last_tokens(SEPARATOR.join(a.text for a in actions), available) + triggered = match_cards(adventure.story_cards, trigger_window) + + card_budget = int(available * CARD_BUDGET_SHARE) + card_records = [] + lore_lines: list[str] = [] + used = 0 + for match in triggered: + line = f"World Lore: {match['entry'].strip()}" + tokens = count_tokens(line) + included = used + tokens <= card_budget + if included: + lore_lines.append(line) + used += tokens + card_records.append( + {"id": match["id"], "name": match["name"], "keyword": match["keyword"], + "included": included} + ) + lore_section = ( + Section("world_lore", "\n".join(lore_lines)) if lore_lines else None + ) + + # ----- Story history: newest first until the remaining budget is spent ----- + history_budget = available - used + included_actions: list[models.Action] = [] + spent = 0 + oldest_truncated = False + for action in reversed(actions): + # Budget against the text as it appears in the prompt, which includes + # the state block when this adventure tracks world state. + rendered = _history_text(action) if has_ws else action.text + tokens = count_tokens(rendered) + count_tokens(SEPARATOR) + if spent + tokens > history_budget: + if not included_actions: + # Even the newest action alone is over budget: hard-truncate it. + included_actions.append( + models.Action( + adventure_id=action.adventure_id, + type=action.type, + text=truncate_to_last_tokens(action.text, history_budget), + ) + ) + oldest_truncated = True + break + included_actions.append(action) + spent += tokens + included_actions.reverse() + + # ----- Assemble the story text, with the author's note near the end ----- + # Append each AI turn's state block again. The app strips it before storage, + # and the recent history has to show the model the pattern to follow. + texts = [_history_text(a) if has_ws else a.text for a in included_actions] + note_sections: list[Section] = [] + if authors_note: + pos = max(0, len(texts) - AUTHORS_NOTE_DEPTH) + before, after = texts[:pos], texts[pos:] + if before: + note_sections.append(Section("history", SEPARATOR.join(before))) + note_sections.append(Section("authors_note", authors_note)) + note_sections.append(Section("recent_history", SEPARATOR.join(after))) + else: + note_sections.append(Section("history", SEPARATOR.join(texts))) + # The live sections, ordered from least to most volatile. See the comment + # where they are built. They go below the history so that the history stays + # cached, and above the final sections so that those stay last. + for live in (summary_section, lore_section, memories_section, world_state_section): + if live is not None: + note_sections.append(live) + if front_memory: + note_sections.append(Section("front_memory", front_memory)) + # Place the length hint just above the emit reminder, which keeps the last + # position. The length budget applies to the narration, and the reminder + # applies to the block that follows it, so this is also the order in which + # the model acts. + note_sections.append(Section("length_hint", length_note)) + if has_ws: + # A correction for the previous turn sits directly above the reminder + # to emit a block, which is the instruction it modifies. + if refusal_note: + note_sections.append(Section("world_state_refusals", refusal_note)) + # The emit rule sits in the system block, far from where the model + # generates text, so repeat it last where it has the most effect. + note_sections.append(Section("world_state_reminder", worldstate.EMIT_REMINDER)) + + story_sections = [s for s in note_sections if s.text] + system_text = SEPARATOR.join(s.text for s in system_sections if s.text) + story_text = SEPARATOR.join(s.text for s in story_sections) + + all_sections = [s for s in system_sections if s.text] + story_sections + report = { + "sections": [ + {"label": s.label, "text": s.text, "tokens": s.tokens} for s in all_sections + ], + "prompt": {"system": system_text, "story": story_text}, + "tokens": { + "total": count_tokens(system_text) + count_tokens(story_text), + "budget": settings.context_token_budget, + }, + "cards": card_records, + "memories": memory_bank, + "history": { + "included": len(included_actions), + # The count covers the whole story rather than the window fetched + # above. Insights reports how many of the total actions it + # included, so this number must be the real total. + "total": history.count(adventure, exclude_action_id), + "oldest_truncated": oldest_truncated, + }, + "settings": { + "model": settings.model, + "api_mode": settings.api_mode, + "temperature": settings.temperature, + "max_output_tokens": settings.max_output_tokens, + }, + } + return system_text, story_text, report diff --git a/backend/app/context/cursors.py b/backend/app/context/cursors.py new file mode 100644 index 0000000..599ff49 --- /dev/null +++ b/backend/app/context/cursors.py @@ -0,0 +1,172 @@ +"""Phase 14: tracks how far along a story the derived work has reached. + +Two things are built from the story and stored beside it: the memories and the +story summary. Both need to record where they stopped. + +That mark used to be a count, such as "the first 12 story actions are covered". +A count is a position in a list, and this list changes. If you delete an action +in front of the mark, every later action moves down one slot, so the mark now +covers an action it never read. The rules in `memorybank` for sliding cursors, +rewinding them, and converting between positions and `Action.index` all existed +to correct for that, and each rule was a chance to introduce a silent error. + +A cursor here is an anchor instead. It stores `(branch_id, depth)`, naming the +node up to and including which the work is done. Deleting an action does not +move it, because a depth is a coordinate along a path rather than a position in +a list. The question "what is not covered yet" becomes +`history.count_after(anchor)`, which asks about the story rather than about a +list index, and it stays correct no matter what is deleted in front of it. + +The branch half of the anchor is what makes it survive forking. A depth alone is +ambiguous once two branches both hold a node at depth 41. The anchor names the +branch, and `Path.depth_on` reads it back as a depth on whichever story is being +played. That read caps the depth at the fork, or reports nothing covered if the +anchor sits on a branch this path does not contain. Until forking ships there is +one branch, so this always returns the stored depth. That is the point: the +coordinate system is correct before anything depends on it. + +`NO_DEPTH`, which is -1, means nothing is covered. A new adventure therefore +needs no special case, because every node is deeper than -1. +""" + +from sqlalchemy.orm import Session + +from .. import models +from . import history, lineage + +NO_DEPTH = lineage.NO_DEPTH + + +class Cursor: + """One anchor on the adventure row, for either the memory bank or the summary. + + The anchor is a pair of columns rather than a foreign key to the node. The + node can be deleted, which is what undo does, and the boundary still means + something afterwards. A foreign key would repeatedly fail to resolve. + """ + + def __init__(self, name: str): + self.name = name + self.branch_field = f"{name}_cursor_branch_id" + self.depth_field = f"{name}_cursor_depth" + + # ------------------------------------------------------------- reading + + def stored(self, adventure: models.Adventure) -> tuple[int | None, int]: + """Returns the anchor as written, without resolving it against a path.""" + depth = getattr(adventure, self.depth_field) + return getattr(adventure, self.branch_field), ( + NO_DEPTH if depth is None else depth + ) + + def depth(self, db: Session, adventure: models.Adventure) -> int: + """Returns the anchor as a depth on the story being played now.""" + branch_id, depth = self.stored(adventure) + return lineage.path_of(db, adventure).depth_on(branch_id, depth) + + # ------------------------------------------------------------- writing + + def anchor( + self, adventure: models.Adventure, branch_id: int | None, depth: int + ) -> None: + """Sets the anchor to a coordinate supplied directly. + + This is the plain setter beneath `anchor_at`. Only an import supplies a + coordinate with no node to read it from. A v2 bundle carries the anchor + itself, as described in `app/bundle.py`, and the node it named lives in + a different database. + """ + setattr(adventure, self.branch_field, branch_id) + setattr(adventure, self.depth_field, max(depth, NO_DEPTH)) + + def anchor_at(self, adventure: models.Adventure, node: models.Action) -> None: + """Marks the work done up to and including `node`. + + This uses the node's own branch rather than the adventure's head. A block + of six actions can end before the fork that created the current branch, + and the coverage belongs where those actions are. + """ + self.anchor(adventure, node.branch_id, lineage.NO_DEPTH + if node.depth is None else node.depth) + + def clear(self, adventure: models.Adventure) -> None: + """Clears the anchor, so that nothing counts as covered. + + Call this when the branch the anchor referred to is deleted. On Postgres + a stale branch id would never resolve. On SQLite the next fork can reuse + an id that was just freed, and a stale anchor would then resolve onto a + branch it never saw and report that stretch of story as summarized. + Clearing the anchor costs one re-summarize, which is the safe direction + to be wrong in. + """ + self.anchor(adventure, None, NO_DEPTH) + + def rewind_to( + self, adventure: models.Adventure, branch_id: int | None, depth: int + ) -> None: + """Moves the anchor back to `depth` if it is past that depth. + + The anchor never moves forward here. Moving backward is the only + direction that is safe without knowing what else changed. Covering + ground twice costs one summarizer call. Skipping ground removes a + stretch of story from the memories permanently. + """ + _, current = self.stored(adventure) + if current <= depth: + return + setattr(adventure, self.branch_field, branch_id) + setattr(adventure, self.depth_field, max(depth, NO_DEPTH)) + + +MEMORY = Cursor("memory") +SUMMARY = Cursor("summary") +ALL = (MEMORY, SUMMARY) + + +def rewind_all( + adventure: models.Adventure, branch_id: int | None, depth: int +) -> None: + """Returns a stretch of story to both the memory pass and the summary pass. + + The two move together because they cover the same actions from different + directions. The summary folds in the memories, so withdrawing a memory + without rewinding the summary would leave the summary claiming to have read + something that no longer exists. + """ + for cursor in ALL: + cursor.rewind_to(adventure, branch_id, depth) + + +def anchor_at_position( + adventure: models.Adventure, cursor: Cursor, position: int +) -> None: + """Sets `cursor` from a count of covered story actions. + + A v1 bundle stores its mark as a count, and so does a database written + before the anchors existed. + + The action at `position` in depth order is the node that carries the same + meaning, and it keeps that meaning after something in front of it is + deleted. A position past the end of the story is not an invalid value. An + adventure that was fully caught up under the old rule can hold one, and it + means the same thing as the tip, so this function anchors at the tip. + + `migrations._backfill_cursor_anchors` implements this rule in SQL for every + adventure at once, without loading any of them. The two must agree. + """ + if position <= 0: + return + covered = history.slice_(adventure, position - 1, 1) or history.tail(adventure, 1) + if covered: + cursor.anchor_at(adventure, covered[0]) + + +def position_of(adventure: models.Adventure, depth: int) -> int: + """Returns how many story actions lie at or before `depth`. + + This reads an anchor back as a count. The v1 export bundle stores cursors as + counts, and builds that have never used depths read v1 bundles. This + function is the only remaining code that speaks that coordinate system. The + v2 format introduced in SP6 replaces it. + """ + return max(history.count(adventure) - history.count_after(adventure, depth), 0) diff --git a/backend/app/context/history.py b/backend/app/context/history.py new file mode 100644 index 0000000..df279df --- /dev/null +++ b/backend/app/context/history.py @@ -0,0 +1,458 @@ +"""Reads part of a story without loading all of it. + +`story_actions()` used to walk `adventure.actions`, which loads every row of the +adventure. Every caller then discarded nearly all of those rows. The context +builder joins the story and immediately trims it to the token budget. The +in-scene NPC check reads the last 6 actions. Memory retrieval reads the last 4. +The post-turn cursor clamp needs only a count. A turn on a 200-action adventure +read about 840 KB in order to use about 70 KB, and the cost grew with every +turn. + +This module serves those shapes from SQL directly, as a tail, a slice, or a +count. A read is therefore bounded by the context budget rather than by the +length of the story. + +Three rules hold the module together: + +- There is one definition of a story action. That definition decides both what + a reader sees and what the summarizer receives, so the SQL and the Python must + agree exactly. `_STORY_TEXT` and `is_story_text()` express the same rule + twice. Keep them in step. +- No caller loads the same rows twice. If `adventure.actions` is already in + memory, every helper here slices that collection instead of running a query. + The scripting pipeline hands the whole history to user scripts, as AI Dungeon + does, so a scripted adventure costs no more than it did before. +- Every read applies the branch clause, as of Phase 14. `adventure.actions` + holds the actions of every branch rather than the story being played, so the + in-memory path filters the collection down to the path before slicing it, in + the same way the SQL does. Skipping that filter would build a prompt from two + different stories without reporting an error, which is why `lineage.Path` + owns both forms of the rule. + +Reads order by `depth` rather than `index`. Since SP4, two different rows can +hold the same value for both columns, because the attempts at one turn share +them. Only `depth` together with the `live` test in the branch clause +identifies the row the story uses. + +SP3 added the reads that count from a node rather than from the start: +`count_after`, `after`, and `newest`. The memory bank used to ask for positions +12 through 18 of the story, and the answer to that question changes when an +action in front of those positions is deleted. It now asks for the six actions +after depth 41, which is the question that forking requires in any case. +""" + +from sqlalchemy import func, inspect as sa_inspect +from sqlalchemy.orm import Session, defer, object_session + +from .. import models +from . import lineage + +# How many of the newest actions to read before checking whether they cover the +# token budget. If they do not, the next size comes from the average action +# length just measured rather than from doubling the previous size. Doubling +# overshoots, which means reading hundreds of actions in order to use sixty. +WINDOW_START = 32 +WINDOW_MARGIN = 0.15 # Aim this far past the budget, so a second round is rare. +WINDOW_STEP = 8 # Read at least this many more actions in each round. + +# `depth` is the ordering key, and `id` breaks ties. Without `id`, the database +# would choose the order. Two rows can share a depth: a pre-tree row, which has +# a NULL depth and is invisible to reads, or a pair of sibling attempts. +_OLDEST_FIRST = (models.Action.depth, models.Action.id) +_NEWEST_FIRST = (models.Action.depth.desc(), models.Action.id.desc()) + + +def _sql_stripped(column): + """`column` with leading/trailing whitespace removed, portably. + + SQLite and Postgres both accept `trim()` with a single argument, but that + form removes spaces only. Python's `str.strip()` also removes newlines and + tabs. Without this helper, an action containing only a newline would count + as story text in SQL but not in Python. Both dialects spell `replace()` and + `trim()` the same way, so this function converts the other whitespace to + spaces first. It does not handle form feed or vertical tab, because nothing + produces them. + """ + folded = column + for char in ("\n", "\r", "\t"): + folded = func.replace(folded, char, " ") + return func.trim(folded) + + +_STORY_TEXT = _sql_stripped(models.Action.text) != "" + + +def is_story_text(text: str) -> bool: + """Returns whether `text` counts as story text. + + This is the Python form of `_STORY_TEXT`. Keep the two in step. + """ + return bool(text.strip()) + + +def _loaded_actions(adventure: models.Adventure) -> list[models.Action] | None: + """The adventure's actions if they are already in memory, else None. + + Slicing a collection that is already loaded costs nothing, and running a + query beside it would fetch the same rows a second time. + """ + state = sa_inspect(adventure) + if state.detached or "actions" in state.unloaded: + return None + return list(adventure.actions) + + +def _from_memory( + adventure: models.Adventure, exclude_action_id: int | None +) -> list[models.Action] | None: + """The story, from the already-loaded collection, or None to go to SQL. + + The collection holds the adventure's actions, which means the actions of + every branch. Filtering it down to the path here applies the same rule that + the SQL applies. Without that filter, the context builder would receive a + prompt built from siblings of the story being played. + + Resolving the path requires a session to read the branch row from. If there + is no session, this function returns None so that the caller falls back to + SQL rather than guessing. + """ + loaded = _loaded_actions(adventure) + if loaded is None: + return None + db = _session(adventure) + if db is None: + return None + path = lineage.path_of(db, adventure) + rows = [ + a for a in loaded + if path.contains(a) + and is_story_text(a.text) + and (exclude_action_id is None or a.id != exclude_action_id) + ] + # `Adventure.actions` is ordered by `index`; a path is ordered by depth. + rows.sort(key=path.sort_key) + return rows + + +def _filters( + adventure: models.Adventure, + path: lineage.Path, + exclude_action_id: int | None, + entries: int | None = None, +) -> list: + # `adventure_id` is redundant beside the branch clause, because branch ids + # are unique and a branch already identifies one adventure. The filter + # remains because it costs little, it catches a node written onto another + # adventure's branch, and it makes the query easier to read. + conditions = [ + models.Action.adventure_id == adventure.id, + path.clause(models.Action, count=entries), + _STORY_TEXT, + ] + if exclude_action_id is not None: + conditions.append(models.Action.id != exclude_action_id) + return conditions + + +def _query( + db: Session, + adventure: models.Adventure, + path: lineage.Path, + exclude_action_id: int | None, + entries: int | None = None, +): + # Reasoning traces are never read from replayed history and can be larger + # than the narration itself on a reasoning model. + return ( + db.query(models.Action) + .filter(*_filters(adventure, path, exclude_action_id, entries)) + .options(defer(models.Action.reasoning)) + ) + + +def _count_query( + db: Session, + adventure: models.Adventure, + path: lineage.Path, + exclude_action_id: int | None, + entries: int | None = None, +): + """A real `SELECT count(...)`. + + This function deliberately avoids `_query(...).count()`. That form wraps the + entity select in a subquery, so the emitted SQL names every column, + including the deferred columns that this design keeps off the wire. Neither + form returns those bytes to the client, but the database still reads them, + and an egress guard that inspects the SQL cannot tell the two forms apart. + """ + return db.query(func.count(models.Action.id)).filter( + *_filters(adventure, path, exclude_action_id, entries) + ) + + +def _session(adventure: models.Adventure) -> Session | None: + return object_session(adventure) + + +def _path(db: Session, adventure: models.Adventure) -> lineage.Path: + return lineage.path_of(db, adventure) + + +# ------------------------------------------------------------------ the API + +def story_actions( + adventure: models.Adventure, exclude_action_id: int | None = None +) -> list[models.Action]: + """Every story action, oldest first. + + Call this function when you need the whole story. User scripts receive it, + which matches AI Dungeon's scripting API. Use `tail`, `slice_`, or `count` + when you need only part of the story. + + `exclude_action_id` removes one action from the result. Retry uses it. The + attempt being replaced is still the live node of its turn, because it stays + live until a replacement exists, but it must not appear in the context that + is assembled to replace it. + """ + in_memory = _from_memory(adventure, exclude_action_id) + if in_memory is not None: + return in_memory + db = _session(adventure) + if db is None: + return [] + return ( + _query(db, adventure, _path(db, adventure), exclude_action_id) + .order_by(*_OLDEST_FIRST) + .all() + ) + + +def count(adventure: models.Adventure, exclude_action_id: int | None = None) -> int: + """How many story actions there are, without fetching any of them.""" + in_memory = _from_memory(adventure, exclude_action_id) + if in_memory is not None: + return len(in_memory) + db = _session(adventure) + if db is None: + return 0 + return ( + _count_query(db, adventure, _path(db, adventure), exclude_action_id).scalar() + or 0 + ) + + +def tail_range( + adventure: models.Adventure, + skip: int, + limit: int, + exclude_action_id: int | None = None, +) -> list[models.Action]: + """`limit` story actions ending `skip` actions before the end, oldest first. + + Passing `skip=0` returns the newest slice. Passing `skip=32` and `limit=16` + returns the 16 actions immediately older than the newest 32. A growing + window therefore fetches only the actions it does not already hold. + + This read is the reason the lineage window exists. The path's ranges do not + overlap and they descend, so the newest N nodes come from the newest few + lineage entries and the query never has to name the rest of the ancestry. A + story that has forked 200 times reads its tail with as few clauses as one + that has never forked. + + `prefix_covering` estimates how many entries that takes, using depth + arithmetic alone. The estimate falls short only when an action was deleted + from the middle of the story. In that case this function widens the read to + the whole lineage, at the cost of one more query. + """ + if limit <= 0 or skip < 0: + return [] + in_memory = _from_memory(adventure, exclude_action_id) + if in_memory is not None: + stop = len(in_memory) - skip + return in_memory[max(stop - limit, 0):stop] if stop > 0 else [] + db = _session(adventure) + if db is None: + return [] + path = _path(db, adventure) + entries = path.prefix_covering(skip + limit) + while True: + rows = ( + _query(db, adventure, path, exclude_action_id, entries) + .order_by(*_NEWEST_FIRST) + .offset(skip) + .limit(limit) + .all() + ) + if len(rows) >= limit or entries >= len(path): + break + entries = len(path) # short: widen once, to everything, and re-ask + rows.reverse() + return rows + + +def tail( + adventure: models.Adventure, limit: int, exclude_action_id: int | None = None +) -> list[models.Action]: + """The newest `limit` story actions, returned oldest first.""" + return tail_range(adventure, 0, limit, exclude_action_id) + + +def slice_( + adventure: models.Adventure, + start: int, + length: int, + exclude_action_id: int | None = None, +) -> list[models.Action]: + """Story actions at positions [start, start + length), oldest first. + + Positions index into the same filtered, depth-ordered list that the memory + cursors count in, which is why the SQL filter must match the Python filter + exactly. + + This function counts from the oldest end, so it names the whole lineage. No + prefix of the ancestry contains the first ten actions of the story. + """ + if length <= 0 or start < 0: + return [] + in_memory = _from_memory(adventure, exclude_action_id) + if in_memory is not None: + return in_memory[start:start + length] + db = _session(adventure) + if db is None: + return [] + return ( + _query(db, adventure, _path(db, adventure), exclude_action_id) + .order_by(*_OLDEST_FIRST) + .offset(start) + .limit(length) + .all() + ) + + +def depth_of(action: models.Action) -> int: + """`action.depth`, with the no-depth case spelled once. + + A row with no depth predates the tree. No path contains such a row, so it + appears only in a collection that is already loaded. It sorts before the + story rather than after it. + """ + return action.depth if action.depth is not None else lineage.NO_DEPTH + + +def count_after( + adventure: models.Adventure, depth: int, exclude_action_id: int | None = None +) -> int: + """How many story actions lie past `depth` on the path. + + This replaces the older calculation, which compared the length of the story + with the position of the cursor. Deleting an action in front of the boundary + makes this number smaller, which is correct. It does not move the boundary + to a different action, which is the error that positions produced. + + `covering_after` reports which lineage entries can hold a node deeper than + the boundary, so a cursor near the tip names one branch however many forks + lie below it. + """ + in_memory = _from_memory(adventure, exclude_action_id) + if in_memory is not None: + return sum(1 for a in in_memory if depth_of(a) > depth) + db = _session(adventure) + if db is None: + return 0 + path = _path(db, adventure) + return ( + _count_query( + db, adventure, path, exclude_action_id, path.covering_after(depth) + ) + .filter(models.Action.depth > depth) + .scalar() + or 0 + ) + + +def after( + adventure: models.Adventure, + depth: int, + limit: int, + exclude_action_id: int | None = None, +) -> list[models.Action]: + """The oldest `limit` story actions past `depth`, oldest first. + + This returns the next block that the summarizer has not read. It asks a + question about the story rather than using an offset into a list whose + entries move. + """ + if limit <= 0: + return [] + in_memory = _from_memory(adventure, exclude_action_id) + if in_memory is not None: + return [a for a in in_memory if depth_of(a) > depth][:limit] + db = _session(adventure) + if db is None: + return [] + path = _path(db, adventure) + return ( + _query(db, adventure, path, exclude_action_id, path.covering_after(depth)) + .filter(models.Action.depth > depth) + .order_by(*_OLDEST_FIRST) + .limit(limit) + .all() + ) + + +def newest(adventure: models.Adventure) -> models.Action | None: + """The newest story action, or None on an empty story. + + This returns a row rather than a count and an offset, because it names the + node an anchor moves to when derived work reaches the end of the story. + + It used to return the second newest action. The memory bank held one action + back because retry rewrote a row, so a memory that covered the newest action + could describe narration the player had already replaced. Since SP4, a retry + writes a sibling row instead, and the derived work at a coordinate is + withdrawn when the story at that coordinate changes. There is nothing left + to hold back. + """ + rows = tail(adventure, 1) + return rows[0] if rows else None + + +def window_covering( + adventure: models.Adventure, + budget_tokens: int, + token_counter, + exclude_action_id: int | None = None, +) -> list[models.Action]: + """Returns the newest story actions whose combined text exceeds `budget_tokens`. + + The result always holds at least as much text as the context builder can + include, and never less. + + This function counts tokens rather than estimating a characters-per-token + ratio, so the prompt matches what loading the whole story would produce. It + budgets against the raw text, which is never longer than the rendered + history text, so any error causes it to fetch slightly more than needed. + + Each round fetches only the actions it does not already hold, so no row is + read twice however many rounds the loop takes. + """ + actions: list[models.Action] = [] + tokens = 0 + size = WINDOW_START + while True: + older = tail_range( + adventure, len(actions), size - len(actions), exclude_action_id + ) + if not older: + return actions # The result already holds the whole story. + actions = older + actions + tokens += sum(token_counter(a.text) for a in older) + if len(actions) < size: + return actions # That was the whole story. + if tokens > budget_tokens: + return actions + # The window is still short. Estimate how many actions the budget needs + # at the average length just measured, then read that many. + average = tokens / len(actions) + projected = int(budget_tokens / average * (1 + WINDOW_MARGIN)) + WINDOW_STEP + size = max(projected, size + WINDOW_STEP) diff --git a/backend/app/context/lineage.py b/backend/app/context/lineage.py new file mode 100644 index 0000000..4f994d2 --- /dev/null +++ b/backend/app/context/lineage.py @@ -0,0 +1,278 @@ +"""Phase 14: decides which nodes make up one story. + +`tree.py` decides where a node is written. This module decides which nodes a +read can see, and it is the only place that makes that decision. + +A branch owns the nodes played on it and inherits everything before its fork +point from its ancestors. "The story on branch C" is therefore not a value you +can filter a column on. It is an OR of ranges:: + + (branch_id = C) -- C's own nodes, through to the tip + OR (branch_id = B AND depth <= 5) + OR (branch_id = A AND depth <= 3) + +The `branches.lineage` column records exactly that list, newest first. The fork +computes it once, so no read walks parent pointers to rebuild it. + +The shape of the list gives two properties that the windowed reads depend on: + +- The ranges do not overlap, and they descend. A branch's own nodes always sit + deeper than its fork point, and each lineage entry is capped at the fork depth + of the branch below it. Ordering the whole clause by `depth` descending + therefore reads entry 0's nodes, then entry 1's, then entry 2's. A tail read + can use the newest few entries and stop. +- The number of clauses depends on the size of the context window, not on how + many times the story has forked. A story with 200 forks whose newest branch + runs 40 turns reads with a single clause, because the window is covered before + the second entry is reached. `prefix_covering` implements this, and it is why + `history.window_covering` can keep its current shape. + +Everything in this module reads. Nothing here creates a branch or writes a row. +An adventure with no branch has no story, and the write side repairs that. See +`tree.place_action` and the flush listener in `models.py`. +""" + +from sqlalchemy import and_, false, or_ +from sqlalchemy.orm import Session + +from .. import models + +# The depth of an adventure that has no actions. This mirrors `tree.NO_DEPTH`. +# It is duplicated here so that a read never has to import the write half. +NO_DEPTH = -1 + +# The opening node of an adventure. Depth 0 exists only on the root branch, +# because a fork starts its own nodes after the depth it forked at. This +# constant therefore names one node per adventure rather than one per branch. +# +# Migration 62 also placed every memory written before memories had coordinates +# at depth 0. That is why both places that can retire a memory must handle this +# depth: `memorybank.forget_node`, and a v1 import that has no depth to read. +ROOT_DEPTH = 0 + + +def entries_of(branch: models.Branch) -> list[tuple[int, int | None]]: + """Returns `branch.lineage` as (branch_id, max_depth) pairs, newest first. + + An empty lineage means this branch alone, through to its tip. That is not an + error. It is what a root branch's lineage means, and it is also what a + branch row holds between the moment it is inserted and the moment its own id + is written into the column. The fallback is therefore correct rather than a + guess. + """ + raw = branch.lineage if isinstance(branch.lineage, list) else [] + entries: list[tuple[int, int | None]] = [] + for item in raw: + # JSON round-trips lists, but a hand-written row might hold tuples. + if not isinstance(item, (list, tuple)) or not item: + continue + branch_id = item[0] + max_depth = item[1] if len(item) > 1 else None + if not isinstance(branch_id, int): + continue + entries.append((branch_id, max_depth if isinstance(max_depth, int) else None)) + return entries or [(branch.id, None)] + + +class Path: + """One story, expressed as a SQL clause and as a Python predicate. + + The object holds the lineage entries newest first, plus the depth of the + tip. The tip is used only to estimate how much story each entry covers. + """ + + def __init__(self, entries: list[tuple[int, int | None]], tip: int | None = None): + self.entries = entries + self.tip = tip + + def __bool__(self) -> bool: + return bool(self.entries) + + def __len__(self) -> int: + return len(self.entries) + + # ---------------------------------------------------------------- SQL + + def clause( + self, + model=models.Action, + count: int | None = None, + ): + """Returns the branch clause over `model`, which is `Action` or `Memory`. + + `count` limits the clause to the newest `count` lineage entries, which + produces a windowed read. Pass `None` for the whole lineage. Any caller + that counts from the oldest end, such as a slice or a total, must pass + `None`. + + Every row this clause selects has a depth. Memories were once an + exception, because a hand-written memory had a branch but no depth and + needed an escape clause here to avoid being capped at a fork. SP7 + anchors those memories at the head instead. See `tree.place_memory`. A + memory is now on a path rather than exempt from one, and a row with no + depth is a pre-tree leftover that no read should return. + + Actions must also be live, as of SP4. One coordinate can hold several + attempts at a turn, and the story uses one of them. The other attempts + sit at the same branch and depth, and this clause excludes them once, so + that no read of the story has to account for retries. Only + `app/attempts.py` looks past this filter. + + An empty path returns `false` rather than no filter at all. An adventure + whose nodes carry no branch has no story, and the correct way to show + that is an empty page rather than every branch at once. + """ + entries = self.entries if count is None else self.entries[:count] + if not entries: + return false() + on_path = or_(*[self._entry_clause(model, b, d) for b, d in entries]) + if model is models.Action: + return and_(on_path, models.Action.live.is_(True)) + return on_path + + @staticmethod + def _entry_clause(model, branch_id: int, max_depth: int | None): + if max_depth is None: + return model.branch_id == branch_id + return and_(model.branch_id == branch_id, model.depth <= max_depth) + + # ------------------------------------------------------------- Python + + def contains(self, node) -> bool: + """Returns whether `node` is on this path. + + This is the Python equivalent of `clause()`. Keep the two in step. + + Callers use it where the rows are already in memory. The scripting + pipeline hands user scripts the whole history, so a loaded collection + can be reduced to the path without a second query. + + The method checks `live` first, and only on rows that define the + attribute, because memories have no siblings. + """ + if getattr(node, "live", True) is False: + return False + for branch_id, max_depth in self.entries: + if node.branch_id != branch_id: + continue + if max_depth is None: + return True + if node.depth is not None and node.depth <= max_depth: + return True + return False + + def sort_key(self, node) -> tuple[int, int]: + """Returns a sort key that orders nodes from oldest to newest. + + `depth` is the ordering key. `id` breaks ties, which a pre-tree row with + a NULL depth or a pair of siblings can produce. + """ + return (node.depth if node.depth is not None else NO_DEPTH, node.id or 0) + + # ------------------------------------------------------------ windowing + + def prefix_covering(self, rows: int) -> int: + """Returns how many lineage entries hold the newest `rows` nodes. + + The result is an estimate from depth arithmetic rather than a query. + Entry *i* covers the depths between its own cap and the cap of the entry + below it, and a path holds at most one node per depth. The estimate is + therefore never too large. It is too small only when the story has gaps, + which happens after an action is deleted from the middle. In that case + the caller widens the read to the whole lineage, which costs one extra + query. + """ + total = len(self.entries) + if rows <= 0 or total == 0: + return total + covered = 0 + for i, (_, max_depth) in enumerate(self.entries): + top = self.tip if max_depth is None else max_depth + below = self.entries[i + 1][1] if i + 1 < total else NO_DEPTH + if top is None or below is None: + # Either no tip was recorded, or a hand-written row is missing a + # cap. There is nothing to estimate from, so return every entry. + # Guessing low would hide the older half of the story. + return total + covered += max(top - below, 0) + if covered >= rows: + return i + 1 + return total + + def covering_after(self, depth: int) -> int: + """Returns how many lineage entries can hold a node deeper than `depth`. + + This is the counterpart to `prefix_covering`, and it is exact rather than + an estimate. Entry *i* holds nothing deeper than its own cap, and the + caps descend, so the first entry capped at or below `depth` ends the + search. That entry and every older one fall behind the boundary. As a + result, reading the story after the cursor touches one branch on any + story whose cursor sits on its newest branch, however many times the + story has forked. + """ + for i, (_, max_depth) in enumerate(self.entries): + if max_depth is not None and max_depth <= depth: + return i + return len(self.entries) + + def depth_on(self, branch_id: int | None, depth: int) -> int: + """Reads a stored `(branch_id, depth)` anchor as a depth on this path. + + An anchor records how far along a story some derived work reached, such + as which actions the memories cover or what the summary folded in. The + anchor names a node, so moving to a different path needs an explicit + answer. There are two cases: + + - The anchor's branch is on this path. The depth stands, capped at the + fork where this path leaves that branch, because nothing past the fork + belongs to this story. + - The anchor's branch is not on this path. The work was done on a branch + this story does not contain, so nothing here counts as covered. + + The second case cannot occur while an adventure has one branch, because + the anchor is always set from a node on it. It exists because the safe + answer to an unknown anchor is to redo the work rather than skip it. + """ + if depth <= NO_DEPTH: + return NO_DEPTH + if branch_id is None: + # The anchor predates the tree, or someone set it by hand. There is + # only one story, so the depth is a position in it. + return depth + for entry_branch, max_depth in self.entries: + if entry_branch == branch_id: + return depth if max_depth is None else min(depth, max_depth) + return NO_DEPTH + + +def branch_of(db: Session, adventure: models.Adventure) -> models.Branch | None: + """Returns the branch this adventure is read at, or None if it has none. + + This function is deliberately not `tree.head_branch`, which creates a branch. + A GET request must not write. An adventure with no branch row also has no + nodes that carry a branch, so both answers agree that there is no story. + """ + if adventure.head_branch_id is not None: + branch = db.get(models.Branch, adventure.head_branch_id) + if branch is not None: + return branch + # The head points at a branch that no longer exists. Fall through to the + # root, which is the same recovery that `tree.head_branch` performs on + # the write side. + return ( + db.query(models.Branch) + .filter( + models.Branch.adventure_id == adventure.id, + models.Branch.parent_branch_id.is_(None), + ) + .order_by(models.Branch.id) + .first() + ) + + +def path_of(db: Session, adventure: models.Adventure) -> Path: + """Returns the story that the adventure's head currently sits on.""" + branch = branch_of(db, adventure) + if branch is None: + return Path([], adventure.head_depth) + return Path(entries_of(branch), adventure.head_depth) diff --git a/backend/app/database.py b/backend/app/database.py new file mode 100644 index 0000000..20284ca --- /dev/null +++ b/backend/app/database.py @@ -0,0 +1,74 @@ +import os +from pathlib import Path + +from sqlalchemy import create_engine, event +from sqlalchemy.orm import DeclarativeBase, sessionmaker + +# AIDND_DB_PATH lets deployments (Docker volume, hosted disk) relocate the +# SQLite database; default stays backend/data.db for local runs. The parent +# directory also hosts the auto-generated secret.key (see security.py), so +# DB_PATH stays defined even when Postgres is in use. +_env_db_path = os.environ.get("AIDND_DB_PATH") +DB_PATH = ( + Path(_env_db_path).resolve() + if _env_db_path + else Path(__file__).resolve().parent.parent / "data.db" +) + +# `AIDND_DATABASE_URL`, or the conventional `DATABASE_URL`, switches the app to +# a server database. Any SQLAlchemy URL works, and hosted deploys use Postgres, +# which Phase 9 settled on Neon for. If neither variable is set, the app uses +# SQLite. +DATABASE_URL = ( + os.environ.get("AIDND_DATABASE_URL", "").strip() + or os.environ.get("DATABASE_URL", "").strip() +) + + +def _normalize_url(url: str) -> str: + """Map the postgres:// / postgresql:// schemes hosts hand out to the + psycopg3 driver installed in requirements.txt.""" + for prefix in ("postgres://", "postgresql://"): + if url.startswith(prefix): + return "postgresql+psycopg://" + url[len(prefix):] + return url + + +if DATABASE_URL: + engine = create_engine( + _normalize_url(DATABASE_URL), + # Serverless Postgres (Neon) suspends idle databases; pre-ping + # replaces silently-dead pooled connections instead of erroring. + pool_pre_ping=True, + # Store/read naive UTC like SQLite does, regardless of server default. + connect_args={"options": "-c timezone=UTC"}, + ) +else: + DB_PATH.parent.mkdir(parents=True, exist_ok=True) + engine = create_engine( + f"sqlite:///{DB_PATH}", + connect_args={"check_same_thread": False}, + ) + + @event.listens_for(engine, "connect") + def _enable_sqlite_foreign_keys(dbapi_connection, _record): + # SQLite ships with foreign keys OFF per connection; without this every + # ondelete=CASCADE/SET NULL in models.py is silently ignored. + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA foreign_keys=ON") + cursor.close() + + +SessionLocal = sessionmaker(bind=engine, autoflush=False, expire_on_commit=False) + + +class Base(DeclarativeBase): + pass + + +def get_db(): + db = SessionLocal() + try: + yield db + finally: + db.close() diff --git a/backend/app/debuglog.py b/backend/app/debuglog.py new file mode 100644 index 0000000..bd10e73 --- /dev/null +++ b/backend/app/debuglog.py @@ -0,0 +1,71 @@ +"""In-memory ring buffer of recent provider requests/responses for the debug page. + +API keys never enter the log: only the request body (which carries no +credentials) and response text are recorded, both truncated. +""" + +import itertools +from collections import deque +from datetime import datetime, timezone + +MAX_ENTRIES = 30 +MAX_TEXT = 6000 + +_entries: deque[dict] = deque(maxlen=MAX_ENTRIES) +_ids = itertools.count(1) + + +def _clip(text: str) -> str: + if len(text) <= MAX_TEXT: + return text + return text[:MAX_TEXT] + f"\n… [{len(text) - MAX_TEXT} more chars truncated]" + + +def _clip_obj(obj): + if isinstance(obj, str): + return _clip(obj) + if isinstance(obj, dict): + return {k: _clip_obj(v) for k, v in obj.items()} + if isinstance(obj, list): + return [_clip_obj(v) for v in obj] + return obj + + +def start_entry(url: str, model: str, body: dict) -> dict: + entry = { + "id": next(_ids), + "time": datetime.now(timezone.utc).isoformat(), + "url": url, + "model": model, + "request": _clip_obj(body), + "status": "pending", + "response": "", + "usage": None, + "error": None, + } + _entries.appendleft(entry) + return entry + + +def finish_entry( + entry: dict, + *, + response: str = "", + error: str | None = None, + usage: dict | None = None, +) -> None: + """Finishes a log entry. + + `usage` is the endpoint's own token accounting, when it reported any. On + OpenRouter it carries `prompt_tokens_details.cached_tokens`, which is the + only direct measure of whether the prompt prefix is being cached, and it is + worth seeing next to the request that produced it. + """ + entry["response"] = _clip(response) + entry["usage"] = usage + entry["error"] = error + entry["status"] = "error" if error else "ok" + + +def recent() -> list[dict]: + return list(_entries) diff --git a/backend/app/images.py b/backend/app/images.py new file mode 100644 index 0000000..0afc543 --- /dev/null +++ b/backend/app/images.py @@ -0,0 +1,67 @@ +"""Scenario cover art: inline data URIs in, cacheable URLs out. + +A scenario's `image` column holds either an `https://` URL or a base64 +`data:image/…` URI (the editor downscales uploads before storing one). Sending +those data URIs inside list responses would balloon them, so lists advertise a +`image_url` pointing at `GET /api/scenarios/{id}/image` instead, and the bytes +are fetched once and cached by the browser. +""" + +import base64 +import binascii +import re + +# Only raster formats a browser renders in an . SVG is deliberately absent: +# it can carry script, and these bytes are served from our own origin. +DATA_URI_RE = re.compile( + r"^data:(image/(?:png|jpeg|webp|gif|avif));base64,([A-Za-z0-9+/=\s]+)$", + re.IGNORECASE, +) + + +def public_url(scenario_id: int, image: str, version: object) -> str: + """The URL a client should load for this scenario's art ("" if none). + + `version` is any object with a stable repr, and is normally the row's + `updated_at`. It becomes a cache-buster, so the image response can be marked + immutable and still refresh as soon as the author replaces the picture. + """ + if not image: + return "" + if DATA_URI_RE.match(image): + stamp = int(version.timestamp()) if hasattr(version, "timestamp") else 0 + return f"/api/scenarios/{scenario_id}/image?v={stamp}" + # Anything else must be an absolute https URL. http:// is rejected rather + # than passed through: the deployed app is https, so a browser would block + # it as mixed content and the author would just see a broken image. + return image if image.startswith("https://") else "" + + +def sanitize(value: object, max_length: int) -> str: + """Coerce an untrusted `image` value from an import bundle to a safe one. + + A value that is not a supported data URI or an https URL, or that is too + large to store, becomes "". A hostile or merely unfamiliar bundle therefore + cannot pass in a `javascript:` URI or exceed the column cap. + """ + if not isinstance(value, str) or not value or len(value) > max_length: + return "" + if DATA_URI_RE.match(value): + return value if decode(value) is not None else "" + return value if value.startswith("https://") else "" + + +def decode(image: str) -> tuple[bytes, str] | None: + """`(bytes, content_type)` for a stored data URI, or None if it isn't one.""" + match = DATA_URI_RE.match(image or "") + if not match: + return None + try: + # validate=True rejects anything outside the base64 alphabet, so strip + # the newlines a hand-pasted or line-wrapped URI may carry first. + payload = re.sub(r"\s+", "", match.group(2)) + return base64.b64decode(payload, validate=True), match.group(1).lower() + except (binascii.Error, ValueError): + # The base64 is truncated or hand-edited. Treat it as no image rather + # than return a 500. + return None diff --git a/backend/app/limits.py b/backend/app/limits.py new file mode 100644 index 0000000..43ec35f --- /dev/null +++ b/backend/app/limits.py @@ -0,0 +1,345 @@ +"""Phase 9: abuse guards for hosted, multi-user deployments. + +Rate limits and row caps do nothing in local mode, because a single local player +should never be throttled by their own app. The values are hardcoded on purpose. +They are generous enough that a legitimate player never notices them, and tight +enough that a hostile visitor cannot exhaust the demo key, saturate the CPU, or +fill the database. +""" + +import json +import os +import threading +import time +from collections import defaultdict, deque + +from fastapi import HTTPException, Request +from sqlalchemy import func +from sqlalchemy.orm import Session + +from . import auth, models + +# ---------- Rate limiting ---------- +# Fixed windows per scope and caller. The windows live in memory, which is +# enough for the single-process deployment this app targets. The worst case +# after a restart is a brief extra allowance. + +# Maps a scope to (max requests, window seconds). +RATE_LIMITS: dict[str, tuple[int, int]] = { + "turn": (10, 60), # AI turn generation. The demo key also has a daily cap. + "chat": (30, 60), # The AI Chat scratchpad, for power users. + "script-test": (30, 60), # Sandboxed, but each run costs up to 2s of CPU. + "connection-test": (10, 60), # Outbound HTTP to a user-supplied URL. + "import": (30, 60), # Large writes. + "auth": (10, 300), # Register and login attempts, per IP. + "guest": (30, 300), # New guest users, per IP. Each one is a database row. + # Pageview beacons. The limit is generous, because a real reader clicking + # around a SPA sends a handful a minute, and it is low enough that nobody + # can inflate the traffic numbers faster than by reloading the page. + "analytics": (120, 60), +} + +_windows: dict[tuple[str, str], deque] = defaultdict(deque) +_windows_guard = threading.Lock() + + +# How many proxy hops sit between the app and the real client. On Render, and on +# most platforms, that is one, because the platform's edge appends the connecting +# IP to the right of `X-Forwarded-For`. A client can prepend any value on the +# left, but it cannot push a value past the edge's own append, so the trustworthy +# client IP is the entry that many places from the right rather than uvicorn's +# leftmost choice. Trusting the leftmost entry let anyone rotate +# `X-Forwarded-For` to get a fresh rate-limit bucket per request and bypass the +# auth and guest limits. If the deployment adds more hops, set +# `AIDND_TRUSTED_PROXY_HOPS`. +TRUSTED_PROXY_HOPS = max(1, int(os.environ.get("AIDND_TRUSTED_PROXY_HOPS", "1") or 1)) + + +def client_ip(request: Request) -> str: + """Returns the real client IP, resisting a spoofed `X-Forwarded-For`. + + The function reads the hop the trusted edge appended, which is the rightmost + entry minus any extra trusted hops. If no forwarded header is present, which + happens locally, in development, and on a direct connection, it falls back to + the socket peer. + + The function is public because the access log needs the same answer. Two + functions that each decide which address belongs to the caller is how one of + them ends up trusting a header it should not. + """ + forwarded = request.headers.get("x-forwarded-for") + if forwarded: + parts = [p.strip() for p in forwarded.split(",") if p.strip()] + if parts: + return parts[-min(TRUSTED_PROXY_HOPS, len(parts))] + return request.client.host if request.client else "unknown" + + +def rate_limit(scope: str, request: Request, user: models.User | None = None) -> None: + """Raises a 429 when the caller exceeds the scope's window. + + The window is keyed per user when a user is known, because an account + survives an IP change, and per IP otherwise. + """ + if not auth.MULTI_USER: + return + limit, window_seconds = RATE_LIMITS[scope] + key = (scope, f"u{user.id}" if user else f"ip{client_ip(request)}") + now = time.time() + with _windows_guard: + window = _windows[key] + while window and window[0] < now - window_seconds: + window.popleft() + if len(window) >= limit: + raise HTTPException( + 429, "You're doing that too fast — wait a minute and try again." + ) + window.append(now) + if len(_windows) > 10_000: + _prune(now) + + +# ---------- Per-account login throttle ---------- +# This is defense in depth next to the per-IP `auth` limit. A botnet dilutes +# that limit, because many real source IPs each get their own bucket, so it +# cannot by itself stop a distributed guessing run against one account. This cap +# keys on the target email rather than on the caller, so guessing one account's +# password stays expensive however many addresses the guesses come from. +# +# Only failures count, and a correct password clears the record. The window +# slides over a short period rather than locking the account, so a user who +# mistypes a few times recovers within minutes. The trade-off is that an +# attacker can keep a known account throttled, which is an inconvenience and is +# preferable to letting the account be brute-forced. +LOGIN_FAIL_LIMIT = 8 # Failed attempts per account. +LOGIN_FAIL_WINDOW = 900 # The window in seconds, which is 15 minutes. + +_login_fails: dict[str, deque] = defaultdict(deque) +_login_guard = threading.Lock() + + +def check_login_allowed(email: str) -> None: + """Raises a 429 when an account has too many recent failed logins. + + Call this before verifying the password, so that a guess never reaches the + hash. + """ + if not auth.MULTI_USER: + return + now = time.time() + with _login_guard: + window = _login_fails[email] + while window and window[0] < now - LOGIN_FAIL_WINDOW: + window.popleft() + if len(window) >= LOGIN_FAIL_LIMIT: + raise HTTPException( + 429, + "Too many failed sign-in attempts for this account — " + "wait a few minutes and try again.", + ) + + +def note_login_failure(email: str) -> None: + """Records one failed attempt against `email`.""" + if not auth.MULTI_USER: + return + now = time.time() + with _login_guard: + _login_fails[email].append(now) + if len(_login_fails) > 10_000: # Bound the map against a flood of unique emails. + stale = [ + key for key, window in _login_fails.items() + if not window or window[-1] < now - LOGIN_FAIL_WINDOW + ] + for key in stale: + del _login_fails[key] + + +def note_login_success(email: str) -> None: + """Clears the account's failure record after a correct password.""" + with _login_guard: + _login_fails.pop(email, None) + + +def _prune(now: float) -> None: + """Drops callers whose whole window has expired, so the per-IP dict stays bounded. + + Call this with the guard held. + """ + longest = max(seconds for _, seconds in RATE_LIMITS.values()) + stale = [key for key, window in _windows.items() + if not window or window[-1] < now - longest] + for key in stale: + del _windows[key] + + +# ---------- Per-user row caps ---------- + +MAX_ADVENTURES_PER_USER = 100 +MAX_SCENARIOS_PER_USER = 200 +MAX_SCRIPTS_PER_USER = 200 +MAX_STORY_CARDS_PER_OWNER = 200 # Per scenario or per adventure. +MAX_MEMORIES_PER_ADVENTURE = 1000 +MAX_ACTIONS_PER_ADVENTURE = 5000 +# Phase 14, SP6. A tree holds one branch per divergence somebody built a story +# on, so a tree with more branches than the story has turns came from a file +# rather than from play. The cap applies to imports only. Forking is a POST that +# adds one row and has no cap of its own, and the cap that matters there is +# `MAX_ACTIONS_PER_ADVENTURE` above. +MAX_BRANCHES_PER_ADVENTURE = 1000 + + +def check_row_cap( + kind: str, + db: Session, + user: models.User, + *, + adventure: models.Adventure | None = None, + scenario_id: int | None = None, + adventure_id: int | None = None, +) -> None: + """Raises a 409 when creating one more row of `kind` would exceed its cap. + + The caller has already checked ownership of the scenario or adventure passed + in. + """ + if not auth.MULTI_USER: + return + if kind == "adventures": + count = _count(db, models.Adventure, models.Adventure.user_id == user.id) + cap, subject, hint = ( + MAX_ADVENTURES_PER_USER, "adventures", + "delete one you no longer play to make room", + ) + elif kind == "scenarios": + count = _count(db, models.Scenario, models.Scenario.user_id == user.id) + cap, subject, hint = ( + MAX_SCENARIOS_PER_USER, "scenarios", "delete one to make room" + ) + elif kind == "scripts": + count = _count(db, models.Script, models.Script.user_id == user.id) + cap, subject, hint = ( + MAX_SCRIPTS_PER_USER, "scripts", "delete one to make room" + ) + elif kind == "story_cards": + owner_filter = ( + models.StoryCard.scenario_id == scenario_id + if scenario_id is not None + else models.StoryCard.adventure_id == adventure_id + ) + count = _count(db, models.StoryCard, owner_filter) + cap, subject, hint = ( + MAX_STORY_CARDS_PER_OWNER, "story cards here", "delete one to make room" + ) + elif kind == "memories": + count = _count(db, models.Memory, models.Memory.adventure_id == adventure.id) + cap, subject, hint = ( + MAX_MEMORIES_PER_ADVENTURE, "memories in this adventure", + "delete some to make room", + ) + elif kind == "actions": + # Count every action in the adventure, which is the whole tree rather + # than the path being played. That number is what costs storage, and + # nothing is pruned automatically, so it is the right one to cap. It does + # mean a heavily branched adventure reaches the cap while its story is + # shorter than the cap, which is why the message counts "actions in this + # adventure" rather than turns. + count = _count(db, models.Action, models.Action.adventure_id == adventure.id) + cap, subject, hint = ( + MAX_ACTIONS_PER_ADVENTURE, "actions in this adventure", + "export it and continue in a new adventure", + ) + else: # pragma: no cover. This is a programming error, not user input. + raise ValueError(f"Unknown row cap kind: {kind}") + if count >= cap: + raise HTTPException(409, f"You've reached the limit of {cap} {subject} — {hint}.") + + +def _count(db: Session, model, condition) -> int: + return db.query(func.count(model.id)).filter(condition).scalar() or 0 + + +_BUNDLE_LIST_CAPS = { + "story_cards": MAX_STORY_CARDS_PER_OWNER, + "memories": MAX_MEMORIES_PER_ADVENTURE, + "actions": MAX_ACTIONS_PER_ADVENTURE, + "branches": MAX_BRANCHES_PER_ADVENTURE, +} + + +def check_bundle_lists(**lists) -> None: + """Raises a 409 when an import bundle's lists exceed the caps live creation uses. + + The keyword arguments are `story_cards`, `memories`, `actions`, and + `branches`. + """ + if not auth.MULTI_USER: + return + for name, value in lists.items(): + cap = _BUNDLE_LIST_CAPS[name] + if isinstance(value, list) and len(value) > cap: + noun = name.replace("_", " ") + raise HTTPException( + 409, f"This file contains {len(value)} {noun} — the limit is {cap}." + ) + + +# ---------- Request body size ---------- +# The limit is generous enough for the largest legitimate payload, which is an +# adventure export holding thousands of actions. It applies in every mode, and no +# honest request approaches it. + +MAX_BODY_BYTES = 2 * 1024 * 1024 +MAX_IMPORT_BODY_BYTES = 20 * 1024 * 1024 + + +class BodySizeLimitMiddleware: + """Rejects oversized request bodies by their declared `Content-Length`. + + This is pure ASGI rather than `BaseHTTPMiddleware`, so SSE responses stream + through unchanged. A chunked upload with no length is refused, because every + real client of this API sends `Content-Length`, including browser fetch and + curl with a file. + """ + + def __init__(self, app): + self.app = app + + async def __call__(self, scope, receive, send): + if scope["type"] == "http" and scope.get("method") in ("POST", "PUT", "PATCH"): + headers = {k.decode("latin-1").lower(): v.decode("latin-1") + for k, v in scope.get("headers", [])} + limit = ( + MAX_IMPORT_BODY_BYTES + if scope.get("path", "").endswith("/import") + else MAX_BODY_BYTES + ) + length = headers.get("content-length") + problem = None + if length is None: + if "chunked" in headers.get("transfer-encoding", "").lower(): + problem = (411, "Content-Length is required.") + else: + try: + if int(length) > limit: + problem = ( + 413, + f"Request too large (limit {limit // (1024 * 1024)} MB).", + ) + except ValueError: + problem = (400, "Invalid Content-Length.") + if problem: + await _send_json_error(send, *problem) + return + await self.app(scope, receive, send) + + +async def _send_json_error(send, status: int, detail: str) -> None: + body = json.dumps({"detail": detail}).encode() + await send({ + "type": "http.response.start", + "status": status, + "headers": [(b"content-type", b"application/json"), + (b"content-length", str(len(body)).encode())], + }) + await send({"type": "http.response.body", "body": body}) diff --git a/backend/app/main.py b/backend/app/main.py new file mode 100644 index 0000000..dbce939 --- /dev/null +++ b/backend/app/main.py @@ -0,0 +1,176 @@ +import os +from contextlib import asynccontextmanager +from pathlib import Path + +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware +from fastapi.staticfiles import StaticFiles +from starlette.exceptions import HTTPException as StarletteHTTPException + +from . import analytics, cleanup +from .auth import MULTI_USER +from .database import engine +from .limits import BodySizeLimitMiddleware +from .migrations import bootstrap +from .routers import ( + adventures, analytics as analytics_router, auth, chat, debug, scenarios, scripts, + settings, story_cards, +) +from .seed import seed_public_scenarios + +bootstrap(engine) +seed_public_scenarios(engine) + +# Production serves the SPA same-origin, so CORS only matters for the Vite dev +# server; AIDND_CORS_ORIGINS overrides for any other cross-origin setup. +CORS_ORIGINS = [ + o.strip() + for o in os.environ.get("AIDND_CORS_ORIGINS", "").split(",") + if o.strip() +] or ["http://localhost:5173", "http://127.0.0.1:5173"] + +@asynccontextmanager +async def lifespan(_app: FastAPI): + # Sweeps once on boot, then on an interval. Booting is the reliable + # trigger on Render's free tier, where the service sleeps after ~15 + # minutes and a long-running timer rarely gets to fire. + sweeper = cleanup.start_sweeper() + # Visit counters are buffered in memory and written in batches; this is + # what turns them into rows, and stop_flusher writes out the last batch so + # a deploy doesn't drop it. + flusher = analytics.start_flusher() + try: + yield + finally: + await cleanup.stop_sweeper(sweeper) + await analytics.stop_flusher(flusher) + + +# The interactive API docs stay local-only: in multi-user mode they just hand +# strangers a map of the API surface. +app = FastAPI( + title="AI D&D", + docs_url=None if MULTI_USER else "/docs", + redoc_url=None, + openapi_url=None if MULTI_USER else "/openapi.json", + lifespan=lifespan, +) + +app.add_middleware( + CORSMiddleware, + allow_origins=CORS_ORIGINS, + allow_methods=["*"], + allow_headers=["*"], +) + +app.add_middleware(BodySizeLimitMiddleware) + + +class SecurityHeadersMiddleware: + """Standard hardening headers on every response. Pure ASGI (wraps `send`) + so SSE streams pass through unbuffered. The CSP allows exactly what the + SPA uses: same-origin everything, inline styles (React), Google Fonts.""" + + _HEADERS = [ + (b"x-content-type-options", b"nosniff"), + (b"referrer-policy", b"same-origin"), + (b"x-frame-options", b"DENY"), + ( + b"content-security-policy", + b"default-src 'self'; " + b"script-src 'self'; " + b"style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; " + b"font-src https://fonts.gstatic.com; " + b"img-src 'self' data:; " + b"connect-src 'self'; " + b"frame-ancestors 'none'", + ), + ] + + def __init__(self, app): + self.app = app + + async def __call__(self, scope, receive, send): + if scope["type"] != "http": + return await self.app(scope, receive, send) + + async def send_with_headers(message): + if message["type"] == "http.response.start": + message.setdefault("headers", []) + message["headers"] = list(message["headers"]) + self._HEADERS + await send(message) + + await self.app(scope, receive, send_with_headers) + + +class ApiErrorMiddleware: + """Counts failed API responses for the analytics dashboard. + + This is middleware rather than an exception handler, because it observes + what the client received. A 429 from a rate limiter, a 404 from routing, and + a 500 from a handler that never returned all reach it the same way. It is + pure ASGI for the same reason as the headers above: an SSE turn must not be + buffered on its way out. It watches `/api` only, because a 404 on the SPA + mount is a page load rather than a fault. + """ + + def __init__(self, app): + self.app = app + + async def __call__(self, scope, receive, send): + if scope["type"] != "http" or not scope.get("path", "").startswith("/api"): + return await self.app(scope, receive, send) + + async def send_counting(message): + if message["type"] == "http.response.start" and message["status"] >= 400: + # The router has already put the matched route on the scope by + # the time a response starts, so the label can name the + # endpoint rather than the caller's path. + analytics.record( + analytics.M_ERROR, + analytics.api_route_label(scope, message["status"]), + ) + await send(message) + + await self.app(scope, receive, send_counting) + + +app.add_middleware(ApiErrorMiddleware) +app.add_middleware(SecurityHeadersMiddleware) + +app.include_router(auth.router) +app.include_router(scenarios.router) +app.include_router(adventures.router) +app.include_router(story_cards.router) +app.include_router(scripts.router) +app.include_router(settings.router) +app.include_router(chat.router) +app.include_router(debug.router) +app.include_router(analytics_router.router) + + +@app.get("/api/health") +def health(): + return {"ok": True} + + +# In production, serve the built frontend (frontend/dist) as static files. +class SPAStaticFiles(StaticFiles): + """Serve index.html for unknown paths so client-side routes (/play/3) + survive a page reload. API routes are matched before this mount.""" + + async def get_response(self, path, scope): + try: + response = await super().get_response(path, scope) + except StarletteHTTPException as exc: + if exc.status_code != 404: + raise + return await super().get_response("index.html", scope) + if response.status_code == 404: + return await super().get_response("index.html", scope) + return response + + +frontend_dist = Path(__file__).resolve().parent.parent.parent / "frontend" / "dist" +if frontend_dist.is_dir(): + app.mount("/", SPAStaticFiles(directory=frontend_dist, html=True), name="frontend") diff --git a/backend/app/memorybank.py b/backend/app/memorybank.py new file mode 100644 index 0000000..69d87b4 --- /dev/null +++ b/backend/app/memorybank.py @@ -0,0 +1,828 @@ +"""Phase 6: automatic summarization and the embedding memory bank. + +This follows AI Dungeon's memory system. See +help.aidungeon.com/faq/the-memory-system. + +After each turn, `run_post_turn` runs as a fire-and-forget task with its own +database session. It does three things: + +- Every `MEMORY_INTERVAL` actions, starting once the adventure reaches + `MEMORY_START` actions, it summarizes each uncovered block of actions into a + short memory. A block waits until `SETTLE_SLACK` actions sit past it, so a + memory never ends on the action a retry could replace. +- Every `SUMMARY_INTERVAL` actions, it rewrites the story summary to include the + new memories. The rewrite always starts from the text the user edited and + never discards it. +- It embeds new memories through an OpenAI-compatible `/v1/embeddings` endpoint, + then evicts the bank down to its capacity. Evicted memories are marked as + forgotten and kept so that the UI can still show them. + +When the app generates a turn, `retrieve_memories` embeds the recent story text +and ranks the bank by cosine similarity. The highest-ranked memories become the +Memories section of the context. + +Every AI call in this module is best-effort. A failure is logged to the debug +page and retried on a later turn, because the cursors advance only after a call +succeeds. +""" + +import asyncio +from array import array +from collections import OrderedDict + +from sqlalchemy import func, select, update +from sqlalchemy.orm import Session, defer, object_session + +from . import models, tree, vectors +from .context import ( + cursors, + history, + lineage, + match_cards, + story_actions, + truncate_to_last_tokens, +) +from .database import SessionLocal +from .providers import OpenAICompatibleProvider, ProviderError +from .vectors import cosine # re-exported: the ranking lives here, the maths there + +MEMORY_INTERVAL = 6 # actions per memory +MEMORY_START = 12 # first memory once the adventure reaches this many actions +SUMMARY_INTERVAL = 15 # actions between Story Summary updates +MAX_MEMORIES_PER_RUN = 5 # cap catch-up work (e.g. imported adventures) per turn +MAX_EMBED_BATCH = 32 +RETRIEVAL_WINDOW_TOKENS = 600 # recent story text used as the similarity query +RETRIEVAL_WINDOW_ACTIONS = 4 # ...taken from this many of the newest actions +SUMMARY_MAX_WORDS = 250 +MEMORY_EXCERPT_TOKENS = 2000 # of the block, when a block is longer than this + +# How much story has to sit past a block before that block is summarized. +# +# This is not the pre-SP4 holdback returning. That rule made the newest action +# invisible to the summarizer everywhere, and it existed because a retry +# rewrote an action's text in place, so a memory covering the tip could end up +# describing narration the player had already replaced. Sibling attempts and +# `forget_node` settled that, and correctness does not rest on the number +# below: a delete or an undo anywhere in the story is still repaired by +# withdrawing what the coordinate produced. +# +# This is a cost rule, and it is only about the tip. Retry and take-switching +# both refuse anything but the newest action (`takes.retry_action`, +# `takes.switch_take`), and both withdraw the derived work at the coordinate +# they change. So a memory whose block ends on the tip is the one memory a +# player can still throw away, and every retry of that turn pays for it twice: +# once to write it, once to write it again. A block closes every +# MEMORY_INTERVAL actions and a normal turn writes two of them, so that is one +# turn in three. +# +# One action of slack moves the block end out of reach of both endpoints, and +# it buys nothing back in context quality: the block that just closed is still +# in the history window in full, so a memory of it tells the model what it can +# already read. Memories earn their place once the raw text has scrolled out, +# which is never the turn the block closed. +SETTLE_SLACK = 1 + +# ---- The cast brief (Phase 18b) ---- +# How many characters the brief names, and how much of each description it +# carries. The cast is authored content rather than generated, so it is small in +# practice; these are ceilings against a scenario with a very large cast, not a +# budget anyone is expected to reach. +MAX_CAST_MEMBERS = 8 +CAST_ENTRY_CHARS = 240 +SETTING_TOKENS = 300 # of `adventure.memory`, the plot essentials + +# A ceiling on one memory, in words. Measured, not guessed: with only +# "1-2 plain sentences" to go on, a real model wrote 34 words for one block and +# 105 for the next, and a 105-word memory is a paragraph. Five of those are +# injected per turn at the default `memory_top_k`, so the bank's cost is set +# here. A stated number also holds the length steady between memories, which is +# the same consistency the framing rule buys for the wording. +MEMORY_MAX_WORDS = 50 + +# The framing rule is the larger half of this prompt, and it is worth the +# tokens. Without it the model chooses a person per call, so one bank ends up +# holding "You entered the crypt", "The player entered the crypt" and "He +# entered the crypt" for the same kind of event. +# +# The reason is stated, not just the rule. A memory is retrieved in isolation +# months of story later, and a model told why bare pronouns fail complies far +# more consistently than one handed a bare instruction. +# +# The protagonist's name lives in the user message, in the Cast, rather than +# here. That keeps this prompt constant across every adventure and every call. +MEMORY_SYSTEM_PROMPT = ( + "You compress interactive-fiction story excerpts into memories. Respond with " + "1-2 plain sentences in past tense stating the concrete facts and events " + f"(names, places, items, promises, injuries), in at most {MEMORY_MAX_WORDS} " + "words. Keep the details a later scene could turn on — a name, a promise, an " + "injury, where something is — and drop the ones it could not.\n\n" + "Write in the third person. The excerpt is written in the second person: " + '"you" is the protagonist, who is named in the Cast. Refer to the ' + 'protagonist by that name, never as "you". If the Cast gives no name for ' + 'them, call them "the player". Name the other characters too rather than ' + 'writing "he", "she" or "they" on their own — this memory will be read on ' + "its own, much later, with nothing around it to say who a pronoun meant.\n\n" + "No preamble, no commentary." +) +SUMMARY_SYSTEM_PROMPT = ( + "You maintain the running summary of an interactive-fiction story. Respond " + "with only the updated summary: a single plain-prose overview of the plot " + f"so far, at most {SUMMARY_MAX_WORDS} words. Preserve important established " + "facts; compress older events harder than recent ones.\n\n" + "Write in the third person, and name the characters. Refer to the " + 'protagonist by the name given in the Cast, or as "the player" if the Cast ' + 'gives no name. Never address them as "you".' +) + +# Adventures with a post-turn task currently running (single-process app). +_running: set[int] = set() +# Strong references to tasks that are still running. The event loop holds only +# weak references, so without this set a fire-and-forget task can be garbage +# collected before it finishes. +_tasks: set[asyncio.Task] = set() + + +# Both factories below use the user's own key by construction. They read the +# endpoint and key from `Settings` and never from `auth.DEMO_*`, so +# summarization and embedding cannot spend the shared demo key. Their call sites +# are also skipped when `using_demo` is true. +# +# Do not change these to accept a `ProviderConfig`. `summary_model` and +# `embedding_model` are free-form user input and are not on the demo allowlist. +def summary_provider(settings: models.Settings) -> OpenAICompatibleProvider: + return OpenAICompatibleProvider( + settings.endpoint_url, + settings.api_key_plain, + settings.summary_model or settings.model, + settings.api_mode, + settings.reasoning_max_tokens, + ) + + +def embedding_provider(settings: models.Settings) -> OpenAICompatibleProvider: + return OpenAICompatibleProvider( + settings.endpoint_url, settings.api_key_plain, settings.embedding_model + ) + + +def set_vector(memory: models.Memory, vector: list[float] | None) -> None: + """Store (or clear) a memory's embedding. + + This function updates both columns that describe the vector. The ranking + reads `embedding_blob`, and everything else reads the `embedded` flag. + Routing every write through one function keeps the two in step, and it makes + this the only place a stored vector changes. That is why the cache below can + be invalidated here and nowhere else. + + One caller cannot use this function: the bulk clear in + `routers/settings.py` that runs when the embedding model changes. It sets + the same two columns directly. See the comment there. + """ + memory.embedding_blob = None if vector is None else vectors.pack(vector) + memory.embedded = vector is not None + cached = _vector_cache.get(memory.adventure_id) + if cached is not None: + cached.pop(memory.id, None) + + +# ---------- The vector cache ---------- + +# Maps an adventure id to a dict of memory id to vector, with the +# most recently used adventure last. +# +# Turns for one adventure arrive one after another, and the bank changes little +# between them. Reading every vector on each turn fetches the same 600 KB +# repeatedly. This cache stores vectors as `array("f")`, which uses 4 bytes per +# component and matches the 6 KB the column holds. A list of Python floats would +# use eight times as much. +# +# Two rules keep the cache correct. Any code that changes a vector calls +# `set_vector`, which removes that entry. Any code that removes a memory from +# play removes it from the catalogue query below, and the next read discards +# entries that the catalogue no longer lists. Eviction, deletion, pruning, and +# an edit that clears the vector all work this way. No code path has to remember +# to invalidate the cache, which is the error this design avoids. +# +# The cache lives in the process, so it assumes one worker, which is what the +# deploy runs. With two workers, each keeps its own copy. Both stay correct +# about eviction and deletion, but a vector rewritten by one worker can remain +# stale in the other until that memory leaves the catalogue. +_vector_cache: OrderedDict[int, dict[int, array]] = OrderedDict() +VECTOR_CACHE_ADVENTURES = 8 # ~600 KB each at a 100-memory bank + + +def forget_cached_vectors(adventure_id: int) -> None: + """Drops an adventure's cached vectors. + + Call this only when the adventure itself is deleted. Every other case + corrects itself, as described in the comment above. + """ + _vector_cache.pop(adventure_id, None) + + +def _vectors_for(db: Session, adventure_id: int, ids: list[int]) -> dict[int, array]: + """The vectors for `ids`, reading only the ones not already held.""" + cached = _vector_cache.get(adventure_id) + if cached is None: + cached = _vector_cache[adventure_id] = {} + _vector_cache.move_to_end(adventure_id) + while len(_vector_cache) > VECTOR_CACHE_ADVENTURES: + _vector_cache.popitem(last=False) + + wanted = set(ids) + for gone in set(cached) - wanted: + del cached[gone] + missing = [memory_id for memory_id in ids if memory_id not in cached] + if missing: + rows = db.execute( + select(models.Memory.id, models.Memory.embedding_blob) + .where(models.Memory.id.in_(missing)) + ).all() + for memory_id, blob in rows: + if blob: + cached[memory_id] = vectors.unpack(blob) + return cached + + +def forget_node(db: Session, adventure: models.Adventure, action: models.Action) -> int: + """Withdraw what a node produced, because the node is being removed. + + Call this before deleting `action`, which undo and the delete-action + endpoint both do. A memory attaches to the node its block ends on, so + finding the memories that describe a node is a lookup on `(branch_id, + depth)`. The earlier `prune_dangling_memories` instead scanned for rows + whose covered range no longer existed, so it could only detect the problem + after it occurred. + + Deleting the memory is half the work. The stretch of story it covered still + sits behind the cursors. Without a rewind, those actions count as + summarized while nothing describes them, and nothing reports the problem for + the rest of the adventure. `source_start` records where that stretch began, + so the anchor moves to the node before it. That depth is valid whether or + not a node still occupies it. + + The opening node is the one exception, because migration 62 placed the whole + pre-coordinate bank on it. See the comment on `lineage.ROOT_DEPTH`. + + Returns the number of memories withdrawn. + """ + if action.branch_id is None or action.depth is None: + return 0 # A pre-tree row. No path contains it, so nothing refers to it. + doomed = ( + db.query(models.Memory) + .filter( + models.Memory.adventure_id == adventure.id, + models.Memory.branch_id == action.branch_id, + models.Memory.depth == action.depth, + ) + .all() + ) + if action.depth == lineage.ROOT_DEPTH: + # Special case for the opening node, and only for memories that + # describe no stretch of story. Migration 62 placed every memory written + # before memories had coordinates at depth 0. That choice preserved + # every memory, but it also placed them all on one node, so withdrawing + # that node would delete a player's entire bank in one action. + # + # A memory with no `source_start` was either typed by the player or + # migrated. It describes no actions, so no deletion can invalidate it, + # and it stays. A memory that genuinely summarizes a block ending here is + # still withdrawn, because the text it describes is being deleted. + doomed = [m for m in doomed if m.source_start is not None] + if not doomed: + return 0 + starts = [m.source_start for m in doomed if m.source_start is not None] + for memory in doomed: + db.delete(memory) + if starts: + cursors.rewind_all(adventure, action.branch_id, min(starts) - 1) + return len(doomed) + + +def source_block(db: Session, memory: models.Memory) -> list[models.Action]: + """The actions a memory was written from, oldest first. + + The inverse of what `_create_due_memories` recorded. `source_start` and + `source_end` are depths, and `branch_id` says which path they are depths + on — that branch's own lineage, not the adventure's current path. A memory + written before a fork must still read back from the branch it was written + on, whichever branch the adventure has since moved to. + + Returns `[]` for a memory that describes no stretch of story. Those are + hand-written, or migrated from before memories had coordinates (see + `lineage.ROOT_DEPTH`), and there is no block to read. + + The range may come back shorter than `MEMORY_INTERVAL`. An action inside it + can have been deleted since, and a memory whose block is now partial still + describes the actions that remain. + """ + if memory.source_start is None or memory.source_end is None: + return [] + if memory.branch_id is None: + return [] # A pre-tree row: no path contains it. + branch = db.get(models.Branch, memory.branch_id) + if branch is None: + return [] + path = lineage.Path(lineage.entries_of(branch)) + rows = ( + db.query(models.Action) + .filter( + models.Action.adventure_id == memory.adventure_id, + # This clause excludes the sibling attempts at a retried turn, so + # the block holds the one text the story used. + path.clause(models.Action), + models.Action.depth >= memory.source_start, + models.Action.depth <= memory.source_end, + ) + # `id` breaks a tie on `depth`, as everywhere else that orders actions. + .order_by(models.Action.depth, models.Action.id) + # Reasoning traces are never part of an excerpt and can outweigh the + # narration on a reasoning model. + .options(defer(models.Action.reasoning)) + .all() + ) + return [a for a in rows if history.is_story_text(a.text)] + + +# ---------- The cast brief ---------- + +def _cast_line(name: str, entry: str, *, protagonist: bool = False) -> str: + """One roster line: who they are, and nothing about where they stand now.""" + who = f"{name} — the protagonist" if protagonist else f"{name} —" + entry = " ".join(entry.split()) # collapse newlines: this is a one-line roster + if len(entry) > CAST_ENTRY_CHARS: + entry = entry[:CAST_ENTRY_CHARS].rsplit(" ", 1)[0] + "…" + if protagonist: + return f"- {who}." if not entry else f"- {who}. {entry}" + return f"- {name} — {entry}" if entry else f"- {name}" + + +def cast_brief(adventure: models.Adventure, text: str) -> str: + """Returns who appears in `text`, and what the story is about. + + This is the context the summarizer never had. It was handed six actions of + second-person prose and nothing else, so the only honest memory it could + write for `You push the door open. She grabs your arm.` was "You entered a + room and she stopped you." — which, retrieved forty turns later, names + nobody. + + Three rules hold this together. + + **Fixed descriptions only, never live values.** It is tempting to add + `Gwen: trust 40 (wary)`. That would make the same event summarized at two + different times come out framed differently, which is the fault this whole + change exists to remove. + + **The cast comes from the story cards, not from `stat_schema`.** Every NPC a + scenario defines is already turned into a story card at adventure creation + (`scenario_text.scenario_card_specs`), deduplicated against the hand-written + ones by name. Reading the cards therefore covers the schema NPCs, the + author's own cards, and an adventure with no RPG layer at all, through one + path instead of three. + + **Keyword matching alone is not enough here, which is why the roster is + topped up.** The turn prompt includes a card only when its trigger words + appear, and that is right for lore: a card nobody mentioned is not relevant + to the next sentence. It is wrong for this brief. The block that most needs + a cast is exactly the one written in bare pronouns — "she grabs your arm" + matches no keyword, and the summarizer is then left guessing at precisely + the moment it was given this brief to stop guessing. So matched cards come + first, and any remaining slots are filled with the other **character** + cards. Places and items are not topped up: an unmentioned tavern is not + who "she" was. + + Walking `adventure.story_cards` is a relationship load, which this module + otherwise avoids. It is affordable here for two reasons the memory bank's + own reads were not: a card is five short text columns with no vector, and + this runs once per `MEMORY_INTERVAL` actions in the background task rather + than on every turn. `build_context` already walks the same collection. + """ + lines: list[str] = [] + name = adventure.persona_name.strip() + pronouns = adventure.persona_pronouns.strip() + if name or adventure.persona_desc.strip(): + who = f"{name} ({pronouns})" if name and pronouns else (name or "The player") + lines.append(_cast_line(who, adventure.persona_desc.strip(), protagonist=True)) + + seen = {name.lower()} if name else set() + + def add(card_name: str, entry: str) -> bool: + """Adds one roster line. Returns False once the roster is full.""" + card_name = (card_name or "").strip() + if card_name and card_name.lower() not in seen: + seen.add(card_name.lower()) + lines.append(_cast_line(card_name, (entry or "").strip())) + return len(lines) < MAX_CAST_MEMBERS + + room = True + for card in match_cards(adventure.story_cards, text): + room = add(card["name"], card["entry"]) + if not room: + break + if room: + for card in adventure.story_cards: + if (card.type or "").strip().lower() != "character": + continue + if not add(card.name, card.entry): + break + + parts = [] + if lines: + parts.append("Cast:\n" + "\n".join(lines)) + setting = adventure.memory.strip() + if setting: + parts.append("Setting:\n" + truncate_to_last_tokens(setting, SETTING_TOKENS)) + return "\n\n".join(parts) + + +# ---------- Retrieval (runs inside the turn, before build_context) ---------- + +async def retrieve_memories( + adventure: models.Adventure, + settings: models.Settings, + *, + update_stats: bool, + exclude_action_id: int | None = None, +) -> dict | None: + """Returns the memories to inject, or None when the bank is off. + + The result is a dict of the form + `{"used": [{id, text, similarity, pinned}], "error": str | None}`. It is + None when the memory bank is disabled for this adventure. + + Set `update_stats` to True to increment the use counters. Only real turns + should do this, not the dry runs that Insights performs. + + `exclude_action_id` removes the action being retried from the similarity + query, so that a discarded attempt cannot influence which memories are + returned. + """ + if not adventure.memory_bank_enabled: + return None + if not settings.embedding_model.strip(): + return {"used": [], "error": "No embedding model configured in Settings."} + db = object_session(adventure) + if db is None: + return {"used": [], "error": None} + + # Select which memories are in play, and nothing else about them. This code + # used to walk `adventure.memories`, which loaded every row of the bank, + # including its vector. That cost about 31 KB per memory and about 3 MB per + # turn, which was 96% of everything a turn read. An id and a flag come to + # about eight bytes per row. + # + # The branch clause uses the whole lineage here rather than the window the + # story is read through. Retrieval exists to recall events from far back in + # the story, such as what happened forty turns ago. The full lineage stays + # affordable because memories are sparse, at roughly one per six actions, so + # even a heavily forked story returns only tens of small rows. + catalogue = db.execute( + select(models.Memory.id, models.Memory.pinned).where( + models.Memory.adventure_id == adventure.id, + lineage.path_of(db, adventure).clause(models.Memory), + models.Memory.forgotten.is_(False), + models.Memory.embedded.is_(True), + ) + ).all() + if not catalogue: + return {"used": [], "error": None} + + recent = history.tail(adventure, RETRIEVAL_WINDOW_ACTIONS, exclude_action_id) + query = truncate_to_last_tokens( + "\n\n".join(a.text for a in recent), RETRIEVAL_WINDOW_TOKENS + ) + if not query.strip(): + return {"used": [], "error": None} + + try: + [query_vec] = await embedding_provider(settings).embed([query]) + except ProviderError as exc: + return {"used": [], "error": str(exc)} + + held = _vectors_for(db, adventure.id, [memory_id for memory_id, _ in catalogue]) + scored = sorted( + ( + (cosine(query_vec, held[memory_id]), memory_id, pinned) + for memory_id, pinned in catalogue + if memory_id in held + ), + key=lambda row: row[0], + reverse=True, + ) + # Pinned memories are always used, and they count toward `top_k`, so the + # injected set stays within the budget unless the pinned memories alone + # exceed it. + top_k = max(1, settings.memory_top_k) + used = [row for row in scored if row[2]] + remaining = max(0, top_k - len(used)) + used += [row for row in scored if not row[2]][:remaining] + used.sort(key=lambda row: row[0], reverse=True) + if not used: + return {"used": [], "error": None} + + # Fetch the text only now, and only for the `top_k` rows that were chosen. + used_ids = [memory_id for _, memory_id, _ in used] + texts = dict( + db.execute( + select(models.Memory.id, models.Memory.text) + .where(models.Memory.id.in_(used_ids)) + ).all() + ) + + if update_stats: + # Pass `synchronize_session=False` because nothing in this request + # reads the counters back. Matching the UPDATE against loaded objects + # would require loading those objects, which is the cost this code path + # exists to avoid. + db.execute( + update(models.Memory) + .where(models.Memory.id.in_(used_ids)) + .values(use_count=models.Memory.use_count + 1, last_used_at=models.utcnow()) + .execution_options(synchronize_session=False) + ) + + return { + "used": [ + {"id": memory_id, "text": texts.get(memory_id, ""), + "similarity": round(score, 4), "pinned": pinned} + for score, memory_id, pinned in used + ], + "error": None, + } + + +# ---------- Post-turn background work ---------- + +def schedule_post_turn(adventure: models.Adventure) -> None: + """Fire-and-forget summarization/embedding work after a turn is saved.""" + if not (adventure.auto_summarize or adventure.memory_bank_enabled): + return + if adventure.id in _running: + return + task = asyncio.get_running_loop().create_task(run_post_turn(adventure.id)) + _tasks.add(task) + task.add_done_callback(_tasks.discard) + + +async def run_post_turn(adventure_id: int) -> None: + if adventure_id in _running: + return + _running.add(adventure_id) + db = SessionLocal() + try: + adventure = db.get(models.Adventure, adventure_id) + if adventure is None: + return + # Settings are per-user (Phase 8): use the adventure owner's row. + settings = ( + db.query(models.Settings) + .filter(models.Settings.user_id == adventure.user_id) + .first() + ) + if settings is None: + return + # This code no longer clamps the cursors. Undo can leave the story + # shorter than the mark. When the mark was a position, a value past the + # end of the list stalled the pass until the story grew back, so every + # post-turn run clamped it. That clamp introduced its own error, because + # clamping to the settled count rewound a caught-up adventure by one + # step and covered an action twice. + # + # An anchor past the tip is not an invalid value. `settled_after` + # reports that there is nothing to do, and once the story grows past the + # anchor the pass resumes where it stopped. + if adventure.auto_summarize: + await _create_due_memories(adventure, settings, db) + await _update_story_summary(adventure, settings, db) + if adventure.memory_bank_enabled and settings.embedding_model.strip(): + await _embed_pending(adventure, settings, db) + _evict_over_capacity(adventure, settings, db) + finally: + db.close() + _running.discard(adventure_id) + + +async def summarize_block( + adventure: models.Adventure, + provider: OpenAICompatibleProvider, + block: list[models.Action], +) -> str: + """Writes one memory from one block of story. + + Both callers come through here, which is the point of the function. The + pass below writes a memory as the story reaches it; `tools/rewrite_memories` + rewrites one an older prompt produced. Assembling the prompt in two places + would mean a rewritten memory was written by a prompt that never shipped, + and nothing would report the difference. + + Raises `ProviderError`, which each caller handles its own way: the pass + below leaves the cursor alone and retries next turn, and the tool leaves the + old text in place and moves on. + """ + raw = "\n\n".join(a.text for a in block) + excerpt = truncate_to_last_tokens(raw, MEMORY_EXCERPT_TOKENS) + # Match the cast against the untruncated block. The excerpt is what the + # model reads, but a character named in the part that was trimmed is still + # one the memory may have to name. + brief = cast_brief(adventure, raw) + prompt = f"Story excerpt:\n\n{excerpt}\n\nMemory:" + return await provider.complete( + MEMORY_SYSTEM_PROMPT, f"{brief}\n\n{prompt}" if brief else prompt + ) + + +async def _create_due_memories( + adventure: models.Adventure, settings: models.Settings, db: Session +) -> None: + provider = summary_provider(settings) + for _ in range(MAX_MEMORIES_PER_RUN): + # Re-read the anchor on every pass. Committing a memory does not change + # the story, but this loop is the only code that moves the anchor, so + # both numbers must be current. + anchor = cursors.MEMORY.depth(db, adventure) + if history.count_after(adventure, anchor) < MEMORY_INTERVAL + SETTLE_SLACK: + return # No settled block of story sits past the mark. The block + # itself is still MEMORY_INTERVAL actions; the slack asks + # for story past its end. See `SETTLE_SLACK`. + if history.count(adventure) < MEMORY_START: + return # The adventure is too short to have started summarizing. + # The order of those two checks is deliberate. The usual answer is that + # no memory is due, and the first check settles that without measuring + # the length of the whole story. + block = history.after(adventure, anchor, MEMORY_INTERVAL) + if len(block) < MEMORY_INTERVAL: + return + try: + text = await summarize_block(adventure, provider, block) + except ProviderError: + return # Logged on the debug page. The cursor is unchanged, so the + # next turn retries this block. + if not text: + return + memory = models.Memory( + adventure_id=adventure.id, + text=text, + source_start=block[0].depth, + source_end=block[-1].depth, + ) + # Attach the memory to the node it summarizes, so that a fork inherits + # the memories of the path it forked from and no others. Then move the + # mark to that same node. Both values record how far this pass has + # reached, and taking them from one row keeps them in step even when the + # depths have gaps. + tree.attach_memory(memory, block[-1]) + db.add(memory) + cursors.MEMORY.anchor_at(adventure, block[-1]) + db.commit() + + +async def _update_story_summary( + adventure: models.Adventure, settings: models.Settings, db: Session +) -> None: + anchor = cursors.SUMMARY.depth(db, adventure) + uncovered = history.count_after(adventure, anchor) + if uncovered < SUMMARY_INTERVAL: + return + # Where the summary stands once this run succeeds. Read this before the AI + # call rather than after it. The mark records the end of the story as this + # pass saw it, and a turn that arrives during the call must not be counted + # as read. + caught_up = history.newest(adventure) + if caught_up is None: + return + + # Include the memories for the stretch that the summary has not read, which + # means every memory attached to a node past the anchor. The marks and the + # memories are now depths on one path, so no coordinate conversion remains. + # If memory creation has fallen behind, for example because the last attempt + # failed, this falls back to the raw story text. + new_events = db.execute( + select(models.Memory.text) + .where( + models.Memory.adventure_id == adventure.id, + lineage.path_of(db, adventure).clause(models.Memory), + models.Memory.depth > anchor, + ) + .order_by(models.Memory.depth) + ).scalars().all() + if new_events: + events_text = "\n".join(f"- {t}" for t in new_events) + else: + block = history.after(adventure, anchor, uncovered) + events_text = truncate_to_last_tokens("\n\n".join(a.text for a in block), 2000) + + current = adventure.story_summary.strip() + # The summary is built from the memories, so it inherits their framing for + # free once they are named and third-person. It still gets the brief of its + # own, because the fallback above hands it raw second-person story text + # whenever memory creation has fallen behind. + brief = cast_brief(adventure, f"{current}\n\n{events_text}") + user_prompt = ( + f"Current story summary:\n{current or '(none yet)'}\n\n" + f"New events since the last update:\n{events_text}\n\n" + "Updated summary:" + ) + if brief: + user_prompt = f"{brief}\n\n{user_prompt}" + try: + text = await summary_provider(settings).complete( + SUMMARY_SYSTEM_PROMPT, user_prompt, max_tokens=600 + ) + except ProviderError: + return + if not text: + return + adventure.story_summary = text + cursors.SUMMARY.anchor_at(adventure, caught_up) + db.commit() + + +async def _embed_pending( + adventure: models.Adventure, settings: models.Settings, db: Session +) -> None: + # Use a query rather than walking `adventure.memories`. That walk ran on + # every turn and loaded the whole bank's vectors in order to find the few + # rows with none. + # + # Neither this query nor the eviction below applies a branch clause, and + # that is deliberate. Whether a row is embedded is a fact about the row, not + # about the path being played. Skipping a sibling branch's memories would + # only postpone the work until someone switched branches and needed them + # ranked. Capacity works the same way. The bank belongs to the adventure, + # and the memories of a branch nobody is reading are the right ones to evict + # first. + pending = ( + db.query(models.Memory) + .filter( + models.Memory.adventure_id == adventure.id, + models.Memory.embedded.is_(False), + models.Memory.forgotten.is_(False), + ) + .order_by(models.Memory.id) + .limit(MAX_EMBED_BATCH) + .all() + ) + if not pending: + return + try: + new = await embedding_provider(settings).embed([m.text for m in pending]) + except ProviderError: + return + for memory, vector in zip(pending, new): + set_vector(memory, vector) + db.commit() + + +def _evict_over_capacity( + adventure: models.Adventure, settings: models.Settings, db: Session +) -> None: + # The database performs both the count and the ranking, and returns neither + # the rows nor the vectors. Counting by walking `adventure.memories` fetched + # every vector in the bank on every turn, whether or not the bank was over + # capacity. + in_this_bank = (models.Memory.adventure_id == adventure.id, + models.Memory.forgotten.is_(False)) + active = db.execute( + select(func.count(models.Memory.id)).where(*in_this_bank) + ).scalar() or 0 + overflow = active - max(1, settings.memory_bank_capacity) + if overflow <= 0: + return + # Evict the least recently used memory first, and use the use count only to + # break ties. + # + # Ordering by use count first froze the bank. A memory written on this turn + # has never been used, so once every other memory had been retrieved at + # least once, the new memory held the lowest count in the bank. The same + # post-turn run that wrote it then evicted it, one pass after embedding it. + # Use counts only increase, so the bank never recovered. An adventure kept + # whatever memories it held when the bank first filled, and every later + # memory was summarized, marked as forgotten, and never ranked. + # + # Ordering by recency avoids that. A new memory carries the newest + # timestamp, so it is the last row to be evicted rather than the first, and + # it remains until other memories are used. Demoting the use count costs + # little, because retrieving a useful memory also makes it recent. The two + # orderings differ only for memories that were used once and have not been + # retrieved since, which are the rows a full bank should evict. + doomed = db.execute( + select(models.Memory.id) + .where(*in_this_bank, models.Memory.pinned.is_(False)) + .order_by( + func.coalesce(models.Memory.last_used_at, models.Memory.created_at), + models.Memory.use_count, + ) + .limit(overflow) + ).scalars().all() + if not doomed: + return # Every active memory is pinned, so the pins override capacity. + db.execute( + update(models.Memory) + .where(models.Memory.id.in_(doomed)) + .values(forgotten=True) + .execution_options(synchronize_session=False) + ) + db.commit() + # The bulk UPDATE bypassed the loaded objects, so code that still holds the + # collection would otherwise see the evicted memories as active. + db.expire(adventure, ["memories"]) diff --git a/backend/app/migrations.py b/backend/app/migrations.py new file mode 100644 index 0000000..5270361 --- /dev/null +++ b/backend/app/migrations.py @@ -0,0 +1,1128 @@ +"""Lightweight versioned schema migrations. + +How it works: + +* `Base.metadata.create_all()` creates a fresh database, which is always + current, and stamps it with `LATEST_VERSION`. +* An existing database runs every migration whose version is greater than its + stored version, in order, and is then stamped. + +The version is stored in SQLite's `PRAGMA user_version`, or in a one-row +`schema_version` table on Postgres, which has no `PRAGMA`. + +To change the schema, update `models.py`, which keeps fresh databases current, +and append a `(version, sql)` pair here, which upgrades existing databases. Keep +migrations idempotent where that is cheap, such as with `IF NOT EXISTS`. +Migrations up to 23 predate Postgres support and use SQLite-only syntax. That is +safe, because every Postgres database starts fresh: `create_all` creates it, the +code stamps it `LATEST`, and it never replays those migrations. Migrations added +from Phase 9 on must run on both dialects. +""" + +import json +import re +from datetime import datetime + +from sqlalchemy import ( + JSON, Boolean, Column, DateTime, Integer, MetaData, String, Table, Text, + inspect, text, +) +from sqlalchemy.engine import Engine + +from . import compression, vectors +from .database import Base + +# Each entry is a version and the SQL to run when upgrading past it. Append to +# this list, and never reorder it. The SQL is a string, or a `{dialect: sql}` map +# with a `default` entry when the two dialects need different syntax, such as +# BLOB against BYTEA. +MIGRATIONS: list[tuple[int, str | dict[str, str]]] = [ + # Phase 6: auto-summarization and the memory bank. `create_all` creates the + # `memories` table itself, and it runs for existing databases too. + (2, "ALTER TABLE adventures ADD COLUMN auto_summarize BOOLEAN NOT NULL DEFAULT 0"), + (3, "ALTER TABLE adventures ADD COLUMN memory_bank_enabled BOOLEAN NOT NULL DEFAULT 0"), + (4, "ALTER TABLE adventures ADD COLUMN memory_cursor INTEGER NOT NULL DEFAULT 0"), + (5, "ALTER TABLE adventures ADD COLUMN summary_cursor INTEGER NOT NULL DEFAULT 0"), + (6, "ALTER TABLE settings ADD COLUMN summary_model VARCHAR(200) NOT NULL DEFAULT ''"), + (7, "ALTER TABLE settings ADD COLUMN embedding_model VARCHAR(200) NOT NULL DEFAULT ''"), + (8, "ALTER TABLE settings ADD COLUMN memory_bank_capacity INTEGER NOT NULL DEFAULT 200"), + (9, "ALTER TABLE settings ADD COLUMN memory_top_k INTEGER NOT NULL DEFAULT 5"), + # Repair duplicate action indexes. The player action and the AI action of + # one turn used to get the same index, so renumber each adventure's actions + # to 0..n-1 in their existing order. `UPDATE..FROM` computes the ranks as a + # snapshot before any row is rewritten. A correlated subquery would read + # partially updated rows and could produce duplicates again. + (10, """ + UPDATE actions SET "index" = ranked.new_index + FROM ( + SELECT id, ROW_NUMBER() OVER ( + PARTITION BY adventure_id ORDER BY "index", id + ) - 1 AS new_index + FROM actions + ) AS ranked + WHERE ranked.id = actions.id + """), + # Reasoning-model support: a separate thinking budget and stored reasoning + # text. + (11, "ALTER TABLE settings ADD COLUMN reasoning_max_tokens INTEGER NOT NULL DEFAULT 0"), + (12, "ALTER TABLE actions ADD COLUMN reasoning TEXT"), + # Phase 8: optional accounts. `create_all` creates the `users` table itself. + # These migrations move every pre-existing row under a local user with + # `id=1`, so a single-user install keeps working unchanged. + (13, """ + INSERT INTO users (id, email, password_hash, is_guest, created_at, + demo_turns_used, demo_turns_date) + SELECT 1, NULL, NULL, 0, CURRENT_TIMESTAMP, 0, '' + WHERE NOT EXISTS (SELECT 1 FROM users) + """), + (14, "ALTER TABLE scenarios ADD COLUMN user_id INTEGER REFERENCES users(id) ON DELETE CASCADE"), + (15, "UPDATE scenarios SET user_id = 1"), + (16, "ALTER TABLE scenarios ADD COLUMN is_public BOOLEAN NOT NULL DEFAULT 0"), + (17, "ALTER TABLE scripts ADD COLUMN user_id INTEGER REFERENCES users(id) ON DELETE CASCADE"), + (18, "UPDATE scripts SET user_id = 1"), + (19, "ALTER TABLE adventures ADD COLUMN user_id INTEGER REFERENCES users(id) ON DELETE CASCADE"), + (20, "UPDATE adventures SET user_id = 1"), + (21, "ALTER TABLE settings ADD COLUMN user_id INTEGER REFERENCES users(id) ON DELETE CASCADE"), + (22, "UPDATE settings SET user_id = 1"), + (23, "CREATE UNIQUE INDEX IF NOT EXISTS ix_settings_user_id ON settings (user_id)"), + # Link each adventure-script copy back to its library Script, so a player + # can re-sync it on demand. The column is NULL for a copy made before it + # existed. + (24, "ALTER TABLE adventure_scripts ADD COLUMN source_script_id INTEGER " + "REFERENCES scripts(id) ON DELETE SET NULL"), + # A per-action snapshot of the shared `script_state` as it was before that + # action's hooks ran, so undo and retry can roll the state back. JSON is + # valid on both SQLite and Postgres. + (25, "ALTER TABLE actions ADD COLUMN state_before JSON"), + # Phase 12: RPG world state. `stat_schema` defines a scenario's stats, + # bands, rules, and milestones. `world_state` holds an adventure's live + # values. `world_state_before` snapshots those values per action for undo and + # retry, the same way `state_before` does. JSON is valid on both SQLite and + # Postgres. + (26, "ALTER TABLE scenarios ADD COLUMN stat_schema JSON"), + (27, "ALTER TABLE adventures ADD COLUMN world_state JSON"), + (28, "ALTER TABLE actions ADD COLUMN world_state_before JSON"), + # Raise the default context budget from 4096 to 16384, because Phase 12 adds + # a stat guide and the world state to every turn. This updates only rows + # still on the old default, so anyone who chose a custom value keeps it. + (29, "UPDATE settings SET context_token_budget = 16384 WHERE context_token_budget = 4096"), + # Scenario cover art, either an external URL or an inline base64 data URI. + # The column is TEXT rather than VARCHAR, because a downscaled data URI runs + # to tens of kilobytes. + (30, "ALTER TABLE scenarios ADD COLUMN image TEXT NOT NULL DEFAULT ''"), + # The emoji or glyph fallback, used when `image` is empty. + (31, "ALTER TABLE scenarios ADD COLUMN icon VARCHAR(16) NOT NULL DEFAULT ''"), + # The `${Placeholder}` answers given when the adventure started. They are + # stored so that "Update from scenario" can fill the copied text again. The + # column is NULL for an adventure created before it existed, and that + # adventure prompts for the answers again on its first refresh. + (32, "ALTER TABLE adventures ADD COLUMN placeholders JSON"), + # Which part of the scenario a copied story card came from, either + # `card:` or `npc:`, so that a refresh updates and removes only the + # scenario-derived cards and leaves player-authored ones unchanged. NULL + # means the card is player-authored, or that the copy predates this column. + # A predating copy is matched by name once and then tagged. + (33, "ALTER TABLE story_cards ADD COLUMN source_ref VARCHAR(64)"), + # Retry history: every attempt made for an AI turn, oldest first, so that a + # retry appends rather than deletes. NULL means the turn was never retried + # and the row is its only version, which is also the correct reading for + # every action that predates this column. + (34, "ALTER TABLE actions ADD COLUMN variants JSON"), + (35, "ALTER TABLE actions ADD COLUMN variant_index INTEGER NOT NULL DEFAULT 0"), + # Egress: `context_snapshot` holds the whole assembled prompt, about 74 kB + # per row, and bulk reads loaded it for two small values: the world-change + # chips and the emit block replayed into history. Move that slice into its + # own column so the snapshot can be deferred. `_backfill_world_delta` fills + # the new column. + (36, "ALTER TABLE actions ADD COLUMN world_delta JSON"), + # Egress, part two: `variants` holds every discarded retry attempt, but a + # list response needs only how many there are. Store the count next to it so + # the column can be deferred. Otherwise each retry adds about 5 kB to every + # later load of that adventure. `_backfill_variant_count` fills the new + # column. + (37, "ALTER TABLE actions ADD COLUMN variant_count INTEGER NOT NULL DEFAULT 0"), + # Egress, part three: a 1536-dimension embedding written as a JSON list is + # about 31 kB, and ranking fetches the whole bank every turn. Packed float32 + # holds the same numbers exactly in 6 kB. See `vectors.py`. The dimensions do + # not change, so this is a format conversion with no re-embedding and no API + # calls. `_backfill_embedding_blob` fills the new column. The old JSON column + # stays in place and is still written until a later migration drops it. + (38, {"sqlite": "ALTER TABLE memories ADD COLUMN embedding_blob BLOB", + "default": "ALTER TABLE memories ADD COLUMN embedding_blob BYTEA"}), + # A one-bit flag next to the vector, so the Memories drawer and the embed + # queue can test whether a memory has a vector without fetching one. This + # matches `actions.variant_count` next to `actions.variants`. TRUE, FALSE, + # and a boolean DEFAULT use the same syntax on both dialects, and 0 and 1 + # would not. + (39, "ALTER TABLE memories ADD COLUMN embedded BOOLEAN NOT NULL DEFAULT false"), + (40, "UPDATE memories SET embedded = true WHERE embedding_blob IS NOT NULL"), + # Memory bank capacity from 200 to 80. Only rows still on the old default + # change, so anyone who chose a value keeps it. Migration 29 follows the same + # rule. An adventure already over 80 evicts down on its next turn. + (41, "UPDATE settings SET memory_bank_capacity = 80 WHERE memory_bank_capacity = 200"), + # Drop the JSON vectors. Migration 38 left them in place so a rollback could + # still find them. Production has since been verified reading from + # `embedding_blob`, at schema version 41 with 134 of 134 rows backfilled, so + # the column is 4 MB of a 99.6 MB database that nothing reads. DROP COLUMN + # uses the same syntax on both dialects, and SQLite has supported it since + # 3.35. + (42, "ALTER TABLE memories DROP COLUMN embedding"), + # Compress `context_snapshot`. One column holds 89% of the database. It + # stores assembled prompts that no query filters on and that one screen + # reads, one row at a time. Postgres already TOASTs the column, but pglz + # reaches only 1.7x, while zlib reaches three to four times on the same text. + # Deferring the column already fixed the read cost. This migration is about + # the 512 MB the free tier allows. + # + # This takes three steps, because a column cannot portably change type in + # place. Add the new column, convert into it with + # `_backfill_context_snapshot`, which verifies that every row round-trips + # before the old column is dropped, then swap the names so the model still + # calls it `context_snapshot`. + # + # Postgres does not release the disk space on its own. DROP COLUMN only + # marks the column dropped, and the backfill's UPDATE leaves one dead tuple + # per row, so the table grows before it shrinks. The peak is roughly twice + # the starting size while both columns exist. Autovacuum makes that space + # reusable but does not shrink the files. Run this once after the deploy + # that ships this migration: + # + # VACUUM FULL actions; + # + # That command needs exclusive access and free space equal to the finished + # table. On the 2026-08-17 figures, the database is 99.6 MB, peaks near 200 + # MB, and settles at about 53 MB once vacuumed, against a 512 MB tier. + # Skipping the vacuum is safe and leaves the saving unrealized. + (43, {"sqlite": "ALTER TABLE actions ADD COLUMN context_snapshot_z BLOB", + "default": "ALTER TABLE actions ADD COLUMN context_snapshot_z BYTEA"}), + (44, "ALTER TABLE actions DROP COLUMN context_snapshot"), + (45, "ALTER TABLE actions RENAME COLUMN context_snapshot_z TO context_snapshot"), + # Phase 14, SP1: the story becomes a tree. Every action gains the branch it + # was played on and its depth along that branch, adventures gain a head + # pointer, and memories attach to the node that produced them. `create_all` + # creates the `branches` table itself, as it did for `memories`. + # + # Nothing reads these columns yet. SP2 moves the reads onto them. This + # subphase exists so that every row already has the columns by the time + # anything reads them, including the rows written between the two deploys, + # which `app/tree.py` stamps. The legacy `index`, `variants`, + # `variant_index`, and `variant_count` columns stay in place, unread, until + # the tree is proven live. + # + # This rewrites every row of `actions` twice, once per ADD COLUMN backfill + # pass on Postgres, so run this once after the deploy that ships it: + # + # VACUUM FULL actions; + # + # Run it on the direct endpoint, not on -pooler. This is the 144 MB result + # from 2026-08-17: a rewrite roughly doubles the table, and only a VACUUM + # FULL releases the space. Skipping it is safe and leaves the table large. + (46, "ALTER TABLE actions ADD COLUMN branch_id INTEGER " + "REFERENCES branches(id) ON DELETE CASCADE"), + (47, "ALTER TABLE actions ADD COLUMN depth INTEGER"), + # `head_branch_id` has no REFERENCES clause. `branches.adventure_id` already + # points the other way, and a second constraint would make the pair a cycle + # that `create_all` cannot order. See the column comment in `models.py`. + (48, "ALTER TABLE adventures ADD COLUMN head_branch_id INTEGER"), + (49, "ALTER TABLE adventures ADD COLUMN head_depth INTEGER NOT NULL DEFAULT -1"), + (50, "ALTER TABLE memories ADD COLUMN branch_id INTEGER " + "REFERENCES branches(id) ON DELETE CASCADE"), + (51, "ALTER TABLE memories ADD COLUMN depth INTEGER"), + # The index every branch clause needs, plus the data pass that fills the six + # columns above. `_backfill_tree` runs at this version because it needs all + # six columns to exist. + (52, "CREATE INDEX IF NOT EXISTS ix_actions_branch_depth ON actions (branch_id, depth)"), + # Phase 14, SP3: the memory cursor and the summary cursor stop being + # positions in the story and become nodes in it, as the `(branch, depth)` of + # the last action each pass covered. The legacy `memory_cursor` and + # `summary_cursor` columns stay, unread, until SP8 drops them along with + # `actions.index`. + # + # Unlike migrations 46 to 52, this rewrites `adventures` rather than + # `actions`, which is a few hundred rows against a few hundred thousand, so + # it needs no VACUUM FULL of its own. The one SP1's deploy asks for is still + # outstanding. + (53, "ALTER TABLE adventures ADD COLUMN memory_cursor_branch_id INTEGER"), + (54, "ALTER TABLE adventures ADD COLUMN memory_cursor_depth INTEGER NOT NULL DEFAULT -1"), + (55, "ALTER TABLE adventures ADD COLUMN summary_cursor_branch_id INTEGER"), + (56, "ALTER TABLE adventures ADD COLUMN summary_cursor_depth INTEGER NOT NULL DEFAULT -1"), + # Phase 14, SP4: a retry stops rewriting a row and writes a sibling next to + # it. `live` records which sibling the story tells. `state_after` and + # `world_state_after` give each attempt its own outcome to switch back to. + # The legacy `variants`, `variant_index`, `state_before`, and + # `world_state_before` columns stay, unread, until SP8. + # + # This rewrites every row of `actions` three times, once per ADD COLUMN + # backfill pass on Postgres, and then inserts one row per discarded attempt, + # so run this once after the deploy that ships it: + # + # VACUUM FULL actions; + # + # Run it on the direct endpoint, not on -pooler. This is the same 144 MB + # result as SP1. + (57, "ALTER TABLE actions ADD COLUMN live BOOLEAN NOT NULL DEFAULT true"), + (58, "ALTER TABLE actions ADD COLUMN state_after JSON"), + (59, "ALTER TABLE actions ADD COLUMN world_state_after JSON"), + # The two data passes get a version of their own, so that they run after all + # three columns exist. `_backfill_state_after` derives the after-snapshots + # from the before-snapshots, and `_split_variants_into_siblings` splits each + # `variants` list into sibling rows. The statement below is the index those + # siblings would make worth having, but a group lookup is `(branch_id, + # depth)` over a few rows and `ix_actions_branch_depth` already serves it, so + # the statement does nothing and only gives the passes a version to run + # at. + (60, "CREATE INDEX IF NOT EXISTS ix_actions_branch_depth ON actions (branch_id, depth)"), + # Phase 14, SP7: a branch can be named. NULL means nobody named the branch, + # which is true of every branch that exists when this runs, so there is no + # backfill and nothing to derive. `branches` holds a few rows per adventure + # rather than one per turn, so unlike SP1 and SP4 this rewrite covers a few + # hundred rows against a few hundred thousand and needs no VACUUM FULL of + # its own. + (61, "ALTER TABLE branches ADD COLUMN name VARCHAR(80)"), + # Phase 14, SP7: every memory gets a node. A hand-written memory used to + # keep a NULL depth, which no fork could bound, so it followed the reader + # onto branches whose story it never described. A new memory anchors at the + # head. An existing memory lands at depth 0 of the branch it is on, which is + # the only choice that removes a memory from no path: 0 is at or before every + # fork point, so a memory stays visible from the same paths it is visible + # from today. Anchoring an existing memory at the tip would have removed it + # from every branch forked earlier than the memory was written. + (62, "UPDATE memories SET depth = 0 WHERE depth IS NULL"), + # Phase 14, SP9: the node a node was played after, so a turn's attempts can + # be grouped by parent rather than by coordinate. A coordinate stops + # identifying which attempts belong together as soon as one of them is forked + # onto its own branch, because it leaves its siblings behind and reads 1/1 + # next to their 1/3. + # + # The column is nullable, and the backfill leaves it NULL wherever it cannot + # place a row correctly. See `_backfill_parents`. `attempts.group` falls back + # to the coordinate for those rows, which is the rule they were written + # under. + (63, "ALTER TABLE actions ADD COLUMN parent_id INTEGER REFERENCES actions(id) ON DELETE SET NULL"), + (64, "CREATE INDEX IF NOT EXISTS ix_actions_parent ON actions (parent_id)"), + # Phase 17: `Settings.stream` was dead state. Nothing ever read it, and every + # turn streams. This is item S1 in `docs/self-review.md`. The table holds one + # row per user, so the rewrite is small and needs no VACUUM FULL. + (65, "ALTER TABLE settings DROP COLUMN stream"), + # Phase 17, SP8: drop the eight columns the story tree replaced. Each one was + # kept past the migration that superseded it so that a rollback found a real + # value on the rows the newer build wrote. The tree has run in production + # since Phase 14, so the rollback window is closed. + # + # `actions.index` was the global turn number. `depth` replaced it, and the + # last reader, the number issued to a new row, went with this migration. + # `variants` held the retry history as a repeating group, and `variant_index` + # and `variant_count` described that group's shape. Each attempt is its own + # row now, ordered by `id`. `state_before` and `world_state_before` recorded + # the state a node started from, which every attempt at a turn shares; the + # "after" columns record each attempt's own outcome instead. + # `adventures.memory_cursor` and `summary_cursor` were positions into a flat + # list, and the branch and depth pairs beside them replaced both. + # + # Dropping a column rewrites the toasted values on Postgres and nothing + # reclaims that space on its own. Run `VACUUM FULL actions;` on the direct + # Neon endpoint after the deploy, not the pooled one. It takes an ACCESS + # EXCLUSIVE lock, so the app blocks on `actions` while it runs. + (66, "ALTER TABLE actions DROP COLUMN variants"), + (67, "ALTER TABLE actions DROP COLUMN variant_index"), + (68, "ALTER TABLE actions DROP COLUMN variant_count"), + (69, "ALTER TABLE actions DROP COLUMN state_before"), + (70, "ALTER TABLE actions DROP COLUMN world_state_before"), + # `index` is a keyword in SQLite, so the column name is quoted. + (71, 'ALTER TABLE actions DROP COLUMN "index"'), + (72, "ALTER TABLE adventures DROP COLUMN memory_cursor"), + (73, "ALTER TABLE adventures DROP COLUMN summary_cursor"), + # Phase 18: the persona. Defaulting to '' rather than NULL means every read + # is a plain string test, and an adventure that predates the feature is + # indistinguishable from one whose player left the fields blank. + (74, "ALTER TABLE adventures ADD COLUMN persona_name VARCHAR(80) NOT NULL DEFAULT ''"), + (75, "ALTER TABLE adventures ADD COLUMN persona_pronouns VARCHAR(40) NOT NULL DEFAULT ''"), + (76, "ALTER TABLE adventures ADD COLUMN persona_desc TEXT NOT NULL DEFAULT ''"), +] + +LATEST_VERSION = max((v for v, _ in MIGRATIONS), default=1) + +# Migrations that need a data pass after their DDL, keyed by version. +WORLD_DELTA_VERSION = 36 +VARIANT_COUNT_VERSION = 37 +EMBEDDING_BLOB_VERSION = 38 +SNAPSHOT_COMPRESS_VERSION = 43 +TREE_BACKFILL_VERSION = 52 +CURSOR_ANCHOR_VERSION = 56 +SIBLING_SPLIT_VERSION = 60 +PARENT_BACKFILL_VERSION = 64 + +# An adventure with no actions has no tip. A value of -1 keeps the rule that the +# next node goes at `head_depth + 1` true without a special case. This matches +# `tree.NO_DEPTH`. +NO_DEPTH = -1 + +# How many snapshots are converted per round trip. This is much smaller than +# `BACKFILL_BATCH`, because a vector is 6 kB and a snapshot is 232 kB, so 200 +# snapshots would hold 46 MB at once. +SNAPSHOT_BATCH = 50 + +# How many vectors are converted per round trip. The value is small enough that +# the backfill never holds more than a few megabytes, and large enough that it +# does not run one query per row. +BACKFILL_BATCH = 200 + + +def _backfill_world_delta(conn) -> None: + """Populates `actions.world_delta` from the existing `context_snapshot`. + + The pass runs entirely on the server. The snapshots are the reason this + change exists, so reading about 40 MB of them into Python to rewrite one + slice would defeat its purpose. The SQL is dialect-specific, because SQLite + and Postgres use different JSON syntax and both have to work. SQLite runs + locally and in the tests. + """ + if conn.dialect.name == "sqlite": + sql = """ + UPDATE actions SET world_delta = json_object( + 'delta', json_extract(context_snapshot, '$.world_state.delta'), + 'applied', json_extract(context_snapshot, '$.world_state.report.applied') + ) + WHERE world_delta IS NULL + AND context_snapshot IS NOT NULL + AND json_extract(context_snapshot, '$.world_state') IS NOT NULL + """ + else: + sql = """ + UPDATE actions SET world_delta = jsonb_build_object( + 'delta', context_snapshot::jsonb #> '{world_state,delta}', + 'applied', context_snapshot::jsonb #> '{world_state,report,applied}' + ) + WHERE world_delta IS NULL + AND context_snapshot IS NOT NULL + AND jsonb_exists(context_snapshot::jsonb, 'world_state') + """ + conn.execute(text(sql)) + + +def _backfill_parents(conn) -> None: + """Points every node at the attempt it was played after. + + The pass makes one query, which reads the live node one depth back on the + same branch. That covers a linear story, and every adventure written before + SP9 is linear. Forking reached the screen in SP7, and the tree was found + unusable before anyone forked with it. + + The rows left NULL are the first node of a forked branch, whose parent is on + an ancestor branch and cannot be found without walking `lineage` for each + row. `attempts.group` falls back to the coordinate for a NULL parent, which + is the rule those rows were written under, so the fallback returns the + original answer for them rather than a worse one. Every fork made from SP9 on + sets `parent_id` at write time and does not use this pass. + + The query correlates to the row being updated rather than numbering the + table, so the planner uses `ix_actions_branch_depth`. This follows from + `_backfill_cursor_anchors`, which numbered every action once per adventure. + """ + conn.execute( + text( + """ + UPDATE actions SET parent_id = ( + SELECT prev.id FROM actions AS prev + WHERE prev.adventure_id = actions.adventure_id + AND prev.branch_id = actions.branch_id + AND prev.depth = actions.depth - 1 + AND prev.live = TRUE + ORDER BY prev.id + LIMIT 1 + ) + WHERE parent_id IS NULL + AND branch_id IS NOT NULL + AND depth IS NOT NULL + """ + ) + ) + + +def _backfill_variant_count(conn) -> None: + """Populates `actions.variant_count` from the existing `variants` list. + + The pass runs on the server for the same reason as `_backfill_world_delta`. + `variants` is the column this change removes from responses, so counting it + in Python would read every stored attempt over the network once in order to + stop reading it on every request. + """ + if not _has_columns(conn, "actions", "variants", "variant_count"): + return + if conn.dialect.name == "sqlite": + sql = """ + UPDATE actions SET variant_count = json_array_length(variants) + WHERE variants IS NOT NULL AND json_valid(variants) + """ + else: + sql = """ + UPDATE actions SET variant_count = jsonb_array_length(variants::jsonb) + WHERE variants IS NOT NULL + AND jsonb_typeof(variants::jsonb) = 'array' + """ + conn.execute(text(sql)) + + +def _for_dialect(sql: str | dict[str, str], dialect: str) -> str: + return sql if isinstance(sql, str) else sql.get(dialect, sql["default"]) + + +# Matches the ADD COLUMN migrations in this file. Every one is written above, +# so this pattern parses only SQL this file controls. +_ADD_COLUMN = re.compile(r"^\s*ALTER\s+TABLE\s+(\w+)\s+ADD\s+COLUMN\s+\"?(\w+)\"?", re.I) +_DROP_COLUMN = re.compile(r"^\s*ALTER\s+TABLE\s+(\w+)\s+DROP\s+COLUMN\s+\"?(\w+)\"?", re.I) + + +def _has_columns(conn, table: str, *columns: str) -> bool: + """Returns `True` when `table` has every one of `columns`. + + The data passes below read columns that later migrations drop. A pass only + ever has real work to do on a database old enough to still carry them, and + a `create_all` database is already current, so the guard skips the pass + rather than failing on a column that is not there. This is the same rule + `_column_already_there` applies to DDL, written for the passes. + """ + inspector = inspect(conn) + if table not in inspector.get_table_names(): + return False + present = {col["name"] for col in inspector.get_columns(table)} + return all(column in present for column in columns) + + +def _column_already_gone(conn, sql: str) -> bool: + """Returns `True` when `sql` drops a column the table no longer has. + + This is the counterpart to `_column_already_there`, for the same reason. + `create_all` builds the current schema, which is already missing every + column a migration drops. Replaying from an older stamp against a database + built that way would fail on a column that is not there. + """ + match = _DROP_COLUMN.match(sql) + if match is None: + return False + table, column = match.group(1), match.group(2) + inspector = inspect(conn) + if table not in inspector.get_table_names(): + return False + return column not in {col["name"] for col in inspector.get_columns(table)} + + +def _column_already_there(conn, sql: str) -> bool: + """Returns `True` when `sql` adds a column the table already has. + + This is the `IF NOT EXISTS` the module docstring asks for, written in Python + because SQLite has no syntax for it on ADD COLUMN. Without this check, a + database that carries a newer column than its stamp claims fails on a + duplicate column and never runs the backfill. That database is common: + `create_all` always builds the current schema, so it is what every test that + replays a migration starts from, and SQLite cannot drop those columns again + once a foreign key names them. + """ + match = _ADD_COLUMN.match(sql) + if match is None: + return False + table, column = match.group(1), match.group(2) + inspector = inspect(conn) + if table not in inspector.get_table_names(): + return False + return column in {col["name"] for col in inspector.get_columns(table)} + + +def _backfill_embedding_blob(conn) -> None: + """Repacks `memories.embedding`, a JSON list, into `memories.embedding_blob`. + + This is the only backfill here that runs through Python, because struct + packing has no portable SQL form. Unlike migrations 36 and 37, it pays a + one-time read of every vector, about 4 MB in production, to stop reading + three megabytes every turn. It runs in batches, so the read stays bounded + however large the bank grows. + + The pass reads the JSON defensively, because SQLite returns the raw string + while psycopg has already parsed it into a list. It skips any value that is + not a non-empty list, so one malformed row cannot block the migration. + """ + last_id = 0 + while True: + rows = conn.execute( + text(""" + SELECT id, embedding FROM memories + WHERE embedding IS NOT NULL AND embedding_blob IS NULL AND id > :last + ORDER BY id LIMIT :batch + """), + {"last": last_id, "batch": BACKFILL_BATCH}, + ).all() + if not rows: + return + for row_id, stored in rows: + vector = json.loads(stored) if isinstance(stored, str) else stored + if not isinstance(vector, list) or not vector: + continue + conn.execute( + text("UPDATE memories SET embedding_blob = :blob WHERE id = :id"), + {"blob": vectors.pack(vector), "id": row_id}, + ) + last_id = rows[-1][0] + + +def _backfill_context_snapshot(conn) -> None: + """Compresses `actions.context_snapshot` into `actions.context_snapshot_z`. + + This runs between migrations 43 and 44, which is the only point where both + columns exist. Migration 44 drops the original column, so unlike every other + backfill here this one destroys data if it is wrong, and every row it + converts belongs to somebody's game. Each row is therefore decompressed again + and compared with the input before it counts as converted. A row that fails + to round-trip aborts the run rather than being skipped: the transaction rolls + back, the DROP never runs, and the prompts are still there for another + attempt. + + The pass reads the JSON as defensively as the vector backfill does, because + SQLite returns a raw string and psycopg has already parsed it. + """ + last_id = 0 + while True: + rows = conn.execute( + text(""" + SELECT id, context_snapshot FROM actions + WHERE context_snapshot IS NOT NULL + AND context_snapshot_z IS NULL AND id > :last + ORDER BY id LIMIT :batch + """), + {"last": last_id, "batch": SNAPSHOT_BATCH}, + ).all() + if not rows: + return + params = [] + for row_id, stored in rows: + value = json.loads(stored) if isinstance(stored, str) else stored + if value is None: + continue + packed = compression.pack(value) + if compression.unpack(packed) != value: + raise RuntimeError( + f"context_snapshot for action {row_id} did not survive a " + "compress/decompress round trip; refusing to drop the " + "original column" + ) + params.append({"z": packed, "id": row_id}) + if params: + # Run one executemany per batch rather than one statement per row. + # This runs at container start, before the port opens, against a + # database across a network. A thousand round trips is the + # difference between a deploy that starts and a health check that + # times out waiting for it. + conn.execute( + text("UPDATE actions SET context_snapshot_z = :z WHERE id = :id"), + params, + ) + last_id = rows[-1][0] + + +# Returns SQL for the root branch of the adventure a row belongs to. It uses +# MIN(id) rather than a LIMIT, so the result is a plain scalar subquery on both +# dialects and stays deterministic if a database ends up with two roots for one +# adventure. +def _root_branch_of(column: str) -> str: + return ( + "(SELECT MIN(b.id) FROM branches b " + f"WHERE b.adventure_id = {column} AND b.parent_branch_id IS NULL)" + ) + + +def _backfill_tree(conn) -> None: + """Rewrites every existing adventure's linear story as a tree with one branch. + + The pass creates one root branch per adventure, sets `depth` to the old + `index`, points the head at the tip, and attaches every memory to the node it + summarized. It copies no row, deletes no row, and changes no ordering. + `index` and `depth` hold the same numbers when it finishes, which makes the + claim that current adventures are unaffected testable. + + The pass runs on the server. `actions` is the table that fills the disk, and + reading it into Python to write two integers per row would repeat a mistake + this project has already made twice. Each statement is guarded on its own + target being unset, so a run that fails partway through resumes rather than + applying twice. + """ + if not _has_columns(conn, "actions", "index"): + return + sqlite = conn.dialect.name == "sqlite" + + # 1. A root branch per adventure. Its lineage names the row's own id, which + # does not exist until the row does, so lineage starts as an empty list. + # `create_all` creates `branches` with lineage NOT NULL, so an empty array + # is how an unfilled lineage has to be stored. + conn.execute(text(""" + INSERT INTO branches (adventure_id, parent_branch_id, fork_depth, lineage, created_at) + SELECT a.id, NULL, NULL, '[]', CURRENT_TIMESTAMP + FROM adventures a + WHERE NOT EXISTS (SELECT 1 FROM branches b WHERE b.adventure_id = a.id) + """)) + + # 2. Set lineage to `[[own_id, null]]`, which is one uncapped entry, + # because the root branch is the whole story. The database's own JSON + # functions build it, because binding a JSON string as a parameter has no + # form that means the same thing to SQLite, which sees TEXT, and to + # Postgres, which sees json. The guard tests a length rather than + # `= '[]'`, because Postgres `json` has no equality operator. + conn.execute(text( + "UPDATE branches SET lineage = json_array(json_array(id, null)) " + "WHERE json_array_length(lineage) = 0" + if sqlite else + "UPDATE branches SET lineage = " + "jsonb_build_array(jsonb_build_array(id, null))::json " + "WHERE json_array_length(lineage) = 0" + )) + + # 3. Move every action onto that branch, at the depth its index implies. + # Deleting an action in the middle left gaps in `index`, and those gaps + # carry over on purpose. Depth has to preserve the order the story is read + # in, and renumbering here would move every cursor that points past a + # gap. + conn.execute(text(f""" + UPDATE actions + SET branch_id = {_root_branch_of('actions.adventure_id')}, + depth = "index" + WHERE branch_id IS NULL + """)) + + # 4. Set the head to the root branch and to the depth of its newest node. + conn.execute(text(f""" + UPDATE adventures + SET head_branch_id = {_root_branch_of('adventures.id')}, + head_depth = COALESCE( + (SELECT MAX(a."index") FROM actions a WHERE a.adventure_id = adventures.id), + {NO_DEPTH} + ) + WHERE head_branch_id IS NULL + """)) + + # 5. Attach memories to the node that produced them. `source_end` is the + # index of the last action a memory summarized, so it is that node's + # depth. A hand-written memory has no node and keeps a NULL depth. + conn.execute(text(f""" + UPDATE memories + SET branch_id = {_root_branch_of('memories.adventure_id')}, + depth = source_end + WHERE branch_id IS NULL + """)) + + +# A frozen copy of `context.history._STORY_TEXT` as it stood at version 56, +# which selects text that is not blank once whitespace is stripped. It is written +# out here rather than imported, because a migration has to keep meaning what it +# meant on the day it ran, while the module can change. `char()` is `chr()` on +# Postgres, and there is no third form, so the SQL depends on the dialect. +def _story_text_sql(column: str, sqlite: bool) -> str: + char = "char" if sqlite else "chr" + folded = column + for code in (10, 13, 9): # newline, carriage return, tab + folded = f"replace({folded}, {char}({code}), ' ')" + return f"trim({folded}) <> ''" + + +def _backfill_cursor_anchors(conn) -> None: + """Converts each adventure's two cursors from positions into nodes. + + A `memory_cursor` of 12 meant that the first twelve story actions were + covered. The twelfth story action, in depth order, is the node that means the + same thing and keeps meaning it after an action before it is deleted. The + translation is therefore a `ROW_NUMBER()` over the story plus a lookup at the + cursor's own value. + + The arithmetic has to handle two cases: + + * A cursor past the end of the story. This is legitimate. An adventure that + was caught up under the older rule can have a cursor equal to its action + count, and `run_post_turn` clamped it on every pass. No `rn` matches, so + the query falls back to the deepest node that exists, which still means + caught up. + * A cursor of 0, which is what most adventures have. Nothing is covered, the + column default already records that, and no row is updated. + + The statement is guarded on `_depth = -1`, so that a run which fails partway + through resumes. Every adventure it has already converted is skipped, and one + it has not converted looks the same as an untouched row. + + The row numbering is correlated rather than ranked and then filtered. + Numbering every action in the table and selecting one row from the result + reads all of `actions` once per adventure, because the window function stops + the correlation from being pushed down and the planner cannot make it + cheaper. This runs at boot, inside the single transaction that holds the + schema, against a database with real stories in it. Restricting the scan to + the adventure being updated makes each pass an index lookup on + `actions.adventure_id`, and `PARTITION BY` then partitions one adventure. The + two forms return the same answer, because the rows the partition would + separate are the rows the filter removes. + """ + if not _has_columns(conn, "adventures", "memory_cursor", "summary_cursor"): + return + sqlite = conn.dialect.name == "sqlite" + story = _story_text_sql("text", sqlite) + for name in ("memory", "summary"): + conn.execute(text(f""" + UPDATE adventures + SET {name}_cursor_branch_id = {_root_branch_of('adventures.id')}, + {name}_cursor_depth = COALESCE( + (SELECT ranked.depth FROM ( + SELECT depth, ROW_NUMBER() OVER ( + ORDER BY depth, id + ) AS rn + FROM actions + WHERE {story} + AND actions.adventure_id = adventures.id + ) AS ranked + WHERE ranked.rn = adventures.{name}_cursor), + (SELECT MAX(a.depth) FROM actions a + WHERE a.adventure_id = adventures.id + AND {_story_text_sql('a.text', sqlite)}), + {NO_DEPTH}) + WHERE adventures.{name}_cursor > 0 + AND adventures.{name}_cursor_depth = {NO_DEPTH} + """)) + + +# The per-attempt slices of a context snapshot, frozen here as they stood at +# version 60, where they were `adventures.VARIANT_SNAPSHOT_KEYS` and are now +# `attempts.ATTEMPT_KEYS`. Everything else in a snapshot is the assembled prompt, +# which every attempt at one turn shares, so a discarded attempt's row carries +# only these keys. +_ATTEMPT_KEYS = ("world_state", "script", "raw_output") + + +def _backfill_state_after(conn) -> None: + """Derives the "after" snapshots SP4 reads from each action's "before" ones. + + The state a node left behind is the state the node after it started from, so + `state_after` of action n is exactly `state_before` of action n + 1. The + hooks that ran between them belong to this turn, and both snapshots were + taken with those hooks already applied. The newest action of an adventure has + no action after it, and what it left behind is what the adventure currently + holds. + + Each column takes two statements rather than one COALESCE, because the second + statement is guarded on there being no later action at all. A row whose + successor predates `state_before` has to stay NULL rather than inherit the + tip's state. + + The queries order by `index`, which is still the story's order at version 60. + SP4 is the first migration where `depth` and `index` can disagree, and it has + not run when this pass does. + """ + if not _has_columns(conn, "actions", "state_before", "world_state_before"): + return + for column, live in ( + ("state_after", "script_state"), + ("world_state_after", "world_state"), + ): + before = column.replace("_after", "_before") + conn.execute(text(f""" + UPDATE actions SET {column} = ( + SELECT n.{before} FROM actions n + WHERE n.adventure_id = actions.adventure_id + AND n."index" > actions."index" + ORDER BY n."index", n.id LIMIT 1 + ) + WHERE actions.{column} IS NULL + """)) + conn.execute(text(f""" + UPDATE actions SET {column} = ( + SELECT adv.{live} FROM adventures adv + WHERE adv.id = actions.adventure_id + ) + WHERE actions.{column} IS NULL + AND NOT EXISTS ( + SELECT 1 FROM actions n + WHERE n.adventure_id = actions.adventure_id + AND n."index" > actions."index" + ) + """)) + + +# The `actions` table as migration 60 finds it, declared here rather than read +# from `Base.metadata`. The split pass writes JSON values, and only a column +# type knows how to render a dict on this dialect, so it needs a Table. Reading +# the live one would tie a migration to the current model: migration 66 drops +# five of these columns, and the pass would then fail to compile on a database +# that still has them. This declaration is a snapshot of a past schema and must +# not be updated to track `models.py`. +# +# It carries its own `MetaData`, so `create_all` never sees it. +_ACTIONS_AT_60 = Table( + "actions", MetaData(), + Column("id", Integer, primary_key=True), + Column("adventure_id", Integer), + Column("index", Integer), + Column("branch_id", Integer), + Column("depth", Integer), + Column("live", Boolean), + Column("type", String(20)), + Column("text", Text), + Column("reasoning", Text), + Column("context_snapshot", compression.CompressedJSON), + Column("world_delta", JSON), + Column("state_before", JSON), + Column("world_state_before", JSON), + Column("state_after", JSON), + Column("world_state_after", JSON), + Column("variants", JSON), + Column("variant_count", Integer), + Column("variant_index", Integer), + Column("created_at", DateTime), +) + + +def _split_variants_into_siblings(conn) -> None: + """Gives every discarded retry attempt its own row. + + `actions.variants` is a JSON repeating group holding the attempts at one + turn, oldest first, with `variant_index` naming the one the row's own `text` + duplicates. SP4 makes each attempt a node on the same branch at the same + depth, with `live` set on exactly one of them, so this pass reads the list one + last time and writes it out as siblings. + + The existing row keeps its own attempt and its context snapshot. That + snapshot is the assembled prompt for the whole turn and is stored once per + turn, never once per attempt. A sibling's snapshot carries only the slices + that differ between attempts, which is what the JSON entry held, so this pass + changes how the bytes are arranged rather than how many there are. + + The pass runs in Python rather than SQL, because the work iterates a JSON + array and inserts one row per element, which the two dialects express + differently and neither expresses well. Its cost is bounded by the number of + turns anyone has retried rather than by the size of the table, and it reads no + `context_snapshot`. + + The pass is resumable. A group that already has as many rows as its + `variant_count` claims has been split, so it is skipped. + """ + if not _has_columns(conn, "actions", "variants", "variant_index"): + return + actions = _ACTIONS_AT_60 + last_id = 0 + while True: + rows = conn.execute( + text(""" + SELECT id, adventure_id, branch_id, depth, "index", type, + created_at, variants, variant_index + FROM actions + WHERE variants IS NOT NULL AND id > :last + ORDER BY id LIMIT :batch + """), + {"last": last_id, "batch": BACKFILL_BATCH}, + ).mappings().all() + if not rows: + return + last_id = rows[-1]["id"] + for row in rows: + entries = row["variants"] + if isinstance(entries, str): + entries = json.loads(entries) + if not isinstance(entries, list) or len(entries) < 2: + continue + _split_one_action(conn, actions, row, entries) + + +def _split_one_action(conn, actions, row, entries: list) -> None: + live_index = row["variant_index"] or 0 + live_index = min(max(live_index, 0), len(entries) - 1) + already = conn.execute( + text(""" + SELECT COUNT(*) FROM actions + WHERE adventure_id = :adv AND branch_id = :branch AND depth = :depth + """), + {"adv": row["adventure_id"], "branch": row["branch_id"], "depth": row["depth"]}, + ).scalar() + if already and already >= len(entries): + return # A previous run of this pass already split it. + # Prefer the live attempt's own recorded outcome over the one + # `_backfill_state_after` derived from the turn after it. The two agree, but + # only the recorded one is a fact about this attempt. If the entry has no + # outcome to give, which happens when an adventure has no RPG layer and + # stores no world state per attempt, leave the derived value in place rather + # than overwrite it with NULL. + kept = {"live": True, "variant_count": len(entries), "variant_index": live_index} + live_state = _entry_script_state(entries[live_index]) + live_world = _entry_world_state(entries[live_index]) + if live_state is not None: + kept["state_after"] = live_state + if live_world is not None: + kept["world_state_after"] = live_world + # Use the Table rather than `text()`, here and below. These values are dicts + # bound for JSON columns, and the column type is the only thing that knows + # how to render one on this dialect. The table is `_ACTIONS_AT_60`, frozen + # at the schema this pass runs against. + conn.execute(actions.update().where(actions.c.id == row["id"]).values(**kept)) + siblings = [ + { + "adventure_id": row["adventure_id"], + "index": row["index"], + "branch_id": row["branch_id"], + "depth": row["depth"], + "live": False, + "type": row["type"], + "text": str(entry.get("text") or ""), + "reasoning": entry.get("reasoning"), + "context_snapshot": _attempt_snapshot(entry) or None, + "world_delta": _entry_world_delta(entry), + "state_before": None, + "world_state_before": None, + "state_after": _entry_script_state(entry), + "world_state_after": _entry_world_state(entry), + "variants": None, + "variant_count": len(entries), + "variant_index": i, + "created_at": _entry_created_at(entry, row["created_at"]), + } + for i, entry in enumerate(entries) + if i != live_index and isinstance(entry, dict) + ] + if siblings: + conn.execute(actions.insert(), siblings) + + +def _entry_script_state(entry) -> dict | None: + state = (entry or {}).get("script_state") + return state if isinstance(state, dict) else None + + +def _entry_world_state(entry) -> dict | None: + state = ((entry or {}).get("world_state") or {}).get("state") + return state if isinstance(state, dict) else None + + +def _entry_world_delta(entry) -> dict | None: + ws = (entry or {}).get("world_state") + if not isinstance(ws, dict): + return None + return { + "delta": ws.get("delta") or {}, + "applied": (ws.get("report") or {}).get("applied") or [], + } + + +def _attempt_snapshot(entry) -> dict: + return {k: entry[k] for k in _ATTEMPT_KEYS if isinstance(entry, dict) and k in entry} + + +def _entry_created_at(entry, fallback): + """Returns the attempt's own timestamp, or the row's timestamp as a fallback. + + The value is cosmetic. Sibling order comes from `variant_index`, which this + pass writes explicitly, so that nothing depends on two attempts made in the + same second sorting in the order they were made. + """ + raw = (entry or {}).get("created_at") + if isinstance(raw, str) and raw: + try: + return datetime.fromisoformat(raw) + except ValueError: + pass + if isinstance(fallback, str): + try: + return datetime.fromisoformat(fallback) + except ValueError: + return None + return fallback + + +def _get_version(conn) -> int: + if conn.dialect.name == "sqlite": + return conn.execute(text("PRAGMA user_version")).scalar() or 1 + conn.execute(text( + "CREATE TABLE IF NOT EXISTS schema_version (version INTEGER NOT NULL)" + )) + version = conn.execute(text("SELECT version FROM schema_version")).scalar() + # An existing database with no stamp can only have been created by an + # earlier `create_all` of this same codebase, so it is already at LATEST. + return version if version is not None else LATEST_VERSION + + +def _set_version(conn, version: int) -> None: + if conn.dialect.name == "sqlite": + conn.execute(text(f"PRAGMA user_version = {version}")) + return + conn.execute(text( + "CREATE TABLE IF NOT EXISTS schema_version (version INTEGER NOT NULL)" + )) + if conn.execute(text("SELECT version FROM schema_version")).scalar() is None: + conn.execute( + text("INSERT INTO schema_version (version) VALUES (:v)"), {"v": version} + ) + else: + conn.execute(text("UPDATE schema_version SET version = :v"), {"v": version}) + + +def bootstrap(engine: Engine, through: int = LATEST_VERSION) -> None: + """Brings the database up to `through`, which defaults to the newest version. + + The app always takes the default. A test passes an older version when it + asserts on something a later migration removes. A migration test that stops + at the version it is about keeps reading the columns that migration wrote, + rather than the schema those columns became several versions later. + """ + fresh = not inspect(engine).get_table_names() + Base.metadata.create_all(bind=engine) + with engine.begin() as conn: + if fresh: + _set_version(conn, through) + return + current = _get_version(conn) + for version, sql in MIGRATIONS: + if current < version <= through: + statement = _for_dialect(sql, conn.dialect.name) + # Skip the DDL when it has already run. The data pass below it + # still runs. + if not (_column_already_there(conn, statement) + or _column_already_gone(conn, statement)): + conn.execute(text(statement)) + if version == WORLD_DELTA_VERSION: + _backfill_world_delta(conn) + if version == VARIANT_COUNT_VERSION: + _backfill_variant_count(conn) + if version == EMBEDDING_BLOB_VERSION: + _backfill_embedding_blob(conn) + # This has to run between migration 43, which adds the column, + # and migration 44, which drops the old one. The loop is one + # transaction, so if this raises, the DROP rolls back with it + # and the prompts are still there. + if version == SNAPSHOT_COMPRESS_VERSION: + _backfill_context_snapshot(conn) + if version == TREE_BACKFILL_VERSION: + _backfill_tree(conn) + if version == CURSOR_ANCHOR_VERSION: + _backfill_cursor_anchors(conn) + # The order matters. The split reads what the first pass wrote + # for the rows it does not change, and overwrites it for the + # rows it does, because an attempt's own outcome takes priority + # over one derived from the turn after it. + if version == SIBLING_SPLIT_VERSION: + _backfill_state_after(conn) + _split_variants_into_siblings(conn) + # Migration 63 adds the column and migration 64 indexes it. + # The UPDATE is what the index serves, so it runs once both + # exist. + if version == PARENT_BACKFILL_VERSION: + _backfill_parents(conn) + current = version + _set_version(conn, current) + _encrypt_plaintext_api_keys(conn) + + +def _encrypt_plaintext_api_keys(conn) -> None: + """Encrypts API keys saved before encryption at rest existed (Phase 8). + + Those keys are stored in plain text, so this pass wraps them in Fernet. Plain + SQL cannot do it. The pass runs on every start, and it matches no rows once + every row carries the `enc:` prefix. + """ + from . import security # Deferred: security derives its key from DB_PATH setup. + + rows = conn.execute(text( + "SELECT id, api_key FROM settings WHERE api_key != '' AND api_key NOT LIKE 'enc:%'" + )).all() + for row_id, plain in rows: + conn.execute( + text("UPDATE settings SET api_key = :key WHERE id = :id"), + {"key": security.encrypt_secret(plain), "id": row_id}, + ) diff --git a/backend/app/models.py b/backend/app/models.py new file mode 100644 index 0000000..dba938a --- /dev/null +++ b/backend/app/models.py @@ -0,0 +1,720 @@ +from datetime import datetime, timezone + +from sqlalchemy import ( + JSON, Boolean, Column, DateTime, Float, ForeignKey, Index, Integer, LargeBinary, + String, Table, Text, UniqueConstraint, event, +) +from sqlalchemy.orm import Mapped, Session, mapped_column, relationship + +from .compression import CompressedJSON +from .database import Base + + +def utcnow() -> datetime: + return datetime.now(timezone.utc) + + +class User(Base): + """Phase 8: optional accounts. + + Three kinds of row share this table: + + - The local user has a NULL email and `is_guest` set to False. Single-user + mode creates this row automatically. It owns everything that a database + from before Phase 8 contained. + - Guests have a NULL email and `is_guest` set to True. Multi-user mode + creates one on a visitor's first visit and identifies it only by the + session cookie. + - Registered users have an email. Registration upgrades a guest row in + place, so the guest's data survives without being reassigned. + """ + + __tablename__ = "users" + + id: Mapped[int] = mapped_column(primary_key=True) + email: Mapped[str | None] = mapped_column(String(320), unique=True, nullable=True) + password_hash: Mapped[str | None] = mapped_column(String(300), nullable=True) + is_guest: Mapped[bool] = mapped_column(Boolean, default=True) + created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow) + last_seen_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) + # Shared demo key usage (resets when the UTC date changes). + demo_turns_used: Mapped[int] = mapped_column(Integer, default=0) + demo_turns_date: Mapped[str] = mapped_column(String(10), default="") + + +scenario_scripts = Table( + "scenario_scripts", + Base.metadata, + Column("scenario_id", ForeignKey("scenarios.id", ondelete="CASCADE"), primary_key=True), + Column("script_id", ForeignKey("scripts.id", ondelete="CASCADE"), primary_key=True), +) + + +class Scenario(Base): + __tablename__ = "scenarios" + + id: Mapped[int] = mapped_column(primary_key=True) + # NULL owner + is_public = seeded demo content, readable by everyone. + user_id: Mapped[int | None] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), nullable=True + ) + is_public: Mapped[bool] = mapped_column(Boolean, default=False) + title: Mapped[str] = mapped_column(String(200), default="Untitled Scenario") + description: Mapped[str] = mapped_column(Text, default="") + prompt: Mapped[str] = mapped_column(Text, default="") + # Plot components (AI Dungeon terminology; `memory` == Plot Essentials) + memory: Mapped[str] = mapped_column(Text, default="") + authors_note: Mapped[str] = mapped_column(Text, default="") + ai_instructions: Mapped[str] = mapped_column(Text, default="") + tags: Mapped[str] = mapped_column(String(500), default="") + # Cover art. The value is either an "https://" URL or an inline + # "data:image/...;base64,..." URI. The editor downscales uploads before + # storing them. An empty value tells the UI to fall back to an emoji sigil + # or to generated art. The image is stored in the row rather than on disk, + # because Render's free tier provides no persistent volume. Storing it here + # also keeps export bundles self-contained. + image: Mapped[str] = mapped_column(Text, default="") + # A single emoji or glyph, used when `image` is empty. This is a separate + # column because the value is a character rather than a location, so + # nothing needs to fetch or cache it. + icon: Mapped[str] = mapped_column(String(16), default="") + # Phase 12: the RPG world-state template. It holds stat definitions, which + # include bands and rules, and milestones. A NULL or empty value means the + # scenario has no RPG layer. + stat_schema: Mapped[dict | None] = mapped_column(JSON, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow) + updated_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow, onupdate=utcnow) + + story_cards: Mapped[list["StoryCard"]] = relationship( + back_populates="scenario", cascade="all, delete-orphan" + ) + adventures: Mapped[list["Adventure"]] = relationship(back_populates="scenario") + scripts: Mapped[list["Script"]] = relationship(secondary=scenario_scripts) + + +class Adventure(Base): + __tablename__ = "adventures" + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int | None] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), nullable=True + ) + scenario_id: Mapped[int | None] = mapped_column( + ForeignKey("scenarios.id", ondelete="SET NULL"), nullable=True + ) + title: Mapped[str] = mapped_column(String(200), default="Untitled Adventure") + memory: Mapped[str] = mapped_column(Text, default="") + authors_note: Mapped[str] = mapped_column(Text, default="") + ai_instructions: Mapped[str] = mapped_column(Text, default="") + story_summary: Mapped[str] = mapped_column(Text, default="") + # Phase 18: who the player is playing as. The AI never writes these — they + # are user-only, which is what lets them sit in the cached system block + # rather than below the history with the values that change. An empty + # `persona_name` means the adventure has no persona, and every read below + # falls back to the wording used before this existed. + # + # These are adventure columns rather than part of the scenario's + # `stat_schema`, for two reasons. An adventure with no RPG layer still has a + # protagonist, and `worldstate.schema._initials` treats every dict inside a + # stat section as a stat definition, so a persona placed there would be + # instantiated, rendered and given an `initial` value as though it were one. + persona_name: Mapped[str] = mapped_column(String(80), default="") + persona_pronouns: Mapped[str] = mapped_column(String(40), default="") + persona_desc: Mapped[str] = mapped_column(Text, default="") + script_state: Mapped[dict] = mapped_column(JSON, default=dict) + # Phase 12: live RPG world state (world/player/npc stats + milestones), + # instantiated from the scenario's stat_schema. Empty when there's no RPG layer. + world_state: Mapped[dict] = mapped_column(JSON, default=dict) + # The ${Placeholder} answers collected when this adventure was started, kept + # so "Update from scenario" can re-fill freshly copied scenario text with the + # same values. NULL for adventures created before this column existed. + placeholders: Mapped[dict | None] = mapped_column(JSON, nullable=True) + # Phase 6: opt-in per adventure (extra AI calls) + auto_summarize: Mapped[bool] = mapped_column(Boolean, default=False) + memory_bank_enabled: Mapped[bool] = mapped_column(Boolean, default=False) + # Phase 14, SP3: how far the memory pass and the summary pass have read, + # each as a coordinate. Each pair + # holds the branch and depth of the last action that pass covered. A + # position moves when an action in front of it is deleted, so the mark + # silently starts covering an action it never read. A depth is a coordinate + # along a path, so deleting an action does not move it. NO_DEPTH, which is + # -1, means that nothing is covered yet, so the first block needs no special + # case. These are plain integers rather than foreign keys, for the reason + # given on `head_branch_id` below. See `context/cursors.py`. + memory_cursor_branch_id: Mapped[int | None] = mapped_column(Integer, nullable=True) + memory_cursor_depth: Mapped[int] = mapped_column(Integer, default=-1) + summary_cursor_branch_id: Mapped[int | None] = mapped_column(Integer, nullable=True) + summary_cursor_depth: Mapped[int] = mapped_column(Integer, default=-1) + # Phase 14: where the story is being played. `head_branch_id` names the + # branch, and `head_depth` gives the depth of its newest node. + # + # `head_branch_id` is deliberately not a ForeignKey. `branches.adventure_id` + # already points from branches to adventures, so a constraint in this + # direction would make the two tables a cycle that `create_all` cannot + # order. The usual fix is `use_alter`, which needs an ALTER statement that + # SQLite does not provide. The column caches a pointer, and + # `tree.head_branch` treats a head that names a missing branch as a bug to + # recover from rather than a state to preserve. + head_branch_id: Mapped[int | None] = mapped_column(Integer, nullable=True) + # The depth of the tip, so the next node is always head_depth + 1. + # NO_DEPTH (-1) for an adventure with no actions yet. + head_depth: Mapped[int] = mapped_column(Integer, default=-1) + created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow) + updated_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow, onupdate=utcnow) + + scenario: Mapped[Scenario | None] = relationship(back_populates="adventures") + story_cards: Mapped[list["StoryCard"]] = relationship( + back_populates="adventure", cascade="all, delete-orphan" + ) + # Every action in the adventure, across all branches. This collection is + # the tree, not the story being played. Ordering it by depth does not make + # it a story, because a path is a selection out of the tree. Code that shows + # a story to a reader goes through `context.history`, which applies the + # branch clause. This relationship exists for ownership and for the + # delete-orphan cascade. + actions: Mapped[list["Action"]] = relationship( + back_populates="adventure", + cascade="all, delete-orphan", + order_by="Action.id", + ) + scripts: Mapped[list["AdventureScript"]] = relationship( + back_populates="adventure", + cascade="all, delete-orphan", + order_by="AdventureScript.position", + ) + memories: Mapped[list["Memory"]] = relationship( + back_populates="adventure", + cascade="all, delete-orphan", + order_by="Memory.id", + ) + + +class Branch(Base): + """Phase 14: one path through an adventure's story tree. + + A branch does not own a copy of the story. It holds the nodes played on it, + and it inherits everything before its fork point from its ancestors. Reading + branch C means reading C's nodes, then B's nodes up to the depth where C + forked, then A's nodes up to the depth where B forked. The `lineage` column + records that list, so a read becomes one OR clause per entry instead of a + walk up parent pointers. + + Until forking ships, each adventure has one root branch and every node + belongs to it. This is not a partly migrated state. A linear story is a tree + with one branch, which is why writing these columns changes nothing that a + reader can observe. + + This class defines no ORM relationships, by design. `actions.branch_id` and + `memories.branch_id` both use ON DELETE CASCADE, so the database removes a + deleted branch's nodes. A relationship would make SQLAlchemy load those rows + first, and loading every action of a branch is what the windowed reads exist + to avoid. + """ + + __tablename__ = "branches" + + id: Mapped[int] = mapped_column(primary_key=True) + adventure_id: Mapped[int] = mapped_column(ForeignKey("adventures.id", ondelete="CASCADE")) + # NULL on a root branch. + parent_branch_id: Mapped[int | None] = mapped_column( + ForeignKey("branches.id", ondelete="CASCADE"), nullable=True + ) + # The depth at which this branch left its parent. The fork records this + # value, and no code infers it later. Deriving it from the first depth where + # two branches' nodes differ would produce a wrong answer whenever an + # attempt repeats its parent's text. + fork_depth: Mapped[int | None] = mapped_column(Integer, nullable=True) + # The ancestry, newest first, as [[branch_id, max_depth], ...]. A NULL + # `max_depth` means the entry extends to the tip of that branch. Any other + # value is the fork depth of the branch below it, inclusive. The fork + # computes this list once from the parent's lineage plus one entry, so no + # read has to reconstruct it. + lineage: Mapped[list] = mapped_column(JSON, default=list) + # The name the player gave this line of the story, or NULL if no one named + # it. The column stores NULL rather than a generated name such as + # "branch 4", because it records what the player chose rather than what the + # app derived. A stored default would also become wrong as soon as an + # earlier branch is deleted and the ordinals shift. The UI labels an unnamed + # branch by its fork depth, which deleting a branch does not change. + name: Mapped[str | None] = mapped_column(String(80), nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow) + + +class Memory(Base): + """Phase 6: an auto-summarized (or hand-written) fact about the adventure. + + The vector lives in `embedding_blob` as packed float32 (see vectors.py). + NULL until embedded, which also marks it for backfill when an embedding + model becomes available. + + Cosine ranking runs in Python, so the vectors travel over the wire. Measure + that cost in bytes rather than in rows. A few hundred vectors stored as JSON + come to about 10 MB, fetched again on every turn. Size any new column by the + bytes it adds, not by the number of rows. + """ + + __tablename__ = "memories" + + id: Mapped[int] = mapped_column(primary_key=True) + adventure_id: Mapped[int] = mapped_column(ForeignKey("adventures.id", ondelete="CASCADE")) + text: Mapped[str] = mapped_column(Text, default="") + # The vector, stored as little-endian float32. This column is deferred + # because it is wider than the rest of the row combined and only one code + # path reads it. Code that loads memories in bulk, such as the Memories + # drawer, eviction, and the embed queue, must select the columns it needs + # instead of loading whole entities. + embedding_blob: Mapped[bytes | None] = mapped_column( + LargeBinary, nullable=True, deferred=True + ) + # The stretch of story this memory summarizes, given as depths on + # `branch_id`. Both are NULL for a hand-written memory, which summarizes no + # actions. `source_end` is the depth of the node the memory + # attaches to, and `depth` below mirrors it. `source_start` is where the + # stretch begins, which is where the summarizer resumes if the memory is + # withdrawn. + source_start: Mapped[int | None] = mapped_column(Integer, nullable=True) + source_end: Mapped[int | None] = mapped_column(Integer, nullable=True) + # Phase 14: the node that produced this memory, meaning the last action the + # memory summarizes. Derived data attaches to the node it came from, which + # is what makes forking cheap. Memories on a shared ancestor are shared + # automatically, and a memory that covers part of branch B is not visible + # from any path that does not go through B. + # + # Every memory has a coordinate, including a hand-written one, which takes + # the head as of the moment it was written (SP7). A NULL depth used to mean + # that the memory belonged to the adventure rather than to a path. A fork + # cannot cap a NULL, so such a memory followed the reader onto branches + # whose story it did not describe. + branch_id: Mapped[int | None] = mapped_column( + ForeignKey("branches.id", ondelete="CASCADE"), nullable=True + ) + depth: Mapped[int | None] = mapped_column(Integer, nullable=True) + # Whether `embedding_blob` is set. `memorybank.set_vector` keeps this column + # current. Readers need only the yes-or-no answer, and fetching six + # kilobytes of vector to get it is too expensive. + embedded: Mapped[bool] = mapped_column(Boolean, default=False) + pinned: Mapped[bool] = mapped_column(Boolean, default=False) + forgotten: Mapped[bool] = mapped_column(Boolean, default=False) # evicted, kept for UI + use_count: Mapped[int] = mapped_column(Integer, default=0) + last_used_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow) + + adventure: Mapped[Adventure] = relationship(back_populates="memories") + + +class StoryCard(Base): + """Owned by either a scenario or an adventure (exactly one set).""" + + __tablename__ = "story_cards" + + id: Mapped[int] = mapped_column(primary_key=True) + scenario_id: Mapped[int | None] = mapped_column( + ForeignKey("scenarios.id", ondelete="CASCADE"), nullable=True + ) + adventure_id: Mapped[int | None] = mapped_column( + ForeignKey("adventures.id", ondelete="CASCADE"), nullable=True + ) + type: Mapped[str] = mapped_column(String(100), default="") + name: Mapped[str] = mapped_column(String(200), default="") + keys: Mapped[str] = mapped_column(Text, default="") # comma-separated triggers + entry: Mapped[str] = mapped_column(Text, default="") + notes: Mapped[str] = mapped_column(Text, default="") + # Set on adventure copies only. It records which piece of the scenario the + # card came from, as either "card:" or "npc:". + # The "Update from scenario" action refreshes or removes exactly these + # cards. A NULL value means the player wrote the card, so the update leaves + # it alone, or that the copy predates this column, in which case the update + # matches it by name and then sets this value. + source_ref: Mapped[str | None] = mapped_column(String(64), nullable=True) + + scenario: Mapped[Scenario | None] = relationship(back_populates="story_cards") + adventure: Mapped[Adventure | None] = relationship(back_populates="story_cards") + + +def _change_label(parts: list[str]) -> str: + """Names a world-state path for the inline turn summary. + + `npc.gwen.trust` becomes "gwen trust". Every other shape uses its last + segment, so `player.hp` becomes "hp". + """ + if len(parts) == 3 and parts[0] == "npc": + return f"{parts[1]} {parts[2]}" + return parts[-1] if parts else "" + + +class Action(Base): + __tablename__ = "actions" + # Phase 14: every story read selects one branch up to one depth, then + # another branch up to another depth, and so on. The pair (branch_id, depth) + # is the index those clauses need. + __table_args__ = (Index("ix_actions_branch_depth", "branch_id", "depth"),) + + id: Mapped[int] = mapped_column(primary_key=True) + adventure_id: Mapped[int] = mapped_column(ForeignKey("adventures.id", ondelete="CASCADE")) + # Phase 14: the node's place in the tree. `depth` is a position along one + # path rather than a global turn number. Node A4 and node B4 are + # alternatives, not duplicates. + # + # Both columns are nullable because ALTER TABLE cannot add a NOT NULL column + # without a default, and no default makes sense for a branch. The migration + # fills these columns for existing rows, and `tree.place_action` fills them + # for new rows. From SP2 onward, a NULL `branch_id` marks a row that no read + # can see. + branch_id: Mapped[int | None] = mapped_column( + ForeignKey("branches.id", ondelete="CASCADE"), nullable=True + ) + depth: Mapped[int | None] = mapped_column(Integer, nullable=True) + # Phase 14, SP9: the node that this node was played after. It names the take + # that was live when this row was written, not whatever sits at depth - 1 + # now. + # + # This column answers one question: which takes belong to the same turn. A + # coordinate cannot answer it. A take that is forked onto its own branch + # leaves the coordinate that its siblings still occupy, so the pager would + # show it as 1/1 next to their 1/3. Forking a branch does not change a + # node's parent. + # + # Code reads this column only to group takes, using one indexed lookup + # rather than a walk. Paths still resolve through `lineage`, which is why + # adding this column required no change to any read of the story. + # + # The value is NULL on a root node, and on pre-SP9 rows that the migration + # could not place. For those rows, `attempts.group` falls back to the + # coordinate, which is how they were written. + parent_id: Mapped[int | None] = mapped_column( + ForeignKey("actions.id", ondelete="SET NULL"), nullable=True, index=True + ) + # Phase 14, SP4: whether this node is the one the story uses at its + # coordinate. Retry no longer rewrites a row. It writes a sibling at the + # same branch and depth, so one coordinate can hold several attempts while + # exactly one of them is on the path. `lineage.Path.clause` is the only + # place that reads this column, for the same reason it is the only place + # that knows about branches. If a discarded attempt reaches a read, the page + # renders the same turn twice. + # + # A node with no siblings is live, so the default is True and every pre-SP4 + # row is already correct. The migration does not need to visit them. + live: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) + type: Mapped[str] = mapped_column(String(20)) # start|do|say|story|continue|ai + text: Mapped[str] = mapped_column(Text, default="") + # Reasoning-model "thinking" that preceded the text (AI actions only). + reasoning: Mapped[str | None] = mapped_column(Text, nullable=True) + # The full assembled prompt for this turn, used by the Insights viewer. + # This is the largest column in the database. It averages 163 KB per row in + # production and 232 KB on the longest adventure, and it accounts for 89% of + # everything stored. Only one endpoint reads it, one action at a time. + # + # The column is expensive in two ways, so it has two protections. + # `deferred=True` protects reads, because SQLAlchemy loads the column only + # when code touches the attribute. A page load therefore costs nothing. Code + # that reads actions in bulk must not touch this attribute, which is why + # `world_delta` below exists. `CompressedJSON` protects storage, because + # this column determines when the free tier's 512 MB limit is reached. The + # attribute behaves like a plain dict in both cases. See compression.py. + context_snapshot: Mapped[dict | None] = mapped_column( + CompressedJSON, nullable=True, deferred=True + ) + # The small slice of the snapshot that IS needed in bulk: this turn's RPG + # state changes, for the inline chips under an AI message (world_changes) + # and for re-attaching the emit block when replaying history to the model. + # Mirrors the active variant, same as text/reasoning/context_snapshot. + world_delta: Mapped[dict | None] = mapped_column(JSON, nullable=True) + # Phase 14, SP4: the shared script state and the RPG world state as they + # stood after this node was played. These columns record the node's outcome + # rather than its starting position. + # + # Two operations need this outcome, and neither can use a snapshot taken + # before the turn. Switching between siblings must restore the state that + # the chosen attempt produced, and the attempts differ in exactly that. + # Rolling back to before a turn means restoring the state that the preceding + # node left behind, which is one lookup along the path. + # + # The value is NULL on pre-SP4 rows for which the migration could not derive + # one. Every caller tolerates that. A missing snapshot means that the caller + # leaves the live state unchanged. It never means reset the state. + # + # These columns are deferred, because code reads them only for the single + # node being switched to, undone, or retried past. + state_after: Mapped[dict | None] = mapped_column( + JSON, nullable=True, deferred=True + ) + world_state_after: Mapped[dict | None] = mapped_column( + JSON, nullable=True, deferred=True + ) + created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow) + + adventure: Mapped[Adventure] = relationship(back_populates="actions") + + @property + def world_changes(self) -> list[dict]: + """Compact per-turn RPG state changes (Phase 12), for the inline summary + under an AI message. Labels are path-based (no schema needed): + `npc.gwen.trust` -> "gwen trust". + + The summary reports refused changes as well as accepted ones. A stat the + engine clamped carries `clamped`, and a stat it refused outright becomes + a `rejected` entry carrying the reason. Reporting only the accepted + changes made a clamp indistinguishable from a change that never + happened: a value the model pushed past its ceiling came back as a + delta of 0 and rendered as an ordinary chip, so a refused update read on + screen as an applied one. + + Reads `world_delta`, never `context_snapshot`. This runs for every + action in a list response, and touching the deferred snapshot here would + drag the entire prompt archive out of the database.""" + wd = self.world_delta if isinstance(self.world_delta, dict) else None + if wd is None: + return [] + clamped_paths = { + str(e.get("path", "")) for e in (wd.get("clamped") or []) if isinstance(e, dict) + } + out: list[dict] = [] + for entry in wd.get("applied") or []: + path = str(entry.get("path", "")) + parts = path.split(".") + section, name = parts[0], parts[-1] + if section == "flags": + out.append({"kind": "flag", "label": name, "on": bool(entry.get("new"))}) + elif section == "milestones": + out.append({"kind": "milestone", "label": name}) + else: + old, new = entry.get("old"), entry.get("new") + delta = new - old if isinstance(old, (int, float)) and isinstance(new, (int, float)) else None + chip = { + "kind": "stat", + "label": _change_label(parts), + "delta": delta, + "value": new, + "clamped": path in clamped_paths, + } + # Carried only when the engine wrote one. It is empty for every + # accepted change, and a key per chip per action is paid on + # every page load. + if entry.get("fix"): + chip["fix"] = str(entry["fix"]) + out.append(chip) + for entry in wd.get("rejected") or []: + if not isinstance(entry, dict): + continue + parts = str(entry.get("path", "")).split(".") + chip = { + "kind": "rejected", + "label": _change_label(parts), + "reason": str(entry.get("reason", "")), + } + if entry.get("fix"): + chip["fix"] = str(entry["fix"]) + out.append(chip) + return out + + +class Script(Base): + __tablename__ = "scripts" + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int | None] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), nullable=True + ) + name: Mapped[str] = mapped_column(String(200), default="Untitled Script") + description: Mapped[str] = mapped_column(Text, default="") + library_js: Mapped[str] = mapped_column(Text, default="") + input_js: Mapped[str] = mapped_column(Text, default="") + context_js: Mapped[str] = mapped_column(Text, default="") + output_js: Mapped[str] = mapped_column(Text, default="") + created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow) + updated_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow, onupdate=utcnow) + + +class AdventureScript(Base): + """A script copied into an adventure at creation, so library edits don't + change running adventures unless the player explicitly re-syncs it from + `source_script_id`. `state` lives on Adventure.script_state (one shared + state per adventure, as in AI Dungeon).""" + + __tablename__ = "adventure_scripts" + + id: Mapped[int] = mapped_column(primary_key=True) + adventure_id: Mapped[int] = mapped_column(ForeignKey("adventures.id", ondelete="CASCADE")) + # The library Script that this copy was made from, which lets the player + # re-sync it on demand. The value is NULL for legacy copies that predate + # this column, and for demo-derived copies whose source the player does not + # own. Those copies fall back to matching by name, or cannot be synced. + source_script_id: Mapped[int | None] = mapped_column( + ForeignKey("scripts.id", ondelete="SET NULL"), nullable=True + ) + position: Mapped[int] = mapped_column(Integer, default=0) + enabled: Mapped[bool] = mapped_column(Boolean, default=True) + name: Mapped[str] = mapped_column(String(200), default="Untitled Script") + description: Mapped[str] = mapped_column(Text, default="") + library_js: Mapped[str] = mapped_column(Text, default="") + input_js: Mapped[str] = mapped_column(Text, default="") + context_js: Mapped[str] = mapped_column(Text, default="") + output_js: Mapped[str] = mapped_column(Text, default="") + + adventure: Mapped[Adventure] = relationship(back_populates="scripts") + + +class Settings(Base): + __tablename__ = "settings" + + id: Mapped[int] = mapped_column(primary_key=True) + # Phase 8: one row per user (pre-Phase-8 DBs had a single id=1 row, which + # the migration assigns to the local user). + user_id: Mapped[int | None] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), nullable=True, unique=True + ) + endpoint_url: Mapped[str] = mapped_column(String(500), default="http://localhost:11434/v1") + # Encrypted at rest with Fernet, which produces a value that starts with + # "enc:". See security.py. To read the key, use `api_key_plain`. + api_key: Mapped[str] = mapped_column(String(500), default="") + model: Mapped[str] = mapped_column(String(200), default="") + api_mode: Mapped[str] = mapped_column(String(20), default="chat") # chat|completion + temperature: Mapped[float] = mapped_column(Float, default=0.8) + # 800 leaves room for a full scene; 400 tended to truncate mid-paragraph + # and left reasoning models with nothing after their thinking. + max_output_tokens: Mapped[int] = mapped_column(Integer, default=800) + # Separate thinking budget for reasoning models (OpenRouter-style + # `reasoning: {max_tokens}`); 0 = param not sent, -1 = reasoning explicitly + # off (`reasoning: {effort: none}`). Added on top of + # max_output_tokens so story output keeps its full budget. + reasoning_max_tokens: Mapped[int] = mapped_column(Integer, default=0) + context_token_budget: Mapped[int] = mapped_column(Integer, default=16384) + narrator_prompt: Mapped[str] = mapped_column( + Text, + default=( + "You are a masterful storyteller continuing an interactive adventure. " + "Continue the story naturally in second person, staying consistent with " + "everything established so far. Write vivid prose. Never speak for the " + "player or break character. Do not conclude the story; always leave room " + "for the player's next action." + ), + ) + # Phase 6: auto-summarization + memory bank + summary_model: Mapped[str] = mapped_column(String(200), default="") # "" = main model + embedding_model: Mapped[str] = mapped_column(String(200), default="") # "" = bank disabled + # This was 200. It was lowered mainly to improve retrieval quality. Ranking + # 200 memories to choose 5 selects from a large amount of noise, and the + # oldest memories describe a part of the story that the player has left + # behind. Cheaper reads are a secondary benefit rather than the reason. + memory_bank_capacity: Mapped[int] = mapped_column(Integer, default=80) + memory_top_k: Mapped[int] = mapped_column(Integer, default=5) + + @property + def has_api_key(self) -> bool: + return bool(self.api_key) + + @property + def api_key_plain(self) -> str: + from . import security # local import: models is imported before security + + return security.decrypt_secret(self.api_key) + + +# ---------- Visit analytics (see analytics.py) ---------- +# Two intentionally simple tables. Neither can hold text that a player wrote, +# and neither can be joined back to a `users` row, because the visitor column +# holds an HMAC and has no foreign key. When guest cleanup deletes an account, +# the history that account contributed remains intact and anonymous. + + +class AnalyticsDaily(Base): + """One counter: how many times `label` happened within `metric` on `day`. + + The table stores a generic triple of metric, label, and hits rather than one + column per statistic. Measuring something new therefore costs a constant + rather than a migration. The only writer is an UPSERT that runs from a + buffer. See `analytics.flush`. + """ + + __tablename__ = "analytics_daily" + + id: Mapped[int] = mapped_column(primary_key=True) + day: Mapped[str] = mapped_column(String(10), index=True) # YYYY-MM-DD, UTC + metric: Mapped[str] = mapped_column(String(32)) + label: Mapped[str] = mapped_column(String(80), default="") + hits: Mapped[int] = mapped_column(Integer, default=0) + + # The upsert target: one row per bucket per day, created or incremented. + __table_args__ = ( + UniqueConstraint("day", "metric", "label", name="uq_analytics_daily_bucket"), + ) + + +class AnalyticsVisitorDay(Base): + """One visitor, one day, and which funnel steps they reached on it. + + This table exists so that the funnel counts people rather than clicks. A + player who starts six adventures counts as one person who started an + adventure. `is_new` is set when the visitor has no earlier row, which is why + the visitor column also has an index of its own. + """ + + __tablename__ = "analytics_visitor_days" + + id: Mapped[int] = mapped_column(primary_key=True) + day: Mapped[str] = mapped_column(String(10)) + # HMAC of the user id under the app secret; not reversible, not a key. + visitor: Mapped[str] = mapped_column(String(32)) + is_new: Mapped[bool] = mapped_column(Boolean, default=False) + opened: Mapped[bool] = mapped_column(Boolean, default=False) + created: Mapped[bool] = mapped_column(Boolean, default=False) + played: Mapped[bool] = mapped_column(Boolean, default=False) + signed_up: Mapped[bool] = mapped_column(Boolean, default=False) + + __table_args__ = ( + UniqueConstraint("day", "visitor", name="uq_analytics_visitor_day"), + Index("ix_analytics_visitor", "visitor"), + ) + + +class AccessEvent(Base): + """One sign-in, registration, failed attempt, or session first-seen. + + This table is the counterpart to the two above, and it is kept separate from + them on purpose. It identifies people by design, recording address, email, + and device. Keeping it in its own table and its own module means that the + structure enforces the anonymity of the counters rather than a convention. + + `user_id` is a plain integer with no foreign key. An access log that + disappeared when the account did would not serve its purpose, and guest + cleanup deletes accounts on a schedule. `who` and `is_guest` are snapshots + for the same reason, so a row still reads correctly after the account is + gone. + """ + + __tablename__ = "access_events" + + id: Mapped[int] = mapped_column(primary_key=True) + at: Mapped[datetime] = mapped_column(DateTime, default=utcnow, index=True) + # session | login | register | login_failed + kind: Mapped[str] = mapped_column(String(16)) + user_id: Mapped[int | None] = mapped_column(Integer, nullable=True) + # The email for a registered account, or a label such as "Guest #12" + # otherwise. For a failed sign-in, this holds the address that was tried, + # which is the reason the row exists. + who: Mapped[str] = mapped_column(String(320), default="") + is_guest: Mapped[bool] = mapped_column(Boolean, default=False) + ip: Mapped[str] = mapped_column(String(45), default="") # 45 = max IPv6 + country: Mapped[str] = mapped_column(String(16), default="") + device: Mapped[str] = mapped_column(String(16), default="") + user_agent: Mapped[str] = mapped_column(String(200), default="") + + +# Phase 14: the fallback under `tree.place_action`. +# +# Since SP2, reads filter on `branch_id` and `depth`. A node written without +# them is invisible to every page, every context build, and every memory pass. +# The failure is silent, because nothing raises an error. Every current writer +# places its nodes explicitly, but relying on that would also mean relying on +# every fixture, script, and test written from now on. The session therefore +# enforces the rule as rows travel to the database. +# +# This listener is registered here rather than in tree.py so that importing the +# models is enough to enable it. The invariant belongs to the rows, not to the +# module that usually writes them. The import sits inside the callback because +# tree.py imports this module. +@event.listens_for(Session, "before_flush") +def _place_new_nodes_on_the_tree(session, flush_context, instances): + from . import tree + + tree.place_new_nodes(session) diff --git a/backend/app/netguard.py b/backend/app/netguard.py new file mode 100644 index 0000000..068d843 --- /dev/null +++ b/backend/app/netguard.py @@ -0,0 +1,52 @@ +"""SSRF guard for the one place the server makes an outbound request to a +user-supplied address: the BYOK `endpoint_url` (connection test + turns/chat). + +Without this guard, a hosted user could point `endpoint_url` at an internal +service or at the cloud metadata endpoint, 169.254.169.254, and have the server +fetch it. The connection test even returns part of the response. The guard +therefore refuses any URL that resolves to a non-public address. + +The guard does nothing in local mode. A local install talking to +http://localhost:11434, which is Ollama, is the intended case. The guard applies +only to a hosted, multi-user deployment, where the endpoint comes from an +untrusted visitor. +""" + +import ipaddress +import socket +from urllib.parse import urlparse + +from . import auth + + +def endpoint_block_reason(url: str) -> str | None: + """A human-readable reason this URL must NOT be fetched server-side, or None + if it's allowed. Resolves the host and rejects it if any resulting address + is non-public (private, loopback, link-local/metadata, reserved, …). + + Checking at request time (not just on save) is deliberate: it resists a DNS + record that flips to a private IP after the value was stored. + """ + if not auth.MULTI_USER: + return None + parsed = urlparse(url) + if parsed.scheme not in ("http", "https"): + return "the endpoint URL must start with http:// or https://" + host = parsed.hostname + if not host: + return "the endpoint URL has no host" + port = parsed.port or (443 if parsed.scheme == "https" else 80) + try: + infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM) + except socket.gaierror: + return "the endpoint host could not be resolved" + for info in infos: + try: + ip = ipaddress.ip_address(info[4][0]) + except ValueError: + return "the endpoint host resolved to an unrecognized address" + # is_global is the strict allowlist: private/loopback/link-local/CGNAT + # all report False, so this one check covers the metadata IP too. + if not ip.is_global or ip.is_multicast or ip.is_reserved: + return "the endpoint URL resolves to a non-public address" + return None diff --git a/backend/app/providers/__init__.py b/backend/app/providers/__init__.py new file mode 100644 index 0000000..98a1b84 --- /dev/null +++ b/backend/app/providers/__init__.py @@ -0,0 +1,4 @@ +from .base import PromptParts, Provider, ProviderError +from .openai_compatible import OpenAICompatibleProvider + +__all__ = ["PromptParts", "Provider", "ProviderError", "OpenAICompatibleProvider"] diff --git a/backend/app/providers/base.py b/backend/app/providers/base.py new file mode 100644 index 0000000..187c4b1 --- /dev/null +++ b/backend/app/providers/base.py @@ -0,0 +1,34 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import AsyncIterator + + +@dataclass +class PromptParts: + """Assembled context, provider-agnostic. Providers map this to their wire format.""" + + system: str # narrator prompt + AI instructions + memory + story: str # the story text so far (already token-budgeted) + + +class ProviderError(Exception): + """User-presentable provider failure (connection refused, bad key, model not found…).""" + + +class Provider(ABC): + # The endpoint's own token accounting for the most recent call, when it + # reported any. It notably carries `prompt_tokens_details.cached_tokens`, + # which is the only direct measure of whether the prompt prefix is being + # cached. A caller reads it after the call it made, and one provider is built + # per request, so nothing races. + last_usage: dict | None = None + + @abstractmethod + def generate( + self, + parts: PromptParts, + *, + temperature: float, + max_tokens: int, + ) -> AsyncIterator[tuple[str, str]]: + """Yield ("text" | "reasoning", chunk) pairs. Raises ProviderError on failure.""" diff --git a/backend/app/providers/openai_compatible.py b/backend/app/providers/openai_compatible.py new file mode 100644 index 0000000..ce4da43 --- /dev/null +++ b/backend/app/providers/openai_compatible.py @@ -0,0 +1,423 @@ +import json +from typing import AsyncIterator + +import httpx + +from .. import debuglog, netguard +from .base import PromptParts, Provider, ProviderError + +# Appended after the story text in chat mode, so a chat-tuned model continues +# the prose rather than replying conversationally. +CHAT_CONTINUE_HINT = "\n\n[Continue the story directly. Output only story text.]" + +# OpenRouter serves one model from whichever upstream is available, and every +# upstream holds its own prompt cache, so a request routed somewhere new starts +# with a cold cache however stable the prompt is. Naming a preferred upstream +# makes routing deterministic, which is what allows a cache hit at all. +# +# `allow_fallbacks` stays at its default of true on purpose, because this is a +# preference rather than a restriction. If the named upstream is down, the +# request still goes elsewhere and only misses the cache, which is the behavior +# without this setting. +# +# This is a list rather than a value derived from the model slug. The vendor half +# of a slug is usually the provider slug, such as "deepseek/..." mapping to +# "deepseek", which was verified against /api/v1/providers, but not reliably. +# Google's models are served by "google-ai-studio" and "google-vertex", and there +# is no "google". Look a vendor up on the model's Providers tab before adding it +# here. A slug that does not exist is a routing preference that, at best, does +# nothing. +_OPENROUTER_HOST = "openrouter.ai" +_PREFERRED_UPSTREAM = {"deepseek": "deepseek"} + + +# Completion endpoints have no roles, so a chat has to be flattened into one +# labeled transcript that ends on "Assistant:" for the model to continue. +_ROLE_LABELS = {"system": "System", "user": "User", "assistant": "Assistant"} + + +def flatten_messages(messages: list[dict]) -> str: + turns = "\n\n".join( + f"{_ROLE_LABELS.get(m['role'], m['role'])}: {m['content']}" for m in messages + ) + return f"{turns}\n\nAssistant:" + + +class OpenAICompatibleProvider(Provider): + """Adapter for any /v1-style endpoint. + + This covers Ollama, LM Studio, OpenAI, OpenRouter, vLLM, and Groq, among + others. + """ + + def __init__( + self, + endpoint_url: str, + api_key: str, + model: str, + api_mode: str = "chat", + reasoning_max_tokens: int = 0, + ): + self.base_url = endpoint_url.rstrip("/") + self.api_key = api_key + self.model = model + self.api_mode = api_mode # Either "chat" or "completion". + # The thinking budget for reasoning models, on top of `max_tokens`. A + # value of 0 means the `reasoning` parameter is not sent, because an + # endpoint that does not know the field may reject it. A negative value + # asks the endpoint to turn reasoning off. + self.reasoning_max_tokens = reasoning_max_tokens + # The token accounting from the last call, when the endpoint reported + # any. It holds the prompt and completion counts, plus, on OpenRouter, + # `prompt_tokens_details.cached_tokens`, which is the number of prompt + # tokens read from cache rather than billed in full. Every request method + # writes it, so a caller reads it after the call it made. One provider is + # built per request. + self.last_usage: dict | None = None + + def _headers(self) -> dict: + headers = {"Content-Type": "application/json"} + if self.api_key: + headers["Authorization"] = f"Bearer {self.api_key}" + return headers + + def _apply_reasoning_budget(self, body: dict) -> None: + """Gives reasoning models their own thinking budget, in the OpenRouter style. + + The method raises `max_tokens`, so the output keeps its full budget. + + A negative budget does the opposite. It sends `effort: "none"` to turn + reasoning off on a model that reasons by default, such as DeepSeek V4 + Flash. That differs from `exclude: true`, which still reasons and still + bills for it while hiding the trace. Zero still means send nothing, so an + endpoint that rejects unknown fields, such as Ollama, keeps working. + """ + if self.api_mode != "chat": + return + if self.reasoning_max_tokens < 0: + body["reasoning"] = {"effort": "none"} + elif self.reasoning_max_tokens > 0: + body["reasoning"] = {"max_tokens": self.reasoning_max_tokens} + body["max_tokens"] += self.reasoning_max_tokens + + def _apply_provider_routing(self, body: dict) -> None: + """Prefers one upstream on OpenRouter, so the prompt cache stays warm. + + The method does nothing anywhere else. `provider` is an OpenRouter + extension, and Ollama and similar servers reject fields they do not know. + The `reasoning` parameter above is written around the same constraint. + """ + if _OPENROUTER_HOST not in self.base_url: + return + upstream = _PREFERRED_UPSTREAM.get(self.model.split("/", 1)[0].lower()) + if upstream: + body["provider"] = {"order": [upstream]} + + def _record_usage(self, payload: dict) -> None: + """Records the endpoint's own token accounting, if it reported any. + + OpenRouter now always reports usage, and `usage: {include: true}` and + `stream_options` are deprecated and do nothing. In a stream the usage + arrives on a final chunk that carries no choices, which is why this is + read separately from the text extraction. + """ + usage = payload.get("usage") + if isinstance(usage, dict) and usage: + self.last_usage = usage + + def _request(self, parts: PromptParts, temperature: float, max_tokens: int) -> tuple[str, dict]: + if self.api_mode == "completion": + url = f"{self.base_url}/completions" + body = { + "model": self.model, + "prompt": f"{parts.system}\n\n{parts.story}", + "temperature": temperature, + "max_tokens": max_tokens, + "stream": True, + } + else: + url = f"{self.base_url}/chat/completions" + body = { + "model": self.model, + "messages": [ + {"role": "system", "content": parts.system}, + {"role": "user", "content": parts.story + CHAT_CONTINUE_HINT}, + ], + "temperature": temperature, + "max_tokens": max_tokens, + "stream": True, + } + self._apply_reasoning_budget(body) + self._apply_provider_routing(body) + return url, body + + @staticmethod + def _extract_chunk(payload: dict) -> str: + choices = payload.get("choices") or [] + if not choices: + return "" + choice = choices[0] + # A chat stream uses `delta.content`, and a completion stream uses + # `text`. The non-stream fallbacks are `message.content` and `text`. + delta = choice.get("delta") or {} + return ( + delta.get("content") + or choice.get("text") + or (choice.get("message") or {}).get("content") + or "" + ) + + @staticmethod + def _extract_reasoning(payload: dict) -> str: + """Returns a reasoning model's thinking text. + + OpenRouter normalizes it to `reasoning`, and DeepSeek-style servers use + `reasoning_content`. + """ + choices = payload.get("choices") or [] + if not choices: + return "" + choice = choices[0] + delta = choice.get("delta") or {} + message = choice.get("message") or {} + return ( + delta.get("reasoning") + or delta.get("reasoning_content") + or message.get("reasoning") + or message.get("reasoning_content") + or "" + ) + + async def generate( + self, parts: PromptParts, *, temperature: float, max_tokens: int + ) -> AsyncIterator[tuple[str, str]]: + """Yields `("text", chunk)` and `("reasoning", chunk)` pairs.""" + if not self.model: + raise ProviderError("No model configured — set one in Settings.") + url, body = self._request(parts, temperature, max_tokens) + async for event in self._stream(url, body): + yield event + + async def chat( + self, messages: list[dict], *, temperature: float, max_tokens: int + ) -> AsyncIterator[tuple[str, str]]: + """Runs a plain multi-turn chat, with no story framing and no context + assembly. + + The method sends `[{"role", "content"}, ...]` straight to the endpoint. + The AI Chat scratchpad uses it, and the turn engine uses `generate()`. + """ + if not self.model: + raise ProviderError("No model configured — set one in Settings.") + if self.api_mode == "completion": + url = f"{self.base_url}/completions" + body = { + "model": self.model, + "prompt": flatten_messages(messages), + "temperature": temperature, + "max_tokens": max_tokens, + "stream": True, + } + else: + url = f"{self.base_url}/chat/completions" + body = { + "model": self.model, + "messages": messages, + "temperature": temperature, + "max_tokens": max_tokens, + "stream": True, + } + self._apply_reasoning_budget(body) + self._apply_provider_routing(body) + async for event in self._stream(url, body): + yield event + + async def _stream(self, url: str, body: dict) -> AsyncIterator[tuple[str, str]]: + """Runs the shared SSE request for `generate()` and `chat()`. + + The method POSTs a streaming request, yields `("text", chunk)` and + `("reasoning", chunk)` pairs, and logs the exchange. + """ + # SSRF guard for hosted mode. A user-supplied `endpoint_url` must not + # point at an internal or metadata address. This does nothing for a + # local install. + reason = netguard.endpoint_block_reason(url) + if reason: + raise ProviderError(f"This endpoint can't be used — {reason}.") + log = debuglog.start_entry(url, self.model, body) + received: list[str] = [] + try: + async with httpx.AsyncClient(timeout=httpx.Timeout(120, connect=10)) as client: + async with client.stream("POST", url, json=body, headers=self._headers()) as resp: + if resp.status_code != 200: + detail = (await resp.aread()).decode(errors="replace")[:500] + raise ProviderError(self._friendly_http_error(resp.status_code, detail)) + # Some servers ignore `stream=true` and return one plain + # JSON body, so buffer the non-SSE lines to fall back to. + saw_sse = False + raw_lines: list[str] = [] + async for line in resp.aiter_lines(): + if not line.startswith("data:"): + if not saw_sse: + raw_lines.append(line) + continue + saw_sse = True + data = line[5:].strip() + if data == "[DONE]": + debuglog.finish_entry( + log, response="".join(received), usage=self.last_usage + ) + return + try: + payload = json.loads(data) + except ValueError: + continue + self._record_usage(payload) + reasoning = self._extract_reasoning(payload) + if reasoning: + yield "reasoning", reasoning + chunk = self._extract_chunk(payload) + if chunk: + received.append(chunk) + yield "text", chunk + if not saw_sse: + body_text = "\n".join(raw_lines).strip() + try: + payload = json.loads(body_text) + except ValueError: + raise ProviderError( + "AI endpoint returned neither an SSE stream nor JSON: " + + body_text[:200] + ) + self._record_usage(payload) + reasoning = self._extract_reasoning(payload) + if reasoning: + yield "reasoning", reasoning + chunk = self._extract_chunk(payload) + if chunk: + received.append(chunk) + yield "text", chunk + if not received: + raise ProviderError( + "AI endpoint returned a response with no text: " + + body_text[:200] + ) + debuglog.finish_entry(log, response="".join(received), usage=self.last_usage) + except httpx.ConnectError as exc: + error = f"Could not connect to {self.base_url} — is the AI server running?" + debuglog.finish_entry(log, response="".join(received), error=error) + raise ProviderError(error) from exc + except httpx.TimeoutException as exc: + debuglog.finish_entry(log, response="".join(received), error="Timed out") + raise ProviderError("The AI endpoint timed out.") from exc + except httpx.HTTPError as exc: + debuglog.finish_entry(log, response="".join(received), error=str(exc)) + raise ProviderError(f"Request to AI endpoint failed: {exc}") from exc + except (ProviderError, GeneratorExit, BaseException) as exc: + status = "cancelled" if isinstance(exc, GeneratorExit) else str(exc) + debuglog.finish_entry(log, response="".join(received), error=status) + raise + + async def complete( + self, system: str, user: str, *, temperature: float = 0.3, max_tokens: int = 400 + ) -> str: + """Runs a single non-streaming completion, for background calls such as + summarization. + + Unlike `generate()`, this adds no story-continuation framing. + """ + if not self.model: + raise ProviderError("No model configured — set one in Settings.") + if self.api_mode == "completion": + url = f"{self.base_url}/completions" + body = { + "model": self.model, + "prompt": f"{system}\n\n{user}", + "temperature": temperature, + "max_tokens": max_tokens, + "stream": False, + } + else: + url = f"{self.base_url}/chat/completions" + body = { + "model": self.model, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + "temperature": temperature, + "max_tokens": max_tokens, + "stream": False, + } + self._apply_reasoning_budget(body) + self._apply_provider_routing(body) + + log = debuglog.start_entry(url, self.model, body) + try: + async with httpx.AsyncClient(timeout=httpx.Timeout(120, connect=10)) as client: + resp = await client.post(url, json=body, headers=self._headers()) + except httpx.HTTPError as exc: + debuglog.finish_entry(log, error=str(exc)) + raise ProviderError(f"Request to AI endpoint failed: {exc}") from exc + if resp.status_code != 200: + error = self._friendly_http_error(resp.status_code, resp.text[:500]) + debuglog.finish_entry(log, error=error) + raise ProviderError(error) + try: + payload = resp.json() + except ValueError as exc: + debuglog.finish_entry(log, error="Invalid JSON response") + raise ProviderError("AI endpoint returned invalid JSON.") from exc + self._record_usage(payload) + text = self._extract_chunk(payload) + debuglog.finish_entry(log, response=text, usage=self.last_usage) + return text.strip() + + async def embed(self, texts: list[str]) -> list[list[float]]: + """POSTs to /v1/embeddings. Here `self.model` is the embedding model.""" + if not self.model: + raise ProviderError("No embedding model configured — set one in Settings.") + url = f"{self.base_url}/embeddings" + body = {"model": self.model, "input": texts} + log = debuglog.start_entry(url, self.model, body) + try: + async with httpx.AsyncClient(timeout=httpx.Timeout(60, connect=10)) as client: + resp = await client.post(url, json=body, headers=self._headers()) + except httpx.HTTPError as exc: + debuglog.finish_entry(log, error=str(exc)) + raise ProviderError(f"Embedding request failed: {exc}") from exc + if resp.status_code != 200: + error = self._friendly_http_error(resp.status_code, resp.text[:500]) + debuglog.finish_entry(log, error=error) + raise ProviderError(error) + try: + data = resp.json().get("data", []) + vectors = [item["embedding"] for item in sorted(data, key=lambda d: d.get("index", 0))] + except (ValueError, KeyError, TypeError) as exc: + debuglog.finish_entry(log, error="Malformed embeddings response") + raise ProviderError("AI endpoint returned malformed embeddings.") from exc + if len(vectors) != len(texts): + debuglog.finish_entry(log, error="Embedding count mismatch") + raise ProviderError("AI endpoint returned the wrong number of embeddings.") + debuglog.finish_entry(log, response=f"{len(vectors)} vectors × {len(vectors[0]) if vectors else 0} dims") + return vectors + + def _friendly_http_error(self, status: int, detail: str) -> str: + if status == 401: + return "Authentication failed — check your API key in Settings." + if status == 404: + return ( + f"Endpoint or model not found (HTTP 404). Check the endpoint URL and that " + f"model '{self.model}' exists. {detail}" + ) + if status == 429: + # OpenRouter's shared free tier has a per-day cap. Distinguish it + # from a short-term burst limit, so the message tells the reader what + # to do. + if "free-models-per-day" in detail: + return ( + "The free demo has hit its daily request limit (resets at " + "00:00 UTC). Please try again later." + ) + return "The AI is getting too many requests right now — wait a moment and try again." + return f"AI endpoint returned HTTP {status}: {detail}" diff --git a/backend/app/routers/__init__.py b/backend/app/routers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/routers/adventures/__init__.py b/backend/app/routers/adventures/__init__.py new file mode 100644 index 0000000..0996c11 --- /dev/null +++ b/backend/app/routers/adventures/__init__.py @@ -0,0 +1,56 @@ +"""The adventure endpoints, split across one module per group of routes. + +Importing this package registers every route, because each endpoint module +decorates its handlers with the shared `router` from `deps`. The import order +below is the order FastAPI matches paths in. No two routes here shadow each +other, so the order is for reading rather than for correctness. + +Read the modules in this order to follow a turn from end to end: + + deps the router and the ownership check every endpoint runs + paging reading a window of actions and numbering its attempts + nodes moving around the story tree + turns playing a turn, and the lock that allows only one at a time + takes retries and the attempts that collect at one coordinate + branches where a story splits + +What this package re-exports, and what it deliberately does not: + +Pure helpers and handlers are re-exported below, so `adventures.ACTION_PAGE` and +`adventures.undo_turn` keep working. The names a test replaces are not, and you +must reach those as `adventures.turns.`. Rebinding a re-exported alias +changes only the alias, so patching `adventures.generate_turn` would leave every +caller reading the original. Leaving those names off raises `AttributeError` +instead, which is the failure you want. +""" +from .deps import router + +# Imported for the side effect of registering routes. The names are unused here. +from . import ( # noqa: F401 + crud, + turns, + takes, + branches, + bundle_io, + scripts, + refresh, + insights, + memories, + actions, +) +from ... import limits # noqa: F401 `adventures.limits` is patched by tests. +from .crud import SNIPPET_MAX, _snippet +from .paging import ACTION_PAGE +from .takes import retry_action, undo_turn +from .turns import world_delta_of + +__all__ = [ + "ACTION_PAGE", + "SNIPPET_MAX", + "_snippet", + "limits", + "retry_action", + "router", + "undo_turn", + "world_delta_of", +] diff --git a/backend/app/routers/adventures/actions.py b/backend/app/routers/adventures/actions.py new file mode 100644 index 0000000..7c46ce1 --- /dev/null +++ b/backend/app/routers/adventures/actions.py @@ -0,0 +1,105 @@ +"""Reading, editing, and deleting individual actions. + +`list_actions` pages through the current branch. The edit and delete endpoints +change one node, and deleting one removes the whole attempt group at its +coordinate through `nodes.delete_turn`. +""" + +from fastapi import Depends, HTTPException +from sqlalchemy.orm import Session + +from ... import attempts, models, schemas, tree +from ...database import get_db + +from . import turns +from .deps import CurrentUser, current_adventure, router +from .nodes import db_tip, delete_turn +from .paging import ACTION_PAGE, action_window, annotate_takes + + +@router.get("/{adventure_id}/actions", response_model=schemas.ActionPage) +def list_actions( + before_id: int | None = None, + limit: int = ACTION_PAGE, + db: Session = Depends(get_db), + adventure: models.Adventure = Depends(current_adventure), +): + """Returns a page of the story, working backwards from the newest action. + + `before_id` is the oldest action the caller already holds, so scrolling up + asks for what comes before it. Omit `before_id` for the newest window. See + `action_window` for why this anchors on a row rather than an offset. + """ + limit = max(1, min(limit, ACTION_PAGE * 4)) + actions, total, has_more = action_window( + db, adventure, before_id=before_id, limit=limit + ) + return schemas.ActionPage( + actions=[ + schemas.ActionOut.model_validate(a) + for a in annotate_takes(db, adventure.id, actions) + ], + total=total, + has_more=has_more, + ) + + +@router.patch("/{adventure_id}/actions/{action_id}", response_model=schemas.ActionOut) +def update_action( + adventure_id: int, + action_id: int, + payload: schemas.ActionUpdate, + db: Session = Depends(get_db), + adventure: models.Adventure = Depends(current_adventure), +): + action = db.get(models.Action, action_id) + if action is None or action.adventure_id != adventure_id: + raise HTTPException(404, "Action not found") + # One row holds one text. Nothing mirrors it now, so nothing else has to be + # updated. The edit used to have to be written into the live variant entry + # as well, or paging away and back reverted it. + action.text = payload.text + db.commit() + return action + + +@router.delete("/{adventure_id}/actions/{action_id}", status_code=204) +def delete_action( + adventure_id: int, + action_id: int, + db: Session = Depends(get_db), + adventure: models.Adventure = Depends(current_adventure), +): + action = db.get(models.Action, action_id) + if action is None or action.adventure_id != adventure_id: + raise HTTPException(404, "Action not found") + # The lock is held for the same reason undo holds it: this endpoint puts + # the shared state back, and a turn that is still generating is about to + # write it. + turns.acquire_turn_lock(adventure_id) + try: + # This works like undo. The turn is deleted with all of its attempts, + # and whatever it produced is withdrawn. The marks are depths, and a + # depth does not move when an action before it is deleted. + delete_turn(db, adventure, action) + db.flush() + db.expire(adventure, ["actions"]) + # Deleting the newest action moves the tip. Deleting an action in the + # middle leaves a gap in the depths, which is intended. See + # `_backfill_tree`. + tree.refresh_head(db, adventure) + # The script state and the world state belong to the adventure, not to + # the node, so deleting the node does not take back what it did to + # them. Put them back to what the story now ends with, which is the + # same restore a branch switch does. + # + # The world state carries the cooldown clock in `_meta.last_changed`, + # and that clock is a depth. Leaving it set marked the deleted turn's + # changes as having happened at the depth the next turn is played at, + # so the referee refused them as changed too recently — on a turn the + # story no longer contains. Deleting a middle action restores the tip's + # own outcome, which is the state the adventure is already in. + attempts.restore_state(adventure, db_tip(db, adventure)) + db.commit() + finally: + turns._active_turns.discard(adventure_id) diff --git a/backend/app/routers/adventures/branches.py b/backend/app/routers/adventures/branches.py new file mode 100644 index 0000000..a09f73a --- /dev/null +++ b/backend/app/routers/adventures/branches.py @@ -0,0 +1,259 @@ +"""The branch endpoints: list, rename, delete, and switch. + +Attempts accumulate at the tip as siblings, which costs nothing. An attempt +becomes a branch only when the player continues the story from it and leaves the +line that moved past it. That is the same event as playing a turn past the +attempt. Creating the branch then rather than on the next turn means a branch +exists only for a divergence someone built on, and the line being left is not +modified. +""" + +from fastapi import Depends, HTTPException +from sqlalchemy import func +from sqlalchemy.orm import Session + +from ... import attempts, models, schemas, tree +from ...context import cursors +from ...context import lineage +from ...database import get_db + +from . import turns +from .deps import CurrentUser, current_adventure, router +from .nodes import db_tip +from .paging import current_window + + +@router.get("/{adventure_id}/branches", response_model=list[schemas.BranchOut]) +def list_branches( + db: Session = Depends(get_db), + adventure: models.Adventure = Depends(current_adventure), +): + """Returns every branch of the adventure and where each one leaves its parent. + + A tree view is drawn from this shape. `fork_depth` gives the depth where the + line splits off, and `depth` gives the depth where it currently ends. The + whole picture costs one query over `branches` plus one grouped query over + `actions`, never one query per branch, so a view of a hundred forks does not + cost a hundred round trips. + """ + branches = ( + db.query(models.Branch) + .filter(models.Branch.adventure_id == adventure.id) + .order_by(models.Branch.id) + .all() + ) + owned = { + branch_id: (count, tip) + for branch_id, count, tip in db.query( + models.Action.branch_id, + func.count(models.Action.id), + func.max(models.Action.depth), + ) + .filter( + models.Action.adventure_id == adventure.id, + models.Action.live.is_(True), + ) + .group_by(models.Action.branch_id) + .all() + } + out = [] + for branch in branches: + count, tip = owned.get(branch.id, (0, None)) + out.append(schemas.BranchOut( + id=branch.id, + parent_branch_id=branch.parent_branch_id, + fork_depth=branch.fork_depth, + # A branch with no nodes of its own sits at its fork point. That + # node is the last one its story contains. The node is borrowed, but + # it is still the tip. This matches `tree.refresh_head`. + depth=tip if tip is not None else ( + branch.fork_depth if branch.fork_depth is not None else tree.NO_DEPTH + ), + own_actions=count, + is_head=(branch.id == adventure.head_branch_id), + name=branch.name, + created_at=branch.created_at, + )) + return out + + +def get_branch_or_404( + adventure: models.Adventure, branch_id: int, db: Session +) -> models.Branch: + """Returns one branch of this adventure. + + If the branch belongs to another adventure, the 404 does not confirm that the + branch exists. + """ + branch = db.get(models.Branch, branch_id) + if branch is None or branch.adventure_id != adventure.id: + raise HTTPException(404, "Branch not found") + return branch + + +@router.patch( + "/{adventure_id}/branches/{branch_id}", response_model=schemas.BranchOut +) +def rename_branch( + branch_id: int, + payload: schemas.BranchRename, + db: Session = Depends(get_db), + adventure: models.Adventure = Depends(current_adventure), +): + """Names a branch, or clears the name to leave it unnamed. + + A blank string means the same thing as `null`. A name of only spaces is not a + name anyone chose, and storing one gives the client an empty label to draw + instead of the fork depth. + """ + branch = get_branch_or_404(adventure, branch_id, db) + name = (payload.name or "").strip() + branch.name = name or None + adventure.updated_at = models.utcnow() + db.commit() + db.refresh(branch) + # Read both numbers in one pass, and count them the way `list_branches` + # counts them, as live rows on this branch. A renamed branch is the same + # branch, so this response has to match the row the panel would fetch. + tip, own = ( + db.query(func.max(models.Action.depth), func.count(models.Action.id)) + .filter( + models.Action.adventure_id == adventure.id, + models.Action.branch_id == branch.id, + models.Action.live.is_(True), + ) + .one() + ) + return schemas.BranchOut( + id=branch.id, + parent_branch_id=branch.parent_branch_id, + fork_depth=branch.fork_depth, + depth=tip if tip is not None else ( + branch.fork_depth if branch.fork_depth is not None else tree.NO_DEPTH + ), + own_actions=own, + is_head=(branch.id == adventure.head_branch_id), + name=branch.name, + created_at=branch.created_at, + ) + + +@router.delete("/{adventure_id}/branches/{branch_id}", status_code=204) +def delete_branch( + adventure_id: int, + branch_id: int, + db: Session = Depends(get_db), + adventure: models.Adventure = Depends(current_adventure), +): + """Deletes a branch and everything forked from it. + + Nothing prunes the tree automatically, so this endpoint is what keeps a + heavily retried adventure from growing without bound. That is why it ships + with the view that first lets anyone create a fork rather than after it. + + Two kinds of branch cannot be deleted. The root cannot, because it holds the + turns every other branch borrows, so deleting it deletes the whole story. The + branch currently being read cannot, and neither can any branch it was forked + from, because the cascade would remove the head under the player and leave + `head_branch_id` dangling. Switch branches first. + + Nodes and memories are deleted by `ON DELETE CASCADE`, and descendants by the + cascade on `branches.parent_branch_id`, so the delete is a single statement + however deep the subtree is. + """ + branch = get_branch_or_404(adventure, branch_id, db) + if branch.parent_branch_id is None: + raise HTTPException( + 400, "This is the story's first branch — deleting it would delete " + "the adventure. Delete the adventure itself instead.", + ) + head = db.get(models.Branch, adventure.head_branch_id) + # The head's lineage lists itself and every branch it borrows from, so one + # membership test covers both the branch being read and any branch forked + # from it. + if head is not None and branch.id in { + entry_id for entry_id, _ in lineage.entries_of(head) + }: + raise HTTPException( + 400, "You are reading this branch, or one forked from it. Switch to " + "another branch first.", + ) + turns.acquire_turn_lock(adventure_id) + try: + # Collect the subtree before the delete, because afterwards there is no + # way to ask which branches were removed. A cursor left pointing at a + # deleted branch is harmless on Postgres, which never reuses ids, but it + # is a bug on SQLite, where the next fork can receive the id that was + # just freed. A stale anchor then resolves onto a branch it never saw. + doomed = _branch_subtree(db, adventure, branch) + for cursor in cursors.ALL: + stored_branch, _ = cursor.stored(adventure) + if stored_branch in doomed: + cursor.clear(adventure) + # The deleted branch's memories are deleted with it, and their cached + # vectors drop out of the catalogue on the next read, so no + # invalidation call is needed. See the note on the `memorybank` cache. + db.delete(branch) + adventure.updated_at = models.utcnow() + db.commit() + finally: + turns._active_turns.discard(adventure_id) + +def _branch_subtree( + db: Session, adventure: models.Adventure, root: models.Branch +) -> set[int]: + """Returns `root` and every branch descended from it, following parent pointers. + + The walk runs over the adventure's own branch rows rather than one query per + level. An adventure has few branches, so the walk costs one round trip, and a + recursive CTE would have to be written twice for the two dialects this + codebase supports. + """ + children: dict[int | None, list[int]] = {} + for bid, parent in db.query(models.Branch.id, models.Branch.parent_branch_id).filter( + models.Branch.adventure_id == adventure.id + ): + children.setdefault(parent, []).append(bid) + found: set[int] = set() + stack = [root.id] + while stack: + current = stack.pop() + if current in found: + continue + found.add(current) + stack.extend(children.get(current, ())) + return found + + +@router.post( + "/{adventure_id}/branches/{branch_id}/switch", response_model=schemas.ActionPage +) +def switch_branch( + adventure_id: int, + branch_id: int, + db: Session = Depends(get_db), + adventure: models.Adventure = Depends(current_adventure), +): + """Reads and plays a different branch of the story. + + No row is copied and no row is rewritten. The head pointer moves, and the + shared script state and world state are restored to what that branch's tip + left behind. The restore is what makes a switch safe. Both states are stored + per adventure, so a branch that did not restore them would be played with + another branch's numbers, including the world-state cooldown clock inside the + snapshot. + """ + branch = db.get(models.Branch, branch_id) + if branch is None or branch.adventure_id != adventure.id: + raise HTTPException(404, "Branch not found") + turns.acquire_turn_lock(adventure_id) + try: + adventure.head_branch_id = branch.id + tree.refresh_head(db, adventure) + attempts.restore_state(adventure, db_tip(db, adventure)) + adventure.updated_at = models.utcnow() + db.commit() + db.refresh(adventure) + return current_window(db, adventure) + finally: + turns._active_turns.discard(adventure_id) diff --git a/backend/app/routers/adventures/bundle_io.py b/backend/app/routers/adventures/bundle_io.py new file mode 100644 index 0000000..3c1fd50 --- /dev/null +++ b/backend/app/routers/adventures/bundle_io.py @@ -0,0 +1,70 @@ +"""Exporting an adventure to a bundle, and importing one back. + +`app/bundle.py` owns the format and the version handling. These two endpoints +only check ownership and hand the work over. +""" + +from fastapi import Body, Depends, Request +from sqlalchemy.orm import Session + +from ... import analytics, bundle, limits, models, schemas +from ...database import get_db + +from .deps import CurrentUser, current_adventure, router + + +@router.get("/{adventure_id}/export") +def export_adventure( + db: Session = Depends(get_db), + adv: models.Adventure = Depends(current_adventure), +): + """Returns a full backup: plot components, story cards, scripts, state, and tree. + + `app/bundle.py` owns the format, in both of its versions. A backup outlives + the schema, so no call site decides anything about its shape. + """ + return bundle.export(db, adv) + + +@router.post("/import", response_model=schemas.AdventureOut, status_code=201) +def import_adventure( + request: Request, + payload: dict = Body(...), + db: Session = Depends(get_db), + user: models.User = CurrentUser, +): + version = bundle.check_format(payload) + limits.rate_limit("import", request, user) + limits.check_row_cap("adventures", db, user) + limits.check_bundle_lists( + story_cards=payload.get("storyCards"), + memories=payload.get("memories"), + actions=payload.get("actions"), + branches=payload.get("branches"), + ) + # Check the tree before the adventure row exists, so that an inconsistent + # file returns a 400 rather than leaving a half-imported adventure with a + # gap in its story. + story = bundle.plan(payload, version) + # Count again, this time over what is written. The check above reads the + # file's own lists, and in a v1 file one turn is one entry that carries its + # retries in a `variants` array. `plan()` expands that into one row per + # attempt, because SP4 made every attempt a node. A file of 5,000 turns with + # ten attempts each therefore passes a 5,000-action cap and writes 50,000 + # rows, well inside the 20 MB body limit. `plan()` has no side effects and + # the adventure does not exist yet, so this check costs only the planning. + limits.check_bundle_lists( + actions=story["nodes"], + memories=story["memories"], + branches=story["branches"], + ) + + adventure = bundle.materialize(db, payload, story, user.id) + + db.commit() + db.refresh(adventure) + # This is not a funnel step. A returning player imports a bundle, so it + # says nothing about how far a first-time visitor got. It is counted anyway, + # because it is the clearest evidence that anyone uses the export format. + analytics.record_event(analytics.EV_IMPORT, user) + return adventure diff --git a/backend/app/routers/adventures/crud.py b/backend/app/routers/adventures/crud.py new file mode 100644 index 0000000..e2d2876 --- /dev/null +++ b/backend/app/routers/adventures/crud.py @@ -0,0 +1,340 @@ +"""Listing, creating, reading, renaming, and deleting adventures. + +The world-state and script-state readers are here too, because they report an +adventure's stored state rather than play a turn. +""" + +from fastapi import Body, Depends, HTTPException +from sqlalchemy import func +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import set_committed_value + +from ... import analytics, attempts, images, limits, memorybank, models, schemas, tree, worldstate +from ...database import get_db + +from .deps import CurrentUser, current_adventure, router +from .paging import action_window, annotate_takes +from .scenario_text import fill_placeholders, scenario_card_specs + + +# How many characters of the last narration a Continue card shows. The limit is +# long enough to re-establish the scene and short enough to keep the card +# small. +SNIPPET_MAX = 220 + + +def _snippet(text: str) -> str: + """Condenses stored action text into a single line for a card.""" + # `turns._generate_turn` strips the world-state block before storing AI text, + # so this function only has to normalize whitespace. + collapsed = " ".join((text or "").split()) + if len(collapsed) <= SNIPPET_MAX: + return collapsed + # Cut at a word boundary rather than mid-word. CSS adds the ellipsis. + cut = collapsed[:SNIPPET_MAX].rsplit(" ", 1)[0] + return f"{cut}…" + + +# Action types that read as narration. `start` is the scenario's opening prompt, +# which is the only text a newly created adventure has. Without `start`, a new +# story's card would show no text at all. `do` and `say` are excluded because +# the card quotes the story's voice rather than the player's. +NARRATION_TYPES = ("ai", "story", "start") + + +def _latest_narration(db: Session, head_branches: dict[int, int | None]) -> dict[int, str]: + """Maps each adventure id to the text of its most recent narrated action. + + This runs one window-function query rather than one lookup per adventure, so + the list endpoint makes a fixed number of round trips. + + The query is scoped by head branch rather than by the full lineage, and this + is the only place in the codebase that does so. A lineage clause per + adventure would add a hundred OR terms to the index screen's query to select + one row each. The two scopes differ only for a branch with no nodes of its + own, and playing a turn onto a branch is what creates it, so that state does + not occur. An adventure with no branch has no story to quote. + """ + branch_ids = [b for b in head_branches.values() if b is not None] + if not branch_ids: + return {} + ranked = ( + db.query( + models.Action.adventure_id.label("adventure_id"), + models.Action.text.label("text"), + func.row_number() + .over( + partition_by=models.Action.adventure_id, + order_by=(models.Action.depth.desc(), models.Action.id.desc()), + ) + .label("rank"), + ) + .filter( + models.Action.adventure_id.in_(list(head_branches)), + models.Action.branch_id.in_(branch_ids), + models.Action.type.in_(NARRATION_TYPES), + # Sibling attempts share a depth, and the newest has the highest + # id. Without this filter the snippet quotes the attempt written + # last rather than the one the story tells. After you switch back to + # an earlier attempt, the index screen would quote the discarded one + # and disagree with the story on screen. + models.Action.live.is_(True), + ) + .subquery() + ) + rows = db.query(ranked.c.adventure_id, ranked.c.text).filter(ranked.c.rank == 1).all() + return {adventure_id: text for adventure_id, text in rows} + + +@router.get("", response_model=list[schemas.AdventureListItem]) +def list_adventures(db: Session = Depends(get_db), user: models.User = CurrentUser): + # Select named columns rather than the whole Adventure entity. The entity is + # sixteen columns wide and includes `script_state`, `world_state`, + # `placeholders`, `story_summary`, `memory`, `authors_note`, and + # `ai_instructions`. That is about 15 kB per row in production, fetched once + # per adventure on every index load, and this screen uses none of it. Naming + # the columns also means a wide column added to Adventure later has to opt + # in to being listed here. + rows = ( + db.query( + models.Adventure.id, + models.Adventure.scenario_id, + models.Adventure.title, + models.Adventure.updated_at, + models.Adventure.head_branch_id, + func.count(models.Action.id), + models.Scenario.title, + models.Scenario.image, + models.Scenario.icon, + models.Scenario.updated_at, + ) + .outerjoin(models.Action) + .outerjoin(models.Scenario, models.Adventure.scenario_id == models.Scenario.id) + .filter(models.Adventure.user_id == user.id) + # Group by both primary keys. Postgres requires every selected column + # to be grouped or aggregated. The Adventure columns are covered by its + # own grouped primary key, but the Scenario columns come from a joined + # table and have to be listed as well. SQLite accepts the shorter form, + # and Postgres rejects it. + .group_by( + models.Adventure.id, + models.Scenario.id, + models.Scenario.title, + models.Scenario.image, + models.Scenario.icon, + models.Scenario.updated_at, + ) + .order_by(models.Adventure.updated_at.desc()) + .all() + ) + narration = _latest_narration(db, {row[0]: row[4] for row in rows}) + return [ + schemas.AdventureListItem( + id=adv_id, + scenario_id=scenario_id, + scenario_title=scenario_title, + title=title, + updated_at=updated_at, + action_count=count, + snippet=_snippet(narration.get(adv_id, "")), + # The art belongs to the scenario, so the cache-busting stamp uses + # the scenario's `updated_at`, not the adventure's. + image_url=images.public_url(scenario_id, image or "", scenario_updated), + icon=icon or "", + ) + # `count` counts every action in the adventure, not only the ones on + # the path. With one branch the two numbers are equal. After forking + # ships, the index screen overstates a story that has sibling branches. + # The fix belongs to SP5, which is where a fork can first exist. + for (adv_id, scenario_id, title, updated_at, _head_branch_id, count, + scenario_title, image, icon, scenario_updated) in rows + ] + + +@router.post("", response_model=schemas.AdventureOut, status_code=201) +def create_adventure( + payload: schemas.AdventureCreate, + db: Session = Depends(get_db), + user: models.User = CurrentUser, +): + limits.check_row_cap("adventures", db, user) + scenario = None + if payload.scenario_id is not None: + scenario = db.get(models.Scenario, payload.scenario_id) + # A scenario is playable if the user owns it or if it is public. + if scenario is None or (scenario.user_id != user.id and not scenario.is_public): + raise HTTPException(404, "Scenario not found") + + values = payload.placeholders + adventure = models.Adventure( + user_id=user.id, + scenario_id=scenario.id if scenario else None, + title=payload.title or (scenario.title if scenario else "Untitled Adventure"), + memory=fill_placeholders(scenario.memory, values) if scenario else "", + authors_note=fill_placeholders(scenario.authors_note, values) if scenario else "", + ai_instructions=fill_placeholders(scenario.ai_instructions, values) if scenario else "", + # Phase 12: seed the live RPG state from the scenario's template. + world_state=worldstate.instantiate(scenario.stat_schema) if scenario else {}, + # Stored so that a later "Update from scenario" fills the copied text + # with the same answers instead of inserting literal `${...}` tokens. + placeholders=dict(values), + # Phase 18: the persona the player named before starting. It belongs to + # the adventure rather than the scenario, so nothing is copied here and + # "Update from scenario" never touches it. + persona_name=payload.persona_name.strip(), + persona_pronouns=payload.persona_pronouns.strip(), + persona_desc=payload.persona_desc.strip(), + ) + db.add(adventure) + db.flush() + # Give every adventure a story tree as soon as it exists, before anything + # is played onto it. Otherwise the tree code has to tolerate a NULL head + # everywhere, which buys nothing. + tree.head_branch(db, adventure) + + if scenario: + for ref, spec in scenario_card_specs(scenario, values).items(): + db.add(models.StoryCard(adventure_id=adventure.id, source_ref=ref, **spec)) + for position, script in enumerate(scenario.scripts): + db.add( + models.AdventureScript( + adventure_id=adventure.id, + source_script_id=script.id, + position=position, + name=script.name, + description=script.description, + library_js=script.library_js, + input_js=script.input_js, + context_js=script.context_js, + output_js=script.output_js, + ) + ) + if scenario.prompt.strip(): + opening = models.Action( + adventure_id=adventure.id, + type="start", + text=fill_placeholders(scenario.prompt, values), + ) + # Record the starting state on the opening node, so undoing or + # retrying the first turn has a state to roll back to. + attempts.snapshot_outcome(adventure, opening) + tree.place_action(db, adventure, opening) + db.add(opening) + + db.commit() + db.refresh(adventure) + analytics.record_event(analytics.EV_ADVENTURE, user) + # Track which shared scenarios players pick. This is the only content this + # module records, and it records only public scenarios. A player's own + # scenario titles stay private. + if scenario is not None and scenario.is_public: + analytics.record(analytics.M_SCENARIO, scenario.title) + return adventure + + +@router.get("/{adventure_id}", response_model=schemas.AdventureOut) +def get_adventure( + db: Session = Depends(get_db), + adventure: models.Adventure = Depends(current_adventure), +): + """Returns the adventure and the newest window of its story. + + `actions` holds the last `ACTION_PAGE` actions, not all of them. + `action_count` reports the real total, so the reader knows that more actions + exist above. `GET /{id}/actions` serves the older pages as the reader + scrolls up. + """ + actions, total, _ = action_window(db, adventure) + # Annotate before handing over the window. This path serializes through the + # relationship rather than building `ActionOut` itself, so the pager numbers + # have to be on the rows before Pydantic reads them. + annotate_takes(db, adventure.id, actions) + # Attach the window as if the relationship had loaded it. + # `set_committed_value` is the only safe way to do this. Assigning + # `adventure.actions = [...]` marks the collection dirty, and the + # relationship cascades delete-orphan, so the next flush deletes every + # action outside the window. `set_committed_value` records the rows as the + # already-loaded, unmodified value, so serialization triggers no lazy load + # and leaves nothing pending. + set_committed_value(adventure, "actions", actions) + out = schemas.AdventureOut.model_validate(adventure) + out.action_count = total + return out + + +@router.get("/{adventure_id}/script-state") +def get_script_state( + adventure: models.Adventure = Depends(current_adventure), +): + """Returns the scripting `state` object. + + The object holds every variable that scripts read and write through + `state.x`, persisted after each hook. It stays `{}` until a script sets a + variable. + """ + state = adventure.script_state if isinstance(adventure.script_state, dict) else {} + return {"state": state} + + +@router.get("/{adventure_id}/world-state") +def get_world_state( + adventure: models.Adventure = Depends(current_adventure), +): + """Returns the live RPG world state and the scenario's `stat_schema`. + + The play view uses both to render the character sheet and the milestones. + `schema` is null when the adventure has no RPG layer. + """ + schema = adventure.scenario.stat_schema if adventure.scenario else None + state = adventure.world_state if isinstance(adventure.world_state, dict) else {} + return { + "state": state, + "schema": schema if worldstate.has_schema(schema) else None, + } + + +@router.put("/{adventure_id}/world-state") +def override_world_state( + overrides: dict = Body(...), + db: Session = Depends(get_db), + adventure: models.Adventure = Depends(current_adventure), +): + """Edits the live RPG values directly, as a manual correction rather than a turn. + + `overrides` maps paths such as `player.hp`, `npc.gwen.trust`, `flags.x`, and + `milestones.y` to their new absolute values. The endpoint rejects unknown + paths and wrong types one at a time, and applies the rest. + """ + schema = adventure.scenario.stat_schema if adventure.scenario else None + if not worldstate.has_schema(schema): + raise HTTPException(400, "This adventure has no RPG world-state layer") + state = adventure.world_state if isinstance(adventure.world_state, dict) else {} + new_state, report = worldstate.apply_override(state, schema, overrides) + adventure.world_state = new_state + db.commit() + return {"state": new_state, "report": report} + + +@router.patch("/{adventure_id}", response_model=schemas.AdventureOut) +def update_adventure( + payload: schemas.AdventureUpdate, + db: Session = Depends(get_db), + adventure: models.Adventure = Depends(current_adventure), +): + for field, value in payload.model_dump(exclude_unset=True).items(): + setattr(adventure, field, value) + db.commit() + return adventure + + +@router.delete("/{adventure_id}", status_code=204) +def delete_adventure( + adventure_id: int, + db: Session = Depends(get_db), + adventure: models.Adventure = Depends(current_adventure), +): + db.delete(adventure) + db.commit() + # No later request reads this adventure's vectors, so drop them now. The + # cache would otherwise hold them until the process restarted. + memorybank.forget_cached_vectors(adventure_id) diff --git a/backend/app/routers/adventures/deps.py b/backend/app/routers/adventures/deps.py new file mode 100644 index 0000000..d5e3137 --- /dev/null +++ b/backend/app/routers/adventures/deps.py @@ -0,0 +1,42 @@ +"""The router every endpoint module registers on, and the dependencies they share. + +This module imports nothing else in the package. Keeping it at the bottom of the +import graph is what lets each endpoint module import the router without +importing its siblings. +""" + +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy.orm import Session + +from ... import auth, models +from ...database import get_db + + +router = APIRouter(prefix="/api/adventures", tags=["adventures"]) + +CurrentUser = Depends(auth.get_current_user) + + +def get_adventure_or_404( + adventure_id: int, db: Session, user: models.User +) -> models.Adventure: + adventure = db.get(models.Adventure, adventure_id) + if adventure is None or adventure.user_id != user.id: + raise HTTPException(404, "Adventure not found") + return adventure + + +def current_adventure( + adventure_id: int, + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +) -> models.Adventure: + """Resolves the `{adventure_id}` in the path to the caller's own adventure. + + Declare this in a handler's signature. The ownership check then reads in the + signature, where you look for it, and it runs before the handler body. + + FastAPI caches a dependency for the length of one request, so the handler's + own `db` is the same session this one loaded the adventure from. + """ + return get_adventure_or_404(adventure_id, db, user) diff --git a/backend/app/routers/adventures/insights.py b/backend/app/routers/adventures/insights.py new file mode 100644 index 0000000..254f676 --- /dev/null +++ b/backend/app/routers/adventures/insights.py @@ -0,0 +1,50 @@ +"""Read-only views of the context an adventure would send, or did send. + +The dry run assembles a prompt without calling the model. The per-action endpoint +returns the prompt a turn was actually generated from. Neither writes anything. +""" + +from fastapi import Depends, HTTPException +from sqlalchemy.orm import Session + +from ... import auth, memorybank, models +from ...context import build_context +from ...database import get_db +from ..settings import get_settings + +from .deps import CurrentUser, current_adventure, router + + +@router.get("/{adventure_id}/context") +async def dry_run_context( + db: Session = Depends(get_db), + user: models.User = CurrentUser, + adventure: models.Adventure = Depends(current_adventure), +): + """Returns what the app would send to the AI if the player continued now.""" + settings = get_settings(db, user) + if auth.resolve_provider_config(settings).using_demo: + memories = ( + {"used": [], "error": "Memory bank is unavailable on the shared demo key."} + if adventure.memory_bank_enabled + else None + ) + else: + memories = await memorybank.retrieve_memories(adventure, settings, update_stats=False) + _, _, report = build_context(adventure, settings, memories) + return report + + +@router.get("/{adventure_id}/actions/{action_id}/context") +def action_context( + adventure_id: int, + action_id: int, + db: Session = Depends(get_db), + adventure: models.Adventure = Depends(current_adventure), +): + action = db.get(models.Action, action_id) + if action is None or action.adventure_id != adventure_id: + raise HTTPException(404, "Action not found") + if action.context_snapshot is None: + raise HTTPException(404, "No context snapshot for this action") + return action.context_snapshot diff --git a/backend/app/routers/adventures/memories.py b/backend/app/routers/adventures/memories.py new file mode 100644 index 0000000..119b8d3 --- /dev/null +++ b/backend/app/routers/adventures/memories.py @@ -0,0 +1,116 @@ +"""CRUD for the memory bank entries attached to one adventure. + +`app/memorybank.py` owns embedding and retrieval. These endpoints only edit the +rows. +""" + +from fastapi import Depends, HTTPException +from sqlalchemy.orm import Session, load_only + +from ... import limits, memorybank, models, schemas, tree +from ...context import lineage +from ...database import get_db + +from .deps import CurrentUser, current_adventure, router + + +# The columns `schemas.MemoryOut` renders. `embedded` is a real column and +# belongs here. The vector it describes does not. +MEMORY_LIST_COLUMNS = ( + models.Memory.adventure_id, + models.Memory.text, + models.Memory.pinned, + models.Memory.forgotten, + models.Memory.embedded, + models.Memory.use_count, + models.Memory.last_used_at, + models.Memory.source_start, + models.Memory.source_end, + models.Memory.created_at, +) + + +@router.get("/{adventure_id}/memories", response_model=list[schemas.MemoryOut]) +def list_memories( + adventure_id: int, + db: Session = Depends(get_db), + adventure: models.Adventure = Depends(current_adventure), +): + # Name the columns in a query rather than walk `adventure.memories`. + # Retrieval used to walk the relationship, which is why a turn cost + # megabytes: a relationship load returns whole entities, so it reads + # whatever the model carries. `embedding_blob` is deferred and would stay + # out today, so this rule is about the next wide column rather than that + # one. + # + # The drawer shows the same bank the model reads. The filter uses the same + # clause retrieval uses, so the drawer answers one question rather than two. + # An adventure-wide list would show memories from branches this story never + # went down, which are never retrieved, and a reader cannot tell those apart + # from the ones in play. No memory becomes unreachable, because a memory + # belongs to a branch: switching to that branch shows it, and deleting the + # branch deletes its memories. + return ( + db.query(models.Memory) + .options(load_only(*MEMORY_LIST_COLUMNS)) + .filter( + models.Memory.adventure_id == adventure_id, + lineage.path_of(db, adventure).clause(models.Memory), + ) + .order_by(models.Memory.id) + .all() + ) + + +@router.post("/{adventure_id}/memories", response_model=schemas.MemoryOut, status_code=201) +def create_memory( + payload: schemas.MemoryCreate, + db: Session = Depends(get_db), + user: models.User = CurrentUser, + adventure: models.Adventure = Depends(current_adventure), +): + """Adds a memory manually. The next post-turn pass embeds it.""" + limits.check_row_cap("memories", db, user, adventure=adventure) + if not payload.text.strip(): + raise HTTPException(400, "Memory text cannot be empty") + memory = models.Memory(adventure_id=adventure.id, text=payload.text.strip()) + # No node produced this memory, so it gets a branch but no depth. + tree.place_memory(db, adventure, memory) + db.add(memory) + db.commit() + db.refresh(memory) + return memory + + +@router.patch("/{adventure_id}/memories/{memory_id}", response_model=schemas.MemoryOut) +def update_memory( + adventure_id: int, + memory_id: int, + payload: schemas.MemoryUpdate, + db: Session = Depends(get_db), + adventure: models.Adventure = Depends(current_adventure), +): + memory = db.get(models.Memory, memory_id) + if memory is None or memory.adventure_id != adventure_id: + raise HTTPException(404, "Memory not found") + fields = {k: v for k, v in payload.model_dump(exclude_unset=True).items() if v is not None} + if "text" in fields and fields["text"].strip() != memory.text: + memorybank.set_vector(memory, None) # Re-embed on the next post-turn pass. + for field, value in fields.items(): + setattr(memory, field, value) + db.commit() + return memory + + +@router.delete("/{adventure_id}/memories/{memory_id}", status_code=204) +def delete_memory( + adventure_id: int, + memory_id: int, + db: Session = Depends(get_db), + adventure: models.Adventure = Depends(current_adventure), +): + memory = db.get(models.Memory, memory_id) + if memory is None or memory.adventure_id != adventure_id: + raise HTTPException(404, "Memory not found") + db.delete(memory) + db.commit() diff --git a/backend/app/routers/adventures/nodes.py b/backend/app/routers/adventures/nodes.py new file mode 100644 index 0000000..c55b279 --- /dev/null +++ b/backend/app/routers/adventures/nodes.py @@ -0,0 +1,139 @@ +"""Moving around the story tree: what is newest, what comes next, what to remove. + +These functions answer questions about action nodes without knowing which +endpoint asked. They do not touch the turn lock and they do not stream, so any +module in the package can import them. +""" + +from fastapi import HTTPException +from sqlalchemy.orm import Session, undefer + +from ... import attempts, memorybank, models, tree +from ...context import cursors +from ...context import lineage + + +def next_depth(adventure: models.Adventure) -> int: + """Returns the depth for the next node played onto this story, one past the tip. + + A depth is a position along one story, and the branch is what makes it + unambiguous. Two branches each hold a node at depth 4, and they are + alternatives rather than duplicates. + """ + return adventure.head_depth + 1 + + +def last_action(adventure: models.Adventure, db: Session) -> models.Action | None: + """Returns the newest action of any kind on the story being played, or `None`. + + This runs a query rather than reading `adventure.actions[-1]`, which loads + the entire story to read one row. That collection also holds every branch's + actions, so it sometimes returns a row from the wrong branch. + """ + return ( + db.query(models.Action) + .filter( + models.Action.adventure_id == adventure.id, + lineage.path_of(db, adventure).clause(models.Action), + ) + .order_by(models.Action.depth.desc(), models.Action.id.desc()) + .first() + ) + + +def _move_to_after( + db: Session, adventure: models.Adventure, after_id: int | None +) -> None: + """Moves the story to `after_id` before the turn is played. + + This is where a branch is created (SP9). Reading an attempt that the story + moved past changes nothing on the server. Writing below one is the first time + the player states which line they mean, and that is when the fork happens. + + An attempt already on the path needs no move, because the story is already + there. + """ + if after_id is None: + return + node = db.get(models.Action, after_id) + if node is None or node.adventure_id != adventure.id: + raise HTTPException(404, "Action not found") + if node.live and lineage.path_of(db, adventure).contains(node): + return + if not node.live and len(attempts.group(db, node)) < 2: + # The pager cannot reach this node, so no legitimate action put the + # player here. + raise HTTPException(400, "That take is not one of this turn's.") + stand_on(db, adventure, node) + db.commit() + db.refresh(adventure) + + +def delete_turn( + db: Session, adventure: models.Adventure, node: models.Action +) -> None: + """Removes a turn, including every attempt at it and not only the one on screen. + + A discarded attempt is a leaf at the same coordinate, and the only way to + reach it is through that coordinate. Leaving it behind when the turn is + deleted orphans a row that no read can reach. Whatever the turn produced is + withdrawn once, because a memory is attached to the coordinate rather than to + one attempt. + """ + memorybank.forget_node(db, adventure, node) + # Scoped to this node's branch (SP9). Groups span branches now, and an + # attempt forked onto its own line belongs to another branch's story. See + # `attempts.on_branch`. + for attempt in attempts.on_branch(attempts.group(db, node), node): + db.delete(attempt) + + +def db_tip(db: Session, adventure: models.Adventure) -> models.Action | None: + """Returns the newest node of the story as it stands, with its outcome loaded.""" + return ( + db.query(models.Action) + .filter( + models.Action.adventure_id == adventure.id, + lineage.path_of(db, adventure).clause(models.Action), + ) + .options( + undefer(models.Action.state_after), + undefer(models.Action.world_state_after), + ) + .order_by(models.Action.depth.desc(), models.Action.id.desc()) + .first() + ) + + +def stand_on( + db: Session, adventure: models.Adventure, action: models.Action +) -> None: + """Makes `action` the attempt the story tells, forking only if that is needed. + + There are two cases, and the caller does not have to know which one applies. + While the turn is still the tip, its attempts are leaves that nothing was + built on, so this is a switch and no branch is created. Once the story has + moved past the turn, the line being left keeps every turn it has, so the + attempt needs a branch of its own. + + The fork endpoint calls this function, and so does a turn played below an + attempt the story moved past. Both are the same operation, once as a request + and once as a step on the way to writing (SP9). + """ + newest = last_action(adventure, db) + at_the_tip = ( + newest is not None + and newest.branch_id == action.branch_id + and newest.depth == action.depth + ) + if at_the_tip: + # The story at this coordinate is about to change, so withdraw whatever + # was derived from it. A retry does the same thing. A fork needs none of + # this, because it leaves the coordinate and its memory where they are. + # See `tree.fork`. + memorybank.forget_node(db, adventure, action) + cursors.rewind_all(adventure, action.branch_id, (action.depth or 0) - 1) + attempts.make_live(db, adventure, action) + else: + tree.fork(db, adventure, action) + attempts.restore_state(adventure, action) diff --git a/backend/app/routers/adventures/paging.py b/backend/app/routers/adventures/paging.py new file mode 100644 index 0000000..92c0c23 --- /dev/null +++ b/backend/app/routers/adventures/paging.py @@ -0,0 +1,168 @@ +"""Reading a window of actions, and numbering the attempts inside it. + +Three callers build an action window: the adventure GET, the action list, and +every endpoint that returns a page after changing the story. They read the same +columns and apply the same numbering, so both live here. +""" + +from sqlalchemy import func +from sqlalchemy.orm import Session, load_only + +from ... import models, schemas +from ...context import lineage + + +# The columns `schemas.ActionOut` renders, listed explicitly. +# +# `deferred=True` in `models.py` keeps the four heavy columns out of bulk reads, +# but each new column then has to opt in to staying narrow. Both egress +# regressions this project has had came from a column that did not opt in. This +# tuple inverts the default: a new column costs nothing until you add it here. +# +# `world_delta` is listed because `ActionOut.world_changes` is computed from it. +# Omitting it saves no bytes. It converts one bulk read into one lazy load per +# row. +ACTION_LIST_COLUMNS = ( + models.Action.adventure_id, + models.Action.type, + models.Action.text, + models.Action.reasoning, + models.Action.world_delta, + # SP9: the pager's key. If `parent_id` were deferred, every row on the page + # would cost a lazy load, which is the cost `load_only` is here to prevent. + # `branch_id` is listed for the same reason. The pager reads it to tell a + # local step from a branch switch. + models.Action.parent_id, + models.Action.branch_id, + models.Action.created_at, +) + + +# How many actions an adventure opens with, and how many arrive per scroll. +# +# Opening a finished adventure once fetched the whole story in one response. +# That reached 589.5 kB for the longest story in production, and it grew as +# stories grew. 60 actions is a few screens of reading. The common case of +# opening a story, reading the end, and taking a turn never pages, and the worst +# case is bounded by the window size rather than by the length of the story. +ACTION_PAGE = 60 + + +def action_window( + db: Session, + adventure: models.Adventure, + before_id: int | None = None, + limit: int = ACTION_PAGE, +) -> tuple[list[models.Action], int, bool]: + """Returns the `limit` actions immediately older than `before_id`, oldest first. + + The return value is `(actions, total, has_more)`. If `before_id` is `None`, + the newest window is returned. + + The query is scoped to the head branch's lineage, which is the story being + played, rather than to the adventure. A sibling branch's turns therefore + never appear in the transcript. `total` counts the same path, because it is + what tells the reader that more actions exist above. + + The window is anchored on an action, never on a count or on arithmetic over + depth, for two reasons: + + * Appends. Counting back from the newest action shifts every older position + when a turn lands. A reader who scrolls up while a turn is generating gets + a window that is one row off, which re-sends one action and skips another. + An anchor is stable, because "older than this action" means the same thing + before and after the story grows. + * The story tree. Depth is dense today, and branching ends that. Comparing + depths to order a path still works, but treating them as positions does + not. + + `has_more` comes from requesting one row past the window rather than from a + second count, so it costs one row instead of a scan. + """ + path = lineage.path_of(db, adventure) + on_path = ( + models.Action.adventure_id == adventure.id, + path.clause(models.Action), + ) + total = db.query(func.count(models.Action.id)).filter(*on_path).scalar() + if limit <= 0: + return [], total, total > 0 + + query = db.query(models.Action).options(load_only(*ACTION_LIST_COLUMNS)).filter(*on_path) + if before_id is not None: + anchor = ( + db.query(models.Action.depth) + .filter(models.Action.id == before_id, *on_path) + .scalar() + ) + if anchor is None: + # The anchor was deleted while the reader scrolled, by an undo or + # by an edited turn, or it belongs to a branch this story is not on. + # No row can be older than a row that is not present, so report the + # end of the story rather than guess and return a duplicate page. + return [], total, False + query = query.filter(models.Action.depth < anchor) + + rows = ( + query.order_by(models.Action.depth.desc(), models.Action.id.desc()) + .limit(limit + 1) + .all() + ) + has_more = len(rows) > limit + rows = rows[:limit] + rows.reverse() + return rows, total, has_more + + +def annotate_takes( + db: Session, adventure_id: int, actions: list[models.Action] +) -> list[models.Action]: + """Sets the `2/4` pager numbers on every action on a page (SP9). + + This runs one query for the whole page rather than one per row. The pager + needs the shape of each turn's attempt group, and calling `attempts.group` + per action costs one query per message on screen. + + This function reads the siblings rather than counting them. A group holds + only a few attempts, the page is bounded, and a count still needs a second + query for the ordinal. It fetches only the id and the parent, so it stays + cheap even when the text is large. + """ + parents = {a.parent_id for a in actions if a.parent_id is not None} + if parents: + rows = ( + db.query(models.Action.id, models.Action.parent_id) + .filter( + models.Action.adventure_id == adventure_id, + models.Action.parent_id.in_(parents), + ) + .order_by(models.Action.id) + .all() + ) + else: + rows = [] + siblings: dict[int, list[int]] = {} + for row_id, parent_id in rows: + siblings.setdefault(parent_id, []).append(row_id) + for action in actions: + ids = siblings.get(action.parent_id) if action.parent_id else None + if not ids: + # A root node, or a pre-SP9 row that the backfill could not place. + # It has one attempt, which is how it was written. + action.take_count, action.take_index = 1, 0 + continue + action.take_count = len(ids) + action.take_index = ids.index(action.id) if action.id in ids else 0 + return actions + + +def current_window(db: Session, adventure: models.Adventure) -> schemas.ActionPage: + actions, total, has_more = action_window(db, adventure) + return schemas.ActionPage( + actions=[ + schemas.ActionOut.model_validate(a) + for a in annotate_takes(db, adventure.id, actions) + ], + total=total, + has_more=has_more, + ) diff --git a/backend/app/routers/adventures/refresh.py b/backend/app/routers/adventures/refresh.py new file mode 100644 index 0000000..c83fdff --- /dev/null +++ b/backend/app/routers/adventures/refresh.py @@ -0,0 +1,195 @@ +"""Copying a scenario's current content over an adventure's copy. + +An adventure copies the scenario's plot text and story cards when it is created, +so that later authoring does not change a story in progress. The per-script "Sync +from library" in `scripts` works the same way. This module is the explicit +opt-out. The preview endpoint reports what would change, and the write endpoint +applies it. +""" + +from fastapi import Body, Depends, HTTPException +from sqlalchemy.orm import Session + +from ... import models, schemas, worldstate +from ...database import get_db + +from . import turns +from .deps import CurrentUser, current_adventure, router +from .scenario_text import ( + CARD_FIELDS, SCENARIO_TEXT_FIELDS, fill_placeholders, scenario_card_specs, + scenario_placeholder_names, +) + + +def resolve_source_scenario( + adventure: models.Adventure, db: Session, user: models.User +) -> models.Scenario | None: + """Returns the scenario an adventure can refresh from. + + The result is the scenario the adventure was started from, if that scenario + still exists and the user can still read it, which means the user owns it or + it is public. The result is `None` after the scenario is deleted, which sets + `scenario_id` to NULL, or after it stops being shared. + """ + if adventure.scenario_id is None: + return None + scenario = db.get(models.Scenario, adventure.scenario_id) + if scenario is None or (scenario.user_id != user.id and not scenario.is_public): + return None + return scenario + + +def _scenario_cards(adventure: models.Adventure) -> dict[str, models.StoryCard]: + """Returns the adventure's scenario-derived cards, keyed by `source_ref`. + + An adventure created before `source_ref` existed has none, so it falls back + to matching the scenario's cards by name. The fallback runs only when the + adventure has no tagged cards at all. Otherwise a player-authored card that + shares a scenario card's name would be adopted and overwritten. + """ + return {c.source_ref: c for c in adventure.story_cards if c.source_ref} + + +def _match_legacy( + adventure: models.Adventure, specs: dict[str, dict] +) -> dict[str, models.StoryCard]: + by_name: dict[str, models.StoryCard] = {} + for card in adventure.story_cards: + by_name.setdefault((card.name or "").strip().lower(), card) + matched: dict[str, models.StoryCard] = {} + for ref, spec in specs.items(): + card = by_name.get((spec["name"] or "").strip().lower()) + if card is not None: + matched[ref] = card + return matched + + +def plan_refresh( + adventure: models.Adventure, scenario: models.Scenario, values: dict[str, str] +) -> tuple[dict, dict, dict]: + """Computes what a refresh would change, without modifying anything. + + The return value is `(plan, specs, matched)`. `plan` is the summary the UI + shows, `specs` holds the scenario's card specs by ref, and `matched` holds + the existing adventure card for each ref that already has one. + """ + fields = { + field: {"old": getattr(adventure, field), "new": fill_placeholders( + getattr(scenario, field), values)} + for field in SCENARIO_TEXT_FIELDS + } + changed_fields = {f: v for f, v in fields.items() if v["old"] != v["new"]} + + specs = scenario_card_specs(scenario, values) + tagged = _scenario_cards(adventure) + matched = tagged or _match_legacy(adventure, specs) + + added, updated = [], [] + for ref, spec in specs.items(): + card = matched.get(ref) + if card is None: + added.append(spec["name"]) + elif any(getattr(card, f) != spec[f] for f in CARD_FIELDS): + updated.append(card.name or spec["name"]) + # Only a card the scenario produced is removable. A player-authored card + # has no `source_ref` and is never modified. + removed = [c.name for ref, c in tagged.items() if ref not in specs] + + _, world = worldstate.reconcile(adventure.world_state, scenario.stat_schema) + + plan = { + "scenario_id": scenario.id, + "scenario_title": scenario.title, + "fields": changed_fields, + "cards": {"added": added, "updated": updated, "removed": removed}, + "world_state": world, + } + plan["has_changes"] = bool( + changed_fields or added or updated or removed + or world["added"] or world["removed"] + ) + return plan, specs, matched + + +@router.get("/{adventure_id}/refresh", response_model=schemas.RefreshPlan) +def preview_refresh( + db: Session = Depends(get_db), + user: models.User = CurrentUser, + adventure: models.Adventure = Depends(current_adventure), +): + """Returns what "Update from scenario" would change, for the confirm dialog.""" + scenario = resolve_source_scenario(adventure, db, user) + if scenario is None: + raise HTTPException(404, "No scenario to update from") + stored = adventure.placeholders if isinstance(adventure.placeholders, dict) else {} + plan, _, _ = plan_refresh(adventure, scenario, stored) + # An adventure started before placeholder answers were stored has none, and + # an author can add a new `${...}` later. In both cases the player is asked + # for the missing names, and the answers are saved for next time. + plan["placeholders_needed"] = [ + n for n in scenario_placeholder_names(scenario) if n not in stored + ] + return plan + + +@router.post("/{adventure_id}/refresh", response_model=schemas.AdventureOut) +def refresh_from_scenario( + adventure_id: int, + payload: schemas.AdventureRefresh = Body(default=schemas.AdventureRefresh()), + db: Session = Depends(get_db), + user: models.User = CurrentUser, + adventure: models.Adventure = Depends(current_adventure), +): + """Copies the scenario's current plot text, story cards, and stat schema over + this adventure's copy. + + The refresh overwrites the plot fields and every scenario-derived card, adds + what the scenario gained, and removes what it dropped. + + The refresh leaves these unchanged: the opening `start` action, because the + story is built on it and it is already part of the memories and the summary; + the adventure's own title; its story summary; its player-authored story + cards; and, through `worldstate.reconcile`, the live value of every stat the + schema still defines. + """ + scenario = resolve_source_scenario(adventure, db, user) + if scenario is None: + raise HTTPException(404, "No scenario to update from") + + values = {**(adventure.placeholders if isinstance(adventure.placeholders, dict) else {}), + **payload.placeholders} + + # A refresh rewrites the same state a turn is part-way through changing, so + # it takes the turn slot rather than run at the same time as the + # generator. + turns.acquire_turn_lock(adventure_id) + try: + _, specs, matched = plan_refresh(adventure, scenario, values) + + for field in SCENARIO_TEXT_FIELDS: + setattr(adventure, field, fill_placeholders(getattr(scenario, field), values)) + + for ref, spec in specs.items(): + card = matched.get(ref) + if card is None: + db.add(models.StoryCard(adventure_id=adventure.id, source_ref=ref, **spec)) + continue + for field in CARD_FIELDS: + setattr(card, field, spec[field]) + # Store the ref, so that a name-matched legacy card syncs by id + # next time. + card.source_ref = ref + for ref, card in _scenario_cards(adventure).items(): + if ref not in specs: + db.delete(card) + + adventure.world_state, _ = worldstate.reconcile( + adventure.world_state, scenario.stat_schema + ) + adventure.placeholders = values + db.commit() + finally: + turns._active_turns.discard(adventure_id) + + db.refresh(adventure) + return adventure diff --git a/backend/app/routers/adventures/scenario_text.py b/backend/app/routers/adventures/scenario_text.py new file mode 100644 index 0000000..9812230 --- /dev/null +++ b/backend/app/routers/adventures/scenario_text.py @@ -0,0 +1,106 @@ +"""Copying a scenario's text and story cards onto an adventure. + +An adventure holds its own copy of the scenario's plot text and cards, so that +later authoring does not change a story in progress. Two callers make that copy: +`crud.create_adventure` on the way in, and `refresh` when the player asks for the +scenario's current content. +""" +import re + + +from ... import models, worldstate + + +PLACEHOLDER_RE = re.compile(r"\$\{([^}]+)\}") + + +def fill_placeholders(text: str, values: dict[str, str]) -> str: + """Replaces `${Name}` with the player-provided value. + + Unknown names are left unchanged. + """ + if not text or not values: + return text + return PLACEHOLDER_RE.sub( + lambda m: values.get(m.group(1).strip(), m.group(0)), text + ) + + +# Adventure fields that start as a copy of the scenario's text, so "Update from +# scenario" can copy them again. `title` is excluded because it is the +# adventure's own name, which players rename. `story_summary` is excluded +# because it is play output rather than scenario content. +SCENARIO_TEXT_FIELDS = ("memory", "authors_note", "ai_instructions") + + +# Story-card fields that are copied from the scenario and compared to detect +# drift. +CARD_FIELDS = ("type", "name", "keys", "entry", "notes") + + +def scenario_card_specs(scenario: models.Scenario, values: dict[str, str]) -> dict[str, dict]: + """Returns every story card a scenario implies, keyed by a stable `source_ref`. + + The result holds the scenario's own cards, keyed `card:`, plus one card + per NPC defined in its `stat_schema`, keyed `npc:`. Placeholders are + already filled in. + + Adventure creation and refresh both call this function, so the two cannot + diverge. + """ + specs: dict[str, dict] = {} + existing_names = {(c.name or "").strip().lower() for c in scenario.story_cards} + for card in scenario.story_cards: + specs[f"card:{card.id}"] = { + "type": card.type, + "name": card.name, + "keys": fill_placeholders(card.keys, values), + "entry": fill_placeholders(card.entry, values), + "notes": card.notes, + } + # Phase 12: each defined NPC gets a story card, so its description works as + # lore and can trigger in a scene. If a card with that name already exists, + # skip the NPC. + for npc_key, ndef in (scenario.stat_schema or {}).get("npcs", {}).items(): + if not isinstance(ndef, dict): + continue + name = worldstate.npc_name(ndef, npc_key) + if name.strip().lower() in existing_names: + continue + specs[f"npc:{npc_key}"] = { + "type": "character", + "name": name, + "keys": fill_placeholders(str(ndef.get("keys") or name), values), + "entry": fill_placeholders(str(ndef.get("desc") or ""), values), + "notes": "", + } + return specs + + +def _placeholder_names(*texts: str) -> list[str]: + """Returns the unique `${Placeholder}` names across the given texts. + + The order is first appearance first. This matches the frontend's + `extractPlaceholders`. + """ + names: list[str] = [] + for text in texts: + for match in PLACEHOLDER_RE.finditer(text or ""): + name = match.group(1).strip() + if name and name not in names: + names.append(name) + return names + + +def scenario_placeholder_names(scenario: models.Scenario) -> list[str]: + """Returns every placeholder the scenario's refreshable content asks for. + + The opening prompt is excluded, because a refresh never rewrites it. + """ + texts = [scenario.memory, scenario.authors_note, scenario.ai_instructions] + for card in scenario.story_cards: + texts += [card.keys, card.entry] + for ndef in (scenario.stat_schema or {}).get("npcs", {}).values(): + if isinstance(ndef, dict): + texts += [str(ndef.get("keys") or ""), str(ndef.get("desc") or "")] + return _placeholder_names(*texts) diff --git a/backend/app/routers/adventures/scripts.py b/backend/app/routers/adventures/scripts.py new file mode 100644 index 0000000..5e878ec --- /dev/null +++ b/backend/app/routers/adventures/scripts.py @@ -0,0 +1,116 @@ +"""The per-adventure copies of library scripts. + +An adventure snapshots a library `Script` when it starts, so editing the library +does not change a story in progress. These endpoints report whether a snapshot +has fallen behind its library original, and copy the original over on request. +""" + +from fastapi import Depends, HTTPException +from sqlalchemy.orm import Session + +from ... import models, schemas +from ...database import get_db + +from .deps import CurrentUser, current_adventure, router + + +# Fields that are copied from a library Script into its adventure-script +# snapshot, and compared to decide whether a copy is out of date. +SYNC_FIELDS = ("name", "description", "library_js", "input_js", "context_js", "output_js") + + +def resolve_library_script( + adv_script: models.AdventureScript, db: Session, user: models.User +) -> models.Script | None: + """Returns the library Script an adventure script can re-sync from. + + The result is the script this copy was made from. For a legacy copy with no + link, it is one of the player's own scripts with the same name. Only the + player's own scripts are considered, so a copy derived from a demo scenario + has nothing to sync to. + """ + if adv_script.source_script_id is not None: + script = db.get(models.Script, adv_script.source_script_id) + if script is not None and script.user_id == user.id: + return script + return ( + db.query(models.Script) + .filter(models.Script.user_id == user.id, models.Script.name == adv_script.name) + .order_by(models.Script.updated_at.desc()) + .first() + ) + + +def _mark_out_of_date( + adv_script: models.AdventureScript, db: Session, user: models.User +) -> models.AdventureScript: + """Attaches a transient `out_of_date` flag, which `AdventureScriptOut` reads. + + The flag is `True` or `False` when a syncable library version exists, and + `None` when none exists. + """ + library = resolve_library_script(adv_script, db, user) + adv_script.out_of_date = ( + None if library is None + else any(getattr(adv_script, f) != getattr(library, f) for f in SYNC_FIELDS) + ) + return adv_script + + +@router.get("/{adventure_id}/scripts", response_model=list[schemas.AdventureScriptOut]) +def list_adventure_scripts( + db: Session = Depends(get_db), + user: models.User = CurrentUser, + adventure: models.Adventure = Depends(current_adventure), +): + return [_mark_out_of_date(s, db, user) for s in adventure.scripts] + + +@router.post( + "/{adventure_id}/scripts/{adv_script_id}/sync", + response_model=schemas.AdventureScriptOut, +) +def sync_adventure_script( + adventure_id: int, + adv_script_id: int, + db: Session = Depends(get_db), + user: models.User = CurrentUser, + adventure: models.Adventure = Depends(current_adventure), +): + """Overwrites this copy's code with the latest from its library script. + + `enabled`, `position`, and the adventure's shared `script_state` are kept. + """ + script = db.get(models.AdventureScript, adv_script_id) + if script is None or script.adventure_id != adventure_id: + raise HTTPException(404, "Script not found") + library = resolve_library_script(script, db, user) + if library is None: + raise HTTPException(404, "No library script to sync from") + for field in SYNC_FIELDS: + setattr(script, field, getattr(library, field)) + # Store the link, so that a name-matched legacy copy syncs by id next + # time. + script.source_script_id = library.id + db.commit() + db.refresh(script) + return _mark_out_of_date(script, db, user) + + +@router.patch( + "/{adventure_id}/scripts/{adv_script_id}", response_model=schemas.AdventureScriptOut +) +def update_adventure_script( + adventure_id: int, + adv_script_id: int, + payload: schemas.AdventureScriptUpdate, + db: Session = Depends(get_db), + adventure: models.Adventure = Depends(current_adventure), +): + script = db.get(models.AdventureScript, adv_script_id) + if script is None or script.adventure_id != adventure_id: + raise HTTPException(404, "Script not found") + for field, value in payload.model_dump(exclude_unset=True).items(): + setattr(script, field, value) + db.commit() + return script diff --git a/backend/app/routers/adventures/takes.py b/backend/app/routers/adventures/takes.py new file mode 100644 index 0000000..13bfb37 --- /dev/null +++ b/backend/app/routers/adventures/takes.py @@ -0,0 +1,388 @@ +"""Retries, takes, and the attempts that pile up at one coordinate. + +Retry first deleted the AI action and generated a replacement. A later version +kept the row and appended each attempt to a JSON list on it. Now every attempt is +its own node on the same branch at the same depth, and exactly one of them is +`live`. `app/attempts.py` owns the group and its invariants. The endpoints here +only query it. +""" + +from fastapi import Depends, HTTPException, Request +from fastapi.responses import StreamingResponse +from sqlalchemy.orm import Session + +from ... import attempts, limits, memorybank, models, schemas, tree +from ...context import cursors +from ...context import lineage +from ...database import get_db +from ...scripting import ScriptPipeline +from ...sse import SSE_HEADERS + +from . import turns +from .deps import CurrentUser, current_adventure, router +from .nodes import delete_turn, last_action, stand_on +from .paging import action_window, annotate_takes, current_window + + +@router.post("/{adventure_id}/retry") +def retry_action( + adventure_id: int, + request: Request, + db: Session = Depends(get_db), + user: models.User = CurrentUser, + adventure: models.Adventure = Depends(current_adventure), +): + """Regenerates the last AI action and keeps the discarded attempt. + + The attempt on screen stays as it was written. The shared script state and + world state roll back to what the node before it left behind, and the new + attempt is stored as a sibling at the same coordinate. No text the AI wrote + is rewritten or deleted. + """ + limits.rate_limit("turn", request, user) + turns.check_demo_cap(db, user) + turns.acquire_turn_lock(adventure_id) + last_ai = None + try: + newest = last_action(adventure, db) + if newest is not None and newest.type == "ai": + last_ai = newest + # Roll the state back to before this AI turn's hooks ran, so that + # regenerating starts from a clean state rather than applying output + # mutations on top of the attempt being replaced. If the preceding + # node has no snapshot, which happens for a pre-SP4 row that the + # migration could not derive one for, this call does nothing and + # leaves the state as it is. + attempts.roll_back_before(db, adventure, last_ai) + db.commit() + db.refresh(adventure) + except BaseException: + turns._active_turns.discard(adventure_id) + raise + return StreamingResponse( + turns.with_turn_lock( + adventure_id, + turns.generate_turn( + adventure, db, ScriptPipeline(adventure, db), user, retry_of=last_ai + ), + ), + media_type="text/event-stream", + headers=SSE_HEADERS, + ) + + +@router.get( + "/{adventure_id}/actions/{action_id}/variants", + response_model=list[schemas.VariantOut], +) +def list_variants( + adventure_id: int, + action_id: int, + db: Session = Depends(get_db), + adventure: models.Adventure = Depends(current_adventure), +): + """Returns every attempt made for one AI turn. + + The client fetches these on demand, because the adventure payload carries + only the counts. That keeps old narration out of every page load. + + You can address the turn by any of its attempts, not only the live one. + Switching changes which row the story tells, and a client that holds an id it + received a moment ago still has to be able to ask about the same turn. + """ + action = db.get(models.Action, action_id) + if action is None or action.adventure_id != adventure_id: + raise HTTPException(404, "Action not found") + rows = attempts.group(db, action) + if len(rows) < 2: + return [] # Never retried, so the turn has one attempt. + return [ + schemas.VariantOut( + id=row.id, + index=i, + text=row.text, + reasoning=row.reasoning, + branch_id=row.branch_id, + created_at=row.created_at.isoformat() if row.created_at else None, + active=row.live, + ) + for i, row in enumerate(rows) + ] + + +@router.post( + "/{adventure_id}/actions/{action_id}/variant", response_model=schemas.ActionOut +) +def select_variant( + adventure_id: int, + action_id: int, + payload: schemas.VariantSelect, + db: Session = Depends(get_db), + adventure: models.Adventure = Depends(current_adventure), +): + """Makes an earlier attempt live again and restores the state it produced. + + The restored state covers both the script state and the world state. + + Only the last action can be switched. The turns after an older action were + written to continue the text that is currently active, so replacing that text + would leave the story contradicting itself. The attempts of earlier turns + stay readable through `list_variants`. + """ + action = db.get(models.Action, action_id) + if action is None or action.adventure_id != adventure_id: + raise HTTPException(404, "Action not found") + rows = attempts.group(db, action) + if not 0 <= payload.index < len(rows) or len(rows) < 2: + raise HTTPException(400, "No such attempt for this action") + newest = last_action(adventure, db) + if newest is None or newest.depth != action.depth or newest.branch_id != action.branch_id: + raise HTTPException( + 400, + "Only the latest message can be switched — the story has already " + "continued from this one.", + ) + turns.acquire_turn_lock(adventure_id) + try: + chosen = rows[payload.index] + if not chosen.live: + # The story at this coordinate is about to change, so withdraw + # anything derived from the previous text. A retry does the same + # thing for the same reason. + memorybank.forget_node(db, adventure, chosen) + cursors.rewind_all(adventure, chosen.branch_id, (chosen.depth or 0) - 1) + attempts.make_live(db, adventure, chosen) + adventure.updated_at = models.utcnow() + db.commit() + db.refresh(chosen) + # Return the row that is now in the story, which is a different row + # from the one the request addressed. An attempt is a node, so choosing + # one moves the story onto it rather than rewriting a row. + return chosen + finally: + turns._active_turns.discard(adventure_id) + + +@router.post( + "/{adventure_id}/actions/{action_id}/fork", response_model=schemas.ActionPage +) +def fork_from_attempt( + adventure_id: int, + action_id: int, + db: Session = Depends(get_db), + adventure: models.Adventure = Depends(current_adventure), +): + """Continues the story from this attempt, forking a branch if one is needed. + + There are three cases, and the first two do not fork: + + * The attempt is already the one the story tells, so there is nothing to do. + * Its turn is the tip, so the attempts are still leaves that nothing was + built on. The endpoint switches, as `/variant` does, and creates no branch. + * The story has moved past its turn, so the endpoint forks. The attempt gets + a branch of its own, and the line it leaves keeps every turn it has. + """ + action = db.get(models.Action, action_id) + if action is None or action.adventure_id != adventure_id: + raise HTTPException(404, "Action not found") + # Check this before checking the shape of the turn, because a fork leaves + # the promoted attempt alone on its branch. A client that repeats the call, + # after a double click or a retried request, has to get the same answer + # rather than an error saying the turn it just forked has nothing to fork + # to. + if action.live: + # A live node already holds what its coordinate says, so there is no + # attempt here to promote. On the path being read this call does + # nothing, and it has to stay that way, so that a repeated call after a + # double click or a retried request gets the same answer. Off the path + # the node belongs to another line's story, and moving there is a branch + # switch. + # + # The membership test covers the whole lineage, not `head_branch_id`. A + # head borrows its ancestors' turns, so a live node on an ancestor is + # already being read. Forking it would move the live row off the parent + # and promote a sibling in its place, which rewrites the story on a + # branch nobody asked about and on this one, which borrows that depth. + if lineage.path_of(db, adventure).contains(action): + return current_window(db, adventure) + raise HTTPException( + 400, + "That take is already the story on another branch. Switch to that " + "branch to read it.", + ) + if len(attempts.group(db, action)) < 2: + raise HTTPException( + 400, "This turn has only one take, so there is nothing to fork to." + ) + turns.acquire_turn_lock(adventure_id) + try: + stand_on(db, adventure, action) + adventure.updated_at = models.utcnow() + db.commit() + db.refresh(adventure) + return current_window(db, adventure) + finally: + turns._active_turns.discard(adventure_id) + + +@router.post("/{adventure_id}/actions/{action_id}/takes") +def add_take( + adventure_id: int, + action_id: int, + payload: schemas.TakeCreate, + request: Request, + db: Session = Depends(get_db), + user: models.User = CurrentUser, + adventure: models.Adventure = Depends(current_adventure), +): + """Plays a turn again, whoever wrote it. + + This endpoint replaces two earlier operations. `retry` gave an AI turn + another attempt, but only for the newest turn, and a player's own message had + no attempts at all, so changing text you had typed meant overwriting it and + losing the story it led to. Here an AI turn regenerates, a player turn takes + the text you supply, and neither depends on where in the story it sits. + + The tip is the only case that needs no branch, and only for an AI turn, + because nothing was played after it and its attempts are still leaves. A + player turn is never at the tip, since the reply to it is, so a player turn + that has been answered always takes a branch. + + A branch is needed here for the same reason `fork` needs one. The turn being + replayed already has a story after it, and that story was written as a + continuation of the old text. `branch_at` leaves the path just before this + turn, so the new attempt is written at the same depth under the same parent, + and the line it leaves is unchanged. No node below is copied. + """ + limits.rate_limit("turn", request, user) + limits.check_row_cap("actions", db, user, adventure=adventure) + turns.check_demo_cap(db, user) + action = db.get(models.Action, action_id) + if action is None or action.adventure_id != adventure_id: + raise HTTPException(404, "Action not found") + if action.type not in ("do", "say", "story", "continue", "ai"): + # The opening is not a turn anyone played, so it has no second attempt. + # Editing the scenario is what changes it. + raise HTTPException(400, "The opening of a story has no other take.") + if action.depth is None or not lineage.path_of(db, adventure).contains(action): + raise HTTPException(400, "That turn is not on the story you are reading.") + turns.acquire_turn_lock(adventure_id) + retry_of = None + try: + newest = last_action(adventure, db) + at_the_tip = newest is not None and newest.id == action.id + if at_the_tip and action.type == "ai": + # Nothing was played after it, so its attempts are still leaves and + # a branch would serve no purpose. This is the `retry` path. + retry_of = action + attempts.roll_back_before(db, adventure, action) + else: + # The turn has a story after it, written as a continuation of the + # text that is there now. The new attempt leaves the path just + # before the turn, so that story keeps the attempt it was written + # for. + tree.branch_at(db, adventure, action.depth - 1) + attempts.roll_back_before(db, adventure, action) + adventure.updated_at = models.utcnow() + db.commit() + db.refresh(adventure) + except BaseException: + turns._active_turns.discard(adventure_id) + raise + if action.type == "ai": + # There is no player action to write. The action this turn answers is + # already on the path, borrowed from the line being left. + stream = turns.generate_turn( + adventure, db, ScriptPipeline(adventure, db), user, retry_of=retry_of + ) + else: + stream = turns.run_player_turn( + adventure, + db, + schemas.ActionCreate(type=action.type, text=payload.text), + user, + # The client seeded its editor from the stored text, which already + # carries the "> You ..." conventions. + preformatted=True, + ) + return StreamingResponse( + turns.with_turn_lock(adventure_id, stream), + media_type="text/event-stream", + headers=SSE_HEADERS, + ) + + +@router.post("/{adventure_id}/undo", response_model=schemas.ActionPage) +def undo_turn( + adventure_id: int, + db: Session = Depends(get_db), + adventure: models.Adventure = Depends(current_adventure), +): + """Deletes the last turn: the trailing AI action and its player action, if any. + + The endpoint also rolls the shared `script_state` back to before that turn + ran, and it prunes any memory that summarized the removed actions. The turn + lock prevents an undo while a turn is still generating. + """ + turns.acquire_turn_lock(adventure_id) + try: + # Only the last turn is removed, so fetch the two actions it can + # consist of rather than the whole story. + newest = ( + db.query(models.Action) + .filter( + models.Action.adventure_id == adventure.id, + lineage.path_of(db, adventure).clause(models.Action), + ) + .order_by(models.Action.depth.desc(), models.Action.id.desc()) + .limit(2) + .all() + ) + if not newest or newest[0].type == "start": + raise HTTPException(400, "Nothing to undo") + last = newest[0] + before_that = newest[1] if len(newest) > 1 else None + # Undo only what this branch owns. Everything before the fork is + # borrowed from an ancestor and is part of that ancestor's story too, so + # an undo here must never delete a turn out of another branch. The test + # reads the row's own branch rather than the fork depth, because the + # branch is what decides the case. + if last.branch_id != adventure.head_branch_id: + raise HTTPException( + 400, "Nothing to undo on this branch — the turns before it " + "belong to the branch it was forked from.", + ) + first_removed = last + if (last.type == "ai" and before_that is not None + and before_that.type in ("do", "say", "story") + and before_that.branch_id == adventure.head_branch_id): + first_removed = before_that + # The state the story returns to once the turn is gone, which is what + # the node before the earliest removed one left behind. Read it before + # the deletes, while those rows are still in the story. + restore_to = attempts.preceding(db, adventure, first_removed) + delete_turn(db, adventure, last) + if first_removed is not last: + delete_turn(db, adventure, first_removed) + attempts.restore_state(adventure, restore_to) + db.flush() # Apply the deletes before anything reads the story back. + db.expire(adventure, ["actions"]) + # The tip moves back with the deleted rows. + tree.refresh_head(db, adventure) + db.commit() + db.refresh(adventure) + # Return the newest window rather than the whole story. The client + # replaces its transcript with this response, and the transcript is a + # window. Returning everything would defeat the paging on the action a + # player is most likely to repeat several times in a row. + actions, total, has_more = action_window(db, adventure) + return schemas.ActionPage( + actions=[ + schemas.ActionOut.model_validate(a) + for a in annotate_takes(db, adventure.id, actions) + ], + total=total, + has_more=has_more, + ) + finally: + turns._active_turns.discard(adventure_id) diff --git a/backend/app/routers/adventures/turns.py b/backend/app/routers/adventures/turns.py new file mode 100644 index 0000000..3f24d15 --- /dev/null +++ b/backend/app/routers/adventures/turns.py @@ -0,0 +1,421 @@ +"""Playing a turn: the model call, the SSE stream, and the one-turn-at-a-time lock. + +Everything a test needs to intercept lives here, and other modules reach it as +`turns.` rather than importing it by value. That matters twice. The turn +lock guards one set only while one module owns it. And a test that replaces +`OpenAICompatibleProvider`, `generate_turn`, or `check_demo_cap` patches this +module, which every caller reads through. +""" +import threading + +from fastapi import Depends, HTTPException, Request +from fastapi.responses import StreamingResponse +from sqlalchemy.orm import Session + +from ... import ( + analytics, attempts, auth, limits, memorybank, models, schemas, tree, worldstate, +) +from ...context import build_context, cursors +from ...database import get_db +from ...providers import OpenAICompatibleProvider, PromptParts, ProviderError +from ...scripting import ScriptPipeline +from ...sse import SSE_HEADERS, sse, turn_error +from ..settings import get_settings + +from .deps import CurrentUser, current_adventure, router +from .nodes import _move_to_after, next_depth +from .paging import annotate_takes + + +def world_delta_of(snapshot: dict | None) -> dict | None: + """Returns the bulk-read slice of a context snapshot, for `Action.world_delta`. + + `context_snapshot` is deferred because it holds the whole assembled prompt. + The parts that every action needs get their own small column instead: the + world-change chips, the emit block replayed into history, and the refusal + note fed back to the model. Update this function wherever a snapshot is + written. + + Carry all three report lists, not just `applied`. `Action.world_changes` + marks a chip from `clamped` and builds its refusal chips from `rejected`, + and `worldstate.refusals` reads both. Storing `applied` alone left every + consumer unable to tell a refused change from one that worked, which is the + distinction this column exists to carry. The two extra lists are subsets of + one turn's block, so they cost a few hundred bytes per action at most. + """ + ws = (snapshot or {}).get("world_state") + if not isinstance(ws, dict): + return None + report = ws.get("report") or {} + return { + "delta": ws.get("delta") or {}, + "applied": report.get("applied") or [], + "clamped": report.get("clamped") or [], + "rejected": report.get("rejected") or [], + } + + +# One turn at a time per adventure. The set lives in memory, which is enough for +# a single-process local app. Sync endpoints run in a threadpool, so the +# check-and-add needs a lock. The check also has to run during the request +# rather than when the SSE generator first runs. Otherwise two rapid requests +# both pass the check and generate concurrently. +_active_turns: set[int] = set() +_active_turns_guard = threading.Lock() + + +def acquire_turn_lock(adventure_id: int): + """Claims the adventure's turn slot atomically. `with_turn_lock` releases it.""" + with _active_turns_guard: + if adventure_id in _active_turns: + raise HTTPException(409, "A turn is already generating for this adventure.") + _active_turns.add(adventure_id) + + +async def with_turn_lock(adventure_id: int, gen): + """Wraps an SSE generator so that it releases the `acquire_turn_lock` lock.""" + try: + async for event in gen: + yield event + finally: + _active_turns.discard(adventure_id) + + +def format_player_input(action_type: str, text: str) -> str: + """Formats player input the way AI Dungeon does.""" + text = text.strip() + if action_type == "say": + text = text.strip('"') + if text and text[-1] not in ".!?…": + text += "." + return f'> You say "{text}"' + if action_type == "do": + if text.lower().startswith("you "): + text = text[4:] + if text and text[-1] not in ".!?…": + text += "." + return f"> You {text}" + return text # The "story" type is appended as raw text. + + +def action_json(action: models.Action, db: Session | None = None) -> dict: + """Serializes one action for the wire. + + Passing `db` fills in the pager numbers, and a turn that was just played must + pass it. The attempt that turn created is often the second one at its + coordinate, so the message needs a pager that the client cannot infer from a + count of one. Without `db`, a retry showed no pager until the page reloaded. + The adventure GET has the same requirement and builds its window a third way. + """ + if db is not None: + annotate_takes(db, action.adventure_id, [action]) + return schemas.ActionOut.model_validate(action).model_dump(mode="json") + + +async def generate_turn( + adventure: models.Adventure, + db: Session, + pipeline: ScriptPipeline, + user: models.User, + retry_of: models.Action | None = None, +): + """Streams the AI continuation as SSE, then stores the result. + + The continuation passes through the `context` and `output` script hooks. + + If `retry_of` is set, the result is stored as a sibling of that AI action, at + the same turn and the same coordinate, and the discarded attempt stays where + it was written. Before calling, the caller must roll the adventure back to + the state before that turn. See `retry_action`. If this generator ends + without saving, the rollback is undone, so the state cannot diverge from the + text on screen. + """ + saved = False + try: + async for event in _generate_turn(adventure, db, pipeline, user, retry_of): + if event is _SAVED: + saved = True + continue + yield event + finally: + if retry_of is not None and not saved: + # The turn failed with a provider error, an empty reply, a script + # stop, or a disconnected client. No sibling was written, so the + # attempt on screen is still the live one. Restore the state it + # produced. + attempts.restore_state(adventure, retry_of) + db.commit() + + +# `_generate_turn` yields this sentinel once the action is committed. It tells +# the wrapper above to leave the rollback in place rather than reverse it. +_SAVED = object() + + +async def _generate_turn( + adventure: models.Adventure, + db: Session, + pipeline: ScriptPipeline, + user: models.User, + retry_of: models.Action | None = None, +): + settings = get_settings(db, user) + cfg = auth.resolve_provider_config(settings) + # On a retry, the attempt being replaced is still the live node of its turn, + # because it stays live until a replacement exists. Filter it out of the + # context. Otherwise the model reads the attempt it is replacing as + # established story and writes a sequel to it. + replacing_id = retry_of.id if retry_of is not None else None + if cfg.using_demo: + # The server-funded key makes no embedding or summarization calls, so + # memory retrieval is skipped. If the bank is on, return a note. + memories = ( + {"used": [], "error": "Memory bank is unavailable on the shared demo key — add your own API key in Settings."} + if adventure.memory_bank_enabled + else None + ) + else: + memories = await memorybank.retrieve_memories( + adventure, settings, update_stats=True, exclude_action_id=replacing_id + ) + system_text, story_text, snapshot = build_context( + adventure, settings, memories, exclude_action_id=replacing_id + ) + + # onModelContext: scripts read, and can rewrite, the whole assembled + # context. + combined = f"{system_text}\n\n{story_text}" if system_text else story_text + modified, stop = pipeline.run("context", combined) + if stop: + yield sse({"type": "stopped", "script": pipeline.report()}) + return + context_changed = modified != combined + parts = ( + PromptParts(system="", story=modified) + if context_changed + else PromptParts(system=system_text, story=story_text) + ) + snapshot["script"] = pipeline.report() | { + "context_changed": context_changed, + "context_before": combined if context_changed else None, + "context_after": modified if context_changed else None, + } + + provider = OpenAICompatibleProvider( + cfg.endpoint_url, cfg.api_key, cfg.model, settings.api_mode, + settings.reasoning_max_tokens, + ) + chunks: list[str] = [] + reasoning_chunks: list[str] = [] + try: + async for kind, chunk in provider.generate( + parts, temperature=settings.temperature, max_tokens=settings.max_output_tokens + ): + if kind == "reasoning": + reasoning_chunks.append(chunk) + yield sse({"type": "reasoning", "text": chunk}) + else: + chunks.append(chunk) + yield sse({"type": "chunk", "text": chunk}) + except ProviderError as exc: + yield turn_error(str(exc)) + return + + text = "".join(chunks).strip() + # The model's literal reply, kept for the Insights "Raw AI output" view. It + # still contains the world-state block, which the code below strips. + raw_output = text + if not text: + # The model streamed reasoning but no story text, so it spent its whole + # budget on reasoning. Report that rather than "empty response". + if reasoning_chunks: + detail = ( + "The model used its entire token budget on reasoning and returned no " + 'story text. Raise "Max output tokens" in Settings, set a "Reasoning ' + 'max tokens" cap, or switch to a non-reasoning model.' + ) + else: + detail = "The AI returned an empty response." + yield turn_error(detail) + return + + # onOutput + text, _ = pipeline.run("output", text) + if not text.strip(): + yield turn_error("A script's output modifier returned empty text.") + return + snapshot["script"] = snapshot["script"] | pipeline.report() + + # RPG world state (Phase 12): read the AI's state delta out of the reply, + # apply it through the engine, and strip the block from the displayed text. + # + # A retry re-runs the same turn, so it is played at that turn's depth. The + # cooldown rules run on a position in the story, and a second attempt at turn + # 12 is still turn 12. This was `retry_of.index`, which held the same number + # until SP4. Depth stays correct once a branch has its own numbering. + ai_depth = retry_of.depth if retry_of is not None else next_depth(adventure) + stat_schema = adventure.scenario.stat_schema if adventure.scenario else None + if worldstate.has_schema(stat_schema): + text, delta = worldstate.extract_delta(text) + if not text.strip(): + yield turn_error("The AI returned only a state update and no story text.") + return + new_world_state, ws_report = worldstate.apply_delta( + adventure.world_state, stat_schema, delta, ai_depth + ) + adventure.world_state = new_world_state + snapshot["world_state"] = {"delta": delta, "report": ws_report, "state": new_world_state} + + snapshot["raw_output"] = raw_output + # The cost the endpoint reports for the call, including how much of the + # prompt came from cache rather than being billed in full. This is recorded + # per attempt, next to the prompt it priced. + snapshot["usage"] = provider.last_usage + + reasoning = "".join(reasoning_chunks).strip() or None + ai_action = models.Action( + adventure_id=adventure.id, + depth=ai_depth, + type="ai", + text=text, + reasoning=reasoning, + context_snapshot=snapshot, + world_delta=world_delta_of(snapshot), + ) + attempts.snapshot_outcome(adventure, ai_action) + if retry_of is not None: + attempts.add_attempt(db, adventure, retry_of, ai_action) + db.add(ai_action) + # The text at this coordinate changed, so anything derived from it no + # longer describes the story. Withdraw the memory attached to the node + # and return that stretch to both passes. Before SP4 this code was + # unreachable, because the summarizer held the newest action back until + # a turn landed on top of it. `memorybank.SETTLE_SLACK` keeps a memory + # off the tip again, for cost rather than for correctness, so this is + # now the rare case: undo or delete can carry a summarized node back to + # the tip, and then a retry of it lands here. See `memorybank`. + memorybank.forget_node(db, adventure, retry_of) + cursors.rewind_all(adventure, retry_of.branch_id, ai_depth - 1) + # Flush so the new attempt has an id. The session does not autoflush, + # and attempts page in id order, so a read taken before this point puts + # the newest attempt nowhere. + db.flush() + else: + tree.place_action(db, adventure, ai_action) + db.add(ai_action) + adventure.updated_at = models.utcnow() + if cfg.using_demo: + # Successful demo turns count against the daily cap, which the endpoint + # checks before the turn starts. A failed provider call above returns + # before this line. + auth.count_demo_turn(user) + db.commit() + # Count the turn here, after every path on which it could still have failed, + # so the number means "stories advanced" rather than "requests attempted". + # The demo tally counts those same turns as spend on the server-funded key. + analytics.record_event(analytics.EV_TURN, user) + if cfg.using_demo: + analytics.record(analytics.M_EVENT, analytics.EV_DEMO_TURN) + db.refresh(ai_action) + yield _SAVED + yield sse({"type": "done", "action": action_json(ai_action, db), "script": pipeline.report()}) + # Phase 6: schedule summarization and embedding without waiting for them. + # The task opens its own database session. It is skipped on the demo key, + # because background AI calls are unmetered spend on the server-funded + # key. + if not cfg.using_demo: + memorybank.schedule_post_turn(adventure) + + +def check_demo_cap(db: Session, user: models.User) -> None: + """Checks the demo cap before a turn starts. + + Checking first avoids storing a capped player's input and then leaving it + without a reply. + """ + settings = get_settings(db, user) + if auth.resolve_provider_config(settings).using_demo and auth.demo_turns_left(user) <= 0: + raise HTTPException(429, auth.DEMO_CAP_MESSAGE) + + +async def run_player_turn( + adventure: models.Adventure, + db: Session, + payload: schemas.ActionCreate, + user: models.User, + preformatted: bool = False, +): + """Plays a player's turn: their action, then the reply to it. + + `preformatted` means the text already carries the `> You ...` conventions and + is written as-is. That applies when the player retakes a turn they already + played (SP9). The editor is seeded with the stored text, which is already + formatted, and a plain edit puts that same text in the box and writes it back + verbatim. Formatting it a second time produces `> You > You ...`. + """ + pipeline = ScriptPipeline(adventure, db) + + # An empty do, say, or story action behaves as a continue. + if payload.type != "continue" and payload.text.strip(): + # onInput reads the formatted text, as in AI Dungeon: "> You ...". + formatted = ( + payload.text.strip() if preformatted + else format_player_input(payload.type, payload.text) + ) + modified, stop = pipeline.run("input", formatted) + if not modified.strip(): + yield turn_error("A script's input modifier returned empty text.", + script=pipeline.report()) + return + player_action = models.Action( + adventure_id=adventure.id, + depth=next_depth(adventure), + type=payload.type, + text=modified, + ) + # The state after the input hook has run. The node leaves this state + # behind. The AI turn after it starts here, and a retry of that turn + # rolls back to here. + attempts.snapshot_outcome(adventure, player_action) + tree.place_action(db, adventure, player_action) + db.add(player_action) + db.commit() + db.refresh(player_action) + # The new action was added through its foreign key, so the loaded + # `adventure.actions` collection is stale. Without this expire, + # `build_context` for the AI action does not see the player action + # that was just saved. + db.expire(adventure, ["actions"]) + yield sse({"type": "player", "action": action_json(player_action, db)}) + if stop: + # If onInput returns `{ stop: true }`, skip the AI call. + yield sse({"type": "stopped", "script": pipeline.report()}) + return + + async for event in generate_turn(adventure, db, pipeline, user): + yield event + + +@router.post("/{adventure_id}/actions") +def create_action( + adventure_id: int, + payload: schemas.ActionCreate, + request: Request, + db: Session = Depends(get_db), + user: models.User = CurrentUser, + adventure: models.Adventure = Depends(current_adventure), +): + limits.rate_limit("turn", request, user) + limits.check_row_cap("actions", db, user, adventure=adventure) + check_demo_cap(db, user) + acquire_turn_lock(adventure_id) + try: + _move_to_after(db, adventure, payload.after_id) + except BaseException: + _active_turns.discard(adventure_id) + raise + return StreamingResponse( + with_turn_lock(adventure_id, run_player_turn(adventure, db, payload, user)), + media_type="text/event-stream", + headers=SSE_HEADERS, + ) diff --git a/backend/app/routers/analytics.py b/backend/app/routers/analytics.py new file mode 100644 index 0000000..79bf8be --- /dev/null +++ b/backend/app/routers/analytics.py @@ -0,0 +1,140 @@ +"""Visit analytics: one endpoint the browser writes to, one the owner reads. + +The split matters. `/collect` is public and accepts one fact, which page was +viewed, because anything a stranger can POST is a number a stranger can invent. +Everything the dashboard relies on, meaning turns, adventures, sign-ups, demo +spend, and errors, is recorded on the server by the code that performs it, so +those counts are as trustworthy as the app itself. + +The two reading endpoints are owner-only and 404 for everyone else, the same +way the AI Chat router does: a feature nobody else can use is better off not +appearing to exist. `/summary` serves the anonymous counters (analytics.py, +which stores nothing that points at a person) and `/access` serves the access +log (accesslog.py, which identifies people on purpose). +""" + +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response +from pydantic import BaseModel, Field +from sqlalchemy.orm import Session + +from .. import accesslog, analytics, auth, limits, models +from ..database import get_db + +router = APIRouter(prefix="/api/analytics", tags=["analytics"]) + + +def owner( + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +) -> models.User: + """Gates the reading half. It returns 404 rather than 403. See the module + docstring.""" + if not auth.is_owner(user): + raise HTTPException(404, "Not found") + return user + + +Owner = Depends(owner) + + +class Pageview(BaseModel): + """What the SPA reports on a page load or a route change. + + `first` marks a real page load rather than a client-side navigation. The + facts that describe a visit rather than a view, which are where it came from, + on what kind of device, and from which country, are recorded only on a page + load, so a visitor who clicks through five pages is still one referral. + """ + + path: str = Field("", max_length=300) + referrer: str = Field("", max_length=500) + first: bool = False + + +@router.post("/collect", status_code=204) +def collect( + payload: Pageview, + request: Request, + db: Session = Depends(get_db), +) -> Response: + """Record one pageview. Always 204, even when nothing was counted: the + browser has no business knowing whether it was.""" + limits.rate_limit("analytics", request) + # Resolved by hand rather than through get_current_user: a pageview that + # arrives before /auth/me has minted a session should still be counted as a + # view, not turned into a 401 the SPA has to handle. + user = ( + auth.resolve_session_user(request, db) + if auth.MULTI_USER + else auth.local_user(db) + ) + # The operator's own clicks are not traffic. This applies only in + # multi-user mode. Locally every user is the owner, and excluding them would + # leave the dashboard empty on the machine the app is developed on. + if auth.MULTI_USER and user is not None and auth.is_owner(user): + return Response(status_code=204) + + analytics.record(analytics.M_PAGE, analytics.normalize_route(payload.path)) + analytics.record_visit(user) + if payload.first: + referrer = analytics.normalize_referrer( + payload.referrer, request.url.hostname or "" + ) + if referrer: # "" means same-origin, which is not a referral + analytics.record(analytics.M_REFERRER, referrer) + analytics.record( + analytics.M_DEVICE, + analytics.device_of(request.headers.get("user-agent", "")), + ) + analytics.record(analytics.M_COUNTRY, analytics.country_of(request.headers)) + return Response(status_code=204) + + +@router.get("/summary") +def summary( + days: int = Query(30, ge=1, le=365), + db: Session = Depends(get_db), + _user: models.User = Owner, +) -> dict: + """Returns the whole dashboard in one aggregate response. + + The response is a few kilobytes however much traffic is behind it. + """ + return analytics.summary(db, days) + + +@router.get("/access") +def access_log( + limit: int = Query(50, ge=1, le=200), + before_id: int | None = Query(None), + kind: str | None = Query(None), + q: str | None = Query(None, max_length=120), + db: Session = Depends(get_db), + _user: models.User = Owner, +) -> dict: + """A page of the access log, newest first. + + Unlike `/summary`, this returns rows about people, which is what it is for. + It is therefore behind the same owner gate, it is paged rather than returned + in full, and the people it describes have no endpoint that reaches it. + """ + page = accesslog.recent( + db, limit=limit, before_id=before_id, kind=kind, query=q + ) + return { + "events": [ + { + "id": event.id, + "at": event.at.isoformat(), + "kind": event.kind, + "who": event.who, + "is_guest": event.is_guest, + "ip": event.ip, + "country": event.country, + "device": event.device, + "user_agent": event.user_agent, + } + for event in page["events"] + ], + "has_more": page["has_more"], + } diff --git a/backend/app/routers/auth.py b/backend/app/routers/auth.py new file mode 100644 index 0000000..138f200 --- /dev/null +++ b/backend/app/routers/auth.py @@ -0,0 +1,158 @@ +import re + +from fastapi import APIRouter, Depends, HTTPException, Request, Response +from sqlalchemy.orm import Session + +from .. import (accesslog, analytics, auth, cleanup, limits, models, schemas, + security, starter) +from ..database import get_db +from .settings import get_settings + +router = APIRouter(prefix="/api/auth", tags=["auth"]) + +EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$") + + +def _set_session_cookie(response: Response, user_id: int) -> None: + response.set_cookie( + auth.SESSION_COOKIE, + security.sign_session(user_id), + max_age=auth.COOKIE_MAX_AGE, + httponly=True, + samesite="lax", + secure=auth.COOKIE_SECURE, + path="/", + ) + + +def me_payload(user: models.User, db: Session) -> dict: + settings = get_settings(db, user) + cfg = auth.resolve_provider_config(settings) + return { + "multi_user": auth.MULTI_USER, + "id": user.id, + "email": user.email, + "is_guest": user.is_guest, + # Trusted testers: unmetered demo turns, plus the AI Chat scratchpad. + "power_user": auth.is_power_user(user), + # Separate allowlist: shows the visit-analytics page and its nav link. + "analytics": auth.is_owner(user), + # How long an idle guest is kept before cleanup deletes it (None when + # the policy is off). Served rather than hardcoded in the UI so the + # number a guest is shown is the number actually enforced. + "guest_retention_days": cleanup.RETENTION_DAYS if cleanup.enabled() else None, + "demo": { + "enabled": auth.demo_enabled(), + "using_demo": cfg.using_demo, + "model": cfg.model if cfg.using_demo else None, + "turns_per_day": auth.DEMO_TURNS_PER_DAY, + "turns_left": auth.demo_turns_left(user) if auth.demo_enabled() else None, + "models": auth.DEMO_MODELS if auth.demo_enabled() else [], + }, + } + + +@router.get("/me") +def me(request: Request, response: Response, db: Session = Depends(get_db)): + """Returns the current user. + + In multi-user mode this also establishes the session. If the cookie is + missing or invalid, the endpoint creates a guest user and sets a cookie. The + frontend calls it on load and after any 401. + """ + if not auth.MULTI_USER: + user = auth.local_user(db) + else: + user = auth.resolve_session_user(request, db) + if user is None: + # Each new guest is a database row, so cap how fast one IP can + # create them. + limits.rate_limit("guest", request) + user = models.User(is_guest=True) + db.add(user) + db.commit() + # The guest is committed first, so a failure while copying the + # starter adventure still leaves them with an account. + starter.give(db, user) + db.commit() + _set_session_cookie(response, user.id) + # This endpoint is the SPA's bootstrap call, so it is where a session first + # shows itself; accesslog thins the rows down to one per day per address. + accesslog.note_session(db, user, request) + return me_payload(user, db) + + +@router.post("/register") +def register( + payload: schemas.AuthCredentials, + request: Request, + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + """Upgrades the current guest in place. + + The `user_id` does not change, so every adventure, scenario, script, and + setting they created as a guest is kept. + """ + if not auth.MULTI_USER: + raise HTTPException(400, "Accounts are disabled in local mode.") + limits.rate_limit("auth", request) + email = payload.email.strip().lower() + if not EMAIL_RE.match(email): + raise HTTPException(422, "Enter a valid email address.") + if len(payload.password) < 8: + raise HTTPException(422, "Password must be at least 8 characters.") + if not user.is_guest: + raise HTTPException(400, "This session is already registered.") + if db.query(models.User).filter(models.User.email == email).first(): + raise HTTPException(409, "An account with this email already exists — log in instead.") + user.email = email + user.password_hash = security.hash_password(payload.password) + user.is_guest = False + db.commit() + analytics.record_event(analytics.EV_SIGNUP, user) + accesslog.record(db, accesslog.REGISTER, request, user=user) + return me_payload(user, db) + + +@router.post("/login") +def login( + payload: schemas.AuthCredentials, + request: Request, + response: Response, + db: Session = Depends(get_db), +): + """Point this browser's session at an existing account. Any current guest + session is simply abandoned (its data stays under the guest user).""" + if not auth.MULTI_USER: + raise HTTPException(400, "Accounts are disabled in local mode.") + limits.rate_limit("auth", request) + email = payload.email.strip().lower() + # Per-account throttle: stops distributed guessing against one email even + # when the per-IP limit above is diluted across many source addresses. + limits.check_login_allowed(email) + user = db.query(models.User).filter(models.User.email == email).first() + if ( + user is None + or not user.password_hash + or not security.verify_password(payload.password, user.password_hash) + ): + limits.note_login_failure(email) + # Logged with the address that was tried, not the account that owns it: + # a guessing run against an address that has no account is exactly the + # thing worth being able to see. + accesslog.record(db, accesslog.LOGIN_FAILED, request, who=email) + raise HTTPException(401, "Incorrect email or password.") + limits.note_login_success(email) + _set_session_cookie(response, user.id) + analytics.record_event(analytics.EV_LOGIN, user) + accesslog.record(db, accesslog.LOGIN, request, user=user) + return me_payload(user, db) + + +@router.post("/logout") +def logout(response: Response): + if not auth.MULTI_USER: + raise HTTPException(400, "Accounts are disabled in local mode.") + response.delete_cookie(auth.SESSION_COOKIE, path="/") + return {"ok": True} diff --git a/backend/app/routers/chat.py b/backend/app/routers/chat.py new file mode 100644 index 0000000..379db67 --- /dev/null +++ b/backend/app/routers/chat.py @@ -0,0 +1,169 @@ +"""AI Chat: a plain scratchpad for talking to a model directly. + +Power users reach it, which means the `AIDND_POWER_USERS` email allowlist. It is +deliberately thin. It adds no story context, no scripts, and no world state, and +it persists nothing. The conversation lives in the browser and is posted in full +on each turn. It exists for testing models, prompts, and endpoints without +starting an adventure. + +Model choice is free when the user brought their own API key. On the shared demo +key the model stays pinned to the `AIDND_DEMO_MODELS` allowlist, exactly as it is +for turns. The server funds that key, so this page must not let it reach paid +models. +""" + +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import StreamingResponse +from sqlalchemy.orm import Session + +from .. import auth, limits, models, schemas +from ..database import get_db +from ..providers import OpenAICompatibleProvider, ProviderError +from ..sse import SSE_HEADERS, sse +from .settings import get_settings, list_endpoint_models + +router = APIRouter(prefix="/api/chat", tags=["chat"]) + + +def power_user( + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +) -> models.User: + """Gate for the whole router. 404 rather than 403 so the feature simply + doesn't appear to exist for everyone else.""" + if not auth.is_power_user(user): + raise HTTPException(404, "Not found") + return user + + +PowerUser = Depends(power_user) + + +def _resolve_model( + settings: models.Settings, requested: str | None +) -> tuple[auth.ProviderConfig, str | None]: + """Returns the provider config for this chat, plus a note when the requested + model was not used. + + The pinning rule lives in `resolve_provider_config`. This function only + reports the substitution that call made, so one place decides what the demo + key may talk to. + """ + cfg = auth.resolve_provider_config(settings, model_override=requested) + wanted = (requested or "").strip() + if wanted and wanted != cfg.model: + return cfg, ( + f"'{wanted}' isn't available on the shared demo key — using " + f"{cfg.model}. Add your own API key in Settings to use any model." + ) + return cfg, None + + +@router.get("/config") +async def chat_config( + db: Session = Depends(get_db), + user: models.User = PowerUser, +): + """Returns what this page can talk to. + + The response holds the resolved endpoint and model, whether model choice is + pinned to the demo allowlist, and the endpoint's model listing. The listing + is best effort, and an unreachable endpoint returns an empty list. + """ + settings = get_settings(db, user) + cfg = auth.resolve_provider_config(settings) + listing = await list_endpoint_models(cfg) + return { + "endpoint_url": cfg.endpoint_url, + "model": cfg.model, + "using_demo": cfg.using_demo, + "api_mode": settings.api_mode, + "temperature": settings.temperature, + "max_tokens": settings.max_output_tokens, + # On the demo key the whitelist IS the list of choices; otherwise it's + # whatever the endpoint advertises (suggestions, not a restriction). + "models": auth.DEMO_MODELS if cfg.using_demo else listing.get("models", []), + "models_error": None if listing.get("ok") else listing.get("detail"), + } + + +async def run_chat(cfg: auth.ProviderConfig, settings: models.Settings, payload: schemas.ChatRequest, + note: str | None, db: Session, user: models.User): + """Streams the reply as SSE, using the turn stream's event shape. + + The generator emits `reasoning` and `chunk` events while generating and then + a `done` event, so the frontend reuses the same code. + """ + if note: + yield sse({"type": "note", "detail": note}) + provider = OpenAICompatibleProvider( + cfg.endpoint_url, cfg.api_key, cfg.model, settings.api_mode, + settings.reasoning_max_tokens, + ) + messages = [m.model_dump() for m in payload.messages] + chunks: list[str] = [] + reasoning_chunks: list[str] = [] + try: + async for kind, chunk in provider.chat( + messages, + temperature=payload.temperature if payload.temperature is not None else settings.temperature, + max_tokens=payload.max_tokens or settings.max_output_tokens, + ): + if kind == "reasoning": + reasoning_chunks.append(chunk) + yield sse({"type": "reasoning", "text": chunk}) + else: + chunks.append(chunk) + yield sse({"type": "chunk", "text": chunk}) + except ProviderError as exc: + yield sse({"type": "error", "detail": str(exc)}) + return + + text = "".join(chunks).strip() + if not text: + detail = ( + "The model used its entire token budget on reasoning and returned no " + "reply — raise max tokens, cap the reasoning budget in Settings, or " + "use a non-reasoning model." + if reasoning_chunks + else "The AI returned an empty response." + ) + yield sse({"type": "error", "detail": detail}) + return + + if cfg.using_demo: + # Unmetered for power users (count_demo_turn is a no-op for them), but + # keep the call so the accounting stays right if the gate ever widens. + auth.count_demo_turn(user) + db.commit() + yield sse({ + "type": "done", + "text": text, + "reasoning": "".join(reasoning_chunks).strip() or None, + "model": cfg.model, + }) + + +@router.post("/stream") +def chat_stream( + payload: schemas.ChatRequest, + request: Request, + db: Session = Depends(get_db), + user: models.User = PowerUser, +): + total = sum(len(m.content) for m in payload.messages) + if total > schemas.CHAT_TOTAL_MAX: + raise HTTPException( + 413, f"This conversation is too long to send ({total:,} characters) — " + "clear it or start a new one." + ) + limits.rate_limit("chat", request, user) + settings = get_settings(db, user) + cfg, note = _resolve_model(settings, payload.model) + if not cfg.model: + raise HTTPException(400, "No model configured — set one in Settings or pick one here.") + return StreamingResponse( + run_chat(cfg, settings, payload, note, db, user), + media_type="text/event-stream", + headers=SSE_HEADERS, + ) diff --git a/backend/app/routers/debug.py b/backend/app/routers/debug.py new file mode 100644 index 0000000..37f26d4 --- /dev/null +++ b/backend/app/routers/debug.py @@ -0,0 +1,19 @@ +from fastapi import APIRouter, HTTPException + +from .. import auth, debuglog + +router = APIRouter(prefix="/api/debug", tags=["debug"]) + + +@router.get("/requests") +def recent_requests(): + """Most-recent-first log of provider requests/responses (no API keys). + + The log is a single process-wide ring buffer with no per-user attribution, + so in multi-user mode, which is how a hosted deployment runs, it would expose + other players' prompts. It is disabled there and available on a local + install. + """ + if auth.MULTI_USER: + raise HTTPException(403, "The debug log is only available on local installs.") + return debuglog.recent() diff --git a/backend/app/routers/scenarios.py b/backend/app/routers/scenarios.py new file mode 100644 index 0000000..738624b --- /dev/null +++ b/backend/app/routers/scenarios.py @@ -0,0 +1,271 @@ +from fastapi import APIRouter, Body, Depends, HTTPException, Request +from fastapi.responses import Response +from sqlalchemy import or_ +from sqlalchemy.orm import Session + +from .. import analytics, auth, images, limits, models, schemas +from ..database import get_db + +router = APIRouter(prefix="/api/scenarios", tags=["scenarios"]) + + +def get_scenario_or_404( + scenario_id: int, db: Session, user: models.User, *, edit: bool = False +) -> models.Scenario: + """Visible = owned or public; editable = owned only.""" + scenario = db.get(models.Scenario, scenario_id) + if scenario is None or (scenario.user_id != user.id and not scenario.is_public): + raise HTTPException(404, "Scenario not found") + if edit and scenario.user_id != user.id: + raise HTTPException(403, "This is a shared demo scenario — it can't be edited. Start an adventure from it, or duplicate it.") + return scenario + + +@router.get("", response_model=list[schemas.ScenarioListItem]) +def list_scenarios( + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + return ( + db.query(models.Scenario) + .filter(or_(models.Scenario.user_id == user.id, models.Scenario.is_public)) + .order_by(models.Scenario.updated_at.desc()) + .all() + ) + + +@router.post("", response_model=schemas.ScenarioOut, status_code=201) +def create_scenario( + payload: schemas.ScenarioCreate, + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + limits.check_row_cap("scenarios", db, user) + scenario = models.Scenario(**payload.model_dump(), user_id=user.id) + db.add(scenario) + db.commit() + return scenario + + +@router.get("/{scenario_id}", response_model=schemas.ScenarioOut) +def get_scenario( + scenario_id: int, + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + scenario = get_scenario_or_404(scenario_id, db, user) + # A funnel step, recorded for shared scenarios only. Opening one is the + # first sign that a visitor is interested, and someone editing their own + # scenario is already past this point. Their titles are theirs rather than a + # statistic. + if scenario.is_public: + analytics.record_event(analytics.EV_SCENARIO_OPEN, user) + return scenario + + +@router.get("/{scenario_id}/image") +def get_scenario_image( + scenario_id: int, + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + """Serve an uploaded cover image as real bytes. + + Lists point here instead of inlining the data URI. The response is marked + immutable and the URL carries a `?v=` stamp, so browsers cache + it indefinitely but pick up a new picture the moment the author saves one. + """ + scenario = get_scenario_or_404(scenario_id, db, user) + decoded = images.decode(scenario.image) + if decoded is None: + raise HTTPException(404, "This scenario has no uploaded image") + data, content_type = decoded + return Response( + content=data, + media_type=content_type, + headers={"Cache-Control": "private, max-age=31536000, immutable"}, + ) + + +@router.patch("/{scenario_id}", response_model=schemas.ScenarioOut) +def update_scenario( + scenario_id: int, + payload: schemas.ScenarioUpdate, + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + scenario = get_scenario_or_404(scenario_id, db, user, edit=True) + data = payload.model_dump(exclude_unset=True) + script_ids = data.pop("script_ids", None) + for field, value in data.items(): + setattr(scenario, field, value) + if script_ids is not None: + scripts = ( + db.query(models.Script) + .filter(models.Script.id.in_(script_ids), models.Script.user_id == user.id) + .all() + ) + if len(scripts) != len(set(script_ids)): + raise HTTPException(404, "One or more scripts not found") + scenario.scripts = sorted(scripts, key=lambda s: script_ids.index(s.id)) + db.commit() + return scenario + + +@router.delete("/{scenario_id}", status_code=204) +def delete_scenario( + scenario_id: int, + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + scenario = get_scenario_or_404(scenario_id, db, user, edit=True) + db.delete(scenario) + db.commit() + + +# ---------- Import / Export ---------- + +@router.get("/{scenario_id}/export") +def export_scenario( + scenario_id: int, + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + s = get_scenario_or_404(scenario_id, db, user) + return { + "format": "ai-dnd-scenario-v1", + "title": s.title, + "description": s.description, + "prompt": s.prompt, + "memory": s.memory, + "authorsNote": s.authors_note, + "aiInstructions": s.ai_instructions, + "tags": s.tags, + "image": s.image, + "icon": s.icon, + "statSchema": s.stat_schema, + "storyCards": [ + {"type": c.type, "name": c.name, "keys": c.keys, "entry": c.entry, "notes": c.notes} + for c in s.story_cards + ], + "scripts": [ + { + "name": sc.name, "description": sc.description, "library": sc.library_js, + "input": sc.input_js, "context": sc.context_js, "output": sc.output_js, + } + for sc in s.scripts + ], + } + + +# Key aliases seen in AI Dungeon scenario exports, mapped best-effort. +_SCENARIO_KEYS = { + "title": "title", + "description": "description", + "prompt": "prompt", + "memory": "memory", + "authorsNote": "authors_note", + "authors_note": "authors_note", + "authorsNoteText": "authors_note", + "aiInstructions": "ai_instructions", + "ai_instructions": "ai_instructions", + "instructions": "ai_instructions", +} +_IGNORED_KEYS = {"format", "storyCards", "worldInfo", "worldInformation", "scripts", "tags", + "statSchema", "stat_schema", "image", "icon", + "createdAt", "updatedAt", "id", "publicId", "nsfw", "type", "options"} + + +@router.post("/import", status_code=201) +def import_scenario( + request: Request, + bundle: dict = Body(...), + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + """Accepts our export format and AI Dungeon scenario exports best-effort; + reports any keys it didn't understand.""" + limits.rate_limit("import", request, user) + limits.check_row_cap("scenarios", db, user) + fields: dict = {} + unmapped: list[str] = [] + for key, value in bundle.items(): + if key in _SCENARIO_KEYS and isinstance(value, str): + fields[_SCENARIO_KEYS[key]] = value + elif key not in _IGNORED_KEYS: + unmapped.append(key) + + tags = bundle.get("tags") + if isinstance(tags, list): + fields["tags"] = ", ".join(str(t) for t in tags) + elif isinstance(tags, str): + fields["tags"] = tags + + schema = bundle.get("statSchema") or bundle.get("stat_schema") + if isinstance(schema, dict): + fields["stat_schema"] = schema + + # An AI Dungeon bundle also carries an `image`, so this reads it. The value + # is untrusted input, so it goes through `sanitize()` rather than a direct + # assignment. + image = images.sanitize(bundle.get("image"), schemas.IMAGE_MAX) + if image: + fields["image"] = image + icon = bundle.get("icon") + if isinstance(icon, str) and icon: + fields["icon"] = icon[:schemas.ICON_MAX] + + scenario = models.Scenario(**fields, user_id=user.id) + if not scenario.title: + scenario.title = "Imported Scenario" + # A raw-dict import bypasses the schemas, so truncate to the VARCHAR widths. + # Postgres enforces them. See `schemas.py`. Column defaults have not been + # applied yet, because that happens at flush, so a bundle with no `tags` key + # leaves the attribute None, which is why the code says `or ""`. + scenario.title = scenario.title[:schemas.NAME_MAX] + scenario.tags = (scenario.tags or "")[:schemas.TAGS_MAX] + db.add(scenario) + db.flush() + + # AI Dungeon exports have used all three names for the same list. + cards = ( + bundle.get("storyCards") + or bundle.get("worldInfo") + or bundle.get("worldInformation") + or [] + ) + limits.check_bundle_lists(story_cards=cards) + for card in cards: + if not isinstance(card, dict): + continue + db.add( + models.StoryCard( + scenario_id=scenario.id, + type=str(card.get("type") or "")[:schemas.CARD_TYPE_MAX], + name=str(card.get("name") or card.get("title") or "")[:schemas.NAME_MAX], + keys=str(card.get("keys") or ""), + # AI Dungeon world info uses "value"; story cards use "entry". + entry=str(card.get("entry") or card.get("value") or ""), + notes=str(card.get("notes") or card.get("description") or ""), + ) + ) + + for item in bundle.get("scripts") or []: + if not isinstance(item, dict): + continue + script = models.Script( + user_id=user.id, + name=str(item.get("name") or "Imported Script")[:schemas.NAME_MAX], + description=str(item.get("description") or ""), + library_js=str(item.get("library") or item.get("sharedLibrary") or ""), + input_js=str(item.get("input") or item.get("onInput") or ""), + context_js=str(item.get("context") or item.get("onModelContext") or ""), + output_js=str(item.get("output") or item.get("onOutput") or ""), + ) + db.add(script) + db.flush() + scenario.scripts.append(script) + + db.commit() + out = schemas.ScenarioOut.model_validate(scenario).model_dump(mode="json") + return {"scenario": out, "unmapped_keys": unmapped} diff --git a/backend/app/routers/scripts.py b/backend/app/routers/scripts.py new file mode 100644 index 0000000..24ba879 --- /dev/null +++ b/backend/app/routers/scripts.py @@ -0,0 +1,159 @@ +from fastapi import APIRouter, Body, Depends, HTTPException, Request +from sqlalchemy.orm import Session + +from .. import auth, limits, models, schemas +from ..database import get_db +from ..scripting import run_hook + +router = APIRouter(prefix="/api/scripts", tags=["scripts"]) + +HOOK_FIELDS = {"input": "input_js", "context": "context_js", "output": "output_js"} + + +def get_script_or_404(script_id: int, db: Session, user: models.User) -> models.Script: + script = db.get(models.Script, script_id) + if script is None or script.user_id != user.id: + raise HTTPException(404, "Script not found") + return script + + +@router.get("", response_model=list[schemas.ScriptOut]) +def list_scripts( + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + return ( + db.query(models.Script) + .filter(models.Script.user_id == user.id) + .order_by(models.Script.updated_at.desc()) + .all() + ) + + +@router.post("", response_model=schemas.ScriptOut, status_code=201) +def create_script( + payload: schemas.ScriptCreate, + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + limits.check_row_cap("scripts", db, user) + script = models.Script(**payload.model_dump(), user_id=user.id) + db.add(script) + db.commit() + return script + + +@router.get("/{script_id}", response_model=schemas.ScriptOut) +def get_script( + script_id: int, + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + return get_script_or_404(script_id, db, user) + + +@router.patch("/{script_id}", response_model=schemas.ScriptOut) +def update_script( + script_id: int, + payload: schemas.ScriptUpdate, + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + script = get_script_or_404(script_id, db, user) + for field, value in payload.model_dump(exclude_unset=True).items(): + setattr(script, field, value) + db.commit() + return script + + +@router.delete("/{script_id}", status_code=204) +def delete_script( + script_id: int, + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + db.delete(get_script_or_404(script_id, db, user)) + db.commit() + + +@router.post("/{script_id}/test") +def test_script( + script_id: int, + payload: schemas.ScriptTestRequest, + request: Request, + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + """Runs one hook against sample text, making no AI call and storing nothing.""" + script = get_script_or_404(script_id, db, user) + limits.rate_limit("script-test", request, user) + result = run_hook( + script.library_js, + getattr(script, HOOK_FIELDS[payload.hook]), + payload.text, + payload.state, + history=[], + story_cards=[], + info={"actionCount": 0, "characterNames": [], "memoryLength": 0, "maxChars": 0}, + ) + return { + "text": result.text, + "stop": result.stop, + "state": result.state, + "storyCards": result.story_cards, + "logs": result.logs, + "error": result.error, + } + + +# ---------- Import / Export ---------- + +@router.get("/{script_id}/export") +def export_script( + script_id: int, + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + """JSON bundle matching how AI Dungeon scripts circulate.""" + script = get_script_or_404(script_id, db, user) + return { + "name": script.name, + "description": script.description, + "library": script.library_js, + "input": script.input_js, + "context": script.context_js, + "output": script.output_js, + } + + +@router.post("/import", response_model=schemas.ScriptOut, status_code=201) +def import_script( + request: Request, + bundle: dict = Body(...), + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + """Accepts our export bundle; tolerates *_js key names too.""" + limits.rate_limit("import", request, user) + limits.check_row_cap("scripts", db, user) + def pick(*keys: str) -> str: + for key in keys: + value = bundle.get(key) + if isinstance(value, str): + return value + return "" + + script = models.Script( + user_id=user.id, + # A raw-dict import bypasses the schemas, so truncate to the VARCHAR + # width. + name=(pick("name") or "Imported Script")[:schemas.NAME_MAX], + description=pick("description"), + library_js=pick("library", "library_js", "sharedLibrary"), + input_js=pick("input", "input_js", "onInput"), + context_js=pick("context", "context_js", "onModelContext"), + output_js=pick("output", "output_js", "onOutput"), + ) + db.add(script) + db.commit() + return script diff --git a/backend/app/routers/settings.py b/backend/app/routers/settings.py new file mode 100644 index 0000000..b288061 --- /dev/null +++ b/backend/app/routers/settings.py @@ -0,0 +1,129 @@ +import httpx +from fastapi import APIRouter, Depends, Request +from sqlalchemy.orm import Session +from starlette.concurrency import run_in_threadpool + +from .. import auth, limits, models, netguard, schemas, security +from ..database import get_db + +router = APIRouter(prefix="/api/settings", tags=["settings"]) + + +def get_settings(db: Session, user: models.User) -> models.Settings: + """Returns the user's settings row, creating it on first access. + + Phase 8 made settings per user rather than global. They cover the endpoint, + the key, the models, and the memory configuration. + """ + settings = ( + db.query(models.Settings).filter(models.Settings.user_id == user.id).first() + ) + if settings is None: + settings = models.Settings(user_id=user.id) + db.add(settings) + db.commit() + return settings + + +@router.get("", response_model=schemas.SettingsOut) +def read_settings( + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + return get_settings(db, user) + + +@router.put("", response_model=schemas.SettingsOut) +def update_settings( + payload: schemas.SettingsUpdate, + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + settings = get_settings(db, user) + fields = payload.model_dump(exclude_unset=True) + # Write-only API key: absent = unchanged, "" = cleared, else encrypted. + if "api_key" in fields: + fields["api_key"] = security.encrypt_secret(fields["api_key"].strip()) + embedding_model_changed = ( + "embedding_model" in fields + and fields["embedding_model"] != settings.embedding_model + ) + for field, value in fields.items(): + setattr(settings, field, value) + if embedding_model_changed: + # Vectors from the old model have a different dimensionality/space; + # clear them so the post-turn task re-embeds with the new model. + # This covers only this user's adventures, because settings are per + # user now. + # + # Both columns, and the flag. This is the one place that clears vectors + # in bulk rather than through memorybank.set_vector, and when the + # vectors moved to embedding_blob it kept nulling the old JSON column + # alone. The blob survived, `embedded` stayed true, and + # `_embed_pending`, which selects rows where `embedded IS FALSE`, never + # found the rows. The bank kept ranking against the previous model's + # vectors. + owned = ( + db.query(models.Adventure.id) + .filter(models.Adventure.user_id == user.id) + .scalar_subquery() + ) + db.query(models.Memory).filter(models.Memory.adventure_id.in_(owned)).update( + {"embedding_blob": None, "embedded": False}, synchronize_session=False + ) + # No cache invalidation needed, and deliberately none added: clearing + # `embedded` drops these rows out of the catalogue query, so retrieval + # stops asking for them, and by the time _embed_pending puts one back + # it has gone through set_vector, which evicts that entry. The rule + # holds: anything that removes a memory from play corrects itself. + db.commit() + return settings + + +async def list_endpoint_models(cfg: auth.ProviderConfig) -> dict: + """Fetches the endpoint's /models listing. + + The call also serves as a connectivity check, so a failure returns + `{"ok": False, "detail": ...}` rather than raising. + """ + # SSRF guard. Never probe a non-public address the user supplied. + reason = await run_in_threadpool(netguard.endpoint_block_reason, cfg.endpoint_url) + if reason: + return {"ok": False, "detail": f"Can't reach that endpoint — {reason}."} + url = cfg.endpoint_url.rstrip("/") + "/models" + headers = {} + if cfg.api_key: + headers["Authorization"] = f"Bearer {cfg.api_key}" + try: + async with httpx.AsyncClient(timeout=10) as client: + resp = await client.get(url, headers=headers) + except httpx.HTTPError as exc: + return {"ok": False, "detail": f"Connection failed: {exc}"} + + if resp.status_code != 200: + return {"ok": False, "detail": f"HTTP {resp.status_code}: {resp.text[:300]}"} + + models_available: list[str] = [] + try: + data = resp.json() + models_available = [m.get("id", "?") for m in data.get("data", [])] + except (ValueError, AttributeError, TypeError): + pass # The body is not JSON or has an unexpected shape. The endpoint + # is still reachable. + return {"ok": True, "models": models_available} + + +@router.post("/test") +async def test_connection( + request: Request, + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + """Runs a cheap connectivity check against whatever the turn engine would use. + + That includes the shared demo endpoint, when the user has no key of their + own. + """ + limits.rate_limit("connection-test", request, user) + settings = get_settings(db, user) + return await list_endpoint_models(auth.resolve_provider_config(settings)) diff --git a/backend/app/routers/story_cards.py b/backend/app/routers/story_cards.py new file mode 100644 index 0000000..991a085 --- /dev/null +++ b/backend/app/routers/story_cards.py @@ -0,0 +1,187 @@ +from fastapi import APIRouter, Body, Depends, HTTPException, Request +from sqlalchemy.orm import Session + +from .. import auth, limits, models, schemas +from ..database import get_db + +router = APIRouter(prefix="/api/story-cards", tags=["story-cards"]) + +# The AI Dungeon world-info and story-card array format. Its field names differ +# from this app's columns: `value` maps to `entry`, `title` maps to `name`, and +# `description` maps to `notes`. The extra `useForCharacterCreation` flag has no +# equivalent here. It is ignored on import and written as false on export, so a +# round trip through AI Dungeon stays valid. + + +def _visible_owner(scenario_id, adventure_id, db, user): + """Resolves the scenario or adventure a caller may read cards from. + + Public demo scenarios are included. The function raises a 404 or a 422 when + it cannot resolve one. + """ + if (scenario_id is None) == (adventure_id is None): + raise HTTPException(422, "Provide exactly one of scenario_id or adventure_id") + if scenario_id is not None: + scenario = db.get(models.Scenario, scenario_id) + if scenario is None or (scenario.user_id != user.id and not scenario.is_public): + raise HTTPException(404, "Owner not found") + return scenario + adventure = db.get(models.Adventure, adventure_id) + if adventure is None or adventure.user_id != user.id: + raise HTTPException(404, "Owner not found") + return adventure + + +def _card_editable_or_404(card: models.StoryCard | None, user: models.User) -> models.StoryCard: + """Cards inherit their scope from the owning scenario/adventure. Public + (demo) scenarios are visible to everyone but editable by no one.""" + if card is not None: + owner = card.scenario if card.scenario_id is not None else card.adventure + if owner is not None and owner.user_id == user.id: + return card + raise HTTPException(404, "Story card not found") + + +@router.get("", response_model=list[schemas.StoryCardOut]) +def list_story_cards( + scenario_id: int | None = None, + adventure_id: int | None = None, + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + if (scenario_id is None) == (adventure_id is None): + raise HTTPException(422, "Provide exactly one of scenario_id or adventure_id") + if scenario_id is not None: + scenario = db.get(models.Scenario, scenario_id) + if scenario is None or (scenario.user_id != user.id and not scenario.is_public): + raise HTTPException(404, "Owner not found") + return sorted(scenario.story_cards, key=lambda c: c.id) + adventure = db.get(models.Adventure, adventure_id) + if adventure is None or adventure.user_id != user.id: + raise HTTPException(404, "Owner not found") + return sorted(adventure.story_cards, key=lambda c: c.id) + + +@router.post("", response_model=schemas.StoryCardOut, status_code=201) +def create_story_card( + payload: schemas.StoryCardCreate, + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + if (payload.scenario_id is None) == (payload.adventure_id is None): + raise HTTPException(422, "Provide exactly one of scenario_id or adventure_id") + owner_model = models.Scenario if payload.scenario_id else models.Adventure + owner_id = payload.scenario_id or payload.adventure_id + owner = db.get(owner_model, owner_id) + if owner is None or owner.user_id != user.id: + raise HTTPException(404, "Owner not found") + limits.check_row_cap( + "story_cards", db, user, + scenario_id=payload.scenario_id, adventure_id=payload.adventure_id, + ) + card = models.StoryCard(**payload.model_dump()) + db.add(card) + db.commit() + return card + + +@router.patch("/{card_id}", response_model=schemas.StoryCardOut) +def update_story_card( + card_id: int, + payload: schemas.StoryCardUpdate, + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + card = _card_editable_or_404(db.get(models.StoryCard, card_id), user) + for field, value in payload.model_dump(exclude_unset=True).items(): + setattr(card, field, value) + db.commit() + return card + + +@router.delete("/{card_id}", status_code=204) +def delete_story_card( + card_id: int, + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + card = _card_editable_or_404(db.get(models.StoryCard, card_id), user) + db.delete(card) + db.commit() + + +# ---------- Bulk import / export (AI Dungeon world-info format) ---------- + + +@router.get("/export") +def export_story_cards( + scenario_id: int | None = None, + adventure_id: int | None = None, + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + owner = _visible_owner(scenario_id, adventure_id, db, user) + return [ + { + "keys": c.keys, + "value": c.entry, + "type": c.type, + "title": c.name, + "description": c.notes, + "useForCharacterCreation": False, + } + for c in sorted(owner.story_cards, key=lambda c: c.id) + ] + + +@router.post("/import", response_model=list[schemas.StoryCardOut], status_code=201) +def import_story_cards( + request: Request, + payload: dict = Body(...), + db: Session = Depends(get_db), + user: models.User = Depends(auth.get_current_user), +): + """Append a list of story cards (AI Dungeon world-info format, or our own + export) to a scenario/adventure the caller owns.""" + scenario_id = payload.get("scenario_id") + adventure_id = payload.get("adventure_id") + if (scenario_id is None) == (adventure_id is None): + raise HTTPException(422, "Provide exactly one of scenario_id or adventure_id") + owner_model = models.Scenario if scenario_id is not None else models.Adventure + owner = db.get(owner_model, scenario_id if scenario_id is not None else adventure_id) + if owner is None or owner.user_id != user.id: + raise HTTPException(404, "Owner not found") + + # Accept a bare array or {"cards": [...]} / {"storyCards": [...]}, so an + # AI Dungeon world-info file dropped in as-is still works. + cards_in = payload.get("cards") or payload.get("storyCards") or payload.get("worldInfo") + if not isinstance(cards_in, list): + raise HTTPException(422, 'Expected a "cards" array of story cards.') + cards_in = [c for c in cards_in if isinstance(c, dict)] + + limits.rate_limit("import", request, user) + limits.check_bundle_lists(story_cards=cards_in) + existing = len(owner.story_cards) + if auth.MULTI_USER and existing + len(cards_in) > limits.MAX_STORY_CARDS_PER_OWNER: + raise HTTPException( + 409, + f"Importing {len(cards_in)} would exceed the limit of " + f"{limits.MAX_STORY_CARDS_PER_OWNER} story cards here " + f"({existing} already present) — remove some first.", + ) + + created = [] + for card in cards_in: + obj = models.StoryCard( + scenario_id=scenario_id, + adventure_id=adventure_id, + type=str(card.get("type") or "")[:schemas.CARD_TYPE_MAX], + name=str(card.get("title") or card.get("name") or "")[:schemas.NAME_MAX], + keys=str(card.get("keys") or "")[:schemas.PROSE_MAX], + entry=str(card.get("value") or card.get("entry") or "")[:schemas.PROSE_MAX], + notes=str(card.get("description") or card.get("notes") or "")[:schemas.PROSE_MAX], + ) + db.add(obj) + created.append(obj) + db.commit() + return created diff --git a/backend/app/schemas.py b/backend/app/schemas.py new file mode 100644 index 0000000..9ff2c28 --- /dev/null +++ b/backend/app/schemas.py @@ -0,0 +1,510 @@ +from datetime import datetime +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, Field, computed_field + +from . import images + +# Length caps (Phase 9). The VARCHAR caps are a correctness requirement rather +# than only an abuse limit. Postgres enforces column lengths and SQLite never +# did, so a longer value has to be a 422 here rather than a 500 at INSERT. The +# text-column caps are generous abuse ceilings that a legitimate player does not +# reach. +NAME_MAX = 200 # Titles and names. VARCHAR(200). +TAGS_MAX = 500 # VARCHAR(500). +CARD_TYPE_MAX = 100 # VARCHAR(100). +PROSE_MAX = 50_000 # Memory, author's note, prompts, entries, and notes. +SCRIPT_MAX = 200_000 # One JavaScript source file. +ACTION_MAX = 20_000 # One player action. +MEMORY_TEXT_MAX = 5_000 +# A scenario cover image, stored inline as a base64 data URI. A 400x300 WebP at +# the quality the editor encodes runs about 20 to 40 kB. A cap of 400 kB leaves +# room for a client that downscales less aggressively, and it stops anyone from +# storing a multi-megabyte PNG in a row that every list request reads. +IMAGE_MAX = 400_000 +ICON_MAX = 16 # One emoji or glyph. VARCHAR(16). +BRANCH_NAME_MAX = 80 # What a player called one line of the story. VARCHAR(80). +PERSONA_NAME_MAX = 80 # The protagonist's name. VARCHAR(80). +PERSONA_PRONOUNS_MAX = 40 # "they/them" and the like. VARCHAR(40). + +Name = Annotated[str, Field(max_length=NAME_MAX)] +Tags = Annotated[str, Field(max_length=TAGS_MAX)] +CardType = Annotated[str, Field(max_length=CARD_TYPE_MAX)] +Prose = Annotated[str, Field(max_length=PROSE_MAX)] +ScriptSource = Annotated[str, Field(max_length=SCRIPT_MAX)] +ActionText = Annotated[str, Field(max_length=ACTION_MAX)] +Image = Annotated[str, Field(max_length=IMAGE_MAX)] +Icon = Annotated[str, Field(max_length=ICON_MAX)] +PersonaName = Annotated[str, Field(max_length=PERSONA_NAME_MAX)] +PersonaPronouns = Annotated[str, Field(max_length=PERSONA_PRONOUNS_MAX)] + + +class ORMModel(BaseModel): + model_config = ConfigDict(from_attributes=True) + + +# ---------- Story cards ---------- + +class StoryCardBase(BaseModel): + type: CardType = "" + name: Name = "" + keys: Prose = "" + entry: Prose = "" + notes: Prose = "" + + +class StoryCardCreate(StoryCardBase): + scenario_id: int | None = None + adventure_id: int | None = None + + +class StoryCardUpdate(BaseModel): + type: CardType | None = None + name: Name | None = None + keys: Prose | None = None + entry: Prose | None = None + notes: Prose | None = None + + +class StoryCardOut(ORMModel, StoryCardBase): + id: int + scenario_id: int | None + adventure_id: int | None + + +# ---------- Scenarios ---------- + +class ScenarioBase(BaseModel): + title: Name = "Untitled Scenario" + description: Prose = "" + prompt: Prose = "" + memory: Prose = "" + authors_note: Prose = "" + ai_instructions: Prose = "" + tags: Tags = "" + # Cover art, either an https URL or a base64 data URI. See `app/images.py`. + image: Image = "" + # The emoji or glyph shown when `image` is empty. + icon: Icon = "" + # Phase 12: the RPG world-state template, holding stat definitions, bands, + # rules, and milestones. `None` means the scenario has no RPG layer. + stat_schema: dict | None = None + + +class ScenarioCreate(ScenarioBase): + pass + + +class ScenarioUpdate(BaseModel): + title: Name | None = None + description: Prose | None = None + prompt: Prose | None = None + memory: Prose | None = None + authors_note: Prose | None = None + ai_instructions: Prose | None = None + tags: Tags | None = None + image: Image | None = None + icon: Icon | None = None + stat_schema: dict | None = None + script_ids: list[int] | None = None + + +class ScenarioOut(ORMModel, ScenarioBase): + id: int + is_public: bool = False # Shared demo content, read-only for everyone. + created_at: datetime + updated_at: datetime + story_cards: list[StoryCardOut] = [] + scripts: list["ScriptOut"] = [] + + +class ScenarioListItem(ORMModel): + id: int + title: str + description: str + tags: str + is_public: bool = False + updated_at: datetime + # Read from the row so that `image_url` can be derived, and excluded from + # the response, because a list of base64 data URIs would be megabytes of + # JSON. + image: str = Field("", exclude=True) + icon: str = "" + + @computed_field + @property + def image_url(self) -> str: + return images.public_url(self.id, self.image, self.updated_at) + + +# ---------- Adventures ---------- + +class AdventureCreate(BaseModel): + scenario_id: int | None = None + title: Name | None = None + # The `${Placeholder}` values collected from the player at the start, which + # is the AI Dungeon behavior. + placeholders: dict[str, str] = {} + # Phase 18: who the player is playing as, collected by the same modal. These + # are independent of `placeholders`: a scenario that asks for `${Name}` is + # asking its own question, and nothing here fills it in. + persona_name: PersonaName = "" + persona_pronouns: PersonaPronouns = "" + persona_desc: Prose = "" + + +class AdventureUpdate(BaseModel): + title: Name | None = None + memory: Prose | None = None + authors_note: Prose | None = None + ai_instructions: Prose | None = None + story_summary: Prose | None = None + auto_summarize: bool | None = None + memory_bank_enabled: bool | None = None + persona_name: PersonaName | None = None + persona_pronouns: PersonaPronouns | None = None + persona_desc: Prose | None = None + + +class AdventureRefresh(BaseModel): + """The body for "Update from scenario". + + `placeholders` supplies answers the adventure has no stored value for. See + `AdventureCreate.placeholders`. The answers are merged over the stored ones + and saved. + """ + + placeholders: dict[str, str] = {} + + +class RefreshPlan(BaseModel): + """What a refresh would change. The confirm dialog is built from this.""" + + scenario_id: int + scenario_title: str + has_changes: bool + # Maps a field name to `{"old": ..., "new": ...}`, for differing fields + # only. + fields: dict[str, dict] = {} + # Maps "added", "updated", or "removed" to a list of card names. + cards: dict[str, list[str]] = {} + # Maps "added" or "removed" to a list of stat paths. Live values are + # otherwise kept. + world_state: dict[str, list[str]] = {} + # The `${Placeholder}` names the scenario asks for that the adventure has + # no stored answer to. The client collects these and sends them back. + placeholders_needed: list[str] = [] + + +class ActionOut(ORMModel): + id: int + adventure_id: int + type: str + text: str + reasoning: str | None = None + # Phase 12: the compact RPG state changes for this turn, read from the + # model property. + world_changes: list[dict] = [] + # SP9: the pager, such as `2/4`. It reports how many attempts this turn has + # and which one is on screen. It is keyed on the parent, so it counts the + # attempts of this turn rather than every node that shares a depth, and it + # keeps counting them after one has been forked onto its own branch. + # + # A turn nobody has retaken reads 1/1, which is most turns, and the client + # draws no pager for a count of one. The attempts themselves come from + # `GET /actions/{id}/variants`, so this payload stays small. + take_count: int = 1 + take_index: int = 0 + # Which line this node is on, so the pager can distinguish the two kinds of + # step without asking the server. An attempt on this branch is a leaf with + # nothing below it, so showing it is a local change. An attempt on another + # branch has a story of its own, so moving to it is a branch switch. + branch_id: int | None = None + created_at: datetime + + +class VariantOut(BaseModel): + # Since SP4 every attempt is its own node, so each one has an id, and the + # client needs that id. A fork is addressed by the attempt being promoted, + # not by its position in a group that renumbers whenever an attempt is + # added. + id: int + index: int + text: str + reasoning: str | None = None + # See `ActionOut.branch_id`. It decides whether choosing this attempt is a + # local step or a branch switch. + branch_id: int | None = None + created_at: str | None = None + active: bool = False + + +class VariantSelect(BaseModel): + index: int = Field(ge=0) + + +class BranchOut(ORMModel): + """One line through the story tree (Phase 14, SP5). + + This carries enough to draw the tree and nothing more. `fork_depth` is where + this line leaves its parent, and `depth` is where it currently ends, so a + fork is two numbers rather than a walk. `own_actions` counts the turns played + on this branch itself. The rest of its story is borrowed from its ancestors, + which is why the number is smaller than a reader expects. + """ + + id: int + parent_branch_id: int | None = None + fork_depth: int | None = None + depth: int + own_actions: int = 0 + is_head: bool = False + # NULL for a branch nobody has named. The client labels those from the fork + # depth rather than the server inventing a name. See the column comment. + name: str | None = None + created_at: datetime + + +class BranchRename(BaseModel): + """A name a player chose, or `null` to make the branch unnamed again.""" + + name: Annotated[str, Field(max_length=BRANCH_NAME_MAX)] | None = None + + +class ActionUpdate(BaseModel): + text: ActionText + + +class ActionCreate(BaseModel): + type: Literal["do", "say", "story", "continue"] + text: ActionText = "" + # The node this action is played after (SP9). Omitting it means the tip, + # which is what every ordinary turn uses. + # + # Naming an attempt the story moved past is what creates a branch. Stepping + # between attempts costs nothing and creates nothing, and the fork happens + # on the first text written below one. That is the first moment the player + # states which line they mean. Before it, they were reading. + after_id: int | None = None + + +class TakeCreate(BaseModel): + """Another attempt at a turn (SP9). + + `text` is what the player says instead, and it applies only when the turn was + the player's. An AI turn's other attempt is generated, so the field is + ignored there rather than rejected. The client makes the same request for + both, and the node type decides what happens. + """ + + text: ActionText = "" + + +class AdventureOut(ORMModel): + id: int + scenario_id: int | None + title: str + memory: str + authors_note: str + ai_instructions: str + story_summary: str + auto_summarize: bool + memory_bank_enabled: bool + persona_name: str + persona_pronouns: str + persona_desc: str + created_at: datetime + updated_at: datetime + story_cards: list[StoryCardOut] = [] + # The newest window of the story, not all of it. Older pages arrive from + # `GET /{id}/actions` as the reader scrolls up. `action_count` is the whole + # story's length, which is how the client knows more actions exist above. + actions: list[ActionOut] = [] + action_count: int = 0 + + +class ActionPage(BaseModel): + """A slice of the story, counted back from the newest action.""" + + actions: list[ActionOut] = [] + total: int = 0 + # Whether anything older than this slice exists. The server computes it, so + # the client never has to do arithmetic on positions to find the end. + has_more: bool = False + + +# ---------- Memory bank (Phase 6) ---------- + +class MemoryOut(ORMModel): + id: int + adventure_id: int + text: str + pinned: bool + forgotten: bool + embedded: bool # model property: embedding vector present + use_count: int + last_used_at: datetime | None + source_start: int | None + source_end: int | None + created_at: datetime + + +class MemoryCreate(BaseModel): + text: Annotated[str, Field(max_length=MEMORY_TEXT_MAX)] + + +class MemoryUpdate(BaseModel): + text: Annotated[str, Field(max_length=MEMORY_TEXT_MAX)] | None = None + pinned: bool | None = None + forgotten: bool | None = None + + +class AdventureListItem(ORMModel): + id: int + scenario_id: int | None + scenario_title: str | None = None + title: str + updated_at: datetime + action_count: int = 0 + # The end of the most recent narration, so a Continue card can show the + # story rather than only a turn count. + snippet: str = "" + # Cover art inherited from the parent scenario. See `app/images.py`. + image_url: str = "" + icon: str = "" + + +# ---------- Scripts ---------- + +class ScriptBase(BaseModel): + name: Name = "Untitled Script" + description: Prose = "" + library_js: ScriptSource = "" + input_js: ScriptSource = "" + context_js: ScriptSource = "" + output_js: ScriptSource = "" + + +class ScriptCreate(ScriptBase): + pass + + +class ScriptUpdate(BaseModel): + name: Name | None = None + description: Prose | None = None + library_js: ScriptSource | None = None + input_js: ScriptSource | None = None + context_js: ScriptSource | None = None + output_js: ScriptSource | None = None + + +class ScriptOut(ORMModel, ScriptBase): + id: int + created_at: datetime + updated_at: datetime + + +class ScriptTestRequest(BaseModel): + hook: Literal["input", "context", "output"] + text: Prose = "" + state: dict = {} + + +class AdventureScriptOut(ORMModel): + id: int + adventure_id: int + position: int + enabled: bool + name: str + description: str + library_js: str + input_js: str + context_js: str + output_js: str + # The router sets this field, which is not stored. It is `True` when a + # syncable library version exists whose code differs from this copy, and + # `None` when there is nothing to sync from. + out_of_date: bool | None = None + + +class AdventureScriptUpdate(BaseModel): + enabled: bool | None = None + library_js: ScriptSource | None = None + input_js: ScriptSource | None = None + context_js: ScriptSource | None = None + output_js: ScriptSource | None = None + + +# ---------- Auth (Phase 8) ---------- + +class AuthCredentials(BaseModel): + email: Annotated[str, Field(max_length=320)] # VARCHAR(320). + # The upper bound keeps the scrypt cost constant. Without it, hashing a + # megabyte password would give an attacker free CPU time. + password: Annotated[str, Field(max_length=128)] + + +# ---------- Settings ---------- + +class SettingsOut(ORMModel): + endpoint_url: str + # The key itself is never returned. It is encrypted at rest and + # write-only. + has_api_key: bool + model: str + api_mode: str + temperature: float + max_output_tokens: int + reasoning_max_tokens: int + context_token_budget: int + narrator_prompt: str + summary_model: str + embedding_model: str + memory_bank_capacity: int + memory_top_k: int + + +ScenarioOut.model_rebuild() + + +# ---------- AI Chat (power users) ---------- +# A scratchpad for talking to a model directly, with no story framing. The +# server persists nothing, so these caps are per-request abuse limits only. + +CHAT_MESSAGE_MAX = 100_000 # One message. +CHAT_TOTAL_MAX = 400_000 # The whole conversation sent per request. +CHAT_MESSAGES_MAX = 200 # Turns per request. + + +class ChatMessage(BaseModel): + role: Literal["system", "user", "assistant"] + content: Annotated[str, Field(max_length=CHAT_MESSAGE_MAX)] + + +class ChatRequest(BaseModel): + messages: Annotated[list[ChatMessage], Field(min_length=1, max_length=CHAT_MESSAGES_MAX)] + # If this field is empty or omitted, the user's configured model is used. + model: Name | None = None + temperature: Annotated[float, Field(ge=0, le=5)] | None = None + max_tokens: Annotated[int, Field(ge=1, le=100_000)] | None = None + + +class SettingsUpdate(BaseModel): + endpoint_url: Annotated[str, Field(max_length=500)] | None = None # VARCHAR(500). + # Encryption expands the stored value by about four thirds into the same + # VARCHAR(500), so 256 plaintext characters is the largest safe input. The + # stored form is "enc:" plus Fernet plus base64. + api_key: Annotated[str, Field(max_length=256)] | None = None + model: Name | None = None + api_mode: Annotated[str, Field(max_length=20)] | None = None + temperature: Annotated[float, Field(ge=0, le=5)] | None = None + max_output_tokens: Annotated[int, Field(ge=1, le=100_000)] | None = None + # A value of -1 turns reasoning off explicitly, which sends + # `reasoning: {effort: none}`. A value of 0 sends nothing. + reasoning_max_tokens: Annotated[int, Field(ge=-1, le=100_000)] | None = None + context_token_budget: Annotated[int, Field(ge=256, le=200_000)] | None = None + narrator_prompt: Prose | None = None + summary_model: Name | None = None + embedding_model: Name | None = None + memory_bank_capacity: Annotated[int, Field(ge=1, le=1000)] | None = None + memory_top_k: Annotated[int, Field(ge=1, le=50)] | None = None diff --git a/backend/app/scripting/__init__.py b/backend/app/scripting/__init__.py new file mode 100644 index 0000000..d8814e6 --- /dev/null +++ b/backend/app/scripting/__init__.py @@ -0,0 +1,4 @@ +from .engine import HookResult, run_hook +from .pipeline import ScriptPipeline + +__all__ = ["HookResult", "ScriptPipeline", "run_hook"] diff --git a/backend/app/scripting/engine.py b/backend/app/scripting/engine.py new file mode 100644 index 0000000..774a117 --- /dev/null +++ b/backend/app/scripting/engine.py @@ -0,0 +1,146 @@ +"""AI Dungeon-compatible script execution in an embedded QuickJS sandbox. + +Each hook run is fully isolated (fresh Context), capped at 16 MB memory and +2 seconds CPU, with no filesystem/network/process access (QuickJS has none by +default). Scripts follow the AI Dungeon contract: define a `modifier(text)` +and call it as the last line; its return value `{ text, stop }` is the result. +""" + +import json +from dataclasses import dataclass, field + +import quickjs + +MEMORY_LIMIT = 16 * 1024 * 1024 +TIME_LIMIT_SECONDS = 2 +HISTORY_WINDOW = 100 # recent actions exposed as `history` + +# Globals per the official docs: text, state, history, storyCards, info, +# log/console.log, story card functions, plus legacy worldInfo aliases. +PRELUDE = """ +"use strict"; +var __logs = []; +var state = __DATA__.state; +var text = __DATA__.text; +var history = __DATA__.history; +var storyCards = __DATA__.storyCards; +var info = __DATA__.info; + +function log(msg) { + __logs.push(typeof msg === "string" ? msg : JSON.stringify(msg)); +} +var console = { log: log }; + +// Returns the new card's index, or false if a card with those keys exists — +// matching real AI Dungeon. Note index 0 is falsy; that quirk is upstream's. +function addStoryCard(keys, entry, type) { + for (var i = 0; i < storyCards.length; i++) { + if (storyCards[i].keys === keys) return false; + } + storyCards.push({ id: null, keys: keys || "", entry: entry || "", type: type || "" }); + return storyCards.length - 1; +} +function updateStoryCard(index, keys, entry, type) { + var card = storyCards[index]; + if (!card) throw new Error("Story card not found"); + card.keys = keys; + card.entry = entry; + card.type = type; +} +function removeStoryCard(index) { + if (!storyCards[index]) throw new Error("Story card not found"); + storyCards.splice(index, 1); +} + +// Legacy aliases used by older AI Dungeon scripts. +var worldInfo = storyCards; +var worldEntries = storyCards; +function addWorldEntry(keys, entry) { return addStoryCard(keys, entry, ""); } +function updateWorldEntry(index, keys, entry) { + var card = storyCards[index]; + if (!card) throw new Error("World entry not found"); + card.keys = keys; + card.entry = entry; +} +function removeWorldEntry(index) { return removeStoryCard(index); } +""" + +COLLECT = """ +JSON.stringify({ + result: (typeof __result === "undefined" || __result === null) ? null : __result, + state: state, + storyCards: storyCards, + logs: __logs +}) +""" + + +@dataclass +class HookResult: + text: str + stop: bool = False + state: dict = field(default_factory=dict) + story_cards: list = field(default_factory=list) + logs: list = field(default_factory=list) + error: str | None = None + + +def run_hook( + library_js: str, + hook_js: str, + text: str, + state: dict, + history: list[dict], + story_cards: list[dict], + info: dict, +) -> HookResult: + """Run one modifier hook. This function never raises. Failures return as + `.error` with text, state, and cards unchanged, so a bad script cannot + break a turn.""" + unchanged = HookResult(text=text, state=state, story_cards=story_cards) + source = f"{library_js}\n;\n{hook_js}" if library_js.strip() else hook_js + if not source.strip(): + return unchanged + + data = { + "state": state, + "text": text, + "history": history[-HISTORY_WINDOW:], + "storyCards": story_cards, + "info": info, + } + try: + ctx = quickjs.Context() + ctx.set_memory_limit(MEMORY_LIMIT) + ctx.set_time_limit(TIME_LIMIT_SECONDS) + ctx.eval(f"var __DATA__ = {json.dumps(data)};") + ctx.eval(PRELUDE) + ctx.eval(f"var __SRC__ = {json.dumps(source)};") + # Indirect eval keeps the script in global scope, so `modifier(text)` as the + # script's final expression statement becomes the completion value. + ctx.eval("var __result = (0, eval)(__SRC__);") + collected = json.loads(ctx.eval(COLLECT)) + except quickjs.JSException as exc: + unchanged.error = f"Script error: {exc}" + return unchanged + except Exception as exc: # memory limit, invalid JSON state, engine faults + unchanged.error = f"Script execution failed: {exc}" + return unchanged + + result = collected.get("result") + new_text, stop = text, False + if isinstance(result, dict): + if isinstance(result.get("text"), str): + new_text = result["text"] + stop = bool(result.get("stop")) + elif isinstance(result, str): + new_text = result + + new_state = collected.get("state") + return HookResult( + text=new_text, + stop=stop, + state=new_state if isinstance(new_state, dict) else {}, + story_cards=collected.get("storyCards") or [], + logs=collected.get("logs") or [], + ) diff --git a/backend/app/scripting/pipeline.py b/backend/app/scripting/pipeline.py new file mode 100644 index 0000000..7dee364 --- /dev/null +++ b/backend/app/scripting/pipeline.py @@ -0,0 +1,112 @@ +"""Runs an adventure's enabled scripts through a turn's hook points, applying +state and story-card mutations back to the database after each hook.""" + +from sqlalchemy.orm import Session + +from .. import models +from ..context import history as context_history +from .engine import run_hook + +MAX_STORY_CARDS = 5000 # AI Dungeon's per-adventure sanity cap + + +class ScriptPipeline: + def __init__(self, adventure: models.Adventure, db: Session): + self.adventure = adventure + self.db = db + self.logs: list[str] = [] + self.errors: list[str] = [] + + @property + def message(self) -> str | None: + state = self.adventure.script_state + msg = state.get("message") if isinstance(state, dict) else None + return msg if isinstance(msg, str) and msg.strip() else None + + def _history(self) -> list[dict]: + # Read the path rather than `adventure.actions`. That collection holds + # every branch's actions, and this is the documented history API a user + # script reads. Giving a script the siblings of the turn it is running on + # would be the same bug as building a prompt from them, and visible to + # the user. + # + # `story_actions` also drops rows with blank text, which + # `adventure.actions` kept, so this array is shorter than it was for an + # adventure that has any such rows. `info.actionCount` counts the same + # way. That is intended. A row with no text is this app's bookkeeping, it + # has no counterpart in the AI Dungeon history a ported script was + # written against, and the prompt has never included one. A script keyed + # on every N actions lands on different turns than it did before phase + # 14, and no reading of this is compatible with both. + return [ + {"text": a.text, "rawText": a.text, "type": a.type} + for a in context_history.story_actions(self.adventure) + ] + + def _cards(self) -> list[dict]: + return [ + {"id": c.id, "keys": c.keys, "entry": c.entry, "type": c.type} + for c in self.adventure.story_cards + ] + + def _info(self) -> dict: + return { + "actionCount": context_history.count(self.adventure), + "characterNames": [], + "memoryLength": len(self.adventure.memory), + "maxChars": 0, + } + + def _apply_cards(self, returned: list) -> None: + existing = {c.id: c for c in self.adventure.story_cards} + seen_ids = set() + added = 0 + for item in returned: + if not isinstance(item, dict): + continue + card_id = item.get("id") + keys = str(item.get("keys") or "") + entry = str(item.get("entry") or "") + card_type = str(item.get("type") or "") + if card_id in existing: + seen_ids.add(card_id) + card = existing[card_id] + card.keys, card.entry, card.type = keys, entry, card_type + elif len(existing) + added < MAX_STORY_CARDS: + self.db.add( + models.StoryCard( + adventure_id=self.adventure.id, + keys=keys, entry=entry, type=card_type, + ) + ) + added += 1 + for card_id, card in existing.items(): + if card_id not in seen_ids: + self.db.delete(card) + + def run(self, hook: str, text: str) -> tuple[str, bool]: + """Chain `hook` across all enabled scripts. Returns (text, stop).""" + state = self.adventure.script_state if isinstance(self.adventure.script_state, dict) else {} + for script in self.adventure.scripts: + hook_js = getattr(script, f"{hook}_js") + if not script.enabled or not hook_js.strip(): + continue + result = run_hook( + script.library_js, hook_js, text, state, + self._history(), self._cards(), self._info(), + ) + if result.error: + self.errors.append(f"{script.name} ({hook}): {result.error}") + continue # a broken script never breaks the turn + self.logs.extend(f"[{script.name}/{hook}] {line}" for line in result.logs) + self._apply_cards(result.story_cards) + state = result.state + self.adventure.script_state = state + self.db.commit() + text = result.text + if result.stop: + return text, True + return text, False + + def report(self) -> dict: + return {"logs": self.logs, "errors": self.errors, "message": self.message} diff --git a/backend/app/security.py b/backend/app/security.py new file mode 100644 index 0000000..c41f5a4 --- /dev/null +++ b/backend/app/security.py @@ -0,0 +1,131 @@ +"""Phase 8: secrets and crypto primitives for optional accounts. + +Everything derives from one server-side secret: + +* Session cookies are HMAC-signed with it. +* Stored LLM API keys are Fernet-encrypted with a key derived from it. + +The secret comes from `AIDND_SECRET_KEY`, or it is generated once into +`secret.key` next to the database, so a local install and a Docker volume work +with no configuration. Losing that file logs everyone out and makes the stored +API keys unreadable, and users then re-enter them. A multi-user deployment has +to set the environment variable, because a hosted filesystem is ephemeral and a +`secret.key` regenerated on every deploy would log out every user each time. + +Passwords use `hashlib.scrypt`, which is in the standard library and backed by +OpenSSL, so this needs no separate hashing dependency. +""" + +import base64 +import hashlib +import hmac +import os +import secrets + +from cryptography.fernet import Fernet, InvalidToken + +from .database import DB_PATH + +_SECRET_FILE = DB_PATH.parent / "secret.key" + + +def _load_secret() -> bytes: + env = os.environ.get("AIDND_SECRET_KEY", "").strip() + if env: + return env.encode() + # Same flag parse as auth.MULTI_USER (auth imports this module, so it + # can't be imported from there). + if os.environ.get("AIDND_MULTI_USER", "").strip().lower() in ("1", "true", "yes", "on"): + raise RuntimeError( + "AIDND_SECRET_KEY must be set when AIDND_MULTI_USER is on: an " + "auto-generated secret.key on an ephemeral hosted filesystem would " + "rotate on every deploy, logging out every user and orphaning " + "their stored API keys. Generate one with: " + "python -c \"import secrets; print(secrets.token_urlsafe(48))\"" + ) + if _SECRET_FILE.exists(): + return _SECRET_FILE.read_bytes().strip() + secret = secrets.token_urlsafe(48).encode() + _SECRET_FILE.write_bytes(secret) + return secret + + +SECRET_KEY = _load_secret() +_fernet = Fernet(base64.urlsafe_b64encode(hashlib.sha256(SECRET_KEY).digest())) + + +# ---------- Password hashing (scrypt) ---------- + +_SCRYPT_N, _SCRYPT_R, _SCRYPT_P = 2**14, 8, 1 + + +def hash_password(password: str) -> str: + salt = secrets.token_bytes(16) + key = hashlib.scrypt( + password.encode(), salt=salt, n=_SCRYPT_N, r=_SCRYPT_R, p=_SCRYPT_P + ) + return f"scrypt${_SCRYPT_N}${_SCRYPT_R}${_SCRYPT_P}${salt.hex()}${key.hex()}" + + +def verify_password(password: str, stored: str) -> bool: + try: + scheme, n, r, p, salt_hex, key_hex = stored.split("$") + if scheme != "scrypt": + return False + key = hashlib.scrypt( + password.encode(), salt=bytes.fromhex(salt_hex), + n=int(n), r=int(r), p=int(p), + ) + return hmac.compare_digest(key, bytes.fromhex(key_hex)) + except (ValueError, AttributeError): + return False + + +# ---------- Session tokens ---------- +# The token is "v1..". It does not expire, because a long-lived +# guest session is what this is for. + +def sign_session(user_id: int) -> str: + payload = f"v1.{user_id}" + sig = hmac.new(SECRET_KEY, payload.encode(), hashlib.sha256).hexdigest() + return f"{payload}.{sig}" + + +def verify_session(token: str) -> int | None: + try: + version, user_id, sig = token.split(".") + if version != "v1": + return None + payload = f"{version}.{user_id}" + expected = hmac.new(SECRET_KEY, payload.encode(), hashlib.sha256).hexdigest() + if not hmac.compare_digest(sig, expected): + return None + return int(user_id) + except (ValueError, AttributeError): + return None + + +# ---------- API-key encryption at rest ---------- +# Stored values carry an "enc:" prefix so plaintext keys from pre-Phase-8 +# databases can be recognized and migrated. + +ENC_PREFIX = "enc:" + + +def encrypt_secret(plain: str) -> str: + if not plain: + return "" + return ENC_PREFIX + _fernet.encrypt(plain.encode()).decode() + + +def decrypt_secret(stored: str) -> str: + """Returns the plaintext key. Tolerates legacy plaintext values (returned + as-is) and undecryptable tokens (secret rotated → treated as unset).""" + if not stored: + return "" + if not stored.startswith(ENC_PREFIX): + return stored + try: + return _fernet.decrypt(stored[len(ENC_PREFIX):].encode()).decode() + except (InvalidToken, ValueError): + return "" diff --git a/backend/app/seed.py b/backend/app/seed.py new file mode 100644 index 0000000..3b14cb1 --- /dev/null +++ b/backend/app/seed.py @@ -0,0 +1,249 @@ +"""Seed public demo scenarios on startup. + +Every JSON file in ``seed_data/`` describes one demo scenario in the same +model-native shape the export endpoint produces. Seeded scenarios have a NULL +owner and ``is_public=True``, so every visitor (including guests) sees them and +can start an adventure from them, while nobody can edit them. Starting an +adventure copies the scenario's story cards and scripts into the adventure, so +the seeded scripts run for guests too. + +Seed files are the source of truth for demo content: a scenario is inserted if +missing, reconciled in place when a seed file's content changes, and deleted +when no file claims its title any more, so an edit ships on the next deploy. +Rename a seed by changing its `title` and listing the old one under +`previous_titles`, which moves the rename onto the existing row. When a seed already matches, nothing is written, so +this stays cheap to run on every boot. An adventure already started from a demo +keeps its own copied cards and scripts and is unchanged. Only a new adventure +picks up the updated content. +""" + +import json +import logging +from pathlib import Path + +from sqlalchemy.engine import Engine + +from . import models +from .database import SessionLocal + +logger = logging.getLogger(__name__) + +SEED_DIR = Path(__file__).resolve().parent / "seed_data" + +# `title` is in this list so that a rename found through `previous_titles` is +# both detected by `_matches` and written by `_apply_scalars`. +_SCALARS = ("title", "description", "prompt", "memory", "authors_note", "ai_instructions", + "tags", "image", "icon") +_CARD_FIELDS = ("type", "name", "keys", "entry", "notes") +_SCRIPT_FIELDS = ("name", "library_js", "input_js", "context_js", "output_js") + + +def seed_public_scenarios(engine: Engine) -> None: + if not SEED_DIR.is_dir(): + return + files = sorted(SEED_DIR.glob("*.json")) + if not files: + return + + db = SessionLocal() + try: + changed = 0 + # Every title the files claim, including the ones they used to use. The + # sweep below deletes the seeded rows this set does not name. + claimed: set[str] = set() + complete = True + for path in files: + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + logger.warning("Skipping seed file %s: %s", path.name, exc) + complete = False + continue + + title = (data.get("title") or "").strip() + if not title: + continue + + claimed.add(title) + claimed.update(str(old) for old in data.get("previous_titles") or []) + + existing = find_seeded(db, title) or _find_renamed(db, data) + if existing is None: + _insert_scenario(db, data) + changed += 1 + elif not _matches(existing, data): + _update_scenario(db, existing, data) + changed += 1 + + changed += _sweep_unclaimed(db, claimed) if complete else 0 + if changed: + db.commit() + logger.info("Seeded/updated %d public demo scenario(s).", changed) + except Exception: + db.rollback() + # A seed failure must never take the app down; log and carry on. + logger.exception("Seeding public scenarios failed; continuing without them.") + finally: + db.close() + + +def _sweep_unclaimed(db, claimed: set[str]) -> int: + """Deletes seeded scenarios no seed file claims any more, and returns how + many went. + + A rename used to strand the row it left behind. `previous_titles` stops new + ones appearing, and this removes the ones already out there, which was + otherwise hand-work on every deployment. Only rows with a NULL owner and + `is_public` are considered, and a player's own scenario is neither, so + nothing anybody created can be reached from here. + + An adventure started from a deleted demo survives. `adventures.scenario_id` + is `ON DELETE SET NULL`, so the story, its cards, and its scripts are its + own copies and stay; the adventure loses the cover art it inherited. + + The caller skips this when a seed file failed to parse. A file that cannot + be read claims no title, and deleting on that basis would treat a syntax + error as an instruction to remove live content. An empty seed directory + never reaches here at all, for the same reason. + """ + stale = ( + db.query(models.Scenario) + .filter( + models.Scenario.user_id.is_(None), + models.Scenario.is_public.is_(True), + models.Scenario.title.notin_(claimed) if claimed else True, + ) + .all() + ) + for scenario in stale: + logger.info("Removing seeded scenario %r; no seed file claims it.", scenario.title) + # The scripts are joined through a secondary table, so nothing cascades + # to them. They have a NULL owner and no other reader. + for script in list(scenario.scripts): + db.delete(script) + scenario.scripts = [] + db.delete(scenario) + return len(stale) + + +def _card_tuple(source, get) -> tuple: + return tuple(get(source, f) for f in _CARD_FIELDS) + + +def _script_tuple(source, get) -> tuple: + return tuple(get(source, f) for f in _SCRIPT_FIELDS) + + +def find_seeded(db, title: str) -> models.Scenario | None: + """Returns the seeded scenario with this exact title, if there is one.""" + return ( + db.query(models.Scenario) + .filter( + models.Scenario.title == title, + models.Scenario.user_id.is_(None), + models.Scenario.is_public.is_(True), + ) + .first() + ) + + +def _find_renamed(db, data: dict) -> models.Scenario | None: + """Returns the row a renamed seed file used to own, so the rename lands on it. + + A seed is matched by title, so renaming one inserts a second scenario and + strands the first. The stranded row stays public forever and has to be + deleted by hand on every deployment. List the old title under + `previous_titles` in the seed file and the rename updates the existing row + instead, which also keeps the adventures already started from it pointing at + a scenario that still exists. + + Drop a `previous_titles` entry once every deployment has booted past it. + """ + for old in data.get("previous_titles") or []: + found = find_seeded(db, str(old)) + if found is not None: + return found + return None + + +def _matches(scenario: models.Scenario, data: dict) -> bool: + """True when the DB scenario already equals the seed file, so we can skip + the write and avoid churning rows on every boot.""" + if any(getattr(scenario, f) != data.get(f, "") for f in _SCALARS): + return False + if (scenario.stat_schema or None) != (data.get("stat_schema") or None): + return False + have_cards = sorted(_card_tuple(c, lambda o, f: getattr(o, f)) for c in scenario.story_cards) + want_cards = sorted( + _card_tuple(c, lambda o, f: o.get(f, "")) + for c in (data.get("story_cards") or []) if isinstance(c, dict) + ) + if have_cards != want_cards: + return False + have_scripts = sorted(_script_tuple(s, lambda o, f: getattr(o, f)) for s in scenario.scripts) + want_scripts = sorted( + _script_tuple(s, lambda o, f: (o.get(f, "") or ("Script" if f == "name" else ""))) + for s in (data.get("scripts") or []) if isinstance(s, dict) + ) + return have_scripts == want_scripts + + +def _insert_scenario(db, data: dict) -> None: + scenario = models.Scenario(user_id=None, is_public=True, title=data.get("title", "")) + _apply_scalars(scenario, data) + db.add(scenario) + db.flush() + _populate_children(db, scenario, data) + + +def _update_scenario(db, scenario: models.Scenario, data: dict) -> None: + _apply_scalars(scenario, data) + # Replace the child content in full. Demo content is owned by the server and + # cheap to rebuild, and replacing it this way keeps the scenario row, and the + # adventure foreign keys that point at it, intact. + for card in list(scenario.story_cards): + db.delete(card) + for script in list(scenario.scripts): + db.delete(script) + scenario.scripts = [] + db.flush() + _populate_children(db, scenario, data) + + +def _apply_scalars(scenario: models.Scenario, data: dict) -> None: + for field in _SCALARS: + setattr(scenario, field, data.get(field, "")) + # Phase 12: RPG world-state template (a JSON dict, not a scalar string). + scenario.stat_schema = data.get("stat_schema") or None + + +def _populate_children(db, scenario: models.Scenario, data: dict) -> None: + for card in data.get("story_cards") or []: + if not isinstance(card, dict): + continue + db.add( + models.StoryCard( + scenario_id=scenario.id, + type=card.get("type", ""), + name=card.get("name", ""), + keys=card.get("keys", ""), + entry=card.get("entry", ""), + notes=card.get("notes", ""), + ) + ) + + for item in data.get("scripts") or []: + if not isinstance(item, dict): + continue + script = models.Script( + user_id=None, + name=item.get("name", "Script"), + description=item.get("description", ""), + library_js=item.get("library_js", ""), + input_js=item.get("input_js", ""), + context_js=item.get("context_js", ""), + output_js=item.get("output_js", ""), + ) + db.add(script) + db.flush() + scenario.scripts.append(script) diff --git a/backend/app/seed_data/01-sunken-crypt-of-vharos.json b/backend/app/seed_data/01-sunken-crypt-of-vharos.json new file mode 100644 index 0000000..dc4f105 --- /dev/null +++ b/backend/app/seed_data/01-sunken-crypt-of-vharos.json @@ -0,0 +1,67 @@ +{ + "title": "[Demo] The Sunken Crypt of Vharos", + "description": "A short dungeon-crawl demo scenario with story cards and one of each script hook, for testing the app end to end.", + "prompt": "Rain hammers the moors as you descend the moss-slick steps beneath the ruined chapel. Your torch gutters in the stale air. Below, the Sunken Crypt of Vharos waits — its iron door ajar, as if someone (or something) expected you. Mira's warning rings in your ears: bring back the Ember Amulet before nightfall, or the village of Hollowmere burns.\n\nYou stand before the iron door, water pooling around your boots.", + "memory": "The player is an adventurer hired by Mira, blacksmith of Hollowmere, to retrieve the Ember Amulet from the Sunken Crypt of Vharos before nightfall. The crypt is flooded, dark, and haunted. Tone: classic D&D dungeon crawl, dangerous but fair.", + "authors_note": "Keep scenes tense and grounded; offer clear choices; consequences matter.", + "ai_instructions": "Write in second person, present tense. End each response at a moment where the player can act.", + "tags": "demo, dungeon, fantasy, short", + "story_cards": [ + { + "type": "character", + "name": "Mira the Blacksmith", + "keys": "Mira, blacksmith", + "entry": "Mira is Hollowmere's blacksmith: broad-shouldered, gray-braided, practical. She hired the player and paid half up front. She knows more about the crypt than she has admitted.", + "notes": "Secretly a descendant of Vharos." + }, + { + "type": "location", + "name": "The Sunken Crypt", + "keys": "crypt, tomb, Vharos", + "entry": "A flooded burial complex beneath a ruined chapel. Knee-deep black water, collapsed pillars, and phosphorescent moss. Three chambers: the Drowned Hall, the Ossuary, and the sealed Reliquary where the Ember Amulet rests.", + "notes": "" + }, + { + "type": "item", + "name": "The Ember Amulet", + "keys": "amulet, ember", + "entry": "A fist-sized garnet on a bronze chain that glows like a coal. It keeps Hollowmere's protective hearth-ward burning. Touching it bare-handed brands the flesh but does no lasting harm.", + "notes": "" + } + ], + "scripts": [ + { + "name": "[Demo] Dice Roller", + "description": "Input hook + shared library. Type '!roll 2d6' or '!roll d20' in a Do/Say/Story action and the command is replaced with the rolled result.", + "library_js": "// Shared library: available to every hook of this script.\nfunction rollDice(count, sides) {\n var total = 0, rolls = [];\n for (var i = 0; i < count; i++) {\n var r = Math.floor(Math.random() * sides) + 1;\n rolls.push(r);\n total += r;\n }\n return { total: total, rolls: rolls };\n}\n", + "input_js": "const modifier = (text) => {\n // Replace every \"!roll NdS\" (N optional) with the roll result.\n var out = text.replace(/!roll\\s+(\\d*)d(\\d+)/gi, function (m, n, s) {\n var count = parseInt(n || \"1\", 10);\n var sides = parseInt(s, 10);\n var res = rollDice(count, sides);\n log(\"Rolled \" + count + \"d\" + sides + \": [\" + res.rolls.join(\", \") + \"] = \" + res.total);\n return \"(rolled \" + count + \"d\" + sides + \": \" + res.total + \")\";\n });\n return { text: out };\n};\nmodifier(text);\n", + "context_js": "", + "output_js": "" + }, + { + "name": "[Demo] Turn & HP Tracker", + "description": "Demonstrates persistent state. Counts turns; '!hp -3' or '!hp +5' in input adjusts HP (starts at 20). Current stats appear in state.message.", + "library_js": "", + "input_js": "const modifier = (text) => {\n if (state.hp === undefined) state.hp = 20;\n state.turns = (state.turns || 0) + 1;\n\n var out = text.replace(/!hp\\s*([+-]\\d+)/gi, function (m, delta) {\n state.hp += parseInt(delta, 10);\n return \"\";\n });\n\n state.message = \"Turn \" + state.turns + \" | HP: \" + state.hp + \"/20\";\n log(state.message);\n\n if (state.hp <= 0) {\n // stop:true ends the turn before the AI is called.\n return { text: out + \"\\n\\nYou have fallen. (HP reached 0 — turn stopped by script.)\", stop: true };\n }\n return { text: out };\n};\nmodifier(text);\n", + "context_js": "", + "output_js": "" + }, + { + "name": "[Demo] Context Inspector", + "description": "Context hook: logs the size of the assembled context each turn and appends a style directive. Check the logs/context in the action's context snapshot.", + "library_js": "", + "input_js": "", + "context_js": "const modifier = (text) => {\n log(\"Context size: \" + text.length + \" chars, actions so far: \" + info.actionCount\n + \", story cards: \" + storyCards.length);\n // Anything returned here replaces what is sent to the model.\n return { text: text + \"\\n[Style: keep the response under three paragraphs.]\" };\n};\nmodifier(text);\n", + "output_js": "" + }, + { + "name": "[Demo] Output Polish + Card Discovery", + "description": "Output hook: trims a trailing incomplete sentence from the AI reply, and auto-creates a story card the first time the ghost Vharos is mentioned.", + "library_js": "", + "input_js": "", + "context_js": "", + "output_js": "const modifier = (text) => {\n var out = text;\n\n // Drop a trailing sentence fragment (no ending punctuation).\n var m = out.match(/^([\\s\\S]*[.!?\"'\\u2026])[^.!?\"'\\u2026]*$/);\n if (m && m[1].length > 40) {\n if (m[1].length < out.length) log(\"Trimmed incomplete final sentence.\");\n out = m[1];\n }\n\n // Demonstrate script-created story cards.\n if (/vharos/i.test(out)) {\n var added = addStoryCard(\n \"Vharos, ghost, spirit\",\n \"Vharos was the crypt's architect, now a restless ghost bound to the amulet he was buried with. He speaks in echoes and cannot lie.\",\n \"character\"\n );\n if (added !== false) log(\"Vharos mentioned — story card created.\");\n }\n\n return { text: out };\n};\nmodifier(text);\n" + } + ], + "icon": "🗝" +} diff --git a/backend/app/seed_data/02-signal-from-the-derelict.json b/backend/app/seed_data/02-signal-from-the-derelict.json new file mode 100644 index 0000000..61ba5db --- /dev/null +++ b/backend/app/seed_data/02-signal-from-the-derelict.json @@ -0,0 +1,34 @@ +{ + "title": "[Demo] Signal from the Derelict", + "description": "A tense sci-fi mystery aboard a drifting research station. Showcases the memory and story-card context engine with no scripts required.", + "prompt": "The airlock cycles with a hiss, and the cold of the Meridian station folds around your suit. Six days ago it stopped answering hails; three hours ago your ship's antenna caught a single repeating pulse from inside — a heartbeat rhythm, too regular to be an accident. Emergency lighting stutters red down the corridor ahead. Your suit reads breathable air, faint traces of ozone, and no life signs.\n\nYou float at the threshold, the pulse ticking softly in your headset.", + "memory": "The player is a salvage contractor who boarded the derelict research station Meridian after it went silent six days ago. A repeating signal is coming from somewhere inside. The crew of nine is missing. Tone: grounded, tense sci-fi mystery — quiet dread over action.", + "authors_note": "Keep the atmosphere claustrophobic and quiet; reward careful investigation; make the station feel physical and specific.", + "ai_instructions": "Write in second person, present tense. Favor sensory detail and restraint. End each response where the player can act.", + "tags": "demo, sci-fi, mystery, short", + "story_cards": [ + { + "type": "location", + "name": "Meridian Station", + "keys": "Meridian, station, ship", + "entry": "A mid-sized orbital research station in a decaying orbit around a gas giant. Ring layout: docking spur, crew habitat, the labs, and the sealed reactor core. Power is failing; artificial gravity is intermittent. It smells faintly of ozone and antiseptic.", + "notes": "Orbit decays fully in roughly 40 hours — a soft clock." + }, + { + "type": "character", + "name": "Dr. Reyes", + "keys": "Reyes, doctor, chief scientist", + "entry": "Dr. Ana Reyes was Meridian's chief scientist: meticulous, secretive, protective of her work. Her personal logs are locked. The last public entry, six days ago, reads only: 'It answered back.'", + "notes": "Whereabouts unknown. Her lab is sealed from the inside." + }, + { + "type": "item", + "name": "The Signal", + "keys": "signal, pulse, heartbeat", + "entry": "A repeating low-frequency pulse broadcasting from deep in the station, roughly one beat per second. It is not a standard distress code. The closer you get to the labs, the stronger it becomes.", + "notes": "" + } + ], + "scripts": [], + "icon": "🛰" +} diff --git a/backend/app/seed_data/03-hp-system-demo.json b/backend/app/seed_data/03-hp-system-demo.json new file mode 100644 index 0000000..b57cc00 --- /dev/null +++ b/backend/app/seed_data/03-hp-system-demo.json @@ -0,0 +1,21 @@ +{ + "title": "[Demo] The Goblin Ambush (HP script)", + "description": "A tiny combat scene that shows script-driven state: the AI narrates and decides damage, a JavaScript script tracks your HP. Open the State panel on the left to watch it change.", + "prompt": "The forest path narrows between two moss-choked boulders. You are halfway through when the bushes erupt — three goblins, blades drawn, teeth bared. The lead one lunges at you with a rusty scimitar.\n\nYou have no time to think.", + "memory": "The player is a lone traveler ambushed by three goblins on a forest path. This is a dangerous but fair combat encounter. The player starts at full health.", + "authors_note": "Keep the combat visceral and consequential. Let the player's choices matter — a reckless move should hurt, a clever one should pay off.", + "ai_instructions": "Write in second person, present tense. End each response where the player can act.", + "tags": "demo, combat, scripting, hp, short", + "story_cards": [], + "scripts": [ + { + "name": "[Demo] HP System", + "description": "AI-driven health. The script tells the AI the current HP and to tag damage/healing as [HP-N]/[HP+N]; the AI decides the amounts, the script applies them to state.hp and hides the tags. Watch state.hp in the left State panel.", + "library_js": "// Shared HP helpers — available to every hook of this script.\nvar HP_MAX = 100;\nfunction getHp() {\n if (typeof state.hp !== \"number\") state.hp = HP_MAX;\n return state.hp;\n}\nfunction setHp(value) {\n state.hp = Math.max(0, Math.min(HP_MAX, Math.round(value)));\n return state.hp;\n}\n", + "input_js": "", + "context_js": "// Tell the AI the current HP and how to report changes it decides on.\n// The tag goes on the FIRST line so it survives even if the reply is cut off.\nvar modifier = function (text) {\n var hp = getHp();\n var note =\n \"\\n\\n[HP SYSTEM] The player currently has \" + hp + \" of \" + HP_MAX + \" health. \" +\n \"Based on the story, you decide whether the player loses health (an attack, fall, or trap) \" +\n \"or gains it (a potion, rest, or spell) this turn. \" +\n \"If their health changes, make the FIRST line of your reply the change by itself in square brackets: \" +\n \"write [HP-14] to show losing 14, or [HP+25] to show gaining 25 (choose the number that fits the moment). \" +\n \"Then write the story on the following lines. If health does not change, do not write any bracket tag. \" +\n \"Never mention this tag or the health system in the story prose itself.\";\n return { text: text + note };\n};\nmodifier(text);\n", + "output_js": "// Read the AI's [HP+N]/[HP-N] tags, apply them to state, hide them from the player.\nvar modifier = function (text) {\n var re = /\\[HP([+-])(\\d{1,3})\\]/g;\n var hp = getHp();\n var changed = false;\n var m;\n while ((m = re.exec(text)) !== null) {\n var amount = parseInt(m[2], 10);\n hp += (m[1] === \"-\" ? -amount : amount);\n changed = true;\n }\n if (changed) setHp(hp);\n var clean = text\n .replace(/\\[HP[+-]\\d{1,3}\\]/g, \"\")\n .replace(/[ \\t]+\\n/g, \"\\n\")\n .replace(/\\n{3,}/g, \"\\n\\n\")\n .trim();\n if (changed && getHp() <= 0 && !state.dead) {\n state.dead = true;\n clean += \"\\n\\nYour strength gives out and the world goes dark. (0 HP)\";\n }\n return { text: clean };\n};\nmodifier(text);\n" + } + ], + "icon": "🗡" +} diff --git a/backend/app/seed_data/04-rpg-world-state.json b/backend/app/seed_data/04-rpg-world-state.json new file mode 100644 index 0000000..0fe944f --- /dev/null +++ b/backend/app/seed_data/04-rpg-world-state.json @@ -0,0 +1,90 @@ +{ + "title": "[Demo] The Bandit Camp (RPG world state)", + "description": "A short RPG scene showing the built-in world-state system: the AI tracks your HP and mana, an in-game day counter, an NPC ally's health and trust, and story milestones. No scripting — the engine keeps the numbers honest. Open the World State panel on the left to watch it change.", + "prompt": "Dawn breaks grey over the treeline as you and Gwen crouch at the edge of the bandit camp. Smoke curls from a dying fire; three bedrolls lie empty. Somewhere ahead, the stolen caravan strongbox waits.\n\nGwen checks her bowstring and looks to you. \"Quiet, or loud?\"", + "memory": "The player and Gwen, a loyal ranger ally, are raiding a bandit camp to recover a stolen strongbox. The player is a capable adventurer. This is a dangerous but winnable encounter.", + "authors_note": "Keep it tense and consequential. Reckless moves should cost health; clever ones should pay off. Gwen reacts to how the player treats her.", + "ai_instructions": "Write in second person, present tense. End each reply where the player can act. Let the world state guide the fiction — if the player is badly hurt, show it. Reflect what happens in the numbers each turn: change health when someone is hurt or healed, shift Gwen's trust based on how the player treats her, move the leader's aggression as the situation escalates or calms, and mark milestones as they are reached.", + "tags": "demo, rpg, world-state, combat, short", + "icon": "🏹", + "stat_schema": { + "world": { + "day": { "type": "counter", "min": 1, "initial": 1, "desc": "Which in-game day it is; only ever counts up." } + }, + "player": { + "hp": { + "desc": "The player's physical health. At 0 they fall.", + "min": 0, "max": 100, "initial": 100, "max_delta_per_turn": 35, + "bands": [[0, 1, "dead"], [1, 25, "gravely wounded"], [25, 50, "hurt"], + [50, 80, "scratched"], [80, 100, "healthy"], [100, 101, "full health"]] + }, + "mana": { + "desc": "Magical energy for spells; spent casting, restored by resting.", + "min": 0, "max": 50, "initial": 30, "max_delta_per_turn": 25, + "bands": [[0, 10, "drained"], [10, 30, "steady"], [30, 51, "brimming"]] + }, + "outfit": { + "type": "text", "initial": "worn leather armor, travel-stained", + "desc": "What the player is currently wearing/carrying visibly; update it when gear is gained, lost, or changed." + } + }, + "flags": { + "alarm_raised": { "desc": "True once the bandits know they're under attack; stealth is blown.", "initial": false }, + "player_hidden": { "desc": "True while the player is out of sight in cover.", "initial": true } + }, + "npcs": { + "gwen": { + "name": "Gwen", + "keys": "Gwen, ranger, her", + "desc": "A loyal ranger and the player's ally. Quick with a bow, dry-humoured, fiercely protective. Her trust rises when the player fights smart and watches her back, and falls when they are reckless with her life.", + "stats": { + "health": { + "desc": "Gwen's physical health.", + "min": 0, "max": 100, "initial": 100, "max_delta_per_turn": 35, + "bands": [[0, 1, "dead"], [1, 25, "gravely wounded"], [25, 50, "hurt"], + [50, 80, "scratched"], [80, 101, "healthy"]] + }, + "trust": { + "desc": "How much Gwen trusts the player; rises with smart, loyal play and falls with recklessness.", + "min": -100, "max": 100, "initial": 20, "max_delta_per_turn": 20, + "bands": [[-100, -30, "hostile"], [-30, 30, "wary"], [30, 70, "friendly"], + [70, 101, "devoted"]] + } + } + }, + "bandit_leader": { + "name": "Bandit Leader", + "keys": "leader, chief, boss, scarred", + "desc": "The scarred leader of the bandit camp, guarding the strongbox. Fights harder the more cornered he becomes.", + "stats": { + "health": { + "desc": "The leader's physical health.", + "min": 0, "max": 120, "initial": 120, "max_delta_per_turn": 40, + "bands": [[0, 1, "dead"], [1, 30, "near death"], [30, 70, "bloodied"], + [70, 121, "unhurt"]] + }, + "aggression": { + "desc": "How aggressively the leader fights; climbs as the fight turns against him.", + "min": 0, "max": 100, "initial": 40, "max_delta_per_turn": 25, + "bands": [[0, 30, "cautious"], [30, 70, "fierce"], [70, 101, "berserk"]] + } + } + } + }, + "milestones": { + "camp_cleared": { "desc": "Clear the bandit camp of enemies" }, + "strongbox_found": { "desc": "Recover the stolen strongbox" }, + "gwen_survives": { "desc": "Escape with Gwen still alive" } + } + }, + "story_cards": [ + { + "type": "location", + "name": "Bandit Camp", + "keys": "camp, bandits, strongbox", + "entry": "A rough camp of bandits in a forest clearing, holding a stolen caravan strongbox.", + "notes": "" + } + ], + "scripts": [] +} diff --git a/backend/app/seed_data/05-league-championship.json b/backend/app/seed_data/05-league-championship.json new file mode 100644 index 0000000..6685c8e --- /dev/null +++ b/backend/app/seed_data/05-league-championship.json @@ -0,0 +1,170 @@ +{ + "title": "[Demo] Pokemon League Championship: Round One", + "previous_titles": ["[Demo] League Championship: Round One"], + "description": "A full Pokémon battle run on the built-in world-state system. Every member of your team is tracked as its own entity with HP and a status condition, a text field names who is currently out, and your potion supply runs down as you use it. The opponent's active Pokémon is tracked the same way. No scripting — the engine keeps the numbers honest. Open the World State panel on the left to watch the battle play out.", + "prompt": "The floodlights come up over the championship field and the referee raises both flags. Round One of eight. Beat the trainer in front of you or go home.\n\nAcross the packed rock terrain, Trainer Milo tosses his first Poké Ball. Graveler lands hard enough to crack the stone under it, four arms flexing, and settles into a low stance.\n\n\"Rock-types all the way down,\" Milo calls over the noise. \"Hope you brought something that hits harder than it looks.\"\n\nYour Pidgeotto is already in the air, wings locked into a slow circle above the field. It is fast, but Rock-type moves will tear it out of the sky, and Graveler is built to take a hit. Three Hyper Potions sit in your belt pouch.\n\nGraveler grinds one fist into its palm. The referee drops the flag.\n\nWhat's your move?", + "memory": "The player is a Trainer in Round One of an eight-round League Championship. The Round One opponent is Milo, a Rock-type specialist with three Pokémon: Graveler (Rock/Ground), Onix (Rock/Ground), and Kabutops (Rock/Water). The player's team is Pidgeotto (Normal/Flying), Ivysaur (Grass/Poison), Wartortle (Water), Machoke (Fighting), and Pikachu (Electric). Pidgeotto is out first. The player carries Hyper Potions, which restore HP and can be swapped for a status heal when needed. A battle is lost when every Pokémon on one side has fainted.", + "authors_note": "Play it like a real battle. Type matchups decide damage, status conditions stick around and cost the afflicted Pokémon turns, and switching costs the player a turn while the opponent attacks freely. Milo is competent: he switches to punish bad matchups and targets whatever is weakest.", + "ai_instructions": "Write in second person, present tense. Narrate the battle move by move, naming the active Pokémon on each side and the move each one uses. End every reply at a decision point where the player can choose a move, switch Pokémon, or use an item.\n\nReflect type advantage and disadvantage in the damage you narrate and in the HP numbers: a super-effective hit should take roughly double a neutral hit, a resisted hit roughly half. Ground and Rock moves hit Pikachu and Pidgeotto hard. Water, Grass, and Fighting moves are strong against Milo's Rock-types.\n\nEvery turn, update the state. Every number you send is a change to add to the current value, never the value you want it to end up at. The world state lines show you totals; the block you write carries the difference. A Pokémon at 96 HP that takes 30 damage is `-30`, not `66`.\n- Add 1 to `world.turn` in every reply, including this one. It counts the turns this battle has lasted, so it moves whether or not anything else does.\n- Send a negative number for HP lost and a positive number for HP restored on both sides (`npc..hp` for the player's own Pokémon, e.g. npc.pidgeotto.hp; `npc.milo.active_hp` for Milo's). Every Pokémon starts at its full HP, so a positive number on an undamaged one changes nothing.\n- Set a Pokémon's `status` stat (`npc..status`, or `npc.milo.active_status`) to the condition it just picked up: poisoned, burned, paralyzed, frozen, asleep, or none when it is cured or wakes up. Status matters. Poison and burn shave HP at the end of each turn, paralysis and freeze cost turns, sleep stops a Pokémon acting until it wakes.\n- When the player switches, set `player.active_pokemon` to the incoming Pokémon's name in full (not a delta).\n- When the player uses a potion, send `-1` for `player.potions` and a positive number for the healed Pokémon's HP. The count falls when a potion is spent. It never rises. Potions cannot be used once the count reaches 0.\n- When a Pokémon's HP hits 0 it faints, its status becomes `fainted`, and its trainer must send out a replacement (the player picks the next `player.active_pokemon`). A fainted Pokémon cannot be healed or switched back in.\n- Whenever Milo sends out a new Pokémon, set `npc.milo.active_pokemon` and `npc.milo.active_status` to their new values in full, and send `npc.milo.active_hp` as the positive change that carries it from where it stands to the newcomer's full HP. From a fainted Pokémon at 0 that is the newcomer's full HP: Graveler 98, Onix 90, Kabutops 88.\n\nWhen one of Milo's Pokémon faints, do both of these in the same reply: send `1` for `npc.milo.pokemon_fainted`, which adds one to the count rather than setting it, and set `npc.milo.active_pokemon` with a fresh `npc.milo.active_hp` and `npc.milo.active_status` for the one he sends out next. `npc.milo.pokemon_fainted` counts up from 0 and never goes down.\n\nMark each milestone as it happens, using the id shown beside the goal in the world state. Do not decide the whole battle in one reply; give the player a turn between every exchange.", + "tags": "demo, pokemon, battle, tournament, world-state, combat", + "icon": "◓", + "image": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMAAAADACAYAAABS3GwHAAAMc0lEQVR42u2du44byRWG+w0m712bNtbrsQ0DTKyYMPQAhHIDjA0IYOZACRMHyggnCreTyTYYKNyoM2cCA4cK6FSJGSqkeWa7Z6nRcKZO3bou3wF+QIA4zWb3f6rOvZoGQRAEQRAEQcJL27azExYPMOPJIDmTej4QeTOgH7A74WiJ3dl1xuvKd8x54shURL86I/qtI8FdsRvuYVSMK94QEsJcWZ3QnbCfkOym2A/3usKsQmxJvzxhmwnhTRRCfsuSN4s8R3pZOQ8FkP4SDsNvRBmQO9IvKiD9c8qwgAn1ObHrQswbn2bSGie6/DBlB9mfRUeYtTwzp4fYavSYRxDfCX9pvz0u21/f4e/tb47/bL8zgnx2/Du5BoqAJEv86/abO6IKcX9ovz/+1F4f/9P+8fi/b/7sFXJNubZ8h3yXfKd8N4qAjMSfxSC+EO8f7W+PP7a/D0J0G8WQe5F7knuLpAgk2BKL6mxDmjEj4acmuylGhQhsPm2JGk1P/lWIGP5f218d/9X+LokV3scOIb9FflOgXMIKJhZg7owrfQmkf0oZAu0MmEURyb/2uer/rZ1lZd74NJPkt3veDdYwNINVX6Inpa/22l3BY0SJ3SAA+Zc+Vn15yRJC/G/7p+qJ/xDyTOTZeFKEAwV3/si/hfjZKsIWBruFN3cQP3tF2BEu1ZN/7mrySDkBNr4fH0GepQeTiAI7RWzfKYYvJQOQ1y/kmXrIJZAzCGXvj+YOZA0LD2YRfsEF8ncu9TmYO3HNIse6ow7Gf+ns3to+TFb9aXcDByW4rd45don0SDr/3+0fIOLEkHfgUFpRb4TIhfySwie0mVbI1KGsoj4lcCG/VDZCujQh7wYlCER+iTzUWLSWY5GdZZSoDiWwcXjlgWLv5+UXWCrBLaHORxJb2Pt5+gWWibOuVPJvIT9KUGWyzKa8AfJXrwSrUsg/tylkgzxlwbKgbp47+a+0VZ2yWkCYMmGxExyyjgxpw52YPZhDj4VHq3B6IT9KUIxTPPTwquL8kL8uJbDIEyxzIf9MY/eT5CJZpvAHZjkogGp0CeUNdZdNaEeupE7+NYVtIHAB3boI00fKZyEAEChLqdM0hTSmjzRQ4PSCc6dY2VTTp0Z+VakDTi94zCnOslRCm+2lhxd46jFOI0usSXjJJAFeNHgKymkT2xQcX+N4P6NLgMnIFWV+YJaF44vpAwKZQv1U5F9Q4QkSqRxdJL36M6sT2MwiTXYX0Kz+NLeASE00i+RWfxxfENEh7pNb/XF8QWSHeJHU6k+5A4jcO9CHJv+c1R8kvgvMQypAx+oPEt8FulDkv2L1B5nsAlchFGDN6g8y2QXWIRRgb/Llcuo4LwyEgHDLUAH2k4U+ifuDkHmBSUKips4vbY4gofbJzqcCHJjwADKbJHGIOuRKejp5QSAGFP3Dy2jmD87v0xEMqW6UUJ48J+l6egzyf/IZ+SyRNC/OcBfN/MH5/brJW16U5Wz8+z4KuQZDBKyd4UMU84eGl1/wQ/u9E+mfesZybZ6xumFm6aIAWya8mU84szwYTn1gIM9bNVFuGzz5VbP5I/a6w0np1pDvrLnTTmEG7W3JPyP687Rjq3DGgkHuoVaHWbHwzGwUYEX057KDG8LOd/EPanSUFQvQKlj4s7bkl8Mh0MF9g9qUQJEU64LZ/5AfJZgSQfwA09r/mkYd+ib/y5cvj69evbqD/BslCD5K8cp79Wct9r8r+a+vr4+vX78+vn///vjx48fj58+fH4X8n3xGPit/gxJ49QMWGgXYYP//Eu2xdXhldb+5ublI+OcgfyvXsHWMa4gOKfyAjUYBbon/q1eYe7x48eJuJbcl/kPIteSaNiFS8gH3uPV6wLVss4zn+xpivnz69Mkb+UfINeXa2vupIVlmaJ7uNAqAA6xLtNzBxdzRmEXajDGO8M/wOvun9MkP2tMLY5DfVglKrx1STIyYe4sAlV6ZqIn6xCS/jRKUbq4KF71FgkwjQCXblooHemeXxyb/CI1PUPKCpfDVNt4UoOQIkGnYUyIzIRxejWNsGh0quWdDEQkyUoC+5hIIzXGdPkOdLiFSjqc1LonovShAyZEF07i/JKimJv8I02RZyXkBw4hd7y0HcKmxO3eYOr9TOL6uDjHvzSAXkGqlY1JVl9fXyZB/hEvtUE1AATxgysiPj4gQCoACOCEF59fFGUYBUAAnPFXSPBXknng3jgpg2ghfO1Ij/wjejWODvGYMeq2Q7q1UFcBnZ1nBWKAADkgp/m+bD0ABUAAUAAVAATCBUACcYJxgnGDCoIRBCYOiACTCUAAUgFIIFIAHRTFc1QqwMyHAONavNJgSKctyaN7bzktDjLThpRoJccWbN2+KbYiR31bqezNsDe29tUSW+iA/fPhQbEuk/LZS35vPlshNrqHA2AmlnJriU07gRQwB+5sK0fd9sQ9UM3Mnl7EoKfksviFc9KkAi9ofqDaqkvxgrASjVhM9i4W30Yhv374t+qG+e/eumNGI8ltKflfCRW+jEU1zASlXRUaOLCQ9HLfkiJ02CuZ3PHrh26rStkx2PHrJvprSXN15PyCj5EiQNi+Q4gEZJcf9LSJAqgMyNrkWhYVYeW3r7Kc8IknuecoQbYJ5ENURSQtWmC+TY06H1sU+JO/0tyUnvSx3aNUheVc4wn6VINoxqRWRX9kGan5Mquag7FoedAglCFGlWhP5FSUQuoOyBwXo8APyUYIaya+w/zsbBVjhB1xWgpQa0OVeaiO/0v5f2SjAjETL09EhmxCpb8g91BDtcUxUzhobMfUDasgHPJUss4nRu0K+s4Ykl4f4/76xldMfb6k1Ma8diuEbyHfwvFW1WlsXBVjWXmtuk7wK4R/INUuvwA00BGzZuMjpAgfMIDtHWexzF2WQv5Vr1OjgejJ/Do2rmIZDa4wGaRxmsdelbFee06XGbvk/+Yx8tlbH1nP0p/OhAEuiQSDT6M+y8SGmZlBtSTGQdPLr0PgSUzNotVrxkkBQCMeimT82Y9NxhkEiA4AXjU8xTYrhDIMEnN9941tOF12bJmqIYIAQkTRFonEdQgGuTLef0idGgKQnP+hr/307w+wCYMLVv2tCienMIHYBMOHqP29CiunwXHYBMMHq3zehRRMSZRcAkVf/RRNDNLsAeQHgEvdPavW32QVSPE8L5AHlJLxFE1NMd4FaxvOBScdS9k1s0ewCNMyAgA0v8Vd/m10AhxgEcnz7ZioxnRyBQwwCOb72Ex88KsFWMyyWlww8jTp0a3j3XCN0wBQCkU2fQ7Can1BT5Go4rhPYDxFQDg1YNSmJxiGe+nhRkF65g3K4WN+kJoNDbGwK0T4JLNocR9Nn1qQopk0zTJQDFhPewjW7TGUKMUmCCQ9K8vdN6qI1hWqcaw+szllI1/SxHaZF7wA1/gosm5xEkyCr6VRDyG916ua2yVFMDtxGCSC/twOuE1QAVZaYylEqPJPN9sZopKeJhuaWqA3uqZZKYA5Vb/akV+oQ2ylGCaom/7YpUUwHa6EEVZO/a0qW0w+85RBoklwXcNuULkNkaGejBJRN5FHeYEn+XfYRn9BKQAFdcYVt9ZHfhxJI+Sx+QVr2vrKkuW7y+1ACaaDAL0jD3lc2s0D+R5Tg1vYsXXqMs+nh/crhrZ78riHS82kTjFyJO7pEOb2hrlBnzGTZeZSI3SDOqm8Z5Sk7yTVl2cTDxBmzSMPM6rRMbJVb3hC4gO7g8rCl+AqzyI+5Y1nI9rCqcw6zI0WIHppFhEztQpsezB0iPVP6BSjCpMTH3veoBEtXkwhFiEr8Q3Y9vBkowUw7cuUpRZBTx/ERfrbx5Vl4Iv5xeEczGBtOEdY+doPzsooai+zkNzuUL1xa9dcwNLPd4Ly0ovRdYVztHUoXWPUTzBkcPL/Mu3i3VDaWoAzyG+S3eIjhX1r1ie0nEC7dBni5X+wMOZlJcq+BVvovIjyENws3iy7VHY0KkcIOIfcwEt6xPgdzpxBFWMRQhPOIkhBPQog3Nzd3JQMhFEOuKdeW75Dvku/0GLkxJf4ChqEIKvNJiCqQcgIhrgnks+PfBTZjID6KACB+HYowd+k5qAgdhWvlR40kmbaH7PfYD8+EqE6F5lEXIpeQAQ7Db8fMQe4L7kpXhpH0FKohzyrDthAzaT/8FkiPWCnDbCi56DJRiP1wrysSVkgoJ1p8h80w2mM3Idl3wz1shnvCiUUmDbOOirEZ4ui9o4Lszq6zOSM6YUoke7Nq8QCYKwiCIAiCIEgM+T+tNIWP5O3yJwAAAABJRU5ErkJggg==", + "stat_schema": { + "world": { + "round": { "type": "counter", "min": 1, "max": 8, "initial": 1, "desc": "Which round of the eight-round championship this is; only ever counts up." }, + "turn": { "type": "counter", "min": 0, "initial": 0, "max_delta_per_turn": 1, "desc": "How many turns the current battle has lasted; only ever counts up." } + }, + "player": { + "potions": { + "desc": "Hyper Potions left in the player's bag. Each one restores HP or cures a status condition, and is used up when it is used. Send -1 when the player uses one. Never send the number remaining.", + "min": 0, "max": 5, "initial": 3, "max_delta_per_turn": 1, + "bands": [[0, 1, "out of potions"], [1, 2, "one left"], [2, 4, "a few left"], [4, 6, "well stocked"]] + }, + "active_pokemon": { + "type": "text", "initial": "Pidgeotto", + "desc": "Which of the player's Pokémon is currently on the field: Pidgeotto, Ivysaur, Wartortle, Machoke, or Pikachu. Replace this the moment the player switches." + } + }, + "flags": { + "sandstorm_active": { "desc": "True while a sandstorm is whipping across the field, chipping HP off any Pokémon that is not Rock, Ground, or Steel each turn.", "initial": false } + }, + "npcs": { + "pidgeotto": { + "name": "Pidgeotto", + "keys": "Pidgeotto", + "desc": "The player's own Pokémon. Normal/Flying. Fast, but Rock-type moves hit it for four times damage.", + "stats": { + "hp": { + "desc": "Pidgeotto's HP. Send the change, not the new total: a negative number for damage taken, a positive number for HP restored.", + "min": 0, "max": 84, "initial": 84, "max_delta_per_turn": 60, + "bands": [[0, 1, "fainted"], [1, 20, "critical"], [20, 45, "hurt"], [45, 70, "scuffed"], [70, 85, "healthy"]] + }, + "status": { + "type": "text", "initial": "none", + "desc": "Status condition: none, poisoned, burned, paralyzed, frozen, asleep, or fainted." + } + } + }, + "ivysaur": { + "name": "Ivysaur", + "keys": "Ivysaur", + "desc": "The player's own Pokémon. Grass/Poison. Super effective on Milo's Rock/Ground types; can inflict poison or sleep.", + "stats": { + "hp": { + "desc": "Ivysaur's HP. Send the change, not the new total: a negative number for damage taken, a positive number for HP restored.", + "min": 0, "max": 96, "initial": 96, "max_delta_per_turn": 60, + "bands": [[0, 1, "fainted"], [1, 24, "critical"], [24, 50, "hurt"], [50, 78, "scuffed"], [78, 97, "healthy"]] + }, + "status": { + "type": "text", "initial": "none", + "desc": "Status condition: none, poisoned, burned, paralyzed, frozen, asleep, or fainted." + } + } + }, + "wartortle": { + "name": "Wartortle", + "keys": "Wartortle", + "desc": "The player's own Pokémon. Water. Bulky, and super effective on the whole of Milo's team.", + "stats": { + "hp": { + "desc": "Wartortle's HP. Send the change, not the new total: a negative number for damage taken, a positive number for HP restored.", + "min": 0, "max": 92, "initial": 92, "max_delta_per_turn": 60, + "bands": [[0, 1, "fainted"], [1, 23, "critical"], [23, 48, "hurt"], [48, 75, "scuffed"], [75, 93, "healthy"]] + }, + "status": { + "type": "text", "initial": "none", + "desc": "Status condition: none, poisoned, burned, paralyzed, frozen, asleep, or fainted." + } + } + }, + "machoke": { + "name": "Machoke", + "keys": "Machoke", + "desc": "The player's own Pokémon. Fighting. Hits Rock-types hardest of the team but is slow and takes Ground moves badly.", + "stats": { + "hp": { + "desc": "Machoke's HP. Send the change, not the new total: a negative number for damage taken, a positive number for HP restored.", + "min": 0, "max": 104, "initial": 104, "max_delta_per_turn": 65, + "bands": [[0, 1, "fainted"], [1, 26, "critical"], [26, 54, "hurt"], [54, 84, "scuffed"], [84, 105, "healthy"]] + }, + "status": { + "type": "text", "initial": "none", + "desc": "Status condition: none, poisoned, burned, paralyzed, frozen, asleep, or fainted." + } + } + }, + "pikachu": { + "name": "Pikachu", + "keys": "Pikachu", + "desc": "The player's own Pokémon. Electric. Frail, and its Electric moves do nothing at all to Ground-types.", + "stats": { + "hp": { + "desc": "Pikachu's HP. Send the change, not the new total: a negative number for damage taken, a positive number for HP restored.", + "min": 0, "max": 76, "initial": 76, "max_delta_per_turn": 60, + "bands": [[0, 1, "fainted"], [1, 19, "critical"], [19, 40, "hurt"], [40, 62, "scuffed"], [62, 77, "healthy"]] + }, + "status": { + "type": "text", "initial": "none", + "desc": "Status condition: none, poisoned, burned, paralyzed, frozen, asleep, or fainted." + } + } + }, + "milo": { + "name": "Trainer Milo", + "keys": "Milo, opponent, Graveler, Onix, Kabutops, rock", + "desc": "The Round One opponent, a Rock-type specialist with three Pokémon: Graveler, Onix, and Kabutops. He leads with Graveler, switches to punish a bad matchup, and aims for whichever of the player's Pokémon is closest to fainting.", + "stats": { + "active_pokemon": { + "type": "text", "initial": "Graveler", + "desc": "Which Pokémon Milo currently has on the field: Graveler, Onix, or Kabutops. Replace this the moment he sends out a different one." + }, + "active_hp": { + "desc": "The HP of Milo's active Pokémon. Send the change, not the new total: a negative number for damage taken, a positive number for HP restored. When Milo sends out a different Pokémon, send the positive change that carries this from where it stands to the newcomer's full HP: Graveler 98, Onix 90, Kabutops 88.", + "min": 0, "max": 98, "initial": 98, "max_delta_per_turn": 98, + "bands": [[0, 1, "fainted"], [1, 25, "critical"], [25, 50, "hurt"], [50, 78, "scuffed"], [78, 99, "healthy"]] + }, + "active_status": { + "type": "text", "initial": "none", + "desc": "The status condition on Milo's active Pokémon: none, poisoned, burned, paralyzed, frozen, asleep, or fainted." + }, + "pokemon_fainted": { + "desc": "How many of Milo's three Pokémon have fainted. Add 1 each time one goes down. Milo loses the round when this reaches 3.", + "type": "counter", + "min": 0, "max": 3, "initial": 0, "max_delta_per_turn": 1, + "bands": [[0, 1, "full team"], [1, 2, "two left"], [2, 3, "last Pokémon"], [3, 4, "defeated"]] + } + } + } + }, + "milestones": { + "graveler_defeated": { "desc": "Knock out Milo's lead Graveler" }, + "status_landed": { "desc": "Inflict a status condition on one of Milo's Pokémon" }, + "type_advantage_used": { "desc": "Land a super-effective hit by switching in the right Pokémon" }, + "round_one_won": { "desc": "Knock out all three of Milo's Pokémon and advance to Round Two" } + } + }, + "story_cards": [ + { + "type": "location", + "name": "Championship Stadium", + "keys": "stadium, arena, battlefield, field, terrain", + "entry": "The League Championship's main battlefield. The floor is broken rock terrain, which gives Rock- and Ground-type moves solid footing and gives grounded Pokémon cover to duck behind. Eight rounds stand between any Trainer and the Champion's title, and this is Round One.", + "notes": "Rocky terrain. Rock and Ground moves get a slight edge. Boulders can be used as cover or thrown." + }, + { + "type": "character", + "name": "Trainer Milo", + "keys": "Milo, opponent, rock trainer", + "entry": "Milo is the Round One opponent, a Rock-type specialist who won his qualifiers on raw defense. His team is Graveler (Rock/Ground), Onix (Rock/Ground), and Kabutops (Rock/Water). He leads with Graveler, likes Sandstorm and Rock Slide, and switches out the moment a matchup turns against him.", + "notes": "Graveler and Onix are weak to Water, Grass, Fighting, Ground, Ice, and Steel, and are immune to Electric. Kabutops is weak to Grass, Fighting, Ground, and Electric. All three resist Normal and Flying." + }, + { + "type": "item", + "name": "Hyper Potion", + "keys": "potion, hyper potion, heal, item, bag", + "entry": "A pressurized spray that closes wounds fast. One dose restores a large chunk of a Pokémon's HP, or clears a single status condition instead. Using one costs your turn, and the opponent attacks freely while you do it. You start the round with three and cannot restock mid-battle.", + "notes": "Decrement player.potions by 1 per use. A potion cannot revive a fainted Pokémon." + } + ], + "scripts": [] +} diff --git a/backend/app/sse.py b/backend/app/sse.py new file mode 100644 index 0000000..21365d4 --- /dev/null +++ b/backend/app/sse.py @@ -0,0 +1,30 @@ +"""Server-sent events: the wire format, the headers, and the error frame. + +Two routers stream: the turn engine in `routers/adventures/turns.py` and the +chat scratchpad in `routers/chat.py`. Both send JSON objects as SSE `data:` +frames, so the format lives here rather than in either one. +""" +import json + +from . import analytics + +# `no-cache` stops an intermediary from caching the stream. `X-Accel-Buffering` +# makes nginx-style reverse proxies, which hosted deploys use, flush each event +# immediately rather than buffer it. +SSE_HEADERS = {"Cache-Control": "no-cache", "X-Accel-Buffering": "no"} + + +def sse(obj: dict) -> str: + """Returns one SSE frame carrying `obj` as JSON.""" + return f"data: {json.dumps(obj)}\n\n" + + +def turn_error(detail: str, **extra) -> str: + """Returns an SSE error for a turn that could not be produced, and counts it. + + A failed turn is still an HTTP 200 response, so the middleware's status-code + tally cannot see it. This metric exists so that a demo whose model refuses + every request does not report as healthy. + """ + analytics.record(analytics.M_EVENT, analytics.EV_TURN_ERROR) + return sse({"type": "error", "detail": detail, **extra}) diff --git a/backend/app/starter.py b/backend/app/starter.py new file mode 100644 index 0000000..3b123dd --- /dev/null +++ b/backend/app/starter.py @@ -0,0 +1,92 @@ +"""Gives every new guest a short adventure that is already played. + +A guest arriving on an empty account has nothing to look at, and the demo turns +are limited, so learning what the app does used to cost one of them. This copies +a small pre-played adventure into the new account instead. It opens on a story +with real turns in it, and the turn summaries show what the world-state engine +records: the changes it applied, and the changes it refused. + +The file in `starter_data/` is an ordinary export bundle, produced by +`GET /api/adventures/{id}/export` and trimmed to its first two exchanges. To +replace it, play a new adventure, export it, and overwrite the file. Nothing +else here knows what the story contains. + +The copy is the guest's own from the first moment: they can edit it, branch it, +delete it, or export it, and nothing links it back to the file. A returning +guest is not given a second one, because this runs only where a guest row is +created. +""" + +import json +import logging +from pathlib import Path + +from sqlalchemy.orm import Session + +from . import bundle, models, seed + +logger = logging.getLogger(__name__) + +STARTER_FILE = Path(__file__).resolve().parent / "starter_data" / "pokemon-league.json" + + +def _load() -> dict | None: + """Returns the starter bundle, or None when the file is missing or invalid. + + The file ships with the server, so a failure here is a packaging mistake + rather than bad input. It is still tolerated: a guest with no starter + adventure can play, and a guest who cannot be created cannot. + """ + try: + return json.loads(STARTER_FILE.read_text(encoding="utf-8")) + except (OSError, ValueError): + logger.exception("Starter adventure is missing or unreadable; skipping it.") + return None + + +def give(db: Session, user: models.User) -> models.Adventure | None: + """Copies the starter adventure into `user`, and returns it. + + The caller commits. This writes rows and does not commit them, so a guest + and their first adventure land in one transaction: an account is never left + half-populated. + + Failures are logged and swallowed. The starter is a convenience, and losing + it must not cost the visitor their session. + """ + payload = _load() + if payload is None: + return None + try: + # A savepoint, so a failure halfway through writing the tree discards + # only the starter's rows. Without it the caller's next commit would + # flush whatever part of the adventure the session still held. + with db.begin_nested(): + story = bundle.plan(payload, bundle.check_format(payload)) + adventure = bundle.materialize(db, payload, story, user.id) + _link_scenario(db, adventure, payload) + return adventure + except Exception: + logger.exception("Could not give user %s the starter adventure.", user.id) + return None + + +def _link_scenario(db: Session, adventure: models.Adventure, payload: dict) -> None: + """Points the copy at the demo scenario it came from, when that seed exists. + + A bundle does not carry a scenario id, because the id is local to one + database and an export outlives it. The link matters anyway: an adventure + has no cover art of its own, and inherits the scenario's. Without it the + starter shows a monogram tile while the demo it came from shows its own + artwork. + + The name is read from `scenarioTitle` in the file. If no seeded scenario + answers to it, the adventure keeps a NULL `scenario_id`, which is the same + state an imported bundle is in and costs only the artwork. + """ + title = str(payload.get("scenarioTitle") or "").strip() + if not title: + return + scenario = seed.find_seeded(db, title) + if scenario is not None: + adventure.scenario_id = scenario.id diff --git a/backend/app/starter_data/pokemon-league.json b/backend/app/starter_data/pokemon-league.json new file mode 100644 index 0000000..97399d3 --- /dev/null +++ b/backend/app/starter_data/pokemon-league.json @@ -0,0 +1,516 @@ +{ + "title": "Pokemon League Championship: Round One", + "scenarioTitle": "[Demo] Pokemon League Championship: Round One", + "format": "ai-dnd-adventure-v2", + "memory": "The player is a Trainer in Round One of an eight-round League Championship. The Round One opponent is Milo, a Rock-type specialist with three Pokémon: Graveler (Rock/Ground), Onix (Rock/Ground), and Kabutops (Rock/Water). The player's team is Pidgeotto (Normal/Flying), Ivysaur (Grass/Poison), Wartortle (Water), Machoke (Fighting), and Pikachu (Electric). Pidgeotto is out first. The player carries Hyper Potions, which restore HP and can be swapped for a status heal when needed. A battle is lost when every Pokémon on one side has fainted.", + "authorsNote": "Play it like a real battle. Type matchups decide damage, status conditions stick around and cost the afflicted Pokémon turns, and switching costs the player a turn while the opponent attacks freely. Milo is competent: he switches to punish bad matchups and targets whatever is weakest.", + "aiInstructions": "Write in second person, present tense. Narrate the battle move by move, naming the active Pokémon on each side and the move each one uses. End every reply at a decision point where the player can choose a move, switch Pokémon, or use an item.\n\nReflect type advantage and disadvantage in the damage you narrate and in the HP numbers: a super-effective hit should take roughly double a neutral hit, a resisted hit roughly half. Ground and Rock moves hit Pikachu and Pidgeotto hard. Water, Grass, and Fighting moves are strong against Milo's Rock-types.\n\nEvery turn, update the state. Every number you send is a change to add to the current value, never the value you want it to end up at. The world state lines show you totals; the block you write carries the difference. A Pokémon at 96 HP that takes 30 damage is `-30`, not `66`.\n- Add 1 to `world.turn` in every reply, including this one. It counts the turns this battle has lasted, so it moves whether or not anything else does.\n- Send a negative number for HP lost and a positive number for HP restored on both sides (`npc..hp` for the player's own Pokémon, e.g. npc.pidgeotto.hp; `npc.milo.active_hp` for Milo's). Every Pokémon starts at its full HP, so a positive number on an undamaged one changes nothing.\n- Set a Pokémon's `status` stat (`npc..status`, or `npc.milo.active_status`) to the condition it just picked up: poisoned, burned, paralyzed, frozen, asleep, or none when it is cured or wakes up. Status matters. Poison and burn shave HP at the end of each turn, paralysis and freeze cost turns, sleep stops a Pokémon acting until it wakes.\n- When the player switches, set `player.active_pokemon` to the incoming Pokémon's name in full (not a delta).\n- When the player uses a potion, send `-1` for `player.potions` and a positive number for the healed Pokémon's HP. The count falls when a potion is spent. It never rises. Potions cannot be used once the count reaches 0.\n- When a Pokémon's HP hits 0 it faints, its status becomes `fainted`, and its trainer must send out a replacement (the player picks the next `player.active_pokemon`). A fainted Pokémon cannot be healed or switched back in.\n- Whenever Milo sends out a new Pokémon, set `npc.milo.active_pokemon` and `npc.milo.active_status` to their new values in full, and send `npc.milo.active_hp` as the positive change that carries it from where it stands to the newcomer's full HP. From a fainted Pokémon at 0 that is the newcomer's full HP: Graveler 98, Onix 90, Kabutops 88.\n\nWhen one of Milo's Pokémon faints, do both of these in the same reply: send `1` for `npc.milo.pokemon_fainted`, which adds one to the count rather than setting it, and set `npc.milo.active_pokemon` with a fresh `npc.milo.active_hp` and `npc.milo.active_status` for the one he sends out next. `npc.milo.pokemon_fainted` counts up from 0 and never goes down.\n\nMark each milestone as it happens, using the id shown beside the goal in the world state. Do not decide the whole battle in one reply; give the player a turn between every exchange.", + "storySummary": "", + "scriptState": {}, + "worldState": { + "world": { + "round": 1, + "turn": 2 + }, + "player": { + "potions": 3, + "active_pokemon": "Wartortle" + }, + "npc": { + "pidgeotto": { + "hp": 84, + "status": "none" + }, + "ivysaur": { + "hp": 96, + "status": "none" + }, + "wartortle": { + "hp": 84, + "status": "none" + }, + "machoke": { + "hp": 104, + "status": "none" + }, + "pikachu": { + "hp": 76, + "status": "none" + }, + "milo": { + "active_pokemon": "Onix", + "active_hp": 90, + "active_status": "none", + "pokemon_fainted": 1 + } + }, + "milestones": { + "graveler_defeated": { + "reached": true, + "at": 4 + } + }, + "flags": { + "sandstorm_active": false + }, + "_meta": { + "last_changed": { + "world.turn": 4, + "player.active_pokemon": 2, + "npc.wartortle.hp": 2, + "npc.milo.active_hp": 4, + "npc.milo.pokemon_fainted": 4, + "npc.milo.active_pokemon": 4 + } + } + }, + "autoSummarize": false, + "memoryBankEnabled": false, + "branches": [ + { + "parent": null, + "forkDepth": null + } + ], + "headBranch": 0, + "memoryCursor": { + "branch": null, + "depth": -1 + }, + "summaryCursor": { + "branch": null, + "depth": -1 + }, + "memories": [], + "storyCards": [ + { + "type": "location", + "name": "Championship Stadium", + "keys": "stadium, arena, battlefield, field, terrain", + "entry": "The League Championship's main battlefield. The floor is broken rock terrain, which gives Rock- and Ground-type moves solid footing and gives grounded Pokémon cover to duck behind. Eight rounds stand between any Trainer and the Champion's title, and this is Round One.", + "notes": "Rocky terrain. Rock and Ground moves get a slight edge. Boulders can be used as cover or thrown." + }, + { + "type": "character", + "name": "Trainer Milo", + "keys": "Milo, opponent, rock trainer", + "entry": "Milo is the Round One opponent, a Rock-type specialist who won his qualifiers on raw defense. His team is Graveler (Rock/Ground), Onix (Rock/Ground), and Kabutops (Rock/Water). He leads with Graveler, likes Sandstorm and Rock Slide, and switches out the moment a matchup turns against him.", + "notes": "Graveler and Onix are weak to Water, Grass, Fighting, Ground, Ice, and Steel, and are immune to Electric. Kabutops is weak to Grass, Fighting, Ground, and Electric. All three resist Normal and Flying." + }, + { + "type": "item", + "name": "Hyper Potion", + "keys": "potion, hyper potion, heal, item, bag", + "entry": "A pressurized spray that closes wounds fast. One dose restores a large chunk of a Pokémon's HP, or clears a single status condition instead. Using one costs your turn, and the opponent attacks freely while you do it. You start the round with three and cannot restock mid-battle.", + "notes": "Decrement player.potions by 1 per use. A potion cannot revive a fainted Pokémon." + }, + { + "type": "character", + "name": "Pidgeotto", + "keys": "Pidgeotto", + "entry": "The player's own Pokémon. Normal/Flying. Fast, but Rock-type moves hit it for four times damage.", + "notes": "" + }, + { + "type": "character", + "name": "Ivysaur", + "keys": "Ivysaur", + "entry": "The player's own Pokémon. Grass/Poison. Super effective on Milo's Rock/Ground types; can inflict poison or sleep.", + "notes": "" + }, + { + "type": "character", + "name": "Wartortle", + "keys": "Wartortle", + "entry": "The player's own Pokémon. Water. Bulky, and super effective on the whole of Milo's team.", + "notes": "" + }, + { + "type": "character", + "name": "Machoke", + "keys": "Machoke", + "entry": "The player's own Pokémon. Fighting. Hits Rock-types hardest of the team but is slow and takes Ground moves badly.", + "notes": "" + }, + { + "type": "character", + "name": "Pikachu", + "keys": "Pikachu", + "entry": "The player's own Pokémon. Electric. Frail, and its Electric moves do nothing at all to Ground-types.", + "notes": "" + } + ], + "scripts": [], + "actions": [ + { + "branch": 0, + "depth": 0, + "live": true, + "type": "start", + "text": "The floodlights come up over the championship field and the referee raises both flags. Round One of eight. Beat the trainer in front of you or go home.\n\nAcross the packed rock terrain, Trainer Milo tosses his first Poké Ball. Graveler lands hard enough to crack the stone under it, four arms flexing, and settles into a low stance.\n\n\"Rock-types all the way down,\" Milo calls over the noise. \"Hope you brought something that hits harder than it looks.\"\n\nYour Pidgeotto is already in the air, wings locked into a slow circle above the field. It is fast, but Rock-type moves will tear it out of the sky, and Graveler is built to take a hit. Three Hyper Potions sit in your belt pouch.\n\nGraveler grinds one fist into its palm. The referee drops the flag.\n\nWhat's your move?", + "createdAt": "2026-08-28T12:24:29.635984", + "stateAfter": {}, + "worldStateAfter": { + "world": { + "round": 1, + "turn": 0 + }, + "player": { + "potions": 3, + "active_pokemon": "Pidgeotto" + }, + "npc": { + "pidgeotto": { + "hp": 84, + "status": "none" + }, + "ivysaur": { + "hp": 96, + "status": "none" + }, + "wartortle": { + "hp": 92, + "status": "none" + }, + "machoke": { + "hp": 104, + "status": "none" + }, + "pikachu": { + "hp": 76, + "status": "none" + }, + "milo": { + "active_pokemon": "Graveler", + "active_hp": 98, + "active_status": "none", + "pokemon_fainted": 0 + } + }, + "milestones": {}, + "flags": { + "sandstorm_active": false + }, + "_meta": { + "last_changed": {} + } + } + }, + { + "branch": 0, + "depth": 1, + "live": true, + "type": "do", + "text": "> You Recall Pidgeotto and send out Wartortle, then hit Graveler with a full-power Water Gun.", + "createdAt": "2026-08-28T12:24:48.236580", + "stateAfter": {}, + "worldStateAfter": { + "world": { + "round": 1, + "turn": 0 + }, + "player": { + "potions": 3, + "active_pokemon": "Pidgeotto" + }, + "npc": { + "pidgeotto": { + "hp": 84, + "status": "none" + }, + "ivysaur": { + "hp": 96, + "status": "none" + }, + "wartortle": { + "hp": 92, + "status": "none" + }, + "machoke": { + "hp": 104, + "status": "none" + }, + "pikachu": { + "hp": 76, + "status": "none" + }, + "milo": { + "active_pokemon": "Graveler", + "active_hp": 98, + "active_status": "none", + "pokemon_fainted": 0 + } + }, + "milestones": {}, + "flags": { + "sandstorm_active": false + }, + "_meta": { + "last_changed": {} + } + } + }, + { + "branch": 0, + "depth": 2, + "live": true, + "type": "ai", + "text": "You call Pidgeotto back with a flash of red light, and it banks gratefully out of Graveler's range before the crack of a Rock Throw can find it. Wartortle drops onto the broken stone in its place, planting its feet, shell catching the floodlights.\n\n\"Smart,\" Milo grunts, \"but Graveler doesn't fold that easy.\" He points. \"Rock Throw!\"\n\nGraveler wrenches a chunk of the battlefield loose and hurls it in a flat, brutal arc. But Wartortle's already moving, ducking low behind a natural outcrop for a half-second of cover before the shard clips its shoulder — a glancing hit, more noise than damage.\n\n\"Wartortle, Water Gun — full power!\"\n\nWartortle plants both feet and unleashes a roaring jet of water straight into Graveler's chest. It's not a graceful hit, it's a drowning one — the stream slams Graveler backward, hissing against hot rock, and the four-armed boulder staggers, one knee grinding into the dirt before it forces itself back upright, dripping and furious.\n\n\"That's the matchup I didn't want to see,\" Milo mutters, though he's grinning like he half-expected it.\n\nGraveler shakes itself off, glaring, arms flexing back into a battle-ready stance. It's hurt — badly — but still standing, and still dangerous at close range.\n\nThe referee's flag stays down. It's still your move.", + "createdAt": "2026-08-28T12:25:00.242852", + "stateAfter": {}, + "worldStateAfter": { + "world": { + "round": 1, + "turn": 1 + }, + "player": { + "potions": 3, + "active_pokemon": "Wartortle" + }, + "npc": { + "pidgeotto": { + "hp": 84, + "status": "none" + }, + "ivysaur": { + "hp": 96, + "status": "none" + }, + "wartortle": { + "hp": 84, + "status": "none" + }, + "machoke": { + "hp": 104, + "status": "none" + }, + "pikachu": { + "hp": 76, + "status": "none" + }, + "milo": { + "active_pokemon": "Graveler", + "active_hp": 52, + "active_status": "none", + "pokemon_fainted": 0 + } + }, + "milestones": {}, + "flags": { + "sandstorm_active": false + }, + "_meta": { + "last_changed": { + "world.turn": 2, + "player.active_pokemon": 2, + "npc.wartortle.hp": 2, + "npc.milo.active_hp": 2 + } + } + }, + "worldDelta": { + "delta": { + "world.turn": 1, + "player.active_pokemon": "Wartortle", + "npc.wartortle.hp": -8, + "npc.milo.active_hp": -46, + "flags.type_advantage_used": true + }, + "applied": [ + { + "path": "world.turn", + "old": 0, + "new": 1 + }, + { + "path": "player.active_pokemon", + "old": "Pidgeotto", + "new": "Wartortle" + }, + { + "path": "npc.wartortle.hp", + "old": 92, + "new": 84 + }, + { + "path": "npc.milo.active_hp", + "old": 98, + "new": 52 + } + ], + "clamped": [], + "rejected": [ + { + "path": "flags.type_advantage_used", + "reason": "unknown flag", + "fix": "There is no flag `type_advantage_used`. The flags are: `sandstorm_active`." + } + ] + } + }, + { + "branch": 0, + "depth": 3, + "live": true, + "type": "do", + "text": "> You Wartortle hits Graveler with everything it has left, a point-blank Hydro Pump to finish it.", + "createdAt": "2026-08-28T12:25:21.664571", + "stateAfter": {}, + "worldStateAfter": { + "world": { + "round": 1, + "turn": 1 + }, + "player": { + "potions": 3, + "active_pokemon": "Wartortle" + }, + "npc": { + "pidgeotto": { + "hp": 84, + "status": "none" + }, + "ivysaur": { + "hp": 96, + "status": "none" + }, + "wartortle": { + "hp": 84, + "status": "none" + }, + "machoke": { + "hp": 104, + "status": "none" + }, + "pikachu": { + "hp": 76, + "status": "none" + }, + "milo": { + "active_pokemon": "Graveler", + "active_hp": 52, + "active_status": "none", + "pokemon_fainted": 0 + } + }, + "milestones": {}, + "flags": { + "sandstorm_active": false + }, + "_meta": { + "last_changed": { + "world.turn": 2, + "player.active_pokemon": 2, + "npc.wartortle.hp": 2, + "npc.milo.active_hp": 2 + } + } + } + }, + { + "branch": 0, + "depth": 4, + "live": true, + "type": "ai", + "text": "Wartortle plants its stubby legs against the cracked stone, shell rattling as it draws breath for the biggest attack it has. \"Finish it — Hydro Pump!\" you shout.\n\nThe blast that follows isn't a stream, it's a cannon — a roaring column of pressurized water that closes the gap before Graveler can even brace. It catches the rock-type square in the chest and drives it backward across the broken terrain, gouging a furrow through the stone floor. Graveler's arms windmill once, twice, searching for purchase that isn't there, and then it simply goes down, crashing onto its back in a spreading puddle, swirl-eyed and still.\n\nMilo's jaw tightens, but he's already reaching for his second ball. \"Didn't want to see that either. Fine — you earned that one.\" He hurls it out. \"Onix, let's go!\"\n\nThe ground itself seems to shudder as Onix erupts from the earth in a towering column of jointed rock, easily three stories high, boulder head swinging around to fix its small eyes on Wartortle. Dust and loose gravel rain down from the impact of its arrival, pattering off your shoulders even from across the field.\n\n\"Rock/Ground,\" you murmur to yourself, but Wartortle doesn't flinch — Water still chews through that typing just as easily, and the referee's flag is already dropping to signal the next exchange.\n\nWartortle shakes water from its shell, tired from the Hydro Pump but still standing tall, eyeing the massive serpent of stone now looming over it.", + "createdAt": "2026-08-28T12:25:34.346927", + "stateAfter": {}, + "worldStateAfter": { + "world": { + "round": 1, + "turn": 2 + }, + "player": { + "potions": 3, + "active_pokemon": "Wartortle" + }, + "npc": { + "pidgeotto": { + "hp": 84, + "status": "none" + }, + "ivysaur": { + "hp": 96, + "status": "none" + }, + "wartortle": { + "hp": 84, + "status": "none" + }, + "machoke": { + "hp": 104, + "status": "none" + }, + "pikachu": { + "hp": 76, + "status": "none" + }, + "milo": { + "active_pokemon": "Onix", + "active_hp": 90, + "active_status": "none", + "pokemon_fainted": 1 + } + }, + "milestones": { + "graveler_defeated": { + "reached": true, + "at": 4 + } + }, + "flags": { + "sandstorm_active": false + }, + "_meta": { + "last_changed": { + "world.turn": 4, + "player.active_pokemon": 2, + "npc.wartortle.hp": 2, + "npc.milo.active_hp": 4, + "npc.milo.pokemon_fainted": 4, + "npc.milo.active_pokemon": 4 + } + } + }, + "worldDelta": { + "delta": { + "world.turn": 1, + "npc.milo.active_hp": -52, + "npc.milo.pokemon_fainted": 1, + "npc.milo.active_pokemon": "Onix", + "npc.milo.active_status": "none", + "milestones.graveler_defeated": true + }, + "applied": [ + { + "path": "world.turn", + "old": 1, + "new": 2 + }, + { + "path": "npc.milo.active_hp", + "old": 52, + "new": 0 + }, + { + "path": "npc.milo.pokemon_fainted", + "old": 0, + "new": 1 + }, + { + "path": "npc.milo.active_pokemon", + "old": "Graveler", + "new": "Onix" + }, + { + "path": "milestones.graveler_defeated", + "old": false, + "new": true + } + ], + "clamped": [], + "rejected": [] + } + } + ] +} diff --git a/backend/app/tree.py b/backend/app/tree.py new file mode 100644 index 0000000..78a194c --- /dev/null +++ b/backend/app/tree.py @@ -0,0 +1,412 @@ +"""Phase 14: writes nodes onto the story tree. + +This module is the write half of the tree. It decides which branch a new node +goes on, what depth the node gets, and where the adventure's head points. All +three decisions live here because a mistake in any of them is silent. A node +written without a branch is invisible to every read, and nothing raises an +error. + +The read half is `context/lineage.py`. It turns a branch into the set of nodes +that make up one story. + +Until forking ships, each adventure has one branch and `depth` mirrors `index`, +so nothing here changes observable behavior yet. That is intentional. By the +time reads depend on these columns, every row already has them, including the +rows written between the two deploys that no migration visits. + +SP2 added `place_new_nodes`, which runs on every flush. Wiring up individual +call sites worked while nothing read the columns. Now that reads filter on them, +relying on each writer to remember would also mean relying on every fixture, +script, and test. A missed call produces a story with missing turns, so the +flush enforces the rule instead. +""" + +import copy + +from sqlalchemy import func, insert, update +from sqlalchemy.orm import Session + +from . import models +from .context import lineage + +# Head depth of an adventure that has no actions. Using -1 keeps the rule "the +# next node goes at head_depth + 1" true without a special case. This matches +# `migrations.NO_DEPTH`. +NO_DEPTH = -1 + + +def root_branch(db: Session, adventure: models.Adventure) -> models.Branch: + """Returns the adventure's root branch, creating it on first use. + + This function gets or creates the branch instead of creating it alongside + the adventure. The adventures that need a root branch are usually ones that + already exist: an imported bundle, a fixture built through the ORM, or a + database migrated before this code shipped. + """ + branch = ( + db.query(models.Branch) + .filter( + models.Branch.adventure_id == adventure.id, + models.Branch.parent_branch_id.is_(None), + ) + .order_by(models.Branch.id) + .first() + ) + if branch is not None: + return branch + # Use a Core insert instead of the ORM. `place_new_nodes` can call this + # during a flush, and a nested ORM flush raises an error. Both paths share + # one transaction. The lineage refers to the branch's own id, so it needs a + # second statement, which runs once per adventure. + new_id = db.execute( + insert(models.Branch).values( + adventure_id=adventure.id, + parent_branch_id=None, + fork_depth=None, + lineage=[], + created_at=models.utcnow(), + ) + ).inserted_primary_key[0] + db.execute( + update(models.Branch) + .where(models.Branch.id == new_id) + .values(lineage=[[new_id, None]]) + ) + return db.get(models.Branch, new_id) + + +def head_branch(db: Session, adventure: models.Adventure) -> models.Branch: + """Returns the branch that new nodes are played onto.""" + if adventure.head_branch_id is not None: + branch = db.get(models.Branch, adventure.head_branch_id) + if branch is not None: + return branch + # The head points at a branch that no longer exists, which means a bug + # elsewhere. Fall back to the root instead of refusing to play, + # otherwise the adventure becomes unusable. + branch = root_branch(db, adventure) + adventure.head_branch_id = branch.id + return branch + + +def fork(db: Session, adventure: models.Adventure, node: models.Action) -> models.Branch: + """Moves `node` onto a new branch so the story can continue from it. + + `node` is a discarded attempt at a turn that the story has already moved + past. Making it live where it stands would orphan every turn played after + it, because those turns continue the attempt that won. Instead, `node` moves + to a new branch that forks from the depth just before it. The parent branch + keeps its story unchanged. The new branch inherits everything up to the fork + and owns this one node. + + This function inserts one row and moves one row. It copies nothing, so a + fork costs one `branches` row plus the ancestry cached on it, regardless of + how long the story is. + + Derived data stays on the parent, by design. A memory attaches to the node + its block ends on, and that node does not move. From the new branch, the + lineage caps the parent at `fork_depth`, so the memory sits one depth past + the border. Neither retrieval nor the cursors can see it, and the block is + summarized again from the text this branch contains. + + The other attempts at this turn also stay on the parent, because they are + still takes on the parent's turn. If none of them is live, the oldest one + becomes live, so the parent keeps the line it was written on. + """ + parent = db.get(models.Branch, node.branch_id) + if parent is None or node.depth is None: + raise ValueError("cannot fork from a node that is not on a branch") + fork_depth = node.depth - 1 + # Read the sibling attempts before moving the node. The session does not + # autoflush, so a later read still finds the node here and hands the live + # flag back to it. + remaining = [ + row for row in db.query(models.Action) + .filter( + models.Action.adventure_id == adventure.id, + models.Action.branch_id == parent.id, + models.Action.depth == node.depth, + ) + .order_by(models.Action.id) + .all() + if row is not node + ] + # The parent's ancestry, with every entry capped at the fork depth. Only the + # first entry can change in practice, because older entries are already + # capped at a shallower depth. Capping all of them states the invariant + # directly. + inherited = [ + [branch_id, fork_depth if cap is None else min(cap, fork_depth)] + for branch_id, cap in lineage.entries_of(parent) + ] + # Core insert with the lineage written second, for the reason given in + # `root_branch`. This code can run inside a flush, and the lineage refers to + # the new row's own id. + new_id = db.execute( + insert(models.Branch).values( + adventure_id=adventure.id, + parent_branch_id=parent.id, + fork_depth=fork_depth, + lineage=[], + created_at=models.utcnow(), + ) + ).inserted_primary_key[0] + db.execute( + update(models.Branch) + .where(models.Branch.id == new_id) + .values(lineage=[[new_id, None]] + inherited) + ) + + depth = node.depth + node.branch_id = new_id + node.live = True + + # The group the node left needs a live attempt again. Taking the oldest is + # arbitrary, and it has to be somebody: a coordinate with no live attempt + # disappears from the story on the branch it was left on. + if remaining and not any(row.live for row in remaining): + remaining[0].live = True + + adventure.head_branch_id = new_id + adventure.head_depth = depth + return db.get(models.Branch, new_id) + + +def branch_at( + db: Session, adventure: models.Adventure, fork_depth: int +) -> models.Branch: + """Creates an empty branch that leaves the current path at `fork_depth`. + + `fork` moves an existing node onto its own branch. This function creates the + same kind of branch with no nodes on it yet, for the case where the take + that will live there does not exist. A player asking for another take of a + turn the story has moved past reaches this path (SP9). + + The head lands at `fork_depth`, so the next node written becomes the new + take. That node gets the same depth as the original and the same parent, + which `place_action` derives from the path. + + This function does not modify the branch being left. That branch keeps its + node at that depth, the node stays live, and every turn played after it + stays in place. + """ + parent = head_branch(db, adventure) + inherited = [ + [branch_id, fork_depth if cap is None else min(cap, fork_depth)] + for branch_id, cap in lineage.entries_of(parent) + ] + # Core insert with the lineage written second, for the reason given in + # `root_branch`. This code can run inside a flush, and the lineage refers to + # the new row's own id. + new_id = db.execute( + insert(models.Branch).values( + adventure_id=adventure.id, + parent_branch_id=parent.id, + fork_depth=fork_depth, + lineage=[], + created_at=models.utcnow(), + ) + ).inserted_primary_key[0] + db.execute( + update(models.Branch) + .where(models.Branch.id == new_id) + .values(lineage=[[new_id, None]] + inherited) + ) + adventure.head_branch_id = new_id + adventure.head_depth = fork_depth + return db.get(models.Branch, new_id) + + +def place_action( + db: Session, + adventure: models.Adventure, + action: models.Action, + branch: models.Branch | None = None, + parent: models.Action | None = None, +) -> models.Branch: + """Puts `action` on the head branch and moves the head to it. + + An action with no `depth` of its own goes one step past the tip, which is + where the next turn belongs. The opening of a new adventure lands at 0 that + way, because an adventure with nothing played has a head depth of + `NO_DEPTH`. + + Pass `branch` when you have already resolved the head and are placing + several nodes at once. See `place_new_nodes` for why that is worth doing. + + `parent` is the take that this node was played after (SP9), and it is what + groups the takes of one turn. If you omit it, this function derives it from + the path, which is the correct default: a node written now follows the story + the player is reading now. If you place several nodes in one flush, chain + `parent` explicitly, because the database has not seen any of them yet. + """ + branch = branch or head_branch(db, adventure) + action.branch_id = branch.id + if action.depth is None: + action.depth = adventure.head_depth + 1 + if parent is not None: + action.parent_id = parent.id + elif action.parent_id is None and action.depth: + action.parent_id = _preceding_id(db, adventure, branch, action.depth) + adventure.head_branch_id = branch.id + if action.depth is not None and action.depth > adventure.head_depth: + adventure.head_depth = action.depth + return branch + + +def _preceding_id( + db: Session, adventure: models.Adventure, branch: models.Branch, depth: int +) -> int | None: + """Returns the id of the live node one step back along `branch`'s path. + + The query searches the whole lineage instead of `branch` alone, because a + branch inherits the story before its fork point. The node in front of a + forked branch's first turn lives on an ancestor, and that node is the parent + a pager needs in order to find siblings. + """ + path = lineage.Path(lineage.entries_of(branch)) + return ( + db.query(models.Action.id) + .filter( + models.Action.adventure_id == adventure.id, + models.Action.depth == depth - 1, + models.Action.live.is_(True), + path.clause(), + ) + .order_by(models.Action.id) + .limit(1) + .scalar() + ) + + +def place_memory( + db: Session, + adventure: models.Adventure, + memory: models.Memory, + branch: models.Branch | None = None, +) -> models.Branch: + """Attaches a memory to the node it belongs to. + + `source_end` is the index of the last action the memory summarizes, which is + that node's depth. A hand-written memory summarizes no actions, so it uses + the head instead. That records the story the author was reading at the time. + + Giving every memory a coordinate is what makes its scope unambiguous (SP7). + Before SP7, hand-written memories had a NULL depth and belonged to the + adventure rather than to a path. A fork cannot cap a NULL, so those memories + followed the reader onto branches whose story they did not describe. Now the + question "is this memory part of the story I am reading?" has one answer for + every row in the bank, and it is the same answer the lineage gives for + nodes. + """ + branch = branch or head_branch(db, adventure) + memory.branch_id = branch.id + if memory.depth is None: + memory.depth = ( + memory.source_end if memory.source_end is not None + else adventure.head_depth + ) + return branch + + +def attach_memory(memory: models.Memory, node: models.Action) -> None: + """Attaches a memory to the node it was derived from. + + This follows the general rule for derived data, and the memory bank is the + first case of it. Anything derived from the story attaches to the node that + produced it, and is then visible from exactly the paths that contain that + node. A fork inherits its ancestors' memories because it inherits their + nodes, so nothing is copied. A memory made on a sibling branch is not + visible, because that node is not on this path. + + This function differs from `place_memory` because it takes the branch from + `node`, which is not always the head. A block of story can end before the + fork that created the current branch, and the memory belongs where that + block is. + """ + memory.branch_id = node.branch_id + memory.depth = node.depth + + +def stamp_outcome(adventure: models.Adventure, action: models.Action) -> None: + """Records the state a node left behind, if the writer did not record it. + + This is the fallback under `attempts.snapshot_outcome`, and it exists for + the same reason `place_action` has one. Since SP4, undo and retry cannot + roll back past a node that has no outcome, and the failure is silent: the + script state and world state stay where they were. The turn engine records + the outcome itself, so this function skips those rows. It catches fixtures, + scripts, and imports that write a story directly through the ORM. + + The values written are the state as of the flush. That is correct, because a + writer that changes no state between two nodes leaves the same state behind + both of them. + """ + if action.state_after is None: + state = adventure.script_state if isinstance(adventure.script_state, dict) else {} + action.state_after = copy.deepcopy(state) + if action.world_state_after is None: + world = adventure.world_state if isinstance(adventure.world_state, dict) else {} + action.world_state_after = copy.deepcopy(world) + + +def place_new_nodes(session: Session) -> None: + """Places every unplaced node that is about to be inserted. + + This runs on every flush. Call sites still call `place_action` and + `place_memory` directly, and they should. Placing a node at the call site + happens before the surrounding code reads the row back, and the explicit + call makes that ordering visible. This function is the fallback under those + calls, so a fixture, script, test, or a call site added later still gets a + branch without knowing the tree exists. + + Nodes whose adventure has not been inserted yet are skipped, because there + is no id to attach a branch to. This case does not arise in practice, since + an Action needs `adventure_id` before it can be written at all. + + The head is resolved once per adventure per flush and cached in `heads`. + This saves more than a dictionary lookup. The identity map holds weak + references, so a branch row that nothing else refers to is collected between + two nodes and read from the database again for the next one. Resolving the + head per node turned a fixture that wrote 200 actions in one flush into 200 + separate queries. + """ + heads: dict[int, models.Branch] = {} + for obj in list(session.new): + if isinstance(obj, models.Action): + place = place_action + elif isinstance(obj, models.Memory): + place = place_memory + else: + continue + if obj.adventure_id is None: + continue + adventure = session.get(models.Adventure, obj.adventure_id) + if adventure is None: + continue + if isinstance(obj, models.Action): + stamp_outcome(adventure, obj) + if obj.branch_id is not None: + continue # already placed at its call site + head = heads.get(adventure.id) + if head is None: + head = heads[adventure.id] = head_branch(session, adventure) + place(session, adventure, obj, head) + + +def refresh_head(db: Session, adventure: models.Adventure) -> None: + """Recomputes the head depth after nodes are removed by undo or delete. + + A branch with no nodes of its own sits at its fork point, because that is + the last node its story contains. The node is inherited from the parent, but + it is still the tip. A root branch with no nodes has no story at all. + """ + branch = head_branch(db, adventure) + tip = ( + db.query(func.max(models.Action.depth)) + .filter(models.Action.branch_id == branch.id) + .scalar() + ) + if tip is None: + tip = branch.fork_depth if branch.fork_depth is not None else NO_DEPTH + adventure.head_depth = tip diff --git a/backend/app/vectors.py b/backend/app/vectors.py new file mode 100644 index 0000000..5a7d4f4 --- /dev/null +++ b/backend/app/vectors.py @@ -0,0 +1,52 @@ +"""Storing and comparing embedding vectors. + +A 1536-dimension vector written as a JSON list is about 31 kB, because every +component is written as a decimal string of about seventeen digits. The same +vector as packed float32 is 6,144 bytes, which is five times smaller, and the +memory bank is read in full on every turn, so those bytes are paid repeatedly. + +Float32 is not an approximation here. The embedding endpoints return vectors +computed in float32 and render them into JSON as the shortest decimal string +that round-trips through a double. Converting that back to float32 recovers the +original bits exactly. Nothing that was present is lost, which is why the +conversion needs no re-embedding and carries no risk to retrieval quality. + +Dimensions are deliberately unchanged. Dropping to 512 or 768 would have saved +another 3x and cost an API call per stored memory to re-embed, against a bank +that the packing and the in-process cache together already make cheap. +""" + +import math +import struct +import sys +from array import array + + +def pack(vector) -> bytes: + """A vector as little-endian float32.""" + return struct.pack(f"<{len(vector)}f", *vector) + + +def unpack(blob: bytes) -> array: + """The inverse of `pack`. Length is implied: four bytes per component. + + Returns an `array("f")` rather than a list, because these are held in + memory between turns: the array is the same 4 bytes a component the column + is, where a list of Python floats is eight times that. It indexes, zips and + lens like a list, which is all the ranking needs. + """ + vector = array("f") + vector.frombytes(blob) + if sys.byteorder != "little": + vector.byteswap() + return vector + + +def cosine(a: list[float], b: list[float]) -> float: + # Different lengths means the embedding model changed since this vector was + # stored; zip() would silently score garbage. + if len(a) != len(b): + return 0.0 + dot = sum(x * y for x, y in zip(a, b)) + norm = math.sqrt(sum(x * x for x in a)) * math.sqrt(sum(y * y for y in b)) + return dot / norm if norm else 0.0 diff --git a/backend/app/worldstate/__init__.py b/backend/app/worldstate/__init__.py new file mode 100644 index 0000000..ff4bc5c --- /dev/null +++ b/backend/app/worldstate/__init__.py @@ -0,0 +1,55 @@ +"""Phase 12: RPG world state. + +The scenario carries a `stat_schema`, which is the template: which stats exist, +their bands and rules, and the milestones. An adventure carries a live +`world_state` instantiated from it. Each turn the AI proposes a delta holding +only what changed. `apply_delta` decides what the delta is allowed to do. It +clamps values to min and max, caps the change per turn, enforces cooldowns, and +makes milestones sticky. + +Nothing here raises on bad AI output. A malformed delta returns `{}` and the turn +continues, the same way a broken script never breaks a turn. + +The work is split four ways, and each module reads on its own: + + schema what a scenario's `stat_schema` allows, and building state from it + parse the block the model writes, and reading it back + apply every write to world state, and the limits it is held to + render turning state and schema into prompt text + +Import the names from this package rather than from those modules. The split is +an implementation detail, and the names below are the interface. +""" + +from .apply import apply_delta, apply_override +from .parse import ( + EMIT_REMINDER, + EMIT_RULE, + applied_delta, + extract_delta, + refusals, + render_delta_block, + render_refusals, +) +from .render import render_reference, render_state_section +from .schema import band_label, has_schema, instantiate, npc_name, npc_triggers, reconcile + +__all__ = [ + "EMIT_REMINDER", + "EMIT_RULE", + "applied_delta", + "apply_delta", + "apply_override", + "band_label", + "extract_delta", + "has_schema", + "instantiate", + "npc_name", + "npc_triggers", + "reconcile", + "refusals", + "render_delta_block", + "render_reference", + "render_refusals", + "render_state_section", +] diff --git a/backend/app/worldstate/apply.py b/backend/app/worldstate/apply.py new file mode 100644 index 0000000..336ab7a --- /dev/null +++ b/backend/app/worldstate/apply.py @@ -0,0 +1,407 @@ +"""Applying a delta or an override to world state, within the schema's limits. + +Every write to world state goes through here, so the clamping rules live in one +place. A proposed change that breaks a limit is recorded as refused rather than +dropped, because the player and the model both need to see that it did not land. +""" +import copy +from typing import NamedTuple + +from .schema import STAT_SECTIONS, _initials, instantiate, npc_name + + +class _Target(NamedTuple): + """Where one path writes, and what the schema says about the value there.""" + + kind: str # "flag", "milestone", or "stat" + section: str # "flags", "milestones", "player", "world", or "npc" + key: str # the key inside the container + stat_def: dict | None # the stat's definition, or None for a flag or milestone + npc_id: str | None # the character, for `npc..`, else None + npc_stats: dict | None # that character's stat defs, used to build a new block + + def container(self, ws: dict) -> dict: + """Returns the dict this path writes into, and creates it if it is missing. + + Call this only when you are about to write. Creating the container is a + side effect, and a rejected path must not leave an empty section behind. + """ + if self.npc_id is None: + return ws.setdefault(self.section, {}) + return ws.setdefault("npc", {}).setdefault(self.npc_id, _initials(self.npc_stats)) + + +def _resolve(path: str, stat_schema: dict) -> tuple[_Target | None, dict | None]: + """Routes one path to the value it names, without deciding what happens to it. + + Returns `(target, None)` when the schema defines the path, and + `(None, rejection)` when it does not. The rejection is the entry that goes + into a report's `rejected` list, with its reason and its fix already worded. + + `apply_delta` and `apply_override` route paths identically and differ only in + what they write, so the routing lives here and each function keeps its own + write rule. + """ + parts = path.split(".") + + # `flags.` is a boolean the scenario declares. + if parts[0] == "flags" and len(parts) == 2: + flag_defs = stat_schema.get("flags") or {} + if parts[1] not in flag_defs: + return None, { + "path": path, "reason": "unknown flag", + "fix": f"There is no flag `{parts[1]}`. {_names_phrase('flag', flag_defs)}", + } + return _Target("flag", "flags", parts[1], None, None, None), None + + # `milestones.` records that a milestone is reached. + if parts[0] == "milestones" and len(parts) == 2: + milestones = stat_schema.get("milestones") or {} + if parts[1] not in milestones: + return None, { + "path": path, "reason": "unknown milestone", + "fix": f"There is no milestone `{parts[1]}`. " + f"{_names_phrase('milestone', milestones)}", + } + return _Target("milestone", "milestones", parts[1], None, None, None), None + + # `world.` and `player.`. + if parts[0] in STAT_SECTIONS and len(parts) == 2: + stat_defs = stat_schema.get(parts[0]) or {} + stat_def = stat_defs.get(parts[1]) + if not isinstance(stat_def, dict): + return None, { + "path": path, "reason": "unknown stat", + "fix": f"`{parts[0]}` has no stat `{parts[1]}`. " + f"{_names_phrase('stat', stat_defs)}", + } + return _Target("stat", parts[0], parts[1], stat_def, None, None), None + + # `npc..`. Each character carries its own stat definitions. + if parts[0] == "npc" and len(parts) == 3: + npcs = stat_schema.get("npcs") or {} + ndef = npcs.get(parts[1]) + if not isinstance(ndef, dict): + return None, { + "path": path, "reason": "unknown npc", + "fix": f"There is no character `{parts[1]}`. " + f"{_names_phrase('character', npcs)}", + } + stat_defs = ndef.get("stats") or {} + stat_def = stat_defs.get(parts[2]) + if not isinstance(stat_def, dict): + return None, { + "path": path, "reason": "unknown npc stat", + "fix": f"`{npc_name(ndef, parts[1])}` has no stat `{parts[2]}`. " + f"{_names_phrase('stat', stat_defs)}", + } + return _Target("stat", "npc", parts[2], stat_def, parts[1], stat_defs), None + + return None, { + "path": path, "reason": "unknown path", + "fix": f"`{path}` is not a tracked value. Use player., " + f"world., npc.., flags. or milestones..", + } + + +def _coerce_number(value): + if isinstance(value, bool): # `bool` is an `int` subclass, so reject it here. + return None + if isinstance(value, (int, float)): + return value + if isinstance(value, str): + try: + return float(value.strip()) + except ValueError: + return None + return None + + +def _names_phrase(kind: str, defs: dict, limit: int = 12) -> str: + """Lists what the model could have written instead, for a wrong name. + + A rejection that only says a name is unknown leaves the model guessing + again. Naming the alternatives turns it into a correction it can act on. + """ + names = [k for k in (defs or {}) if isinstance(k, str)] + if not names: + return f"This scenario tracks no {kind}." + shown = ", ".join(f"`{n}`" for n in names[:limit]) + more = f", and {len(names) - limit} more" if len(names) > limit else "" + plural = f"{kind}s" if not kind.endswith("s") else kind + return f"The {plural} are: {shown}{more}." + + +def _limits_phrase(stat_def: dict) -> str: + """Names a stat's numeric limits, for a correction sent back to the model.""" + lo, hi = stat_def.get("min"), stat_def.get("max") + cap = stat_def.get("max_delta_per_turn") + bits = [] + if isinstance(lo, (int, float)) and isinstance(hi, (int, float)): + bits.append(f"it runs from {lo} to {hi}") + elif isinstance(lo, (int, float)): + bits.append(f"it never goes below {lo}") + elif isinstance(hi, (int, float)): + bits.append(f"it never goes above {hi}") + if isinstance(cap, (int, float)): + bits.append(f"it moves at most {cap} per turn") + return "; ".join(bits) + + +def _apply_stat(container: dict, key: str, stat_def: dict, change, + path: str, action_index: int, meta: dict, report: dict) -> None: + delta = _coerce_number(change) + if delta is None: + report["rejected"].append({ + "path": path, "reason": "not a number", + "fix": f"`{path}` takes a number, written as a change such as -5 or 8.", + }) + return + + cooldown = stat_def.get("cooldown") or 0 + last = meta["last_changed"].get(path) + if cooldown and last is not None and action_index - last < cooldown: + waited = action_index - last + report["rejected"].append({ + "path": path, "reason": "cooldown", + "fix": f"`{path}` changed {waited} turn(s) ago and cannot change again " + f"until {cooldown} turns have passed.", + }) + return + + if stat_def.get("type") == "counter" and delta < 0: + report["rejected"].append({ + "path": path, "reason": "counter can't decrease", + "fix": f"`{path}` only counts up. Send a positive change such as 1, " + f"never a negative and never the running total.", + }) + return + + clamped = False + cap = stat_def.get("max_delta_per_turn") + if cap is not None and abs(delta) > cap: + delta = cap if delta > 0 else -cap + clamped = True + + old = container.get(key, stat_def.get("initial", 0)) + new = old + delta + lo, hi = stat_def.get("min"), stat_def.get("max") + if lo is not None and new < lo: + new, clamped = lo, True + if hi is not None and new > hi: + new, clamped = hi, True + # Keep ints integral for display. + if isinstance(old, int) and float(new).is_integer(): + new = int(new) + + container[key] = new + meta["last_changed"][path] = action_index + entry = {"path": path, "old": old, "new": new} + if clamped and new == old: + # The clamp cancelled the change. Nothing moved, so the model needs the + # same correction a rejection gets: without it the only evidence is a + # value that stayed put, which reads as the change never being asked for. + edge = "maximum" if hi is not None and new == hi else "minimum" + entry["fix"] = ( + f"`{path}` did not move. It is already at its {edge} of {new}" + + (f" ({_limits_phrase(stat_def)})." if _limits_phrase(stat_def) else ".") + ) + report["applied"].append(entry) + if clamped: + report["clamped"].append(entry) + + +def _apply_text_stat(container: dict, key: str, stat_def: dict, change, + path: str, action_index: int, meta: dict, report: dict) -> None: + """Applies a free-text stat, which replaces rather than adds. + + The AI sends the new value in full rather than a delta. No clamping and no + bands apply. Only an optional cooldown and an optional `max_length` + truncation apply. + """ + if not isinstance(change, str): + report["rejected"].append({ + "path": path, "reason": "not a string", + "fix": f"`{path}` holds text. Send its new value in full, not a number " + f"and not a change.", + }) + return + + cooldown = stat_def.get("cooldown") or 0 + last = meta["last_changed"].get(path) + if cooldown and last is not None and action_index - last < cooldown: + waited = action_index - last + report["rejected"].append({ + "path": path, "reason": "cooldown", + "fix": f"`{path}` changed {waited} turn(s) ago and cannot change again " + f"until {cooldown} turns have passed.", + }) + return + + new = change.strip() + max_len = stat_def.get("max_length") + if isinstance(max_len, int) and max_len > 0 and len(new) > max_len: + new = new[:max_len] + + old = container.get(key, stat_def.get("initial", "")) + if new == old: + return # Nothing changed, so do nothing. + + container[key] = new + meta["last_changed"][path] = action_index + report["applied"].append({"path": path, "old": old, "new": new}) + + +def apply_override(world_state: dict, stat_schema: dict, overrides: dict) -> tuple[dict, dict]: + """Sets live values directly, as a manual author edit rather than an AI turn. + + This differs from `apply_delta` in three ways. Numeric stats are set rather + than added to. `cooldown`, `max_delta_per_turn`, and the rule that a counter + cannot decrease are all ignored, because this is a deliberate correction + rather than an AI move to check. Milestones can be toggled in both + directions rather than only marked reached. + + Values are still validated against the schema, so an unknown path or a wrong + type is rejected, and numeric values still clamp to min and max. + """ + ws = copy.deepcopy(world_state) if isinstance(world_state, dict) else {} + if not ws: + ws = instantiate(stat_schema) + report: dict = {"applied": [], "rejected": []} + + if not isinstance(overrides, dict): + return ws, report + + def set_stat(container: dict, key: str, stat_def: dict, value, path: str) -> None: + if stat_def.get("type") == "text": + if not isinstance(value, str): + report["rejected"].append({"path": path, "reason": "not a string"}) + return + new = value.strip() + max_len = stat_def.get("max_length") + if isinstance(max_len, int) and max_len > 0 and len(new) > max_len: + new = new[:max_len] + old = container.get(key, stat_def.get("initial", "")) + container[key] = new + report["applied"].append({"path": path, "old": old, "new": new}) + return + + num = _coerce_number(value) + if num is None: + report["rejected"].append({"path": path, "reason": "not a number"}) + return + old = container.get(key, stat_def.get("initial", 0)) + lo, hi = stat_def.get("min"), stat_def.get("max") + if lo is not None and num < lo: + num = lo + if hi is not None and num > hi: + num = hi + if isinstance(old, int) and float(num).is_integer(): + num = int(num) + container[key] = num + report["applied"].append({"path": path, "old": old, "new": num}) + + for raw_path, value in overrides.items(): + path = str(raw_path) + target, rejection = _resolve(path, stat_schema) + if rejection is not None: + report["rejected"].append(rejection) + continue + + if target.kind == "flag": + if not isinstance(value, bool): + report["rejected"].append({ + "path": path, "reason": "not a boolean", + "fix": f"`{path}` takes true or false.", + }) + continue + flags = target.container(ws) + old = bool(flags.get(target.key, False)) + flags[target.key] = value + report["applied"].append({"path": path, "old": old, "new": value}) + + elif target.kind == "milestone": + # An override toggles a milestone in both directions, so unlike a + # delta it takes false as well as true. + if not isinstance(value, bool): + report["rejected"].append({ + "path": path, "reason": "not a boolean", + "fix": f"`{path}` takes true or false.", + }) + continue + reached = target.container(ws) + old = bool(reached.get(target.key, {}).get("reached")) + if value: + reached[target.key] = {"reached": True} + else: + reached.pop(target.key, None) + report["applied"].append({"path": path, "old": old, "new": value}) + + else: + set_stat(target.container(ws), target.key, target.stat_def, value, path) + + return ws, report + + +def apply_delta(world_state: dict, stat_schema: dict, delta: dict, + action_index: int) -> tuple[dict, dict]: + """Validates and clamps `delta` against `stat_schema`, then applies it. + + The delta is applied to a copy of `world_state`. The return value is + `(new_world_state, report)`. + """ + ws = copy.deepcopy(world_state) if isinstance(world_state, dict) else {} + if not ws: + ws = instantiate(stat_schema) + ws.setdefault("_meta", {}).setdefault("last_changed", {}) + meta = ws["_meta"] + report: dict = {"applied": [], "clamped": [], "rejected": []} + + if not isinstance(delta, dict): + return ws, report + + for raw_path, change in delta.items(): + path = str(raw_path) + target, rejection = _resolve(path, stat_schema) + if rejection is not None: + report["rejected"].append(rejection) + continue + + if target.kind == "flag": + # A flag goes both ways, so either value is accepted. + if not isinstance(change, bool): + report["rejected"].append({ + "path": path, "reason": "not a boolean", + "fix": f"`{path}` takes true or false.", + }) + continue + flags = target.container(ws) + old = bool(flags.get(target.key, False)) + if change != old: + flags[target.key] = change + report["applied"].append({"path": path, "old": old, "new": change}) + + elif target.kind == "milestone": + # A milestone is sticky, so only true is accepted. + if change is not True: + report["rejected"].append({ + "path": path, "reason": "not true", + "fix": f"`{path}` can only be set to true. A milestone is " + f"reached once and never taken back.", + }) + continue + reached = target.container(ws) + if reached.get(target.key, {}).get("reached"): + continue # Already reached, so do nothing. + reached[target.key] = {"reached": True, "at": action_index} + report["applied"].append({"path": path, "old": False, "new": True}) + + elif target.stat_def.get("type") == "text": + _apply_text_stat(target.container(ws), target.key, target.stat_def, change, + path, action_index, meta, report) + + else: + _apply_stat(target.container(ws), target.key, target.stat_def, change, + path, action_index, meta, report) + + return ws, report diff --git a/backend/app/worldstate/parse.py b/backend/app/worldstate/parse.py new file mode 100644 index 0000000..6c91e75 --- /dev/null +++ b/backend/app/worldstate/parse.py @@ -0,0 +1,165 @@ +"""The delta block: the rule the model is given, and reading back what it sent. + +The model writes one fenced block per turn. This module holds the instruction +text, the parser that survives the ways a model gets the format wrong, and the +report rendering that tells a player which changes were refused. +""" +import json +import re + + +# Appended once to the system prompt, so the model knows how to report +# changes. +EMIT_RULE = ( + "You maintain a numeric world state. Treat your own narration as authoritative: " + "whenever what you write implies a change to any tracked value — health or resources " + "going up or down, time passing, a relationship or mood shifting, a status turning on " + "or off, progress toward a goal, an item or piece of information gained or lost — you " + "MUST record it, including the numbers, not just on/off flags. After your narration, " + "append a fenced code block labelled `state` with a JSON object of the CHANGES ONLY, " + "as deltas (not new totals). Update every value the scene affected this turn, not only " + "the obvious ones. Read the range and band labels shown for each stat and keep every " + "change proportionate to the moment: an ordinary or minor event nudges a value slightly, " + "while a large change — or reaching a stat's minimum or maximum — is reserved for a " + "genuinely pivotal, defining moment (a passing remark shifts a relationship a little; a " + "lasting act of loyalty or betrayal shifts it a lot). Do not move a value across most of " + 'its range in a single ordinary turn. Every stat is listed by its exact path in the ' + 'stat guide and again beside its live value — copy a path from there rather than ' + 'building one out of a name. The shapes are "player.", ' + '"world.", "npc.." (use the id in parentheses, e.g. npc.gwen.trust, ' + 'not the display name); "flags.": true or false to toggle an on/off state; and ' + '"milestones.": true when an objective is completed. Some stats marked (free text) in ' + "the stat guide hold a short string instead of a number — for those, send the new value " + "in full (not a delta), e.g. what the player is now wearing or holding; only send it when " + "it actually changed. Send only things that actually " + "changed and never restate unchanged values; if truly nothing changed, omit the block. " + "Example:\n" + '```state\n{"player.hp": -15, "npc.gwen.trust": 5, "milestones.escaped": true, ' + '"player.outfit": "torn traveling cloak"}\n```' +) + + +# A short reminder placed at the end of the prompt, which is the strongest +# recency position, so the emit rule is close to where the model generates. +EMIT_REMINDER = ( + "[Reminder: end your reply with a ```state block of the changes this turn " + "(deltas only), or omit it if truly nothing changed.]" +) + + +def render_delta_block(delta: dict) -> str: + """Renders a stored delta back into the fenced `state` block the AI emitted. + + The caller replays past turns into the context with this, so the model copies + the format. An empty delta returns an empty string, which means the turn + changed nothing. + """ + if not isinstance(delta, dict) or not delta: + return "" + return "```state\n" + json.dumps(delta, ensure_ascii=False) + "\n```" + + +def applied_delta(world_delta: dict | None) -> dict: + """Returns the changes the engine accepted, shaped as the AI sends them. + + Replaying the delta the AI sent would show it a refused change standing as + though it had been applied, contradicted by the live values in the same + prompt. The model has no way to read that as a correction, so it repeats + the change. Replaying what was accepted removes the contradiction. + + A numeric change that ended where it started is omitted, because it moved + nothing and a zero in the replayed block reads as a value worth sending. + """ + if not isinstance(world_delta, dict): + return {} + out: dict = {} + for entry in world_delta.get("applied") or []: + if not isinstance(entry, dict): + continue + path = str(entry.get("path", "")) + old, new = entry.get("old"), entry.get("new") + if path.startswith("milestones."): + out[path] = True + elif path.startswith("flags."): + out[path] = bool(new) + elif isinstance(old, (int, float)) and isinstance(new, (int, float)): + if new != old: + out[path] = new - old + else: + out[path] = new + return out + + +def refusals(world_delta: dict | None) -> list[str]: + """Returns a correction line for each change the engine did not carry out. + + Covers the changes that were lost: a rejection, and a clamp that left the + value where it started. A clamp that reduced a change but still moved the + value is deliberately absent. The model's intent landed in that case, and + reporting the shortfall invites it to send the remainder on the next turn, + which is the swing `max_delta_per_turn` exists to prevent. + """ + if not isinstance(world_delta, dict): + return [] + lines = [] + for entry in world_delta.get("rejected") or []: + if isinstance(entry, dict) and entry.get("fix"): + lines.append(str(entry["fix"])) + for entry in world_delta.get("clamped") or []: + if isinstance(entry, dict) and entry.get("fix"): + lines.append(str(entry["fix"])) + return lines + + +def render_refusals(world_delta: dict | None) -> str: + """Renders `refusals` as the note appended after the most recent AI turn.""" + lines = refusals(world_delta) + if not lines: + return "" + body = "\n".join(f"- {ln}" for ln in lines) + return ("[Part of your last state block was not applied. Correct it in this " + f"turn's block:\n{body}]") + + +# ```state { ... } ``` (also tolerates ```json or an unlabelled fence); DOTALL. +_FENCE_RE = re.compile(r"```(?:state|json)?\s*(\{.*?\})\s*```", re.DOTALL | re.IGNORECASE) + + +# Fallback: a bare JSON object hugging the end of the text. +_TRAILING_RE = re.compile(r"(\{[^{}]*\})\s*$", re.DOTALL) + + +def _tolerant_load(blob: str) -> dict: + # Strip trailing commas and leading + on numbers, both of which weaker + # free models emit and strict JSON rejects. + cleaned = re.sub(r",(\s*[}\]])", r"\1", blob) + cleaned = re.sub(r"(:\s*)\+(\d)", r"\1\2", cleaned) + try: + parsed = json.loads(cleaned) + except (json.JSONDecodeError, ValueError): + return {} + return parsed if isinstance(parsed, dict) else {} + + +def extract_delta(text: str) -> tuple[str, dict]: + """Removes the trailing state block from an AI response. + + The return value is `(clean_text, delta)`. `delta` is `{}` when there is no + block or the block cannot be parsed, and `clean_text` has the block removed. + A bare trailing object is stripped only when it parses to a delta, so + ordinary prose that ends in `}` is left alone. + """ + matches = list(_FENCE_RE.finditer(text)) + if matches: + m = matches[-1] + delta = _tolerant_load(m.group(1)) + clean = (text[: m.start()] + text[m.end():]).strip() + return clean, delta + + m = _TRAILING_RE.search(text) + if m: + delta = _tolerant_load(m.group(1)) + if delta and all("." in str(k) for k in delta): + clean = text[: m.start()].strip() + return clean, delta + return text.strip(), {} diff --git a/backend/app/worldstate/render.py b/backend/app/worldstate/render.py new file mode 100644 index 0000000..708970c --- /dev/null +++ b/backend/app/worldstate/render.py @@ -0,0 +1,187 @@ +"""Turning world state into the text the model and the player read. + +`render_state_section` writes the current state into the prompt. +`render_reference` writes the schema itself, so the model knows which stats +exist and what they are called. +""" +from .schema import STAT_SECTIONS, _initials, band_label, npc_name + + +def _stat_line(defs: dict, values: dict) -> str: + parts = [] + for name, d in defs.items(): + if not isinstance(d, dict): + continue + if d.get("type") == "text": + val = values.get(name, d.get("initial", "")) + parts.append(f'{name} "{val}"' if val else f"{name} (unset)") + continue + val = values.get(name, d.get("initial", 0)) + hi = d.get("max") + shown = f"{val}/{hi}" if hi is not None else f"{val}" + label = band_label(d, val) + parts.append(f"{name} {shown}" + (f" ({label})" if label else "")) + return ", ".join(parts) + + +def render_state_section(world_state: dict, stat_schema: dict, + visible_npcs: dict[str, str], + player_name: str = "") -> str: + """Returns the compact context block that every turn includes. + + `visible_npcs` maps a card id to a display name, for the NPCs currently in + the scene. + + `player_name` is the persona's name (Phase 18), used to label the player's + stat block. It is empty for an adventure with no persona, and the block then + reads `You:` as it did before personas existed. + """ + ws = world_state if isinstance(world_state, dict) else {} + lines: list[str] = [] + + world_defs = stat_schema.get("world") or {} + world_line = _stat_line(world_defs, ws.get("world") or {}) + header = "World state" + (f" — {world_line}." if world_line else ".") + lines.append(header) + + player_defs = stat_schema.get("player") or {} + player_line = _stat_line(player_defs, ws.get("player") or {}) + if player_line: + # Name the block after the persona, and show the path beside it exactly + # as the NPC lines below do. The model reads the name in the narration, + # so without the path in view it writes `kaelen.hp` and the delta is + # refused as an unknown path. + label = f"{player_name} (player)" if player_name else "You" + lines.append(f"{label}: {player_line}.") + + npcs = stat_schema.get("npcs") or {} + npc_state = ws.get("npc") or {} + for npc_key, name in visible_npcs.items(): + ndef = npcs.get(npc_key) or {} + stat_defs = ndef.get("stats") or {} + values = npc_state.get(npc_key) or _initials(stat_defs) + npc_line = _stat_line(stat_defs, values) + if npc_line: + # Show the id so the AI can address it as npc... + lines.append(f"{name} (npc.{npc_key}): {npc_line}.") + + flag_defs = stat_schema.get("flags") or {} + flag_state = ws.get("flags") or {} + flag_parts = [ + f"{name} {'yes' if flag_state.get(name, bool(d.get('initial', False))) else 'no'}" + for name, d in flag_defs.items() if isinstance(d, dict) + ] + if flag_parts: + lines.append("Flags: " + ", ".join(flag_parts) + ".") + + # Show the id beside each goal, the same as NPCs and flags. The AI marks a + # milestone as `milestones.`, and `apply_delta` rejects an id the schema + # does not define, so a goal listed by description alone gives the model no + # way to name it and it can only guess. + milestones = stat_schema.get("milestones") or {} + reached = ws.get("milestones") or {} + goals = [f"{mid} — {d.get('desc', mid)}" for mid, d in milestones.items() + if not reached.get(mid, {}).get("reached")] + done = [f"{mid} — {d.get('desc', mid)}" for mid, d in milestones.items() + if reached.get(mid, {}).get("reached")] + if goals: + lines.append("Goals (mark with milestones.): " + "; ".join(goals) + ".") + if done: + lines.append("Achieved: " + "; ".join(done) + ".") + + return "\n".join(lines) + + +def _describe_stat(name: str, d: dict) -> str | None: + """Returns one reference line for a stat, named by the path the AI writes. + + `name` is that path. The guide is the model's only complete list of what + exists, so a stat named any other way leaves it to work the path out from + the live values, and those cover only what is on screen this turn. + + The description and the band ladder are independent, and each is included + only when present, so a stat may have either, both, or neither. + """ + is_text = d.get("type") == "text" + # `EMIT_RULE` sends the model here to find out which stats take a whole + # value instead of a change, and it names this marker, so the two have to + # be written the same way. + label = f"{name} (free text)" if is_text else name + bits: list[str] = [] + desc = d.get("desc") + if isinstance(desc, str) and desc.strip(): + # Fragments are joined with "; " and end with a single ".", so remove + # any trailing period the author put on the description. + bits.append(desc.strip().rstrip(".")) + if is_text: + # Free text has no range and no bands, so the marker is the whole + # entry. It still earns a line without a description, because the + # marker is what stops the model sending a delta. + return f"{label} — {'; '.join(bits)}." if bits else f"{label}." + lo, hi = d.get("min"), d.get("max") + if isinstance(lo, (int, float)) and isinstance(hi, (int, float)): + bits.append(f"range {lo}–{hi}") + bands = d.get("bands") + if isinstance(bands, list) and bands: + ladder = ", ".join( + f"{b[0]}–{b[1]} {b[2]}" + for b in bands if isinstance(b, list) and len(b) == 3 + ) + if ladder: + bits.append(f"bands: {ladder}") + return f"{label} — {'; '.join(bits)}." if bits else None + + +def render_reference(stat_schema: dict, player_name: str = "") -> str: + """Returns a fixed, per-scenario legend for the stats. + + Each line gives what a stat means, from its `desc`, and its band ladder. The + legend does not change from turn to turn, and it is separate from the live + values. + + `player_name` names the persona, so that the guide ties the name the model + reads in the story to the `player.` paths it has to write. The persona's + description is deliberately not repeated here: it has its own section, and + this guide is about paths. + """ + lines: list[str] = [] + if player_name: + lines.append( + f"Protagonist {player_name} — the player character; their stats are " + f"addressed as player.." + ) + for section in STAT_SECTIONS: + for name, d in (stat_schema.get(section) or {}).items(): + if isinstance(d, dict): + row = _describe_stat(f"{section}.{name}", d) + if row: + lines.append(row) + for npc_key, ndef in (stat_schema.get("npcs") or {}).items(): + if not isinstance(ndef, dict): + continue + name = npc_name(ndef, npc_key) + desc = ndef.get("desc") + # The header is written even for an NPC with no description, because it + # is the only line that ties a display name to the id the AI has to + # address. The live values state it too, but only for the NPCs a scene + # has mentioned, so without this an NPC off screen can only be guessed + # at — and a guess is refused as a character or a stat that does not + # exist. + head = f"NPC {name} (npc.{npc_key})" + lines.append( + f"{head} — {desc.strip().rstrip('.')}." + if isinstance(desc, str) and desc.strip() else f"{head}." + ) + for sname, sdef in (ndef.get("stats") or {}).items(): + if isinstance(sdef, dict): + row = _describe_stat(f"npc.{npc_key}.{sname}", sdef) + if row: + lines.append(row) + for name, d in (stat_schema.get("flags") or {}).items(): + if isinstance(d, dict): + desc = d.get("desc") + if isinstance(desc, str) and desc.strip(): + lines.append(f"flags.{name} — {desc.strip().rstrip('.')}.") + if not lines: + return "" + return "Stat guide (fixed reference):\n" + "\n".join(f"- {ln}" for ln in lines) diff --git a/backend/app/worldstate/schema.py b/backend/app/worldstate/schema.py new file mode 100644 index 0000000..f8b3a4c --- /dev/null +++ b/backend/app/worldstate/schema.py @@ -0,0 +1,198 @@ +"""Reading a scenario's `stat_schema` and building state that matches it. + +Nothing here decides what a turn changes. These functions answer what the schema +allows, what a fresh state looks like, and how an edited schema maps onto state +that already exists. +""" +import copy + + +# The `stat_schema` top-level sections that hold stat definitions. +STAT_SECTIONS = ("world", "player") + + +def has_schema(stat_schema: dict | None) -> bool: + """Returns `True` when a scenario defines an RPG layer.""" + if not isinstance(stat_schema, dict): + return False + return any( + isinstance(stat_schema.get(k), dict) and stat_schema[k] + for k in (*STAT_SECTIONS, "npcs", "milestones", "flags") + ) + + +def npc_name(ndef: dict, key: str) -> str: + name = ndef.get("name") + return name.strip() if isinstance(name, str) and name.strip() else key + + +def npc_triggers(ndef: dict, key: str) -> list[str]: + """Returns the lowercased trigger words that detect an NPC in a scene. + + The words come from the NPC's `keys` field, or from its display name when + `keys` is empty. + """ + raw = ndef.get("keys") or npc_name(ndef, key) + return [k.strip().lower() for k in str(raw).split(",") if k.strip()] + + +def _initials(defs: dict) -> dict: + return { + name: d.get("initial", "" if d.get("type") == "text" else 0) + for name, d in defs.items() + if isinstance(d, dict) + } + + +def instantiate(stat_schema: dict | None) -> dict: + """Builds a fresh live `world_state` from a schema, using initial values only.""" + if not has_schema(stat_schema): + return {} + ws: dict = {} + for section in STAT_SECTIONS: + ws[section] = _initials(stat_schema.get(section) or {}) + # Each defined NPC gets its own stat block from its own `stats` defs. + ws["npc"] = { + key: _initials(ndef.get("stats") or {}) + for key, ndef in (stat_schema.get("npcs") or {}).items() + if isinstance(ndef, dict) + } + ws["milestones"] = {} # only reached ones are stored + ws["flags"] = { + name: bool(d.get("initial", False)) + for name, d in (stat_schema.get("flags") or {}).items() + if isinstance(d, dict) + } + ws["_meta"] = {"last_changed": {}} + return ws + + +def reconcile(world_state: dict | None, stat_schema: dict | None) -> tuple[dict, dict]: + """Brings a live `world_state` back in line with an edited schema. + + This is not `instantiate`. A value the schema still defines keeps whatever it + reached in play, because re-instantiating would restore the player to full + health and clear their milestones. Only the difference is applied. Stats, + NPCs, flags, and milestones the schema gained appear at their initial value, + and ones it no longer defines are removed along with their `last_changed` + bookkeeping. The return value is `(new_state, report)`, where the report + lists paths under `added` and `removed`, so the UI can show what a refresh + would do. + + The additions are mostly cosmetic. Rendering and delta application both fall + back to a stat definition's `initial` when the live state has no value for + it, so a newly added stat already behaves correctly. This function stores the + value, and unlike those read-through paths it also removes what the schema + dropped. + """ + report: dict = {"added": [], "removed": []} + if not has_schema(stat_schema): + # The scenario dropped its RPG layer, so the adventure drops it too. + stale = bool(world_state) + if stale: + report["removed"].append("(all world state)") + return {}, report + + ws = copy.deepcopy(world_state) if isinstance(world_state, dict) else {} + if not ws: + return instantiate(stat_schema), report + + def sync_section(container: dict, defs: dict, prefix: str) -> dict: + out = {} + for name, d in defs.items(): + if not isinstance(d, dict): + continue + if name in container: + out[name] = container[name] + else: + out[name] = d.get("initial", "" if d.get("type") == "text" else 0) + report["added"].append(f"{prefix}.{name}") + for name in container: + if name not in out: + report["removed"].append(f"{prefix}.{name}") + return out + + for section in STAT_SECTIONS: + ws[section] = sync_section( + ws.get(section) if isinstance(ws.get(section), dict) else {}, + stat_schema.get(section) or {}, + section, + ) + + npc_defs = stat_schema.get("npcs") or {} + old_npcs = ws.get("npc") if isinstance(ws.get("npc"), dict) else {} + new_npcs = {} + for key, ndef in npc_defs.items(): + if not isinstance(ndef, dict): + continue + old = old_npcs.get(key) if isinstance(old_npcs.get(key), dict) else {} + new_npcs[key] = sync_section(old, ndef.get("stats") or {}, f"npc.{key}") + for key in old_npcs: + if key not in new_npcs: + report["removed"].append(f"npc.{key}") + ws["npc"] = new_npcs + + flag_defs = stat_schema.get("flags") or {} + old_flags = ws.get("flags") if isinstance(ws.get("flags"), dict) else {} + new_flags = {} + for name, d in flag_defs.items(): + if not isinstance(d, dict): + continue + if name in old_flags: + new_flags[name] = bool(old_flags[name]) + else: + new_flags[name] = bool(d.get("initial", False)) + report["added"].append(f"flags.{name}") + for name in old_flags: + if name not in new_flags: + report["removed"].append(f"flags.{name}") + ws["flags"] = new_flags + + # Milestones store only the ones reached, so there is nothing to add here. + # An unreached milestone is absent. Drop reached milestones the scenario no + # longer defines, because they would otherwise stay in "Achieved" with no + # label. + milestone_defs = stat_schema.get("milestones") or {} + reached = ws.get("milestones") if isinstance(ws.get("milestones"), dict) else {} + ws["milestones"] = {k: v for k, v in reached.items() if k in milestone_defs} + for k in reached: + if k not in milestone_defs: + report["removed"].append(f"milestones.{k}") + + # Cooldown bookkeeping for paths that no longer exist is never read, but it + # accumulates in every stored snapshot, so remove it with the rest. + meta = ws.setdefault("_meta", {}) + last_changed = meta.get("last_changed") + if isinstance(last_changed, dict): + removed = set(report["removed"]) + meta["last_changed"] = { + path: at for path, at in last_changed.items() + if not any(path == r or path.startswith(f"{r}.") for r in removed) + } + else: + meta["last_changed"] = {} + + return ws, report + + +def band_label(stat_def: dict, value) -> str | None: + """Returns the word label for `value` from a stat definition's bands, if any. + + A band is `[lo, hi, label]` and matches when `lo <= value < hi`. The top band + includes its upper bound, so a stat at its maximum still gets a label. + """ + bands = stat_def.get("bands") + if not isinstance(bands, list) or not isinstance(value, (int, float)): + return None + last_hi = None + for band in bands: + if not (isinstance(band, list) and len(band) == 3): + continue + lo, hi, label = band + last_hi = hi + if lo <= value < hi: + return str(label) + # Inclusive top edge. + if bands and value == last_hi: + return str(bands[-1][2]) + return None diff --git a/backend/requirements-dev.txt b/backend/requirements-dev.txt new file mode 100644 index 0000000..6645938 --- /dev/null +++ b/backend/requirements-dev.txt @@ -0,0 +1,4 @@ +# Dev/test-only dependencies (not installed on the Render deploy). +# pip install -r requirements.txt -r requirements-dev.txt +# python -m pytest tests/ +pytest>=8.0 diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 0000000..fba864e --- /dev/null +++ b/backend/requirements.txt @@ -0,0 +1,9 @@ +fastapi>=0.115 +uvicorn[standard]>=0.30 +sqlalchemy>=2.0 +pydantic>=2.7 +httpx>=0.27 +tiktoken>=0.7 +quickjs>=1.19 +cryptography>=42 +psycopg[binary]>=3.2 diff --git a/backend/seed_demo.py b/backend/seed_demo.py new file mode 100644 index 0000000..0266406 --- /dev/null +++ b/backend/seed_demo.py @@ -0,0 +1,267 @@ +"""Seed a demo scenario + adventure with sample scripts for testing. + +Run from the backend folder: .venv\\Scripts\\python.exe seed_demo.py +Safe to rerun: it deletes any previous rows titled "[Demo] ..." first. + +Phase 8: the scenario is seeded as PUBLIC (user_id NULL + is_public), so in +multi-user mode every guest sees it as read-only starter content. The sample +adventure and script-library copies belong to the local user (only relevant +on single-user installs). +""" + +from app import auth, models, migrations, tree +from app.database import SessionLocal, engine + +# create_all + user_version stamp; plain create_all would leave a fresh DB at +# version 0 and the server would replay every ALTER TABLE migration on start. +migrations.bootstrap(engine) + +DEMO_PREFIX = "[Demo]" + +# --------------------------------------------------------------------------- +# Sample scripts. The AI Dungeon contract is to define `modifier(text)` and call +# it last. +# --------------------------------------------------------------------------- + +DICE_ROLLER = dict( + name=f"{DEMO_PREFIX} Dice Roller", + description=( + "Input hook + shared library. Type '!roll 2d6' or '!roll d20' in a Do/Say/" + "Story action and the command is replaced with the rolled result." + ), + library_js="""\ +// Shared library: available to every hook of this script. +function rollDice(count, sides) { + var total = 0, rolls = []; + for (var i = 0; i < count; i++) { + var r = Math.floor(Math.random() * sides) + 1; + rolls.push(r); + total += r; + } + return { total: total, rolls: rolls }; +} +""", + input_js="""\ +const modifier = (text) => { + // Replace every "!roll NdS" (N optional) with the roll result. + var out = text.replace(/!roll\\s+(\\d*)d(\\d+)/gi, function (m, n, s) { + var count = parseInt(n || "1", 10); + var sides = parseInt(s, 10); + var res = rollDice(count, sides); + log("Rolled " + count + "d" + sides + ": [" + res.rolls.join(", ") + "] = " + res.total); + return "(rolled " + count + "d" + sides + ": " + res.total + ")"; + }); + return { text: out }; +}; +modifier(text); +""", +) + +TURN_TRACKER = dict( + name=f"{DEMO_PREFIX} Turn & HP Tracker", + description=( + "Demonstrates persistent state. Counts turns; '!hp -3' or '!hp +5' in input " + "adjusts HP (starts at 20). Current stats appear in state.message." + ), + input_js="""\ +const modifier = (text) => { + if (state.hp === undefined) state.hp = 20; + state.turns = (state.turns || 0) + 1; + + var out = text.replace(/!hp\\s*([+-]\\d+)/gi, function (m, delta) { + state.hp += parseInt(delta, 10); + return ""; + }); + + state.message = "Turn " + state.turns + " | HP: " + state.hp + "/20"; + log(state.message); + + if (state.hp <= 0) { + // stop:true ends the turn before the AI is called. + return { text: out + "\\n\\nYou have fallen. (HP reached 0 — turn stopped by script.)", stop: true }; + } + return { text: out }; +}; +modifier(text); +""", +) + +CONTEXT_INSPECTOR = dict( + name=f"{DEMO_PREFIX} Context Inspector", + description=( + "Context hook: logs the size of the assembled context each turn and appends " + "a style directive. Check the logs/context in the action's context snapshot." + ), + context_js="""\ +const modifier = (text) => { + log("Context size: " + text.length + " chars, actions so far: " + info.actionCount + + ", story cards: " + storyCards.length); + // Anything returned here replaces what is sent to the model. + return { text: text + "\\n[Style: keep the response under three paragraphs.]" }; +}; +modifier(text); +""", +) + +OUTPUT_POLISH = dict( + name=f"{DEMO_PREFIX} Output Polish + Card Discovery", + description=( + "Output hook: trims a trailing incomplete sentence from the AI reply, and " + "auto-creates a story card the first time the ghost Vharos is mentioned." + ), + output_js="""\ +const modifier = (text) => { + var out = text; + + // Drop a trailing sentence fragment (no ending punctuation). + var m = out.match(/^([\\s\\S]*[.!?"'\\u2026])[^.!?"'\\u2026]*$/); + if (m && m[1].length > 40) { + if (m[1].length < out.length) log("Trimmed incomplete final sentence."); + out = m[1]; + } + + // Demonstrate script-created story cards. + if (/vharos/i.test(out)) { + var added = addStoryCard( + "Vharos, ghost, spirit", + "Vharos was the crypt's architect, now a restless ghost bound to the amulet he was buried with. He speaks in echoes and cannot lie.", + "character" + ); + if (added !== false) log("Vharos mentioned — story card created."); + } + + return { text: out }; +}; +modifier(text); +""", +) + +SCRIPTS = [DICE_ROLLER, TURN_TRACKER, CONTEXT_INSPECTOR, OUTPUT_POLISH] + +# --------------------------------------------------------------------------- +# Scenario content +# --------------------------------------------------------------------------- + +SCENARIO = dict( + title=f"{DEMO_PREFIX} The Sunken Crypt of Vharos", + description=( + "A short dungeon-crawl demo scenario with story cards and one of each " + "script hook, for testing the app end to end." + ), + prompt=( + "Rain hammers the moors as you descend the moss-slick steps beneath the " + "ruined chapel. Your torch gutters in the stale air. Below, the Sunken " + "Crypt of Vharos waits — its iron door ajar, as if someone (or something) " + "expected you. Mira's warning rings in your ears: bring back the Ember " + "Amulet before nightfall, or the village of Hollowmere burns.\n\n" + "You stand before the iron door, water pooling around your boots." + ), + memory=( + "The player is an adventurer hired by Mira, blacksmith of Hollowmere, to " + "retrieve the Ember Amulet from the Sunken Crypt of Vharos before " + "nightfall. The crypt is flooded, dark, and haunted. Tone: classic D&D " + "dungeon crawl, dangerous but fair." + ), + authors_note="Keep scenes tense and grounded; offer clear choices; consequences matter.", + ai_instructions=( + "Write in second person, present tense. End each response at a moment " + "where the player can act." + ), + tags="demo, dungeon, fantasy, short", +) + +STORY_CARDS = [ + dict( + type="character", + name="Mira the Blacksmith", + keys="Mira, blacksmith", + entry=( + "Mira is Hollowmere's blacksmith: broad-shouldered, gray-braided, " + "practical. She hired the player and paid half up front. She knows " + "more about the crypt than she has admitted." + ), + notes="Secretly a descendant of Vharos.", + ), + dict( + type="location", + name="The Sunken Crypt", + keys="crypt, tomb, Vharos", + entry=( + "A flooded burial complex beneath a ruined chapel. Knee-deep black " + "water, collapsed pillars, and phosphorescent moss. Three chambers: " + "the Drowned Hall, the Ossuary, and the sealed Reliquary where the " + "Ember Amulet rests." + ), + notes="", + ), + dict( + type="item", + name="The Ember Amulet", + keys="amulet, ember", + entry=( + "A fist-sized garnet on a bronze chain that glows like a coal. It " + "keeps Hollowmere's protective hearth-ward burning. Touching it bare-" + "handed brands the flesh but does no lasting harm." + ), + notes="", + ), +] + +# --------------------------------------------------------------------------- + +db = SessionLocal() +try: + owner = auth.local_user(db) + + # Remove earlier demo rows so reruns stay clean. + for adv in db.query(models.Adventure).filter(models.Adventure.title.like(f"{DEMO_PREFIX}%")): + db.delete(adv) + for sc in db.query(models.Scenario).filter(models.Scenario.title.like(f"{DEMO_PREFIX}%")): + db.delete(sc) + for s in db.query(models.Script).filter(models.Script.name.like(f"{DEMO_PREFIX}%")): + db.delete(s) + db.commit() + + # Scripts attached to the public scenario are unowned (user_id NULL) so + # they ship with it everywhere; they're copied into each adventure at + # creation, so they never need to appear in anyone's script library. + scripts = [models.Script(**s) for s in SCRIPTS] + db.add_all(scripts) + + # A scenario with cards and scripts attached, as public starter content. + scenario = models.Scenario(**SCENARIO, is_public=True) + scenario.scripts = scripts + db.add(scenario) + db.flush() + for card in STORY_CARDS: + db.add(models.StoryCard(scenario_id=scenario.id, **card)) + + # Adventure created from the scenario, mirroring POST /api/adventures + adventure = models.Adventure( + user_id=owner.id, + scenario_id=scenario.id, + title=scenario.title, + memory=scenario.memory, + authors_note=scenario.authors_note, + ai_instructions=scenario.ai_instructions, + ) + db.add(adventure) + db.flush() + for card in STORY_CARDS: + db.add(models.StoryCard(adventure_id=adventure.id, **card)) + for position, s in enumerate(SCRIPTS): + db.add(models.AdventureScript(adventure_id=adventure.id, position=position, **s)) + opening = models.Action( + adventure_id=adventure.id, type="start", text=scenario.prompt + ) + # Through the same door create_adventure uses, so the seeded adventure has a + # story tree like every other one. + tree.place_action(db, adventure, opening) + db.add(opening) + db.commit() + + print(f"Scenario id={scenario.id}: {scenario.title}") + print(f"Adventure id={adventure.id}: {adventure.title}") + print(f"Scripts: {', '.join(s.name for s in scripts)}") +finally: + db.close() diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py new file mode 100644 index 0000000..b63f213 --- /dev/null +++ b/backend/tests/conftest.py @@ -0,0 +1,57 @@ +"""Shared setup for the test suite. + +pytest imports this file before it imports any test module, which is the only +reason the database redirection below works. `app.database` reads +`AIDND_DB_PATH` at import and builds `engine` from it once, so the variable has +to be set before the first `from app...` line anywhere in the suite. + +Every test module used to carry its own copy of that redirection. Only the first +one to be imported ever took effect, because the engine already existed by the +time the second one ran. The other copies created a temp file that nothing +opened and nothing deleted. One copy here does the job, and it cleans up after +itself. + +The tests share one database. That is not new: they already did. Each `client` +fixture calls `Base.metadata.create_all` on setup and `drop_all` on teardown, so +no test sees another test's rows. +""" +import os +import tempfile + +_tmp = tempfile.NamedTemporaryFile(suffix=".db", delete=False) +_tmp.close() +os.environ["AIDND_DB_PATH"] = _tmp.name +# A real `AIDND_DATABASE_URL` or `DATABASE_URL` in the developer's shell points +# at Postgres, and `app.database` prefers either over the SQLite path above. +# Clear both, so running the suite never touches a server database. +os.environ.pop("AIDND_DATABASE_URL", None) +os.environ.pop("DATABASE_URL", None) + +import pytest # noqa: E402 Import order is load-bearing; see above. + +from fakes import ScriptedProvider # noqa: E402 + + +@pytest.fixture(autouse=True) +def reset_scripted_provider(): + """Clears the fake provider's state between tests. + + `ScriptedProvider` keeps its replies and its call count on the class, because + the code under test constructs the provider itself and a test cannot reach + the instance. Class state outlives a test, so reset it here rather than + trusting every fixture to remember. + """ + ScriptedProvider.replies = [] + ScriptedProvider.calls = 0 + ScriptedProvider.prompts = [] + yield + + +def pytest_sessionfinish(session, exitstatus): + """Deletes the temporary database once the run ends.""" + try: + os.unlink(_tmp.name) + except OSError: + # The file is already gone, or Windows still holds a handle on it. It is + # in the temp directory either way, so leaving it costs nothing. + pass diff --git a/backend/tests/fakes.py b/backend/tests/fakes.py new file mode 100644 index 0000000..8b035c9 --- /dev/null +++ b/backend/tests/fakes.py @@ -0,0 +1,41 @@ +"""Stand-ins for the parts of the app a test must not really call. + +Import these rather than writing another copy. Nine test modules each carried +their own `ScriptedProvider`, and the copies had drifted into four different +feature sets, so a test that needed to raise a provider error had to be written +in one of the files whose copy supported it. +""" + + +class ScriptedProvider: + """Streams canned replies in place of `OpenAICompatibleProvider`. + + Set `replies` to the texts the model returns, one per call. The last entry + repeats once the list runs out, so a test that plays more turns than it + scripted still gets text. To drive the provider-error path, put an + `Exception` in the list. It is raised rather than streamed. + + State lives on the class, not on the instance, because the turn engine + constructs the provider itself and a test never sees the object. The autouse + `reset_scripted_provider` fixture in `conftest.py` clears it between tests. + + `prompts` records every assembled `(system, story)` pair, which is what a + test asserts on to check what the model was shown. + """ + + last_usage = None + replies: list = [] + calls = 0 + prompts: list = [] + + def __init__(self, *a, **k): + pass + + async def generate(self, parts, *, temperature, max_tokens): + index = min(ScriptedProvider.calls, len(ScriptedProvider.replies) - 1) + ScriptedProvider.calls += 1 + ScriptedProvider.prompts.append((parts.system, parts.story)) + reply = ScriptedProvider.replies[index] + if isinstance(reply, Exception): + raise reply + yield ("text", reply) diff --git a/backend/tests/schema_rewind.py b/backend/tests/schema_rewind.py new file mode 100644 index 0000000..77f1249 --- /dev/null +++ b/backend/tests/schema_rewind.py @@ -0,0 +1,47 @@ +"""Make a database look like an older schema version, so a migration can run. + +`create_all` always builds the current schema. A test that wants to watch a +migration run must remove the newer columns first and then stamp an older +version. Otherwise the migration finds a column that already exists and +fails on a duplicate. + +Rewinding the stamp alone worked for a while, so two test files did exactly +that. It stopped working when another `ADD COLUMN` migration landed. Without +this rewind, the replay runs migrations the tests never intended to +exercise, against columns `create_all` already added. This module rewinds +properly in one place. Adding a migration now means adding its inverse here, +instead of tracking down failures in three unrelated test files. + +This module supports SQLite only. Every test that replays migrations runs on +a temp file, and SQLite stores the stamp in `PRAGMA user_version`. +Migrations that change a column's type (43-45, JSON to compressed bytes) +have no clean inverse, so this list omits them. Those migrations replay +as-is, which is what the tests using them already expect. +""" + +from sqlalchemy import text +from sqlalchemy.engine import Engine + +# Each entry is (version that added the column, statements that remove it). +# The list is ordered newest first. +# +# Phase 14's `branch_id` columns are missing on purpose. SQLite refuses to +# drop a column that a foreign key references, so a current-schema database +# cannot be rewound past them. `migrations._column_already_there` handles +# this case. It skips DDL that already ran, so the tree migrations run their +# backfill against a schema that already has the columns. +_UNDO: list[tuple[int, tuple[str, ...]]] = [ + # Packed float32 vectors and the flag beside them. + (39, ("ALTER TABLE memories DROP COLUMN embedded",)), + (38, ("ALTER TABLE memories DROP COLUMN embedding_blob",)), +] + + +def rewind_to(engine: Engine, version: int) -> None: + """Drop everything added after `version`, then stamp the database at it.""" + with engine.begin() as conn: + for added_at, statements in _UNDO: + if added_at > version: + for sql in statements: + conn.execute(text(sql)) + conn.execute(text(f"PRAGMA user_version = {version}")) diff --git a/backend/tests/test_accesslog.py b/backend/tests/test_accesslog.py new file mode 100644 index 0000000..5a1da0c --- /dev/null +++ b/backend/tests/test_accesslog.py @@ -0,0 +1,208 @@ +"""The access log: app/accesslog.py and GET /api/analytics/access. + +This is the half of the analytics work that identifies people on purpose, +so these tests pin the details that would quietly make it wrong. The +address recorded must be the hardened one, not a header a client chose. +Session rows must be thinned instead of written on every page load. And a +row must outlive the account it describes, because guest cleanup deletes +accounts on a schedule, and a log that deletes itself is not a log. + + python -m pytest tests/test_accesslog.py -v +""" +import pytest +from fastapi import Depends +from fastapi.testclient import TestClient + +from app import accesslog, auth, limits, models, security +from app.database import Base, SessionLocal, engine, get_db +from app.main import app + +EDGE = "198.51.100.77" # what the trusted proxy appended +SPOOF = "10.0.0.1" # what a client put in front of it + + +@pytest.fixture(autouse=True) +def clean_state(): + accesslog._last_session.clear() + yield + accesslog._last_session.clear() + + +@pytest.fixture() +def client(monkeypatch): + Base.metadata.create_all(bind=engine) + setup = SessionLocal() + owner = models.User(is_guest=False, email="owner@example.com") + member = models.User( + is_guest=False, email="player@example.com", + password_hash=security.hash_password("hunter2long"), + ) + setup.add_all([owner, member]) + setup.commit() + ids = {"owner": owner.id, "member": member.id} + setup.close() + + monkeypatch.setattr(limits, "rate_limit", lambda *a, **k: None) + monkeypatch.setattr(limits, "check_login_allowed", lambda *a, **k: None) + monkeypatch.setattr(auth, "MULTI_USER", True) + monkeypatch.setattr(auth, "ANALYTICS_EMAILS", {"owner@example.com"}) + + # /auth/me resolves its own session, so the cookie flow below is the real + # one. Every other endpoint goes through get_current_user, and `act_as` + # decides who that is. + acting = {"id": ids["owner"]} + + def _current(db=Depends(get_db)): + return db.get(models.User, acting["id"]) + + app.dependency_overrides[auth.get_current_user] = _current + try: + test_client = TestClient(app) + test_client.ids = ids + test_client.act_as = lambda user_id: acting.update(id=user_id) + yield test_client + finally: + app.dependency_overrides.clear() + Base.metadata.drop_all(bind=engine) + + +def visit(client, ip=EDGE, ua="Mozilla/5.0 (Windows NT 10.0; Win64; x64)"): + return client.get( + "/api/auth/me", + headers={"x-forwarded-for": f"{SPOOF}, {ip}", "user-agent": ua}, + ) + + +def rows(kind=None): + db = SessionLocal() + try: + query = db.query(models.AccessEvent).order_by(models.AccessEvent.id) + if kind: + query = query.filter_by(kind=kind) + return query.all() + finally: + db.close() + + +def read_log(client, **params): + return client.get("/api/analytics/access", params=params) + + +# ---------- Writing ---------- + +def test_a_new_session_is_logged(client): + visit(client) + logged = rows() + assert len(logged) == 1 + entry = logged[0] + assert entry.kind == accesslog.SESSION + assert entry.is_guest and entry.who.startswith("Guest #") + assert entry.device == "desktop" + + +def test_the_address_is_the_hardened_one_not_the_clients(client): + visit(client) + # The client prepended its own value. Only the hop the edge appended counts. + # Recording the leftmost value would make every row forgeable, which is + # worse for a log than having no log at all. + assert rows()[0].ip == EDGE + + +def test_session_rows_are_thinned_to_one_per_day_per_address(client): + for _ in range(4): + visit(client) + assert len(rows(accesslog.SESSION)) == 1 + + +def test_a_changed_address_writes_a_new_row(client): + visit(client) + visit(client, ip="203.0.113.9") + logged = rows(accesslog.SESSION) + assert [entry.ip for entry in logged] == [EDGE, "203.0.113.9"] + # Same session throughout, so both rows name the same visitor. + assert logged[0].who == logged[1].who + + +def test_sign_in_and_failure_are_both_logged(client): + client.post("/api/auth/login", json={"email": "player@example.com", "password": "wrong"}, + headers={"x-forwarded-for": EDGE}) + client.post("/api/auth/login", json={"email": "player@example.com", "password": "hunter2long"}, + headers={"x-forwarded-for": EDGE}) + kinds = [entry.kind for entry in rows()] + assert accesslog.LOGIN_FAILED in kinds and accesslog.LOGIN in kinds + + failure = rows(accesslog.LOGIN_FAILED)[0] + # This records the address that was tried, not the account it belongs to. + # A failed attempt against an address with no matching account is + # exactly what this row exists to capture. + assert failure.who == "player@example.com" + assert failure.user_id is None + assert rows(accesslog.LOGIN)[0].user_id == client.ids["member"] + + +def test_registering_is_logged_against_the_upgraded_account(client): + visit(client) # creates the guest whose session then registers + client.act_as(rows()[0].user_id) + client.post("/api/auth/register", json={"email": "new@example.com", "password": "hunter2long"}) + entry = rows(accesslog.REGISTER)[0] + assert entry.who == "new@example.com" and not entry.is_guest + + +def test_a_row_outlives_the_account_it_describes(client): + visit(client) + entry = rows()[0] + db = SessionLocal() + try: + db.delete(db.get(models.User, entry.user_id)) + db.commit() + finally: + db.close() + # There is no foreign key, and `who` is a snapshot. Guest cleanup deletes + # accounts on a schedule, and a log that vanishes along with them is not + # a log. + survivor = rows()[0] + assert survivor.who == entry.who and survivor.ip == EDGE + + +def test_a_long_user_agent_is_truncated(client): + visit(client, ua="Mozilla/" + "x" * 500) + assert len(rows()[0].user_agent) == accesslog.MAX_UA + + +def test_a_logging_failure_does_not_break_the_request(client, monkeypatch): + monkeypatch.setattr(accesslog, "_client_ip", lambda request: 1 / 0) + # The log observes sign-in. A logging failure must not block the request. + assert visit(client).status_code == 200 + + +# ---------- Reading ---------- + +def test_the_log_is_invisible_to_everyone_but_the_owner(client): + visit(client) + assert read_log(client).status_code == 200 + client.act_as(client.ids["member"]) + assert read_log(client).status_code == 404 + + +def test_the_log_reads_newest_first_and_pages_backwards(client): + for index in range(5): + visit(client, ip=f"203.0.113.{index}") + first = read_log(client, limit=2).json() + assert [event["ip"] for event in first["events"]] == ["203.0.113.4", "203.0.113.3"] + assert first["has_more"] + + older = read_log(client, limit=2, before_id=first["events"][-1]["id"]).json() + assert [event["ip"] for event in older["events"]] == ["203.0.113.2", "203.0.113.1"] + + +def test_the_log_filters_by_kind_and_searches(client): + visit(client) + client.post("/api/auth/login", json={"email": "player@example.com", "password": "hunter2long"}, + headers={"x-forwarded-for": "203.0.113.44"}) + + assert len(read_log(client, kind="login").json()["events"]) == 1 + by_email = read_log(client, q="player@example.com").json()["events"] + assert len(by_email) == 1 and by_email[0]["kind"] == "login" + by_ip = read_log(client, q="203.0.113.44").json()["events"] + assert len(by_ip) == 1 + assert read_log(client, q="nobody@example.com").json()["events"] == [] diff --git a/backend/tests/test_action_paging.py b/backend/tests/test_action_paging.py new file mode 100644 index 0000000..f314f42 --- /dev/null +++ b/backend/tests/test_action_paging.py @@ -0,0 +1,246 @@ +"""Opening an adventure fetches a window, not the whole story. + +A story only ever gets longer. Production's longest is 607 actions and +589.5 kB in one response, and that number never decreases on its own. The +page load returns the newest `ACTION_PAGE` window, and the reader pages +upward from there. + +The paging anchors on an action id rather than an offset, and these tests +cover why. An offset counted back from the newest shifts every older +position the moment a turn lands, which is exactly when a reader is likely +to be scrolling. An anchor means the same thing before and after. + + python -m pytest tests/test_action_paging.py -v +""" +import re + +import pytest +from fastapi import Depends +from fastapi.testclient import TestClient + +from app import auth, limits, models +from app.database import Base, SessionLocal, engine, get_db +from app.main import app +from app.routers.adventures import ACTION_PAGE +from tools import dbmeter + +TOTAL = ACTION_PAGE * 3 + 7 # deliberately not a whole number of pages + + +@pytest.fixture() +def client(monkeypatch): + Base.metadata.create_all(bind=engine) + setup = SessionLocal() + user = models.User(is_guest=False, email="paging@example.com") + setup.add(user) + setup.flush() + setup.add(models.Settings(user_id=user.id, api_key="enc:dummy", model="m")) + adventure = models.Adventure(user_id=user.id, title="Cave", script_state={}) + setup.add(adventure) + setup.flush() + for i in range(TOTAL): + setup.add(models.Action( + adventure_id=adventure.id, + type="start" if i == 0 else ("ai" if i % 2 else "do"), + text=f"Action {i}." + "word " * 200, + )) + setup.commit() + adv_id, user_id = adventure.id, user.id + setup.close() + + monkeypatch.setattr(limits, "rate_limit", lambda *a, **k: None) + monkeypatch.setattr(limits, "check_row_cap", lambda *a, **k: None) + + def _current_user(db=Depends(get_db)): + return db.get(models.User, user_id) + + app.dependency_overrides[auth.get_current_user] = _current_user + c = TestClient(app) + c.adv_id = adv_id + try: + yield c + finally: + app.dependency_overrides.clear() + Base.metadata.drop_all(bind=engine) + + +def ordinal(action) -> int: + """Returns which turn this is, read out of the fixture's own text. + + The payload used to carry `index`, a story-wide turn number that SP8 + dropped. Nothing replaced it: a depth is a position along one branch, and + the pager keys on ids. The fixture numbers its own actions, so these tests + read the number back rather than reintroduce one. + """ + return int(re.match(r"Action (\d+)\.", action["text"]).group(1)) + + +def page(client, before_id=None, limit=None): + params = {} + if before_id is not None: + params["before_id"] = before_id + if limit is not None: + params["limit"] = limit + r = client.get(f"/api/adventures/{client.adv_id}/actions", params=params) + assert r.status_code == 200, r.text + return r.json() + + +def add_action(client, text="A new turn.") -> int: + db = SessionLocal() + try: + action = models.Action( + adventure_id=client.adv_id, type="ai", text=text + ) + db.add(action) + db.commit() + return action.id + finally: + db.close() + + +# ------------------------------------------------------------- the page load + +def test_the_page_load_returns_only_the_newest_window(client): + r = client.get(f"/api/adventures/{client.adv_id}") + assert r.status_code == 200 + body = r.json() + assert len(body["actions"]) == ACTION_PAGE + assert body["action_count"] == TOTAL + # It is the newest window, ending on the last action. + assert ordinal(body["actions"][-1]) == TOTAL - 1 + assert ordinal(body["actions"][0]) == TOTAL - ACTION_PAGE + + +def test_a_short_story_is_returned_whole(client): + db = SessionLocal() + try: + db.query(models.Action).filter(models.Action.depth >= 5).delete() + db.commit() + finally: + db.close() + + body = client.get(f"/api/adventures/{client.adv_id}").json() + assert len(body["actions"]) == 5 + assert body["action_count"] == 5 + + +def test_the_page_load_does_not_grow_with_the_story(client): + """Confirm that opening a story costs a window, regardless of the + story's length.""" + meter = dbmeter.Meter() + meter.attach(engine) + try: + with meter.scope("page load"): + client.get(f"/api/adventures/{client.adv_id}") + windowed = meter.scopes[-1].total.fetched + finally: + meter.detach() + + # Each action carries about 1 KB of text, and there are 187 of them. A + # window holds 60 actions. This ceiling is generous but still far below + # the size of the whole story. + assert windowed < ACTION_PAGE * 2_000, f"{windowed:,} B for one window" + assert windowed < TOTAL * 500, ( + f"{windowed:,} B — that is the whole story, not a window" + ) + + +# ------------------------------------------------------------------ paging up + +def test_the_first_page_is_the_newest(client): + body = page(client) + assert len(body["actions"]) == ACTION_PAGE + assert body["total"] == TOTAL + assert body["has_more"] is True + assert ordinal(body["actions"][-1]) == TOTAL - 1 + + +def test_paging_up_covers_the_whole_story_exactly_once(client): + seen = [] + body = page(client) + seen = [ordinal(a) for a in body["actions"]] + guard = 0 + while body["has_more"]: + guard += 1 + assert guard < 20, "paging did not terminate" + body = page(client, before_id=body["actions"][0]["id"]) + seen = [ordinal(a) for a in body["actions"]] + seen + + assert seen == list(range(TOTAL)), "gap, duplicate or reordering while paging" + + +def test_has_more_is_false_at_the_beginning_of_the_story(client): + body = page(client) + while body["has_more"]: + body = page(client, before_id=body["actions"][0]["id"]) + assert ordinal(body["actions"][0]) == 0 + + +def test_each_page_is_ordered_oldest_first(client): + body = page(client) + indices = [ordinal(a) for a in body["actions"]] + assert indices == sorted(indices) + + +# ------------------------------------------------- the reason for the anchor + +def test_a_turn_arriving_mid_scroll_does_not_shift_the_next_page(client): + """Reproduce the failure an offset-based scheme would have. Read the + newest page, let a turn land, then page up. The reader must get exactly + what precedes the actions they already hold, with no duplicate and no + skipped action.""" + first = page(client) + oldest_held = first["actions"][0] + + add_action(client) + + older = page(client, before_id=oldest_held["id"]) + assert ordinal(older["actions"][-1]) == ordinal(oldest_held) - 1, ( + "the page shifted when a turn landed" + ) + assert all(ordinal(a) < ordinal(oldest_held) for a in older["actions"]) + # The new turn changes the total, which is expected. It must not move the window. + assert older["total"] == TOTAL + 1 + + +def test_a_deleted_anchor_reports_the_end_rather_than_a_duplicate_page(client): + """Undo can remove the action a slow scroll was anchored to. The endpoint + must stop instead of returning a page the reader already has.""" + body = page(client) + anchor = body["actions"][0] + + db = SessionLocal() + try: + db.query(models.Action).filter(models.Action.id == anchor["id"]).delete() + db.commit() + finally: + db.close() + + after = page(client, before_id=anchor["id"]) + assert after["actions"] == [] + assert after["has_more"] is False + + +# ------------------------------------------------------------------- limits + +def test_limit_is_honoured_and_capped(client): + assert len(page(client, limit=5)["actions"]) == 5 + # A client that asks for the whole story cannot bypass the paging cap. + assert len(page(client, limit=100_000)["actions"]) <= ACTION_PAGE * 4 + + +def test_a_nonsense_limit_still_returns_something(client): + assert len(page(client, limit=0)["actions"]) >= 1 + assert len(page(client, limit=-5)["actions"]) >= 1 + + +# --------------------------------------------------------------------- undo + +def test_undo_returns_a_window_not_the_story(client): + r = client.post(f"/api/adventures/{client.adv_id}/undo") + assert r.status_code == 200, r.text + body = r.json() + assert len(body["actions"]) == ACTION_PAGE + assert body["total"] == TOTAL - 1 + assert body["has_more"] is True diff --git a/backend/tests/test_analytics.py b/backend/tests/test_analytics.py new file mode 100644 index 0000000..f0e3eb4 --- /dev/null +++ b/backend/tests/test_analytics.py @@ -0,0 +1,385 @@ +"""Visit analytics: app/analytics.py and the two endpoints in front of it. + +This file tests three things, and the rest is arithmetic. The counters must +survive the buffer/UPSERT round trip: a flush adds to what is already +stored instead of replacing it, or every number would show only the last +minute. The funnel counts people rather than clicks, which is the only +reason the visitor-day table exists. The gate holds: a stranger cannot read +the dashboard, and cannot inflate what it reports beyond hitting the page. + + python -m pytest tests/test_analytics.py -v +""" +from datetime import timedelta + +import pytest +from fastapi import Depends +from fastapi.testclient import TestClient + +from app import analytics, auth, limits, models +from app.database import Base, SessionLocal, engine, get_db +from app.main import app + + +@pytest.fixture(autouse=True) +def clean_buffer(): + """The buffer is process-wide, so a test that leaves counts in it would + show up inside the next one's flush.""" + analytics._counts.clear() + analytics._visits.clear() + analytics._labels_seen.clear() + yield + analytics._counts.clear() + analytics._visits.clear() + analytics._labels_seen.clear() + + +@pytest.fixture() +def db(): + Base.metadata.create_all(bind=engine) + session = SessionLocal() + try: + yield session + finally: + session.close() + Base.metadata.drop_all(bind=engine) + + +def counter(db, metric, label): + row = ( + db.query(models.AnalyticsDaily) + .filter_by(metric=metric, label=label) + .one_or_none() + ) + return row.hits if row else 0 + + +def make_user(db, email=None): + user = models.User(is_guest=email is None, email=email) + db.add(user) + db.commit() + return user + + +# ---------- The buffer and its flush ---------- + +def test_counts_accumulate_across_flushes(db): + analytics.record(analytics.M_PAGE, "/") + analytics.record(analytics.M_PAGE, "/") + analytics.flush(db) + analytics.record(analytics.M_PAGE, "/") + analytics.flush(db) + # The second flush has to find the existing row and add to it. Replacing it + # would leave every counter showing only the newest minute of traffic. + assert counter(db, analytics.M_PAGE, "/") == 3 + + +def test_flush_is_a_no_op_when_nothing_happened(db): + analytics.flush(db) + assert db.query(models.AnalyticsDaily).count() == 0 + + +def test_a_failed_flush_keeps_the_counts(db, monkeypatch): + analytics.record(analytics.M_PAGE, "/") + monkeypatch.setattr(analytics, "_write_counts", lambda *a: 1 / 0) + analytics.flush(db) # must not raise + monkeypatch.undo() + analytics.flush(db) + assert counter(db, analytics.M_PAGE, "/") == 1 + + +def test_label_cardinality_is_capped(db): + for i in range(analytics.MAX_LABELS_PER_METRIC + 25): + analytics.record(analytics.M_REFERRER, f"host{i}.example") + analytics.flush(db) + labels = db.query(models.AnalyticsDaily).filter_by(metric=analytics.M_REFERRER).count() + # Everything past the cap is folded into one bucket, so a referrer flood + # cannot create unlimited rows. + assert labels == analytics.MAX_LABELS_PER_METRIC + 1 + assert counter(db, analytics.M_REFERRER, analytics.OTHER) == 25 + + +# ---------- Visitors ---------- + +def test_visitor_id_is_stable_and_keyed(db, monkeypatch): + user = make_user(db) + handle = analytics.visitor_id(user) + assert handle == analytics.visitor_id(user) # a returning visitor + assert handle != analytics.visitor_id(make_user(db)) # is still one visitor + assert len(handle) == 32 and int(handle, 16) >= 0 # opaque hex, not an id + # Keyed on the app secret, not a bare hash of the user id. Otherwise + # anyone holding this table could rebuild the mapping by hashing + # sequential ids. + monkeypatch.setattr(analytics.security, "SECRET_KEY", b"a-different-secret") + assert analytics.visitor_id(user) != handle + + +def test_a_repeat_visitor_is_new_only_once(db): + user = make_user(db) + analytics.record_visit(user) + analytics.flush(db) + rows = db.query(models.AnalyticsVisitorDay).all() + assert len(rows) == 1 and rows[0].is_new + + # Same visitor, a later day: seen before, so not new. The row is not + # merged into the first day's row either. + tomorrow = (models.utcnow().date() + timedelta(days=1)).isoformat() + analytics._visits[(tomorrow, analytics.visitor_id(user))] = set() + analytics.flush(db) + rows = db.query(models.AnalyticsVisitorDay).order_by(models.AnalyticsVisitorDay.day).all() + assert [row.is_new for row in rows] == [True, False] + + +def test_one_row_per_visitor_per_day_however_much_they_do(db): + user = make_user(db) + for _ in range(5): + analytics.record_event(analytics.EV_ADVENTURE, user) + analytics.flush(db) + assert db.query(models.AnalyticsVisitorDay).count() == 1 + assert counter(db, analytics.M_EVENT, analytics.EV_ADVENTURE) == 5 + + +def test_funnel_flags_only_ever_turn_on(db): + user = make_user(db) + analytics.record_event(analytics.EV_TURN, user) + analytics.flush(db) + # A later visit that reaches no funnel step must not clear the earlier one. + analytics.record_visit(user) + analytics.flush(db) + row = db.query(models.AnalyticsVisitorDay).one() + assert row.played and not row.created + + +def test_purge_drops_only_rows_past_the_horizon(db): + old = (models.utcnow().date() - timedelta(days=analytics.RETENTION_DAYS + 1)).isoformat() + db.add(models.AnalyticsVisitorDay(day=old, visitor="a" * 32)) + db.add(models.AnalyticsVisitorDay(day=analytics._today(), visitor="b" * 32)) + db.commit() + assert analytics.purge_old_visitor_days(db) == 1 + assert [r.visitor for r in db.query(models.AnalyticsVisitorDay)] == ["b" * 32] + + +# ---------- Normalizing what a browser claims ---------- + +@pytest.mark.parametrize("path, expected", [ + ("/", "/"), + ("/adventures", "/adventures"), + ("/adventures/", "/adventures"), + ("/play/12?x=1", "/play/:id"), + ("/scenarios/9#top", "/scenarios/:id"), + ("/wp-admin", "(other)"), + ("/play/../../etc", "(other)"), + ("", "/"), +]) +def test_route_normalization(path, expected): + assert analytics.normalize_route(path) == expected + + +@pytest.mark.parametrize("referrer, expected", [ + ("", "(direct)"), + ("https://news.ycombinator.com/item?id=1", "news.ycombinator.com"), + ("https://www.google.com/", "google.com"), + ("https://ai-dnd.example/scenarios", ""), # our own host: not a referral + ("javascript:alert(1)", "(other)"), + ("https://" + "x" * 200 + ".com", "(other)"), +]) +def test_referrer_normalization(referrer, expected): + assert analytics.normalize_referrer(referrer, "ai-dnd.example") == expected + + +@pytest.mark.parametrize("ua, expected", [ + ("Mozilla/5.0 (iPhone; CPU iPhone OS 17_0) AppleWebKit", "mobile"), + ("Mozilla/5.0 (iPad; CPU OS 17_0) AppleWebKit", "tablet"), + ("Mozilla/5.0 (Windows NT 10.0; Win64; x64)", "desktop"), + ("Googlebot/2.1", "bot"), + ("", "(unknown)"), +]) +def test_device_detection(ua, expected): + assert analytics.device_of(ua) == expected + + +def test_only_iso_looking_country_headers_are_trusted(): + assert analytics.country_of({"cf-ipcountry": "de"}) == "DE" + assert analytics.country_of({"cf-ipcountry": "Norway"}) == analytics.UNKNOWN + assert analytics.country_of({"cf-ipcountry": "XX"}) == analytics.UNKNOWN + assert analytics.country_of({}) == analytics.UNKNOWN + + +def test_error_labels_use_the_route_not_the_path(): + class Route: + path = "/api/adventures/{adventure_id}" + + assert analytics.api_route_label({"route": Route()}, 500) == "500 /api/adventures/{adventure_id}" + # An unmatched path is entirely attacker-chosen, so it never becomes a label. + assert analytics.api_route_label({}, 404) == "404 (unmatched)" + + +# ---------- The summary ---------- + +def test_summary_counts_people_once_per_step(db): + one, two = make_user(db), make_user(db) + for _ in range(3): + analytics.record_event(analytics.EV_SCENARIO_OPEN, one) + analytics.record_event(analytics.EV_TURN, one) + analytics.record_event(analytics.EV_SCENARIO_OPEN, two) + + result = analytics.summary(db, days=7) + steps = {row["step"]: row["count"] for row in result["funnel"]} + assert steps["Visited"] == 2 + assert steps["Opened a scenario"] == 2 + assert steps["Played a turn"] == 1 # not 3, because one person made three turns + assert steps["Signed up"] == 0 + # Raw event totals still count every occurrence. + assert result["totals"]["turns"] == 3 + assert result["totals"]["visitors"] == 2 + + +def test_summary_series_covers_every_day_including_empty_ones(db): + analytics.record(analytics.M_PAGE, "/") + result = analytics.summary(db, days=7) + assert len(result["series"]) == 7 + assert result["series"][-1]["day"] == models.utcnow().date().isoformat() + assert result["series"][-1]["pageviews"] == 1 + assert result["series"][0]["pageviews"] == 0 + + +def test_summary_flushes_before_reading(db): + analytics.record(analytics.M_EVENT, analytics.EV_TURN) + # Never flushed by hand: the dashboard must not be up to a minute stale. + assert analytics.summary(db, days=1)["totals"]["turns"] == 1 + + +def test_summary_reports_pages_referrers_and_errors(db): + analytics.record(analytics.M_PAGE, "/play/:id", n=4) + analytics.record(analytics.M_REFERRER, "news.ycombinator.com", n=2) + analytics.record(analytics.M_ERROR, "500 /api/adventures/{adventure_id}") + result = analytics.summary(db, days=30) + assert result["pages"][0] == {"label": "/play/:id", "hits": 4} + assert result["referrers"][0]["label"] == "news.ycombinator.com" + assert result["totals"]["errors"] == 1 + + +# ---------- The endpoints ---------- + +@pytest.fixture() +def client(monkeypatch): + Base.metadata.create_all(bind=engine) + setup = SessionLocal() + visitor = models.User(is_guest=True) + owner = models.User(is_guest=False, email="owner@example.com") + setup.add_all([visitor, owner]) + setup.commit() + ids = {"visitor": visitor.id, "owner": owner.id} + setup.close() + + monkeypatch.setattr(limits, "rate_limit", lambda *a, **k: None) + # Multi-user is what makes the gate mean anything: local mode trusts + # whoever is at the keyboard, because it is the operator's own machine. + monkeypatch.setattr(auth, "MULTI_USER", True) + monkeypatch.setattr(auth, "ANALYTICS_EMAILS", {"owner@example.com"}) + + current = {"id": ids["visitor"]} + + def _current_user(db=Depends(get_db)): + return db.get(models.User, current["id"]) + + app.dependency_overrides[auth.get_current_user] = _current_user + monkeypatch.setattr( + auth, "resolve_session_user", lambda request, db: db.get(models.User, current["id"]) + ) + try: + client = TestClient(app) + client.ids, client.current = ids, current + yield client + finally: + app.dependency_overrides.clear() + Base.metadata.drop_all(bind=engine) + + +def read_summary(client, days=30): + return client.get(f"/api/analytics/summary?days={days}") + + +def test_dashboard_is_invisible_to_everyone_but_the_owner(client): + assert read_summary(client).status_code == 404 + client.current["id"] = client.ids["owner"] + assert read_summary(client).status_code == 200 + + +def test_collect_records_a_pageview_and_the_visit(client): + resp = client.post("/api/analytics/collect", json={"path": "/play/7", "first": True, + "referrer": "https://news.ycombinator.com/"}) + assert resp.status_code == 204 + client.current["id"] = client.ids["owner"] + body = read_summary(client).json() + assert body["pages"][0] == {"label": "/play/:id", "hits": 1} + assert body["referrers"][0]["label"] == "news.ycombinator.com" + assert body["totals"]["visitors"] == 1 + + +def test_referrer_and_device_are_recorded_once_per_visit_not_per_view(client): + for path in ("/", "/scenarios", "/adventures"): + client.post("/api/analytics/collect", json={"path": path, "first": path == "/"}) + client.current["id"] = client.ids["owner"] + body = read_summary(client).json() + assert body["totals"]["pageviews"] == 3 + # Three views, one visit: the referral and the device are facts about the + # visit, so counting them per view would multiply every one of them. + assert sum(row["hits"] for row in body["devices"]) == 1 + + +def test_the_owners_own_visits_are_not_traffic(client): + client.current["id"] = client.ids["owner"] + client.post("/api/analytics/collect", json={"path": "/", "first": True}) + assert read_summary(client).json()["totals"]["pageviews"] == 0 + + +def test_a_client_cannot_invent_pages_or_events(client): + client.post("/api/analytics/collect", json={"path": "/../../admin", "first": True}) + # There is no field for it, so a made-up event is not even expressible. + client.post("/api/analytics/collect", json={"path": "/", "event": "signup"}) + client.current["id"] = client.ids["owner"] + body = read_summary(client).json() + assert {row["label"] for row in body["pages"]} == {"(other)", "/"} + assert body["totals"]["signups"] == 0 + + +def test_api_errors_are_counted_by_route(client): + client.get("/api/adventures/999999") + client.current["id"] = client.ids["owner"] + errors = read_summary(client).json()["errors"] + assert errors and errors[0]["label"].startswith("404 /api/adventures/") + + +# ---------- The dialect the tests never run on ---------- + +def test_the_upserts_compile_for_postgres(): + """Prod runs on Neon, but these tests run on SQLite, and a failed flush + is caught and logged instead of raised. A dialect mistake would + therefore stay invisible until the dashboard quietly stayed empty. This + test compiles both statements against Postgres without connecting to + one. + """ + from sqlalchemy import create_engine + from sqlalchemy.dialects import postgresql + from sqlalchemy.orm import sessionmaker + + session = sessionmaker(bind=create_engine("postgresql+psycopg://u:p@localhost/db"))() + compiled = [] + + def capture(statement, *args, **kwargs): + compiled.append(str(statement.compile(dialect=postgresql.dialect()))) + + session.execute = capture + session.scalars = lambda *a, **k: [] + + analytics._write_counts(session, {("2026-01-01", "pageview", "/"): 2}) + analytics._write_visits(session, {("2026-01-01", "f" * 32): {"played"}}) + + counts, visits = compiled + assert "ON CONFLICT (day, metric, label) DO UPDATE" in counts + assert "analytics_daily.hits + excluded.hits" in counts + assert "ON CONFLICT (day, visitor) DO UPDATE" in visits + assert "analytics_visitor_days.played OR excluded.played" in visits + # is_new is settled by the first write of a visitor's first day and must + # not be in the update clause at all. + assert "is_new" not in visits.split("DO UPDATE")[1] diff --git a/backend/tests/test_attempt_siblings.py b/backend/tests/test_attempt_siblings.py new file mode 100644 index 0000000..c8c97ec --- /dev/null +++ b/backend/tests/test_attempt_siblings.py @@ -0,0 +1,444 @@ +"""Phase 14 SP4: a retry writes a sibling node instead of rewriting a row. + +`test_retry_variants.py` is the behavioral contract from before the tree +existed, and it still passes unchanged: the same URLs, the same payload +shape, the same outcomes. This file asserts the things that are true only of +the new storage. A turn can be several rows, exactly one of them is the +story, and the arrangement costs neither an extra prompt nor an extra turn. + + python -m pytest tests/test_attempt_siblings.py -v +""" +import pytest +from fastapi import Depends +from fastapi.testclient import TestClient +from sqlalchemy.orm import undefer + +from app import attempts, auth, limits, models, tree +from app.context import cursors, history +from app.database import Base, SessionLocal, engine, get_db +from app.main import app +from app.routers import adventures + +from fakes import ScriptedProvider + +SCHEMA = {"player": {"hp": {"min": 0, "max": 100, "initial": 100}}} + +GOLD_SCRIPT = """ +const modifier = (text) => { + state.gold = (state.gold || 0) + 10; + return { text }; +}; +modifier(text); +""" + + +@pytest.fixture() +def client(monkeypatch): + Base.metadata.create_all(bind=engine) + setup = SessionLocal() + user = models.User(is_guest=False, email="siblings@example.com") + setup.add(user) + setup.flush() + setup.add(models.Settings(user_id=user.id, api_key="enc:dummy", model="test-model")) + scenario = models.Scenario(user_id=user.id, title="S", stat_schema=SCHEMA) + setup.add(scenario) + setup.flush() + adv = models.Adventure( + user_id=user.id, title="Cave", scenario_id=scenario.id, + script_state={}, world_state={"player": {"hp": 100}}, + ) + setup.add(adv) + setup.flush() + setup.add(models.Action(adventure_id=adv.id, type="start", text="You enter a cave.")) + setup.add(models.AdventureScript( + adventure_id=adv.id, position=0, enabled=True, name="Gold", output_js=GOLD_SCRIPT, + )) + setup.commit() + adv_id, user_id = adv.id, user.id + setup.close() + + ScriptedProvider.replies = ["Attempt one."] + ScriptedProvider.calls = 0 + ScriptedProvider.prompts = [] + monkeypatch.setattr(adventures.turns, "OpenAICompatibleProvider", ScriptedProvider) + monkeypatch.setattr(auth, "resolve_provider_config", lambda s: auth.ProviderConfig( + "http://fake", "k", "test-model", False)) + monkeypatch.setattr(limits, "rate_limit", lambda *a, **k: None) + monkeypatch.setattr(limits, "check_row_cap", lambda *a, **k: None) + + def _current_user(db=Depends(get_db)): + return db.get(models.User, user_id) + + app.dependency_overrides[auth.get_current_user] = _current_user + c = TestClient(app) + c.adv_id = adv_id + try: + yield c + finally: + app.dependency_overrides.clear() + adventures.turns._active_turns.clear() + Base.metadata.drop_all(bind=engine) + + +def _play(client, text="look around", type="do"): + r = client.post(f"/api/adventures/{client.adv_id}/actions", + json={"type": type, "text": text}) + assert r.status_code == 200, r.text + + +def _retry(client): + r = client.post(f"/api/adventures/{client.adv_id}/retry") + assert r.status_code == 200, r.text + + +def _page(client) -> dict: + return client.get(f"/api/adventures/{client.adv_id}").json() + + +def _rows(adv_id) -> list[models.Action]: + """Every action row of the adventure, story or not, live or not. + + The query undefers these columns because the session closes before the + caller reads the result. This file specifically tests the columns that a + page load never loads. + """ + db = SessionLocal() + try: + return ( + db.query(models.Action) + .filter(models.Action.adventure_id == adv_id) + .options( + undefer(models.Action.state_after), + undefer(models.Action.world_state_after), + undefer(models.Action.context_snapshot), + ) + .order_by(models.Action.depth, models.Action.id) + .all() + ) + finally: + db.close() + + +# ------------------------------------------------------------- the sibling + +def test_a_retry_writes_a_second_row_at_the_same_coordinate(client): + ScriptedProvider.replies = ["Attempt one.", "Attempt two."] + _play(client) + _retry(client) + + rows = _rows(client.adv_id) + ai = [a for a in rows if a.type == "ai"] + assert len(ai) == 2, "a retry is a node, not a rewrite" + assert {(a.branch_id, a.depth) for a in ai} == {(ai[0].branch_id, ai[0].depth)} + assert [a.text for a in ai] == ["Attempt one.", "Attempt two."] + # Exactly one of them is the story, and it is the newer take. + assert [a.live for a in ai] == [False, True] + # The discarded attempt is untouched, not a copy of anything. + assert ai[0].state_after is not None + + +def test_the_story_shows_and_counts_the_turn_once(client): + ScriptedProvider.replies = ["Attempt one.", "Attempt two."] + _play(client) + before = _page(client)["action_count"] + _retry(client) + after = _page(client) + + assert after["action_count"] == before, "a discarded attempt is not a turn" + assert [a["type"] for a in after["actions"]] == ["start", "do", "ai"] + assert after["actions"][-1]["text"] == "Attempt two." + + +def test_a_discarded_attempt_never_reaches_the_prompt(client): + """This is the failure case the branch clause exists to prevent, at + sibling scale. The losing attempt sits at the same branch and depth as + the live one, so anything reading the story by coordinate alone would + replay both.""" + ScriptedProvider.replies = ["Attempt one.", "Attempt two.", "Next turn."] + _play(client) + _retry(client) + _play(client, "go deeper") + + story = ScriptedProvider.prompts[-1][1] + assert "Attempt two." in story + assert "Attempt one." not in story + + +def test_switching_moves_the_story_onto_the_other_row(client): + ScriptedProvider.replies = [ + "A scratch.\n```state\n{\"player.hp\": -5}\n```", + "A beating.\n```state\n{\"player.hp\": -40}\n```", + ] + _play(client) + _retry(client) + newest_id = _page(client)["actions"][-1]["id"] + + r = client.post( + f"/api/adventures/{client.adv_id}/actions/{newest_id}/variant", json={"index": 0}) + assert r.status_code == 200, r.text + # A different row answers the request. That is the only change. + assert r.json()["id"] != newest_id + assert r.json()["text"].startswith("A scratch") + + rows = _rows(client.adv_id) + ai = [a for a in rows if a.type == "ai"] + assert [a.live for a in ai] == [True, False] + # Both takes remain unchanged in the database. + assert [a.text.split(".")[0] for a in ai] == ["A scratch", "A beating"] + + +def test_the_assembled_prompt_is_stored_once_per_turn(client): + """A snapshot holds about 160 kB of prompt that every attempt at a turn + shares. Giving each sibling its own copy would make retry multiply the + size of the largest column in the database. Instead, the prompt moves + with the live flag.""" + ScriptedProvider.replies = ["Attempt one.", "Attempt two."] + _play(client) + _retry(client) + + def holders(): + return [ + a.id for a in _rows(client.adv_id) + if a.type == "ai" and "sections" in (a.context_snapshot or {}) + ] + + live_holder = holders() + assert len(live_holder) == 1 + newest = _page(client)["actions"][-1] + assert live_holder == [newest["id"]] + + client.post(f"/api/adventures/{client.adv_id}/actions/{newest['id']}/variant", + json={"index": 0}) + moved = holders() + assert len(moved) == 1 and moved != live_holder, "the prompt follows the story" + + +# ------------------------------------------------------- removing the turn + +def test_undo_takes_every_attempt_with_it(client): + ScriptedProvider.replies = ["One.", "Two.", "Three."] + _play(client) + _retry(client) + _retry(client) + assert len([a for a in _rows(client.adv_id) if a.type == "ai"]) == 3 + + r = client.post(f"/api/adventures/{client.adv_id}/undo") + assert r.status_code == 200, r.text + assert [a.type for a in _rows(client.adv_id)] == ["start"] + + +def test_deleting_a_retried_turn_deletes_its_attempts(client): + ScriptedProvider.replies = ["One.", "Two."] + _play(client) + _retry(client) + newest = _page(client)["actions"][-1] + + r = client.delete(f"/api/adventures/{client.adv_id}/actions/{newest['id']}") + assert r.status_code == 204, r.text + assert [a.type for a in _rows(client.adv_id)] == ["start", "do"] + + +def test_deleting_a_turn_through_a_discarded_attempt_still_takes_the_turn(client): + """The pager hands out whichever id it last saw, and a switch changes which + row that is. Deleting through the losing sibling must not leave the story + holding a turn with no attempts.""" + ScriptedProvider.replies = ["One.", "Two."] + _play(client) + _retry(client) + discarded = [a for a in _rows(client.adv_id) if a.type == "ai" and not a.live][0] + + r = client.delete(f"/api/adventures/{client.adv_id}/actions/{discarded.id}") + assert r.status_code == 204, r.text + assert [a.type for a in _rows(client.adv_id)] == ["start", "do"] + + +# ------------------------------------------- what the holdback used to cover + +def test_retrying_withdraws_the_memory_the_turn_produced(client): + """Why summarization no longer holds the newest action back. + + A memory covering the newest turn used to be unreachable by + construction. The summarizer stopped one action short, because a retry + rewrote the row under a mark that had already moved past it. Now the + mark and the memory both name the node, so replacing what a node says + withdraws them. Undo and delete already had this repair; the holdback + was the only gap a retry still needed to close. + """ + ScriptedProvider.replies = ["One.", "Two."] + _play(client) + + db = SessionLocal() + try: + adventure = db.get(models.Adventure, client.adv_id) + newest = history.newest(adventure) + memory = models.Memory( + adventure_id=adventure.id, text="You looked around.", + source_start=1, source_end=newest.depth, + ) + tree.attach_memory(memory, newest) + db.add(memory) + cursors.MEMORY.anchor_at(adventure, newest) + cursors.SUMMARY.anchor_at(adventure, newest) + db.commit() + covered_depth = newest.depth + finally: + db.close() + + _retry(client) + + db = SessionLocal() + try: + adventure = db.get(models.Adventure, client.adv_id) + assert db.query(models.Memory).count() == 0, "the withdrawn memory is gone" + # The depth range it covered is released, so the block is + # summarized again from where it began instead of being skipped. + assert cursors.MEMORY.depth(db, adventure) == 0 + assert cursors.SUMMARY.depth(db, adventure) == 0 + assert covered_depth > 0 + finally: + db.close() + + +def test_a_memory_on_an_earlier_turn_survives_a_retry(client): + """Only the coordinate whose text changed is withdrawn.""" + ScriptedProvider.replies = ["One.", "Two.", "Three."] + _play(client) + _play(client, "go deeper") + + db = SessionLocal() + try: + adventure = db.get(models.Adventure, client.adv_id) + earlier = history.tail(adventure, 3)[0] + memory = models.Memory( + adventure_id=adventure.id, text="An earlier block.", + source_start=0, source_end=earlier.depth, + ) + tree.attach_memory(memory, earlier) + db.add(memory) + db.commit() + finally: + db.close() + + _retry(client) + + db = SessionLocal() + try: + assert [m.text for m in db.query(models.Memory).all()] == ["An earlier block."] + finally: + db.close() + + +# -------------------------------------------------------------- the group + +def test_the_group_grows_in_the_order_the_attempts_arrive(client): + """The attempts page in the order they were made, and the pager counts them. + + Ordering used to come from `variant_index`, an explicit ordinal that SP8 + dropped. `id` carries the same order, because a row is inserted when its + attempt is made. + """ + ScriptedProvider.replies = ["One.", "Two.", "Three."] + _play(client) + assert _page(client)["actions"][-1]["take_count"] == 1 # never retried + _retry(client) + _retry(client) + + ai = [a for a in _rows(client.adv_id) if a.type == "ai"] + assert [a.text for a in ai] == ["One.", "Two.", "Three."] + assert _page(client)["actions"][-1]["take_count"] == 3 + + +def test_attempts_module_agrees_with_the_endpoint(client): + ScriptedProvider.replies = ["One.", "Two."] + _play(client) + _retry(client) + newest = _page(client)["actions"][-1] + + listed = client.get( + f"/api/adventures/{client.adv_id}/actions/{newest['id']}/variants").json() + db = SessionLocal() + try: + node = db.get(models.Action, newest["id"]) + group = attempts.group(db, node) + assert [a.text for a in group] == [v["text"] for v in listed] + assert attempts.live_in(group).id == newest["id"] + finally: + db.close() + + +def test_export_carries_every_attempt_as_its_own_node(client): + """SP6 changed the answer here, for the same reasons as the rest of that + subphase. + + A v1 bundle had one entry per turn and folded the group back into a + `variants` array, because the format had nowhere else to put a second + take. A v2 bundle has coordinates, so an attempt is a node in the file + exactly as it is a node in the database, and `live` says which one is + the story. + """ + ScriptedProvider.replies = ["One.", "Two."] + _play(client) + _retry(client) + + bundle = client.get(f"/api/adventures/{client.adv_id}/export").json() + ai = [a for a in bundle["actions"] if a["type"] == "ai"] + assert [(a["text"], a["live"]) for a in ai] == [("One.", False), ("Two.", True)] + assert len({(a["branch"], a["depth"]) for a in ai}) == 1, "one turn, two takes" + assert "variants" not in ai[0], "nothing writes the repeating group any more" + + # Importing the bundle puts the group back exactly as it stood. + imported = client.post("/api/adventures/import", json=bundle).json()["id"] + rows = _rows(imported) + ai_rows = [a for a in rows if a.type == "ai"] + assert [(a.text, a.live) for a in ai_rows] == [("One.", False), ("Two.", True)] + assert len({(a.branch_id, a.depth) for a in ai_rows}) == 1 + + +def test_a_retry_after_switching_back_files_the_new_attempt_last(client): + """The group stays in the order the attempts were made. + + `add_attempt` used to number a new take one past the take it replaced. + That numbering is correct only when the story is standing on the newest + take. Switch a three-take turn back to the first and retry, and the new + attempt collided with take 2, which put it between takes 2 and 3. The + group orders by `id` now, so a new attempt is always last. + """ + ScriptedProvider.replies = ["One.", "Two.", "Three.", "Four."] + _play(client) + _retry(client) + _retry(client) + assert [a.text for a in _rows(client.adv_id) if a.type == "ai"] == [ + "One.", "Two.", "Three."] + + live = _page(client)["actions"][-1] + r = client.post( + f"/api/adventures/{client.adv_id}/actions/{live['id']}/variant", + json={"index": 0}) + assert r.status_code == 200, r.text + + _retry(client) + ai = [a for a in _rows(client.adv_id) if a.type == "ai"] + assert [a.text for a in ai] == ["One.", "Two.", "Three.", "Four."] + assert attempts.live_in(ai).text == "Four." + + +def test_the_adventure_list_quotes_the_take_the_story_tells(client): + """The index screen and the story have to agree. + + Siblings share a depth, and the newest of them has the highest id. A + snippet ordered by `(depth, id)` alone quotes whichever attempt was + written last. After switching back, that attempt is the one the player + discarded. + """ + ScriptedProvider.replies = ["One.", "Two."] + _play(client) + _retry(client) + live = _page(client)["actions"][-1] + assert live["text"] == "Two." + client.post(f"/api/adventures/{client.adv_id}/actions/{live['id']}/variant", + json={"index": 0}) + + listed = client.get("/api/adventures").json() + row = [a for a in listed if a["id"] == client.adv_id][0] + assert "One." in row["snippet"] + assert "Two." not in row["snippet"] diff --git a/backend/tests/test_branch_clause.py b/backend/tests/test_branch_clause.py new file mode 100644 index 0000000..d557211 --- /dev/null +++ b/backend/tests/test_branch_clause.py @@ -0,0 +1,558 @@ +"""Phase 14 SP2: a read sees one story, and knows which one. + +These tests build the fork by hand. Three branch rows and their nodes are +written straight to the database, arranged as the design doc's own worked +example. That was the only way to build one when this file was written, +because nothing forked until SP5. It stays that way now that `tree.fork` +exists, because a fixture built with the same code under test could not +catch that code being wrong. `test_branch_forking.py` checks the two +against each other. + + branch C, tip at depth 7, lineage [(C, 7), (B, 5), (A, 3)] + -> A0 A1 A2 A3 B4 B5 C6 C7 + +The point of building the fixture by hand is that every read in the app +must go through one module. A forgotten clause does not raise an error. It +silently shows a story assembled out of two different branches. The +fixture deliberately leaves nodes where a forgotten clause would pick them +up: A kept playing past the fork (A4, A5), B kept playing past its own +(B6), and a second adventure holds a whole story of its own. None of these +nodes may appear on C. + + python -m pytest tests/test_branch_clause.py -v +""" +import pytest +from fastapi import Depends +from fastapi.testclient import TestClient +from sqlalchemy import event + +from app import auth, limits, models, tree +from app.context import history, lineage +from app.database import Base, SessionLocal, engine, get_db +from app.main import app +from app.scripting import ScriptPipeline +from tools import dbmeter + + +# --------------------------------------------------------------- the fixture + +def make_branch(db, adventure, parent=None, fork_depth=None): + """A branch row whose lineage is its parent's, capped, plus itself. + + This function performs the same computation SP5 does at fork time. It + is written out here so the fixture cannot pass by repeating a bug in + the code under test. + """ + branch = models.Branch( + adventure_id=adventure.id, + parent_branch_id=parent.id if parent else None, + fork_depth=fork_depth, + lineage=[], + ) + db.add(branch) + db.flush() + inherited = [] + if parent is not None: + for ancestor_id, cap in lineage.entries_of(parent): + capped = fork_depth if cap is None else min(cap, fork_depth) + inherited.append([ancestor_id, capped]) + branch.lineage = [[branch.id, None]] + inherited + db.flush() + return branch + + +def add_node(db, adventure, branch, depth, label, index=None): + action = models.Action( + adventure_id=adventure.id, + branch_id=branch.id, + depth=depth, + type="start" if depth == 0 else ("ai" if depth % 2 else "do"), + text=f"{label}{depth}", + ) + db.add(action) + return action + + +def make_adventure(db, user, title): + adventure = models.Adventure(user_id=user.id, title=title, script_state={}) + db.add(adventure) + db.flush() + return adventure + + +@pytest.fixture() +def forked(): + """The worked example, plus everything a forgotten clause would expose.""" + Base.metadata.create_all(bind=engine) + db = SessionLocal() + user = models.User(is_guest=False, email="branch@example.com") + db.add(user) + db.flush() + db.add(models.Settings(user_id=user.id, api_key="enc:dummy", model="m")) + + adventure = make_adventure(db, user, "Forked") + a = make_branch(db, adventure) + b = make_branch(db, adventure, parent=a, fork_depth=3) + c = make_branch(db, adventure, parent=b, fork_depth=5) + + for depth in range(4): + add_node(db, adventure, a, depth, "A") + # A did not stop when B forked off it: these two are siblings of B4/B5. + for depth in (4, 5): + add_node(db, adventure, a, depth, "A", index=100 + depth) + for depth in (4, 5): + add_node(db, adventure, b, depth, "B") + add_node(db, adventure, b, 6, "B", index=200 + 6) # B's own continuation + for depth in (6, 7): + add_node(db, adventure, c, depth, "C") + + # A second adventure, so "does the clause remember its adventure?" has an + # answer. A database holding one adventure cannot tell you. + other = make_adventure(db, user, "Elsewhere") + other_branch = make_branch(db, other) + for depth in range(4): + add_node(db, other, other_branch, depth, "X") + + adventure.head_branch_id = c.id + adventure.head_depth = 7 + other.head_branch_id = other_branch.id + other.head_depth = 3 + db.commit() + + ids = {"a": a.id, "b": b.id, "c": c.id, "adventure": adventure.id, + "other": other.id, "user": user.id} + try: + yield db, adventure, ids + finally: + db.close() + Base.metadata.drop_all(bind=engine) + + +def switch_to(db, adventure, branch_id, tip): + """Move the head, the way SP7's branch picker will.""" + adventure.head_branch_id = branch_id + adventure.head_depth = tip + db.commit() + + +def labels(actions): + return [a.text for a in actions] + + +# ------------------------------------------------------------- the story read + +def test_the_worked_example_reads_back_as_the_design_doc_says(forked): + db, adventure, _ = forked + assert labels(history.story_actions(adventure)) == [ + "A0", "A1", "A2", "A3", "B4", "B5", "C6", "C7" + ] + + +def test_a_siblings_nodes_are_invisible(forked): + db, adventure, _ = forked + seen = labels(history.story_actions(adventure)) + # A4 and A5 are A's own continuation past B's fork. B6 is B's own + # continuation past C's fork. + assert "A4" not in seen and "A5" not in seen and "B6" not in seen + # Nothing from the other adventure appears either. + assert not [text for text in seen if text.startswith("X")] + + +def test_each_branch_reads_its_own_story(forked): + db, adventure, ids = forked + switch_to(db, adventure, ids["a"], 5) + assert labels(history.story_actions(adventure)) == [ + "A0", "A1", "A2", "A3", "A4", "A5" + ] + switch_to(db, adventure, ids["b"], 6) + assert labels(history.story_actions(adventure)) == [ + "A0", "A1", "A2", "A3", "B4", "B5", "B6" + ] + switch_to(db, adventure, ids["c"], 7) + assert labels(history.story_actions(adventure)) == [ + "A0", "A1", "A2", "A3", "B4", "B5", "C6", "C7" + ] + + +def test_the_ancestors_shared_nodes_are_shared_not_copied(forked): + db, adventure, ids = forked + # A0..A3 appear on all three stories and exist exactly once in the table. + rows = ( + db.query(models.Action) + .filter(models.Action.adventure_id == adventure.id, + models.Action.text == "A2") + .all() + ) + assert len(rows) == 1 + assert rows[0].branch_id == ids["a"] + + +def test_count_and_tail_and_slice_agree_with_the_path(forked): + db, adventure, _ = forked + assert history.count(adventure) == 8 + assert labels(history.tail(adventure, 3)) == ["B5", "C6", "C7"] + assert labels(history.tail_range(adventure, 2, 2)) == ["B4", "B5"] + assert labels(history.slice_(adventure, 3, 3)) == ["A3", "B4", "B5"] + + +def test_a_window_that_reaches_past_the_fork_still_reads_in_order(forked): + db, adventure, _ = forked + # 32 is history.WINDOW_START: more than the whole story, so the read has to + # widen through all three lineage entries and still come back in order. + assert labels(history.window_covering(adventure, 10 ** 6, len)) == [ + "A0", "A1", "A2", "A3", "B4", "B5", "C6", "C7" + ] + + +# ------------------------------------------- the loaded-collection short cut + +def test_an_already_loaded_collection_is_cut_down_to_the_path(forked): + """Tests `history._from_memory`'s shortcut, the highest-risk line here. + + `adventure.actions` returns every branch's actions. Slicing it without + the path would assemble a prompt out of two different stories, and + nothing would raise an error. This test loads the collection + deliberately and checks that the answer is still the path. + """ + db, adventure, _ = forked + loaded = list(adventure.actions) # every branch, ordered by index + assert len(loaded) == 11 # the path's 8, plus A4, A5 and B6 + assert labels(history.story_actions(adventure)) == [ + "A0", "A1", "A2", "A3", "B4", "B5", "C6", "C7" + ] + assert history.count(adventure) == 8 + assert labels(history.tail(adventure, 3)) == ["B5", "C6", "C7"] + + +def test_user_scripts_are_handed_the_path(forked): + """The same risk one layer up, in code visible to users: + `pipeline._history()` is the documented scripting history API.""" + db, adventure, _ = forked + list(adventure.actions) # the pipeline's caller has usually loaded these + pipeline = ScriptPipeline(adventure, db) + assert [h["text"] for h in pipeline._history()] == [ + "A0", "A1", "A2", "A3", "B4", "B5", "C6", "C7" + ] + assert pipeline._info()["actionCount"] == 8 + + +# ------------------------------------------------------------ over the wire + +@pytest.fixture() +def client(forked, monkeypatch): + db, adventure, ids = forked + monkeypatch.setattr(limits, "rate_limit", lambda *a, **k: None) + monkeypatch.setattr(limits, "check_row_cap", lambda *a, **k: None) + + def _current_user(session=Depends(get_db)): + return session.get(models.User, ids["user"]) + + app.dependency_overrides[auth.get_current_user] = _current_user + c = TestClient(app) + try: + yield c, db, adventure, ids + finally: + app.dependency_overrides.clear() + + +def test_the_page_the_reader_opens_is_the_path(client): + c, db, adventure, ids = client + r = c.get(f"/api/adventures/{adventure.id}") + assert r.status_code == 200, r.text + body = r.json() + assert [a["text"] for a in body["actions"]] == [ + "A0", "A1", "A2", "A3", "B4", "B5", "C6", "C7" + ] + # `action_count` tells the reader whether more actions exist above. It + # counts the path too: 8, not the 13 rows the adventure holds. + assert body["action_count"] == 8 + + +def test_paging_up_walks_the_path_across_the_forks(client): + c, db, adventure, ids = client + first = c.get(f"/api/adventures/{adventure.id}/actions", params={"limit": 3}).json() + assert [a["text"] for a in first["actions"]] == ["B5", "C6", "C7"] + assert first["has_more"] is True + older = c.get( + f"/api/adventures/{adventure.id}/actions", + params={"limit": 3, "before_id": first["actions"][0]["id"]}, + ).json() + assert [a["text"] for a in older["actions"]] == ["A2", "A3", "B4"] + oldest = c.get( + f"/api/adventures/{adventure.id}/actions", + params={"limit": 3, "before_id": older["actions"][0]["id"]}, + ).json() + assert [a["text"] for a in oldest["actions"]] == ["A0", "A1"] + assert oldest["has_more"] is False + + +def test_paging_from_an_anchor_off_the_path_reports_the_end(client): + """A stale client holding an id from another branch gets an empty page, + not that branch's story.""" + c, db, adventure, ids = client + off_path = ( + db.query(models.Action) + .filter(models.Action.adventure_id == adventure.id, + models.Action.text == "A5") + .one() + ) + page = c.get( + f"/api/adventures/{adventure.id}/actions", + params={"before_id": off_path.id}, + ).json() + assert page["actions"] == [] + assert page["has_more"] is False + + +def test_the_index_screen_quotes_the_branch_being_played(client): + c, db, adventure, ids = client + listed = {row["id"]: row for row in c.get("/api/adventures").json()} + # The newest *narrated* node on C: do/say are the player's voice, and the + # fixture alternates types, so C7 is the one that reads as story. + assert listed[adventure.id]["snippet"] == "C7" + switch_to(db, adventure, ids["a"], 5) + listed = {row["id"]: row for row in c.get("/api/adventures").json()} + assert listed[adventure.id]["snippet"] == "A5" + assert listed[ids["other"]]["snippet"] == "X3" + + +# ------------------------------------------------------- the flush guard + +def test_a_node_written_without_a_branch_is_placed_anyway(forked): + """SP1 wired the writers. Since SP2, an unplaced node is an invisible one. + + This behavior lets a fixture, a script, or a test built straight through + the ORM keep working. It is also why the baseline contract still passes + with its actions written directly to the database. + """ + db, adventure, ids = forked + written = models.Action( + adventure_id=adventure.id, type="do", text="C8" + ) + db.add(written) + db.commit() + assert written.branch_id == ids["c"] + # The node lands one step past the tip of C. SP8 dropped `index`, which is + # where a caller used to name its own depth, so an unplaced node now always + # follows the head. + assert written.depth == adventure.head_depth + assert adventure.head_depth == 8 + assert labels(history.tail(adventure, 2)) == ["C7", "C8"] + + +def test_a_memory_written_without_a_branch_is_placed_anyway(forked): + db, adventure, ids = forked + memory = models.Memory( + adventure_id=adventure.id, text="The cave was cold.", source_start=0, source_end=3 + ) + db.add(memory) + db.commit() + assert memory.branch_id == ids["c"] + assert memory.depth == 3 + + +def test_placing_a_flush_of_nodes_reads_the_branch_once(forked, emitted_sql): + """The guard resolves the head once per flush, not once per node. + + The identity map holds weak references. A branch row with no strong + reference gets collected between two nodes and read back again for the + next one. Writing two hundred actions in one flush ran two hundred + SELECTs on `branches` before the head lookup moved outside the loop. + The test result alone would not have shown this. + """ + db, adventure, _ = forked + emitted_sql.clear() + for i in range(50): + db.add(models.Action( + adventure_id=adventure.id, type="do", text=f"bulk {i}" + )) + db.commit() + branch_reads = [s for s in emitted_sql if s.startswith("SELECT") and "FROM branches" in s] + assert len(branch_reads) <= 2, ( + f"{len(branch_reads)} reads of `branches` to place 50 nodes" + ) + + +def test_an_adventure_with_no_branch_at_all_reads_as_empty(forked): + """A missing branch must fail loudly: nothing, rather than everything. + + A row with no branch cannot be shown without guessing which story it + belongs to, and a wrong guess here puts a sibling's turns into a + prompt. + """ + db, adventure, ids = forked + stray = make_adventure(db, db.get(models.User, ids["user"]), "Stray") + db.query(models.Action).filter(models.Action.text == "A0").update( + {"adventure_id": stray.id}, synchronize_session=False + ) + db.commit() + db.expire_all() + stray = db.get(models.Adventure, stray.id) + assert lineage.path_of(db, stray).entries == [] + assert history.story_actions(stray) == [] + assert history.count(stray) == 0 + + +# --------------------------------------------------- the cost of forking + +@pytest.fixture() +def deeply_forked(): + """A story forked twenty times, then played forty turns past the last one. + + This shape tests the design's core assumption: reading the tail of this + story must cost the same as reading the tail of an unforked story, + because the window is covered long before the ancestry runs out. + """ + Base.metadata.create_all(bind=engine) + db = SessionLocal() + user = models.User(is_guest=False, email="deep@example.com") + db.add(user) + db.flush() + adventure = make_adventure(db, user, "Deep") + branch = make_branch(db, adventure) + depth = 0 + for _ in range(4): + add_node(db, adventure, branch, depth, "n") + depth += 1 + for fork in range(20): + branch = make_branch(db, adventure, parent=branch, fork_depth=depth - 1) + for _ in range(2): + add_node(db, adventure, branch, depth, "n") + depth += 1 + tip_branch = make_branch(db, adventure, parent=branch, fork_depth=depth - 1) + for _ in range(40): + add_node(db, adventure, tip_branch, depth, "n") + depth += 1 + adventure.head_branch_id = tip_branch.id + adventure.head_depth = depth - 1 + db.commit() + try: + yield db, adventure + finally: + db.close() + Base.metadata.drop_all(bind=engine) + + +@pytest.fixture() +def emitted_sql(): + """Every statement the connection runs, so a clause can be counted.""" + seen = [] + + def on_execute(conn, cursor, statement, parameters, context, executemany): + seen.append(statement) + + event.listen(engine, "before_cursor_execute", on_execute) + try: + yield seen + finally: + event.remove(engine, "before_cursor_execute", on_execute) + + +def branch_terms(statement: str) -> int: + return statement.count("actions.branch_id =") + + +def test_a_tail_read_names_one_branch_however_many_forks_there_were( + deeply_forked, emitted_sql +): + db, adventure = deeply_forked + assert len(lineage.path_of(db, adventure)) == 22 # the whole ancestry + emitted_sql.clear() + rows = history.tail(adventure, 32) + assert len(rows) == 32 + selects = [s for s in emitted_sql if "FROM actions" in s and branch_terms(s)] + assert selects, "no action read was emitted" + # Clause count is bounded by the context window, not by fork count: the + # newest branch alone holds forty turns, so one entry covers a window of + # thirty-two and the other twenty-one are never named. + assert max(branch_terms(s) for s in selects) == 1 + + +def test_a_window_reaching_past_the_forks_names_only_what_it_needs( + deeply_forked, emitted_sql +): + db, adventure = deeply_forked + emitted_sql.clear() + rows = history.tail(adventure, 41) # 40 on the tip branch, one older + assert len(rows) == 41 + selects = [s for s in emitted_sql if "FROM actions" in s and branch_terms(s)] + # Two lineage entries reach 41 deep (40 + 2). The other twenty stay + # unnamed. Every fork past the window costs the query nothing. + assert max(branch_terms(s) for s in selects) == 2 + + +def test_the_estimate_is_arithmetic_not_a_query(deeply_forked): + db, adventure = deeply_forked + path = lineage.path_of(db, adventure) + assert path.prefix_covering(1) == 1 + assert path.prefix_covering(40) == 1 + assert path.prefix_covering(41) == 2 + assert path.prefix_covering(43) == 3 + assert path.prefix_covering(10 ** 6) == len(path) + + +def test_forking_twenty_times_costs_the_same_bytes_as_never_forking( + deeply_forked, +): + """The design's cost assumption, measured in bytes. + + Two stories of the same length, one played straight through and one + forked twenty times, cost the same to read their newest window. The + window is covered by the newest lineage entry either way, and the + ancestry is never named. The forked read pays for one extra row: the + branch it reads the lineage from. + """ + db, forked_adventure = deeply_forked + flat = make_adventure(db, db.get(models.User, forked_adventure.user_id), "Flat") + branch = make_branch(db, flat) + for depth in range(84): # the same 84 nodes the forked story is long + add_node(db, flat, branch, depth, "n") + flat.head_branch_id = branch.id + flat.head_depth = 83 + flat_id, forked_id = flat.id, forked_adventure.id + # Commit and release the connection. The meter wraps the connection + # pool's factory, so a connection checked out before the meter attaches + # is never visible to it. Building the fixture is a write path the test + # does not measure, and it is not charged to either scope. + db.commit() + db.expire_all() + + meter = dbmeter.Meter() + meter.attach(engine) + try: + with meter.scope("flat"): + assert len(history.tail(db.get(models.Adventure, flat_id), 32)) == 32 + flat_bytes = meter.scopes[-1].total.fetched + with meter.scope("forked"): + assert len(history.tail(db.get(models.Adventure, forked_id), 32)) == 32 + forked_bytes = meter.scopes[-1].total.fetched + finally: + meter.detach() + + assert flat_bytes > 0, "the meter saw nothing; it is measuring the wrong connection" + + assert forked_bytes < flat_bytes * 1.25, ( + f"reading a 20-fork story cost {forked_bytes:,} B against the " + f"{flat_bytes:,} B an unforked one of the same length cost" + ) + + +def test_a_gap_in_the_story_widens_the_read_rather_than_shortening_it( + deeply_forked, emitted_sql +): + """The estimate counts depths, and a deleted action leaves a depth with + no row behind it. The read must notice it came up short and widen.""" + db, adventure = deeply_forked + victim = ( + db.query(models.Action) + .filter(models.Action.branch_id == adventure.head_branch_id) + .order_by(models.Action.depth) + .first() + ) + db.delete(victim) + db.commit() + emitted_sql.clear() + rows = history.tail(adventure, 40) + assert len(rows) == 40 # 39 on the tip branch, one borrowed from above + selects = [s for s in emitted_sql if "FROM actions" in s and branch_terms(s)] + assert max(branch_terms(s) for s in selects) == len(lineage.path_of(db, adventure)) diff --git a/backend/tests/test_branch_forking.py b/backend/tests/test_branch_forking.py new file mode 100644 index 0000000..8b5bb40 --- /dev/null +++ b/backend/tests/test_branch_forking.py @@ -0,0 +1,562 @@ +"""Phase 14 SP5: continuing from a discarded attempt forks a branch. + +SP4 made every attempt at a turn a node. While the attempts sit at the +tip, they are leaves and cost nothing: switching between them just moves +the `live` flag. The moment the player continues from an attempt the line +has already moved past, the two futures must coexist. That is a branch. + +This file tests the claim the whole design rests on: a fork inserts one +row and moves one row, no matter how large the story behind it is. +Everything before the fork is borrowed, not copied, and the arithmetic +that makes borrowing possible lives in `lineage`. + + python -m pytest tests/test_branch_forking.py -v +""" +import pytest +from fastapi import Depends +from fastapi.testclient import TestClient + +from app import auth, limits, models +from app.context import cursors, lineage +from app.database import Base, SessionLocal, engine, get_db +from app.main import app +from app.routers import adventures + +from fakes import ScriptedProvider + +# `hp` moves freely. `mana` has a cooldown of 2 turns, so an incorrect +# advance shows up as a change the referee should have rejected. +SCHEMA = { + "player": { + "hp": {"min": 0, "max": 100, "initial": 100}, + "mana": {"min": 0, "max": 50, "initial": 50, "cooldown": 2}, + } +} + +GOLD_SCRIPT = """ +const modifier = (text) => { + state.gold = (state.gold || 0) + 10; + return { text }; +}; +modifier(text); +""" + + +@pytest.fixture() +def client(monkeypatch): + Base.metadata.create_all(bind=engine) + setup = SessionLocal() + user = models.User(is_guest=False, email="fork@example.com") + setup.add(user) + setup.flush() + setup.add(models.Settings(user_id=user.id, api_key="enc:dummy", model="test-model")) + scenario = models.Scenario(user_id=user.id, title="S", stat_schema=SCHEMA) + setup.add(scenario) + setup.flush() + adv = models.Adventure( + user_id=user.id, title="Cave", scenario_id=scenario.id, + script_state={}, world_state={"player": {"hp": 100, "mana": 50}}, + ) + setup.add(adv) + setup.flush() + setup.add(models.Action(adventure_id=adv.id, type="start", text="You enter a cave.")) + setup.add(models.AdventureScript( + adventure_id=adv.id, position=0, enabled=True, name="Gold", output_js=GOLD_SCRIPT, + )) + setup.commit() + adv_id, user_id = adv.id, user.id + setup.close() + + ScriptedProvider.replies = ["A reply."] + ScriptedProvider.calls = 0 + ScriptedProvider.prompts = [] + monkeypatch.setattr(adventures.turns, "OpenAICompatibleProvider", ScriptedProvider) + monkeypatch.setattr(auth, "resolve_provider_config", lambda s: auth.ProviderConfig( + "http://fake", "k", "test-model", False)) + monkeypatch.setattr(limits, "rate_limit", lambda *a, **k: None) + monkeypatch.setattr(limits, "check_row_cap", lambda *a, **k: None) + + def _current_user(db=Depends(get_db)): + return db.get(models.User, user_id) + + app.dependency_overrides[auth.get_current_user] = _current_user + c = TestClient(app) + c.adv_id = adv_id + try: + yield c + finally: + app.dependency_overrides.clear() + adventures.turns._active_turns.clear() + Base.metadata.drop_all(bind=engine) + + +# ------------------------------------------------------------------ helpers + +def _play(client, text="look around", type="do"): + r = client.post(f"/api/adventures/{client.adv_id}/actions", + json={"type": type, "text": text}) + assert r.status_code == 200, r.text + + +def _retry(client): + r = client.post(f"/api/adventures/{client.adv_id}/retry") + assert r.status_code == 200, r.text + + +def _texts(client) -> list[str]: + return [a["text"] for a in client.get(f"/api/adventures/{client.adv_id}").json()["actions"]] + + +def _branches(client) -> list[dict]: + r = client.get(f"/api/adventures/{client.adv_id}/branches") + assert r.status_code == 200, r.text + return r.json() + + +def _fork(client, action_id): + return client.post(f"/api/adventures/{client.adv_id}/actions/{action_id}/fork") + + +def _state(adv_id): + db = SessionLocal() + try: + adv = db.get(models.Adventure, adv_id) + return adv.script_state, adv.world_state + finally: + db.close() + + +def _rows(adv_id) -> list[models.Action]: + db = SessionLocal() + try: + return ( + db.query(models.Action) + .filter(models.Action.adventure_id == adv_id) + .order_by(models.Action.depth, models.Action.id) + .all() + ) + finally: + db.close() + + +def _divergent_story(client): + """A story that retried turn 2, continued from the newer take, and left + the older one behind as a leaf. + + start > do > [attempt one | ATTEMPT TWO] > do > next turn + + Returns the id of the discarded attempt. + """ + ScriptedProvider.replies = ["Attempt one.", "Attempt two.", "Next turn."] + _play(client) + _retry(client) + _play(client, "go deeper") + return [a.id for a in _rows(client.adv_id) if a.type == "ai" and not a.live][0] + + +# ---------------------------------------------------------------- the fork + +def test_a_fork_inserts_one_branch_row_and_copies_no_actions(client): + discarded = _divergent_story(client) + rows_before = [a.id for a in _rows(client.adv_id)] + assert len(_branches(client)) == 1 + + r = _fork(client, discarded) + assert r.status_code == 200, r.text + + branches = _branches(client) + assert len(branches) == 2, "exactly one branch row per divergence built on" + assert [a.id for a in _rows(client.adv_id)] == rows_before, "a fork copies nothing" + forked = [b for b in branches if b["parent_branch_id"] is not None][0] + assert forked["is_head"] is True + assert forked["own_actions"] == 1, "the promoted attempt, and nothing else" + # The fork point is the depth just before the attempt. The code stores + # this value instead of inferring it from where two branches first + # differ, because that inference would guess wrong as soon as an + # attempt repeats its parent's text. + assert forked["fork_depth"] == forked["depth"] - 1 + + +def test_the_lineage_is_capped_at_the_fork_depth(client): + discarded = _divergent_story(client) + _fork(client, discarded) + + db = SessionLocal() + try: + adventure = db.get(models.Adventure, client.adv_id) + branch = lineage.branch_of(db, adventure) + entries = lineage.entries_of(branch) + assert entries[0] == (branch.id, None), "its own nodes, to the tip" + assert entries[1] == (branch.parent_branch_id, branch.fork_depth) + assert len(entries) == 2 + finally: + db.close() + + +def test_both_branches_read_independently(client): + discarded = _divergent_story(client) + parent = _branches(client)[0]["id"] + _fork(client, discarded) + + # The fork's story: everything up to the divergence, then the other take. + assert _texts(client) == [ + "You enter a cave.", "> You look around.", "Attempt one.", + ] + # The branch it left behind is unchanged, including turns after the fork. + r = client.post(f"/api/adventures/{client.adv_id}/branches/{parent}/switch") + assert r.status_code == 200, r.text + assert _texts(client) == [ + "You enter a cave.", "> You look around.", "Attempt two.", + "> You go deeper.", "Next turn.", + ] + + +def test_the_parent_keeps_a_live_attempt_where_the_fork_left(client): + """Promoting the other attempt must not leave the parent with a gap in + its story. A coordinate with no live node is a turn that disappears + from the read.""" + discarded = _divergent_story(client) + _fork(client, discarded) + + db = SessionLocal() + try: + adventure = db.get(models.Adventure, client.adv_id) + parent_id = db.query(models.Branch).filter_by( + adventure_id=adventure.id, parent_branch_id=None).one().id + per_coordinate = {} + for row in _rows(client.adv_id): + per_coordinate.setdefault((row.branch_id, row.depth), []).append(row) + for (branch_id, depth), group in per_coordinate.items(): + live = [a for a in group if a.live] + assert len(live) == 1, f"branch {branch_id} depth {depth}" + # The parent's turn 2 is now a single take, so the pager no longer + # offers a page through attempts that diverged onto another branch. + parent_turn = per_coordinate[(parent_id, 2)] + assert len(parent_turn) == 1 + finally: + db.close() + + +def test_playing_on_a_fork_continues_that_branchs_depths(client): + """A depth is a position along this story. Numbering the next node from + the adventure-wide index would leave a gap where the other branch's + turns are, and every windowing estimate would then have to work around + that gap.""" + discarded = _divergent_story(client) + _fork(client, discarded) + ScriptedProvider.replies = ["Onward."] + _play(client, "turn back") + + db = SessionLocal() + try: + adventure = db.get(models.Adventure, client.adv_id) + path = lineage.path_of(db, adventure) + rows = [a for a in _rows(client.adv_id) if path.contains(a)] + assert [a.depth for a in sorted(rows, key=lambda a: a.depth)] == [0, 1, 2, 3, 4] + finally: + db.close() + assert _texts(client)[-1] == "Onward." + + +# ------------------------------------------------------------- not a fork + +def test_forking_at_the_tip_switches_without_making_a_branch(client): + """Attempts nobody has built on stay leaves. This is what keeps the + lineage a list of divergences instead of a list of every retry.""" + ScriptedProvider.replies = ["Attempt one.", "Attempt two."] + _play(client) + _retry(client) + discarded = [a.id for a in _rows(client.adv_id) if a.type == "ai" and not a.live][0] + + r = _fork(client, discarded) + assert r.status_code == 200, r.text + assert len(_branches(client)) == 1, "no branch for an attempt at the tip" + assert _texts(client)[-1] == "Attempt one." + + +def test_forking_the_attempt_already_in_the_story_does_nothing(client): + """This call is idempotent. A client that has lost track of which take + is live must not create a new branch on every click.""" + discarded = _divergent_story(client) + _fork(client, discarded) + promoted = [a.id for a in _rows(client.adv_id) if a.type == "ai" and a.live + and a.branch_id != _branches(client)[0]["id"]][0] + before = _texts(client) + + for _ in range(3): + r = _fork(client, promoted) + assert r.status_code == 200, r.text + assert len(_branches(client)) == 2 + assert _texts(client) == before + + +def test_forking_a_turn_that_is_already_the_story_is_a_no_op(client): + ScriptedProvider.replies = ["Only take."] + _play(client) + only = [a.id for a in _rows(client.adv_id) if a.type == "ai"][0] + + r = _fork(client, only) + assert r.status_code == 200, r.text + assert len(_branches(client)) == 1 + + +def test_forking_a_live_node_on_another_branch_is_refused(client): + """A live node off the path belongs to another branch's story, not to a + spare attempt on this one. The refusal names the tool that actually + switches branches. It used to answer "only one take", which was true + of the attempt group but useless here: the caller does not want + another take, it wants the branch this node is on.""" + discarded = _divergent_story(client) + _fork(client, discarded) + parent_id = [b for b in _branches(client) if b["parent_branch_id"] is None][0]["id"] + stranded = [a.id for a in _rows(client.adv_id) + if a.branch_id == parent_id and a.depth == 2][0] + + r = _fork(client, stranded) + assert r.status_code == 400 + assert "another branch" in r.json()["detail"] + # Refusing must leave the tree alone. The bug this guards against is a + # fork that promotes a sibling on the branch it was called against. + assert len(_branches(client)) == 2 + + +# -------------------------------------------------------------- the state + +def test_switching_restores_the_script_and_world_state(client): + ScriptedProvider.replies = [ + "A scratch.\n```state\n{\"player.hp\": -5}\n```", + "A beating.\n```state\n{\"player.hp\": -40}\n```", + "Onward.", + ] + _play(client) + _retry(client) + _play(client, "go deeper") + parent = _branches(client)[0]["id"] + on_parent = _state(client.adv_id) + + discarded = [a.id for a in _rows(client.adv_id) if a.type == "ai" and not a.live][0] + _fork(client, discarded) + script_state, world_state = _state(client.adv_id) + assert world_state["player"]["hp"] == 95, "the attempt this branch tells" + assert script_state == {"gold": 10}, "one turn of gold, not three" + + client.post(f"/api/adventures/{client.adv_id}/branches/{parent}/switch") + assert _state(client.adv_id) == on_parent + + +def test_the_cooldown_clock_travels_with_the_branch(client): + """The world-state clock is a depth, and depths repeat across branches, + so it can only be correct if each branch carries its own. It does, + without extra work: the clock lives inside `_meta.last_changed`, which + is part of the world state a switch restores.""" + ScriptedProvider.replies = [ + "Drained.\n```state\n{\"player.mana\": -10}\n```", + "Untouched.", + "Onward.", + ] + _play(client) + _retry(client) + _play(client, "go deeper") + discarded = [a.id for a in _rows(client.adv_id) if a.type == "ai" and not a.live][0] + on_parent = _state(client.adv_id)[1] + assert on_parent["_meta"]["last_changed"].get("player.mana") is None + + _fork(client, discarded) + forked = _state(client.adv_id)[1] + assert forked["player"]["mana"] == 40 + assert forked["_meta"]["last_changed"]["player.mana"] == 2 + + parent = [b for b in _branches(client) if b["parent_branch_id"] is None][0]["id"] + client.post(f"/api/adventures/{client.adv_id}/branches/{parent}/switch") + assert _state(client.adv_id)[1] == on_parent + + +def test_a_retry_does_not_advance_the_cooldown_clock(client): + """SP5's one carried-over open item. A retry re-runs the same turn, so + the clock the cooldown rules read must not move. The reused `index` + used to guarantee this; the reused depth guarantees it now.""" + ScriptedProvider.replies = [ + "Drained.\n```state\n{\"player.mana\": -10}\n```", + "Drained again.\n```state\n{\"player.mana\": -10}\n```", + ] + _play(client) + first = _state(client.adv_id)[1]["_meta"]["last_changed"]["player.mana"] + _retry(client) + assert _state(client.adv_id)[1]["_meta"]["last_changed"]["player.mana"] == first + # The second attempt's drain must land, instead of being rejected for a + # cooldown it was never actually subject to. + assert _state(client.adv_id)[1]["player"]["mana"] == 40 + + +# --------------------------------------------------------- derived work + +def test_a_memory_on_the_line_left_behind_is_out_of_range_on_the_fork(client): + """Nothing is moved or removed when a branch forks. The memory attaches + to the coordinate the parent's attempt still occupies, and the lineage + caps the parent one depth short of it. The fork therefore cannot see + this memory, and resummarizes that span from the text it actually + contains.""" + from app import tree + + discarded = _divergent_story(client) + db = SessionLocal() + try: + adventure = db.get(models.Adventure, client.adv_id) + winner = db.query(models.Action).filter_by( + adventure_id=adventure.id, type="ai", live=True).order_by( + models.Action.depth).first() + memory = models.Memory( + adventure_id=adventure.id, text="Attempt two happened.", + source_start=0, source_end=winner.depth, + ) + tree.attach_memory(memory, winner) + db.add(memory) + cursors.MEMORY.anchor_at(adventure, winner) + db.commit() + finally: + db.close() + + _fork(client, discarded) + + db = SessionLocal() + try: + adventure = db.get(models.Adventure, client.adv_id) + # The memory is still there, untouched, because it describes the + # parent's story, which is unchanged. + assert [m.text for m in db.query(models.Memory).all()] == ["Attempt two happened."] + path = lineage.path_of(db, adventure) + visible = db.query(models.Memory).filter( + models.Memory.adventure_id == adventure.id, + path.clause(models.Memory), + ).all() + assert visible == [], "a sibling's memory reached this branch" + # The cursor reads one depth short of the memory, so this branch + # treats the block as due again instead of silently marking it read. + assert cursors.MEMORY.depth(db, adventure) == 1 + finally: + db.close() + + +# ------------------------------------------------------------------- undo + +def test_undo_stops_at_the_fork(client): + """Undoing a turn on a fork must never reach into the branch it forked + from. Those turns belong to that branch's story too.""" + discarded = _divergent_story(client) + _fork(client, discarded) + rows_before = len(_rows(client.adv_id)) + + r = client.post(f"/api/adventures/{client.adv_id}/undo") + assert r.status_code == 200, r.text + # The promoted attempt is removed, and the player action before it + # stays, because that action belongs to the parent and the parent + # still has it. + assert len(_rows(client.adv_id)) == rows_before - 1 + assert _texts(client) == ["You enter a cave.", "> You look around."] + + # Nothing is left of this branch's own turns, so undo must refuse + # instead of removing the parent's turns. + r = client.post(f"/api/adventures/{client.adv_id}/undo") + assert r.status_code == 400 + assert "forked from" in r.json()["detail"] + + +# ----------------------------------------------------------- the tree view + +def test_the_branch_list_is_the_tree(client): + discarded = _divergent_story(client) + _fork(client, discarded) + ScriptedProvider.replies = ["Onward."] + _play(client, "turn back") + + branches = _branches(client) + root = [b for b in branches if b["parent_branch_id"] is None][0] + forked = [b for b in branches if b["parent_branch_id"] == root["id"]][0] + assert root["fork_depth"] is None and root["depth"] == 4 + assert forked["fork_depth"] == 1 and forked["depth"] == 4 + assert root["own_actions"] == 5 and forked["own_actions"] == 3 + assert forked["is_head"] is True and root["is_head"] is False + + +def test_fork_agrees_with_the_lineage_computed_by_hand(client): + """`test_branch_clause.make_branch` has computed a fork's lineage by + hand since SP2, precisely so the fixture could not pass by repeating a + bug in the code under test. SP5 introduces that code, so this test + checks the two against each other instead of letting them drift + apart.""" + from tests.test_branch_clause import make_branch + + discarded = _divergent_story(client) + _fork(client, discarded) + + db = SessionLocal() + try: + adventure = db.get(models.Adventure, client.adv_id) + real = lineage.branch_of(db, adventure) + parent = db.get(models.Branch, real.parent_branch_id) + by_hand = make_branch(db, adventure, parent=parent, fork_depth=real.fork_depth) + # The two branches share a shape but not an id, so compare only the + # ancestry, which is the arithmetic part rather than the allocated id. + assert lineage.entries_of(real)[1:] == lineage.entries_of(by_hand)[1:] + assert lineage.entries_of(real)[0] == (real.id, None) + finally: + db.close() + + +def test_a_deep_fork_chain_reads_for_what_one_branch_costs(client): + """Clause count is bounded by the window, not by fork count. This is + the property the whole lineage cache exists for, now measured through + real forks instead of hand-built rows.""" + from tools import dbmeter + + ScriptedProvider.replies = ["First take.", "Second take.", "Onward."] + _play(client) + for _ in range(8): + _retry(client) + _play(client, "onward") + discarded = [ + a.id for a in _rows(client.adv_id) + if a.type == "ai" and not a.live + ] + if discarded: + assert _fork(client, discarded[-1]).status_code == 200 + _play(client, "onward") + branches = _branches(client) + assert len(branches) > 4, "the fixture did not actually fork" + + meter = dbmeter.Meter() + meter.attach(engine) + try: + with meter.scope("page"): + client.get(f"/api/adventures/{client.adv_id}") + page_bytes = meter.scopes[-1].total.fetched + finally: + meter.detach() + assert page_bytes > 0, "the meter saw nothing" + + db = SessionLocal() + try: + adventure = db.get(models.Adventure, client.adv_id) + entries = lineage.entries_of(lineage.branch_of(db, adventure)) + # The whole ancestry is available to be named. + assert len(entries) == len(branches) + # The windowed read names as few of them as the window needs. + path = lineage.path_of(db, adventure) + assert path.prefix_covering(60) <= len(entries) + finally: + db.close() + + +def test_switching_to_a_branch_of_another_adventure_is_a_404(client): + """A branch id names one adventure, so the two ids in the URL must + agree. Otherwise a guessed number reads somebody else's story.""" + other = client.post("/api/adventures", json={"title": "Elsewhere"}).json()["id"] + ScriptedProvider.replies = ["Elsewhere."] + r = client.post(f"/api/adventures/{other}/actions", json={"type": "do", "text": "wait"}) + assert r.status_code == 200, r.text + stranger = client.get(f"/api/adventures/{other}/branches").json() + assert len(stranger) == 1 + + assert client.post( + f"/api/adventures/{client.adv_id}/branches/{stranger[0]['id']}/switch" + ).status_code == 404 diff --git a/backend/tests/test_branch_management.py b/backend/tests/test_branch_management.py new file mode 100644 index 0000000..9d94090 --- /dev/null +++ b/backend/tests/test_branch_management.py @@ -0,0 +1,494 @@ +"""Phase 14 SP7: renaming a branch and deleting one. + +SP5 gave the tree a fork and a switch. Neither operation removes anything, +and nothing in the design prunes a tree automatically, so an adventure that +is retried and forked enough grows without a limit. Delete is what limits +that growth. It ships with the same view that first makes a fork reachable, +rather than in a later subphase. + +Two rules carry most of this file: + +* A name is chosen, so the database stores it. A label is derived, so the + database does not. An unnamed branch keeps NULL, and the client draws the + label from its fork depth. A generated label such as "branch 4" stored in + the column would become wrong the moment branch 3 is deleted. +* Delete must never remove a branch the reader depends on. Refusing to + delete the head is the obvious case. Refusing to delete an ancestor of the + head is the same rule applied one level up: deleting it would leave + `head_branch_id` pointing at a row the cascade removed. + + python -m pytest tests/test_branch_management.py -v +""" +import pytest +from fastapi import Depends +from fastapi.testclient import TestClient +from sqlalchemy import select + +from app import auth, limits, models, schemas +from app.context import cursors, lineage +from app.database import Base, SessionLocal, engine, get_db +from app.main import app +from app.routers import adventures + +from fakes import ScriptedProvider + + +@pytest.fixture() +def client(monkeypatch): + Base.metadata.create_all(bind=engine) + setup = SessionLocal() + user = models.User(is_guest=False, email="branches@example.com") + setup.add(user) + setup.flush() + setup.add(models.Settings(user_id=user.id, api_key="enc:dummy", model="test-model")) + adv = models.Adventure( + user_id=user.id, title="Cave", script_state={}, world_state={}, + ) + setup.add(adv) + setup.flush() + setup.add(models.Action( + adventure_id=adv.id, type="start", text="You enter a cave.")) + setup.commit() + adv_id, user_id = adv.id, user.id + setup.close() + + ScriptedProvider.replies = ["Attempt one.", "Attempt two.", "Next turn."] + ScriptedProvider.calls = 0 + monkeypatch.setattr(adventures.turns, "OpenAICompatibleProvider", ScriptedProvider) + monkeypatch.setattr(auth, "resolve_provider_config", lambda s: auth.ProviderConfig( + "http://fake", "k", "test-model", False)) + monkeypatch.setattr(limits, "rate_limit", lambda *a, **k: None) + monkeypatch.setattr(limits, "check_row_cap", lambda *a, **k: None) + + def _current_user(db=Depends(get_db)): + return db.get(models.User, user_id) + + app.dependency_overrides[auth.get_current_user] = _current_user + c = TestClient(app) + c.adv_id = adv_id + try: + yield c + finally: + app.dependency_overrides.clear() + adventures.turns._active_turns.clear() + Base.metadata.drop_all(bind=engine) + + +# ------------------------------------------------------------------ helpers + +def _play(client, text="look around"): + r = client.post(f"/api/adventures/{client.adv_id}/actions", + json={"type": "do", "text": text}) + assert r.status_code == 200, r.text + + +def _retry(client): + r = client.post(f"/api/adventures/{client.adv_id}/retry") + assert r.status_code == 200, r.text + + +def _branches(client) -> list[dict]: + r = client.get(f"/api/adventures/{client.adv_id}/branches") + assert r.status_code == 200, r.text + return r.json() + + +def _rename(client, branch_id, name): + return client.patch(f"/api/adventures/{client.adv_id}/branches/{branch_id}", + json={"name": name}) + + +def _delete(client, branch_id): + return client.delete(f"/api/adventures/{client.adv_id}/branches/{branch_id}") + + +def _switch(client, branch_id): + return client.post(f"/api/adventures/{client.adv_id}/branches/{branch_id}/switch") + + +def _texts(client) -> list[str]: + return [ + a["text"] + for a in client.get(f"/api/adventures/{client.adv_id}").json()["actions"] + ] + + +def _discarded_on(adv_id, branch_id=None) -> int: + """An AI attempt with no turn built on it, optionally restricted to one branch.""" + db = SessionLocal() + try: + q = db.query(models.Action).filter( + models.Action.adventure_id == adv_id, + models.Action.type == "ai", + models.Action.live.is_(False), + ) + if branch_id is not None: + q = q.filter(models.Action.branch_id == branch_id) + return q.order_by(models.Action.id).first().id + finally: + db.close() + + +def _forked(client): + """A story with one fork. Returns (root id, forked id). The fork is head. + + start · do · [attempt one | ATTEMPT TWO] · do · next turn + └── forked here + """ + _play(client) + _retry(client) + _play(client, "go deeper") + root = _branches(client)[0]["id"] + r = client.post( + f"/api/adventures/{client.adv_id}/actions/{_discarded_on(client.adv_id)}/fork") + assert r.status_code == 200, r.text + forked = [b for b in _branches(client) if b["id"] != root][0]["id"] + return root, forked + + +def _counts(adv_id, branch_ids): + db = SessionLocal() + try: + return ( + db.query(models.Action) + .filter(models.Action.branch_id.in_(branch_ids)).count(), + db.query(models.Memory) + .filter(models.Memory.branch_id.in_(branch_ids)).count(), + ) + finally: + db.close() + + +# ------------------------------------------------------------------ naming + +def test_a_branch_starts_unnamed(client): + """NULL, not a generated label. The client draws a label from the fork depth. + + A name stored here would go stale the moment a branch before it is + deleted and the ordinals shift. + """ + root, forked = _forked(client) + assert [b["name"] for b in _branches(client)] == [None, None] + + +def test_a_name_is_stored_and_read_back(client): + root, forked = _forked(client) + r = _rename(client, forked, "the cellar") + assert r.status_code == 200, r.text + assert r.json()["name"] == "the cellar" + assert {b["id"]: b["name"] for b in _branches(client)} == { + root: None, forked: "the cellar", + } + + +def test_a_blank_name_goes_back_to_unnamed(client): + """A name of spaces is not a name anyone chose. + + Storing it would give the client an empty label instead of falling back + to the fork depth. The branch would look nameless and appear broken. + """ + root, forked = _forked(client) + _rename(client, forked, "briefly named") + assert _rename(client, forked, " ").json()["name"] is None + assert _rename(client, forked, None).json()["name"] is None + + +def test_a_name_longer_than_the_column_is_refused(client): + """422 here rather than a 500 at INSERT: Postgres enforces VARCHAR(80).""" + root, forked = _forked(client) + assert _rename(client, forked, "x" * (schemas.BRANCH_NAME_MAX + 1)).status_code == 422 + assert _rename(client, forked, "x" * schemas.BRANCH_NAME_MAX).status_code == 200 + + +def test_a_rename_hands_back_the_row_the_listing_would_give(client): + """Renaming a branch does not change how many turns are on it. + + `own_actions` was hard-coded to 0 in this response. That bug stayed + invisible because the panel discards the response body and refetches. + Anything that trusted the reply would show a branch that had just lost + its turns. + """ + root, forked = _forked(client) + listed = {b["id"]: b for b in _branches(client)} + + renamed = _rename(client, forked, "the cellar").json() + + assert renamed["own_actions"] == listed[forked]["own_actions"] + assert renamed["own_actions"] > 0, "the fixture put turns on this branch" + assert renamed["depth"] == listed[forked]["depth"] + assert renamed["is_head"] == listed[forked]["is_head"] + + +def test_naming_a_branch_of_another_adventure_is_a_404(client): + root, forked = _forked(client) + db = SessionLocal() + try: + other = models.Adventure( + user_id=db.get(models.Adventure, client.adv_id).user_id, + title="Elsewhere", script_state={}, world_state={}, + ) + db.add(other) + db.commit() + other_id = other.id + finally: + db.close() + r = client.patch(f"/api/adventures/{other_id}/branches/{forked}", json={"name": "x"}) + assert r.status_code == 404 + + +# ----------------------------------------------------------------- deleting + +def test_the_root_branch_cannot_be_deleted(client): + """It holds the turns every other branch borrows.""" + root, forked = _forked(client) + r = _delete(client, root) + assert r.status_code == 400 + assert "adventure" in r.json()["detail"].lower() + assert len(_branches(client)) == 2 + + +def test_the_branch_being_read_cannot_be_deleted(client): + root, forked = _forked(client) + assert [b["is_head"] for b in _branches(client) if b["id"] == forked] == [True] + r = _delete(client, forked) + assert r.status_code == 400 + assert "switch" in r.json()["detail"].lower() + + +def test_an_ancestor_of_the_branch_being_read_cannot_be_deleted(client): + """Deleting an ancestor of the head is the same mistake as deleting the head. + + `parent_branch_id` cascades, so deleting a branch the head was forked + from would delete the head too, and leave `head_branch_id` pointing at + nothing. + """ + root, forked = _forked(client) + # A fork of the fork, so `forked` is an ancestor of the head rather than + # the head itself. + _retry(client) + _play(client, "press on") + nested = _discarded_on(client.adv_id, branch_id=forked) + r = client.post(f"/api/adventures/{client.adv_id}/actions/{nested}/fork") + assert r.status_code == 200, r.text + assert len(_branches(client)) == 3 + + r = _delete(client, forked) + assert r.status_code == 400 + assert "forked from it" in r.json()["detail"] + assert len(_branches(client)) == 3 + + +def test_deleting_a_branch_leaves_the_line_it_forked_from_untouched(client): + root, forked = _forked(client) + _switch(client, root) + kept = _texts(client) + + assert _delete(client, forked).status_code == 204 + assert [b["id"] for b in _branches(client)] == [root] + assert _texts(client) == kept, "the parent keeps every turn it had" + + +def test_deleting_a_branch_takes_its_nodes_and_its_descendants(client): + """One statement deletes the whole subtree, regardless of depth. The + cascade performs the traversal.""" + root, forked = _forked(client) + _retry(client) + _play(client, "press on") + nested_attempt = _discarded_on(client.adv_id, branch_id=forked) + client.post(f"/api/adventures/{client.adv_id}/actions/{nested_attempt}/fork") + nested = [b["id"] for b in _branches(client) if b["id"] not in (root, forked)][0] + + doomed_actions, _ = _counts(client.adv_id, [forked, nested]) + assert doomed_actions > 0 + root_actions_before, _ = _counts(client.adv_id, [root]) + + _switch(client, root) + assert _delete(client, forked).status_code == 204 + + assert [b["id"] for b in _branches(client)] == [root] + assert _counts(client.adv_id, [forked, nested]) == (0, 0) + assert _counts(client.adv_id, [root])[0] == root_actions_before + + +def test_deleting_a_branch_clears_a_cursor_that_stood_on_it(client): + """Harmless on Postgres, a real bug on SQLite. + + Postgres never reuses a branch id, so a stale anchor simply never + resolves. SQLite assigns the freed id to the next fork. The anchor then + resolves onto a branch it never saw, and reports a stretch of story as + already summarized. That stretch is then permanently excluded from the + memories. + """ + root, forked = _forked(client) + db = SessionLocal() + try: + adventure = db.get(models.Adventure, client.adv_id) + cursors.MEMORY.anchor(adventure, forked, 3) + cursors.SUMMARY.anchor(adventure, root, 1) + db.commit() + finally: + db.close() + + _switch(client, root) + assert _delete(client, forked).status_code == 204 + + db = SessionLocal() + try: + adventure = db.get(models.Adventure, client.adv_id) + assert cursors.MEMORY.stored(adventure) == (None, cursors.NO_DEPTH) + # The cursor on a branch that still exists is left exactly where it was. + assert cursors.SUMMARY.stored(adventure) == (root, 1) + finally: + db.close() + + +def test_deleting_an_unknown_branch_is_a_404(client): + root, forked = _forked(client) + assert _delete(client, forked + 9999).status_code == 404 + + +# --------------------------------------------------------- the bank vs a path + +def _memories(client) -> list[dict]: + r = client.get(f"/api/adventures/{client.adv_id}/memories") + assert r.status_code == 200, r.text + return r.json() + + +def _add_memory(client, text): + r = client.post(f"/api/adventures/{client.adv_id}/memories", json={"text": text}) + assert r.status_code == 201, r.text + return r.json()["id"] + + +def test_a_hand_written_memory_is_anchored_where_it_was_written(client): + """It takes the head, so it is a memory *of a story* rather than of an + adventure. A NULL depth is a coordinate no fork can cap.""" + root, forked = _forked(client) + memory_id = _add_memory(client, "Took the other door.") + + db = SessionLocal() + try: + memory = db.get(models.Memory, memory_id) + adventure = db.get(models.Adventure, client.adv_id) + assert memory.branch_id == forked, "the branch being read" + assert memory.depth is not None, "never NULL again" + assert memory.depth == adventure.head_depth + finally: + db.close() + + +def test_the_drawer_shows_the_path_being_read_and_nothing_else(client): + """The memories a reader can see match the memories the model can retrieve. + + An adventure-wide list would include memories from branches this story + never took. Those memories are never retrieved, and a reader could not + distinguish them from the ones actually in play. + """ + root, forked = _forked(client) + on_the_fork = _add_memory(client, "Took the other door.") + _switch(client, root) + on_the_root = _add_memory(client, "Went the long way instead.") + + assert {m["id"] for m in _memories(client)} == {on_the_root}, \ + "the fork's memory is not on this story" + + # Switching to the fork shows its own memory and the root's. A fork + # borrows its ancestors up to the point where it diverged from them. + # The relationship is asymmetric on purpose: the parent never took the + # fork's path. + _switch(client, forked) + listed = {m["id"] for m in _memories(client)} + assert on_the_fork in listed + assert on_the_root not in listed, "written after the fork left this branch" + + +def test_the_drawer_and_retrieval_agree_on_what_is_visible(client): + """One predicate decides visibility, so a memory can never be listed but + unretrievable, or the reverse. Two separate definitions of "on this + path" would eventually diverge.""" + root, forked = _forked(client) + _add_memory(client, "Took the other door.") + _switch(client, root) + _add_memory(client, "Went the long way instead.") + + db = SessionLocal() + try: + adventure = db.get(models.Adventure, client.adv_id) + retrievable = { + row[0] for row in db.execute( + select(models.Memory.id).where( + models.Memory.adventure_id == adventure.id, + lineage.path_of(db, adventure).clause(models.Memory), + ) + ) + } + finally: + db.close() + + assert {m["id"] for m in _memories(client)} == retrievable + + +def test_deleting_a_branch_deletes_the_memories_written_on_it(client): + """Deleting a branch removes its memories from the database, not just + from view: the cascade deletes the row along with the branch. This + keeps a hidden memory from becoming unreachable in a different way: a + memory you cannot currently see is on a branch you can still switch to, + and deleting that branch deletes the memory permanently.""" + root, forked = _forked(client) + doomed = _add_memory(client, "Took the other door.") + _switch(client, root) + + db = SessionLocal() + try: + assert db.get(models.Memory, doomed) is not None, "still on its own branch" + finally: + db.close() + + assert _delete(client, forked).status_code == 204 + + db = SessionLocal() + try: + assert db.get(models.Memory, doomed) is None, "gone with the branch" + finally: + db.close() + + +# ------------------------------------------------------------------- backup + +def test_a_bundle_carries_the_name_a_player_chose(client): + """A name is a decision, so the export includes it. This is the rule the + v2 format is built on. + + `lineage` and the head depth stay out of the export because the + importer can compute them from what the file already carries. A name + is computed from nothing, so the export must carry it. + """ + root, forked = _forked(client) + _rename(client, root, "the long way") + _rename(client, forked, "the cellar") + + exported = client.get(f"/api/adventures/{client.adv_id}/export").json() + assert [b.get("name") for b in exported["branches"]] == ["the long way", "the cellar"] + + r = client.post("/api/adventures/import", json=exported) + assert r.status_code == 201, r.text + restored = client.get(f"/api/adventures/{r.json()['id']}/branches").json() + assert [b["name"] for b in restored] == ["the long way", "the cellar"] + + +def test_an_unnamed_tree_exports_no_name_key(client): + """Unchanged from the file SP6 wrote, for a tree nobody has named.""" + _forked(client) + exported = client.get(f"/api/adventures/{client.adv_id}/export").json() + assert all("name" not in b for b in exported["branches"]) + + +def test_a_bundle_naming_a_branch_with_a_number_is_refused(client): + """400 from the planner, not a database error three branches in.""" + _forked(client) + exported = client.get(f"/api/adventures/{client.adv_id}/export").json() + exported["branches"][1]["name"] = 7 + r = client.post("/api/adventures/import", json=exported) + assert r.status_code == 400 + assert "not text" in r.json()["detail"] diff --git a/backend/tests/test_bundle_v2.py b/backend/tests/test_bundle_v2.py new file mode 100644 index 0000000..9d97c0c --- /dev/null +++ b/backend/tests/test_bundle_v2.py @@ -0,0 +1,635 @@ +"""Phase 14 SP6: the export bundle carries the tree. + +A bundle is the only part of this phase a migration can never reach: the +file already exists on somebody's disk. So there are two formats, and the +two halves of this file check different properties. + +v2 must be lossless for a story that forked. v1 could not be, because it +stored one list for two stories, interleaved by `index`, which read back +as a mangled story. Losslessness here means the tree: every branch, the +fork point it left on its parent, which attempt at each turn is the +story, and what each node left behind. That last item is what a branch +switch restores, and a tree nobody can switch inside is not the tree +that was exported. + +v1 must still import, because a backup that stops importing is not a +backup. + +Both formats follow one rule: a bundle carries what was chosen and never +what is derived. The lineage, the head depth, the legacy `index`, and the +variant ordinals are all rebuilt on the way in, so a hand-edited file +cannot disagree with itself. The tests that matter most here hand the +importer a file that does disagree with itself. + + python -m pytest tests/test_bundle_v2.py -v +""" +import pytest +from fastapi import Depends +from fastapi.testclient import TestClient + +from app import auth, bundle, limits, models +from app.context import lineage +from app.database import Base, SessionLocal, engine, get_db +from app.main import app +from app.routers import adventures + +from fakes import ScriptedProvider + +SCHEMA = {"player": {"hp": {"min": 0, "max": 100, "initial": 100}}} + +# Ten gold a turn, so the stored gold total tells how many turns the +# story behind it played. This makes an after-snapshot visible from outside. +GOLD_SCRIPT = """ +const modifier = (text) => { + state.gold = (state.gold || 0) + 10; + return { text }; +}; +modifier(text); +""" + +OPENING = "You enter a cave." + + +@pytest.fixture() +def client(monkeypatch): + Base.metadata.create_all(bind=engine) + setup = SessionLocal() + user = models.User(is_guest=False, email="bundle@example.com") + setup.add(user) + setup.flush() + setup.add(models.Settings(user_id=user.id, api_key="enc:dummy", model="test-model")) + scenario = models.Scenario(user_id=user.id, title="S", stat_schema=SCHEMA) + setup.add(scenario) + setup.flush() + adv = models.Adventure( + user_id=user.id, title="Cave", scenario_id=scenario.id, + script_state={}, world_state={"player": {"hp": 100}}, + ) + setup.add(adv) + setup.flush() + setup.add(models.Action(adventure_id=adv.id, type="start", text=OPENING)) + setup.add(models.AdventureScript( + adventure_id=adv.id, position=0, enabled=True, name="Gold", output_js=GOLD_SCRIPT, + )) + setup.commit() + adv_id, user_id = adv.id, user.id + setup.close() + + ScriptedProvider.replies = ["A reply."] + ScriptedProvider.calls = 0 + monkeypatch.setattr(adventures.turns, "OpenAICompatibleProvider", ScriptedProvider) + monkeypatch.setattr(auth, "resolve_provider_config", lambda s: auth.ProviderConfig( + "http://fake", "k", "test-model", False)) + monkeypatch.setattr(limits, "rate_limit", lambda *a, **k: None) + monkeypatch.setattr(limits, "check_row_cap", lambda *a, **k: None) + + def _current_user(db=Depends(get_db)): + return db.get(models.User, user_id) + + app.dependency_overrides[auth.get_current_user] = _current_user + c = TestClient(app) + c.adv_id = adv_id + try: + yield c + finally: + app.dependency_overrides.clear() + adventures.turns._active_turns.clear() + Base.metadata.drop_all(bind=engine) + + +# ------------------------------------------------------------------ helpers + +def _play(client, adv_id, text="look around", type="do"): + r = client.post(f"/api/adventures/{adv_id}/actions", json={"type": type, "text": text}) + assert r.status_code == 200, r.text + + +def _retry(client, adv_id): + assert client.post(f"/api/adventures/{adv_id}/retry").status_code == 200 + + +def _export(client, adv_id) -> dict: + r = client.get(f"/api/adventures/{adv_id}/export") + assert r.status_code == 200, r.text + return r.json() + + +def _import(client, payload): + return client.post("/api/adventures/import", json=payload) + + +def _imported(client, payload) -> int: + r = _import(client, payload) + assert r.status_code == 201, r.text + return r.json()["id"] + + +def _branches(client, adv_id) -> list[dict]: + r = client.get(f"/api/adventures/{adv_id}/branches") + assert r.status_code == 200, r.text + return r.json() + + +def _switch(client, adv_id, branch_id): + r = client.post(f"/api/adventures/{adv_id}/branches/{branch_id}/switch") + assert r.status_code == 200, r.text + + +def _texts(client, adv_id) -> list[str]: + return [a["text"] for a in client.get(f"/api/adventures/{adv_id}").json()["actions"]] + + +def _every_branch_story(client, adv_id) -> list[list[str]]: + """What each branch tells, in branch order. This is the whole tree as text.""" + stories = [] + for branch in _branches(client, adv_id): + _switch(client, adv_id, branch["id"]) + stories.append(_texts(client, adv_id)) + return stories + + +def _adventure_count() -> int: + db = SessionLocal() + try: + return db.query(models.Adventure).count() + finally: + db.close() + + +def _rows(adv_id) -> list[models.Action]: + db = SessionLocal() + try: + return ( + db.query(models.Action) + .filter(models.Action.adventure_id == adv_id) + .order_by(models.Action.branch_id, models.Action.depth, + models.Action.id) + .all() + ) + finally: + db.close() + + +def _branch_rows(adv_id) -> list[models.Branch]: + db = SessionLocal() + try: + return ( + db.query(models.Branch) + .filter(models.Branch.adventure_id == adv_id) + .order_by(models.Branch.id) + .all() + ) + finally: + db.close() + + +def _script_state(adv_id) -> dict: + db = SessionLocal() + try: + return db.get(models.Adventure, adv_id).script_state + finally: + db.close() + + +def _forked_story(client) -> int: + """A story that went two ways and stayed both. + + root: start > do > [attempt two] > do > next turn + fork: [ATTEMPT ONE] > do > elsewhere + + The discarded attempt is the one this function promotes, because a + fork moves the attempt being left for and leaves the line it came + from untouched. + + Returns the adventure id, with the head on the fork. + """ + adv_id = client.adv_id + ScriptedProvider.replies = ["Attempt one.", "Attempt two.", "Next turn.", "Elsewhere."] + _play(client, adv_id) + _retry(client, adv_id) + _play(client, adv_id, "go deeper") + discarded = [a.id for a in _rows(adv_id) if a.type == "ai" and not a.live][0] + assert client.post(f"/api/adventures/{adv_id}/actions/{discarded}/fork").status_code == 200 + _play(client, adv_id, "go sideways") + return adv_id + + +# ------------------------------------------------------- the round trip (v2) + +def test_a_forked_story_survives_the_round_trip(client): + """The main claim: both futures come back, and both are still readable. + + This is the thing v1 could not do. The check is not "the same rows", + since the ids are new, but "the same stories", read the way a player + reads them: by switching to a branch and looking at what it says. + """ + original = _forked_story(client) + before = _every_branch_story(client, original) + assert len(before) == 2, "the fixture forked" + assert before[0] != before[1], "and the two branches tell different stories" + + copy = _imported(client, _export(client, original)) + assert copy != original + assert _every_branch_story(client, copy) == before + + +def test_the_fork_point_comes_back_where_it_was_put(client): + """`fork_depth` is stored, never inferred, including through a file. + + Inferring it from where two branches' nodes first differ would guess + at how the story was played, and that guess fails as soon as an + attempt happens to repeat its parent's text. + """ + original = _forked_story(client) + before = [(b["parent_branch_id"] is None, b["fork_depth"]) for b in _branches(client, original)] + + copy = _imported(client, _export(client, original)) + assert [(b["parent_branch_id"] is None, b["fork_depth"]) + for b in _branches(client, copy)] == before + + +def test_the_head_comes_back_on_the_branch_it_was_left_on(client): + original = _forked_story(client) + head_before = [b["is_head"] for b in _branches(client, original)] + assert head_before == [False, True], "the fixture left the head on the fork" + + copy = _imported(client, _export(client, original)) + assert [b["is_head"] for b in _branches(client, copy)] == head_before + # The tip it sits at is derived from the nodes that arrived, not + # read from the file. The bundle never states how deep a branch goes. + assert _texts(client, copy) == _texts(client, original) + + +def test_a_switch_in_the_copy_restores_what_that_branch_left_behind(client): + """This test justifies why the bundle carries after-snapshots. + + The gold script adds ten a turn, so the stored gold total counts the + turns behind it. A bundle that carried the actions but not the + outcomes would import a tree that reads correctly but switches to the + wrong state. + """ + original = _forked_story(client) + # Play one more turn on the fork, so the two tips end up at + # genuinely different totals. Turn for turn, both branches earn the + # same gold, so a switch that restored nothing would still look right. + ScriptedProvider.replies = ["Further still."] + _play(client, original, "press on") + + per_branch = [] + for branch in _branches(client, original): + _switch(client, original, branch["id"]) + per_branch.append(_script_state(original).get("gold")) + assert len(set(per_branch)) == len(per_branch), "the tips are at different totals" + + copy = _imported(client, _export(client, original)) + restored = [] + for branch in _branches(client, copy): + _switch(client, copy, branch["id"]) + restored.append(_script_state(copy).get("gold")) + assert restored == per_branch + + +def test_a_memory_comes_back_on_the_node_it_hangs_off(client): + """Derived work is addressed by coordinate, so the coordinate is carried.""" + original = _forked_story(client) + tip = [a for a in _rows(original) if a.live][-1] + db = SessionLocal() + try: + db.add(models.Memory( + adventure_id=original, text="They met a goblin.", + source_start=0, source_end=tip.depth, + branch_id=tip.branch_id, depth=tip.depth, + )) + db.commit() + finally: + db.close() + + exported = _export(client, original) + assert [(m["branch"], m["depth"]) for m in exported["memories"]] == [(1, tip.depth)] + + copy = _imported(client, exported) + db = SessionLocal() + try: + memories = db.query(models.Memory).filter( + models.Memory.adventure_id == copy).all() + branches = [b.id for b in _branch_rows(copy)] + assert [(branches.index(m.branch_id), m.depth) for m in memories] == [(1, tip.depth)] + finally: + db.close() + + +# --------------------------------------------------- what is not in the file + +def test_the_lineage_is_rebuilt_rather_than_carried(client): + """A cache of `parent` plus `fork_depth` is not a second thing to ship. + + The file states where each branch forked. The ancestry that makes the + fork readable is computed from that value on the way in, capped at + the fork exactly as `tree.fork` caps it. Shipping the cache too would + put two sources of truth for one fact in a file anybody can hand-edit. + """ + original = _forked_story(client) + exported = _export(client, original) + assert all("lineage" not in b for b in exported["branches"]) + + root, forked = _branch_rows(_imported(client, exported)) + assert root.lineage == [[root.id, None]] + assert forked.lineage == [[forked.id, None], [root.id, forked.fork_depth]] + # This is the arithmetic the reader depends on. The parent is capped + # one depth short of the attempt that was promoted, so the fork + # cannot see it. + assert lineage.entries_of(forked) == [(forked.id, None), (root.id, forked.fork_depth)] + + +def test_two_branches_each_keep_their_own_node_at_one_depth(client): + """A depth describes a path, not the adventure. + + The fork and the branch it left both hold a turn at depth 2, and they are + not the same turn. The import used to issue a global `index` per turn as + well, so that every coordinate had a number nothing else held. SP8 dropped + that column, so the coordinate is all there is, and it has to survive the + round trip on its own. + """ + copy = _imported(client, _export(client, _forked_story(client))) + rows = _rows(copy) + + at_depth_2 = [r for r in rows if r.depth == 2] + assert len({r.branch_id for r in at_depth_2}) == 2 + + +# ------------------------------------------------------- a file that is wrong + +def test_a_node_naming_a_branch_the_file_does_not_list_is_refused(client): + """The import must refuse this file rather than half-apply it. A tree + missing a branch is a story that silently stops, which is the failure + this whole phase exists to prevent.""" + payload = _export(client, _forked_story(client)) + payload["branches"] = payload["branches"][:1] + before = _adventure_count() + + r = _import(client, payload) + assert r.status_code == 400, r.text + assert "branch" in r.json()["detail"].lower() + assert _adventure_count() == before, "nothing was created" + + +def test_a_branch_forking_from_one_listed_after_it_is_refused(client): + """The ordering rule guarantees no cycles at the cost of one + comparison. Without it, a cycle would produce an import that never + returns instead of one that fails cleanly.""" + payload = _export(client, _forked_story(client)) + payload["branches"] = [{"parent": 1, "forkDepth": 0}, {"parent": None, "forkDepth": None}] + before = _adventure_count() + + r = _import(client, payload) + assert r.status_code == 400, r.text + assert _adventure_count() == before + + +def test_a_fork_with_no_depth_is_refused(client): + payload = _export(client, _forked_story(client)) + payload["branches"][1].pop("forkDepth") + before = _adventure_count() + + r = _import(client, payload) + assert r.status_code == 400, r.text + assert "depth" in r.json()["detail"].lower() + assert _adventure_count() == before + + +def test_more_branches_than_the_cap_is_refused(client, monkeypatch): + monkeypatch.setattr(auth, "MULTI_USER", True) + payload = { + "format": bundle.FORMAT, "title": "Too many", + "branches": [{"parent": None, "forkDepth": None}] + * (limits.MAX_BRANCHES_PER_ADVENTURE + 1), + "actions": [], + } + before = _adventure_count() + + r = _import(client, payload) + assert r.status_code == 409, r.text + assert _adventure_count() == before + + +def test_a_turn_the_file_gives_no_live_attempt_still_tells_one(client): + """A coordinate with nothing live is a turn no read can see. + + The file is allowed to be wrong about this, since it is a text file + someone can edit. The import picks the first attempt instead of + importing a story with a gap. + """ + payload = _export(client, _forked_story(client)) + for node in payload["actions"]: + node["live"] = False + copy = _imported(client, payload) + + assert _texts(client, copy), "the story is readable" + live = [(r.branch_id, r.depth) for r in _rows(copy) if r.live] + assert len(live) == len(set(live)), "exactly one live attempt per coordinate" + assert len(live) == len({(r.branch_id, r.depth) for r in _rows(copy)}) + + +# ------------------------------------------------------------- the v1 reader + +def test_a_v1_bundle_still_imports(client): + """The v1 reader must remain even after the v1 writer is gone, because + those files already exist and are saved.""" + payload = { + "format": bundle.LEGACY_FORMAT, + "title": "Old backup", + "memoryCursor": 0, "summaryCursor": 0, + "actions": [ + {"index": 0, "type": "start", "text": OPENING}, + {"index": 1, "type": "do", "text": "> You go north."}, + { + "index": 2, "type": "ai", "text": "Two.", + "variants": [{"text": "One."}, {"text": "Two."}], + "variantIndex": 1, + }, + ], + } + copy = _imported(client, payload) + + assert _texts(client, copy) == [OPENING, "> You go north.", "Two."] + # The import produces one branch, and the `variants` array splits + # back into the sibling group it always described. + assert len(_branches(client, copy)) == 1 + ai = [r for r in _rows(copy) if r.type == "ai"] + assert [(r.text, r.live) for r in ai] == [("One.", False), ("Two.", True)] + assert len({(r.branch_id, r.depth) for r in ai}) == 1 + + +def test_a_v1_bundle_with_a_cursor_lands_it_on_a_node(client): + """v1 counts covered actions. The tree anchors them to a node instead. + The translation needs the nodes to exist first, so it runs after they + are written.""" + payload = { + "format": bundle.LEGACY_FORMAT, "title": "Old backup", + "memoryCursor": 2, "summaryCursor": 2, + "actions": [ + {"index": 0, "type": "start", "text": OPENING}, + {"index": 1, "type": "story", "text": "A corridor."}, + {"index": 2, "type": "story", "text": "A door."}, + ], + } + copy = _imported(client, payload) + + db = SessionLocal() + try: + adventure = db.get(models.Adventure, copy) + assert adventure.memory_cursor_branch_id is not None + assert adventure.memory_cursor_depth == 1, "the second story action" + finally: + db.close() + + +def test_a_v2_bundle_brings_its_anchors_back(client): + """The other direction: v2 carries the anchor directly.""" + original = _forked_story(client) + tip = [a for a in _rows(original) if a.live][-1] + db = SessionLocal() + try: + adventure = db.get(models.Adventure, original) + adventure.memory_cursor_branch_id = tip.branch_id + adventure.memory_cursor_depth = tip.depth + db.commit() + finally: + db.close() + + exported = _export(client, original) + assert exported["memoryCursor"] == {"branch": 1, "depth": tip.depth} + + copy = _imported(client, exported) + db = SessionLocal() + try: + adventure = db.get(models.Adventure, copy) + branches = [b.id for b in _branch_rows(copy)] + assert branches.index(adventure.memory_cursor_branch_id) == 1 + assert adventure.memory_cursor_depth == tip.depth + finally: + db.close() + + +def test_a_v1_memory_that_summarises_nothing_lands_on_the_root(client): + """The import must answer the question migration 62 already answered. + + A v1 file has no depths, and a memory the player typed has no + `sourceEnd` to derive one from. It used to come back with a NULL + depth, the exact state SP7 removed from the schema. + `Path._entry_clause` compares `depth <= max_depth`, and a NULL fails + that comparison. The memory would read fine until the imported + adventure forked, then vanish from the new branch. + """ + payload = { + "format": bundle.LEGACY_FORMAT, "title": "Old backup", + "memoryCursor": 0, "summaryCursor": 0, + "actions": [ + {"index": 0, "type": "start", "text": OPENING}, + {"index": 1, "type": "story", "text": "A corridor."}, + ], + "memories": [ + {"text": "Kira is the innkeeper's daughter"}, # typed + {"text": "The corridor, summarised", "sourceStart": 1, "sourceEnd": 1}, + ], + } + copy = _imported(client, payload) + + db = SessionLocal() + try: + rows = {m.text: m for m in db.query(models.Memory) + .filter(models.Memory.adventure_id == copy).all()} + assert rows["Kira is the innkeeper's daughter"].depth == 0, ( + "a typed memory anchors at the root, which every branch can see" + ) + assert rows["The corridor, summarised"].depth == 1, "derived from its range" + assert all(m.depth is not None for m in rows.values()) + assert all(m.branch_id is not None for m in rows.values()) + finally: + db.close() + + +def test_the_action_cap_counts_the_rows_a_v1_file_expands_into(client, monkeypatch): + """The cap must count what gets written, not what the file lists. + + A v1 turn carries its retries in a `variants` array, and SP4 made + every attempt a row, so one entry can expand into ten. Counting + entries instead of rows would let a file inside the cap write a + multiple of it. The body-size limit does not help here: the text is + tiny, and the row count is the actual cost. + """ + monkeypatch.setattr(auth, "MULTI_USER", True) + monkeypatch.setattr(limits, "MAX_ACTIONS_PER_ADVENTURE", 6) + monkeypatch.setattr(limits, "_BUNDLE_LIST_CAPS", + {**limits._BUNDLE_LIST_CAPS, "actions": 6}) + payload = { + "format": bundle.LEGACY_FORMAT, "title": "Small file, many rows", + "memoryCursor": 0, "summaryCursor": 0, + "actions": [{"index": 0, "type": "start", "text": OPENING}] + [ + { + "index": i, "type": "ai", "text": "Take four.", + "variants": [{"text": f"Take {n}."} for n in range(4)], + "variantIndex": 3, + } + for i in range(1, 4) + ], + } + assert len(payload["actions"]) <= 6, "the file itself is inside the cap" + before = _adventure_count() + + r = _import(client, payload) + assert r.status_code == 409, r.text + assert _adventure_count() == before, "and nothing was written" + + +def test_an_unknown_format_is_refused(client): + r = _import(client, {"format": "ai-dnd-adventure-v3", "title": "From the future"}) + assert r.status_code == 400, r.text + assert bundle.FORMAT in r.json()["detail"] + + +# ------------------------------------------------------- the persona (Phase 18) + +def test_the_persona_survives_the_round_trip(client): + adv_id = client.adv_id + assert client.patch(f"/api/adventures/{adv_id}", json={ + "persona_name": "Kaelen", + "persona_pronouns": "he/him", + "persona_desc": "A half-elf ranger, exiled from the northern holds.", + }).status_code == 200 + + payload = _export(client, adv_id) + assert payload["persona"] == { + "name": "Kaelen", + "pronouns": "he/him", + "desc": "A half-elf ranger, exiled from the northern holds.", + } + + copy_id = _imported(client, payload) + r = client.get(f"/api/adventures/{copy_id}") + assert r.status_code == 200, r.text + copy = r.json() + assert copy["persona_name"] == "Kaelen" + assert copy["persona_pronouns"] == "he/him" + assert copy["persona_desc"].startswith("A half-elf ranger") + + +def test_a_bundle_written_before_personas_still_imports(client): + """The key is read with `.get`, so a v2 file from before Phase 18 lands + with an empty persona rather than failing. No FORMAT bump was needed.""" + payload = _export(client, client.adv_id) + del payload["persona"] + copy = client.get(f"/api/adventures/{_imported(client, payload)}").json() + assert copy["persona_name"] == "" + assert copy["persona_pronouns"] == "" + assert copy["persona_desc"] == "" + + +def test_an_over_long_persona_in_a_raw_bundle_is_truncated(client): + """A raw-dict import bypasses the schemas, so the widths the columns + declare are enforced here instead. Postgres would reject the INSERT.""" + payload = _export(client, client.adv_id) + payload["persona"] = {"name": "K" * 500, "pronouns": "p" * 500, "desc": "d"} + copy = client.get(f"/api/adventures/{_imported(client, payload)}").json() + assert len(copy["persona_name"]) == 80 + assert len(copy["persona_pronouns"]) == 40 diff --git a/backend/tests/test_cast_brief.py b/backend/tests/test_cast_brief.py new file mode 100644 index 0000000..1a7e157 --- /dev/null +++ b/backend/tests/test_cast_brief.py @@ -0,0 +1,234 @@ +"""Phase 18b: what the summarizer is told about who the characters are. + +Before this, `_create_due_memories` sent six actions of second-person prose and +nothing else. The model had no way to know who "you" was or who "she" was, so +neither did any memory it wrote, and the summary built from those memories +inherited the problem. + +The rules this file holds in place: + +* The brief carries fixed descriptions only. A live stat in it would make the + same event, summarized twice, come out framed differently — the exact fault + the change exists to remove. +* Places and items are not topped up into the roster. Only characters are. +* An adventure with no persona and no cards still summarizes, with the prompt + it had before. + + python -m pytest tests/test_cast_brief.py -v +""" +import asyncio + +import pytest + +from app import memorybank, models +from app.database import Base, SessionLocal, engine + +GWEN = "A loyal ranger and the player's ally. Quick with a bow, fiercely protective." +LEADER = "Scarred, patient, and the one holding the strongbox key." +TAVERN = "A tavern three days south of the camp." + + +class StubSummarizer: + """Records every (system, user) pair handed to the summarizer.""" + + def __init__(self): + self.calls: list[tuple[str, str]] = [] + + async def complete(self, system, user, **kwargs): + self.calls.append((system, user)) + return f"Memory {len(self.calls)}." + + +@pytest.fixture() +def db(): + Base.metadata.create_all(bind=engine) + session = SessionLocal() + try: + yield session + finally: + session.close() + Base.metadata.drop_all(bind=engine) + + +def make_adventure(db, *, actions=0, persona=True, cards=True, memory=True): + user = models.User(is_guest=False, email="cast@example.com") + db.add(user) + db.flush() + db.add(models.Settings(user_id=user.id, api_key="enc:dummy", model="test-model")) + adventure = models.Adventure( + user_id=user.id, title="Camp", script_state={}, auto_summarize=True, + memory=("The player and Gwen are raiding a bandit camp to recover a " + "stolen strongbox.") if memory else "", + persona_name="Kaelen" if persona else "", + persona_pronouns="he/him" if persona else "", + persona_desc="A half-elf ranger, exiled from the northern holds." if persona else "", + ) + db.add(adventure) + db.flush() + if cards: + db.add(models.StoryCard(adventure_id=adventure.id, name="Gwen", + keys="Gwen, ranger, her", type="character", entry=GWEN)) + db.add(models.StoryCard(adventure_id=adventure.id, name="Bandit Leader", + keys="Bandit Leader, leader", type="character", entry=LEADER)) + db.add(models.StoryCard(adventure_id=adventure.id, name="The Rusted Tankard", + keys="tankard, tavern", type="location", entry=TAVERN)) + for i in range(actions): + db.add(models.Action(adventure_id=adventure.id, + type="ai" if i % 2 else "do", + text=f"You walk on. Action {i}.")) + db.commit() + db.refresh(adventure) + return adventure + + +# ------------------------------------------------------------ the brief itself + +def test_the_protagonist_is_named_and_marked_as_such(db): + adventure = make_adventure(db) + brief = memorybank.cast_brief(adventure, "You walk into the camp.") + assert "- Kaelen (he/him) — the protagonist." in brief + assert "A half-elf ranger" in brief + + +def test_a_named_character_in_the_text_is_in_the_roster(db): + adventure = make_adventure(db) + brief = memorybank.cast_brief(adventure, "Gwen checks her bowstring.") + assert "- Gwen — " in brief and "Quick with a bow" in brief + + +def test_a_character_referred_to_only_by_pronoun_is_still_in_the_roster(db): + """The block that most needs a cast is the one written in bare pronouns. + Keyword matching alone finds nothing here, so the roster is topped up.""" + adventure = make_adventure(db) + brief = memorybank.cast_brief(adventure, "She grabs your arm before you step through.") + assert "- Gwen — " in brief + assert "- Bandit Leader — " in brief + + +def test_places_are_not_topped_up(db): + """An unmentioned tavern is not who "she" was.""" + adventure = make_adventure(db) + brief = memorybank.cast_brief(adventure, "She grabs your arm.") + assert "Rusted Tankard" not in brief + + +def test_a_place_that_is_mentioned_does_appear(db): + """Topping up is limited to characters. Matching is not.""" + adventure = make_adventure(db) + brief = memorybank.cast_brief(adventure, "You push into the tavern, breathless.") + assert "The Rusted Tankard" in brief + + +def test_the_setting_is_the_plot_essentials(db): + adventure = make_adventure(db) + brief = memorybank.cast_brief(adventure, "You walk on.") + assert "Setting:\nThe player and Gwen are raiding a bandit camp" in brief + + +def test_no_persona_and_no_cards_gives_no_brief(db): + adventure = make_adventure(db, persona=False, cards=False, memory=False) + assert memorybank.cast_brief(adventure, "You walk on.") == "" + + +def test_a_persona_alone_is_enough_for_a_brief(db): + adventure = make_adventure(db, cards=False, memory=False) + brief = memorybank.cast_brief(adventure, "You walk on.") + assert brief.startswith("Cast:\n- Kaelen (he/him) — the protagonist.") + + +def test_an_unnamed_protagonist_is_called_the_player(db): + adventure = make_adventure(db, cards=False, memory=False) + adventure.persona_name = "" + adventure.persona_pronouns = "" + db.commit() + assert "- The player — the protagonist." in memorybank.cast_brief(adventure, "x") + + +def test_the_roster_is_capped(db, monkeypatch): + monkeypatch.setattr(memorybank, "MAX_CAST_MEMBERS", 2) + adventure = make_adventure(db) + brief = memorybank.cast_brief(adventure, "She grabs your arm.") + assert len([ln for ln in brief.splitlines() if ln.startswith("- ")]) == 2 + + +def test_a_long_description_is_trimmed(db): + adventure = make_adventure(db, cards=False, memory=False) + adventure.persona_desc = "word " * 400 + db.commit() + brief = memorybank.cast_brief(adventure, "x") + assert "…" in brief + assert len(brief) < 600 + + +def test_a_character_is_never_listed_twice(db): + """The persona and a card can carry the same name.""" + adventure = make_adventure(db, cards=False, memory=False) + db.add(models.StoryCard(adventure_id=adventure.id, name="Kaelen", + keys="Kaelen", type="character", entry="Also Kaelen.")) + db.commit() + db.refresh(adventure) + brief = memorybank.cast_brief(adventure, "Kaelen walks on.") + assert brief.lower().count("kaelen") == 1 + + +def test_the_brief_holds_no_live_values(db): + """Fixed descriptions only. A stat here would frame the same event two ways + depending on when it happened to be summarized.""" + adventure = make_adventure(db) + brief = memorybank.cast_brief(adventure, "Gwen checks her bowstring.") + for live in ("hp", "trust", "/100", "wary", "healthy"): + assert live not in brief.lower() + + +# ------------------------------------------------- what actually gets sent + +def _run_memory_pass(db, adventure, monkeypatch, stub): + monkeypatch.setattr(memorybank, "MEMORY_START", 0) + monkeypatch.setattr(memorybank, "MEMORY_INTERVAL", 4) + monkeypatch.setattr(memorybank, "MAX_MEMORIES_PER_RUN", 1) + monkeypatch.setattr(memorybank, "summary_provider", lambda s: stub) + settings = db.query(models.Settings).first() + asyncio.run(memorybank._create_due_memories(adventure, settings, db)) + assert stub.calls, "the summarizer was never called" + return stub.calls[0] + + +def test_the_memory_prompt_carries_the_brief_above_the_excerpt(db, monkeypatch): + adventure = make_adventure(db, actions=6) + system, user = _run_memory_pass(db, adventure, monkeypatch, StubSummarizer()) + assert user.index("Cast:") < user.index("Story excerpt:") + assert "Kaelen (he/him) — the protagonist" in user + assert "Setting:" in user + + +def test_the_memory_prompt_demands_third_person_and_names(db, monkeypatch): + adventure = make_adventure(db, actions=6) + system, _ = _run_memory_pass(db, adventure, monkeypatch, StubSummarizer()) + assert "third person" in system + assert 'never as "you"' in system + + +def test_an_adventure_with_nothing_to_say_sends_the_old_prompt(db, monkeypatch): + """No persona, no cards, no plot essentials: the user message is exactly + what it was before this change, with no stray blank lines.""" + adventure = make_adventure(db, actions=6, persona=False, cards=False, memory=False) + _, user = _run_memory_pass(db, adventure, monkeypatch, StubSummarizer()) + assert user.startswith("Story excerpt:") + assert "Cast:" not in user + + +def test_the_summary_prompt_carries_the_brief_too(db, monkeypatch): + """It builds from the memories, so it inherits their framing — but the + fallback hands it raw second-person story text, which needs the brief.""" + adventure = make_adventure(db, actions=20) + stub = StubSummarizer() + monkeypatch.setattr(memorybank, "SUMMARY_INTERVAL", 1) + monkeypatch.setattr(memorybank, "summary_provider", lambda s: stub) + settings = db.query(models.Settings).first() + asyncio.run(memorybank._update_story_summary(adventure, settings, db)) + assert stub.calls, "the summarizer was never called" + system, user = stub.calls[0] + assert user.index("Cast:") < user.index("Current story summary:") + assert "Kaelen (he/him) — the protagonist" in user + assert "third person" in system + assert adventure.story_summary == "Memory 1." diff --git a/backend/tests/test_change_visibility.py b/backend/tests/test_change_visibility.py new file mode 100644 index 0000000..fa9d07c --- /dev/null +++ b/backend/tests/test_change_visibility.py @@ -0,0 +1,261 @@ +"""Tests for the changes the engine refuses, and for naming a milestone. + +A refused change used to leave no trace a player could see. The turn summary +read `world_delta["applied"]` alone, so a value the model pushed past its +ceiling came back with a delta of 0 and rendered as an ordinary chip. The +engine had recorded the refusal and nothing showed it. + +The milestone half is the same shape. `apply_delta` matches a milestone by its +schema id, and the context named goals by description only, so the model had no +way to learn the id it was being asked to send. + + python -m pytest tests/test_change_visibility.py -v +""" +import json +import pathlib + +from app import models +from app import worldstate as w +from app.routers.adventures import world_delta_of + +SEED = pathlib.Path(__file__).resolve().parents[1] / "app" / "seed_data" + +SCHEMA = { + "world": {"day": {"type": "counter", "min": 1, "initial": 1}}, + "player": { + "hp": {"min": 0, "max": 100, "initial": 100, "max_delta_per_turn": 30}, + # Starts at its own ceiling and only ever falls, which is the shape that + # turns a wrong-signed delta into a change of nothing. + "arrows": {"min": 0, "max": 6, "initial": 6, "max_delta_per_turn": 1}, + }, + "npcs": {"gwen": {"name": "Gwen", "stats": {"trust": {"min": -100, "max": 100, "initial": 0}}}}, + "flags": {"has_key": {"desc": "Holds the key", "initial": False}}, + "milestones": { + "rescue_gwen": {"desc": "Rescue Gwen"}, + "escape_keep": {"desc": "Escape the keep"}, + }, +} + + +def action(delta, index=1): + """Returns an unsaved Action carrying the report `apply_delta` produced. + + The column is filled through `world_delta_of`, the same function the turn + endpoint uses, rather than by assembling the dict here. An earlier version + of this helper built the shape by hand with every report list present. That + hid a live bug: `world_delta_of` stored `applied` alone, so no refusal ever + reached a chip or the next prompt while all of these tests passed. + """ + ws, report = w.apply_delta(w.instantiate(SCHEMA), SCHEMA, delta, index) + snapshot = {"world_state": {"delta": delta, "report": report, "state": ws}} + return models.Action(world_delta=world_delta_of(snapshot)) + + +def test_world_delta_of_keeps_every_report_list(): + """The stored column must carry the refusals, not just the successes. + + `Action.world_changes` marks a clamped chip from `clamped` and builds its + refusal chips from `rejected`, and `worldstate.refusals` reads both. Dropping + either list leaves every consumer unable to tell a refused change from one + that worked. + """ + _, report = w.apply_delta( + w.instantiate(SCHEMA), SCHEMA, + {"player.arrows": 3, "player.nonesuch": 1, "player.hp": -5}, 1, + ) + stored = world_delta_of({"world_state": {"delta": {}, "report": report}}) + assert [e["path"] for e in stored["applied"]] == ["player.arrows", "player.hp"] + assert [e["path"] for e in stored["clamped"]] == ["player.arrows"] + assert [e["path"] for e in stored["rejected"]] == ["player.nonesuch"] + + +def test_world_delta_of_survives_a_snapshot_with_no_world_state(): + assert world_delta_of({"story": "s"}) is None + assert world_delta_of(None) is None + + +# --------------------------------------------------------------------------- # +# The goals line names the id the AI has to send +# --------------------------------------------------------------------------- # + +def test_goals_line_names_the_milestone_id(): + line = w.render_state_section(w.instantiate(SCHEMA), SCHEMA, {}) + assert "rescue_gwen — Rescue Gwen" in line + assert "escape_keep — Escape the keep" in line + # The path is spelled out, because the id alone does not say how to send it. + assert "milestones." in line + + +def test_achieved_line_names_the_id_too(): + ws, _ = w.apply_delta(w.instantiate(SCHEMA), SCHEMA, {"milestones.rescue_gwen": True}, 1) + line = w.render_state_section(ws, SCHEMA, {}) + assert "Achieved: rescue_gwen — Rescue Gwen." in line + # A reached milestone leaves the goals list. + assert "rescue_gwen — Rescue Gwen;" not in line + + +def test_every_milestone_in_the_demo_is_named_to_the_model(): + """The demo that exposed this must not lose the ids again.""" + schema = json.loads((SEED / "05-league-championship.json").read_text(encoding="utf-8"))["stat_schema"] + line = w.render_state_section(w.instantiate(schema), schema, {}) + for mid in schema["milestones"]: + assert mid in line + + +# --------------------------------------------------------------------------- # +# A refused change reaches the turn summary +# --------------------------------------------------------------------------- # + +def test_a_clamp_that_changes_nothing_is_still_reported(): + """The Pokémon bug: a positive delta on a stat already at its ceiling. + + `arrows` sits at 6 of a maximum 6, so `+2` caps to `+1`, reaches 7, and + clamps back to 6. The value never moves, and the summary has to say so. + """ + chips = action({"player.arrows": 2}).world_changes + arrows = [c for c in chips if c["label"] == "arrows"] + assert len(arrows) == 1 + assert arrows[0]["delta"] == 0 + assert arrows[0]["clamped"] is True + + +def test_a_clamp_that_still_moves_the_value_is_marked(): + chips = action({"player.hp": -80}).world_changes + hp = [c for c in chips if c["label"] == "hp"][0] + assert hp["delta"] == -30 # max_delta_per_turn + assert hp["clamped"] is True + + +def test_an_accepted_change_is_not_marked(): + chips = action({"player.hp": -10}).world_changes + hp = [c for c in chips if c["label"] == "hp"][0] + assert hp["delta"] == -10 + assert hp["clamped"] is False + + +def test_a_refusal_becomes_its_own_entry_carrying_the_reason(): + chips = action({"world.day": -1, "player.stamina": 5}).world_changes + refused = {c["label"]: c["reason"] for c in chips if c["kind"] == "rejected"} + assert refused == {"day": "counter can't decrease", "stamina": "unknown stat"} + + +def test_a_refused_npc_stat_keeps_the_npc_in_its_label(): + chips = action({"npc.gwen.bogus": 5}).world_changes + refused = [c for c in chips if c["kind"] == "rejected"][0] + assert refused["label"] == "gwen bogus" + + +def test_accepted_and_refused_changes_appear_together(): + chips = action({"player.hp": -10, "world.day": -1}).world_changes + kinds = [c["kind"] for c in chips] + assert "stat" in kinds and "rejected" in kinds + + +def test_a_turn_that_changed_nothing_still_has_no_chips(): + assert models.Action(world_delta=None).world_changes == [] + + +def test_flags_and_milestones_are_unchanged_by_the_new_fields(): + chips = action({"flags.has_key": True, "milestones.rescue_gwen": True}).world_changes + assert {"kind": "flag", "label": "has_key", "on": True} in chips + assert {"kind": "milestone", "label": "rescue_gwen"} in chips + + +# --------------------------------------------------------------------------- # +# The demo stat counts up, so a wrong sign is refused rather than absorbed +# --------------------------------------------------------------------------- # + +def test_milos_faint_counter_counts_up_from_zero(): + """A stat that starts at its ceiling cannot report a wrong sign. + + `pokemon_left` began at 3 of a maximum 3, so the model sending the count it + had left rather than a delta clamped to no change at all. Counting the + faints up from 0 puts the wrong direction on the counter rule, which + refuses it out loud. + """ + schema = json.loads((SEED / "05-league-championship.json").read_text(encoding="utf-8"))["stat_schema"] + stat = schema["npcs"]["milo"]["stats"]["pokemon_fainted"] + assert stat["initial"] == 0 and stat["type"] == "counter" + + ws = w.instantiate(schema) + after, report = w.apply_delta(ws, schema, {"npc.milo.pokemon_fainted": 1}, 1) + assert after["npc"]["milo"]["pokemon_fainted"] == 1 + + _, report = w.apply_delta(ws, schema, {"npc.milo.pokemon_fainted": -1}, 1) + assert len(report["rejected"]) == 1 + refused = report["rejected"][0] + assert refused["path"] == "npc.milo.pokemon_fainted" + assert refused["reason"] == "counter can't decrease" + + +def test_the_demo_no_longer_mentions_the_old_stat(): + """The instructions name the stat, so a rename has to reach them too.""" + raw = (SEED / "05-league-championship.json").read_text(encoding="utf-8") + assert "pokemon_left" not in raw + assert "pokemon_fainted" in json.loads(raw)["ai_instructions"] + + +def test_the_demo_asks_for_the_turn_counter(): + """`world.turn` sat at 0 for a whole playtest: nothing told the model to + move it. The schema defining a stat is not an instruction to update it.""" + d = json.loads((SEED / "05-league-championship.json").read_text(encoding="utf-8")) + assert "world.turn" in d["stat_schema"]["world"] or "turn" in d["stat_schema"]["world"] + assert "world.turn" in d["ai_instructions"] + + +# --------------------------------------------------------------------------- # +# What the model is told about its own refused changes +# --------------------------------------------------------------------------- # + +def test_history_replays_what_was_accepted_not_what_was_sent(): + """The contradiction that taught the model to repeat itself. + + `arrows` is at its ceiling, so `+2` changes nothing. Replaying the sent + delta showed the model a change the live values disagreed with. + """ + from app.context.builder import _history_text + + a = action({"player.arrows": 2, "player.hp": -10}) + a.text = "The arrow flies." + replayed = _history_text(a) + assert '"player.hp": -10' in replayed + assert "arrows" not in replayed + + +def test_history_replay_keeps_flags_and_milestones_and_text(): + from app.context.builder import _history_text + + a = action({"flags.has_key": True, "milestones.rescue_gwen": True}) + a.text = "The lock gives." + replayed = _history_text(a) + assert '"flags.has_key": true' in replayed + assert '"milestones.rescue_gwen": true' in replayed + + +def test_a_refusal_reaches_the_model_with_the_valid_names(): + note = w.render_refusals(action({"milestones.bogus": True}).world_delta) + assert "There is no milestone `bogus`" in note + # The correction has to name what it could have sent instead. + assert "`rescue_gwen`" in note and "`escape_keep`" in note + + +def test_a_wrong_sign_on_a_counter_is_explained(): + note = w.render_refusals(action({"world.day": -1}).world_delta) + assert "only counts up" in note + + +def test_a_clamp_that_moved_nothing_quotes_the_limit(): + note = w.render_refusals(action({"player.arrows": 2}).world_delta) + assert "did not move" in note + assert "maximum of 6" in note and "it runs from 0 to 6" in note + + +def test_a_clamp_that_still_moved_the_value_says_nothing(): + """Reporting a trimmed change invites the model to send the remainder next + turn, which is the swing `max_delta_per_turn` exists to prevent.""" + assert w.render_refusals(action({"player.hp": -80}).world_delta) == "" + + +def test_a_clean_turn_adds_no_note(): + assert w.render_refusals(action({"player.hp": -10}).world_delta) == "" + assert w.render_refusals(None) == "" diff --git a/backend/tests/test_chat.py b/backend/tests/test_chat.py new file mode 100644 index 0000000..42acfce --- /dev/null +++ b/backend/tests/test_chat.py @@ -0,0 +1,248 @@ +"""HTTP tests for the AI Chat scratchpad (power users only). + +Covers the access gate, the streamed reply, and the demo-key model pinning. +This pinning must not let a public visitor reach paid models through this +page. + + python -m pytest tests/test_chat.py -v +""" +import pytest +from fastapi import Depends +from fastapi.testclient import TestClient + +from app import auth, limits, models +from app.database import Base, SessionLocal, engine, get_db +from app.main import app +from app.routers import chat + + +class FakeProvider: + """Records what it was constructed with, then streams a fixed reply. Stands + in for the real egress point, so asserting on last_key/last_model is + asserting on exactly what would have gone over the wire.""" + last_usage = None + last_model = None + last_key = None + last_endpoint = None + last_messages = None + + def __init__(self, endpoint_url, api_key, model, api_mode="chat", reasoning_max_tokens=0): + FakeProvider.last_model = model + FakeProvider.last_key = api_key + FakeProvider.last_endpoint = endpoint_url + + async def chat(self, messages, *, temperature, max_tokens): + FakeProvider.last_messages = messages + yield ("reasoning", "hmm") + yield ("text", "Hello back.") + + +@pytest.fixture() +def client(monkeypatch): + Base.metadata.create_all(bind=engine) + setup = SessionLocal() + user = models.User(is_guest=False, email="power@example.com") + setup.add(user) + setup.flush() + setup.add(models.Settings(user_id=user.id, api_key="enc:dummy", model="test-model")) + setup.commit() + user_id = user.id + setup.close() + + monkeypatch.setattr(chat, "OpenAICompatibleProvider", FakeProvider) + monkeypatch.setattr(limits, "rate_limit", lambda *a, **k: None) + # Multi-user mode is what makes the power-user gate meaningful, because + # local mode trusts everyone. The allowlist is set per test. + monkeypatch.setattr(auth, "MULTI_USER", True) + monkeypatch.setattr(auth, "POWER_USERS", {"power@example.com"}) + # These tests deliberately do not stub resolve_provider_config. The + # point is to exercise the real BYOK-vs-demo decision, since that + # decision is what keeps the shared key off paid models. Each test + # picks a mode with _byok/_demo below. + + def _current_user(db=Depends(get_db)): + return db.get(models.User, user_id) + + app.dependency_overrides[auth.get_current_user] = _current_user + try: + yield TestClient(app) + finally: + app.dependency_overrides.clear() + Base.metadata.drop_all(bind=engine) + + +def _send(client, **extra): + return client.post("/api/chat/stream", json={"messages": [{"role": "user", "content": "hi"}], **extra}) + + +def _byok(monkeypatch): + """The user brought their own key: no demo key in play, any model allowed.""" + monkeypatch.setattr(auth, "demo_enabled", lambda: False) + db = SessionLocal() + try: + settings = db.query(models.Settings).first() + settings.api_key = "sk-my-own-key" # legacy-plaintext path: used as-is + db.commit() + finally: + db.close() + + +def _demo(monkeypatch, whitelist=("free/allowed",)): + """The user has no key, so turns run on the server-funded demo key.""" + monkeypatch.setattr(auth, "demo_enabled", lambda: True) + monkeypatch.setattr(auth, "DEMO_API_KEY", "demo-key") + monkeypatch.setattr(auth, "DEMO_ENDPOINT_URL", "http://demo") + monkeypatch.setattr(auth, "DEMO_MODELS", list(whitelist)) + + +def test_non_power_user_gets_404(client, monkeypatch): + monkeypatch.setattr(auth, "POWER_USERS", set()) + assert _send(client).status_code == 404 + assert client.get("/api/chat/config").status_code == 404 + + +def test_power_user_streams_a_reply(client, monkeypatch): + _byok(monkeypatch) + resp = _send(client) + assert resp.status_code == 200, resp.text + assert '"type": "reasoning"' in resp.text + assert "Hello back." in resp.text + assert '"type": "done"' in resp.text + assert FakeProvider.last_messages == [{"role": "user", "content": "hi"}] + + +def test_system_prompt_and_model_override_are_honoured(client, monkeypatch): + _byok(monkeypatch) + resp = client.post("/api/chat/stream", json={ + "messages": [ + {"role": "system", "content": "Be terse."}, + {"role": "user", "content": "hi"}, + ], + "model": "some/other-model", + }) + assert resp.status_code == 200, resp.text + # BYOK: any model the user names is passed straight through, on their key. + assert FakeProvider.last_model == "some/other-model" + assert FakeProvider.last_key == "sk-my-own-key" + assert FakeProvider.last_messages[0] == {"role": "system", "content": "Be terse."} + + +def test_demo_key_pins_model_to_whitelist(client, monkeypatch): + _demo(monkeypatch) + resp = _send(client, model="expensive/paid-model") + assert resp.status_code == 200, resp.text + # Refused visibly: the whitelisted model runs instead, with a note. The + # paid slug must never reach the wire alongside the server-funded key. + assert FakeProvider.last_model == "free/allowed" + assert FakeProvider.last_key == "demo-key" + assert '"type": "note"' in resp.text + + # A whitelisted model is still selectable on the demo key. + _demo(monkeypatch, ["free/allowed", "free/second"]) + resp = _send(client, model="free/second") + assert resp.status_code == 200, resp.text + assert FakeProvider.last_model == "free/second" + + +def test_demo_key_ignores_an_off_whitelist_settings_model(client, monkeypatch): + """The override is not the only untrusted input. `Settings.model` is + also user-set, and it must be pinned the same way when there is no + BYOK key.""" + _demo(monkeypatch) + db = SessionLocal() + try: + db.query(models.Settings).first().model = "expensive/paid-model" + db.commit() + finally: + db.close() + resp = _send(client) + assert resp.status_code == 200, resp.text + assert FakeProvider.last_model == "free/allowed" + + +def test_demo_key_endpoint_cannot_be_redirected(client, monkeypatch): + """A user-controlled `endpoint_url` would leak the key itself, which is + worse than spending it. The demo branch pins the URL too.""" + _demo(monkeypatch) + db = SessionLocal() + try: + db.query(models.Settings).first().endpoint_url = "http://attacker.example/v1" + db.commit() + finally: + db.close() + assert _send(client).status_code == 200 + assert FakeProvider.last_endpoint == "http://demo" + assert FakeProvider.last_key == "demo-key" + + +def test_provider_config_refuses_server_funded_paid_model(monkeypatch): + """The structural backstop: a hand-built config (a future code path that + forgets to go through resolve_provider_config) cannot run a + server-funded turn on an off-whitelist model.""" + monkeypatch.setattr(auth, "DEMO_API_KEY", "demo-key") + monkeypatch.setattr(auth, "DEMO_MODELS", ["free/allowed"]) + with pytest.raises(ValueError): + auth.ProviderConfig("http://demo", "demo-key", "expensive/paid-model", True) + auth.ProviderConfig("http://demo", "demo-key", "free/allowed", True) # whitelisted: fine + # The user's own key with any model stays fine. + auth.ProviderConfig("http://any", "sk-mine", "expensive/paid-model", False) + + +def test_byok_user_may_reuse_the_demo_keys_value(client, monkeypatch): + """Regression: the demo key is just an OpenRouter key, so a user can paste + that same value into their own Settings. That is still BYOK, because + the user is paying, and it must not trip the guard. It used to raise + on every resolution, which returned a 500 from `GET /auth/me` and + broke the entire SPA (no nav, no chat).""" + monkeypatch.setattr(auth, "demo_enabled", lambda: True) + monkeypatch.setattr(auth, "DEMO_API_KEY", "shared-key") + monkeypatch.setattr(auth, "DEMO_ENDPOINT_URL", "http://demo") + monkeypatch.setattr(auth, "DEMO_MODELS", ["free/allowed"]) + db = SessionLocal() + try: + settings = db.query(models.Settings).first() + settings.api_key = "shared-key" # same value, but supplied by the user + settings.model = "expensive/paid-model" # their spend, their choice + db.commit() + finally: + db.close() + + assert client.get("/api/auth/me").status_code == 200 + assert client.get("/api/chat/config").status_code == 200 + resp = _send(client) + assert resp.status_code == 200, resp.text + assert FakeProvider.last_model == "expensive/paid-model" + assert FakeProvider.last_key == "shared-key" + + +def test_resolve_provider_config_is_the_single_choke_point(monkeypatch): + """Turns, AI Chat, and the connection test all resolve through this one + function, so pinning it here pins every caller. No DB or HTTP needed.""" + monkeypatch.setattr(auth, "demo_enabled", lambda: True) + monkeypatch.setattr(auth, "DEMO_API_KEY", "demo-key") + monkeypatch.setattr(auth, "DEMO_ENDPOINT_URL", "http://demo") + monkeypatch.setattr(auth, "DEMO_MODELS", ["free/allowed"]) + + # No key of their own: both endpoint and model are pinned, regardless + # of what they set. + no_key = models.Settings(endpoint_url="http://mine/v1", api_key="", model="expensive/paid") + assert auth.resolve_provider_config(no_key) == auth.ProviderConfig( + "http://demo", "demo-key", "free/allowed", True) + assert auth.resolve_provider_config( + no_key, model_override="expensive/paid").model == "free/allowed" + assert auth.resolve_provider_config( + no_key, model_override="free/allowed").model == "free/allowed" + + # Their own key: their endpoint, their key, their choice of model. + byok = models.Settings(endpoint_url="http://mine/v1", api_key="sk-mine", model="expensive/paid") + assert auth.resolve_provider_config(byok) == auth.ProviderConfig( + "http://mine/v1", "sk-mine", "expensive/paid", False) + + +def test_oversized_conversation_is_refused(client, monkeypatch): + _byok(monkeypatch) + huge = "x" * 90_000 + resp = client.post("/api/chat/stream", json={ + "messages": [{"role": "user", "content": huge} for _ in range(5)], + }) + assert resp.status_code == 413, resp.text diff --git a/backend/tests/test_delete_state.py b/backend/tests/test_delete_state.py new file mode 100644 index 0000000..533d121 --- /dev/null +++ b/backend/tests/test_delete_state.py @@ -0,0 +1,214 @@ +"""Deleting a turn puts the shared state back. + +`script_state` and `world_state` belong to the adventure, not to the node +that changed them. Undo, retry, a take and a branch switch all restore them; +the delete endpoint did not. Deleting an AI turn removed the text and left +everything the turn did to the numbers standing. + +The visible symptom was the cooldown clock. It lives in +`world_state._meta.last_changed` and it holds a depth. A deleted turn left +its depth there, and the turn played in its place is played at that same +depth, so the referee refused the change as one that had happened this very +turn — on a turn the story no longer contains. Delete the AI reply because +you did not like the stat change it proposed, press Continue, and the same +change comes back refused as "changed too recently". + + python -m pytest tests/test_delete_state.py -v +""" +import pytest +from fastapi import Depends +from fastapi.testclient import TestClient + +from app import auth, limits, models +from app.database import Base, SessionLocal, engine, get_db +from app.main import app +from app.routers import adventures +from fakes import ScriptedProvider + +# `mana` carries a cooldown, so a clock that was not rolled back shows up as +# a refusal rather than as a number that is merely off. +SCHEMA = { + "player": { + "hp": {"min": 0, "max": 100, "initial": 100}, + "mana": {"min": 0, "max": 50, "initial": 50, "cooldown": 2}, + } +} + +# Ten gold a turn. A total that only ever climbs makes a missing rollback +# obvious: it is off by exactly one turn's worth. +GOLD_SCRIPT = """ +const modifier = (text) => { + state.gold = (state.gold || 0) + 10; + return { text }; +}; +modifier(text); +""" + +DRAIN = 'Drained.\n```state\n{"player.mana": -10}\n```' + + +@pytest.fixture() +def client(monkeypatch): + Base.metadata.create_all(bind=engine) + setup = SessionLocal() + user = models.User(is_guest=False, email="delstate@example.com") + setup.add(user) + setup.flush() + setup.add(models.Settings(user_id=user.id, api_key="enc:dummy", model="test-model")) + scenario = models.Scenario(user_id=user.id, title="S", stat_schema=SCHEMA) + setup.add(scenario) + setup.flush() + adv = models.Adventure( + user_id=user.id, title="Tower", scenario_id=scenario.id, + script_state={}, world_state={"player": {"hp": 100, "mana": 50}}, + ) + setup.add(adv) + setup.flush() + setup.add(models.Action(adventure_id=adv.id, type="start", text="You begin.")) + setup.add(models.AdventureScript( + adventure_id=adv.id, position=0, enabled=True, name="Gold", output_js=GOLD_SCRIPT, + )) + setup.commit() + adv_id, user_id = adv.id, user.id + setup.close() + + ScriptedProvider.replies = [DRAIN] + ScriptedProvider.calls = 0 + monkeypatch.setattr(adventures.turns, "OpenAICompatibleProvider", ScriptedProvider) + monkeypatch.setattr(auth, "resolve_provider_config", lambda s: auth.ProviderConfig( + "http://fake", "k", "test-model", False)) + monkeypatch.setattr(limits, "rate_limit", lambda *a, **k: None) + monkeypatch.setattr(limits, "check_row_cap", lambda *a, **k: None) + + def _current_user(db=Depends(get_db)): + return db.get(models.User, user_id) + + app.dependency_overrides[auth.get_current_user] = _current_user + c = TestClient(app) + c.adv_id = adv_id + try: + yield c + finally: + app.dependency_overrides.clear() + adventures.turns._active_turns.clear() + Base.metadata.drop_all(bind=engine) + + +# ------------------------------------------------------------------ helpers + +def _play(client, text="look around"): + r = client.post(f"/api/adventures/{client.adv_id}/actions", + json={"type": "do", "text": text}) + assert r.status_code == 200, r.text + + +def _continue(client): + r = client.post(f"/api/adventures/{client.adv_id}/actions", + json={"type": "continue", "text": ""}) + assert r.status_code == 200, r.text + + +def _delete(client, action_id): + return client.delete(f"/api/adventures/{client.adv_id}/actions/{action_id}") + + +def _state(adv_id): + db = SessionLocal() + try: + adv = db.get(models.Adventure, adv_id) + return adv.script_state, adv.world_state + finally: + db.close() + + +def _ai_rows(adv_id): + db = SessionLocal() + try: + return ( + db.query(models.Action) + .filter(models.Action.adventure_id == adv_id, models.Action.type == "ai") + .order_by(models.Action.depth, models.Action.id) + .all() + ) + finally: + db.close() + + +def _last_changes(adv_id): + db = SessionLocal() + try: + adv = db.get(models.Adventure, adv_id) + return adv.actions[-1].world_changes + finally: + db.close() + + +# ------------------------------------------------------- the reported bug + +def test_deleting_the_ai_turn_rewinds_the_world_state(client): + _play(client) + assert _state(client.adv_id)[1]["player"]["mana"] == 40 + + _delete(client, _ai_rows(client.adv_id)[-1].id) + + _, world = _state(client.adv_id) + assert world["player"]["mana"] == 50, "the drain went with the turn" + assert not (world.get("_meta") or {}).get("last_changed"), "and so did its clock" + + +def test_the_next_turn_is_not_refused_for_a_deleted_turn_s_cooldown(client): + """The bug as a player meets it: delete the reply, press Continue, and + the change it proposes is refused as one that already happened.""" + _play(client) + _delete(client, _ai_rows(client.adv_id)[-1].id) + + _continue(client) + + assert _state(client.adv_id)[1]["player"]["mana"] == 40, "the drain lands" + assert [c for c in _last_changes(client.adv_id) if c["kind"] == "rejected"] == [] + + +def test_deleting_the_ai_turn_rewinds_the_script_state(client): + """The same restore, on the other half of the shared state. Without it a + replayed turn stacks its script run on top of the deleted one's.""" + _play(client) + assert _state(client.adv_id)[0] == {"gold": 10} + + _delete(client, _ai_rows(client.adv_id)[-1].id) + assert _state(client.adv_id)[0] == {} + + _continue(client) + assert _state(client.adv_id)[0] == {"gold": 10}, "one turn of gold, not two" + + +# ------------------------------------------------- deleting further back + +def test_deleting_a_turn_the_story_moved_past_leaves_the_tip_alone(client): + """A restore reads the tip's own outcome, not the deleted node's + neighbour, so removing a turn from the middle of the story does not roll + the numbers back to that point. The text goes; the state stays.""" + _play(client) + _play(client, "press on") + before = _state(client.adv_id) + assert before[0] == {"gold": 20} + + first_ai = _ai_rows(client.adv_id)[0] + assert _delete(client, first_ai.id).status_code == 204 + + assert _state(client.adv_id) == before + + +def test_delete_is_blocked_while_a_turn_is_generating(client): + """The endpoint writes the shared state now, so it takes the same lock + undo takes rather than racing the turn that is about to write it.""" + _play(client) + action_id = _ai_rows(client.adv_id)[-1].id + + adventures.turns.acquire_turn_lock(client.adv_id) # a turn is "generating" + try: + assert _delete(client, action_id).status_code == 409 + # The refused delete must not have released someone else's lock. + assert client.adv_id in adventures.turns._active_turns + finally: + adventures.turns._active_turns.discard(client.adv_id) + assert len(_ai_rows(client.adv_id)) == 1, "and the turn is still there" diff --git a/backend/tests/test_egress.py b/backend/tests/test_egress.py new file mode 100644 index 0000000..25e5bd1 --- /dev/null +++ b/backend/tests/test_egress.py @@ -0,0 +1,463 @@ +"""Guards on how much the database is asked for. + +`context_snapshot` holds the entire assembled prompt for a turn. That is 163 kB +per row averaged over production, 232 kB on the longest adventure, and 89% of the +database. It used to be fetched for every action on every adventure load and +every turn, to read two small values out of it. These tests fail if that +returns. + +Two kinds of guard live here, and both are needed: + +* Column guards assert which columns a statement names. That is the shape both + of this project's egress regressions took: one query carrying a column nobody + read. +* Byte ceilings assert what a request costs. Every column guard would still pass + if a response grew tenfold within the columns it is allowed to read, which is + what a story that keeps getting longer does. + + python -m pytest tests/test_egress.py -v +""" +import json +import random + +import pytest +from fastapi import Depends +from fastapi.testclient import TestClient +from sqlalchemy import event, text +from sqlalchemy.orm import undefer + +from app import auth, limits, migrations, models +from app.context import history +from app.database import Base, SessionLocal, engine, get_db +from app.main import app +from tools import dbmeter +from tools.fakeprose import prose + +# A stand-in for the real thing: the assembled prompt, which is what makes the +# column enormous, plus the small world_state slice the UI actually needs. +# +# Varied text, not `"x" * 20_000`. The column is stored compressed now +# by migration 43, and a repeated character compresses about a thousandfold. +# That would make the byte ceilings below pass against a fixture that costs +# nothing, which tests nothing. Prose-shaped filler compresses like the prompts +# it stands in for. +_SNAPSHOT_RNG = random.Random(20_260_817) +BIG_SNAPSHOT = { + "system": prose(_SNAPSHOT_RNG, 20_000), + "story": prose(_SNAPSHOT_RNG, 40_000), + "world_state": { + "delta": {"player.hp": -15}, + "report": {"applied": [{"path": "player.hp", "old": 100, "new": 85}]}, + "state": {"player": {"hp": 85}}, + }, +} + +# Retry history: each discarded attempt keeps its full narration, so an action +# retried a few times carries several KB that a list response only ever counts. +BIG_VARIANTS = [ + {"text": "z" * 4_000, "reasoning": None, "script_state": {}, + "created_at": "2026-01-01T00:00:00"} + for _ in range(3) +] + + +@pytest.fixture() +def sql_log(): + """Every statement the ORM sends, for asserting on what was selected.""" + statements: list[str] = [] + + def record(conn, cursor, statement, parameters, context, executemany): + statements.append(statement) + + event.listen(engine, "before_cursor_execute", record) + try: + yield statements + finally: + event.remove(engine, "before_cursor_execute", record) + + +@pytest.fixture() +def client(monkeypatch): + Base.metadata.create_all(bind=engine) + setup = SessionLocal() + user = models.User(is_guest=False, email="egress@example.com") + setup.add(user) + setup.flush() + setup.add(models.Settings(user_id=user.id, api_key="enc:dummy", model="test-model")) + adventure = models.Adventure(user_id=user.id, title="Cave", script_state={}) + setup.add(adventure) + setup.flush() + for i in range(12): + setup.add(models.Action( + adventure_id=adventure.id, + type="ai" if i % 2 else "do", text=f"Action {i}.", + context_snapshot=BIG_SNAPSHOT, + world_delta={"delta": {"player.hp": -15}, + "applied": [{"path": "player.hp", "old": 100, "new": 85}]}, + )) + setup.commit() + adv_id, user_id = adventure.id, user.id + setup.close() + + monkeypatch.setattr(limits, "rate_limit", lambda *a, **k: None) + monkeypatch.setattr(limits, "check_row_cap", lambda *a, **k: None) + + def _current_user(db=Depends(get_db)): + return db.get(models.User, user_id) + + app.dependency_overrides[auth.get_current_user] = _current_user + c = TestClient(app) + c.adv_id = adv_id + try: + yield c + finally: + app.dependency_overrides.clear() + Base.metadata.drop_all(bind=engine) + + +def action_selects(statements: list[str]) -> list[str]: + return [s for s in statements if "FROM actions" in s and s.lstrip().upper().startswith("SELECT")] + + +# ------------------------------------------------------- the deferred columns + +def test_loading_an_adventure_does_not_fetch_context_snapshot(client, sql_log): + r = client.get(f"/api/adventures/{client.adv_id}") + assert r.status_code == 200, r.text + assert len(r.json()["actions"]) == 12 + + selects = action_selects(sql_log) + assert selects, "expected at least one SELECT against actions" + offenders = [s for s in selects if "context_snapshot" in s] + assert offenders == [], f"context_snapshot was fetched in bulk:\n{offenders[0][:400]}" + + +def test_the_state_snapshots_are_not_fetched_in_bulk(client, sql_log): + """Both are rollback snapshots, only ever needed for the single node being + undone, retried past or switched to. A page load must pay for neither. + """ + client.get(f"/api/adventures/{client.adv_id}") + selects = action_selects(sql_log) + for column in ("state_after", "world_state_after"): + offenders = [s for s in selects if column in s] + assert offenders == [], f"{column} was fetched in bulk" + + +def test_world_changes_still_works_without_the_snapshot(client): + """The chips under an AI message must survive the snapshot being deferred.""" + r = client.get(f"/api/adventures/{client.adv_id}") + ai = [a for a in r.json()["actions"] if a["type"] == "ai"] + assert ai, "fixture should have AI actions" + assert ai[0]["world_changes"] == [ + {"kind": "stat", "label": "hp", "delta": -15, "value": 85, "clamped": False} + ] + + +def test_counting_actions_does_not_name_the_deferred_columns(client, sql_log): + """A count that wraps the entity select in a subquery names every column in + the emitted SQL. + + No bytes come back, but the database still reads them, and the guard above + cannot distinguish that from a real bulk fetch. + """ + db = SessionLocal() + try: + adventure = db.get(models.Adventure, client.adv_id) + sql_log.clear() + assert history.count(adventure) == 12 + counts = [s for s in sql_log if "count" in s.lower()] + assert counts, "expected a COUNT to be emitted" + for column in ("context_snapshot", "state_after", "world_state_after"): + assert not any(column in s for s in counts), ( + f"{column} is named by the count query:\n{counts[0][:400]}" + ) + finally: + db.close() + + +def test_snapshot_is_still_reachable_on_demand(client): + """Deferred means lazy rather than absent. Insights still gets the whole + snapshot.""" + r = client.get(f"/api/adventures/{client.adv_id}") + action_id = r.json()["actions"][0]["id"] + r = client.get(f"/api/adventures/{client.adv_id}/actions/{action_id}/context") + assert r.status_code == 200, r.text + # Round-tripped through zlib and back to a dict, byte for byte. + assert r.json()["system"] == BIG_SNAPSHOT["system"] + assert r.json()["story"] == BIG_SNAPSHOT["story"] + + +# ------------------------------------------------------------------ backfill + +def as_json_snapshot_column(db) -> None: + """Put actions.context_snapshot back as JSON, the way it was before 43. + + Migration 36 lifts world_delta out of the snapshot with SQL JSON + functions, so it can only run while the column still *is* JSON. In a real + upgrade it always is, because 36 runs seven migrations before 43 compresses + the column into a BLOB. `create_all` builds today's schema, so a test that + calls that backfill has to rebuild the schema it was written against. + """ + db.execute(text("ALTER TABLE actions DROP COLUMN context_snapshot")) + db.execute(text("ALTER TABLE actions ADD COLUMN context_snapshot JSON")) + db.execute( + text("UPDATE actions SET context_snapshot = :snapshot"), + {"snapshot": json.dumps(BIG_SNAPSHOT)}, + ) + db.commit() + + +def test_backfill_populates_world_delta_from_existing_snapshots(client): + """Migration 36 lifts the slice out server-side, without reading the + snapshots into Python.""" + db = SessionLocal() + try: + as_json_snapshot_column(db) + db.execute(text("UPDATE actions SET world_delta = NULL")) + db.commit() + assert db.query(models.Action).filter(models.Action.world_delta.isnot(None)).count() == 0 + + with engine.begin() as conn: + migrations._backfill_world_delta(conn) + + db.expire_all() + actions = db.query(models.Action).all() + assert all(a.world_delta is not None for a in actions) + assert actions[0].world_delta["delta"] == {"player.hp": -15} + assert actions[0].world_delta["applied"] == [ + {"path": "player.hp", "old": 100, "new": 85} + ] + finally: + db.close() + + +def test_backfill_populates_variant_count_from_existing_variants(client): + """Migration 37 counts the lists on the server. + + Reading them into Python to count them would fetch the column over the wire + once in order to stop fetching it on every request. + + Migration 68 drops `variant_count` and 66 drops `variants`, so this test + puts both columns back before it calls the pass, the same way + `as_json_snapshot_column` above rebuilds the column migration 36 needs. The + assertions read raw SQL, because the model no longer has either attribute. + """ + db = SessionLocal() + try: + db.execute(text("ALTER TABLE actions ADD COLUMN variants JSON")) + db.execute(text( + "ALTER TABLE actions ADD COLUMN variant_count INTEGER NOT NULL DEFAULT 0" + )) + db.execute( + text("UPDATE actions SET variants = :v WHERE type = 'ai'"), + {"v": json.dumps(BIG_VARIANTS)}, + ) + db.commit() + + with engine.begin() as conn: + migrations._backfill_variant_count(conn) + + rows = db.execute(text( + "SELECT type, variant_count FROM actions ORDER BY id" + )).all() + assert rows, "fixture should have actions" + for kind, count in rows: + assert count == (len(BIG_VARIANTS) if kind == "ai" else 0) + finally: + db.close() + + +def test_backfill_leaves_actions_without_world_state_alone(client): + db = SessionLocal() + try: + db.execute(text("UPDATE actions SET world_delta = NULL, context_snapshot = '{\"story\": \"s\"}'")) + db.commit() + with engine.begin() as conn: + migrations._backfill_world_delta(conn) + db.expire_all() + assert all(a.world_delta is None for a in db.query(models.Action).all()) + finally: + db.close() + + +# ---------------------------------------------------------------- byte ceilings +# +# The tests above assert which columns a statement names, which is the shape both +# of this project's egress regressions took. They would all still pass if a +# response grew tenfold within the columns it is allowed to read, and a story +# that keeps getting longer does that. These tests put a number on it. +# +# The ceilings are per action rather than absolute, so they mean the same thing +# whatever size the fixture is, and they are generous. They exist to catch a +# tenfold regression rather than to freeze today's byte count. + +ACTIONS_IN_FIXTURE = 12 + +# 3 kB an action against a real 994 B, measured on production 2026-08-17. +# Anything that pulls a deferred column blows past this by two orders of +# magnitude. See `test_the_ceiling_discriminates` below. +PAGE_LOAD_BYTES_PER_ACTION = 3_000 + + +@pytest.fixture() +def meter(): + """A byte meter on the shared engine, removed again afterwards. + + A test requests this after `client` in its arguments, so that building the + fixture, which is a write path no player takes, is not charged to any + scope. + """ + m = dbmeter.Meter() + m.attach(engine) + try: + yield m + finally: + m.detach() + + +def fetched(meter) -> int: + return meter.scopes[-1].total.fetched + + +def test_page_load_stays_under_its_byte_ceiling(client, meter): + with meter.scope("page load"): + r = client.get(f"/api/adventures/{client.adv_id}") + assert r.status_code == 200 + + budget = ACTIONS_IN_FIXTURE * PAGE_LOAD_BYTES_PER_ACTION + assert fetched(meter) < budget, ( + f"page load fetched {fetched(meter):,} B for {ACTIONS_IN_FIXTURE} " + f"actions, over the {budget:,} B budget" + ) + + +def test_the_action_list_stays_under_its_byte_ceiling(client, meter): + with meter.scope("action list"): + r = client.get(f"/api/adventures/{client.adv_id}/actions") + assert r.status_code == 200 + + budget = ACTIONS_IN_FIXTURE * PAGE_LOAD_BYTES_PER_ACTION + assert fetched(meter) < budget, ( + f"the action list fetched {fetched(meter):,} B, over {budget:,} B" + ) + + +def test_reading_one_action_does_not_cost_the_whole_story(client, meter): + """The snapshot is reachable on demand, and that request pays for one row + rather than for the whole adventure.""" + db = SessionLocal() + try: + action_id = db.query(models.Action.id).order_by(models.Action.id).first()[0] + finally: + db.close() + + with meter.scope("one snapshot"): + r = client.get(f"/api/adventures/{client.adv_id}/actions/{action_id}/context") + assert r.status_code == 200, r.text + + one_snapshot = len(json.dumps(BIG_SNAPSHOT)) + assert fetched(meter) < one_snapshot * 2, ( + f"fetching one action's snapshot cost {fetched(meter):,} B; one " + f"snapshot is {one_snapshot:,} B" + ) + + +def _fat_adventures(user_id: int, count: int = 5, body: int = 20_000) -> None: + """Adventures whose bodies are heavy and whose index cards are not. + + script_state, world_state and story_summary belong to the play screen. The + index shows a title, a stamp and a snippet, and used to load all of it. + """ + db = SessionLocal() + try: + for i in range(count): + db.add(models.Adventure( + user_id=user_id, + title=f"Adventure {i}", + script_state={"log": "s" * body}, + world_state={"player": {"notes": "w" * body}}, + story_summary="y" * body, + memory="m" * body, + )) + db.commit() + finally: + db.close() + + +def test_the_index_does_not_read_the_adventure_body(client, sql_log): + db = SessionLocal() + try: + user_id = db.query(models.User.id).first()[0] + finally: + db.close() + _fat_adventures(user_id) + + r = client.get("/api/adventures") + assert r.status_code == 200 + assert len(r.json()) == 6 # the fixture's one, plus five + + listing = [ + s for s in sql_log + if "FROM adventures" in s and s.lstrip().upper().startswith("SELECT") + ] + assert listing, "expected a listing query" + for column in ("script_state", "world_state", "story_summary", "memory", + "authors_note", "ai_instructions", "placeholders"): + assert not any(column in s for s in listing), ( + f"the index read adventures.{column}, which nothing on that " + f"screen displays" + ) + + +def test_the_index_stays_under_its_byte_ceiling(client, meter): + db = SessionLocal() + try: + user_id = db.query(models.User.id).first()[0] + finally: + db.close() + _fat_adventures(user_id) + + with meter.scope("index"): + r = client.get("/api/adventures") + assert r.status_code == 200 + + # Six adventures carrying 80 kB of body each. A card is a title, a stamp + # and a 220-character snippet; 4 kB apiece is already generous. + budget = 6 * 4_000 + assert fetched(meter) < budget, ( + f"the index fetched {fetched(meter):,} B for six adventures, over " + f"{budget:,} B — it is reading the bodies again" + ) + + +def test_the_ceiling_discriminates(client, meter): + """A ceiling is only worth having if the thing it excludes would breach it. + + This is the regression the byte tests exist to catch, performed on purpose: + undefer the snapshot and the same twelve rows cost several times the whole + budget. If this ever stops exceeding it, the fixture has gone too small for + the tests above to mean anything. + + The margin used to be a hundredfold and is now about six. The guard has not + weakened. Migration 43 compresses the column, and the fixture text is + prose-shaped, so it compresses like a real prompt rather than like a repeated + character. + """ + budget = ACTIONS_IN_FIXTURE * PAGE_LOAD_BYTES_PER_ACTION + db = SessionLocal() + try: + with meter.scope("undeferred"): + rows = ( + db.query(models.Action) + .options(undefer(models.Action.context_snapshot)) + .all() + ) + assert len(rows) == ACTIONS_IN_FIXTURE + finally: + db.close() + + assert fetched(meter) > budget * 3, ( + "undeferring the snapshot cost only " + f"{fetched(meter):,} B against a {budget:,} B budget — the fixture is " + "too small for the byte ceilings above to catch anything" + ) diff --git a/backend/tests/test_embedding_blob.py b/backend/tests/test_embedding_blob.py new file mode 100644 index 0000000..6fa82a5 --- /dev/null +++ b/backend/tests/test_embedding_blob.py @@ -0,0 +1,300 @@ +"""Migration 38: embeddings move from a JSON list to packed float32. + +The conversion has to be exact, because nothing re-embeds. A memory whose +vector shifts is silently ranked wrong forever, with no error anywhere to +report it. These tests check that the numbers survive the round trip bit +for bit, and that the migration reaches every row, however many there are. + + python -m pytest tests/test_embedding_blob.py -v +""" +import struct + + +import json +import random + +import pytest +from sqlalchemy import text + +from app import memorybank, migrations, models, vectors +from app.database import Base, SessionLocal, engine +from tests import schema_rewind + + +def float32(value: float) -> float: + """`value` as the double nearest to its float32 truncation. This is + what an embedding endpoint's JSON actually holds.""" + return struct.unpack(" list[float]: + return [float32(rng.uniform(-1.0, 1.0)) for _ in range(dims)] + + +@pytest.fixture() +def db(): + Base.metadata.create_all(bind=engine) + session = SessionLocal() + try: + yield session + finally: + session.close() + Base.metadata.drop_all(bind=engine) + + +@pytest.fixture() +def adventure(db): + user = models.User(is_guest=False, email="vectors@example.com") + db.add(user) + db.flush() + adv = models.Adventure(user_id=user.id, title="Cave", script_state={}) + db.add(adv) + db.commit() + return adv + + +# ------------------------------------------------------------------- packing + +def test_pack_round_trips_exactly(): + """Not "close enough": embedding endpoints compute in float32 and render + that into JSON, so packing back to float32 must be lossless.""" + rng = random.Random(1) + vector = sample_vector(rng) + assert list(vectors.unpack(vectors.pack(vector))) == vector + + +def test_packed_vector_is_four_bytes_per_dimension(): + """1536 dims takes 6 KB packed, against about 31 KB as JSON.""" + vector = sample_vector(random.Random(2)) + blob = vectors.pack(vector) + assert len(blob) == 1536 * 4 + assert len(blob) < len(json.dumps(vector).encode()) / 4 + + +def test_pack_handles_the_extremes(): + vector = [float32(v) for v in (0.0, -0.0, 1.0, -1.0, 3.4028234663852886e38, 1e-38)] + assert list(vectors.unpack(vectors.pack(vector))) == vector + + +def test_unpack_returns_a_compact_array(): + """These vectors stay in memory between turns, so the container type + matters. An array("f") stores each component in 4 bytes, the same width + as the column. A list of Python floats uses eight times that.""" + vector = sample_vector(random.Random(9)) + unpacked = vectors.unpack(vectors.pack(vector)) + assert unpacked.typecode == "f" + assert unpacked.itemsize == 4 + assert len(unpacked) == len(vector) + + +def test_pack_rounds_a_value_float32_cannot_hold(): + """The exactness guarantee applies only to vectors that came from an + embedding model, which computes in float32, not to arbitrary doubles. + This test pins down that boundary, because it is where the round-trip + claim holds.""" + assert vectors.unpack(vectors.pack([1e-38]))[0] != 1e-38 + assert vectors.unpack(vectors.pack([1e-38]))[0] == pytest.approx(1e-38) + + +def test_cosine_moved_but_still_reachable_from_memorybank(): + """Callers import it from memorybank. The math lives in vectors.""" + assert memorybank.cosine is vectors.cosine + assert vectors.cosine([1.0, 0.0], [1.0, 0.0]) == pytest.approx(1.0) + assert vectors.cosine([1.0, 0.0], [0.0, 1.0]) == pytest.approx(0.0) + assert vectors.cosine([1.0, 0.0], [1.0, 0.0, 0.0]) == 0.0 # length mismatch + + +# -------------------------------------------------------------- set_vector + +def test_set_vector_writes_the_blob_and_the_flag(db, adventure): + """The two columns that describe a vector move together, or a reader that + trusts `embedded` gets a NULL blob.""" + memory = models.Memory(adventure_id=adventure.id, text="a fact") + db.add(memory) + db.commit() + + vector = sample_vector(random.Random(3), dims=8) + memorybank.set_vector(memory, vector) + db.commit() + db.expire_all() + + assert list(vectors.unpack(memory.embedding_blob)) == vector + assert memory.embedded is True + + +def test_set_vector_none_clears_both(db, adventure): + """Editing a memory's text drops its vector so the next pass re-embeds. A + blob left behind would keep ranking the old text.""" + memory = models.Memory(adventure_id=adventure.id, text="a fact") + db.add(memory) + memorybank.set_vector(memory, sample_vector(random.Random(4), dims=8)) + db.commit() + + memorybank.set_vector(memory, None) + db.commit() + db.expire_all() + + assert memory.embedding_blob is None + assert memory.embedded is False + + +# --------------------------------------------------------------- the backfill + +def add_legacy_json_column(db) -> None: + """Put `memories.embedding` back for the length of a test. + + Migration 42 dropped it, and the model no longer declares it, so + `create_all` does not produce it. Everything below tests the upgrade + from a database that still has the column, which is the only state + where the backfill has any work to do. Re-adding it by hand keeps these + tests honest about the schema they claim to start from. + """ + db.execute(text("ALTER TABLE memories ADD COLUMN embedding JSON")) + db.commit() + + +def seed_json_only(db, adventure, count: int, dims: int = 64) -> dict[int, list[float]]: + """Memories as they exist before the migration: JSON vector, no blob.""" + add_legacy_json_column(db) + rng = random.Random(count) + expected = {} + for i in range(count): + vector = sample_vector(rng, dims) + memory = models.Memory(adventure_id=adventure.id, text=f"fact {i}") + db.add(memory) + db.flush() + # Raw, because the ORM no longer knows this column exists. + db.execute( + text("UPDATE memories SET embedding = :v WHERE id = :id"), + {"v": json.dumps(vector), "id": memory.id}, + ) + expected[memory.id] = vector + db.commit() + db.execute(text("UPDATE memories SET embedding_blob = NULL, embedded = false")) + db.commit() + return expected + + +def test_backfill_converts_every_existing_vector(db, adventure): + expected = seed_json_only(db, adventure, count=5) + + with engine.begin() as conn: + migrations._backfill_embedding_blob(conn) + + db.expire_all() + for memory in db.query(models.Memory).all(): + assert list(vectors.unpack(memory.embedding_blob)) == expected[memory.id] + + +def test_backfill_reaches_past_one_batch(db, adventure): + """It loops on id, and an off-by-one there would silently leave the + tail of a big bank unconverted. That failure reads as "not embedded + yet".""" + count = migrations.BACKFILL_BATCH * 2 + 3 + expected = seed_json_only(db, adventure, count=count, dims=4) + + with engine.begin() as conn: + migrations._backfill_embedding_blob(conn) + + db.expire_all() + memories = db.query(models.Memory).all() + assert len(memories) == count + assert all(m.embedding_blob is not None for m in memories) + assert all(list(vectors.unpack(m.embedding_blob)) == expected[m.id] for m in memories) + + +def test_backfill_leaves_unembedded_memories_alone(db, adventure): + add_legacy_json_column(db) + db.add(models.Memory(adventure_id=adventure.id, text="not embedded yet")) + db.commit() + + with engine.begin() as conn: + migrations._backfill_embedding_blob(conn) + + db.expire_all() + assert db.query(models.Memory).one().embedding_blob is None + + +def test_backfill_is_idempotent(db, adventure): + """It runs once from bootstrap, but a half-finished run must be safe to + repeat, and rows already converted must not be rewritten.""" + seed_json_only(db, adventure, count=3) + + with engine.begin() as conn: + migrations._backfill_embedding_blob(conn) + db.expire_all() + first = {m.id: m.embedding_blob for m in db.query(models.Memory).all()} + + with engine.begin() as conn: + migrations._backfill_embedding_blob(conn) + db.expire_all() + assert {m.id: m.embedding_blob for m in db.query(models.Memory).all()} == first + + +def test_backfill_skips_a_malformed_row_without_stopping(db, adventure): + """One bad row must not strand every row after it. The loop orders by + id, so an exception here would leave the rest of the bank unconverted.""" + expected = seed_json_only(db, adventure, count=2) + broken = models.Memory(adventure_id=adventure.id, text="broken") + db.add(broken) + db.commit() + db.execute( + text("UPDATE memories SET embedding = :bad WHERE id = :id"), + {"bad": '"not a list"', "id": broken.id}, + ) + db.commit() + + with engine.begin() as conn: + migrations._backfill_embedding_blob(conn) + + db.expire_all() + assert db.get(models.Memory, broken.id).embedding_blob is None + for memory_id, vector in expected.items(): + blob = db.get(models.Memory, memory_id).embedding_blob + assert list(vectors.unpack(blob)) == vector + + +# ------------------------------------------------------- the upgrade in full + +def test_bootstrap_adds_the_columns_and_backfills_them(db, adventure): + """The path a deployed database actually takes: sitting at 37 with neither + new column, then started on this build.""" + expected = seed_json_only(db, adventure, count=4) + unembedded = models.Memory(adventure_id=adventure.id, text="no vector yet") + db.add(unembedded) + db.commit() + unembedded_id = unembedded.id + db.close() + + schema_rewind.rewind_to(engine, migrations.EMBEDDING_BLOB_VERSION - 1) + + migrations.bootstrap(engine) + + with engine.begin() as conn: + assert conn.execute(text("PRAGMA user_version")).scalar() == migrations.LATEST_VERSION + rows = conn.execute(text("SELECT id, embedding_blob, embedded FROM memories")).all() + by_id = {row[0]: (row[1], row[2]) for row in rows} + assert len(by_id) == len(expected) + 1 + for memory_id, vector in expected.items(): + blob, embedded = by_id[memory_id] + assert list(vectors.unpack(blob)) == vector + assert embedded + # The flag has to follow the vector, not the row: a memory that was never + # embedded must still read as not embedded afterwards. + assert by_id[unembedded_id] == (None, False) + + # Migration 42, at the end of the same run, removes the JSON column. + # Ordering matters: 38 reads it, 42 drops it, and an upgrade that ran + # them in the other order would arrive with an empty bank. + with engine.begin() as conn: + columns = {row[1] for row in conn.execute(text("PRAGMA table_info(memories)"))} + assert "embedding" not in columns + assert {"embedding_blob", "embedded"} <= columns + + +def test_migration_38_is_spelled_for_both_dialects(): + """Every Postgres deploy replays migrations from 24 on, so a SQLite-only + ALTER here would break the live database and nothing else would notice.""" + sql = dict(migrations.MIGRATIONS)[migrations.EMBEDDING_BLOB_VERSION] + assert "BLOB" in migrations._for_dialect(sql, "sqlite") + assert "BYTEA" in migrations._for_dialect(sql, "postgresql") diff --git a/backend/tests/test_embedding_model_switch.py b/backend/tests/test_embedding_model_switch.py new file mode 100644 index 0000000..292d62d --- /dev/null +++ b/backend/tests/test_embedding_model_switch.py @@ -0,0 +1,161 @@ +"""Switching embedding models must re-embed the bank. + +Vectors from two different models are not comparable. They live in +different spaces and often have different widths. Changing the model must +discard the stored vectors and let the post-turn pass rebuild them. + +This worked while the vectors lived in `memories.embedding`. The settings +route nulled that column, and the embed queue picked up the rows. Migration +38 moved the vectors to `embedding_blob` and added an `embedded` flag beside +them, but the bulk clear kept nulling only the old column. The blob +survived, the flag stayed true, and `_embed_pending` (which filters on +`embedded IS FALSE`) never saw the rows. The bank kept ranking against the +previous model's vectors. + +Nothing reports this failure. `cosine` returns 0.0 on a width mismatch, so a +different-width model scores every memory zero, and retrieval silently +returns whichever rows sort first. A same-width model scores plausible +garbage instead. + + python -m pytest tests/test_embedding_model_switch.py -v +""" +import asyncio + +import pytest +from fastapi import Depends +from fastapi.testclient import TestClient + +from app import auth, limits, memorybank, models +from app.database import Base, SessionLocal, engine, get_db +from app.main import app + +DIMS = 8 + + +@pytest.fixture() +def client(monkeypatch): + Base.metadata.create_all(bind=engine) + memorybank._vector_cache.clear() + setup = SessionLocal() + user = models.User(is_guest=False, email="switch@example.com") + setup.add(user) + setup.flush() + setup.add(models.Settings( + user_id=user.id, api_key="enc:dummy", model="test-model", + embedding_model="model-a", + )) + adventure = models.Adventure( + user_id=user.id, title="Cave", script_state={}, memory_bank_enabled=True + ) + setup.add(adventure) + setup.flush() + for i in range(5): + memory = models.Memory(adventure_id=adventure.id, text=f"Memory {i}") + memorybank.set_vector(memory, [float(i)] + [0.0] * (DIMS - 1)) + setup.add(memory) + setup.commit() + adv_id, user_id = adventure.id, user.id + setup.close() + + monkeypatch.setattr(limits, "rate_limit", lambda *a, **k: None) + monkeypatch.setattr(limits, "check_row_cap", lambda *a, **k: None) + + def _current_user(db=Depends(get_db)): + return db.get(models.User, user_id) + + app.dependency_overrides[auth.get_current_user] = _current_user + c = TestClient(app) + c.adv_id = adv_id + try: + yield c + finally: + app.dependency_overrides.clear() + memorybank._vector_cache.clear() + Base.metadata.drop_all(bind=engine) + + +def memories(db): + return db.query(models.Memory).order_by(models.Memory.id).all() + + +def test_the_bank_starts_embedded(client): + db = SessionLocal() + try: + rows = memories(db) + assert len(rows) == 5 + assert all(m.embedded for m in rows) + assert all(m.embedding_blob for m in rows) + finally: + db.close() + + +def test_changing_the_model_clears_every_vector(client): + r = client.put("/api/settings", json={"embedding_model": "model-b"}) + assert r.status_code == 200, r.text + + db = SessionLocal() + try: + rows = memories(db) + assert [m.embedding_blob for m in rows] == [None] * 5, \ + "the blob survived the model change" + assert not any(m.embedded for m in rows), \ + "`embedded` stayed true, so nothing will ever re-embed these" + finally: + db.close() + + +def test_cleared_memories_are_queued_for_re_embedding(client): + """The `embedded` flag is not cosmetic. It is the only condition + `_embed_pending` filters on, so this test confirms the bank actually + recovers.""" + client.put("/api/settings", json={"embedding_model": "model-b"}) + + db = SessionLocal() + try: + pending = ( + db.query(models.Memory) + .filter(models.Memory.embedded.is_(False), + models.Memory.forgotten.is_(False)) + .all() + ) + assert len(pending) == 5 + finally: + db.close() + + +def test_retrieval_uses_no_stale_vector_after_the_switch(client, monkeypatch): + """Until the re-embed runs, the bank must return nothing rather than + ranking against the old model's vectors.""" + client.put("/api/settings", json={"embedding_model": "model-b"}) + + class Embedder: + async def embed(self, texts): + return [[1.0] + [0.0] * (DIMS - 1) for _ in texts] + + monkeypatch.setattr(memorybank, "embedding_provider", lambda s: Embedder()) + + db = SessionLocal() + try: + adventure = db.get(models.Adventure, client.adv_id) + settings = db.query(models.Settings).first() + result = asyncio.run( + memorybank.retrieve_memories(adventure, settings, update_stats=False) + ) + assert result["used"] == [] + finally: + db.close() + + +def test_an_unrelated_settings_change_keeps_the_vectors(client): + """Only an embedding-model change may clear the bank. Re-embedding costs + an API call per memory.""" + r = client.put("/api/settings", json={"model": "some-other-chat-model"}) + assert r.status_code == 200, r.text + + db = SessionLocal() + try: + rows = memories(db) + assert all(m.embedded for m in rows) + assert all(m.embedding_blob for m in rows) + finally: + db.close() diff --git a/backend/tests/test_guest_cleanup.py b/backend/tests/test_guest_cleanup.py new file mode 100644 index 0000000..7e11b30 --- /dev/null +++ b/backend/tests/test_guest_cleanup.py @@ -0,0 +1,205 @@ +"""Guest retention policy: app/cleanup.py. + +These tests cover the two things that matter. Idle guests and their whole +data graph must actually be deleted, and nothing else must ever be +deleted. + + python -m pytest tests/test_guest_cleanup.py -v +""" +from datetime import timedelta + + +import pytest +from sqlalchemy import create_engine, event +from sqlalchemy.orm import sessionmaker + +from app import cleanup, models +from app.database import Base +from app.migrations import bootstrap + + +@pytest.fixture() +def db(tmp_path): + engine = create_engine(f"sqlite:///{tmp_path/'t.db'}", connect_args={"check_same_thread": False}) + + @event.listens_for(engine, "connect") + def _fk(dbapi_connection, _record): + # The whole policy relies on ON DELETE CASCADE. SQLite ignores every + # one of them unless this is set, the same as database.py does. + cur = dbapi_connection.cursor() + cur.execute("PRAGMA foreign_keys=ON") + cur.close() + + bootstrap(engine) + Base.metadata.create_all(bind=engine) + session = sessionmaker(bind=engine, autoflush=False, expire_on_commit=False)() + yield session + session.close() + + +NOW = models.utcnow().replace(tzinfo=None) + + +def make_user(db, *, days_idle=None, days_old=0, guest=True, email=None): + """A user last seen `days_idle` ago (None = never seen, only created).""" + user = models.User( + is_guest=guest, + email=email, + password_hash=None if email is None else "x", + created_at=NOW - timedelta(days=days_old), + last_seen_at=None if days_idle is None else NOW - timedelta(days=days_idle), + ) + db.add(user) + db.commit() + return user + + +def sweep(db): + return cleanup.delete_stale_guests(db, now=NOW) + + +def alive(db, user_id): + # A count, not db.get: the sweep deletes with synchronize_session=False, so + # the session's identity map still holds the object and db.get would answer + # from memory without ever asking the database. + return db.query(models.User).filter(models.User.id == user_id).count() == 1 + + +# ---------- what goes ---------- + +def test_deletes_guest_idle_past_the_window(db): + user = make_user(db, days_idle=6) + assert sweep(db) == 1 + assert not alive(db, user.id) + + +def test_keeps_guest_inside_the_window(db): + user = make_user(db, days_idle=4) + assert sweep(db) == 0 + assert alive(db, user.id) + + +def test_boundary_is_not_yet_stale(db): + # Exactly 5 days survives. The comparison is strict. + user = make_user(db, days_idle=cleanup.RETENTION_DAYS) + assert sweep(db) == 0 + assert alive(db, user.id) + + +def test_never_seen_guest_falls_back_to_created_at(db): + """last_seen_at is NULL until a guest's second request (auth._touch runs + hourly), so a coalesce-less query would delete brand-new visitors.""" + fresh = make_user(db, days_idle=None, days_old=0) + stale = make_user(db, days_idle=None, days_old=9) + assert sweep(db) == 1 + assert alive(db, fresh.id) + assert not alive(db, stale.id) + + +def test_recent_visit_beats_an_old_created_at(db): + # A long-standing guest who came back yesterday stays. + user = make_user(db, days_idle=1, days_old=90) + assert sweep(db) == 0 + assert alive(db, user.id) + + +# ---------- what must never go ---------- + +def test_spares_registered_users(db): + """Registering upgrades the guest row in place. An idle account here is + a real user with real data, which is what signing up is meant to + protect.""" + user = make_user(db, days_idle=400, guest=False, email="a@b.com") + assert sweep(db) == 0 + assert alive(db, user.id) + + +def test_spares_the_local_mode_user(db): + # email NULL but is_guest False: local mode's implicit owner of everything. + user = make_user(db, days_idle=400, guest=False) + assert sweep(db) == 0 + assert alive(db, user.id) + + +def test_spares_a_guest_flagged_row_that_has_an_email(db): + # This row should not exist, but both clauses are checked so it cannot be collected. + user = make_user(db, days_idle=400, guest=True, email="odd@b.com") + assert sweep(db) == 0 + assert alive(db, user.id) + + +def test_leaves_seeded_public_scenarios_alone(db): + """Seeded demo content has user_id NULL, so it is outside the filter.""" + seeded = models.Scenario(user_id=None, is_public=True, title="Demo") + db.add(seeded) + make_user(db, days_idle=30) + db.commit() + assert sweep(db) == 1 + assert db.query(models.Scenario).filter(models.Scenario.id == seeded.id).count() == 1 + + +def test_disabled_when_retention_is_zero(db, monkeypatch): + monkeypatch.setattr(cleanup, "RETENTION_DAYS", 0) + user = make_user(db, days_idle=999) + assert sweep(db) == 0 + assert alive(db, user.id) + + +def test_enabled_requires_multi_user(monkeypatch): + from app import auth + monkeypatch.setattr(auth, "MULTI_USER", False) + assert cleanup.enabled() is False + monkeypatch.setattr(auth, "MULTI_USER", True) + monkeypatch.setattr(cleanup, "RETENTION_DAYS", 5) + assert cleanup.enabled() is True + monkeypatch.setattr(cleanup, "RETENTION_DAYS", 0) + assert cleanup.enabled() is False + + +# ---------- the cascade ---------- + +def test_deletes_the_whole_data_graph(db): + """One DELETE must remove the adventure, its actions and memories, the + story cards, and the settings row. Nothing is loaded into Python, so if + the FK cascade does not reach a table, its rows are silently orphaned, + or the statement fails, instead of being removed.""" + user = make_user(db, days_idle=30) + scenario = models.Scenario(user_id=user.id, title="S") + db.add(scenario) + db.commit() + adventure = models.Adventure(user_id=user.id, scenario_id=scenario.id, title="A") + db.add(adventure) + db.commit() + db.add_all([ + models.Action(adventure_id=adventure.id, type="ai", text="t"), + models.Memory(adventure_id=adventure.id, text="m", source_start=0, source_end=0), + models.StoryCard(adventure_id=adventure.id, name="c"), + models.Settings(user_id=user.id), + ]) + db.commit() + + assert sweep(db) == 1 + + for model in (models.Scenario, models.Adventure, models.Action, + models.Memory, models.StoryCard, models.Settings): + assert db.query(model).count() == 0, f"{model.__name__} rows survived" + + +def test_one_users_cleanup_does_not_touch_another(db): + keeper = make_user(db, days_idle=1) + keep_adv = models.Adventure(user_id=keeper.id, title="mine") + goner = make_user(db, days_idle=30) + db.add_all([keep_adv, models.Adventure(user_id=goner.id, title="theirs")]) + db.commit() + + assert sweep(db) == 1 + remaining = db.query(models.Adventure).all() + assert [a.title for a in remaining] == ["mine"] + + +def test_sweep_swallows_errors(monkeypatch): + """A broken cleanup must not take the app down (same rule as seeding).""" + monkeypatch.setattr(cleanup.auth, "MULTI_USER", True) + monkeypatch.setattr(cleanup, "delete_stale_guests", + lambda *a, **k: (_ for _ in ()).throw(RuntimeError("boom"))) + assert cleanup.sweep() == 0 diff --git a/backend/tests/test_history_window.py b/backend/tests/test_history_window.py new file mode 100644 index 0000000..b050523 --- /dev/null +++ b/backend/tests/test_history_window.py @@ -0,0 +1,257 @@ +"""The context builder reads a window of the story, not all of it. + +Walking `adventure.actions` every turn made the turn cost O(story length). +A long adventure read hundreds of KB to use only the tail of it, and the +cost grew with every turn played. `app.context.history` serves tails, +slices, and counts from SQL instead. + +Two things must hold, and both are easy to break by accident: + +* The window must produce exactly the prompt the full story produced. + Otherwise, the change alters behavior even though it looks like a pure + optimization. +* The helpers must agree with the old list arithmetic, because + memorybank's cursors are positions in that list. A cursor off by one + silently summarizes the wrong actions. + + python -m pytest tests/test_history_window.py -v +""" +import pytest +from sqlalchemy import event + +from app import memorybank, models +from app.context import builder, history +from app.database import Base, SessionLocal, engine + +# Long enough that a window is much smaller than the whole story. +ACTION_COUNT = 200 +NARRATION = ( + "The scrub gives way to a shallow bowl of land where woodsmoke hangs in " + "flat grey layers, and somewhere behind the largest tent a woman is " + "arguing, low and fast. " +) * 3 + +SCHEMA = { + "player": {"hp": {"min": 0, "max": 100, "initial": 100, "desc": "Health"}}, + "npcs": { + "gwen": {"name": "Gwen", "keys": ["gwen"], "desc": "A scout.", + "stats": {"trust": {"min": 0, "max": 100, "initial": 30}}}, + }, +} + + +@pytest.fixture() +def story(): + """An adventure with ACTION_COUNT actions, plus its settings.""" + Base.metadata.create_all(bind=engine) + db = SessionLocal() + user = models.User(is_guest=False, email="window@example.com") + db.add(user) + db.flush() + settings = models.Settings(user_id=user.id, api_key="enc:dummy", model="m") + db.add(settings) + scenario = models.Scenario(user_id=user.id, title="S", stat_schema=SCHEMA, + prompt="A long road." * 50) + db.add(scenario) + db.flush() + adventure = models.Adventure( + user_id=user.id, title="Long", scenario_id=scenario.id, script_state={}, + memory="The hero is hunting bandits. " * 20, + world_state={"player": {"hp": 100}, "npc": {"gwen": {"trust": 30}}, + "milestones": {}, "flags": {}, "_meta": {"last_changed": {}}}, + ) + db.add(adventure) + db.flush() + db.add(models.StoryCard(adventure_id=adventure.id, name="Gwen", keys="gwen", + entry="A scout with sharp eyes.", type="lore")) + for i in range(ACTION_COUNT): + db.add(models.Action( + adventure_id=adventure.id, + type="ai" if i % 2 else "do", + text=f"[{i}] {NARRATION}", + world_delta={"delta": {"player.hp": -1}, + "applied": [{"path": "player.hp", "old": 100, "new": 99}]}, + )) + db.commit() + db.expire_all() + adventure = db.get(models.Adventure, adventure.id) + settings = db.get(models.Settings, settings.id) + try: + yield db, adventure, settings + finally: + db.close() + Base.metadata.drop_all(bind=engine) + + +def full_window(adventure, budget_tokens, token_counter, exclude_action_id=None): + """Stand-in for `window_covering` that returns the entire story. This is + the behavior this module replaced.""" + return history.story_actions(adventure, exclude_action_id) + + +@pytest.fixture() +def actions_loaded(): + """Counts the `Action` rows the ORM materializes, which shows how much of + the story was actually fetched. `rowcount` is meaningless for a SELECT + on SQLite, so this counts the objects the mapper builds instead.""" + loaded = {"n": 0} + + def on_load(target, context): + loaded["n"] += 1 + + event.listen(models.Action, "load", on_load) + try: + yield loaded + finally: + event.remove(models.Action, "load", on_load) + + +# ------------------------------------------------------- the prompt is equal + +@pytest.mark.parametrize("budget", [1024, 4096, 8192, 16384, 65536]) +def test_window_builds_the_same_prompt_as_the_whole_story(story, budget, monkeypatch): + db, adventure, settings = story + settings.context_token_budget = budget + + windowed = builder.build_context(adventure, settings) + monkeypatch.setattr(builder.history, "window_covering", full_window) + db.expire(adventure) + everything = builder.build_context(adventure, settings) + + assert windowed[0] == everything[0], "system prompt differs" + assert windowed[1] == everything[1], "story prompt differs" + assert windowed[2]["history"] == everything[2]["history"] + assert windowed[2]["cards"] == everything[2]["cards"] + + +def test_window_matches_on_the_retry_shape(story, monkeypatch): + """Retry excludes the action being regenerated. The exclusion must reach + the window query, not just the in-memory filter.""" + db, adventure, settings = story + last = history.tail(adventure, 1)[0] + + windowed = builder.build_context(adventure, settings, exclude_action_id=last.id) + assert f"[{last.depth}]" not in windowed[1] + + monkeypatch.setattr(builder.history, "window_covering", full_window) + db.expire(adventure) + everything = builder.build_context(adventure, settings, exclude_action_id=last.id) + assert windowed[1] == everything[1] + + +def test_reported_total_is_the_whole_story_not_the_window(story): + """Insights reports "N of M actions included." M must not become the + window size.""" + db, adventure, settings = story + settings.context_token_budget = 4096 + report = builder.build_context(adventure, settings)[2] + assert report["history"]["total"] == ACTION_COUNT + assert report["history"]["included"] < ACTION_COUNT + + +# ------------------------------------------------------------ it is bounded + +def test_building_context_reads_far_less_than_the_whole_story(story, actions_loaded): + db, adventure, settings = story + # Expire first. Expiring afterward would discard the unflushed change and + # silently put the budget back to its default. + db.expire_all() + # Small enough that the budget, not the length of the story, decides. + settings.context_token_budget = 4096 + actions_loaded["n"] = 0 + + report = builder.build_context(adventure, settings)[2] + included = report["history"]["included"] + + assert included < ACTION_COUNT, "fixture is too short to prove anything" + # The window targets a margin past the budget and requests more if it + # falls short, so it reads somewhat more than it includes. What matters + # is that the read depends on the token budget, not on the length of + # the story. + assert actions_loaded["n"] < ACTION_COUNT // 2, ( + f"read {actions_loaded['n']} action rows out of {ACTION_COUNT} to " + f"include {included} — the window is not bounding the read" + ) + + +def test_window_is_ordered_and_free_of_duplicates(story): + """The window grows by fetching only what it does not already hold, so an + off-by-one in the offset would show up as a repeated or missing action.""" + db, adventure, settings = story + window = history.window_covering(adventure, 16384, builder.count_tokens) + ids = [a.id for a in window] + assert len(ids) == len(set(ids)), "the same action appeared twice in the window" + assert ids == sorted(ids), "window must be oldest-first" + + +# ------------------------------------------- the node-anchored reads agree + +def test_helpers_agree_with_the_full_list(story): + db, adventure, settings = story + actions = history.story_actions(adventure) + assert len(actions) == ACTION_COUNT + + assert history.count(adventure) == len(actions) + assert [a.id for a in history.tail(adventure, 4)] == [a.id for a in actions[-4:]] + assert [a.id for a in history.slice_(adventure, 10, 6)] == [a.id for a in actions[10:16]] + assert [a.id for a in history.tail_range(adventure, 5, 3)] == \ + [a.id for a in actions[-8:-5]] + assert history.newest(adventure).id == actions[-1].id + + for probe in (0, 1, ACTION_COUNT // 2, ACTION_COUNT - 1): + boundary = actions[probe].depth + assert history.count_after(adventure, boundary) == ACTION_COUNT - probe - 1 + assert [a.id for a in history.after(adventure, boundary, 3)] == \ + [a.id for a in actions[probe + 1:probe + 4]] + + +def test_a_depth_boundary_survives_a_middle_action_being_deleted(story): + """The case that broke the cursors twice before. This is the reason the + cursors are depths now. + + A position answers "how much story is past this point?" by counting + from the start. Deleting anything in front of the mark changes which + action the mark names. A depth names the same node either way. The + only thing that changes is the count of what comes after, and that + count is the one thing that should change here. + """ + db, adventure, settings = story + actions = history.story_actions(adventure) + mark = actions[30].depth + before = history.count_after(adventure, mark) + next_three = [a.id for a in history.after(adventure, mark, 3)] + + victim = actions[10] # in front of the mark + db.delete(victim) + db.commit() + db.expire(adventure) + + assert history.count(adventure) == ACTION_COUNT - 1 + assert history.count_after(adventure, mark) == before, "the mark moved" + assert [a.id for a in history.after(adventure, mark, 3)] == next_three + + # Deleting something after the mark is the one change that does affect + # the count, because that count reflects the story, not the coordinate + # system. + db.delete(history.after(adventure, mark, 1)[0]) + db.commit() + db.expire(adventure) + assert history.count_after(adventure, mark) == before - 1 + + +def test_blank_actions_are_excluded_the_same_way_in_sql_and_python(story): + """SQL and Python must agree on membership or a cursor points elsewhere.""" + db, adventure, settings = story + for blank in ("", " ", "\n", "\t\n "): + db.add(models.Action(adventure_id=adventure.id, + type="story", text=blank)) + db.commit() + db.expire(adventure) + + # SQL path (relationship not loaded) + from_sql = history.count(adventure) + # Python path (relationship loaded) + adventure.actions # noqa: B018 - force the collection into memory + from_python = history.count(adventure) + + assert from_sql == from_python == ACTION_COUNT diff --git a/backend/tests/test_length_hint.py b/backend/tests/test_length_hint.py new file mode 100644 index 0000000..b092956 --- /dev/null +++ b/backend/tests/test_length_hint.py @@ -0,0 +1,245 @@ +"""The turn prompt asks for a turn that fits inside `max_output_tokens`. + +`max_output_tokens` is a hard limit the endpoint enforces mid-sentence. The +state block is emitted after the narration, so a long turn hits the limit +partway through the block, and the deltas are lost. Nothing reads +`finish_reason`, so this loss happens silently. The prompt now carries a +word budget derived from the cap, so the model lands just inside it. + +Two things are easy to break here: + +* The hint must be stated in words, not tokens. A model cannot count its + own tokens, and a hint it cannot follow is wasted budget. +* The hint must not displace `EMIT_REMINDER` from the last position, which + is the whole mechanism that keeps the state block emitted at all (see + test_worldstate.py and the emit-reliability fix). + + python -m pytest tests/test_length_hint.py -v +""" +import re + + +import pytest + +from app import models, worldstate +from app.context import builder +from app.database import Base, SessionLocal, engine + +SCHEMA = { + "player": {"hp": {"min": 0, "max": 100, "initial": 100, "desc": "Health"}}, +} + + +@pytest.fixture() +def story(): + """A short adventure, with and without a stat schema on demand.""" + Base.metadata.create_all(bind=engine) + db = SessionLocal() + user = models.User(is_guest=False, email="length@example.com") + db.add(user) + db.flush() + settings = models.Settings(user_id=user.id, api_key="enc:dummy", model="m") + db.add(settings) + scenario = models.Scenario(user_id=user.id, title="S", prompt="A road.") + db.add(scenario) + db.flush() + adventure = models.Adventure( + user_id=user.id, title="A", scenario_id=scenario.id, script_state={}, + memory="The hero hunts bandits.", + ) + db.add(adventure) + db.flush() + for i in range(4): + db.add(models.Action(adventure_id=adventure.id, + type="ai" if i % 2 else "do", text=f"[{i}] Onward.")) + db.commit() + db.expire_all() + adventure = db.get(models.Adventure, adventure.id) + settings = db.get(models.Settings, settings.id) + try: + yield db, adventure, settings, scenario.id + finally: + db.close() + Base.metadata.drop_all(bind=engine) + + +def with_schema(db, scenario_id, adventure): + scenario = db.get(models.Scenario, scenario_id) + scenario.stat_schema = SCHEMA + adventure.world_state = worldstate.instantiate(SCHEMA) + db.commit() + db.expire_all() + + +# ----------------------------------------------------- the hint itself + +def test_budget_is_the_cap_minus_headroom_and_buffer_in_words(): + """800-token cap → 750 after headroom → ~562 words → 506 after the buffer.""" + hint = builder.length_hint(800, has_ws=True) + assert "506" in hint + assert "token" not in hint.lower(), "a model cannot count its own tokens" + + +def test_budget_tracks_the_setting(): + small = builder.length_hint(800, has_ws=True) + large = builder.length_hint(2400, has_ws=True) + assert small != large + assert "1586" in large + + +def asked_words(cap): + return int(re.search(r"(\d+) words", builder.length_hint(cap, has_ws=True)).group(1)) + + +def test_buffer_leaves_room_for_overshoot(): + """The stated number must sit meaningfully under the real ceiling, or an + on-target-but-slightly-long turn still hits the limit.""" + for cap in (400, 800, 1500, 2400): + asked = asked_words(cap) + ceiling = (cap - builder.LENGTH_HEADROOM) * builder.WORDS_PER_TOKEN + assert asked < ceiling + assert asked >= ceiling * 0.85, "buffer so large the hint wastes the cap" + + +def test_hint_is_phrased_as_a_ceiling_not_a_budget(): + """Measured: budget phrasing ("keep this turn under about N words") + reads as a target to fill. It moved the mean turn from 174 to 246 + words, toward the limit it exists to avoid. The limit framing must + survive future prompt edits.""" + hint = builder.length_hint(800, has_ws=True) + assert "must not exceed" in hint + assert "under about" not in hint + assert "lower end" in hint, "without this the number still reads as a target" + + +def test_hint_states_a_floor_as_well_as_a_ceiling(): + """A ceiling alone is one-sided: a terse model has nothing to act on but + the "only as much as the moment needs" clause and produces only two + paragraphs. The floor is what makes the same prompt produce a similar + length across models with different tendencies.""" + hint = builder.length_hint(800, has_ws=True) + assert "506" in hint and "177" in hint + assert "should not stop short of" in hint + # Asymmetric on purpose: the ceiling is a hard limit and the floor is a + # soft target, and neither is phrased as a specific number to reach. + assert hint.index("must not exceed") < hint.index("should not stop short of") + + +def test_floor_stays_well_under_the_ceiling(): + for cap in (400, 800, 1500, 2400): + hint = builder.length_hint(cap, has_ws=True) + ceiling, floor = (int(n) for n in re.findall(r"(\d+)", hint)[:2]) + assert floor < ceiling * 0.5 + + +def test_floor_is_dropped_when_the_cap_is_too_tight_for_one(): + """At a tight cap a short turn is the correct turn. The tight-cap + wording is the one measured to keep the state block from being + truncated (0/6 truncations at cap 250, against 2/6 unhinted), so it + is left exactly as it was.""" + hint = builder.length_hint(250, has_ws=True) + assert "should not stop short of" not in hint + assert "much shorter" in hint + + +def test_floor_does_not_grow_without_bound(): + """A big cap means "long turns are allowed", not "every turn must be an essay": + the share alone would demand 555 words minimum at cap 2400.""" + hint = builder.length_hint(2400, has_ws=True) + assert str(builder.MAX_LENGTH_FLOOR_WORDS) in hint + + +def test_no_hint_when_the_cap_is_too_small_to_phrase(): + """Under the floor the hint is noise the model pays for in context.""" + assert builder.length_hint(100, has_ws=True) == "" + assert builder.length_hint(builder.LENGTH_HEADROOM, has_ws=True) == "" + assert builder.length_hint(0, has_ws=True) == "" + + +def test_no_negative_word_budget(): + """A cap below the headroom must not ask for a negative number of words.""" + for cap in (1, 10, 49, 51): + assert builder.length_hint(cap, has_ws=True) == "" + + +def test_reason_given_matches_whether_state_is_tracked(): + assert "state block" in builder.length_hint(800, has_ws=True) + assert "state block" not in builder.length_hint(800, has_ws=False) + + +# ----------------------------------------------------- in the assembled prompt + +def test_hint_reaches_the_story_prompt(story): + db, adventure, settings, _ = story + settings.max_output_tokens = 800 + _, story_text, report = builder.build_context(adventure, settings) + + assert "506" in story_text + labels = [s["label"] for s in report["sections"]] + assert "length_hint" in labels + + +def test_emit_reminder_keeps_the_last_word(story): + """The hint sits above the emit reminder: the reminder's whole value is the + recency slot, and the model has to write the narration before the block.""" + db, adventure, settings, scenario_id = story + with_schema(db, scenario_id, adventure) + settings.max_output_tokens = 800 + + _, story_text, report = builder.build_context(adventure, settings) + + assert story_text.rstrip().endswith(worldstate.EMIT_REMINDER.rstrip()) + labels = [s["label"] for s in report["sections"]] + assert labels.index("length_hint") < labels.index("world_state_reminder") + + +def test_prompt_stays_inside_the_budget_on_a_long_story(story): + """Regression guard: the hint is appended after history has already + spent the budget, so it must be reserved up front like `EMIT_REMINDER` + is. + + This check is weak on purpose. The history loop stops before crossing + its budget, so it leaves about one action of slack, and the roughly + 30-token hint fits inside that slack. This catches a hint that grows + large, not a missing reservation. The reservation itself is not + observable from the outside.""" + db, adventure, settings, _ = story + for i in range(4, 120): + db.add(models.Action( + adventure_id=adventure.id, type="ai" if i % 2 else "do", + text=f"[{i}] " + "The road bends past the burnt mill and the smoke. " * 12, + )) + db.commit() + db.expire_all() + adventure = db.get(models.Adventure, adventure.id) + + settings.max_output_tokens = 2400 + settings.context_token_budget = 2048 + + _, _, report = builder.build_context(adventure, settings) + assert report["history"]["included"] < 120, "budget was never actually filled" + assert report["tokens"]["total"] <= report["tokens"]["budget"] + + +def test_hint_is_counted_in_the_reported_totals(story): + """Insights reports what the turn actually costs. A section that + reaches the model but not the accounting makes that reported cost + inaccurate.""" + db, adventure, settings, _ = story + settings.max_output_tokens = 800 + + _, _, report = builder.build_context(adventure, settings) + hint = next(s for s in report["sections"] if s["label"] == "length_hint") + assert hint["tokens"] > 0 + assert hint["text"] in report["prompt"]["story"] + assert builder.count_tokens(report["prompt"]["story"]) <= report["tokens"]["total"] + + +def test_no_hint_section_when_the_cap_is_tiny(story): + """An empty hint drops out entirely rather than leaving a blank section.""" + db, adventure, settings, _ = story + settings.max_output_tokens = 60 + + _, story_text, report = builder.build_context(adventure, settings) + assert "length_hint" not in [s["label"] for s in report["sections"]] + assert "Keep this turn" not in story_text diff --git a/backend/tests/test_memory_nodes.py b/backend/tests/test_memory_nodes.py new file mode 100644 index 0000000..bcc4ede --- /dev/null +++ b/backend/tests/test_memory_nodes.py @@ -0,0 +1,506 @@ +"""Phase 14 SP3: memories attach to nodes, and the marks are nodes too. + +Two claims, and neither fails loudly if it is wrong: + +* A memory belongs to the path that produced it. A memory made on branch B + must be invisible from A, and the memories of a shared ancestor must be + visible from both, without anything being copied when a fork happens. The + failure mode is a prompt that quietly carries a summary of a story the + player abandoned. +* Retrieval reads the whole lineage, and that stays affordable. The story + is read through a window, but recall is long-range by definition and + cannot use one. So the clause names every ancestor. The bet is that + memories are sparse enough, one per six actions, for that to stay tens of + small rows even twenty forks deep. This file measures that bet below + rather than asserting it. + +Nothing in the product forks yet, so this file builds the fork by hand, +exactly as `test_branch_clause.py` builds it. + + python -m pytest tests/test_memory_nodes.py -v +""" +import asyncio + +import pytest + +from app import memorybank, models, tree +from app.context import cursors, lineage +from app.database import Base, SessionLocal, engine +from tools import dbmeter + + +class StubEmbedder: + """Returns whatever vector the test set, for any text.""" + + def __init__(self, vector=(1.0, 0.0, 0.0)): + self.vector = list(vector) + + async def embed(self, texts): + return [list(self.vector) for _ in texts] + + +# --------------------------------------------------------------- the fixture + +def make_branch(db, adventure, parent=None, fork_depth=None): + """A branch row whose lineage is its parent's lineage, capped, plus + itself. This is the computation SP5 performs at fork time. The fixture + reimplements it here so it cannot pass by agreeing with a bug in the + code under test.""" + branch = models.Branch( + adventure_id=adventure.id, + parent_branch_id=parent.id if parent else None, + fork_depth=fork_depth, + lineage=[], + ) + db.add(branch) + db.flush() + inherited = [] + if parent is not None: + for ancestor_id, cap in lineage.entries_of(parent): + capped = fork_depth if cap is None else min(cap, fork_depth) + inherited.append([ancestor_id, capped]) + branch.lineage = [[branch.id, None]] + inherited + db.flush() + return branch + + +def add_node(db, adventure, branch, depth, label, index=None): + action = models.Action( + adventure_id=adventure.id, + branch_id=branch.id, + depth=depth, + type="ai" if depth % 2 else "do", + text=f"{label}{depth}", + ) + db.add(action) + return action + + +def add_memory(db, adventure, text, node, vector=(1.0, 0.0, 0.0), **kwargs): + """A memory of the block ending on `node`, attached the way the post-turn + pass attaches one.""" + memory = models.Memory( + adventure_id=adventure.id, text=text, + source_start=None if node is None else node.depth, + source_end=None if node is None else node.depth, + **kwargs, + ) + if node is not None: + tree.attach_memory(memory, node) + else: + tree.place_memory(db, adventure, memory) + db.add(memory) + db.flush() + memorybank.set_vector(memory, list(vector)) + db.commit() + return memory + + +@pytest.fixture() +def forked(): + """A0..A3, then B4 B5 off A3, then C6 C7 off B5, with a memory attached + to one node of each branch. A keeps playing past the fork point where B + left it. + + The head is C, so the story is A0 A1 A2 A3 B4 B5 C6 C7, and the + memories in play are A's, B's, and C's. The one on A5 is excluded: it + is on a sibling of B4 and belongs to a story nobody is reading. + """ + Base.metadata.create_all(bind=engine) + db = SessionLocal() + user = models.User(is_guest=False, email="nodes@example.com") + db.add(user) + db.flush() + settings = models.Settings( + user_id=user.id, api_key="enc:dummy", model="m", + embedding_model="text-embedding-3-small", memory_top_k=10, + memory_bank_capacity=80, + ) + db.add(settings) + adventure = models.Adventure( + user_id=user.id, title="Forked", script_state={}, memory_bank_enabled=True, + auto_summarize=True, + ) + db.add(adventure) + db.flush() + + a = make_branch(db, adventure) + b = make_branch(db, adventure, parent=a, fork_depth=3) + c = make_branch(db, adventure, parent=b, fork_depth=5) + nodes = {} + for depth in range(4): + nodes[f"A{depth}"] = add_node(db, adventure, a, depth, "A") + for depth in (4, 5): # A kept playing: siblings of B4/B5 + nodes[f"A{depth}"] = add_node(db, adventure, a, depth, "A", index=100 + depth) + for depth in (4, 5): + nodes[f"B{depth}"] = add_node(db, adventure, b, depth, "B") + for depth in (6, 7): + nodes[f"C{depth}"] = add_node(db, adventure, c, depth, "C") + db.flush() + + memories = { + "shared": add_memory(db, adventure, "on the shared trunk", nodes["A3"]), + "sibling": add_memory(db, adventure, "on A's own continuation", nodes["A5"]), + "b": add_memory(db, adventure, "on B", nodes["B5"]), + "c": add_memory(db, adventure, "on C", nodes["C7"]), + } + adventure.head_branch_id = c.id + adventure.head_depth = 7 + db.commit() + + ids = {"a": a.id, "b": b.id, "c": c.id, "nodes": nodes, "memories": memories} + try: + yield db, adventure, settings, ids + finally: + db.close() + Base.metadata.drop_all(bind=engine) + + +def switch_to(db, adventure, branch_id, tip): + adventure.head_branch_id = branch_id + adventure.head_depth = tip + db.commit() + + +def retrieved(adventure, settings) -> set[str]: + memorybank.embedding_provider = lambda s: StubEmbedder() + result = asyncio.run( + memorybank.retrieve_memories(adventure, settings, update_stats=False) + ) + assert result["error"] is None, result["error"] + return {m["text"] for m in result["used"]} + + +# ------------------------------------------------------------- the isolation + +def test_a_memory_on_a_sibling_is_not_retrieved(forked): + """The whole point of this file. A5 is a node of the story that was + abandoned when B forked, and the memory attached to it must not reach a + prompt on C.""" + db, adventure, settings, ids = forked + assert retrieved(adventure, settings) == { + "on the shared trunk", "on B", "on C" + } + + +def test_a_shared_ancestor_is_visible_from_both_branches(forked): + """Nothing is copied at a fork, so the trunk's memories are shared by + construction rather than by duplication.""" + db, adventure, settings, ids = forked + switch_to(db, adventure, ids["a"], 5) + from_a = retrieved(adventure, settings) + assert "on the shared trunk" in from_a + # From A, the branches taken off it are the ones out of reach. + assert from_a == {"on the shared trunk", "on A's own continuation"} + + +def test_the_lineage_is_read_whole_not_windowed(forked): + """The story is read through a window. Recall is not. The trunk memory + is four nodes and two forks back, and is still a candidate.""" + db, adventure, settings, ids = forked + path = lineage.path_of(db, adventure) + assert len(path) == 3 + # The window a story read would use here names one entry. Retrieval + # names all three, which is the difference this test checks. + assert path.prefix_covering(2) == 1 + assert "on the shared trunk" in retrieved(adventure, settings) + + +def test_a_hand_written_memory_is_anchored_where_it_was_typed(forked): + """SP7: a typed memory takes the head, so it obeys the same rule as a + summarized one. + + It used to carry no depth, which sounded like "belongs to the whole + adventure" and behaved like "cannot be capped at a fork." It followed + the reader onto branches whose story it never described. Anchoring it + makes the bank answer one question instead of two. + """ + db, adventure, settings, ids = forked + switch_to(db, adventure, ids["a"], 5) + typed = add_memory(db, adventure, "typed by hand", None) + assert (typed.branch_id, typed.depth) == (ids["a"], 5), "the head it was typed at" + + +def test_a_typed_memory_survives_a_fork_of_the_ground_it_was_typed_on(forked): + """The half of the old behavior that was correct, kept. + + A memory typed on the shared trunk is still there after forking away, + but only because the fork's path goes through that node, not because + the memory is exempt from being capped. + """ + db, adventure, settings, ids = forked + switch_to(db, adventure, ids["a"], 3) # the node B, and so C, forked from + add_memory(db, adventure, "typed on the trunk", None) + + switch_to(db, adventure, ids["c"], 7) + assert "typed on the trunk" in retrieved(adventure, settings) + + +def test_a_typed_memory_does_not_follow_you_onto_a_path_it_is_not_on(forked): + """The other half of the old behavior, which was wrong, is now fixed. + + A5 is A's own continuation past the point where B left it, so it is a + sibling of the story C tells. This is exactly where the `sibling` + memory sits, and it is excluded for the same reason. Typing a memory + instead of summarizing it grants no exemption from the path rule. + """ + db, adventure, settings, ids = forked + switch_to(db, adventure, ids["a"], 5) + add_memory(db, adventure, "typed off the path", None) + + switch_to(db, adventure, ids["c"], 7) + assert "typed off the path" not in retrieved(adventure, settings) + + +# ------------------------------------------------------------------ the marks + +def test_a_mark_moves_to_the_node_the_memory_covers(forked): + """The mark and the memory are one statement about where the pass got to, + so they are written from the same row.""" + db, adventure, settings, ids = forked + cursors.MEMORY.anchor_at(adventure, ids["nodes"]["B5"]) + db.commit() + assert cursors.MEMORY.stored(adventure) == (ids["b"], 5) + assert cursors.MEMORY.depth(db, adventure) == 5 + + +def test_a_mark_from_a_sibling_reads_as_nothing_covered(forked): + """A mark is a node, so switching to another story must resolve the + mark's meaning rather than assume it. A path segment this story never + took is not covered, and the fallback for "not covered" must be redoing + the work, not skipping it.""" + db, adventure, settings, ids = forked + cursors.MEMORY.anchor_at(adventure, ids["nodes"]["C7"]) + db.commit() + switch_to(db, adventure, ids["a"], 5) + assert cursors.MEMORY.depth(db, adventure) == cursors.NO_DEPTH + + +def test_a_mark_on_an_ancestor_is_capped_at_the_fork(forked): + """A6 and A7 are past where this path left A, so a mark deeper than the + fork cannot mean 'covered' for anything on this story.""" + db, adventure, settings, ids = forked + cursors.MEMORY.anchor_at(adventure, ids["nodes"]["A5"]) + db.commit() + assert cursors.MEMORY.depth(db, adventure) == 3 # C forks off B forks off A@3 + + +def test_a_mark_never_moves_forward_on_a_rewind(forked): + db, adventure, settings, ids = forked + cursors.MEMORY.anchor_at(adventure, ids["nodes"]["A3"]) + cursors.rewind_all(adventure, ids["c"], 6) + assert cursors.MEMORY.stored(adventure) == (ids["a"], 3) + + +# ---------------------------------------------------- what the passes read + +def test_the_summary_folds_in_only_the_path_it_is_on(forked, monkeypatch): + """`_update_story_summary` gathers the memories past its mark. On C + that is B's and C's memories. It never includes the one on A's own + continuation, even though that memory's depth would otherwise put it + inside the range.""" + db, adventure, settings, ids = forked + monkeypatch.setattr(memorybank, "SUMMARY_INTERVAL", 1) + + class Stub: + def __init__(self): + self.prompts = [] + + async def complete(self, system, user, **kwargs): + self.prompts.append(user) + return "A summary." + + stub = Stub() + monkeypatch.setattr(memorybank, "summary_provider", lambda s: stub) + cursors.SUMMARY.anchor_at(adventure, ids["nodes"]["A3"]) + db.commit() + + asyncio.run(memorybank._update_story_summary(adventure, settings, db)) + + [prompt] = stub.prompts + assert "on B" in prompt and "on C" in prompt + assert "on A's own continuation" not in prompt + assert "on the shared trunk" not in prompt # behind the mark + # Caught up to the end of the story. Until SP4 that was C6: the newest + # action was held back because retrying it rewrote the row underneath the + # mark. A retry writes a sibling now, and the withdrawal that follows takes + # the mark back with it, so there is nothing to hold back. + assert cursors.SUMMARY.stored(adventure) == (ids["c"], 7) + + +def test_a_block_is_summarized_from_the_path_and_hung_off_its_last_node( + forked, monkeypatch +): + db, adventure, settings, ids = forked + monkeypatch.setattr(memorybank, "MEMORY_START", 0) + monkeypatch.setattr(memorybank, "MEMORY_INTERVAL", 4) + # This test is about which actions a block is read from, not about when a + # block forms, so switch the settling slack off and let the path end on a + # block boundary. `test_memory_settling` owns the timing rule. + monkeypatch.setattr(memorybank, "SETTLE_SLACK", 0) + + class Stub: + def __init__(self): + self.excerpts = [] + + async def complete(self, system, user, **kwargs): + self.excerpts.append(user) + return f"Memory {len(self.excerpts)}." + + stub = Stub() + monkeypatch.setattr(memorybank, "summary_provider", lambda s: stub) + asyncio.run(memorybank._create_due_memories(adventure, settings, db)) + + # Two blocks of four from a path of eight, both formed in one pass. + first, second = stub.excerpts + assert "A5" not in first + second, "a sibling's narration reached the summarizer" + assert ["A0", "A1", "A2", "A3"] == [line for line in first.split() if line[0] in "ABC"] + assert ["B4", "B5", "C6", "C7"] == [line for line in second.split() if line[0] in "ABC"] + made = db.query(models.Memory).filter_by(text="Memory 1.").one() + assert (made.branch_id, made.depth) == (ids["a"], 3) + # The mark ends up on the node the second block attaches to, which is the tip. + assert cursors.MEMORY.stored(adventure) == (ids["c"], 7) + + +# ------------------------------------------------------ the cost of forking + +@pytest.fixture() +def deeply_forked(): + """A story forked twenty times, with a memory every six actions. This + is the density the post-turn pass actually produces.""" + Base.metadata.create_all(bind=engine) + db = SessionLocal() + user = models.User(is_guest=False, email="deepmem@example.com") + db.add(user) + db.flush() + db.add(models.Settings( + user_id=user.id, api_key="enc:dummy", model="m", + embedding_model="text-embedding-3-small", + # Every candidate is injected, so the measurement covers fetching the + # texts too and not only ranking them. + memory_top_k=50, + )) + + def story(title, forks): + adventure = models.Adventure( + user_id=user.id, title=title, script_state={}, memory_bank_enabled=True, + ) + db.add(adventure) + db.flush() + branch = make_branch(db, adventure) + depth = 0 + nodes = [] + for _ in range(4): + nodes.append(add_node(db, adventure, branch, depth, "n")) + depth += 1 + for _ in range(forks): + branch = make_branch(db, adventure, parent=branch, fork_depth=depth - 1) + for _ in range(2): + nodes.append(add_node(db, adventure, branch, depth, "n")) + depth += 1 + if forks: + branch = make_branch(db, adventure, parent=branch, fork_depth=depth - 1) + for _ in range(84 - depth): + nodes.append(add_node(db, adventure, branch, depth, "n")) + depth += 1 + db.flush() + for node in nodes[5::6]: # one memory per six actions, as the pass makes them + add_memory(db, adventure, f"memory at {node.depth}", node) + adventure.head_branch_id = branch.id + adventure.head_depth = depth - 1 + return adventure + + forked_story = story("Forked", 20) + flat_story = story("Flat", 0) + db.commit() + try: + yield db, flat_story, forked_story + finally: + db.close() + Base.metadata.drop_all(bind=engine) + + +def test_retrieving_from_a_deep_fork_costs_what_a_flat_story_costs(deeply_forked): + """The bet from the module docstring, measured in bytes. Retrieval + names all twenty-two branches instead of one, but it fetches only an id + and a flag per memory, and both stories return the same fourteen + memories. The clause is where the difference shows up, and the clause + is not what crosses the wire.""" + db, flat_story, forked_story = deeply_forked + settings = db.query(models.Settings).one() + flat_id, forked_id = flat_story.id, forked_story.id + db.commit() + db.expire_all() + + meter = dbmeter.Meter() + meter.attach(engine) + try: + with meter.scope("flat"): + assert len(retrieved(db.get(models.Adventure, flat_id), settings)) == 14 + flat_bytes = meter.scopes[-1].total.fetched + with meter.scope("forked"): + assert len(retrieved(db.get(models.Adventure, forked_id), settings)) == 14 + forked_bytes = meter.scopes[-1].total.fetched + finally: + meter.detach() + + # Measured 2026-08-18: 1,807 B against 1,823 B. Both figures cover the + # same fourteen rows, named through twenty-two branch terms instead of one. + assert flat_bytes > 0, "the meter saw nothing; it is measuring the wrong connection" + assert forked_bytes < flat_bytes * 1.5, ( + f"retrieval on a 20-fork story cost {forked_bytes:,} B against the " + f"{flat_bytes:,} B a flat story of the same length cost" + ) + + +# ------------------------------------------------------- the opening node + +def test_a_typed_memory_on_the_opening_node_survives_that_node_going(forked): + """The one exception where a node and its memories are not withdrawn + together. + + A memory anchored to a node is withdrawn with the node. This is the + rule, and it is deliberate: the memory described that turn, and the + turn is leaving. But migration 62 parked every memory written before + memories had coordinates on depth 0, the only landing spot visible from + every branch. As a result, the opening node carries a whole bank of + memories it never produced. Withdrawing it would delete all of those + memories at once, for every adventure that predates the tree. + + A memory with no `source_start` covers no stretch of story, so nothing + about it can go stale. It stays. + """ + db, adventure, settings, ids = forked + typed = models.Memory( + adventure_id=adventure.id, text="Kira is the innkeeper's daughter", + source_start=None, source_end=None, + ) + typed.branch_id, typed.depth = ids["a"], 0 + db.add(typed) + db.commit() + typed_id = typed.id + + withdrawn = memorybank.forget_node(db, adventure, ids["nodes"]["A0"]) + db.commit() + + assert withdrawn == 0 + assert db.get(models.Memory, typed_id) is not None + + +def test_a_summary_of_the_opening_node_is_still_withdrawn(forked): + """The exception is about memories that describe nothing, not about depth 0. + + A summary that genuinely ends on the opening node describes text that + is being removed, so the summary is removed too. Otherwise the root + would collect exactly the dangling rows that `forget_node` replaced + `prune_dangling_memories` to prevent. + """ + db, adventure, settings, ids = forked + derived = add_memory(db, adventure, "the opening, summarised", ids["nodes"]["A0"]) + derived_id = derived.id + + withdrawn = memorybank.forget_node(db, adventure, ids["nodes"]["A0"]) + db.commit() + + assert withdrawn == 1 + assert db.get(models.Memory, derived_id) is None diff --git a/backend/tests/test_memory_retrieval.py b/backend/tests/test_memory_retrieval.py new file mode 100644 index 0000000..ba4cfe4 --- /dev/null +++ b/backend/tests/test_memory_retrieval.py @@ -0,0 +1,452 @@ +"""Ranking the memory bank without reading the memory bank. + +Retrieval used to walk `adventure.memories`, which loaded every row with +its vector. That vector data was 96% of everything a turn read. Retrieval +now asks SQL which memories are in play, holds their vectors in process, +and fetches text for only the five it picks. + +Three things have to stay true for that to be safe, and each is a separate +failure that no error message would report: + +* the ranking picks the same memories it always did; +* nothing bulk-reads a vector column again; +* a cached vector is never served after the stored one changed. + + python -m pytest tests/test_memory_retrieval.py -v +""" +import asyncio +from datetime import timedelta + +import pytest +from sqlalchemy import event, inspect as sa_inspect + +from app import memorybank, models +from app.database import Base, SessionLocal, engine + + +class StubEmbedder: + """Returns whatever vector the test set, and counts calls.""" + + def __init__(self, vector=(1.0, 0.0, 0.0)): + self.vector = list(vector) + self.calls = 0 + + async def embed(self, texts): + self.calls += 1 + return [list(self.vector) for _ in texts] + + +@pytest.fixture() +def db(): + Base.metadata.create_all(bind=engine) + memorybank._vector_cache.clear() + session = SessionLocal() + try: + yield session + finally: + session.close() + memorybank._vector_cache.clear() + Base.metadata.drop_all(bind=engine) + + +@pytest.fixture() +def settings(db): + user = models.User(is_guest=False, email="rank@example.com") + db.add(user) + db.flush() + row = models.Settings( + user_id=user.id, api_key="enc:dummy", model="m", + embedding_model="text-embedding-3-small", memory_top_k=2, + memory_bank_capacity=80, + ) + db.add(row) + db.commit() + return row + + +@pytest.fixture() +def adventure(db, settings): + adv = models.Adventure( + user_id=settings.user_id, title="Cave", script_state={}, + memory_bank_enabled=True, + ) + db.add(adv) + db.flush() + # Retrieval builds its query from the newest actions. With none, it + # returns before ranking anything. + for i in range(2): + db.add(models.Action( + adventure_id=adv.id, type="ai", text=f"Something happened {i}." + )) + db.commit() + return adv + + +def add_memory(db, adventure, text, vector, **kwargs): + memory = models.Memory(adventure_id=adventure.id, text=text, **kwargs) + db.add(memory) + db.flush() + if vector is not None: + memorybank.set_vector(memory, list(vector)) + db.commit() + return memory + + +@pytest.fixture() +def bank(db, adventure): + """Three orthogonal vectors, so a query vector picks one unambiguously.""" + return { + "x": add_memory(db, adventure, "about x", (1.0, 0.0, 0.0)), + "y": add_memory(db, adventure, "about y", (0.0, 1.0, 0.0)), + "z": add_memory(db, adventure, "about z", (0.0, 0.0, 1.0)), + } + + +def retrieve(adventure, settings, embedder, **kwargs): + memorybank.embedding_provider = lambda s: embedder + kwargs.setdefault("update_stats", False) + return asyncio.run(memorybank.retrieve_memories(adventure, settings, **kwargs)) + + +@pytest.fixture() +def sql_log(): + statements: list[str] = [] + + def record(conn, cursor, statement, parameters, context, executemany): + statements.append(statement) + + event.listen(engine, "before_cursor_execute", record) + try: + yield statements + finally: + event.remove(engine, "before_cursor_execute", record) + + +# ------------------------------------------------------------------- ranking + +def test_ranks_by_cosine_similarity(db, adventure, settings, bank): + result = retrieve(adventure, settings, StubEmbedder((1.0, 0.0, 0.0))) + assert [m["id"] for m in result["used"]][0] == bank["x"].id + assert result["used"][0]["similarity"] == pytest.approx(1.0) + assert result["used"][0]["text"] == "about x" + + +def test_honours_top_k(db, adventure, settings, bank): + settings.memory_top_k = 1 + db.commit() + assert len(retrieve(adventure, settings, StubEmbedder())["used"]) == 1 + + +def test_pinned_memories_are_always_used(db, adventure, settings, bank): + """A pin means "always in context", however badly it scores.""" + settings.memory_top_k = 1 + bank["z"].pinned = True + db.commit() + + used = retrieve(adventure, settings, StubEmbedder((1.0, 0.0, 0.0)))["used"] + assert [m["id"] for m in used] == [bank["z"].id] + assert used[0]["pinned"] is True + + +def test_forgotten_and_unembedded_memories_are_not_candidates(db, adventure, settings): + live = add_memory(db, adventure, "live", (1.0, 0.0, 0.0)) + add_memory(db, adventure, "evicted", (1.0, 0.0, 0.0), forgotten=True) + add_memory(db, adventure, "no vector yet", None) + + used = retrieve(adventure, settings, StubEmbedder())["used"] + assert [m["id"] for m in used] == [live.id] + + +def test_empty_bank_returns_no_error(db, adventure, settings): + assert retrieve(adventure, settings, StubEmbedder()) == {"used": [], "error": None} + + +def test_missing_embedding_model_is_reported(db, adventure, settings, bank): + settings.embedding_model = "" + db.commit() + result = retrieve(adventure, settings, StubEmbedder()) + assert result["used"] == [] and "embedding model" in result["error"] + + +def test_update_stats_bumps_only_the_used(db, adventure, settings, bank): + settings.memory_top_k = 1 + db.commit() + retrieve(adventure, settings, StubEmbedder((1.0, 0.0, 0.0)), update_stats=True) + db.commit() + db.expire_all() + + assert db.get(models.Memory, bank["x"].id).use_count == 1 + assert db.get(models.Memory, bank["x"].id).last_used_at is not None + assert db.get(models.Memory, bank["y"].id).use_count == 0 + + +def test_dry_runs_do_not_bump_the_counters(db, adventure, settings, bank): + """Insights assembles a context without spending a turn. It must not + look like the memories were used.""" + retrieve(adventure, settings, StubEmbedder(), update_stats=False) + db.commit() + db.expire_all() + assert all(db.get(models.Memory, m.id).use_count == 0 for m in bank.values()) + + +# --------------------------------------------------------------- the egress + +def memory_selects(statements): + return [ + s for s in statements + if "FROM memories" in s and s.lstrip().upper().startswith("SELECT") + ] + + +def test_the_json_column_is_gone(db): + """`memories.embedding` held the vectors before migration 38, and + nothing read it afterward. Migration 42 dropped it. Restoring it would + bring back 4 MB of dead weight and a second place vectors can be + written from. That second place is how the model-switch bug happened + (test_embedding_model_switch.py).""" + columns = {c["name"] for c in sa_inspect(engine).get_columns("memories")} + assert "embedding" not in columns + assert {"embedding_blob", "embedded"} <= columns + + +def test_the_catalogue_query_carries_no_vectors(db, adventure, settings, bank, sql_log): + """The query that decides which memories are in play must stay tiny. + This is the query that used to pull the whole bank across the wire.""" + retrieve(adventure, settings, StubEmbedder()) + catalogue = [s for s in memory_selects(sql_log) if "memories.pinned" in s] + assert catalogue, "expected a catalogue query" + assert not any("embedding_blob" in s for s in catalogue) + + +def test_a_second_turn_reads_no_vectors_at_all(db, adventure, settings, bank, sql_log): + """The point of the cache: back-to-back turns on one adventure pay for the + vectors once.""" + retrieve(adventure, settings, StubEmbedder()) + sql_log.clear() + retrieve(adventure, settings, StubEmbedder()) + assert not any("embedding_blob" in s for s in memory_selects(sql_log)) + + +def test_only_the_new_memory_is_fetched_after_one_is_added(db, adventure, settings, bank, sql_log): + """A growing bank must not re-read the vectors it already holds.""" + retrieve(adventure, settings, StubEmbedder()) + added = add_memory(db, adventure, "about w", (0.5, 0.5, 0.0)) + + sql_log.clear() + retrieve(adventure, settings, StubEmbedder()) + vector_reads = [s for s in memory_selects(sql_log) if "embedding_blob" in s] + assert len(vector_reads) == 1 + # One placeholder means one id: the new memory, and nothing else. + assert vector_reads[0].count("?") == 1 + assert added.id in {m["id"] for m in + retrieve(adventure, settings, StubEmbedder((0.5, 0.5, 0.0)))["used"]} + + +def test_only_top_k_texts_are_fetched(db, adventure, settings, bank, sql_log): + settings.memory_top_k = 1 + db.commit() + retrieve(adventure, settings, StubEmbedder()) + text_reads = [s for s in memory_selects(sql_log) if "memories.text" in s] + assert len(text_reads) == 1 + assert text_reads[0].count("?") == 1 + + +# ---------------------------------------------------------------- staleness + +def test_a_rewritten_vector_is_not_served_from_cache(db, adventure, settings, bank): + """The cache's one genuine hazard: a memory keeps its id while its + vector changes, so an id-set check alone would continue serving the + old vector. Editing a memory's text and re-embedding it does exactly + that. + """ + settings.memory_top_k = 1 + db.commit() + first = retrieve(adventure, settings, StubEmbedder((0.0, 0.0, 1.0))) + assert [m["id"] for m in first["used"]] == [bank["z"].id] + + # z is re-embedded onto the x axis, all within one gap between turns. + memorybank.set_vector(bank["z"], [1.0, 0.0, 0.0]) + db.commit() + + again = retrieve(adventure, settings, StubEmbedder((0.0, 0.0, 1.0))) + assert [m["id"] for m in again["used"]] != [bank["z"].id] + + +def test_a_deleted_memory_leaves_the_cache(db, adventure, settings, bank): + retrieve(adventure, settings, StubEmbedder()) + db.delete(bank["x"]) + db.commit() + + used = retrieve(adventure, settings, StubEmbedder((1.0, 0.0, 0.0)))["used"] + assert bank["x"].id not in {m["id"] for m in used} + assert memorybank._vector_cache[adventure.id].keys() == {bank["y"].id, bank["z"].id} + + +def test_the_cache_is_bounded(db, adventure, settings, bank): + """It holds vectors indefinitely, so without a bound a long-running process + accumulates every adventure ever played.""" + retrieve(adventure, settings, StubEmbedder()) + for fake_id in range(1000, 1000 + memorybank.VECTOR_CACHE_ADVENTURES + 2): + memorybank._vectors_for(db, fake_id, []) + assert len(memorybank._vector_cache) == memorybank.VECTOR_CACHE_ADVENTURES + assert adventure.id not in memorybank._vector_cache # evicted, least recent + + +def test_forget_cached_vectors_drops_an_adventure(db, adventure, settings, bank): + retrieve(adventure, settings, StubEmbedder()) + assert adventure.id in memorybank._vector_cache + memorybank.forget_cached_vectors(adventure.id) + assert adventure.id not in memorybank._vector_cache + + +# ----------------------------------------------------------------- eviction + +def test_eviction_marks_the_least_recently_used(db, adventure, settings): + settings.memory_bank_capacity = 1 + db.commit() + now = models.utcnow() + recent = add_memory(db, adventure, "recent", (1.0, 0.0, 0.0), use_count=1) + stale = add_memory(db, adventure, "stale", (0.0, 1.0, 0.0), use_count=1) + recent.last_used_at = now + stale.last_used_at = now - timedelta(days=30) + db.commit() + + memorybank._evict_over_capacity(adventure, settings, db) + db.expire_all() + + assert db.get(models.Memory, stale.id).forgotten is True + assert db.get(models.Memory, recent.id).forgotten is False + + +def test_eviction_breaks_ties_on_use_count(db, adventure, settings): + """Two memories last used at the same moment: the one the story has + used less is the one that goes. This must be only a tiebreak. Ranking + on the count first is what used to freeze the bank (see below).""" + settings.memory_bank_capacity = 1 + db.commit() + now = models.utcnow() + keep = add_memory(db, adventure, "used often", (1.0, 0.0, 0.0), use_count=5) + drop = add_memory(db, adventure, "used once", (0.0, 1.0, 0.0), use_count=1) + keep.last_used_at = drop.last_used_at = now + db.commit() + + memorybank._evict_over_capacity(adventure, settings, db) + db.expire_all() + + assert db.get(models.Memory, drop.id).forgotten is True + assert db.get(models.Memory, keep.id).forgotten is False + + +def test_a_newborn_is_not_evicted_by_the_bank_it_joins(db, adventure, settings): + """The bank used to stop accepting new memories. Eviction ranked on + use_count first, and a memory written this turn has never been used. + Once every existing memory had been retrieved even once, the newborn + became the lowest-ranked row in the bank. Eviction then removed it in + the same post-turn run that wrote it, before retrieval ever saw it. + That state never recovers: counts only go up, so no memory written + after it could get in either.""" + settings.memory_bank_capacity = 3 + db.commit() + now = models.utcnow() + established = [ + add_memory(db, adventure, f"used once {i}", (1.0, 0.0, 0.0), + use_count=1, last_used_at=now - timedelta(minutes=3 - i)) + for i in range(3) + ] + newborn = add_memory(db, adventure, "written this turn", (0.0, 1.0, 0.0)) + + memorybank._evict_over_capacity(adventure, settings, db) + db.expire_all() + + assert db.get(models.Memory, newborn.id).forgotten is False + # the least recently used goes instead + assert db.get(models.Memory, established[0].id).forgotten is True + assert db.get(models.Memory, established[2].id).forgotten is False + + +def test_a_full_bank_still_turns_over(db, adventure, settings): + """The same failure seen over several turns: a bank at capacity must + keep accepting new memories, or the adventure stops remembering + anything past the point where it filled up.""" + settings.memory_bank_capacity = 3 + now = models.utcnow() + db.commit() + for i in range(3): + add_memory(db, adventure, f"opening {i}", (1.0, 0.0, 0.0), + use_count=1, last_used_at=now - timedelta(minutes=10 - i)) + + later = [] + for turn in range(4): + later.append(add_memory(db, adventure, f"turn {turn}", (0.0, 1.0, 0.0))) + memorybank._evict_over_capacity(adventure, settings, db) + db.expire_all() + + active = {m.text for m in db.query(models.Memory).filter( + models.Memory.adventure_id == adventure.id, + models.Memory.forgotten.is_(False), + )} + assert active == {"turn 1", "turn 2", "turn 3"} + + +def test_eviction_never_touches_a_pin(db, adventure, settings): + """Capacity yields to pins: if everything active is pinned there is nothing + to evict, and the bank is allowed to sit over capacity.""" + settings.memory_bank_capacity = 1 + db.commit() + pins = [add_memory(db, adventure, f"pin {i}", (1.0, 0.0, 0.0), pinned=True) + for i in range(3)] + + memorybank._evict_over_capacity(adventure, settings, db) + db.expire_all() + + assert all(db.get(models.Memory, p.id).forgotten is False for p in pins) + + +def test_eviction_reads_no_vectors(db, adventure, settings, bank, sql_log): + """It ran every turn and pulled the whole bank to count it.""" + settings.memory_bank_capacity = 1 + db.commit() + sql_log.clear() + memorybank._evict_over_capacity(adventure, settings, db) + assert not any("embedding_blob" in s for s in memory_selects(sql_log)) + + +def test_evicted_memories_drop_out_of_the_cache(db, adventure, settings, bank): + retrieve(adventure, settings, StubEmbedder()) + settings.memory_bank_capacity = 1 + db.commit() + memorybank._evict_over_capacity(adventure, settings, db) + + used = retrieve(adventure, settings, StubEmbedder())["used"] + assert len(used) == 1 + assert len(memorybank._vector_cache[adventure.id]) == 1 + + +# ------------------------------------------------------------- embed queue + +def test_embed_pending_picks_only_unembedded_memories(db, adventure, settings): + done = add_memory(db, adventure, "already done", (1.0, 0.0, 0.0)) + todo = add_memory(db, adventure, "needs a vector", None) + add_memory(db, adventure, "evicted, skip", None, forgotten=True) + + embedder = StubEmbedder((0.0, 1.0, 0.0)) + memorybank.embedding_provider = lambda s: embedder + asyncio.run(memorybank._embed_pending(adventure, settings, db)) + db.expire_all() + + assert db.get(models.Memory, todo.id).embedded is True + assert embedder.calls == 1 + # The one already embedded keeps the vector it had. + assert db.get(models.Memory, done.id).embedding_blob == memorybank.vectors.pack( + [1.0, 0.0, 0.0] + ) + + +def test_embed_pending_reads_no_vectors(db, adventure, settings, bank, sql_log): + add_memory(db, adventure, "needs a vector", None) + embedder = StubEmbedder() + memorybank.embedding_provider = lambda s: embedder + + sql_log.clear() + asyncio.run(memorybank._embed_pending(adventure, settings, db)) + assert not any("embedding_blob" in s for s in memory_selects(sql_log)) diff --git a/backend/tests/test_memory_rewrite.py b/backend/tests/test_memory_rewrite.py new file mode 100644 index 0000000..efc0e5b --- /dev/null +++ b/backend/tests/test_memory_rewrite.py @@ -0,0 +1,419 @@ +"""Phase 18b: rewriting memories an older prompt wrote. + +The prompt change only reaches memories written after it. A bank filled before +it keeps its unnamed, second-person entries and injects them into every turn +from then on, so there has to be a way to run the new prompt back over them. + +Two halves, and the first is the one that can be wrong quietly: + +* `memorybank.source_block` reads a memory's block back out of the story. + Nothing in the app has ever had to do that. It has to read on the branch the + memory was written on rather than the one the adventure is playing now, skip + the sibling attempts at a retried turn, and cope with a memory whose actions + have since been deleted. +* `tools/rewrite_memories` replaces the text and clears the vector, leaves a + hand-written memory alone, and writes nothing at all without `--write`. + + python -m pytest tests/test_memory_rewrite.py -v +""" +import argparse +import asyncio + +import pytest + +from app import memorybank, models, tree +from app.context import lineage +from app.database import Base, SessionLocal, engine +from tools import rewrite_memories + + +class StubSummarizer: + """Returns a numbered memory, and records what it was asked. + + The default text is what the new prompt asks for and the `OLD` text is what + the old one produced, so an assertion says which prompt wrote a memory + rather than counting calls. + """ + + OLD = "You entered the crypt {n}." + + def __init__(self, template="Kaelen entered the crypt {n}.", error=None): + self.calls: list[tuple[str, str]] = [] + self.template = template + self.error = error + + async def complete(self, system, user, **kwargs): + self.calls.append((system, user)) + if self.error is not None: + raise self.error + return self.template.format(n=len(self.calls)) + + +class StubEmbedder: + def __init__(self): + self.texts: list[str] = [] + + async def embed(self, texts): + self.texts.extend(texts) + return [[1.0, 0.0, 0.0] for _ in texts] + + +@pytest.fixture() +def db(): + Base.metadata.create_all(bind=engine) + session = SessionLocal() + try: + yield session + finally: + session.close() + Base.metadata.drop_all(bind=engine) + + +def make_adventure(db, *, actions=13, key="sk-test-key", email="rewrite@example.com"): + """An adventure whose bank was written by the old prompt. + + Thirteen actions, not twelve: two blocks of six, plus the one action that + settles the second of them. A block is not summarized while it ends on the + newest action. See `memorybank.SETTLE_SLACK`. + """ + user = models.User(is_guest=False, email=email) + db.add(user) + db.flush() + # A key with no `enc:` prefix is stored plaintext and read back as-is; see + # `security.decrypt_secret`. That keeps the fixture off the crypto path. + db.add(models.Settings(user_id=user.id, api_key=key, model="test-model", + embedding_model="text-embedding-3-small")) + adventure = models.Adventure( + user_id=user.id, title="Camp", script_state={}, auto_summarize=True, + memory="The player and Gwen are raiding a bandit camp.", + persona_name="Kaelen", persona_pronouns="he/him", + persona_desc="A half-elf ranger.", + ) + db.add(adventure) + db.flush() + db.add(models.StoryCard(adventure_id=adventure.id, name="Gwen", + keys="Gwen, her", type="character", + entry="A loyal ranger and the player's ally.")) + for i in range(actions): + db.add(models.Action(adventure_id=adventure.id, + type="ai" if i % 2 else "do", + text=f"You walk on. Action {i}.")) + db.commit() + db.refresh(adventure) + return adventure + + +def fill_bank(db, adventure, *, count=2): + """Runs the real pass, so the memories carry the coordinates the real ones + carry rather than coordinates this file made up.""" + stub = StubSummarizer(template=StubSummarizer.OLD) + original = memorybank.summary_provider + memorybank.summary_provider = lambda s: stub + try: + settings = (db.query(models.Settings) + .filter(models.Settings.user_id == adventure.user_id).one()) + asyncio.run(memorybank._create_due_memories(adventure, settings, db)) + finally: + memorybank.summary_provider = original + memories = (db.query(models.Memory) + .filter(models.Memory.adventure_id == adventure.id) + .order_by(models.Memory.id).all()) + assert len(memories) == count, f"expected {count} memories, got {len(memories)}" + for memory in memories: + memorybank.set_vector(memory, [1.0, 0.0, 0.0]) + db.commit() + return memories + + +def options(**overrides): + args = dict(write=False, adventure=None, email=None, limit=None, + include_forgotten=False, embed=False, endpoint=None, + model=None, api_key=None) + args.update(overrides) + return argparse.Namespace(**args) + + +def run_tool(args) -> int: + return asyncio.run(rewrite_memories.main(args)) + + +# ------------------------------------------------------ reading the block back + +def test_the_block_is_the_actions_the_memory_covers(db): + adventure = make_adventure(db) + first, second = fill_bank(db, adventure) + block = memorybank.source_block(db, first) + assert [a.depth for a in block] == list( + range(first.source_start, first.source_end + 1)) + assert len(block) == memorybank.MEMORY_INTERVAL + assert [a.text for a in block] == [f"You walk on. Action {i}." for i in range(6)] + assert [a.text for a in memorybank.source_block(db, second)] == [ + f"You walk on. Action {i}." for i in range(6, 12)] + + +def test_a_hand_written_memory_has_no_block(db): + """It summarizes no actions, so there is nothing to rewrite it from — and + the player may have typed it.""" + adventure = make_adventure(db) + memory = models.Memory(adventure_id=adventure.id, text="Kaelen owes a debt.") + tree.place_memory(db, adventure, memory) + db.add(memory) + db.commit() + assert memorybank.source_block(db, memory) == [] + + +def test_a_deleted_action_shortens_the_block(db): + """A memory whose block is now partial still describes what remains.""" + adventure = make_adventure(db) + first, _ = fill_bank(db, adventure) + db.delete(memorybank.source_block(db, first)[2]) + db.commit() + block = memorybank.source_block(db, first) + assert len(block) == memorybank.MEMORY_INTERVAL - 1 + assert "Action 2." not in [a.text for a in block] + + +def test_a_retried_turn_contributes_only_its_live_attempt(db): + """Sibling attempts share a coordinate. The block holds the one the story + used, not both.""" + adventure = make_adventure(db) + first, _ = fill_bank(db, adventure) + kept = memorybank.source_block(db, first)[3] + db.add(models.Action(adventure_id=adventure.id, branch_id=kept.branch_id, + depth=kept.depth, live=False, type=kept.type, + text="A discarded attempt.")) + db.commit() + texts = [a.text for a in memorybank.source_block(db, first)] + assert "A discarded attempt." not in texts + assert len(texts) == memorybank.MEMORY_INTERVAL + + +def test_the_block_is_read_on_the_branch_the_memory_was_written_on(db): + """The adventure has moved to a fork since. A memory on the trunk still + reads back, and one left on the abandoned continuation reads that + continuation rather than the branch now being played.""" + adventure = make_adventure(db) + trunk_memory, _ = fill_bank(db, adventure) + trunk = tree.head_branch(db, adventure) + + fork = models.Branch(adventure_id=adventure.id, parent_branch_id=trunk.id, + fork_depth=5, lineage=[]) + db.add(fork) + db.flush() + fork.lineage = [[fork.id, None], [trunk.id, 5]] + for depth in (6, 7, 8, 9, 10, 11): + db.add(models.Action(adventure_id=adventure.id, branch_id=fork.id, + depth=depth, type="do", text=f"Fork action {depth}.")) + db.flush() + tip = (db.query(models.Action) + .filter(models.Action.branch_id == fork.id) + .order_by(models.Action.depth.desc()).first()) + fork_memory = models.Memory(adventure_id=adventure.id, text="On the fork.", + source_start=6, source_end=11) + tree.attach_memory(fork_memory, tip) + db.add(fork_memory) + adventure.head_branch_id = fork.id + adventure.head_depth = 11 + db.commit() + + # The trunk memory predates the fork and is inherited, so it reads the same + # actions from either branch. + assert [a.text for a in memorybank.source_block(db, trunk_memory)] == [ + f"You walk on. Action {i}." for i in range(6)] + # The abandoned continuation's memory covers depths 6..11 on the trunk. The + # fork covers the same depths with different actions, and the head is on the + # fork, so a read that used the adventure's path would return the fork's. + abandoned = (db.query(models.Memory) + .filter(models.Memory.branch_id == trunk.id, + models.Memory.source_start == 6).one()) + assert [a.text for a in memorybank.source_block(db, abandoned)] == [ + f"You walk on. Action {i}." for i in range(6, 12)] + assert [a.text for a in memorybank.source_block(db, fork_memory)] == [ + f"Fork action {i}." for i in range(6, 12)] + + +def test_the_rewrite_prompt_is_the_one_the_app_sends(db): + """`summarize_block` is the app's own prompt assembly. A rewrite that built + its own would be written by a prompt that never shipped.""" + adventure = make_adventure(db) + first, _ = fill_bank(db, adventure) + stub = StubSummarizer() + asyncio.run(memorybank.summarize_block( + adventure, stub, memorybank.source_block(db, first))) + system, user = stub.calls[0] + assert "third person" in system + assert user.index("Cast:") < user.index("Story excerpt:") + assert "Kaelen (he/him) — the protagonist" in user + + +# --------------------------------------------------------------------- the tool + +def test_the_database_line_carries_no_password(): + """The report names the database it is about to rewrite. That line ends up + in a console, a screenshot or a pasted bug report.""" + shown = rewrite_memories.safe_dsn( + "postgresql://parth:hunter2@ep-cool-frost.us-east-1.aws.neon.tech/aidnd" + "?sslmode=require") + assert "hunter2" not in shown + assert "sslmode" not in shown # a password can be passed there too + assert shown == ("postgresql://parth@ep-cool-frost.us-east-1.aws.neon.tech" + "/aidnd") + + +def test_an_unparseable_database_url_shows_nothing_at_all(): + assert rewrite_memories.safe_dsn("not-a-url") == "(configured)" + + + +def test_without_write_nothing_changes(db, monkeypatch): + adventure = make_adventure(db) + first, _ = fill_bank(db, adventure) + stub = StubSummarizer() + monkeypatch.setattr(memorybank, "summary_provider", lambda s: stub) + assert run_tool(options()) == 0 + db.expire_all() + assert stub.calls == [], "a dry run must not call the model" + assert first.text == "You entered the crypt 1." + assert first.embedded is True + + +def test_write_replaces_the_text_and_clears_the_vector(db, monkeypatch): + adventure = make_adventure(db) + first, second = fill_bank(db, adventure) + stub = StubSummarizer() + monkeypatch.setattr(memorybank, "summary_provider", lambda s: stub) + assert run_tool(options(write=True)) == 0 + db.expire_all() + assert [first.text, second.text] == ["Kaelen entered the crypt 1.", "Kaelen entered the crypt 2."] + # The stored vector describes wording that no longer exists, so the memory + # leaves the ranked bank until something embeds the new text. + assert (first.embedded, first.embedding_blob) == (False, None) + + +def test_embed_puts_the_rewritten_memories_back_in_the_bank(db, monkeypatch): + adventure = make_adventure(db) + first, _ = fill_bank(db, adventure) + embedder = StubEmbedder() + summarizer = StubSummarizer() + monkeypatch.setattr(memorybank, "summary_provider", lambda s: summarizer) + monkeypatch.setattr(memorybank, "embedding_provider", lambda s: embedder) + assert run_tool(options(write=True, embed=True)) == 0 + db.expire_all() + assert embedder.texts == ["Kaelen entered the crypt 1.", "Kaelen entered the crypt 2."] + assert first.embedded is True + + +def test_a_hand_written_memory_is_left_alone(db, monkeypatch): + adventure = make_adventure(db) + fill_bank(db, adventure) + typed = models.Memory(adventure_id=adventure.id, text="Kaelen owes a debt.") + tree.place_memory(db, adventure, typed) + db.add(typed) + db.commit() + monkeypatch.setattr(memorybank, "summary_provider", lambda s: StubSummarizer()) + assert run_tool(options(write=True)) == 0 + db.expire_all() + assert typed.text == "Kaelen owes a debt." + + +def test_an_owner_with_no_api_key_is_skipped(db, monkeypatch): + """Summarization spends the user's own key and never the shared demo key.""" + adventure = make_adventure(db, key="") + first, _ = fill_bank(db, adventure) + stub = StubSummarizer() + monkeypatch.setattr(memorybank, "summary_provider", lambda s: stub) + assert run_tool(options(write=True)) == 0 + db.expire_all() + assert stub.calls == [] + assert first.text == "You entered the crypt 1." + + +def test_an_api_key_on_the_command_line_covers_that_owner(db, monkeypatch): + """The way to summarize for an owner who has no key of their own, and the + way to point a run at the local Claude shim instead of a paid endpoint.""" + adventure = make_adventure(db, key="") + first, _ = fill_bank(db, adventure) + built: list[tuple] = [] + + def build(*args, **kwargs): + built.append(args) + return StubSummarizer() + + monkeypatch.setattr("app.providers.OpenAICompatibleProvider", build) + assert run_tool(options(write=True, api_key="sk-cli", model="sonnet", + endpoint="http://127.0.0.1:8787/v1")) == 0 + db.expire_all() + assert first.text == "Kaelen entered the crypt 1." + assert built[0][:3] == ("http://127.0.0.1:8787/v1", "sk-cli", "sonnet") + + +def test_limit_stops_early(db, monkeypatch): + adventure = make_adventure(db) + first, second = fill_bank(db, adventure) + monkeypatch.setattr(memorybank, "summary_provider", lambda s: StubSummarizer()) + assert run_tool(options(write=True, limit=1)) == 0 + db.expire_all() + assert first.text == "Kaelen entered the crypt 1." + assert second.text == "You entered the crypt 2." + + +def test_a_provider_error_leaves_the_old_text_and_reports_it(db, monkeypatch): + from app.providers import ProviderError + adventure = make_adventure(db) + first, _ = fill_bank(db, adventure) + monkeypatch.setattr(memorybank, "summary_provider", + lambda s: StubSummarizer(error=ProviderError("nope"))) + assert run_tool(options(write=True)) == 1 + db.expire_all() + assert first.text == "You entered the crypt 1." + assert first.embedded is True + + +def test_only_the_named_adventure_is_touched(db, monkeypatch): + one = make_adventure(db) + two = make_adventure(db, email="other@example.com") + kept, _ = fill_bank(db, one) + changed, _ = fill_bank(db, two) + monkeypatch.setattr(memorybank, "summary_provider", lambda s: StubSummarizer()) + assert run_tool(options(write=True, adventure=[two.id])) == 0 + db.expire_all() + assert kept.text == "You entered the crypt 1." + assert changed.text == "Kaelen entered the crypt 1." + + +def test_only_the_named_account_is_touched(db, monkeypatch): + """The hosted database holds other people's stories, and each adventure is + summarized with its owner's key.""" + mine = make_adventure(db, email="mine@example.com") + theirs = make_adventure(db, email="theirs@example.com") + kept, _ = fill_bank(db, theirs) + changed, _ = fill_bank(db, mine) + monkeypatch.setattr(memorybank, "summary_provider", lambda s: StubSummarizer()) + assert run_tool(options(write=True, email=["MINE@example.com"])) == 0 + db.expire_all() + assert kept.text == "You entered the crypt 1." + assert changed.text == "Kaelen entered the crypt 1." + + +def test_an_unknown_email_is_an_error(db): + make_adventure(db) + assert run_tool(options(email=["nobody@example.com"])) == 2 + + +def test_an_unknown_adventure_id_is_an_error(db): + make_adventure(db) + assert run_tool(options(adventure=[9999])) == 2 + + +def test_an_evicted_memory_is_left_out_unless_asked_for(db, monkeypatch): + adventure = make_adventure(db) + first, second = fill_bank(db, adventure) + second.forgotten = True + db.commit() + monkeypatch.setattr(memorybank, "summary_provider", lambda s: StubSummarizer()) + assert run_tool(options(write=True)) == 0 + db.expire_all() + assert second.text == "You entered the crypt 2." + assert run_tool(options(write=True, include_forgotten=True)) == 0 + db.expire_all() + assert second.text.startswith("Kaelen entered") diff --git a/backend/tests/test_memory_settling.py b/backend/tests/test_memory_settling.py new file mode 100644 index 0000000..88ff77a --- /dev/null +++ b/backend/tests/test_memory_settling.py @@ -0,0 +1,350 @@ +"""Memories must never describe narration that is no longer in the story, and +must never skip a stretch of it. + +For six phases, the answer was a holdback. Summarization stopped one action +short of the newest, because only the last action was retryable, and a +retry rewrote `Action.text` under a mark that had already moved past it. +SP4 ended that: a retry writes a sibling node, and the coordinate's derived +work is withdrawn as it happens, using the same repair that undo and delete +already made. Correctness has rested on that withdrawal ever since, and the +second half of this file is where it is asserted. + +The first half is about what the withdrawal costs. Redoing a block is +correct and it is also paid for twice, so `memorybank.SETTLE_SLACK` keeps a +block from ending on the newest action, which is the only action retry and +take-switching can reach. A block forms as soon as one action has settled +past it, and changing what a coordinate says still takes back what was +derived from it. + +Phase 14 SP3 changed what the mark is. It used to be a count of covered +story actions, and the second half of this file is the cost of that. +Deleting an action from in front of a position slid a never-summarized +action into the covered range, so every delete had to slide the cursors +too. The mark is a node now, `(branch_id, depth)`, and a node does not move +when something in front of it is deleted. Those tests now assert that +nothing happens, where they used to assert that the right correction +happened. + + python -m pytest tests/test_memory_settling.py -v +""" +import asyncio + + +import pytest + +from app import memorybank, models, tree +from app.context import cursors, history +from app.database import Base, SessionLocal, engine + + +class StubSummarizer: + """Records every excerpt handed to the summarizer.""" + + def __init__(self): + self.excerpts: list[str] = [] + + async def complete(self, system, user, **kwargs): + self.excerpts.append(user) + return f"Memory {len(self.excerpts)}." + + +@pytest.fixture() +def db(): + Base.metadata.create_all(bind=engine) + session = SessionLocal() + try: + yield session + finally: + session.close() + Base.metadata.drop_all(bind=engine) + + +def make_adventure(db, action_count: int) -> models.Adventure: + """An adventure whose actions alternate player/AI, newest last.""" + user = models.User(is_guest=False, email="memory@example.com") + db.add(user) + db.flush() + settings = models.Settings(user_id=user.id, api_key="enc:dummy", model="test-model") + db.add(settings) + adventure = models.Adventure( + user_id=user.id, title="Cave", script_state={}, auto_summarize=True + ) + db.add(adventure) + db.flush() + for i in range(action_count): + db.add(models.Action( + adventure_id=adventure.id, + type="ai" if i % 2 else "do", text=f"Action {i}.", + )) + db.commit() + db.refresh(adventure) + return adventure + + +def cover(db, adventure, position: int) -> None: + """Mark the first `position` story actions as already summarized. + + Written as a position and translated to the node it names, because that is + what every adventure in the database looked like before SP3 and what a v1 + bundle still carries. + """ + cursors.anchor_at_position(adventure, cursors.MEMORY, position) + cursors.anchor_at_position(adventure, cursors.SUMMARY, position) + db.commit() + + +def covered_depth(db, adventure) -> int: + """The memory mark, as a depth on the story being played.""" + return cursors.MEMORY.depth(db, adventure) + + +def run_memories(db, adventure, stub, monkeypatch): + monkeypatch.setattr(memorybank, "summary_provider", lambda s: stub) + settings = db.query(models.Settings).first() + asyncio.run(memorybank._create_due_memories(adventure, settings, db)) + + +# ------------------------------------------- when a block forms (SETTLE_SLACK) + +def test_a_block_that_ends_on_the_newest_action_waits(db, monkeypatch): + """Covered to action 5 with 12 actions: block 6-11 is full, but it ends on + the newest action, so it is not written yet. + + Writing it would be correct. A retry of node 11 withdraws it on its way + past, which is the same repair as + `test_deleting_a_summarized_node_withdraws_its_memory`. It would also mean + the block was summarized once, thrown away, and summarized again, for a + memory that says nothing the history window does not still carry. + """ + adventure = make_adventure(db, 12) + cover(db, adventure, 6) + + stub = StubSummarizer() + run_memories(db, adventure, stub, monkeypatch) + + assert stub.excerpts == [] + assert covered_depth(db, adventure) == 5 # the mark stays where it was + + # One more action settles the block. It is written from the same six: the + # slack asks for story past the block, it does not grow the block. + db.add(models.Action(adventure_id=adventure.id, type="ai", text="Action 12.")) + db.commit() + db.refresh(adventure) + run_memories(db, adventure, stub, monkeypatch) + + assert len(stub.excerpts) == 1 + assert "Action 11." in stub.excerpts[0] + assert "Action 12." not in stub.excerpts[0] + memory = db.query(models.Memory).one() + assert (memory.source_start, memory.source_end) == (6, 11) + # The mark and the memory name the same node. That is what keeps them + # from drifting apart, however gappy the underlying depths are. + assert (memory.branch_id, memory.depth) == cursors.MEMORY.stored(adventure) + assert covered_depth(db, adventure) == 11 + + +def test_a_retry_at_the_tip_has_no_memory_to_withdraw(db, monkeypatch): + """The slack, stated as the case it removes. + + Retry and take-switching both refuse anything but the newest action + (`takes.retry_action`, `takes.switch_take`), so holding the block end one + action back puts every memory out of their reach. Nothing here is a claim + about the withdrawal path, which undo and delete still reach at any node. + """ + adventure = make_adventure(db, 13) + run_memories(db, adventure, StubSummarizer(), monkeypatch) + + assert db.query(models.Memory).count() == 2 # blocks 0-5 and 6-11 + assert covered_depth(db, adventure) == 11 + + tip = history.newest(adventure) + assert tip.depth == 12 # the deepest memory is a node behind it + assert memorybank.forget_node(db, adventure, tip) == 0 + assert covered_depth(db, adventure) == 11 # so a retry rewinds nothing + + +def test_the_first_memory_lands_at_memory_start(db, monkeypatch): + adventure = make_adventure(db, memorybank.MEMORY_START - 1) + stub = StubSummarizer() + run_memories(db, adventure, stub, monkeypatch) + assert stub.excerpts == [] # too short to have started at all + + db.add(models.Action( + adventure_id=adventure.id, + type="do", text="Later.", + )) + db.commit() + db.refresh(adventure) + run_memories(db, adventure, stub, monkeypatch) + # MEMORY_START is 12 actions = two full blocks. The first is settled, + # because the second sits past it. The second ends on the newest action + # and waits. + assert len(stub.excerpts) == 1 + assert "Later." not in stub.excerpts[-1] + assert covered_depth(db, adventure) == memorybank.MEMORY_INTERVAL - 1 + + # One action past the second block settles it too, and both are caught up + # in one run (MAX_MEMORIES_PER_RUN allows 5). + db.add(models.Action(adventure_id=adventure.id, type="ai", text="Even later.")) + db.commit() + db.refresh(adventure) + run_memories(db, adventure, stub, monkeypatch) + assert len(stub.excerpts) == 2 + assert "Later." in stub.excerpts[-1] + assert covered_depth(db, adventure) == memorybank.MEMORY_START - 1 + + +def test_legacy_caught_up_adventure_is_not_rewound(db, monkeypatch): + """An adventure summarized under the old rule carries a cursor equal to + its action count, one past the end of the story. That used to require a + clamp on every post-turn pass, and clamping it to the settled count + re-covered an action. + + A mark that names a node has no such edge. The newest action is the + node, and "everything after it" is empty until the story grows. + """ + adventure = make_adventure(db, 12) + db.add(models.Memory(adventure_id=adventure.id, text="A", source_start=0, source_end=5)) + db.add(models.Memory(adventure_id=adventure.id, text="B", source_start=6, source_end=11)) + cover(db, adventure, 12) + + assert covered_depth(db, adventure) == 11 # the newest action, not one past it + assert history.count_after(adventure, covered_depth(db, adventure)) == 0 + + # Grow the story and let the next block form. + for i in range(12, 25): + db.add(models.Action(adventure_id=adventure.id, type="do", text=f"Action {i}.")) + db.commit() + db.refresh(adventure) + run_memories(db, adventure, StubSummarizer(), monkeypatch) + + covered: dict[int, int] = {} + for m in db.query(models.Memory).all(): + for i in range(m.source_start, m.source_end + 1): + covered[i] = covered.get(i, 0) + 1 + assert [i for i, c in covered.items() if c > 1] == [] # no action summarized twice + assert [i for i in range(max(covered) + 1) if i not in covered] == [] # no gaps + + +def test_no_memories_before_memory_start(db, monkeypatch): + adventure = make_adventure(db, 8) + stub = StubSummarizer() + run_memories(db, adventure, stub, monkeypatch) + assert stub.excerpts == [] + + +# ------------------------------------------- deleting already-summarized ground + +def orphans(db, adventure) -> list[int]: + """Depths the mark calls summarized that no memory describes. + + The failure this whole section is about, stated once: an action behind the + mark with nothing covering it is never summarized again, and nothing ever + reports it. + """ + covered: set[int] = set() + for m in db.query(models.Memory).filter_by(adventure_id=adventure.id): + covered |= set(range(m.source_start, m.source_end + 1)) + mark = cursors.MEMORY.depth(db, adventure) + return [ + a.depth for a in memorybank.story_actions(adventure) + if a.depth <= mark and a.depth not in covered + ] + + +def summarized_adventure(db): + """13 actions with two memories covering depths 0-11, the mark on node 11.""" + adventure = make_adventure(db, 13) + for text, start, end in (("A", 0, 5), ("B", 6, 11)): + node = db.query(models.Action).filter_by( + adventure_id=adventure.id, depth=end + ).one() + memory = models.Memory( + adventure_id=adventure.id, text=text, source_start=start, source_end=end + ) + tree.attach_memory(memory, node) + db.add(memory) + cover(db, adventure, 12) + db.refresh(adventure) + return adventure + + +def test_deleting_a_middle_action_leaves_the_mark_where_it_was(db): + """The bug that motivated the old machinery, and the reason it is gone. + + A position cursor counted actions from the start, so deleting an earlier + one slid a never-summarized action into the covered range and every delete + had to correct for it. A depth is not a count: node 11 is still node 11 + with node 5 gone. + """ + adventure = summarized_adventure(db) + # Node 4 is inside memory A's block but is not the node it hangs off, + # so nothing is withdrawn. The old code read it the same way: only a + # memory whose end had fallen off the story was pruned. + victim = db.query(models.Action).filter_by(adventure_id=adventure.id, depth=4).one() + + assert memorybank.forget_node(db, adventure, victim) == 0 + db.delete(victim) + db.commit() + db.refresh(adventure) + + assert covered_depth(db, adventure) == 11 + assert [m.text for m in db.query(models.Memory).all()] == ["A", "B"] + assert orphans(db, adventure) == [] + + +def test_deleting_a_later_action_leaves_the_mark_alone(db): + adventure = summarized_adventure(db) + victim = db.query(models.Action).filter_by(adventure_id=adventure.id, depth=12).one() + + memorybank.forget_node(db, adventure, victim) + db.delete(victim) + db.commit() + db.refresh(adventure) + + assert covered_depth(db, adventure) == 11 + assert orphans(db, adventure) == [] + + +def test_deleting_a_summarized_node_withdraws_its_memory(db): + """Discarding the memory is not enough. The story it covered is still + behind the mark, so the mark has to move back to where that block began. + + Memory B ends on node 11, so deleting node 11 withdraws it. The old code + found this by scanning for a memory whose covered range had fallen off + the end of the story. Now the memory hangs off the node, so finding it + is a lookup. + """ + adventure = summarized_adventure(db) + victim = db.query(models.Action).filter_by(adventure_id=adventure.id, depth=11).one() + + assert memorybank.forget_node(db, adventure, victim) == 1 + db.delete(victim) + db.commit() + db.refresh(adventure) + + assert [m.text for m in db.query(models.Memory).all()] == ["A"] + assert covered_depth(db, adventure) == 5 # back to where the discarded memory began + assert cursors.SUMMARY.depth(db, adventure) == 5 # and the summary with it + assert orphans(db, adventure) == [] + + +def test_repeated_deletes_never_orphan_an_action(db): + """The scenario that motivated this: undo/delete-last, over and over. + + No clamp in the loop any more, and no bookkeeping call per delete beyond + withdrawing what the node produced. + """ + adventure = summarized_adventure(db) + for _ in range(6): + actions = memorybank.story_actions(adventure) + if not actions: + break + victim = max(actions, key=lambda a: a.depth) + memorybank.forget_node(db, adventure, victim) + db.delete(victim) + db.flush() + db.expire(adventure, ["actions"]) + db.commit() + db.refresh(adventure) + assert orphans(db, adventure) == [] diff --git a/backend/tests/test_netguard.py b/backend/tests/test_netguard.py new file mode 100644 index 0000000..4b4e962 --- /dev/null +++ b/backend/tests/test_netguard.py @@ -0,0 +1,60 @@ +"""Tests for the SSRF guard on the user-supplied BYOK endpoint_url. + + python -m pytest tests/test_netguard.py -v +""" +import pytest + +from app import auth, netguard + + +@pytest.fixture +def hosted(monkeypatch): + monkeypatch.setattr(auth, "MULTI_USER", True) + + +def _resolves_to(monkeypatch, ip: str): + """Pin `getaddrinfo` so the test exercises the address decision, not real DNS.""" + monkeypatch.setattr( + netguard.socket, "getaddrinfo", + lambda *a, **k: [(2, 1, 6, "", (ip, 443))], + ) + + +@pytest.mark.parametrize("ip", [ + "127.0.0.1", # loopback + "169.254.169.254", # cloud metadata (link-local) + "10.0.0.5", # RFC1918 + "192.168.1.1", # RFC1918 + "172.16.0.9", # RFC1918 + "0.0.0.0", # unspecified + "100.64.0.1", # carrier-grade NAT + "::1", # IPv6 loopback + "fd00::1", # IPv6 unique-local +]) +def test_blocks_non_public_addresses(hosted, monkeypatch, ip): + _resolves_to(monkeypatch, ip) + assert netguard.endpoint_block_reason("https://evil.example.com/v1") is not None + + +def test_allows_public_address(hosted, monkeypatch): + _resolves_to(monkeypatch, "104.18.0.1") # a public IP + assert netguard.endpoint_block_reason("https://openrouter.ai/api/v1") is None + + +def test_rejects_non_http_scheme(hosted): + assert netguard.endpoint_block_reason("file:///etc/passwd") is not None + assert netguard.endpoint_block_reason("gopher://x/") is not None + + +def test_unresolvable_host_is_blocked(hosted, monkeypatch): + def boom(*a, **k): + raise netguard.socket.gaierror("no such host") + monkeypatch.setattr(netguard.socket, "getaddrinfo", boom) + assert netguard.endpoint_block_reason("https://nope.invalid/v1") is not None + + +def test_noop_in_local_mode(monkeypatch): + monkeypatch.setattr(auth, "MULTI_USER", False) + # Local installs must reach localhost (Ollama). The guard never blocks local mode. + assert netguard.endpoint_block_reason("http://localhost:11434/v1") is None + assert netguard.endpoint_block_reason("http://127.0.0.1:11434/v1") is None diff --git a/backend/tests/test_persona.py b/backend/tests/test_persona.py new file mode 100644 index 0000000..9eec972 --- /dev/null +++ b/backend/tests/test_persona.py @@ -0,0 +1,359 @@ +"""Phase 18: the adventure's persona — who the player is playing as. + +Before this existed the protagonist had no name anywhere in the prompt. The +player's stat block rendered as `You: hp 100/100` and the summarizer was handed +second-person prose with nothing to say who "you" was. + +The rules this file holds in place: + +* An empty `persona_name` behaves exactly as the app did before personas + existed, so an adventure that predates the migration is not a special case. +* The persona is in the **system** block. It is user-only, so it never changes + during a story, and putting it in the cached prefix is what makes it free. +* A persona works without an RPG layer. That is the case it was added for. +* Naming the block does not rename the path. The model still writes + `player.hp`, and a delta aimed at `persona.*` is refused. + + python -m pytest tests/test_persona.py -v +""" +import pytest +from fastapi import Depends +from fastapi.testclient import TestClient + +from app import auth, limits, models, worldstate +from app.context import builder +from app.database import Base, SessionLocal, engine, get_db +from app.main import app + +SCHEMA = { + "player": {"hp": {"min": 0, "max": 100, "initial": 100, "desc": "Health"}}, + "world": {"day": {"min": 1, "initial": 1, "desc": "Day"}}, +} + + +def _persona(**kwargs) -> models.Adventure: + """An unsaved adventure carrying only the fields `render_persona` reads.""" + return models.Adventure( + persona_name=kwargs.get("name", ""), + persona_pronouns=kwargs.get("pronouns", ""), + persona_desc=kwargs.get("desc", ""), + ) + + +# ------------------------------------------------------------- the rendering + +def test_no_persona_renders_nothing(): + assert builder.render_persona(_persona()) == "" + + +def test_whitespace_only_is_the_same_as_empty(): + """The columns default to '', but a player can clear a field to spaces.""" + assert builder.render_persona(_persona(name=" ", pronouns=" ", desc="\n")) == "" + + +def test_name_and_pronouns_and_description(): + text = builder.render_persona( + _persona(name="Kaelen", pronouns="he/him", desc="A half-elf ranger.") + ) + assert text.startswith("Player character:\n") + assert "You are Kaelen (he/him)." in text + assert "A half-elf ranger." in text + + +def test_name_alone_is_enough(): + text = builder.render_persona(_persona(name="Kaelen")) + assert "You are Kaelen." in text + assert "(" not in text, "no empty pronoun bracket" + + +def test_description_alone_is_enough(): + """A player who names no character still gets their description through.""" + text = builder.render_persona(_persona(desc="A nameless wanderer.")) + assert "A nameless wanderer." in text + assert "You are" not in text + + +# ---------------------------------------------------- the world-state labels + +def test_state_section_labels_the_block_with_the_name(): + state = worldstate.instantiate(SCHEMA) + block = worldstate.render_state_section(state, SCHEMA, {}, "Kaelen") + # The path travels with the name. Without it the model reads "Kaelen" in + # the prose and writes `kaelen.hp`, which `apply_delta` then refuses. + assert "Kaelen (player): hp 100/100" in block + + +def test_state_section_falls_back_to_you(): + state = worldstate.instantiate(SCHEMA) + block = worldstate.render_state_section(state, SCHEMA, {}, "") + assert "You: hp 100/100" in block + assert "(player)" not in block + + +def test_reference_ties_the_name_to_the_path(): + guide = worldstate.render_reference(SCHEMA, "Kaelen") + assert "Protagonist Kaelen" in guide + assert "player." in guide + + +def test_reference_without_a_persona_is_unchanged(): + assert "Protagonist" not in worldstate.render_reference(SCHEMA, "") + + +def test_reference_does_not_repeat_the_description(): + """The description has its own section. The guide is about paths.""" + guide = worldstate.render_reference(SCHEMA, "Kaelen") + assert "half-elf" not in guide + + +# ------------------------------------------------------ the assembled prompt + +@pytest.fixture() +def story(): + """An adventure with an RPG layer, a persona, and a little history.""" + Base.metadata.create_all(bind=engine) + db = SessionLocal() + user = models.User(is_guest=False, email="persona@example.com") + db.add(user) + db.flush() + settings = models.Settings(user_id=user.id, api_key="enc:dummy", model="m") + db.add(settings) + scenario = models.Scenario( + user_id=user.id, title="S", prompt="A road.", stat_schema=SCHEMA + ) + db.add(scenario) + db.flush() + adventure = models.Adventure( + user_id=user.id, title="A", scenario_id=scenario.id, script_state={}, + memory="The hero hunts bandits.", + ai_instructions="Write in second person.", + story_summary="The hero left the village.", + world_state=worldstate.instantiate(SCHEMA), + persona_name="Kaelen", + persona_pronouns="he/him", + persona_desc="A half-elf ranger, exiled from the northern holds.", + ) + db.add(adventure) + db.flush() + for i in range(6): + db.add(models.Action(adventure_id=adventure.id, + type="ai" if i % 2 else "do", + text=f"[{i}] The road bends onward past the treeline.")) + db.commit() + db.expire_all() + adventure = db.get(models.Adventure, adventure.id) + settings = db.get(models.Settings, settings.id) + try: + yield db, adventure, settings + finally: + db.close() + Base.metadata.drop_all(bind=engine) + + +def test_persona_is_in_the_system_block(story): + """It is user-only, so it belongs in the cached prefix rather than below + the history with the values that move.""" + db, adventure, settings = story + system_text, story_text, _ = builder.build_context(adventure, settings) + assert "You are Kaelen (he/him)." in system_text + assert "A half-elf ranger" in system_text + assert "Kaelen (he/him)" not in story_text + + +def test_persona_does_not_move_between_turns(story): + """The static block has to be byte-identical across turns. A persona that + changed with the world state would re-price the whole history.""" + db, adventure, settings = story + before, _, _ = builder.build_context(adventure, settings) + adventure.world_state = {**adventure.world_state, "player": {"hp": 40}} + db.commit() + after, story_text, _ = builder.build_context(adventure, settings) + assert before == after + assert "Kaelen (player): hp 40/100" in story_text + + +def test_persona_sits_above_the_plot_essentials(story): + db, adventure, settings = story + _, _, report = builder.build_context(adventure, settings) + labels = [s["label"] for s in report["sections"]] + assert labels.index("persona") < labels.index("plot_essentials") + + +def test_no_persona_section_when_the_fields_are_blank(story): + db, adventure, settings = story + adventure.persona_name = "" + adventure.persona_pronouns = "" + adventure.persona_desc = "" + db.commit() + system_text, story_text, report = builder.build_context(adventure, settings) + assert "persona" not in [s["label"] for s in report["sections"]] + assert "Player character" not in system_text + assert "You: hp 100/100" in story_text + + +def test_persona_is_charged_to_the_token_budget(story): + """It moved into `system_sections`, and everything there is counted in + `reserved`. If it were not, the history would overrun the budget.""" + db, adventure, settings = story + _, _, report = builder.build_context(adventure, settings) + persona = next(s for s in report["sections"] if s["label"] == "persona") + assert persona["tokens"] > 0 + assert report["tokens"]["total"] >= persona["tokens"] + + +# --------------------------------------------- a persona with no RPG layer + +@pytest.fixture() +def plain(): + """A blank adventure: no scenario, so no `stat_schema` at all.""" + Base.metadata.create_all(bind=engine) + db = SessionLocal() + user = models.User(is_guest=False, email="plain@example.com") + db.add(user) + db.flush() + settings = models.Settings(user_id=user.id, api_key="enc:dummy", model="m") + db.add(settings) + adventure = models.Adventure( + user_id=user.id, title="A", script_state={}, world_state={}, + persona_name="Wren", persona_pronouns="they/them", + persona_desc="A courier who reads other people's letters.", + ) + db.add(adventure) + db.flush() + db.add(models.Action(adventure_id=adventure.id, type="do", text="Walk east.")) + db.commit() + db.expire_all() + adventure = db.get(models.Adventure, adventure.id) + settings = db.get(models.Settings, settings.id) + try: + yield db, adventure, settings + finally: + db.close() + Base.metadata.drop_all(bind=engine) + + +def test_persona_works_without_an_rpg_layer(plain): + """The case the persona exists for. It must not be gated behind `has_ws`.""" + db, adventure, settings = plain + system_text, _, _ = builder.build_context(adventure, settings) + assert "You are Wren (they/them)." in system_text + assert "A courier who reads other people's letters." in system_text + + +# ------------------------------------------------- the AI cannot rewrite it + +@pytest.mark.parametrize("path", ["persona.name", "persona.desc", "persona"]) +def test_a_delta_aimed_at_the_persona_is_refused(path): + """Nothing was built for this: `_resolve` rejects unknown paths already. + The test holds the behavior in place, because the persona sitting in the + cached prefix depends on the model being unable to move it.""" + state = worldstate.instantiate(SCHEMA) + _, report = worldstate.apply_delta(state, SCHEMA, {path: "Bob"}, 1) + assert [r["path"] for r in report["rejected"]] == [path] + assert not report["applied"] + + +def test_the_player_path_still_works_with_a_persona_set(): + """Naming the block does not rename the path.""" + state = worldstate.instantiate(SCHEMA) + new_state, report = worldstate.apply_delta(state, SCHEMA, {"player.hp": -15}, 1) + assert new_state["player"]["hp"] == 85 + assert [r["path"] for r in report["applied"]] == ["player.hp"] + + +# ------------------------------------- setting it at the start, editing it later + +@pytest.fixture() +def client(monkeypatch): + Base.metadata.create_all(bind=engine) + setup = SessionLocal() + user = models.User(is_guest=False, email="persona-api@example.com") + setup.add(user) + setup.flush() + setup.add(models.Settings(user_id=user.id, api_key="enc:dummy", model="test-model")) + setup.commit() + user_id = user.id + setup.close() + + monkeypatch.setattr(limits, "rate_limit", lambda *a, **k: None) + monkeypatch.setattr(limits, "check_row_cap", lambda *a, **k: None) + + def _current_user(db=Depends(get_db)): + return db.get(models.User, user_id) + + app.dependency_overrides[auth.get_current_user] = _current_user + c = TestClient(app) + try: + yield c + finally: + app.dependency_overrides.clear() + Base.metadata.drop_all(bind=engine) + + +def test_a_persona_can_be_named_when_the_adventure_is_created(client): + """The modal that used to appear only for `${Placeholder}` scenarios now + always opens, and this is what it posts.""" + r = client.post("/api/adventures", json={ + "title": "A", "persona_name": "Kaelen", "persona_pronouns": "he/him", + "persona_desc": "A half-elf ranger.", + }) + assert r.status_code == 201, r.text + body = r.json() + assert body["persona_name"] == "Kaelen" + assert body["persona_pronouns"] == "he/him" + assert body["persona_desc"] == "A half-elf ranger." + + +def test_creating_without_a_persona_still_works(client): + """Every field is optional. A blank adventure is the old behavior.""" + r = client.post("/api/adventures", json={"title": "A"}) + assert r.status_code == 201, r.text + assert r.json()["persona_name"] == "" + + +def test_the_persona_is_stripped_on_the_way_in(client): + r = client.post("/api/adventures", json={ + "title": "A", "persona_name": " Kaelen ", "persona_desc": " ranger \n", + }) + assert r.json()["persona_name"] == "Kaelen" + assert r.json()["persona_desc"] == "ranger" + + +def test_the_persona_can_be_edited_later(client): + """A player renames their character mid-story; nothing else moves.""" + adv_id = client.post("/api/adventures", json={ + "title": "A", "persona_name": "Kaelen", + }).json()["id"] + r = client.patch(f"/api/adventures/{adv_id}", json={"persona_name": "Aria"}) + assert r.status_code == 200, r.text + assert r.json()["persona_name"] == "Aria" + assert client.get(f"/api/adventures/{adv_id}").json()["persona_name"] == "Aria" + + +def test_an_over_long_name_is_refused_rather_than_truncated(client): + """The column is VARCHAR(80). A 422 here is a 500 at INSERT otherwise.""" + r = client.post("/api/adventures", json={"title": "A", "persona_name": "K" * 81}) + assert r.status_code == 422 + + +def test_placeholders_and_the_persona_are_independent(client): + """A scenario asking for `${Name}` is asking its own question. Nothing + fills it in from the persona, and nothing fills the persona from it.""" + db = SessionLocal() + scenario = models.Scenario( + user_id=db.query(models.User).first().id, + title="S", prompt="A guard sneers at ${Name}.", + ) + db.add(scenario) + db.commit() + scenario_id = scenario.id + db.close() + + adv = client.post("/api/adventures", json={ + "scenario_id": scenario_id, + "persona_name": "Kaelen", + "placeholders": {"Name": "Wren"}, + }).json() + assert adv["persona_name"] == "Kaelen" + opening = next(a for a in adv["actions"] if a["type"] == "start") + assert "A guard sneers at Wren." == opening["text"] diff --git a/backend/tests/test_prompt_caching.py b/backend/tests/test_prompt_caching.py new file mode 100644 index 0000000..0aa6c29 --- /dev/null +++ b/backend/tests/test_prompt_caching.py @@ -0,0 +1,246 @@ +"""Prompt caching: the prompt has to start with the same bytes every turn. + +Every endpoint that caches prompts caches a prefix. It reuses the request +up to the first byte that differs from last time, and no further. The cost +of a turn is therefore decided by layout: one section that changes each +turn, placed near the top, re-prices everything underneath it, and +underneath it is the story history, which makes up most of the prompt. + +Three things must hold, and each is easy to undo by accident: + +* The static block is byte-identical across turns. Adding a section that + moves (live stats, retrieved memories, a rewritten summary) to + `system_sections` is the mistake this file exists to catch. +* The sections that move sit after the history, but still before the tail + that is last for its own reasons: front memory, the length hint, and + `EMIT_REMINDER`, which is what keeps the state block emitted at all. +* Moving a section out of the system block does not drop it from the + token budget. It is still in the prompt. + +This file also covers two request-level concerns: preferring one +OpenRouter upstream, since each upstream holds its own cache, and reading +back the usage the endpoint reports, so the hit rate is measurable rather +than assumed. + + python -m pytest tests/test_prompt_caching.py -v +""" +import os + + +import pytest + +from app import models, worldstate +from app.context import builder +from app.database import Base, SessionLocal, engine +from app.providers.openai_compatible import OpenAICompatibleProvider + +SCHEMA = { + "player": {"hp": {"min": 0, "max": 100, "initial": 100, "desc": "Health"}}, + "world": {"alarm": {"min": 0, "max": 10, "initial": 0, "desc": "Alarm level"}}, +} + + +# ------------------------------------------------- preferring one upstream + +def _routed(endpoint, model): + provider = OpenAICompatibleProvider(endpoint, "k", model, "chat", 0) + body = {"max_tokens": 100} + provider._apply_provider_routing(body) + return body + + +def test_openrouter_deepseek_pins_the_upstream(): + """Each upstream has its own cache, so routing has to be deterministic.""" + body = _routed("https://openrouter.ai/api/v1", "deepseek/deepseek-v4-flash-0731") + assert body["provider"] == {"order": ["deepseek"]} + + +def test_fallbacks_stay_on(): + """A preference, not a restriction: if deepseek is down the turn still runs + somewhere else and merely misses the cache.""" + body = _routed("https://openrouter.ai/api/v1", "deepseek/deepseek-v4-flash-0731") + assert "allow_fallbacks" not in body["provider"] + + +def test_non_openrouter_endpoints_get_no_provider_field(): + """Ollama and other providers reject fields they do not know. This is + the same problem the `reasoning` param works around.""" + body = _routed("http://localhost:11434/v1", "deepseek/deepseek-v4-flash-0731") + assert "provider" not in body + + +def test_unknown_vendors_are_left_alone(): + """The vendor half of a slug is not reliably a provider slug: Google's + models are served by "google-ai-studio", and there is no "google". A guess + would be a routing preference naming an upstream that does not exist.""" + body = _routed("https://openrouter.ai/api/v1", "google/gemma-4-26b-a4b-it:free") + assert "provider" not in body + + +# ------------------------------------------------------ reading usage back + +def test_usage_is_recorded_from_a_final_chunk(): + """In a stream, the usage block arrives in a final chunk that carries + no choices, which is why it is read separately from the text + extraction.""" + provider = OpenAICompatibleProvider("https://openrouter.ai/api/v1", "k", "m") + assert provider.last_usage is None + provider._record_usage({"choices": [{"delta": {"content": "hi"}}]}) + assert provider.last_usage is None + provider._record_usage( + {"choices": [], "usage": {"prompt_tokens": 900, + "prompt_tokens_details": {"cached_tokens": 768}}} + ) + assert provider.last_usage["prompt_tokens_details"]["cached_tokens"] == 768 + + +def test_a_later_chunk_without_usage_does_not_erase_it(): + provider = OpenAICompatibleProvider("https://openrouter.ai/api/v1", "k", "m") + provider._record_usage({"usage": {"prompt_tokens": 5}}) + provider._record_usage({"choices": [{"delta": {"content": "x"}}]}) + provider._record_usage({"usage": {}}) + assert provider.last_usage == {"prompt_tokens": 5} + + +# ------------------------------------------------------------ prompt layout + +def _with_hp(world_state, hp): + """`world_state` is nested by group, and the JSON column only detects a + whole new object. Build a new one instead of mutating in place.""" + return {**world_state, "player": {**world_state["player"], "hp": hp}} + + +@pytest.fixture() +def story(): + Base.metadata.create_all(bind=engine) + db = SessionLocal() + user = models.User(is_guest=False, email="cache@example.com") + db.add(user) + db.flush() + settings = models.Settings(user_id=user.id, api_key="enc:dummy", model="m") + db.add(settings) + scenario = models.Scenario( + user_id=user.id, title="S", prompt="A road.", stat_schema=SCHEMA + ) + db.add(scenario) + db.flush() + adventure = models.Adventure( + user_id=user.id, title="A", scenario_id=scenario.id, script_state={}, + memory="The hero hunts bandits.", + ai_instructions="Write in second person.", + story_summary="The hero left the village.", + world_state=worldstate.instantiate(SCHEMA), + # Phase 18. Set here so that every test in this file runs with a + # persona present: it is user-only, so it belongs in the static block, + # and this is the file that guards what may live there. + persona_name="Kaelen", + persona_pronouns="he/him", + persona_desc="A half-elf ranger.", + ) + db.add(adventure) + db.flush() + for i in range(6): + db.add(models.Action(adventure_id=adventure.id, + type="ai" if i % 2 else "do", + text=f"[{i}] The road bends onward past the treeline.")) + db.commit() + db.expire_all() + adventure = db.get(models.Adventure, adventure.id) + settings = db.get(models.Settings, settings.id) + try: + yield db, adventure, settings + finally: + db.close() + Base.metadata.drop_all(bind=engine) + + +def test_changing_a_stat_leaves_the_static_block_untouched(story): + """The whole point. Live values used to sit third from the top, so a + single point of damage re-priced the instructions, the plot, and the + history.""" + db, adventure, settings = story + before, _, _ = builder.build_context(adventure, settings) + adventure.world_state = _with_hp(adventure.world_state, 40) + db.commit() + after, story_text, _ = builder.build_context(adventure, settings) + assert before == after + assert "hp 40/100" in story_text, "the new value still has to reach the model" + + +def test_the_static_block_holds_the_things_that_do_not_move(story): + db, adventure, settings = story + system_text, story_text, _ = builder.build_context(adventure, settings) + for fixed in ("Write in second person.", "The hero hunts bandits.", + "You are Kaelen (he/him).", "A half-elf ranger."): + assert fixed in system_text + # The stat guide is derived from the schema, so it is fixed. The live + # values it describes are not fixed, and belong to the story text. + assert "Stat guide" in system_text + for moves in ("The hero left the village.", "hp 100/100"): + assert moves not in system_text + assert moves in story_text + + +def test_volatile_sections_sit_after_the_history(story): + db, adventure, settings = story + _, story_text, _ = builder.build_context(adventure, settings) + history_at = story_text.index("[5] The road bends") + for label in ("Story summary:", "World state"): + assert story_text.index(label) > history_at, label + + +def test_the_tail_stays_the_tail(story): + """Front memory, the length hint, and `EMIT_REMINDER` are last for + reasons of their own, and the live sections must not have displaced + them.""" + db, adventure, settings = story + _, story_text, report = builder.build_context(adventure, settings) + labels = [s["label"] for s in report["sections"]] + assert labels[-1] == "world_state_reminder" + assert labels[-2] == "length_hint" + assert labels.index("world_state") < labels.index("length_hint") + assert story_text.rstrip().endswith(worldstate.EMIT_REMINDER.rstrip()) + + +def test_live_sections_are_still_charged_to_the_budget(story): + """They moved out of `system_sections`, so it would be easy to stop + counting them in `reserved`. If that happened, the history, which is + budgeted with what is left over, would quietly overrun.""" + db, adventure, settings = story + for i in range(6, 90): + db.add(models.Action( + adventure_id=adventure.id, type="do", + text=f"[{i}] " + "The road bends onward past the treeline. " * 6, + )) + settings.context_token_budget = 4000 + db.commit() + db.expire_all() + adventure = db.get(models.Adventure, adventure.id) + settings = db.get(models.Settings, settings.id) + + _, _, lean = builder.build_context(adventure, settings) + adventure.story_summary = "The hero left the village. " * 150 + db.commit() + _, _, fat = builder.build_context(adventure, settings) + + assert fat["history"]["included"] < lean["history"]["included"], ( + "a bigger summary has to leave less room for history" + ) + + +def test_a_new_turn_only_appends_to_the_cached_prefix(story): + """Playing on must extend the previous prompt, not rewrite it: the shared + prefix has to still contain the whole static block and the older history.""" + db, adventure, settings = story + system_a, story_a, _ = builder.build_context(adventure, settings) + db.add(models.Action(adventure_id=adventure.id, type="do", + text="[6] You step into the clearing.")) + db.commit() + db.expire_all() + adventure = db.get(models.Adventure, adventure.id) + system_b, story_b, _ = builder.build_context(adventure, settings) + + assert system_a == system_b + shared = os.path.commonprefix([story_a, story_b]) + assert "[0] The road bends" in shared + assert "[5] The road bends" in shared diff --git a/backend/tests/test_ratelimit_hardening.py b/backend/tests/test_ratelimit_hardening.py new file mode 100644 index 0000000..a83455f --- /dev/null +++ b/backend/tests/test_ratelimit_hardening.py @@ -0,0 +1,99 @@ +"""Regression tests for the X-Forwarded-For rate-limit bypass and the +per-account login throttle added to close it. + +Background: uvicorn's `--forwarded-allow-ips "*"` trusted the leftmost +`X-Forwarded-For` entry. The client controls that entry, so rotating the +header issued a fresh rate-limit bucket on every request. `client_ip` now +reads the hop the trusted edge appends, which is the rightmost one. Login +also has an email-keyed throttle that no IP trick can weaken. + + python -m pytest tests/test_ratelimit_hardening.py -v +""" +import pytest + +from app import auth, limits + + +class _Req: + """Minimal stand-in for starlette's Request: a header lookup and a peer.""" + + def __init__(self, xff: str | None, peer: str | None = "10.0.0.1"): + self.headers = {} if xff is None else {"x-forwarded-for": xff} + self.client = None if peer is None else type("C", (), {"host": peer})() + + +# ---------- client_ip: the spoof-resistant hop ---------- + +def test_client_ip_takes_appended_rightmost_hop(monkeypatch): + monkeypatch.setattr(limits, "TRUSTED_PROXY_HOPS", 1) + # An attacker prepends a fake IP. The edge appends the real one on the right. + req = _Req("203.0.113.9, 198.51.100.77") + assert limits.client_ip(req) == "198.51.100.77" + + +def test_client_ip_ignores_spoofed_leftmost(monkeypatch): + monkeypatch.setattr(limits, "TRUSTED_PROXY_HOPS", 1) + # The keyed IP stays the real hop regardless of what the client adds on + # the left, so rotating that value no longer creates a new bucket. + a = limits.client_ip(_Req("1.1.1.1, 198.51.100.77")) + b = limits.client_ip(_Req("2.2.2.2, 198.51.100.77")) + c = limits.client_ip(_Req("evil, junk, 198.51.100.77")) + assert a == b == c == "198.51.100.77" + + +def test_client_ip_honours_extra_trusted_hops(monkeypatch): + monkeypatch.setattr(limits, "TRUSTED_PROXY_HOPS", 2) + # Two trusted hops: real client is second from the right. + req = _Req("9.9.9.9, 203.0.113.5, 198.51.100.77") + assert limits.client_ip(req) == "203.0.113.5" + + +def test_client_ip_falls_back_to_socket_peer(): + assert limits.client_ip(_Req(None, peer="172.16.0.4")) == "172.16.0.4" + assert limits.client_ip(_Req(None, peer=None)) == "unknown" + + +# ---------- per-account login throttle ---------- + +@pytest.fixture(autouse=True) +def _multi_user(monkeypatch): + monkeypatch.setattr(auth, "MULTI_USER", True) + # Isolate the module-level failure map for each test. + from collections import defaultdict, deque + monkeypatch.setattr(limits, "_login_fails", defaultdict(deque)) + + +def test_login_throttle_blocks_after_limit(): + email = "victim@example.com" + # Each attempt up to the limit is allowed and recorded as a failure. + for _ in range(limits.LOGIN_FAIL_LIMIT): + limits.check_login_allowed(email) # does not raise + limits.note_login_failure(email) + # One more failure exceeds the limit. + with pytest.raises(limits.HTTPException) as exc: + limits.check_login_allowed(email) + assert exc.value.status_code == 429 + + +def test_login_throttle_is_per_account(): + for _ in range(limits.LOGIN_FAIL_LIMIT): + limits.note_login_failure("a@example.com") + with pytest.raises(limits.HTTPException): + limits.check_login_allowed("a@example.com") + # A different account is unaffected because the throttle keys on email, not IP. + limits.check_login_allowed("b@example.com") # must not raise + + +def test_successful_login_clears_the_streak(): + email = "typo@example.com" + for _ in range(limits.LOGIN_FAIL_LIMIT): + limits.note_login_failure(email) + limits.note_login_success(email) + limits.check_login_allowed(email) # failure streak cleared, must not raise + + +def test_throttle_is_noop_in_local_mode(monkeypatch): + monkeypatch.setattr(auth, "MULTI_USER", False) + for _ in range(limits.LOGIN_FAIL_LIMIT * 3): + limits.note_login_failure("solo@example.com") + limits.check_login_allowed("solo@example.com") # never throttled locally diff --git a/backend/tests/test_reasoning_param.py b/backend/tests/test_reasoning_param.py new file mode 100644 index 0000000..4b50baf --- /dev/null +++ b/backend/tests/test_reasoning_param.py @@ -0,0 +1,45 @@ +"""What the provider puts in the `reasoning` request field for each budget +setting: a positive budget asks for thinking, 0 stays silent, -1 turns it off. + + python -m pytest tests/test_reasoning_param.py -v +""" +from app.providers.openai_compatible import OpenAICompatibleProvider + + +def _body(reasoning_max_tokens, api_mode="chat", max_tokens=1000): + provider = OpenAICompatibleProvider( + "https://openrouter.ai/api/v1", "k", "deepseek/deepseek-v4-flash-0731", + api_mode, reasoning_max_tokens, + ) + body = {"max_tokens": max_tokens} + provider._apply_reasoning_budget(body) + return body + + +def test_zero_sends_nothing(): + """Ollama and other providers reject unknown fields. Sending 0 must not add a `reasoning` field.""" + assert "reasoning" not in _body(0) + + +def test_positive_budget_adds_thinking_tokens(): + body = _body(500) + assert body["reasoning"] == {"max_tokens": 500} + # the story output keeps its own full budget on top of the thinking budget + assert body["max_tokens"] == 1500 + + +def test_negative_turns_reasoning_off(): + body = _body(-1) + assert body["reasoning"] == {"effort": "none"} + # "off" must not inflate the output budget + assert body["max_tokens"] == 1000 + + +def test_off_is_not_merely_excluded(): + """`exclude: true` still generates and bills for reasoning tokens. The off setting must omit the field entirely instead of relying on `exclude`.""" + assert _body(-1)["reasoning"].get("exclude") is None + + +def test_completion_mode_never_sends_reasoning(): + for budget in (-1, 0, 500): + assert "reasoning" not in _body(budget, api_mode="completion") diff --git a/backend/tests/test_retry_variants.py b/backend/tests/test_retry_variants.py new file mode 100644 index 0000000..484188f --- /dev/null +++ b/backend/tests/test_retry_variants.py @@ -0,0 +1,363 @@ +"""End-to-end HTTP tests for retry history: every attempt at an AI turn is kept +as a variant of the same action, browsable and (for the last message) +switchable, restoring the world/script state that attempt produced. + + python -m pytest tests/test_retry_variants.py -v +""" +import pytest +from fastapi import Depends +from fastapi.testclient import TestClient + +from app import auth, limits, models +from app.database import Base, SessionLocal, engine, get_db +from app.main import app +from app.providers import ProviderError +from app.routers import adventures + +from fakes import ScriptedProvider + +SCHEMA = {"player": {"hp": {"min": 0, "max": 100, "initial": 100}}} + +# Each turn spends 10 gold, so a double-applied or un-rolled-back attempt shows. +GOLD_SCRIPT = """ +const modifier = (text) => { + state.gold = (state.gold || 0) + 10; + return { text }; +}; +modifier(text); +""" + + +@pytest.fixture() +def client(monkeypatch): + Base.metadata.create_all(bind=engine) + setup = SessionLocal() + user = models.User(is_guest=False, email="variants@example.com") + setup.add(user) + setup.flush() + setup.add(models.Settings(user_id=user.id, api_key="enc:dummy", model="test-model")) + scenario = models.Scenario(user_id=user.id, title="S", stat_schema=SCHEMA) + setup.add(scenario) + setup.flush() + adv = models.Adventure( + user_id=user.id, title="Cave", scenario_id=scenario.id, + script_state={}, world_state={"player": {"hp": 100}}, + ) + setup.add(adv) + setup.flush() + setup.add(models.Action(adventure_id=adv.id, type="start", text="You enter a cave.")) + setup.add(models.AdventureScript( + adventure_id=adv.id, position=0, enabled=True, name="Gold", output_js=GOLD_SCRIPT, + )) + setup.commit() + adv_id, user_id = adv.id, user.id + setup.close() + + ScriptedProvider.replies = ["Attempt one."] + ScriptedProvider.calls = 0 + ScriptedProvider.prompts = [] + monkeypatch.setattr(adventures.turns, "OpenAICompatibleProvider", ScriptedProvider) + monkeypatch.setattr(auth, "resolve_provider_config", lambda s: auth.ProviderConfig( + "http://fake", "k", "test-model", False)) + monkeypatch.setattr(limits, "rate_limit", lambda *a, **k: None) + monkeypatch.setattr(limits, "check_row_cap", lambda *a, **k: None) + + def _current_user(db=Depends(get_db)): + return db.get(models.User, user_id) + + app.dependency_overrides[auth.get_current_user] = _current_user + c = TestClient(app) + c.adv_id = adv_id + try: + yield c + finally: + app.dependency_overrides.clear() + adventures.turns._active_turns.clear() + Base.metadata.drop_all(bind=engine) + + +def _adv(adv_id): + db = SessionLocal() + try: + adv = db.get(models.Adventure, adv_id) + return adv.script_state, adv.world_state + finally: + db.close() + + +def _actions(client): + return client.get(f"/api/adventures/{client.adv_id}").json()["actions"] + + +def _play(client, text="look around"): + r = client.post(f"/api/adventures/{client.adv_id}/actions", + json={"type": "do", "text": text}) + assert r.status_code == 200, r.text + + +def _retry(client): + r = client.post(f"/api/adventures/{client.adv_id}/retry") + assert r.status_code == 200, r.text + return r + + +# ---------------------------------------------------------------- keeping them + +def test_retry_keeps_the_discarded_attempt(client): + ScriptedProvider.replies = ["Attempt one.", "Attempt two."] + _play(client) + assert _actions(client)[-1]["text"] == "Attempt one." + + _retry(client) + actions = _actions(client) + # One AI action still, not two. The retry replaced the live text in place. + assert [a["type"] for a in actions] == ["start", "do", "ai"] + last = actions[-1] + assert last["text"] == "Attempt two." + assert last["take_count"] == 2 + assert last["take_index"] == 1 + + r = client.get(f"/api/adventures/{client.adv_id}/actions/{last['id']}/variants") + assert r.status_code == 200, r.text + assert [v["text"] for v in r.json()] == ["Attempt one.", "Attempt two."] + assert [v["active"] for v in r.json()] == [False, True] + + +def test_retry_context_excludes_the_attempt_being_replaced(client): + """A retry produces a fresh take on the same turn. The row survives the + retry because it holds the variant history, so it is still in + `adventure.actions` while the replacement context is assembled. The + context builder must filter it out, or the model continues past the + attempt it is replacing and writes a sequel that blends both.""" + ScriptedProvider.replies = ["Attempt one.", "Attempt two."] + _play(client) + _retry(client) + + retry_story = ScriptedProvider.prompts[-1][1] + assert "Attempt one." not in retry_story + # The turn's own player action must still be there. It is what the model responds to. + assert "look around" in retry_story + assert "You enter a cave." in retry_story + + +def test_retry_context_keeps_earlier_ai_turns(client): + """Only the action being retried is dropped, not AI history in general.""" + ScriptedProvider.replies = ["First turn.", "Second turn.", "Second, again."] + _play(client, "go north") + _play(client, "go south") + _retry(client) + + retry_story = ScriptedProvider.prompts[-1][1] + assert "First turn." in retry_story + assert "Second turn." not in retry_story + + +def test_never_retried_action_has_no_variants(client): + _play(client) + last = _actions(client)[-1] + assert last["take_count"] == 1 # itself, and nothing to page to + assert client.get( + f"/api/adventures/{client.adv_id}/actions/{last['id']}/variants").json() == [] + + +def test_three_attempts_all_kept_in_order(client): + ScriptedProvider.replies = ["One.", "Two.", "Three."] + _play(client) + _retry(client) + _retry(client) + last = _actions(client)[-1] + assert last["take_count"] == 3 + assert last["take_index"] == 2 + variants = client.get( + f"/api/adventures/{client.adv_id}/actions/{last['id']}/variants").json() + assert [v["text"] for v in variants] == ["One.", "Two.", "Three."] + + +# ---------------------------------------------------------------- switching + +def test_switching_back_restores_that_attempt_state(client): + ScriptedProvider.replies = [ + "You take a scratch.\n```state\n{\"player.hp\": -5}\n```", + "You take a beating.\n```state\n{\"player.hp\": -40}\n```", + ] + _play(client) + assert _adv(client.adv_id)[1]["player"]["hp"] == 95 + _retry(client) + script_state, world_state = _adv(client.adv_id) + assert world_state["player"]["hp"] == 60 + assert script_state == {"gold": 10} # rolled back, not stacked to 20 + + last = _actions(client)[-1] + r = client.post( + f"/api/adventures/{client.adv_id}/actions/{last['id']}/variant", json={"index": 0}) + assert r.status_code == 200, r.text + assert r.json()["text"].startswith("You take a scratch") + assert r.json()["take_index"] == 0 + # The stats follow the narration back. + script_state, world_state = _adv(client.adv_id) + assert world_state["player"]["hp"] == 95 + assert script_state == {"gold": 10} + + # And forward again. + client.post(f"/api/adventures/{client.adv_id}/actions/{last['id']}/variant", + json={"index": 1}) + assert _adv(client.adv_id)[1]["player"]["hp"] == 60 + + +def test_switching_updates_the_world_change_chips(client): + ScriptedProvider.replies = [ + "A scratch.\n```state\n{\"player.hp\": -5}\n```", + "A beating.\n```state\n{\"player.hp\": -40}\n```", + ] + _play(client) + _retry(client) + last = _actions(client)[-1] + assert last["world_changes"][0]["delta"] == -40 + + client.post(f"/api/adventures/{client.adv_id}/actions/{last['id']}/variant", + json={"index": 0}) + assert _actions(client)[-1]["world_changes"][0]["delta"] == -5 + + +def test_cannot_switch_a_turn_the_story_moved_past(client): + ScriptedProvider.replies = ["One.", "Two.", "Three."] + _play(client) + _retry(client) + retried = _actions(client)[-1] + _play(client) # story continues from "Two." + + r = client.post( + f"/api/adventures/{client.adv_id}/actions/{retried['id']}/variant", json={"index": 0}) + assert r.status_code == 400 + assert "latest message" in r.json()["detail"] + # The variant is still readable. Keeping every attempt browsable is why it still exists. + variants = client.get( + f"/api/adventures/{client.adv_id}/actions/{retried['id']}/variants").json() + assert [v["text"] for v in variants] == ["One.", "Two."] + + +def test_switching_to_a_missing_index_is_rejected(client): + ScriptedProvider.replies = ["One.", "Two."] + _play(client) + _retry(client) + last = _actions(client)[-1] + r = client.post( + f"/api/adventures/{client.adv_id}/actions/{last['id']}/variant", json={"index": 7}) + assert r.status_code == 400 + + +# ---------------------------------------------------------------- edge cases + +def test_failed_retry_leaves_the_previous_attempt_in_charge(client): + """A provider error mid-retry must undo the rollback, or the stats on + screen would silently disagree with the text still shown.""" + ScriptedProvider.replies = ["Attempt one.", ProviderError("upstream is down")] + _play(client) + assert _adv(client.adv_id)[0] == {"gold": 10} + + client.post(f"/api/adventures/{client.adv_id}/retry") + + actions = _actions(client) + assert actions[-1]["text"] == "Attempt one." # text never lost + assert _adv(client.adv_id)[0] == {"gold": 10} # and the state still matches it + + +def test_undo_removes_the_action_and_its_history(client): + ScriptedProvider.replies = ["One.", "Two."] + _play(client) + _retry(client) + r = client.post(f"/api/adventures/{client.adv_id}/undo") + assert r.status_code == 200, r.text + # Undo returns the newest window now, not the whole story. + assert [a["type"] for a in r.json()["actions"]] == ["start"] + assert _adv(client.adv_id)[0] == {} + + +def test_editing_the_text_updates_the_live_variant(client): + ScriptedProvider.replies = ["One.", "Two."] + _play(client) + _retry(client) + last = _actions(client)[-1] + client.patch(f"/api/adventures/{client.adv_id}/actions/{last['id']}", + json={"text": "Two, but better."}) + + # Page away and back: the edit must survive, not be reverted by the switch. + client.post(f"/api/adventures/{client.adv_id}/actions/{last['id']}/variant", + json={"index": 0}) + client.post(f"/api/adventures/{client.adv_id}/actions/{last['id']}/variant", + json={"index": 1}) + assert _actions(client)[-1]["text"] == "Two, but better." + + +def _live_depth(client) -> int: + """Returns the depth of the newest action the story tells.""" + db = SessionLocal() + try: + return ( + db.query(models.Action.depth) + .filter(models.Action.adventure_id == client.adv_id, + models.Action.live.is_(True)) + .order_by(models.Action.depth.desc()) + .first()[0] + ) + finally: + db.close() + + +def test_retry_keeps_the_turn_depth(client): + """A retry re-runs the same turn, so the world-state clock (which drives + cooldowns) must not advance. + + The depth is read from the row rather than the payload. It is a coordinate + on one branch, and the client pages by id, so it is not exposed. + """ + ScriptedProvider.replies = ["One.", "Two."] + _play(client) + before = _live_depth(client) + _retry(client) + assert _live_depth(client) == before + + +def test_export_and_import_round_trips_variants(client): + ScriptedProvider.replies = ["One.", "Two."] + _play(client) + _retry(client) + bundle = client.get(f"/api/adventures/{client.adv_id}/export").json() + # SP6: the attempts are nodes in the bundle too, sharing one coordinate, + # and `live` says which of them the story tells. The `variants` array + # survives only in the v1 reader. See the hand-edited bundle below. + ai = [a for a in bundle["actions"] if a["type"] == "ai"] + assert [(a["text"], a["live"]) for a in ai] == [("One.", False), ("Two.", True)] + assert len({(a["branch"], a["depth"]) for a in ai}) == 1 + + r = client.post("/api/adventures/import", json=bundle) + assert r.status_code == 201, r.text + imported = r.json()["id"] + actions = client.get(f"/api/adventures/{imported}").json()["actions"] + assert [a["type"] for a in actions] == ["start", "do", "ai"] + assert actions[-1]["text"] == "Two." + + # The pager reads 1/1 on the copy, because the import writes no + # `parent_id` and `annotate_takes` groups on it. The attempts are both + # there, at one coordinate, and `GET .../variants` still lists them. This + # is a gap in the import rather than in the drop: `take_count` has been the + # only number the client reads since SP9, and the import has never set the + # column it is derived from. + assert actions[-1]["take_count"] == 1 + variants = client.get( + f"/api/adventures/{imported}/actions/{actions[-1]['id']}/variants").json() + assert [v["text"] for v in variants] == ["One.", "Two."] + + +def test_import_clamps_an_out_of_range_variant_index(client): + bundle = { + "format": "ai-dnd-adventure-v1", "title": "Hand-edited", + "actions": [{ + "index": 0, "type": "ai", "text": "Only.", + "variants": [{"text": "Only."}], "variantIndex": 9, + }], + } + r = client.post("/api/adventures/import", json=bundle) + assert r.status_code == 201, r.text + actions = client.get(f"/api/adventures/{r.json()['id']}").json()["actions"] + assert actions[0]["take_index"] == 0 diff --git a/backend/tests/test_scenario_art.py b/backend/tests/test_scenario_art.py new file mode 100644 index 0000000..5a0ad78 --- /dev/null +++ b/backend/tests/test_scenario_art.py @@ -0,0 +1,205 @@ +"""Scenario cover art and the Continue-card snippet. + +Unit tests for the data-URI handling in app/images.py. Then HTTP tests +confirm that the list endpoints advertise a cacheable `image_url` instead +of the inline base64, that the image route serves real bytes, and that an +adventure's snippet comes from the latest narration rather than the +player's last line. + + python -m pytest tests/test_scenario_art.py -v +""" +import base64 + + +import pytest +from fastapi import Depends +from fastapi.testclient import TestClient + +from app import auth, images, limits, models, schemas +from app.database import Base, SessionLocal, engine, get_db +from app.main import app + +# Smallest valid PNG: a single transparent pixel. +PNG_BYTES = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFAAH/q842iQAAAABJRU5ErkJggg==" +) +PNG_URI = "data:image/png;base64," + base64.b64encode(PNG_BYTES).decode() + + +# --------------------------------------------------------------------------- # +# app/images.py +# --------------------------------------------------------------------------- # + +def test_decode_returns_bytes_and_content_type(): + assert images.decode(PNG_URI) == (PNG_BYTES, "image/png") + + +def test_decode_tolerates_wrapped_base64(): + """A hand-pasted URI can carry newlines. `b64decode(validate=True)` rejects them.""" + wrapped = "data:image/png;base64," + "\n".join( + base64.b64encode(PNG_BYTES).decode()[i:i + 24] for i in range(0, 100, 24) + ) + # This test only confirms that the call does not raise and does not + # silently return a partial decode of a truncated payload. The wrapped + # prefix here is not the whole image, so a None result is also + # acceptable. What matters is that no exception occurs. + images.decode(wrapped) + + +def test_decode_rejects_non_data_uris_and_garbage(): + assert images.decode("https://example.com/a.png") is None + assert images.decode("data:image/png;base64,!!!not base64!!!") is None + assert images.decode("") is None + assert images.decode(None) is None + + +def test_decode_rejects_svg(): + """SVG can carry script, and the app serves these bytes from its own origin.""" + svg = "data:image/svg+xml;base64," + base64.b64encode(b"").decode() + assert images.decode(svg) is None + + +def test_public_url_points_at_the_endpoint_for_data_uris(): + class Stamp: + def timestamp(self): + return 1700000000.0 + + assert images.public_url(7, PNG_URI, Stamp()) == "/api/scenarios/7/image?v=1700000000" + + +def test_public_url_passes_through_https_but_not_http(): + stamp = None + assert images.public_url(1, "https://cdn.example.com/a.png", stamp) == \ + "https://cdn.example.com/a.png" + # http:// would be blocked as mixed content on the deployed https app. + assert images.public_url(1, "http://cdn.example.com/a.png", stamp) == "" + assert images.public_url(1, "", stamp) == "" + + +def test_sanitize_rejects_hostile_and_oversized_values(): + assert images.sanitize("javascript:alert(1)", 1000) == "" + assert images.sanitize("http://example.com/a.png", 1000) == "" + assert images.sanitize(PNG_URI, len(PNG_URI) - 1) == "" # over the cap + assert images.sanitize(12345, 1000) == "" + assert images.sanitize(None, 1000) == "" + # Well-formed values survive. + assert images.sanitize(PNG_URI, schemas.IMAGE_MAX) == PNG_URI + assert images.sanitize("https://example.com/a.png", 1000) == "https://example.com/a.png" + + +# --------------------------------------------------------------------------- # +# HTTP +# --------------------------------------------------------------------------- # + +@pytest.fixture() +def client(monkeypatch): + Base.metadata.create_all(bind=engine) + setup = SessionLocal() + user = models.User(is_guest=False, email="art@example.com") + setup.add(user) + setup.flush() + setup.add(models.Settings(user_id=user.id, api_key="enc:dummy", model="test-model")) + + # One scenario with an uploaded picture, one with only an emoji. + pictured = models.Scenario(user_id=user.id, title="Pictured", image=PNG_URI) + emoji_only = models.Scenario(user_id=user.id, title="Emoji", icon="🐉") + setup.add_all([pictured, emoji_only]) + setup.flush() + + adventure = models.Adventure( + user_id=user.id, scenario_id=pictured.id, title="A Run", + ) + setup.add(adventure) + setup.flush() + # A full turn: opening narration, the player's line, then the AI's reply. + setup.add_all([ + models.Action(adventure_id=adventure.id, type="ai", + text="The door groans open."), + models.Action(adventure_id=adventure.id, type="do", + text="I draw my sword."), + models.Action(adventure_id=adventure.id, type="ai", + text="Steel rings. The\ncorridor answers."), + ]) + setup.commit() + ids = {"scenario": pictured.id, "emoji": emoji_only.id, "adventure": adventure.id} + user_id = user.id + setup.close() + + monkeypatch.setattr(limits, "rate_limit", lambda *a, **k: None) + monkeypatch.setattr(limits, "check_row_cap", lambda *a, **k: None) + + def _current_user(db=Depends(get_db)): + return db.get(models.User, user_id) + + app.dependency_overrides[auth.get_current_user] = _current_user + c = TestClient(app) + c.ids = ids + try: + yield c + finally: + app.dependency_overrides.clear() + Base.metadata.drop_all(bind=engine) + + +def test_scenario_list_advertises_a_url_and_hides_the_base64(client): + rows = {row["title"]: row for row in client.get("/api/scenarios").json()} + + pictured = rows["Pictured"] + assert pictured["image_url"].startswith(f"/api/scenarios/{client.ids['scenario']}/image?v=") + # A list response must never carry the inline image. + assert "image" not in pictured + + assert rows["Emoji"]["image_url"] == "" + assert rows["Emoji"]["icon"] == "🐉" + + +def test_image_endpoint_serves_the_decoded_bytes(client): + resp = client.get(f"/api/scenarios/{client.ids['scenario']}/image") + assert resp.status_code == 200 + assert resp.headers["content-type"] == "image/png" + assert resp.content == PNG_BYTES + assert "immutable" in resp.headers["cache-control"] + + +def test_image_endpoint_404s_without_an_upload(client): + resp = client.get(f"/api/scenarios/{client.ids['emoji']}/image") + assert resp.status_code == 404 + + +def test_single_scenario_still_returns_the_raw_uri_for_editing(client): + """The editor needs the actual value to preview and to clear.""" + body = client.get(f"/api/scenarios/{client.ids['scenario']}").json() + assert body["image"] == PNG_URI + + +def test_adventure_list_carries_snippet_and_inherited_art(client): + row = next(r for r in client.get("/api/adventures").json() + if r["id"] == client.ids["adventure"]) + # Latest narration, whitespace collapsed. Not the player's line, "I draw my sword." + assert row["snippet"] == "Steel rings. The corridor answers." + assert row["image_url"].startswith(f"/api/scenarios/{client.ids['scenario']}/image?v=") + assert row["action_count"] == 3 + + +def test_snippet_is_truncated_on_a_word_boundary(client): + from app.routers.adventures import SNIPPET_MAX, _snippet + + long_text = "word " * 200 + out = _snippet(long_text) + assert len(out) <= SNIPPET_MAX + 1 # +1 for the ellipsis + assert out.endswith("…") + assert "wor…" not in out # never cuts mid-word + + +def test_scenario_export_import_round_trips_the_art(client): + bundle = client.get(f"/api/scenarios/{client.ids['scenario']}/export").json() + assert bundle["image"] == PNG_URI + + created = client.post("/api/scenarios/import", json=bundle).json()["scenario"] + assert created["image"] == PNG_URI + + +def test_import_drops_a_hostile_image_value(client): + bundle = {"title": "Sneaky", "image": "javascript:alert(1)"} + created = client.post("/api/scenarios/import", json=bundle).json()["scenario"] + assert created["image"] == "" diff --git a/backend/tests/test_scenario_refresh.py b/backend/tests/test_scenario_refresh.py new file mode 100644 index 0000000..40e5dc8 --- /dev/null +++ b/backend/tests/test_scenario_refresh.py @@ -0,0 +1,374 @@ +"""Tests for "Update from scenario" — pulling an edited scenario's plot text, +story cards and stat schema back down over a running adventure's copy. + + python -m pytest tests/test_scenario_refresh.py -v +""" +import pytest +from fastapi import Depends +from fastapi.testclient import TestClient + +from app import auth, limits, models, worldstate +from app.database import Base, SessionLocal, engine, get_db +from app.main import app + +SCHEMA = { + "player": {"hp": {"min": 0, "max": 100, "initial": 100}}, + "npcs": { + "gwen": {"name": "Gwen", "keys": "Gwen, ranger", "desc": "A loyal ranger.", + "stats": {"trust": {"min": -100, "max": 100, "initial": 0}}}, + }, + "flags": {"alarm": {"desc": "Alerted", "initial": False}}, + "milestones": {"win": {"desc": "Win the fight"}}, +} + + +@pytest.fixture() +def client(monkeypatch): + Base.metadata.create_all(bind=engine) + setup = SessionLocal() + user = models.User(is_guest=False, email="refresh@example.com") + setup.add(user) + setup.flush() + setup.add(models.Settings(user_id=user.id, api_key="enc:dummy", model="test-model")) + setup.commit() + user_id = user.id + setup.close() + + monkeypatch.setattr(limits, "rate_limit", lambda *a, **k: None) + monkeypatch.setattr(limits, "check_row_cap", lambda *a, **k: None) + + def _current_user(db=Depends(get_db)): + return db.get(models.User, user_id) + + app.dependency_overrides[auth.get_current_user] = _current_user + c = TestClient(app) + c.user_id = user_id + try: + yield c + finally: + app.dependency_overrides.clear() + Base.metadata.drop_all(bind=engine) + + +def make_scenario(client, **kwargs): + db = SessionLocal() + try: + cards = kwargs.pop("cards", []) + scenario = models.Scenario(user_id=client.user_id, **kwargs) + db.add(scenario) + db.flush() + for card in cards: + db.add(models.StoryCard(scenario_id=scenario.id, **card)) + db.commit() + return scenario.id + finally: + db.close() + + +def edit_scenario(scenario_id, cards=None, **fields): + """Author-side edit, straight to the database. The scenario API is + tested elsewhere. + + `cards` is the scenario's full card list afterwards. Cards are matched + to existing rows by name and edited in place, exactly as ScenarioEditor + does (PATCH /story-cards/{id}). Card ids stay stable across authoring, + which is what source_ref tracking relies on. + """ + db = SessionLocal() + try: + scenario = db.get(models.Scenario, scenario_id) + for field, value in fields.items(): + setattr(scenario, field, value) + if cards is not None: + existing = {c.name: c for c in scenario.story_cards} + for spec in cards: + card = existing.pop(spec["name"], None) + if card is None: + db.add(models.StoryCard(scenario_id=scenario_id, **spec)) + else: + for field, value in spec.items(): + setattr(card, field, value) + for card in existing.values(): + db.delete(card) + db.commit() + finally: + db.close() + + +def start(client, scenario_id, placeholders=None): + r = client.post("/api/adventures", json={ + "scenario_id": scenario_id, "placeholders": placeholders or {}}) + assert r.status_code == 201, r.text + return r.json()["id"] + + +def get_adventure(client, adv_id): + r = client.get(f"/api/adventures/{adv_id}") + assert r.status_code == 200, r.text + return r.json() + + +def preview(client, adv_id): + r = client.get(f"/api/adventures/{adv_id}/refresh") + assert r.status_code == 200, r.text + return r.json() + + +def refresh(client, adv_id, placeholders=None): + r = client.post(f"/api/adventures/{adv_id}/refresh", + json={"placeholders": placeholders or {}}) + assert r.status_code == 200, r.text + return r.json() + + +def card_named(adventure, name): + return next((c for c in adventure["story_cards"] if c["name"] == name), None) + + +# --------------------------------------------------------------------------- # +# Plot text +# --------------------------------------------------------------------------- # + +def test_refresh_overwrites_plot_text_including_player_edits(client): + sid = make_scenario(client, title="Keep", memory="Old memory", + authors_note="Old note", ai_instructions="Old rules") + adv_id = start(client, sid) + + # The player tunes the adventure's copy, then the author edits the scenario. + client.patch(f"/api/adventures/{adv_id}", json={"memory": "Player's own memory"}) + edit_scenario(sid, memory="New memory", authors_note="New note") + + plan = preview(client, adv_id) + assert plan["has_changes"] is True + assert plan["fields"]["memory"] == {"old": "Player's own memory", "new": "New memory"} + assert "ai_instructions" not in plan["fields"] # unchanged fields aren't listed + + adventure = refresh(client, adv_id) + assert adventure["memory"] == "New memory" + assert adventure["authors_note"] == "New note" + assert adventure["ai_instructions"] == "Old rules" + + +def test_refresh_leaves_the_opening_action_and_title_alone(client): + sid = make_scenario(client, title="Keep", prompt="You stand at the gate.") + adv_id = start(client, sid) + client.patch(f"/api/adventures/{adv_id}", json={"title": "My run"}) + edit_scenario(sid, title="Fortress", prompt="You stand in the throne room.") + + adventure = refresh(client, adv_id) + # The story is built on the opening beat, and it is baked into memories and + # the summary, so a refresh must never rewrite it. + assert adventure["actions"][0]["text"] == "You stand at the gate." + assert adventure["title"] == "My run" + + +def test_no_changes_reported_when_nothing_was_edited(client): + sid = make_scenario(client, title="Keep", memory="Same", stat_schema=SCHEMA, + cards=[{"name": "Gate", "keys": "gate", "entry": "Iron."}]) + adv_id = start(client, sid) + assert preview(client, adv_id)["has_changes"] is False + + +# --------------------------------------------------------------------------- # +# Story cards +# --------------------------------------------------------------------------- # + +def test_cards_added_updated_and_removed_but_player_cards_survive(client): + sid = make_scenario(client, title="Keep", cards=[ + {"name": "Gate", "keys": "gate", "entry": "An iron gate."}, + {"name": "Well", "keys": "well", "entry": "A dry well."}, + ]) + adv_id = start(client, sid) + + # The player writes their own card mid-play. + client.post("/api/story-cards", json={ + "adventure_id": adv_id, "name": "My horse", "keys": "horse", "entry": "Bess."}) + + # The author rewrites the Gate, drops the Well, adds a Tower. + edit_scenario(sid, cards=[ + {"name": "Gate", "keys": "gate, portcullis", "entry": "A rusted iron gate."}, + {"name": "Tower", "keys": "tower", "entry": "It leans."}, + ]) + + plan = preview(client, adv_id) + assert plan["cards"] == {"added": ["Tower"], "updated": ["Gate"], "removed": ["Well"]} + + adventure = refresh(client, adv_id) + names = sorted(c["name"] for c in adventure["story_cards"]) + assert names == ["Gate", "My horse", "Tower"] + assert card_named(adventure, "Gate")["entry"] == "A rusted iron gate." + # A player-authored card is never touched, even though the scenario has no + # matching card — only scenario-derived copies are managed. + assert card_named(adventure, "My horse")["entry"] == "Bess." + + +def test_a_renamed_scenario_card_is_updated_not_duplicated(client): + sid = make_scenario(client, title="Keep", + cards=[{"name": "Gate", "keys": "gate", "entry": "Iron."}]) + adv_id = start(client, sid) + db = SessionLocal() + try: + card = db.query(models.StoryCard).filter_by(scenario_id=sid).one() + card.name = "Portcullis" + db.commit() + finally: + db.close() + + adventure = refresh(client, adv_id) + # Tracked by source_ref, so a rename is a rename — not a delete plus an add. + assert [c["name"] for c in adventure["story_cards"]] == ["Portcullis"] + + +def test_legacy_cards_with_no_source_ref_are_matched_by_name_and_adopted(client): + sid = make_scenario(client, title="Keep", + cards=[{"name": "Gate", "keys": "gate", "entry": "Iron."}]) + adv_id = start(client, sid) + # Simulate an adventure created before source_ref existed. + db = SessionLocal() + try: + for card in db.query(models.StoryCard).filter_by(adventure_id=adv_id): + card.source_ref = None + db.commit() + finally: + db.close() + + edit_scenario(sid, cards=[{"name": "Gate", "keys": "gate", "entry": "Rusted iron."}]) + adventure = refresh(client, adv_id) + assert len(adventure["story_cards"]) == 1 + assert card_named(adventure, "Gate")["entry"] == "Rusted iron." + + db = SessionLocal() + try: + adopted = db.query(models.StoryCard).filter_by(adventure_id=adv_id).one() + assert adopted.source_ref is not None # syncs by id from here on + finally: + db.close() + + +def test_npc_cards_are_created_for_npcs_added_after_the_adventure_started(client): + sid = make_scenario(client, title="Keep", stat_schema={"player": SCHEMA["player"]}) + adv_id = start(client, sid) + assert get_adventure(client, adv_id)["story_cards"] == [] + + edit_scenario(sid, stat_schema=SCHEMA) + adventure = refresh(client, adv_id) + gwen = card_named(adventure, "Gwen") + assert gwen is not None and gwen["keys"] == "Gwen, ranger" + + +# --------------------------------------------------------------------------- # +# World state +# --------------------------------------------------------------------------- # + +def test_refresh_keeps_live_values_but_syncs_the_shape(client): + sid = make_scenario(client, title="Keep", stat_schema=SCHEMA) + adv_id = start(client, sid) + # Play happens: damage taken, a milestone reached. + client.put(f"/api/adventures/{adv_id}/world-state", + json={"player.hp": 40, "npc.gwen.trust": 25, "milestones.win": True}) + + edited = { + "player": {"hp": SCHEMA["player"]["hp"], "gold": {"min": 0, "initial": 10}}, + "npcs": {"gwen": {"name": "Gwen", "keys": "Gwen, ranger", "desc": "A loyal ranger.", + "stats": {"trust": {"min": -100, "max": 100, "initial": 0}}}}, + "flags": {}, # the alarm flag is gone + "milestones": SCHEMA["milestones"], + } + edit_scenario(sid, stat_schema=edited) + + plan = preview(client, adv_id) + assert plan["world_state"] == {"added": ["player.gold"], "removed": ["flags.alarm"]} + + refresh(client, adv_id) + r = client.get(f"/api/adventures/{adv_id}/world-state") + state = r.json()["state"] + # Values the schema still defines survive — a refresh is not a reset. + assert state["player"]["hp"] == 40 + assert state["npc"]["gwen"]["trust"] == 25 + assert state["milestones"]["win"]["reached"] is True + assert state["player"]["gold"] == 10 # new stat, at its initial + assert state["flags"] == {} # dropped stat, cleaned up + + +def test_reconcile_drops_a_removed_stats_cooldown_bookkeeping(): + state = worldstate.instantiate(SCHEMA) + state["_meta"]["last_changed"] = {"player.hp": 3, "npc.gwen.trust": 4} + reduced = {"player": SCHEMA["player"], "milestones": SCHEMA["milestones"]} + + new_state, report = worldstate.reconcile(state, reduced) + assert "npc.gwen" in report["removed"] + assert new_state["_meta"]["last_changed"] == {"player.hp": 3} + + +def test_reconcile_clears_everything_when_the_rpg_layer_is_removed(): + new_state, report = worldstate.reconcile(worldstate.instantiate(SCHEMA), None) + assert new_state == {} + assert report["removed"] == ["(all world state)"] + + +# --------------------------------------------------------------------------- # +# Placeholders +# --------------------------------------------------------------------------- # + +def test_stored_placeholder_answers_are_reused_on_refresh(client): + sid = make_scenario(client, title="Keep", memory="Your name is ${Hero}.", + cards=[{"name": "Sword", "keys": "sword", + "entry": "${Hero}'s blade."}]) + adv_id = start(client, sid, {"Hero": "Wren"}) + assert get_adventure(client, adv_id)["memory"] == "Your name is Wren." + + assert preview(client, adv_id)["placeholders_needed"] == [] + edit_scenario(sid, memory="${Hero}, you are late.") + adventure = refresh(client, adv_id) + # The answer is reused rather than re-injecting a literal ${Hero}. + assert adventure["memory"] == "Wren, you are late." + assert card_named(adventure, "Sword")["entry"] == "Wren's blade." + + +def test_missing_placeholder_answers_are_requested_then_remembered(client): + sid = make_scenario(client, title="Keep", memory="Your name is ${Hero}.") + adv_id = start(client, sid, {"Hero": "Wren"}) + # An adventure predating the stored-answers column. + db = SessionLocal() + try: + db.get(models.Adventure, adv_id).placeholders = None + db.commit() + finally: + db.close() + + assert preview(client, adv_id)["placeholders_needed"] == ["Hero"] + adventure = refresh(client, adv_id, {"Hero": "Wren"}) + assert adventure["memory"] == "Your name is Wren." + # Saved, so the player is only asked once. + assert preview(client, adv_id)["placeholders_needed"] == [] + + +# --------------------------------------------------------------------------- # +# Availability +# --------------------------------------------------------------------------- # + +def test_refresh_is_unavailable_once_the_scenario_is_gone(client): + sid = make_scenario(client, title="Keep", memory="Old") + adv_id = start(client, sid) + db = SessionLocal() + try: + db.delete(db.get(models.Scenario, sid)) + db.commit() + finally: + db.close() + + assert client.get(f"/api/adventures/{adv_id}/refresh").status_code == 404 + assert client.post(f"/api/adventures/{adv_id}/refresh", json={}).status_code == 404 + + +def test_refresh_is_rejected_while_a_turn_is_generating(client): + from app.routers import adventures + + sid = make_scenario(client, title="Keep", memory="Old") + adv_id = start(client, sid) + adventures.turns._active_turns.add(adv_id) + try: + r = client.post(f"/api/adventures/{adv_id}/refresh", json={}) + assert r.status_code == 409 + finally: + adventures.turns._active_turns.discard(adv_id) diff --git a/backend/tests/test_seed_sweep.py b/backend/tests/test_seed_sweep.py new file mode 100644 index 0000000..0ff965d --- /dev/null +++ b/backend/tests/test_seed_sweep.py @@ -0,0 +1,155 @@ +"""Seeded demo scenarios: renames land in place, and orphans are removed. + +A seed file is matched to its scenario by title, so renaming one used to insert +a second scenario and leave the first public forever. Both halves of the fix are +here: `previous_titles` moves the rename onto the existing row, and the sweep +deletes seeded rows no file claims any more. + + python -m pytest tests/test_seed_sweep.py -v +""" +import json + + +import pytest + +from app import models, seed +from app.database import Base, SessionLocal, engine + + +@pytest.fixture() +def db(): + Base.metadata.create_all(bind=engine) + session = SessionLocal() + try: + yield session + finally: + session.close() + Base.metadata.drop_all(bind=engine) + + +@pytest.fixture() +def seed_dir(tmp_path, monkeypatch): + monkeypatch.setattr(seed, "SEED_DIR", tmp_path) + return tmp_path + + +def write_seed(seed_dir, name: str, **fields) -> None: + data = {"title": fields.pop("title"), "description": "d", "prompt": "p"} + data.update(fields) + (seed_dir / name).write_text(json.dumps(data), encoding="utf-8") + + +def titles(db) -> set[str]: + return {s.title for s in db.query(models.Scenario).all()} + + +def test_a_seed_is_inserted_once(db, seed_dir): + write_seed(seed_dir, "a.json", title="Alpha") + seed.seed_public_scenarios(engine) + seed.seed_public_scenarios(engine) + assert titles(db) == {"Alpha"} + + +def test_a_rename_moves_the_existing_row(db, seed_dir): + """The whole point: one row, one id, and no orphan left behind.""" + write_seed(seed_dir, "a.json", title="Alpha") + seed.seed_public_scenarios(engine) + original = db.query(models.Scenario).one().id + + write_seed(seed_dir, "a.json", title="Alpha Prime", previous_titles=["Alpha"]) + seed.seed_public_scenarios(engine) + db.expire_all() + row = db.query(models.Scenario).one() + assert (row.id, row.title) == (original, "Alpha Prime") + + +def test_a_seeded_row_no_file_claims_is_deleted(db, seed_dir): + """The stale demo this sweep exists for, reproduced.""" + write_seed(seed_dir, "a.json", title="Alpha") + write_seed(seed_dir, "b.json", title="Beta") + seed.seed_public_scenarios(engine) + assert titles(db) == {"Alpha", "Beta"} + + (seed_dir / "b.json").unlink() + seed.seed_public_scenarios(engine) + db.expire_all() + assert titles(db) == {"Alpha"} + + +def test_a_previous_title_still_counts_as_claimed(db, seed_dir): + """A rename runs the sweep in the same pass, and must not eat its own row.""" + write_seed(seed_dir, "a.json", title="Alpha") + seed.seed_public_scenarios(engine) + write_seed(seed_dir, "a.json", title="Alpha Prime", previous_titles=["Alpha"]) + seed.seed_public_scenarios(engine) + db.expire_all() + assert titles(db) == {"Alpha Prime"} + + +def test_a_players_own_scenario_is_never_touched(db, seed_dir): + """Only a NULL owner and `is_public` make a row seeded. A player's is + neither, so nothing anybody created is reachable from the sweep.""" + user = models.User(is_guest=True) + db.add(user) + db.flush() + db.add(models.Scenario(user_id=user.id, is_public=True, title="Mine")) + db.add(models.Scenario(user_id=user.id, is_public=False, title="Also mine")) + db.commit() + + write_seed(seed_dir, "a.json", title="Alpha") + seed.seed_public_scenarios(engine) + db.expire_all() + assert titles(db) == {"Alpha", "Mine", "Also mine"} + + +def test_an_unreadable_seed_file_stops_the_sweep(db, seed_dir): + """A syntax error is not an instruction to delete live demo content. + + A file that will not parse claims no title, so sweeping on that pass would + remove the scenario it describes, and the next deploy would put it back. + """ + write_seed(seed_dir, "a.json", title="Alpha") + write_seed(seed_dir, "b.json", title="Beta") + seed.seed_public_scenarios(engine) + + (seed_dir / "b.json").write_text("{ not json", encoding="utf-8") + seed.seed_public_scenarios(engine) + db.expire_all() + assert titles(db) == {"Alpha", "Beta"} + + +def test_an_adventure_outlives_the_demo_it_started_from(db, seed_dir): + """`adventures.scenario_id` is ON DELETE SET NULL, so the story survives and + loses only the cover art it inherited.""" + write_seed(seed_dir, "a.json", title="Alpha") + write_seed(seed_dir, "keep.json", title="Kept") + seed.seed_public_scenarios(engine) + scenario = db.query(models.Scenario).filter_by(title="Alpha").one() + + user = models.User(is_guest=True) + db.add(user) + db.flush() + adventure = models.Adventure(user_id=user.id, scenario_id=scenario.id, title="Mine") + db.add(adventure) + db.commit() + adventure_id = adventure.id + + (seed_dir / "a.json").unlink() + seed.seed_public_scenarios(engine) + db.expire_all() + assert titles(db) == {"Kept"} + survivor = db.get(models.Adventure, adventure_id) + assert survivor is not None + assert survivor.scenario_id is None + + +def test_an_empty_seed_directory_deletes_nothing(db, seed_dir): + """No files at all reads as a packaging failure, not as a request to remove + every demo. The seeder returns before the sweep.""" + write_seed(seed_dir, "a.json", title="Alpha") + seed.seed_public_scenarios(engine) + + (seed_dir / "a.json").unlink() + seed.seed_public_scenarios(engine) + db.expire_all() + assert titles(db) == {"Alpha"} diff --git a/backend/tests/test_snapshot_compression.py b/backend/tests/test_snapshot_compression.py new file mode 100644 index 0000000..92ecd7f --- /dev/null +++ b/backend/tests/test_snapshot_compression.py @@ -0,0 +1,257 @@ +"""context_snapshot, stored compressed. + +The column makes up 89% of the database, and the free tier allows only +512 MB. Deferring the column already solved the read cost, so this is +about the storage ceiling. Postgres already TOASTs the column and gets +only a 1.7x ratio, because pglz favors fast decompression for data a query +might filter on. Nothing ever filters on an assembled prompt. + +Three things must hold, and only the first is obvious: + +* What goes in comes back out exactly, including a snapshot written before + the conversion and one that is NULL. +* The model still hands callers a dict, so no call site changes. +* Migration 44 drops the original column, so the backfill is the one + destructive step in this file. It must convert every row or abort. + + python -m pytest tests/test_snapshot_compression.py -v +""" +import json +import random +import zlib + + +import pytest +from sqlalchemy import text + +from app import compression, migrations, models +from app.database import Base, SessionLocal, engine +from tests import schema_rewind +from tools.fakeprose import prose + + +def snapshot(seed: int, nbytes: int = 60_000) -> dict: + rng = random.Random(seed) + return { + "system": prose(rng, nbytes // 3), + "story": prose(rng, nbytes - nbytes // 3), + "world_state": {"delta": {"player.hp": -3}}, + } + + +@pytest.fixture() +def db(): + Base.metadata.create_all(bind=engine) + session = SessionLocal() + try: + yield session + finally: + session.close() + Base.metadata.drop_all(bind=engine) + + +@pytest.fixture() +def adventure(db): + user = models.User(is_guest=False, email="snap@example.com") + db.add(user) + db.flush() + adv = models.Adventure(user_id=user.id, title="Cave", script_state={}) + db.add(adv) + db.commit() + return adv + + +# ------------------------------------------------------------------- packing + +def test_pack_round_trips_exactly(): + value = snapshot(1) + assert compression.unpack(compression.pack(value)) == value + + +def test_pack_handles_the_awkward_values(): + for value in ({}, {"a": None}, {"nested": {"deep": [1, 2, {"x": "é"}]}}): + assert compression.unpack(compression.pack(value)) == value + + +def test_pack_actually_shrinks_prose(): + """The whole justification. If this ratio collapses, the migration is + spending CPU for nothing.""" + value = snapshot(2, 200_000) + raw = json.dumps(value, separators=(",", ":")).encode() + packed = compression.pack(value) + assert len(packed) < len(raw) / 3, ( + f"{len(raw):,} B compressed to {len(packed):,} B — under 3x" + ) + + +def test_unpack_rejects_nothing_it_wrote(): + packed = compression.pack({"a": "b"}) + assert compression.unpack(bytearray(packed)) == {"a": "b"} + assert compression.unpack(memoryview(bytes(packed))) == {"a": "b"} + + +# --------------------------------------------------------------- the column + +def test_the_column_stores_bytes_and_returns_a_dict(db, adventure): + value = snapshot(3) + action = models.Action( + adventure_id=adventure.id, type="ai", text="t", + context_snapshot=value, + ) + db.add(action) + db.commit() + db.expire_all() + + assert action.context_snapshot == value + + stored = db.execute( + text("SELECT context_snapshot FROM actions WHERE id = :id"), + {"id": action.id}, + ).scalar() + assert isinstance(stored, (bytes, bytearray, memoryview)) + assert json.loads(zlib.decompress(bytes(stored))) == value + + +def test_the_column_is_smaller_than_the_json_it_holds(db, adventure): + value = snapshot(4, 200_000) + action = models.Action( + adventure_id=adventure.id, type="ai", text="t", + context_snapshot=value, + ) + db.add(action) + db.commit() + + stored = db.execute( + text("SELECT length(context_snapshot) FROM actions WHERE id = :id"), + {"id": action.id}, + ).scalar() + assert stored < len(json.dumps(value)) / 3 + + +def test_null_stays_null(db, adventure): + action = models.Action( + adventure_id=adventure.id, type="do", text="t", + context_snapshot=None, + ) + db.add(action) + db.commit() + db.expire_all() + assert action.context_snapshot is None + + +def test_an_unreadable_snapshot_reads_as_none_rather_than_raising(db, adventure): + """One corrupt row must not return a 500 error for the turn that loads + it. The snapshot is a debugging view. The story is what actually + matters.""" + action = models.Action( + adventure_id=adventure.id, type="ai", text="t", + context_snapshot={"a": "b"}, + ) + db.add(action) + db.commit() + db.execute( + text("UPDATE actions SET context_snapshot = :junk WHERE id = :id"), + {"junk": b"not zlib at all", "id": action.id}, + ) + db.commit() + db.expire_all() + + assert db.get(models.Action, action.id).context_snapshot is None + + +# ------------------------------------------------------- the upgrade in full + +def as_json_column(db, rows: dict[int, dict]) -> None: + """The pre-43 schema: context_snapshot as a JSON column, populated.""" + db.execute(text("ALTER TABLE actions DROP COLUMN context_snapshot")) + db.execute(text("ALTER TABLE actions ADD COLUMN context_snapshot JSON")) + for action_id, value in rows.items(): + db.execute( + text("UPDATE actions SET context_snapshot = :v WHERE id = :id"), + {"v": json.dumps(value), "id": action_id}, + ) + db.commit() + + +def seed_pre_43(db, adventure, count: int = 4) -> dict[int, dict]: + ids = [] + for i in range(count): + action = models.Action( + adventure_id=adventure.id, type="ai", text=f"t{i}" + ) + db.add(action) + db.flush() + ids.append(action.id) + # One action with no snapshot at all, which must survive as NULL. + plain = models.Action( + adventure_id=adventure.id, type="do", text="look" + ) + db.add(plain) + db.commit() + + expected = {action_id: snapshot(action_id) for action_id in ids} + as_json_column(db, expected) + db.commit() + # Take the later migrations' columns back off too, not just the stamp: + # replaying 43-45 also replays everything appended after them. + schema_rewind.rewind_to(engine, migrations.SNAPSHOT_COMPRESS_VERSION - 1) + return expected + + +def test_bootstrap_converts_every_snapshot(db, adventure): + expected = seed_pre_43(db, adventure) + db.close() + + migrations.bootstrap(engine) + + check = SessionLocal() + try: + for action_id, value in expected.items(): + assert check.get(models.Action, action_id).context_snapshot == value + assert check.execute( + text("SELECT count(*) FROM actions WHERE context_snapshot IS NULL") + ).scalar() == 1 + finally: + check.close() + + +def test_bootstrap_leaves_the_column_named_context_snapshot(db, adventure): + seed_pre_43(db, adventure) + db.close() + + migrations.bootstrap(engine) + + with engine.begin() as conn: + columns = {row[1] for row in conn.execute(text("PRAGMA table_info(actions)"))} + assert "context_snapshot" in columns + assert "context_snapshot_z" not in columns, "the swap left the scratch column behind" + + +def test_the_backfill_aborts_rather_than_dropping_unconvertible_data( + db, adventure, monkeypatch +): + """Migration 44 destroys the original column. If anything fails to + convert, the whole run must roll back with the column still there. + Otherwise a bug in this file could lose someone's prompts.""" + seed_pre_43(db, adventure) + db.close() + + def broken_unpack(_blob): + return {"not": "what went in"} + + monkeypatch.setattr(compression, "unpack", broken_unpack) + + with pytest.raises(RuntimeError, match="round trip"): + migrations.bootstrap(engine) + + with engine.begin() as conn: + columns = {row[1] for row in conn.execute(text("PRAGMA table_info(actions)"))} + version = conn.execute(text("PRAGMA user_version")).scalar() + surviving = conn.execute( + text("SELECT count(*) FROM actions WHERE context_snapshot IS NOT NULL") + ).scalar() + + assert "context_snapshot" in columns + assert version < migrations.SNAPSHOT_COMPRESS_VERSION, \ + "the version advanced past a backfill that failed" + assert surviving == 4, "the prompts did not survive the rollback" diff --git a/backend/tests/test_starter_adventure.py b/backend/tests/test_starter_adventure.py new file mode 100644 index 0000000..3ca393e --- /dev/null +++ b/backend/tests/test_starter_adventure.py @@ -0,0 +1,152 @@ +"""The pre-played adventure every new guest is given: app/starter.py. + +The starter is a shipped export bundle, so the two things that can break it are +the file and the import path. A bundle that no longer plans, or a story trimmed +to a state a player cannot continue from, both leave the guest worse off than an +empty account would. + + python -m pytest tests/test_starter_adventure.py -v +""" +import json + + +import pytest +from sqlalchemy import create_engine, event +from sqlalchemy.orm import sessionmaker + +from app import bundle, models, seed, starter +from app.database import Base +from app.migrations import bootstrap + + +@pytest.fixture() +def db(tmp_path): + engine = create_engine(f"sqlite:///{tmp_path/'t.db'}", + connect_args={"check_same_thread": False}) + + @event.listens_for(engine, "connect") + def _fk(dbapi_connection, _record): + cur = dbapi_connection.cursor() + cur.execute("PRAGMA foreign_keys=ON") + cur.close() + + bootstrap(engine) + Base.metadata.create_all(bind=engine) + session = sessionmaker(bind=engine, autoflush=False, expire_on_commit=False)() + yield session + session.close() + + +@pytest.fixture() +def guest(db): + user = models.User(is_guest=True) + db.add(user) + db.commit() + return user + + +def payload() -> dict: + return json.loads(starter.STARTER_FILE.read_text(encoding="utf-8")) + + +def test_the_shipped_file_is_a_bundle_this_build_can_import(): + """The file is written by an export, so a format change can strand it.""" + data = payload() + version = bundle.check_format(data) + assert version == bundle.FORMAT + story = bundle.plan(data, version) + assert story["nodes"] + + +def test_a_guest_gets_the_adventure_and_owns_it(db, guest): + adventure = starter.give(db, guest) + db.commit() + assert adventure is not None + assert adventure.user_id == guest.id + assert "Pokemon" in adventure.title + assert len(adventure.actions) == len(payload()["actions"]) + + +def test_the_story_cards_come_with_it(db, guest): + """A copy without the cards would drop out of character on the next turn.""" + adventure = starter.give(db, guest) + db.commit() + assert {c.name for c in adventure.story_cards} == { + c["name"] for c in payload()["storyCards"] + } + + +def test_the_turns_carry_what_the_engine_recorded(db, guest): + """The point of shipping a played story: the summaries have chips in them. + + An applied change, a refused one, and a milestone all appear in the first + two exchanges, which is what a visitor sees before spending a demo turn. + """ + adventure = starter.give(db, guest) + db.commit() + chips = [chip for action in adventure.actions for chip in action.world_changes] + kinds = {chip["kind"] for chip in chips} + assert "stat" in kinds + assert "rejected" in kinds + assert "milestone" in kinds + + +def test_the_state_left_behind_can_be_played_from(db, guest): + """The opponent has to be on the field with HP, or every hit is refused. + + A Pokemon sent in at 0 HP sits at the floor of `active_hp`, so the engine + refuses each later change and the story stops moving. That state reached a + playtest once, and it must not be what a guest inherits. + """ + adventure = starter.give(db, guest) + db.commit() + milo = adventure.world_state["npc"]["milo"] + assert milo["active_hp"] > 0 + assert milo["active_pokemon"] + assert adventure.actions[-1].world_state_after["npc"]["milo"] == milo + + +def test_a_missing_file_costs_the_guest_nothing(db, guest, monkeypatch): + """A packaging mistake must not stop an account from being created.""" + monkeypatch.setattr(starter, "STARTER_FILE", starter.STARTER_FILE.with_name("gone.json")) + assert starter.give(db, guest) is None + + +def test_a_broken_bundle_leaves_no_half_written_adventure(db, guest, monkeypatch): + """The savepoint: a failure partway through discards the rows it wrote.""" + monkeypatch.setattr(bundle, "materialize", + lambda *a, **k: (_ for _ in ()).throw(RuntimeError("boom"))) + assert starter.give(db, guest) is None + db.commit() + assert db.query(models.Adventure).count() == 0 + + +def test_the_copy_inherits_the_demo_scenario_art(db, guest): + """An adventure has no cover art of its own; it inherits the scenario's. + + A bundle carries no scenario id, so the link is made by title. Without it + the starter card shows a monogram while the demo it came from shows its + artwork. + """ + title = payload()["scenarioTitle"] + data = json.loads((seed.SEED_DIR / "05-league-championship.json").read_text(encoding="utf-8")) + # The demo the copy came from, as the seeder would have written it. The + # seeder itself opens its own session against the app's engine, so it + # cannot be pointed at this fixture's database. + assert data["title"] == title, "the starter names a scenario no seed file ships" + db.add(models.Scenario(user_id=None, is_public=True, title=title, image=data["image"])) + db.commit() + + adventure = starter.give(db, guest) + db.commit() + scenario = seed.find_seeded(db, title) + assert scenario is not None and scenario.image + assert adventure.scenario_id == scenario.id + + +def test_a_missing_demo_scenario_only_costs_the_art(db, guest): + """Nothing seeds the scenarios in this fixture, so the link finds nothing.""" + adventure = starter.give(db, guest) + db.commit() + assert adventure is not None + assert adventure.scenario_id is None diff --git a/backend/tests/test_state_revert.py b/backend/tests/test_state_revert.py new file mode 100644 index 0000000..7a2f45e --- /dev/null +++ b/backend/tests/test_state_revert.py @@ -0,0 +1,241 @@ +"""Tests for undo and retry rolling back the shared `script_state` +(plan/11-state-revert-and-retry-fix.md). + +Phase 14 SP4 reversed the snapshots. An action used to carry the state as +it stood before it ran, and rolling back read the snapshot off the action +being removed. Now it carries the state it left behind, and rolling back +reads that state off the node in front of it. This is the same value +reached from the other direction, and it is the only version a retry can +use, because attempts at one turn share a starting position and differ +only in their outcome. + +Run from the backend dir: python -m pytest tests/test_state_revert.py -v +""" + +# Point the app at a throwaway SQLite file before importing anything that +# binds the engine at import time. `app.database` reads `AIDND_DB_PATH` +# on import. + +import pytest +from fastapi import HTTPException + +from app import attempts, memorybank, models +from app.database import Base, SessionLocal, engine +from app.routers import adventures + + +@pytest.fixture() +def db(): + Base.metadata.create_all(bind=engine) + session = SessionLocal() + try: + yield session + finally: + session.close() + Base.metadata.drop_all(bind=engine) + adventures.turns._active_turns.clear() + + +def _make_adventure(db, script_state): + user = models.User(is_guest=False) + db.add(user) + db.flush() + adv = models.Adventure(user_id=user.id, title="T", script_state=script_state) + db.add(adv) + db.flush() + return user, adv + + +def _add(db, adv, index, type_, text="x", state_after=None): + a = models.Action( + adventure_id=adv.id, type=type_, text=text, + state_after=state_after, + ) + db.add(a) + db.flush() + return a + + +def _forget_snapshots(db, adv): + """Blank every outcome, the way a row written before SP4 looks. + + Straight SQL, because `tree.stamp_outcome` runs on every flush precisely so + that a node written through the ORM cannot end up without one. + """ + db.query(models.Action).filter_by(adventure_id=adv.id).update( + {"state_after": None, "world_state_after": None}, synchronize_session=False + ) + db.commit() + db.expire_all() + + +# ---------------------------------------------------------------- undo + +def test_undo_reverts_state_to_before_the_turn(db): + # A turn moved script_state from {gold:0} to {gold:10}. The node in + # front of the turn records where it started. The current state is + # the mutated one. + user, adv = _make_adventure(db, {"gold": 10}) + _add(db, adv, 0, "start", state_after={"gold": 0}) + _add(db, adv, 1, "do", state_after={"gold": 0}) + _add(db, adv, 2, "ai", state_after={"gold": 10}) + db.commit() + + adventures.undo_turn(adv.id, db=db, adventure=adv) + + assert adv.script_state == {"gold": 0} + assert [a.type for a in adv.actions] == ["start"] + + +def test_undo_of_bare_continue_uses_the_node_in_front(db): + # A "continue" turn has no player action, so the opening is what the story + # falls back to. + user, adv = _make_adventure(db, {"gold": 5}) + _add(db, adv, 0, "start", state_after={"gold": 0}) + _add(db, adv, 1, "ai", state_after={"gold": 5}) + db.commit() + + adventures.undo_turn(adv.id, db=db, adventure=adv) + + assert adv.script_state == {"gold": 0} + assert [a.type for a in adv.actions] == ["start"] + + +def test_undo_leaves_state_untouched_when_snapshot_missing(db): + # A row the SP4 migration could not derive an outcome for: leave the live + # state alone rather than resetting it to nothing. + user, adv = _make_adventure(db, {"gold": 10}) + _add(db, adv, 0, "start") + _add(db, adv, 1, "do") + _add(db, adv, 2, "ai") + _forget_snapshots(db, adv) + + adventures.undo_turn(adv.id, db=db, adventure=adv) + + assert adv.script_state == {"gold": 10} + + +def test_undo_raises_when_nothing_to_undo(db): + user, adv = _make_adventure(db, {}) + _add(db, adv, 0, "start") + db.commit() + with pytest.raises(HTTPException) as exc: + adventures.undo_turn(adv.id, db=db, adventure=adv) + assert exc.value.status_code == 400 + + +def test_undo_blocked_by_active_turn_lock(db): + user, adv = _make_adventure(db, {}) + _add(db, adv, 0, "start") + _add(db, adv, 1, "ai", state_after={}) + db.commit() + + adventures.turns.acquire_turn_lock(adv.id) # a turn is "generating" + try: + with pytest.raises(HTTPException) as exc: + adventures.undo_turn(adv.id, db=db, adventure=adv) + assert exc.value.status_code == 409 + # The failed undo must not have released someone else's lock. + assert adv.id in adventures.turns._active_turns + finally: + adventures.turns._active_turns.discard(adv.id) + + +def test_undo_prunes_memory_covering_removed_actions(db): + user, adv = _make_adventure(db, {}) + for i in range(4): + _add(db, adv, i, "ai" if i % 2 else "do", state_after={}) + # A memory summarizing actions up to index 3, which undo will delete. + covering = models.Memory(adventure_id=adv.id, text="m", source_start=0, source_end=3) + keep = models.Memory(adventure_id=adv.id, text="k", source_start=0, source_end=1) + db.add_all([covering, keep]) + db.commit() + + adventures.undo_turn(adv.id, db=db, adventure=adv) # removes indexes 2 & 3 + + texts = {m.text for m in adv.memories} + assert texts == {"k"} + + +# -------------------------------------------------------- withdrawing a node + +def test_forget_node_withdraws_only_what_that_node_produced(db): + """Phase 14 SP3: a memory attaches to the node where its block ends, so + removing a node is a lookup rather than a scan for memories that + reference actions the story no longer has.""" + user, adv = _make_adventure(db, {}) + _add(db, adv, 0, "do") + second = _add(db, adv, 1, "ai") + db.add_all([ + models.Memory(adventure_id=adv.id, text="hangs off node 1", + source_start=0, source_end=1), + models.Memory(adventure_id=adv.id, text="hangs off node 0", + source_start=0, source_end=0), + ]) + db.commit() + + removed = memorybank.forget_node(db, adv, second) + db.commit() + db.refresh(adv) # expire_on_commit=False: reload the memories collection + + assert removed == 1 + assert {m.text for m in adv.memories} == {"hangs off node 0"} + + +# ---------------------------------------------------------------- snapshot + +def test_snapshot_outcome_is_an_independent_deep_copy(db): + _, adv = _make_adventure(db, {"nested": {"n": 1}}) + node = models.Action(adventure_id=adv.id, type="ai", text="x") + attempts.snapshot_outcome(adv, node) + adv.script_state["nested"]["n"] = 99 + assert node.state_after == {"nested": {"n": 1}} # unaffected by later mutation + + +def test_snapshot_outcome_handles_non_dict(db): + _, adv = _make_adventure(db, {}) + adv.script_state = None + node = models.Action(adventure_id=adv.id, type="ai", text="x") + attempts.snapshot_outcome(adv, node) + assert node.state_after == {} + + +def test_restore_state_ignores_a_node_with_no_outcome(db): + _, adv = _make_adventure(db, {"gold": 7}) + attempts.restore_state(adv, models.Action(adventure_id=adv.id, type="ai")) + assert adv.script_state == {"gold": 7} + attempts.restore_state(adv, None) + assert adv.script_state == {"gold": 7} + + +# ---------------------------------------------------------------- retry + +def test_retry_restores_the_state_the_turn_started_from(db, monkeypatch): + # Retry must roll script_state back to what the node in front of the AI + # action left behind, so regeneration does not stack output mutations + # on top of the attempt being replaced. + user, adv = _make_adventure(db, {"gold": 20}) # 20 = double-applied bug value + _add(db, adv, 0, "start", state_after={"gold": 0}) + _add(db, adv, 1, "do", state_after={"gold": 10}) + _add(db, adv, 2, "ai", state_after={"gold": 20}) + db.commit() + + monkeypatch.setattr(adventures.limits, "rate_limit", lambda *a, **k: None) + monkeypatch.setattr(adventures.turns, "check_demo_cap", lambda *a, **k: None) + + async def _noop(*a, **k): + if False: + yield # make it an async generator + monkeypatch.setattr(adventures.turns, "generate_turn", _noop) + + adventures.retry_action(adv.id, request=None, db=db, user=user, adventure=adv) + + assert adv.script_state == {"gold": 10} + # Nothing is written until a replacement actually arrives: the attempt on + # screen is left exactly as it was, and stays the live one. + assert [a.type for a in adv.actions] == ["start", "do", "ai"] + last = adv.actions[-1] + assert last.live is True + assert len(attempts.group(db, last)) == 1 # no sibling was filed + assert last.state_after == {"gold": 20} # its own outcome, untouched + adventures.turns._active_turns.discard(adv.id) diff --git a/backend/tests/test_story_tree_baseline.py b/backend/tests/test_story_tree_baseline.py new file mode 100644 index 0000000..0f2b699 --- /dev/null +++ b/backend/tests/test_story_tree_baseline.py @@ -0,0 +1,481 @@ +"""Phase 14 SP0: the regression contract for the story tree. + +This file exists to answer one question repeatedly, as the storage model is +replaced underneath the app: do existing adventures still behave exactly as +they did? + +It drives the product the way a player does, over HTTP, asserting only on +API responses. It never reaches into the ORM to check how something is +stored. Everything it asserts is true of the linear implementation today and +must stay true of the tree, because a linear story is a tree with one branch. + + python -m pytest tests/test_story_tree_baseline.py -v + +This file must pass unmodified through SP1 (schema), SP2 (branch clause), +and SP3 (memories on nodes). If a change here looks necessary in one of +those subphases, the change is wrong, not the test. SP4 is the first +subphase allowed to move it, and only for the variant-count semantics +called out in plan/14. +""" +import pytest +from fastapi import Depends +from fastapi.testclient import TestClient + +from app import auth, limits, models +from app.database import Base, SessionLocal, engine, get_db +from app.main import app +from app.routers import adventures + +from fakes import ScriptedProvider + +# A world-state schema, so the RPG layer is exercised rather than skipped. +SCHEMA = {"player": {"hp": {"min": 0, "max": 100, "initial": 100}}} + +# Ten gold a turn. A turn that double-applies its hooks, or one that fails to +# roll back on retry, shows up here as a wrong total rather than as nothing. +GOLD_SCRIPT = """ +const modifier = (text) => { + state.gold = (state.gold || 0) + 10; + return { text }; +}; +modifier(text); +""" + +OPENING = "You enter a cave." + + +def _make_world(monkeypatch, *, seeded_actions: int = 0): + """Create a user, a scenario, and an adventure, with `seeded_actions` + extra story actions written straight to the database. Paging tests need + more turns than it is practical to play one at a time.""" + Base.metadata.create_all(bind=engine) + setup = SessionLocal() + user = models.User(is_guest=False, email="baseline@example.com") + setup.add(user) + setup.flush() + setup.add(models.Settings(user_id=user.id, api_key="enc:dummy", model="test-model")) + scenario = models.Scenario(user_id=user.id, title="S", stat_schema=SCHEMA) + setup.add(scenario) + setup.flush() + adv = models.Adventure( + user_id=user.id, title="Cave", scenario_id=scenario.id, + script_state={}, world_state={"player": {"hp": 100}}, + ) + setup.add(adv) + setup.flush() + setup.add(models.Action(adventure_id=adv.id, type="start", text=OPENING)) + for i in range(seeded_actions): + setup.add(models.Action( + adventure_id=adv.id, + type="ai" if i % 2 else "do", + text=f"Seeded turn {i}.", + )) + setup.add(models.AdventureScript( + adventure_id=adv.id, position=0, enabled=True, name="Gold", output_js=GOLD_SCRIPT, + )) + setup.commit() + adv_id, user_id = adv.id, user.id + setup.close() + + ScriptedProvider.replies = ["A reply."] + ScriptedProvider.calls = 0 + ScriptedProvider.prompts = [] + monkeypatch.setattr(adventures.turns, "OpenAICompatibleProvider", ScriptedProvider) + monkeypatch.setattr(auth, "resolve_provider_config", lambda s: auth.ProviderConfig( + "http://fake", "k", "test-model", False)) + monkeypatch.setattr(limits, "rate_limit", lambda *a, **k: None) + monkeypatch.setattr(limits, "check_row_cap", lambda *a, **k: None) + + def _current_user(db=Depends(get_db)): + return db.get(models.User, user_id) + + app.dependency_overrides[auth.get_current_user] = _current_user + c = TestClient(app) + c.adv_id = adv_id + return c + + +@pytest.fixture() +def client(monkeypatch): + c = _make_world(monkeypatch) + try: + yield c + finally: + app.dependency_overrides.clear() + adventures.turns._active_turns.clear() + Base.metadata.drop_all(bind=engine) + + +@pytest.fixture() +def long_client(monkeypatch): + """An adventure longer than one window, so paging is real.""" + c = _make_world(monkeypatch, seeded_actions=adventures.ACTION_PAGE + 10) + try: + yield c + finally: + app.dependency_overrides.clear() + adventures.turns._active_turns.clear() + Base.metadata.drop_all(bind=engine) + + +# ------------------------------------------------------------------ helpers + +def _open(client): + r = client.get(f"/api/adventures/{client.adv_id}") + assert r.status_code == 200, r.text + return r.json() + + +def _actions(client): + return _open(client)["actions"] + + +def _texts(client): + return [a["text"] for a in _actions(client)] + + +def _play(client, text="look around", type="do"): + r = client.post(f"/api/adventures/{client.adv_id}/actions", + json={"type": type, "text": text}) + assert r.status_code == 200, r.text + return r + + +def _state(adv_id): + db = SessionLocal() + try: + adv = db.get(models.Adventure, adv_id) + return adv.script_state, adv.world_state + finally: + db.close() + + +# ------------------------------------------------------- opening the story + +def test_adventure_opens_on_a_window_with_a_total(long_client): + """The payload includes the newest window and the whole story's length. + That is how the client knows more actions exist above it.""" + payload = _open(long_client) + seeded = adventures.ACTION_PAGE + 11 # + the start action + assert payload["action_count"] == seeded + assert len(payload["actions"]) == adventures.ACTION_PAGE + # The newest window, so it ends at the newest action. + assert payload["actions"][-1]["text"] == f"Seeded turn {seeded - 2}." + + +def test_short_adventure_returns_everything(client): + payload = _open(client) + assert payload["action_count"] == 1 + assert [a["text"] for a in payload["actions"]] == [OPENING] + + +# --------------------------------------------------------------- the turn + +def test_a_turn_appends_the_player_action_then_the_ai_action(client): + ScriptedProvider.replies = ["The dark presses in."] + _play(client, "light a torch") + actions = _actions(client) + assert [a["type"] for a in actions] == ["start", "do", "ai"] + assert actions[1]["text"] == "> You light a torch." + assert actions[2]["text"] == "The dark presses in." + + +def test_say_and_story_and_continue_all_work(client): + ScriptedProvider.replies = ["One.", "Two.", "Three."] + _play(client, "hello", type="say") + _play(client, "The wind rises.", type="story") + _play(client, "", type="continue") + types = [a["type"] for a in _actions(client)] + assert types == ["start", "say", "ai", "story", "ai", "ai"] + + +def test_the_story_so_far_is_replayed_into_the_prompt(client): + ScriptedProvider.replies = ["First.", "Second."] + _play(client, "go north") + _play(client, "go south") + story = ScriptedProvider.prompts[-1][1] + assert OPENING in story + assert "First." in story + assert "go north" in story + + +def test_scripts_run_once_per_turn(client): + """The gold script adds ten a turn. Two turns is twenty — not forty.""" + ScriptedProvider.replies = ["One.", "Two."] + _play(client) + _play(client) + script_state, _ = _state(client.adv_id) + assert script_state["gold"] == 20 + + +# ------------------------------------------------------------------ retry + +def test_retry_replaces_the_text_and_keeps_the_attempt(client): + ScriptedProvider.replies = ["Attempt one.", "Attempt two."] + _play(client) + assert _texts(client)[-1] == "Attempt one." + + r = client.post(f"/api/adventures/{client.adv_id}/retry") + assert r.status_code == 200, r.text + + actions = _actions(client) + # This turn still has one AI action. A retry creates a new take, not a new turn. + assert [a["type"] for a in actions] == ["start", "do", "ai"] + assert actions[-1]["text"] == "Attempt two." + + r = client.get( + f"/api/adventures/{client.adv_id}/actions/{actions[-1]['id']}/variants") + assert r.status_code == 200, r.text + assert [v["text"] for v in r.json()] == ["Attempt one.", "Attempt two."] + assert [v["active"] for v in r.json()] == [False, True] + + +def test_retry_does_not_stack_script_effects(client): + """The discarded attempt's ten gold is rolled back, so one turn plus one + retry is still ten, not twenty.""" + ScriptedProvider.replies = ["Attempt one.", "Attempt two."] + _play(client) + client.post(f"/api/adventures/{client.adv_id}/retry") + script_state, _ = _state(client.adv_id) + assert script_state["gold"] == 10 + + +def test_switching_back_to_an_earlier_attempt_restores_it(client): + ScriptedProvider.replies = ["Attempt one.", "Attempt two."] + _play(client) + client.post(f"/api/adventures/{client.adv_id}/retry") + action_id = _actions(client)[-1]["id"] + + r = client.post( + f"/api/adventures/{client.adv_id}/actions/{action_id}/variant", + json={"index": 0}) + assert r.status_code == 200, r.text + assert _texts(client)[-1] == "Attempt one." + # The script state that attempt produced comes back with it. + script_state, _ = _state(client.adv_id) + assert script_state["gold"] == 10 + + +def test_only_the_newest_turn_can_be_switched(client): + """An older turn's alternatives stay readable but not selectable. The + story after it continues from what is live.""" + ScriptedProvider.replies = ["One.", "Again.", "Two."] + _play(client) + client.post(f"/api/adventures/{client.adv_id}/retry") + older_id = _actions(client)[-1]["id"] + _play(client) # the story moves on + + r = client.post( + f"/api/adventures/{client.adv_id}/actions/{older_id}/variant", + json={"index": 0}) + assert r.status_code == 400 + + +# ------------------------------------------------------------------- undo + +def test_undo_removes_the_whole_turn_and_rolls_state_back(client): + ScriptedProvider.replies = ["One.", "Two."] + _play(client, "go north") + _play(client, "go south") + assert len(_actions(client)) == 5 + + r = client.post(f"/api/adventures/{client.adv_id}/undo") + assert r.status_code == 200, r.text + page = r.json() + # Both the AI action and the player action that prompted it are gone. + assert [a["type"] for a in page["actions"]] == ["start", "do", "ai"] + assert page["total"] == 3 + # The second turn's ten gold is also gone. + script_state, _ = _state(client.adv_id) + assert script_state["gold"] == 10 + + +def test_undo_returns_a_window_not_the_whole_story(long_client): + before = _open(long_client)["action_count"] + r = long_client.post(f"/api/adventures/{long_client.adv_id}/undo") + assert r.status_code == 200, r.text + page = r.json() + assert len(page["actions"]) == adventures.ACTION_PAGE + # The seeded story ends on an `ai` preceded by a `do`, so a turn is two + # actions and undo takes both. + assert page["total"] == before - 2 + assert page["has_more"] is True + + +def test_the_opening_cannot_be_undone(client): + r = client.post(f"/api/adventures/{client.adv_id}/undo") + assert r.status_code == 400 + + +# ------------------------------------------------------------------ paging + +def test_paging_walks_back_to_the_start_without_gaps_or_repeats(long_client): + """The property that matters: paging all the way up must show every + action exactly once, in order. A tree must preserve this invariant. The + anchor is always an action, never a position.""" + payload = _open(long_client) + total = payload["action_count"] + seen = [a["id"] for a in payload["actions"]] + + has_more = len(seen) < total + while has_more: + r = long_client.get( + f"/api/adventures/{long_client.adv_id}/actions", + params={"before_id": seen[0]}) + assert r.status_code == 200, r.text + page = r.json() + ids = [a["id"] for a in page["actions"]] + assert ids, "a page claiming more must return some" + seen = ids + seen + has_more = page["has_more"] + + assert len(seen) == total + assert len(set(seen)) == total, "an action was served twice" + assert seen == sorted(seen), "pages came back out of order" + + +def test_a_page_is_bounded_by_the_requested_limit(long_client): + r = long_client.get(f"/api/adventures/{long_client.adv_id}/actions", + params={"limit": 5}) + assert r.status_code == 200, r.text + page = r.json() + assert len(page["actions"]) == 5 + assert page["has_more"] is True + assert page["total"] == adventures.ACTION_PAGE + 11 + + +def test_paging_past_the_start_reports_the_end(long_client): + r = long_client.get(f"/api/adventures/{long_client.adv_id}/actions", + params={"limit": 5}) + oldest = r.json()["actions"][0]["id"] + # Continue paging until it reaches the oldest action. + seen_all = False + cursor = oldest + for _ in range(100): + page = long_client.get( + f"/api/adventures/{long_client.adv_id}/actions", + params={"before_id": cursor, "limit": 20}).json() + if not page["has_more"]: + seen_all = True + break + cursor = page["actions"][0]["id"] + assert seen_all + + +# ------------------------------------------------------- editing and deleting + +def test_editing_an_action_sticks(client): + ScriptedProvider.replies = ["Original."] + _play(client) + action_id = _actions(client)[-1]["id"] + r = client.patch(f"/api/adventures/{client.adv_id}/actions/{action_id}", + json={"text": "Edited."}) + assert r.status_code == 200, r.text + # Re-read from the database instead of trusting the response. An edit + # that only appears in the response has already reverted for anyone who + # reloads. + assert _texts(client)[-1] == "Edited." + + +def test_editing_a_retried_action_survives_a_reload(client): + """The edit has to reach the live attempt too, or paging away and back + reverts it.""" + ScriptedProvider.replies = ["One.", "Two."] + _play(client) + client.post(f"/api/adventures/{client.adv_id}/retry") + action_id = _actions(client)[-1]["id"] + client.patch(f"/api/adventures/{client.adv_id}/actions/{action_id}", + json={"text": "Edited."}) + assert _texts(client)[-1] == "Edited." + + +def test_deleting_an_action_removes_it(client): + ScriptedProvider.replies = ["One."] + _play(client) + action_id = _actions(client)[-1]["id"] + r = client.delete(f"/api/adventures/{client.adv_id}/actions/{action_id}") + assert r.status_code == 204, r.text + assert len(_actions(client)) == 2 + + +# ---------------------------------------------------------------- memories + +def test_memories_are_created_listed_and_deleted(client): + r = client.post(f"/api/adventures/{client.adv_id}/memories", + json={"text": "The torch went out."}) + assert r.status_code == 201, r.text + memory_id = r.json()["id"] + + listed = client.get(f"/api/adventures/{client.adv_id}/memories").json() + assert [m["text"] for m in listed] == ["The torch went out."] + + r = client.patch( + f"/api/adventures/{client.adv_id}/memories/{memory_id}", + json={"pinned": True}) + assert r.status_code == 200, r.text + assert r.json()["pinned"] is True + + r = client.delete(f"/api/adventures/{client.adv_id}/memories/{memory_id}") + assert r.status_code == 204, r.text + assert client.get(f"/api/adventures/{client.adv_id}/memories").json() == [] + + +# ------------------------------------------------------------------ export + +def test_export_carries_the_whole_story(client): + """A bundle is a backup: it contains every action, not just the window. + + SP6 changed the format assertion below, as this note said it would. It + also changed one more line than the note allowed for: the `variants` + array in `test_export_keeps_retry_attempts`. That change reflects the + same fact: a bundle that stores coordinates has no use for a repeating + group. Everything else here still passes unmodified. + """ + ScriptedProvider.replies = ["One.", "Two."] + _play(client, "go north") + _play(client, "go south") + + r = client.get(f"/api/adventures/{client.adv_id}/export") + assert r.status_code == 200, r.text + bundle = r.json() + assert bundle["format"] == "ai-dnd-adventure-v2" + assert bundle["title"] == "Cave" + assert [a["text"] for a in bundle["actions"]] == [ + OPENING, "> You go north.", "One.", "> You go south.", "Two.", + ] + + +def test_export_round_trips_through_import(client): + ScriptedProvider.replies = ["One."] + _play(client, "go north") + bundle = client.get(f"/api/adventures/{client.adv_id}/export").json() + + r = client.post("/api/adventures/import", json=bundle) + assert r.status_code == 201, r.text + imported = r.json() + assert imported["id"] != client.adv_id + assert imported["title"] == "Cave" + assert [a["text"] for a in imported["actions"]] == [ + OPENING, "> You go north.", "One.", + ] + + +def test_export_keeps_retry_attempts(client): + ScriptedProvider.replies = ["Attempt one.", "Attempt two."] + _play(client) + client.post(f"/api/adventures/{client.adv_id}/retry") + + bundle = client.get(f"/api/adventures/{client.adv_id}/export").json() + ai = [a for a in bundle["actions"] if a["type"] == "ai"] + assert [a["text"] for a in ai] == ["Attempt one.", "Attempt two."] + assert [a["live"] for a in ai] == [False, True] + + +# -------------------------------------------------------------- world state + +def test_world_state_is_readable_and_survives_a_turn(client): + ScriptedProvider.replies = ["Nothing changes."] + _play(client) + r = client.get(f"/api/adventures/{client.adv_id}/world-state") + assert r.status_code == 200, r.text + assert r.json()["state"]["player"]["hp"] == 100 diff --git a/backend/tests/test_take_edit.py b/backend/tests/test_take_edit.py new file mode 100644 index 0000000..6dd505e --- /dev/null +++ b/backend/tests/test_take_edit.py @@ -0,0 +1,180 @@ +"""Phase 14 SP9: editing the take you are actually reading. + +The pager can display a turn on take 2 of 4. The transcript row for that +turn stays keyed by the live take, because that is the row the transcript +reports and the one the window carries. The take currently displayed is +known only to the client, by its own node id. + +So "edit this" has two ids to choose from, and the page shipped with the +wrong one. It opened the editor on the live take's text and saved over it, +even from a row that was displaying take 2. This is a client bug, and it +is fixed in Play.jsx. The fix depends on a guarantee only the server can +make: + + A take is an ordinary row to the edit endpoint, addressed by its own id, + whether or not it is the one the path runs through. + +The fix also depends on the group listing reporting this correctly +afterward. This file asserts both guarantees, so the client's fix cannot +be silently broken. + + python -m pytest tests/test_take_edit.py -v +""" +import pytest +from fastapi import Depends +from fastapi.testclient import TestClient + +from app import auth, limits, models +from app.database import Base, SessionLocal, engine, get_db +from app.main import app +from app.routers import adventures + +from fakes import ScriptedProvider + + +@pytest.fixture() +def client(monkeypatch): + Base.metadata.create_all(bind=engine) + setup = SessionLocal() + user = models.User(is_guest=False, email="takeedit@example.com") + setup.add(user) + setup.flush() + setup.add(models.Settings(user_id=user.id, api_key="enc:dummy", model="test-model")) + scenario = models.Scenario(user_id=user.id, title="S") + setup.add(scenario) + setup.flush() + adv = models.Adventure( + user_id=user.id, title="Vault", scenario_id=scenario.id, + script_state={}, world_state={}, + ) + setup.add(adv) + setup.flush() + setup.add(models.Action(adventure_id=adv.id, type="start", text="You begin.")) + setup.commit() + adv_id, user_id = adv.id, user.id + setup.close() + + ScriptedProvider.replies = [f"Take {n}." for n in range(1, 40)] + ScriptedProvider.calls = 0 + monkeypatch.setattr(adventures.turns, "OpenAICompatibleProvider", ScriptedProvider) + monkeypatch.setattr(auth, "resolve_provider_config", lambda s: auth.ProviderConfig( + "http://fake", "k", "test-model", False)) + monkeypatch.setattr(limits, "rate_limit", lambda *a, **k: None) + monkeypatch.setattr(limits, "check_row_cap", lambda *a, **k: None) + + def _current_user(db=Depends(get_db)): + return db.get(models.User, user_id) + + app.dependency_overrides[auth.get_current_user] = _current_user + c = TestClient(app) + c.adv_id = adv_id + try: + yield c + finally: + app.dependency_overrides.clear() + adventures.turns._active_turns.clear() + Base.metadata.drop_all(bind=engine) + + +def _play(client, text="look around"): + r = client.post(f"/api/adventures/{client.adv_id}/actions", + json={"type": "do", "text": text}) + assert r.status_code == 200, r.text + + +def _retry(client): + r = client.post(f"/api/adventures/{client.adv_id}/retry") + assert r.status_code == 200, r.text + + +def _takes(client, action_id): + r = client.get(f"/api/adventures/{client.adv_id}/actions/{action_id}/variants") + assert r.status_code == 200, r.text + return r.json() + + +def _edit(client, action_id, text): + return client.patch(f"/api/adventures/{client.adv_id}/actions/{action_id}", + json={"text": text}) + + +def _live_ai(client): + """The AI row the transcript currently reports as live.""" + adv = client.get(f"/api/adventures/{client.adv_id}").json() + return [a for a in adv["actions"] if a["type"] == "ai"][-1] + + +def _four_takes(client): + """One turn, played four times. Returns (row the page holds, take list).""" + _play(client) + for _ in range(3): + _retry(client) + row = _live_ai(client) + assert row["take_count"] == 4 + return row, _takes(client, row["id"]) + + +def test_the_pager_reads_four_distinct_takes(client): + """The premise: 2/4 and 4/4 are different rows with different words.""" + row, takes = _four_takes(client) + assert [t["text"] for t in takes] == ["Take 1.", "Take 2.", "Take 3.", "Take 4."] + assert takes[3]["id"] == row["id"], "the newest take is the one the story tells" + assert takes[1]["id"] != row["id"], "2/4 is not the row the transcript is keyed by" + + +def test_editing_a_take_that_is_not_live_edits_that_take(client): + """The bug, at the level the client's fix depends on. + + Saving against take 2's own id must land on take 2. The request must + not be refused for being off the path, and must not be redirected onto + the live row. + """ + row, takes = _four_takes(client) + second = takes[1] + + r = _edit(client, second["id"], "Take 2, rewritten.") + assert r.status_code == 200, r.text + assert r.json()["text"] == "Take 2, rewritten." + + after = _takes(client, row["id"]) + assert [t["text"] for t in after] == [ + "Take 1.", "Take 2, rewritten.", "Take 3.", "Take 4.", + ], "one take changed, and only the one addressed" + + +def test_editing_a_take_leaves_the_live_one_alone(client): + """What the page did instead: the reader saw 2/4 and 4/4 was overwritten.""" + row, takes = _four_takes(client) + + _edit(client, takes[1]["id"], "Take 2, rewritten.") + + assert _live_ai(client)["text"] == "Take 4.", "the story still tells what it told" + + +def test_a_take_edit_survives_paging_away_and_back(client): + """The listing is the pager's only source, so the edit must appear in it. + + The client caches this list per message, and the fix drops that cache + after an edit. If the server ever started answering from a stale copy + of its own, stepping away and back would show the text before the + edit, and this test would not catch it. That is why this test makes + the round trip. + """ + row, takes = _four_takes(client) + _edit(client, takes[1]["id"], "Take 2, rewritten.") + + # Addressed by the live row, as the pager does, and by the edited take + # itself, as a client holding that id would. + assert _takes(client, row["id"])[1]["text"] == "Take 2, rewritten." + assert _takes(client, takes[1]["id"])[1]["text"] == "Take 2, rewritten." + + +def test_an_edited_take_is_still_the_take_it_was(client): + """Editing text is not switching, forking, or reordering.""" + row, takes = _four_takes(client) + _edit(client, takes[1]["id"], "Take 2, rewritten.") + + after = _takes(client, row["id"]) + assert [t["id"] for t in after] == [t["id"] for t in takes], "same rows, same order" + assert [t["active"] for t in after] == [False, False, False, True] + assert _live_ai(client)["take_index"] == 3, "still 4/4 on screen" diff --git a/backend/tests/test_take_parentage.py b/backend/tests/test_take_parentage.py new file mode 100644 index 0000000..b965ff7 --- /dev/null +++ b/backend/tests/test_take_parentage.py @@ -0,0 +1,482 @@ +"""Phase 14 SP9: a turn's takes are grouped by their parent, not by where +they sit. + +SP4 made every take a node at the same (branch, depth). That coordinate +answers "which takes belong to this turn" until one of them forks onto its +own branch. At that point it leaves the coordinate and reads as the only +take of its turn, with its siblings unreachable from the line it was taken +on. + +The parent does not move when a branch does, which is the whole fix. It +also gets the nesting right without extra work: takes under C1 and takes +under C2 share a depth and, until one forks, a branch. Only the parent +separates them. + +The branch itself is lazy now. Stepping between takes creates nothing: +looking is free. The fork happens on the first write below a take the +story moved past, which is the first moment the player has said which line +they mean. + + python -m pytest tests/test_take_parentage.py -v +""" +import json + + +import pytest +from fastapi import Depends +from fastapi.testclient import TestClient + +from app import attempts, auth, limits, models +from app.database import Base, SessionLocal, engine, get_db +from app.main import app +from app.routers import adventures + +from fakes import ScriptedProvider + + +@pytest.fixture() +def client(monkeypatch): + Base.metadata.create_all(bind=engine) + setup = SessionLocal() + user = models.User(is_guest=False, email="takes@example.com") + setup.add(user) + setup.flush() + setup.add(models.Settings(user_id=user.id, api_key="enc:dummy", model="test-model")) + adv = models.Adventure( + user_id=user.id, title="Cave", script_state={}, world_state={} + ) + setup.add(adv) + setup.flush() + setup.add( + models.Action(adventure_id=adv.id, type="start", text="You enter.") + ) + setup.commit() + adv_id, user_id = adv.id, user.id + setup.close() + + # Distinct replies so a take can be told apart from its siblings by text. + ScriptedProvider.replies = [f"Take {n}." for n in range(1, 40)] + ScriptedProvider.calls = 0 + monkeypatch.setattr(adventures.turns, "OpenAICompatibleProvider", ScriptedProvider) + monkeypatch.setattr( + auth, + "resolve_provider_config", + lambda s: auth.ProviderConfig("http://fake", "k", "test-model", False), + ) + monkeypatch.setattr(limits, "rate_limit", lambda *a, **k: None) + monkeypatch.setattr(limits, "check_row_cap", lambda *a, **k: None) + + def _current_user(db=Depends(get_db)): + return db.get(models.User, user_id) + + app.dependency_overrides[auth.get_current_user] = _current_user + c = TestClient(app) + c.adv_id = adv_id + try: + yield c + finally: + app.dependency_overrides.clear() + adventures.turns._active_turns.clear() + Base.metadata.drop_all(bind=engine) + + +# ------------------------------------------------------------------ helpers + +def _play(client, text="look around", after_id=None): + body = {"type": "do", "text": text} + if after_id is not None: + body["after_id"] = after_id + r = client.post(f"/api/adventures/{client.adv_id}/actions", json=body) + assert r.status_code == 200, r.text + return r + + +def _retry(client): + r = client.post(f"/api/adventures/{client.adv_id}/retry") + assert r.status_code == 200, r.text + + +def _branch_count(adv_id) -> int: + db = SessionLocal() + try: + return ( + db.query(models.Branch).filter(models.Branch.adventure_id == adv_id).count() + ) + finally: + db.close() + + +def _ai_rows(adv_id) -> list[models.Action]: + """Every AI node ever written, oldest first. Includes both live and + discarded nodes, on any branch.""" + db = SessionLocal() + try: + return ( + db.query(models.Action) + .filter( + models.Action.adventure_id == adv_id, + models.Action.type == "ai", + ) + .order_by(models.Action.id) + .all() + ) + finally: + db.close() + + +def _group_size(action_id: int) -> int: + db = SessionLocal() + try: + node = db.get(models.Action, action_id) + return len(attempts.group(db, node)) + finally: + db.close() + + +# ------------------------------------------------------------------ tests + +def test_retaken_turn_groups_all_its_takes(client): + """The baseline the rest of the file relies on: three takes, one turn.""" + _play(client) + _retry(client) + _retry(client) + + takes = _ai_rows(client.adv_id) + assert len(takes) == 3 + for take in takes: + assert _group_size(take.id) == 3, "every take sees the whole turn" + + +def test_a_forked_take_keeps_its_siblings(client): + """The bug this subphase exists for. + + Under coordinate grouping the forked take reads 1/1: it has left the + (branch, depth) the others are still at. The parent does not move with it. + """ + _play(client) + _retry(client) + _retry(client) + # The story moves past the turn, so taking a different take is now a fork. + _play(client, "press on") + + first_take = _ai_rows(client.adv_id)[0] + assert first_take.live is False + + # Writing below it is what forks. See the next test. + _play(client, "go back and try this instead", after_id=first_take.id) + + assert _group_size(first_take.id) == 3, ( + "a take forked onto its own branch is still a take of the same turn" + ) + + +def test_stepping_between_takes_creates_no_branch(client): + """Looking is free. Only writing commits to a line.""" + _play(client) + _retry(client) + _play(client, "press on") + before = _branch_count(client.adv_id) + + first_take = _ai_rows(client.adv_id)[0] + # Reading a take: the pager fetches the turn's takes and shows one. + r = client.get( + f"/api/adventures/{client.adv_id}/actions/{first_take.id}/variants" + ) + assert r.status_code == 200, r.text + assert len(r.json()) == 2 + + assert _branch_count(client.adv_id) == before, "reading forked nothing" + + +def test_writing_below_a_passed_take_forks_exactly_once(client): + _play(client) + _retry(client) + _play(client, "press on") + before = _branch_count(client.adv_id) + + first_take = _ai_rows(client.adv_id)[0] + _play(client, "a different way", after_id=first_take.id) + + assert _branch_count(client.adv_id) == before + 1, "one write, one branch" + + +def test_takes_under_one_parent_do_not_count_takes_under_its_sibling(client): + """The player's own example: 3/3 on one line, 2/2 on the other. + + C1 and C2 are takes of the same turn. What is played below each of them + is a different turn, and the two turns must not merge. They share a + depth, and until the fork they share a branch too. + """ + _play(client) + _retry(client) # two takes at this turn: C1, C2 (C2 live) + c1, c2 = _ai_rows(client.adv_id)[:2] + + # Below C2, the live one: a turn with three takes. + _play(client, "down the second path") + _retry(client) + _retry(client) + under_c2 = [a for a in _ai_rows(client.adv_id) if a.parent_id is not None] + under_c2 = [a for a in under_c2 if a.id not in (c1.id, c2.id)] + assert len(under_c2) == 3 + + # Now take C1 instead, and play a turn with two takes below it. + _play(client, "down the first path", after_id=c1.id) + _retry(client) + + everything = _ai_rows(client.adv_id) + under_c1 = [a for a in everything if a.parent_id not in (None,) + and a.id not in (c1.id, c2.id) and a.id not in {x.id for x in under_c2}] + assert len(under_c1) == 2 + + assert _group_size(under_c2[0].id) == 3, "C2's line keeps its three takes" + assert _group_size(under_c1[0].id) == 2, "C1's line counts only its own two" + + +def _page(client) -> list[dict]: + return client.get(f"/api/adventures/{client.adv_id}").json()["actions"] + + +def test_the_page_carries_the_pager_numbers(client): + """`2/4` arrives with the page, not from a query per message.""" + _play(client) + _retry(client) + _retry(client) + + ai = [a for a in _page(client) if a["type"] == "ai"] + assert len(ai) == 1, "one take is on the path; the others are behind it" + assert ai[0]["take_count"] == 3 + assert ai[0]["take_index"] == 2, "the newest take is the one being read" + + +def _done_action(response) -> dict: + """The action carried by the `done` event of a turn's SSE stream.""" + for line in response.text.splitlines(): + if not line.startswith("data: "): + continue + event = json.loads(line[len("data: "):]) + if event.get("type") == "done": + return event["action"] + raise AssertionError("no done event in the stream") + + +def test_the_streamed_action_carries_the_pager_too(client): + """Found while exercising the app manually, not by a targeted test. + + A retry's reply is the second take of its turn, so it arrives needing a + pager. The stream builds its own ActionOut and missed the annotation. + The pager appeared only once the page was reloaded, which is exactly + the moment nobody reloads. + """ + _play(client) + r = client.post(f"/api/adventures/{client.adv_id}/retry") + assert r.status_code == 200, r.text + + action = _done_action(r) + assert action["take_count"] == 2, "the take that just landed knows it has a sibling" + assert action["take_index"] == 1 + + +def test_a_turn_nobody_retook_reads_one_of_one(client): + _play(client) + for action in _page(client): + assert action["take_count"] == 1 + assert action["take_index"] == 0 + + +def test_the_pager_still_counts_a_take_that_was_forked_away(client): + """The 1/3 case, seen from the wire rather than from `attempts.group`.""" + _play(client) + _retry(client) + _retry(client) + _play(client, "press on") + + first_take = _ai_rows(client.adv_id)[0] + _play(client, "a different way", after_id=first_take.id) + + ai = [a for a in _page(client) if a["id"] == first_take.id] + assert ai, "the forked take is what this branch now tells" + assert ai[0]["take_count"] == 3 + assert ai[0]["take_index"] == 0 + + +def _take(client, action_id, text): + return client.post( + f"/api/adventures/{client.adv_id}/actions/{action_id}/takes", + json={"text": text}, + ) + + +def _path_texts(client) -> list[str]: + return [ + a["text"] + for a in client.get(f"/api/adventures/{client.adv_id}").json()["actions"] + ] + + +def _user_rows(adv_id) -> list[models.Action]: + db = SessionLocal() + try: + return ( + db.query(models.Action) + .filter( + models.Action.adventure_id == adv_id, + models.Action.type == "do", + ) + .order_by(models.Action.id) + .all() + ) + finally: + db.close() + + +def test_a_players_own_turn_can_be_played_again(client): + """The gap SP7 left: nothing could give a player's own message another take.""" + _play(client, "open the door") + _play(client, "press on") + before = _branch_count(client.adv_id) + + first = _user_rows(client.adv_id)[0] + r = _take(client, first.id, "smash the door instead") + assert r.status_code == 200, r.text + + assert _branch_count(client.adv_id) == before + 1 + # Stored text carries the player-action formatting ("> You ..."), so these + # are substring checks rather than equality. + blob = "\n".join(_path_texts(client)) + assert "smash the door instead" in blob + assert "open the door" not in blob, "the new take replaces it on this line" + assert "press on" not in blob, "what followed the old text stays behind" + + +def test_a_retaken_player_turn_is_not_formatted_twice(client): + """Found while exercising the app manually: "> You > You open the door." + + The editor is seeded from the stored text, which is already in the + formatted form. Plain edit puts that same text in the box and writes + it back verbatim. Running it through the formatter again doubles the + prefix. + """ + _play(client, "open the door") + _play(client, "press on") + + first = _user_rows(client.adv_id)[0] + assert first.text.startswith("> You "), "stored formatted, which is the premise" + r = _take(client, first.id, "> You smash the door instead.") + assert r.status_code == 200, r.text + + new_take = [a for a in _user_rows(client.adv_id) + if "smash the door instead" in a.text][0] + assert new_take.text == "> You smash the door instead." + assert "> You > You" not in new_take.text + + +def test_the_line_left_behind_keeps_its_whole_story(client): + _play(client, "open the door") + _play(client, "press on") + first = _user_rows(client.adv_id)[0] + _take(client, first.id, "smash the door instead") + + # Everything written on the original line is still there, untouched. + kept = "\n".join(a.text for a in _user_rows(client.adv_id)) + for written in ("open the door", "press on", "smash the door instead"): + assert written in kept + + +def test_both_takes_of_a_players_turn_are_one_group(client): + _play(client, "open the door") + _play(client, "press on") + first = _user_rows(client.adv_id)[0] + _take(client, first.id, "smash the door instead") + + new_take = [a for a in _user_rows(client.adv_id) + if "smash the door instead" in a.text][0] + assert _group_size(new_take.id) == 2, "a pager here reads 2/2" + assert _group_size(first.id) == 2, "and reads the same from the other take" + + +def test_an_ai_turn_at_the_tip_takes_no_branch(client): + """Its takes are still leaves. This is `retry`, reached the other way.""" + _play(client) + before = _branch_count(client.adv_id) + + ai = _ai_rows(client.adv_id)[0] + r = _take(client, ai.id, "") + assert r.status_code == 200, r.text + + assert _branch_count(client.adv_id) == before + assert len(_ai_rows(client.adv_id)) == 2, "a second take, beside the first" + + +def test_an_ai_turn_the_story_moved_past_takes_a_branch(client): + """Retry could never reach this case: it only ever saw the newest turn.""" + _play(client) + _play(client, "press on") + before = _branch_count(client.adv_id) + first_ai = _ai_rows(client.adv_id)[0] + + r = _take(client, first_ai.id, "") + assert r.status_code == 200, r.text + + assert _branch_count(client.adv_id) == before + 1 + # The turn the story moved past now has two takes, and the line it was on + # keeps the one it was written for. + assert _group_size(first_ai.id) == 2 + assert first_ai.id in {a.id for a in _ai_rows(client.adv_id)} + + +def test_the_old_line_still_has_its_continuation(client): + _play(client, "open the door") + _play(client, "press on") + first_ai = _ai_rows(client.adv_id)[0] + _take(client, first_ai.id, "") + + # The new take is what this branch tells. "press on" belonged to the other. + blob = "\n".join(_path_texts(client)) + assert "press on" not in blob + kept = "\n".join(a.text for a in _user_rows(client.adv_id)) + assert "press on" in kept, "still there, on the line it was played on" + + +def test_the_opening_has_no_other_take(client): + db = SessionLocal() + try: + start = ( + db.query(models.Action) + .filter(models.Action.adventure_id == client.adv_id, + models.Action.type == "start") + .one() + ) + start_id = start.id + finally: + db.close() + r = _take(client, start_id, "a different beginning") + assert r.status_code == 400 + + +def test_retaking_even_the_newest_player_turn_forks(client): + """A player turn is never the tip: the reply to it is. + + Retaking the last thing the player typed still puts existing story at + risk: the AI answered it, and that answer was written for the old + text. So this case must fork too. The guard against forking for + nothing only fires for a player action with no reply under it. + """ + _play(client, "open the door") + before = _branch_count(client.adv_id) + + first = _user_rows(client.adv_id)[0] + r = _take(client, first.id, "knock politely") + assert r.status_code == 200, r.text + + assert _branch_count(client.adv_id) == before + 1 + + +def test_naming_a_take_that_is_already_the_story_just_plays_on(client): + """`after_id` pointing at the tip is an ordinary turn, and forks nothing.""" + _play(client) + before = _branch_count(client.adv_id) + + live = [a for a in _ai_rows(client.adv_id) if a.live][-1] + _play(client, "carry on", after_id=live.id) + + assert _branch_count(client.adv_id) == before diff --git a/backend/tests/test_take_state.py b/backend/tests/test_take_state.py new file mode 100644 index 0000000..34c2d9c --- /dev/null +++ b/backend/tests/test_take_state.py @@ -0,0 +1,196 @@ +"""Phase 14 SP9: what a take does to the shared state. + +A turn does not only write text. A script mutates `script_state`, and the +referee mutates `world_state`. Both are shared: they belong to the +adventure, not to the node. Playing a turn again must put them back to +where they were before that turn ran. Otherwise the new take stacks its +mutations on top of the one it replaces, and the numbers drift every time +the player asks for another take. + +`retry` has provided this guarantee since SP4 (`attempts.roll_back_before`). +These tests confirm the same guarantee for the two roads SP9 opened: a take +of a turn the story moved past, and a take of the player's own turn. Both +create a branch, which matters because the rollback must survive leaving +the line it was on. + + python -m pytest tests/test_take_state.py -v +""" +import pytest +from fastapi import Depends +from fastapi.testclient import TestClient + +from app import auth, limits, models +from app.database import Base, SessionLocal, engine, get_db +from app.main import app +from app.routers import adventures + +from fakes import ScriptedProvider + +SCHEMA = {"player": {"hp": {"min": 0, "max": 100, "initial": 100}}} + +# Ten gold a turn, every turn. A number that only ever increases makes a +# rollback failure obvious: if a take stacks instead of replacing, the gold +# total is off by exactly one turn's worth. +GOLD_SCRIPT = """ +const modifier = (text) => { + state.gold = (state.gold || 0) + 10; + return { text }; +}; +modifier(text); +""" + + +@pytest.fixture() +def client(monkeypatch): + Base.metadata.create_all(bind=engine) + setup = SessionLocal() + user = models.User(is_guest=False, email="takestate@example.com") + setup.add(user) + setup.flush() + setup.add(models.Settings(user_id=user.id, api_key="enc:dummy", model="test-model")) + scenario = models.Scenario(user_id=user.id, title="S", stat_schema=SCHEMA) + setup.add(scenario) + setup.flush() + adv = models.Adventure( + user_id=user.id, title="Vault", scenario_id=scenario.id, + script_state={}, world_state={"player": {"hp": 100}}, + ) + setup.add(adv) + setup.flush() + setup.add(models.Action(adventure_id=adv.id, type="start", text="You begin.")) + setup.add(models.AdventureScript( + adventure_id=adv.id, position=0, enabled=True, name="Gold", output_js=GOLD_SCRIPT, + )) + setup.commit() + adv_id, user_id = adv.id, user.id + setup.close() + + ScriptedProvider.replies = [f"Take {n}." for n in range(1, 40)] + ScriptedProvider.calls = 0 + monkeypatch.setattr(adventures.turns, "OpenAICompatibleProvider", ScriptedProvider) + monkeypatch.setattr(auth, "resolve_provider_config", lambda s: auth.ProviderConfig( + "http://fake", "k", "test-model", False)) + monkeypatch.setattr(limits, "rate_limit", lambda *a, **k: None) + monkeypatch.setattr(limits, "check_row_cap", lambda *a, **k: None) + + def _current_user(db=Depends(get_db)): + return db.get(models.User, user_id) + + app.dependency_overrides[auth.get_current_user] = _current_user + c = TestClient(app) + c.adv_id = adv_id + try: + yield c + finally: + app.dependency_overrides.clear() + adventures.turns._active_turns.clear() + Base.metadata.drop_all(bind=engine) + + +def _play(client, text="look around", after_id=None): + body = {"type": "do", "text": text} + if after_id is not None: + body["after_id"] = after_id + r = client.post(f"/api/adventures/{client.adv_id}/actions", json=body) + assert r.status_code == 200, r.text + + +def _take(client, action_id, text=""): + r = client.post( + f"/api/adventures/{client.adv_id}/actions/{action_id}/takes", + json={"text": text}, + ) + assert r.status_code == 200, r.text + return r + + +def _gold(adv_id) -> int: + db = SessionLocal() + try: + return (db.get(models.Adventure, adv_id).script_state or {}).get("gold", 0) + finally: + db.close() + + +def _rows(adv_id, type_): + db = SessionLocal() + try: + return ( + db.query(models.Action) + .filter(models.Action.adventure_id == adv_id, models.Action.type == type_) + .order_by(models.Action.id) + .all() + ) + finally: + db.close() + + +def test_the_script_runs_once_a_turn(client): + """This test establishes the baseline the rest of the file depends on.""" + _play(client) + assert _gold(client.adv_id) == 10 + _play(client, "press on") + assert _gold(client.adv_id) == 20 + + +def test_a_take_of_a_past_ai_turn_does_not_stack_its_script(client): + """Two turns played, then the first one taken again. + + The take leaves the path just before turn one. The state it starts from + is the state turn one started from: zero gold, not the twenty that two + turns accumulated. The take's own run then adds ten. + """ + _play(client) + _play(client, "press on") + assert _gold(client.adv_id) == 20 + + first_ai = _rows(client.adv_id, "ai")[0] + _take(client, first_ai.id) + + assert _gold(client.adv_id) == 10, "rolled back to before that turn, then run once" + + +def test_a_take_of_a_player_turn_does_not_stack_its_script(client): + _play(client) + _play(client, "press on") + assert _gold(client.adv_id) == 20 + + first_player = _rows(client.adv_id, "do")[0] + _take(client, first_player.id, "> You do something else.") + + assert _gold(client.adv_id) == 10 + + +def test_writing_below_a_passed_take_starts_from_that_take_s_state(client): + """The `after_id` path, which forks while writing. + + The take being written under produced a state of ten gold, from its own + turn. The turn played on top of it adds another ten. The twenty gold + that the abandoned line reached has no effect on this branch. + """ + _play(client) + r = client.post(f"/api/adventures/{client.adv_id}/retry") + assert r.status_code == 200, r.text + _play(client, "press on") + assert _gold(client.adv_id) == 20 + + discarded = [a for a in _rows(client.adv_id, "ai") if not a.live][0] + _play(client, "a different way", after_id=discarded.id) + + assert _gold(client.adv_id) == 20, "that take's ten, plus this turn's ten" + + +def test_the_line_left_behind_keeps_the_state_it_reached(client): + """Switching back finds the abandoned line's numbers where it left them.""" + _play(client) + _play(client, "press on") + first_ai = _rows(client.adv_id, "ai")[0] + _take(client, first_ai.id) + assert _gold(client.adv_id) == 10 + + branches = client.get(f"/api/adventures/{client.adv_id}/branches").json() + root = [b for b in branches if b["parent_branch_id"] is None][0] + r = client.post(f"/api/adventures/{client.adv_id}/branches/{root['id']}/switch") + assert r.status_code == 200, r.text + + assert _gold(client.adv_id) == 20, "the first telling still has its two turns" diff --git a/backend/tests/test_tree_migration.py b/backend/tests/test_tree_migration.py new file mode 100644 index 0000000..4047bdb --- /dev/null +++ b/backend/tests/test_tree_migration.py @@ -0,0 +1,796 @@ +"""Phase 14 SP1: every existing adventure becomes a tree with one branch. + +The migration this file tests cannot be re-run. It reads `index` and +writes `depth`, and from SP2 on, the reads follow `depth`. If it mis-maps +a row, that row does not raise an error. It disappears from the story, +which is why the assertions here check every row instead of a sample. + +The fixture is a genuine schema-45 database, not a current one with an +old stamp. `create_all` always builds the current schema, so the three +tables the tree touches are dropped and rebuilt from the frozen pre-tree +DDL below. The migration then runs its real ALTER statements against +them, including the one that adds a foreign key. A pre-migration database +built any other way, such as a rewound stamp with the columns left in +place, would silently skip the DDL and test only half the change. + + python -m pytest tests/test_tree_migration.py -v +""" +import json + + +import pytest +from fastapi import Depends +from fastapi.testclient import TestClient +from sqlalchemy import text + +from app import auth, compression, limits, migrations, models, tree +from app.context import history +from app.database import Base, SessionLocal, engine, get_db +from app.main import app + +# The three tables as they stood at schema 45, frozen. This is a snapshot +# of a past schema. It must not be updated to track `models.py`, because +# the whole point is that it lacks what SP1 adds. This DDL uses SQLite +# syntax only. The migration's Postgres half is exercised against a real +# server at deploy time (see plan/14). +PRE_TREE_DDL = ( + """ + CREATE TABLE adventures ( + id INTEGER NOT NULL PRIMARY KEY, + user_id INTEGER REFERENCES users(id) ON DELETE CASCADE, + scenario_id INTEGER, + title VARCHAR(200) NOT NULL DEFAULT 'Untitled Adventure', + memory TEXT NOT NULL DEFAULT '', + authors_note TEXT NOT NULL DEFAULT '', + ai_instructions TEXT NOT NULL DEFAULT '', + story_summary TEXT NOT NULL DEFAULT '', + script_state JSON NOT NULL DEFAULT '{}', + world_state JSON NOT NULL DEFAULT '{}', + placeholders JSON, + auto_summarize BOOLEAN NOT NULL DEFAULT 0, + memory_bank_enabled BOOLEAN NOT NULL DEFAULT 0, + memory_cursor INTEGER NOT NULL DEFAULT 0, + summary_cursor INTEGER NOT NULL DEFAULT 0, + created_at DATETIME, + updated_at DATETIME + ) + """, + """ + CREATE TABLE actions ( + id INTEGER NOT NULL PRIMARY KEY, + adventure_id INTEGER NOT NULL REFERENCES adventures(id) ON DELETE CASCADE, + "index" INTEGER NOT NULL, + type VARCHAR(20) NOT NULL, + text TEXT NOT NULL DEFAULT '', + reasoning TEXT, + context_snapshot BLOB, + world_delta JSON, + state_before JSON, + world_state_before JSON, + variants JSON, + variant_count INTEGER NOT NULL DEFAULT 0, + variant_index INTEGER NOT NULL DEFAULT 0, + created_at DATETIME + ) + """, + """ + CREATE TABLE memories ( + id INTEGER NOT NULL PRIMARY KEY, + adventure_id INTEGER NOT NULL REFERENCES adventures(id) ON DELETE CASCADE, + text TEXT NOT NULL DEFAULT '', + embedding_blob BLOB, + source_start INTEGER, + source_end INTEGER, + embedded BOOLEAN NOT NULL DEFAULT 0, + pinned BOOLEAN NOT NULL DEFAULT 0, + forgotten BOOLEAN NOT NULL DEFAULT 0, + use_count INTEGER NOT NULL DEFAULT 0, + last_used_at DATETIME, + created_at DATETIME + ) + """, +) + +# The story of adventure "Gapped": index 3 is missing, because deleting a +# middle action never renumbered the ones after it. The migration must +# preserve the gap. +GAPPED_INDEXES = (0, 1, 2, 4) +STRAIGHT_INDEXES = (0, 1) +# "Blank" holds an action whose text is nothing but whitespace. It is a +# row of the adventure but not of the story, so a cursor counting covered +# actions never counted it. Migration 56 must skip it the same way, using +# a frozen copy of the story-text predicate. This predicate is the one +# duplicated definition in the change, so this is the one test case that +# can catch it drifting. +BLANK_INDEXES = (0, 1, 2, 3) +BLANK_AT = 2 + + +@pytest.fixture() +def pre_tree(): + """A schema-45 database with three adventures in it, returned as the ids + (gapped, straight, empty) their stories were written under.""" + # Every test in this module shares one temp file, and a setup that + # fails before its yield never reaches a teardown. Start from empty + # instead of from whatever the previous test left. + Base.metadata.drop_all(bind=engine) + Base.metadata.create_all(bind=engine) + with engine.begin() as conn: + # `branches` and the six new columns never existed at schema 45. + # Dropping the tables is the only way to remove the columns. + # SQLite refuses to drop a column that a foreign key references, + # and that is exactly the case for `branch_id`. + for table in ("actions", "memories", "branches", "adventures"): + conn.execute(text(f"DROP TABLE IF EXISTS {table}")) + for ddl in PRE_TREE_DDL: + conn.execute(text(ddl)) + conn.execute(text( + "INSERT INTO users (id, email, is_guest, created_at, demo_turns_used, " + "demo_turns_date) VALUES (1, 'v45@example.com', 0, CURRENT_TIMESTAMP, 0, '')" + )) + + # The cursors as schema 45 held them: counts of covered story actions. + # Gapped's story is 0, 1, 2, 4, so "3 covered" is the node at depth + # 2 and "4 covered" is the node at depth 4. This is the whole + # reason a count and a depth are not the same number. Straight is + # caught up past its own end (5 covered, 2 actions), a state the + # older rule left behind that a clamp used to mask on every + # post-turn pass. + cursors_at = {"Gapped": (3, 4), "Straight": (5, 0), "Empty": (0, 0), + "Blank": (3, 0)} + ids = {} + for name in ("Gapped", "Straight", "Empty", "Blank"): + conn.execute(text( + "INSERT INTO adventures (user_id, title, memory_cursor, summary_cursor) " + "VALUES (1, :title, :mc, :sc)" + ), {"title": name, "mc": cursors_at[name][0], "sc": cursors_at[name][1]}) + ids[name] = conn.execute(text( + "SELECT id FROM adventures WHERE title = :title" + ), {"title": name}).scalar() + + for adventure_id, indexes in ( + (ids["Gapped"], GAPPED_INDEXES), + (ids["Straight"], STRAIGHT_INDEXES), + (ids["Blank"], BLANK_INDEXES), + ): + for index in indexes: + blank = adventure_id == ids["Blank"] and index == BLANK_AT + conn.execute(text( + 'INSERT INTO actions (adventure_id, "index", type, text) ' + "VALUES (:a, :i, :t, :x)" + ), {"a": adventure_id, "i": index, + "t": "start" if index == 0 else "do", + "x": " \n\t " if blank else f"Turn {index}."}) + + # One memory that summarised a block of story, and one written by hand, + # which summarised nothing and so belongs to no node. + conn.execute(text( + "INSERT INTO memories (adventure_id, text, source_start, source_end) " + "VALUES (:a, 'The gate opened.', 0, 1)" + ), {"a": ids["Gapped"]}) + conn.execute(text( + "INSERT INTO memories (adventure_id, text) VALUES (:a, 'Hand-written.')" + ), {"a": ids["Gapped"]}) + + conn.execute(text("PRAGMA user_version = 45")) + + try: + yield ids + finally: + Base.metadata.drop_all(bind=engine) + + +def rows(sql: str, **params) -> list[tuple]: + with engine.begin() as conn: + return conn.execute(text(sql), params).all() + + +def scalar(sql: str, **params): + with engine.begin() as conn: + return conn.execute(text(sql), params).scalar() + + +# ------------------------------------------------------- the migration itself + +def test_the_stamp_reaches_the_current_version(pre_tree): + migrations.bootstrap(engine) + assert scalar("PRAGMA user_version") == migrations.LATEST_VERSION + + +def test_every_action_lands_on_its_adventure_root_branch(pre_tree): + before = scalar("SELECT count(*) FROM actions") + + migrations.bootstrap(engine) + + assert scalar("SELECT count(*) FROM actions") == before, "the migration lost a row" + assert scalar("SELECT count(*) FROM actions WHERE branch_id IS NULL") == 0 + assert scalar("SELECT count(*) FROM actions WHERE depth IS NULL") == 0 + # Each action's branch belongs to that action's own adventure. A branch + # clause that forgot its adventure would still look right on a database + # holding one, which is why the fixture holds three. + mismatched = scalar(""" + SELECT count(*) FROM actions a JOIN branches b ON b.id = a.branch_id + WHERE b.adventure_id != a.adventure_id + """) + assert mismatched == 0 + + +def test_depth_is_the_old_index_gaps_included(pre_tree): + migrations.bootstrap(engine, through=migrations.TREE_BACKFILL_VERSION) + + assert rows('SELECT "index", depth FROM actions WHERE depth != "index"') == [] + depths = [ + row[0] for row in rows( + "SELECT depth FROM actions WHERE adventure_id = :a ORDER BY depth", + a=pre_tree["Gapped"], + ) + ] + # 3 is still missing. Renumbering here would silently move every cursor + # pointing past the gap, and the reads only need the order, not density. + assert depths == list(GAPPED_INDEXES) + + +def test_one_root_branch_per_adventure_with_its_own_lineage(pre_tree): + migrations.bootstrap(engine) + + branches = rows( + "SELECT id, adventure_id, parent_branch_id, fork_depth, lineage FROM branches" + ) + assert len(branches) == 4, "one branch per adventure, including the empty one" + for branch_id, _adventure_id, parent, fork_depth, lineage in branches: + assert parent is None, "a migrated branch is a root; nothing forked yet" + assert fork_depth is None + # The whole story, uncapped: one entry, itself, no ceiling. + assert json.loads(lineage) == [[branch_id, None]] + + +def test_the_head_points_at_the_tip_of_the_root_branch(pre_tree): + migrations.bootstrap(engine) + + heads = dict(rows("SELECT title, head_depth FROM adventures")) + assert heads["Gapped"] == max(GAPPED_INDEXES) + assert heads["Straight"] == max(STRAIGHT_INDEXES) + # No actions, no tip. -1 keeps "the next node goes at head_depth + 1" true + # without a special case anywhere else. + assert heads["Empty"] == tree.NO_DEPTH + assert scalar("SELECT count(*) FROM adventures WHERE head_branch_id IS NULL") == 0 + dangling = scalar(""" + SELECT count(*) FROM adventures a + WHERE NOT EXISTS ( + SELECT 1 FROM branches b + WHERE b.id = a.head_branch_id AND b.adventure_id = a.id + ) + """) + assert dangling == 0, "a head pointing outside its own adventure" + + +def test_memories_attach_to_the_node_they_summarised(pre_tree): + migrations.bootstrap(engine) + + summarised = rows( + "SELECT source_end, depth, branch_id FROM memories WHERE source_end IS NOT NULL" + ) + assert summarised, "the fixture is supposed to have one" + for source_end, depth, branch_id in summarised: + assert depth == source_end, "the memory hangs off the last action it covered" + assert branch_id is not None + + # A hand-written memory summarised no node, so SP7's migration 62 + # lands it at depth 0 of its branch. Depth 0 is at or before every + # fork point, so the memory stays visible from exactly the paths it + # was visible from before. Anchoring it this way removes nothing from + # anybody's existing memory bank. + manual = rows("SELECT depth, branch_id FROM memories WHERE source_end IS NULL") + assert manual and all(depth == 0 and branch is not None for depth, branch in manual) + + +def test_the_cursors_become_the_nodes_they_named(pre_tree): + """SP3, migration 56. A count of covered actions and a depth become + different numbers as soon as the story has a gap in it, and every + adventure with a deleted action has one.""" + migrations.bootstrap(engine, through=migrations.CURSOR_ANCHOR_VERSION) + + def marks(title): + [row] = rows( + "SELECT memory_cursor_depth, summary_cursor_depth, " + "memory_cursor_branch_id, summary_cursor_branch_id " + "FROM adventures WHERE title = :t", t=title + ) + return row + + # Gapped's story is 0, 1, 2, 4. "3 covered" is the third action, at + # depth 2. Reading the count as a depth would hand the summarizer node + # 3, which does not exist, and silently skip node 4 forever. + memory_depth, summary_depth, memory_branch, summary_branch = marks("Gapped") + assert (memory_depth, summary_depth) == (2, 4) + root = scalar( + "SELECT id FROM branches WHERE adventure_id = " + "(SELECT id FROM adventures WHERE title = 'Gapped')" + ) + assert memory_branch == summary_branch == root + + # Straight was caught up under the older rule: 5 covered, 2 actions. There + # is no fifth node to name, and the number meant "caught up", so it lands + # on the tip rather than on nothing. + memory_depth, summary_depth, _, summary_branch = marks("Straight") + assert memory_depth == 1 + assert (summary_depth, summary_branch) == (migrations.NO_DEPTH, None) + + # Nothing covered stays nothing covered, and names no branch. + assert marks("Empty") == (migrations.NO_DEPTH, migrations.NO_DEPTH, None, None) + + # A whitespace-only action is a row but not a story action, so it was + # never counted. "3 covered" of 0, 1, [blank], 3 is the node at depth + # 3, not 2. The migration's copy of the story-text predicate is the + # only place that rule is written twice, so this is the test case + # that catches it drifting. + assert marks("Blank")[0] == 3 + + # The legacy columns are left exactly as they were: a rolled-back build + # reads them, and this migration is not the one that drops them. + assert rows( + "SELECT memory_cursor, summary_cursor FROM adventures ORDER BY title" + ) == [(3, 0), (0, 0), (3, 4), (5, 0)] # Blank, Empty, Gapped, Straight + + +def test_the_branch_clause_index_exists(pre_tree): + """SP2's reads are only cheap if this index exists. `create_all` does + not add an index to a table it did not create, which is what + migration 52 handles.""" + migrations.bootstrap(engine) + + assert scalar( + "SELECT count(*) FROM sqlite_master " + "WHERE type = 'index' AND name = 'ix_actions_branch_depth'" + ) == 1 + + +def test_running_it_again_changes_nothing(pre_tree): + migrations.bootstrap(engine, through=migrations.CURSOR_ANCHOR_VERSION) + snapshot = ( + rows("SELECT id, branch_id, depth FROM actions ORDER BY id"), + rows("SELECT id, adventure_id, lineage FROM branches ORDER BY id"), + rows("SELECT id, head_branch_id, head_depth, memory_cursor_branch_id, " + "memory_cursor_depth, summary_cursor_branch_id, summary_cursor_depth " + "FROM adventures ORDER BY id"), + rows("SELECT id, branch_id, depth FROM memories ORDER BY id"), + ) + + # Run the deploy path twice, then the data pass on its own. The stamp + # stops the first run, the NULL guards stop the second, and a + # migration that only survives because of the stamp is one bad rescue + # away from doubling every branch. + migrations.bootstrap(engine, through=migrations.CURSOR_ANCHOR_VERSION) + with engine.begin() as conn: + migrations._backfill_tree(conn) + migrations._backfill_cursor_anchors(conn) + + assert ( + rows("SELECT id, branch_id, depth FROM actions ORDER BY id"), + rows("SELECT id, adventure_id, lineage FROM branches ORDER BY id"), + rows("SELECT id, head_branch_id, head_depth, memory_cursor_branch_id, " + "memory_cursor_depth, summary_cursor_branch_id, summary_cursor_depth " + "FROM adventures ORDER BY id"), + rows("SELECT id, branch_id, depth FROM memories ORDER BY id"), + ) == snapshot + + +# -------------------------------------------------- rows written *after* it + +@pytest.fixture() +def client(monkeypatch): + """The app on a migrated database, so new rows go through the real writers. + + Fixing existing rows is only half the job. No migration ever revisits + a row written after it ran, and a row without a branch is a row no + read can see. + """ + Base.metadata.drop_all(bind=engine) + Base.metadata.create_all(bind=engine) + setup = SessionLocal() + user = models.User(is_guest=False, email="writer@example.com") + setup.add(user) + setup.flush() + scenario = models.Scenario(user_id=user.id, title="S", prompt="You enter a cave.") + setup.add(scenario) + setup.commit() + user_id, scenario_id = user.id, scenario.id + setup.close() + + monkeypatch.setattr(limits, "rate_limit", lambda *a, **k: None) + monkeypatch.setattr(limits, "check_row_cap", lambda *a, **k: None) + + def _current_user(db=Depends(get_db)): + return db.get(models.User, user_id) + + app.dependency_overrides[auth.get_current_user] = _current_user + c = TestClient(app) + c.scenario_id = scenario_id + try: + yield c + finally: + app.dependency_overrides.clear() + Base.metadata.drop_all(bind=engine) + + +def test_a_new_adventure_gets_a_branch_and_its_opening_sits_on_it(client): + response = client.post("/api/adventures", json={"scenario_id": client.scenario_id}) + assert response.status_code == 201 + adventure_id = response.json()["id"] + + db = SessionLocal() + try: + adventure = db.get(models.Adventure, adventure_id) + branch = db.query(models.Branch).filter_by(adventure_id=adventure_id).one() + assert adventure.head_branch_id == branch.id + assert branch.lineage == [[branch.id, None]] + opening = db.query(models.Action).filter_by(adventure_id=adventure_id).one() + assert (opening.branch_id, opening.depth) == (branch.id, 0) + assert adventure.head_depth == 0 + finally: + db.close() + + +def test_a_blank_adventure_has_a_branch_before_anything_is_played(client): + adventure_id = client.post("/api/adventures", json={}).json()["id"] + + db = SessionLocal() + try: + adventure = db.get(models.Adventure, adventure_id) + assert adventure.head_branch_id is not None + assert adventure.head_depth == tree.NO_DEPTH + finally: + db.close() + + +def test_a_hand_written_memory_is_anchored_at_the_head(client): + """SP7: nothing carries a NULL depth anymore. + + On an adventure with no story yet, the head is `NO_DEPTH` (-1), which + reads as "before the first node" and so falls in range of every + branch. This is correct for a note written before anything has + happened. + """ + adventure_id = client.post("/api/adventures", json={}).json()["id"] + + created = client.post( + f"/api/adventures/{adventure_id}/memories", json={"text": "Remember the gate."} + ) + assert created.status_code == 201 + + db = SessionLocal() + try: + memory = db.query(models.Memory).filter_by(adventure_id=adventure_id).one() + adventure = db.get(models.Adventure, adventure_id) + assert memory.branch_id is not None + assert memory.depth == adventure.head_depth == -1 + finally: + db.close() + + +def test_deleting_a_branch_takes_its_nodes_with_it(client): + """`ON DELETE CASCADE` on both `branch_id` columns means the database + removes a branch's nodes instead of relying on application code to + remember to. SP7 builds delete-a-branch directly on top of this, and + no other code needs to load a branch to do it.""" + adventure_id = client.post( + "/api/adventures", json={"scenario_id": client.scenario_id} + ).json()["id"] + + db = SessionLocal() + try: + adventure = db.get(models.Adventure, adventure_id) + memory = models.Memory( + adventure_id=adventure_id, text="m", source_start=0, source_end=0 + ) + tree.place_memory(db, adventure, memory) + db.add(memory) + db.commit() + branch_id = adventure.head_branch_id + + db.execute( + models.Branch.__table__.delete().where(models.Branch.id == branch_id) + ) + db.commit() + assert db.query(models.Action).filter_by(adventure_id=adventure_id).count() == 0 + assert db.query(models.Memory).filter_by(adventure_id=adventure_id).count() == 0 + finally: + db.close() + + +def test_deleting_an_adventure_takes_its_branch_with_it(client): + adventure_id = client.post( + "/api/adventures", json={"scenario_id": client.scenario_id} + ).json()["id"] + + assert client.delete(f"/api/adventures/{adventure_id}").status_code == 204 + + db = SessionLocal() + try: + assert db.query(models.Branch).filter_by(adventure_id=adventure_id).count() == 0 + finally: + db.close() + + +def test_deleting_the_newest_action_moves_the_head_back(client): + """The head is a cache. A cache that only ever moves forward becomes + wrong the first time someone undoes a turn.""" + adventure_id = client.post( + "/api/adventures", json={"scenario_id": client.scenario_id} + ).json()["id"] + + db = SessionLocal() + try: + adventure = db.get(models.Adventure, adventure_id) + extra = models.Action(adventure_id=adventure_id, type="do", text="Look.") + tree.place_action(db, adventure, extra) + db.add(extra) + db.commit() + assert adventure.head_depth == 1 + action_id = extra.id + finally: + db.close() + + assert client.delete( + f"/api/adventures/{adventure_id}/actions/{action_id}" + ).status_code == 204 + + db = SessionLocal() + try: + assert db.get(models.Adventure, adventure_id).head_depth == 0 + finally: + db.close() + + +# ---------------------------------------- SP4: variants become sibling rows + +# One turn's retry history as schema 45 stored it: a JSON array on the +# AI row, with `variant_index` naming the entry `text` mirrors. The live +# entry is deliberately not the last one written. A migration that +# assumed it was would still pass on every fixture where the player +# never paged back. +RETRY_VARIANTS = [ + {"text": "Attempt one.", "reasoning": None, + "script_state": {"gold": 10}, "created_at": "2026-01-01T00:00:00", + "raw_output": "Attempt one.", + "world_state": {"delta": {"player.hp": -5}, + "report": {"applied": [{"path": "player.hp", "old": 100, "new": 95}]}, + "state": {"player": {"hp": 95}}}}, + {"text": "Attempt two.", "reasoning": "thinking", + "script_state": {"gold": 20}, "created_at": "2026-01-01T00:01:00", + "raw_output": "Attempt two.", + "world_state": {"delta": {"player.hp": -40}, + "report": {"applied": [{"path": "player.hp", "old": 100, "new": 60}]}, + "state": {"player": {"hp": 60}}}}, + {"text": "Attempt three.", "reasoning": None, + "script_state": {"gold": 30}, "created_at": "2026-01-01T00:02:00", + "raw_output": "Attempt three."}, +] +LIVE_VARIANT = 1 + +# The whole turn's assembled prompt, stored once. The attempts differ +# only in the three slices above, and SP4 must preserve that arrangement. +# Giving each sibling a copy of this prompt would multiply the largest +# column in the database by the retry count. +RETRY_SNAPSHOT = { + "sections": [{"label": "history", "text": "A long prompt.", "tokens": 4}], + "prompt": {"system": "S", "story": "A long prompt."}, + "raw_output": "Attempt two.", + "script": {"logs": []}, + "world_state": RETRY_VARIANTS[LIVE_VARIANT]["world_state"], +} + + +@pytest.fixture() +def pre_split(): + """A schema-45 adventure with one retried turn, plus a plain turn on + each side. + + This fixture is separate from `pre_tree` so SP1's assertions keep + counting what they were written to count. The story is: 0 start, 1 + do, 2 ai (three attempts), 3 do. The adventure's live state is the + one attempt 1 produced. + """ + Base.metadata.drop_all(bind=engine) + Base.metadata.create_all(bind=engine) + with engine.begin() as conn: + for table in ("actions", "memories", "branches", "adventures"): + conn.execute(text(f"DROP TABLE IF EXISTS {table}")) + for ddl in PRE_TREE_DDL: + conn.execute(text(ddl)) + conn.execute(text( + "INSERT INTO users (id, email, is_guest, created_at, demo_turns_used, " + "demo_turns_date) VALUES (1, 'v45@example.com', 0, CURRENT_TIMESTAMP, 0, '')" + )) + conn.execute(text( + "INSERT INTO adventures (user_id, title, script_state, world_state) " + "VALUES (1, 'Retried', :script, :world)" + ), {"script": json.dumps({"gold": 20}), + "world": json.dumps({"player": {"hp": 60}})}) + adventure_id = conn.execute( + text("SELECT id FROM adventures WHERE title = 'Retried'") + ).scalar() + # `state_before` on each row records the script state as that + # action found it. SP4 reads each row's `state_before` from the + # next row to build the `state_after` pair. + for index, kind, before in ( + (0, "start", None), (1, "do", {"gold": 0}), + (2, "ai", {"gold": 0}), (3, "do", {"gold": 20}), + ): + conn.execute(text( + 'INSERT INTO actions (adventure_id, "index", type, text, reasoning, ' + "state_before, context_snapshot, variants, variant_count, variant_index) " + "VALUES (:a, :i, :t, :x, :r, :sb, :cs, :v, :vc, :vi)" + ), { + "a": adventure_id, "i": index, "t": kind, + "x": RETRY_VARIANTS[LIVE_VARIANT]["text"] if kind == "ai" else f"Turn {index}.", + "r": RETRY_VARIANTS[LIVE_VARIANT]["reasoning"] if kind == "ai" else None, + "sb": None if before is None else json.dumps(before), + "cs": compression.pack(RETRY_SNAPSHOT) if kind == "ai" else None, + "v": json.dumps(RETRY_VARIANTS) if kind == "ai" else None, + "vc": len(RETRY_VARIANTS) if kind == "ai" else 0, + "vi": LIVE_VARIANT if kind == "ai" else 0, + }) + conn.execute(text("PRAGMA user_version = 45")) + try: + yield adventure_id + finally: + Base.metadata.drop_all(bind=engine) + + +def _attempts(adventure_id) -> list[tuple]: + return rows( + "SELECT variant_index, text, live, variant_count FROM actions " + 'WHERE adventure_id = :a AND "index" = 2 ORDER BY variant_index', + a=adventure_id, + ) + + +def test_each_attempt_becomes_a_row_at_the_turns_coordinate(pre_split): + migrations.bootstrap(engine, through=migrations.SIBLING_SPLIT_VERSION) + + assert _attempts(pre_split) == [ + (0, "Attempt one.", 0, 3), + (1, "Attempt two.", 1, 3), + (2, "Attempt three.", 0, 3), + ] + # One turn, one coordinate: the siblings share a branch and a depth, and + # keep the legacy index that says which turn they are all takes on. + coordinates = rows( + 'SELECT DISTINCT branch_id, depth FROM actions WHERE adventure_id = :a ' + 'AND "index" = 2', a=pre_split, + ) + assert len(coordinates) == 1 + # The rest of the story is untouched, still one row per turn. + assert scalar("SELECT count(*) FROM actions WHERE adventure_id = :a", a=pre_split) == 6 + + +def test_the_live_attempt_is_the_one_the_row_was_mirroring(pre_split): + """`variant_index` is the only record of which take the player was reading, + and it survives as the `live` flag. Guessing "the newest" instead would + silently rewrite the story of anyone who had paged back.""" + migrations.bootstrap(engine, through=migrations.SIBLING_SPLIT_VERSION) + + live = rows( + "SELECT text FROM actions WHERE adventure_id = :a AND live = 1 " + 'AND "index" = 2', a=pre_split, + ) + assert live == [("Attempt two.",)] + + +def test_the_prompt_stays_on_the_live_attempt_and_nowhere_else(pre_split): + migrations.bootstrap(engine, through=migrations.SIBLING_SPLIT_VERSION) + + holders = [] + for variant_index, snapshot in rows( + 'SELECT variant_index, context_snapshot FROM actions WHERE adventure_id = :a ' + 'AND "index" = 2 ORDER BY variant_index', a=pre_split, + ): + stored = compression.unpack(snapshot) if snapshot else {} + if "sections" in stored: + holders.append(variant_index) + else: + # A superseded attempt keeps only what was its own. + assert set(stored) <= set(migrations._ATTEMPT_KEYS) + assert holders == [LIVE_VARIANT] + + +def test_each_attempt_keeps_the_outcome_it_produced(pre_split): + migrations.bootstrap(engine, through=migrations.SIBLING_SPLIT_VERSION) + + parsed = [ + (i, json.loads(state), json.loads(world) if world else None) + for i, state, world in rows( + "SELECT variant_index, state_after, world_state_after FROM actions " + 'WHERE adventure_id = :a AND "index" = 2 ORDER BY variant_index', + a=pre_split, + ) + ] + assert [(i, s) for i, s, _ in parsed] == [ + (0, {"gold": 10}), (1, {"gold": 20}), (2, {"gold": 30}) + ] + assert parsed[0][2] == {"player": {"hp": 95}} + assert parsed[1][2] == {"player": {"hp": 60}} + # Attempt three recorded no world state: either an adventure with no + # RPG layer, or a take made before the column existed. It stays NULL + # instead of borrowing a neighbor's, so switching to it leaves the RPG + # layer alone. This matches what `apply_variant` did with an entry + # that had no world state. + assert parsed[2][2] is None + + +def test_state_after_is_the_state_before_of_the_turn_in_front(pre_split): + migrations.bootstrap(engine, through=migrations.SIBLING_SPLIT_VERSION) + + after = dict(rows( + 'SELECT "index", state_after FROM actions WHERE adventure_id = :a ' + "AND live = 1 ORDER BY depth", a=pre_split, + )) + # Action 1's outcome is action 2's starting position, exactly. + assert json.loads(after[1]) == {"gold": 0} + # The tip has nothing in front of it, so what it left behind is what the + # adventure is carrying now. + assert json.loads(after[3]) == {"gold": 20} + + +def test_the_split_survives_being_run_again(pre_split): + migrations.bootstrap(engine, through=migrations.SIBLING_SPLIT_VERSION) + snapshot = _attempts(pre_split) + before = scalar("SELECT count(*) FROM actions") + + migrations.bootstrap(engine, through=migrations.SIBLING_SPLIT_VERSION) + with engine.begin() as conn: + migrations._backfill_state_after(conn) + migrations._split_variants_into_siblings(conn) + + assert scalar("SELECT count(*) FROM actions") == before, "attempts were duplicated" + assert _attempts(pre_split) == snapshot + + +def test_the_migrated_story_reads_back_as_one_turn(pre_split): + """This is the point of the whole migration: reads see a four-action + story, not six.""" + migrations.bootstrap(engine) + + db = SessionLocal() + try: + adventure = db.get(models.Adventure, pre_split) + assert [a.text for a in history.story_actions(adventure)] == [ + "Turn 0.", "Turn 1.", "Attempt two.", "Turn 3.", + ] + assert history.count(adventure) == 4 + finally: + db.close() + + +# ------------------------------------------------------- SP8: the columns go + +# The columns migrations 66 to 73 drop, paired with the table they sat on. +DROPPED_COLUMNS = [ + ("actions", "index"), + ("actions", "variants"), + ("actions", "variant_index"), + ("actions", "variant_count"), + ("actions", "state_before"), + ("actions", "world_state_before"), + ("adventures", "memory_cursor"), + ("adventures", "summary_cursor"), +] + + +@pytest.mark.parametrize("table,column", DROPPED_COLUMNS) +def test_the_legacy_columns_are_dropped(pre_tree, table, column): + """A pre-tree database that migrates all the way ends without these eight. + + The fixture is a real schema-45 database, so each column starts present and + the `ALTER TABLE` statements have something to remove. A `create_all` + database never has them, and would pass this test without running the + migration at all. + """ + before = {row[1] for row in rows(f"PRAGMA table_info({table})")} + assert column in before, "the fixture must start with the column" + + migrations.bootstrap(engine) + + after = {row[1] for row in rows(f"PRAGMA table_info({table})")} + assert column not in after diff --git a/backend/tests/test_turn_flow_integration.py b/backend/tests/test_turn_flow_integration.py new file mode 100644 index 0000000..54bb25f --- /dev/null +++ b/backend/tests/test_turn_flow_integration.py @@ -0,0 +1,125 @@ +"""End-to-end HTTP tests for undo and retry state revert. These tests drive +real turns through the actual routes and scripting engine, with only the LLM +provider mocked. + +A script's output hook adds 10 gold each turn. These tests confirm that the +adventure's stored gold total stays correct across play, undo, and retry. + + python -m pytest tests/test_turn_flow_integration.py -v +""" +import pytest +from fastapi import Depends +from fastapi.testclient import TestClient + +from app import auth, limits, models +from app.database import Base, SessionLocal, engine, get_db +from app.main import app +from app.routers import adventures + +from fakes import ScriptedProvider + +GOLD_SCRIPT = """ +const modifier = (text) => { + state.gold = (state.gold || 0) + 10; + return { text }; +}; +modifier(text); +""" + + +AI_REPLY = "The torch flickers as you press onward." + + +@pytest.fixture() +def client(monkeypatch): + Base.metadata.create_all(bind=engine) + setup = SessionLocal() + user = models.User(is_guest=False, email="tester@example.com") + setup.add(user) + setup.flush() + setup.add(models.Settings(user_id=user.id, api_key="enc:dummy", model="test-model")) + adv = models.Adventure(user_id=user.id, title="Cave", script_state={}) + setup.add(adv) + setup.flush() + setup.add(models.Action(adventure_id=adv.id, type="start", text="You enter a cave.")) + setup.add(models.AdventureScript( + adventure_id=adv.id, position=0, enabled=True, name="Gold", + output_js=GOLD_SCRIPT, + )) + setup.commit() + adv_id, user_id = adv.id, user.id + setup.close() + + # Force a real, non-demo turn that uses the fake provider. + ScriptedProvider.replies = [AI_REPLY] + monkeypatch.setattr(adventures.turns, "OpenAICompatibleProvider", ScriptedProvider) + monkeypatch.setattr(auth, "resolve_provider_config", lambda s: auth.ProviderConfig( + "http://fake", "k", "test-model", False)) + monkeypatch.setattr(limits, "rate_limit", lambda *a, **k: None) + monkeypatch.setattr(limits, "check_row_cap", lambda *a, **k: None) + + def _current_user(db=Depends(get_db)): + return db.get(models.User, user_id) + + app.dependency_overrides[auth.get_current_user] = _current_user + + c = TestClient(app) + c.adv_id = adv_id + try: + yield c + finally: + app.dependency_overrides.clear() + adventures.turns._active_turns.clear() + Base.metadata.drop_all(bind=engine) + + +def _state(adv_id): + db = SessionLocal() + try: + return db.get(models.Adventure, adv_id).script_state + finally: + db.close() + + +def _play(client, type_="do", text="look around"): + r = client.post(f"/api/adventures/{client.adv_id}/actions", json={"type": type_, "text": text}) + assert r.status_code == 200, r.text + return r + + +def test_play_then_undo_reverts_gold(client): + assert _state(client.adv_id) == {} + _play(client) + assert _state(client.adv_id) == {"gold": 10} + + r = client.post(f"/api/adventures/{client.adv_id}/undo") + assert r.status_code == 200, r.text + assert _state(client.adv_id) == {} # gold reverted to zero + + +def test_two_turns_then_undo_reverts_only_last(client): + _play(client) + _play(client) + assert _state(client.adv_id) == {"gold": 20} + + client.post(f"/api/adventures/{client.adv_id}/undo") + assert _state(client.adv_id) == {"gold": 10} # back to after turn 1, not 0 + + +def test_retry_does_not_double_apply_gold(client): + _play(client) + assert _state(client.adv_id) == {"gold": 10} + + # Before the fix, this produced 20 because the output hook ran twice. + # Now it stays 10. + r = client.post(f"/api/adventures/{client.adv_id}/retry") + assert r.status_code == 200, r.text + assert _state(client.adv_id) == {"gold": 10} + + +def test_retry_then_undo_still_clean(client): + _play(client) + client.post(f"/api/adventures/{client.adv_id}/retry") + assert _state(client.adv_id) == {"gold": 10} + client.post(f"/api/adventures/{client.adv_id}/undo") + assert _state(client.adv_id) == {} diff --git a/backend/tests/test_worldstate.py b/backend/tests/test_worldstate.py new file mode 100644 index 0000000..a9528b4 --- /dev/null +++ b/backend/tests/test_worldstate.py @@ -0,0 +1,305 @@ +"""Unit tests for the RPG world-state engine (Phase 12): delta extraction and +the code that enforces clamp, cooldown, and milestone rules. + + python -m pytest tests/test_worldstate.py -v +""" +from app import worldstate as w + +SCHEMA = { + "world": {"day": {"type": "counter", "min": 1, "initial": 1}}, + "player": { + "hp": {"min": 0, "max": 100, "initial": 100, "max_delta_per_turn": 30, + "bands": [[0, 20, "very weak"], [20, 40, "hurt"], + [40, 60, "minor damage"], [60, 90, "healthy"], + [90, 100, "full health"]]}, + "outfit": {"type": "text", "initial": "traveling clothes", "desc": "What the player is wearing"}, + }, + "npcs": { + "gwen": { + "name": "Gwen", + "keys": "Gwen, ranger", + "desc": "A loyal ranger", + "stats": {"trust": {"min": -100, "max": 100, "initial": 0, "cooldown": 2}}, + }, + "drake": { + "name": "The Drake", + "stats": {"ferocity": {"min": 0, "max": 100, "initial": 50}}, + }, + }, + "flags": { + "has_key": {"desc": "Holds the key", "initial": False}, + "disguised": {"desc": "In disguise"}, + }, + "milestones": {"rescue_gwen": {"desc": "Rescue Gwen"}}, +} + + +def fresh(): + return w.instantiate(SCHEMA) + + +def test_instantiate_uses_initials(): + ws = fresh() + assert ws["world"] == {"day": 1} + assert ws["player"] == {"hp": 100, "outfit": "traveling clothes"} + # Each defined NPC is instantiated up front with its own stats. + assert ws["npc"] == {"gwen": {"trust": 0}, "drake": {"ferocity": 50}} + assert ws["milestones"] == {} + + +def test_text_stat_replaces_value(): + ws, report = w.apply_delta(fresh(), SCHEMA, {"player.outfit": "muddy cloak"}, 1) + assert ws["player"]["outfit"] == "muddy cloak" + assert report["applied"][0] == {"path": "player.outfit", "old": "traveling clothes", "new": "muddy cloak"} + + +def test_text_stat_rejects_non_string(): + ws, report = w.apply_delta(fresh(), SCHEMA, {"player.outfit": 5}, 1) + assert ws["player"]["outfit"] == "traveling clothes" + assert report["rejected"][0]["reason"] == "not a string" + + +def test_text_stat_noop_when_unchanged(): + ws, report = w.apply_delta(fresh(), SCHEMA, {"player.outfit": "traveling clothes"}, 1) + assert report["applied"] == [] + + +def test_per_npc_distinct_stats(): + ws, _ = w.apply_delta(fresh(), SCHEMA, {"npc.drake.ferocity": 20}, 1) + assert ws["npc"]["drake"]["ferocity"] == 70 + # gwen has no `ferocity` stat, and drake has no `trust` stat. Cross paths + # are rejected. + ws, report = w.apply_delta(ws, SCHEMA, {"npc.gwen.ferocity": 5, "npc.bogus.trust": 5}, 2) + reasons = {r["reason"] for r in report["rejected"]} + assert reasons == {"unknown npc stat", "unknown npc"} + + +def test_has_schema(): + assert w.has_schema(SCHEMA) + assert not w.has_schema(None) + assert not w.has_schema({}) + assert not w.has_schema({"npc_card_types": ["npc"]}) # config only, no stats + + +def test_max_delta_per_turn_clamps(): + ws, report = w.apply_delta(fresh(), SCHEMA, {"player.hp": -50}, 5) + assert ws["player"]["hp"] == 70 # -50 capped to -30 + assert report["clamped"] + + +def test_clamp_to_min(): + ws, _ = w.apply_delta(fresh(), SCHEMA, {"player.hp": -30}, 1) + ws, _ = w.apply_delta(ws, SCHEMA, {"player.hp": -30}, 3) + ws, _ = w.apply_delta(ws, SCHEMA, {"player.hp": -30}, 5) + ws, report = w.apply_delta(ws, SCHEMA, {"player.hp": -30}, 7) + assert ws["player"]["hp"] == 0 # 100-30-30-30-30 clamps at min 0 + assert report["clamped"] + + +def test_counter_rejects_negative(): + ws, report = w.apply_delta(fresh(), SCHEMA, {"world.day": -1}, 3) + assert ws["world"]["day"] == 1 + assert report["rejected"][0]["reason"] == "counter can't decrease" + ws, _ = w.apply_delta(ws, SCHEMA, {"world.day": 1}, 4) + assert ws["world"]["day"] == 2 + + +def test_npc_cooldown(): + ws, _ = w.apply_delta(fresh(), SCHEMA, {"npc.gwen.trust": 10}, 7) + assert ws["npc"]["gwen"]["trust"] == 10 + # cooldown 2: another change at index 8 is too soon. + ws, report = w.apply_delta(ws, SCHEMA, {"npc.gwen.trust": 10}, 8) + assert ws["npc"]["gwen"]["trust"] == 10 + assert report["rejected"][0]["reason"] == "cooldown" + # far enough later, it applies. + ws, _ = w.apply_delta(ws, SCHEMA, {"npc.gwen.trust": 10}, 10) + assert ws["npc"]["gwen"]["trust"] == 20 + + +def test_milestone_sticky(): + ws, report = w.apply_delta(fresh(), SCHEMA, {"milestones.rescue_gwen": True}, 9) + assert ws["milestones"]["rescue_gwen"] == {"reached": True, "at": 9} + assert report["applied"] + # second set is a silent no-op. + ws, report = w.apply_delta(ws, SCHEMA, {"milestones.rescue_gwen": True}, 11) + assert ws["milestones"]["rescue_gwen"]["at"] == 9 + assert not report["applied"] + # false is ignored. + ws, report = w.apply_delta(ws, SCHEMA, {"milestones.rescue_gwen": False}, 13) + assert ws["milestones"]["rescue_gwen"]["reached"] is True + + +def test_flags_toggle_both_ways(): + ws = fresh() + assert ws["flags"] == {"has_key": False, "disguised": False} # initials + ws, report = w.apply_delta(ws, SCHEMA, {"flags.has_key": True}, 1) + assert ws["flags"]["has_key"] is True + assert report["applied"] + # Setting it back to false works, because flags are two-way, unlike + # sticky milestones. + ws, _ = w.apply_delta(ws, SCHEMA, {"flags.has_key": False}, 2) + assert ws["flags"]["has_key"] is False + # setting to the same value is a no-op. + ws, report = w.apply_delta(ws, SCHEMA, {"flags.has_key": False}, 3) + assert not report["applied"] + + +def test_flag_rejects_non_bool_and_unknown(): + ws, report = w.apply_delta(fresh(), SCHEMA, {"flags.has_key": 1, "flags.nope": True}, 1) + reasons = {r["reason"] for r in report["rejected"]} + assert reasons == {"not a boolean", "unknown flag"} + assert ws["flags"]["has_key"] is False + + +def test_override_sets_absolute_value_bypassing_cap(): + # A manual override does not obey max_delta_per_turn the way a turn + # does. It sets the value directly, though the value is still clamped + # to min/max. + ws, report = w.apply_override(fresh(), SCHEMA, {"player.hp": 10}) + assert ws["player"]["hp"] == 10 + assert report["applied"] == [{"path": "player.hp", "old": 100, "new": 10}] + ws, report = w.apply_override(ws, SCHEMA, {"player.hp": 999}) + assert ws["player"]["hp"] == 100 # still clamps to max + + +def test_override_bypasses_cooldown_and_counter_rule(): + ws, _ = w.apply_override(fresh(), SCHEMA, {"npc.gwen.trust": 10}) + # a second override immediately after would be blocked by cooldown under + # apply_delta, but override ignores cooldown entirely. + ws, report = w.apply_override(ws, SCHEMA, {"npc.gwen.trust": -5}) + assert ws["npc"]["gwen"]["trust"] == -5 + assert report["applied"] + # counters can be set down directly too (a correction, not a turn). + ws, report = w.apply_override(ws, SCHEMA, {"world.day": 1}) + assert ws["world"]["day"] == 1 + assert report["applied"] + + +def test_override_text_stat_replaces(): + ws, report = w.apply_override(fresh(), SCHEMA, {"player.outfit": "knight's plate"}) + assert ws["player"]["outfit"] == "knight's plate" + assert report["applied"] + + +def test_override_milestone_toggles_both_ways(): + ws, _ = w.apply_override(fresh(), SCHEMA, {"milestones.rescue_gwen": True}) + assert ws["milestones"]["rescue_gwen"]["reached"] is True + # unlike apply_delta, override can un-set a milestone. + ws, report = w.apply_override(ws, SCHEMA, {"milestones.rescue_gwen": False}) + assert "rescue_gwen" not in ws["milestones"] + assert report["applied"] + + +def test_override_rejects_unknown_and_bad_type(): + ws, report = w.apply_override(fresh(), SCHEMA, { + "player.hp": "not a number", + "npc.bogus.trust": 5, + "flags.nope": True, + }) + reasons = {r["path"]: r["reason"] for r in report["rejected"]} + assert reasons["player.hp"] == "not a number" + assert reasons["npc.bogus.trust"] == "unknown npc" + assert reasons["flags.nope"] == "unknown flag" + assert ws["player"]["hp"] == 100 # untouched + + +def test_reference_includes_desc_and_bands_independently(): + guide = w.render_reference(SCHEMA) + # hp has both a description and a set of value bands. + assert "very weak" in guide and "range 0–100" in guide + # A counter like day has no desc or bands, so it contributes nothing + # here. Flags do show their desc. + assert "flags.has_key — Holds the key." in guide + # NPCs contribute their own description and per-NPC stat lines. + assert "NPC Gwen (npc.gwen) — A loyal ranger." in guide + assert "npc.gwen.trust" in guide and "npc.drake.ferocity" in guide + + +def test_reference_names_every_stat_by_its_path(): + """The guide is the model's only complete list of what exists, so it has + to name each stat the way a state block must name it. Naming a stat after + its owner's display name ("Trainer Milo active_status") left the model to + build the path itself, and the paths it built were refused as stats and + characters that do not exist.""" + guide = w.render_reference(SCHEMA) + assert "player.hp" in guide + assert "player.outfit" in guide + # The old wording, which read as prose rather than as an address. + assert "Gwen trust" not in guide + assert "The Drake ferocity" not in guide + + +def test_reference_names_an_npc_that_has_no_description(): + """The live values name an NPC only while a scene mentions them, so the + guide is the only place an off-screen NPC's id is stated. The Drake has no + `desc`, and used to reach the model as "The Drake ferocity" alone.""" + guide = w.render_reference(SCHEMA) + assert "NPC The Drake (npc.drake)." in guide + + +def test_reference_marks_free_text_stats_the_way_the_emit_rule_names_them(): + """`EMIT_RULE` tells the model that a stat "marked (free text) in the stat + guide" takes a whole value rather than a delta, so the guide has to carry + that marker literally.""" + assert "(free text)" in w.EMIT_RULE + guide = w.render_reference(SCHEMA) + assert "player.outfit (free text) — What the player is wearing." in guide + + +def test_reference_lists_a_free_text_stat_with_no_description(): + """The marker alone is the entry. Dropping the line would leave the model + sending a number for a stat that holds a string.""" + schema = {"player": {"holding": {"type": "text", "initial": ""}}} + assert "player.holding (free text)." in w.render_reference(schema) + + +def test_unknown_paths_rejected_not_fatal(): + ws, report = w.apply_delta(fresh(), SCHEMA, {"player.stamina": -5, "bogus": 1}, 2) + reasons = {r["reason"] for r in report["rejected"]} + assert reasons == {"unknown stat", "unknown path"} + assert ws["player"]["hp"] == 100 # untouched + + +def test_extract_fenced_delta_tolerates_mess(): + text = 'You strike.\n\n```state\n{"player.hp": -15, "npc.12.trust": +5,}\n```' + clean, delta = w.extract_delta(text) + assert clean == "You strike." + assert delta == {"player.hp": -15, "npc.12.trust": 5} + + +def test_extract_no_block(): + clean, delta = w.extract_delta("Just prose that ends normally.") + assert delta == {} + assert clean == "Just prose that ends normally." + + +def test_extract_prose_ending_in_brace_not_eaten(): + # A bare object with no dotted keys is not a delta, so the text stays + # as is. + clean, delta = w.extract_delta('He said {this}') + assert delta == {} + assert clean == "He said {this}" + + +def test_band_label(): + d = SCHEMA["player"]["hp"] + assert w.band_label(d, 10) == "very weak" + assert w.band_label(d, 55) == "minor damage" + assert w.band_label(d, 100) == "full health" # inclusive top edge + + +def test_render_delta_block_roundtrips_through_extract(): + # A rendered block re-injected into history must parse back to the same delta, + # so the model sees valid examples of its own emit format. + delta = {"player.hp": -15, "milestones.escaped": True} + block = w.render_delta_block(delta) + assert block.startswith("```state") + clean, parsed = w.extract_delta(f"You flee into the night.\n{block}") + assert clean == "You flee into the night." + assert parsed == delta + + +def test_render_delta_block_empty_is_blank(): + # No change this turn -> nothing appended to history. + assert w.render_delta_block({}) == "" + assert w.render_delta_block(None) == "" diff --git a/backend/tests/test_worldstate_integration.py b/backend/tests/test_worldstate_integration.py new file mode 100644 index 0000000..d6473d9 --- /dev/null +++ b/backend/tests/test_worldstate_integration.py @@ -0,0 +1,182 @@ +"""End-to-end HTTP test for RPG world state (Phase 12): a scenario with a +stat_schema, a turn whose (faked) AI reply carries a state delta block, and +undo rolling the world state back. + + python -m pytest tests/test_worldstate_integration.py -v +""" +import pytest +from fastapi import Depends +from fastapi.testclient import TestClient + +from app import auth, limits, models, worldstate +from app.database import Base, SessionLocal, engine, get_db +from app.main import app +from app.routers import adventures + +from fakes import ScriptedProvider + +SCHEMA = { + "player": {"hp": {"min": 0, "max": 100, "initial": 100, "max_delta_per_turn": 30}}, + "npcs": { + "gwen": { + "name": "Gwen", "keys": "Gwen", + "desc": "A loyal ranger ally.", + "stats": {"trust": {"min": -100, "max": 100, "initial": 0}}, + }, + }, + "flags": {"alarm": {"desc": "The enemy is alerted", "initial": False}}, + "milestones": {"win": {"desc": "Win the fight"}}, +} + +# The faked model narrates and appends a delta that exceeds the per-turn +# cap, so the test can confirm the engine clamps it. It also flips a flag +# and completes a milestone. +AI_REPLY = ( + "The goblin's blade bites deep and Gwen nods at your resolve.\n\n" + '```state\n{"player.hp": -80, "npc.gwen.trust": 15, "flags.alarm": true, "milestones.win": true}\n```' +) + + +@pytest.fixture() +def client(monkeypatch): + Base.metadata.create_all(bind=engine) + setup = SessionLocal() + user = models.User(is_guest=False, email="rpg@example.com") + setup.add(user) + setup.flush() + setup.add(models.Settings(user_id=user.id, api_key="enc:dummy", model="test-model")) + scenario = models.Scenario(user_id=user.id, title="Dungeon", stat_schema=SCHEMA) + setup.add(scenario) + setup.flush() + adv = models.Adventure( + user_id=user.id, scenario_id=scenario.id, title="Run", + world_state=worldstate.instantiate(SCHEMA), + ) + setup.add(adv) + setup.flush() + # "Gwen" in the story text makes her NPC in-scene (matches the "gwen" npc's keys). + setup.add(models.Action(adventure_id=adv.id, type="start", + text="You face a goblin. Gwen watches.")) + setup.commit() + adv_id, user_id = adv.id, user.id + setup.close() + + ScriptedProvider.replies = [AI_REPLY] + monkeypatch.setattr(adventures.turns, "OpenAICompatibleProvider", ScriptedProvider) + monkeypatch.setattr(auth, "resolve_provider_config", lambda s: auth.ProviderConfig( + "http://fake", "k", "test-model", False)) + monkeypatch.setattr(limits, "rate_limit", lambda *a, **k: None) + monkeypatch.setattr(limits, "check_row_cap", lambda *a, **k: None) + + def _current_user(db=Depends(get_db)): + return db.get(models.User, user_id) + + app.dependency_overrides[auth.get_current_user] = _current_user + + c = TestClient(app) + c.adv_id = adv_id + try: + yield c + finally: + app.dependency_overrides.clear() + adventures.turns._active_turns.clear() + Base.metadata.drop_all(bind=engine) + + +def _world(adv_id): + db = SessionLocal() + try: + return db.get(models.Adventure, adv_id).world_state + finally: + db.close() + + +def _last_ai_text(adv_id): + db = SessionLocal() + try: + adv = db.get(models.Adventure, adv_id) + return adv.actions[-1].text + finally: + db.close() + + +def _play(client, text="attack the goblin"): + r = client.post(f"/api/adventures/{client.adv_id}/actions", json={"type": "do", "text": text}) + assert r.status_code == 200, r.text + return r + + +def test_turn_applies_clamped_delta_and_strips_block(client): + _play(client) + ws = _world(client.adv_id) + assert ws["player"]["hp"] == 70 # -80 capped to -30 + assert ws["npc"]["gwen"]["trust"] == 15 + assert ws["flags"]["alarm"] is True + assert ws["milestones"]["win"]["reached"] is True + # The state block is not shown to the player. + assert "```state" not in _last_ai_text(client.adv_id) + assert "goblin's blade" in _last_ai_text(client.adv_id) + # The raw model reply, including the block, is kept for the Insights view. + db = SessionLocal() + try: + snap = db.get(models.Adventure, client.adv_id).actions[-1].context_snapshot + finally: + db.close() + assert "```state" in snap["raw_output"] + assert '"player.hp": -80' in snap["raw_output"] + + +def test_action_world_changes_summary(client): + _play(client) + db = SessionLocal() + try: + changes = db.get(models.Adventure, client.adv_id).actions[-1].world_changes + finally: + db.close() + by_label = {c["label"]: c for c in changes} + assert by_label["hp"]["delta"] == -30 # clamped stat, signed delta + assert by_label["gwen trust"]["delta"] == 15 # npc.. -> "id stat" + assert by_label["alarm"] == {"kind": "flag", "label": "alarm", "on": True} + assert by_label["win"]["kind"] == "milestone" + + +def test_world_state_endpoint(client): + _play(client) + r = client.get(f"/api/adventures/{client.adv_id}/world-state") + assert r.status_code == 200, r.text + body = r.json() + assert body["schema"]["player"]["hp"]["max"] == 100 + assert body["state"]["player"]["hp"] == 70 + + +def test_override_world_state_endpoint(client): + r = client.put(f"/api/adventures/{client.adv_id}/world-state", + json={"player.hp": 5, "flags.alarm": True, "npc.bogus.trust": 1}) + assert r.status_code == 200, r.text + body = r.json() + assert body["state"]["player"]["hp"] == 5 + assert body["state"]["flags"]["alarm"] is True + assert body["report"]["rejected"][0]["reason"] == "unknown npc" + # persisted to the DB, not just the response. + assert _world(client.adv_id)["player"]["hp"] == 5 + + # This bypasses max_delta_per_turn (30) because it is a direct correction, not a turn. + r = client.put(f"/api/adventures/{client.adv_id}/world-state", json={"player.hp": 100}) + assert r.json()["state"]["player"]["hp"] == 100 + + +def test_undo_reverts_world_state(client): + _play(client) + assert _world(client.adv_id)["player"]["hp"] == 70 + r = client.post(f"/api/adventures/{client.adv_id}/undo") + assert r.status_code == 200, r.text + assert _world(client.adv_id)["player"]["hp"] == 100 # back to initial + assert _world(client.adv_id)["milestones"] == {} + + +def test_retry_does_not_double_apply(client): + _play(client) + assert _world(client.adv_id)["player"]["hp"] == 70 + r = client.post(f"/api/adventures/{client.adv_id}/retry") + assert r.status_code == 200, r.text + assert _world(client.adv_id)["player"]["hp"] == 70 # not 40 diff --git a/backend/tools/__init__.py b/backend/tools/__init__.py new file mode 100644 index 0000000..3ca1587 --- /dev/null +++ b/backend/tools/__init__.py @@ -0,0 +1,4 @@ +"""Developer tools that are not part of the deployed app. + +Nothing under `app/` may import from here. +""" diff --git a/backend/tools/branch_fixture.py b/backend/tools/branch_fixture.py new file mode 100644 index 0000000..582ba46 --- /dev/null +++ b/backend/tools/branch_fixture.py @@ -0,0 +1,174 @@ +"""Build a bootable adventure that has actually gone two ways (Phase 14, SP7). + +`tools.stress_session --keep` answers the scrolling question and nothing else. +Its world state and script state are both empty, so it cannot show whether a +branch switch restores them. This script builds the small counterpart: a stat +schema, a script that counts gold, two attempts at one turn that do visibly +different damage, a fork, and a hand-written memory on each branch. + +**The two branches are deliberately the same length.** Both paths are five +actions, so `actions.length` is identical either side of a switch. That is what +makes this fixture worth keeping: every panel that refreshed on the length of +the story looked correct until something asked it to tell two equal-length +branches apart, and then four of them went on showing the branch just left. + + cd backend + .venv/Scripts/python.exe -m tools.branch_fixture /tmp/branches.db + AIDND_DB_PATH=/tmp/branches.db .venv/Scripts/python.exe \\ + -m uvicorn app.main:app --port 8010 + +Then open http://127.0.0.1:8010/. The SPA is served out of `frontend/dist`, so +run `npm run build` first if that directory is stale. Expect hp 60 on "The hard +way down" and hp 95 on the fork, and expect both to change as soon as you +switch. + +No LLM is called: the provider is scripted and its replies carry their own +fenced `state` blocks. +""" +import os +import sys + +# Run from `backend/`; Python puts this script's own directory on the path, not +# the working one. +sys.path.insert(0, os.getcwd()) + +OUT = sys.argv[1] if len(sys.argv) > 1 else "branchfixture.db" +if os.path.exists(OUT): + os.remove(OUT) +os.environ["AIDND_DB_PATH"] = OUT +os.environ.pop("AIDND_DATABASE_URL", None) +os.environ.pop("DATABASE_URL", None) +os.environ.pop("AIDND_MULTI_USER", None) + +from fastapi.testclient import TestClient # noqa: E402 +from sqlalchemy import text # noqa: E402 + +from app import auth, limits, models # noqa: E402 +from app.database import Base, SessionLocal, engine # noqa: E402 +from app.main import app # noqa: E402 +from app.migrations import LATEST_VERSION # noqa: E402 +from app.routers import adventures # noqa: E402 + +SCHEMA = { + "player": { + "hp": {"min": 0, "max": 100, "initial": 100}, + "mana": {"min": 0, "max": 50, "initial": 50, "cooldown": 2}, + } +} + +GOLD_SCRIPT = """ +const modifier = (text) => { + state.gold = (state.gold || 0) + 10; + return { text }; +}; +modifier(text); +""" + + +class ScriptedProvider: + replies: list = [] + # The turn engine records the cost of the call, so a stand-in provider has + # to carry this attribute even when it never calls anything. + last_usage = None + calls = 0 + + def __init__(self, *a, **k): + pass + + async def generate(self, parts, *, temperature, max_tokens): + i = min(ScriptedProvider.calls, len(ScriptedProvider.replies) - 1) + ScriptedProvider.calls += 1 + yield ("text", ScriptedProvider.replies[i]) + + +Base.metadata.create_all(bind=engine) +db = SessionLocal() +# Local mode looks for the row with email IS NULL and is_guest false. +user = models.User(is_guest=False, email=None) +db.add(user) +db.flush() +db.add(models.Settings(user_id=user.id, api_key="enc:dummy", model="test-model")) +scenario = models.Scenario(user_id=user.id, title="Thornwick", stat_schema=SCHEMA) +db.add(scenario) +db.flush() +adv = models.Adventure( + user_id=user.id, title="The Hollow Beneath Thornwick", scenario_id=scenario.id, + script_state={}, world_state={"player": {"hp": 100, "mana": 50}}, + memory_bank_enabled=True, +) +db.add(adv) +db.flush() +db.add(models.Action( + adventure_id=adv.id, type="start", + text="The cellar door has been shut since your grandmother died. " + "Tonight the lantern is lit and the key is in your hand.")) +db.add(models.AdventureScript( + adventure_id=adv.id, position=0, enabled=True, name="Gold", output_js=GOLD_SCRIPT)) +db.commit() +adv_id = adv.id +db.close() + +adventures.turns.OpenAICompatibleProvider = ScriptedProvider +auth.resolve_provider_config = lambda s: auth.ProviderConfig( + "http://fake", "k", "test-model", False) +limits.rate_limit = lambda *a, **k: None +limits.check_row_cap = lambda *a, **k: None + +client = TestClient(app) +base = f"/api/adventures/{adv_id}" + + +def play(text): + r = client.post(f"{base}/actions", json={"type": "do", "text": text}) + assert r.status_code == 200, r.text + + +# Turn one is a scratch, retried into a beating. The story continues from the +# beating, so the scratch is the attempt left behind. The two differ by 35 hit +# points, which is the number a switch has to restore. +ScriptedProvider.replies = [ + "You ease the door open and a nail catches your wrist.\n" + "```state\n{\"player.hp\": -5}\n```", + "The door slams back and takes you off your feet, down four steps onto " + "stone.\n```state\n{\"player.hp\": -40}\n```", + "The cellar is colder than the night outside, and it smells faintly sweet.", + "Something moves along the far wall, keeping the dark between you.", +] +play("light the lantern and open the cellar door") +r = client.post(f"{base}/retry") +assert r.status_code == 200, r.text +play("go down, one hand on the wall") + +db = SessionLocal() +discarded = [ + a.id for a in db.query(models.Action) + .filter(models.Action.adventure_id == adv_id, models.Action.type == "ai") + .order_by(models.Action.id) if not a.live +][0] +db.close() + +client.post(f"{base}/memories", json={ + "text": "Fell down the cellar stairs; badly hurt, moving slowly."}) + +r = client.post(f"{base}/actions/{discarded}/fork") +assert r.status_code == 200, r.text +play("keep the lantern high and look for the far wall") +client.post(f"{base}/memories", json={ + "text": "Only a scratched wrist; still quick on your feet."}) + +# Leave the reader on the original line, so the difference is one click away. +root = client.get(f"{base}/branches").json()[0]["id"] +client.patch(f"{base}/branches/{root}", json={"name": "The hard way down"}) +client.post(f"{base}/branches/{root}/switch") + +with engine.begin() as conn: + conn.execute(text(f"PRAGMA user_version = {LATEST_VERSION}")) + +for b in client.get(f"{base}/branches").json(): + print(f" branch {b['id']}: name={b['name']!r} fork_depth={b['fork_depth']} " + f"own={b['own_actions']} head={b['is_head']}") +state = client.get(f"{base}/world-state").json()["state"] +print(f" world state on head: {state.get('player')}") +print(f" script state on head: {client.get(f'{base}/script-state').json()['state']}") +print(f" memories: {len(client.get(f'{base}/memories').json())}") +print(f"fixture written: {OUT}") diff --git a/backend/tools/claude_shim.py b/backend/tools/claude_shim.py new file mode 100644 index 0000000..c23e3a8 --- /dev/null +++ b/backend/tools/claude_shim.py @@ -0,0 +1,272 @@ +"""An OpenAI-compatible endpoint backed by the local `claude` CLI. + +Point AI-DnD's Settings at `http://127.0.0.1:8787/v1` and the turn engine talks +to Claude through your Claude Code subscription instead of an API key. Use it to +play the demos against a real model when you do not want to spend API credit, +and to check prompt changes against the model the deployed app actually meets. +The shim speaks the small part of the OpenAI protocol the app uses: a model +listing, and `chat/completions` in both streaming and non-streaming form. + +Each request spawns one `claude --print` process. The CLI holds no conversation +state here, which suits AI-DnD, because the app assembles the whole prompt every +turn and expects a stateless endpoint. + +Embeddings are not served. Leave the embedding model blank in Settings, or point +the memory bank at a real endpoint. + +To use it: + +1. Start the shim from the backend virtualenv, which already has FastAPI and + uvicorn: + + backend/.venv/Scripts/python.exe tools/claude_shim.py + +2. In Settings, set the provider to OpenAI-compatible, the base URL to + `http://127.0.0.1:8787/v1`, and the API key to any non-empty string. The + shim ignores the key. It authenticates as you, through the CLI. + +3. Pick `sonnet` as the model, then run the connection test. + +Set the reasoning budget to `0` or `-1`. A positive budget sends a +`reasoning.max_tokens` field that Claude 5 models reject with a 400. + +Options: + + --port Port to listen on. Defaults to 8787. + --claude Path to the `claude` executable. Defaults to `$AIDND_CLAUDE_BIN`, + then to whatever is on `PATH`, then to the per-user install + under `~/.local/bin`. + +Run this only against a local backend. The endpoint has no authentication, and +anything that reaches it spends your Claude subscription quota. The SSRF guard +in `app/netguard.py` blocks localhost endpoints when `AIDND_MULTI_USER` is set, +so a deployed instance cannot be pointed here. +""" +import argparse +import asyncio +import json +import os +import shutil +import tempfile +import time +import uuid + +import uvicorn +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, StreamingResponse + + +def find_claude() -> str: + """Returns the path to the `claude` executable. + + The per-user install is the last resort, because `PATH` is what a developer + controls. Windows needs the `.exe` spelled out, since `create_subprocess_exec` + does no extension search. + """ + override = os.environ.get("AIDND_CLAUDE_BIN") + if override: + return override + found = shutil.which("claude") + if found: + return found + for name in ("claude.exe", "claude"): + candidate = os.path.expanduser(os.path.join("~", ".local", "bin", name)) + if os.path.exists(candidate): + return candidate + return "claude" + + +CLAUDE = find_claude() +DEFAULT_PORT = 8787 + +# The listing the Settings page shows. The CLI accepts both the aliases and the +# full ids, so both are offered. +MODELS = [ + "opus", + "sonnet", + "haiku", + "claude-opus-5", + "claude-sonnet-5", + "claude-haiku-4-5", +] + +# Flags that strip the coding agent down to a text generator. `--system-prompt-file` +# replaces Claude Code's own system prompt, `--restricted` removes the tools that +# run commands, and the empty MCP config keeps the connected servers out of the +# tool list. An empty `--mcp-config` value is rejected: the key has to be present. +BASE_FLAGS = [ + "--print", + "--restricted", + "--strict-mcp-config", + "--mcp-config", '{"mcpServers": {}}', + "--disallowed-tools", + "Read Write Edit Glob Grep WebSearch WebFetch Task Skill ToolSearch", +] + +app = FastAPI() + + +@app.get("/v1/models") +async def list_models(): + return {"object": "list", "data": [{"id": m, "object": "model"} for m in MODELS]} + + +@app.post("/v1/embeddings") +async def embeddings(): + return JSONResponse( + {"error": {"message": "This shim does not serve embeddings. Leave the " + "embedding model blank, or point the memory bank " + "at a real endpoint."}}, + status_code=501, + ) + + +def split_prompt(messages: list[dict]) -> tuple[str, str]: + """Splits an OpenAI message list into a system prompt and a user prompt. + + The system messages become the CLI's `--system-prompt-file`. Everything else + is joined into one prompt sent on stdin. AI-DnD sends exactly one system and + one user message per turn, so the labeling only matters for the AI Chat page. + """ + system_parts, turn_parts = [], [] + for m in messages: + content = m.get("content") or "" + if isinstance(content, list): # Content-block form, if a caller uses it. + content = "".join(b.get("text", "") for b in content if isinstance(b, dict)) + if m.get("role") == "system": + system_parts.append(content) + elif len(messages) <= 2: + turn_parts.append(content) + else: + turn_parts.append(f"{m.get('role', 'user').title()}: {content}") + return "\n\n".join(system_parts), "\n\n".join(turn_parts) + + +async def run_claude(system: str, prompt: str, model: str): + """Yields `(kind, text)` pairs from one `claude --print` run. + + `kind` is `"text"` for narration, `"usage"` for the final token accounting, + and `"error"` for a failure the CLI reported in its result record. + """ + fd, sys_path = tempfile.mkstemp(suffix=".txt", text=True) + with os.fdopen(fd, "w", encoding="utf-8") as fh: + fh.write(system or "You are a helpful assistant.") + try: + args = [ + CLAUDE, *BASE_FLAGS, + "--system-prompt-file", sys_path, + "--model", model, + "--output-format", "stream-json", + "--include-partial-messages", + "--verbose", + ] + proc = await asyncio.create_subprocess_exec( + *args, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=tempfile.gettempdir(), + ) + proc.stdin.write(prompt.encode("utf-8")) + await proc.stdin.drain() + proc.stdin.close() + + async for raw in proc.stdout: + line = raw.decode("utf-8", errors="replace").strip() + if not line: + continue + try: + rec = json.loads(line) + except ValueError: + continue + if rec.get("type") == "stream_event": + event = rec.get("event") or {} + if event.get("type") == "content_block_delta": + delta = event.get("delta") or {} + if delta.get("type") == "text_delta": + yield "text", delta.get("text", "") + elif rec.get("type") == "result": + if rec.get("is_error"): + yield "error", str(rec.get("result") or "claude failed") + usage = rec.get("usage") or {} + yield "usage", { + "prompt_tokens": (usage.get("input_tokens", 0) + + usage.get("cache_read_input_tokens", 0) + + usage.get("cache_creation_input_tokens", 0)), + "completion_tokens": usage.get("output_tokens", 0), + "total_tokens": 0, + "prompt_tokens_details": { + "cached_tokens": usage.get("cache_read_input_tokens", 0), + }, + "cost_usd": rec.get("total_cost_usd"), + } + await proc.wait() + if proc.returncode != 0: + detail = (await proc.stderr.read()).decode("utf-8", errors="replace") + yield "error", f"claude exited {proc.returncode}: {detail[:500]}" + finally: + os.unlink(sys_path) + + +@app.post("/v1/chat/completions") +async def chat_completions(request: Request): + body = await request.json() + model = body.get("model") or "sonnet" + system, prompt = split_prompt(body.get("messages") or []) + cid = f"chatcmpl-{uuid.uuid4().hex[:24]}" + created = int(time.time()) + + if not body.get("stream"): + text, usage, error = "", {}, None + async for kind, value in run_claude(system, prompt, model): + if kind == "text": + text += value + elif kind == "usage": + usage = value + else: + error = value + if error and not text: + return JSONResponse({"error": {"message": error}}, status_code=502) + return { + "id": cid, "object": "chat.completion", "created": created, "model": model, + "choices": [{"index": 0, "finish_reason": "stop", + "message": {"role": "assistant", "content": text}}], + "usage": usage, + } + + async def sse(): + def chunk(delta: dict, finish=None, usage=None) -> str: + payload = { + "id": cid, "object": "chat.completion.chunk", "created": created, + "model": model, + "choices": [{"index": 0, "delta": delta, "finish_reason": finish}], + } + if usage: + payload["usage"] = usage + return f"data: {json.dumps(payload)}\n\n" + + yield chunk({"role": "assistant", "content": ""}) + sent = False + async for kind, value in run_claude(system, prompt, model): + if kind == "text" and value: + sent = True + yield chunk({"content": value}) + elif kind == "usage": + yield chunk({}, finish="stop", usage=value) + elif kind == "error" and not sent: + yield chunk({"content": f"[shim error] {value}"}, finish="stop") + yield "data: [DONE]\n\n" + + return StreamingResponse(sse(), media_type="text/event-stream", + headers={"Cache-Control": "no-cache"}) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--port", type=int, default=DEFAULT_PORT) + parser.add_argument("--claude", default=CLAUDE, help="Path to the claude executable.") + args = parser.parse_args() + CLAUDE = args.claude + print(f"claude binary: {CLAUDE}") + print(f"Point AI-DnD Settings at http://127.0.0.1:{args.port}/v1") + uvicorn.run(app, host="127.0.0.1", port=args.port, log_level="info") diff --git a/backend/tools/dbmeter.py b/backend/tools/dbmeter.py new file mode 100644 index 0000000..d82443f --- /dev/null +++ b/backend/tools/dbmeter.py @@ -0,0 +1,344 @@ +"""How many bytes the database was actually asked for. + +Query *counts* are easy to see and have never been the problem here. Both +egress blowouts this project has had were one query fetching a column nobody +read: `context_snapshot` on every action, then every memory's embedding on +every turn. Counting statements would have shown nothing wrong in either case, +and at development scale, meaning ten rows and no embeddings, so would a +stopwatch. + +So this meter measures bytes, and it measures them at the only place the truth +is available: the DBAPI cursor, after the driver has decoded a row and before +SQLAlchemy has turned it into anything. Every value that crosses that line +crossed the wire. + +Usage: + + meter = Meter() + meter.attach(engine) + with meter.scope("turn"): + ... # anything that touches the database + print(meter.render()) + +`attach()` wraps the pool's connection factory, so it reuses the engine the app +already configured rather than building a second one. Nothing about connect +args, pre-ping, or the SQLite foreign-key pragma is repeated here, and the +metered engine cannot diverge from the real one. +Existing pooled connections are dropped first, so a connection opened before +attaching cannot quietly stay unmetered. + +Sizes are of the decoded values, not of the protocol framing: the count is what +the payload weighs, and ignores per-row and per-packet overhead. Text and bytes +are exact. Numbers are charged their binary width, which is what the Postgres +binary protocol sends and close enough elsewhere. JSON is the case that matters +most and the one to be careful about: SQLite hands back the raw string (exact), +while psycopg parses `json`/`jsonb` into Python before this sees it, so the +value is re-serialised to size it. Re-serialising is within a byte or two of +the original for machine-written JSON, which is all this project stores. +""" + +from __future__ import annotations + +import json +import re +from collections import defaultdict +from contextlib import contextmanager +from dataclasses import dataclass, field + +# The table a statement is charged to. Only the first match is used: a join +# reads mostly from its driving table, and splitting a row across tables would +# need the result metadata, which is more machinery than the answer is worth. +_TABLE_RE = re.compile( + r"\b(?:FROM|JOIN|INTO|UPDATE)\s+\"?([A-Za-z_][A-Za-z_0-9]*)\"?", re.IGNORECASE +) +_WHITESPACE_RE = re.compile(r"\s+") + + +def value_bytes(value) -> int: + """The payload weight of one decoded column value.""" + if value is None: + return 0 + if isinstance(value, (bytes, bytearray)): + return len(value) + if isinstance(value, memoryview): + return value.nbytes + if isinstance(value, str): + return len(value.encode("utf-8")) + if isinstance(value, bool): + return 1 + if isinstance(value, (int, float)): + return 8 + if isinstance(value, (dict, list)): + # A JSON column the driver already parsed. psycopg parses it and SQLite + # does not. The separators match what a database emits, with no spaces. + return len(json.dumps(value, separators=(",", ":"), default=str).encode("utf-8")) + return len(str(value).encode("utf-8")) + + +def row_bytes(row) -> int: + return sum(value_bytes(v) for v in row) + + +def _table_of(statement: str) -> str: + match = _TABLE_RE.search(statement) + if match: + return match.group(1).lower() + # The statement names no table, such as BEGIN, a PRAGMA, or a savepoint. + # Group those under the keyword, so they stay countable rather than collapse + # into one "?" bucket. + head = statement.strip().split(None, 1) + return head[0].lower()[:20] if head else "(empty)" + + +def _one_line(statement: str, width: int = 132) -> str: + collapsed = _WHITESPACE_RE.sub(" ", statement).strip() + return collapsed if len(collapsed) <= width else collapsed[: width - 1] + "…" + + +@dataclass +class Tally: + statements: int = 0 + rows: int = 0 + fetched: int = 0 # bytes + + +@dataclass +class Scope: + """What one measured stretch of work asked the database for.""" + + name: str + total: Tally = field(default_factory=Tally) + by_table: dict[str, Tally] = field(default_factory=lambda: defaultdict(Tally)) + by_statement: dict[str, Tally] = field(default_factory=lambda: defaultdict(Tally)) + + def add_statement(self, statement: str) -> None: + self.total.statements += 1 + self.by_table[_table_of(statement)].statements += 1 + self.by_statement[statement].statements += 1 + + def add_rows(self, statement: str, rows: int, nbytes: int) -> None: + for tally in ( + self.total, + self.by_table[_table_of(statement)], + self.by_statement[statement], + ): + tally.rows += rows + tally.fetched += nbytes + + +class Meter: + """Collects what the metered cursors report, grouped by scope. + + Scopes nest: a statement is charged to every scope currently open, so an + inner "retrieve memories" and an outer "turn" both see it. + """ + + def __init__(self) -> None: + self._open: list[Scope] = [] + self.scopes: list[Scope] = [] + self._attached_pools: list = [] + + # -------------------------------------------------------------- recording + + @contextmanager + def scope(self, name: str): + scope = Scope(name) + self._open.append(scope) + try: + yield scope + finally: + self._open.pop() + self.scopes.append(scope) + + def note_statement(self, statement: str) -> None: + for scope in self._open: + scope.add_statement(statement) + + def note_rows(self, statement: str, rows: int, nbytes: int) -> None: + for scope in self._open: + scope.add_rows(statement, rows, nbytes) + + # ------------------------------------------------------------- attaching + + def attach(self, engine) -> None: + """Meter every connection this engine opens from now on.""" + # Drop pooled connections created before now; they were built by the + # unwrapped creator and would go on reporting nothing. + engine.dispose() + pool = engine.pool + creator = pool._creator # the factory create_engine() built from the URL + if getattr(creator, "_dbmeter", None) is self: + return + meter = self + + def metered_creator(): + return _MeteredConnection(creator(), meter) + + metered_creator._dbmeter = self + pool._creator = metered_creator + self._attached_pools.append((engine, pool, creator)) + + def detach(self) -> None: + """Put every metered engine back as it was. + + A script exits and takes the wrapping with it; a test does not, and one + test leaving the shared engine metered would go on charging bytes to a + scope nobody opened. Pooled connections are dropped again on the way + out for the same reason attach drops them on the way in. + """ + while self._attached_pools: + engine, pool, creator = self._attached_pools.pop() + pool._creator = creator + engine.dispose() + + def __enter__(self) -> "Meter": + return self + + def __exit__(self, *exc) -> None: + self.detach() + + # ------------------------------------------------------------- reporting + + def render(self, *, statements: int = 5) -> str: + return "\n".join(render_scope(s, statements=statements) for s in self.scopes) + + +# ------------------------------------------------------------ DBAPI wrappers + + +class _MeteredConnection: + """A DBAPI connection that hands out metered cursors. + + Everything else is delegated: the dialects reach for driver-specific + attributes (`isolation_level` on SQLite, `info` and `autocommit` on + psycopg) and this must stay transparent to all of them. + """ + + def __init__(self, connection, meter: Meter) -> None: + object.__setattr__(self, "_connection", connection) + object.__setattr__(self, "_meter", meter) + + def cursor(self, *args, **kwargs): + return _MeteredCursor(self._connection.cursor(*args, **kwargs), self._meter) + + def __getattr__(self, name): + return getattr(object.__getattribute__(self, "_connection"), name) + + def __setattr__(self, name, value): + setattr(self._connection, name, value) + + def __enter__(self): + self._connection.__enter__() + return self + + def __exit__(self, *exc): + return self._connection.__exit__(*exc) + + +class _MeteredCursor: + """Counts the bytes of every row handed back. + + The wrapper covers the fetch methods rather than `execute`, because what a + statement costs is not known when it is sent. `SELECT * FROM memories` and + `SELECT count(*) FROM memories` look alike going out and differ by three + megabytes coming back. + """ + + def __init__(self, cursor, meter: Meter) -> None: + self.__dict__["_cursor"] = cursor + self.__dict__["_meter"] = meter + self.__dict__["_statement"] = "" + + # -- execution + + def execute(self, statement, *args, **kwargs): + self.__dict__["_statement"] = statement + self._meter.note_statement(statement) + return self._cursor.execute(statement, *args, **kwargs) + + def executemany(self, statement, *args, **kwargs): + self.__dict__["_statement"] = statement + self._meter.note_statement(statement) + return self._cursor.executemany(statement, *args, **kwargs) + + # -- fetching + + def _charge(self, rows) -> None: + self._meter.note_rows( + self._statement, len(rows), sum(row_bytes(r) for r in rows) + ) + + def fetchone(self): + row = self._cursor.fetchone() + if row is not None: + self._charge([row]) + return row + + def fetchmany(self, *args, **kwargs): + rows = self._cursor.fetchmany(*args, **kwargs) + self._charge(rows) + return rows + + def fetchall(self): + rows = self._cursor.fetchall() + self._charge(rows) + return rows + + def __iter__(self): + # Special methods are looked up on the type, so this cannot be left to + # __getattr__ the way the rest of the driver surface is. + for row in self._cursor: + self._charge([row]) + yield row + + def __getattr__(self, name): + return getattr(self.__dict__["_cursor"], name) + + def __setattr__(self, name, value): + setattr(self.__dict__["_cursor"], name, value) + + def __enter__(self): + self._cursor.__enter__() + return self + + def __exit__(self, *exc): + return self._cursor.__exit__(*exc) + + +# --------------------------------------------------------------- formatting + + +def kb(nbytes: int) -> str: + return f"{nbytes / 1024:,.1f} kB" + + +def render_scope(scope: Scope, *, statements: int = 5) -> str: + lines = [ + "", + f"── {scope.name} " + "─" * max(0, 62 - len(scope.name)), + f" {scope.total.statements} statements · {scope.total.rows} rows · " + f"{kb(scope.total.fetched)} fetched", + ] + + if scope.by_table: + lines += ["", f" {'table':<20}{'stmts':>7}{'rows':>8}{'fetched':>16}"] + ranked = sorted( + scope.by_table.items(), key=lambda kv: kv[1].fetched, reverse=True + ) + for table, tally in ranked: + share = tally.fetched / scope.total.fetched if scope.total.fetched else 0 + lines.append( + f" {table:<20}{tally.statements:>7}{tally.rows:>8}" + f"{kb(tally.fetched):>16}{share:>7.0%}" + ) + + heavy = sorted( + scope.by_statement.items(), key=lambda kv: kv[1].fetched, reverse=True + )[:statements] + heavy = [(s, t) for s, t in heavy if t.fetched] + if heavy: + lines += ["", " heaviest statements"] + for statement, tally in heavy: + lines.append(f" {kb(tally.fetched):>14} {tally.statements}x {_one_line(statement)}") + + return "\n".join(lines) diff --git a/backend/tools/fakeprose.py b/backend/tools/fakeprose.py new file mode 100644 index 0000000..a0ca9e7 --- /dev/null +++ b/backend/tools/fakeprose.py @@ -0,0 +1,38 @@ +"""Filler text that behaves like prose under compression. + +Shared by the stress harness and the egress tests, because both now measure +something a repeated string would answer wrongly. `"x" * 20_000` compresses +about a thousandfold and English three- or fourfold, so a fixture built from +repeats makes a compressed column look free and any ceiling drawn around it +meaningless. + +Not a language model, and does not need to be. What matters is the symbol +distribution, not the sense. +""" +from __future__ import annotations + +import random + +WORDS = ( + "corridor narrows shoulders brush wet stone torchlight gutters draught " + "smells cold iron somewhere ahead water moving count nine paces passage " + "opens chamber ceiling lost dark sound breathing comes back half second " + "late Gwen catches sleeve without word points floor line pale grit laid " + "across threshold deliberate arc quartermaster bandit camp above ford " + "tunnels exchange key lantern rope knife bread rain mud hill road gate " + "watchman silver debt promise fever horse cart river bridge mill barley " + "smoke rafters bench ale ledger seal wax parchment ink candle shutter " + "hinge bolt cellar barrel salt fish nets harbour tide gull mast canvas" +).split() + + +def prose(rng: random.Random, nbytes: int) -> str: + """Roughly `nbytes` of varied sentences, deterministic for a given rng.""" + out: list[str] = [] + total = 0 + while total < nbytes: + sentence = " ".join(rng.choice(WORDS) for _ in range(rng.randint(8, 18))) + chunk = sentence.capitalize() + ". " + out.append(chunk) + total += len(chunk) + return "".join(out)[:nbytes] diff --git a/backend/tools/make_pokeball.py b/backend/tools/make_pokeball.py new file mode 100644 index 0000000..b46d53a --- /dev/null +++ b/backend/tools/make_pokeball.py @@ -0,0 +1,89 @@ +"""Draws a Pokeball as a PNG and prints it as a base64 data URI. + +`app/images.py` accepts only raster data URIs, so the seed file cannot carry an +SVG. This script writes the ball with `zlib` alone, so the seed's cover art +needs no image library and no build step. + +Run it, then paste `pokeball_uri.txt` into the seed file's `image` field: + + python make_pokeball.py +""" +import base64 +import math +import struct +import zlib + +SIZE = 192 # Drawn at `SCALE` times this size, then averaged down. +SCALE = 4 +N = SIZE * SCALE + +RED = (238, 27, 44) +WHITE = (247, 247, 247) +BLACK = (26, 26, 26) + + +def sample(x: float, y: float): + """Returns the `(r, g, b, a)` of the ball at one point of the canvas.""" + cx = cy = N / 2 + dx, dy = x - cx, y - cy + dist = math.hypot(dx, dy) + radius = N / 2 - N * 0.02 + + if dist > radius: + return (0, 0, 0, 0) + if dist > radius - N * 0.05: # Outer rim. + return (*BLACK, 255) + if dist < N * 0.10: # Center button. + return (*WHITE, 255) + if dist < N * 0.15: # Ring around the button. + return (*BLACK, 255) + if abs(dy) < N * 0.04: # Belt across the middle. + return (*BLACK, 255) + return (*(RED if dy < 0 else WHITE), 255) + + +def render(): + """Returns `SIZE` rows of `SIZE` RGBA pixels, box filtered from `SCALE`.""" + rows = [] + for py in range(SIZE): + row = bytearray() + for px in range(SIZE): + r = g = b = a = 0 + for sy in range(SCALE): + for sx in range(SCALE): + pr, pg, pb, pa = sample(px * SCALE + sx + 0.5, py * SCALE + sy + 0.5) + # Premultiply, so a transparent sample adds no color. + r += pr * pa + g += pg * pa + b += pb * pa + a += pa + if a: + row += bytes((r // a, g // a, b // a, a // (SCALE * SCALE))) + else: + row += b"\0\0\0\0" + rows.append(bytes(row)) + return rows + + +def png(rows) -> bytes: + raw = b"".join(b"\0" + r for r in rows) + + def chunk(tag: bytes, data: bytes) -> bytes: + return (struct.pack(">I", len(data)) + tag + data + + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF)) + + header = struct.pack(">IIBBBBB", SIZE, SIZE, 8, 6, 0, 0, 0) + return (b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", header) + + chunk(b"IDAT", zlib.compress(raw, 9)) + + chunk(b"IEND", b"")) + + +if __name__ == "__main__": + data = png(render()) + uri = "data:image/png;base64," + base64.b64encode(data).decode("ascii") + with open("pokeball.png", "wb") as fh: + fh.write(data) + with open("pokeball_uri.txt", "w", encoding="utf-8") as fh: + fh.write(uri) + print(len(data), "bytes;", len(uri), "chars") diff --git a/backend/tools/measure_bundle.py b/backend/tools/measure_bundle.py new file mode 100644 index 0000000..eb91489 --- /dev/null +++ b/backend/tools/measure_bundle.py @@ -0,0 +1,151 @@ +"""What a v2 bundle costs, measured on a production-sized adventure. + +SP6 gave the bundle coordinates, and coordinates cost bytes: a branch number +and a depth on every node, plus the two after-snapshots a switch needs to put +back. This puts a number on that, against the v1 shape it replaces, on the same +fixture the egress work is measured on. + + python -m tools.measure_bundle --actions 600 --rich + python -m tools.measure_bundle --actions 600 --rich --forks 20 + +Reuses `tools.stress_session`'s fixture, so read that module's warnings first: +a `--rich` run is a correctness fixture, and its byte figures are not +comparable to a plain one. +""" +import json +import random +import sys + +# Import `stress_session` first. This is not a style preference. Importing it is +# what points `AIDND_DB_PATH` at a throwaway file, and `app.database` reads that +# at module scope, so an `app` import above this line runs the whole fixture +# against `backend/data.db` instead. The failure looks like success: the first +# run seeds a synthetic user and adventure into the local database and reports +# plausible numbers, and only the second run fails on the unique email. +from tools import stress_session as stress # noqa: I001 (see above) + +from app import bundle, models, tree +from app.database import SessionLocal + + +def _v1_shape(v2: dict) -> dict: + """The same story as v1 would have written it, for a like-for-like count. + + There is one entry per turn, with the siblings collected back into a + `variants` array, no coordinates, and no outcomes. That is what version 1 + could carry. + """ + turns: dict[tuple[int, int], list[dict]] = {} + order: list[tuple[int, int]] = [] + for node in v2["actions"]: + key = (node["branch"], node["depth"]) + if key not in turns: + turns[key] = [] + order.append(key) + turns[key].append(node) + actions = [] + for i, key in enumerate(order): + group = turns[key] + live = next((n for n in group if n["live"]), group[0]) + actions.append({ + "index": i, "type": live["type"], "text": live["text"], + "reasoning": live.get("reasoning"), + "variants": [ + {"text": n["text"], "reasoning": n.get("reasoning"), + "createdAt": n["createdAt"]} + for n in group + ] if len(group) > 1 else None, + "variantIndex": group.index(live), + "createdAt": live["createdAt"], + }) + old = {k: v for k, v in v2.items() if k not in ("branches", "headBranch")} + old["format"] = bundle.LEGACY_FORMAT + old["actions"] = actions + old["memories"] = [ + {k: v for k, v in m.items() if k not in ("branch", "depth")} + for m in v2["memories"] + ] + old["memoryCursor"] = 0 + old["summaryCursor"] = 0 + return old + + +def _bytes(obj) -> int: + return len(json.dumps(obj).encode("utf-8")) + + +def main(argv=None) -> int: + argv = list(sys.argv[1:] if argv is None else argv) + forks = 0 + if "--forks" in argv: + i = argv.index("--forks") + forks = int(argv[i + 1]) + del argv[i:i + 2] + args = stress.parse_args(argv) + rng = random.Random(args.seed) + stress._build_text(args, random.Random(args.seed ^ 0x5F5F)) + adv_id, _ = stress.build_fixture(args, rng) + + db = SessionLocal() + try: + adventure = db.get(models.Adventure, adv_id) + if forks: + # Twenty divergences off one line, each a little deeper, which is + # the shape SP5 measured the fork cost on. The nodes are chosen up + # front and the session is flushed after every fork. `fork` moves a + # row onto its new branch, and with `autoflush=False` a query issued + # before that move is written still finds the node at its old + # location. + root = adventure.head_branch_id + candidates = [ + node.id for node in + db.query(models.Action) + .filter( + models.Action.adventure_id == adventure.id, + models.Action.branch_id == root, + models.Action.depth > 1, + ) + .order_by(models.Action.depth) + .all() + ] + step = max(len(candidates) // (forks + 1), 1) + made = 0 + for node_id in candidates[::step][:forks]: + tree.fork(db, adventure, db.get(models.Action, node_id)) + db.flush() + made += 1 + db.commit() + print(f"forked {made} times") + + v2 = bundle.export(db, adventure) + v1 = _v1_shape(v2) + nodes = len(v2["actions"]) + turns = len({(n["branch"], n["depth"]) for n in v2["actions"]}) + branches = len(v2["branches"]) + + stripped = json.loads(json.dumps(v2)) + for node in stripped["actions"]: + node.pop("stateAfter", None) + node.pop("worldStateAfter", None) + node.pop("worldDelta", None) + + v2_bytes, v1_bytes, bare = _bytes(v2), _bytes(v1), _bytes(stripped) + print(f"{turns} turns · {nodes} nodes · {branches} branches") + print(f"v1 shape {v1_bytes:>12,} B") + print(f"v2 {v2_bytes:>12,} B " + f"{v2_bytes / v1_bytes:.3f}× v1") + print(f"v2 w/o outcomes {bare:>12,} B " + f"{bare / v1_bytes:.3f}× v1") + print(f"the outcomes {v2_bytes - bare:>12,} B " + f"{(v2_bytes - bare) / nodes:.1f} B/node") + print(f"coordinates {bare - v1_bytes:>12,} B " + f"{(bare - v1_bytes) / nodes:+.1f} B/node") + print(f"import cap {bundle.__name__}: " + f"{v2_bytes / (20 * 1024 * 1024):.1%} of MAX_IMPORT_BODY_BYTES") + finally: + db.close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/tools/memory_ab.py b/backend/tools/memory_ab.py new file mode 100644 index 0000000..65c2efe --- /dev/null +++ b/backend/tools/memory_ab.py @@ -0,0 +1,257 @@ +"""A/B the memory prompt against a real model, on one story held constant. + +Reviewing a prompt tells you what it asks for. It does not tell you what a model +does with it. This runs the real pipeline twice over the *same* blocks of the +*same* story, changing only the prompt, and prints the memories side by side. + +What it showed the first time it was run, and why `MEMORY_MAX_WORDS` exists, is +written up in `plan/18-persona-and-memory-quality.md`. Two consecutive memories +from one story came back in two different persons, and the same model wrote 34 +words for one block and 105 for the next. + +Three things make it a fair test rather than a demonstration: + +- **The story is generated once**, through the app's own `build_context`, and + both arms summarize the same actions. The prompt is the only variable. +- **The control is read from git**, at the commit named by `--before`, so it is + the prompt that actually shipped rather than a paraphrase of it. +- **Every call goes through `OpenAICompatibleProvider`** to the endpoint in + `--endpoint`, so this exercises the provider, the streaming path and + `complete()` rather than a stub. + +Point it at `tools/claude_shim.py` to spend a Claude subscription instead of API +credit: + + python tools/claude_shim.py --port 8787 & + python tools/memory_ab.py --out /tmp/ab.md + +Point `--endpoint` at the provider the deployed app really uses to learn +something this cannot tell you: whether a weaker model follows the framing rule +as well as a Claude model does. + +Options: + + --endpoint OpenAI-compatible base URL. Defaults to the local shim. + --model Model name to ask that endpoint for. Defaults to `sonnet`. + --before Git commit holding the prompt to compare against. + --turns How many player turns to generate. Six gives two blocks. + --out Write the full transcript here as Markdown. + +This writes to a scratch SQLite file and drops it afterwards. It never touches +`backend/data.db`. +""" +import argparse +import asyncio +import os +import re +import subprocess +import sys +import tempfile +import textwrap +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent.parent + +# The moves the player makes. Fixed, so that a re-run summarizes comparable +# story rather than a different adventure. +PLAYER_TURNS = [ + "go quiet and signal Gwen to circle around the right flank", + "search the nearest bedroll for anything useful", + "move toward the strongbox, keeping low", + "grab her wrist and pull her down behind the woodpile", + "wait for the bandit to turn, then move", + "ask her whether she still trusts my read on this", +] + + +def prompt_at(commit: str, name: str) -> str: + """Reads one prompt constant out of `memorybank.py` as of `commit`. + + Read from git rather than pasted here, so that the control cannot drift out + of step with what was actually shipped. + """ + source = subprocess.run( + ["git", "show", f"{commit}:backend/app/memorybank.py"], + capture_output=True, text=True, cwd=REPO, check=True, + ).stdout + match = re.search(rf"^{name} = \((.*?)^\)$", source, re.S | re.M) + if not match: + raise SystemExit(f"{name} not found in memorybank.py at {commit}") + namespace: dict = {"MEMORY_MAX_WORDS": 50, "SUMMARY_MAX_WORDS": 250} + return eval(f"({match.group(1)})", namespace) # noqa: S307 — our own source + + +def words(text: str) -> int: + return len(text.split()) + + +def framing(text: str) -> str: + """How this memory refers to the protagonist. The thing being measured.""" + tags = [] + if re.search(r"\byou(r)?\b", text, re.I): + tags.append('second person ("you")') + if re.search(r"\bthe player\b", text, re.I): + tags.append('"the player"') + return ", ".join(tags) or "named" + + +async def main(args) -> None: + from app import memorybank, models, tree, worldstate + from app.context import build_context, truncate_to_last_tokens + from app.database import Base, SessionLocal, engine + from app.providers import OpenAICompatibleProvider + from app.routers.adventures.scenario_text import scenario_card_specs + from app.seed import seed_public_scenarios + + def provider(model: str | None = None) -> OpenAICompatibleProvider: + return OpenAICompatibleProvider( + args.endpoint, "unused-by-the-shim", model or args.model, "chat", 0) + + async def generate(system: str, story: str) -> str: + """One turn, through the provider's real streaming path.""" + from app.providers.base import PromptParts + out = [] + async for kind, piece in provider().generate( + PromptParts(system=system, story=story), temperature=0.8, + max_tokens=args.max_tokens, + ): + if kind == "text": + out.append(piece) + return "".join(out).strip() + + Base.metadata.create_all(bind=engine) + seed_public_scenarios(engine) + db = SessionLocal() + user = models.User(is_guest=False, email="memory-ab@example.com") + db.add(user) + db.flush() + settings = models.Settings(user_id=user.id, api_key="enc:d", + model=args.model, max_output_tokens=args.max_tokens) + db.add(settings) + scenario = db.query(models.Scenario).filter( + models.Scenario.title.like("%Bandit Camp%")).one() + + adventure = models.Adventure( + user_id=user.id, title=scenario.title, scenario_id=scenario.id, + script_state={}, auto_summarize=True, memory=scenario.memory, + authors_note=scenario.authors_note, + ai_instructions=scenario.ai_instructions, + world_state=worldstate.instantiate(scenario.stat_schema), + persona_name="Kaelen", persona_pronouns="he/him", + persona_desc=("A half-elf ranger, exiled from the northern holds for a " + "killing he still won't explain. Wary of nobles, soft on " + "strays."), + ) + db.add(adventure) + db.flush() + tree.head_branch(db, adventure) + for ref, spec in scenario_card_specs(scenario, {}).items(): + db.add(models.StoryCard(adventure_id=adventure.id, source_ref=ref, **spec)) + opening = models.Action(adventure_id=adventure.id, type="start", + text=scenario.prompt) + tree.place_action(db, adventure, opening) + db.add(opening) + db.commit() + db.refresh(adventure) + + print(f"Generating {args.turns} turns through build_context…", flush=True) + for i, move in enumerate(PLAYER_TURNS[:args.turns], 1): + player = models.Action(adventure_id=adventure.id, type="do", text=move) + tree.place_action(db, adventure, player) + db.add(player) + db.commit() + db.refresh(adventure) + + system_text, story_text, _ = build_context(adventure, settings) + clean, delta = worldstate.extract_delta(await generate(system_text, story_text)) + ai = models.Action(adventure_id=adventure.id, type="ai", text=clean) + tree.place_action(db, adventure, ai) + db.add(ai) + if delta: + adventure.world_state, report = worldstate.apply_delta( + adventure.world_state, scenario.stat_schema, delta, i) + ai.world_delta = report + db.commit() + db.refresh(adventure) + print(f" [{i}] {move} → {words(clean)} words" + + (f", state {delta}" if delta else ""), flush=True) + + old_system = prompt_at(args.before, "MEMORY_SYSTEM_PROMPT") + actions = sorted( + (a for a in adventure.actions if a.type in ("start", "do", "ai")), + key=lambda a: (a.depth if a.depth is not None else 0, a.id), + ) + blocks = [actions[i:i + memorybank.MEMORY_INTERVAL] + for i in range(0, len(actions) - memorybank.MEMORY_INTERVAL + 1, + memorybank.MEMORY_INTERVAL)] + print(f"\n{len(actions)} actions → {len(blocks)} blocks. Summarizing each twice…", + flush=True) + + rows = [] + for n, block in enumerate(blocks, 1): + raw = "\n\n".join(a.text for a in block) + plain = (f"Story excerpt:\n\n" + f"{truncate_to_last_tokens(raw, 2000)}\n\nMemory:") + brief = memorybank.cast_brief(adventure, raw) + before = await provider(args.model).complete(old_system, plain) + after = await provider(args.model).complete( + memorybank.MEMORY_SYSTEM_PROMPT, + f"{brief}\n\n{plain}" if brief else plain) + rows.append((n, before.strip(), after.strip())) + print(f" block {n}: before {words(before)}w ({framing(before)}), " + f"after {words(after)}w ({framing(after)})", flush=True) + + print("\n" + "=" * 74) + print(f"{'':16}{'words':>6} framing") + print("-" * 74) + for n, before, after in rows: + for label, text in (("before", before), ("after", after)): + print(f"memory {n} {label:7}{words(text):>6} {framing(text)}") + print("=" * 74) + + if args.out: + with open(args.out, "w", encoding="utf-8") as fh: + fh.write("# Memory prompt A/B\n\n") + fh.write(f"One story, generated through `build_context` against " + f"`{args.model}` at `{args.endpoint}`. Both prompts then " + f"summarize the same blocks, so the prompt is the only " + f"variable. The control is `MEMORY_SYSTEM_PROMPT` as of " + f"`{args.before}`.\n\n") + fh.write("| memory | arm | words | framing |\n|---|---|---|---|\n") + for n, before, after in rows: + for label, text in (("before", before), ("after", after)): + fh.write(f"| {n} | {label} | {words(text)} | {framing(text)} |\n") + for n, before, after in rows: + fh.write(f"\n## Memory {n}\n\n**Before:** {before}\n\n" + f"**After:** {after}\n") + fh.write("\n## The story both arms summarized\n\n") + for a in actions: + fh.write(f"**{a.type}:** {a.text}\n\n") + print(f"\nwritten to {args.out}") + + db.close() + Base.metadata.drop_all(bind=engine) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument("--endpoint", default="http://127.0.0.1:8787/v1") + parser.add_argument("--model", default="sonnet") + parser.add_argument("--before", default="9cdcb55", + help="commit holding the prompt to compare against") + parser.add_argument("--turns", type=int, default=6) + parser.add_argument("--max-tokens", type=int, default=700) + parser.add_argument("--out") + args = parser.parse_args() + + # A scratch database, so a run never touches the developer's own. + handle = tempfile.NamedTemporaryFile(suffix=".db", delete=False) + handle.close() + os.environ["AIDND_DB_PATH"] = handle.name + os.environ.pop("AIDND_DATABASE_URL", None) + os.environ.pop("DATABASE_URL", None) + sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + try: + asyncio.run(main(args)) + finally: + Path(handle.name).unlink(missing_ok=True) diff --git a/backend/tools/rewrite_memories.py b/backend/tools/rewrite_memories.py new file mode 100644 index 0000000..f30faeb --- /dev/null +++ b/backend/tools/rewrite_memories.py @@ -0,0 +1,347 @@ +"""Rewrite memories already in the bank with the current summarizer prompt. + +Phase 18b changed what the summarizer is told: a cast brief naming the +protagonist and the other characters, a rule to write in the third person, and +`MEMORY_MAX_WORDS`. Only memories written after that change get any of it. An +adventure played before it keeps a bank of "You entered the crypt" — unnamed, +in the second person, and occasionally a hundred-word paragraph — and those are +the memories that get injected into every turn from now on. + +This rewrites them in place, from the same actions they were written from. It is +a one-off backfill, not part of the app. + +**In place, rather than deleting and letting the app re-summarize.** A memory +row carries more than its text: whether it is pinned, how often it has been +retrieved, and the node it hangs off, which is what makes a fork inherit the +right memories. Rewriting `text` keeps all of that. Deleting the bank and +rewinding the cursor would lose it, and would then trickle memories back at +`MAX_MEMORIES_PER_RUN` per turn. + +**The prompt is not assembled here.** `memorybank.summarize_block` is what the +app itself calls, so a rewritten memory is written by the prompt that is +actually shipping rather than by a copy of it that can drift. + +Reading the block back is the one thing the app has never had to do. A memory +records `source_start`, `source_end` and `branch_id`; `memorybank.source_block` +turns those back into actions, on the branch the memory was written on rather +than whichever branch the adventure is on now. + +What it will not touch: + +- A memory with no source range: hand-written, or migrated from before memories + had coordinates. There is no block to rewrite it from, and the player may have + typed it. +- A memory whose actions have since been deleted. +- An adventure whose owner has no API key in Settings. Summarization spends the + user's own key by construction and never the shared demo key (see the comment + above `memorybank.summary_provider`), and this holds to that. Pass + `--endpoint`/`--model`/`--api-key` to summarize with something else. + +The vector is cleared for every memory it rewrites, because the stored one +describes the old wording. `--embed` re-embeds them here; without it the app's +own post-turn pass does it, `MAX_EMBED_BATCH` per turn, and until then those +memories are out of the ranked bank. **Stop the app before running with +`--embed`.** A running process caches vectors by memory id and expects to be the +one writing them (see `memorybank._vector_cache`), so a vector written from +outside it can sit behind a stale cached copy until it restarts. + +`--embed` always uses the embedding model and endpoint in the owner's Settings, +never `--endpoint`. A vector is only meaningful against the vectors it is ranked +beside, so re-embedding two memories with a different model would quietly put +two coordinate systems in one bank. `--endpoint` moves the summarizer alone. + +Usage, from `backend/`: + + python -m tools.rewrite_memories # what would change + python -m tools.rewrite_memories --write --limit 3 # try three of them + python -m tools.rewrite_memories --write --embed # the whole backfill + +Without `--write` it makes no model calls, spends nothing, and only reports the +scope. Every run reads the database the app reads: `AIDND_DB_PATH`, or +`DATABASE_URL` for a hosted Postgres. Take a copy of it first — the old text is +overwritten and is not kept anywhere. + +**On the hosted deploy, name whose adventures you mean.** That database holds +other people's stories, and each adventure is summarized with *its owner's* key, +so an unfiltered `--write` spends other people's money on memories they did not +ask to have rewritten. `--email` restricts the run to the accounts you name and +`--adventure` to single adventures; a dry run costs nothing and lists both, with +the owner of each. Guests have no email and can only be reached by id. + +Two environment variables reach that database from a checkout. In PowerShell, +which is where this project is developed, they are set first and then persist +for the rest of the session: + + $env:AIDND_DATABASE_URL = Read-Host 'Neon URL' + $env:AIDND_SECRET_KEY = Read-Host 'Secret key' + .venv\Scripts\python.exe -m tools.rewrite_memories --email you@example.com + +`Read-Host` keeps both values out of the PowerShell history file. On a POSIX +shell the same thing is one line: + + AIDND_DATABASE_URL=... AIDND_SECRET_KEY=... \ + python -m tools.rewrite_memories --email you@example.com + +`AIDND_SECRET_KEY` is not optional there. Stored API keys are encrypted with it, +and with the wrong one `decrypt_secret` returns "" and every adventure is skipped +as having no key (see `security.py`). The deployed image does not carry this +directory — the Dockerfile copies `backend/app` alone — so run it from a +checkout against the hosted database rather than from a shell on the box. +""" +import argparse +import asyncio +import sys +from pathlib import Path + +from urllib.parse import urlsplit + +from sqlalchemy import func, inspect as sa_inspect, select + + +def safe_dsn(url: str) -> str: + """A connection string with the credentials taken out. + + The report says which database it is about to rewrite, which is worth + printing. The password in a Neon URL is not: this output goes to a console, + a screenshot, or a pasted bug report, and the operator has no way to know + the line carried a credential until it is somewhere else. + """ + parsed = urlsplit(url) + if not parsed.hostname: + return "(configured)" + who = f"{parsed.username}@" if parsed.username else "" + port = f":{parsed.port}" if parsed.port else "" + # The query string is dropped whole. `sslmode` is the only part anyone + # wants to see, and some drivers accept a password there too. + return f"{parsed.scheme}://{who}{parsed.hostname}{port}{parsed.path}" + + +def words(text: str) -> int: + return len(text.split()) + + +def one_line(text: str, width: int = 96) -> str: + flat = " ".join(text.split()) + return flat if len(flat) <= width else flat[: width - 1] + "…" + + +async def main(args) -> int: + from app import memorybank, models + from app.database import DB_PATH, DATABASE_URL, SessionLocal + from app.providers import OpenAICompatibleProvider, ProviderError + + db = SessionLocal() + print(f"database: {safe_dsn(DATABASE_URL) if DATABASE_URL else DB_PATH}") + if not sa_inspect(db.get_bind()).has_table(models.Adventure.__tablename__): + # A mistyped path creates an empty SQLite file rather than failing, so + # say what is wrong instead of raising "no such table: adventures". + print("There are no tables here. Point AIDND_DB_PATH, or DATABASE_URL " + "for a hosted deploy, at the database the app uses.") + return 2 + + adventures = db.query(models.Adventure).order_by(models.Adventure.id) + if args.adventure: + adventures = adventures.filter(models.Adventure.id.in_(args.adventure)) + if args.email: + # Compared case-insensitively: an address is typed on the command line + # here and was typed into a registration form there. + wanted = [e.strip().lower() for e in args.email] + owners = db.execute( + select(models.User.id, func.lower(models.User.email)) + .where(func.lower(models.User.email).in_(wanted)) + ).all() + unknown = sorted(set(wanted) - {email for _, email in owners}) + if unknown: + print(f"no account with that email: {', '.join(unknown)}") + return 2 + adventures = adventures.filter( + models.Adventure.user_id.in_([user_id for user_id, _ in owners])) + adventures = adventures.all() + if args.adventure and len(adventures) != len(set(args.adventure)): + found = {a.id for a in adventures} + missing = sorted(set(args.adventure) - found) + print(f"no such adventure: {', '.join(str(i) for i in missing)}") + return 2 + + # Settings are per user, and several adventures usually share one owner. + settings_by_user: dict[int, models.Settings | None] = {} + + def settings_for(user_id: int) -> models.Settings | None: + if user_id not in settings_by_user: + settings_by_user[user_id] = ( + db.query(models.Settings) + .filter(models.Settings.user_id == user_id) + .first() + ) + return settings_by_user[user_id] + + def provider_for(settings: models.Settings) -> OpenAICompatibleProvider: + """The adventure owner's own summarizer, unless the run overrides it.""" + if not (args.endpoint or args.model or args.api_key): + return memorybank.summary_provider(settings) + return OpenAICompatibleProvider( + args.endpoint or settings.endpoint_url, + args.api_key or settings.api_key_plain, + args.model or settings.summary_model or settings.model, + settings.api_mode, + settings.reasoning_max_tokens, + ) + + totals = {"rewritten": 0, "would rewrite": 0, "no source": 0, + "no story left": 0, "no key": 0, "failed": 0, "empty reply": 0} + rewritten: list[models.Memory] = [] + budget = args.limit + + for adventure in adventures: + memories = ( + db.query(models.Memory) + .filter(models.Memory.adventure_id == adventure.id) + .order_by(models.Memory.id) + ) + if not args.include_forgotten: + memories = memories.filter(models.Memory.forgotten.is_(False)) + memories = memories.all() + if not memories: + continue + + settings = settings_for(adventure.user_id) + # An adventure whose owner has no key is reported rather than skipped + # silently: it is the one reason a memory this tool can rewrite is left + # alone, and the operator can fix it with --api-key. + usable = settings is not None and bool( + args.api_key or args.endpoint or settings.api_key_plain + ) + owner = db.get(models.User, adventure.user_id) + who = (owner.email if owner and owner.email + else f"guest #{adventure.user_id}") + print(f"\nadventure {adventure.id}: {adventure.title!r} ({who}) " + f"— {len(memories)} memories" + + ("" if usable else " [owner has no API key in Settings]")) + + for memory in memories: + if memory.source_start is None or memory.source_end is None: + totals["no source"] += 1 + print(f" #{memory.id} skipped: hand-written, no source range") + continue + block = memorybank.source_block(db, memory) + if not block: + totals["no story left"] += 1 + print(f" #{memory.id} skipped: the actions it summarized are gone") + continue + if not usable: + totals["no key"] += 1 + continue + if not args.write: + totals["would rewrite"] += 1 + print(f" #{memory.id} would rewrite from {len(block)} actions " + f"({words(memory.text)}w): {one_line(memory.text, 70)}") + continue + if budget is not None and budget <= 0: + break + try: + text = (await memorybank.summarize_block( + adventure, provider_for(settings), block)).strip() + except ProviderError as exc: + totals["failed"] += 1 + print(f" #{memory.id} FAILED: {exc}") + continue + if not text: + totals["empty reply"] += 1 + print(f" #{memory.id} FAILED: the model returned nothing") + continue + print(f" #{memory.id} {words(memory.text)}w → {words(text)}w") + print(f" was: {one_line(memory.text)}") + print(f" now: {one_line(text)}") + memory.text = text + # The stored vector describes the wording that has just been + # replaced. Clearing it takes the memory out of the ranked bank + # (`retrieve_memories` selects on `embedded`) until something + # embeds the new text. + memorybank.set_vector(memory, None) + # Commit each one. A run that dies halfway keeps what it has done, + # and re-running it costs only the memories still to do. + db.commit() + rewritten.append(memory) + totals["rewritten"] += 1 + if budget is not None: + budget -= 1 + if budget is not None and budget <= 0 and args.write: + print("\n--limit reached.") + break + + embedded = 0 + if args.write and args.embed and rewritten: + by_adventure: dict[int, list[models.Memory]] = {} + for memory in rewritten: + by_adventure.setdefault(memory.adventure_id, []).append(memory) + print() + for adventure_id, group in by_adventure.items(): + adventure = db.get(models.Adventure, adventure_id) + settings = settings_for(adventure.user_id) + if settings is None or not settings.embedding_model.strip(): + print(f"adventure {adventure_id}: no embedding model in Settings; " + f"{len(group)} memories left for the app to embed") + continue + done = 0 + for start in range(0, len(group), memorybank.MAX_EMBED_BATCH): + batch = group[start:start + memorybank.MAX_EMBED_BATCH] + try: + new = await memorybank.embedding_provider(settings).embed( + [m.text for m in batch]) + except ProviderError as exc: + print(f"adventure {adventure_id}: embedding failed ({exc}); " + f"{len(group) - done} left for the app to embed") + break + for memory, vector in zip(batch, new): + memorybank.set_vector(memory, vector) + db.commit() + done += len(batch) + embedded += done + if done: + print(f"adventure {adventure_id}: embedded {done}") + + print("\n" + "-" * 60) + for label, count in totals.items(): + if count: + print(f"{label:>14}: {count}") + if not args.write: + print("\nNothing was written. Re-run with --write to rewrite these.") + elif totals["rewritten"] - embedded > 0: + left = totals["rewritten"] - embedded + print(f"\n{left} rewritten {'memory is' if left == 1 else 'memories are'}" + " unembedded, so they are out of the ranked bank until the app's " + f"post-turn pass embeds them ({memorybank.MAX_EMBED_BATCH} per " + "turn).") + db.close() + return 1 if totals["failed"] or totals["empty reply"] else 0 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument("--write", action="store_true", + help="rewrite. Without it, only report what would change.") + parser.add_argument("--adventure", type=int, action="append", + help="restrict to this adventure id; repeatable.") + parser.add_argument("--email", action="append", + help="restrict to this account's adventures; repeatable. " + "Use it on a shared database.") + parser.add_argument("--limit", type=int, + help="stop after rewriting this many memories.") + parser.add_argument("--include-forgotten", action="store_true", + help="also rewrite evicted memories, which are out of play.") + parser.add_argument("--embed", action="store_true", + help="re-embed here. Stop the app first; see the docstring.") + parser.add_argument("--endpoint", help="override the owner's endpoint URL.") + parser.add_argument("--model", help="override the owner's summary model.") + parser.add_argument("--api-key", help="override the owner's API key.") + args = parser.parse_args() + + # A Windows console defaults to cp1252, which cannot encode the arrow this + # prints, let alone whatever is in a story's prose. Ask for UTF-8 and + # replace what the terminal still cannot render, so a run never dies + # halfway through on an encoding error. + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + + sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + raise SystemExit(asyncio.run(main(args))) diff --git a/backend/tools/shots_fixture.py b/backend/tools/shots_fixture.py new file mode 100644 index 0000000..2dc0de1 --- /dev/null +++ b/backend/tools/shots_fixture.py @@ -0,0 +1,321 @@ +"""The story the README screenshots are taken of. + +`tools.tree_fixture` builds a tree with a shape worth drawing, on a scenario +with one stat, because a map only cares about the shape. A screenshot cares +about everything else: the world-state rail wants a scenario with bands, flags, +milestones and a named cast, and the story wants prose somebody would read. + +So this script drives the seeded Bandit Camp demo through eight turns of written +prose and written deltas. That is the same scenario the older images were shot +on, which keeps the set consistent. It then forks three discarded attempts onto +branches of their own, one of them off a branch, so the map has to nest. It +leaves the reader on the first telling, on a turn that has a second attempt, so +one screen shows the world state, the attempt pager, and the branch rail at +once. + + cd backend + .venv/Scripts/python.exe -m tools.shots_fixture /tmp/shots.db + AIDND_DB_PATH=/tmp/shots.db .venv/Scripts/python.exe \ + -m uvicorn app.main:app --port 8010 + +Then open http://127.0.0.1:8010/play/1. The SPA is served out of +`frontend/dist`, so run `npm run build` first if it is stale. + +No LLM is called: the provider is scripted, and every delta below is the one +the referee is being shown clamping. +""" +import os +import sys + +sys.path.insert(0, os.getcwd()) + +OUT = sys.argv[1] if len(sys.argv) > 1 else "shotsfixture.db" +if os.path.exists(OUT): + os.remove(OUT) +os.environ["AIDND_DB_PATH"] = OUT +os.environ.pop("AIDND_DATABASE_URL", None) +os.environ.pop("DATABASE_URL", None) +os.environ.pop("AIDND_MULTI_USER", None) + +import json # noqa: E402 + +from fastapi.testclient import TestClient # noqa: E402 +from sqlalchemy import text # noqa: E402 + +from app import auth, limits, models # noqa: E402 +from app.database import SessionLocal # noqa: E402 +from app.database import engine # noqa: E402 +from app.main import app # noqa: E402 +from app.migrations import LATEST_VERSION # noqa: E402 +from app.routers import adventures # noqa: E402 + +SCENARIO_TITLE = "[Demo] The Bandit Camp (RPG world state)" + +# (what the player typed, what the model said, what it claimed changed). +# +# The deltas are chosen so the rail has something to show at every level: a +# counter that only climbs, two flags that flip both ways, a text stat that is +# rewritten rather than added to, a milestone that sticks, and one delta the +# per-turn cap has to cut down (-60 hp against a max_delta_per_turn of 35). +TURNS = [ + ( + "Signal Gwen to circle left, and keep my eyes on the fire.", + "Gwen goes without a sound, low along the ditch, and the fire keeps its " + "own counsel. Two bedrolls, not three. Whoever owns the third is awake " + "somewhere in the dark, and that is the one worth knowing about.", + {"npc.gwen.trust": 8}, + ), + ( + "Cut the horse line so they scatter through the camp.", + "The rope parts under the knife and eleven hands of frightened horse go " + "through the fire pit sideways. Somebody shouts your name for a thing " + "you have not done yet. The camp is awake now, all of it at once.", + {"flags.alarm_raised": True, "flags.player_hidden": False, + "npc.bandit_leader.aggression": 20}, + ), + ( + "Get behind the wagon before anyone finds the ditch.", + "You make the wagon's shadow with a spear-length to spare and put your " + "back against a wheel that has not turned in a season. Gwen's arrow " + "answers from the treeline — once, and then not again, which is her way " + "of saying she is fine and busy.", + {"flags.player_hidden": True, "player.mana": -6}, + ), + ( + "Wait for the leader to pass, then take him from behind.", + "You wait a beat too long. He turns for a noise Gwen is making and walks " + "onto the knife himself, and it is the least honourable thing you have " + "ever done well. He is dead before the surprise finishes crossing his " + "face.", + {"npc.bandit_leader.health": -120, "npc.gwen.trust": -10, + "milestones.camp_cleared": True}, + ), + ( + "Fall back to Gwen and let her finish it.", + "You give ground the way she taught you, keeping him square to the " + "treeline, and the arrow takes him through the shoulder blade at " + "eleven paces. He sits down in the ash of his own fire and does not " + "get up. The camp goes quiet in pieces.", + {"npc.bandit_leader.health": -70, "npc.gwen.trust": 15, + "milestones.camp_cleared": True, "flags.alarm_raised": False}, + ), + ( + "Search the wagon for the strongbox.", + "It is under the false floor, of course it is, and it is heavier than " + "the caravan master implied. The lock has been worked at by somebody " + "patient and unsuccessful. You take the dead man's coat as well; yours " + "is one long tear from armpit to hip.", + {"milestones.strongbox_found": True, + "player.outfit": "a bandit's tarred coat over torn leather, strongbox under one arm"}, + ), + ( + "Bind my side before we move.", + "Gwen does it, badly and fast, with a strip off the same coat. \"You " + "went in alone,\" she says, in the voice she uses when she has decided " + "not to have the argument. The bleeding stops. The rest of it does not.", + {"player.hp": 18, "npc.gwen.trust": -5}, + ), + ( + "Take the north road while it's still dark.", + "You are two miles out when the sky starts telling on you, and the camp " + "behind is only smoke by then. Gwen walks ahead where she can see the " + "road, which means she is still angry, which means she is still here.", + {"world.day": 1, "milestones.gwen_survives": True, "player.mana": 9}, + ), +] + +# The attempts the story did not keep. Each one is retried at the turn with the +# same index below, and then forked onto a branch of its own. A discarded attempt +# is the only thing a fork can be made from. +RETAKES = { + 3: ( + "He passes close enough that you can smell the tar on his coat, and the " + "knife goes in under the arm where the plate is not. He is not a man who " + "goes down for it. He turns inside the blow and opens your side with the " + "back-swing, and the two of you come apart bleeding.", + {"player.hp": -60, "npc.bandit_leader.health": -45, + "npc.bandit_leader.aggression": 25}, + ), + 6: ( + "You tell her you will do it yourself and she lets you, which is worse " + "than the argument. The knot is bad. You will feel it in the morning, " + "and she will watch you feel it and say nothing at all.", + {"player.hp": 9, "npc.gwen.trust": -18}, + ), +} + + +class ScriptedProvider: + """Serves whatever the driver loaded, so a retry can differ from the attempt + it replaces. + + That difference is what the screenshots show. + """ + # The turn engine records the cost of the call, so a stand-in provider has + # to carry this attribute even when it never calls anything. + last_usage = None + + next_reply = ("", {}) + + def __init__(self, *a, **k): + pass + + async def generate(self, parts, *, temperature, max_tokens): + prose, delta = ScriptedProvider.next_reply + block = f"\n```state\n{json.dumps(delta)}\n```" if delta else "" + yield ("text", prose + block) + + +adventures.turns.OpenAICompatibleProvider = ScriptedProvider +auth.resolve_provider_config = lambda s: auth.ProviderConfig( + "http://fake", "k", "test-model", False) +limits.rate_limit = lambda *a, **k: None +limits.check_row_cap = lambda *a, **k: None + +# `bootstrap()` runs at import and seeds the public demo scenarios, so the Bandit +# Camp already exists. Shooting on it matters because it is the scenario a +# visitor meets. +client = TestClient(app) + +db = SessionLocal() +user = models.User(is_guest=False, email=None) +db.add(user) +db.flush() +db.add(models.Settings(user_id=user.id, api_key="enc:dummy", model="test-model")) +scenario = db.query(models.Scenario).filter( + models.Scenario.title == SCENARIO_TITLE).one() +db.commit() +scenario_id = scenario.id +db.close() + +r = client.post("/api/adventures", json={ + "scenario_id": scenario_id, + "title": "The Bandit Camp", + "placeholders": {}, +}) +assert r.status_code == 201, r.text +adv_id = r.json()["id"] +base = f"/api/adventures/{adv_id}" + + +def play(i): + """One turn of the written story.""" + player, prose, delta = TURNS[i] + ScriptedProvider.next_reply = (prose, delta) + r = client.post(f"{base}/actions", json={"type": "do", "text": player}) + assert r.status_code == 200, r.text + + +def retake(i): + """Retry turn `i` with the other reply, and hand back the take it left.""" + ScriptedProvider.next_reply = RETAKES[i] + return retry_last() + + +def retry_last(): + """Retries whatever is at the tip with whatever reply is loaded. + + Returns the attempt the story just left, which is the only thing a fork can + be made from. + """ + before = live_ai_ids() + r = client.post(f"{base}/retry") + assert r.status_code == 200, r.text + dead = [a for a in all_ai() if not a.live and a.id in before] + assert dead, "retry left no discarded take" + return dead[-1].id + + +def all_ai(): + db = SessionLocal() + out = db.query(models.Action).filter( + models.Action.adventure_id == adv_id, + models.Action.type == "ai").order_by(models.Action.id).all() + db.expunge_all() + db.close() + return out + + +def live_ai_ids(): + return [a.id for a in all_ai() if a.live] + + +def branches(): + return client.get(f"{base}/branches").json() + + +def name(branch_id, label): + r = client.patch(f"{base}/branches/{branch_id}", json={"name": label}) + assert r.status_code == 200, r.text + + +# ---- the first telling, with the knife-in-the-back turn retried ---- +for i in range(4): + play(i) +knife = retake(3) +for i in range(4, 7): + play(i) +binding = retake(6) +play(7) +root = branches()[0]["id"] +name(root, "The quiet way in") + +# ---- the take where he died easy, given a line of its own ---- +r = client.post(f"{base}/actions/{knife}/fork") +assert r.status_code == 200, r.text +ScriptedProvider.next_reply = ( + "The camp finds him before you have finished wiping the knife, and there " + "is nothing quiet about the next four minutes.", {"flags.alarm_raised": True, + "player.hp": -22}) +r = client.post(f"{base}/actions", json={"type": "do", "text": "Take his horn and blow it."}) +assert r.status_code == 200, r.text +name([b for b in branches() if b["is_head"]][0]["id"], "Loud, and early") + +# ---- and a line that left that one again, so the map has to nest ---- +# +# Forking the attempt at the tip only switches to it, because the attempts there +# are still leaves that nothing was built on. The story is therefore moved one +# turn past it first, and only then does the attempt it left deserve a branch. +ScriptedProvider.next_reply = ( + "Nobody comes. The horn was the wrong horn, or the camp has been empty of " + "anyone who cares since before you got here.", {"npc.gwen.trust": -4}) +horn = retry_last() +ScriptedProvider.next_reply = ( + "The tents are as empty as the sound was. Somebody left in a hurry and did " + "not take the good rope.", {"flags.player_hidden": True}) +r = client.post(f"{base}/actions", json={"type": "do", "text": "Search the tents."}) +assert r.status_code == 200, r.text + +r = client.post(f"{base}/actions/{horn}/fork") +assert r.status_code == 200, r.text +ScriptedProvider.next_reply = ( + "They come at the sound the way water finds a crack, and you spend the next " + "hour learning the camp by running through it.", + {"player.hp": -28, "npc.bandit_leader.aggression": 15}) +r = client.post(f"{base}/actions", json={"type": "do", "text": "Run for the treeline."}) +assert r.status_code == 200, r.text +name([b for b in branches() if b["is_head"]][0]["id"], "Through the camp") + +# ---- and the argument that was never had ---- +client.post(f"{base}/branches/{root}/switch") +r = client.post(f"{base}/actions/{binding}/fork") +assert r.status_code == 200, r.text +ScriptedProvider.next_reply = ( + "She lets it go, and the road out is quieter for it than either of you " + "wanted.", {"npc.gwen.trust": -6}) +r = client.post(f"{base}/actions", json={"type": "do", "text": "Say nothing and walk."}) +assert r.status_code == 200, r.text +name([b for b in branches() if b["is_head"]][0]["id"], "Said nothing") + +# Leave the reader on the first telling, on the turn that has two attempts. That +# is the one screen showing the rail, the pager, and the branches. +client.post(f"{base}/branches/{root}/switch") + +with engine.begin() as conn: + conn.execute(text(f"PRAGMA user_version = {LATEST_VERSION}")) + +for b in branches(): + print(f" branch {b['id']}: name={b['name']!r} parent={b['parent_branch_id']} " + f"fork_depth={b['fork_depth']} depth={b['depth']} own={b['own_actions']} " + f"head={b['is_head']}") +print(f"fixture written: {OUT} (adventure {adv_id})") diff --git a/backend/tools/stress_session.py b/backend/tools/stress_session.py new file mode 100644 index 0000000..3ff9e56 --- /dev/null +++ b/backend/tools/stress_session.py @@ -0,0 +1,848 @@ +"""Drives a production-sized adventure through the real routes and reports what +each one costs in database bytes. + + cd backend + .venv/Scripts/python.exe -m tools.stress_session + .venv/Scripts/python.exe -m tools.stress_session --actions 200 --memories 100 + .venv/Scripts/python.exe -m tools.stress_session --no-embeddings + +The memory bank is on by default, and that matters. The round-two stress +harness ran with no embedding model configured, and embedding providers are +BYOK-only by construction, so `retrieve_memories` returned early every time. +That run measured the turn loop with its heaviest read disabled and reported +23 MB for a playthrough that costs an order of magnitude more. +`--no-embeddings` reproduces that configuration on purpose, to show the gap. It +is never the default, and it prints a warning. + +Everything here is synthetic. The fixture is generated to the shape of +production data, with 1536-dimension embeddings, context snapshots of about +74 kB, and retry variants. It never reads a real adventure, user, or backup. + +Only the network is faked, which means the LLM and the embedding endpoint. +Routing, sessions, the ORM, the scripting engine, and the context builder are +the real ones, because the bugs this harness exists to catch are in the layer a +mock would replace. + +It runs on a throwaway SQLite file by default. The measurement is which columns +of which rows a code path requests, and the ORM decides that identically on both +dialects. The dialects differ in how a value is encoded on the wire, especially +JSON, so treat the absolute figures as production-shaped rather than +production-exact, and compare one run against another. + +To measure the encodings SQLite cannot reach, which are bytea for the packed +vectors and json columns that psycopg parses before the meter sees them, set +`AIDND_STRESS_DATABASE_URL` to a throwaway Postgres database: + + AIDND_STRESS_DATABASE_URL=postgresql://…/stress_scratch \ + .venv/Scripts/python.exe -m tools.stress_session + +The harness writes, so it refuses any target whose database name does not +contain 'stress' or 'scratch'. Never point it at a database holding real users. + +Calibration. The fixture is sized from production and was re-measured on +2026-08-17 against the live Neon database, reading aggregates only: counts and +`octet_length` sums, never row contents. + + per action, text 886 B -> --narration-bytes 1700, alternating + with a one-line player input + longest adventure 607 actions -> --actions 600 + context_snapshot 232 KB/row -> --snapshot-bytes 232000 + memory bank, largest 100 memories, 6,144 B a vector + +The previous defaults were wrong in both directions at once and happened to +land near the right total. Actions were modeled at about 2.1 kB against a real +886 B, and stories at 200 actions against a real 607. The width was too large +and the length was too small, and the length is what a page load pays for. + +Filler text is generated word by word rather than repeated. A repeated sentence +compresses about a hundredfold and prose three- or fourfold, so the old fixture +would have made any compression measurement on `context_snapshot` meaningless. +""" + +import os +import sys +import tempfile +from pathlib import Path + + +def _early_keep(argv: list[str]) -> str: + """Reads `--keep` before argparse exists. + + The database location has to be decided before `app.database` is imported, + and that import is a few lines below. argparse still declares the flag, so + `--help` documents it and a typo is still an error. + """ + for i, arg in enumerate(argv): + if arg == "--keep" and i + 1 < len(argv): + return argv[i + 1] + if arg.startswith("--keep="): + return arg.split("=", 1)[1] + return "" + + +_keep = _early_keep(sys.argv[1:]) + +# This has to run before the app import, because `database.py` reads these at +# module scope. +# +# The default is a throwaway SQLite file. `AIDND_STRESS_DATABASE_URL` points the +# harness at a real Postgres instead, which is the only way to reach the +# encodings SQLite cannot exercise: bytea for the packed vectors, and json +# columns that psycopg parses into Python before the meter sees them. +# +# The name guard matters. This harness writes a whole synthetic adventure, so a +# URL that pointed at the production database would seed it with fake users and +# fake play. The target has to name itself as disposable. +_stress_url = os.environ.get("AIDND_STRESS_DATABASE_URL", "").strip() +if _stress_url: + if _keep: + sys.exit( + "--keep writes a SQLite file for the app to serve; it cannot be\n" + "combined with AIDND_STRESS_DATABASE_URL." + ) + _dbname = _stress_url.rsplit("/", 1)[-1].split("?")[0] + if not any(mark in _dbname.lower() for mark in ("stress", "scratch")): + sys.exit( + f"refusing to run against database {_dbname!r}.\n" + "This harness writes a synthetic adventure, so its target must be a\n" + "throwaway database with 'stress' or 'scratch' in the name." + ) + os.environ["AIDND_DATABASE_URL"] = _stress_url + os.environ.pop("DATABASE_URL", None) +elif _keep: + # A fixture to start the app against, rather than a temporary file the + # report discards. Every run rebuilds it from empty, because + # `build_fixture()` assumes an empty database on the SQLite path and a + # second run would otherwise add a second adventure next to the first. + _keep_path = Path(_keep).resolve() + _keep_path.parent.mkdir(parents=True, exist_ok=True) + _keep_path.unlink(missing_ok=True) + os.environ["AIDND_DB_PATH"] = str(_keep_path) + os.environ.pop("AIDND_DATABASE_URL", None) + os.environ.pop("DATABASE_URL", None) +else: + _tmp = tempfile.NamedTemporaryFile(suffix=".db", delete=False) + _tmp.close() + os.environ["AIDND_DB_PATH"] = _tmp.name + os.environ.pop("AIDND_DATABASE_URL", None) + os.environ.pop("DATABASE_URL", None) + +import argparse +import asyncio +import json +import random + +from fastapi import Depends +from fastapi.testclient import TestClient +from sqlalchemy import text + +from app import auth, limits, memorybank, models, security, seed, tree, worldstate +from app.context import cursors +from app.database import Base, SessionLocal, engine, get_db +from app.main import app +from app.providers import PromptParts +from app.routers import adventures + +from .dbmeter import Meter, kb +from .fakeprose import prose + +EMBEDDING_DIMS = 1536 + +# About 232 kB, measured on the largest adventure in production on 2026-08-17. +# The old figure here was 74 kB, taken from the comment in `models.py`. The real +# column averages 163 kB per row across the whole table and 232 kB on the +# adventure that matters, because the assembled prompt grows with the story +# behind it. +# +# The text is varied rather than one sentence repeated. A repeated sentence +# compresses about a hundredfold and real prose three- or fourfold, so a fixture +# built from repeats would make any compression measurement meaningless, and +# shrinking this column is the open question the fixture exists to answer. +SNAPSHOT_SYSTEM = None # set by _build_text() +SNAPSHOT_STORY = None + +PLAYER_INPUT = "> You crouch and look more closely at the grit on the floor." + +# `_build_text()` sets all three from the fixture arguments. +NARRATION = None +MEMORY_TEXT = ( + "You found a bandit camp above the ford and agreed to guide Gwen through " + "the tunnels in exchange for the iron key she took from the quartermaster." +) + + +def _build_text(args, rng: random.Random) -> None: + """Sizes the three variable-length fixture strings from the arguments. + + This is separate from `build_fixture` so that the sizes are decided once, + before anything is written, and so that a shape's cost depends on the flags + rather than on how many rows were generated first. + """ + global NARRATION, SNAPSHOT_SYSTEM, SNAPSHOT_STORY + NARRATION = prose(rng, args.narration_bytes) + # The assembled prompt is a system block plus the story so far. The split + # is roughly one to five in production. + SNAPSHOT_SYSTEM = prose(rng, args.snapshot_bytes // 6) + SNAPSHOT_STORY = prose(rng, args.snapshot_bytes - args.snapshot_bytes // 6) + + +# --------------------------------------------------------------- fake network + + +class FakeProvider: + """Stands in for the LLM, streaming one fixed line. + + It makes no network call, costs nothing, and returns the same text every + time. + """ + + # The turn engine records the cost of the call, so a stand-in provider has + # to carry this attribute even when it never calls anything. + last_usage = None + + def __init__(self, *a, **k): + pass + + async def generate(self, parts: PromptParts, *, temperature, max_tokens): + yield ("text", NARRATION) + + async def complete(self, system, user, *, max_tokens=None): + return MEMORY_TEXT + + +class FakeEmbeddings: + """Stands in for the embedding endpoint. + + It returns vectors of the production width, so what the turn writes back is + the size production writes back. + """ + + def __init__(self, rng: random.Random): + self.rng = rng + + async def embed(self, texts: list[str]) -> list[list[float]]: + return [ + [self.rng.uniform(-1.0, 1.0) for _ in range(EMBEDDING_DIMS)] + for _ in texts + ] + + +# -------------------------------------------------------------- rich fixture + +# The correctness fixture, as opposed to the scale fixture. +# +# The default fixture is sized from production and exists to measure bytes, so it +# leaves every column it does not measure at its default. That makes it a poor +# test of semantics, and phase 14 replaces the storage model underneath all of +# it. On a freshly built default fixture, the columns a story tree has to migrate +# correctly hold this: +# +# state_after, world_state_after The same value on all 600 rows. They +# were state_before, and NULL, until SP4 +# reversed them. A rollback over identical +# snapshots tests nothing. +# scenario_id, world_state Absent. There is no RPG layer, so the +# cooldown clock SP5 must not advance +# never runs. +# adventure_scripts None. A branch switch reuses the +# script_state rollback. +# memory_cursor, summary_cursor Both 0. SP3 replaces the cursors. +# sibling attempts Two attempts of byte-identical text with +# the first always live, so the one +# question SP4 has to answer, which +# attempt is live, has no observable +# answer. +# +# `--rich` fills in those columns and changes nothing else, so the measuring +# fixture's numbers stay comparable from run to run. It is a correctness fixture, +# so keep it small, such as `--rich --actions 30`. It provides variety per row +# rather than many rows. + +RICH_SUMMARY = ( + "You tracked the bandits to a camp above the ford, freed Gwen from the " + "quartermaster's cage, and struck a bargain: the iron key for safe passage " + "through the tunnels. The alarm has not yet been raised." +) + +RICH_CARDS = [ + ("character", "Gwen", "Gwen, ranger, her", + "A loyal ranger who owes you her life and says so rarely."), + ("location", "The Ford", "ford, river, crossing", + "A shallow crossing overlooked by the bandit camp."), + ("item", "Iron Key", "iron key, key", + "Taken from the quartermaster. Opens something below the camp."), +] + +# Ten gold a turn, so a state snapshot that failed to roll back shows a wrong +# total rather than no value. The retry tests use the same shape. +RICH_SCRIPT = """ +const modifier = (text) => { + state.gold = (state.gold || 0) + 10; + return { text }; +}; +modifier(text); +""" + + +def rich_stat_schema() -> dict: + """Returns the demo RPG schema, read from the seed data rather than invented. + + Using the real schema means the fixture exercises bands, cooldowns, + `max_delta_per_turn`, and NPC stat blocks in their real shapes. An invented + schema would diverge from the one it stands in for. + """ + path = seed.SEED_DIR / "04-rpg-world-state.json" + return json.loads(path.read_text(encoding="utf-8"))["stat_schema"] + + +def rich_script_state(turn: int) -> dict: + """Returns the script state as of `turn`. + + The values increase with the turn, so any snapshot identifies the turn it was + taken at, which is what makes an incorrect rollback visible. + """ + return {"gold": turn * 10, "turn": turn} + + +def rich_world_state(schema: dict, turn: int) -> dict: + """Returns a live world state that has been played to `turn`. + + `instantiate` returns the initial state, and a fixture whose every row holds + that same state cannot distinguish a restored snapshot from an unrestored + one. Here hp declines, mana declines, and a flag changes partway through. + """ + ws = worldstate.instantiate(schema) + ws["player"]["hp"] = max(20, 100 - turn) + ws["player"]["mana"] = max(0, 30 - turn // 2) + ws["world"]["day"] = 1 + turn // 20 + ws["flags"]["alarm_raised"] = turn > 12 + ws["flags"]["player_hidden"] = turn <= 12 + ws["_meta"]["last_changed"] = {"player.hp": turn} + return ws + + +def rich_attempts(rng: random.Random, index: int) -> tuple[list[str], int]: + """Returns distinguishable retry attempts, and which one the story tells. + + Every attempt in the default fixture carries the same text, with the live one + fixed at index 0. That is the behavior SP4 has to get right and the behavior + that fixture cannot test, so here the texts differ, the counts differ, and + the live attempt is often not the last. + """ + count = 3 if index % 12 == 1 else 2 + texts = [ + f"[attempt {n + 1} of {count} at turn {index}] {prose(rng, 240)}" + for n in range(count) + ] + # The live attempt is not always the newest. A player who retried twice and + # then went back to the first attempt is the case that breaks code assuming + # the live attempt is the last one written. + return texts, (0 if index % 18 == 1 else count - 1) + + +def rich_cursor_positions(action_count: int) -> tuple[int, int]: + """Return the memory and summary mark positions for the rich fixture. + + The marks start out as positions because that is the form a pre-SP3 + database holds. `build_fixture` turns them into anchors once the actions + they point at exist. + """ + return max(0, action_count - 8), max(0, action_count - 20) + + +def add_rich_extras(db, args, rng: random.Random, user, adventure) -> None: + """Adds everything `--rich` contributes apart from the actions themselves. + + The caller flushes the adventure first and writes the actions afterwards, + because the actions need the schema to snapshot a world state from. + """ + schema = rich_stat_schema() + scenario = models.Scenario( + user_id=user.id, + title="Stress (RPG)", + prompt="You crouch at the ford, watching the bandit camp.", + stat_schema=schema, + ) + db.add(scenario) + db.flush() + + adventure.scenario_id = scenario.id + adventure.world_state = rich_world_state(schema, args.actions) + adventure.script_state = rich_script_state(args.actions) + # The post-turn passes do nothing unless summarization is on and the marks + # are past the start. `build_fixture` translates the marks into anchors + # once the actions exist. See `rich_cursor_positions`. + adventure.auto_summarize = True + adventure.story_summary = RICH_SUMMARY + + for kind, name, keys, entry in RICH_CARDS: + db.add(models.StoryCard( + adventure_id=adventure.id, type=kind, name=name, keys=keys, entry=entry, + )) + + db.add(models.AdventureScript( + adventure_id=adventure.id, position=0, enabled=True, + name="Gold", description="Ten gold a turn.", output_js=RICH_SCRIPT, + )) + return schema + + +def add_second_adventure(db, rng: random.Random, user) -> int: + """Adds a short second adventure, so the fixture can detect cross-adventure + leaks. + + A tree scopes every read by branch, and a branch clause that omitted its + adventure would still look correct on a database holding one adventure. + """ + other = models.Adventure( + user_id=user.id, title="Stress (second)", script_state={}, + memory_bank_enabled=True, auto_summarize=False, + ) + db.add(other) + db.flush() + for i in range(6): + action = models.Action( + adventure_id=other.id, + type="ai" if i % 2 else "do", + text=f"[second adventure] turn {i}. {prose(rng, 200)}", + state_after=rich_script_state(i), + ) + tree.place_action(db, other, action) + db.add(action) + memory = models.Memory( + adventure_id=other.id, + text="This memory belongs to the second adventure and must never be " + "retrieved for the first.", + source_start=0, source_end=5, + ) + memorybank.set_vector(memory, [rng.uniform(-1.0, 1.0) for _ in range(EMBEDDING_DIMS)]) + tree.place_memory(db, other, memory) + db.add(memory) + return other.id + + +def _assert_live_variant_invariant(db, adventure_id: int) -> None: + """Checks that exactly one attempt per turn is live, on every turn. + + The check runs here rather than being assumed, because a coordinate with two + live siblings renders its turn twice and a coordinate with none omits the + turn. Both failures produce wrong output rather than an exception, so a + fixture that broke the rule would let incorrect code look correct. + """ + groups: dict[tuple, list[models.Action]] = {} + for action in ( + db.query(models.Action) + .filter(models.Action.adventure_id == adventure_id) + .all() + ): + groups.setdefault((action.branch_id, action.depth), []).append(action) + retried = 0 + for (branch_id, depth), rows in groups.items(): + live = [a for a in rows if a.live] + if len(live) != 1: + sys.exit( + f"fixture is inconsistent: branch {branch_id} depth {depth} has " + f"{len(live)} live attempts out of {len(rows)}." + ) + if len(rows) > 1: + retried += 1 + if not retried: + sys.exit("--rich built no retried actions; raise --actions above 6.") + + +# ------------------------------------------------------------------- fixture + + +def build_fixture(args, rng: random.Random) -> tuple[int, int]: + """Builds a user, settings, and one adventure at production scale. + + Returns `(adventure_id, user_id)`. + """ + # A SQLite run gets a new temporary file every time, so the fixture can + # assume an empty database. A Postgres scratch target persists between runs, + # and a second run would collide on the fixture user's unique email, so empty + # it first. This runs only for a target whose name passed the 'stress' or + # 'scratch' guard at the top of this module. + if _stress_url: + Base.metadata.drop_all(bind=engine) + Base.metadata.create_all(bind=engine) + db = SessionLocal() + try: + user = models.User(is_guest=False, email="stress@example.invalid") + db.add(user) + db.flush() + db.add(models.Settings( + user_id=user.id, + api_key=security.encrypt_secret("stress-key"), + model="stress-model", + endpoint_url="https://fake.invalid/v1", + # The default this tool exists to keep anyone from forgetting. + embedding_model="" if args.no_embeddings else "openai/text-embedding-3-small", + memory_bank_capacity=args.capacity, + )) + adventure = models.Adventure( + user_id=user.id, + title="Stress", + script_state={}, + memory_bank_enabled=True, + # Off, so that a turn measures only the turn. The post-turn pass + # is its own shape below, and running it during the measurement + # would combine the two. + auto_summarize=False, + ) + db.add(adventure) + db.flush() + + schema = add_rich_extras(db, args, rng, user, adventure) if args.rich else None + + for i in range(args.actions): + is_ai = bool(i % 2) + retried = is_ai and i % 6 == 1 + if args.rich: + # Distinguishable attempts, with a live one that is often not + # the last written. + texts, live = rich_attempts(rng, i) if retried else ([], 0) + if not retried: + texts = [f"[turn {i}] {NARRATION}" if is_ai + else f"[turn {i}] {PLAYER_INPUT}"] + else: + texts = [NARRATION, NARRATION] if retried else [ + NARRATION if is_ai else PLAYER_INPUT + ] + live = 0 + # The attempts at one turn share that turn's coordinate. Only the + # first one moves the head, so the rest copy the placement of the + # first instead of going through `tree.place_action`, which would + # read each of them as a new turn. This mirrors + # `attempts.add_attempt`, which the fixture cannot call directly + # because it makes the newest attempt live and the fixture needs a + # live attempt that is often not the newest. + first_attempt = None + for n, body in enumerate(texts): + action = models.Action( + adventure_id=adventure.id, + type="ai" if is_ai else "do", + text=body, + # The turn's assembled prompt is stored once, on the + # attempt the story tells. A superseded sibling keeps only + # its own slices. `app/attempts.py` maintains that + # invariant, and a fixture that ignored it would multiply + # the largest column in the database by the retry count. + context_snapshot=( + {"system": SNAPSHOT_SYSTEM, "story": SNAPSHOT_STORY} + if n == live else + {"raw_output": body} + ), + world_delta={"delta": {"player.hp": -3}, + "applied": [{"path": "player.hp", "old": 88, "new": 85}]}, + # Every third AI turn was retried once, so a turn is + # sometimes several rows sharing one coordinate. + live=(n == live), + # Under `--rich` the value increases with the turn, so an + # incorrect rollback shows a wrong number rather than no + # value. Otherwise `tree.stamp_outcome` writes the + # adventure's state, which does not change and therefore + # tests nothing. + state_after=rich_script_state(i) if args.rich else None, + world_state_after=( + rich_world_state(schema, i) if args.rich else None + ), + ) + # `create_all` builds a fresh database and stamps it LATEST, + # so no migration runs against it and the tree backfill never + # sees it. The fixture has to stamp its own nodes, or it would + # be the one database in the project whose actions have no + # branch. + if first_attempt is None: + tree.place_action(db, adventure, action) + first_attempt = action + else: + action.branch_id = first_attempt.branch_id + action.depth = first_attempt.depth + action.parent_id = first_attempt.parent_id + db.add(action) + + for i in range(args.memories): + memory = models.Memory( + adventure_id=adventure.id, + text=f"{MEMORY_TEXT} ({i})", + source_start=i * memorybank.MEMORY_INTERVAL, + source_end=i * memorybank.MEMORY_INTERVAL + memorybank.MEMORY_INTERVAL - 1, + # A bank in play is not uniformly active. Some memories are + # pinned, which means they are always retrieved, and some are + # evicted but kept for the UI. Both states have to survive being + # re-attached to nodes. + pinned=bool(args.rich and i % 9 == 0), + forgotten=bool(args.rich and i % 11 == 5), + ) + # Use the same call the app uses, so the fixture cannot store + # vectors in a shape production never produces. + memorybank.set_vector( + memory, [rng.uniform(-1.0, 1.0) for _ in range(EMBEDDING_DIMS)] + ) + tree.place_memory(db, adventure, memory) + db.add(memory) + + if args.rich: + # Translate the memory mark and the summary mark from the + # positions `add_rich_layers` set into nodes, now that actions exist + # to point at. This uses the same call the v1 importer uses. A + # fixture stamped LATEST never runs a migration, so skipping this + # would leave the one database whose marks are only positions. + db.flush() + memory_position, summary_position = rich_cursor_positions(args.actions) + cursors.anchor_at_position(adventure, cursors.MEMORY, memory_position) + cursors.anchor_at_position(adventure, cursors.SUMMARY, summary_position) + add_second_adventure(db, rng, user) + _assert_live_variant_invariant(db, adventure.id) + + db.commit() + return adventure.id, user.id + finally: + db.close() + + +def install_fakes(user_id: int, rng: random.Random) -> None: + embeddings = FakeEmbeddings(rng) + adventures.turns.OpenAICompatibleProvider = FakeProvider + memorybank.embedding_provider = lambda settings: embeddings + memorybank.summary_provider = lambda settings: FakeProvider() + auth.resolve_provider_config = lambda s, **k: auth.ProviderConfig( + "https://fake.invalid/v1", "stress-key", "stress-model", False + ) + limits.rate_limit = lambda *a, **k: None + limits.check_row_cap = lambda *a, **k: None + # Post-turn work scheduled in the background would be counted inside + # whatever scope was open. It is measured deliberately, as its own shape. + memorybank.schedule_post_turn = lambda adventure: None + + def _current_user(db=Depends(get_db)): + return db.get(models.User, user_id) + + app.dependency_overrides[auth.get_current_user] = _current_user + + +# --------------------------------------------------------------------- shapes + + +def shape_list(client, meter, adv_id): + """Measures the adventures index, which returns each adventure's latest + narration.""" + with meter.scope("GET /adventures (index)"): + r = client.get("/api/adventures") + _check(r) + + +def shape_load(client, meter, adv_id): + """Measures opening a finished adventure, which returns the whole story in + one response.""" + with meter.scope(f"GET /adventures/{{id}} (page load)"): + r = client.get(f"/api/adventures/{adv_id}") + _check(r) + + +def shape_turn(client, meter, adv_id): + """Measures one played turn, including memory retrieval.""" + with meter.scope("POST /adventures/{id}/actions (one turn)"): + r = client.post( + f"/api/adventures/{adv_id}/actions", + json={"type": "do", "text": "look more closely at the grit"}, + ) + _check(r) + + +def shape_insights(client, meter, adv_id): + """Measures the Insights dry run, which assembles a context without playing + a turn.""" + with meter.scope("GET /adventures/{id}/context (insights)"): + r = client.get(f"/api/adventures/{adv_id}/context") + _check(r) + + +def shape_memories(client, meter, adv_id): + """Measures the Memories drawer, which returns every memory and no + vectors.""" + with meter.scope("GET /adventures/{id}/memories (drawer)"): + r = client.get(f"/api/adventures/{adv_id}/memories") + _check(r) + + +def shape_post_turn(client, meter, adv_id): + """Measures summarization, embedding, and eviction after the turn is + saved.""" + with meter.scope("run_post_turn (background)"): + asyncio.run(memorybank.run_post_turn(adv_id)) + + +SHAPES = { + "list": shape_list, + "load": shape_load, + "turn": shape_turn, + "insights": shape_insights, + "memories": shape_memories, + "post_turn": shape_post_turn, +} + + +def _check(response) -> None: + if response.status_code >= 400: + sys.exit(f"shape failed: {response.status_code} {response.text[:400]}") + + +# ------------------------------------------------------------------- --keep + + +def make_bootable() -> None: + """Applies the two edits that make a measurement fixture servable by the app. + + Both edits are needed because `build_fixture()` builds a database for the + meter rather than for a browser. + """ + from app.migrations import LATEST_VERSION + + with engine.begin() as conn: + # `create_all()` builds the current schema but leaves the stamp at its + # default, and `bootstrap()` reads an unstamped existing database as + # very old. It would replay every migration against a schema that + # already has every column, and fail on the first one. + conn.execute(text(f"PRAGMA user_version = {LATEST_VERSION}")) + # In local mode, which is when `AIDND_MULTI_USER` is unset, + # `get_current_user()` looks for the row with `email IS NULL` and + # `is_guest` false. The fixture's user is a registered one, so without + # this update nothing owns the adventure and the app opens on an empty + # library. + conn.execute(text("UPDATE users SET email = NULL, is_guest = 0")) + + +def print_keep_notes(path: str, actions: int) -> None: + port = 8010 + print() + print(f"fixture kept: {path}") + print(f" {actions} actions, bootable in local mode. To scroll it:") + print() + print(f" cd backend && AIDND_DB_PATH={path} \\") + print(f" .venv/Scripts/python.exe -m uvicorn app.main:app --port {port}") + print(f" cd frontend && AIDND_API_PORT={port} npm run dev") + print() + # 8000 is the vite proxy's default, and another local app already listens + # there. That app shadows this API with its own SPA catch-all route, which + # looks like an empty database rather than a proxy problem. + print(f" Port {port} rather than 8000 on purpose; AIDND_API_PORT points vite at it.") + + +# ----------------------------------------------------------------------- main + + +def parse_args(argv=None): + p = argparse.ArgumentParser( + prog="tools.stress_session", description=__doc__.splitlines()[0] + ) + # 607 is the longest adventure in production as of 2026-08-17. Length is + # the dimension the old default of 200 got wrong. Real actions are smaller + # than this fixture used to make them, but real stories run three times + # longer, and length is what a page load pays for. + p.add_argument("--actions", type=int, default=600, + help="story actions in the fixture (default: 600, " + "production's longest adventure is 607)") + p.add_argument("--memories", type=int, default=100, + help="memories, all embedded (default: 100)") + # This is not the app's default of 80. A measuring instrument holds the + # fixture at the requested size rather than evict from it during a run. + p.add_argument("--capacity", type=int, default=200, + help="Settings.memory_bank_capacity; lower it below " + "--memories to exercise eviction (default: 200)") + # The longest adventure in production carries 886 B of text per action, + # averaged over both kinds. AI actions alternate with a one-line player + # input, so the AI half has to be about twice that. + p.add_argument("--narration-bytes", type=int, default=1700, + help="length of an AI action's text; alternating with a " + "one-line player input this averages ~890 B/action, " + "which is what production measures (default: 1700)") + p.add_argument("--snapshot-bytes", type=int, default=232_000, + help="context_snapshot per action; 232 KB is the average " + "on production's longest adventure, 163 KB is the " + "average across the whole table (default: 232000)") + p.add_argument("--shapes", default=",".join(SHAPES), + help=f"comma-separated subset of: {', '.join(SHAPES)}") + p.add_argument("--repeat", type=int, default=1, + help="run each shape this many times (default: 1)") + p.add_argument("--no-embeddings", action="store_true", + help="unset the embedding model — reproduces the round-two " + "blind spot, where the bank's cost is invisible") + p.add_argument("--rich", action="store_true", + help="populate the columns the measuring fixture leaves at " + "their defaults: state_after/world_state_after, an " + "RPG scenario and live world state, adventure scripts, " + "non-zero memory/summary cursors, pinned and forgotten " + "memories, distinguishable retry attempts, and a second " + "adventure. A correctness fixture rather than a scale " + "one — prefer it small (--rich --actions 30). Changes " + "what the shapes cost, so do not compare a --rich run " + "against a plain one") + # `_early_keep` also reads this flag at import time, because the database + # location has to be decided before `app.database` loads. It is declared + # here so that it appears in `--help` and an unknown flag is still + # rejected. + p.add_argument("--keep", metavar="PATH", default="", + help="write the fixture to PATH and leave it bootable, so " + "the app can serve it in a browser (default: a temp " + "file, discarded). SQLite only") + p.add_argument("--seed", type=int, default=7) + p.add_argument("--statements", type=int, default=5, + help="heaviest statements to print per shape (default: 5)") + return p.parse_args(argv) + + +def main(argv=None) -> int: + args = parse_args(argv) + chosen = [s.strip() for s in args.shapes.split(",") if s.strip()] + unknown = [s for s in chosen if s not in SHAPES] + if unknown: + sys.exit(f"unknown shape(s): {', '.join(unknown)}") + + rng = random.Random(args.seed) + _build_text(args, random.Random(args.seed ^ 0x5F5F)) + adv_id, user_id = build_fixture(args, rng) + install_fakes(user_id, rng) + + meter = Meter() + # Attach after the fixture is built. Building it is a write path no player + # takes, and its bytes would hide everything the shapes report. + meter.attach(engine) + + print(f"fixture: {args.actions} actions × {args.narration_bytes} B " + f"(+{args.snapshot_bytes // 1024} kB snapshot, deferred) · " + f"{args.memories} memories × {EMBEDDING_DIMS} dims · " + f"capacity {args.capacity}") + if args.rich: + print("RICH: correctness fixture — RPG scenario, per-action state " + "snapshots, scripts, cursors, distinguishable attempts, a second " + "adventure. Byte figures below are NOT comparable to a plain run.") + if args.no_embeddings: + print("WARNING: embedding model unset — memory retrieval will return " + "early and the bank's cost will not appear below.") + else: + print("memory bank: ON (embedding model configured)") + + with TestClient(app) as client: + for _ in range(args.repeat): + for name in chosen: + SHAPES[name](client, meter, adv_id) + + print(meter.render(statements=args.statements)) + print() + print(f"{'total across all shapes':<44}{kb(sum(s.total.fetched for s in meter.scopes)):>16}") + + app.dependency_overrides.clear() + adventures.turns._active_turns.clear() + + # After the shapes, not before: make_bootable() writes, and the meter is + # still attached until the report above is rendered. + if args.keep: + make_bootable() + print_keep_notes(os.environ["AIDND_DB_PATH"], args.actions) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/tools/tree_fixture.py b/backend/tools/tree_fixture.py new file mode 100644 index 0000000..6210b89 --- /dev/null +++ b/backend/tools/tree_fixture.py @@ -0,0 +1,181 @@ +"""A tree with a shape worth drawing, for driving the branch map by hand. + +`tools.branch_fixture` builds two branches of equal length, which is the case +the panel-refresh bug needed and the smallest tree that proves a switch. A map +needs the other case: several lines, leaving at different moments, one of them +forked off a fork, so lanes have to nest and the moment axis has to mean +something. Four branches, depths 26 / 20 / 22 / 22, forks at 5, 15 and 17. + +There is no frontend test runner, so this is the whole of the map's coverage: +it exists to be looked at. + + cd backend + .venv/Scripts/python.exe -m tools.tree_fixture /tmp/tree.db + AIDND_DB_PATH=/tmp/tree.db .venv/Scripts/python.exe \ + -m uvicorn app.main:app --port 8010 + +Then open http://127.0.0.1:8010/play/1, Branches → See the tree. The SPA is +served out of `frontend/dist`, so run `npm run build` first if it is stale. + +No LLM is called: the provider is scripted. +""" +import os +import sys + +sys.path.insert(0, os.getcwd()) + +OUT = sys.argv[1] if len(sys.argv) > 1 else "treefixture.db" +if os.path.exists(OUT): + os.remove(OUT) +os.environ["AIDND_DB_PATH"] = OUT +os.environ.pop("AIDND_DATABASE_URL", None) +os.environ.pop("DATABASE_URL", None) +os.environ.pop("AIDND_MULTI_USER", None) + +from fastapi.testclient import TestClient # noqa: E402 +from sqlalchemy import text # noqa: E402 + +from app import auth, limits, models # noqa: E402 +from app.database import Base, SessionLocal, engine # noqa: E402 +from app.main import app # noqa: E402 +from app.migrations import LATEST_VERSION # noqa: E402 +from app.routers import adventures # noqa: E402 + +SCHEMA = {"player": {"hp": {"min": 0, "max": 100, "initial": 100}}} + +PROSE = [ + "The stair turns and the lantern light goes with it.", + "Water somewhere ahead, and it is not running.", + "A door, and the frame around it is warm to the touch.", + "Something has been eating the salt off the stones.", + "The passage opens and the ceiling goes out of reach.", + "A shape at the edge of the light declines to move.", + "The floor here has been swept, recently, by something patient.", + "You find the second lantern, still warm, and nobody holding it.", +] + + +class ScriptedProvider: + calls = 0 + # The turn engine records the cost of the call, so a stand-in provider has + # to carry this attribute even when it never calls anything. + last_usage = None + + def __init__(self, *a, **k): + pass + + async def generate(self, parts, *, temperature, max_tokens): + i = ScriptedProvider.calls + ScriptedProvider.calls += 1 + yield ("text", f"{PROSE[i % len(PROSE)]}\n```state\n{{\"player.hp\": -2}}\n```") + + +Base.metadata.create_all(bind=engine) +db = SessionLocal() +user = models.User(is_guest=False, email=None) +db.add(user) +db.flush() +db.add(models.Settings(user_id=user.id, api_key="enc:dummy", model="test-model")) +scenario = models.Scenario(user_id=user.id, title="Thornwick", stat_schema=SCHEMA) +db.add(scenario) +db.flush() +adv = models.Adventure( + user_id=user.id, title="The Hollow Beneath Thornwick", scenario_id=scenario.id, + script_state={}, world_state={"player": {"hp": 100}}, memory_bank_enabled=True, +) +db.add(adv) +db.flush() +db.add(models.Action( + adventure_id=adv.id, type="start", + text="The cellar door has been shut since your grandmother died.")) +db.commit() +adv_id = adv.id +db.close() + +adventures.turns.OpenAICompatibleProvider = ScriptedProvider +auth.resolve_provider_config = lambda s: auth.ProviderConfig( + "http://fake", "k", "test-model", False) +limits.rate_limit = lambda *a, **k: None +limits.check_row_cap = lambda *a, **k: None + +client = TestClient(app) +base = f"/api/adventures/{adv_id}" + + +def play(n=1): + for _ in range(n): + r = client.post(f"{base}/actions", json={"type": "do", "text": "go on"}) + assert r.status_code == 200, r.text + + +def retry_and_get_discarded(): + """Retry the last turn and hand back the take the story left behind.""" + before = live_ai_ids() + r = client.post(f"{base}/retry") + assert r.status_code == 200, r.text + db = SessionLocal() + dead = [a.id for a in db.query(models.Action) + .filter(models.Action.adventure_id == adv_id, models.Action.type == "ai") + .order_by(models.Action.id) if not a.live] + db.close() + assert dead, "retry left no discarded take" + # The one that was live a moment ago is the one this retry replaced. + return [i for i in dead if i in before][-1] + + +def live_ai_ids(): + db = SessionLocal() + out = [a.id for a in db.query(models.Action) + .filter(models.Action.adventure_id == adv_id, models.Action.type == "ai") + .order_by(models.Action.id) if a.live] + db.close() + return out + + +def fork(action_id): + r = client.post(f"{base}/actions/{action_id}/fork") + assert r.status_code == 200, r.text + + +def branches(): + return client.get(f"{base}/branches").json() + + +# ---- the first telling: a long line, with two turns retried along it ---- +play(3) +early = retry_and_get_discarded() +play(6) +late = retry_and_get_discarded() +play(4) +root = branches()[0]["id"] +client.patch(f"{base}/branches/{root}", json={"name": "The hard way down"}) + +# ---- a line that left early, and a line that left it again ---- +fork(early) # now reading the early attempt, on its own branch +play(5) +mid = retry_and_get_discarded() +play(2) +second = [b for b in branches() if b["is_head"]][0]["id"] +client.patch(f"{base}/branches/{second}", json={"name": "Down the other stair"}) + +fork(mid) # a fork off a fork +play(3) + +# ---- and one more off the first telling, much later ---- +client.post(f"{base}/branches/{root}/switch") +fork(late) +play(2) + +# Leave the reader on the deepest line, so the map opens on a nested lane. +deep = [b for b in branches() if b["parent_branch_id"] not in (None,) + and b["name"] is None][0] +client.post(f"{base}/branches/{deep['id']}/switch") + +with engine.begin() as conn: + conn.execute(text(f"PRAGMA user_version = {LATEST_VERSION}")) + +for b in branches(): + print(f" branch {b['id']}: name={b['name']!r} parent={b['parent_branch_id']} " + f"fork_depth={b['fork_depth']} depth={b['depth']} own={b['own_actions']} " + f"head={b['is_head']}") +print(f"fixture written: {OUT} (adventure {adv_id})") diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..93b65ef --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,11 @@ +services: + ai-dnd: + build: . + ports: + - "8000:8000" + volumes: + - ai-dnd-data:/data + restart: unless-stopped + +volumes: + ai-dnd-data: diff --git a/docs/GUIDE.md b/docs/GUIDE.md new file mode 100644 index 0000000..40d4d10 --- /dev/null +++ b/docs/GUIDE.md @@ -0,0 +1,1227 @@ +# AI D&D — design notes + +How the engine works and why it is built this way. The README covers what the project does +and how to run it; this covers the reasoning behind the parts that had a real choice in +them. + +Part 1 is the AI layer, which is where most of the design effort went. Parts 2 and 3 are +what makes it a service rather than a demo. Part 4 is the web plumbing, kept short. + +--- + +## Contents + +- [Part 0 — Orientation](#part-0--orientation) +- [Part 1 — The AI layer](#part-1--the-ai-layer) + - [1.1 The turn pipeline](#11-the-turn-pipeline) + - [1.2 Context assembly is a budget problem](#12-context-assembly-is-a-budget-problem) + - [1.3 World state: the AI proposes, Python referees](#13-world-state-the-ai-proposes-python-referees) + - [1.4 Output length, by measurement](#14-output-length-by-measurement) + - [1.5 The memory bank](#15-the-memory-bank) + - [1.6 Streaming](#16-streaming) + - [1.7 The scripting sandbox](#17-the-scripting-sandbox) + - [1.8 Why there is no agent framework](#18-why-there-is-no-agent-framework) +- [Part 2 — Data and correctness](#part-2--data-and-correctness) +- [Part 3 — Production concerns](#part-3--production-concerns) + - [3.6 Counting visits](#36-counting-visits) +- [Part 4 — The web plumbing, briefly](#part-4--the-web-plumbing-briefly) +- [Part 5 — Measured results and known limitations](#part-5--measured-results-and-known-limitations) + +--- + +# Part 0 — Orientation + +## What the thing is + +An AI Dungeon clone. You write a scenario, then play an open-ended text adventure where a +language model narrates the world. You type "I open the door", the model writes what +happens next, and it remembers what came before. + +Four things make it more than a chat wrapper: + +1. **A context engine.** The model has a limited input window. The app decides, every + single turn, which pieces of the story get to be in the prompt and which get dropped. +2. **A world-state engine.** The scenario declares stats (`hp`, `trust`, `day`). The model + proposes changes to them each turn; a Python engine decides what actually sticks. +3. **A story tree.** The story is not a list. Any turn can hold more than one take, and + writing below one that isn't the live one starts a branch that borrows every turn above + the fork rather than copying it. +4. **A scripting sandbox.** Real AI Dungeon JavaScript scripts import and run, inside an + embedded QuickJS interpreter. + +Runs locally against Ollama for free, or hosted against any OpenAI-compatible endpoint. + +## The stack, and what each part is doing + +| Piece | What it does here | +|---|---| +| **FastAPI** (Python) | The HTTP server. Every URL like `/api/adventures/3/actions` maps to a Python function. Also does the SSE streaming. | +| **SQLAlchemy** | The ORM. `Adventure`, `Action`, `Memory` are Python classes; SQLAlchemy turns them into tables and turns attribute access into `SELECT`s. | +| **SQLite / Postgres** | The database. SQLite is a single file on disk (local). Postgres is a server (hosted, on Neon). Same code talks to both. | +| **React** (JavaScript) | The UI. Describes what the screen should look like for a given state; when the state changes it re-renders. | +| **Vite** | The frontend build tool and dev server. Bundles React into plain JS the browser can load. | +| **httpx** | The HTTP client used to call the model endpoint. | +| **tiktoken** | Counts tokens, so the budgeting is arithmetic rather than a guess. | +| **QuickJS** | A small embeddable JavaScript engine, used as a sandbox for user scripts. | + +The whole thing is one process in production: FastAPI serves the API *and* the built React +files from the same port. + +## The shape of one request + +``` +you tap "Do" + → browser sends POST /api/adventures/3/actions {type:"do", text:"open the door"} + → FastAPI route: check ownership, rate limit, turn lock + → assemble the prompt + → POST to the model endpoint with stream=true + → tokens come back one at a time + → each token is forwarded to the browser as a Server-Sent Event + → React appends it to the screen as it arrives + → when the stream ends: parse the state block, referee it, save the action +``` + +--- + +# Part 1 — The AI layer + +## 1.1 The turn pipeline + +Everything that happens between "player pressed a button" and "text is on screen". +Source: `backend/app/routers/adventures.py` (`_generate_turn`). + +``` +player input + → onInput script hook (user JS may rewrite or block it) + → store the player action + → retrieve memories (embed recent story, cosine-rank the bank) + → build_context() (the budget allocator) + → onModelContext script hook (user JS may rewrite the whole prompt) + → snapshot the exact prompt (for the Insights panel) + → provider.generate() (streamed, token by token) + → onOutput script hook + → extract the fenced state block, referee the delta, strip it from the prose + → save the action, stamping on it the script + world state it leaves behind + → fire-and-forget: summarize + embed in the background +``` + +Two design choices are visible in that list before any of the details. + +**The prompt is snapshotted, not reconstructed.** Every AI action stores the exact text +that was sent to the model. That's what powers the Insights panel: open any turn and see +each context component, its token cost, and why it was included. It's also what makes +prompt bugs findable. The cost is storage (~74 KB per turn), which turns into a real +performance problem later. See [2.5](#25-the-189x-egress-fix). + +**Every node records the state it leaves behind.** `state_after` and `world_state_after` +are attached to the action once its hooks and its delta have run, so a node carries the +stats and the RPG values as they stood when that turn finished. Rewinding to *before* +a turn is then a read of the node in front of it, which is the same operation as switching +to another branch: one mechanism, and the reason undo, retry, and branch switching all put +the numbers back instead of only rewriting text. (These were `*_before` fields originally; a +tree needs the *after* value, because a branch's tip is what a reader standing on it should +see.) + +--- + +## 1.2 Context assembly is a budget problem + +Source: `backend/app/context/builder.py`. + +### The problem + +The model can only read so much. Say the budget is 8,000 tokens. A 200-turn adventure has +far more story than that. Something has to be dropped, and *what* gets dropped decides +whether the story stays coherent. + +### The naive version + +Send the last N turns. That breaks in two directions: N turns of short exchanges wastes the +window, and N turns of long ones overflows it. It also throws away the things that matter +most: the premise, the character sheet, the fact that you promised the innkeeper you'd +return. + +### What this app does + +Split the prompt into **fixed** sections and **elastic** ones. + +Fixed (always included, whatever they cost): + +| Section | What it is | +|---|---| +| `narrator` | The system prompt: how to write. | +| `world_state_guide` | The stat legend: what each stat means, its range, its bands. | +| `world_state` | Current values of every stat, plus NPCs in scene. | +| `world_state_rule` | How to report changes. | +| `ai_instructions` | Per-adventure steering. | +| `persona` | Who the player is playing as: name, pronouns, description. Only the player edits it, so it never moves. | +| `plot_essentials` | AI Dungeon's "Memory": the premise. | +| `story_summary` | The auto-maintained running summary. | +| `used_memories` | Top-K retrievals from the memory bank. | + +Elastic (fit into what's left): + +| Section | Rule | +|---|---| +| `world_lore` | Story cards triggered by keywords in recent text. Capped at 40% of the remaining budget. | +| `history` | Story turns, newest first, until the budget runs out. | + +The algorithm is three lines of arithmetic: + +```python +reserved = sum of every fixed section + author's note + length hint + reminder +available = max(256, context_token_budget - reserved) +``` + +Then cards spend up to `available * 0.4`, and history spends `available - cards_used`, +filling backwards from the newest turn. + +### The details that are decisions + +**Cards are capped at 40% of the elastic budget.** Story cards are triggered by keyword +match, so a scene mentioning six named things could pull in six lore entries and leave no +room for the story itself. The cap makes the failure mode "some lore is missing" instead +of "the model has no idea what just happened". Cards that don't fit are still *reported* +to Insights with `included: false`, so the UI can show the lore that got squeezed out. + +**History fills newest-first and stops.** Oldest turns fall out. This is the right +direction because the old material is not actually lost: it has been summarized into +memories and the running summary, which are in the fixed section. + +**If even the single newest turn is over budget, it gets hard-truncated** rather than +dropped. A prompt with no story at all would produce nonsense; a prompt with the tail end +of the last turn produces something. + +**The author's note is injected 3 actions from the end**, not at the top. +`AUTHORS_NOTE_DEPTH = 3`. Instructions placed near the end of a prompt have more influence +on what comes next than instructions at the top, because of recency. The author's note is a +steering control ("keep it tense"), so it goes where steering works. + +**The world-state reminder goes dead last.** The full emit rule lives up in the system +block, hundreds of tokens away from where the model starts writing. A one-line reminder +occupies the final slot. Same recency logic, applied to the thing most likely to be +forgotten. + +**Past AI turns get their state block re-attached.** The state block is stripped from the +text before it's stored, so a replayed history would show the model twenty of its own past +turns that *contain no state block*. That teaches it, by imitation, to stop emitting one. +So `_history_text()` reconstructs the block from the stored delta and re-appends it when +building history. The model sees its own pattern and keeps following it. + +### The performance trap hiding in this + +Building the context needs the newest ~6,000 tokens of story. The obvious implementation +reads `adventure.actions`, which loads every row of the adventure, and then throws 90% of +it away. At turn 200 that was 839 KB of database reads to use maybe 70 KB, and it grew +every single turn. + +`backend/app/context/history.py` fixes it by serving three shapes directly from SQL: a +tail, a slice, and a count. `window_covering()` fetches the newest 32 actions, measures +their real token count, and if that's short of the budget it *projects* how many more it +needs from the average length just measured rather than blindly doubling: + +```python +average = tokens / len(actions) +projected = int(budget / average * 1.15) + 8 +``` + +Each round fetches only what it doesn't already hold, so no row is read twice. Result: the +same turn costs 129 KB instead of 839 KB, and stops growing at around turn 50. The cost is +bounded by the context budget instead of by the length of the story. + +There's a second rule in that module worth naming: **if the actions are already loaded in +memory, slice them instead of querying.** The scripting pipeline hands the whole history to +user scripts (AI Dungeon's API requires it), so on a scripted adventure the rows are already +there. Issuing a query beside them would mean paying twice. + +--- + +## 1.3 World state: the AI proposes, Python referees + +Source: `backend/app/worldstate/engine.py`, `plan/12-phase-rpg-world-state.md`. + +### The problem + +You want an RPG layer: hit points, trust, quest progress. Who owns the numbers? + +### Three options, and why two lose + +**Option A: a deterministic dice engine.** The player types "attack the goblin", the +engine rolls, applies damage, and the model narrates the result. This is what a real RPG +does. It loses here because the action space is unbounded: the player can type anything, and +mapping arbitrary natural language onto a fixed rules system is a harder problem than the +one being solved. + +**Option B: the model owns the numbers.** Let it track hp in the prose and trust it. This +fails immediately. Models are bad at arithmetic, worse at remembering a number across +twenty turns, and completely unable to obey their own frequency rules. Tell one "only +change this every 5 turns" and it will change it every turn. + +**Option C, chosen: the model proposes, the engine disposes.** The model narrates and +appends a JSON delta of what changed. Python validates and clamps it before anything is +stored. + +```` +narration: "The blade catches your shoulder. Gwen shouts and drags you back." + +```state +{"player.hp": -15, "npc.gwen.trust": 5, "milestones.escaped": true} +``` +```` + +The engine then applies, in order: + +| Rule | What it stops | +|---|---| +| Path must exist in the schema | Hallucinated stats | +| Value must be the right type | `"a lot"` instead of `-15` | +| Cooldown | Changing a stat more often than the scenario allows | +| Counters can't decrease | The in-game day going backwards | +| `max_delta_per_turn` | Losing 90 hp to a stubbed toe | +| Clamp to `min`/`max` | Negative hp, trust above 100 | +| Milestones are sticky, `true` only | Un-completing a quest | +| Flags are two-way booleans | (Deliberately unrestricted: that's what flags are for) | + +Everything it rejects is *reported*, not silently swallowed. The Insights panel shows +applied, clamped, and rejected paths per turn, and the chip under each narration shows what +actually changed. + +### The reliability mechanism: word bands + +A stat can carry **bands**: + +```json +"hp": { "min": 0, "max": 100, "initial": 100, + "bands": [[0,20,"very weak"],[20,40,"hurt"],[40,60,"minor damage"], + [60,90,"healthy"],[90,100,"full health"]] } +``` + +Two things use them. The live state block shows the current band label, `hp 55/100 (minor +damage)`, so the model reads a *word*, not just a number. And the stat guide shows the +whole ladder once per turn, so the model can see the full scale it's reasoning across. + +The point: models reason well over semantics and badly over arithmetic. "He's badly hurt, +so a solid hit should take him to very weak" is a judgment a model can make. "55 minus 22 is +33" is one it will get wrong often enough to matter. + +### The failure philosophy + +Nothing in the world-state engine raises. A malformed delta returns `{}` and the turn +continues. The parser is deliberately tolerant: it strips trailing commas and leading `+` +signs on numbers, both of which weaker free models emit and strict JSON rejects. It accepts +a fence labeled `state`, one labeled `json`, or an unlabeled one, and falls back to a bare +JSON object at the end of the text, but only if it parses into something that looks like a +delta, so prose ending in `}` is never eaten. + +This matters because the hosted demo runs on free-tier models. A stricter parser would mean +a good model works and a free one doesn't. + +### One call, not two + +The model narrates *and* emits the delta in a single request. The alternative, narrating and +then making a second call to extract structured state, is more reliable per call and costs +twice the latency and twice the rate-limit budget. On the free tier (20 requests/minute) +that would halve the playable turn rate. The tolerant parser plus the terminal reminder was +the cheaper way to buy the same reliability. + +--- + +## 1.4 Output length, by measurement + +### The problem + +`max_output_tokens` is a hard wall the endpoint enforces mid-sentence. Hit it and whatever +is being written gets cut off. Since the state block is emitted *last*, the state block is +what gets lost. The turn narrates fine and silently records nothing. + +### First attempt + +Tell the model its budget: *"keep this turn under about N words"*. + +### What the measurement showed + +Average turn length went from **174 words to 246**, and every run was longer than every +unhinted run (n=5). Phrased as a budget, the number reads as a *target to fill*. The hint +pushed turns toward the very wall it existed to protect. + +### The fix + +Phrase it as a ceiling, and say explicitly that a typical turn is much shorter: + +``` +[Hard limit: this turn must not exceed 412 words. Write only as much as the +moment needs — a typical turn is much shorter. Finish the narration and append +the state block well inside the limit.] +``` + +Average came back to 170 words, and the state block survived at tight caps. + +### And the arithmetic around it + +```python +words = int((max_output_tokens - 50) * 0.75 * 0.90) +``` + +- `- 50` (`LENGTH_HEADROOM`): tokens held back for the state block itself. +- `* 0.75` (`WORDS_PER_TOKEN`): models can't count their own tokens, but they do follow a + word budget. English prose is roughly 0.75 words per token. +- `* 0.90` (`LENGTH_BUFFER`): a word budget is a suggestion the model overshoots, and the cap + it protects is a hard wall. Aim 10% short so the overshoot lands in slack. +- Below 40 words the hint is dropped entirely: it stops earning its tokens. + +--- + +## 1.5 The memory bank + +Source: `backend/app/memorybank.py`. + +### The problem + +Story history falls out of the context window as the adventure grows. Turn 4 said you +promised the innkeeper you'd return. At turn 90 that's long gone from the prompt, but if +you walk back into the inn, it should come back. + +### The three layers + +``` +raw turns → memories → story summary +(verbatim) (every 6 turns) (rewritten every 15 turns) + ↓ + embeddings → cosine similarity → top-K into the prompt +``` + +| Layer | Cadence | Purpose | +|---|---|---| +| **Memory** | Every 6 actions, starting at 12, once one action sits past the block | One or two past-tense sentences of concrete fact. | +| **Story summary** | Every 15 actions | A single ≤250-word overview of the whole plot, rewritten by folding in the new memories. | +| **Retrieval** | Every turn | Embed the last 4 actions (≤600 tokens), cosine-rank the bank, inject the top K (default 5). | + +Retrieval is the part that answers the innkeeper problem: the promise is a memory, the +memory has a vector, walking into the inn produces a query vector near it, and it comes +back into the prompt. + +### The decisions inside it + +**A memory attaches to the node whose block it ends on.** Not to the adventure, and not to +a *position* in a list of actions, but to a `(branch_id, depth)` coordinate. That is what +makes "which memories described this turn?" an indexed lookup rather than a scan for rows +whose covered range has fallen off the end of the story, and it is what makes memories +inherit correctly across a fork: the ones above the fork point already sit on ancestors +both lines read. + +This is also how repair works. When a turn's text is replaced or removed (a retry, an undo, +a deleted action), `forget_node` withdraws the memory attached to that coordinate and +rewinds both marks to just before the stretch it covered, so the ground is summarized again +from what the story now says. An earlier version instead held the newest action back a turn +so it could never be summarized before it stopped being retryable. Correctness no longer +rests on that, because the repair exists whether or not the invalidation happens at the tip. + +**A block still waits for one action to settle past it** (`SETTLE_SLACK`), and that is a +cost rule rather than a correctness one. Retry and take-switching both refuse anything but +the newest action, so a memory whose block ends on the tip is the one memory a player can +still throw away: every retry of that turn writes it, withdraws it, and writes it again. A +block closes every 6 actions and a normal turn writes 2, so without the slack that is one +turn in three. The slack costs nothing in return, because the block that just closed is +still in the history window in full — a memory of it says what the model can already read. +Memories earn their place once the raw text has scrolled out, which is never the turn the +block closed. + +**Cursors only advance on success.** Every AI call in this module is best-effort. If +summarization fails, the function returns and the cursor is unchanged, so the same block is +retried on a later turn. There is no retry loop, no dead-letter queue, and no backoff: the +cadence *is* the retry mechanism. Failures are logged to the debug page. + +**Summarization is fire-and-forget, in a background task with its own DB session.** The +player's turn is already on screen; making them wait for a summarization call would add a +second or two of latency to every sixth turn for no visible benefit. The task holds strong +references to itself (the event loop only keeps weak ones, so a fire-and-forget task can +otherwise be garbage-collected mid-run) and a per-adventure guard set stops two from +overlapping. + +**Pinned memories count toward `top_k`.** Pinned ones are always injected; unpinned ones +fill up to `top_k - len(pinned)`. Without that, 6 pinned memories plus `top_k=5` injects 11 +and blows the budget the whole context engine exists to respect. + +**A dimension mismatch scores 0.0, it doesn't crash.** If the user changes their embedding +model, old 768-dim vectors get compared against a new 1536-dim query. `zip()` would happily +truncate and score garbage silently. An explicit length check returns 0.0 instead. + +**Eviction is LRU-ish, and evicted memories are kept.** Over capacity (default 200), the +least-used, least-recently-used unpinned memories are marked `forgotten` rather than +deleted, so the UI can still show them and you can un-forget one. + +**Background calls never spend the shared demo key.** The summarization and embedding +providers are built directly from the user's own settings and never from the demo config, +and their call sites are skipped when the turn is running on the demo key. Summarization is +unmetered background spend; the demo key is server-funded. Both facts together would be a +bill. + +--- + +## 1.6 Streaming + +The model produces tokens one at a time. Waiting for the whole reply before showing +anything makes a 20-second generation feel broken. + +**Server-Sent Events (SSE)** is the mechanism: an HTTP response that stays open and pushes +`data: {...}` lines as they become available. It's one-directional (server → browser), +which is exactly what's needed here. WebSockets would be a bidirectional connection for a +unidirectional problem. + +The chain: + +``` +model endpoint --SSE--> FastAPI --SSE--> browser --> React state --> screen +``` + +FastAPI reads the provider's stream, and for each chunk yields +`data: {"type":"chunk","text":"..."}`. The frontend reads the response body with a +`ReadableStream` reader, buffers on `\n\n` boundaries, and dispatches each parsed event. + +Event types: `player` (the stored player action), `reasoning` (thinking-model traces, which +stream into a separate collapsible panel with their own token budget), `chunk` (story +text), `stopped` (a script blocked the turn), `error`, `done`. + +Two production details that only show up when hosted: + +- `X-Accel-Buffering: no`: nginx-style reverse proxies buffer responses by default, which + turns a stream into one big delivery at the end. This header tells them to flush each + event. +- The security-headers and body-size middlewares are written as **pure ASGI** rather than + Starlette's `BaseHTTPMiddleware`, because the latter buffers the response body and would + break streaming. + +**The empty-reply case is diagnosed, not reported as "empty".** If a reasoning model +streams thinking but no story text, it spent its whole budget thinking. The error says so +and tells you which three settings to change. + +--- + +## 1.7 The scripting sandbox + +Source: `backend/app/scripting/`. + +Real AI Dungeon scripts are JavaScript files defining `modifier(text)` and calling it as +the last line, with globals like `state`, `history`, `storyCards`. To be compatible, this +app runs the same contract in an embedded **QuickJS** interpreter. + +The safety properties are mostly structural: + +| Property | How | +|---|---| +| No filesystem, network, or process access | QuickJS has none by default: nothing was removed, nothing was added | +| Memory cap | 16 MB per run | +| CPU cap | 2 seconds per run | +| No shared state between runs | A fresh `Context` per hook execution | +| A broken script can't break a turn | Every failure comes back as `.error` with text/state/cards unchanged; the pipeline logs it and continues | + +Data crosses the boundary as JSON: Python serializes `{state, text, history, storyCards, +info}` in, and the script's results out. There is no object bridge to exploit. + +One deliberate bug-compatibility: `addStoryCard` returns the new card's *index*, so the +first card returns `0`, which is falsy, so `if (!addStoryCard(...))` misfires. That's +upstream AI Dungeon's behavior. It's documented in the code and left alone, because +matching real scripts is the whole point of the feature. + +--- + +## 1.8 Why there is no agent framework + +Graph-based agent frameworks (LangGraph and similar) earn their complexity with +**branching, cyclic, multi-step control flow**: a graph of nodes where the path depends on +what the model decides, with loops, retries, tool calls, and persisted state between steps. + +This turn pipeline is a **fixed linear sequence with exactly one model call**. There is no +routing decision, no tool selection, no loop. The "graph" is: + +``` +hook → retrieve → build → hook → call → hook → referee → store +``` + +Every turn takes that path. Adding a graph framework would mean carrying its state +abstraction, its serialization model, and its debugging surface to express a straight line. + +There is also a specific reason a framework's context handling wouldn't fit: **the +budgeting logic is the product.** Buffer-window and summary-memory abstractions are +opinionated about how to fit history into a window. Here the Insights panel exposes each +context component, its token cost, and the trigger word that pulled it in, so the assembly +has to be explicit and inspectable. + +**When it would be the right call:** if the design went toward the two-call version, where +narration is followed by a separate structured-extraction step, with a retry branch when +extraction fails and a tool-calling path for dice, that is a graph, and hand-rolling it +would get ugly fast. + +**This is not the same branching as the story tree (§2.2).** "No branching" here +describes *control flow*: the code path a single turn takes through the backend. That path +never forks. It is not true of the *data* the app stores. When a player rewinds and plays a +turn differently, that action creates a new branch in the saved history. One turn's +execution is a straight line. The sequence of turns across a playthrough is a tree. The two +claims are about different things and do not conflict. + +--- + +# Part 2 — Data and correctness + +## 2.1 The domain model + +``` +User + ├─ Scenario (the template) ── stat_schema, prompt, memory, author's note + │ └─ StoryCard, Script + └─ Adventure (the playthrough) ── world_state, script_state, head_branch_id/head_depth + ├─ Branch (one line of it) ── parent_branch_id, fork_depth, lineage, name + ├─ Action (one node) ── branch_id, depth, parent_id, live, + │ text, context_snapshot, state_after + ├─ StoryCard (its own copy) + ├─ Memory (text, embedding, branch_id, depth, use_count) + └─ AdventureScript +``` + +**The decision that shapes everything: template vs instance.** A scenario declares what +stats *exist*; an adventure holds what they *are* right now. Creating an adventure copies +the scenario's story cards, scripts and plot fields into it, so editing a scenario later +never mutates a game in progress. (There's an explicit opt-in "Update from scenario" flow +for when you *do* want that, which diffs the two and shows you what would change.) + +Same reasoning as instantiating a class: shared definition, independent state. + +## 2.2 The story is a tree + +The largest structural change the project has had, and the one with the most reasoning +behind it. + +### The problem + +The story used to be a list, and a mutable one. Retry rewrote the last entry in place; +undo and delete removed entries from the middle. Everything derived from the story, such as +the memories, the running summary, and the two marks saying how far each had got, was +indexed by *position in that list*, and a position means something different after +anything in front of it is deleted. + +That single fact produced a family of bugs that all looked different: + +- Deleting a middle action slid a never-summarized action down into the "already + covered" range, so a *recent* action silently never became a memory. +- Discarding a memory left its actions behind the mark, describing nothing. +- Retry rewrote an action's text after the mark had passed it, so its memory described + narration that was no longer in the story. +- The retried row was still attached to the adventure while its replacement was being + written, so the model was shown the attempt it was meant to replace and wrote a + *continuation* of it. That exclusion had to be threaded through four separate readers. +- Attempts lived in a JSON array on the row with a mirrored copy of the live one in the + ordinary columns, which is a repeating group and a denormalisation in one. + +Each was fixed where it was found. The pattern only becomes visible when you line them +up: **they are all the same bug, and it is that the story is a list nobody may reorder.** + +### The shape + +Make the story a tree, and none of them are reachable. + +Every action is a **node** with a `branch_id` and a `depth`. A **branch** is one line +through the tree; it holds the nodes played on it and *borrows* everything before its +fork point from its ancestors. Nothing is ever copied, and apart from an explicit delete, +nothing is ever removed. + +``` +branches(id, adventure_id, parent_branch_id, fork_depth, lineage, name) +actions(id, adventure_id, branch_id, depth, parent_id, live, text, …, state_after) +memories(…, branch_id, depth) +adventures(…, head_branch_id, head_depth) +``` + +`depth` is a position along *a* path, not a global turn number: `A4` and `B4` are two +alternatives, not two turns. Reading branch C, whose tip is at depth 7 and which left B +at 5, which left A at 3: + +```sql +SELECT * FROM actions +WHERE (branch_id = 'C') + OR (branch_id = 'B' AND depth <= 5) + OR (branch_id = 'A' AND depth <= 3) +ORDER BY depth DESC LIMIT 32 +``` + +→ `A0 A1 A2 A3 B4 B5 C6 C7`. + +**Why `branch_id` + `depth` rather than parent pointers alone.** Parent pointers are the +obvious way to store a tree and the wrong way to read one: reading a story would be N +round trips up a chain, which throws away the windowed history work (§1.2) that made a +turn's read cost flat. Depth replaces the old `index` as the ordering key, so the reads +keep the shape they already had. + +### The lineage, and why fork count doesn't cost anything + +The OR-clause above is not reconstructed per read. It is stored on the branch row as +`lineage`, for example `[(C, ∞), (B, 5), (A, 3)]`, computed once when the fork happens, +from the parent's lineage plus one entry. `context/lineage.py` is the only module that +knows how to turn it into a query, which is deliberate: one forgotten clause shows the +wrong story and reports nothing. + +Two properties of the shape do the real work: + +- **The ranges are disjoint and descending.** A branch's own nodes always sit deeper than + its fork point, and each ancestor is capped at the fork depth of the branch beneath it. + So ordering the whole clause by `depth DESC` reads entry 0's nodes, then entry 1's, + then entry 2's, which means a tail read can use the newest few entries and stop. +- **Clause count is bounded by the context window, not by fork count.** A 200-fork story + whose newest branch is 40 turns long reads with *one* clause, because the window is + covered before the second entry is reached. + +A branch stores no story of its own, so a fork costs an id, a parent, a fork depth and a +cached ancestry. Measured on a 40-turn story forked twenty times against the same story +flat: a page load of **31,652 B against 31,433 B, a 1.007× ratio**, or about **103 bytes +per branch**. No migration, no vacuum, no copy. + +### What a player actually does + +None of the above is what the screen shows. In the player's words: + +> Any turn can gain another **take**. On an AI turn that means regenerate; on your own +> message it means type something else. Stepping between takes with `‹ 2/4 ›` is free: +> the story below simply empties, because that take has no children yet. **A branch is +> created when you write below a take that is not the live one**, never before. + +That rule collapses two operations into one and deletes a distinction from the UI. The +first version of this screen had a chip that *switched* at the tip and only *previewed* +above it, with a second button to take that line: one control whose meaning depended on +where the reader was standing. The rule above replaced it with a pager that only ever +steps, a fork button on every turn, and no tip-versus-past distinction at all. The +distinction survives in the implementation, where it decides whether a write needs a +branch: at the tip the attempts are still leaves nobody has built on, so taking one is a +switch and no branch is created. + +### Takes are grouped by parent, not by coordinate + +The load-bearing detail, and the one that is not obvious. + +The natural way to find "the other takes of this turn" is by coordinate: same branch, +same depth. It is wrong in both directions: + +``` +B ── C C1 C2 <- three takes, one parent (B) + │ └── D1' D2' <- two takes, parent C2 + └── D1 D2 D3 <- three takes, parent C1 +``` + +Standing on the C2 path at that depth must read `2/2`, not `5`. Coordinate grouping gets +that one right by accident, because writing under a non-live take forks and the two sets +land on different branches. It gets `C` wrong: once C has been forked onto a branch of +its own it is alone at its coordinate and reads `1/1`, having lost C1 and C2 from a pager +that must still say `1/3`. + +So a node carries `parent_id`, read for nothing but this. The alternative, making a +branch's fork point a *node* rather than a depth so a promoted take never moves, was +rejected: the whole point of `lineage` is that a read is an OR-clause per branch instead +of a walk up parent pointers, and re-pointing the fork at a node changes path resolution +itself, dragging in the cursors, memory depths and both bundle formats. `parent_id` is +one indexed lookup, never a walk, and nothing about how a path resolves changes. + +### Cursors become anchors + +The two marks, how far the memory bank has got and how far the summary has got, used to +be counts. A count is a position in a list, and every rule about sliding them, rewinding +them and translating between positions and `Action.index` existed to patch up the fact +that the list moves. + +A cursor is now an **anchor**: `(branch_id, depth)`, the node up to and including which +the work is done. Deleting an action does not move it, because a depth is a coordinate +along a path rather than a slot in a list. "What is not covered yet" becomes a question +about the story instead of about a list index, and it answers correctly whatever has been +deleted in front of it. The branch half is what makes it survive forking: a depth alone +is ambiguous once two branches both have a node 41. + +`position_of_index`, `note_action_removed`, `settled_story_actions` and the cursor-rewind +machinery were **deleted**, not left unused. So was the one-turn memory holdback that +existed because a retry could rewrite an action the mark had already passed. (`SETTLE_SLACK` +later put one action of slack back, for what redoing a block costs rather than for what it +could get wrong. See "The memory system".) + +### Derived work attaches to the node that produced it + +Generalize the rule and a lot falls out: *anything derived attaches to the node that +produced it*. A memory covering depths 37–42 attaches to that branch's node 42 and is +invisible to any path that does not run through it. Shared ancestors are therefore shared +automatically, so **a fork needs nothing recreated**: the memories above the fork point +are already on the ancestors both lines read. + +The subtle case is the memory sitting *at* the forked coordinate. The first cut moved it +onto the new branch and re-anchored the marks naming it. Both are wrong for the same +reason: that memory describes whichever attempt was live at that coordinate, which is the +one staying on the parent. The right answer needs no code: the lineage caps the parent one +depth short of the fork, so the memory is simply out of range from the new branch, +invisible to both the retrieval clause and the anchor read. The new line summarizes that +ground again, from the text it actually tells. + +Hand-written memories obey the same rule. One used to carry a NULL depth, described as +"belongs to the adventure rather than to a path". That sounds harmless and is not: a NULL +is a coordinate no fork can cap, so a note typed on one line followed the reader onto +branches whose events it never described. They are anchored at the head instead: *the +story you were reading when you wrote it*. + +### Deleting, and why the branch UI was a hard dependency + +Nothing is ever auto-pruned. That is the guarantee the whole design rests on, and it is +also why branch management could not be a nice-to-have: without a way to delete a line, +storage grows without limit. + +The delete rule has two halves and the second is easy to miss. Refusing to delete the +line being read is obvious. The other half is refusing any line it was **forked from**: +`parent_branch_id` cascades, so deleting an ancestor takes the head with it and leaves +`head_branch_id` pointing at a row that is gone. One membership test against the head's +own lineage covers both, because a lineage already names itself and every branch it +borrows from. The server is the authority; the client computes the same set only so a +button can say so before it is pressed. + +### The migration, and what it deliberately did not do + +There is no feature flag. **A linear story is a tree with one branch**, so the +intermediate states were not half-migrated: they were the same product with a superset +schema underneath, which made "existing adventures are unaffected" a literal, testable +pass condition at every step. A flag would have bought two live code paths through the +context builder, the memory bank, undo and retry at once. + +The legacy columns (`index`, `variants`, `variant_index`, the two `*_before` snapshots) +were kept unread for a release rather than dropped with the migration that stopped using +them, so that a redeploy of the previous build is still a way out. Dropping columns is the +one step that isn't. + +One operational note that generalizes: on Postgres, a migration that rewrites every row of +`actions` roughly doubles the table, and only `VACUUM FULL` gives it back: 79 MB reclaimed +in 5.5 s on one occasion. But bloat scales with the **heap**, and `context_snapshot` is 94% +of this table and lives out of line, so a migration touching only small columns reuses the +existing TOAST pointer and costs a tenth of that. Read the sizes from `sum(octet_length())` +per column, not from `n_live_tup`, which is a stale estimate in exactly the direction that +makes bloat look smaller. + +### What this is honest about + +- **The two marks are one pair on the adventure**, not one per branch. Switching branches + makes the mark on the line being left unreadable from the new one, and that ground is + summarized again. It answers "nothing covered", which is the safe direction: redo the + work, never skip it. But switching back and forth costs AI calls. Per-branch cursors + are the fix if it ever matters. +- **Story cards stay adventure-wide.** A card invented on branch B shows on branch A. + Event-sourcing card changes onto nodes was considered and rejected. +- **Editing an already-summarized action still leaves its memory stale.** The machinery to + fix it now exists: an edit could write a sibling take and switch to it, which is a retry + the player typed. It does not do that yet. + +## 2.3 Undo and retry that actually rewind + +Most implementations of undo delete the last message. That's wrong here, because a turn +mutates three things: the text, the scripting scoreboard (`script_state`), and the RPG +stats (`world_state`). + +**The mechanism:** every node carries `state_after` and `world_state_after`, deep copies +of what the adventure looked like once that turn had played. Rewinding to before a turn is +a read of the node in front of it, so undo, retry and a branch switch are the same +restore. The cooldown clock comes along for free: it lives inside the world state, in +`_meta.last_changed`, so each line of the story carries its own without anything having to +know there is one. + +**Nothing a retry replaces is discarded.** The old attempt stays as another **take** of +that turn, a sibling node at the same coordinate with `live` set to false, and the pager +steps between them. Retry is not a special case: it is the tree, with the branch not yet +created. See [2.2](#22-the-story-is-a-tree). + +Three details that are easy to get wrong: + +**The turn being retried is excluded from its own context.** Its takes are still attached +to the adventure, so without `exclude_action_id` the model would be shown the attempt it is +replacing as established story and would write a continuation of it. The exclusion had +leaked into four readers, not one: history replay, story-card trigger matching, in-scene +NPC detection, and the memory-bank similarity query. The invariant is worth stating flatly: +*anything reading the story during generation takes the exclusion.* + +**A retry reuses the turn's depth**, not the next one. Cooldowns are measured along the +path, so allocating a new depth would advance the clock the cooldown rules run on and a +retry would quietly unlock stats that should still be waiting. + +**`delete_turn` used to mean "every take at this coordinate".** Once a take can be forked +onto a branch of its own, the group spans branches, and undo reached across and deleted a +take belonging to a line nobody asked about. Anything that reads a take group and then +*writes* has to say whether it means the turn or the coordinate. + +**If the regeneration fails, the rollback is reversed.** `generate_turn` wraps the +generator in a `try/finally`: if it ends without saving, whether from a provider error, an +empty reply, a script `stop`, or the browser hanging up, the previous take is put back in +charge. Otherwise the state on the server would drift from the text still on the user's +screen. + +## 2.4 The turn lock + +One turn at a time per adventure. Double-clicking "Continue" must not run two generations. + +The subtlety: the check has to happen in the **request phase**, not when the SSE generator +first runs. A `StreamingResponse` doesn't start iterating its generator until the response +begins, so a check-inside-the-generator lets two rapid requests both pass before either one +claims the slot. And because sync FastAPI endpoints run in a threadpool, the test-and-set +needs a real `threading.Lock`. + +```python +def acquire_turn_lock(adventure_id): # in the request handler + with _active_turns_guard: + if adventure_id in _active_turns: + raise HTTPException(409, "A turn is already generating…") + _active_turns.add(adventure_id) + +async def with_turn_lock(adventure_id, gen): # wraps the SSE generator + try: + async for event in gen: yield event + finally: + _active_turns.discard(adventure_id) +``` + +In-memory, so it's a single-process guarantee. That's honest for the deployment this +targets: one Render web service. Two processes would need the lock in the database. + +## 2.5 The 189x egress fix + +**The setup:** `Action.context_snapshot` holds the entire assembled prompt for a turn, +about 74 KB per row, 94% of the database. + +**The bug:** every adventure load pulled that column for every action, to read two small +fields out of it (the world-state delta, for the "what changed" chip, and the applied +report). SQLAlchemy loads all columns by default. + +**The fix, in three parts:** + +1. Move the two small things that *are* needed for every action into their own column + (`Action.world_delta`). +2. Mark the heavy columns `deferred` (`context_snapshot`, `variants`, `reasoning`), so + they're only fetched when explicitly asked for. +3. Backfill the new column with dialect-specific server-side SQL, so the old data is + extracted inside the database and never crosses the wire. + +**The result:** one adventure load went from **38.5 MB to 0.20 MB**. + +**The part that makes it stick:** `tests/test_egress.py` hooks into SQLAlchemy's +`before_cursor_execute` event, captures every statement the ORM sends, and fails if a bulk +load ever names those columns again. The regression is caught by asserting on the *SQL*, +not on a timing. + +One more detail from that test's design: the count query is written as a real +`SELECT count(...)` rather than `query.count()`, because SQLAlchemy's `.count()` wraps the +entity select in a subquery, so the emitted SQL names every column, including the deferred +ones. No bytes come back either way, but the database still has to read them, and a guard +that greps SQL cannot tell the two apart. + +There's a companion denormalization for the same reason: the pager has to know how many +takes a turn has without fetching any of them, so `variant_index` and `variant_count` are +cached on the row and refreshed by one function (`attempts.renumber`), precisely so they +can't drift and the pager can't lie. `variant_count` is 0 rather than 1 for a turn nobody +retried, because the question it answers is "is there anything to page through?" + +## 2.6 Migrations, hand-rolled + +No Alembic. An append-only list of `(version, SQL)` pairs, with the current version stored +in SQLite's `PRAGMA user_version` or a one-row table on Postgres. 64 versions so far. + +- A **fresh** database is created by `Base.metadata.create_all()` (always current) and + stamped at the latest version; it never replays history. +- An **existing** database runs every migration above its stored version, in order. + +Why this and not Alembic: for a single-file SQLite app that a user might have been running +for months, the entire requirement is "add a column, don't lose their data". Alembic's +autogenerate, branching, and down-migrations are machinery for a team with a staging +environment. This is 250 lines and you can read all of it. + +The constraint it creates is written at the top of the file: change `models.py` (so fresh +databases are current) *and* append a pair here (so existing ones upgrade). Migrations 2–23 +predate Postgres support and use SQLite-only syntax. This is harmless, because every +Postgres database starts fresh and never replays them, but anything added since must run +on both dialects. + +One migration worth reading (#10, repairing duplicate action indexes) uses `UPDATE … FROM` +with a window function rather than a correlated subquery, because SQLite may evaluate a +correlated subquery against partially-updated rows and produce duplicates again while +"repairing" them. + +--- + +# Part 3 — Production concerns + +## 3.1 Two modes, one codebase + +`AIDND_MULTI_USER` switches the whole app between two personalities: + +| | Local (default) | Hosted | +|---|---|---| +| Users | One auto-created "local user" | Guest on first visit, optional account | +| Auth | None: no cookies, no login UI | Signed session cookie | +| Rate limits | Off | On | +| Row caps | Off | On | +| API docs (`/docs`) | On | Off | +| Provider | Whatever Settings points at | User's key, or the shared demo key | + +The reasoning: a person running this on their own laptop should never be throttled by their +own app, never see a login screen, and should get the interactive API docs. A hosted +deployment needs all four of those to be the opposite. Rather than two builds, the +differences are gated at each site. + +**Guests upgrade in place.** A visitor gets a guest `User` row on first load. Registering +sets `email` and `password_hash` on that *same row*, so every adventure they played as a +guest survives with no re-parenting and no migration step. Three kinds of row share the +users table: local (email NULL, not guest), guest (email NULL, guest), registered (email +set). + +**A seed file owns its scenario's whole life.** `seed.py` inserts a demo that is +missing, reconciles one whose file changed, and deletes a seeded row no file claims any +more. Matching is by title, so a rename needs the old name under `previous_titles` or it +inserts a second scenario and strands the first. Only rows with a NULL owner and +`is_public` are seeded rows, which is what keeps the sweep away from anything a player +made. An adventure started from a demo that is later removed survives: +`adventures.scenario_id` is `ON DELETE SET NULL`, and the adventure holds its own copies +of the cards and scripts, so it loses only the inherited cover art. + +**Guests start with a story already in progress.** `starter.py` copies a shipped export +bundle into each new guest account at the same point the row is created. An empty account +gives a visitor nothing to read, and the daily demo turns are limited, so learning what +the app does used to cost one of them. The copy is the guest's own from the first moment: +they can edit, branch, delete, or export it, and nothing links it back to the file. The +guest row is committed before the copy is attempted, so a failure there still leaves them +with an account, and the copy itself runs inside a savepoint. + +**Guests expire; accounts don't.** One row per curious visitor adds up, so `cleanup.py` +deletes guests idle for `AIDND_GUEST_RETENTION_DAYS` (default 5), measured as +`COALESCE(last_seen_at, created_at)`, because `_touch` only writes `last_seen_at` hourly +and a guest minted by `/auth/me` has NULL until its second request. The filter requires +both `is_guest` *and* `email IS NULL`, so upgrading in place is also how you opt out of +expiry. It runs once at startup (the reliable trigger on a host that sleeps) and then +every few hours. + +It's a single Core `DELETE`, not `db.delete(user)`: the ORM path would SELECT every +adventure, action and memory into Python purely to delete them, and the FK graph is +`ON DELETE CASCADE` from `users` all the way down, so the database can do the whole graph +in one statement. Nothing a guest owns is visible to anyone else either: `is_public` is +output-only, so shared content is exactly the seeded scenarios, which have `user_id NULL` +and never match the filter. + +## 3.2 The shared demo key + +The demo lets people play with no signup and no API key, on a key the server pays for. That +is a spending surface, so it's the most defended code in the project. + +`resolve_provider_config()` is the single place the BYOK-vs-demo decision is made, and on +the demo branch it pins **two** things: + +- **The model**, pinned to a whitelist. A caller-supplied override or a hand-edited + settings row can't aim a server-funded key at an expensive model. Anything unrecognized + falls back to the first whitelisted model. +- **The endpoint**, pinned to the configured demo URL. Otherwise the key could be + redirected to a URL the user controls and harvested. + +Plus a daily per-user turn cap (default 20), checked *before* the player's input is stored +so a capped player doesn't get their message saved with no reply, and counted only after a +successful turn. + +There's a defensive `__post_init__` on the config object that raises if a demo config +somehow carries a non-whitelisted model. The comment on it records a real bug: the check +tests `using_demo`, **not** `api_key == DEMO_API_KEY`. Keying on the key value looks +stricter but is wrong: the demo key is an ordinary OpenRouter key, so a user can +legitimately paste that same key into their own settings as BYOK, and then every resolution +raised, 500ing even `GET /auth/me` and taking the whole SPA down. `using_demo` is what +actually means "the server is paying". + +Background work (summarization, embeddings) is excluded from the demo key entirely: those +are unmetered calls, and unmetered calls on a server-funded key is a bill. + +## 3.3 Secrets + +Everything derives from one server-side secret (`AIDND_SECRET_KEY`). + +| Thing | Mechanism | +|---|---| +| Passwords | `hashlib.scrypt`, N=2^14, r=8, p=1, per-password salt, constant-time compare. Stdlib, so no extra dependency. | +| Sessions | `v1..`, no expiry: long-lived guest sessions are the point. A cookie can outlive a swept guest row; that resolves to a 401, which the frontend already turns into a fresh session. | +| Stored LLM API keys | Fernet (AES) encryption at rest, key derived from the secret, `enc:` prefix so legacy plaintext rows are recognizable and migratable. | + +The secret auto-generates into a file next to the database for local installs (zero config), +but **multi-user mode refuses to start without the env var**, with an error message that +explains why and gives you the command to generate one. Hosted filesystems are ephemeral; a +regenerated secret on every deploy would silently log out every user and orphan their stored +API keys. + +A rotated secret makes stored keys undecryptable. `decrypt_secret` treats that as "unset" +rather than raising, so the user just re-enters their key instead of hitting a 500. + +## 3.4 Abuse guards + +| Guard | Value | +|---|---| +| Turn generation | 10 / minute | +| Auth attempts | 10 / 5 min, per IP | +| Guest creation | 30 / 5 min, per IP (each guest is a DB row) | +| Script test runs | 30 / minute (each costs up to 2s CPU) | +| Connection test | 10 / minute (outbound HTTP to a user-supplied URL) | +| Adventures / scenarios / scripts per user | 100 / 200 / 200 | +| Actions per adventure | 5,000 | +| Request body | 2 MB, 20 MB on import endpoints | + +Rate limits are keyed per user when one is known (accounts survive IP changes) and per IP +otherwise, in fixed windows held in memory, with a pruning pass so the per-IP dict can't +grow without bound. Import endpoints check bundle list lengths against the same caps live +creation enforces, otherwise the cap is trivially bypassed by uploading a file. + +Security headers on every response: `nosniff`, `X-Frame-Options: DENY`, +`Referrer-Policy: same-origin`, and a CSP allowing exactly what the SPA uses: same-origin +everything, inline styles (React needs them), and Google Fonts. + +## 3.5 Deployment + +One Docker web service on Render, serving the SPA and the API same-origin, with Postgres on +Neon. + +The Postgres decision was forced: Render's free tier has no persistent disk, so a SQLite +file wouldn't survive a deploy. The database lives off-box on Neon's free tier. + +Two things worth knowing about the free tier: + +- The service **sleeps after ~15 minutes idle**, and the first request then takes 30–60s. +- `/api/health` deliberately **doesn't touch the database**, so a keep-warm pinger wakes the + web service without waking the database. Waking a database around the clock costs far more + than the cold start is worth. + +CI runs the backend tests, the frontend lint and build, and a Docker image build on every +push. + +## 3.6 Counting visits + +A hosted demo raises a question a local app never does: is anyone using it, and do they get +anywhere? The answer is an owner-only dashboard at `/analytics`, gated on +`AIDND_ANALYTICS_EMAILS`, a list kept separate from `AIDND_POWER_USERS`, since an unmetered +tester is not automatically someone who should see the traffic. + +**Why it isn't a third-party script.** The CSP allows `script-src 'self'`, so a tracker would +mean loosening it. Ad blockers block the popular ones, which silently biases exactly the +technical audience this project is shown to. And none of them can see the measurement that +matters here: a *turn*. The interesting funnel step is not a pageview. + +**Egress is the budget.** After the 189x fix (§2.5) it would be perverse to add a feature +that reads rows per request. So counts accumulate in a process-local dict and flush every 60 +seconds as UPSERTs: **a visit is a write and never a read**. Storage is a generic +`(day, metric, label) -> hits` counter plus one row per visitor per day for the funnel flags. +Every dashboard query is a `GROUP BY` that returns tens of rows regardless of the traffic +behind it, so a month costs a few kilobytes to read back. The cost of the buffer is that a +hard restart can lose up to a minute; the flusher also runs on shutdown, and on a tier that +sleeps when idle, the buffer it sleeps on is empty anyway. + +**The numbers are the server's, not the browser's.** The client reports one fact, which page +was viewed, and even that is normalized to a route (`/play/12` → `/play/:id`) against a +whitelist, so the page list cannot be polluted by anything a stranger posts. Everything that +means something, such as a turn, an adventure, or a sign-up, is recorded by the code that +performs it. That also fixes a blind spot: a failed turn is an HTTP 200 with a bad ending, so +a status-code tally cannot see it, and a demo whose model has started refusing looks +perfectly healthy from outside. `turn_error` is counted where the SSE error is written. + +The funnel counts **people, not clicks**: a player who starts six adventures is one person +who started an adventure, which is the entire reason the per-visitor-day table exists. + +One smaller decision worth naming: error buckets are labeled by the matched *route +template*, never the requested path. That gives one bucket per endpoint instead of one per +adventure id. The reason it isn't merely tidier is that an unmatched path is entirely +attacker-chosen, so labeling by it would let anyone mint rows. + +--- + +# Part 4 — The web plumbing, briefly + +For the parts that are just how the web works, not decisions. + +**Frontend and backend are two programs.** In development they're two servers: Vite on 5173 +serving React, and FastAPI on 8000 serving the API. Vite proxies `/api` to FastAPI so +the browser thinks it's all one origin, which avoids CORS entirely. In production there's +one server: FastAPI serves the built React files as static assets from the same port. + +**SPA routing.** React Router handles URLs like `/play/3` in the browser without a round +trip. But if you *reload* that URL, the browser asks the server for `/play/3`, which isn't a +file. So `SPAStaticFiles` catches the 404 and returns `index.html`, letting React take over +and read the URL itself. API routes are matched before the static mount, so they're +unaffected. + +**Sessions.** A cookie is a small value the browser stores and automatically attaches to +every request to that site. Here it holds `v1..`. The server doesn't +store sessions anywhere; it re-verifies the signature on each request, which is why there's +no session table. + +**The 401 retry.** If the cookie is missing or stale, any API call returns 401. The frontend +catches that once, calls `/api/auth/me` (which mints a fresh guest session), and retries the +original request. So a returning visitor with an expired cookie never sees an error. + +**React, in one paragraph.** A component is a function that returns a description of some +UI. `useState` holds a value; changing it re-renders the component. The streaming turn is +the clearest example: each SSE chunk appends to a state string, React re-renders, and the +text appears to type itself. + +--- + +# Part 5 — Measured results and known limitations + +## Measured results + +| | | +|---|---| +| Database egress per adventure load | 38.5 MB → **0.20 MB** (~189x) | +| Prompt snapshot size | ~74 KB/turn, 94% of the database | +| Turn read cost at turn 200 | 839 KB → **129 KB**, flat after ~turn 50 | +| Cost of a branch | ~**103 B**; 20 forks load at **1.007×** the same story flat | +| Length-hint phrasing | 174 → 246 words phrased as a budget; **170** phrased as a ceiling (n=5) | +| Backend tests | 440, LLM mocked, real QuickJS engine | +| Schema versions | 64 | +| Sandbox limits | 16 MB, 2 s CPU, fresh context per run | +| Context defaults | author's note at depth 3, cards capped at 40% of elastic budget | +| Memory cadence | memory / 6 turns, summary / 15 turns, top-5 retrieval | + +Two of the tests encode a performance property rather than a behavior: +`test_egress.py` asserts on the SQL the ORM emits, and `test_history_window.py` asserts +that the read cost stops growing with story length. + +## Known limitations + +Deliberate trades for a single-user-first app that also happens to be hosted, listed so +nobody has to discover them the hard way. + +- **Single process.** The turn lock, the rate limiter and the summarization task all assume + one worker. A second worker would need the lock in the database (a row-level advisory + lock) and the rate limiter in Redis. +- **No vector index.** Retrieval does cosine similarity in Python over the whole bank. Fine + at the 200-memory cap; at 10,000 it would want pgvector. +- **Prompt snapshots are heavy** even after the egress fix; they're deferred, not smaller. + Compressing them or expiring old ones is the real fix. +- **In-memory rate-limit windows reset on restart**, so a restart grants a brief extra + allowance. +- **Background summarization is a fire-and-forget asyncio task**, so it does not survive a + restart. At real load it belongs in a queue. +- **The demo key depends on a free-tier provider's daily cap**, which the app can only + detect after the fact by string-matching the 429 body. +- **The two memory marks are one pair on the adventure, not one per branch.** Switching + lines makes the mark on the line being left unreadable from the new one, so that ground is + summarized again. It fails in the safe direction: redo, never skip. But switching back + and forth costs AI calls. Per-branch cursors are the fix if it matters. +- **Story cards are adventure-wide**, so a card invented on one branch shows on all of them. +- **Editing an already-summarized turn leaves its memory stale.** Replacing a turn withdraws + what was derived from it; editing one in place does not. + +## Cleanup backlog + +`docs/self-review.md` carries an open list of non-bugs (reuse, simplification, and +efficiency items) kept deliberately separate from the correctness list, which is empty. +The largest ones: + +- `Section.tokens` is uncached, so the context gets tokenized two or three times a turn. +- `onModelContext` flattens system and story into one string before handing it to user + scripts; if a script modifies it, the structure is gone and everything ships as user + content. Passing structure through the hook would be better but would break AI Dungeon + compatibility, which is the point of the feature. +- The import endpoints hand-coerce raw dicts instead of using Pydantic bundle schemas. +- The legacy pre-tree columns (`index`, `variants`, `variant_index`, and the two `*_before` + snapshots) are still on `actions`, unread, kept for one release so redeploying the previous + build remains a way out. Dropping them is a migration that rewrites every row, so it owes a + `VACUUM FULL actions;` after it. + +--- + +*Source: [github.com/parththakkar106/AI-DnD](https://github.com/parththakkar106/AI-DnD) · +[Project page](https://parththakkar106.github.io/AI-DnD/)* diff --git a/docs/architecture.html b/docs/architecture.html new file mode 100644 index 0000000..49b14f2 --- /dev/null +++ b/docs/architecture.html @@ -0,0 +1,1233 @@ + + + + + +AI D&D Engine Room + + + + + + + + + + + + + +
+ + + +
+ +
+

AI-DnD · architecture notes

+

The Engine Room

+

An AI Dungeon clone is a chat wrapper until four things are true. The prompt is budgeted. The numbers are refereed. The story is a tree. Someone else’s JavaScript runs in a sandbox. This page shows how each one is built, and what it cost to learn.

+
+
440backend tests
+
64schema versions
+
189×egress cut
+
103 Bcost of a branch
+
1model call per turn
+
+
+ +
+ +
+
HLD
+

0.1  What the thing is

+

You write a scenario, then play an open-ended text adventure where a language model narrates the world. You type “I open the door”, the model writes what happens next, and it remembers what came before.

+

Four subsystems carry the weight, and every hard problem in the codebase belongs to one of them:

+
    +
  1. A context engine. The model has a finite input window. The app decides, every single turn, which pieces of the story get into the prompt and which get dropped.
    2. +
  2. A world-state engine. The scenario declares stats (hp, trust, day). The model proposes changes each turn; Python decides what actually sticks.
    3. +
  3. A story tree. The story is not a list. Any turn can hold more than one take, and writing below a take that isn’t live starts a branch that borrows every turn above the fork rather than copying it.
    4. +
  4. A scripting sandbox. Real AI Dungeon JavaScript imports and runs, inside an embedded QuickJS interpreter.
    5. +
+

It runs locally against Ollama for free, or hosted against any OpenAI-compatible endpoint.

+
+ +
+
HLD
+

0.2  The stack, and what each part is doing

+
+ + + + + + + + + + + +
PieceJob in this system
FastAPIHTTP server. Routes, plus the SSE streaming.
SQLAlchemyORM. Adventure, Action, Memory are Python classes; attribute access becomes SELECTs — which is exactly how the egress bug happened.
SQLite / PostgresOne file on disk locally; Neon Postgres when hosted. Same code, two dialects.
React + ViteThe SPA. Built files are served by FastAPI, same origin, same port.
httpxCalls the model endpoint, streaming.
tiktokenCounts tokens, so budgeting is arithmetic rather than a guess.
QuickJSEmbeddable JS engine used as the user-script sandbox.
+
+
+ +
+ +
+

Part 1 — The high-level design

+

What the boxes are, who talks to whom, and the two or three decisions that every later decision inherits.

+
+ +
+ +
+
HLD
+

1.1  One process, two things outside it

+

Production is a single Docker web service on Render. It serves both the API and the built SPA. Only two things live off-box: the database and the model endpoint.

+

This shape is not an accident. It is what makes the in-process turn lock and the in-memory rate limiter honest. It is also what the Known limitations list is measured against.

+
+ +
+ +
+
+ + + + + + + + + + + + + Browser + React SPA + ReadableStream + reader + localStorage + + + + POST + + SSE tokens + + + + One Render web service + uvicorn · single worker + + + FastAPI routes + + static SPA + (same origin) + + + context builder + + referee + + QuickJS + 16 MB / 2 s + + + in-memory: turn lock · rate windows + visit counters · summarize tasks + all single-process by design + + + + SQL + + Neon Postgres + scale-to-zero, 5 min + egress is the bill + + + + https + + Model endpoint + OpenAI-compatible + Ollama / OpenRouter + + + + GET /api/health never touches the DB — a keep-warm + pinger wakes the web service without waking Postgres. + +
+
The whole system. The brass box is one process: everything inside it shares memory, which is why the turn lock and rate limiter work and also why a second worker would break both.
+
+ +
+ +
+
HLDbackend/app/models.py
+

1.2  The domain model: template versus instance

+ 
User
+ ├─ Scenario   (the template)     ── stat_schema, prompt, memory, author's note
+ │    └─ StoryCard, Script
+ └─ Adventure  (the playthrough)  ── world_state, script_state,
+      │                              head_branch_id, head_depth
+      ├─ Branch  (one line of it) ── parent_branch_id, fork_depth, lineage, name
+      ├─ Action  (one node)       ── branch_id, depth, parent_id, live,
+      │                              text, context_snapshot, state_after
+      ├─ StoryCard  (its own copy)
+      ├─ Memory     (text, embedding, branch_id, depth, use_count)
+      └─ AdventureScript
+

The decision that shapes everything else: a scenario declares what stats exist. An adventure holds what they are right now. Creating an adventure copies the scenario’s story cards, scripts, and plot fields into it. Editing a scenario later never mutates a game in progress. This is the same reasoning as instantiating a class: shared definition, independent state.

+

The opt-in escape hatch is Update from scenario. GET /adventures/{id}/refresh returns a diff: per-field old and new values, card additions, updates and removals, and world-state paths added or removed. POST to the same path applies it under the turn lock. It never touches the opening action, the adventure’s title, its summary, or player-authored cards.

+
+ The trap that made it possible +

Re-copying scenario text would have re-injected literal ${Hero} placeholders. The answers were consumed once at creation and thrown away. And without story_cards.source_ref (card:<id> / npc:<key>, NULL for player-authored), there was no link back. A rename read as delete-plus-add, and player cards would have been clobbered. Two migrations were the price of one feature.

+
+
+ +
+
HLDrouters/adventures.py · _generate_turn
+

1.3  The turn pipeline

+

Everything between “player pressed a button” and “text is on screen”. Teal steps are user-script hooks — the points where someone else’s JavaScript gets to rewrite the turn.

+
    +
  1. onInput hookUser JS may rewrite or block the input outright.
    2. +
  2. store the player actionWritten before generation, so a failed turn still shows what you typed.
    3. +
  3. retrieve memoriesEmbed the last 4 actions, cosine-rank the bank, take the top K.
    4. +
  4. build_context()The budget allocator. Fixed sections first, elastic ones into what is left.
    5. +
  5. onModelContext hookUser JS may rewrite the entire assembled prompt.
    6. +
  6. snapshot the exact promptStored on the action. Powers Insights; costs ~74 KB a turn, which becomes §3.1.
    7. +
  7. provider.generate()One streamed call. Tokens forwarded to the browser as they arrive.
    8. +
  8. onOutput hookLast chance for user JS to touch the text.
    9. +
  9. extract + referee the state blockParse the fenced JSON delta, clamp it, strip it from the prose.
    10. +
  10. save the nodeStamped with state_after and world_state_after — the scoreboard as this turn leaves it.
    11. +
  11. fire-and-forget: summarize + embedBackground task, own DB session, never on the shared demo key.
    12. +
+

Two choices are visible in that list before any detail. The prompt is snapshotted, not reconstructed. That makes prompt bugs findable, and it makes the database expensive. Every node also records the state it leaves behind. Rewinding to before a turn is a read of the node in front of it. That single move is why undo, retry, and a branch switch are the same restore.

+
+ +
+
HLDAIDND_MULTI_USER
+

1.4  Two personalities, one codebase

+

One env var switches the whole app. Not two builds — the differences are gated at each site.

+
+ + + + + + + + + + +
Local (default)Hosted
UsersOne auto-created local userGuest on first visit, optional account
AuthNone — no cookies, no loginSigned session cookie
Rate limitsOffOn
Row capsOffOn
/docsOnOff
ProviderWhatever Settings points atUser’s key, or the shared demo key
+
+

Someone running this on their own laptop should never be throttled by their own app. They should never see a login screen, and they should get the interactive API docs. A hosted deployment needs all four of those to be the opposite.

+

Guests upgrade in place. A visitor gets a guest User row on first load. Registering sets email and password_hash on that same row, so every adventure played as a guest survives with no re-parenting.

+

Guests idle past AIDND_GUEST_RETENTION_DAYS are swept by a single Core DELETE. The FK graph is ON DELETE CASCADE the whole way down. Without it, the ORM path would have pulled every action and memory into Python just to delete them.

+
+ +
+
HLD
+

1.5  Why there is no agent framework

+

Graph-based agent frameworks earn their complexity with branching, cyclic, multi-step control flow. Their path depends on what the model decides, with loops, tool calls, and persisted state between steps. This pipeline is a fixed linear sequence with exactly one model call. There is no routing decision, no tool selection, no loop.

+

There is a second, more specific reason. The budgeting logic is the product. Buffer-window and summary-memory abstractions are opinionated about how to fit history into a window. Here, the Insights panel exposes each context component, its token cost, and the trigger word that pulled it in. Assembly has to be explicit and inspectable.

+
When it would flip: imagine the design used two calls instead. First narrate, then run a separate structured-extraction step, with a retry branch when extraction fails and a tool path for dice. That is a graph, and hand-rolling it would get ugly fast.
+
This is not the same branching as the story tree. “No branching” here describes control flow: the code path a single turn takes through the backend. That path never forks. “No branching” is not true of the data the app stores. See §2.6: when a player rewinds and plays a turn differently, that action does create a new branch in the saved history. One turn's execution is a straight line. The sequence of turns across a playthrough is a tree. The two claims are about different things and do not conflict.
+
+ +
+ +
+

Part 2 — The low-level design

+

The arithmetic, the schemas and the invariants. Each of these is a place where the obvious implementation is wrong, and the reason it is wrong was found by measurement or by a bug.

+
+ +
+ +
+
LLDbackend/app/context/builder.py
+

2.1  Context assembly is a budget problem

+

Say the budget is 8,000 tokens. A 200-turn adventure has far more story than that. The naive fix is to send the last N turns, but that breaks in both directions. N short exchanges waste the window. N long ones overflow it. Either way, it throws away the premise and the promise you made to the innkeeper.

+

So the prompt is split into two kinds of sections. Fixed sections are included whatever they cost. Elastic ones fit into what is left.

+ +
+
+ fixed sections + cards ≤ 40% + history, newest first +
+
+ fixed → narrator · stat guide · live state · emit rule · ai_instructions · plot · summary · memories + elastic → story cards, then story turns +
+
+ + 
reserved  = every fixed section + author's note + length hint + emit reminder
+available = max(256, context_token_budget - reserved)
+cards      spend up to available * 0.4
+history    spends available - cards_used, filling backwards from newest
+ +

The details that are decisions

+
    +
  • Cards are capped at 40% of the elastic budget. Story cards trigger on keyword match. A scene naming six things could pull six lore entries and leave no room for the story. The cap turns that failure into “some lore is missing” instead of “the model has no idea what just happened”. Cards that do not fit are still reported to Insights with included: false.
    • +
  • History fills newest-first and stops. Old material is not lost. It has already been summarized into memories and the running summary, both of which sit in the fixed block.
    • +
  • If even the newest turn is over budget, it is hard-truncated, not dropped. A prompt with no story produces nonsense. A prompt with the tail of the last turn produces something.
    • +
  • The author’s note is injected 3 actions from the end (AUTHORS_NOTE_DEPTH = 3), not at the top. It is a steering control, so it goes where steering works best: recency.
    • +
  • The world-state reminder takes the very last slot. The full emit rule lives hundreds of tokens up in the system block. A one-line reminder occupies the position closest to where the model starts writing.
    • +
+ +
+ Reliability by imitation +

The state block is stripped from text before storage. Replayed history then showed the model twenty of its own past turns with no state block, teaching it by example to stop emitting one. Once it missed a turn, it never recovered, and retry did not help.

+

The fix has two halves, both gated on the adventure actually having a schema. First, a one-line EMIT_REMINDER sits in the recency slot. Second, _history_text() reconstructs each past turn’s delta block from the stored delta and re-appends it. Action.text stays clean, so the UI, the embeddings, and card trigger-matching are unaffected. History carries only the per-turn delta, for format imitation. The full scoreboard is rendered once, up top.

+
+
+ +
+
LLDbackend/app/context/history.py
+

2.2  The performance trap hiding inside that

+

Building the context needs only the newest ~6,000 tokens of story. The obvious implementation reads adventure.actions, which loads every row of the adventure and then discards 90% of it. At turn 200 that was 839 KB read to use about 70 KB, and it grew every turn.

+

history.py serves three shapes straight from SQL: a tail, a slice, and a count. window_covering() fetches the newest 32 actions and measures their real token count. If that falls short, it projects how many more it needs from the average it just measured, instead of blindly doubling:

+ 
average   = tokens / len(actions)
+projected = int(budget / average * 1.15) + 8
+

Each round fetches only what it does not already hold, so no row is read twice. The same turn now costs 129 KB, flat from about turn 50. The cost is bounded by the context budget, not by the length of the story.

+
Second rule in that module: if the actions are already loaded, slice them instead of querying. The scripting pipeline hands the whole history to user scripts because AI Dungeon’s API requires it. On a scripted adventure the rows are already in memory, so a query beside them would mean paying twice.
+
+ +
+
LLDbackend/app/worldstate/engine.py
+

2.3  World state: the AI proposes, Python referees

+

Three options, and two of them lose.

+
    +
  • A deterministic dice engine. What a real RPG does. It loses here because the action space is unbounded. Mapping arbitrary natural language onto a fixed rules system is harder than the problem being solved.
    • +
  • Let the model own the numbers. Fails immediately. Models are bad at arithmetic, worse at holding a number across twenty turns, and unable to obey their own frequency rules. Tell one “change this at most every 5 turns” and it changes it every turn.
    • +
  • The model proposes, the engine disposes. Chosen. The model narrates and appends a JSON delta. Python validates and clamps it before anything is stored.
    • +
+ 
narration: "The blade catches your shoulder. Gwen shouts and drags you back."
+
+```state
+{"player.hp": -15, "npc.gwen.trust": 5, "milestones.escaped": true}
+```
+
+ + + + + + + + + + + + +
Rule applied, in orderWhat it stops
Path must exist in the schemaHallucinated stats
Value must be the right type"a lot" instead of -15
CooldownChanging a stat more often than the scenario allows
Counters can’t decreaseThe in-game day going backwards
max_delta_per_turnLosing 90 hp to a stubbed toe
Clamp to min/maxNegative hp, trust above 100
Milestones are sticky, true onlyUn-completing a quest
Flags are two-way booleansNothing — deliberately unrestricted
+
+

Everything rejected is reported, not silently swallowed: Insights shows applied, clamped and rejected paths per turn, and a chip under each narration shows what actually changed.

+ +

Word bands are the reliability mechanism

+ 
"hp": { "min": 0, "max": 100, "initial": 100,
+        "bands": [[0,20,"very weak"], [20,40,"hurt"], [40,60,"minor damage"],
+                  [60,90,"healthy"], [90,100,"full health"]] }
+

The live state line shows the current band label, like hp 55/100 (minor damage). The model reads a word, not just a number. The stat guide also prints the whole ladder once per turn. Models reason well over semantics and badly over arithmetic. “He’s badly hurt, so a solid hit takes him to very weak” is a judgment a model can make. “55 minus 22 is 33” is one it gets wrong often enough to matter.

+ +

Two philosophies underneath

+

Nothing in this engine raises. A malformed delta returns {} and the turn continues. The parser strips trailing commas and leading + signs. It accepts a fence labelled state, one labelled json, or an unlabelled one. It falls back to a bare object hugging the end of the text, but only if that object parses into something delta-shaped, so prose ending in } is never eaten. This tolerance exists because the hosted demo runs on free-tier models. A stricter parser would mean good models work and free ones don’t.

+

One call, not two. Narrate-then-extract is more reliable per call, but it costs twice the latency and twice the rate-limit budget. On a 20 req/min free tier, that halves the playable turn rate. The tolerant parser plus the terminal reminder buys the same reliability for less cost.

+
+ +
+
LLDbuilder.length_hint()
+

2.4  Output length, decided by measurement

+

The hint’s whole job is protecting the state block, which is emitted last and is therefore what truncation eats. So it must shorten output. The first version lengthened it.

+
+ + + + + + + +
Phrasing, cap 800, n=5Mean words
No hint174
“Keep this turn under about 506 words.”246
“Hard limit … must not exceed 506 words … a typical turn is much shorter.”170
+
+

A budget reads to the model as a target to fill. Every one of the five budget runs was longer than every unhinted run, 41% longer on average. That pushes output toward the wall the hint exists to avoid. Ceiling phrasing is statistically indistinguishable from no hint at loose caps, but it still works at tight ones. At cap 250, unhinted runs hit finish_reason: length 2 times in 6. Ceiling-hinted runs hit it 0 times in 6.

+

A one-sided ceiling turned out to be half a fix. Across other models, the same prompt gave wildly different lengths. A terse model has nothing to act on but “much shorter”, so it collapses to two paragraphs. The hint is now a band with deliberately asymmetric bounds, so neither side reads as a number to hit:

+ 
must not exceed 506 words, and it should not stop short of about 177.
+Prefer the lower end of that range unless the scene genuinely needs more.
+
+LENGTH_FLOOR_SHARE   = 0.35 of the ceiling
+MIN_LENGTH_FLOOR_WORDS = 60    # below this the floor is dropped and the
+MAX_LENGTH_FLOOR_WORDS = 300   # tight-cap string stays byte-identical
+
The transferable rule: phrase every number in a prompt as a bound, never a target. Give it two sides. A one-sided hint just moves each model further in whichever direction it already leaned. Test it against no-hint on at least two models with opposite verbosity biases before trusting it.
+
+ Shipped unmeasured, and known to be +

No test was run on the band wording. Two risks remain open. A stated range may invite landing mid-range on verbose models. Truncation is still silent, since nothing in the app reads finish_reason yet. The better design is a target_length preference separate from max_output_tokens, which is currently a safety wall doubling as the length dial. It was rejected as too big for the ask.

+
+
+ +
+
LLDbackend/app/memorybank.py
+

2.5  The memory bank

+

Turn 4 said you promised the innkeeper you’d return. At turn 90, that is long gone from the prompt. But if you walk back into the inn, it should come back. Three layers make that happen:

+
+ + + + + + + +
LayerCadenceWhat it is
Memoryevery 6 actions, from 12One or two past-tense sentences of concrete fact.
Story summaryevery 15 actionsA single ≤250-word overview, rewritten by folding in the new memories.
Retrievalevery turnEmbed the last 4 actions (≤600 tokens), cosine-rank the bank, inject the top 5.
+
+

Retrieval is what answers the innkeeper problem. The promise is a memory. The memory has a vector. Walking into the inn produces a query vector near it.

+
    +
  • A memory hangs off the node whose block it ends on. That is a (branch_id, depth) coordinate, not the adventure and not a position in a list. This makes “which memories described this turn?” an indexed lookup. It is also why memories inherit correctly across a fork: the ones above the fork point already sit on ancestors both lines read.
    • +
  • Cursors only advance on success. Every AI call here is best-effort. If summarization fails, the cursor is unchanged, and the same block is retried on a later turn. There is no retry loop, no backoff, no dead-letter queue. The cadence is the retry mechanism.
    • +
  • Pinned memories count toward top_k. Otherwise, 6 pinned plus top_k=5 injects 11 and blows the budget the whole context engine exists to respect.
    • +
  • A dimension mismatch scores 0.0, it does not crash. Change your embedding model, and old 768-dim vectors meet a 1536-dim query. zip() would happily truncate and score garbage silently.
    • +
  • Eviction is LRU-ish and non-destructive. Over capacity (default 200), the least-used unpinned memories are marked forgotten instead of deleted, so you can un-forget one.
    • +
  • Background calls never spend the shared demo key. Summarization and embedding providers are built from the user’s own settings, never from the demo config. Unmetered background calls on a server-funded key would be a bill.
    • +
+
+ The measurement that had been running with the feature off +

Two full rounds of egress work ran with the memory bank effectively disabled. Retrieval needs an embedding model, and embedding providers are BYOK-only by construction. So the demo never embeds, and the stress harness had none configured. Measured later in production: 134 memories, 1536 dims, ~31 KB each as JSON text. The ranking walked adventure.memories, so the whole bank crossed the wire every turn just to pick five. On a 100-memory adventure that was 3,024 KB per turn against 129 KB for everything else, about 96% of a turn. Break-even is 4.2 memories.

+

This is not a repeat of the earlier fix. There is no repeating group, no denormalization. It is a format problem (JSON floats at 20 bytes where a float is 4) plus a fetch-frequency problem. Rule: any egress measurement must run with an embedding model configured.

+
+
+ +
+ +
+
+
LLDHLDcontext/lineage.py · tree.py · attempts.py
+

2.6  The story is a tree

+

The largest structural change the project has had, and the one with the most reasoning behind it.

+

The story used to be a list, and a mutable one. Retry rewrote the last entry in place. Undo and delete removed entries from the middle. Everything derived from the story was indexed by position in that list: memories, the running summary, the two marks saying how far each had got. A position means something different once anything in front of it is deleted. That one fact produced a family of bugs that all looked different:

+
    +
  • Deleting a middle action slid a never-summarized action into the “already covered” range. A recent action silently never became a memory.
    • +
  • Discarding a memory left its actions behind the mark, describing nothing.
    • +
  • Retry rewrote text after the mark had passed it. The memory then described narration no longer in the story.
    • +
  • The retried row stayed attached while its replacement was written. The model was shown the attempt it was meant to replace and wrote a continuation of it. That exclusion had to be threaded through four separate readers.
    • +
  • Attempts lived in a JSON array on the row, with a mirrored copy of the live one in the ordinary columns. That is a repeating group and a denormalization in one.
    • +
+
Each was fixed where it was found. Lined up, the pattern becomes visible: they are all the same bug. The story is a list nobody may reorder.
+
+
+ +
+
+ + + + + + + + + depth + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + + + + A + + + + + + + not on C’s path + + + + + + B + + + + + not on C’s path + + + + + + C + + + head + + + + branches.lineage on C, computed once at fork, never walked: + [ (C, ∞), (B, 5), (A, 3) ] + → ORDER BY depth DESC reads entry 0, then 1, then 2 — and stops. + +
+
Reading branch C. Filled nodes are on the path; the ranges are disjoint and descending, so a tail read consumes the newest entries and never reaches the rest. Clause count is bounded by the context window, not by fork count.
+
+ +
+
+

The shape

+ 
branches(id, adventure_id, parent_branch_id, fork_depth, lineage, name)
+actions (id, adventure_id, branch_id, depth, parent_id, live, text, …, state_after)
+memories(…, branch_id, depth)
+adventures(…, head_branch_id, head_depth)
+

depth is a position along a path, not a global turn number: A4 and B4 are two alternatives, not two turns. Reading branch C is one query:

+ 
SELECT * FROM actions
+WHERE (branch_id = 'C')
+   OR (branch_id = 'B' AND depth <= 5)
+   OR (branch_id = 'A' AND depth <= 3)
+ORDER BY depth DESC LIMIT 32          -- → A0 A1 A2 A3 B4 B5 C6 C7
+

Why branch_id + depth, and not parent pointers alone. Parent pointers are the obvious way to store a tree, but they are the wrong way to read one. Reading a story would need N round trips up a chain, which throws away the windowing work of §2.2. Depth replaces the old index as the ordering key, so reads keep the shape they already had. ltree was rejected because it breaks SQLite dev parity and its path column grows per row. Closure tables were rejected for being O(n²).

+

A branch stores no story of its own. A fork costs only an id, a parent, a fork depth, and a cached ancestry. Measured on a 40-turn story forked twenty times, against the same story flat: a page load of 31,652 B against 31,433 B, or 1.007×, about 103 bytes per branch. No migration, no vacuum, no copy.

+ +

What a player actually does

+
Any turn can gain another take. On an AI turn that means regenerate. On your own message it means type something else. Stepping between takes with ‹ 2/4 › is free, since the story below simply empties: that take has no children yet. A branch is created when you write below a take that is not the live one, never before.
+

That rule collapses two operations into one and deletes a distinction from the UI. The first version of this screen had a chip that switched at the tip and only previewed above it, with a second button to take that line. One control's meaning depended on where the reader was standing. It shipped, was driven by hand, and was found unusable. The replacement is a pager that only ever steps, a fork button on every turn, and no tip-versus-past distinction at all.

+ +

Takes are grouped by parent, not by coordinate

+

This is the load-bearing detail, and it is not obvious. The natural way to find “the other takes of this turn” is by coordinate: same branch, same depth. That is wrong in both directions:

+ 
B ── C   C1   C2              <- three takes, one parent (B)
+          │    └── D1' D2'    <- two takes, parent C2
+          └── D1 D2 D3        <- three takes, parent C1
+

Standing on the C2 path at that depth must read 2/2, not 5. Coordinate grouping gets that right by accident: writing under a non-live take forks, and the two sets land on different branches. It gets C wrong. Once C is forked onto a branch of its own, it sits alone at its coordinate and reads 1/1, having lost C1 and C2 from a pager that must still say 1/3.

+

So a node carries parent_id, read for nothing else. The alternative was making a branch’s fork point a node rather than a depth, and it was rejected. The whole point of lineage is that a read is an OR-clause per branch instead of a walk. Re-pointing the fork at a node changes path resolution itself, dragging in the cursors, the memory depths, and both bundle formats. parent_id is one indexed lookup, never a walk.

+ +

Cursors become anchors

+

The two marks, how far the memory bank has got and how far the summary has got, used to be counts. A count is a position in a list. Every rule about sliding, rewinding, and translating between positions and Action.index existed to patch up the fact that the list moves.

+

A cursor is now an anchor: (branch_id, depth), the node up to and including which the work is done. Deleting an action does not move it. “What is not covered yet?” becomes a question about the story instead of a list index, and it answers correctly no matter what has been deleted in front of it. The branch half is what makes it survive forking. A depth alone is ambiguous once two branches both have a node 41.

+

position_of_index, note_action_removed, settled_story_actions, and the cursor-rewind machinery were deleted, not left unused. So was the one-turn memory holdback that existed because a retry could rewrite an action the mark had already passed. (SETTLE_SLACK later put one action of slack back, for what redoing a block costs rather than for what it could get wrong.)

+ +
+ Three findings only hand-driving produced +

A sibling group breaks every query that assumed one row per depth. _latest_narration ordered by (depth desc, id desc) and got the newest attempt rather than the live one. The index screen quoted a take the player had thrown away. Anything ranking actions by coordinate needs live in the filter.

+

delete_turn meant “every take at this coordinate”. Once the group spans branches, undo reached onto another line and deleted a take nobody asked about. Anything that reads a take group and then writes has to say whether it means the turn or the coordinate.

+

The adventure GET does not build ActionOut. It hands the window to the relationship with set_committed_value and lets Pydantic walk it. Patching every place that builds ActionOut still misses this one path, and every page load takes it.

+
+ +

One review finding was rejected, and that is the part worth keeping. A memory the player types lands on the head’s coordinate. A retry withdraws every memory at that coordinate, so the note disappears. That is reproduced and real, but it is the rule working: a memory anchored to a node describes that node and goes when the node goes. The root node is the one exception. The pre-tree bank was parked on depth 0, the one depth every branch can see, so withdrawing it would retire a whole bank in a click.

+
+ +
+
LLD
+

2.7  Undo and retry that actually rewind

+

Most implementations of undo delete the last message. That is wrong here, because a turn mutates three things: the text, the scripting scoreboard (script_state), and the RPG stats (world_state).

+

Every node carries state_after and world_state_after, deep copies of the adventure once that turn had played. Rewinding to before a turn is a read of the node in front of it, so undo, retry, and a branch switch are the same restore. The cooldown clock comes along free. It lives inside the world state at _meta.last_changed, so each line of the story carries its own without anything having to know there is one.

+

Nothing a retry replaces is thrown away. The old attempt stays as another take, a sibling node with live false, and the pager steps between them. Retry is not a special case. It is the tree with the branch not yet created.

+
    +
  • The turn being retried is excluded from its own context. Its takes are still attached to the adventure. Without exclude_action_id, the model is shown the attempt it is replacing as established story. The exclusion had leaked into four readers: history replay, story-card trigger matching, in-scene NPC detection, and the memory-bank similarity query. Anything reading the story during generation takes the exclusion.
    • +
  • A retry reuses the turn’s depth, not the next one. Cooldowns are measured along the path. A new depth would advance the clock the cooldown rules run on, and it would quietly unlock stats that should still be waiting.
    • +
  • If regeneration fails, the rollback is reversed. generate_turn wraps the generator in try/finally. A provider error, an empty reply, a script stop, or the browser hanging up puts the previous take back in charge. Otherwise server state drifts from the text still on the user’s screen.
    • +
+
+ +
+
LLD
+

2.8  The turn lock

+

One turn at a time per adventure. The subtlety is where the check goes. A StreamingResponse does not start iterating its generator until the response begins. A check inside the generator would let two rapid requests both pass before either claims the slot. And because sync FastAPI endpoints run in a threadpool, the test-and-set needs a real threading.Lock.

+ 
def acquire_turn_lock(adventure_id):          # in the REQUEST handler
+    with _active_turns_guard:
+        if adventure_id in _active_turns:
+            raise HTTPException(409, "A turn is already generating…")
+        _active_turns.add(adventure_id)
+
+async def with_turn_lock(adventure_id, gen):  # wraps the SSE generator
+    try:
+        async for event in gen: yield event
+    finally:
+        _active_turns.discard(adventure_id)
+

The lock is in-memory, so it is a single-process guarantee. That is honest for the deployment it targets. Two workers would need the lock in the database.

+
+ +
+
LLD
+

2.9  Streaming

+

The app uses Server-Sent Events, not WebSockets. Traffic is one-directional, and a bidirectional connection for a unidirectional problem is cost with no return. FastAPI reads the provider’s stream and yields data: {"type":"chunk","text":"…"}. The frontend reads the body with a ReadableStream reader, buffers on \n\n boundaries, and dispatches each event. Event types: player, reasoning (thinking traces, into their own collapsible panel with their own budget), chunk, stopped, error, done.

+
+ Two things that only appear when hosted +

X-Accel-Buffering: no. nginx-style reverse proxies buffer by default, turning a stream into one delivery at the end.

+

The security-headers and body-size middlewares are written as pure ASGI rather than Starlette’s BaseHTTPMiddleware. The latter buffers the response body, which would break streaming outright.

+
+

The empty-reply case is diagnosed, not just reported as “empty”. If a reasoning model streams thinking but no story text, it spent its whole budget thinking. The error says so and names the three settings to change.

+
+ +
+
LLDbackend/app/scripting/
+

2.10  The QuickJS sandbox

+

Real AI Dungeon scripts define modifier(text) and call it as the last line, with globals like state, history, and storyCards. The same contract runs here, inside an embedded QuickJS interpreter. The safety properties are mostly structural. Nothing was removed, because nothing was there to begin with.

+
+ + + + + + + + + +
PropertyHow
No filesystem, network or process accessQuickJS has none by default
Memory cap16 MB per run
CPU cap2 seconds per run
No shared state between runsA fresh Context per hook execution
A broken script can’t break a turnEvery failure returns .error with text, state and cards unchanged
+
+

Data crosses the boundary as JSON. Python serializes {state, text, history, storyCards, info} in, and the script’s results come back out the same way. There is no object bridge to exploit.

+

One bug-compatibility is deliberate: addStoryCard returns the new card’s index. The first card returns 0, which is falsy, so if (!addStoryCard(...)) misfires. That is upstream AI Dungeon’s behavior. Matching real scripts is the entire point of the feature.

+
+ +
+ +
+

Part 3 — Running it in public

+

A hosted demo on a free tier turns three things into engineering problems that a local app never has: bytes on the wire, a spending surface, and strangers.

+
+ +
+ +
+
LLDmigration 36 · tests/test_egress.py
+

3.1  The 189× egress fix

+

Neon’s free tier allows 5 GB of network transfer per project per billing period, and it hard-blocks at connection time. There is no read-only grace. You cannot even pg_dump your way out once it trips. This app tripped it.

+

The diagnosis came from the monitoring tab, not a guess. The database was ~55 MB, and had transferred 5 GB. Compute was idle most of the day, and the pooler never exceeded three connections. That is the whole database, ninety times over: a payload-per-request problem, not traffic volume and not a leak.

+

Action.context_snapshot holds the entire assembled prompt, about 74 KB per row and 94% of the database. Every adventure load pulled it for every action, just to read two small fields out of it. SQLAlchemy loads all columns by default.

+
+ + + + + + + +
StepWhat it does
1Move the two things actually needed per action into their own column, Action.world_delta.
2Mark the heavy columns deferred — context_snapshot, variants, reasoning — so they load only when asked for.
3Backfill with dialect-specific server-side SQL (json_extract on SQLite, #> on Postgres) so the 39 MB never crosses the wire.
+
+

One adventure load went from 38.5 MB to 0.20 MB. A second round found two more problems. Action.variants was bulk-fetched just to compute a count, fixed the same way plus a real variant_count column. And story_actions() walked the relationship, so a turn was O(story length), which is what §2.2 replaced. A 200-turn playthrough went from 84.5 MB to 23.0 MB. A delete went from 115 KB to 5 KB.

+
What makes it stick: tests/test_egress.py hooks SQLAlchemy’s before_cursor_execute. It captures every statement the ORM emits and fails if a bulk load ever names those columns again. The regression is caught by asserting on the SQL, not on a timing. It was verified by sabotage.
+
+ Four traps from that work +

Query.count() wraps the entity select in a subquery, so the emitted SQL names every deferred column. No bytes come back, but the database still reads them. A SQL-grepping guard cannot tell that apart from a real bulk fetch. Use db.query(func.count(Action.id)) instead.

+

SQL and Python must agree exactly on what counts as a story action, or cursors point at the wrong one. trim() strips only spaces, so fold \n\r\t with replace() first.

+

Read table sizes from column sums, not n_live_tup. After a migration rewrites a table, that estimate goes stale in the direction that makes bloat look smaller. It said ~40 MB was reclaimable; the real figure was 79 MB. sum(octet_length(col)) is the honest number.

+

After a migration that rewrites actions, run one VACUUM FULL actions; on the direct endpoint, not the pooler. It takes an ACCESS EXCLUSIVE lock. And DROP COLUMN is metadata-only in Postgres, so it frees nothing by itself.

+
+
The cost trap, recorded so nobody undoes the fix: Neon compute is ~95% of the bill and depends only on awake hours. Never add an uptime pinger that touches the database to dodge Render’s cold start. It holds the database awake around the clock and turns ~$0.30/month into ~$19. Point any warmer at /api/health instead, which deliberately does not query the database.
+
+ +
+
LLDbackend/app/migrations.py
+

3.2  Migrations, hand-rolled

+

No Alembic. An append-only list of (version, SQL) pairs, with the current version in SQLite’s PRAGMA user_version or a one-row table on Postgres. 64 versions so far.

+
    +
  • A fresh database is created by Base.metadata.create_all() — always current — and stamped at the latest version. It never replays history.
    • +
  • An existing database runs every migration above its stored version, in order.
    • +
+

Someone may run this single-file SQLite app for months. The entire requirement is “add a column, don’t lose their data”. Alembic’s autogenerate, branching, and down-migrations are machinery for a team with a staging environment. This is 250 lines, and you can read all of it.

+

The constraint it creates is written at the top of the file. Change models.py so fresh databases are current, and append a pair so existing ones upgrade. Migrations 2–23 predate Postgres support and use SQLite-only syntax. That is harmless, because every Postgres database starts fresh, but anything added since must run on both dialects.

+
+ Migration #10, and a subtle correctness bug +

Repairing duplicate action indexes uses UPDATE … FROM with a window function rather than a correlated subquery. SQLite may evaluate a correlated subquery against partially-updated rows, which would produce fresh duplicates while “repairing” them.

+

Separately, the test suite only ever builds fresh databases. It does not cover migrations as upgrades. Check the upgrade path by hand whenever one is added.

+
+
+ +
+
LLDHLD
+

3.3  The spending surface, and the strangers

+

The demo lets people play with no signup and no API key, on a key the server pays for. That makes it the most defended code in the project.

+

resolve_provider_config() is the single place the BYOK-versus-demo decision is made. On the demo branch it pins two things. It pins the model to a whitelist, so no caller-supplied override can aim a server-funded key at an expensive model. It pins the endpoint to the configured demo URL, so the key cannot be redirected to a URL the user controls and harvested. A defensive __post_init__ raises if a demo config somehow carries a non-whitelisted model.

+
+ The check that took the site down +

That backstop tests using_demo, not api_key == DEMO_API_KEY. Keying on the key value looks stricter, but it is wrong. The demo key is an ordinary OpenRouter key, so a user can legitimately paste that same value into their own settings as BYOK. Every resolution then raised, 500ing even GET /auth/me, which is the SPA’s bootstrap call. Nothing rendered at all.

+

/auth/me is a single point of failure for the whole frontend. Anything it touches must not be able to raise. And using_demo is what actually means “the server is paying”.

+
+ +

Secrets

+

Everything derives from one server-side secret. Passwords use hashlib.scrypt (N=214, r=8, p=1, per-password salt, constant-time compare); this is a stdlib function, so it needs no extra dependency. Sessions are v1.<user_id>.<HMAC-SHA256> with no expiry, because long-lived guest sessions are the point. Stored LLM keys are Fernet-encrypted at rest with an enc: prefix, so legacy plaintext rows stay recognizable. The secret auto-generates for local installs, but multi-user mode refuses to start without the env var. Hosted filesystems are ephemeral. A regenerated secret on every deploy would silently log out every user and orphan their stored API keys.

+ +

Two findings from an authorized pen-test

+

Rate limits were bypassable (High, confirmed live). The Dockerfile ran uvicorn with --forwarded-allow-ips "*", which trusts the leftmost X-Forwarded-For value. Render forwards rather than strips the inbound client XFF, appending the real client IP on the right. So the leftmost hop was fully attacker-controlled, and every rotation bought a fresh rate-limit bucket. Proof: a fixed IP hit 429 after 10 login attempts; rotating a spoofed XFF passed 14 of 14. That defeated the only anti-brute-force control in the app.

+

Fixed in two independent layers. limits._client_ip() now reads the rightmost hop, the one Render appends, which a client cannot push past. It is tunable via AIDND_TRUSTED_PROXY_HOPS. A new per-account throttle also applies: 8 failures per 15 minutes, keyed on the target email, so a botnet with many real IPs still cannot brute one account. The accepted tradeoff is that an attacker can keep a known account in a 15-minute cooldown. That is a nuisance, and it is strictly better than brute force.

+

SSRF via the BYOK endpoint (Medium). A hosted user could point Settings.endpoint_url at an internal or metadata address, and the connection test echoed part of the response back. netguard.endpoint_block_reason resolves the host and refuses anything where not ip.is_global. It checks at request time, so it resists DNS rebinding. It is a deliberate no-op in local mode, since a local install reaching localhost:11434 is the intended case.

+

What held, and is worth naming: per-object authorization was solid throughout. Every {id} route filters on user_id, and sub-resources re-check they belong to their parent. The API key is write-only in every response shape. Session tokens are unforgeable. The sandbox has no host bindings.

+ +

The guard rails, in numbers

+
+ + + + + + + + + + + + + +
GuardLimit
Turn generation10 / min
Auth attempts, per IP10 / 5 min
Guest creation, per IP (each is a DB row)30 / 5 min
Script test runs (each costs up to 2 s CPU)30 / min
Connection test (outbound HTTP to a user URL)10 / min
Adventures / scenarios / scripts per user100 / 200 / 200
Actions per adventure5,000
Request body, and on import endpoints2 MB / 20 MB
Demo turns per user per day20
+
+

Import endpoints check bundle list lengths against the same caps live creation enforces. Otherwise the cap is bypassed by uploading a file. That check had a bug worth remembering: a cap has to count what gets written, not what the file says. The import counted a v1 file’s turns, but each turn expands into a row per saved attempt. A file inside a 5,000-action cap could write 50,000 rows.

+
+ +
+
LLDanalytics.py · accesslog.py
+

3.4  Counting visits without undoing §3.1

+

The obvious way to build analytics is to write a row per request and read rows per dashboard query. That would have undone the entire egress fix. So a visit is a write and never a read. Counts accumulate in a process-local dict and flush every 60 seconds as UPSERTs into a generic (day, metric, label) → hits table, plus one row per visitor per day for the funnel flags. Every dashboard query is a GROUP BY. It returns tens of rows no matter how much traffic sits behind it.

+

The counters are anonymous and the access log beside them is not, on purpose. A visitor in the counter tables is HMAC(secret, "visitor:<user id>") truncated to 32 chars. It is one-way, so those tables cannot be joined back to users, and keyed, so no client can compute one. Story content never reaches that module. The identifying half lives in a separate module and a separate table, so the anonymity of the counters is a property of the code rather than a convention.

+
    +
  • The funnel counts people, not clicks. A player who starts six adventures is one person who started an adventure. That is the whole reason the per-visitor-day table exists. Its flags only ever turn on.
    • +
  • A failed turn is an HTTP 200 with a bad ending. Status-code middleware cannot see one. A demo whose model started refusing every request would look perfectly healthy. All five SSE error paths now go through one turn_error() helper.
    • +
  • The tests run on SQLite; production is Neon. A flush that raises is caught and logged. A dialect mistake in the UPSERTs would have stayed invisible while the dashboard quietly stayed empty. One test compiles both statements against the Postgres dialect without connecting to one.
    • +
+
+ +
+ +
+

Part 4 — The scoreboard

+

What the work bought, and what it deliberately did not.

+
+ +
+ +
+
HLD
+

Measured results

+
+ + + + + + + + + + + + + + + + +
PropertyFigure
Database egress per adventure load38.5 MB → 0.20 MB
Turn read cost at turn 200839 KB → 129 KB, flat
200-turn playthrough, total reads84.5 MB → 23.0 MB
Cost of a branch~103 B · 20 forks load at 1.007×
Prompt snapshot size~74 KB/turn, 94% of the DB
Length hint, budget vs ceiling phrasing246 vs 170 words (n=5)
Backend tests440
Schema versions64
Sandbox limits16 MB, 2 s, fresh context
Context defaultsnote at depth 3, cards ≤ 40%
Memory cadencememory /6, summary /15, top-5
Actual hosting bill~$0.30/mo, mostly $0 collected
+
+

Two of those tests encode a performance property rather than a behavior: test_egress.py asserts on the SQL the ORM emits, and test_history_window.py asserts that the read cost stops growing with story length.

+
+ +
+
HLD
+

Known limitations

+

Deliberate trades for a single-user-first app that also happens to be hosted, written down so nobody has to discover them the hard way.

+
    +
  • Single process. The turn lock, the rate limiter and the summarization task all assume one worker. A second would need a row-level advisory lock and Redis.
    • +
  • No vector index. Retrieval does cosine similarity in Python over the whole bank. Fine at the 200-memory cap; at 10,000 it wants pgvector.
    • +
  • Prompt snapshots are heavy even after the egress fix. They are deferred, not smaller. Compressing or expiring them is the real fix.
    • +
  • In-memory rate-limit windows reset on restart, so a restart grants a brief extra allowance.
    • +
  • Background summarization is a fire-and-forget asyncio task, so it does not survive a restart. At real load it belongs in a queue.
    • +
  • The two memory marks are one pair on the adventure, not one per branch. Switching lines makes the mark on the line being left unreadable, so that ground is summarized again. It fails in the safe direction — redo, never skip — but switching back and forth costs AI calls.
    • +
  • Story cards are adventure-wide, so a card invented on one branch shows on all of them.
    • +
  • Editing an already-summarized turn leaves its memory stale. Replacing a turn withdraws what was derived from it; editing one in place does not.
    • +
  • The frontend has no test runner. This is the standing reason the project keeps finding UI bugs by hand — every one of the tree-UI findings above was invisible to 440 green backend tests.
    • +
  • Truncation is silent. Nothing reads finish_reason yet, which is the obvious next step for §2.4.
    • +
+
+ + + +
+ +
+
+ + diff --git a/docs/guide.html b/docs/guide.html new file mode 100644 index 0000000..1db98b8 --- /dev/null +++ b/docs/guide.html @@ -0,0 +1,1577 @@ + + + + + +AI D&D — the engineering guide + + + + + + + + + +
+ +
+ ⚔ AI D&D + Repo → +
+ +
+ +
+

Engineering guide

+

How this thing works, and why it works that way

+

An AI Dungeon-style storytelling engine. The chat loop is the boring part — + the interesting parts are the token-budget allocator, the world-state referee, the story tree + that lets a turn have more than one answer, and the memory system that decides what the model is + allowed to remember.

+

+ Written to be read end to end. Every section states the decision, the reasoning behind + it, and what it cost. · + Project page · + Source +

+
+ +
+
+ Contents +
    +
  1. Part 0 — Orientation
    2. +
  2. Part 1 — The AI layer
    3. +
  3. 1.1 The turn pipeline
    4. +
  4. 1.2 Context assembly is a budget problem
    5. +
  5. 1.3 World state: the AI proposes, Python referees
    6. +
  6. 1.4 Output length, by measurement
    7. +
  7. 1.5 The memory bank
    8. +
  8. 1.6 Streaming
    9. +
  9. 1.7 The scripting sandbox
    10. +
  10. 1.8 Why there is no agent framework
    11. +
  11. Part 2 — Data and correctness
    12. +
  12. 2.2 The story is a tree
    13. +
  13. 2.3 Undo and retry that rewind
    14. +
  14. 2.5 The 189× egress fix
    15. +
  15. Part 3 — Production concerns
    16. +
  16. Part 4 — The web plumbing
    17. +
  17. Part 5 — Results and limitations
    18. +
+
+
+ + +
+

Part 0

+

Orientation

+

What the thing is, in the fewest words that are still true.

+
+ +
+ +

0.1What it is

+ +

An AI Dungeon clone. You write a scenario, then play an open-ended text adventure where a +language model narrates the world. You type “I open the door”, the model writes what happens +next, and it remembers what came before.

+ +

Four things make it more than a chat wrapper:

+ +
    +
  1. A context engine. The model has a limited input window. The app decides, + every single turn, which pieces of the story get to be in the prompt and which get dropped.
    2. +
  2. A world-state engine. The scenario declares stats — hp, + trust, day. The model proposes changes each turn; a Python engine + decides what actually sticks.
    3. +
  3. A story tree. The story is not a list. Any turn can hold more than one + take, and writing below one that isn’t the live one starts a branch — which borrows every turn + above the fork instead of copying it.
    4. +
  4. A scripting sandbox. Real AI Dungeon JavaScript scripts import and run, + inside an embedded QuickJS interpreter.
    5. +
+ +

Runs locally against Ollama for free, or hosted against any OpenAI-compatible endpoint.

+ +

0.2The stack, and what each part is doing

+ +
+ + + + + + + + + + + + +
PieceWhat it actually does here
FastAPIThe HTTP server. Every URL like /api/adventures/3/actions maps to a Python function. Also does the SSE streaming.
SQLAlchemyLets you write Python classes instead of SQL. Adventure, Action, Memory are Python classes; SQLAlchemy turns them into tables and turns attribute access into SELECTs.
SQLite / PostgresThe database. SQLite is one file on disk (local). Postgres is a server (hosted, on Neon). Same code talks to both.
ReactThe UI. Describes what the screen should look like for a given state; when the state changes it re-renders.
ViteThe frontend build tool and dev server. Bundles React into plain JS the browser can load.
httpxThe Python HTTP client used to call the model endpoint.
tiktokenCounts tokens, so the budgeting is real arithmetic and not a guess.
QuickJSA small embeddable JavaScript engine, used as a sandbox for user scripts.
+
+ +

The whole thing is one process in production: FastAPI serves the API and the built +React files from the same port.

+ +

The shape of one request

+ +
you tap "Do"
+  → POST /api/adventures/3/actions
+       {type: "do", text: "open the door"}
+  → check ownership, rate limit, turn lock
+  → assemble the prompt      ← the interesting part
+  → POST to the model endpoint, stream=true
+  → tokens come back one at a time
+  → each is forwarded on as a Server-Sent Event
+  → React appends it to the screen as it arrives
+  → stream ends: parse the state block, referee
+       it, save the action
+ +
+ + +
+

Part 1

+

The AI layer

+

Where most of the design effort went. Everything here is a decision someone could + reasonably disagree with.

+
+ +
+ +

1.1The turn pipeline

+ +

Everything that happens between “player pressed a button” and “text is on screen”. +Source: backend/app/routers/adventures.py.

+ +
+
onInput — user JS may rewrite or block the input
+
store the player action
+
retrieve memories — embed recent story, cosine-rank the bank
+
build_context() — the budget allocator
+
onModelContext — user JS may rewrite the whole prompt
+
snapshot the exact prompt — for the Insights panel
+
provider.generate() — streamed, token by token
+
onOutput
+
extract + referee the state block — then strip it from the prose
+
save the action — stamped with the state it leaves behind
+
background: summarize + embed — fire-and-forget
+
+ model or prompt work + user script hook + persistence +
+
+ +

Two design choices are visible in that list before any of the details.

+ +

The prompt is snapshotted, not reconstructed. Every AI action stores the +exact text that was sent to the model. That’s what powers the Insights panel — open any turn and +see each context component, its token cost, and why it was included. It’s also what makes prompt +bugs findable. The cost is storage, about 74 KB per turn, which turns into a real performance +problem later (see 2.5).

+ +

Every node records the state it leaves behind. state_after and +world_state_after are stapled onto the action once its hooks and its delta have run, +so a node carries the scoreboard and the RPG stats as they stood when that turn finished. Rewinding +to before a turn is then a read of the node in front of it — the same move as switching to +another branch. One mechanism, and it is why undo, retry and a branch switch all put the numbers +back rather than only rewriting text.

+ +

1.2Context assembly is a budget problem

+ +

Source: backend/app/context/builder.py.

+ +

The problem

+ +

The model can only read so much. Say the budget is 8,000 tokens. A 200-turn adventure has far +more story than that. Something has to be dropped, and what gets dropped decides whether +the story stays coherent.

+ +

The naive version, and why it breaks

+ +

Send the last N turns. That fails in two directions: N turns of short exchanges wastes the +window, and N turns of long ones overflows it. Worse, “the last N turns” throws away the things +that matter most — the premise, the character sheet, the fact that you promised the innkeeper +you’d return.

+ +

What this app does

+ +

Split the prompt into fixed sections and elastic ones.

+ +
+ + + + + + + CONTEXT TOKEN BUDGET + + + + + + + + + + + + RESERVED + CARDS ≤ 40% + HISTORY, NEWEST FIRST + + + + available = budget − reserved + + narrator · stat guide · world state · emit rule · ai instructions + plot essentials · story summary · retrieved memories + ↑ these are always included, whatever they cost + +
Fixed sections are reserved first and never dropped. What’s left is the elastic +budget: triggered story cards may take up to 40% of it, and story history spends the remainder +filling backwards from the newest turn.
+
+ +

The algorithm is three lines of arithmetic:

+ +
reserved  = every fixed section + note + hint + reminder
+available = max(256, token_budget - reserved)
+
+cards   ≤ available * 0.4
+history = available - cards_used, newest first
+ +

The details that are actually decisions

+ +

Cards are capped at 40% of the elastic budget. Story cards are triggered by +keyword match, so a scene mentioning six named things could pull in six lore entries and leave no +room for the story itself. The cap makes the failure mode “some lore is missing” instead of “the +model has no idea what just happened”. Cards that don’t fit are still reported to +Insights with included: false, so the UI can show the lore that got squeezed out.

+ +

History fills newest-first and stops. Oldest turns fall out. That’s the right +direction because the old material isn’t actually lost — it’s been summarized into memories and +the running summary, which live in the fixed section.

+ +

If even the single newest turn is over budget, it gets hard-truncated rather +than dropped. A prompt with no story at all produces nonsense; a prompt with the tail end of the +last turn produces something.

+ +

The author’s note is injected three actions from the end, not at the top. +Instructions placed near the end of a prompt have more influence on what comes next than +instructions at the top — recency. The author’s note is a steering control (“keep it tense”), so +it goes where steering works.

+ +

The world-state reminder goes dead last. The full emit rule lives up in the +system block, hundreds of tokens away from where the model starts writing. A one-line reminder +occupies the final slot. Same recency logic, applied to the thing most likely to be forgotten.

+ +
+ The subtle one +

Past AI turns get their state block re-attached. The block is stripped from + the text before storage, so a replayed history would show the model twenty of its own past + turns that contain no state block — teaching it, by imitation, to stop emitting one. + So the history builder reconstructs the block from the stored delta and re-appends it. The + model sees its own pattern and keeps following it.

+
+ +

The performance trap hiding in this

+ +

Building the context needs the newest ~6,000 tokens of story. The obvious implementation reads +adventure.actions — which loads every row of the adventure — then throws 90% of it +away. At turn 200 that was 839 KB of database reads to use maybe 70 KB, growing every turn.

+ +

context/history.py fixes it by serving three shapes directly from SQL: a tail, a +slice, and a count. window_covering() fetches the newest 32 actions, measures their +real token count, and if that’s short of the budget it projects how many more it needs +from the average length just measured, rather than blindly doubling:

+ +
average   = tokens / len(actions)
+projected = int(budget / average * 1.15) + 8
+ +

Each round fetches only what it doesn’t already hold, so no row is read twice. The same turn +costs 129 KB instead of 839 KB, and stops growing at around turn 50 — the cost is bounded by the +context budget instead of by the length of the story.

+ +

There’s a second rule in that module worth naming: if the actions are already loaded +in memory, slice them instead of querying. The scripting pipeline hands the whole history +to user scripts, because AI Dungeon’s API requires it, so on a scripted adventure the rows are +already there — issuing a query beside them would mean paying twice.

+ +

1.3World state: the AI proposes, Python referees

+ +

Source: backend/app/worldstate/engine.py.

+ +

The question

+ +

You want an RPG layer — hit points, trust, quest progress. Who owns the numbers?

+ +
+
+
Option A — a deterministic dice engine
+
The engine rolls and applies damage, the model narrates the result. This is what a real + RPG does. It loses here because the action space is unbounded: the player can type anything, + and mapping arbitrary natural language onto a fixed rules system is a harder problem than the + one being solved.
+
+
+
Option B — the model owns the numbers
+
Track hp in the prose and trust it. Fails immediately. Models are bad at arithmetic, worse + at holding a number across twenty turns, and completely unable to obey their own frequency + rules — tell one “only change this every 5 turns” and it changes it every turn.
+
+
+
Option C — chosen: propose and dispose
+
The model narrates and appends a JSON delta of what changed. Python validates and clamps + it before anything is stored. The model owns intent; the engine owns arithmetic.
+
+
+ +
narration: "The blade catches your shoulder. Gwen shouts and drags you back."
+
+```state
+{"player.hp": -15, "npc.gwen.trust": 5, "milestones.escaped": true}
+```
+ +

The engine then applies, in order:

+ +
+ + + + + + + + + + + + +
RuleWhat it stops
Path must exist in the schemaHallucinated stats
Value must be the right type"a lot" instead of -15
CooldownChanging a stat more often than the scenario allows
Counters can’t decreaseThe in-game day going backwards
max_delta_per_turnLosing 90 hp to a stubbed toe
Clamp to min/maxNegative hp, trust above 100
Milestones sticky, true onlyUn-completing a quest
Flags are two-way booleansDeliberately unrestricted — that’s what flags are for
+
+ +

Everything rejected is reported, not silently swallowed. The Insights panel shows +applied, clamped and rejected paths per turn, and the chip under each narration shows what +actually changed.

+ +

The reliability mechanism: word bands

+ +

A stat can carry bands:

+ +
"hp": { "min": 0, "max": 100, "initial": 100,
+        "bands": [[0,20,"very weak"],[20,40,"hurt"],[40,60,"minor damage"],
+                  [60,90,"healthy"],[90,100,"full health"]] }
+ +

Two things use them. The live state block shows the current band label — +hp 55/100 (minor damage) — so the model reads a word, not just a number. And +the stat guide shows the whole ladder once per turn, so the model can see the full scale it’s +reasoning across.

+ +

The point: models reason well over semantics and badly over arithmetic. “He’s badly hurt, so a +solid hit should take him to very weak” is a judgement a model can make. “55 minus 22 is 33” is +one it will get wrong often enough to matter.

+ +

The failure philosophy

+ +

Nothing in the world-state engine raises. A malformed delta returns {} and the +turn continues. The parser is deliberately tolerant — it strips trailing commas and leading ++ signs on numbers, both of which weaker free models emit and strict JSON rejects. It +accepts a state, json or unlabelled fence, and falls back to a bare JSON +object at the end of the text, but only if it parses into something that looks like a delta, so +prose ending in } is never eaten.

+ +

This matters because the public demo runs on free-tier models. A stricter parser would mean a +good model works and a free one doesn’t.

+ +

One call, not two

+ +

The model narrates and emits the delta in a single request. The alternative — narrate, +then a second call to extract structured state — is more reliable per call and costs twice the +latency and twice the rate-limit budget. On the free tier (20 requests/minute) that would halve +the playable turn rate. The tolerant parser plus the terminal reminder was the cheaper way to buy +the same reliability.

+ +

1.4Output length, by measurement

+ +

The problem

+ +

max_output_tokens is a hard wall the endpoint enforces mid-sentence. Hit it and +whatever is being written gets cut off. Since the state block is emitted last, the state +block is what gets lost. The turn narrates fine and silently records nothing.

+ +

First attempt, and the measurement

+ +

Tell the model its budget: “keep this turn under about N words”.

+ +
+
174 → 246
average words per turn once the “budget” hint was added — every run longer than every unhinted run (n=5)
+
170
average after rephrasing the same number as a hard ceiling
+
+ +

Phrased as a budget, the number reads as a target to fill. The hint pushed turns +toward the very wall it existed to protect.

+ +

The fix

+ +
[Hard limit: this turn must not exceed 412 words. Write only as much as the
+moment needs — a typical turn is much shorter. Finish the narration and append
+the state block well inside the limit.]
+ +

And the arithmetic around it

+ +
words = int((max_output_tokens - 50) * 0.75 * 0.90)
+ +
    +
  • - 50 — tokens held back for the state block itself.
    • +
  • * 0.75 — models can’t count their own tokens, but they do follow a word budget. + English prose is roughly 0.75 words per token.
    • +
  • * 0.90 — a word budget is a suggestion the model overshoots; the cap it protects + is a hard wall. Aim 10% short so the overshoot lands in slack.
    • +
  • Below 40 words the hint is dropped entirely — it stops earning its tokens.
    • +
+ +

1.5The memory bank

+ +

Source: backend/app/memorybank.py.

+ +

The problem

+ +

Story history falls out of the context window as the adventure grows. Turn 4 said you promised +the innkeeper you’d return. At turn 90 that’s long gone from the prompt — but if you walk back +into the inn, it should come back.

+ +

Three layers

+ +
+ + + + + + + +
LayerCadencePurpose
Memoryevery 6 actions, from 12, once one action sits past the blockOne or two past-tense sentences of concrete fact.
Story summaryevery 15 actionsA single ≤250-word overview, rewritten by folding in the new memories.
Retrievalevery turnEmbed the last 4 actions (≤600 tokens), cosine-rank the bank, inject the top 5.
+
+ +

Retrieval is what answers the innkeeper problem: the promise is a memory, the memory has a +vector, walking into the inn produces a query vector near it, and it comes back into the prompt.

+ +

The decisions inside it

+ +

A memory hangs off the node whose block it ends on. Not off the adventure, and +not off a position in a list of actions — off a (branch_id, depth) coordinate. +That makes “which memories described this turn?” an indexed lookup rather than a scan for rows whose +covered range has fallen off the end of the story, and it is what makes memories inherit correctly +across a fork: the ones above the fork point already sit on ancestors both lines read.

+ +

It is also the repair. When a turn’s text is replaced or removed — a retry, an undo, a deleted +action — forget_node withdraws the memory hanging off that coordinate and +rewinds both marks to just before the stretch it covered, so that ground is summarized again from +what the story now says. An earlier version instead held the newest action back a turn so it could +never be summarized before it stopped being retryable; correctness no longer rests on that, because +the repair exists whether or not the invalidation happens at the tip.

+ +

A block still waits for one action to settle past it +(SETTLE_SLACK), and that is a cost rule rather than a correctness one. Retry and +take-switching both refuse anything but the newest action, so a memory whose block ends on the tip +is the one memory a player can still throw away: every retry of that turn writes it, withdraws it, +and writes it again. A block closes every 6 actions and a normal turn writes 2, so without the +slack that is one turn in three. The slack costs nothing in return, because the block that just +closed is still in the history window in full — a memory of it says what the model can already +read. Memories earn their place once the raw text has scrolled out, which is never the turn the +block closed.

+ +

Cursors only advance on success. Every AI call here is best-effort. If +summarization fails, the function returns and the cursor is unchanged, so the same block is +retried on a later turn. There’s no retry loop, no dead-letter queue, no backoff — the cadence +is the retry mechanism.

+ +

Summarization is fire-and-forget, in a background task with its own DB session. +The player’s turn is already on screen; making them wait would add a second or two of latency +every sixth turn for no visible benefit. The task holds a strong reference to itself — the event +loop only keeps weak ones, so a fire-and-forget task can otherwise be garbage-collected mid-run — +and a per-adventure guard stops two from overlapping.

+ +

Pinned memories count toward top_k. Pinned ones are always +injected; unpinned fill up to top_k − len(pinned). Without that, 6 pinned memories +plus top_k=5 injects 11 and blows the budget the whole context engine exists to +respect.

+ +

A dimension mismatch scores 0.0, it doesn’t crash. If the user changes their +embedding model, old 768-dim vectors get compared against a new 1536-dim query. +zip() would happily truncate and score garbage, silently. An explicit length check +returns 0.0 instead.

+ +

Eviction is LRU-ish, and evicted memories are kept. Over capacity (default +200), the least-used unpinned memories are marked forgotten rather than deleted — so +the UI can still show them and you can un-forget one.

+ +
+ Money trap +

Background calls never spend the shared demo key. The summarization and + embedding providers are built directly from the user’s own settings, never from the demo config, + and their call sites are skipped when the turn is running on the demo key. Summarization is + unmetered background spend; the demo key is server-funded. Both facts together would be a bill.

+
+ +

1.6Streaming

+ +

The model produces tokens one at a time. Waiting for the whole reply before showing anything +makes a 20-second generation feel broken.

+ +

Server-Sent Events is the mechanism: an HTTP response that stays open and +pushes data: {...} lines as they become available. It’s one-directional +(server → browser), which is exactly the shape of this problem — WebSockets would be a +bidirectional connection for a unidirectional need.

+ +
model endpoint  --SSE-->  FastAPI  --SSE-->  browser  -->  React state  -->  screen
+ +

FastAPI reads the provider’s stream and for each chunk yields +data: {"type":"chunk","text":"…"}. The frontend reads the response body with a +ReadableStream reader, buffers on \n\n boundaries, and dispatches each +parsed event. Event types: player, reasoning (thinking-model traces, +which stream into a separate collapsible panel with their own token budget), chunk, +stopped, error, done.

+ +

Two production details that only show up when hosted:

+ +
    +
  • X-Accel-Buffering: no — nginx-style reverse proxies buffer responses by default, + which turns a stream into one big delivery at the end. This header tells them to flush each event.
    • +
  • The security-headers and body-size middlewares are written as pure ASGI + rather than Starlette’s BaseHTTPMiddleware, because the latter buffers the response + body and would break streaming.
    • +
+ +

The empty-reply case is diagnosed, not reported as “empty”. If a reasoning +model streams thinking but no story text, it spent its whole budget thinking — the error says so +and names the three settings that fix it.

+ +

1.7The scripting sandbox

+ +

Real AI Dungeon scripts are JavaScript files defining modifier(text) and calling it +as the last line, with globals like state, history, +storyCards. To be compatible, this app runs the same contract in an embedded +QuickJS interpreter.

+ +

The safety properties are mostly structural:

+ +
+ + + + + + + + + +
PropertyHow
No filesystem, network or process accessQuickJS has none by default — nothing was removed, nothing was added
Memory cap16 MB per run
CPU cap2 seconds per run
No shared state between runsA fresh context per hook execution
A broken script can’t break a turnEvery failure returns as .error with text, state and cards unchanged; the pipeline logs it and continues
+
+ +

Data crosses the boundary as JSON — Python serializes {state, text, history, storyCards, +info} in and the results out. There is no object bridge to exploit.

+ +

One deliberate bug-compatibility: addStoryCard returns the new card’s +index, so the first card returns 0, which is falsy, so +if (!addStoryCard(…)) misfires. That’s upstream AI Dungeon’s behaviour. It’s +documented in the code and left alone, because matching real scripts is the entire point of the +feature.

+ +

1.8Why there is no agent framework

+ +

Graph-based agent frameworks (LangGraph and similar) earn their complexity with +branching, cyclic, multi-step control flow +— a graph of nodes where the path depends on what the model decides, with loops, retries, tool +calls, and persisted state between steps.

+ +

This turn pipeline is a fixed linear sequence with exactly one model call. +There is no routing decision, no tool selection, no loop. Every turn takes the same path. Adding a +graph framework would mean carrying its state abstraction, its serialization model and its +debugging surface to express a straight line.

+ +

There’s also a specific reason a framework’s context handling wouldn’t fit here: +the budgeting logic is the product. Buffer-window and summary-memory abstractions +are opinionated about how to fit history into a window. This app shows the user every context +component, its token cost, and the trigger word that pulled it in — which means the assembly has +to be explicit and inspectable.

+ +

When it would be the right call: if the design went toward the +two-call version — narrate, then a separate structured-extraction step, with a retry branch when +extraction fails and a tool-calling path for dice — that is a graph, and hand-rolling it would get +ugly fast.

+ +
+ + +
+

Part 2

+

Data and correctness

+

The bugs in this section are the kind that don’t crash. They just quietly produce the wrong + answer, which is why each one has a test.

+
+ +
+ +

2.1The domain model

+ +
User
+ ├─ Scenario   (the template)      ── stat_schema, prompt, memory, author's note
+ │    └─ StoryCard, Script
+ └─ Adventure  (the playthrough)   ── world_state, script_state, head_branch_id/head_depth
+      ├─ Branch  (one line of it)  ── parent_branch_id, fork_depth, lineage, name
+      ├─ Action  (one node)        ── branch_id, depth, parent_id, live,
+      │                               text, context_snapshot, state_after
+      ├─ StoryCard  (its own copy)
+      ├─ Memory     (text, embedding, branch_id, depth, use_count)
+      └─ AdventureScript
+ +

The one decision that shapes everything: template vs instance. A scenario +declares what stats exist; an adventure holds what they are right now. Creating +an adventure copies the scenario’s story cards, scripts and plot fields into it, so editing a +scenario later never mutates a game in progress. There’s an explicit opt-in “Update from scenario” +flow for when you do want that, which diffs the two and shows what would change.

+ +

Same reasoning as instantiating a class: shared definition, independent state.

+ +

2.2The story is a tree

+ +

The largest structural change the project has had, and the one with the most reasoning behind +it.

+ +

The problem

+ +

The story used to be a list, and a mutable one. Retry rewrote the last entry in place; undo and +delete removed entries from the middle. Everything derived from the story — the memories, the +running summary, the two marks saying how far each had got — was indexed by position in that +list, and a position means something different after anything in front of it is deleted.

+ +

That single fact produced a family of bugs that all looked different:

+ +
    +
  • Deleting a middle action slid a never-summarized action down into the “already covered” + range, so a recent action silently never became a memory.
    • +
  • Discarding a memory left its actions behind the mark, describing nothing.
    • +
  • Retry rewrote an action’s text after the mark had passed it, so its memory described + narration that was no longer in the story.
    • +
  • The retried row was still attached to the adventure while its replacement was being written, + so the model was shown the attempt it was meant to replace and wrote a continuation of + it. That exclusion had to be threaded through four separate readers.
    • +
  • Attempts lived in a JSON array on the row with a mirrored copy of the live one in the + ordinary columns — a repeating group and a denormalisation in one.
    • +
+ +
+ The pattern +

Each was fixed where it was found. The shape only becomes visible when you line them up: + they are all the same bug, and it is that the story is a list nobody may reorder.

+
+ +

The shape

+ +

Make the story a tree and none of them are reachable. Every action is a node +with a branch_id and a depth. A branch is one line +through the tree: it holds the nodes played on it and borrows everything before its fork +point from its ancestors. Nothing is ever copied, and — apart from an explicit delete — nothing is +ever removed.

+ +
branches(id, adventure_id, parent_branch_id, fork_depth, lineage, name)
+actions(id, adventure_id, branch_id, depth, parent_id, live, text, …, state_after)
+memories(…, branch_id, depth)
+adventures(…, head_branch_id, head_depth)
+ +

depth is a position along a path, not a global turn number: +A4 and B4 are two alternatives, not two turns. Reading branch C, whose +tip is at depth 7 and which left B at 5, which left A at 3:

+ +
SELECT * FROM actions
+WHERE (branch_id = 'C')
+   OR (branch_id = 'B' AND depth <= 5)
+   OR (branch_id = 'A' AND depth <= 3)
+ORDER BY depth DESC LIMIT 32
+ +

→ A0 A1 A2 A3 B4 B5 C6 C7.

+ +

Why branch_id + depth rather than parent pointers alone. +Parent pointers are the obvious way to store a tree and the wrong way to read one: reading a story +would be N round trips up a chain, which throws away the windowed history work (1.2) +that made a turn’s read cost flat. Depth replaces the old index as the ordering key, so +the reads keep the shape they already had.

+ +

The lineage, and why fork count costs nothing

+ +

The OR-clause above is not reconstructed per read. It is stored on the branch row as +lineage — [(C, ∞), (B, 5), (A, 3)] — computed once when the fork happens, +from the parent’s lineage plus one entry. One module knows how to turn it into a query, which is +deliberate: one forgotten clause shows the wrong story and reports nothing.

+ +

Two properties of the shape do the real work:

+ +
    +
  • The ranges are disjoint and descending. A branch’s own nodes always sit + deeper than its fork point, and each ancestor is capped at the fork depth of the branch beneath + it. So ordering the whole clause by depth DESC reads entry 0’s nodes, then entry 1’s, + then entry 2’s — which lets a tail read use the newest few entries and stop.
    • +
  • Clause count is bounded by the context window, not by fork count. A 200-fork + story whose newest branch is 40 turns long reads with one clause, because the window is + covered before the second entry is reached.
    • +
+ +
+
1.007×
page load of a 40-turn story forked twenty times, against the same story flat — 31,652 B vs 31,433 B
+
~103 B
what one branch costs: an id, a parent, a fork depth and a cached ancestry. No copy, no migration, no vacuum.
+
+ +

What a player actually does

+ +

None of the above is what the screen shows. In the player’s words:

+ +
+

Any turn can gain another take. On an AI turn that means regenerate; on your + own message it means type something else. Stepping between takes with ‹ 2/4 › is free + — the story below simply empties, because that take has no children yet. A branch is + created when you write below a take that is not the live one, never before.

+
+ +

That rule collapses two operations into one and deletes a distinction from the UI. The first +version of this screen had a chip that switched at the tip and only previewed +above it, with a second button to take that line — one control whose meaning depended on where the +reader was standing. The rule above replaced it with a pager that only ever steps, a fork button on +every turn, and no tip-versus-past distinction at all. The distinction survives in the +implementation, where it decides whether a write needs a branch: at the tip the attempts are still +leaves nobody has built on, so taking one is a switch and no branch is created.

+ +

Takes are grouped by parent, not by coordinate

+ +

The load-bearing detail, and the one that isn’t obvious. The natural way to find “the other +takes of this turn” is by coordinate — same branch, same depth. It is wrong in both directions:

+ +
B ── C   C1   C2              <- three takes, one parent (B)
+          │    └── D1' D2'    <- two takes, parent C2
+          └── D1 D2 D3        <- three takes, parent C1
+ +

Standing on the C2 path at that depth must read 2/2, not 5. Coordinate +grouping gets that one right by accident, because writing under a non-live take forks and the two +sets land on different branches. It gets C wrong: once C has been forked onto a branch +of its own it is alone at its coordinate and reads 1/1, having lost C1 and C2 from a +pager that must still say 1/3.

+ +

So a node carries parent_id, read for nothing but this. The alternative — making a +branch’s fork point a node rather than a depth, so a promoted take never moves — was +rejected: the whole point of lineage is that a read is an OR-clause per branch instead +of a walk up parent pointers, and re-pointing the fork at a node changes path resolution itself, +dragging in the cursors, memory depths and both bundle formats. parent_id is one +indexed lookup, never a walk, and nothing about how a path resolves changes.

+ +

Cursors become anchors

+ +

The two marks — how far the memory bank has got, how far the summary has got — used to be +counts. A count is a position in a list, and every rule about sliding them, rewinding them and +translating between positions and Action.index existed to patch up the fact that the +list moves.

+ +

A cursor is now an anchor: (branch_id, depth), the node up to and +including which the work is done. Deleting an action doesn’t move it, because a depth is a +coordinate along a path rather than a slot in a list. “What is not covered yet” becomes a question +about the story instead of about a list index, and it answers correctly whatever has been deleted +in front of it. The branch half is what makes it survive forking: a depth alone is ambiguous once +two branches both have a node 41.

+ +

position_of_index, note_action_removed, +settled_story_actions and the cursor-rewind machinery were deleted, +not left unused.

+ +

Derived work attaches to the node that produced it

+ +

Generalise the rule and a lot falls out: anything derived hangs off the node that produced +it. A memory covering depths 37–42 hangs off that branch’s node 42 and is invisible to any path +that doesn’t run through it. Shared ancestors are therefore shared automatically, so a fork +needs nothing recreated — the memories above the fork point are already on the ancestors +both lines read.

+ +
+ The subtle case +

The memory sitting at the forked coordinate. The first cut moved it onto the new + branch and re-anchored the marks naming it. Both are wrong for the same reason: that memory + describes whichever attempt was live at that coordinate, which is the one staying on the parent.

+

The right answer needs no code. The lineage caps the parent one depth short of the fork, so + the memory is simply out of range from the new branch — invisible to both the retrieval clause and + the anchor read. The new line summarizes that ground again, from the text it actually tells.

+
+ +

Hand-written memories obey the same rule. One used to carry a NULL depth, described as “belongs +to the adventure rather than to a path” — which sounds harmless and is not: a NULL is a coordinate +no fork can cap, so a note typed on one line followed the reader onto branches whose events it never +described. They are anchored at the head instead: the story you were reading when you wrote +it.

+ +

Deleting, and why the branch UI was a hard dependency

+ +

Nothing is ever auto-pruned. That is the guarantee the whole design rests on, and it is also why +branch management couldn’t be a nice-to-have: without a way to delete a line, storage grows without +limit.

+ +

The delete rule has two halves and the second is easy to miss. Refusing to delete the line being +read is obvious. The other half is refusing any line it was forked from — +parent_branch_id cascades, so deleting an ancestor takes the head with it and leaves +head_branch_id pointing at a row that is gone. One membership test against the head’s +own lineage covers both, because a lineage already names itself and every branch it borrows from. +The server is the authority; the client computes the same set only so a button can say so before it +is pressed.

+ +

The migration, and what it deliberately did not do

+ +

There is no feature flag. A linear story is a tree with one branch, so the +intermediate states weren’t half-migrated — they were the same product with a superset schema +underneath, which made “existing adventures are unaffected” a literal, testable pass condition at +every step. A flag would have bought two live code paths through the context builder, the memory +bank, undo and retry at once.

+ +

The legacy columns (index, variants, variant_index, the +two *_before snapshots) were kept unread for a release rather than dropped with the +migration that stopped using them, so that redeploying the previous build is still a way out. +Dropping columns is the one step that isn’t.

+ +
+ Operational, and it generalises +

On Postgres a migration that rewrites every row of actions roughly doubles the + table, and only VACUUM FULL gives it back — 79 MB reclaimed in 5.5 s on one occasion. + But bloat scales with the heap, and context_snapshot is 94% of this + table and lives out of line, so a migration touching only small columns reuses the existing TOAST + pointer and costs a tenth of that.

+

Read the sizes from sum(octet_length(col)) per column, not from + n_live_tup — that one is a stale estimate in exactly the direction that makes bloat + look smaller.

+
+ +

What this is honest about

+ +
    +
  • The two marks are one pair on the adventure, not one per branch. Switching + lines makes the mark on the line being left unreadable from the new one, and that ground is + summarized again. It answers “nothing covered”, which is the safe direction — redo the work, never + skip it — but switching back and forth costs AI calls.
    • +
  • Story cards stay adventure-wide. A card invented on branch B shows on branch + A. Event-sourcing card changes onto nodes was considered and rejected.
    • +
  • Editing an already-summarized action still leaves its memory stale. The + machinery to fix it now exists — an edit could write a sibling take and switch to it, which is a + retry the player typed — but it doesn’t do that yet.
    • +
+ +

2.3Undo and retry that actually rewind

+ +

Most implementations of undo delete the last message. That’s wrong here, because a turn mutates +three things: the text, the scripting scoreboard, and the RPG stats.

+ +

The mechanism: every node carries state_after and +world_state_after — deep copies of what the adventure looked like once that turn had +played. Rewinding to before a turn is a read of the node in front of it, so undo, retry and a branch +switch are the same restore. The cooldown clock comes along for free: it lives inside the world +state, so each line of the story carries its own without anything having to know there is one.

+ +

Nothing a retry replaces is thrown away. The old attempt stays as another +take of that turn — a sibling node at the same coordinate, live false — and the +pager steps between them. Which is to say retry isn’t a special case: it is the tree, with the branch +not yet created (2.2).

+ +

Four details that are easy to get wrong:

+ +
    +
  • The turn being retried is excluded from its own context. Its takes are still + attached to the adventure, so without an explicit exclusion the model would be shown the attempt + it’s replacing as established story — and would write a continuation of it. The exclusion had + leaked into four readers, not one: history replay, story-card trigger matching, in-scene NPC + detection, and the memory-bank similarity query. Anything reading the story during generation + takes the exclusion.
    • +
  • A retry reuses the turn’s depth, not the next one. Cooldowns are measured + along the path, so allocating a new depth would advance the clock the cooldown rules run on and a + retry would quietly unlock stats that should still be waiting.
    • +
  • delete_turn used to mean “every take at this coordinate”. Once a + take can be forked onto a branch of its own the group spans branches, and undo reached across and + deleted a take belonging to a line nobody asked about. Anything that reads a take group and then + writes has to say whether it means the turn or the coordinate.
    • +
  • If the regeneration fails, the rollback is reversed. The generator is wrapped + in a try/finally: if it ends without saving — provider error, empty reply, a script + stop, or the browser hanging up — the previous take is put back in charge. Otherwise + the state on the server drifts from the text still on the user’s screen.
    • +
+ +

2.4The turn lock

+ +

One turn at a time per adventure. Double-clicking “Continue” must not run two generations.

+ +

The subtlety: the check has to happen in the request phase, not when the SSE +generator first runs. A streaming response doesn’t start iterating its generator until the response +begins, so a check inside the generator lets two rapid requests both pass before either claims the +slot. And because sync FastAPI endpoints run in a threadpool, the test-and-set needs a real lock.

+ +
def acquire_turn_lock(adventure_id):          # in the request handler
+    with _active_turns_guard:
+        if adventure_id in _active_turns:
+            raise HTTPException(409, "A turn is already generating…")
+        _active_turns.add(adventure_id)
+
+async def with_turn_lock(adventure_id, gen):  # wraps the SSE generator
+    try:
+        async for event in gen: yield event
+    finally:
+        _active_turns.discard(adventure_id)
+ +

In-memory, so it’s a single-process guarantee. That’s honest for the deployment this targets — +one Render web service. Two processes would need the lock in the database.

+ +

2.5The 189× egress fix

+ +
+
38.5 MB → 0.20 MB
database egress for one adventure load
+
~74 KB
per-turn prompt snapshot — 94% of the database
+
+ +

The bug: Action.context_snapshot holds the entire assembled prompt +for a turn. Every adventure load pulled that column for every action, to read two small fields out +of it — the world-state delta for the “what changed” chip, and the applied report. SQLAlchemy loads +all columns by default.

+ +

The fix, in three parts:

+ +
    +
  1. Move the two small things that are needed for every action into their own column.
    2. +
  2. Mark the heavy columns deferred — snapshot, variants, reasoning — so they’re only + fetched when explicitly asked for.
    3. +
  3. Backfill the new column with dialect-specific server-side SQL, so the old data is extracted + inside the database and never crosses the wire.
    4. +
+ +

The part that makes it stick: tests/test_egress.py hooks into +SQLAlchemy’s before_cursor_execute event, captures every statement the ORM sends, and +fails if a bulk load ever names those columns again. The regression is caught by asserting on the +SQL, not on a timing.

+ +

One more detail from that test’s design: the count query is written as a real +SELECT count(…) rather than query.count(), because SQLAlchemy’s +.count() wraps the entity select in a subquery whose SQL names every column — +including the deferred ones. No bytes come back either way, but the database still reads them, and +a guard that greps SQL can’t tell the two apart.

+ +

There’s a companion denormalization for the same reason: the pager has to know how many takes a +turn has without fetching any of them, so variant_index and variant_count +are cached on the row and refreshed by exactly one function, precisely so they can’t drift and the +pager can’t lie. variant_count is 0 rather than 1 for a turn nobody retried, because +the question it answers is “is there anything to page through?”

+ +

2.6Migrations, hand-rolled

+ +

No Alembic. An append-only list of (version, SQL) pairs, with the current version +stored in SQLite’s PRAGMA user_version or a one-row table on Postgres. 64 versions so +far.

+ +
    +
  • A fresh database is created by create_all() — always current — + and stamped at the latest version. It never replays history.
    • +
  • An existing database runs every migration above its stored version, in order.
    • +
+ +

Why this and not Alembic: for a single-file SQLite app someone may have been running for months, +the entire requirement is “add a column, don’t lose their data”. Alembic’s autogenerate, branching +and down-migrations are machinery for a team with a staging environment. This is 250 lines and you +can read all of it.

+ +

The constraint it creates is written at the top of the file: change models.py so +fresh databases are current, and append a pair here so existing ones upgrade. Migrations +2–23 predate Postgres support and use SQLite-only syntax — harmless, because every Postgres +database starts fresh and never replays them, but anything added since must run on both dialects.

+ +

One migration worth reading (repairing duplicate action indexes) uses UPDATE … FROM +with a window function rather than a correlated subquery, because SQLite may evaluate a correlated +subquery against partially-updated rows and produce duplicates again while “repairing” them.

+ +
+ + +
+

Part 3

+

Production concerns

+

What changes when the app stops being yours and starts being a URL strangers can open.

+
+ +
+ +

3.1Two modes, one codebase

+ +

AIDND_MULTI_USER switches the whole app between two personalities:

+ +
+ + + + + + + + + + +
Local (default)Hosted
UsersOne auto-created local userGuest on first visit, optional account
AuthNone — no cookies, no login UISigned session cookie
Rate limitsOffOn
Row capsOffOn
API docsOnOff
ProviderWhatever Settings points atUser’s key, or the shared demo key
+
+ +

The reasoning: someone running this on their own laptop should never be throttled by their own +app, never see a login screen, and should get the interactive API docs. A hosted deployment needs +all four to be the opposite. Rather than two builds, the differences are gated at each site.

+ +

Guests upgrade in place. A visitor gets a guest User row on first +load. Registering sets email and password_hash on that same row +— so every adventure they played as a guest survives with no re-parenting and no migration step. +Three kinds of row share the users table: local, guest, and registered.

+ +

Guests expire; accounts don't. One row per curious visitor adds up, so +cleanup.py deletes guests idle for AIDND_GUEST_RETENTION_DAYS +(default 5) — measured as COALESCE(last_seen_at, created_at), since +_touch only writes last_seen_at hourly and a freshly minted guest +has NULL until its second request. The filter requires both is_guest +and email IS NULL, so upgrading in place is also how you opt out of +expiry. It sweeps once at startup — the reliable trigger on a host that sleeps — and then +every few hours.

+ +

It's a single Core DELETE, not db.delete(user): the ORM path +would SELECT every adventure, action and memory into Python purely to delete them, and the +foreign keys are ON DELETE CASCADE from users all the way down, so +the database does the whole graph in one statement. Nothing a guest owns is visible to +anyone else either — is_public is output-only, so shared content is exactly the +seeded scenarios, which have user_id NULL and never match the filter.

+ +

3.2The shared demo key

+ +

The demo lets people play with no signup and no API key, on a key the server pays for. That is a +spending surface, so it’s the most defended code in the project.

+ +

One function makes the BYOK-vs-demo decision, and on the demo branch it pins two +things:

+ +
    +
  • The model — to a whitelist. A caller-supplied override or a hand-edited + settings row can’t aim a server-funded key at an expensive model. Anything unrecognised falls + back to the first whitelisted model.
    • +
  • The endpoint — to the configured demo URL. Otherwise the key could be + redirected to a URL the user controls and harvested.
    • +
+ +

Plus a daily per-user turn cap (default 20), checked before the player’s input is stored +so a capped player doesn’t get their message saved with no reply, and counted only after a +successful turn.

+ +
+ A real bug, recorded in a comment +

There’s a defensive check that raises if a demo config somehow carries a non-whitelisted + model. It tests using_demo, not + api_key == DEMO_API_KEY. Keying on the key value looks stricter but is wrong — the + demo key is an ordinary OpenRouter key, so a user can legitimately paste that same key into their + own settings as BYOK, and then every resolution raised, 500ing even GET /auth/me and + taking the whole SPA down. using_demo is what actually means “the server is paying”.

+
+ +

3.3Secrets

+ +

Everything derives from one server-side secret.

+ +
+ + + + + + + +
ThingMechanism
Passwordshashlib.scrypt, N=2¹⁴, r=8, p=1, per-password salt, constant-time compare. Stdlib, so no extra dependency.
Sessionsv1.<user_id>.<HMAC-SHA256>, no expiry — long-lived guest sessions are the point. A cookie can outlive a swept guest row; that resolves to a 401, which the frontend already turns into a fresh session.
Stored LLM API keysFernet encryption at rest, key derived from the secret, enc: prefix so legacy plaintext rows are recognisable and migratable.
+
+ +

The secret auto-generates into a file next to the database for local installs (zero config), but +multi-user mode refuses to start without the env var — with an error that explains +why and gives the command to generate one. Hosted filesystems are ephemeral; a regenerated secret +on every deploy would silently log out every user and orphan their stored API keys.

+ +

A rotated secret makes stored keys undecryptable. Decryption treats that as “unset” rather than +raising, so the user just re-enters their key instead of hitting a 500.

+ +

3.4Abuse guards

+ +
+ + + + + + + + + + + + +
GuardLimit
Turn generation10 / min
Auth attempts (per IP)10 / 5 min
Guest creation (per IP)30 / 5 min
Script test runs30 / min
Connection test10 / min
Adventures / scenarios / scripts per user100 / 200 / 200
Actions per adventure5,000
Request body2 MB (20 MB on import)
+
+ +

Rate limits are keyed per user when one is known — accounts survive IP changes — and per IP +otherwise, in fixed windows held in memory, with a pruning pass so the per-IP dict can’t grow +without bound. Import endpoints check bundle list lengths against the same caps live creation +enforces, otherwise the cap is trivially bypassed by uploading a file.

+ +

Security headers on every response: nosniff, X-Frame-Options: DENY, +Referrer-Policy: same-origin, and a CSP allowing exactly what the SPA uses.

+ +

3.5Deployment

+ +

One Docker web service on Render, serving the SPA and the API same-origin, with Postgres on Neon.

+ +

The Postgres decision was forced: Render’s free tier has no persistent disk, so a SQLite file +wouldn’t survive a deploy. The database lives off-box.

+ +

Two things worth knowing about the free tier:

+ +
    +
  • The service sleeps after ~15 minutes idle, and the first request then takes + 30–60 seconds.
    • +
  • /api/health deliberately doesn’t touch the database, so a + keep-warm pinger wakes the web service without waking the database. Waking a database around the + clock costs far more than the cold start is worth.
    • +
+ +

CI runs the backend tests, the frontend lint and build, and a Docker image build on every push.

+ +
+ + +
+

Part 4

+

The web plumbing, briefly

+

The parts that are just how the web works, not decisions.

+
+ +
+ +

Frontend and backend are two programs. In development they’re two servers — +Vite on 5173 serving React, FastAPI on 8000 serving the API — and Vite proxies /api to +FastAPI so the browser thinks it’s all one origin, which avoids CORS entirely. In production +there’s one server: FastAPI serves the built React files as static assets from the same port.

+ +

SPA routing. React Router handles URLs like /play/3 in the browser +without a round trip. But if you reload that URL, the browser asks the server for +/play/3, which isn’t a file. So the static-file handler catches the 404 and returns +index.html, letting React take over and read the URL itself. API routes are matched +before the static mount, so they’re unaffected.

+ +

Sessions. A cookie is a small value the browser stores and automatically +attaches to every request to that site. Here it holds v1.<user_id>.<signature>. +The server doesn’t store sessions anywhere — it re-verifies the signature on each request, which is +why there’s no session table.

+ +

The 401 retry. If the cookie is missing or stale, any API call returns 401. The +frontend catches that once, calls /api/auth/me — which mints a fresh guest session — +and retries the original request. So a returning visitor with an expired cookie never sees an +error.

+ +

React, in one paragraph. A component is a function that returns a description +of some UI. useState holds a value; changing it re-renders the component. The +streaming turn is the clearest example: each SSE chunk appends to a state string, React re-renders, +and the text appears to type itself.

+ +
+ + +
+

Part 5

+

Results and limitations

+

What was measured, and what this design knowingly does not do.

+
+ +
+ +

5.1Measured results

+ +
+ + + + + + + + + + + + + +
Database egress per adventure load38.5 MB → 0.20 MB (~189×)
Prompt snapshot size~74 KB/turn, 94% of the DB
Turn read cost at turn 200839 KB → 129 KB, flat after ~turn 50
Cost of a branch~103 B; 20 forks load at 1.007× the same story flat
Length-hint phrasing174 → 246 words as a budget; 170 as a ceiling (n=5)
Backend tests440, LLM mocked, real QuickJS engine
Schema versions64
Sandbox limits16 MB, 2 s CPU, fresh context per run
Context defaultsauthor’s note at depth 3; cards ≤ 40% of elastic budget
Memory cadencememory / 6 turns, summary / 15 turns, top-5 retrieval
+
+ +

Two of the tests encode a performance property rather than a behaviour: +test_egress.py asserts on the SQL the ORM emits, and +test_history_window.py asserts that the read cost stops growing with story +length.

+ +

5.2Known limitations

+ +

Deliberate trades for a single-user-first app that also happens to be hosted, listed so +nobody has to discover them the hard way.

+ +
    +
  • Single process. The turn lock, the rate limiter and the summarization task + all assume one worker. A second worker would need the lock in the database — a row-level + advisory lock — and the rate limiter in Redis.
    • +
  • No vector index. Retrieval does cosine similarity in Python over the whole + bank. Fine at the 200-memory cap; at 10,000 it would want pgvector.
    • +
  • Prompt snapshots are heavy even after the egress fix — they’re deferred, not + smaller. Compressing them or expiring old ones is the real fix.
    • +
  • In-memory rate-limit windows reset on restart, so a restart grants a brief + extra allowance.
    • +
  • Background summarization is a fire-and-forget asyncio task, so it does not + survive a restart. At real load it belongs in a queue.
    • +
  • The demo key depends on a free-tier provider’s daily cap, which the app can + only detect after the fact by string-matching the 429 body.
    • +
  • The two memory marks are one pair on the adventure, not one per branch. + Switching lines makes the mark on the line being left unreadable from the new one, so that ground + is summarized again. It fails in the safe direction — redo, never skip — but switching back and + forth costs AI calls.
    • +
  • Story cards are adventure-wide, so a card invented on one branch shows on all + of them.
    • +
  • Editing an already-summarized turn leaves its memory stale. Replacing a turn + withdraws what was derived from it; editing one in place does not.
    • +
+ +

5.3Cleanup backlog

+ +

docs/self-review.md carries an open list of non-bugs — reuse, simplification and +efficiency items — kept deliberately separate from the correctness list, which is empty. The +largest ones:

+ +
    +
  • Section.tokens is uncached, so the context gets tokenized two or three times a + turn.
    • +
  • onModelContext flattens system and story into one string before handing it to + user scripts; if a script modifies it, the structure is gone and everything ships as user + content. Passing structure through the hook would be better but would break AI Dungeon + compatibility, which is the point of the feature.
    • +
  • The import endpoints hand-coerce raw dicts instead of using Pydantic bundle schemas.
    • +
  • The legacy pre-tree columns (index, variants, + variant_index, and the two *_before snapshots) are still on + actions, unread, kept for one release so redeploying the previous build remains a way + out. Dropping them is a migration that rewrites every row, so it owes a + VACUUM FULL actions; after it.
    • +
+ +
+ +
+
+

AI D&D — design notes · + Source on GitHub · + Project page

+

Full source for every claim here is in the repo; the file paths are named inline.

+
+
+ +
+ + + + + diff --git a/docs/images/branch-map.jpg b/docs/images/branch-map.jpg new file mode 100644 index 0000000..369f801 Binary files /dev/null and b/docs/images/branch-map.jpg differ diff --git a/docs/images/branches-panel.jpg b/docs/images/branches-panel.jpg new file mode 100644 index 0000000..119de2c Binary files /dev/null and b/docs/images/branches-panel.jpg differ diff --git a/docs/images/home.jpg b/docs/images/home.jpg new file mode 100644 index 0000000..da3ab8f Binary files /dev/null and b/docs/images/home.jpg differ diff --git a/docs/images/insights.jpg b/docs/images/insights.jpg new file mode 100644 index 0000000..a18b69e Binary files /dev/null and b/docs/images/insights.jpg differ diff --git a/docs/images/play-world-state.jpg b/docs/images/play-world-state.jpg new file mode 100644 index 0000000..6185183 Binary files /dev/null and b/docs/images/play-world-state.jpg differ diff --git a/docs/images/scenario-editor-npcs.jpg b/docs/images/scenario-editor-npcs.jpg new file mode 100644 index 0000000..cd85ae4 Binary files /dev/null and b/docs/images/scenario-editor-npcs.jpg differ diff --git a/docs/images/script-editor.jpg b/docs/images/script-editor.jpg new file mode 100644 index 0000000..8931e1c Binary files /dev/null and b/docs/images/script-editor.jpg differ diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 0000000..0c9d168 --- /dev/null +++ b/docs/index.html @@ -0,0 +1,267 @@ + + + + + +AI D&D — an AI Dungeon-style storytelling engine + + + + + + + + + + + + + + +
+ +
+
⚔ Interactive fiction, refereed
+

AI D&D

+

Open-ended adventures narrated by an LLM — with an engine that keeps the numbers honest.

+

Create a world, play it in second person, and let the model improvise the story while a Python + referee enforces what's actually true: hit points, an ally's trust, a raised alarm, a quest milestone. + Bring your own model, or play the demo with none.

+ +
+ Launch the live demo → + Read the design notes + HLD & LLD walkthrough + View the source +
+

No sign-up, no API key. Hosted on a free tier that sleeps — the first load takes ~30–60s to wake.

+ +
+ The play screen with the world-state rail open, showing HP, mana, an NPC's trust and a raised alarm flag +
The left rail is live world state. The model proposes what changed this turn; the engine decides + what sticks, and the chip under the narration reports the result. The ‹ 2/2 › + under a turn steps between the takes it has.
+
+
+ +
+

What makes it more than a chat wrapper

+

Four things a plain "talk to a model" app doesn't do.

+ +
+
+

The story is a tree

+

Any turn can hold more than one take. Stepping between them is free — the story below simply + empties, and the server is told nothing. Writing below a take that isn't the live one is what makes a + branch, and a branch stores no turns of its own: it records where it left its parent and borrows + everything above that. Twenty forks cost 1.007× the page load of the same story flat. Switch lines and + the world state, the script scoreboard and the cooldown clocks all come back to what that line left.

+ The branch map: one horizontal lane per line of the story, each joined to its parent by an elbow at the moment it forked +
+ +
+

The AI proposes, Python referees

+

A scenario declares stats, flags, milestones and a named cast. Each turn the model appends the changes + it thinks happened — and the engine clamps them to range, enforces per-turn caps and cooldowns, keeps + counters monotonic and milestones sticky, then strips the machine-readable block out of the prose. + Word-labelled bands (40–60: minor damage) are what make the model reliable at it. + No dice, no scripting required.

+ The scenario editor showing NPC stats with ranges, per-turn caps, cooldowns and labelled bands +
+ +
+

You can see the entire prompt

+

Every turn stores exactly what was sent to the model. Open Insights on any action to see each context + component, what it cost in tokens, and why it was there — including which trigger word pulled in each + story card and the similarity score behind each retrieved memory.

+ The Insights panel showing the assembled prompt broken into components with token counts +
+ +
+

Real AI Dungeon scripts run

+

The three familiar hooks — onInput, onModelContext, onOutput — + with shared persistent state and a worldEntries API, executed in an embedded + quickjs sandbox. Scripts written for AI Dungeon import and work, and there's a CodeMirror editor in the app.

+ The in-app script editor showing an input hook written in JavaScript +
+ +
+

Memory that survives a long story

+

The modern AI Dungeon memory system: AI-generated memories every few actions, a running story summary, + and embedding-based retrieval that pulls an old-but-relevant fact back into context when it matters. + Undo and retry roll the world state back to a per-action snapshot rather than only rewriting the text. + Every story stays where you left it, and the home page opens on its most recent line.

+ The home page, showing stories in progress alongside scenarios to start from +
+
+
+ +
+

How a turn works

+

Player input goes through the script pipeline, into a token-budgeted context, out to whichever + model you configured, and back through the referee.

+
player input
+  → onInput script modifier
+  → assemble context:  [narrator prompt] + [world state + stat guide] + [AI instructions]
+                       + [plot essentials] + [story summary] + [retrieved memories]
+                       + [triggered story cards] + [history along this branch, token-budgeted]
+                       + [author's note] + [player action]
+  → onModelContext script modifier
+  → snapshot context (Insights)
+  → provider adapter → AI (streamed)
+  → extract + referee the world-state delta block, strip it from the prose
+  → onOutput script modifier
+  → store & render
+ +
+ FastAPI + SQLAlchemy + React + Vite + Postgres / SQLite + quickjs sandbox + Server-sent events + Docker + Any OpenAI-compatible endpoint +
+
+ +
+

Engineering notes

+

The parts that were measured rather than guessed at.

+ +
+
189×
less database egress per adventure load
+
440
backend tests, run by CI on every push
+
64
schema migrations, applied in order on boot
+
$0
to run it locally against Ollama
+
+ +
    +
  • Database egress, cut ~189×. Every adventure load was pulling the entire assembled + prompt — about 74 KB per turn — just to read two small fields off it. Moving those into their own columns + and deferring the heavy ones took one load from 38.5 MB to 0.20 MB. A test hooks into SQLAlchemy's cursor + events and fails if a bulk load ever names those columns again.
    • +
  • Turn cost, made flat. Assembling a turn walked the whole story, so it grew with story + length — 839 KB of reads by turn 200. History is now served as tails and slices from SQL: the same turn + costs 129 KB and stops growing at around turn 50.
    • +
  • Branching that costs 103 bytes. A branch stores where it left its parent and borrows + every turn above that, so nothing is copied on a fork. A 40-turn story forked twenty times loads in + 31,652 B against 31,433 B for the same story flat — 1.007×. Reads stay cheap because the ancestry is + windowed the way the history is: the number of SQL clauses is bounded by the context window, not by how + many times the story has forked.
    • +
  • A shared demo key that can't be drained. The hosted demo funds a model for visitors, so + model selection is pinned server-side with a structural backstop that raises if any code path tries to + resolve a model outside the allowed set — plus a daily per-visitor turn cap.
    • +
+
+ + + +
+ + diff --git a/docs/self-review.md b/docs/self-review.md new file mode 100644 index 0000000..2e09a02 --- /dev/null +++ b/docs/self-review.md @@ -0,0 +1,156 @@ +# Self-review log + +**Every correctness bug on this page is resolved.** 20 are fixed, and 1 is intentionally +skipped, with the reasoning recorded below. The record is kept because the reasoning outlives +the verdicts, and several of these are traps worth remembering. The *cleanup backlog* at the +bottom is a deliberately open list of non-bugs (reuse, simplification, efficiency), not +outstanding defects. + +Original review: 2026-07-05, whole project in scope (no git history at the time). +Status key: `fixed` means applied, `skipped` means intentionally not fixed, and `pending` means +outstanding (none remain). + +**2026-07-06 update (branch `bugfix-code-review`):** every finding was re-verified against +current code. #1, #2, #3, #4, #6, and #12 had already been fixed in earlier sessions, so their +statuses were stale. #5, #7, #8, #9, #10, #13, #14 (backend) and #15 through #21 (frontend) +were fixed in this pass. #11 is skipped: real AI Dungeon's `addStoryCard` also returns the new +card's index (0-falsy included) per the official scripting guidebook, so changing it would +break compatibility. This is documented in `engine.py`'s prelude instead. The cleanup backlog +(R/S/E/A items) below remains open by design. + +## Correctness bugs + +### 1. [fixed] seed_demo.py doesn't stamp schema version → server crashes on next start +- `backend/seed_demo.py:10` +- A fresh DB created via `Base.metadata.create_all` leaves `PRAGMA user_version` at 0. The next server start sees the tables exist and replays every ALTER TABLE migration, causing a `duplicate column name` crash. +- Fix: stamp user_version to latest after create_all (reuse migrations.bootstrap logic). + +### 2. [fixed] Turn-lock race: two simultaneous turns can run on the same adventure +- `backend/app/routers/adventures.py:302` +- `ensure_not_generating()` runs in the route handler, but `_active_turns.add()` only happens when the StreamingResponse generator is first iterated. Double-clicking Continue sends both requests past the 409 check, producing duplicate Action.index rows and interleaved generations. +- Fix: atomically test-and-set the lock in the request phase, release in the stream's `finally`. + +### 3. [fixed] Migration 10 renumbers indexes with a correlated subquery on the table being updated +- `backend/app/migrations.py:34` +- SQLite may evaluate the subquery against partially updated rows, so duplicate indexes can survive the "repair". +- Fix: compute new indexes in Python (SELECT ordered, then UPDATE per row). + +### 4. [fixed] SQLite foreign keys never enabled → CASCADE/SET NULL clauses are dead +- `backend/app/database.py:8` +- Deleting a Script leaves orphaned `scenario_scripts` rows. SQLite rowid reuse can then attach a future script to an old scenario. +- Fix: `PRAGMA foreign_keys=ON` via engine connect event. + +### 5. [fixed] Provider generate() silently yields nothing for non-SSE 200 responses +- `backend/app/providers/openai_compatible.py:84` +- A server that ignores `stream=true` and returns plain JSON produces no `data:` lines, so the result is an empty AI action with no error. +- Fix: buffer the non-SSE body and fall back to parsing it as a single JSON completion. + +### 6. [fixed] Fire-and-forget asyncio task can be GC'd mid-run and wedge the memory bank +- `backend/app/memorybank.py:146` +- The result of `asyncio.create_task` is not referenced, so the task can vanish silently and leave an adventure ID stuck in `_running`. +- Fix: keep strong refs in a set, discard in done-callback. + +### 7. [fixed] Memory cursors are list positions but Memory.source_start/end are Action.index values +- `backend/app/memorybank.py:213` +- After any action deletion, indexes keep gaps and positions shift, so summarization skips or duplicates blocks and `_update_story_summary` folds the wrong memories. +- Fix: use one space consistently. Track cursors by Action.index (position-independent), or renumber on delete. + +### 8. [fixed] Pinned memories don't count toward memory_top_k cap +- `backend/app/memorybank.py:120` +- 6 pinned plus top_k=5 injects 11 memories, blowing the token budget. +- Fix: fill with unpinned only up to `top_k - len(pinned)` (min 0). + +### 9. [fixed] Embedding-model change → cosine() zips different-dimension vectors silently +- `backend/app/memorybank.py:69` +- Old 768-dimension embeddings scored against a new 1536-dimension query produce garbage similarity with no error, and are never re-embedded. +- Fix: return 0.0 on length mismatch (and ideally clear stale embeddings so _embed_pending redoes them). + +### 10. [fixed] MAX_STORY_CARDS cap is a no-op for cards created in one hook +- `backend/app/scripting/pipeline.py:59` +- `len(existing) + len(seen_ids) < MAX...` never counts newly added cards, since seen_ids is a subset of existing. A script can insert an unbounded number of cards in one turn. +- Fix: count inserts made during the loop. + +### 11. [skipped] addStoryCard returns 0 (falsy) for the first card, indistinguishable from `false` rejection +- `backend/app/scripting/engine.py:39` +- `if (!addStoryCard(...))` misfires when the card list was empty. +- Fix: return `storyCards.length` (1-based, always truthy) or `true`; document. + +### 12. [fixed] scenario_id=0 truthiness bug in create_story_card +- `backend/app/routers/story_cards.py:28` +- `scenario_id or ...` picks the wrong owner when id is 0. Use `is not None`. + +### 13. [fixed] test_connection 500s on non-dict JSON from /models +- `backend/app/routers/settings.py:54` +- Only ValueError is caught. `data.get`/`m.get` on non-dict input raises AttributeError, producing a 500 instead of `{ok:false}`. +- Fix: catch (ValueError, AttributeError, TypeError) or validate shapes. + +### 14. [fixed] AI Dungeon exports with `worldInformation` key lose all story cards silently +- `backend/app/routers/scenarios.py:133` +- Import reads only `storyCards`/`worldInfo`. `worldInformation` is in _IGNORED_KEYS, so it is dropped without being reported. +- Fix: accept `worldInformation` as a card source too. + +### 15. [fixed] Shared debounce timer loses edits (Play PlotPanel) +- `frontend/src/pages/Play.jsx:28` +- One `saveTimer` is shared by all plot fields and story-card saves. Editing a second field within 600ms cancels the first pending PATCH, causing silent data loss. +- Fix: per-key timers (e.g. a Map keyed by field/card id). + +### 16. [fixed] Same shared-debounce data loss in ScenarioEditor +- `frontend/src/pages/ScenarioEditor.jsx:22` +- Same fix as #15. + +### 17. [fixed] Continue button silently discards typed input text +- `frontend/src/pages/Play.jsx:469` +- Clicking Continue with text in the box sends type 'continue' (the backend ignores the text) and clears the input. +- Fix: don't clear input on continue (or treat non-empty input as a normal send). + +### 18. [fixed] retry() optimistically deletes last AI action with no rollback on failure +- `frontend/src/pages/Play.jsx:475` +- A failed retry (409 or network error) leaves the UI missing an action that still exists server-side. +- Fix: restore the removed action in the catch path (or only remove on first stream event). + +### 19. [fixed] Settings test()/save() have no error handling → stuck on "Testing…" +- `frontend/src/pages/Settings.jsx:66` +- A rejection leaves `{pending:true}` forever and produces an unhandled rejection. +- Fix: try/catch → setTestResult({ok:false, error:msg}). + +### 20. [fixed] InsightsPanel race: slow earlier request overwrites newer report +- `frontend/src/pages/Play.jsx:302` +- There is no staleness guard, so a slow getAdventureContext call can overwrite a newer action snapshot. +- Fix: track a request id or cancelled flag in the effect. + +### 21. [fixed] extractPlaceholders ignores ${...} in story-card trigger keys +- `frontend/src/pages/Scenarios.jsx:50` +- The backend fills placeholders in card.keys, but the modal never prompts for those names, so a literal `${hero}` key never matches. +- Fix: also scan card.keys when collecting placeholder names. + +## Cleanup backlog (reuse / simplification / efficiency / altitude — not bugs, apply later) + +- **R1** ~~three copies of the debounced-autosave handler.~~ + **Half applied in phase 17 (2026-08).** All three use + `frontend/src/hooks/useDebouncedSave.js`. A shared StoryCardList component is + still open. +- **R2** `backend/seed_demo.py:228`: re-implements create_adventure. Call the router logic instead. +- **R3** `backend/app/providers/openai_compatible.py:122`: complete() duplicates _request()'s body building. Add a `stream` param to _request(). +- **R4** ~~six copies of child-resource get, owner-check, and 404.~~ + **Applied in phase 17 (2026-08).** All 32 handlers take the `current_adventure` + dependency from `routers/adventures/deps.py`. +- **R5** `frontend/src/api.js:26`: streamSSE duplicates request()'s error extraction. Extract `throwIfNotOk(resp)`. +- **S1** ~~`Settings.stream` is dead state (never read).~~ + **Applied in phase 17 (2026-08).** The column, both schema fields, and migration 65 + drop it. `R4` went with it: the ownership check is the `current_adventure` dependency. +- **S2** `frontend/src/pages/Play.jsx:6`: MODES and PLAYER_TYPES are identical constants; lastIsAi/canUndo are computed twice. +- **E1** `backend/app/context/builder.py:119`: joins and tokenizes the entire adventure history every turn for the trigger window. Walk reversed(actions) until budget instead. +- **E2** `backend/app/scripting/pipeline.py:77`: rebuilds full history dicts, JSON, and a blocking commit per script per hook. Build once per hook, slice to HISTORY_WINDOW first, and commit once. +- **E3** `frontend/src/pages/Play.jsx:561`: every SSE chunk re-renders all action rows. Isolate streaming text in a child component or React.memo rows. +- **E4** `backend/app/context/builder.py:48`: Section.tokens is uncached, so the whole context gets tokenized 2 to 3 times per turn. Cache counts and sum sections. +- **E5** `frontend/src/pages/Play.jsx:490`: the keydown effect has no dependency array, so the listener is re-registered every render. +- **E6** `backend/app/memorybank.py:182`: catch-up summarization awaits blocks sequentially. Gather independent blocks instead. +- **A1** ~~no UniqueConstraint('adventure_id','index'); index allocation is ad-hoc per writer.~~ + **Overtaken by phase 14 (2026-08).** `index` is a legacy column that nothing reads: ordering + is `(branch_id, depth)` now, allocated in one place (`tree.place_action`). The column is kept + unread for one release and then dropped, so a constraint on it would apply to a column that + no longer does anything. +- **A2** `backend/app/providers/openai_compatible.py:45`: CHAT_CONTINUE_HINT is appended below the budgeting layer. Assemble prompts in the context builder instead. +- **A3** `backend/app/routers/adventures.py:390`: import endpoints hand-coerce raw dicts. Use a Pydantic bundle schema. +- **A4** `backend/app/routers/adventures.py:207`: onModelContext flattens (system, story) and ships everything as user content if modified. Pass structure through the hook instead. +- **A5** `frontend/src/pages/Home.jsx:87`: the client appends 'Z' to naive datetimes. Emit ISO-8601 with an offset from the API instead. diff --git a/frontend/.gitignore b/frontend/.gitignore new file mode 100644 index 0000000..a547bf3 --- /dev/null +++ b/frontend/.gitignore @@ -0,0 +1,24 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? diff --git a/frontend/.oxlintrc.json b/frontend/.oxlintrc.json new file mode 100644 index 0000000..1255078 --- /dev/null +++ b/frontend/.oxlintrc.json @@ -0,0 +1,8 @@ +{ + "$schema": "./node_modules/oxlint/configuration_schema.json", + "plugins": ["react", "oxc"], + "rules": { + "react/rules-of-hooks": "error", + "react/only-export-components": ["warn", { "allowConstantExport": true }] + } +} diff --git a/frontend/README.md b/frontend/README.md new file mode 100644 index 0000000..be50862 --- /dev/null +++ b/frontend/README.md @@ -0,0 +1,16 @@ +# React + Vite + +This template provides a minimal setup to get React working in Vite with HMR and some Oxlint rules. + +Currently, two official plugins are available: + +- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react) uses [Oxc](https://oxc.rs) +- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react-swc) uses [SWC](https://swc.rs/) + +## React Compiler + +The React Compiler is not enabled on this template because of its impact on dev & build performances. To add it, see [this documentation](https://react.dev/learn/react-compiler/installation). + +## Expanding the Oxlint configuration + +If you are developing a production application, use TypeScript with type-aware lint rules enabled. See the [TS template](https://github.com/vitejs/vite/tree/main/packages/create-vite/template-react-ts) for how to integrate TypeScript and Oxlint's TypeScript-related rules in your project. diff --git a/frontend/index.html b/frontend/index.html new file mode 100644 index 0000000..0134879 --- /dev/null +++ b/frontend/index.html @@ -0,0 +1,26 @@ + + + + + + + + + + + AI D&D + + +
+ + + diff --git a/frontend/package-lock.json b/frontend/package-lock.json new file mode 100644 index 0000000..68ac0b7 --- /dev/null +++ b/frontend/package-lock.json @@ -0,0 +1,1662 @@ +{ + "name": "frontend", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "frontend", + "version": "0.0.0", + "dependencies": { + "@codemirror/lang-javascript": "^6.2.5", + "@codemirror/theme-one-dark": "^6.1.3", + "@uiw/react-codemirror": "^4.25.10", + "react": "^19.2.7", + "react-dom": "^19.2.7", + "react-router-dom": "^7.18.1" + }, + "devDependencies": { + "@types/react": "^19.2.17", + "@types/react-dom": "^19.2.3", + "@vitejs/plugin-react": "^6.0.3", + "oxlint": "^1.71.0", + "vite": "^8.1.1" + } + }, + "node_modules/@babel/runtime": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.7.tgz", + "integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@codemirror/autocomplete": { + "version": "6.20.3", + "resolved": "https://registry.npmjs.org/@codemirror/autocomplete/-/autocomplete-6.20.3.tgz", + "integrity": "sha512-tlosUqb+3BbxCxZdu4tKeRghPFC+QM7q4X5YhKV2eCmPG+1r2F3f4AaSz5sCrFqUtX4Jh20VFTKecl16MgiV9g==", + "license": "MIT", + "dependencies": { + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.17.0", + "@lezer/common": "^1.0.0" + } + }, + "node_modules/@codemirror/commands": { + "version": "6.10.4", + "resolved": "https://registry.npmjs.org/@codemirror/commands/-/commands-6.10.4.tgz", + "integrity": "sha512-Ryk9y9T0FFVF0cUGhAknveAyUOl/A1qReTFi+qPKtOh2Z9F4AUBz3XOrYD4ZEgZirdugVzHvd/2/Wcwy5OliTg==", + "license": "MIT", + "dependencies": { + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.7.0", + "@codemirror/view": "^6.27.0", + "@lezer/common": "^1.1.0" + } + }, + "node_modules/@codemirror/lang-javascript": { + "version": "6.2.5", + "resolved": "https://registry.npmjs.org/@codemirror/lang-javascript/-/lang-javascript-6.2.5.tgz", + "integrity": "sha512-zD4e5mS+50htS7F+TYjBPsiIFGanfVqg4HyUz6WNFikgOPf2BgKlx+TQedI1w6n/IqRBVBbBWmGFdLB/7uxO4A==", + "license": "MIT", + "dependencies": { + "@codemirror/autocomplete": "^6.0.0", + "@codemirror/language": "^6.6.0", + "@codemirror/lint": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.17.0", + "@lezer/common": "^1.0.0", + "@lezer/javascript": "^1.0.0" + } + }, + "node_modules/@codemirror/language": { + "version": "6.12.4", + "resolved": "https://registry.npmjs.org/@codemirror/language/-/language-6.12.4.tgz", + "integrity": "sha512-1q4PaT+o6PbgpkJt4Q8Fv5XJxTy4FUZ4MWETtyiDw3J0Pyr9E2vqcKL+k9wcvjNTIsauxvE7OfmWj3FRPHQ76A==", + "license": "MIT", + "dependencies": { + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.23.0", + "@lezer/common": "^1.5.0", + "@lezer/highlight": "^1.0.0", + "@lezer/lr": "^1.0.0", + "style-mod": "^4.0.0" + } + }, + "node_modules/@codemirror/lint": { + "version": "6.9.7", + "resolved": "https://registry.npmjs.org/@codemirror/lint/-/lint-6.9.7.tgz", + "integrity": "sha512-28/+iWLYxKxsvGYhSYL7zaCZqLz5+FFFDq9tVsvGv9kv8RY4fFAchJ5WX9M3YrrRlTIsECjsXPqeNgnSmNP2dg==", + "license": "MIT", + "dependencies": { + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.42.0", + "crelt": "^1.0.5" + } + }, + "node_modules/@codemirror/search": { + "version": "6.7.1", + "resolved": "https://registry.npmjs.org/@codemirror/search/-/search-6.7.1.tgz", + "integrity": "sha512-uMe5UO6PamJtSHrXhhHOzSX3ReWtiJrva6GnPMwSOrZtiExb5X5eExhr2OUZQVvdxPsKpY3Ro2mFbQadpPWmHA==", + "license": "MIT", + "dependencies": { + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.37.0", + "crelt": "^1.0.5" + } + }, + "node_modules/@codemirror/state": { + "version": "6.7.0", + "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.7.0.tgz", + "integrity": "sha512-Zbl9NyscLMZkfXPQnNAIIAFftidrA1UbcJEIMp24C0Bukc2I5T8wJS0wsXYsnDOqCFJUeJ1BITGNs5CqPDSmSg==", + "license": "MIT", + "dependencies": { + "@marijn/find-cluster-break": "^1.0.0" + } + }, + "node_modules/@codemirror/theme-one-dark": { + "version": "6.1.3", + "resolved": "https://registry.npmjs.org/@codemirror/theme-one-dark/-/theme-one-dark-6.1.3.tgz", + "integrity": "sha512-NzBdIvEJmx6fjeremiGp3t/okrLPYT0d9orIc7AFun8oZcRk58aejkqhv6spnz4MLAevrKNPMQYXEWMg4s+sKA==", + "license": "MIT", + "dependencies": { + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0", + "@lezer/highlight": "^1.0.0" + } + }, + "node_modules/@codemirror/view": { + "version": "6.43.5", + "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.43.5.tgz", + "integrity": "sha512-7uT/vUgH6dfXWn3WqOe23KneILMvGy5wQjNMEcRXLKzziJ9NOktpW6tGoyQpwVkBgE5Gj6hKkCcsddbnkaWrOQ==", + "license": "MIT", + "dependencies": { + "@codemirror/state": "^6.7.0", + "crelt": "^1.0.6", + "style-mod": "^4.1.0", + "w3c-keyname": "^2.2.4" + } + }, + "node_modules/@emnapi/core": { + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.2.tgz", + "integrity": "sha512-TC8MkTuZUtcTSiFeuC0ksCh9QIJ5+F21MvZ4Wn4ORfYaFJ/0dsiudv5tVkejgwZlwQ39jL9WWDe2lz8x0WglOA==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.2", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.2.tgz", + "integrity": "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", + "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@lezer/common": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/@lezer/common/-/common-1.5.2.tgz", + "integrity": "sha512-sxQE460fPZyU3sdc8lafxiPwJHBzZRy/udNFynGQky1SePYBdhkBl1kOagA9uT3pxR8K09bOrmTUqA9wb/PjSQ==", + "license": "MIT" + }, + "node_modules/@lezer/highlight": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-1.2.3.tgz", + "integrity": "sha512-qXdH7UqTvGfdVBINrgKhDsVTJTxactNNxLk7+UMwZhU13lMHaOBlJe9Vqp907ya56Y3+ed2tlqzys7jDkTmW0g==", + "license": "MIT", + "dependencies": { + "@lezer/common": "^1.3.0" + } + }, + "node_modules/@lezer/javascript": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/@lezer/javascript/-/javascript-1.5.4.tgz", + "integrity": "sha512-vvYx3MhWqeZtGPwDStM2dwgljd5smolYD2lR2UyFcHfxbBQebqx8yjmFmxtJ/E6nN6u1D9srOiVWm3Rb4tmcUA==", + "license": "MIT", + "dependencies": { + "@lezer/common": "^1.2.0", + "@lezer/highlight": "^1.1.3", + "@lezer/lr": "^1.3.0" + } + }, + "node_modules/@lezer/lr": { + "version": "1.4.10", + "resolved": "https://registry.npmjs.org/@lezer/lr/-/lr-1.4.10.tgz", + "integrity": "sha512-rnCpTIBafOx4mRp43xOxDJbFipJm/c0cia/V5TiGlhmMa+wsSdoGmUN3w5Bqrks/09Q/D4tNAmWaT8p6NRi77A==", + "license": "MIT", + "dependencies": { + "@lezer/common": "^1.0.0" + } + }, + "node_modules/@marijn/find-cluster-break": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@marijn/find-cluster-break/-/find-cluster-break-1.0.3.tgz", + "integrity": "sha512-FY+MKLBoTsLNJF/eLWaOsXGdz6uh3Iu1axjPf6TUq92IYumcTcXWHoS747JARLkcdlJ/Waiaxc5wQfFO8jC6NA==", + "license": "MIT" + }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.6.tgz", + "integrity": "sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@tybys/wasm-util": "^0.10.3" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.138.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.138.0.tgz", + "integrity": "sha512-1a7ZKmrRTCoN1XMZ4L0PyyqrMnrNlLyPuOkdSX2MZg7IiIGRUyurNhAm73ptDOraoBcIordsIGKNPKUzy3ZmfA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, + "node_modules/@oxlint/binding-android-arm-eabi": { + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm-eabi/-/binding-android-arm-eabi-1.72.0.tgz", + "integrity": "sha512-zhCmvn+1Mj3UchAc/90i99S0t7jJUsHmFVSPg4UWrjO8b8eaSGwscgO6QAUtvHBstkjQwBttQNswEnAF1mIQdA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-android-arm64": { + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm64/-/binding-android-arm64-1.72.0.tgz", + "integrity": "sha512-mtH+aY/ozv1eZoCUC2owjFAtyNBKHpJHygKeEu9zXXnQGW1Q2/qOpvx+I+Lf23+TvTz66F4iiXUbl2cGvoLPCQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-darwin-arm64": { + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-arm64/-/binding-darwin-arm64-1.72.0.tgz", + "integrity": "sha512-EvnajNPDtfknB3ZieeOOyDTwJn9QXDiwfnF4ZDQqART6RG6hjY4WigQcZdGoK2dkB3e1vrmEzN9aYbQCUkh/gQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-darwin-x64": { + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-x64/-/binding-darwin-x64-1.72.0.tgz", + "integrity": "sha512-ZkCdEa/G80A7vEHfeCDz/+L3m33DE73v32mDKhgOIgz8Uwf0DFcK7+uu6qC+7LEhmz5fpOe1osWKyjSNMydFIQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-freebsd-x64": { + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-freebsd-x64/-/binding-freebsd-x64-1.72.0.tgz", + "integrity": "sha512-NroXv2vh+sxVY1uya/rM5pjhx1hm8BzlYpx9q67QP0Xhw5MH2bf5GJylpvLEC+781p1Xli/317EoV9AlGwViag==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm-gnueabihf": { + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.72.0.tgz", + "integrity": "sha512-0NDywYgfj279Ou/BcQuCYSj7NJwBfmWn5qc5uGO/Ny7fUWmXyIpvawqX/8acQlWG6IXelJsJhj+JAy6sjsKj0A==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm-musleabihf": { + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-1.72.0.tgz", + "integrity": "sha512-4vpXB06h65Ezsy4hRyrGjGrfa1SkVPii09yaajiYhmVpgsFiLD+KNxIx/BNAY+XiO+i1yqp9HHdwqM8VTqa5XQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm64-gnu": { + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.72.0.tgz", + "integrity": "sha512-immaN4g2ZGFiOkKrvRX9LvzZdd2GkQM5wR+UyzYyUuyhUTXGQ4HKUJH18xp4G8OfhCVaVAJfKZxwE1r8+4hhaQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm64-musl": { + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.72.0.tgz", + "integrity": "sha512-JGHS9Mnr7iWyyLDxgCv1MhzVpAckgptg00F2gnxt/GD7lQ2SW1BRcxHqhSTaSdDpjWRrBkBxMMh4+Hn3aVtExg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-ppc64-gnu": { + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.72.0.tgz", + "integrity": "sha512-AOYgBZqxNshrg83P9v0RYv+m8s10Cqkj4/PxXFDhcS3k7FqsIG5+CxErshZCIN7G8iy4Y+VGfAsuEdar8AcbBg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-riscv64-gnu": { + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-1.72.0.tgz", + "integrity": "sha512-QMybPS5ij3/vrKG67mqzHwW++91sYxK/PPUVi6SBtNCEzW4niS52fVBdXbQ6nou0wWbUPEpx8Sl/ZjtgE3clXA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-riscv64-musl": { + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-1.72.0.tgz", + "integrity": "sha512-gOc3W7JV0PXRpIL7stUlLe3Wa9Gp0Kdlup87IT3gHDvPKck2xNgMIl/Gs2lldYY2lyXZDC4rWi3hmoLUobkgbQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-s390x-gnu": { + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.72.0.tgz", + "integrity": "sha512-rpGxph+FjjHcYI5q6uxB3Az+tnfmEnDbSA8+PK9ZE/VzyUAkvBOMeuY7ZQMhu5mpZH7YQDsTdW6Cx4kV/msc6w==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-x64-gnu": { + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.72.0.tgz", + "integrity": "sha512-WND+uhf/Ko13SLqQMWQUgsZuLvYYEvL0ZKgg0tgGYfLqxG7l8Ju123fHDMJyYSDl5E3bUbpFUuii/OvMreFQzw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-x64-musl": { + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-musl/-/binding-linux-x64-musl-1.72.0.tgz", + "integrity": "sha512-SrpbrUL70nG9vh6zP4/oKHWgLuHquwsr7MW9XOn0olBVgh10Uqr8qscKhQoBGEn6olK/IUpn5GSKcdQ5AjUhGA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-openharmony-arm64": { + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-openharmony-arm64/-/binding-openharmony-arm64-1.72.0.tgz", + "integrity": "sha512-qkrsEn6NmgFKr7U/QnezQMb+q/vzAy0Dd9Y95gQGQTyjzDLN+HRZMuM5u70iyH4nBLCfKBzhjMsYCehKay2jyg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-win32-arm64-msvc": { + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.72.0.tgz", + "integrity": "sha512-LWR6ZlFZph+KPjXv8opgZsXRDCdrdQe8VL8Cg9zxCoBS73h6znzZpydVgmdnwj8mB9AuSM5jxEgDJDpQkjboeg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-win32-ia32-msvc": { + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-1.72.0.tgz", + "integrity": "sha512-yt6HEh7IsHvtjRWtmeZRX134eaXKHq5Gnqlf1xBJdJl1JtdoRUEJw3nAxpZoUDS860cX/foKbztO441anVBtVQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-win32-x64-msvc": { + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.72.0.tgz", + "integrity": "sha512-b2eKFD2hX7tIwmo/cyH6TDq8vzWRZ2qNHrzoGntUTmq0h3zQh/uX3eTSHCwI8OB/ADQfJCRelLItK8BsxuucDA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.4.tgz", + "integrity": "sha512-EZLpf/8y7GXkkra90ML47kzik/GMP3EMcE9bPyHmRfxLC6z9+aW5A8poCsoxjrT5GfEcNAAvWwUHjvP1pUQkfw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.1.4.tgz", + "integrity": "sha512-aUi+HBvmYb7j8krl1+qJgkG8C17fO79gk3c+jPw4S8glRFc1DTija9S3EyaTSQUm5GJXYKDAsugBEhFHH2vYiQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.1.4.tgz", + "integrity": "sha512-F7hHC3gwY11+vByKPRWqwGbeXWVgKmL+pTGCinaEhdihzBV2aQ0fvZOch9cXYUOKuKKq429HeYXOqQLc7wFCEg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.1.4.tgz", + "integrity": "sha512-sI5yw+7s92SK6odiEhD5lKCBlWcpjHS5qyqpVQbZAJ0fIzEUXrmbl3DH2ybR3PZogulNJF+COLtmA8hUfvkCCQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.1.4.tgz", + "integrity": "sha512-mCi0OKgEieFircrtVYmQAFGszRtMnZ6fpZAXrxanXAu7lqZcsK1E1RAaZNG0uKAnxox3B1f4EyQNnoyMfN1vAA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.1.4.tgz", + "integrity": "sha512-B9Ial3Kv5sh0SHnB1g/QWcUQCEvCF6QKGAl4zXypYj65mVI+B4AhFBwPtSN7pDrJeIx8Z7zdy4ntx+wQABom7w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.1.4.tgz", + "integrity": "sha512-lZVym0PuHE1KZ22gmFTC15lAkrg9iTszR617oYRB/iPY1A56ywoJzVKOJBKaot5RiikCObmur6pogpse3gRcng==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.1.4.tgz", + "integrity": "sha512-t2DNiLJWNTbnEHyUzTumldML6ET4/g16467LZoDDJ3tSxGvguL5/NyC2lCsNKuyRycg9XeDQF5SSv+TNOhQEXg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.1.4.tgz", + "integrity": "sha512-0WIRnL1Uw4BvTZRLQt+PVgo6ZKTJadlC2btP+/EOXv2f/DWbY0rEgl+y834mIVwP1FkTlWVTrGGJXf12lru7EQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.1.4.tgz", + "integrity": "sha512-JWtGshGfX+oENAKonoNkqEJX+7hC8yfhi9GUyPX1VX4mdh1y5r+ZiJLR5XzAB0aoP6s/PcILsGjKq8O0mm24bw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.1.4.tgz", + "integrity": "sha512-rT6yQcxUuXs4CnbofqwHRRV0iem349rLMYpTjkgQGLjrY4ado/eDzwPZPTCgTOlF6Nkp8NEv70yLMTn6qkWxsQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.1.4.tgz", + "integrity": "sha512-KXMGoboq5cyaCQjDA4GLuRiOwBQ0EyFnJoVViLeZ45/3rFItRODEr+NdsBcVpll40hhNArlm/speWGRvj08LzA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.1.4.tgz", + "integrity": "sha512-5K83rb36oJiY7BCyE9zLZtGcPV4g5wvq+xwdO0XPIwDVZI8cyB/AUjkNXGb92/rnmezEkjMOpgY61rtwjQtFwg==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "1.11.1", + "@emnapi/runtime": "1.11.1", + "@napi-rs/wasm-runtime": "^1.1.6" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi/node_modules/@emnapi/core": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.1.tgz", + "integrity": "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.2", + "tslib": "^2.4.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi/node_modules/@emnapi/runtime": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz", + "integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.1.4.tgz", + "integrity": "sha512-PnWBtw3TV5KOg69HQQDR0mnQuyCmSGR2pAB4DC1rPF808fgKeTUMj2EOEyKATpgiuxuR5APQmiDO7PDgEjTFSA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.1.4.tgz", + "integrity": "sha512-M1lpniBePobTfsa7Ks9a199e1akxsXn+GYBUKsEzv3YFzOm1HJAMNwKI3qr0Zq+mxwx9gOZoTdP1yXRYsZUocQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", + "integrity": "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@types/react": { + "version": "19.2.17", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.17.tgz", + "integrity": "sha512-MXfmqaVPEVgkBT/aY0aGCkRWWtByiYQXo3xdQ8r5RzuFrPiRn8Gar2tQdXSUQ2GKV3bkXckek89V8wQBY2Q/Aw==", + "dev": true, + "license": "MIT", + "dependencies": { + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "19.2.3", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.3.tgz", + "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^19.2.0" + } + }, + "node_modules/@uiw/codemirror-extensions-basic-setup": { + "version": "4.25.10", + "resolved": "https://registry.npmjs.org/@uiw/codemirror-extensions-basic-setup/-/codemirror-extensions-basic-setup-4.25.10.tgz", + "integrity": "sha512-P3vytLlpE62KYSWrMUnwDCv2lvaQDuDZzyj03mHntuHo5bSl34fRZpjTY3kQTPGuXHxkGSYpoPFFj+hMTqaaMQ==", + "license": "MIT", + "dependencies": { + "@codemirror/autocomplete": "^6.0.0", + "@codemirror/commands": "^6.0.0", + "@codemirror/language": "^6.0.0", + "@codemirror/lint": "^6.0.0", + "@codemirror/search": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0" + }, + "funding": { + "url": "https://jaywcjlove.github.io/#/sponsor" + }, + "peerDependencies": { + "@codemirror/autocomplete": ">=6.0.0", + "@codemirror/commands": ">=6.0.0", + "@codemirror/language": ">=6.0.0", + "@codemirror/lint": ">=6.0.0", + "@codemirror/search": ">=6.0.0", + "@codemirror/state": ">=6.0.0", + "@codemirror/view": ">=6.0.0" + } + }, + "node_modules/@uiw/react-codemirror": { + "version": "4.25.10", + "resolved": "https://registry.npmjs.org/@uiw/react-codemirror/-/react-codemirror-4.25.10.tgz", + "integrity": "sha512-DzgSMwM5qzB7v1FIb4gEeriYt67iiay756/HIOM9mAbeOVK0MO7rqefHf0O5c0269pJKMW7AH9FjclExD23V9w==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.6", + "@codemirror/commands": "^6.1.0", + "@codemirror/state": "^6.1.1", + "@codemirror/theme-one-dark": "^6.0.0", + "@uiw/codemirror-extensions-basic-setup": "4.25.10", + "codemirror": "^6.0.0" + }, + "funding": { + "url": "https://jaywcjlove.github.io/#/sponsor" + }, + "peerDependencies": { + "@babel/runtime": ">=7.11.0", + "@codemirror/state": ">=6.0.0", + "@codemirror/theme-one-dark": ">=6.0.0", + "@codemirror/view": ">=6.0.0", + "codemirror": ">=6.0.0", + "react": ">=17.0.0", + "react-dom": ">=17.0.0" + } + }, + "node_modules/@vitejs/plugin-react": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-6.0.3.tgz", + "integrity": "sha512-vmFvco5/QuC2f9Oj+wTk0+9XeDFkHxSamwZKYc7MxYwKICfvUvlMhqKI0VuICPltGqh1neqBKDvO4kes1ya8vg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@rolldown/pluginutils": "^1.0.1" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "peerDependencies": { + "@rolldown/plugin-babel": "^0.1.7 || ^0.2.0", + "babel-plugin-react-compiler": "^1.0.0", + "vite": "^8.0.0" + }, + "peerDependenciesMeta": { + "@rolldown/plugin-babel": { + "optional": true + }, + "babel-plugin-react-compiler": { + "optional": true + } + } + }, + "node_modules/codemirror": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/codemirror/-/codemirror-6.0.2.tgz", + "integrity": "sha512-VhydHotNW5w1UGK0Qj96BwSk/Zqbp9WbnyK2W/eVMv4QyF41INRGpjUhFJY7/uDNuudSc33a/PKr4iDqRduvHw==", + "license": "MIT", + "dependencies": { + "@codemirror/autocomplete": "^6.0.0", + "@codemirror/commands": "^6.0.0", + "@codemirror/language": "^6.0.0", + "@codemirror/lint": "^6.0.0", + "@codemirror/search": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0" + } + }, + "node_modules/cookie": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", + "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/crelt": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/crelt/-/crelt-1.0.7.tgz", + "integrity": "sha512-aK6BbWfhf4U/wCcLHKPJl/xa6VkVstRaPywWtMKGwuOLc/wZTyQYuoxgvZnNsBvv7Kg3YTBQYYBCggcviQczuA==", + "license": "MIT" + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/lightningcss": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", + "integrity": "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.32.0", + "lightningcss-darwin-arm64": "1.32.0", + "lightningcss-darwin-x64": "1.32.0", + "lightningcss-freebsd-x64": "1.32.0", + "lightningcss-linux-arm-gnueabihf": "1.32.0", + "lightningcss-linux-arm64-gnu": "1.32.0", + "lightningcss-linux-arm64-musl": "1.32.0", + "lightningcss-linux-x64-gnu": "1.32.0", + "lightningcss-linux-x64-musl": "1.32.0", + "lightningcss-win32-arm64-msvc": "1.32.0", + "lightningcss-win32-x64-msvc": "1.32.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz", + "integrity": "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.32.0.tgz", + "integrity": "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.32.0.tgz", + "integrity": "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.32.0.tgz", + "integrity": "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.32.0.tgz", + "integrity": "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.32.0.tgz", + "integrity": "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.32.0.tgz", + "integrity": "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.32.0.tgz", + "integrity": "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.32.0.tgz", + "integrity": "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.32.0.tgz", + "integrity": "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.32.0.tgz", + "integrity": "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/nanoid": { + "version": "3.3.15", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz", + "integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/oxlint": { + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/oxlint/-/oxlint-1.72.0.tgz", + "integrity": "sha512-1rhdZIP/EvoI91ABIwNU5Q8+bWf8mjrS5UzIOZld4d4bXxJvtlUhlQvaoTogIGin/qdErMOrwaIJvCSIAKTLhA==", + "dev": true, + "license": "MIT", + "bin": { + "oxlint": "bin/oxlint" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/sponsors/Boshen" + }, + "optionalDependencies": { + "@oxlint/binding-android-arm-eabi": "1.72.0", + "@oxlint/binding-android-arm64": "1.72.0", + "@oxlint/binding-darwin-arm64": "1.72.0", + "@oxlint/binding-darwin-x64": "1.72.0", + "@oxlint/binding-freebsd-x64": "1.72.0", + "@oxlint/binding-linux-arm-gnueabihf": "1.72.0", + "@oxlint/binding-linux-arm-musleabihf": "1.72.0", + "@oxlint/binding-linux-arm64-gnu": "1.72.0", + "@oxlint/binding-linux-arm64-musl": "1.72.0", + "@oxlint/binding-linux-ppc64-gnu": "1.72.0", + "@oxlint/binding-linux-riscv64-gnu": "1.72.0", + "@oxlint/binding-linux-riscv64-musl": "1.72.0", + "@oxlint/binding-linux-s390x-gnu": "1.72.0", + "@oxlint/binding-linux-x64-gnu": "1.72.0", + "@oxlint/binding-linux-x64-musl": "1.72.0", + "@oxlint/binding-openharmony-arm64": "1.72.0", + "@oxlint/binding-win32-arm64-msvc": "1.72.0", + "@oxlint/binding-win32-ia32-msvc": "1.72.0", + "@oxlint/binding-win32-x64-msvc": "1.72.0" + }, + "peerDependencies": { + "oxlint-tsgolint": ">=0.22.1", + "vite-plus": "*" + }, + "peerDependenciesMeta": { + "oxlint-tsgolint": { + "optional": true + }, + "vite-plus": { + "optional": true + } + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.16", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.16.tgz", + "integrity": "sha512-vuwillviilfKZsg0VGj5R/YwwcHx4SLsIOI/7K6mQkWx+l5cUHTjj5g0AasTBcyXsbfTgrwsUNmVUb5xVwyPwg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.12", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/react": { + "version": "19.2.7", + "resolved": "https://registry.npmjs.org/react/-/react-19.2.7.tgz", + "integrity": "sha512-HNe9WslTbXmFK8o8cmwgAeJFSBvt1bPdHCVKtaaV+WlAN36mpT4hcRpwbf3fY56ar2oIXzsBpOAiIRHAdY0OlQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.2.7", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.7.tgz", + "integrity": "sha512-t0BRVXvbiE/o20Hfw669rLbMCDWtYZLvmJigy2f0MxsXF+71pxhR3xOkspmsO8h3ZlNzyibAmtCa3l4lYKk6gQ==", + "license": "MIT", + "dependencies": { + "scheduler": "^0.27.0" + }, + "peerDependencies": { + "react": "^19.2.7" + } + }, + "node_modules/react-router": { + "version": "7.18.1", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.18.1.tgz", + "integrity": "sha512-GDLgg3i3uM0aeJO3Fm+TCS+sDQ7gu12T6x0qdTEzcwqEfleci7JwugVNIF3U//0FWKnJT7ptG+20B2jfDqnZAg==", + "license": "MIT", + "dependencies": { + "cookie": "^1.0.1", + "set-cookie-parser": "^2.6.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "react": ">=18", + "react-dom": ">=18" + }, + "peerDependenciesMeta": { + "react-dom": { + "optional": true + } + } + }, + "node_modules/react-router-dom": { + "version": "7.18.1", + "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-7.18.1.tgz", + "integrity": "sha512-KaZh+X/6UtEp28x51AUYZDMg9NGoz2ja3dNHa+ta/tk40vCzKhQ/RypCWBMLbmDr6//E24Vv5uPsrqXFozdkAg==", + "license": "MIT", + "dependencies": { + "react-router": "7.18.1" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "react": ">=18", + "react-dom": ">=18" + } + }, + "node_modules/rolldown": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.1.4.tgz", + "integrity": "sha512-IjZYiLxZwpnhwhdBH2ugdTGVSdhCQUmLxLoqyjiL0JxYjyRst+5a0P3xfrTxJ5F638j4Mvvw5FAX5XE6eHpXbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.138.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm64": "1.1.4", + "@rolldown/binding-darwin-arm64": "1.1.4", + "@rolldown/binding-darwin-x64": "1.1.4", + "@rolldown/binding-freebsd-x64": "1.1.4", + "@rolldown/binding-linux-arm-gnueabihf": "1.1.4", + "@rolldown/binding-linux-arm64-gnu": "1.1.4", + "@rolldown/binding-linux-arm64-musl": "1.1.4", + "@rolldown/binding-linux-ppc64-gnu": "1.1.4", + "@rolldown/binding-linux-s390x-gnu": "1.1.4", + "@rolldown/binding-linux-x64-gnu": "1.1.4", + "@rolldown/binding-linux-x64-musl": "1.1.4", + "@rolldown/binding-openharmony-arm64": "1.1.4", + "@rolldown/binding-wasm32-wasi": "1.1.4", + "@rolldown/binding-win32-arm64-msvc": "1.1.4", + "@rolldown/binding-win32-x64-msvc": "1.1.4" + } + }, + "node_modules/scheduler": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", + "integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==", + "license": "MIT" + }, + "node_modules/set-cookie-parser": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.2.tgz", + "integrity": "sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==", + "license": "MIT" + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/style-mod": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/style-mod/-/style-mod-4.1.3.tgz", + "integrity": "sha512-i/n8VsZydrugj3Iuzll8+x/00GH2vnYsk1eomD8QiRrSAeW6ItbCQDtfXCeJHd0iwiNagqjQkvpvREEPtW3IoQ==", + "license": "MIT" + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD", + "optional": true + }, + "node_modules/vite": { + "version": "8.1.3", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.1.3.tgz", + "integrity": "sha512-Ds+gBRbj0lwRO2Y5hwnUBdxSwlAve9LeRyU4sNnAr0ewW0gWF0n5bgXgUzbgZ49MV9BVUAQUFYVcDUcilUExMA==", + "dev": true, + "license": "MIT", + "dependencies": { + "lightningcss": "^1.32.0", + "picomatch": "^4.0.4", + "postcss": "^8.5.16", + "rolldown": "~1.1.3", + "tinyglobby": "^0.2.17" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.3.0", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/w3c-keyname": { + "version": "2.2.8", + "resolved": "https://registry.npmjs.org/w3c-keyname/-/w3c-keyname-2.2.8.tgz", + "integrity": "sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==", + "license": "MIT" + } + } +} diff --git a/frontend/package.json b/frontend/package.json new file mode 100644 index 0000000..175a306 --- /dev/null +++ b/frontend/package.json @@ -0,0 +1,27 @@ +{ + "name": "frontend", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "lint": "oxlint", + "preview": "vite preview" + }, + "dependencies": { + "@codemirror/lang-javascript": "^6.2.5", + "@codemirror/theme-one-dark": "^6.1.3", + "@uiw/react-codemirror": "^4.25.10", + "react": "^19.2.7", + "react-dom": "^19.2.7", + "react-router-dom": "^7.18.1" + }, + "devDependencies": { + "@types/react": "^19.2.17", + "@types/react-dom": "^19.2.3", + "@vitejs/plugin-react": "^6.0.3", + "oxlint": "^1.71.0", + "vite": "^8.1.1" + } +} diff --git a/frontend/public/favicon.svg b/frontend/public/favicon.svg new file mode 100644 index 0000000..6893eb1 --- /dev/null +++ b/frontend/public/favicon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/public/icons.svg b/frontend/public/icons.svg new file mode 100644 index 0000000..e952219 --- /dev/null +++ b/frontend/public/icons.svg @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx new file mode 100644 index 0000000..a8da339 --- /dev/null +++ b/frontend/src/App.jsx @@ -0,0 +1,120 @@ +import { useEffect, useRef, useState } from 'react' +import { NavLink, Outlet, useLocation } from 'react-router-dom' +import { api } from './api' +import { AuthModal, ToastHost } from './components' +import Embers from './Embers.jsx' + +export default function App() { + // null until /auth/me resolves; in local mode multi_user=false hides all auth UI. + const [me, setMe] = useState(null) + const [authMode, setAuthMode] = useState(null) // 'register' | 'login' | null + const [navOpen, setNavOpen] = useState(false) // mobile hamburger menu + const location = useLocation() + const lastPath = useRef(null) + + useEffect(() => { + api.getMe().then(setMe).catch(() => {}) + }, []) + + // One pageview per route the reader actually lands on. Guarded on the path + // rather than fired on every render: StrictMode runs effects twice in dev, + // and a re-render for unrelated state is not a new page. + useEffect(() => { + if (lastPath.current === location.pathname) return + const first = lastPath.current === null + lastPath.current = location.pathname + // document.referrer survives client-side navigation, so it is only honest + // on the first view — after that this was our own page, not a referral. + api.trackPageview(location.pathname, { + referrer: first ? document.referrer : '', + first, + }) + }, [location.pathname]) + + const onAuthed = (newMe, mode) => { + setAuthMode(null) + if (mode === 'login') { + // Different user now — reload so every page refetches its scoped data. + window.location.reload() + } else { + setMe(newMe) // register upgrades the same user in place; data unchanged + } + } + + const logout = async () => { + try { await api.logout() } catch { /* already logged out */ } + window.location.reload() + } + + return ( + + + + + {authMode && ( + setAuthMode(null)} onAuthed={onAuthed} + retentionDays={me?.guest_retention_days} /> + )} + + ) +} diff --git a/frontend/src/ArtPicker.jsx b/frontend/src/ArtPicker.jsx new file mode 100644 index 0000000..9dc1c47 --- /dev/null +++ b/frontend/src/ArtPicker.jsx @@ -0,0 +1,147 @@ +import { useRef, useState } from 'react' +import { ScenarioArt } from './components' + +// Cards render the plate at 46px (92px on a 2x screen), and the editor preview +// at 76px. 400px square gives headroom for both plus any future larger use, +// while keeping the stored data URI in the tens of kilobytes. +const MAX_EDGE = 400 +const WEBP_QUALITY = 0.82 +// The backend caps the column at 400_000 chars; stay clearly under it so a +// pathological image fails here with a clear message rather than as a 422. +const MAX_DATA_URI = 360_000 + +// A spread of moods rather than a themed set — most scenarios find something +// close enough here and skip hunting for a picture. +const SUGGESTED_ICONS = [ + '⚔️', '🗡️', '🏹', '🛡️', '🔮', '🗝️', '📜', '🏰', + '🐉', '👑', '💀', '🕯️', '🌑', '🌲', '⛰️', '🌊', + '🚀', '🛰️', '🤖', '👁️', '🩸', '🃏', +] + +/** Read a File and re-encode it small, as a WebP data URI. + * + * Downscaling in the browser means the upload never leaves the machine at full + * size and the row stays small — no server-side image library required. + */ +function downscaleToDataURI(file) { + return new Promise((resolve, reject) => { + const reader = new FileReader() + reader.onerror = () => reject(new Error('Could not read that file')) + reader.onload = () => { + const img = new Image() + img.onerror = () => reject(new Error('That file is not an image the browser can read')) + img.onload = () => { + const scale = Math.min(1, MAX_EDGE / Math.max(img.width, img.height)) + const canvas = document.createElement('canvas') + // Math.max(1, …) guards against a 0-dimension canvas, which throws. + canvas.width = Math.max(1, Math.round(img.width * scale)) + canvas.height = Math.max(1, Math.round(img.height * scale)) + const ctx = canvas.getContext('2d') + ctx.drawImage(img, 0, 0, canvas.width, canvas.height) + let uri = canvas.toDataURL('image/webp', WEBP_QUALITY) + // Safari only gained WebP encoding recently; if it silently fell back + // to PNG, retry as JPEG so we don't store a huge lossless blob. + if (!uri.startsWith('data:image/webp')) { + uri = canvas.toDataURL('image/jpeg', WEBP_QUALITY) + } + if (uri.length > MAX_DATA_URI) { + reject(new Error('That image is too detailed to store — try a smaller or simpler one')) + return + } + resolve(uri) + } + img.src = reader.result + } + reader.readAsDataURL(file) + }) +} + +/** Cover-art picker: upload a picture, choose an emoji, or leave it generated. + * + * `image` and `icon` are the stored values; `onChange({ image, icon })` gets + * both on every change so the caller can persist them together. + */ +export default function ArtPicker({ title, image, icon, onChange, disabled }) { + const fileRef = useRef(null) + const [error, setError] = useState('') + + const pick = async (event) => { + const file = event.target.files?.[0] + // Reset immediately so picking the same file twice still fires onChange. + event.target.value = '' + if (!file) return + setError('') + try { + const uri = await downscaleToDataURI(file) + // An uploaded picture wins over an emoji, so clear the icon to make the + // precedence visible rather than leaving a hidden value behind. + onChange({ image: uri, icon: '' }) + } catch (err) { + setError(err.message) + } + } + + const chooseIcon = (glyph) => { + setError('') + onChange({ image: '', icon: icon === glyph ? '' : glyph }) + } + + const clear = () => { + setError('') + onChange({ image: '', icon: '' }) + } + + return ( + + ) +} diff --git a/frontend/src/BranchMap.jsx b/frontend/src/BranchMap.jsx new file mode 100644 index 0000000..2dc4296 --- /dev/null +++ b/frontend/src/BranchMap.jsx @@ -0,0 +1,282 @@ +import { useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react' +import { createPortal } from 'react-dom' +import { CORNER, PAD, ROW_H, branchLabel, headLineage, layoutTree, momentTicks } from './branches' + +// The story tree, drawn. +// +// The Branches panel lists every line; this draws the same list as lanes on one +// clock, which is the thing a list cannot say — *where* two tellings parted, and +// how much story each of them is. A lane runs from the moment its branch left +// its parent to the moment it ends, so length is story and a fork is a corner. +// +// It reads nothing of its own. Everything here comes from the `GET /branches` +// the panel already made, and every operation goes back through the panel's, so +// there is one copy of the rules and one thing to keep honest. +// +// Portalled to : this is opened from inside `.side-panel`, whose panel-in +// animation (fill mode `both`) makes it the containing block for position:fixed +// children — an overlay rendered in place would be trapped in the 420px panel +// and clipped by its overflow. Same trap for anything opened from a drawer. + +// A label is drawn into the space its lane leaves. `perChar` is the average +// width of the face it is drawn in — about 7px for the 15px serif a name uses +// and 5px for the 10.5px UI face under it. Overshooting only costs an ellipsis, +// so this is deliberately a guess rather than a measurement pass. +function clipToWidth(text, px, perChar = 7) { + const max = Math.max(4, Math.floor(px / perChar)) + return text.length <= max ? text : `${text.slice(0, max - 1)}…` +} + +export function BranchMap({ branches, busyId, onSwitch, onRename, onDelete, onClose }) { + const [selectedId, setSelectedId] = useState(() => branches.find((b) => b.is_head)?.id ?? null) + const [renameText, setRenameText] = useState(null) // null = not renaming + const [confirming, setConfirming] = useState(false) + const [width, setWidth] = useState(0) + const boxRef = useRef(null) + + const { lanes, maxDepth, height } = useMemo(() => layoutTree(branches), [branches]) + const lineage = useMemo(() => headLineage(branches), [branches]) + + + // The map is drawn in real pixels rather than scaled from a fixed viewBox, + // so a narrow window gets a narrower map and not smaller writing. + // + // Both halves are load-bearing, and each covers the other's gap. The seed + // measures the *content* box, because that is what the observer reports and + // `clientWidth` is not — it counts the canvas padding, so seeding from it + // drew an svg 24px wider than the box it sits in and the map ran off the + // right edge. The observer then keeps up with a window being dragged; it + // cannot be the only source, because its initial observation is not + // guaranteed to arrive and without the seed the map never drew at all. + useLayoutEffect(() => { + const el = boxRef.current + if (!el) return undefined + const style = getComputedStyle(el) + const padding = parseFloat(style.paddingLeft) + parseFloat(style.paddingRight) + setWidth(el.clientWidth - padding) + const observer = new ResizeObserver(([entry]) => setWidth(entry.contentRect.width)) + observer.observe(el) + return () => observer.disconnect() + }, []) + + // A branch can vanish under the selection — deleting one takes everything + // forked from it, which is more rows than the one that was clicked. + const selected = branches.find((b) => b.id === selectedId) || null + useEffect(() => { + if (!selected) { + setSelectedId(branches.find((b) => b.is_head)?.id ?? null) + setRenameText(null) + setConfirming(false) + } + }, [selected, branches]) + + useEffect(() => { + const onKey = (e) => { + if (e.key !== 'Escape') return + // Escape backs out of the smallest thing that is open, so it never + // throws away a half-typed name along with the map. + if (renameText !== null) setRenameText(null) + else if (confirming) setConfirming(false) + else onClose() + } + window.addEventListener('keydown', onKey) + return () => window.removeEventListener('keydown', onKey) + }, [renameText, confirming, onClose]) + + const W = Math.max(width, 320) + const span = Math.max(maxDepth, 1) + // Ticks are spaced by the room there is to print them in, not by a constant. + const ticks = momentTicks(maxDepth, Math.max(3, Math.round(W / 160))) + const x = (depth) => PAD.left + (depth / span) * (W - PAD.left - PAD.right) + const laneY = (row) => PAD.top + row * ROW_H + 30 + + const pick = (branch) => { + setSelectedId(branch.id) + setRenameText(null) + setConfirming(false) + } + + const busy = busyId !== null && busyId !== undefined + // The server refuses to delete the line being read or anything it was forked + // from; saying so on the button is friendlier than a toast after the click. + const isLoadBearing = selected ? lineage.has(selected.id) : false + const isRoot = selected ? selected.parent_branch_id === null : false + + // Each of these resolves to whether it worked. The panel owns the request + // and reports its own failures, so all this has to decide is whether to put + // the editor away — clearing a half-typed name on a rename that was refused + // would throw away the only copy of it. + const save = async () => { if (await onRename(selected, renameText)) setRenameText(null) } + const drop = async () => { if (await onDelete(selected)) setConfirming(false) } + + return createPortal( +
+
e.stopPropagation()} + role="dialog" aria-modal="true" aria-label="The story so far"> +
+

The story so far

+ + {branches.length} {branches.length === 1 ? 'line' : 'lines'} · {maxDepth + 1} moments + + +
+ +
+ {width > 0 && ( + + {/* The clock the whole map is read against. */} + {ticks.map((depth) => ( + + + {depth + 1} + + ))} + + {lanes.map((lane) => { + const child = lane.branch.parent_branch_id !== null && lane.parentRow !== null + const forkX = x(lane.from) + const start = child ? forkX + CORNER : x(lane.from) + // A branch forked but not yet written past still gets a stub, + // or it would be a corner leading to nothing. + const end = Math.max(x(lane.to), start + 8) + const y = laneY(lane.row) + const b = lane.branch + const label = branchLabel(b) + const meta = `${b.own_actions} of its own` + + (b.parent_branch_id !== null ? ` · left at ${b.fork_depth + 1}` : '') + + ` · ends at ${b.depth + 1}` + // A lane that leaves late has no room to write in to its right, + // so its labels hang back over the fork instead. The row is its + // own band with nothing else in it, and the alternative is text + // running off the edge — which is what a narrow window did. + const roomRight = W - PAD.right - start + const flip = roomRight < 170 && start - PAD.left > roomRight + const textX = flip ? start - 4 : start + const room = flip ? start - 4 - PAD.left : roomRight + const classes = [ + 'bm-lane', + b.is_head ? 'here' : '', + b.id === selectedId ? 'picked' : '', + ].join(' ') + return ( + pick(b)} + onKeyDown={(e) => { + if (e.key === 'Enter' || e.key === ' ') { e.preventDefault(); pick(b) } + }}> + {/* Full-width hit area: the lane itself is 3px tall and a + fork late in a long story is a very small target. */} + + + {child && ( + + )} + {child && } + + + + {/* The tip: a diamond for the line being read, so where you + are standing is findable without reading a word. */} + {b.is_head ? ( + + ) : ( + + )} + + + {clipToWidth(label, room)} + + + {clipToWidth(meta, room, 5)} + + + ) + })} + + )} +
+ + {branches.length === 1 && ( +

+ One thread so far. Retry a turn, then write on from an attempt the + story moved past — that is what makes a second line. +

+ )} + + {selected && ( +
+
+ {renameText !== null ? ( + setRenameText(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') save() + }} + /> + ) : ( + {branchLabel(selected)} + )} + {selected.is_head && reading} +
+ +
+ {selected.own_actions} of its own + {selected.parent_branch_id !== null && ` · forked at moment ${selected.fork_depth + 1}`} + {` · ends at moment ${selected.depth + 1}`} +
+ + {confirming ? ( +
+ Delete this branch and everything forked from it? + + +
+ ) : ( +
+ {!selected.is_head && ( + + )} + {renameText !== null ? ( + <> + + + + ) : ( + + )} + {/* The root holds the turns every other branch borrows, and the + server refuses it — so it is not offered at all. A branch + the head stands on is offered, and says why it cannot go. */} + {!isRoot && ( + + )} +
+ )} +
+ )} +
+
, + document.body, + ) +} diff --git a/frontend/src/Embers.jsx b/frontend/src/Embers.jsx new file mode 100644 index 0000000..21e99ae --- /dev/null +++ b/frontend/src/Embers.jsx @@ -0,0 +1,110 @@ +import { useEffect, useRef } from 'react' + +// Motes per million CSS pixels of viewport. Tuned so a laptop gets ~55 and a +// phone ~20: enough to read as drifting dust, few enough to stay cheap. +const DENSITY = 34 +const MAX_PARTICLES = 90 + +/** Slow-drifting gold motes behind the whole app. + * + * Fixed, non-interactive, and drawn on one canvas. It sits below every page in + * the stacking order and never receives pointer events, so it cannot affect + * layout or intercept clicks. Disabled outright for `prefers-reduced-motion`, + * and paused whenever the tab is hidden so a backgrounded tab costs nothing. + */ +export default function Embers() { + const canvasRef = useRef(null) + + useEffect(() => { + const canvas = canvasRef.current + if (!canvas) return + + const reduced = window.matchMedia('(prefers-reduced-motion: reduce)') + if (reduced.matches) return + + const ctx = canvas.getContext('2d') + if (!ctx) return + + // Cap the backing store at 2x: beyond that the cost is real and nobody can + // see the difference on a blurred mote. + const dpr = Math.min(window.devicePixelRatio || 1, 2) + let width = 0 + let height = 0 + let particles = [] + let frame = 0 + + const spawn = (scattered) => ({ + x: Math.random() * width, + // New motes enter from just below the fold; the first batch is scattered + // across the whole height so the screen isn't empty on load. + y: scattered ? Math.random() * height : height + Math.random() * 40, + radius: 0.6 + Math.random() * 1.6, + speed: 0.08 + Math.random() * 0.3, + sway: Math.random() * Math.PI * 2, + swaySpeed: 0.004 + Math.random() * 0.01, + alpha: 0.12 + Math.random() * 0.4, + }) + + const resize = () => { + width = window.innerWidth + height = window.innerHeight + canvas.width = Math.round(width * dpr) + canvas.height = Math.round(height * dpr) + canvas.style.width = `${width}px` + canvas.style.height = `${height}px` + ctx.setTransform(dpr, 0, 0, dpr, 0, 0) + const target = Math.min( + MAX_PARTICLES, + Math.max(12, Math.round((width * height * DENSITY) / 1_000_000)), + ) + particles = Array.from({ length: target }, () => spawn(true)) + } + + const draw = () => { + ctx.clearRect(0, 0, width, height) + for (let i = 0; i < particles.length; i++) { + const p = particles[i] + p.sway += p.swaySpeed + p.y -= p.speed + p.x += Math.sin(p.sway) * 0.28 + if (p.y < -8) { + particles[i] = spawn(false) + continue + } + // Fade out toward the top of the screen so motes dissolve rather than + // vanishing at the edge. + const fade = Math.min(1, p.y / height) + const glow = ctx.createRadialGradient(p.x, p.y, 0, p.x, p.y, p.radius * 4) + glow.addColorStop(0, `rgba(232, 196, 118, ${(p.alpha * fade).toFixed(3)})`) + glow.addColorStop(1, 'rgba(212, 169, 78, 0)') + ctx.fillStyle = glow + ctx.beginPath() + ctx.arc(p.x, p.y, p.radius * 4, 0, Math.PI * 2) + ctx.fill() + } + frame = requestAnimationFrame(draw) + } + + const start = () => { + if (!frame) frame = requestAnimationFrame(draw) + } + const stop = () => { + cancelAnimationFrame(frame) + frame = 0 + } + const onVisibility = () => (document.hidden ? stop() : start()) + + resize() + start() + window.addEventListener('resize', resize) + document.addEventListener('visibilitychange', onVisibility) + + return () => { + stop() + window.removeEventListener('resize', resize) + document.removeEventListener('visibilitychange', onVisibility) + } + }, []) + + return