Files
interactive-story/backend/app/models.py
T
parththakkar106andClaude Opus 5 2c5909a268 Drop the JSON vector column, and fix what was hiding behind it
Migration 38 left memories.embedding in place so a rollback could still find
the vectors. Production has since been verified reading from embedding_blob,
so migration 42 drops it: 4 MB of a 99.6 MB database holding nothing anyone
reads.

Removing it surfaced a live bug. Changing your embedding model is supposed to
throw the bank's vectors away and let the post-turn pass rebuild them, because
two models' vectors are not comparable. The settings route did that by nulling
memories.embedding -- correct until 38 moved the vectors, after which it
cleared the dead column and left the blob intact with `embedded` still true.
_embed_pending filters on `embedded IS FALSE`, so it never saw those rows and
the bank went on ranking against the old model's vectors permanently.

Nothing would have reported it. cosine returns 0.0 on a width mismatch, so a
different-width model scores every memory zero and retrieval returns whichever
rows happen to sort first; a same-width model scores plausible garbage.

The bulk clear now sets both columns. It stays a bulk UPDATE rather than going
through set_vector -- loading the rows is the cost that whole path exists to
avoid -- so set_vector's docstring now names it as the one caller that
legitimately writes those columns by hand. No cache invalidation is added:
clearing `embedded` drops the rows out of the catalogue query, and set_vector
evicts each entry as the re-embed puts it back.

test_embedding_blob.py now rebuilds the pre-38 schema by hand where it tests
the backfill, since create_all no longer produces the column it converts from,
and asserts 42 removes it at the end of a full bootstrap -- 38 reads that
column and 42 drops it, so an upgrade that reordered them would arrive with an
empty bank.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017Dvvqn9ZDR4ixeFPHNbww7
2026-08-17 13:44:59 +05:30

394 lines
20 KiB
Python

from datetime import datetime, timezone
from sqlalchemy import (
JSON, Boolean, Column, DateTime, Float, ForeignKey, Integer, LargeBinary, String,
Table, Text,
)
from sqlalchemy.orm import Mapped, mapped_column, relationship
from .database import Base
def utcnow() -> datetime:
return datetime.now(timezone.utc)
class User(Base):
"""Phase 8 — optional accounts.
Three kinds of rows share this table:
- the "local user" (email NULL, is_guest False): auto-created in
single-user/local mode; owns everything a pre-Phase-8 DB had;
- guests (email NULL, is_guest True): created on first visit in
multi-user mode, identified only by their session cookie;
- registered users (email set): a guest upgraded in place, so their
data survives registration with no re-parenting.
"""
__tablename__ = "users"
id: Mapped[int] = mapped_column(primary_key=True)
email: Mapped[str | None] = mapped_column(String(320), unique=True, nullable=True)
password_hash: Mapped[str | None] = mapped_column(String(300), nullable=True)
is_guest: Mapped[bool] = mapped_column(Boolean, default=True)
created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow)
last_seen_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
# Shared demo key usage (resets when the UTC date changes).
demo_turns_used: Mapped[int] = mapped_column(Integer, default=0)
demo_turns_date: Mapped[str] = mapped_column(String(10), default="")
scenario_scripts = Table(
"scenario_scripts",
Base.metadata,
Column("scenario_id", ForeignKey("scenarios.id", ondelete="CASCADE"), primary_key=True),
Column("script_id", ForeignKey("scripts.id", ondelete="CASCADE"), primary_key=True),
)
class Scenario(Base):
__tablename__ = "scenarios"
id: Mapped[int] = mapped_column(primary_key=True)
# NULL owner + is_public = seeded demo content, readable by everyone.
user_id: Mapped[int | None] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), nullable=True
)
is_public: Mapped[bool] = mapped_column(Boolean, default=False)
title: Mapped[str] = mapped_column(String(200), default="Untitled Scenario")
description: Mapped[str] = mapped_column(Text, default="")
prompt: Mapped[str] = mapped_column(Text, default="")
# Plot components (AI Dungeon terminology; `memory` == Plot Essentials)
memory: Mapped[str] = mapped_column(Text, default="")
authors_note: Mapped[str] = mapped_column(Text, default="")
ai_instructions: Mapped[str] = mapped_column(Text, default="")
tags: Mapped[str] = mapped_column(String(500), default="")
# Cover art. Either an external "https://…" URL or an inline
# "data:image/…;base64,…" URI (the editor downscales uploads before storing
# one). Empty means the UI falls back to an emoji sigil or generated art.
# Kept in the row rather than on disk because Render's free tier has no
# persistent volume, and it makes export bundles self-contained.
image: Mapped[str] = mapped_column(Text, default="")
# A single emoji or glyph used when there's no `image` — cheap art for
# scenarios nobody wants to find a picture for. Separate from `image`
# because it's a character, not a locator: no fetch, no cache, no bytes.
icon: Mapped[str] = mapped_column(String(16), default="")
# Phase 12: RPG world-state template — stat definitions (bands, rules) and
# milestones. NULL/empty means this scenario has no RPG layer.
stat_schema: Mapped[dict | None] = mapped_column(JSON, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow)
updated_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow, onupdate=utcnow)
story_cards: Mapped[list["StoryCard"]] = relationship(
back_populates="scenario", cascade="all, delete-orphan"
)
adventures: Mapped[list["Adventure"]] = relationship(back_populates="scenario")
scripts: Mapped[list["Script"]] = relationship(secondary=scenario_scripts)
class Adventure(Base):
__tablename__ = "adventures"
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int | None] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), nullable=True
)
scenario_id: Mapped[int | None] = mapped_column(
ForeignKey("scenarios.id", ondelete="SET NULL"), nullable=True
)
title: Mapped[str] = mapped_column(String(200), default="Untitled Adventure")
memory: Mapped[str] = mapped_column(Text, default="")
authors_note: Mapped[str] = mapped_column(Text, default="")
ai_instructions: Mapped[str] = mapped_column(Text, default="")
story_summary: Mapped[str] = mapped_column(Text, default="")
script_state: Mapped[dict] = mapped_column(JSON, default=dict)
# Phase 12: live RPG world state (world/player/npc stats + milestones),
# instantiated from the scenario's stat_schema. Empty when there's no RPG layer.
world_state: Mapped[dict] = mapped_column(JSON, default=dict)
# The ${Placeholder} answers collected when this adventure was started, kept
# so "Update from scenario" can re-fill freshly copied scenario text with the
# same values. NULL for adventures created before this column existed.
placeholders: Mapped[dict | None] = mapped_column(JSON, nullable=True)
# Phase 6: opt-in per adventure (extra AI calls)
auto_summarize: Mapped[bool] = mapped_column(Boolean, default=False)
memory_bank_enabled: Mapped[bool] = mapped_column(Boolean, default=False)
# How many actions have already been folded into memories / the story summary.
memory_cursor: Mapped[int] = mapped_column(Integer, default=0)
summary_cursor: Mapped[int] = mapped_column(Integer, default=0)
created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow)
updated_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow, onupdate=utcnow)
scenario: Mapped[Scenario | None] = relationship(back_populates="adventures")
story_cards: Mapped[list["StoryCard"]] = relationship(
back_populates="adventure", cascade="all, delete-orphan"
)
actions: Mapped[list["Action"]] = relationship(
back_populates="adventure",
cascade="all, delete-orphan",
order_by="Action.index",
)
scripts: Mapped[list["AdventureScript"]] = relationship(
back_populates="adventure",
cascade="all, delete-orphan",
order_by="AdventureScript.position",
)
memories: Mapped[list["Memory"]] = relationship(
back_populates="adventure",
cascade="all, delete-orphan",
order_by="Memory.id",
)
class Memory(Base):
"""Phase 6: an auto-summarized (or hand-written) fact about the adventure.
The vector lives in `embedding_blob` as packed float32 (see vectors.py).
NULL until embedded, which also marks it for backfill when an embedding
model becomes available.
Cosine ranking happens in Python, which means the vectors cross the wire.
The original comment here sized that by count — "fine at a few hundred" —
and it was wrong by the only measure that mattered: a few hundred JSON
vectors is ten megabytes, fetched fresh every turn. Weigh new columns in
bytes.
"""
__tablename__ = "memories"
id: Mapped[int] = mapped_column(primary_key=True)
adventure_id: Mapped[int] = mapped_column(ForeignKey("adventures.id", ondelete="CASCADE"))
text: Mapped[str] = mapped_column(Text, default="")
# The vector, little-endian float32. Deferred because it is wider than the
# rest of the row put together and exactly one code path wants it: anything
# bulk-loading memories (the Memories drawer, eviction, the embed queue)
# must project the columns it needs rather than load whole entities.
embedding_blob: Mapped[bytes | None] = mapped_column(
LargeBinary, nullable=True, deferred=True
)
# Action index range this memory summarizes (null for manual memories).
source_start: Mapped[int | None] = mapped_column(Integer, nullable=True)
source_end: Mapped[int | None] = mapped_column(Integer, nullable=True)
# Whether embedding_blob is set. Maintained on write by memorybank
# .set_vector, for the same reason actions.variant_count exists beside
# actions.variants: every reader wants the one-bit answer and none of them
# should have to fetch six kilobytes of vector to get it.
embedded: Mapped[bool] = mapped_column(Boolean, default=False)
pinned: Mapped[bool] = mapped_column(Boolean, default=False)
forgotten: Mapped[bool] = mapped_column(Boolean, default=False) # evicted, kept for UI
use_count: Mapped[int] = mapped_column(Integer, default=0)
last_used_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow)
adventure: Mapped[Adventure] = relationship(back_populates="memories")
class StoryCard(Base):
"""Owned by either a scenario or an adventure (exactly one set)."""
__tablename__ = "story_cards"
id: Mapped[int] = mapped_column(primary_key=True)
scenario_id: Mapped[int | None] = mapped_column(
ForeignKey("scenarios.id", ondelete="CASCADE"), nullable=True
)
adventure_id: Mapped[int | None] = mapped_column(
ForeignKey("adventures.id", ondelete="CASCADE"), nullable=True
)
type: Mapped[str] = mapped_column(String(100), default="")
name: Mapped[str] = mapped_column(String(200), default="")
keys: Mapped[str] = mapped_column(Text, default="") # comma-separated triggers
entry: Mapped[str] = mapped_column(Text, default="")
notes: Mapped[str] = mapped_column(Text, default="")
# Adventure copies only: which piece of the scenario this card came from —
# "card:<scenario_card_id>" or "npc:<npc_key>". "Update from scenario"
# refreshes/removes exactly these; NULL means player-authored (left alone),
# or a copy predating the column (matched by name, then adopted).
source_ref: Mapped[str | None] = mapped_column(String(64), nullable=True)
scenario: Mapped[Scenario | None] = relationship(back_populates="story_cards")
adventure: Mapped[Adventure | None] = relationship(back_populates="story_cards")
class Action(Base):
__tablename__ = "actions"
id: Mapped[int] = mapped_column(primary_key=True)
adventure_id: Mapped[int] = mapped_column(ForeignKey("adventures.id", ondelete="CASCADE"))
index: Mapped[int] = mapped_column(Integer)
type: Mapped[str] = mapped_column(String(20)) # start|do|say|story|continue|ai
text: Mapped[str] = mapped_column(Text, default="")
# Reasoning-model "thinking" that preceded the text (AI actions only).
reasoning: Mapped[str | None] = mapped_column(Text, nullable=True)
# The full assembled prompt for this turn, for the Insights viewer. By far
# the biggest column in the database (~74 KB/row in production), and needed
# by exactly one endpoint, one action at a time — so it is deferred: never
# loaded unless something actually touches the attribute. Bulk readers must
# NOT touch it; that is what `world_delta` below exists for.
context_snapshot: Mapped[dict | None] = mapped_column(
JSON, nullable=True, deferred=True
)
# The small slice of the snapshot that IS needed in bulk: this turn's RPG
# state changes, for the inline chips under an AI message (world_changes)
# and for re-attaching the emit block when replaying history to the model.
# Mirrors the active variant, same as text/reasoning/context_snapshot.
world_delta: Mapped[dict | None] = mapped_column(JSON, nullable=True)
# Copy of Adventure.script_state as it was immediately BEFORE this action's
# script hooks ran, so undo/retry can roll the shared scoreboard back.
# NULL for actions created before this column existed. Deferred: only ever
# read for the one action being undone or retried.
state_before: Mapped[dict | None] = mapped_column(
JSON, nullable=True, deferred=True
)
# Phase 12: same idea for the RPG world_state, so undo/retry rolls it back too.
world_state_before: Mapped[dict | None] = mapped_column(
JSON, nullable=True, deferred=True
)
# Retry history (AI actions): every attempt made for this turn, oldest
# first, INCLUDING the active one. NULL/empty means never retried — the row
# is its own only version. `variant_index` says which entry `text`,
# `reasoning` and `context_snapshot` currently mirror; retry appends and
# points here instead of deleting the row, so nothing is lost.
#
# Deferred for the same reason as context_snapshot: a list response only
# ever needs the *count* (see variant_count below), but the column holds
# every discarded attempt's full narration, so loading it in bulk made each
# retry a permanent tax on every later page load of that adventure.
variants: Mapped[list | None] = mapped_column(JSON, nullable=True, deferred=True)
# len(variants), maintained on write by set_variants() so the deferred
# column above never has to be fetched just to count it. 0 = never retried.
variant_count: Mapped[int] = mapped_column(Integer, default=0)
variant_index: Mapped[int] = mapped_column(Integer, default=0)
created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow)
adventure: Mapped[Adventure] = relationship(back_populates="actions")
@property
def world_changes(self) -> list[dict]:
"""Compact per-turn RPG state changes (Phase 12), for the inline summary
under an AI message. Labels are path-based (no schema needed):
`npc.gwen.trust` -> "gwen trust".
Reads `world_delta`, never `context_snapshot` — this runs for every
action in a list response, and touching the deferred snapshot here
would drag the whole prompt archive out of the database."""
wd = self.world_delta if isinstance(self.world_delta, dict) else None
if wd is None:
return []
applied = wd.get("applied") or []
out: list[dict] = []
for entry in applied:
parts = str(entry.get("path", "")).split(".")
section, name = parts[0], parts[-1]
if section == "flags":
out.append({"kind": "flag", "label": name, "on": bool(entry.get("new"))})
elif section == "milestones":
out.append({"kind": "milestone", "label": name})
else:
label = f"{parts[1]} {parts[2]}" if section == "npc" and len(parts) == 3 else name
old, new = entry.get("old"), entry.get("new")
delta = new - old if isinstance(old, (int, float)) and isinstance(new, (int, float)) else None
out.append({"kind": "stat", "label": label, "delta": delta, "value": new})
return out
class Script(Base):
__tablename__ = "scripts"
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int | None] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), nullable=True
)
name: Mapped[str] = mapped_column(String(200), default="Untitled Script")
description: Mapped[str] = mapped_column(Text, default="")
library_js: Mapped[str] = mapped_column(Text, default="")
input_js: Mapped[str] = mapped_column(Text, default="")
context_js: Mapped[str] = mapped_column(Text, default="")
output_js: Mapped[str] = mapped_column(Text, default="")
created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow)
updated_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow, onupdate=utcnow)
class AdventureScript(Base):
"""A script copied into an adventure at creation, so library edits don't
change running adventures unless the player explicitly re-syncs it from
`source_script_id`. `state` lives on Adventure.script_state (one shared
state per adventure, as in AI Dungeon)."""
__tablename__ = "adventure_scripts"
id: Mapped[int] = mapped_column(primary_key=True)
adventure_id: Mapped[int] = mapped_column(ForeignKey("adventures.id", ondelete="CASCADE"))
# The library Script this copy was made from, so it can be re-synced on
# demand. NULL for legacy copies (predate this column) and demo-derived
# ones whose source isn't owned by the player — those fall back to a
# name match, or simply aren't syncable.
source_script_id: Mapped[int | None] = mapped_column(
ForeignKey("scripts.id", ondelete="SET NULL"), nullable=True
)
position: Mapped[int] = mapped_column(Integer, default=0)
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
name: Mapped[str] = mapped_column(String(200), default="Untitled Script")
description: Mapped[str] = mapped_column(Text, default="")
library_js: Mapped[str] = mapped_column(Text, default="")
input_js: Mapped[str] = mapped_column(Text, default="")
context_js: Mapped[str] = mapped_column(Text, default="")
output_js: Mapped[str] = mapped_column(Text, default="")
adventure: Mapped[Adventure] = relationship(back_populates="scripts")
class Settings(Base):
__tablename__ = "settings"
id: Mapped[int] = mapped_column(primary_key=True)
# Phase 8: one row per user (pre-Phase-8 DBs had a single id=1 row, which
# the migration assigns to the local user).
user_id: Mapped[int | None] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), nullable=True, unique=True
)
endpoint_url: Mapped[str] = mapped_column(String(500), default="http://localhost:11434/v1")
# Fernet-encrypted at rest ("enc:..." — see security.py); use api_key_plain.
api_key: Mapped[str] = mapped_column(String(500), default="")
model: Mapped[str] = mapped_column(String(200), default="")
api_mode: Mapped[str] = mapped_column(String(20), default="chat") # chat|completion
temperature: Mapped[float] = mapped_column(Float, default=0.8)
# 800 leaves room for a full scene; 400 tended to truncate mid-paragraph
# and left reasoning models with nothing after their thinking.
max_output_tokens: Mapped[int] = mapped_column(Integer, default=800)
# Separate thinking budget for reasoning models (OpenRouter-style
# `reasoning: {max_tokens}`); 0 = param not sent, -1 = reasoning explicitly
# off (`reasoning: {effort: none}`). Added on top of
# max_output_tokens so story output keeps its full budget.
reasoning_max_tokens: Mapped[int] = mapped_column(Integer, default=0)
context_token_budget: Mapped[int] = mapped_column(Integer, default=16384)
narrator_prompt: Mapped[str] = mapped_column(
Text,
default=(
"You are a masterful storyteller continuing an interactive adventure. "
"Continue the story naturally in second person, staying consistent with "
"everything established so far. Write vivid prose. Never speak for the "
"player or break character. Do not conclude the story; always leave room "
"for the player's next action."
),
)
stream: Mapped[bool] = mapped_column(Boolean, default=True)
# Phase 6: auto-summarization + memory bank
summary_model: Mapped[str] = mapped_column(String(200), default="") # "" = main model
embedding_model: Mapped[str] = mapped_column(String(200), default="") # "" = bank disabled
# Was 200. Lowered on retrieval-quality grounds first: ranking two hundred
# memories to pick five means the five are chosen out of a lot of noise,
# and older memories describe a story the player has moved on from. That it
# also cuts what the bank costs to read is the smaller reason.
memory_bank_capacity: Mapped[int] = mapped_column(Integer, default=80)
memory_top_k: Mapped[int] = mapped_column(Integer, default=5)
@property
def has_api_key(self) -> bool:
return bool(self.api_key)
@property
def api_key_plain(self) -> str:
from . import security # local import: models is imported before security
return security.decrypt_secret(self.api_key)