Store embeddings as packed float32 instead of a JSON list

A 1536-dimension vector spelled out as JSON decimals is ~31 KB. The same
numbers packed as float32 are 6,144 bytes, and the whole bank is read on
every turn, so those bytes are paid over and over.

It is a format change, not a precision trade: the endpoints compute in
float32 and render that into JSON, so converting back recovers the original
bits exactly. Nothing is re-embedded and no API call is made -- migration 38
is a pure repack of what is already stored.

Unlike migrations 36 and 37 this backfill cannot be expressed in portable
SQL, so it comes through Python, batched, and pays a one-time read of every
vector to stop paying three megabytes a turn.

The JSON column stays, still written through set_vector, so a rollback finds
the vectors intact. Reading from the blob comes next; a follow-up migration
drops the old column once that is verified.

Migration SQL can now be a {dialect: sql} map -- BLOB and BYTEA have no
common spelling, and every Postgres deploy replays this one.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015CYEJKobJ2Re4Dv7qUoSA7

backend/app/memorybank.py

@@ -20,14 +20,14 @@ on a later turn because the cursors only advance on success.
 """
 
 import asyncio
-import math
 
 from sqlalchemy.orm import Session
 
-from . import models
+from . import models, vectors
 from .context import history, story_actions, truncate_to_last_tokens
 from .database import SessionLocal
 from .providers import OpenAICompatibleProvider, ProviderError
+from .vectors import cosine  # re-exported: the ranking lives here, the maths there
 
 MEMORY_INTERVAL = 6  # actions per memory
 MEMORY_START = 12  # first memory once the adventure reaches this many actions
@@ -79,14 +79,15 @@ def embedding_provider(settings: models.Settings) -> OpenAICompatibleProvider:
     )
 
 
-def cosine(a: list[float], b: list[float]) -> float:
-    # Different lengths means the embedding model changed since this vector was
-    # stored; zip() would silently score garbage.
-    if len(a) != len(b):
-        return 0.0
-    dot = sum(x * y for x, y in zip(a, b))
-    norm = math.sqrt(sum(x * x for x in a)) * math.sqrt(sum(y * y for y in b))
-    return dot / norm if norm else 0.0
+def set_vector(memory: models.Memory, vector: list[float] | None) -> None:
+    """Store (or clear) a memory's embedding.
+
+    Both columns, always together: `embedding_blob` is what will be read, and
+    the JSON `embedding` stays correct behind it until the follow-up migration
+    drops it. Going through one function is what keeps them from drifting.
+    """
+    memory.embedding = vector
+    memory.embedding_blob = None if vector is None else vectors.pack(vector)
 
 
 def settled_count(adventure: models.Adventure) -> int:
@@ -395,11 +396,11 @@ async def _embed_pending(
     if not pending:
         return
     try:
-        vectors = await embedding_provider(settings).embed([m.text for m in pending])
+        new = await embedding_provider(settings).embed([m.text for m in pending])
     except ProviderError:
         return
-    for memory, vector in zip(pending, vectors):
-        memory.embedding = vector
+    for memory, vector in zip(pending, new):
+        set_vector(memory, vector)
     db.commit()
 
 

backend/app/migrations.py

@@ -17,13 +17,18 @@ starts fresh (created by create_all, stamped LATEST, never replays them), but
 migrations added from Phase 9 on must run on both dialects.
 """
 
+import json
+
 from sqlalchemy import inspect, text
 from sqlalchemy.engine import Engine
 
+from . import vectors
 from .database import Base
 
 # (version, SQL to run when upgrading past it) — append only, never reorder.
-MIGRATIONS: list[tuple[int, str]] = [
+# The SQL is a string, or a {dialect: sql} map with a "default" entry when the
+# two dialects have to be spelled differently (BLOB vs BYTEA and the like).
+MIGRATIONS: list[tuple[int, str | dict[str, str]]] = [
     # Phase 6: auto-summarization + memory bank (the `memories` table itself is
     # created by create_all, which runs for existing DBs too).
     (2, "ALTER TABLE adventures ADD COLUMN auto_summarize BOOLEAN NOT NULL DEFAULT 0"),
@@ -121,6 +126,14 @@ MIGRATIONS: list[tuple[int, str]] = [
     # ~5 KB to every later load of that adventure. Backfilled by
     # _backfill_variant_count.
     (37, "ALTER TABLE actions ADD COLUMN variant_count INTEGER NOT NULL DEFAULT 0"),
+    # Egress, round three: a 1536-dimension embedding written as a JSON list is
+    # ~31 KB, and the whole bank is fetched every turn to rank it. Packed
+    # float32 is 6 KB for the same numbers, exactly (see vectors.py). Dimensions
+    # are unchanged, so this is a format conversion — no re-embedding, no API
+    # calls. Backfilled by _backfill_embedding_blob. The old JSON column is left
+    # in place and keeps being written until a follow-up migration drops it.
+    (38, {"sqlite": "ALTER TABLE memories ADD COLUMN embedding_blob BLOB",
+          "default": "ALTER TABLE memories ADD COLUMN embedding_blob BYTEA"}),
 ]
 
 LATEST_VERSION = max((v for v, _ in MIGRATIONS), default=1)
@@ -128,6 +141,11 @@ LATEST_VERSION = max((v for v, _ in MIGRATIONS), default=1)
 # Migrations that need a data pass after their DDL, keyed by version.
 WORLD_DELTA_VERSION = 36
 VARIANT_COUNT_VERSION = 37
+EMBEDDING_BLOB_VERSION = 38
+
+# Vectors converted per round trip. Small enough that the backfill never holds
+# more than a few megabytes, large enough that it isn't a query per row.
+BACKFILL_BATCH = 200
 
 
 def _backfill_world_delta(conn) -> None:
@@ -183,6 +201,46 @@ def _backfill_variant_count(conn) -> None:
     conn.execute(text(sql))
 
 
+def _for_dialect(sql: str | dict[str, str], dialect: str) -> str:
+    return sql if isinstance(sql, str) else sql.get(dialect, sql["default"])
+
+
+def _backfill_embedding_blob(conn) -> None:
+    """Repack memories.embedding (JSON list) into memories.embedding_blob.
+
+    The one backfill here that has to come through Python: struct packing has
+    no portable SQL spelling, so unlike migrations 36 and 37 this pays a
+    one-time read of every vector (~4 MB in production) to stop paying three
+    megabytes every turn. Batched so the read is bounded whatever the bank
+    grows to.
+
+    Reads the JSON defensively — SQLite hands back the raw string while psycopg
+    has already parsed it into a list — and skips anything that isn't a
+    non-empty list, so one malformed row can't strand the migration.
+    """
+    last_id = 0
+    while True:
+        rows = conn.execute(
+            text("""
+                SELECT id, embedding FROM memories
+                WHERE embedding IS NOT NULL AND embedding_blob IS NULL AND id > :last
+                ORDER BY id LIMIT :batch
+            """),
+            {"last": last_id, "batch": BACKFILL_BATCH},
+        ).all()
+        if not rows:
+            return
+        for row_id, stored in rows:
+            vector = json.loads(stored) if isinstance(stored, str) else stored
+            if not isinstance(vector, list) or not vector:
+                continue
+            conn.execute(
+                text("UPDATE memories SET embedding_blob = :blob WHERE id = :id"),
+                {"blob": vectors.pack(vector), "id": row_id},
+            )
+        last_id = rows[-1][0]
+
+
 def _get_version(conn) -> int:
     if conn.dialect.name == "sqlite":
         return conn.execute(text("PRAGMA user_version")).scalar() or 1
@@ -220,11 +278,13 @@ def bootstrap(engine: Engine) -> None:
         current = _get_version(conn)
         for version, sql in MIGRATIONS:
             if version > current:
-                conn.execute(text(sql))
+                conn.execute(text(_for_dialect(sql, conn.dialect.name)))
                 if version == WORLD_DELTA_VERSION:
                     _backfill_world_delta(conn)
                 if version == VARIANT_COUNT_VERSION:
                     _backfill_variant_count(conn)
+                if version == EMBEDDING_BLOB_VERSION:
+                    _backfill_embedding_blob(conn)
                 current = version
         _set_version(conn, current)
         _encrypt_plaintext_api_keys(conn)

backend/app/models.py

@@ -1,7 +1,8 @@
 from datetime import datetime, timezone
 
 from sqlalchemy import (
-    JSON, Boolean, Column, DateTime, Float, ForeignKey, Integer, String, Table, Text,
+    JSON, Boolean, Column, DateTime, Float, ForeignKey, Integer, LargeBinary, String,
+    Table, Text,
 )
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
@@ -141,9 +142,15 @@ class Adventure(Base):
 class Memory(Base):
     """Phase 6: an auto-summarized (or hand-written) fact about the adventure.
 
-    `embedding` is the raw vector as a JSON list (cosine ranking happens in
-    Python — fine at bank sizes of a few hundred). NULL until embedded, which
-    also marks it for backfill when an embedding model becomes available.
+    The vector lives in `embedding_blob` as packed float32 (see vectors.py).
+    NULL until embedded, which also marks it for backfill when an embedding
+    model becomes available.
+
+    Cosine ranking happens in Python, which means the vectors cross the wire.
+    The original comment here sized that by count — "fine at a few hundred" —
+    and it was wrong by the only measure that mattered: a few hundred JSON
+    vectors is ten megabytes, fetched fresh every turn. Weigh new columns in
+    bytes.
     """
 
     __tablename__ = "memories"
@@ -151,7 +158,17 @@ class Memory(Base):
     id: Mapped[int] = mapped_column(primary_key=True)
     adventure_id: Mapped[int] = mapped_column(ForeignKey("adventures.id", ondelete="CASCADE"))
     text: Mapped[str] = mapped_column(Text, default="")
+    # Superseded by embedding_blob and written alongside it, so the two stay in
+    # step until a follow-up migration drops this one. Still the column the
+    # ranking path reads; that moves next.
     embedding: Mapped[list | None] = mapped_column(JSON, nullable=True)
+    # The vector, little-endian float32. Deferred because it is wider than the
+    # rest of the row put together and exactly one code path wants it: anything
+    # bulk-loading memories (the Memories drawer, eviction, the embed queue)
+    # must project the columns it needs rather than load whole entities.
+    embedding_blob: Mapped[bytes | None] = mapped_column(
+        LargeBinary, nullable=True, deferred=True
+    )
     # Action index range this memory summarizes (null for manual memories).
     source_start: Mapped[int | None] = mapped_column(Integer, nullable=True)
     source_end: Mapped[int | None] = mapped_column(Integer, nullable=True)

backend/app/routers/adventures.py

@@ -1488,7 +1488,7 @@ def update_memory(
         raise HTTPException(404, "Memory not found")
     fields = {k: v for k, v in payload.model_dump(exclude_unset=True).items() if v is not None}
     if "text" in fields and fields["text"].strip() != memory.text:
-        memory.embedding = None  # re-embed on the next post-turn pass
+        memorybank.set_vector(memory, None)  # re-embed on the next post-turn pass
     for field, value in fields.items():
         setattr(memory, field, value)
     db.commit()

backend/app/vectors.py

@@ -0,0 +1,40 @@
+"""Storing and comparing embedding vectors.
+
+A 1536-dimension vector written as a JSON list is about 31 KB, because every
+component is spelled out as a decimal string of seventeen-odd digits. The same
+vector as packed float32 is 6,144 bytes — a straight 5x, and the memory bank is
+read in full on every turn, so those bytes are paid over and over.
+
+**Float32 is not an approximation here.** The embedding endpoints return
+vectors computed in float32, rendered into JSON as the shortest decimal string
+that round-trips through a double; converting that back to float32 recovers the
+original bits exactly. Nothing is lost that was ever there, which is why the
+conversion needs no re-embedding and carries no retrieval-quality risk.
+
+Dimensions are deliberately unchanged. Dropping to 512 or 768 would have saved
+another 3x and cost an API call per stored memory to re-embed, against a bank
+that the packing and the in-process cache together already make cheap.
+"""
+
+import math
+import struct
+
+
+def pack(vector: list[float]) -> bytes:
+    """A vector as little-endian float32."""
+    return struct.pack(f"<{len(vector)}f", *vector)
+
+
+def unpack(blob: bytes) -> list[float]:
+    """The inverse of `pack`. Length is implied: four bytes per component."""
+    return list(struct.unpack(f"<{len(blob) // 4}f", blob))
+
+
+def cosine(a: list[float], b: list[float]) -> float:
+    # Different lengths means the embedding model changed since this vector was
+    # stored; zip() would silently score garbage.
+    if len(a) != len(b):
+        return 0.0
+    dot = sum(x * y for x, y in zip(a, b))
+    norm = math.sqrt(sum(x * x for x in a)) * math.sqrt(sum(y * y for y in b))
+    return dot / norm if norm else 0.0