interactive-story/backend/tests/test_m11_migration.py

"""M11 §17: a fresh install and an upgraded database must be the same product.

M10 found the defect this file makes permanent. It shipped a `CREATE INDEX`
migration for an index `create_all` already built from the column, so an
*upgraded* database ended up with two indexes and a fresh one with a single
index — two schemas differing by which path the file took, which is the thing a
migration exists to prevent. Nothing found it except comparing the two.

So the comparison is the test, and it is written to be general rather than about
`visual_profiles`: every table, every column with its type and nullability,
every index and its uniqueness, every foreign key, and the version stamp. A
future migration that diverges the two paths fails here whatever it is about.

The second half is the upgrade itself: a database built by the **previous
supported build** — M10's schema, version 92 — opened by this one, and then
played, exported and imported, because a migration that leaves a campaign
unplayable has not worked.

    python -m pytest tests/test_m11_migration.py -v
"""

import json
import sqlite3
import subprocess
import sys
import tempfile
from pathlib import Path

import pytest
from sqlalchemy import create_engine, inspect, text
from sqlalchemy.orm import sessionmaker

from app import backup, migrations, models
from app.database import Base

BACKEND = Path(__file__).resolve().parent.parent

#: The schema M10 shipped: everything this build has, minus what M11 added.
#: Expressed as the inverse of M11's own migrations, which is what
#: `schema_rewind` does for the suite generally — repeated here as data so this
#: file states plainly what "the previous supported build" means.
M10_VERSION = 92
M11_ADDITIONS = (("adventures", "narration_length"),)


def _describe(engine) -> dict:
    """Everything about a schema that two databases could disagree about."""
    inspector = inspect(engine)
    out: dict = {"tables": {}}
    for table in sorted(inspector.get_table_names()):
        if table.startswith("sqlite_"):
            continue
        columns = {
            c["name"]: {
                "type": str(c["type"]),
                "nullable": bool(c["nullable"]),
                # `default` is rendered differently by different paths (a Python
                # default never reaches the DDL), so it is deliberately not
                # compared; `nullable` and type are what a query can depend on.
            }
            for c in inspector.get_columns(table)
        }
        indexes = {
            i["name"]: {"columns": list(i["column_names"]),
                        "unique": bool(i.get("unique"))}
            for i in inspector.get_indexes(table)
        }
        foreign_keys = sorted(
            (tuple(fk["constrained_columns"]), fk["referred_table"],
             tuple(fk["referred_columns"]))
            for fk in inspector.get_foreign_keys(table)
        )
        out["tables"][table] = {
            "columns": columns, "indexes": indexes, "foreign_keys": foreign_keys,
            "primary_key": inspector.get_pk_constraint(table).get(
                "constrained_columns", []),
        }
    with engine.begin() as conn:
        out["version"] = conn.execute(text("PRAGMA user_version")).scalar()
    return out


@pytest.fixture()
def fresh(tmp_path):
    """A database as a new installation creates one."""
    path = tmp_path / "fresh.db"
    engine = create_engine(f"sqlite:///{path}")
    migrations.bootstrap(engine)
    yield path, engine
    engine.dispose()


@pytest.fixture()
def upgraded(tmp_path):
    """A database as the previous supported build left it, then opened by this one.

    Built by creating the current schema, removing what M11 added, and stamping
    the version M10 ended on — which is what an M10-era file *is*, since M10
    added no migration of its own.
    """
    path = tmp_path / "upgraded.db"
    older = create_engine(f"sqlite:///{path}")
    Base.metadata.create_all(bind=older)
    with sessionmaker(bind=older)() as db:
        # Owned by the implicit local user, which is the row `auth.local_user`
        # resolves to — an email-less, non-guest user. A campaign owned by
        # nobody would not be listed by the server, and the test would be
        # measuring ownership rather than migration.
        owner = models.User(is_guest=False, email=None)
        db.add(owner)
        db.flush()
        adventure = models.Adventure(title="An M10 campaign", user_id=owner.id)
        db.add(adventure)
        db.flush()
        branch = models.Branch(adventure_id=adventure.id, parent_branch_id=None,
                               fork_depth=None, lineage=[])
        db.add(branch)
        db.flush()
        adventure.head_branch_id = branch.id
        adventure.head_depth = 0
        db.add(models.Action(adventure_id=adventure.id, type="start",
                             text="Written before M11 existed.",
                             branch_id=branch.id, depth=0))
        db.add(models.Checkpoint(adventure_id=adventure.id, name="Old point",
                                 branch_id=branch.id, depth=0))
        db.commit()
        adv_id = adventure.id
    with older.begin() as conn:
        for table, column in M11_ADDITIONS:
            conn.execute(text(f"ALTER TABLE {table} DROP COLUMN {column}"))
        conn.execute(text(f"PRAGMA user_version = {M10_VERSION}"))
    older.dispose()

    engine = create_engine(f"sqlite:///{path}")
    yield path, engine, adv_id
    engine.dispose()


# ------------------------------------------------------ the parity comparison

def test_the_two_paths_produce_the_same_schema(fresh, upgraded):
    """M10's defect, as a permanent release regression."""
    fresh_path, fresh_engine = fresh
    up_path, up_engine, _ = upgraded
    migrations.bootstrap(up_engine)

    a, b = _describe(fresh_engine), _describe(up_engine)
    assert set(a["tables"]) == set(b["tables"]), (
        sorted(set(a["tables"]) ^ set(b["tables"])))
    for table in sorted(a["tables"]):
        assert a["tables"][table] == b["tables"][table], (
            f"{table} differs between a fresh install and an upgrade:\n"
            f"fresh:    {json.dumps(a['tables'][table], indent=2, sort_keys=True)}\n"
            f"upgraded: {json.dumps(b['tables'][table], indent=2, sort_keys=True)}"
        )
    assert a["version"] == b["version"] == migrations.LATEST_VERSION


def test_no_table_carries_a_duplicate_index(fresh):
    """The specific shape of M10's defect: two indexes over the same columns."""
    _, engine = fresh
    described = _describe(engine)
    for table, shape in described["tables"].items():
        seen: dict[tuple, str] = {}
        for name, index in shape["indexes"].items():
            key = (tuple(index["columns"]), index["unique"])
            assert key not in seen, (
                f"{table}: {name} duplicates {seen[key]} over {key[0]}")
            seen[key] = name


def test_every_table_the_models_declare_exists(fresh):
    """A missing table is the other way this can go wrong (`visual_profiles`)."""
    _, engine = fresh
    have = set(inspect(engine).get_table_names())
    declared = set(Base.metadata.tables)
    assert declared <= have, sorted(declared - have)
    assert "visual_profiles" in have
    assert "narration_length" in {
        c["name"] for c in inspect(engine).get_columns("adventures")}


# -------------------------------------------------------------- the upgrade

def test_an_m10_database_upgrades_without_losing_anything(upgraded):
    path, engine, adv_id = upgraded
    migrations.bootstrap(engine)
    with engine.begin() as conn:
        assert conn.execute(text("SELECT title FROM adventures")).scalar() == (
            "An M10 campaign")
        assert conn.execute(text("SELECT text FROM actions")).scalar() == (
            "Written before M11 existed.")
        assert conn.execute(text("SELECT name FROM checkpoints")).scalar() == "Old point"
        assert conn.execute(text("PRAGMA foreign_key_check")).fetchall() == []
        assert conn.execute(text("PRAGMA quick_check")).scalar() == "ok"


def test_the_new_column_arrives_with_the_value_that_means_no_choice(upgraded):
    """M11's migration, and why it needs no backfill.

    An empty narration length is not a missing value: it is the campaign saying
    nothing about length, which is exactly what a campaign created before the
    setting existed did say. `length_hint` treats it as it treated everything
    before M11, so no existing campaign's prompt changes under the upgrade.
    """
    path, engine, adv_id = upgraded
    migrations.bootstrap(engine)
    with engine.begin() as conn:
        assert conn.execute(text("SELECT narration_length FROM adventures")).scalar() == ""


def test_opening_an_upgraded_database_repeatedly_changes_nothing(upgraded):
    path, engine, _ = upgraded
    migrations.bootstrap(engine)
    first = _describe(engine)
    for _ in range(3):
        migrations.bootstrap(engine)
    assert _describe(engine) == first


def test_a_migrated_database_still_plays_and_still_travels(upgraded, tmp_path):
    """A migration that leaves a campaign unopenable has not worked.

    Played through a real server process against the migrated file, because the
    claim is about the file rather than about an ORM session.
    """
    sys.path.insert(0, str(BACKEND / "tests"))
    from test_process_restart import Server, _free_port

    path, engine, adv_id = upgraded
    migrations.bootstrap(engine)
    engine.dispose()

    server = Server(str(path), _free_port())
    try:
        server.wait_until_ready()
        listed = server.call("GET", "/adventures", expect=200)
        assert any(a["title"] == "An M10 campaign" for a in listed)
        server.call("POST", f"/adventures/{adv_id}/state/corrections", {
            "events": [{"type": "create_entity", "entity": "aldric",
                        "entity_type": "character", "name": "Aldric"}],
            "note": "after the migration",
        }, expect=201)
        bundle = server.call("GET", f"/adventures/{adv_id}/export", expect=200)
        assert bundle["format"] == "ai-dnd-adventure-v3"
        copy = server.call("POST", "/adventures/import", bundle, expect=201)
        state = server.call("GET", f"/adventures/{copy['id']}/state", expect=200)
        assert state["document"]["entities"]["aldric"]["name"] == "Aldric"
    finally:
        server.stop()


def test_a_backup_of_the_migrated_database_verifies(upgraded):
    """M9's backup, on a file M11 changed the schema of."""
    path, engine, _ = upgraded
    migrations.bootstrap(engine)
    engine.dispose()
    result = backup.create(path)
    try:
        assert result.integrity == "ok"
        with sqlite3.connect(f"file:{result.path}?mode=ro", uri=True) as copy_db:
            assert copy_db.execute("PRAGMA quick_check").fetchone()[0] == "ok"
            assert copy_db.execute(
                "SELECT narration_length FROM adventures").fetchone()[0] == ""
            assert copy_db.execute("PRAGMA user_version").fetchone()[0] == (
                migrations.LATEST_VERSION)
    finally:
        result.path.unlink(missing_ok=True)


def test_a_fresh_install_creates_a_database_from_nothing(tmp_path):
    """§17's first case, through a real process rather than through the ORM."""
    sys.path.insert(0, str(BACKEND / "tests"))
    from test_process_restart import Server, _free_port

    path = tmp_path / "new" / "campaign.db"
    path.parent.mkdir()
    server = Server(str(path), _free_port())
    try:
        server.wait_until_ready()
        assert path.exists(), "no database was created"
        created = server.call("POST", "/adventures",
                              {"title": "Brand new", "opening": "Rain."}, expect=201)
        assert created["narration_length"] == ""
    finally:
        server.stop()
    with sqlite3.connect(f"file:{path}?mode=ro", uri=True) as db:
        assert db.execute("PRAGMA user_version").fetchone()[0] == (
            migrations.LATEST_VERSION)
        tables = {r[0] for r in db.execute(
            "SELECT name FROM sqlite_master WHERE type='table'")}
    assert {"adventures", "actions", "visual_profiles", "summaries",
            "knowledge_sources"} <= tables