interactive-story/backend/tests/test_process_restart.py

"""D11 and L03 across a genuine OS process boundary.

The rest of the suite runs the app in-process through `TestClient`, which is the
right tool for almost everything and the wrong one for exactly one claim:
*durability*. A Save Point that survived only because a Python object was still
alive would pass a same-process test and fail a user's restart. `TestClient`
cannot tell those apart, so the M4 review recorded the shipped D11/L03 tests as
weaker than the acceptance items they were named for (§R B-3).

This module closes that. It starts the real application as a **subprocess**,
plays a story over HTTP, kills the process, starts a **second** process against
the same database file, and only then asks whether the Save Point is still
there. Everything crossing the boundary crosses it as bytes on disk.

Deterministic and local: the spawned server replaces the model with a scripted
provider (`_restart_server.py`), so there is no Ollama, no network and no
sleep-and-hope — readiness is probed, not waited for.

    python -m pytest tests/test_process_restart.py -v
"""
import json
import os
import socket
import subprocess
import sys
import tempfile
import time
import urllib.error
import urllib.request
from pathlib import Path

import pytest

from fakes import GOLD_PER_TURN

HERE = Path(__file__).resolve().parent
SERVER = HERE / "_restart_server.py"

# How long a spawned server may take to answer before the test gives up. The
# process imports the app and runs migrations on a fresh file, which is well
# under a second on this project; the ceiling is for a loaded machine.
STARTUP_TIMEOUT = 60.0


def _free_port() -> int:
    """Returns a port nothing is listening on.

    Bind, read, release. There is a race between releasing and the server
    claiming it, which is why the caller probes for readiness rather than
    assuming success — a lost race shows up as a startup timeout, not as a
    silent pass.
    """
    with socket.socket() as s:
        s.bind(("127.0.0.1", 0))
        return s.getsockname()[1]


class Server:
    """One storyteller process, and the HTTP calls the test makes against it."""

    def __init__(self, db_path: str, port: int):
        self.port = port
        self.proc = subprocess.Popen(
            [sys.executable, str(SERVER), db_path, str(port)],
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT,
            # Never inherit the parent's database redirection; the child is told
            # which file to open on its command line.
            env={**os.environ, "AIDND_DB_PATH": db_path},
        )

    # ------------------------------------------------------------ lifecycle

    def wait_until_ready(self) -> None:
        deadline = time.monotonic() + STARTUP_TIMEOUT
        while time.monotonic() < deadline:
            if self.proc.poll() is not None:
                raise AssertionError(
                    f"server exited early ({self.proc.returncode}):\n{self._output()}"
                )
            try:
                self.call("GET", "/settings")
                return
            except (urllib.error.URLError, ConnectionError, OSError):
                time.sleep(0.05)
        raise AssertionError(f"server never became ready:\n{self._output()}")

    def stop(self) -> None:
        """Ends the process, and does not return until it is actually gone."""
        if self.proc.poll() is None:
            self.proc.terminate()
            try:
                self.proc.wait(timeout=15)
            except subprocess.TimeoutExpired:
                self.proc.kill()
                self.proc.wait(timeout=15)
        if self.proc.stdout is not None:
            self.proc.stdout.close()

    def _output(self) -> str:
        if self.proc.stdout is None:
            return "(no output captured)"
        try:
            return self.proc.stdout.read().decode(errors="replace")[-2000:]
        except Exception:
            return "(output unreadable)"

    def is_listening(self) -> bool:
        try:
            self.call("GET", "/settings")
            return True
        except Exception:
            return False

    # ---------------------------------------------------------------- HTTP

    def call(self, method: str, path: str, payload=None, expect: int | None = None):
        data = json.dumps(payload).encode() if payload is not None else None
        request = urllib.request.Request(
            f"http://127.0.0.1:{self.port}/api{path}",
            data=data,
            method=method,
            headers={"Content-Type": "application/json"},
        )
        try:
            with urllib.request.urlopen(request, timeout=60) as response:
                body, status = response.read(), response.status
        except urllib.error.HTTPError as exc:          # a real answer, not a failure
            body, status = exc.read(), exc.code
        if expect is not None and status != expect:
            raise AssertionError(f"{method} {path} -> {status}: {body[:400]!r}")
        return json.loads(body) if body and status != 204 else None

    def play(self, adventure_id: int, text: str) -> None:
        """Plays one turn through the streaming endpoint, to completion."""
        request = urllib.request.Request(
            f"http://127.0.0.1:{self.port}/api/adventures/{adventure_id}/actions",
            data=json.dumps({"type": "do", "text": text}).encode(),
            method="POST",
            headers={"Content-Type": "application/json"},
        )
        with urllib.request.urlopen(request, timeout=120) as response:
            response.read()

    # ------------------------------------------------------------- reading

    def transcript(self, adventure_id: int) -> list[str]:
        page = self.call("GET", f"/adventures/{adventure_id}", expect=200)
        return [a["text"] for a in page["actions"]]

    def gold(self, adventure_id: int) -> int:
        state = self.call("GET", f"/adventures/{adventure_id}/world-state", expect=200)
        return state["state"]["player"]["gold"]

    def total_rows(self, adventure_id: int) -> int:
        """Every row of the whole tree, head or no head.

        `action_count` is scoped to the path being read, so it falls when the
        head moves back — which is the feature, not a deletion. The export
        carries the entire tree whatever the head is doing, so it is what
        "nothing was deleted" has to be measured against.
        """
        bundle = self.call("GET", f"/adventures/{adventure_id}/export", expect=200)
        return len(bundle["actions"])


@pytest.fixture()
def workspace():
    """A database file, and whichever servers a test starts against it."""
    directory = tempfile.mkdtemp(prefix="m4-restart-")
    db_path = os.path.join(directory, "campaign.db")
    started: list[Server] = []

    def start() -> Server:
        server = Server(db_path, _free_port())
        started.append(server)
        server.wait_until_ready()
        return server

    try:
        yield start
    finally:
        # Every child dies even if the test failed part way through, and each
        # stop() waits, so a later test cannot inherit a live listener.
        for server in started:
            server.stop()


def _campaign_with_a_save_point(server: Server):
    """Three turns, a Save Point on the third, then four more turns.

    Returns everything the second process has to be able to reproduce.
    """
    scenario = server.call("POST", "/scenarios", {
        "title": "Abbey",
        "stat_schema": {"player": {"gold": {"initial": 0, "min": 0, "max": 9999}}},
    }, expect=201)
    adventure = server.call("POST", "/adventures", {
        "title": "The abbey", "scenario_id": scenario["id"],
    }, expect=201)
    adventure_id = adventure["id"]

    for n in range(3):
        server.play(adventure_id, f"turn {n}")
    at_save = {
        "transcript": server.transcript(adventure_id),
        "gold": server.gold(adventure_id),
    }
    save_point = server.call("POST", f"/adventures/{adventure_id}/checkpoints",
                             {"name": "Before entering the abbey"}, expect=201)

    for n in range(4):
        server.play(adventure_id, f"later {n}")
    at_tip = {
        "transcript": server.transcript(adventure_id),
        "gold": server.gold(adventure_id),
        "rows": server.total_rows(adventure_id),
    }
    return adventure_id, save_point, at_save, at_tip


def test_d11_l03_a_save_point_survives_a_real_process_restart(workspace):
    """D11 and L03 together, across a boundary a same-process test cannot cross.

    The first process writes the campaign and exits. The second process is a
    different interpreter with an empty session, an empty identity map and no
    memory of anything — everything it knows, it reads off the disk.
    """
    first = workspace()
    adventure_id, save_point, at_save, at_tip = _campaign_with_a_save_point(first)

    assert at_tip["gold"] == at_save["gold"] + 4 * GOLD_PER_TURN
    assert len(at_tip["transcript"]) == len(at_save["transcript"]) + 8

    # --- the boundary -----------------------------------------------------
    first.stop()
    assert first.proc.poll() is not None, "the first server did not actually exit"
    assert not first.is_listening(), "the first server is still answering"

    second = workspace()
    assert second.proc.pid != first.proc.pid

    # --- D11: the Save Point is still there -------------------------------
    listed = second.call("GET", f"/adventures/{adventure_id}/checkpoints", expect=200)
    assert [c["id"] for c in listed] == [save_point["id"]]
    assert listed[0]["name"] == "Before entering the abbey"
    assert (listed[0]["branch_id"], listed[0]["depth"]) == (
        save_point["branch_id"], save_point["depth"]
    )
    assert listed[0]["resolved"] is True
    # The story came back too, at the position the first process left it.
    assert second.transcript(adventure_id) == at_tip["transcript"]

    rows_before_restore = second.total_rows(adventure_id)
    assert rows_before_restore == at_tip["rows"]

    # --- L03: restoring reconstructs the historical position --------------
    page = second.call(
        "POST", f"/adventures/{adventure_id}/checkpoints/{save_point['id']}/restore",
        expect=200,
    )
    assert [a["text"] for a in page["actions"]] == at_save["transcript"]
    assert second.transcript(adventure_id) == at_save["transcript"]
    assert second.gold(adventure_id) == at_save["gold"]

    # --- restore deleted nothing, and Redo is still available -------------
    assert second.total_rows(adventure_id) == rows_before_restore
    assert page["can_redo"] is True
    assert second.call("GET", f"/adventures/{adventure_id}", expect=200)["can_redo"] is True


def test_the_retained_continuation_is_reachable_after_a_restart(workspace):
    """The later history is not merely present in the database after a restart —
    it is still the continuation the story tells, walkable by ordinary Redo."""
    first = workspace()
    adventure_id, save_point, at_save, at_tip = _campaign_with_a_save_point(first)
    first.stop()

    second = workspace()
    second.call("POST",
                f"/adventures/{adventure_id}/checkpoints/{save_point['id']}/restore",
                expect=200)
    assert second.transcript(adventure_id) == at_save["transcript"]

    steps = 0
    while second.call("GET", f"/adventures/{adventure_id}", expect=200)["can_redo"]:
        second.call("POST", f"/adventures/{adventure_id}/redo", expect=200)
        steps += 1
        assert steps <= 10, "Redo never stopped"

    assert steps == 4
    assert second.transcript(adventure_id) == at_tip["transcript"]
    assert second.gold(adventure_id) == at_tip["gold"]
    assert second.total_rows(adventure_id) == at_tip["rows"]


def test_a_divergent_write_after_a_restart_keeps_the_displaced_future(workspace):
    """D13's second half, across the boundary: the fork still happens on the
    write rather than on the restore, and the displaced turns keep their rows."""
    first = workspace()
    adventure_id, save_point, at_save, at_tip = _campaign_with_a_save_point(first)
    first.stop()

    second = workspace()
    second.call("POST",
                f"/adventures/{adventure_id}/checkpoints/{save_point['id']}/restore",
                expect=200)
    branches_before = second.call("GET", f"/adventures/{adventure_id}/branches",
                                  expect=200)
    assert len(branches_before) == 1, "restore must not fork"

    second.play(adventure_id, "go around the back")

    branches_after = second.call("GET", f"/adventures/{adventure_id}/branches",
                                 expect=200)
    assert len(branches_after) == 2, "the write should have forked"
    # Nothing was deleted to achieve it: the tree grew by the new turn only.
    assert second.total_rows(adventure_id) == at_tip["rows"] + 2
    # Ordinary Redo no longer offers the displaced future.
    assert second.call("GET", f"/adventures/{adventure_id}", expect=200)["can_redo"] is False
    # And the Save Point still names the position it always did.
    listed = second.call("GET", f"/adventures/{adventure_id}/checkpoints", expect=200)
    assert (listed[0]["branch_id"], listed[0]["depth"]) == (
        save_point["branch_id"], save_point["depth"]
    )