interactive-story/backend/tools/v11_release_smoke.py

"""v1.1 release smoke test: the shipped image, as a reader would meet it.

    python -m tools.v11_release_smoke --image <tag> --out <dir under $HOME>

Run from `backend/`. Reads `AIDND_TEST_ENDPOINT` (an **HTTPS** Ollama on the
trusted LAN) and `AIDND_TEST_MODEL`. `--ca` names the private CA to install
inside the container, defaulting to this machine's own.

Supplemental release evidence, not a replacement for the gates: it asks whether
the artefact that ships actually runs, reaches its approved narrator, refuses an
unapproved one, and keeps a campaign across a container restart.

## The two things this is careful about

**The CA is installed, not bypassed.** `app/tlstrust.ssl_context()` is
`ssl.create_default_context()` — the platform's own store — unioned with
certifi's. So the private CA is mounted into
`/usr/local/share/ca-certificates/` and registered with
`update-ca-certificates`, and verification is then ordinary. Nothing sets
`verify=False`, and a check inside the container proves the handshake succeeds
through that store.

**Loopback means the published port.** The process inside the container listens
on `0.0.0.0` because that is the only address a published port can reach
(`docker-compose.yml` says so). What must be loopback-only is the *publish*, so
the container is started with `-p 127.0.0.1:<port>:8000` and the check is that
the host's LAN address refuses the same port.
"""

from __future__ import annotations

import argparse
import json
import os
import socket
import subprocess
import sys
import time
import urllib.error
import urllib.request
from datetime import datetime
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

from tools.m11_webdriver import Browser, free_port, require_under_home  # noqa: E402

ENDPOINT = os.environ.get("AIDND_TEST_ENDPOINT", "")
MODEL = os.environ.get("AIDND_TEST_MODEL", "")
NAME = "v11-release-smoke"
VOLUME = "v11-release-smoke-data"
#: An endpoint the policy must refuse whatever else is true: a public host.
PUBLIC_ENDPOINT = "https://api.openai.com/v1"


class Checks:
    def __init__(self) -> None:
        self.rows: list[dict] = []

    def record(self, name: str, ok: bool, detail: str = "") -> bool:
        self.rows.append({"check": name, "result": "PASS" if ok else "FAIL",
                          "detail": detail})
        print(f"  {'ok  ' if ok else 'FAIL'} {name}" + (f" — {detail}" if detail else ""),
              flush=True)
        return ok

    @property
    def failed(self) -> list[dict]:
        return [r for r in self.rows if r["result"] == "FAIL"]


def run(*args: str, **kwargs) -> subprocess.CompletedProcess:
    return subprocess.run(args, capture_output=True, text=True, **kwargs)


def api(base: str, method: str, path: str, payload=None, timeout=900):
    data = json.dumps(payload).encode() if payload is not None else None
    request = urllib.request.Request(
        f"{base}/api{path}", data=data, method=method,
        headers={"Content-Type": "application/json"} if data else {})
    with urllib.request.urlopen(request, timeout=timeout) as response:
        body = response.read().decode()
    return json.loads(body) if body else None


def stream_turn(base: str, adv: int, text: str) -> list[dict]:
    request = urllib.request.Request(
        f"{base}/api/adventures/{adv}/actions",
        data=json.dumps({"type": "do", "text": text}).encode(),
        method="POST", headers={"Content-Type": "application/json"})
    events: list[dict] = []
    with urllib.request.urlopen(request, timeout=900) as response:
        for raw in response:
            line = raw.decode(errors="replace").strip()
            if line.startswith("data:"):
                try:
                    events.append(json.loads(line[5:].strip()))
                except json.JSONDecodeError:
                    pass
    return events


def lan_address() -> str | None:
    """This machine's own LAN address, for the loopback-only check."""
    probe = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    try:
        probe.connect(("192.0.2.1", 9))  # TEST-NET-1: routed nowhere, sends nothing
        return probe.getsockname()[0]
    except OSError:
        return None
    finally:
        probe.close()


def wait_ready(base: str, *, timeout: float = 180) -> bool:
    deadline = time.monotonic() + timeout
    while time.monotonic() < deadline:
        try:
            urllib.request.urlopen(f"{base}/api/settings", timeout=3)
            return True
        except Exception:
            time.sleep(1)
    return False


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--image", required=True)
    parser.add_argument("--out", required=True)
    parser.add_argument("--ca", default="/usr/local/share/ca-certificates/draco.crt")
    parser.add_argument(
        "--add-host", default="", metavar="NAME:ADDRESS",
        help=("resolve the narrator's hostname inside the container. A `.local` "
              "name is mDNS, and a container has no mDNS resolver, so the "
              "endpoint policy refuses an address it cannot classify and "
              "`PUT /api/settings` answers 400. Mapping the name — rather than "
              "using the address — keeps the hostname the certificate is issued "
              "for, which is the thing this test verifies."))
    args = parser.parse_args()

    if not (ENDPOINT and MODEL):
        print("set AIDND_TEST_ENDPOINT (https://…) and AIDND_TEST_MODEL")
        return 2
    if not ENDPOINT.startswith("https://"):
        print("the smoke test needs an HTTPS endpoint: that is what it verifies")
        return 2
    ca = Path(args.ca)
    if not ca.exists():
        print(f"no CA at {ca}")
        return 2

    out = require_under_home(Path(args.out).expanduser())
    out.mkdir(parents=True, exist_ok=True)
    checks = Checks()
    port = free_port()
    base = f"http://127.0.0.1:{port}"
    started = datetime.now()

    run("docker", "rm", "-f", NAME)
    run("docker", "volume", "rm", VOLUME)
    run("docker", "volume", "create", VOLUME)

    print(f"starting {args.image} on 127.0.0.1:{port} with a fresh volume …")
    start = run(
        "docker", "run", "-d", "--name", NAME,
        "-p", f"127.0.0.1:{port}:8000",
        "-v", f"{VOLUME}:/data",
        "-v", f"{ca}:/usr/local/share/ca-certificates/{ca.name}:ro",
        *(("--add-host", args.add_host) if args.add_host else ()),
        args.image,
        "sh", "-c",
        "update-ca-certificates >/dev/null 2>&1; "
        "exec uvicorn app.main:app --host 0.0.0.0 --port 8000",
    )
    if start.returncode != 0:
        print(start.stderr[:400])
        return 1
    container = start.stdout.strip()[:12]

    try:
        checks.record("the container starts", True, container)
        ready = wait_ready(base)
        if not checks.record("the application answers on loopback", ready, base):
            logs = run("docker", "logs", NAME)
            (out / "container.log").write_text(logs.stdout + logs.stderr)
            return 1

        published = run("docker", "port", NAME).stdout.strip()
        checks.record("the port is published on loopback only",
                      "127.0.0.1" in published and "0.0.0.0" not in published, published)

        lan = lan_address()
        if lan:
            try:
                urllib.request.urlopen(f"http://{lan}:{port}/api/settings", timeout=4)
                reachable = True
            except Exception:
                reachable = False
            checks.record("the LAN address does not serve the application", not reachable,
                          f"port {port} on this machine's LAN address")

        page = urllib.request.urlopen(base + "/", timeout=30)
        html = page.read().decode(errors="replace")
        checks.record("the first page loads", page.status == 200 and "<div id=\"root\"" in html,
                      f"HTTP {page.status}, {len(html)} bytes")
        remote = [chunk for chunk in html.split('"')
                  if chunk.startswith("http://") or chunk.startswith("https://")]
        checks.record("the shell references no remote origin", not remote, str(remote[:3]))
        csp = page.headers.get("content-security-policy") or ""
        checks.record("a CSP is served", bool(csp), csp[:80])

        # The approved endpoint, verified through the private CA *inside* the
        # container, with the application's own trust context and no bypass.
        probe = run("docker", "exec", NAME, "python", "-c",
                    "import json,urllib.request,ssl,sys;"
                    "sys.path.insert(0,'/app/backend');"
                    "from app.tlstrust import ssl_context;"
                    f"r=urllib.request.urlopen('{ENDPOINT}/models',"
                    " timeout=20, context=ssl_context());"
                    "print(r.status)")
        checks.record("the approved HTTPS narrator verifies through the private CA",
                      probe.returncode == 0 and "200" in probe.stdout,
                      (probe.stdout + probe.stderr).strip()[:160])

        api(base, "PUT", "/settings", {"endpoint_url": ENDPOINT, "model": MODEL,
                                       "max_output_tokens": 300,
                                       "model_timeout_seconds": 600})
        try:
            api(base, "PUT", "/settings", {"endpoint_url": PUBLIC_ENDPOINT, "model": MODEL})
            refused = False
            detail = "accepted"
        except urllib.error.HTTPError as exc:
            refused = 400 <= exc.code < 500
            detail = f"HTTP {exc.code}"
        checks.record("a public endpoint is refused", refused, detail)
        # Put the approved one back, whatever happened above.
        api(base, "PUT", "/settings", {"endpoint_url": ENDPOINT, "model": MODEL,
                                       "max_output_tokens": 300,
                                       "model_timeout_seconds": 600})

        created = api(base, "POST", "/adventures", {
            "title": "Release Smoke",
            "opening": "Rain over Westhaven, and the abbey bell tolling.",
            "persona_name": "Aldric"})
        adv = created["id"]
        checks.record("a campaign is created", bool(adv), f"id {adv}")

        events = stream_turn(base, adv, "I ask Mara what the bell means.")
        errors = [e for e in events if e.get("type") == "error"]
        page_after = api(base, "GET", f"/adventures/{adv}/actions?limit=50") or {}
        checks.record("one real narrator turn is accepted",
                      not errors and (page_after.get("total") or 0) >= 2,
                      errors[0].get("detail", "")[:160] if errors else
                      f"{page_after.get('total')} actions")
        before = [(a.get("type"), (a.get("text") or "")[:120])
                  for a in (page_after.get("actions") or [])]
        state_before = api(base, "GET", f"/adventures/{adv}/state") or {}

        print("restarting the container …")
        run("docker", "restart", NAME)
        ready = wait_ready(base)
        checks.record("the container restarts and serves again", ready)

        page_reopened = api(base, "GET", f"/adventures/{adv}/actions?limit=50") or {}
        after = [(a.get("type"), (a.get("text") or "")[:120])
                 for a in (page_reopened.get("actions") or [])]
        checks.record("the transcript survived the restart", after == before,
                      f"{len(before)} -> {len(after)} actions")
        state_after = api(base, "GET", f"/adventures/{adv}/state") or {}
        checks.record("the narrative state survived the restart",
                      state_after == state_before)

        browser = Browser(headless=True, log=out / "geckodriver.log")
        try:
            browser.go(f"{base}/play/{adv}")
            browser.wait_for(".story-controls", timeout=60)
            story = browser.js(
                "const el = document.querySelector('.story');"
                " return el ? el.textContent.trim().length : 0;")
            checks.record("Firefox renders the reopened campaign",
                          isinstance(story, int) and story > 0, f"{story} characters of story")
            browser.screenshot(out / "reopened-campaign.png")
        finally:
            browser.quit()
    finally:
        logs = run("docker", "logs", NAME)
        (out / "container.log").write_text(logs.stdout + logs.stderr)
        run("docker", "rm", "-f", NAME)
        run("docker", "volume", "rm", VOLUME)

    report = {
        "image": args.image,
        "started": started.isoformat(timespec="seconds"),
        "seconds": round((datetime.now() - started).total_seconds()),
        "endpoint_class": "trusted-LAN HTTPS with a private CA",
        "checks": checks.rows,
        "passed": len([r for r in checks.rows if r["result"] == "PASS"]),
        "failed": len(checks.failed),
    }
    (out / "smoke-report.json").write_text(json.dumps(report, indent=2))
    print(f"\n{report['passed']} passed, {report['failed']} failed "
          f"-> {out / 'smoke-report.json'}")
    return 1 if checks.failed else 0


if __name__ == "__main__":
    raise SystemExit(main())