"""v1.1 release smoke test: the shipped image, as a reader would meet it. python -m tools.v11_release_smoke --image --out Run from `backend/`. Reads `AIDND_TEST_ENDPOINT` (an **HTTPS** Ollama on the trusted LAN) and `AIDND_TEST_MODEL`. `--ca` names the private CA to install inside the container, defaulting to this machine's own. Supplemental release evidence, not a replacement for the gates: it asks whether the artefact that ships actually runs, reaches its approved narrator, refuses an unapproved one, and keeps a campaign across a container restart. ## The two things this is careful about **The CA is installed, not bypassed.** `app/tlstrust.ssl_context()` is `ssl.create_default_context()` — the platform's own store — unioned with certifi's. So the private CA is mounted into `/usr/local/share/ca-certificates/` and registered with `update-ca-certificates`, and verification is then ordinary. Nothing sets `verify=False`, and a check inside the container proves the handshake succeeds through that store. **Loopback means the published port.** The process inside the container listens on `0.0.0.0` because that is the only address a published port can reach (`docker-compose.yml` says so). What must be loopback-only is the *publish*, so the container is started with `-p 127.0.0.1::8000` and the check is that the host's LAN address refuses the same port. """ from __future__ import annotations import argparse import json import os import socket import subprocess import sys import time import urllib.error import urllib.request from datetime import datetime from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from tools.m11_webdriver import Browser, free_port, require_under_home # noqa: E402 ENDPOINT = os.environ.get("AIDND_TEST_ENDPOINT", "") MODEL = os.environ.get("AIDND_TEST_MODEL", "") NAME = "v11-release-smoke" VOLUME = "v11-release-smoke-data" #: An endpoint the policy must refuse whatever else is true: a public host. PUBLIC_ENDPOINT = "https://api.openai.com/v1" class Checks: def __init__(self) -> None: self.rows: list[dict] = [] def record(self, name: str, ok: bool, detail: str = "") -> bool: self.rows.append({"check": name, "result": "PASS" if ok else "FAIL", "detail": detail}) print(f" {'ok ' if ok else 'FAIL'} {name}" + (f" — {detail}" if detail else ""), flush=True) return ok @property def failed(self) -> list[dict]: return [r for r in self.rows if r["result"] == "FAIL"] def run(*args: str, **kwargs) -> subprocess.CompletedProcess: return subprocess.run(args, capture_output=True, text=True, **kwargs) def api(base: str, method: str, path: str, payload=None, timeout=900): data = json.dumps(payload).encode() if payload is not None else None request = urllib.request.Request( f"{base}/api{path}", data=data, method=method, headers={"Content-Type": "application/json"} if data else {}) with urllib.request.urlopen(request, timeout=timeout) as response: body = response.read().decode() return json.loads(body) if body else None def stream_turn(base: str, adv: int, text: str) -> list[dict]: request = urllib.request.Request( f"{base}/api/adventures/{adv}/actions", data=json.dumps({"type": "do", "text": text}).encode(), method="POST", headers={"Content-Type": "application/json"}) events: list[dict] = [] with urllib.request.urlopen(request, timeout=900) as response: for raw in response: line = raw.decode(errors="replace").strip() if line.startswith("data:"): try: events.append(json.loads(line[5:].strip())) except json.JSONDecodeError: pass return events def lan_address() -> str | None: """This machine's own LAN address, for the loopback-only check.""" probe = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) try: probe.connect(("192.0.2.1", 9)) # TEST-NET-1: routed nowhere, sends nothing return probe.getsockname()[0] except OSError: return None finally: probe.close() def wait_ready(base: str, *, timeout: float = 180) -> bool: deadline = time.monotonic() + timeout while time.monotonic() < deadline: try: urllib.request.urlopen(f"{base}/api/settings", timeout=3) return True except Exception: time.sleep(1) return False def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--image", required=True) parser.add_argument("--out", required=True) parser.add_argument("--ca", default="/usr/local/share/ca-certificates/draco.crt") parser.add_argument( "--add-host", default="", metavar="NAME:ADDRESS", help=("resolve the narrator's hostname inside the container. A `.local` " "name is mDNS, and a container has no mDNS resolver, so the " "endpoint policy refuses an address it cannot classify and " "`PUT /api/settings` answers 400. Mapping the name — rather than " "using the address — keeps the hostname the certificate is issued " "for, which is the thing this test verifies.")) args = parser.parse_args() if not (ENDPOINT and MODEL): print("set AIDND_TEST_ENDPOINT (https://…) and AIDND_TEST_MODEL") return 2 if not ENDPOINT.startswith("https://"): print("the smoke test needs an HTTPS endpoint: that is what it verifies") return 2 ca = Path(args.ca) if not ca.exists(): print(f"no CA at {ca}") return 2 out = require_under_home(Path(args.out).expanduser()) out.mkdir(parents=True, exist_ok=True) checks = Checks() port = free_port() base = f"http://127.0.0.1:{port}" started = datetime.now() run("docker", "rm", "-f", NAME) run("docker", "volume", "rm", VOLUME) run("docker", "volume", "create", VOLUME) print(f"starting {args.image} on 127.0.0.1:{port} with a fresh volume …") start = run( "docker", "run", "-d", "--name", NAME, "-p", f"127.0.0.1:{port}:8000", "-v", f"{VOLUME}:/data", "-v", f"{ca}:/usr/local/share/ca-certificates/{ca.name}:ro", *(("--add-host", args.add_host) if args.add_host else ()), args.image, "sh", "-c", "update-ca-certificates >/dev/null 2>&1; " "exec uvicorn app.main:app --host 0.0.0.0 --port 8000", ) if start.returncode != 0: print(start.stderr[:400]) return 1 container = start.stdout.strip()[:12] try: checks.record("the container starts", True, container) ready = wait_ready(base) if not checks.record("the application answers on loopback", ready, base): logs = run("docker", "logs", NAME) (out / "container.log").write_text(logs.stdout + logs.stderr) return 1 published = run("docker", "port", NAME).stdout.strip() checks.record("the port is published on loopback only", "127.0.0.1" in published and "0.0.0.0" not in published, published) lan = lan_address() if lan: try: urllib.request.urlopen(f"http://{lan}:{port}/api/settings", timeout=4) reachable = True except Exception: reachable = False checks.record("the LAN address does not serve the application", not reachable, f"port {port} on this machine's LAN address") page = urllib.request.urlopen(base + "/", timeout=30) html = page.read().decode(errors="replace") checks.record("the first page loads", page.status == 200 and "
= 2, errors[0].get("detail", "")[:160] if errors else f"{page_after.get('total')} actions") before = [(a.get("type"), (a.get("text") or "")[:120]) for a in (page_after.get("actions") or [])] state_before = api(base, "GET", f"/adventures/{adv}/state") or {} print("restarting the container …") run("docker", "restart", NAME) ready = wait_ready(base) checks.record("the container restarts and serves again", ready) page_reopened = api(base, "GET", f"/adventures/{adv}/actions?limit=50") or {} after = [(a.get("type"), (a.get("text") or "")[:120]) for a in (page_reopened.get("actions") or [])] checks.record("the transcript survived the restart", after == before, f"{len(before)} -> {len(after)} actions") state_after = api(base, "GET", f"/adventures/{adv}/state") or {} checks.record("the narrative state survived the restart", state_after == state_before) browser = Browser(headless=True, log=out / "geckodriver.log") try: browser.go(f"{base}/play/{adv}") browser.wait_for(".story-controls", timeout=60) story = browser.js( "const el = document.querySelector('.story');" " return el ? el.textContent.trim().length : 0;") checks.record("Firefox renders the reopened campaign", isinstance(story, int) and story > 0, f"{story} characters of story") browser.screenshot(out / "reopened-campaign.png") finally: browser.quit() finally: logs = run("docker", "logs", NAME) (out / "container.log").write_text(logs.stdout + logs.stderr) run("docker", "rm", "-f", NAME) run("docker", "volume", "rm", VOLUME) report = { "image": args.image, "started": started.isoformat(timespec="seconds"), "seconds": round((datetime.now() - started).total_seconds()), "endpoint_class": "trusted-LAN HTTPS with a private CA", "checks": checks.rows, "passed": len([r for r in checks.rows if r["result"] == "PASS"]), "failed": len(checks.failed), } (out / "smoke-report.json").write_text(json.dumps(report, indent=2)) print(f"\n{report['passed']} passed, {report['failed']} failed " f"-> {out / 'smoke-report.json'}") return 1 if checks.failed else 0 if __name__ == "__main__": raise SystemExit(main())