JesseMarkowitz
8c65ae99de
94 files, +1,395 -6,578. Three files are new; twenty-four are gone. The milestone is subtraction, and what is left is the single-user local storyteller the specification describes. Removed in full: campaign scripting and its QuickJS sandbox; multi-user accounts, guest sessions, login, registration and the shared demo key; the visitor-analytics tables, dashboard and page beacon; the access log of sign-ins, addresses and devices; per-IP and per-user rate limiting and quotas; Render deployment config; Postgres and psycopg; cloud inference providers, the API-key field and the key encryption that existed to store it; session-cookie signing. None of it was hidden behind a flag — the routes are gone and answer 404. Two things were kept that the brief allowed keeping. The `users` table and its foreign keys stay as an internal ownership detail, because rewriting them out means a migration across most of the schema to delete a column that costs nothing; nothing creates a second user and no request carries an identity. Five inert tables and four inert columns stay for the same reason, so an M1 campaign database opens unchanged. The one addition is app/endpoints.py, which decides where a story may be sent. Loopback, RFC1918, link-local, unique-local and CGNAT — an explicit allowlist of networks, not a guess at what `ipaddress` means by "private", which calls the documentation ranges private and IPv6 loopback reserved. Every address a hostname resolves to must be in it, so a split answer does not squeak through, and the rule runs both when the endpoint is saved and before every outbound request, because a name that resolved to the LAN this morning can resolve elsewhere this afternoon. Known cloud hosts are named in the refusal so the error says why rather than looking like broken DNS. TLS is never traded against it: M1's shared trust context is intact on all four clients and there is no way to skip verification. The hardcoded 120-second model timeout is now a setting. That was not theoretical — on this GPU-less four-core host a cold load of qwen2.5:3b-instruct took 648.9 seconds to produce the first turn, while turns 2 to 5 of the same campaign took 3.6 to 13.1. Connect stays short at 10s so a wrong address still fails fast; the read timeout defaults to 300s and is bounded at 3600, because "wait longer" must stay a number. Two defects found while testing and fixed here. An unknown /api path fell through the SPA catch-all and came back as HTML with status 200, so a client asking for JSON parsed a web page instead of learning the route was gone. And AIDND_CORS_ORIGINS accepted "*", which on an unauthenticated loopback API would hand every page on the Internet a write handle on the campaign database; it now refuses to start. Verified rather than assumed. Offline, on a network with no route out and no DNS: five turns, retry with both takes retained, restart with an identical transcript digest, a failed model call leaving the accepted AI-turn count untouched, and a capture with zero non-loopback unicast packets. Against a real second machine on the LAN over HTTPS with a private CA: four turns, restart, and a capture showing 289 packets to the approved host, 344 loopback, zero anywhere else, zero DNS queries. Cloud and public endpoints refused with their reasons; no API key settable; every removed route 404. 604 backend tests pass, down from 648 by the fifteen retired with the subsystems they tested and up by the twenty-nine added for the endpoint policy and the removed surface. The scripting tests were not deleted: eight files used a JavaScript counter as instrumentation for the state snapshot and rollback machinery, which M2 does not touch, so the counter moved to the world-state engine and those tests still assert what they always did. Frontend lint and build are clean; the image builds, and its wheel-building stage is gone with quickjs. No M3 work. Undo is still destructive and there is still no Redo.
402 lines
18 KiB
Python
402 lines
18 KiB
Python
import json
|
||
from typing import AsyncIterator
|
||
|
||
import httpx
|
||
|
||
from .. import debuglog, endpoints, tlstrust
|
||
from .base import PromptParts, Provider, ProviderError
|
||
|
||
# Appended after the story text in chat mode, so a chat-tuned model continues
|
||
# the prose rather than replying conversationally.
|
||
CHAT_CONTINUE_HINT = "\n\n[Continue the story directly. Output only story text.]"
|
||
|
||
# A machine that is not listening refuses in milliseconds, so a slow connect
|
||
# means the wrong address rather than a busy model.
|
||
CONNECT_TIMEOUT = 10.0
|
||
|
||
# How long to wait for generation when Settings names no value. Upstream
|
||
# hardcoded 120s, and M1 measured a *cold* load of a 3B model on a GPU-less
|
||
# four-core host exceeding it three times while the same turn took 6-9 seconds
|
||
# once the model was resident. 300s covers a cold start on modest hardware and
|
||
# is still a number: a wedged endpoint fails rather than hanging forever.
|
||
DEFAULT_READ_TIMEOUT = 300.0
|
||
|
||
# Embeddings are short and never cold-load a large model.
|
||
EMBED_READ_TIMEOUT = 60.0
|
||
|
||
|
||
|
||
# Completion endpoints have no roles, so a chat has to be flattened into one
|
||
# labeled transcript that ends on "Assistant:" for the model to continue.
|
||
_ROLE_LABELS = {"system": "System", "user": "User", "assistant": "Assistant"}
|
||
|
||
|
||
def flatten_messages(messages: list[dict]) -> str:
|
||
turns = "\n\n".join(
|
||
f"{_ROLE_LABELS.get(m['role'], m['role'])}: {m['content']}" for m in messages
|
||
)
|
||
return f"{turns}\n\nAssistant:"
|
||
|
||
|
||
class OpenAICompatibleProvider(Provider):
|
||
"""Adapter for Ollama's OpenAI-compatible `/v1` API.
|
||
|
||
The protocol is OpenAI's, which is what the module is named for; the
|
||
product speaks it to Ollama and to nothing else. `endpoints.py` decides
|
||
which addresses may be reached, and every request re-checks — the shape of
|
||
the wire format is not the same thing as permission to use it.
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
endpoint_url: str,
|
||
model: str,
|
||
api_mode: str = "chat",
|
||
read_timeout: float | None = None,
|
||
):
|
||
self.base_url = endpoint_url.rstrip("/")
|
||
self.model = model
|
||
self.api_mode = api_mode # Either "chat" or "completion".
|
||
# How long to wait for the model, in seconds. Cold-loading a model on a
|
||
# CPU-only machine can take minutes, and a fixed short timeout reports
|
||
# that as a failure. See `DEFAULT_READ_TIMEOUT`.
|
||
self.read_timeout = read_timeout or DEFAULT_READ_TIMEOUT
|
||
# The token accounting from the last call, when the endpoint reported
|
||
# any. Every request method writes it, so a caller reads it after the
|
||
# call it made. One provider is built per request.
|
||
self.last_usage: dict | None = None
|
||
|
||
def _headers(self) -> dict:
|
||
# No Authorization header: Ollama does not use one, and this build has
|
||
# no cloud provider to carry a key for.
|
||
return {"Content-Type": "application/json"}
|
||
|
||
def _timeout(self, seconds: float | None = None) -> httpx.Timeout:
|
||
"""Short to connect, patient to read.
|
||
|
||
A machine that is not listening says so in milliseconds, so a slow
|
||
connect is a wrong address rather than a busy model and should fail
|
||
fast. Generation is the opposite: the first token can be minutes away
|
||
while a model loads.
|
||
"""
|
||
return httpx.Timeout(seconds or self.read_timeout, connect=CONNECT_TIMEOUT)
|
||
|
||
def _record_usage(self, payload: dict) -> None:
|
||
"""Records the endpoint's own token accounting, if it reported any.
|
||
|
||
OpenRouter now always reports usage, and `usage: {include: true}` and
|
||
`stream_options` are deprecated and do nothing. In a stream the usage
|
||
arrives on a final chunk that carries no choices, which is why this is
|
||
read separately from the text extraction.
|
||
"""
|
||
usage = payload.get("usage")
|
||
if isinstance(usage, dict) and usage:
|
||
self.last_usage = usage
|
||
|
||
def _request(self, parts: PromptParts, temperature: float, max_tokens: int) -> tuple[str, dict]:
|
||
if self.api_mode == "completion":
|
||
url = f"{self.base_url}/completions"
|
||
body = {
|
||
"model": self.model,
|
||
"prompt": f"{parts.system}\n\n{parts.story}",
|
||
"temperature": temperature,
|
||
"max_tokens": max_tokens,
|
||
"stream": True,
|
||
}
|
||
else:
|
||
url = f"{self.base_url}/chat/completions"
|
||
body = {
|
||
"model": self.model,
|
||
"messages": [
|
||
{"role": "system", "content": parts.system},
|
||
{"role": "user", "content": parts.story + CHAT_CONTINUE_HINT},
|
||
],
|
||
"temperature": temperature,
|
||
"max_tokens": max_tokens,
|
||
"stream": True,
|
||
}
|
||
return url, body
|
||
|
||
@staticmethod
|
||
def _extract_chunk(payload: dict) -> str:
|
||
choices = payload.get("choices") or []
|
||
if not choices:
|
||
return ""
|
||
choice = choices[0]
|
||
# A chat stream uses `delta.content`, and a completion stream uses
|
||
# `text`. The non-stream fallbacks are `message.content` and `text`.
|
||
delta = choice.get("delta") or {}
|
||
return (
|
||
delta.get("content")
|
||
or choice.get("text")
|
||
or (choice.get("message") or {}).get("content")
|
||
or ""
|
||
)
|
||
|
||
@staticmethod
|
||
def _extract_reasoning(payload: dict) -> str:
|
||
"""Returns a reasoning model's thinking text.
|
||
|
||
OpenRouter normalizes it to `reasoning`, and DeepSeek-style servers use
|
||
`reasoning_content`.
|
||
"""
|
||
choices = payload.get("choices") or []
|
||
if not choices:
|
||
return ""
|
||
choice = choices[0]
|
||
delta = choice.get("delta") or {}
|
||
message = choice.get("message") or {}
|
||
return (
|
||
delta.get("reasoning")
|
||
or delta.get("reasoning_content")
|
||
or message.get("reasoning")
|
||
or message.get("reasoning_content")
|
||
or ""
|
||
)
|
||
|
||
async def generate(
|
||
self, parts: PromptParts, *, temperature: float, max_tokens: int
|
||
) -> AsyncIterator[tuple[str, str]]:
|
||
"""Yields `("text", chunk)` and `("reasoning", chunk)` pairs."""
|
||
if not self.model:
|
||
raise ProviderError("No model configured — set one in Settings.")
|
||
url, body = self._request(parts, temperature, max_tokens)
|
||
async for event in self._stream(url, body):
|
||
yield event
|
||
|
||
async def chat(
|
||
self, messages: list[dict], *, temperature: float, max_tokens: int
|
||
) -> AsyncIterator[tuple[str, str]]:
|
||
"""Runs a plain multi-turn chat, with no story framing and no context
|
||
assembly.
|
||
|
||
The method sends `[{"role", "content"}, ...]` straight to the endpoint.
|
||
The AI Chat scratchpad uses it, and the turn engine uses `generate()`.
|
||
"""
|
||
if not self.model:
|
||
raise ProviderError("No model configured — set one in Settings.")
|
||
if self.api_mode == "completion":
|
||
url = f"{self.base_url}/completions"
|
||
body = {
|
||
"model": self.model,
|
||
"prompt": flatten_messages(messages),
|
||
"temperature": temperature,
|
||
"max_tokens": max_tokens,
|
||
"stream": True,
|
||
}
|
||
else:
|
||
url = f"{self.base_url}/chat/completions"
|
||
body = {
|
||
"model": self.model,
|
||
"messages": messages,
|
||
"temperature": temperature,
|
||
"max_tokens": max_tokens,
|
||
"stream": True,
|
||
}
|
||
async for event in self._stream(url, body):
|
||
yield event
|
||
|
||
async def _stream(self, url: str, body: dict) -> AsyncIterator[tuple[str, str]]:
|
||
"""Runs the shared SSE request for `generate()` and `chat()`.
|
||
|
||
The method POSTs a streaming request, yields `("text", chunk)` and
|
||
`("reasoning", chunk)` pairs, and logs the exchange.
|
||
"""
|
||
# Re-checked on every request, not only when the endpoint was saved: a
|
||
# hostname that resolved to a LAN address yesterday can resolve
|
||
# somewhere else today, and a database row can be edited by hand.
|
||
reason = endpoints.rejection_reason(url)
|
||
if reason:
|
||
raise ProviderError(f"This endpoint can't be used — {reason}.")
|
||
log = debuglog.start_entry(url, self.model, body)
|
||
received: list[str] = []
|
||
try:
|
||
async with httpx.AsyncClient(
|
||
timeout=self._timeout(), verify=tlstrust.ssl_context()
|
||
) as client:
|
||
async with client.stream("POST", url, json=body, headers=self._headers()) as resp:
|
||
if resp.status_code != 200:
|
||
detail = (await resp.aread()).decode(errors="replace")[:500]
|
||
raise ProviderError(self._friendly_http_error(resp.status_code, detail))
|
||
# Some servers ignore `stream=true` and return one plain
|
||
# JSON body, so buffer the non-SSE lines to fall back to.
|
||
saw_sse = False
|
||
raw_lines: list[str] = []
|
||
async for line in resp.aiter_lines():
|
||
if not line.startswith("data:"):
|
||
if not saw_sse:
|
||
raw_lines.append(line)
|
||
continue
|
||
saw_sse = True
|
||
data = line[5:].strip()
|
||
if data == "[DONE]":
|
||
debuglog.finish_entry(
|
||
log, response="".join(received), usage=self.last_usage
|
||
)
|
||
return
|
||
try:
|
||
payload = json.loads(data)
|
||
except ValueError:
|
||
continue
|
||
self._record_usage(payload)
|
||
reasoning = self._extract_reasoning(payload)
|
||
if reasoning:
|
||
yield "reasoning", reasoning
|
||
chunk = self._extract_chunk(payload)
|
||
if chunk:
|
||
received.append(chunk)
|
||
yield "text", chunk
|
||
if not saw_sse:
|
||
body_text = "\n".join(raw_lines).strip()
|
||
try:
|
||
payload = json.loads(body_text)
|
||
except ValueError:
|
||
raise ProviderError(
|
||
"AI endpoint returned neither an SSE stream nor JSON: "
|
||
+ body_text[:200]
|
||
)
|
||
self._record_usage(payload)
|
||
reasoning = self._extract_reasoning(payload)
|
||
if reasoning:
|
||
yield "reasoning", reasoning
|
||
chunk = self._extract_chunk(payload)
|
||
if chunk:
|
||
received.append(chunk)
|
||
yield "text", chunk
|
||
if not received:
|
||
raise ProviderError(
|
||
"AI endpoint returned a response with no text: "
|
||
+ body_text[:200]
|
||
)
|
||
debuglog.finish_entry(log, response="".join(received), usage=self.last_usage)
|
||
except httpx.ConnectError as exc:
|
||
error = f"Could not connect to {self.base_url} — is the AI server running?"
|
||
debuglog.finish_entry(log, response="".join(received), error=error)
|
||
raise ProviderError(error) from exc
|
||
except httpx.TimeoutException as exc:
|
||
debuglog.finish_entry(log, response="".join(received), error="Timed out")
|
||
raise ProviderError("The AI endpoint timed out.") from exc
|
||
except httpx.HTTPError as exc:
|
||
debuglog.finish_entry(log, response="".join(received), error=str(exc))
|
||
raise ProviderError(f"Request to AI endpoint failed: {exc}") from exc
|
||
except (ProviderError, GeneratorExit, BaseException) as exc:
|
||
status = "cancelled" if isinstance(exc, GeneratorExit) else str(exc)
|
||
debuglog.finish_entry(log, response="".join(received), error=status)
|
||
raise
|
||
|
||
async def complete(
|
||
self, system: str, user: str, *, temperature: float = 0.3, max_tokens: int = 400
|
||
) -> str:
|
||
"""Runs a single non-streaming completion, for background calls such as
|
||
summarization.
|
||
|
||
Unlike `generate()`, this adds no story-continuation framing.
|
||
"""
|
||
if not self.model:
|
||
raise ProviderError("No model configured — set one in Settings.")
|
||
if self.api_mode == "completion":
|
||
url = f"{self.base_url}/completions"
|
||
body = {
|
||
"model": self.model,
|
||
"prompt": f"{system}\n\n{user}",
|
||
"temperature": temperature,
|
||
"max_tokens": max_tokens,
|
||
"stream": False,
|
||
}
|
||
else:
|
||
url = f"{self.base_url}/chat/completions"
|
||
body = {
|
||
"model": self.model,
|
||
"messages": [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": user},
|
||
],
|
||
"temperature": temperature,
|
||
"max_tokens": max_tokens,
|
||
"stream": False,
|
||
}
|
||
|
||
# Same check as `_stream`: every outbound request re-tests the
|
||
# endpoint, so no path reaches an address the policy refuses.
|
||
reason = endpoints.rejection_reason(url)
|
||
if reason:
|
||
raise ProviderError(f"This endpoint can't be used — {reason}.")
|
||
log = debuglog.start_entry(url, self.model, body)
|
||
try:
|
||
async with httpx.AsyncClient(
|
||
timeout=self._timeout(), verify=tlstrust.ssl_context()
|
||
) as client:
|
||
resp = await client.post(url, json=body, headers=self._headers())
|
||
except httpx.HTTPError as exc:
|
||
debuglog.finish_entry(log, error=str(exc))
|
||
raise ProviderError(f"Request to AI endpoint failed: {exc}") from exc
|
||
if resp.status_code != 200:
|
||
error = self._friendly_http_error(resp.status_code, resp.text[:500])
|
||
debuglog.finish_entry(log, error=error)
|
||
raise ProviderError(error)
|
||
try:
|
||
payload = resp.json()
|
||
except ValueError as exc:
|
||
debuglog.finish_entry(log, error="Invalid JSON response")
|
||
raise ProviderError("AI endpoint returned invalid JSON.") from exc
|
||
self._record_usage(payload)
|
||
text = self._extract_chunk(payload)
|
||
debuglog.finish_entry(log, response=text, usage=self.last_usage)
|
||
return text.strip()
|
||
|
||
async def embed(self, texts: list[str]) -> list[list[float]]:
|
||
"""POSTs to /v1/embeddings. Here `self.model` is the embedding model."""
|
||
if not self.model:
|
||
raise ProviderError("No embedding model configured — set one in Settings.")
|
||
url = f"{self.base_url}/embeddings"
|
||
body = {"model": self.model, "input": texts}
|
||
# Same check as `_stream`: every outbound request re-tests the
|
||
# endpoint, so no path reaches an address the policy refuses.
|
||
reason = endpoints.rejection_reason(url)
|
||
if reason:
|
||
raise ProviderError(f"This endpoint can't be used — {reason}.")
|
||
log = debuglog.start_entry(url, self.model, body)
|
||
try:
|
||
async with httpx.AsyncClient(
|
||
timeout=self._timeout(EMBED_READ_TIMEOUT), verify=tlstrust.ssl_context()
|
||
) as client:
|
||
resp = await client.post(url, json=body, headers=self._headers())
|
||
except httpx.HTTPError as exc:
|
||
debuglog.finish_entry(log, error=str(exc))
|
||
raise ProviderError(f"Embedding request failed: {exc}") from exc
|
||
if resp.status_code != 200:
|
||
error = self._friendly_http_error(resp.status_code, resp.text[:500])
|
||
debuglog.finish_entry(log, error=error)
|
||
raise ProviderError(error)
|
||
try:
|
||
data = resp.json().get("data", [])
|
||
vectors = [item["embedding"] for item in sorted(data, key=lambda d: d.get("index", 0))]
|
||
except (ValueError, KeyError, TypeError) as exc:
|
||
debuglog.finish_entry(log, error="Malformed embeddings response")
|
||
raise ProviderError("AI endpoint returned malformed embeddings.") from exc
|
||
if len(vectors) != len(texts):
|
||
debuglog.finish_entry(log, error="Embedding count mismatch")
|
||
raise ProviderError("AI endpoint returned the wrong number of embeddings.")
|
||
debuglog.finish_entry(log, response=f"{len(vectors)} vectors × {len(vectors[0]) if vectors else 0} dims")
|
||
return vectors
|
||
|
||
def _friendly_http_error(self, status: int, detail: str) -> str:
|
||
if status == 401:
|
||
return "Authentication failed — check your API key in Settings."
|
||
if status == 404:
|
||
return (
|
||
f"Endpoint or model not found (HTTP 404). Check the endpoint URL and that "
|
||
f"model '{self.model}' exists. {detail}"
|
||
)
|
||
if status == 429:
|
||
# OpenRouter's shared free tier has a per-day cap. Distinguish it
|
||
# from a short-term burst limit, so the message tells the reader what
|
||
# to do.
|
||
if "free-models-per-day" in detail:
|
||
return (
|
||
"The free demo has hit its daily request limit (resets at "
|
||
"00:00 UTC). Please try again later."
|
||
)
|
||
return "The AI is getting too many requests right now — wait a moment and try again."
|
||
return f"AI endpoint returned HTTP {status}: {detail}"
|