interactive-story/backend/app/routers/adventures/knowledge.py

"""M7: the imported knowledge library's HTTP surface.

Every route here is scoped to one campaign, twice. `current_adventure` resolves
`{adventure_id}` to an adventure the caller owns or 404s; `_source_or_404` then
requires the source to belong to *that* adventure. A source id from another
campaign is a 404 whichever campaign asks, so guessing ids gets nowhere and
nothing depends on the browser filtering anything
(`IMPORTED-KNOWLEDGE-DESIGN.md` §66).

## The upload takes a file, never a path

`POST .../knowledge` accepts `multipart/form-data` and reads `UploadFile`. There
is no endpoint anywhere that takes a server-side pathname, so H08's traversal
has nothing to traverse: no path is resolved, no root is compared against, no
symlink is followed, because none of those operations exists on this surface.
The filename that arrives is metadata and is cleaned before it is stored.

## Imported text is inert on the way out as well as on the way in

Every response here is JSON, served by FastAPI with `application/json`, and the
browser puts source text into a `<pre>` as a text node. Nothing renders imported
Markdown as HTML, so a `<script>` in a source is a string in a text node and
`javascript:` never becomes an href (H06, H07). `SECURITY-THREAT-MODEL.md` §14
names that the safer default — "render Markdown as sanitized presentation text
only" — and this goes one step further by rendering no Markdown at all: a
Markdown renderer would be attack surface bought for appearance, and appearance
is M8's.
"""

from fastapi import Depends, File, Form, HTTPException, UploadFile
from sqlalchemy import func, select
from sqlalchemy.orm import Session

from ... import models, schemas
from ...database import get_db
from ...knowledge import classes, embeddings, importer

from .deps import CurrentUser, current_adventure, router
from ..settings import get_settings


def _source_or_404(
    db: Session, adventure: models.Adventure, source_id: int
) -> models.KnowledgeSource:
    """One source of *this* campaign, or 404.

    The `adventure_id` test is the isolation rule, and it is written here rather
    than left to a caller because every route needs it and one that forgot would
    be a cross-campaign read.
    """
    source = db.get(models.KnowledgeSource, source_id)
    if source is None or source.adventure_id != adventure.id:
        raise HTTPException(404, "Knowledge source not found")
    return source


def _chunk_counts(db: Session, adventure_id: int) -> dict[int, int]:
    """Passages per source, in one query rather than one per source.

    The list screen shows a count beside every row. Asking the relationship for
    it would be an N+1 across the whole library, which is the shape M5 spent a
    review finding removing and M6 kept out.
    """
    rows = db.execute(
        select(
            models.KnowledgeChunk.source_id, func.count(models.KnowledgeChunk.id)
        )
        .where(models.KnowledgeChunk.adventure_id == adventure_id)
        .group_by(models.KnowledgeChunk.source_id)
    ).all()
    return {source_id: count for source_id, count in rows}


def _embedded_counts(db: Session, adventure_id: int) -> dict[int, int]:
    rows = db.execute(
        select(
            models.KnowledgeChunk.source_id,
            func.count(models.KnowledgeEmbedding.id),
        )
        .join(
            models.KnowledgeEmbedding,
            models.KnowledgeEmbedding.chunk_id == models.KnowledgeChunk.id,
        )
        .where(models.KnowledgeChunk.adventure_id == adventure_id)
        .group_by(models.KnowledgeChunk.source_id)
    ).all()
    return {source_id: count for source_id, count in rows}


def _as_summary(
    source: models.KnowledgeSource, chunks: int, embedded: int
) -> dict:
    return {
        "id": source.id,
        "title": source.title,
        "original_filename": source.original_filename,
        "classification": source.classification,
        "enabled": source.enabled,
        "visibility": source.visibility,
        "always_include": source.always_include,
        "content_hash": source.content_hash,
        "byte_size": source.byte_size,
        "media_type": source.media_type,
        "chunk_count": chunks,
        "embedded_count": embedded,
        "index_state": source.index_state,
        "index_detail": source.index_detail,
        "embed_state": source.embed_state,
        "embed_detail": source.embed_detail,
        "parser_version": source.parser_version,
        "chunking_version": source.chunking_version,
        "imported_at": source.imported_at.isoformat() if source.imported_at else None,
        "updated_at": source.updated_at.isoformat() if source.updated_at else None,
    }


@router.get("/{adventure_id}/knowledge", response_model=list[schemas.KnowledgeSourceOut])
def list_sources(
    db: Session = Depends(get_db),
    adventure: models.Adventure = Depends(current_adventure),
):
    """Every source in this campaign. Never another campaign's.

    The source *content* is deliberately not in this response. A library of
    twenty files would otherwise put a megabyte of prose on a list screen that
    shows none of it; the detail route below serves the text when it is asked
    for.
    """
    counts = _chunk_counts(db, adventure.id)
    embedded = _embedded_counts(db, adventure.id)
    rows = db.execute(
        select(models.KnowledgeSource)
        .where(models.KnowledgeSource.adventure_id == adventure.id)
        .order_by(models.KnowledgeSource.id)
    ).scalars().all()
    return [
        _as_summary(source, counts.get(source.id, 0), embedded.get(source.id, 0))
        for source in rows
    ]


@router.post(
    "/{adventure_id}/knowledge",
    response_model=schemas.KnowledgeSourceOut,
    status_code=201,
)
async def import_source(
    file: UploadFile = File(...),
    classification: str = Form(...),
    title: str = Form(""),
    visibility: str = Form(classes.NORMAL),
    always_include: bool = Form(False),
    allow_duplicate: bool = Form(False),
    db: Session = Depends(get_db),
    user: models.User = CurrentUser,
    adventure: models.Adventure = Depends(current_adventure),
):
    """Imports one local `.txt` or `.md` file as campaign knowledge.

    All of it commits or none of it does. `importer.import_source` raises before
    writing anything when the file is refused, and raises with the session dirty
    when indexing fails; either way the rollback below leaves no source, no
    passages and no index rows — and the reader's file on disk was never opened
    by this process, only received as bytes.
    """
    raw = await file.read()
    try:
        source = importer.import_source(
            db,
            adventure,
            raw=raw,
            filename=file.filename or "",
            classification=classification,
            title=title,
            visibility=visibility,
            always_include=always_include,
            allow_duplicate=allow_duplicate,
        )
    except importer.ImportError_ as exc:
        db.rollback()
        if exc.conflict is not None:
            raise HTTPException(409, {"message": str(exc), "conflict": exc.conflict})
        raise HTTPException(422, str(exc)) from None
    except Exception:
        db.rollback()
        raise
    db.commit()
    db.refresh(source)

    # The vectors, best-effort and after the commit. A source is complete and
    # retrievable lexically at this point; the semantic half is an improvement
    # on it, and an inference host that is down must not cost the reader their
    # import (`IMPORTED-KNOWLEDGE-DESIGN.md` §58).
    settings = get_settings(db, user)
    if embeddings.enabled(settings):
        await embeddings.embed_pending(db, adventure, settings)
        db.commit()
        db.refresh(source)
    return _as_summary(
        source,
        _chunk_counts(db, adventure.id).get(source.id, 0),
        _embedded_counts(db, adventure.id).get(source.id, 0),
    )


@router.get(
    "/{adventure_id}/knowledge/{source_id}",
    response_model=schemas.KnowledgeSourceDetail,
)
def read_source(
    source_id: int,
    db: Session = Depends(get_db),
    adventure: models.Adventure = Depends(current_adventure),
):
    """One source with its text, for the inspector."""
    source = _source_or_404(db, adventure, source_id)
    counts = _chunk_counts(db, adventure.id)
    embedded = _embedded_counts(db, adventure.id)
    return dict(
        _as_summary(source, counts.get(source.id, 0), embedded.get(source.id, 0)),
        content=source.content,
        notes=source.notes,
    )


@router.get(
    "/{adventure_id}/knowledge/{source_id}/chunks",
    response_model=list[schemas.KnowledgeChunkOut],
)
def list_chunks(
    source_id: int,
    db: Session = Depends(get_db),
    adventure: models.Adventure = Depends(current_adventure),
):
    """The passages a source was split into, in order.

    This is what makes chunking inspectable rather than a black box: a reader
    who finds retrieval missing something can see exactly where the boundaries
    fell and what heading each passage was filed under.
    """
    source = _source_or_404(db, adventure, source_id)
    rows = db.execute(
        select(models.KnowledgeChunk, models.KnowledgeEmbedding.model)
        .outerjoin(
            models.KnowledgeEmbedding,
            models.KnowledgeEmbedding.chunk_id == models.KnowledgeChunk.id,
        )
        .where(models.KnowledgeChunk.source_id == source.id)
        .order_by(models.KnowledgeChunk.chunk_index)
    ).all()
    return [
        {
            "id": chunk.id,
            "chunk_index": chunk.chunk_index,
            "heading_path": chunk.heading_path,
            "text": chunk.text,
            "token_count": chunk.token_count,
            "content_hash": chunk.content_hash,
            "embedded": model is not None,
            "embedding_model": model or "",
        }
        for chunk, model in rows
    ]


@router.patch(
    "/{adventure_id}/knowledge/{source_id}",
    response_model=schemas.KnowledgeSourceOut,
)
def update_source(
    source_id: int,
    payload: schemas.KnowledgeSourceUpdate,
    db: Session = Depends(get_db),
    adventure: models.Adventure = Depends(current_adventure),
):
    """Changes a source's classification, state, visibility, flag or title.

    None of these is destructive and none of them requires a reimport. In
    particular:

    * **Reclassifying** rewrites no passage and no index row. The class is read
      at retrieval time, off the source, so a file promoted from Reference to
      Canon starts being framed and weighted as Canon on the very next turn.
    * **Disabling** deletes nothing. The source, its passages, its FTS rows and
      its vectors all stay; every retrieval query filters on `enabled`, so the
      source stops being reachable and starts again the moment it is re-enabled
      (§48, and G04).
    """
    source = _source_or_404(db, adventure, source_id)
    data = payload.model_dump(exclude_unset=True)

    if "classification" in data:
        if not classes.is_class(data["classification"]):
            raise HTTPException(422, "Unknown classification.")
        source.classification = data["classification"]
    if "visibility" in data:
        if not classes.is_visibility(data["visibility"]):
            raise HTTPException(422, "Unknown visibility.")
        source.visibility = data["visibility"]
    if "enabled" in data:
        source.enabled = bool(data["enabled"])
    if "title" in data:
        source.title = (data["title"] or "").strip()[:200] or source.title
    if "notes" in data:
        source.notes = data["notes"] or ""
    if "always_include" in data:
        source.always_include = bool(data["always_include"])
    # Always-include is Canon's alone, wherever the two are set. A source
    # reclassified away from Canon while flagged would otherwise keep asserting
    # itself on every turn as something other than Canon.
    if source.classification != classes.CANON:
        source.always_include = False
    db.commit()
    db.refresh(source)
    counts = _chunk_counts(db, adventure.id)
    embedded = _embedded_counts(db, adventure.id)
    return _as_summary(source, counts.get(source.id, 0), embedded.get(source.id, 0))


@router.delete("/{adventure_id}/knowledge/{source_id}", status_code=204)
def delete_source(
    source_id: int,
    db: Session = Depends(get_db),
    adventure: models.Adventure = Depends(current_adventure),
):
    """Removes a source, its passages, its index rows and its vectors.

    It does not touch a single story row. Turns that used the source keep the
    text they were given, in their own context snapshots, so the record of what
    a past narrator turn was shown survives the source it came from
    (`IMPORTED-KNOWLEDGE-DESIGN.md` §49-50).
    """
    source = _source_or_404(db, adventure, source_id)
    importer.delete_source(db, source)
    db.commit()
    embeddings.forget_cached(adventure.id)
    return None


@router.post("/{adventure_id}/knowledge/reindex")
async def reindex(
    source_id: int | None = None,
    semantic: bool = True,
    db: Session = Depends(get_db),
    user: models.User = CurrentUser,
    adventure: models.Adventure = Depends(current_adventure),
):
    """Rebuilds the derived indexes from the stored source content.

    What it rebuilds is exactly what is rebuildable: passages, FTS rows and,
    when asked, vectors. What it must not change, and does not read at all, is
    source content, classification, visibility, enabled state, story history,
    the active head, the narrative state or any Save Point.

    The lexical rebuild is reported as its own result, and it succeeds or fails
    without reference to the semantic one. `semantic=false` skips embeddings
    entirely; a semantic failure with `semantic=true` still leaves a campaign
    whose lexical retrieval works, and says so.
    """
    sources = [_source_or_404(db, adventure, source_id)] if source_id else (
        db.execute(
            select(models.KnowledgeSource)
            .where(models.KnowledgeSource.adventure_id == adventure.id)
            .order_by(models.KnowledgeSource.id)
        ).scalars().all()
    )
    rebuilt = 0
    failed: list[dict] = []
    for source in sources:
        try:
            rebuilt += importer.build_index(db, source)
        except Exception as exc:  # noqa: BLE001 - recorded on the row, not raised
            db.rollback()
            source = db.get(models.KnowledgeSource, source.id)
            if source is not None:
                source.index_state = "failed"
                source.index_detail = f"{type(exc).__name__}: {exc}"[:2000]
            failed.append({"source_id": source.id if source else None, "detail": str(exc)})
    if semantic:
        embeddings.clear_vectors(db, adventure.id)
    db.commit()
    embeddings.forget_cached(adventure.id)

    embedded = 0
    settings = get_settings(db, user)
    if semantic and embeddings.enabled(settings):
        embedded = await embeddings.embed_pending(db, adventure, settings)
        db.commit()
    return {
        "sources": len(sources),
        "chunks": rebuilt,
        "embedded": embedded,
        "failed": failed,
        "semantic": semantic and embeddings.enabled(settings),
    }


@router.get("/{adventure_id}/knowledge-status")
def knowledge_status(
    db: Session = Depends(get_db),
    user: models.User = CurrentUser,
    adventure: models.Adventure = Depends(current_adventure),
):
    """Whether the library's derived work is healthy, and how much is pending.

    Deliberately distinguishes "nothing was attempted" from "everything
    succeeded" — M6's finding M6-F5 was that reporting `ok` for work that never
    ran reads as a working subsystem. With no embedding model configured this
    answers `semantic_enabled: false` and no status at all, because there is
    nothing to be healthy or unhealthy about.

    It draws the same distinction once more for calibration: a configured model
    this build has not measured reports `semantic_calibrated: false` and
    `semantic_enabled: false`, with the reason, because vectors that exist but
    are never consulted are not a working semantic index.
    """
    settings = get_settings(db, user)
    model = embeddings.model_name(settings)
    # M7 corrective: "a model is configured" and "this build knows what that
    # model's similarity scale means" are different questions, and reporting
    # only the first would tell a reader semantic search is on when it is not.
    calibrated = classes.semantic_floor_for(model) is not None
    sources = db.execute(
        select(models.KnowledgeSource).where(
            models.KnowledgeSource.adventure_id == adventure.id
        )
    ).scalars().all()
    return {
        "sources": len(sources),
        "enabled_sources": sum(1 for s in sources if s.enabled),
        "failed_index": [
            {"id": s.id, "title": s.title, "detail": s.index_detail}
            for s in sources
            if s.index_state == "failed"
        ],
        "failed_embedding": [
            {"id": s.id, "title": s.title, "detail": s.embed_detail}
            for s in sources
            if s.embed_state == "failed"
        ],
        "semantic_enabled": bool(model) and calibrated,
        "embedding_model": model,
        "semantic_calibrated": calibrated,
        "calibrated_models": sorted(classes.SEMANTIC_CALIBRATION),
        "semantic_note": (
            "" if calibrated or not model else
            f"“{model}” has no measured relevance calibration in this build, so "
            "semantic retrieval is disabled and retrieval is lexical only. "
            "Lexical search and story play are unaffected."
        ),
        "pending_embeddings": (
            embeddings.pending_count(db, adventure.id, model) if model else 0
        ),
    }