interactive-story/backend/app/bundle.py

"""Phase 14, SP6: the export bundle, as a tree.

A bundle is the one place a story leaves the database, and the only part of the
tree no migration can reach. A file downloaded today has to still import into a
build shipped next year. The format is therefore versioned, both versions are
defined here, and nothing else in the app knows either of them.

Version 1 is a flat list of turns, each with an optional `variants` array, which
is the repeating group SP4 unpacked into rows. Nothing writes that shape now.
The reader stays, because bundles already on people's disks still use it, and a
backup that stops importing is not a backup.

Version 2 carries the tree. It holds three things version 1 could not, and each
one is required:

* The branches, because a forked adventure is two stories and a flat list holds
  one. A version 1 export interleaved them by turn number, which read as a
  garbled story rather than as lost data.
* `live`, because a coordinate can hold several attempts at one turn and exactly
  one of them is the story.
* Both after-snapshots, because they are what a branch switch and an undo
  restore. A bundle carrying the actions but not the outcomes would import a
  tree nobody could switch inside.

## The rule about what a bundle carries

A bundle carries what was chosen, never what is derived. The head branch, the
fork points, the live flags, and the anchors are decisions somebody made, so
they are in the file. `lineage` and the head depth are computed from those, and
the import recomputes them:

* `lineage` is a cache of `parent` plus `fork_depth`. Shipping it as well would
  put a second source of truth for one fact into a file anyone can hand-edit,
  and the two could then disagree without any read reporting it.
* The head depth used to be derived the same way — the tip of the head branch,
  a fact about the nodes that arrived with it. M3 moved it to the other side of
  the rule. Undo no longer deletes, so a story can be read at a position behind
  its retained tip, and where the reader stopped is a decision nobody can
  recompute from the rows: the same tree exports identically whether the user
  undid three turns or none. `headDepth` is therefore written, and an import
  that ignored it would silently Redo the story to its newest retained turn,
  which is the Phase 0B export finding this milestone exists to close.

A bundle written before M3 has no `headDepth` key, and there is nothing lost in
that: at the time it was written the head could not sit behind the tip, so the
derived answer *was* the recorded one. Such a file imports by deriving, exactly
as it always did.

Every hand-editable coordinate is therefore checked before a row is written, in
`plan`, rather than repaired afterwards. An import that fails partway leaves an
adventure holding half a tree, and a tree missing a branch is a story that stops
without reporting anything.
"""

from datetime import datetime

from fastapi import HTTPException
from sqlalchemy import insert, update
from sqlalchemy.orm import Session, undefer

from . import attempts, models, schemas
from .context import cursors, lineage
from .knowledge import chunking as knowledge_chunking
from .knowledge import classes as knowledge_classes
from .knowledge import importer as knowledge_importer
from .narrative import model as narrative_model

FORMAT = "ai-dnd-adventure-v2"
LEGACY_FORMAT = "ai-dnd-adventure-v1"

# `actions.type` is VARCHAR(20), and a raw-dict import bypasses the schemas.
TYPE_MAX = 20


# ---------------------------------------------------------------- exporting

def export(db: Session, adventure: models.Adventure) -> dict:
    """Returns the whole adventure as a version 2 bundle.

    The export is not scoped to a path. A backup holds the entire tree, not the
    branch its owner is currently reading. Both after-snapshots are undeferred in
    the one query, because they are per-node columns nothing else reads in bulk
    and requesting them a row at a time would cost one query per turn.

    The bundle carries no context snapshots. It has never carried the assembled
    prompts, and it still does not. They run about 163 kB per turn, they explain
    a generation rather than form part of the story, and the Insights viewer they
    feed reads the adventure they came from.
    """
    branches = (
        db.query(models.Branch)
        .filter(models.Branch.adventure_id == adventure.id)
        .order_by(models.Branch.id)
        .all()
    )
    # Branch ids are local to the file and are positions in this list, because
    # the database ids they hold here are already in use on the importing
    # side.
    local = {branch.id: i for i, branch in enumerate(branches)}
    nodes = (
        db.query(models.Action)
        .filter(models.Action.adventure_id == adventure.id)
        .options(
            undefer(models.Action.state_after),
            undefer(models.Action.world_state_after),
        )
        .order_by(
            models.Action.branch_id, models.Action.depth, models.Action.id,
        )
        .all()
    )
    return {
        "format": FORMAT,
        "title": adventure.title,
        "memory": adventure.memory,
        "authorsNote": adventure.authors_note,
        "aiInstructions": adventure.ai_instructions,
        "storySummary": adventure.story_summary,
        # Phase 18. A bundle written before personas existed has no key here,
        # and the import below reads it with `.get`, so it lands with an empty
        # persona — which is the same as having none. No FORMAT bump needed.
        "persona": {
            "name": adventure.persona_name,
            "pronouns": adventure.persona_pronouns,
            "desc": adventure.persona_desc,
        },
        "worldState": adventure.world_state,
        # M5. The authoritative narrative state, and the campaign's own rules.
        # Both are decisions rather than derivations — the state is what the
        # campaign established, and canon is what its owner wrote — so both go
        # in the file by the rule at the top of this module. A bundle written
        # before M5 has neither key and imports with an empty state, which is
        # what such a campaign had.
        "narrativeState": adventure.narrative_state,
        "campaignCanon": adventure.campaign_canon,
        "autoSummarize": adventure.auto_summarize,
        "memoryBankEnabled": adventure.memory_bank_enabled,
        # Write a root entry even for an adventure whose branch row was never
        # created. A story with no branch is a pre-tree story, and the tree it
        # belongs to is the root. `_local` places its nodes there.
        "branches": [_exported_branch(b, local) for b in branches] or [_ROOT],
        "headBranch": local.get(adventure.head_branch_id, 0),
        # M3. Where the story is being read, which is not always where it
        # ends. See the head-depth rule at the top of this module.
        "headDepth": adventure.head_depth,
        "memoryCursor": _exported_anchor(adventure, cursors.MEMORY, local),
        "summaryCursor": _exported_anchor(adventure, cursors.SUMMARY, local),
        "memories": [_exported_memory(m, local) for m in adventure.memories],
        # M4. A Save Point is a decision — someone chose this position and gave
        # it a name — so it goes in the file by the rule at the top of this
        # module. Nothing here is derived: the coordinate is the one stored, not
        # one recomputed from the rows, because the whole point of the pointer
        # is that no amount of reading the turns can tell you which one somebody
        # named. A bundle written before M4 has no key here and imports with no
        # Save Points, which is what such a campaign had.
        "checkpoints": [
            _exported_checkpoint(c, local) for c in _checkpoints_of(db, adventure)
        ],
        "storyCards": [
            {"type": c.type, "name": c.name, "keys": c.keys,
             "entry": c.entry, "notes": c.notes}
            for c in adventure.story_cards
        ],
        # M7. The imported knowledge library, carried by the same rule as
        # everything else here: what somebody chose goes in the file, what a
        # machine derives does not.
        #
        # So the source text and the reader's judgements about it travel —
        # content, classification, enabled, visibility, always-include, the
        # title and filename, the hash. Passages, FTS rows and vectors do not:
        # they are a deterministic function of the content, and the import
        # rebuilds them. That keeps a bundle a readable record of a campaign
        # rather than a database dump, and it keeps a campaign exported on one
        # machine importable on another whose embedding model is different.
        #
        # `contentHash` is exported although it is derivable, because it is the
        # identity the reader can check a restored file against — the one place
        # a derived value earns a place in the file is when its purpose is to
        # detect that the thing it describes has changed underneath it. The
        # import verifies it rather than trusting it.
        #
        # A bundle written before M7 has no key here and imports with an empty
        # library, which is what such a campaign had.
        "knowledge": [_exported_source(k) for k in adventure.knowledge_sources],
        "actions": [_exported_node(a, local) for a in nodes],
    }


def _exported_source(source: models.KnowledgeSource) -> dict:
    """One knowledge source, as it goes into the file."""
    return {
        "title": source.title,
        "originalFilename": source.original_filename,
        "classification": source.classification,
        "enabled": source.enabled,
        "visibility": source.visibility,
        "alwaysInclude": source.always_include,
        "contentHash": source.content_hash,
        "mediaType": source.media_type,
        "notes": source.notes,
        "importedAt": source.imported_at.isoformat() if source.imported_at else None,
        "content": source.content,
    }


_ROOT = {"parent": None, "forkDepth": None}


def _local(branch_id: int | None, local: dict[int, int]) -> int:
    return local.get(branch_id, 0) if branch_id is not None else 0


def _exported_branch(branch: models.Branch, local: dict[int, int]) -> dict:
    parent = (
        local.get(branch.parent_branch_id)
        if branch.parent_branch_id is not None else None
    )
    out = (
        dict(_ROOT) if parent is None
        else {"parent": parent, "forkDepth": branch.fork_depth}
    )
    # A player chose the name, so it goes into the file. That is the same rule
    # that puts the fork points in the file and leaves `lineage` out. An unnamed
    # branch omits the key rather than carry a null, which keeps the file for an
    # unnamed tree byte-identical to the one SP6 wrote.
    if branch.name:
        out["name"] = branch.name
    # M3. A branch the story left behind is a decision too — the depth a
    # divergent write departed at — so it goes in the file by the same rule that
    # puts the fork points in it. A restored backup that could not tell abandoned
    # history from active history would have lost the only thing distinguishing
    # them, since every row of both arrives either way. Omitted when the branch
    # is active, which keeps a file for a never-undone tree as it was.
    if branch.superseded_at is not None:
        out["supersededAt"] = branch.superseded_at.isoformat()
        out["supersededDepth"] = branch.superseded_depth
    return out


def _exported_node(action: models.Action, local: dict[int, int]) -> dict:
    node = {
        "branch": _local(action.branch_id, local),
        "depth": action.depth,
        "live": bool(action.live),
        "type": action.type,
        "text": action.text,
        "createdAt": action.created_at.isoformat() if action.created_at else None,
    }
    if action.reasoning:
        node["reasoning"] = action.reasoning
    # `{}` and an absent key mean different things. `{}` means the node left an
    # empty state behind, and an absent key means the state is unknown and the
    # live state stays as it is. An empty snapshot is therefore written rather
    # than omitted. It costs about eighteen bytes per row, and it decides
    # whether an undo clears a score or leaves it in place.
    if action.state_after is not None:
        node["stateAfter"] = action.state_after
    if action.world_state_after is not None:
        node["worldStateAfter"] = action.world_state_after
    # M5. Without this a restored campaign could be read but not moved around
    # inside: every Undo, Redo and Save Point restore reads the destination
    # node's snapshot, so a bundle carrying the turns and not the snapshots
    # imports a story whose history cannot be walked.
    if action.narrative_state_after is not None:
        node["narrativeStateAfter"] = action.narrative_state_after
    if action.state_changes:
        node["stateChanges"] = action.state_changes
    if action.world_delta:
        node["worldDelta"] = action.world_delta
    return node


def _exported_memory(memory: models.Memory, local: dict[int, int]) -> dict:
    return {
        "text": memory.text, "pinned": memory.pinned, "forgotten": memory.forgotten,
        "sourceStart": memory.source_start, "sourceEnd": memory.source_end,
        "useCount": memory.use_count,
        # The node this memory is attached to. A hand-written memory summarizes
        # no node, so it has a branch and no depth, and it keeps that shape
        # here.
        "branch": _local(memory.branch_id, local),
        "depth": memory.depth,
    }


def _checkpoints_of(db: Session, adventure: models.Adventure) -> list[models.Checkpoint]:
    """Returns the campaign's Save Points in creation order.

    Read with a query rather than through a relationship, for the reason
    `models.Branch` declares none: a relationship on `Adventure` would be loaded
    by anything that touches an adventure, and the export is the only thing in
    the application that wants every Save Point at once.
    """
    return (
        db.query(models.Checkpoint)
        .filter(models.Checkpoint.adventure_id == adventure.id)
        .order_by(models.Checkpoint.id)
        .all()
    )


def _exported_checkpoint(checkpoint: models.Checkpoint, local: dict[int, int]) -> dict:
    return {
        "name": checkpoint.name,
        "note": checkpoint.note,
        # The branch as a position in this file's list, like every other branch
        # reference in the bundle. The depth is a coordinate along it and needs
        # no translation.
        "branch": _local(checkpoint.branch_id, local),
        "depth": checkpoint.depth,
        "createdAt": checkpoint.created_at.isoformat() if checkpoint.created_at else None,
    }


def _imported_persona(persona) -> dict:
    """Reads a bundle's `persona` block into `Adventure` keyword arguments.

    A raw-dict import bypasses the schemas, so the strings are truncated to the
    widths the columns declare, in the same way the rest of `_import` does. A
    bundle written before Phase 18 has no block at all, and an empty persona is
    the same as having none.
    """
    if not isinstance(persona, dict):
        return {}
    return {
        "persona_name": str(persona.get("name") or "")[:schemas.PERSONA_NAME_MAX],
        "persona_pronouns":
            str(persona.get("pronouns") or "")[:schemas.PERSONA_PRONOUNS_MAX],
        "persona_desc": str(persona.get("desc") or ""),
    }


def _exported_anchor(
    adventure: models.Adventure, cursor: cursors.Cursor, local: dict[int, int]
) -> dict:
    branch_id, depth = cursor.stored(adventure)
    return {
        "branch": local.get(branch_id) if branch_id is not None else None,
        "depth": depth,
    }


# ---------------------------------------------------------------- importing

def check_format(bundle: dict) -> str:
    """Returns the bundle's version, or raises a 400."""
    fmt = bundle.get("format")
    if fmt in (FORMAT, LEGACY_FORMAT):
        return fmt
    raise HTTPException(
        400,
        f"Not an adventure export file (expected format {FORMAT} or {LEGACY_FORMAT}).",
    )


def plan(bundle: dict, version: str) -> dict:
    """Returns the bundle's tree, checked and normalized, before a row is written.

    The function has no side effects. It opens no session, needs no adventure,
    and creates nothing. It catches everything a hand-edited file can get wrong
    about the shape of a tree, because the alternative is an import that fails
    partway and leaves an adventure holding a story with a gap in it.

    Both versions produce the same shape, so `write` never learns that there are
    two formats. A version 1 bundle is a linear story, which is a tree with one
    branch, and its `variants` array is a sibling group written the old way.
    """
    branches = (
        _planned_branches(bundle) if version == FORMAT else [dict(_ROOT)]
    )
    nodes = (
        _planned_nodes(bundle, len(branches)) if version == FORMAT
        else _planned_v1_nodes(bundle)
    )
    head = _as_index(bundle.get("headBranch"), len(branches), default=0)
    return {
        "branches": branches,
        "nodes": nodes,
        "memories": _planned_memories(bundle, len(branches)),
        # M4. Empty for a version 1 bundle and for any version 2 bundle written
        # before Save Points existed, which is the same answer: no one had named
        # a position in those campaigns.
        "checkpoints": (
            _planned_checkpoints(bundle, len(branches), nodes)
            if version == FORMAT else []
        ),
        "head": head,
        # None means the file does not say, which is every version 1 bundle and
        # every version 2 bundle written before M3. `_point_the_head` derives it
        # then, which is what those files were written expecting.
        "headDepth": _planned_head_depth(bundle, branches, nodes, head),
        # Version 2 records where the derived work reached. Version 1 counted
        # it, and a count cannot become a node until the nodes exist. See
        # `settle`.
        "anchors": _planned_anchors(bundle, len(branches)) if version == FORMAT else None,
        "positions": None if version == FORMAT else {
            "memory": _as_int(bundle.get("memoryCursor"), 0),
            "summary": _as_int(bundle.get("summaryCursor"), 0),
        },
        # M7. Checked here with everything else, before a row is written, so a
        # hand-edited library fails the import rather than half-landing in it.
        "knowledge": _planned_knowledge(bundle),
    }


def _planned_knowledge(bundle: dict) -> list[dict]:
    """The knowledge sources in a bundle, checked and normalized.

    Every field is validated here rather than at write time, for the same reason
    the tree is: a file anyone can edit must be found wrong before it has
    written anything. A source that fails validation refuses the import — it is
    not silently dropped. A campaign whose imported Canon quietly did not arrive
    is a campaign whose narrator has stopped being told the rules, and the
    reader would have no way to notice.

    The one thing not trusted from the file is the hash. It is recomputed from
    the content that actually arrived, and a mismatch is reported: that is the
    whole reason a derived value is in the file at all.
    """
    entries = bundle.get("knowledge")
    if entries is None:
        return []
    if not isinstance(entries, list):
        raise HTTPException(400, "The knowledge section of this file is not a list.")
    if len(entries) > knowledge_importer.MAX_SOURCES_PER_ADVENTURE:
        raise HTTPException(
            400,
            f"This file contains {len(entries)} knowledge sources — the limit "
            f"is {knowledge_importer.MAX_SOURCES_PER_ADVENTURE}.",
        )
    planned: list[dict] = []
    for i, entry in enumerate(entries):
        if not isinstance(entry, dict):
            raise HTTPException(400, f"Knowledge source {i + 1} is not an object.")
        content = entry.get("content")
        if not isinstance(content, str) or not content.strip():
            raise HTTPException(400, f"Knowledge source {i + 1} carries no content.")
        if len(content.encode("utf-8")) > knowledge_importer.MAX_SOURCE_BYTES:
            raise HTTPException(
                400, f"Knowledge source {i + 1} is larger than the import limit."
            )
        classification = entry.get("classification")
        if not knowledge_classes.is_class(classification):
            raise HTTPException(
                400,
                f"Knowledge source {i + 1} has no valid classification "
                "(expected canon, reference or inspiration).",
            )
        visibility = entry.get("visibility")
        if not knowledge_classes.is_visibility(visibility):
            visibility = knowledge_classes.NORMAL
        filename = knowledge_importer.safe_filename(
            str(entry.get("originalFilename") or "")
        )
        stated = entry.get("contentHash")
        actual = knowledge_chunking.digest(content)
        planned.append({
            "title": str(entry.get("title") or filename or "Imported source")[:200],
            "original_filename": filename,
            "classification": classification,
            "enabled": bool(entry.get("enabled", True)),
            "visibility": visibility,
            "always_include": bool(entry.get("alwaysInclude", False))
            and classification == knowledge_classes.CANON,
            "media_type": (
                str(entry.get("mediaType"))
                if entry.get("mediaType") in ("text/plain", "text/markdown")
                else "text/markdown"
            ),
            "notes": str(entry.get("notes") or ""),
            "imported_at": _as_time(entry.get("importedAt")),
            "content": content,
            "content_hash": actual,
            "hash_mismatch": isinstance(stated, str) and bool(stated) and stated != actual,
        })
    return planned


def _derived_tip(branches: list[dict], nodes: list[dict], head: int) -> int:
    """Returns where the head branch's story ends, which is where a file that
    does not state a head depth is opened.

    This is the rule `tree.refresh_head` applies and the one every bundle
    written before M3 was exported under: the deepest node that arrived on the
    head branch, or, for a branch that carries none of its own, the fork point
    it inherits its story up to.
    """
    depths = [n["depth"] for n in nodes if n["branch"] == head]
    if depths:
        return max(depths)
    fork_depth = branches[head].get("forkDepth")
    return fork_depth if fork_depth is not None else lineage.NO_DEPTH


def _planned_head_depth(
    bundle: dict, branches: list[dict], nodes: list[dict], head: int
) -> int | None:
    """Returns the active head depth the file states, or None if it states none.

    Checked here rather than repaired later, for the reason the module docstring
    gives: a coordinate that disagrees with the rows is not a value any read can
    be given, and an import that discovers it afterwards has already written
    half a tree.

    Two bounds. A head past the retained tip is a file claiming the story is
    read somewhere it does not reach — the shape a truncated or hand-edited
    export takes, and the one that would silently move the reader forward.
    Below, `NO_DEPTH` is the floor, because an adventure with no story at all
    sits one step in front of its first node.

    A head *behind* the tip is not an error. It is the whole point: it is a
    story the user undid and did not redo, and it must import undone.
    """
    stated = bundle.get("headDepth")
    if stated is None:
        return None
    tip = _derived_tip(branches, nodes, head)
    if not _is_int(stated) or stated < lineage.NO_DEPTH:
        raise HTTPException(
            400, f"The file gives the active head depth as {stated!r}."
        )
    if stated > tip:
        raise HTTPException(
            400,
            f"The file reads its story at depth {stated}, but branch {head} "
            f"ends at {tip}.",
        )
    return stated


def _planned_branches(bundle: dict) -> list[dict]:
    raw = bundle.get("branches")
    entries = [b for b in raw if isinstance(b, dict)] if isinstance(raw, list) else []
    if not entries:
        return [dict(_ROOT)]
    specs: list[dict] = []
    for i, entry in enumerate(entries):
        parent = entry.get("parent")
        name = _planned_branch_name(entry, i)
        left = _planned_supersession(entry)
        if parent is None:
            specs.append(dict(_ROOT, **({"name": name} if name else {}), **left))
            continue
        # A branch may fork only from a branch listed before it. The export
        # writes them that way, because branches are numbered in creation order
        # and a parent always exists first. Requiring it here guarantees the
        # graph is acyclic for the cost of one comparison. A lineage is computed
        # by walking to the parent, so a cycle would be an import that never
        # returns.
        if not _is_int(parent) or not 0 <= parent < i:
            raise HTTPException(
                400,
                f"Branch {i} forks from branch {parent!r}, which is not one of "
                f"the {i} branches listed before it.",
            )
        fork_depth = entry.get("forkDepth")
        if not _is_int(fork_depth):
            raise HTTPException(
                400,
                f"Branch {i} forks from branch {parent} but does not say at "
                f"what depth.",
            )
        specs.append({
            "parent": parent, "forkDepth": fork_depth,
            **({"name": name} if name else {}),
            **left,
        })
    return specs


def _planned_supersession(entry: dict) -> dict:
    """Returns the branch's disposition as the file gives it, or nothing.

    Both keys or neither. A time with no depth cannot say what was displaced and
    a depth with no time is not a record of anything having happened, so a file
    carrying one of them is treated as carrying neither rather than half of a
    fact — no read depends on these columns, and inventing the missing half
    would be the only way to get a wrong answer out of them.
    """
    at = _as_time(entry.get("supersededAt"))
    depth = entry.get("supersededDepth")
    if at is None or not _is_int(depth):
        return {}
    return {"supersededAt": at, "supersededDepth": depth}


def _planned_branch_name(entry: dict, i: int) -> str | None:
    """Returns the name a branch entry carries, or `None` if nobody named it.

    The check runs before the row is created rather than being left to the
    column, for the reason the planner exists. A 400 from a function with no side
    effects is better than a half-written adventure and a database error three
    branches in.
    """
    raw = entry.get("name")
    if raw is None:
        return None
    if not isinstance(raw, str):
        raise HTTPException(400, f"Branch {i} has a name that is not text.")
    name = raw.strip()
    if len(name) > schemas.BRANCH_NAME_MAX:
        raise HTTPException(
            400,
            f"Branch {i}'s name is longer than {schemas.BRANCH_NAME_MAX} "
            f"characters.",
        )
    return name or None


def _planned_nodes(bundle: dict, branches: int) -> list[dict]:
    raw = bundle.get("actions")
    nodes: list[dict] = []
    for entry in raw if isinstance(raw, list) else []:
        if not isinstance(entry, dict) or not str(entry.get("text") or ""):
            continue
        branch = entry.get("branch", 0)
        if not _is_int(branch) or not 0 <= branch < branches:
            raise HTTPException(
                400,
                f"An action names branch {branch!r}, but the file lists "
                f"{branches}.",
            )
        depth = entry.get("depth")
        if not _is_int(depth) or depth < 0:
            raise HTTPException(
                400, f"An action on branch {branch} has no depth to sit at."
            )
        nodes.append({
            "branch": branch,
            "depth": depth,
            "live": bool(entry.get("live", True)),
            "narrativeStateAfter": _as_dict(entry.get("narrativeStateAfter")),
            "stateChanges": _as_dict(entry.get("stateChanges")),
            "type": str(entry.get("type") or "story")[:TYPE_MAX],
            "text": str(entry.get("text") or ""),
            "reasoning": _as_text(entry.get("reasoning")),
            "stateAfter": _as_dict(entry.get("stateAfter")),
            "worldStateAfter": _as_dict(entry.get("worldStateAfter")),
            "worldDelta": _as_dict(entry.get("worldDelta")),
            "createdAt": _as_time(entry.get("createdAt")),
        })
    return nodes


def _planned_v1_nodes(bundle: dict) -> list[dict]:
    """Returns a version 1 bundle's turns as nodes, one node per attempt.

    The `variants` array is the repeating group SP4 unpacked, so reading one
    performs the same split that migration 60 does. Every attempt becomes a row
    at the turn's coordinate, and `variantIndex` selects the live one. The index
    is clamped, because a hand-edited bundle can name an attempt its own list
    does not contain, and a turn with no live node is a turn no read can see.

    The depth is the bundle's `index`. A version 1 bundle has one branch, where
    the two numbers agree.
    """
    raw = bundle.get("actions")
    nodes: list[dict] = []
    for i, entry in enumerate(raw if isinstance(raw, list) else []):
        if not isinstance(entry, dict) or not str(entry.get("text") or ""):
            continue
        depth = _as_int(entry.get("index"), i)
        kind = str(entry.get("type") or "story")[:TYPE_MAX]
        variants = [v for v in (entry.get("variants") or []) if isinstance(v, dict)]
        if not variants:
            variants = [{"text": entry["text"], "reasoning": entry.get("reasoning")}]
        live = min(max(_as_int(entry.get("variantIndex"), 0), 0), len(variants) - 1)
        for n, variant in enumerate(variants):
            text = str(variant.get("text") or "")
            if not text:
                continue
            nodes.append({
                "branch": 0,
                "depth": max(depth, 0),
                "live": n == live,
                "type": kind,
                "text": text,
                "reasoning": _as_text(variant.get("reasoning")),
                "stateAfter": None,
                "worldStateAfter": None,
                "worldDelta": None,
                "createdAt": _as_time(variant.get("createdAt")),
            })
    return nodes


def _planned_memories(bundle: dict, branches: int) -> list[dict]:
    raw = bundle.get("memories")
    out: list[dict] = []
    for entry in raw if isinstance(raw, list) else []:
        if not isinstance(entry, dict) or not str(entry.get("text") or "").strip():
            continue
        out.append({
            "text": str(entry["text"]),
            "pinned": bool(entry.get("pinned", False)),
            "forgotten": bool(entry.get("forgotten", False)),
            "sourceStart": entry.get("sourceStart"),
            "sourceEnd": entry.get("sourceEnd"),
            "useCount": _as_int(entry.get("useCount"), 0),
            # A value that is out of range, rather than absent, means the file
            # disagrees with itself. The root is the safe reading, because a
            # memory on a branch nothing can see never reaches a prompt
            # again.
            "branch": _as_index(entry.get("branch"), branches, default=0),
            "depth": entry.get("depth") if _is_int(entry.get("depth")) else None,
        })
    return out


def _planned_checkpoints(
    bundle: dict, branches: int, nodes: list[dict]
) -> list[dict]:
    """Returns the file's Save Points, checked against the tree it also carries.

    A Save Point whose coordinate names no turn in the file is dropped rather
    than imported, and dropped rather than raising. The two halves of that are
    each deliberate:

    * Dropped, because an imported pointer to a position the imported story does
      not contain is a Save Point that can only ever refuse to restore. It would
      be a row that exists to disappoint.
    * Not a 400, unlike the head depth. The head is a position the campaign is
      read at, so a file that misplaces it opens the story in the wrong place
      and every read is affected. A Save Point is a bookmark, and a bad one
      spoils nothing else in the file — refusing to import a whole campaign
      because one bookmark is wrong would lose the story to save the bookmark.

    A name that is blank once trimmed is dropped for the same reason the create
    endpoint refuses one: an unnamed Save Point is not identifiable in a list.
    """
    # Every coordinate the file writes, not only the ones it marks live.
    # `_write_nodes` makes exactly one attempt at each coordinate live whatever
    # the file says, so a coordinate that exists is a coordinate that will
    # resolve — and reading the flags here would drop a Save Point over a
    # question the writer has already settled.
    written = {(n["branch"], n["depth"]) for n in nodes}
    raw = bundle.get("checkpoints")
    out: list[dict] = []
    for entry in raw if isinstance(raw, list) else []:
        if not isinstance(entry, dict):
            continue
        name = str(entry.get("name") or "").strip()[:schemas.CHECKPOINT_NAME_MAX]
        if not name:
            continue
        depth = entry.get("depth")
        if not _is_int(depth):
            continue
        branch = _as_index(entry.get("branch"), branches, default=None)
        if branch is None or (branch, depth) not in written:
            continue
        out.append({
            "name": name,
            "note": str(entry.get("note") or ""),
            "branch": branch,
            "depth": depth,
            "createdAt": _as_time(entry.get("createdAt")),
        })
    return out


def _planned_anchors(bundle: dict, branches: int) -> dict:
    anchors = {}
    for name in ("memory", "summary"):
        raw = bundle.get(f"{name}Cursor")
        raw = raw if isinstance(raw, dict) else {}
        branch = raw.get("branch")
        anchors[name] = (
            _as_index(branch, branches) if branch is not None else None,
            _as_int(raw.get("depth"), lineage.NO_DEPTH),
        )
    return anchors


# ------------------------------------------------------------------ writing

def write(db: Session, adventure: models.Adventure, story: dict) -> None:
    """Writes a planned tree onto a newly created adventure.

    The order is fixed. Branches come first, because a node needs a branch id.
    The nodes come next, because the head and the anchors name a node.
    """
    ids = _write_branches(db, adventure, story["branches"])
    _write_nodes(db, adventure, story["nodes"], ids)
    _write_memories(db, adventure, story["memories"], ids)
    _point_the_head(adventure, story, ids)
    _write_checkpoints(db, adventure, story["checkpoints"], ids)
    _write_anchors(adventure, story, ids)
    _write_knowledge(db, adventure, story.get("knowledge") or [])


def _write_knowledge(
    db: Session, adventure: models.Adventure, specs: list[dict]
) -> None:
    """Restores the imported library, and rebuilds the index it needs.

    The bundle carries the source and not its passages, so this is where they
    come back: `build_index` runs the same deterministic chunker the original
    import ran, against the same text, and produces the same passages. Lexical
    retrieval therefore works the moment the import finishes, with no reindex
    step and no explanation owed to the reader.

    Vectors do not come back, because they were never in the file. The source
    lands `embed_state = "idle"` with no vectors, and the next turn's post-turn
    pass builds them against whatever embedding model *this* machine has — which
    is the right answer, and the reason exporting the vectors would have been
    the wrong one.

    A source whose passages cannot be built is recorded as `failed` with the
    reason rather than raising. By this point the story, its tree, its head and
    its Save Points are already written, and refusing the whole campaign over a
    rebuildable index would trade the valuable thing for the cheap one. The
    failure is visible on the source and in the knowledge status endpoint, and
    Reindex is the repair.
    """
    for spec in specs:
        source = models.KnowledgeSource(
            adventure_id=adventure.id,
            title=spec["title"],
            original_filename=spec["original_filename"],
            classification=spec["classification"],
            enabled=spec["enabled"],
            visibility=spec["visibility"],
            always_include=spec["always_include"],
            content=spec["content"],
            content_hash=spec["content_hash"],
            byte_size=len(spec["content"].encode("utf-8")),
            media_type=spec["media_type"],
            notes=spec["notes"],
            parser_version=knowledge_chunking.PARSER_VERSION,
            chunking_version=knowledge_chunking.CHUNKING_VERSION,
            index_state="pending",
            embed_state="idle",
        )
        if spec["imported_at"] is not None:
            source.imported_at = spec["imported_at"]
        if spec["hash_mismatch"]:
            # Not a refusal. The content is what it is, and the recomputed hash
            # above is the one stored — but the file said something different,
            # which means it was edited after it was written, and the reader
            # should be able to find that out.
            source.notes = (
                f"{source.notes}\n[import] The content hash in the export file "
                "did not match the content it carried; the stored hash was "
                "recomputed from what arrived."
            ).strip()
        db.add(source)
        db.flush()
        try:
            knowledge_importer.build_index(db, source)
        except Exception as exc:  # noqa: BLE001 - recorded, not raised
            source.index_state = "failed"
            source.index_detail = f"{type(exc).__name__}: {exc}"[:2000]


def _write_branches(
    db: Session, adventure: models.Adventure, specs: list[dict]
) -> list[int]:
    """Writes one row per branch, computing the lineage rather than reading it.

    The rows are inserted through Core and the lineage is written second, for the
    reason `tree.root_branch` gives: the lineage names the row's own id. The
    parent's cached ancestry is capped at this fork, which is the arithmetic
    `tree.fork` performs. A fork made now and a fork made a year ago and then
    exported have to produce the same rows.
    """
    ids: list[int] = []
    lineages: list[list[list]] = []
    for spec in specs:
        parent = spec.get("parent")
        fork_depth = spec.get("forkDepth")
        new_id = db.execute(
            insert(models.Branch).values(
                adventure_id=adventure.id,
                parent_branch_id=ids[parent] if parent is not None else None,
                fork_depth=fork_depth if parent is not None else None,
                lineage=[],
                name=spec.get("name"),
                created_at=models.utcnow(),
                superseded_at=spec.get("supersededAt"),
                superseded_depth=spec.get("supersededDepth"),
            )
        ).inserted_primary_key[0]
        entries = [[new_id, None]]
        if parent is not None:
            entries += [
                [branch_id, fork_depth if cap is None else min(cap, fork_depth)]
                for branch_id, cap in lineages[parent]
            ]
        db.execute(
            update(models.Branch)
            .where(models.Branch.id == new_id)
            .values(lineage=entries)
        )
        ids.append(new_id)
        lineages.append(entries)
    return ids


def _write_nodes(
    db: Session, adventure: models.Adventure, specs: list[dict], ids: list[int]
) -> None:
    """Writes the nodes, grouped into the turns they are attempts at.

    One value is decided here rather than read from the file. Exactly one
    attempt in each group is made live, because a file can name none or several,
    and a turn with no live node disappears from the story.
    """
    groups: dict[tuple[int, int], list[models.Action]] = {}
    for spec in specs:
        key = (spec["branch"], spec["depth"])
        action = models.Action(
            adventure_id=adventure.id,
            branch_id=ids[spec["branch"]],
            depth=spec["depth"],
            type=spec["type"],
            text=spec["text"],
            reasoning=spec["reasoning"],
            live=spec["live"],
            state_after=spec["stateAfter"],
            world_state_after=spec["worldStateAfter"],
            world_delta=spec["worldDelta"],
            narrative_state_after=spec.get("narrativeStateAfter"),
            state_changes=spec.get("stateChanges"),
        )
        if spec["createdAt"] is not None:
            action.created_at = spec["createdAt"]
        db.add(action)
        groups.setdefault(key, []).append(action)

    for rows in groups.values():
        live = next((row for row in rows if row.live), rows[0])
        for row in rows:
            row.live = row is live


def _write_memories(
    db: Session, adventure: models.Adventure, specs: list[dict], ids: list[int]
) -> None:
    for spec in specs:
        memory = models.Memory(
            adventure_id=adventure.id,
            text=spec["text"],
            pinned=spec["pinned"],
            forgotten=spec["forgotten"],
            source_start=spec["sourceStart"],
            source_end=spec["sourceEnd"],
            use_count=spec["useCount"],
            branch_id=ids[spec["branch"]],
            depth=spec["depth"],
        )
        # A version 1 memory has no depth of its own. `source_end` is the index
        # of the last action it summarizes, which on one branch is that node's
        # depth.
        if memory.depth is None and memory.source_end is not None:
            memory.depth = memory.source_end
        # A version 1 memory that summarizes nothing, which means one the player
        # typed, has no depth to derive. Leaving it NULL here would recreate on
        # import the state migration 62 exists to end. `Path._entry_clause`
        # compares `depth <= max_depth`, which a NULL fails, so the memory would
        # disappear from every branch as soon as the imported adventure was
        # forked. The root is the answer the migration gives, for the same
        # reason: 0 is at or before every fork point, so the memory is visible
        # from every path this adventure can grow.
        if memory.depth is None:
            memory.depth = lineage.ROOT_DEPTH
        db.add(memory)


def _write_checkpoints(
    db: Session, adventure: models.Adventure, specs: list[dict], ids: list[int]
) -> None:
    """Writes the Save Points, and moves nothing.

    Note what this function does not touch. The head is pointed by
    `_point_the_head` from the file's own `headBranch`/`headDepth`, and importing
    a Save Point must not disturb it — a campaign exported at turn 30 with a
    Save Point at turn 12 opens at turn 30. The bundle records where the story
    was being read and, separately, which positions someone named; restoring one
    of them is a thing the user does afterwards, not a thing an import does for
    them.
    """
    for spec in specs:
        checkpoint = models.Checkpoint(
            adventure_id=adventure.id,
            name=spec["name"],
            note=spec["note"],
            branch_id=ids[spec["branch"]],
            depth=spec["depth"],
        )
        if spec["createdAt"] is not None:
            checkpoint.created_at = spec["createdAt"]
        db.add(checkpoint)


def _point_the_head(
    adventure: models.Adventure, story: dict, ids: list[int]
) -> None:
    """Sets which branch the story is played on, and where it is being read.

    Both come from the file now (M3). A bundle that states its head depth is
    opened exactly where its owner left it, undone turns and all, and the turns
    past that point arrive as the retained future they were exported as.

    A file that states none is opened at the tip of its head branch — whatever
    arrived on it, or, for a branch carrying no nodes of its own, the fork point
    that is the last node its story contains. That is the rule
    `tree.refresh_head` applies and the only answer a pre-M3 bundle can be given,
    because at the time it was written the head could not be anywhere else.
    """
    head = story["head"]
    adventure.head_branch_id = ids[head]
    if story["headDepth"] is not None:
        adventure.head_depth = story["headDepth"]
        return
    adventure.head_depth = _derived_tip(story["branches"], story["nodes"], head)


def _write_anchors(
    adventure: models.Adventure, story: dict, ids: list[int]
) -> None:
    """Records how far the memories and the summary have read. Version 2 only.

    A version 1 bundle stores a count instead, and a count cannot be resolved to
    a node until the nodes are in the database. `settle` does that part
    afterwards.
    """
    anchors = story["anchors"]
    if anchors is None:
        return
    for cursor in cursors.ALL:
        branch, depth = anchors[cursor.name]
        cursor.anchor(adventure, ids[branch] if branch is not None else None, depth)


def settle(adventure: models.Adventure, story: dict) -> None:
    """Resolves a version 1 bundle's counts into anchors, once the nodes exist.

    A version 1 file records how far the memories and the summary have read as
    a count, so the anchor is found by counting that far along the story. A
    version 2 file carries the anchor itself, which `_write_anchors` has
    already stored, so there is nothing left to do.

    The caller runs this after the flush, because counting needs the actions to
    be queryable.
    """
    positions = story["positions"]
    if positions is None:
        return
    for cursor in cursors.ALL:
        cursors.anchor_at_position(adventure, cursor, positions[cursor.name])


def materialize(
    db: Session, payload: dict, story: dict, user_id: int | None
) -> models.Adventure:
    """Writes a planned bundle into a new adventure owned by `user_id`.

    Call `check_format` and `plan` first, and apply any rate or size limits
    before calling this. The split exists because those checks belong to the
    import endpoint alone: the guest starter writes a file the server ships, so
    it has nothing to rate-limit and no untrusted list to cap.

    The caller commits. Two flushes happen here, because the anchors and the
    legacy counts describe one boundary in two coordinate systems, and aligning
    them needs the actions to be queryable.
    """
    # A raw-dict import bypasses the schemas, so truncate strings bound for
    # VARCHAR columns. Postgres enforces the widths. See `schemas.py`.
    adventure = models.Adventure(
        user_id=user_id,
        title=str(payload.get("title") or "Imported Adventure")[:schemas.NAME_MAX],
        memory=str(payload.get("memory") or ""),
        authors_note=str(payload.get("authorsNote") or ""),
        ai_instructions=str(payload.get("aiInstructions") or ""),
        story_summary=str(payload.get("storySummary") or ""),
        world_state=payload.get("worldState") or {},
        # M5. Normalised on the way in, so a hand-edited or truncated state
        # section costs the section rather than the campaign — the story is the
        # valuable thing, and a malformed document should not refuse an import.
        narrative_state=narrative_model.normalize(payload.get("narrativeState"))
        if isinstance(payload.get("narrativeState"), dict) else None,
        campaign_canon=payload.get("campaignCanon")
        if isinstance(payload.get("campaignCanon"), dict) else None,
        auto_summarize=bool(payload.get("autoSummarize", False)),
        memory_bank_enabled=bool(payload.get("memoryBankEnabled", False)),
        **_imported_persona(payload.get("persona")),
    )
    db.add(adventure)
    db.flush()

    for card in payload.get("storyCards") or []:
        if isinstance(card, dict):
            db.add(models.StoryCard(
                adventure_id=adventure.id,
                type=str(card.get("type") or "")[:schemas.CARD_TYPE_MAX],
                name=str(card.get("name") or "")[:schemas.NAME_MAX],
                keys=str(card.get("keys") or ""),
                entry=str(card.get("entry") or ""),
                notes=str(card.get("notes") or ""),
            ))

    # A bundle exported before M2 may carry "scripts" and "scriptState".
    # Campaign scripting is gone, so both are ignored rather than rejected: the
    # story, its tree, its cards and its memories still import intact, which is
    # what the bundle is for.

    write(db, adventure, story)
    db.flush()
    db.expire(adventure, ["actions"])
    settle(adventure, story)
    return adventure

# ------------------------------------------------------------------ reading
# Small coercions. A raw-dict import bypasses the schemas, so every value from a
# bundle is whatever the JSON held.

def _is_int(value) -> bool:
    """Returns whether `value` is an integer. Python counts `True` as an `int`,
    and a coordinate does not."""
    return isinstance(value, int) and not isinstance(value, bool)


def _as_int(value, default: int) -> int:
    return value if _is_int(value) else default


def _as_index(value, count: int, default: int | None = None) -> int | None:
    """Returns a local branch number, or `default` when the file names a branch
    that is not present.

    An out-of-range value means the file disagrees with itself, and it is not a
    value any read can be given.
    """
    return value if _is_int(value) and 0 <= value < count else default


def _as_text(value) -> str | None:
    return str(value) if value else None


def _as_dict(value) -> dict | None:
    return value if isinstance(value, dict) else None


def _as_time(value) -> datetime | None:
    if not isinstance(value, str):
        return None
    try:
        return datetime.fromisoformat(value.replace("Z", "+00:00"))
    except ValueError:
        return None