"""v1.1 WP-B.2: how faithfully a real model's memories keep the facts of their block.

**Diagnostic only. Nothing here is imported by the application, and nothing here
is a release gate.**

Why it exists: the first isolation-valid real-model WP-B run failed at creation.
The whole planting block reached the summariser, and the memory it wrote left the
planted fact out (`V1.1-WP-B2-REPORT.md` §L.2). B2.4 tried the plan's bounded
remedy, a memory prompt instructing the model to keep named facts and objects. It
was measured with this module, did not correct the failure, and was **not
shipped** (§T). The module stays so the limitation can be measured again, on this
model or a different one.

- **Fixtures.** Short story blocks, each built around a fact a later scene could
  turn on, with the ordinary texture a real block carries around it. They are
  genre-neutral (office, contemporary, a science-fiction-neutral station), plus
  the attempt-2 planting block itself. Each names the facts a memory must keep,
  whom each belongs to, and what it must not invent.
- **The checker** (`evaluate`) is deterministic and reads only the memory text.
  It is a heuristic, and says so:
  - a fact counts as kept when one sentence names every part of it;
  - attribution is the nearest named character before the fact's verb;
  - it also reports word count, a leading "Memory:" and second-person "you".
- **The comparison** (`compare`) sends each fixture to a real model through the
  application's own provider and `memorybank.memory_user_prompt`. It scores
  memories under the shipped prompt and under the rejected B2.4 experiment.

A scripted summariser cannot show what a prompt makes a model do. So the
deterministic tests prove only that the checker is right and the fixtures reach
the summariser; model quality is measured here, with inference, and reported.

    # the real-model measurement (inference: ask first)
    .venv/bin/python -m tools.memory_fidelity --endpoint <v1 URL> \\
        --model qwen2.5:3b-instruct-16k --samples 5 --out "$HOME/v11-evidence/<label>"
"""

from __future__ import annotations

import re
from dataclasses import dataclass, field

#: The memory prompt WP-B.2 B2.4 tried and **rejected**. Kept verbatim so the
#: experiment in `V1.1-WP-B2-REPORT.md` §T can be repeated; the application
#: never uses it.
B24_EXPERIMENT_PROMPT = (
    "You compress interactive-fiction story excerpts into memories. Respond with "
    "1-2 plain sentences in past tense, in at most 50 words.\n\n"
    "Keep the concrete facts a later scene could turn on: specific people, "
    "objects and places; where something is; who has, hid, found, knows, saw or "
    "promised what; injuries, clues and commitments. A distinctive fact comes "
    "before mood, scenery, routine movement and small talk. Drop those first, "
    "however much of the excerpt they fill, and never let a later passage crowd "
    "out an earlier fact.\n\n"
    "Keep each fact with the person it belongs to. Never move an action, promise, "
    "possession, statement or piece of knowledge from one character to another, "
    "and never add a fact the excerpt does not state.\n\n"
    'Write in the third person. The narration calls the protagonist "you". The '
    'protagonist\'s own actions and words are the lines that begin with ">", '
    'written as "I" or "You", and what they establish is part of the story just '
    "as the narration is. The protagonist is named in the Cast: refer to them by "
    'that name, never as "you" or "I". If the Cast gives no name for them, call '
    'them "the player". Name the other characters too rather than writing "he", '
    '"she" or "they" on their own — this memory will be read on its own, much '
    "later, with nothing around it to say who a pronoun meant.\n\n"
    "No preamble, no commentary."
)

TARGET_WORDS = 50


@dataclass(frozen=True)
class Fact:
    """One fact a memory must keep.

    `groups`: every group must be matched in one sentence, by any of its terms.
    `verbs`: the relation. When one is in that sentence, the nearest named
    character before it is who the memory says the fact belongs to.
    `actor`: whom it belongs to. `None` for a fact with no owner.
    """

    name: str
    groups: tuple[tuple[str, ...], ...]
    actor: str | None = None
    verbs: tuple[str, ...] = ()


@dataclass(frozen=True)
class Fixture:
    fixture_id: str
    genre: str
    requirement: str  # which measurement this fixture serves
    protagonist: str
    others: tuple[str, ...]
    actions: tuple[tuple[str, str], ...]  # (type, text), oldest first
    facts: tuple[Fact, ...]
    min_facts: int | None = None  # default: all
    #: Regexes a faithful memory must not match. Each is anchored on the wrong
    #: character as the subject ("Marcus promised"), because a faithful memory
    #: may name that character elsewhere in the same sentence ("promised Marcus").
    forbidden: tuple[str, ...] = ()
    #: Hand-written memories for the checker's own tests: one that should pass,
    #: and failures that should not, each with the reason it must report.
    faithful: str = ""
    unfaithful: tuple[tuple[str, str], ...] = field(default_factory=tuple)

    @property
    def cast(self) -> tuple[str, ...]:
        return (self.protagonist, *self.others)

    @property
    def raw(self) -> str:
        return "\n\n".join(text for _, text in self.actions)


# ------------------------------------------------------------------ checker

def _term(term: str) -> re.Pattern:
    body = r"\s+".join(re.escape(part) for part in term.lower().split())
    return re.compile(rf"(?<![a-z]){body}(?:s|es|ed|d)?(?![a-z])")


def _sentences(text: str) -> list[str]:
    return [s.strip() for s in re.split(r"(?<=[.!?])\s+", text or "") if s.strip()]


def _first(low: str, terms) -> int | None:
    found = [m.start() for t in terms for m in [_term(t).search(low)] if m]
    return min(found) if found else None


def _attributed_to(sentence: str, fact: Fact, cast: tuple[str, ...]) -> str | None:
    """The character this sentence gives the fact to, or None if it names none."""
    low = sentence.lower()
    at = _first(low, fact.verbs) if fact.verbs else None
    names = [(m.start(), name) for name in cast for m in _term(name).finditer(low)]
    if at is not None:
        before = [(pos, name) for pos, name in names if pos < at]
        if before:
            return max(before)[1]
    return min(names)[1] if names else None


def evaluate(fixture: Fixture, memory: str) -> dict:
    """What a memory kept, whom it gave each fact to, what it invented, and how
    it is framed."""
    memory = memory or ""
    sentences = _sentences(memory)
    facts = {}
    for fact in fixture.facts:
        holding = [s for s in sentences
                   if all(_first(s.lower(), group) is not None for group in fact.groups)]
        owners = sorted({o for s in holding
                         if (o := _attributed_to(s, fact, fixture.cast)) is not None})
        facts[fact.name] = {
            "kept": bool(holding),
            "attributed_to": owners,
            "attribution_ok": (fact.actor is None or not holding
                               or (owners != [] and set(owners) == {fact.actor})),
        }
    kept = sum(1 for f in facts.values() if f["kept"])
    needed = len(fixture.facts) if fixture.min_facts is None else fixture.min_facts
    inventions = [p for p in fixture.forbidden if re.search(p, memory, re.I)]
    words = len(memory.split())
    misattributed = [name for name, f in facts.items() if f["kept"] and not f["attribution_ok"]]
    return {
        "facts": facts,
        "kept": kept,
        "needed": needed,
        "retained": kept >= needed,
        "misattributed": misattributed,
        "inventions": inventions,
        "words": words,
        "over_target": words > TARGET_WORDS,
        # Framing the shipped prompt's rules exist to prevent.
        "memory_prefix": memory.lstrip().lower().startswith("memory:"),
        "second_person": re.search(r"\byou(r|rs|rself)?\b", memory, re.I) is not None,
        "passed": kept >= needed and not misattributed and not inventions,
    }


# ----------------------------------------------------------------- fixtures

OFFICE_TEXTURE = (
    "The open-plan floor hums with keyboards and the air conditioning rattles in "
    "its vent. Someone has left a birthday card on the printer, and the coffee "
    "machine gurgles through another pot."
)

FIXTURES: tuple[Fixture, ...] = (
    Fixture(
        fixture_id="object_place_office",
        genre="office",
        requirement="distinctive object and place",
        protagonist="Dana", others=("Priya",),
        actions=(
            ("start", "Monday at the insurance office. Dana is covering the late shift."),
            ("do", "> You check the queue of unanswered claims."),
            ("ai", f"{OFFICE_TEXTURE} Priya walks past your desk carrying a stack of folders, "
                   "and slides the red backup drive into the bottom drawer of the grey filing "
                   "cabinet in the archive room before locking it. The phones ring twice and stop. "
                   "Rain streaks the tall windows while the floor slowly empties."),
            ("do", "> You ask Priya whether the audit is still on for Thursday."),
            ("ai", "Priya shrugs, says nobody tells her anything, and goes back to her own desk. "
                   "The cleaners arrive with their carts and the lights dim on a timer."),
            ("do", "> You log off and pack your bag."),
        ),
        facts=(Fact("drive in the cabinet", (("backup drive", "drive"), ("drawer", "filing cabinet", "cabinet")),
                    actor="Priya", verbs=("slid", "slide", "put", "placed", "locked", "hid", "stored", "left")),),
        forbidden=(r"\bdana\s+(had\s+)?(took|takes|has|holds|held|stole|locked|slid|hid)\b[^.]*\bdrive\b",),
        faithful="Priya locked the red backup drive in the bottom drawer of the grey filing cabinet "
                 "in the archive room while Dana covered the late shift.",
        unfaithful=(
            ("Dana covered a quiet late shift at the insurance office while rain fell and the "
             "cleaners arrived.", "not retained"),
            ("Dana locked the red backup drive in the bottom drawer of the filing cabinet.",
             "misattributed"),
        ),
    ),
    Fixture(
        fixture_id="player_fact_station",
        genre="science-fiction-neutral",
        requirement="player-established concrete fact",
        protagonist="Reyes", others=("Okafor",),
        actions=(
            ("start", "Deck four of the relay station, halfway through the night cycle. Reyes is on maintenance duty."),
            ("do", "> You walk the corridor checking the pressure seals."),
            ("ai", "The corridor lights pulse a dim blue. Condensation beads on the pipes, and "
                   "somewhere below a pump cycles on with a shudder. Chief Okafor passes with a "
                   "tablet under one arm and nods without stopping."),
            ("do", "> I watch Chief Okafor seal the coolant sample in locker nine and log it under a false name."),
            ("ai", "The night cycle drags on. The ventilation hisses, a door chimes somewhere "
                   "down the ring, and the viewport shows the same slow turn of stars it always "
                   "does. You finish the seal checks and sign the maintenance sheet, and the "
                   "corridor settles back into its usual hum."),
            ("do", "> You head back to your bunk."),
        ),
        facts=(Fact("sample in locker nine", (("coolant sample", "sample"), ("locker",)),
                    actor="Okafor", verbs=("seal", "sealed", "locked", "put", "stored", "hid", "logged", "placed")),),
        forbidden=(r"\breyes\s+(had\s+)?(sealed|seals|hid|stored|locked|logged)\b[^.]*\bsample\b",),
        faithful="Reyes saw Chief Okafor seal the coolant sample in locker nine and log it under a false name.",
        unfaithful=(
            ("Reyes finished the seal checks on deck four during a quiet night cycle.", "not retained"),
        ),
    ),
    Fixture(
        fixture_id="promise_contemporary",
        genre="contemporary",
        requirement="promise / commitment",
        protagonist="Dana", others=("Marcus",),
        actions=(
            ("start", "A Saturday afternoon at the flat Dana is about to rent from Marcus."),
            ("do", "> You look around the empty living room."),
            ("ai", "Sunlight falls across bare floorboards. The radiator ticks, a neighbour's "
                   "radio plays through the wall, and Marcus jingles a ring of keys while he "
                   "talks about the boiler and the bins."),
            ("do", '> You say "Marcus, I will bring you the signed lease by Friday."'),
            ("ai", "Marcus nods and writes something on the back of an envelope. Outside a bus "
                   "pulls away, a dog barks twice, and the afternoon light moves slowly up the wall."),
            ("do", "> You thank him and leave."),
        ),
        facts=(Fact("lease by Friday", (("lease",), ("friday",)), actor="Dana",
                    verbs=("promise", "promised", "bring", "agreed", "said", "would")),),
        forbidden=(r"\bmarcus\s+(promised|agreed|will\s+bring|would\s+bring)\b[^.]*\blease\b",),
        faithful="Dana promised Marcus she would bring him the signed lease for the flat by Friday.",
        unfaithful=(
            ("Marcus promised to bring Dana the signed lease by Friday.", "misattributed"),
            ("Dana viewed the empty flat on a sunny Saturday while Marcus talked about the boiler.",
             "not retained"),
        ),
    ),
    Fixture(
        fixture_id="attribution_station",
        genre="science-fiction-neutral",
        requirement="attribution",
        protagonist="Reyes", others=("Lena", "Tomas"),
        actions=(
            ("start", "The survey ship's cargo bay, between jumps."),
            ("do", "> You ask who can open the sealed vault."),
            ("ai", "Lena folds her arms. She is the only one aboard who knows the vault door "
                   "code, and she makes it clear she is keeping it to herself. Tomas taps the "
                   "access badge clipped to his jacket; without it the bay lift will not move."),
            ("do", "> You look from one of them to the other."),
            ("ai", "The bay lights flicker as the drive spools. Crates creak against their "
                   "straps, and the air smells of cold metal and oil."),
            ("do", "> You wait for one of them to speak."),
        ),
        facts=(
            Fact("code", (("code",),), actor="Lena", verbs=("knows", "knew", "keeps", "kept", "holds", "held")),
            Fact("badge", (("badge",),), actor="Tomas",
                 verbs=("carries", "carried", "has", "had", "holds", "held", "wore", "wears", "tapped", "taps")),
        ),
        forbidden=(r"\breyes\b[^.]*\b(knew|knows)\b[^.]*\bcode\b",),
        faithful="Lena alone knew the vault door code and kept it to herself; Tomas carried the access "
                 "badge that the bay lift needed.",
        unfaithful=(
            ("Tomas knew the vault door code, and Lena carried the access badge.", "misattributed"),
        ),
    ),
    Fixture(
        fixture_id="clutter_office",
        genre="office",
        requirement="clutter pressure",
        protagonist="Dana", others=("Priya", "Owen"),
        actions=(
            ("start", "The quarterly offsite at a conference hotel by the motorway."),
            ("do", "> You find a seat near the back."),
            ("ai", "The conference room smells of carpet cleaner and burnt coffee. Chairs scrape, "
                   "a projector fan whines, and someone at the front struggles with the clicker. "
                   "Owen talks about his weekend at length, the traffic on the ring road, a new "
                   "sandwich place, the football, and whether it will rain for the barbecue. The "
                   "slides cycle through charts nobody reads. Outside the window lorries hiss past "
                   "on the wet motorway, and the hotel's muzak drifts in whenever the door opens."),
            ("do", "> You go to the refreshment table."),
            ("ai", "Pastries sweat under cling film. Priya stirs her tea, glances around, and "
                   "quietly tells you that she saw Owen shred the signed supplier contract in the "
                   "copy room last night. Then she talks about the weather, the parking, and the "
                   "long drive home, and laughs at a joke from across the room. The afternoon "
                   "session is announced, people drift back to their seats, and the projector "
                   "fan starts whining again over a long talk about quarterly targets."),
            ("do", "> You take your seat for the afternoon session."),
        ),
        facts=(Fact("contract shredded", (("contract",), ("shred", "shredded", "destroyed")),
                    actor="Owen", verbs=("shred", "shredded", "destroyed")),),
        forbidden=(r"\b(priya|dana)\b\s+(had\s+)?(shred|shredded|destroyed)\b",),
        faithful="At the offsite, Priya told Dana she had seen Owen shred the signed supplier contract "
                 "in the copy room the night before.",
        unfaithful=(
            ("Dana sat through a dull offsite of charts, pastries and Owen's talk about the weekend.",
             "not retained"),
            ("Priya shredded the signed supplier contract in the copy room.", "misattributed"),
        ),
    ),
    Fixture(
        fixture_id="no_invention_office",
        genre="office",
        requirement="no invention",
        protagonist="Dana", others=("Owen",),
        actions=(
            ("start", "A short planning meeting in the small room on the third floor."),
            ("do", "> You sit down opposite Owen."),
            ("ai", "A black briefcase sits unclaimed by the door; nobody mentions it. Owen says "
                   "the budget review has moved from Tuesday to Thursday, and asks you to tell "
                   "the team."),
            ("do", "> You agree to pass it on."),
            ("ai", "Owen thanks you, checks his phone, and the meeting ends after ten minutes. "
                   "The briefcase is still by the door when you leave."),
            ("do", "> You walk back to your desk."),
        ),
        facts=(Fact("review moved", (("budget review", "review"), ("thursday",))),),
        forbidden=(
            r"\b(took|taken|stole|hid|hidden|grabbed|pocketed|carried|carries|owns|owned|belong\w*)\b[^.]*\bbriefcase\b",
            r"\bbriefcase\b[^.]*\b(belong\w*|his|her|owen's|dana's|secret|clue)\b",
            r"\b(clue|secret|password|code)\b",
        ),
        faithful="Owen told Dana the budget review had moved from Tuesday to Thursday, and Dana agreed "
                 "to tell the team.",
        unfaithful=(
            ("Owen told Dana the budget review had moved to Thursday and left his secret briefcase by the door.",
             "invented"),
        ),
    ),
    Fixture(
        fixture_id="multiple_facts_station",
        genre="science-fiction-neutral",
        requirement="multiple concrete facts",
        protagonist="Reyes", others=("Hale", "Varga", "Moreau"),
        actions=(
            ("start", "The mess hall of the mining outpost after the shift change."),
            ("do", "> You sit with the day crew."),
            ("ai", "Trays clatter and the recycler drones. Engineer Hale admits, half joking, that "
                   "she hid the spare fuse inside the airlock control panel. Doctor Varga says only "
                   "she knows the reactor override phrase, and changes the subject. Pilot Moreau "
                   "grumbles that he owes Hale two shifts of cover."),
            ("do", "> You finish your meal."),
            ("ai", "The lights dim for the rest cycle and the crew drifts off to their bunks."),
            ("do", "> You head to your quarters."),
        ),
        facts=(
            Fact("fuse in the panel", (("fuse",), ("panel", "airlock")), actor="Hale",
                 verbs=("hid", "hidden", "put", "stashed", "kept")),
            Fact("override phrase", (("override",),), actor="Varga", verbs=("knows", "knew", "keeps")),
            Fact("owes cover", (("owes", "owed", "debt"), ("shift", "cover")), actor="Moreau",
                 verbs=("owes", "owed")),
        ),
        min_facts=2,
        faithful="Hale hid the spare fuse inside the airlock control panel, and Varga alone knew the "
                 "reactor override phrase.",
        unfaithful=(
            ("Varga hid the spare fuse inside the airlock panel, and Hale knew the reactor override phrase.",
             "misattributed"),
        ),
    ),
    Fixture(
        # The planting block of WP-B.2 real-model attempt 2, verbatim: the
        # harness's own fixture campaign, with no identifier in it. It is the
        # failure the reference model has not overcome.
        fixture_id="regression_attempt_2",
        genre="fantasy (the harness campaign)",
        requirement="the actual failed-run block",
        protagonist="Aldric", others=("Mara", "Edrin"),
        actions=(
            ("start", "Rain over Westhaven. Aldric sits in the Crooked Lantern with a silver key in his "
                      "pocket and no-one to give it to."),
            ("do", "> I tell Mara quietly that the silver key opens the crypt beneath the Old Abbey — "
                   "SILVER-KEY-CRYPT-OLD-ABBEY."),
            ("ai", "The lantern's warmth feels cold against Aldric's hands as he fidgets with the silver "
                   "key. Mara observes him without a word, her eyes unreadable. The rain outside "
                   "persists, a backdrop of nature's indifference to the human drama transpiring. "
                   "Aldric's fingers close over the key, the familiar weight making him feel both more "
                   "alone and more determined. He pulls the key from his pocket, the metal feeling cold "
                   "against his skin. \n\n> \"I have the key,\" Aldric whispers to Mara. \"It leads to "
                   "the crypt beneath the abbey.\"\n\nEdrin, having finished his ale, stands. \"Well, now "
                   "that we know the key leads to somewhere... do you think we should see what it "
                   "opens?\"\n\nThe silver key feels heavy in Aldric's hands. He looks at Mara, then at "
                   "the key, and finally at Edrin. He decides. \"Let's do this.\""),
            ("do", "> I watch Mara slip the amber sundial inside the cracked teapot on the tavern's top "
                   "shelf, and she makes me promise to tell no one."),
            ("ai", "Aldric steps closer to the smoky bar, his gaze shifting from the silver key in his "
                   "pocket to Mara. \"Let's not overthink this,\" he says, his voice steady despite the "
                   "gnawing doubt that tugs at him. \"Let's see what the crypt holds.\"\n\nEdrin raises "
                   "an eyebrow, his curiosity piqued. \"Alright, then. But if this leads somewhere "
                   "dangerous, we're splitting up. If we make it back here, I'll tell no one what we "
                   "found.\"\n\nMara nods her agreement, her eyes still unreadable. \"Agreed. But "
                   "remember, we're not the first to look there. We need to tread carefully.\"\n\nWith a "
                   "slight nod, they step out of the tavern. The rain has only intensified, pelting them "
                   "as they walk toward the Old Abbey's shadowed grounds. The crypt, sealed since its "
                   "founding, stands before them, its door ajar, inviting or perhaps foreboding. As "
                   "Aldric approaches, he feels a strange mixture of excitement and fear, his hands "
                   "curling around the silver key's handle. \n\n> He turns the key, and the lock clicks "
                   "open with a satisfying, if ominous, hiss. The door swings inward, revealing a narrow "
                   "passageway. The air inside is musty and cold, the only light filtering in through "
                   "the cracks. \n\nAldric takes a tentative step inside, his breath visible in the air. "
                   "Mara and Edrin follow, their footsteps echoing in the silence. The walls are lined "
                   "with stone and cobwebs, and the smell of damp and decay is overwhelming. They "
                   "advance cautiously, each of them aware of the potential dangers lurking within."),
            ("do", "> I ask Mara what she has heard about the abbey."),
        ),
        facts=(Fact("sundial in the teapot", (("sundial",), ("teapot",)), actor="Mara",
                    verbs=("slip", "slipped", "hid", "hidden", "put", "placed", "tucked", "stashed")),),
        forbidden=(
            r"\b(aldric|edrin)\b\s+(had\s+)?(slipped|hid|placed|put|tucked|stashed)\b[^.]*\bsundial\b",
        ),
        faithful="Aldric watched Mara slip the amber sundial inside the cracked teapot on the tavern's "
                 "top shelf and promised her to tell no one; then Aldric, Mara and Edrin entered the "
                 "crypt beneath the Old Abbey.",
        unfaithful=(
            # The memory attempt 2 actually stored, verbatim.
            ("Aldric sits in the Crooked Lantern with a silver key in his pocket, no-one to give it to. "
             "Mara observed him quietly, her eyes unreadable. Edrin finished his ale and stood, asking if "
             "they should see what the crypt beneath the Old Abbey holds. Aldric decided to go, promising "
             "not to tell anyone. They walked to the Old Abbey's grounds, the crypt door ajar, inviting "
             "and foreboding. Inside, the air was musty and cold, with the smell of damp and decay. They "
             "advanced cautiously, each aware of potential dangers. The silver key, the key to the crypt, "
             "felt heavy in Aldric's hands.", "not retained"),
            ("Aldric slipped the amber sundial inside the cracked teapot on the tavern's top shelf.",
             "misattributed"),
        ),
    ),
)

FIXTURES_BY_ID = {f.fixture_id: f for f in FIXTURES}


def cast_brief_for(fixture: Fixture) -> str:
    """The cast brief `memorybank.cast_brief` would build for this fixture."""
    from app import memorybank

    lines = [memorybank._cast_line(fixture.protagonist, "", protagonist=True)]
    lines += [memorybank._cast_line(name, "") for name in fixture.others]
    return "Cast:\n" + "\n".join(lines)


def user_prompt_for(fixture: Fixture) -> str:
    """Exactly the user message the application sends for this block."""
    from app import memorybank

    return memorybank.memory_user_prompt(cast_brief_for(fixture), memorybank.memory_excerpt(fixture.raw))


# --------------------------------------------------------- the measurement

async def compare(endpoint: str, model: str, samples: int) -> dict:
    """Every fixture, `samples` times, under the shipped prompt and the B2.4 experiment."""
    from app import memorybank, models

    settings = models.Settings(endpoint_url=endpoint, model=model, summary_model="",
                               api_mode=models.Settings.__table__.c.api_mode.default.arg,
                               model_timeout_seconds=300)
    provider = memorybank.summary_provider(settings)
    arms = {"shipped": memorybank.MEMORY_SYSTEM_PROMPT, "b2.4-experiment": B24_EXPERIMENT_PROMPT}
    out: dict = {"endpoint_model": model, "samples": samples, "fixtures": {}}
    for fixture in FIXTURES:
        user = user_prompt_for(fixture)
        row: dict = {"requirement": fixture.requirement, "genre": fixture.genre, "arms": {}}
        for arm, system in arms.items():
            runs = []
            for _ in range(samples):
                text = (await provider.complete(system, user) or "").strip()
                runs.append({"memory": text, **evaluate(fixture, text)})
            summary = {
                "passed": sum(r["passed"] for r in runs),
                "retained": sum(r["retained"] for r in runs),
                "misattributed": sum(bool(r["misattributed"]) for r in runs),
                "invented": sum(bool(r["inventions"]) for r in runs),
                "memory_prefix": sum(r["memory_prefix"] for r in runs),
                "second_person": sum(r["second_person"] for r in runs),
                "words_median": sorted(r["words"] for r in runs)[len(runs) // 2],
                "words_max": max(r["words"] for r in runs),
                "over_target": sum(r["over_target"] for r in runs),
            }
            row["arms"][arm] = {"runs": runs, **summary}
            print(f"{fixture.fixture_id:28} {arm:16} "
                  + " ".join(f"{k} {v}" for k, v in summary.items()), flush=True)
        out["fixtures"][fixture.fixture_id] = row
    return out


def main() -> int:
    import argparse
    import asyncio
    import json
    import os
    import sys
    import tempfile
    from pathlib import Path

    parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
    parser.add_argument("--endpoint", required=True)
    parser.add_argument("--model", required=True)
    parser.add_argument("--samples", type=int, default=5)
    parser.add_argument("--out", required=True)
    args = parser.parse_args()

    handle = tempfile.NamedTemporaryFile(suffix=".db", delete=False)
    handle.close()
    os.environ["AIDND_DB_PATH"] = handle.name  # nothing is written; never the real database
    os.environ.pop("AIDND_DATABASE_URL", None)
    sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
    try:
        result = asyncio.run(compare(args.endpoint, args.model, args.samples))
    finally:
        Path(handle.name).unlink(missing_ok=True)
    out = Path(args.out)
    out.mkdir(parents=True, exist_ok=True)
    (out / "fidelity.json").write_text(json.dumps(result, indent=2))
    print(f"written to {out / 'fidelity.json'}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())