feat: v0.4.0 — rich content support with typed blocks and loss visibility
Extracts per-message content into a typed `blocks` list (text, code, thinking, tool_use, tool_result, image_placeholder, file_placeholder, unknown) and renders them at exporter write time. Voice transcripts, Custom Instructions, and image references now appear in exports instead of being silently dropped. Foundation: - src/blocks.py: pure block constructors, _safe_fence (fence-corruption defense, verified live in Joplin), _blockquote_prefix, render - src/loss_report.py: per-run tally surfaced as INFO summary at end of export so silently-dropped data becomes visible Providers: - ChatGPT: dispatch on content_type produces typed blocks; voice shapes (audio_transcription, audio_asset_pointer, real_time_user_audio_video_ asset_pointer) locked from live DevTools capture; Custom Instructions bug fix (parts-vs-direct-fields); role filter lifted; hidden-context marker driven by is_visually_hidden_from_conversation flag - Claude: defensive dispatch for text/thinking/tool_use/tool_result/image with recursive nested-block flattening; untested against real rich- content data — fix-forward in v0.4.1 Exporter: - Markdown renders from blocks at write time via render_blocks_to_markdown; backward-compat fallback to content for any pre-v0.4.0 cached data Tests: - 27 new tests across providers, exporters, CLI; fixtures rebuilt with real-shape ChatGPT voice + Custom Instructions cases - 181/181 pass Behavior changes (intentional): - JSON output omits content; consumers should read blocks - Per-conversation message counts increase (Custom Instructions, image- only, tool-only messages now appear) - Existing exports not auto-re-rendered; users wanting fresh output run cache --clear then export Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
4798edcea7
commit
473d02f71a
@@ -0,0 +1,85 @@
|
||||
"""Per-export-run tally for content that was dropped or partially extracted.
|
||||
|
||||
Surfaces the loss visibility that the rest of the system promises in its
|
||||
output (visible ``unknown`` blocks). The summary emitted at the end of
|
||||
each export is the load-bearing operator-facing signal: if a real content
|
||||
type starts being silently dropped, this is where it shows up.
|
||||
|
||||
Pass a single instance through ``BaseProvider.normalize_conversation`` and
|
||||
read it back in ``src/main.py`` after the export loop. No global state.
|
||||
"""
|
||||
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
_TOP_N_BREAKDOWN = 5
|
||||
|
||||
|
||||
@dataclass
|
||||
class LossReport:
|
||||
"""Counters for things that didn't render cleanly in an export run."""
|
||||
|
||||
# Type-keyed counters. Values are int counts.
|
||||
unknown_blocks: Counter = field(default_factory=Counter)
|
||||
extraction_failures: Counter = field(default_factory=Counter)
|
||||
filtered_roles: Counter = field(default_factory=Counter)
|
||||
|
||||
# Aggregate counters
|
||||
messages_rendered: int = 0
|
||||
conversations: int = 0
|
||||
|
||||
# Recording -------------------------------------------------------------
|
||||
|
||||
def record_unknown(self, raw_type: str) -> None:
|
||||
self.unknown_blocks[raw_type or "?"] += 1
|
||||
|
||||
def record_extraction_failure(self, raw_type: str) -> None:
|
||||
self.extraction_failures[raw_type or "?"] += 1
|
||||
|
||||
def record_filtered_role(self, role: str) -> None:
|
||||
self.filtered_roles[role or "?"] += 1
|
||||
|
||||
def record_message(self) -> None:
|
||||
self.messages_rendered += 1
|
||||
|
||||
def record_conversation(self) -> None:
|
||||
self.conversations += 1
|
||||
|
||||
# Summary ---------------------------------------------------------------
|
||||
|
||||
def format_summary(self) -> str:
|
||||
"""Return a multi-line summary table suitable for INFO logging.
|
||||
|
||||
Format pinned by plan §Post-export summary — "(none)" sentinel when a
|
||||
counter is empty, top-5 breakdown with "+ N more types" overflow.
|
||||
"""
|
||||
lines: list[str] = ["[export] Run summary:"]
|
||||
lines.append(f" conversations: {self.conversations}")
|
||||
lines.append(f" messages rendered: {self.messages_rendered}")
|
||||
lines.extend(_format_section("unknown blocks: ", self.unknown_blocks))
|
||||
lines.extend(_format_section("extraction failures: ", self.extraction_failures))
|
||||
lines.append(
|
||||
" filtered roles: "
|
||||
"(filter lifted in v0.4.0 — counter retained for future use, expected 0)"
|
||||
)
|
||||
if self.filtered_roles:
|
||||
for role, count in self.filtered_roles.most_common(_TOP_N_BREAKDOWN):
|
||||
lines.append(f" {role}={count}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _format_section(label: str, counter: Counter) -> list[str]:
|
||||
"""Render one counter section: header line + indented breakdown lines."""
|
||||
total = sum(counter.values())
|
||||
header = f" {label} {total}"
|
||||
if total == 0:
|
||||
return [header, " (none)"]
|
||||
|
||||
lines = [header]
|
||||
most_common = counter.most_common()
|
||||
for raw_type, count in most_common[:_TOP_N_BREAKDOWN]:
|
||||
lines.append(f" {raw_type}={count}")
|
||||
if len(most_common) > _TOP_N_BREAKDOWN:
|
||||
remainder = len(most_common) - _TOP_N_BREAKDOWN
|
||||
lines.append(f" + {remainder} more types")
|
||||
return lines
|
||||
Reference in New Issue
Block a user