124 lines
5.1 KiB
Python
124 lines
5.1 KiB
Python
"""Per-export-run tally for content that was dropped or partially extracted.
|
|
|
|
Surfaces the loss visibility that the rest of the system promises in its
|
|
output (visible ``unknown`` blocks). The summary emitted at the end of
|
|
each export is the load-bearing operator-facing signal: if a real content
|
|
type starts being silently dropped, this is where it shows up.
|
|
|
|
Pass a single instance through ``BaseProvider.normalize_conversation`` and
|
|
read it back in ``src/main.py`` after the export loop. No global state.
|
|
"""
|
|
|
|
from collections import Counter
|
|
from dataclasses import dataclass, field
|
|
|
|
_TOP_N_BREAKDOWN = 5
|
|
|
|
|
|
@dataclass
|
|
class LossReport:
|
|
"""Counters for things that didn't render cleanly in an export run."""
|
|
|
|
# Type-keyed counters. Values are int counts.
|
|
unknown_blocks: Counter = field(default_factory=Counter)
|
|
extraction_failures: Counter = field(default_factory=Counter)
|
|
filtered_roles: Counter = field(default_factory=Counter)
|
|
# Messages collapsed/omitted by the EXPORTER_HIDDEN_CONTENT policy —
|
|
# intentional, but still surfaced so the omission is never invisible.
|
|
collapsed: Counter = field(default_factory=Counter)
|
|
collapsed_bytes: int = 0
|
|
|
|
# Aggregate counters
|
|
messages_rendered: int = 0
|
|
conversations: int = 0
|
|
|
|
# Media downloads (EXPORTER_DOWNLOAD_MEDIA): successes / skips / failures
|
|
media_downloaded: int = 0
|
|
media_downloaded_bytes: int = 0
|
|
media_failed: Counter = field(default_factory=Counter)
|
|
|
|
# Recording -------------------------------------------------------------
|
|
|
|
def record_unknown(self, raw_type: str) -> None:
|
|
self.unknown_blocks[raw_type or "?"] += 1
|
|
|
|
def record_extraction_failure(self, raw_type: str) -> None:
|
|
self.extraction_failures[raw_type or "?"] += 1
|
|
|
|
def record_filtered_role(self, role: str) -> None:
|
|
self.filtered_roles[role or "?"] += 1
|
|
|
|
def record_collapsed(self, origin: str, size_bytes: int = 0) -> None:
|
|
self.collapsed[origin or "?"] += 1
|
|
if isinstance(size_bytes, int) and size_bytes > 0:
|
|
self.collapsed_bytes += size_bytes
|
|
|
|
def record_media_downloaded(self, size_bytes: int = 0) -> None:
|
|
self.media_downloaded += 1
|
|
if isinstance(size_bytes, int) and size_bytes > 0:
|
|
self.media_downloaded_bytes += size_bytes
|
|
|
|
def record_media_failed(self, reason: str) -> None:
|
|
self.media_failed[reason or "?"] += 1
|
|
|
|
def record_message(self) -> None:
|
|
self.messages_rendered += 1
|
|
|
|
def record_conversation(self) -> None:
|
|
self.conversations += 1
|
|
|
|
# Summary ---------------------------------------------------------------
|
|
|
|
def format_summary(self) -> str:
|
|
"""Return a multi-line summary table suitable for INFO logging.
|
|
|
|
Format pinned by plan §Post-export summary — "(none)" sentinel when a
|
|
counter is empty, top-5 breakdown with "+ N more types" overflow.
|
|
"""
|
|
lines: list[str] = ["[export] Run summary:"]
|
|
lines.append(f" conversations: {self.conversations}")
|
|
lines.append(f" messages rendered: {self.messages_rendered}")
|
|
lines.extend(_format_section("unknown blocks: ", self.unknown_blocks))
|
|
lines.extend(_format_section("extraction failures: ", self.extraction_failures))
|
|
lines.extend(_format_section("collapsed by policy: ", self.collapsed))
|
|
if self.collapsed:
|
|
lines.append(
|
|
f" ≈{self.collapsed_bytes / 1024:.0f} KB omitted "
|
|
"(intentional — EXPORTER_HIDDEN_CONTENT=full to keep)"
|
|
)
|
|
if self.media_downloaded or self.media_failed:
|
|
lines.append(
|
|
f" media downloaded: {self.media_downloaded} "
|
|
f"(≈{self.media_downloaded_bytes / 1024:.0f} KB)"
|
|
)
|
|
if self.media_failed:
|
|
total_failed = sum(self.media_failed.values())
|
|
lines.append(f" media failed: {total_failed}")
|
|
for reason, count in self.media_failed.most_common(_TOP_N_BREAKDOWN):
|
|
lines.append(f" {reason}={count}")
|
|
lines.append(
|
|
" filtered roles: "
|
|
"(filter lifted in v0.4.0 — counter retained for future use, expected 0)"
|
|
)
|
|
if self.filtered_roles:
|
|
for role, count in self.filtered_roles.most_common(_TOP_N_BREAKDOWN):
|
|
lines.append(f" {role}={count}")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def _format_section(label: str, counter: Counter) -> list[str]:
|
|
"""Render one counter section: header line + indented breakdown lines."""
|
|
total = sum(counter.values())
|
|
header = f" {label} {total}"
|
|
if total == 0:
|
|
return [header, " (none)"]
|
|
|
|
lines = [header]
|
|
most_common = counter.most_common()
|
|
for raw_type, count in most_common[:_TOP_N_BREAKDOWN]:
|
|
lines.append(f" {raw_type}={count}")
|
|
if len(most_common) > _TOP_N_BREAKDOWN:
|
|
remainder = len(most_common) - _TOP_N_BREAKDOWN
|
|
lines.append(f" + {remainder} more types")
|
|
return lines
|