Jesse.Markowitz/AIChatExporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

canary: detect provider API schema drift; release v0.7.0

Browse Source 
The export reads ChatGPT's and Claude's undocumented internal web APIs,
which can change shape without notice; the worst failure for a backup tool
is a silent one (skipped/mis-parsed content with no error). Add a `canary`
command + BaseProvider.check_drift() (overridden by ChatGPT/Claude) that
fetches one listing page + one conversation per provider and asserts only
the normalizer's load-bearing fields — not the full response shape, which
churns harmlessly. The top silent risk it guards is a renamed retrieval-tool
author bypassing the hidden-content collapse. ERROR findings exit non-zero;
WARN findings are surfaced but non-fatal so a backup run is never blocked.

Also retires the in-app watch mode and headless StartOS direction from the
roadmap (tool stays a local, manually-run CLI) and updates docs.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
JesseMarkowitz
2026-06-28 01:37:24 -04:00
co-authored by Claude Opus 4.8
parent ef603cf659
commit 1e016ea652
10 changed files with 693 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main.py
+86 -1
View File
@@ -18,7 +18,7 @@ from src.cache import Cache, CacheError
from src.config import ConfigError
from src.logging_config import setup_logging
from src.loss_report import LossReport
from src.providers.base import ProviderError
from src.providers.base import DRIFT_ERROR, DRIFT_OK, DRIFT_WARN, ProviderError
console = Console()
err_console = Console(stderr=True)
@@ -332,6 +332,91 @@ def doctor(ctx: click.Context) -> None:
sys.exit(1)
@cli.command()
@click.option(
"--provider",
type=click.Choice(["chatgpt", "claude", "all"]),
default="all",
help="Which web-API provider(s) to probe.",
)
@click.pass_context
def canary(ctx: click.Context, provider: str) -> None:
"""Probe provider APIs for schema drift against the fields the parser needs.
Fetches one listing page + one conversation per provider and checks only
the normalizer's load-bearing fields (not the full response shape, which
churns harmlessly). Surfaces the silent-failure risks for a backup tool:
a renamed retrieval-tool author bypassing the hidden-content collapse, a
new content_type, drifted message fields, or ignored attachments.
Exits non-zero if any ERROR finding is reported (a load-bearing field is
missing/mistyped). WARN findings are surfaced but do not fail. See
FUTURE.md §10.
"""
from dotenv import load_dotenv
load_dotenv(override=False)
chatgpt_token = os.getenv("CHATGPT_SESSION_TOKEN", "").strip() or None
claude_key = os.getenv("CLAUDE_SESSION_KEY", "").strip() or None
targets: list[tuple[str, object]] = []
if provider in ("chatgpt", "all") and chatgpt_token:
from src.providers.chatgpt import ChatGPTProvider
ct1 = os.getenv("CHATGPT_SESSION_TOKEN_1", "").strip() or None
targets.append(
("chatgpt", lambda: ChatGPTProvider(chatgpt_token, session_token_1=ct1))
)
if provider in ("claude", "all") and claude_key:
from src.providers.claude import ClaudeProvider
targets.append(("claude", lambda: ClaudeProvider(claude_key)))
if not targets:
err_console.print(
"[yellow]No web-API provider tokens configured "
"(set CHATGPT_SESSION_TOKEN / CLAUDE_SESSION_KEY).[/yellow]"
)
sys.exit(1)
rows: list[tuple[str, dict]] = []
for name, factory in targets:
try:
findings = factory().check_drift()
except ProviderError as e:
findings = [{"severity": DRIFT_ERROR, "check": "probe", "detail": str(e.original)[:100]}]
except Exception as e:
findings = [{"severity": DRIFT_ERROR, "check": "probe", "detail": str(e)[:100]}]
for f in findings:
rows.append((name, f))
_print_canary_table(rows)
if any(f["severity"] == DRIFT_ERROR for _, f in rows):
sys.exit(1)
def _print_canary_table(rows: list[tuple[str, dict]]) -> None:
table = Table(title="API Drift Canary", show_header=True)
table.add_column("Provider", style="bold")
table.add_column("Severity", justify="center")
table.add_column("Check")
table.add_column("Detail")
badge = {
DRIFT_OK: "[green]OK[/green]",
DRIFT_WARN: "[yellow]WARN[/yellow]",
DRIFT_ERROR: "[red]ERROR[/red]",
}
for name, f in rows:
table.add_row(
name,
badge.get(f["severity"], f["severity"]),
f.get("check", ""),
f.get("detail", ""),
)
console.print(table)
def _run_doctor_checks(cache: Cache | None = None) -> list[dict]:
"""Run all doctor checks and return results.
src/providers/base.py
+30
View File
@@ -50,6 +50,27 @@ VALID_HIDDEN_CONTENT_POLICIES = {
HIDDEN_CONTENT_OMIT,
}
# ---------------------------------------------------------------------------
# API-drift canary (FUTURE.md §10)
# ---------------------------------------------------------------------------
# The export depends on undocumented internal web APIs that can change shape
# without notice. The canary fetches a small live sample and asserts only the
# normalizer's load-bearing fields (NOT full shape — provider settings objects
# churn constantly and would be pure noise). Findings carry a severity:
# ERROR — a load-bearing field is missing/mistyped; the parser will break or
# silently lose data. The backup is no longer trustworthy.
# WARN — something unfamiliar appeared (new content_type, new tool author,
# a dormant field went live). Investigate; not necessarily broken.
# OK — the assertion held.
DRIFT_ERROR = "error"
DRIFT_WARN = "warn"
DRIFT_OK = "ok"
def drift_finding(severity: str, check: str, detail: str = "") -> dict:
"""Build one canary finding."""
return {"severity": severity, "check": check, "detail": detail}
def resolve_hidden_content_policy() -> str:
"""Read EXPORTER_HIDDEN_CONTENT from the environment, defaulting to placeholder."""
@@ -159,6 +180,15 @@ class BaseProvider(ABC):
isolation, e.g. from tests, doesn't crash).
"""
def check_drift(self) -> list[dict]:
"""Probe the live API and assert the normalizer's load-bearing fields.
Returns a list of ``drift_finding`` dicts. The default is a no-op
(used by providers with no remote schema risk, e.g. local Claude Code
transcripts); web-API providers override this. See FUTURE.md §10.
"""
return []
# ------------------------------------------------------------------
# Concrete helpers
# ------------------------------------------------------------------
src/providers/chatgpt.py
+96
View File
@@ -45,12 +45,16 @@ from src.blocks import (
from src.loss_report import LossReport
from src.providers.base import (
BaseProvider,
DRIFT_ERROR,
DRIFT_OK,
DRIFT_WARN,
HIDDEN_CONTENT_FULL,
HIDDEN_CONTENT_OMIT,
HIDDEN_CONTENT_PLACEHOLDER,
ProviderError,
REQUEST_TIMEOUT,
VALID_HIDDEN_CONTENT_POLICIES,
drift_finding,
resolve_hidden_content_policy,
)
@@ -91,6 +95,20 @@ def parse_asset_file_id(ref: str) -> str | None:
# myfiles_browser is the legacy name for the same retrieval tool.
_COLLAPSE_TOOL_AUTHORS = {"file_search", "myfiles_browser"}
# ── API-drift canary vocabularies (FUTURE.md §10) ──────────────────────────
# Tool authors seen live 2026-06-28. The collapse keys off author.name, so a
# RENAMED retrieval tool would silently bypass the collapse and re-bloat the
# archive — the canary flags any unfamiliar (role="tool", author.name) pair.
_KNOWN_TOOL_AUTHORS = _COLLAPSE_TOOL_AUTHORS | {"web.run", "python"}
# content_types the normalizer dispatches on (keep in sync with
# _extract_blocks_for_content). A new value already degrades to an `unknown`
# block + LossReport at export time; the canary flags it proactively.
_HANDLED_CONTENT_TYPES = frozenset({
"text", "multimodal_text", "execution_output", "system_error",
"tether_browsing_display", "code", "thoughts", "reasoning_recap",
"user_editable_context", "model_editable_context", "image_asset_pointer",
})
class ChatGPTProvider(BaseProvider):
"""Provider for ChatGPT conversations via the internal web API.
@@ -766,6 +784,84 @@ class ChatGPTProvider(BaseProvider):
"messages": messages,
}
def check_drift(self) -> list[dict]:
"""Probe one listing page + one conversation; assert load-bearing fields.
See FUTURE.md §10. Asserts only the fields the normalizer depends on —
not the full response shape (which churns harmlessly). The top silent
risk is a renamed retrieval tool author bypassing the collapse.
"""
findings: list[dict] = []
page = self.list_conversations(offset=0, limit=5)
if not page:
return [drift_finding(DRIFT_WARN, "listing",
"empty listing — cannot verify shape")]
item = page[0]
for f in ("id", "title"):
if f not in item:
findings.append(drift_finding(
DRIFT_ERROR, "listing", f"summary item missing '{f}'"))
if not (item.get("update_time") or item.get("create_time")):
findings.append(drift_finding(
DRIFT_WARN, "listing", "no update_time/create_time on summary"))
conv_id = item.get("id")
if not conv_id:
return findings or [drift_finding(DRIFT_ERROR, "listing", "no id to fetch")]
raw = self.get_conversation(conv_id)
if not (raw.get("conversation_id") or raw.get("id")):
findings.append(drift_finding(
DRIFT_ERROR, "detail", "no conversation_id/id on detail"))
mapping = raw.get("mapping")
if not isinstance(mapping, dict) or not mapping:
findings.append(drift_finding(
DRIFT_ERROR, "detail", "mapping missing/empty — tree walk will yield 0 messages"))
return findings
saw_text_with_parts = False
seen_new_ct: set[str] = set()
seen_new_tool: set[str] = set()
for node in mapping.values():
if "children" not in node:
findings.append(drift_finding(
DRIFT_ERROR, "mapping", "node missing 'children' — tree walk breaks"))
break
msg = node.get("message")
if not msg:
continue
author = msg.get("author") or {}
role, name = author.get("role"), author.get("name")
content = msg.get("content") or {}
ct = content.get("content_type")
if ct and ct not in _HANDLED_CONTENT_TYPES:
seen_new_ct.add(ct)
if role == "tool" and name and name not in _KNOWN_TOOL_AUTHORS:
seen_new_tool.add(name)
if ct == "text":
parts = content.get("parts") or []
if any(isinstance(p, str) and p.strip() for p in parts):
saw_text_with_parts = True
for name in sorted(seen_new_tool):
findings.append(drift_finding(
DRIFT_WARN, "collapse",
f"unfamiliar tool author {name!r} — if it's a retrieval dump it "
"is NOT being collapsed (silent archive bloat); update "
"_COLLAPSE_TOOL_AUTHORS / _KNOWN_TOOL_AUTHORS"))
for ct in sorted(seen_new_ct):
findings.append(drift_finding(
DRIFT_WARN, "content_type",
f"new content_type {ct!r} — exported as an unknown block; add a handler"))
if not saw_text_with_parts:
findings.append(drift_finding(
DRIFT_WARN, "parts",
"no text message yielded non-empty parts — content.parts may have drifted"))
if not findings:
findings.append(drift_finding(DRIFT_OK, "schema", "all load-bearing fields present"))
return findings
# ---------------------------------------------------------------------------
# Internal helpers
src/providers/claude.py
+72 -1
View File
@@ -16,7 +16,14 @@ from src.blocks import (
make_unknown_block,
)
from src.loss_report import LossReport
from src.providers.base import BaseProvider, ProviderError
from src.providers.base import (
BaseProvider,
DRIFT_ERROR,
DRIFT_OK,
DRIFT_WARN,
ProviderError,
drift_finding,
)
logger = logging.getLogger(__name__)
@@ -232,6 +239,70 @@ class ClaudeProvider(BaseProvider):
"messages": messages,
}
def check_drift(self) -> list[dict]:
"""Probe one listing page + one conversation; assert load-bearing fields.
See FUTURE.md §10. Real Claude messages are flat ``text``/``sender``;
the rich ``content`` block path is dormant, so its activation (content
becoming a list) is itself a drift event worth flagging. ``attachments``
/ ``files`` are ignored by the normalizer — non-empty values are a
silent-loss risk.
"""
findings: list[dict] = []
page = self.list_conversations(offset=0, limit=5)
if not page:
return [drift_finding(DRIFT_WARN, "listing",
"empty listing — cannot verify shape")]
item = page[0]
if not (item.get("uuid") or item.get("id")):
findings.append(drift_finding(DRIFT_ERROR, "listing", "summary missing uuid/id"))
if not (item.get("name") or item.get("title")):
findings.append(drift_finding(DRIFT_WARN, "listing", "no name/title on summary"))
if not (item.get("updated_at") or item.get("update_time")):
findings.append(drift_finding(DRIFT_WARN, "listing", "no updated_at on summary"))
conv_id = item.get("uuid") or item.get("id")
if not conv_id:
return findings or [drift_finding(DRIFT_ERROR, "listing", "no id to fetch")]
raw = self.get_conversation(conv_id)
if not (raw.get("uuid") or raw.get("id")):
findings.append(drift_finding(DRIFT_ERROR, "detail", "no uuid/id on detail"))
msgs = raw.get("chat_messages") or raw.get("messages")
if not isinstance(msgs, list) or not msgs:
findings.append(drift_finding(
DRIFT_ERROR, "detail", "chat_messages missing/empty — 0 messages"))
return findings
saw_sender = saw_text = False
flagged_list = flagged_attach = False
for m in msgs:
if m.get("sender") or m.get("role"):
saw_sender = True
content = m.get("content")
if isinstance(content, list) and not flagged_list:
findings.append(drift_finding(
DRIFT_WARN, "content",
"message.content is now a LIST — Claude switched to typed blocks; "
"the dormant _dispatch_claude_block path is now live, verify rendering"))
flagged_list = True
if m.get("text") or content:
saw_text = True
if (m.get("attachments") or m.get("files")) and not flagged_attach:
findings.append(drift_finding(
DRIFT_WARN, "attachments",
"message has non-empty attachments/files — the normalizer ignores "
"these (silent loss); add handling"))
flagged_attach = True
if not saw_sender:
findings.append(drift_finding(DRIFT_ERROR, "messages", "no sender/role on any message"))
if not saw_text:
findings.append(drift_finding(DRIFT_WARN, "messages", "no text/content found"))
if not findings:
findings.append(drift_finding(DRIFT_OK, "schema", "all load-bearing fields present"))
return findings
# ---------------------------------------------------------------------------
# Internal helpers