From 9e1a8ab7cb74e3f31b01f9e2e92cb01e2d68b63f Mon Sep 17 00:00:00 2001 From: JesseMarkowitz Date: Fri, 12 Jun 2026 18:26:14 -0400 Subject: [PATCH] =?UTF-8?q?feat:=20v0.6.0=20=E2=80=94=20collapse=20policy,?= =?UTF-8?q?=20session=20limiter,=20Claude=20Code=20provider,=20prune,=20br?= =?UTF-8?q?owser=20auth,=20media=20downloads?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 25 ++ CHANGELOG.md | 108 ++++---- FUTURE.md | 365 +++++++++++++++++++++------ README.md | 66 ++++- pyproject.toml | 3 +- requirements.txt | 1 + src/blocks.py | 66 ++++- src/browser_tokens.py | 101 ++++++++ src/cache.py | 35 +++ src/config.py | 76 +++++- src/joplin.py | 86 +++++++ src/loss_report.py | 38 +++ src/main.py | 404 ++++++++++++++++++++++++++--- src/media.py | 202 +++++++++++++++ src/providers/base.py | 78 ++++++ src/providers/chatgpt.py | 168 ++++++++++++- src/providers/claude_code.py | 476 +++++++++++++++++++++++++++++++++++ tests/test_browser_tokens.py | 155 ++++++++++++ tests/test_claude_code.py | 230 +++++++++++++++++ tests/test_cli.py | 105 +++++++- tests/test_config.py | 62 +++++ tests/test_media.py | 248 ++++++++++++++++++ tests/test_providers.py | 232 +++++++++++++++++ 23 files changed, 3133 insertions(+), 197 deletions(-) create mode 100644 src/browser_tokens.py create mode 100644 src/media.py create mode 100644 src/providers/claude_code.py create mode 100644 tests/test_browser_tokens.py create mode 100644 tests/test_claude_code.py create mode 100644 tests/test_media.py diff --git a/.env.example b/.env.example index cb62497..164b4f4 100644 --- a/.env.example +++ b/.env.example @@ -36,6 +36,31 @@ EXPORT_DIR=./exports # provider/year → exports/claude/2024/file.md (ignores projects) OUTPUT_STRUCTURE=provider/project/year +# What to do with content that was invisible in the provider's web UI +# (file-retrieval tool dumps, hidden context like Custom Instructions). +# These dumps can be 90% of a conversation's bytes. Options: +# placeholder (default) → one-line placeholder with tool name and size +# full → keep everything (pre-v0.6.0 behavior) +# omit → drop entirely (still counted in the run summary) +EXPORTER_HIDDEN_CONTENT=placeholder + +# Download conversation assets (images, audio) next to the Markdown, into a +# media/ folder, and inline them. Options: +# images (default) → images only +# all → also audio/voice clips and other files +# off → keep text placeholders, download nothing +# Downloaded media is uploaded to Joplin as resources on the next `joplin` run. +EXPORTER_DOWNLOAD_MEDIA=images + +# Cap how many conversations are downloaded per export run (per provider). +# Runs are resumable — a capped run continues where it stopped next time. +# Keeps big backfills from looking like scraper traffic. Unset = unlimited. +#MAX_CONVERSATIONS_PER_RUN=25 + +# Seconds between consecutive API requests (small random jitter is added). +# Default 1.0; set 0 to disable pacing. +#REQUEST_DELAY=1.0 + # --- Joplin --- # Automate importing exported conversations into Joplin as notes. # Requires Joplin desktop running with the Web Clipper service enabled. diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f17c63..3b176fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,70 +3,66 @@ All notable changes to this project will be documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## [0.4.1] - Unreleased +## [0.6.0] - 2026-06-12 + +First tagged release. The project was developed through several internal +milestones (0.1.0–0.5.0) that were never published; their changes are +consolidated here. + ### Added -- ChatGPT `execution_output` (Code Interpreter / `container.exec` / `python`) renders as a `tool_result` block with `tool_name` from `author.name`, `is_error` from `metadata.aggregate_result.status`, and the optional `summary` line populated from `metadata.reasoning_title`. Captured live during planning. -- ChatGPT `system_error` content (e.g. browse-service 503) renders as an error `tool_result` block with `tool_name` from `author.name` (typically `"web"`). -- ChatGPT `tether_browsing_display` populated case (defensive, not observed in real data) renders as a `tool_result` block; transient spinner placeholders (empty `result`+`summary`) skip silently with DEBUG log. -- `tool_result` block schema gains optional `summary: str | None` field, rendered as italic line between header and fenced output. -- `tool_result` rendering shows `tool_name` in the header when present (e.g. `📤 **Result: container.exec**`); when absent, header stays as `📤 **Result**` (no regression). -- Markdown exporter: `_ROLE_LABELS["tool"] = ("🔧 Tool", "tool")` so tool-role messages render under a recognisable header instead of the generic fallback. -- 11 new tests covering all four cases plus the conv_id fallback (192 total, all passing). -### Fixed -- ChatGPT `normalize_conversation` now reads `conversation_id` as a fallback for `id`. Live ChatGPT detail responses use `conversation_id` at top level; fixtures and listing summaries use `id`. Without the fallback, normalized conversations had empty `id` (visible as blank `conversation_id:` in YAML frontmatter and missing context in WARNING log lines). +**Core export & sync** +- ChatGPT and Claude export via their internal web APIs, with a local cache/manifest for incremental, resumable sync (only new or updated conversations are fetched; each is written to the manifest immediately). +- Markdown and JSON exporters. Markdown rendering happens at exporter-write time (providers produce typed blocks; exporters render them), which keeps the door open for future Obsidian/HTML output. +- CLI: `export`, `list`, `cache`, `doctor`, `auth`, `joplin`, `prune`. +- `--project` filter on `export`/`list`/`joplin` (case-insensitive substring, or `none`). +- ChatGPT Projects support via `CHATGPT_PROJECT_IDS` (project conversations are fetched separately from the default listing). -### Migration -- No new schema breaks; `tool_result` blocks gain a `summary` field that defaults to None on legacy data. Existing exports re-render cleanly with the cache-clear-and-export workflow from v0.4.0. +**Joplin integration** +- `joplin` command syncs exported Markdown to Joplin as notes; notebooks are created automatically and nested per provider+project. Re-running is safe — the Joplin note ID is stored in the manifest, so notes are updated, not duplicated. +- `JOPLIN_API_TOKEN`, `JOPLIN_API_URL`, `JOPLIN_REQUEST_TIMEOUT` config, with actionable error messages on timeout/connection failure. -## [0.4.0] - Unreleased -### Added -- Rich content support: messages now carry an ordered `blocks` list (text, code, thinking, tool_use, tool_result, citation, image_placeholder, file_placeholder, unknown) -- ChatGPT voice mode: `audio_transcription` parts render as text blocks; `audio_asset_pointer` and `real_time_user_audio_video_asset_pointer` render as `📎 File attached` placeholders with size and duration metadata -- ChatGPT Custom Instructions: `user_editable_context` and `model_editable_context` messages now appear in exports (were silently dropped — pre-existing bug fixed); rendered with a `> ℹ️ Hidden context` marker driven by the `is_visually_hidden_from_conversation` flag -- Image placeholders for `image_asset_pointer` parts (uploads + DALL-E) inside `multimodal_text` and at message level -- Defensive Claude block extraction: `text`, `thinking`, `tool_use`, `tool_result` (including nested-block flattening), `image` blocks (untested against real data; will fix-forward in v0.4.1 if real shapes diverge) -- `LossReport` summary table emitted at end of every `export` run, breaking down `unknown blocks` and `extraction failures` by raw type so silently-dropped data becomes visible -- `_safe_fence` helper picks a fence longer than any backtick run in extracted content, preventing embedded triple-backticks from corrupting downstream rendering (verified live in Joplin during planning) -- `unknown` blocks render as `> ⚠️ Unsupported content` with the raw type, observed top-level keys, and reason — so future API additions are visible rather than silent +**Rich content (typed blocks)** +- Messages carry an ordered `blocks` list: text, code, thinking, tool_use, tool_result, citation, image_placeholder, file_placeholder, unknown. +- ChatGPT voice mode: `audio_transcription` parts render as text; audio asset pointers render as `📎 File attached` placeholders with size/duration. Custom Instructions (`user_editable_context`/`model_editable_context`) now appear (were silently dropped) with a `> ℹ️ Hidden context` marker. +- ChatGPT `execution_output`, `system_error`, and `tether_browsing_display` render as `tool_result` blocks (with `tool_name`, `is_error`, optional `summary`); transient browse spinners skip silently. +- Defensive Claude block extraction (text/thinking/tool_use/tool_result/image, including nested-block flattening). +- `_safe_fence` picks a backtick fence longer than any run in the content, so embedded triple-backticks can't corrupt rendering (verified live in Joplin). +- Data-loss visibility: a `LossReport` summary at the end of every `export` run breaks down `unknown blocks` and `extraction failures` by raw type; `unknown` blocks render as visible `> ⚠️ Unsupported content` with the raw type and observed keys. + +**Invisible-content collapse (`EXPORTER_HIDDEN_CONTENT=full|placeholder|omit`, default `placeholder`; `export --hidden-content`)** +- Messages that were invisible in the ChatGPT web UI are collapsed to one-line placeholders instead of exported in full. Two triggers: (a) tool-role retrieval dumps identified by `author.name` (`file_search`, `myfiles_browser`) — ChatGPT re-injects the full text of attached/project files on every tool run, measured at 86% of all content bytes across the three largest conversations and **not** hidden-flagged; (b) messages flagged `is_visually_hidden_from_conversation` (Custom Instructions). Code-execution results, web-search output, and all dialogue are untouched. +- New `collapsed` block renders as `> 🔧 **Tool output** — file_search (15.0 KB) — omitted (EXPORTER_HIDDEN_CONTENT=full to keep)` (ℹ️ Hidden context variant for hidden-flagged messages). The `LossReport` gains a `collapsed by policy` section (count per origin + total KB) so the omission stays visible. + +**Claude Code session provider (`--provider claude-code`)** +- Archives local agent transcripts from `~/.claude/projects/*/*.jsonl` (override with `CLAUDE_CODE_DIR`) — no tokens, no rate limits, no ToS exposure. Prose-only by default per the hidden-content policy: dialogue and deliverables kept; tool traffic grouped into one placeholder per activity run (`> 🔧 Tool output — 14 calls: Read ×9, Bash ×2 (86KB) — omitted`); thinking dropped (counted in the summary); subagent (`isSidechain`) and harness (`isMeta`, snapshots, command tags) records stripped. Titles from the last `ai-title` record; project from the working-directory basename. Joplin notebooks nest under the `AI-Claude` parent. Measured: 19MB of JSONL → 804KB of Markdown across 25 sessions. + +**Binary/image downloads (`EXPORTER_DOWNLOAD_MEDIA=images|all|off`, default `images`; `export --download-media`)** +- Downloads ChatGPT conversation assets into a `media/` folder beside each export and inlines them — images become real `![](media/…)` embeds, files become local links. Two-hop fetch via `/backend-api/files/{id}/download` → signed URL (verified live). `all` also pulls voice-mode audio (≈162MB of clips in this archive, transcripts already in text — hence the images-only default). Idempotent (skips assets already on disk), atomic writes, `600` perms. Expired/missing assets (old generated images 404) keep their placeholder and are counted as `media failed` — never fatal. +- Joplin resource upload: the `joplin` command uploads downloaded media as resources and rewrites `media/…` links to `:/resourceId`, so images render inside notes (verified live: resources created with correct mime/size and linked to the note). Resource IDs are tracked per-conversation in the manifest, so re-syncs reuse them instead of duplicating. + +**Token setup & throughput** +- `auth --from-browser [brave|chrome|chromium|edge|firefox]`: extracts ChatGPT session-token cookies and the Claude `sessionKey` straight from a local browser's cookie store (via `browser-cookie3`; defaults to Brave), validates each against the live API, and writes `.env`. Tokens that fail validation never overwrite a working entry; clear fallback when the browser is absent or the cookie DB is locked. +- Session limiter: `--max-conversations N` / `MAX_CONVERSATIONS_PER_RUN` cap downloads per run (per provider); capped-out conversations are reported as "deferred" and resumed next run. +- Polite pacing: `REQUEST_DELAY` (default 1.0s, ±25% jitter, `0` disables) between consecutive API requests, with the existing 429 backoff as the reactive net. + +**Archive hygiene** +- `prune` deletes export files not referenced by the manifest (old-layout trees, no-ID orphans), with listing, confirmation, `--dry-run`, `-y`, and empty-directory sweep. Refuses to run when the manifest references no files (so `cache --clear` + `prune` can't wipe the archive). First live run removed 420 stale files (9.4 MB). +- `doctor` verifies manifest ↔ disk integrity: every recorded `file_path` must exist on disk. ### Changed -- ChatGPT role filter (previously dropped `tool` and `system` messages) is **lifted**: all roles now route through normal extraction; truly empty messages skip via the existing empty-content guard -- Markdown rendering moves from provider-time to exporter-write-time. Providers produce blocks; exporters call `render_blocks_to_markdown` at write time. This unblocks future Obsidian/HTML exporters -- `BaseProvider.normalize_conversation` signature now accepts an optional `LossReport` parameter (breaking change for any future custom subclass; FileProvider hasn't shipped yet) -- `o1`/`o3` reasoning subparts inside `text` content_type messages remain rendered as plain text (defensive; reclassification to `thinking` block deferred until live shape is captured) +- ChatGPT role filter that dropped `tool`/`system` messages is **lifted**; all roles route through normal extraction (truly empty messages skip via the empty-content guard). +- `BaseProvider.normalize_conversation` accepts an optional `LossReport` parameter. +- `ChatGPTProvider.normalize_conversation` reads `conversation_id` as a fallback for `id` (live detail responses use `conversation_id`; fixtures use `id`). +- `Config` validates `EXPORTER_HIDDEN_CONTENT`, `EXPORTER_DOWNLOAD_MEDIA`, `MAX_CONVERSATIONS_PER_RUN`, and `REQUEST_DELAY`, and logs the active values at startup. +- New dependency: `browser-cookie3==0.20.1`. ### Fixed -- `user_editable_context` / `model_editable_context` extraction (parts-vs-direct-fields mismatch) — Custom Instructions are no longer silently dropped from every conversation +- Custom Instructions (`user_editable_context`/`model_editable_context`) were silently dropped from every conversation (parts-vs-direct-fields mismatch). ### Migration -- Existing exports are not re-rendered automatically. To pick up v0.4.0 rendering for previously exported conversations: - ``` - python -m src.main cache --clear - python -m src.main export --provider all - ``` -- JSON exports: messages now contain `blocks` (typed structured content) and may omit the legacy `content` field. External consumers reading JSON should prefer `blocks`. -- Per-conversation message counts may increase: previously-dropped Custom Instructions, image-only user turns, and tool-only assistant turns now appear. +- JSON exports: messages contain typed `blocks` and may omit the legacy `content` field — external consumers should prefer `blocks`. +- To re-render existing exports with the current rendering and collapse policy: `python -m src.main cache --clear` then `python -m src.main export`, followed by `joplin` to update notes (and upload media resources). Per-conversation message counts may increase as previously-dropped Custom Instructions, image-only turns, and tool-only turns now appear. -### Out of scope (deferred to v0.5.0+) -- Binary downloads of images and audio assets (placeholders show metadata only; `content not preserved in this export`) -- Joplin resource upload for embedded media -- Filename resolution for `file-XYZ` / `sediment://` references -- Speculative ChatGPT types (`tether_browsing_display`, `tether_quote`) and DALL-E assistant images — fall through to `unknown` blocks if encountered - -## [0.2.0] - Unreleased -### Added -- Joplin import automation: `joplin` command syncs exported Markdown files to Joplin as notes -- Notebooks created automatically per provider+project (`ChatGPT - My Project`, etc.) -- Re-running is safe: notes are updated, not duplicated (Joplin note ID stored in manifest) -- `JOPLIN_API_TOKEN`, `JOPLIN_API_URL`, `JOPLIN_REQUEST_TIMEOUT` config variables -- Configurable request timeout with clear error messages and actionable hints on timeout -- `--project` filter on `export` and `list` commands (case-insensitive substring or `none`) -- ChatGPT Projects support via `CHATGPT_PROJECT_IDS` env var - -## [0.1.0] - Unreleased -### Added -- Initial implementation: ChatGPT and Claude export via internal web APIs -- Markdown and JSON exporters -- Local cache/manifest for incremental sync -- CLI with export, list, cache, doctor, and auth commands +### Test suite +- 264 tests, all passing. diff --git a/FUTURE.md b/FUTURE.md index 38fecdd..e95dc52 100644 --- a/FUTURE.md +++ b/FUTURE.md @@ -8,58 +8,198 @@ of these additions straightforward. - v0.1.0 — Core export: ChatGPT + Claude, incremental sync, Markdown + JSON output - v0.2.0 — Joplin import automation (`joplin` command, create/update notes, notebook auto-creation) - v0.4.0 — Rich content support: typed message blocks (text, code, thinking, tool_use, tool_result, image_placeholder, file_placeholder, unknown); ChatGPT voice transcripts as text + audio placeholders; Custom Instructions extraction; data-loss visibility via `LossReport` summary and visible `unknown` blocks +- v0.5.0 — Nested Joplin notebooks, date-prefixed note titles, flat year folders +- v0.6.0 — Collapse tool retrieval dumps & hidden context (`EXPORTER_HIDDEN_CONTENT` policy; roadmap item 1); session limiter + request pacing (`MAX_CONVERSATIONS_PER_RUN`, `REQUEST_DELAY`; roadmap item 6) --- -## Export `--force` Flag (v0.2.x) +# Roadmap (decided 2026-06-12) -Add `--force` to the `export` command to re-export already-cached conversations -without permanently clearing the entire manifest. Useful for re-generating files -after changing the Markdown template or output structure. +Priorities reflect the tool's primary purpose — a trustworthy backup so that +conversation data is not lost if a provider account is ever closed — plus the +day-to-day friction of weekly ChatGPT token refresh, and the long-term goal +of running headless as a StartOS service. -Implementation: pass a `force=True` flag to `cache.get_new_or_updated()`, which -returns all conversations regardless of cache state when force is True. +**Now (in order):** +1. ~~Collapse tool retrieval dumps & hidden context~~ — **shipped in v0.6.0** + (full-archive re-export still pending; see entry below) +2. ~~Brave cookie auto-extraction~~ — **shipped in v0.6.0** (note: runs on + whichever machine hosts the browser — Brave is not on this Linux box) +3. ~~Claude Code session provider~~ — **shipped in v0.6.0** +4. ~~Archive hygiene — `prune` + `doctor` integrity check~~ — **shipped in v0.6.0** -Current workaround: `python -m src.main cache --clear` then re-run export. +**Soon, but later:** -## Joplin `--force` Flag (v0.2.x) +5. ~~Binary content downloads~~ — **shipped in v0.6.0** +6. ~~Per-session download limiter + polite pacing~~ — **shipped in v0.6.0** +7. Scheduled / watch mode → StartOS service packaging (long-term destination) -Similarly, add `--force` to the `joplin` command to re-sync all cached -conversations to Joplin regardless of whether they've been synced before. -Useful after making formatting changes to the Markdown exporter. - -Implementation: in `get_joplin_pending()`, return all entries that have a -`file_path` when `force=True`, ignoring `joplin_synced_at`. - -## Per-Conversation Cache Reset (v0.2.x) - -Add `cache --reset --conversation ` to force re-export or re-sync of a -single conversation without clearing the entire provider cache. - -Current workaround: manually edit `~/.ai-chat-exporter/manifest.json` and -delete the entry, then re-run export. +**Deprioritized** (entries kept at the bottom of this file; revisit on +demand): `--force` flags, per-conversation cache reset, official export-ZIP +fallback, o1/o3 reasoning reclassification, Obsidian output, token expiry +notifications (largely superseded by cookie auto-extraction), search command. +Additional web providers (Gemini/Grok/Perplexity) are explicitly out of +scope — no significant usage to archive. --- -## Official API Fallback (v0.3.0) +## 1. Collapse Tool Retrieval Dumps & Hidden Context — SHIPPED v0.6.0 -If the unofficial internal web API approach breaks, migrate to official export -file parsing as a fallback: -- ChatGPT: parse `conversations.json` from Settings → Export Data -- Claude: parse `conversations.json` from Settings → Privacy → Export Data +**Implemented 2026-06-12** as designed below, with one scoping correction +from live recon: the retrieval dumps are NOT flagged +`is_visually_hidden_from_conversation` — `author.name == "file_search"` is +the discriminator (the hidden flag only marks Custom Instructions and small +system stubs). Verified live: worst files shrink 93% (524KB → 36KB). +Remaining step: full-archive re-export (`cache --clear` + `export`) + +Joplin re-sync, at the user's chosen time. -The `BaseProvider` abstract class is intentionally designed so that a -`FileProvider` subclass can implement the same interface -(`list_conversations`, `get_conversation`, `normalize_conversation`) -without any changes to cache, exporters, or CLI code. +**Problem (measured 2026-06-12 against a full fresh export):** 45% of the +entire 11.2MB archive (260 files) is tool-role messages; 29 files are +majority tool-dump. Worst case: a 524KB conversation where 495KB (94%, 64 +messages) is ChatGPT's file-retrieval tool re-injecting the full text of +the user's own attached documents, the same files dumped dozens of times +per conversation. These messages were invisible in the ChatGPT web UI; +they appear in exports because v0.4.0 lifted the role filter to fix silent +data loss. Custom Instructions hidden-context blocks are a minor secondary +case (~2KB, once per conversation) — the originally planned +`EXPORTER_INCLUDE_HIDDEN_CONTEXT` toggle alone would not help: the worst +file contains zero hidden-context blocks. -To add this: implement `src/providers/file_chatgpt.py` and -`src/providers/file_claude.py`, then add `--input-file` flag to the -export command to accept a pre-downloaded export ZIP or JSON. +**Fix: collapse, don't drop** (consistent with the no-silent-drop rule): ---- +- `EXPORTER_HIDDEN_CONTENT=full|placeholder|omit` env var, default + `placeholder`, plus a `--hidden-content` CLI override on `export`. +- `placeholder` renders affected messages as one line with type and size: + `> 🔧 Tool output (file_search, 24KB) — omitted + (EXPORTER_HIDDEN_CONTENT=full to keep)`. Expected effect: archive + roughly halves; worst files shrink ~90%. +- **Scope — collapse:** (a) tool-role retrieval dumps, identified by raw + `author.name` (`file_search`, `myfiles_browser`, …) in the API response + (the rendered Markdown only shows a generic "🔧 Tool" label, so the + decision must happen in the provider, not the renderer); (b) messages + flagged `is_visually_hidden_from_conversation`, including + `user_editable_context` / `model_editable_context` (Custom + Instructions) — subsumes the old suppress-hidden-context idea. +- **Scope — keep at full size:** code-execution `tool_result` blocks and + web-search results; those are usually content the user wants. +- Count collapsed messages in the post-export summary so the omission + stays visible (mirror the LossReport presentation, but as intentional + policy, not loss). -## Binary Content Downloads (v0.5.0) +Re-export workflow after shipping: `cache --clear` + `export` (same as +the v0.4.0 migration). + +## 2. Brave Cookie Auto-Extraction — SHIPPED v0.6.0 + +**Implemented 2026-06-12**: `auth --from-browser [browser]` +(`src/browser_tokens.py`, browser-cookie3 dependency, defaults to brave; +chrome/chromium/edge/firefox also supported). Extracted tokens are validated +against the live API before `.env` is touched. **Discovered during +implementation: Brave is not installed on this machine** — the logged-in +browser lives elsewhere, so the flag only helps when the exporter runs on +that machine (verified the graceful-failure path here; the happy path is +covered by mocked tests). This strengthens the case for the StartOS +token-push mechanism (entry 8). + +Pull session tokens directly from the Brave browser profile instead of the +weekly manual DevTools dance. `auth` gains an "extract from browser" path +(with the manual flow kept as fallback); a later iteration could let +`doctor` or a 401 handler suggest/perform a re-extract automatically. + +- Brave on Linux follows the Chromium pattern: cookies SQLite at + `~/.config/BraveSoftware/Brave-Browser/Default/Cookies`, values + AES-128-CBC encrypted with a key derived from the OS keyring secret + ("Brave Safe Storage" via SecretService/kwallet, or the `peanuts` v10 + fallback). Well-trodden territory — evaluate a small dependency + (`browser_cookie3`-style) vs. a focused in-repo implementation. +- Extract `__Secure-next-auth.session-token.0` / `.1` from `chatgpt.com` + and `sessionKey` from `claude.ai`; write to `.env` through the existing + auth-wizard write path (tokens never echoed). +- Brave must be the user's logged-in browser (it is); detect a locked / + unreadable cookie DB (Brave running with the DB exclusively locked) and + fall back to the manual flow with a clear message. +- Does **not** solve headless/StartOS — no browser on the server. That + needs a token-push mechanism, tracked under the StartOS entry. + +## 3. Claude Code Session Provider — SHIPPED v0.6.0 + +**Implemented 2026-06-12** as designed below (`src/providers/claude_code.py`, +`--provider claude-code`, `CLAUDE_CODE_DIR` override). Additional findings +during implementation: `isSidechain` records are subagent transcripts (skipped), +`isMeta` marks harness-generated user records (skipped), and listing/normalized +`updated_at` must both use file mtime or the cache would re-export every +session every run. First export: 25 sessions, 19MB JSONL → 804KB Markdown. + +Archive local Claude Code session transcripts. No tokens, no rate limits, +no ToS risk — the data is already on disk but lives in a single JSONL per +session that Claude Code may clean up, and it contains deliverables +(reviews, plans, analyses) that exist nowhere else. + +Decisions (2026-06-12): +- **Rendering: prose-only.** Keep user prompts and assistant text + (including full deliverable write-ups); collapse tool activity to + one-line placeholders (`> 🔧 Tool activity — 14 calls (Read ×9, Bash ×2), + 86KB — omitted`); **exclude thinking blocks**. +- **Joplin: sync enabled.** Each coding project becomes a notebook nested + under an **"AI-Claude"** parent notebook (nested-notebook support shipped + in v0.5.0). + +Data facts (measured 2026-06-12): +- Source: `~/.claude/projects//.jsonl`. + Currently 29 sessions, 18.8MB total, largest 3.8MB. +- Representative 2.1MB session: tool_result 436KB, tool_use 122KB, + thinking 104KB, dialogue prose only ~24KB (~4%) — collapsing tool + activity is what makes these exports readable. +- Record types: `user` / `assistant` (Anthropic-style `message.content` + block arrays) plus harness records: `ai-title` (use for note title and + filename slug), `last-prompt`, `file-history-snapshot`, `attachment`, + `permission-mode`, `system` (skip). Strip harness noise from user + messages (``, `` blocks). + +Implementation shape: new `src/providers/claude_code.py` implementing the +`BaseProvider` interface — `list_conversations` scans project dirs, +`get_conversation` parses the JSONL, `normalize_conversation` maps onto the +existing block schema (content is already block-shaped: text / tool_use / +tool_result / thinking). Incremental sync via file mtime/size recorded in +the existing manifest. Project name derives from the munged cwd dirname. + +## 4. Archive Hygiene: `prune` Command + Manifest Integrity — SHIPPED v0.6.0 + +**Implemented 2026-06-12** as designed below, plus an empty-manifest guard +(refuses to prune right after `cache --clear`). First live run removed 420 +stale files (9.4 MB, old-layout trees + `_.md` orphans); doctor now reports +manifest↔disk integrity (293/293 after the run). + +A backup is only trustworthy if the on-disk tree matches the manifest. +Observed 2026-06-12: pre-v0.5.0 layout trees (`tspc-expertcouncil/2025/`) +and `_.md` no-ID orphans (from the empty-conversation-id bug fixed in +v0.4.1) sit alongside current exports and would double-sync into Joplin. + +- `prune` command: delete export files not referenced by the manifest. + `--dry-run` (default off, but always print the list before deleting) + shows what would be removed and why (old layout / orphan / unknown). +- `doctor` extension: verify every manifest entry's `file_path` exists on + disk; report missing files (re-export candidates) and unreferenced files + (prune candidates). + +## 5. Binary Content Downloads — SHIPPED v0.6.0 + +**Implemented 2026-06-12** (`src/media.py`, `EXPORTER_DOWNLOAD_MEDIA`, +`download_asset`/`parse_asset_file_id` on the ChatGPT provider, Joplin +`create_resource` + `upload_media_and_rewrite`). Live recon settled the +download mechanism: `GET /backend-api/files/{id}/download` returns a signed +`download_url`; a second GET yields the bytes (works for user uploads; +older AI-generated images 404 — expired server-side, handled gracefully). +Asset refs come in three shapes — `sediment://file_…`, +`sediment://#file_…#p_N.png` (generated), `file-service://…`. Archive +scan: 14 images (4 uploads / 10 generated) + 556 audio clips ≈162MB, so +images-only is the default and audio is opt-in via `all`. + +Original notes below. + +**Priority note (2026-06-12): "later, but soon" — under the +backup-if-account-closes goal, embedded images are part of the data that +would be lost; placeholders alone don't preserve them.** v0.4.0 ships placeholders for images and audio assets but does not download the binary content. The `_safe_fence`-wrapped placeholders include the asset @@ -82,24 +222,46 @@ The block-level schema is already in place — only the file-fetch + rewrite layer needs to be added. See the `image_placeholder` and `file_placeholder` block definitions in `src/blocks.py`. -## Reclassify o1/o3 Reasoning Subparts (v0.4.1) +## 6. Per-Session Download Limiter — SHIPPED v0.6.0 -v0.4.0 leaves dict parts inside `text` content_type messages with shape -`{"summary": ..., "content": ...}` rendered as plain text (defensive — the -shape was inferred from a code comment, not captured live). Once a real -reasoning conversation is captured, reclassify these as `thinking` blocks. +**Implemented 2026-06-12** as designed below: `--max-conversations N` / +`MAX_CONVERSATIONS_PER_RUN` session cap with deferred-count reporting, and +`REQUEST_DELAY` pacing (default 1.0s ±25% jitter) in `BaseProvider._request`. +Verified live: a 2-pending run with cap 1 exported one, deferred one, and +the re-run picked it up. -## Suppress Hidden Context (v0.4.x) +Cap how many conversations are downloaded in a single `export` run so the +tool never hammers the ChatGPT/Claude internal APIs with a large burst — +most importantly on the very first export, which otherwise fetches the +entire conversation history in one session. Because every run is resumable +(the manifest records each conversation immediately), a capped run simply +exports the first N pending conversations and the next run picks up where +it left off. This keeps traffic looking like a human-paced session rather +than a scraper, reducing the risk of rate limiting or account flags. -If Custom Instructions duplication across conversations becomes a storage -problem, add `EXPORTER_INCLUDE_HIDDEN_CONTEXT=false` env var. The toggle is -a single `os.getenv()` check at the start of -`_extract_editable_context_blocks` in `src/providers/chatgpt.py` — return -empty list if disabled. +Two complementary pieces: ---- +1. **Session cap** — `--max-conversations N` flag (and + `MAX_CONVERSATIONS_PER_RUN` env default). Implementation: in the + `export` command, slice the pending list after the cache filter: + `to_export = to_export[:n]`. On exit, print exported-vs-remaining + counts (reuse the message format from the 429 early-exit path) and + remind the user to re-run to continue. +2. **Polite pacing** — `REQUEST_DELAY` env var (seconds, with small + random jitter) slept between per-conversation detail fetches in + `BaseProvider`, so even a capped run doesn't fire requests + back-to-back. The existing 429 backoff in `_request` stays as the + reactive safety net. -## Scheduled / Watch Mode (v0.5.0) +Note: the conversation *listing* (paginated, 100/page) still runs in full +each time so the cache comparison works — the cap applies to the heavy +per-conversation detail fetches, which dominate request volume. + +Together with watch mode, this is a stepping stone to the StartOS service: +a scheduled, capped, politely-paced export is the traffic profile a +headless deployment needs. + +## 7. Scheduled / Watch Mode Add a `watch` command (or cron integration helper) to run exports automatically on a schedule: @@ -116,9 +278,84 @@ Implementation: simple loop with `time.sleep()`, or emit a crontab entry string that calls the export and joplin commands in sequence. A `--once` flag would do a single run then exit (useful for cron itself). +Stepping stone to the StartOS service (below). + +## 8. StartOS Service Packaging (long-term destination) + +Run the exporter headless as a StartOS service: scheduled export + sync to +a Joplin Server instance, so the backup happens without a desktop in the +loop. + +- Building blocks needed first: watch mode (#7), session limiter (#6), + and a Joplin Server (vs. desktop Web Clipper) sync target. +- **Hard problem, flagged early:** session-token freshness without a + browser. ChatGPT tokens last ~7 days; there is no browser on the server + to re-extract from. Candidate mechanisms: a service config action the + user pastes fresh tokens into periodically, or a future browser + extension that pushes tokens to the server. Cookie auto-extraction (#2) + only solves the desktop case. + --- -## Obsidian Vault Output (v0.5.0) +# Deprioritized + +Kept for reference; not on the active roadmap. + +## Export `--force` Flag + +Add `--force` to the `export` command to re-export already-cached conversations +without permanently clearing the entire manifest. Useful for re-generating files +after changing the Markdown template or output structure. + +Implementation: pass a `force=True` flag to `cache.get_new_or_updated()`, which +returns all conversations regardless of cache state when force is True. + +Current workaround: `python -m src.main cache --clear` then re-run export. + +## Joplin `--force` Flag + +Similarly, add `--force` to the `joplin` command to re-sync all cached +conversations to Joplin regardless of whether they've been synced before. +Useful after making formatting changes to the Markdown exporter. + +Implementation: in `get_joplin_pending()`, return all entries that have a +`file_path` when `force=True`, ignoring `joplin_synced_at`. + +## Per-Conversation Cache Reset + +Add `cache --reset --conversation ` to force re-export or re-sync of a +single conversation without clearing the entire provider cache. + +Current workaround: manually edit `~/.ai-chat-exporter/manifest.json` and +delete the entry, then re-run export. + +## Official API Fallback + +If the unofficial internal web API approach breaks, migrate to official export +file parsing as a fallback: +- ChatGPT: parse `conversations.json` from Settings → Export Data +- Claude: parse `conversations.json` from Settings → Privacy → Export Data + +The `BaseProvider` abstract class is intentionally designed so that a +`FileProvider` subclass can implement the same interface +(`list_conversations`, `get_conversation`, `normalize_conversation`) +without any changes to cache, exporters, or CLI code. + +To add this: implement `src/providers/file_chatgpt.py` and +`src/providers/file_claude.py`, then add `--input-file` flag to the +export command to accept a pre-downloaded export ZIP or JSON. + +Deprioritized 2026-06-12: the official ChatGPT export does not cover what +this user needs (project data), so it isn't a real fallback here. + +## Reclassify o1/o3 Reasoning Subparts + +v0.4.0 leaves dict parts inside `text` content_type messages with shape +`{"summary": ..., "content": ...}` rendered as plain text (defensive — the +shape was inferred from a code comment, not captured live). Once a real +reasoning conversation is captured, reclassify these as `thinking` blocks. + +## Obsidian Vault Output Add an `obsidian` command (or `--target obsidian` flag) to sync exported conversations into an Obsidian vault directory. The current Markdown format @@ -134,28 +371,7 @@ Obsidian. An `ObsidianSyncer` class (mirroring `JoplinClient`) would simply copy files to the vault directory and maintain a flat or nested folder structure matching the user's Obsidian setup. No API needed — just file I/O. ---- - -## Joplin Nested Notebooks (future) - -Currently notebooks are flat: `ChatGPT - My Project`. Joplin supports nested -notebooks via `parent_id`. A future option (`JOPLIN_NESTED_NOTEBOOKS=true`) -could create a two-level hierarchy: - -``` -ChatGPT/ - My Project/ - No Project/ -Claude/ - Budget Tracker/ -``` - -Implementation: `get_or_create_notebook` would first find/create the provider -notebook, then find/create the project notebook as a child. - ---- - -## Token Expiry Notifications (future) +## Token Expiry Notifications Proactively warn when a token is close to expiry (within 48h for ChatGPT), rather than only surfacing the warning at startup. Options: @@ -165,9 +381,10 @@ rather than only surfacing the warning at startup. Options: - Send a desktop notification via `notify-send` (Linux) or `osascript` (macOS) when a token is within 24h of expiry ---- +Largely superseded by Brave cookie auto-extraction (#2): once refresh is +one command (or automatic), expiry warnings matter much less. -## Search Command (future) +## Search Command Add a `search` command to full-text search across all exported Markdown files: diff --git a/README.md b/README.md index a716497..b421b15 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,17 @@ This checks token presence, format, expiry, directory permissions, disk space, a Session tokens are how your browser stays logged in. This tool uses them to access your chat history on your behalf. +### The fast way: extract from your browser + +If the browser you're logged in with runs on the same machine as this tool: + +```bash +ai-chat-exporter auth --from-browser # Brave (default) +ai-chat-exporter auth --from-browser firefox # or chrome, chromium, edge +``` + +This reads the session cookies straight from the browser's cookie store (decrypted via the OS keyring/DPAPI/Keychain), validates each token against the live API, and writes `.env` — no DevTools, no copy-pasting. A token that fails validation never overwrites a working one. If extraction fails (browser not installed here, cookie DB locked, logged out), fall back to the manual flow below. + ### Token Lifetimes | Provider | Cookie Name | Lifetime | Expiry Detection | @@ -153,6 +164,10 @@ cp .env.example .env |----------|---------|-------------| | `EXPORT_DIR` | `./exports` | Where to write exported Markdown files | | `OUTPUT_STRUCTURE` | `provider/project/year` | Folder structure (see below) | +| `EXPORTER_HIDDEN_CONTENT` | `placeholder` | What to do with content invisible in the provider's web UI (file-retrieval tool dumps, Custom Instructions): `placeholder` collapses to a one-line note with size, `full` keeps everything, `omit` drops it. Dumps can be 90% of a conversation's bytes. | +| `EXPORTER_DOWNLOAD_MEDIA` | `images` | Download conversation assets into a `media/` folder beside each export and inline them: `images` (images only), `all` (also audio/voice clips), `off` (text placeholders only). Downloaded media is uploaded to Joplin as resources on the next `joplin` run. | +| `MAX_CONVERSATIONS_PER_RUN` | unlimited | Cap downloads per export run (per provider). Runs are resumable, so re-running continues where the cap stopped — useful to spread a large first export across several sessions. | +| `REQUEST_DELAY` | `1.0` | Seconds between consecutive API requests, with small random jitter, so traffic stays human-paced. Set `0` to disable. | ### Joplin @@ -192,6 +207,19 @@ The `auth` wizard can also guide you through this step interactively. --- +## Claude Code Sessions + +The `claude-code` provider archives your local [Claude Code](https://claude.com/claude-code) agent transcripts — no tokens, no API, no ToS exposure. Sessions are read from `~/.claude/projects/` (override with `CLAUDE_CODE_DIR` in `.env`). + +```bash +ai-chat-exporter export --provider claude-code +ai-chat-exporter joplin --provider claude-code +``` + +Exports are prose-only by default: your prompts and Claude's write-ups are kept, tool activity is grouped into one-line placeholders (`> 🔧 Tool output — 14 calls: Read ×9, Bash ×2 (86KB) — omitted`), and internal reasoning is dropped (counted in the run summary). Set `EXPORTER_HIDDEN_CONTENT=full` to keep everything. Each coding project becomes a Joplin notebook under the `AI-Claude` parent. The provider is included in `--provider all` whenever the sessions directory exists. + +--- + ## Output Structure All exported files go under `EXPORT_DIR`. The folder structure maps directly to Joplin notebooks. @@ -256,10 +284,11 @@ Each provider+project combination maps to a flat Joplin notebook created automat ### `auth` — Interactive token setup ```bash -ai-chat-exporter auth +ai-chat-exporter auth # manual wizard (DevTools flow) +ai-chat-exporter auth --from-browser # extract from Brave's cookie store ``` -Guided wizard to find and save session tokens and ChatGPT project IDs. Detects OS and shows the correct DevTools shortcut. +Guided wizard to find and save session tokens and ChatGPT project IDs. Detects OS and shows the correct DevTools shortcut. `--from-browser [brave|chrome|chromium|edge|firefox]` skips the wizard and pulls tokens directly from a local browser, validating them live before writing `.env`. ### `doctor` — Health check @@ -300,7 +329,7 @@ ai-chat-exporter export --output /path/to/my/notes ai-chat-exporter export --dry-run ``` -Options: `--provider [chatgpt|claude|all]`, `--format [markdown|json|both]`, `--output PATH`, `--since YYYY-MM-DD`, `--project NAME`, `--dry-run` +Options: `--provider [chatgpt|claude|claude-code|all]`, `--format [markdown|json|both]`, `--output PATH`, `--since YYYY-MM-DD`, `--project NAME`, `--hidden-content [full|placeholder|omit]`, `--download-media [images|all|off]`, `--max-conversations N`, `--dry-run` ### `list` — List conversations @@ -349,6 +378,22 @@ Reads the local export cache and pushes each exported Markdown file to Joplin as Options: `--provider [chatgpt|claude|all]`, `--project NAME`, `--dry-run` +### `prune` — Delete stale export files + +```bash +# Preview what would be deleted +ai-chat-exporter prune --dry-run + +# Delete (asks for confirmation; -y skips the prompt) +ai-chat-exporter prune +``` + +Deletes export files no longer referenced by the cache manifest — leftovers +from old folder layouts or fixed bugs that would otherwise double-sync into +Joplin. Refuses to run when the manifest is empty (e.g. right after +`cache --clear`) so it can never wipe a freshly cleared archive. The `doctor` +command separately verifies that every manifest entry's file exists on disk. + ### `cache` — Manage the sync manifest ```bash @@ -426,7 +471,13 @@ Make sure you've added the project IDs to `CHATGPT_PROJECT_IDS` in your `.env`. The provider's internal API may have changed. Run with `--debug`, sanitize the output (remove any personal content), and check the project's GitHub Issues for known fixes. ### Non-text content warnings -Since v0.4.0, rich content is preserved as typed blocks in the export. ChatGPT voice transcripts render as text and audio assets as `📎 File attached` placeholders with size and duration metadata. Images render as `🖼️ Image attached` placeholders showing the asset reference. Custom Instructions appear under a `> ℹ️ Hidden context` marker. Anything the extractor doesn't recognise renders as a visible `> ⚠️ Unsupported content` block naming the type and observed keys, *and* increments a counter in the post-export summary so you can tell whether real content is being silently skipped. Binary downloads (the actual image/audio bytes) are still deferred — see `FUTURE.md` v0.5.0. +Since v0.4.0, rich content is preserved as typed blocks in the export. ChatGPT voice transcripts render as text and audio assets as `📎 File attached` placeholders with size and duration metadata. Anything the extractor doesn't recognise renders as a visible `> ⚠️ Unsupported content` block naming the type and observed keys, *and* increments a counter in the post-export summary so you can tell whether real content is being silently skipped. + +### Embedded images and media +Since v0.6.0, image attachments are downloaded by default into a `media/` folder beside each export and inlined as `![](media/…)`. Audio and other files download only with `EXPORTER_DOWNLOAD_MEDIA=all`. On the next `joplin` run these are uploaded as Joplin resources so they render inside the note. Assets that have expired on the provider's side (older AI-generated images often do) keep their text placeholder and are tallied as `media failed` in the run summary — they're never fatal to the export. Set `EXPORTER_DOWNLOAD_MEDIA=off` to skip downloads entirely. + +### Tool output / hidden context collapsed +Since v0.6.0, content that was invisible in the ChatGPT web UI is collapsed by default to one-line placeholders like `> 🔧 Tool output — file_search (15.0 KB) — omitted`. This is ChatGPT's file-retrieval tool re-injecting your attached files on every run — it can be 90% of a conversation's bytes while containing none of the dialogue. Custom Instructions collapse the same way (`> ℹ️ Hidden context`). The post-export summary lists every collapsed message by origin with the total KB omitted. To keep everything, set `EXPORTER_HIDDEN_CONTENT=full` in `.env` or pass `--hidden-content full`. ### Empty export / all conversations skipped No new or updated conversations since your last run. To verify: `ai-chat-exporter cache --show`. To force a full re-export: `ai-chat-exporter cache --clear`. @@ -440,12 +491,9 @@ No new or updated conversations since your last run. To verify: `ai-chat-exporte ## Future Work -See `FUTURE.md` for planned features: +See `FUTURE.md` for the full roadmap. Current priorities: -- **v0.2.x** — `export --force` flag; `joplin --force` flag; per-conversation cache reset -- **v0.3.0** — Official API fallback: parse export ZIP files from ChatGPT/Claude settings -- **v0.4.x / v0.5.0** — Binary content downloads (images, audio bytes) and Joplin resource upload; reclassify o1/o3 reasoning subparts; optional `EXPORTER_INCLUDE_HIDDEN_CONTEXT` toggle -- **v0.5.0** — Watch/scheduled mode; Obsidian vault output +- **Watch/scheduled mode** on the way to a headless StartOS service --- diff --git a/pyproject.toml b/pyproject.toml index 3910f8b..a66bf28 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "ai-chat-exporter" -version = "0.4.1" +version = "0.6.0" description = "Export ChatGPT and Claude conversation history to Markdown for personal archival in Joplin" requires-python = ">=3.11" dependencies = [ @@ -15,6 +15,7 @@ dependencies = [ "rich==13.7.1", "python-slugify==8.0.4", "PyJWT==2.8.0", + "browser-cookie3==0.20.1", ] [project.optional-dependencies] diff --git a/requirements.txt b/requirements.txt index dab0d28..64a8018 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,3 +26,4 @@ setuptools==82.0.0 text-unidecode==1.3 urllib3==2.6.3 wheel==0.46.3 +browser-cookie3==0.20.1 diff --git a/src/blocks.py b/src/blocks.py index 9d53526..1399cf7 100644 --- a/src/blocks.py +++ b/src/blocks.py @@ -11,6 +11,7 @@ at the call site — see plan §Data-loss visibility. """ import json +from pathlib import Path from typing import Any BLOCK_TYPE_TEXT = "text" @@ -23,6 +24,10 @@ BLOCK_TYPE_IMAGE_PLACEHOLDER = "image_placeholder" BLOCK_TYPE_FILE_PLACEHOLDER = "file_placeholder" BLOCK_TYPE_UNKNOWN = "unknown" BLOCK_TYPE_HIDDEN_CONTEXT_MARKER = "hidden_context_marker" +BLOCK_TYPE_COLLAPSED = "collapsed" + +COLLAPSED_KIND_TOOL_DUMP = "tool_dump" +COLLAPSED_KIND_HIDDEN_CONTEXT = "hidden_context" UNKNOWN_REASON_UNKNOWN_TYPE = "unknown_type" UNKNOWN_REASON_EXTRACTION_FAILED = "extraction_failed" @@ -164,6 +169,30 @@ def make_unknown_block( } +def make_collapsed_block( + origin: str, + content_type: str, + size_bytes: int, + kind: str, +) -> dict: + """One-line stand-in for a message omitted by the EXPORTER_HIDDEN_CONTENT policy. + + ``kind`` ∈ {COLLAPSED_KIND_TOOL_DUMP, COLLAPSED_KIND_HIDDEN_CONTEXT}. + Unlike ``unknown`` blocks (unexpected loss), a collapsed block records an + intentional policy decision — the content was invisible in the provider's + web UI and the user chose not to keep it. Every call site MUST tally via + ``LossReport.record_collapsed`` so the omission stays visible in the + post-export summary. + """ + return { + "type": BLOCK_TYPE_COLLAPSED, + "origin": origin or "", + "content_type": content_type or "", + "size_bytes": int(size_bytes or 0), + "kind": kind, + } + + def make_hidden_context_marker(content_type: str) -> dict: """A short prepend block that flags the surrounding message as hidden context. @@ -246,6 +275,10 @@ def _render_one(block: dict) -> str: ref = block.get("ref", "") source = block.get("source", "unknown") mime = block.get("mime") + local_path = block.get("local_path") + if local_path: + # Downloaded — render as a real inline image. + return f"![{source or 'image'}]({local_path})" meta_parts = [source] if source else [] if mime: meta_parts.append(mime) @@ -259,14 +292,20 @@ def _render_one(block: dict) -> str: mime = block.get("mime") size_bytes = block.get("size_bytes") duration = block.get("duration_seconds") + local_path = block.get("local_path") meta_parts: list[str] = [] if mime: meta_parts.append(mime) - if isinstance(size_bytes, int) and size_bytes > 0: - kb = size_bytes / 1024 - meta_parts.append(f"{kb:.1f} KB" if kb < 1024 else f"{kb / 1024:.2f} MB") + size_label = _format_size(size_bytes) + if size_label: + meta_parts.append(size_label) if isinstance(duration, (int, float)) and duration > 0: meta_parts.append(f"{duration:.2f}s") + if local_path: + # Downloaded — render as a link to the local copy. + meta = ", ".join(meta_parts) if meta_parts else "" + meta_str = f" ({meta})" if meta else "" + return f"> 📎 **File attached** — [{Path(local_path).name}]({local_path}){meta_str}" meta_parts.append("content not preserved in this export") meta = ", ".join(meta_parts) return f"> 📎 **File attached** — `{label}` ({meta})" @@ -286,6 +325,19 @@ def _render_one(block: dict) -> str: if btype == BLOCK_TYPE_HIDDEN_CONTEXT_MARKER: ctype = block.get("content_type", "") return f"> ℹ️ **Hidden context** — `{ctype}`" + if btype == BLOCK_TYPE_COLLAPSED: + origin = block.get("origin", "") + kind = block.get("kind", "") + if kind == COLLAPSED_KIND_HIDDEN_CONTEXT: + icon, label = "ℹ️", "Hidden context" + else: + icon, label = "🔧", "Tool output" + size_label = _format_size(block.get("size_bytes")) + size_part = f" ({size_label})" if size_label else "" + return ( + f"> {icon} **{label}** — `{origin}`{size_part} — omitted " + "(EXPORTER_HIDDEN_CONTENT=full to keep)" + ) # Defensive: a block of unrecognised local type (shouldn't happen if # constructors are used). Render as visible warning rather than dropping. @@ -297,6 +349,14 @@ def _render_one(block: dict) -> str: # --------------------------------------------------------------------------- +def _format_size(size_bytes: Any) -> str: + """Human-readable size ('24.1 KB', '1.05 MB'), or '' when absent/zero.""" + if not isinstance(size_bytes, int) or size_bytes <= 0: + return "" + kb = size_bytes / 1024 + return f"{kb:.1f} KB" if kb < 1024 else f"{kb / 1024:.2f} MB" + + def _safe_fence(text: str) -> str: """Return a backtick fence longer than the longest run of backticks in ``text``. diff --git a/src/browser_tokens.py b/src/browser_tokens.py new file mode 100644 index 0000000..0bbd3db --- /dev/null +++ b/src/browser_tokens.py @@ -0,0 +1,101 @@ +"""Extract session tokens directly from a local browser's cookie store. + +Default browser is Brave. Uses browser-cookie3, which handles the +platform-specific cookie decryption (Linux: AES key derived from the OS +keyring "Safe Storage" secret; Windows: DPAPI; macOS: Keychain). The browser +must be the one the user is actually logged in with, on the same machine. + +Failure modes worth knowing: +- Browser not installed / profile not found → BrowserTokenError +- Cookie DB locked (browser running with exclusive lock; rare on Linux, + common on Windows) → BrowserTokenError suggesting the browser be closed +- Logged out / cookie expired → BrowserTokenError naming the missing cookie +""" + +import logging + +logger = logging.getLogger(__name__) + +DEFAULT_BROWSER = "brave" +SUPPORTED_BROWSERS = ("brave", "chrome", "chromium", "edge", "firefox") + +# ChatGPT splits large session tokens across two cookies to stay under the +# browser's 4KB cookie limit; small tokens use the unsuffixed name. +_CHATGPT_COOKIE_0 = "__Secure-next-auth.session-token.0" +_CHATGPT_COOKIE_1 = "__Secure-next-auth.session-token.1" +_CHATGPT_COOKIE_SINGLE = "__Secure-next-auth.session-token" +_CLAUDE_COOKIE = "sessionKey" + + +class BrowserTokenError(Exception): + """Raised when tokens cannot be extracted from the browser profile.""" + + +def _load_cookies(browser: str, domain: str) -> dict[str, str]: + """Return {cookie_name: value} for ``domain`` from the given browser.""" + if browser not in SUPPORTED_BROWSERS: + raise BrowserTokenError( + f"Unsupported browser {browser!r}. " + f"Supported: {', '.join(SUPPORTED_BROWSERS)}" + ) + + import browser_cookie3 + + loader = getattr(browser_cookie3, browser) + try: + jar = loader(domain_name=domain) + except Exception as e: + # browser_cookie3 raises BrowserCookieError plus assorted OS/keyring + # errors. All mean the same thing to the caller: fall back to manual. + hint = "" + text = str(e).lower() + if "lock" in text or "database is locked" in text: + hint = " Close the browser and try again." + elif "not find" in text or "no such file" in text: + hint = " Is the browser installed on this machine?" + raise BrowserTokenError( + f"Could not read {browser} cookies for {domain}: {e}.{hint}" + ) from e + + return {cookie.name: cookie.value or "" for cookie in jar} + + +def extract_chatgpt_tokens(browser: str = DEFAULT_BROWSER) -> tuple[str, str | None]: + """Return (session_token_0, session_token_1_or_None) from the browser. + + Raises BrowserTokenError when no session cookie is present (not logged in, + or logged out since the last visit). + """ + cookies = _load_cookies(browser, "chatgpt.com") + token_0 = cookies.get(_CHATGPT_COOKIE_0, "").strip() + token_1 = cookies.get(_CHATGPT_COOKIE_1, "").strip() or None + if token_0: + logger.info( + "[browser-tokens] ChatGPT session cookies found in %s (chunked: %s)", + browser, + "yes" if token_1 else "no", + ) + return token_0, token_1 + + single = cookies.get(_CHATGPT_COOKIE_SINGLE, "").strip() + if single: + logger.info("[browser-tokens] ChatGPT session cookie found in %s (single)", browser) + return single, None + + raise BrowserTokenError( + f"No ChatGPT session cookie in {browser} — open https://chatgpt.com " + "in that browser and log in, then retry." + ) + + +def extract_claude_session(browser: str = DEFAULT_BROWSER) -> str: + """Return the Claude ``sessionKey`` cookie value from the browser.""" + cookies = _load_cookies(browser, "claude.ai") + session_key = cookies.get(_CLAUDE_COOKIE, "").strip() + if session_key: + logger.info("[browser-tokens] Claude sessionKey found in %s", browser) + return session_key + raise BrowserTokenError( + f"No Claude sessionKey cookie in {browser} — open https://claude.ai " + "in that browser and log in, then retry." + ) diff --git a/src/cache.py b/src/cache.py index f14ce43..5d6ca6c 100644 --- a/src/cache.py +++ b/src/cache.py @@ -173,6 +173,22 @@ class Cache: entry["joplin_synced_at"] = datetime.now(tz=timezone.utc).isoformat() self._save() + def get_joplin_resources(self, provider: str, conv_id: str) -> dict[str, str]: + """Return the {relative_media_path: joplin_resource_id} map for a conversation.""" + entry = self._data.get(provider, {}).get(conv_id) + if not isinstance(entry, dict): + return {} + resources = entry.get("joplin_resources") + return dict(resources) if isinstance(resources, dict) else {} + + def set_joplin_resources(self, provider: str, conv_id: str, resources: dict[str, str]) -> None: + """Persist the {relative_media_path: joplin_resource_id} map for a conversation.""" + entry = self._data.get(provider, {}).get(conv_id) + if entry is None: + return + entry["joplin_resources"] = dict(resources) + self._save() + def get_joplin_pending(self, provider: str) -> list[tuple[str, dict]]: """Return (conv_id, entry) pairs that need to be synced to Joplin. @@ -210,6 +226,25 @@ class Cache: return pending + def all_file_paths(self) -> set[str]: + """Return every non-empty ``file_path`` in the manifest, across providers. + + Used by ``prune`` (files on disk but not here are stale) and by the + doctor integrity check (files here but not on disk are missing). + """ + paths: set[str] = set() + for key, entries in self._data.items(): + if not isinstance(entries, dict) or key in ( + "version", + "last_run", + "tos_acknowledged_at", + ): + continue + for entry in entries.values(): + if isinstance(entry, dict) and entry.get("file_path"): + paths.add(entry["file_path"]) + return paths + def last_run(self) -> str | None: """Return the ISO8601 timestamp of the last export run, or None.""" return self._data.get("last_run") diff --git a/src/config.py b/src/config.py index a27c8f1..b9668d9 100644 --- a/src/config.py +++ b/src/config.py @@ -20,6 +20,12 @@ _CLAUDE_PLACEHOLDER = "" # Valid OUTPUT_STRUCTURE values VALID_STRUCTURES = {"provider/project/year", "provider/project", "provider/year"} +# Valid EXPORTER_HIDDEN_CONTENT values (see src/providers/base.py) +VALID_HIDDEN_CONTENT = {"full", "placeholder", "omit"} + +# Valid EXPORTER_DOWNLOAD_MEDIA values (see src/media.py) +VALID_DOWNLOAD_MEDIA = {"images", "all", "off"} + class ConfigError(Exception): """Raised when required configuration is missing or invalid.""" @@ -43,6 +49,16 @@ class Config: # Joplin local REST API settings (Web Clipper service) joplin_api_token: str | None = None joplin_api_url: str = "http://localhost:41184" + # Policy for content invisible in the provider web UI (retrieval dumps, + # hidden context): full | placeholder | omit + hidden_content: str = "placeholder" + # Session cap: max conversations downloaded per export run (None = unlimited). + # Every run is resumable, so a capped run just continues next time. + max_conversations: int | None = None + # Seconds between consecutive API requests (politeness pacing; 0 disables) + request_delay: float = 1.0 + # Asset download policy: images | all | off + download_media: str = "images" def load_config() -> Config: @@ -67,6 +83,12 @@ def load_config() -> Config: joplin_token = os.getenv("JOPLIN_API_TOKEN", "").strip() or None joplin_url = os.getenv("JOPLIN_API_URL", "http://localhost:41184").strip() + hidden_content = os.getenv("EXPORTER_HIDDEN_CONTENT", "").strip().lower() or "placeholder" + + max_conversations_raw = os.getenv("MAX_CONVERSATIONS_PER_RUN", "").strip() + request_delay_raw = os.getenv("REQUEST_DELAY", "").strip() + download_media = os.getenv("EXPORTER_DOWNLOAD_MEDIA", "").strip().lower() or "images" + # Parse CHATGPT_PROJECT_IDS — comma-separated list of gizmo IDs (g-p-xxx) _project_ids_raw = os.getenv("CHATGPT_PROJECT_IDS", "").strip() chatgpt_project_ids = [ @@ -90,6 +112,46 @@ def load_config() -> Config: f"Must be one of: {', '.join(sorted(VALID_STRUCTURES))}" ) + # Validate hidden-content policy + if hidden_content not in VALID_HIDDEN_CONTENT: + errors.append( + f"EXPORTER_HIDDEN_CONTENT '{hidden_content}' is invalid. " + f"Must be one of: {', '.join(sorted(VALID_HIDDEN_CONTENT))}" + ) + + # Validate media download policy + if download_media not in VALID_DOWNLOAD_MEDIA: + errors.append( + f"EXPORTER_DOWNLOAD_MEDIA '{download_media}' is invalid. " + f"Must be one of: {', '.join(sorted(VALID_DOWNLOAD_MEDIA))}" + ) + + # Validate session cap + max_conversations: int | None = None + if max_conversations_raw: + try: + max_conversations = int(max_conversations_raw) + except ValueError: + errors.append( + f"MAX_CONVERSATIONS_PER_RUN '{max_conversations_raw}' is not an integer." + ) + else: + if max_conversations < 1: + errors.append( + f"MAX_CONVERSATIONS_PER_RUN must be at least 1 (got {max_conversations})." + ) + + # Validate request pacing + request_delay = 1.0 + if request_delay_raw: + try: + request_delay = float(request_delay_raw) + except ValueError: + errors.append(f"REQUEST_DELAY '{request_delay_raw}' is not a number.") + else: + if request_delay < 0: + errors.append(f"REQUEST_DELAY must be >= 0 (got {request_delay}).") + # Validate and decode ChatGPT JWT chatgpt_expiry: datetime | None = None if chatgpt_token: @@ -139,6 +201,10 @@ def load_config() -> Config: chatgpt_project_ids=chatgpt_project_ids, joplin_api_token=joplin_token, joplin_api_url=joplin_url, + hidden_content=hidden_content, + max_conversations=max_conversations, + request_delay=request_delay, + download_media=download_media, ) _log_startup_summary(config) @@ -223,7 +289,11 @@ def _log_startup_summary(cfg: Config) -> None: "Joplin: %s | " "export_dir=%s | " "structure=%s | " - "cache_dir=%s", + "cache_dir=%s | " + "hidden_content=%s | " + "max_conversations=%s | " + "request_delay=%.1fs | " + "download_media=%s", chatgpt_status, claude_status, len(cfg.chatgpt_project_ids), @@ -231,4 +301,8 @@ def _log_startup_summary(cfg: Config) -> None: cfg.export_dir, cfg.output_structure, cfg.cache_dir, + cfg.hidden_content, + cfg.max_conversations if cfg.max_conversations is not None else "unlimited", + cfg.request_delay, + cfg.download_media, ) diff --git a/src/joplin.py b/src/joplin.py index 25e1ac0..ce9225d 100644 --- a/src/joplin.py +++ b/src/joplin.py @@ -1,7 +1,10 @@ """Joplin Data API client for importing notes into Joplin desktop.""" +import json import logging import os +import re +from pathlib import Path from typing import Any import requests @@ -190,6 +193,46 @@ class JoplinClient: self._put(f"/notes/{note_id}", {"title": title, "body": body}) logger.info("[joplin] Note updated: %r (%s)", title, note_id) + # ------------------------------------------------------------------ + # Resources (embedded media) + # ------------------------------------------------------------------ + + def create_resource(self, file_path: "Path", title: str | None = None) -> str: + """Upload a local file as a Joplin resource and return its ID. + + Joplin's ``POST /resources`` is multipart: a ``data`` file part plus a + ``props`` JSON part. The returned ID is referenced from note bodies as + ``:/``. + """ + url = f"{self._base_url}/resources" + props = json.dumps({"title": title or file_path.name}) + logger.debug("[joplin] POST /resources (%s)", file_path.name) + try: + with file_path.open("rb") as fh: + resp = requests.post( + url, + params={"token": self._token}, + files={ + "data": (file_path.name, fh), + "props": (None, props), + }, + timeout=_REQUEST_TIMEOUT, + ) + resp.raise_for_status() + resource_id = resp.json()["id"] + logger.info("[joplin] Resource created: %s → %s", file_path.name, resource_id) + return resource_id + except requests.exceptions.ConnectionError as e: + raise JoplinError( + "Cannot connect to Joplin. Is Joplin desktop running with Web Clipper enabled?" + ) from e + except requests.exceptions.Timeout as e: + raise JoplinError(_timeout_message("POST", "/resources")) from e + except requests.exceptions.HTTPError as e: + raise JoplinError(_http_error_message("POST", "/resources", e)) from e + except (requests.exceptions.RequestException, KeyError, OSError) as e: + raise JoplinError(f"Joplin resource upload failed for {file_path.name}: {e}") from e + # ------------------------------------------------------------------ # HTTP helpers # ------------------------------------------------------------------ @@ -304,6 +347,46 @@ def _http_error_message(method: str, path: str, e: requests.exceptions.HTTPError return f"Joplin {method} {path} failed: HTTP {status}{body_snippet}" +# ------------------------------------------------------------------ +# Embedded-media rewriting +# ------------------------------------------------------------------ + +# Markdown image/link targets pointing at the local media/ sibling directory, +# e.g. ![alt](media/file_x.png) or [name](media/clip.wav). +_MEDIA_LINK_RE = re.compile(r"(!?\[[^\]]*\])\((media/[^)\s]+)\)") + + +def upload_media_and_rewrite( + body: str, + note_dir: Path, + client: "JoplinClient", + resource_map: dict[str, str], +) -> tuple[str, dict[str, str]]: + """Rewrite local ``media/…`` links to Joplin ``:/resourceId`` references. + + Uploads each referenced file as a Joplin resource (once), rewriting both + image embeds and file links. ``resource_map`` (relative-path → resource_id) + is the idempotency record from the cache: known paths are reused, new ones + are added and returned so the caller can persist them. Missing files are + left as-is so the link still points at the on-disk copy. + """ + new_map = dict(resource_map) + + def replace(match: re.Match) -> str: + label, rel_path = match.group(1), match.group(2) + resource_id = new_map.get(rel_path) + if resource_id is None: + file_path = (note_dir / rel_path).resolve() + if not file_path.is_file(): + logger.warning("[joplin] Media file missing, leaving link: %s", rel_path) + return match.group(0) + resource_id = client.create_resource(file_path) + new_map[rel_path] = resource_id + return f"{label}(:/{resource_id})" + + return _MEDIA_LINK_RE.sub(replace, body), new_map + + # ------------------------------------------------------------------ # Notebook naming helper # ------------------------------------------------------------------ @@ -312,6 +395,9 @@ def _http_error_message(method: str, path: str, e: requests.exceptions.HTTPError _PROVIDER_DISPLAY = { "chatgpt": "AI-ChatGPT", "claude": "AI-Claude", + # Decision 2026-06-12: Claude Code coding projects nest under the same + # AI-Claude parent, alongside Claude web projects. + "claude-code": "AI-Claude", } diff --git a/src/loss_report.py b/src/loss_report.py index 389666a..54edfbc 100644 --- a/src/loss_report.py +++ b/src/loss_report.py @@ -23,11 +23,20 @@ class LossReport: unknown_blocks: Counter = field(default_factory=Counter) extraction_failures: Counter = field(default_factory=Counter) filtered_roles: Counter = field(default_factory=Counter) + # Messages collapsed/omitted by the EXPORTER_HIDDEN_CONTENT policy — + # intentional, but still surfaced so the omission is never invisible. + collapsed: Counter = field(default_factory=Counter) + collapsed_bytes: int = 0 # Aggregate counters messages_rendered: int = 0 conversations: int = 0 + # Media downloads (EXPORTER_DOWNLOAD_MEDIA): successes / skips / failures + media_downloaded: int = 0 + media_downloaded_bytes: int = 0 + media_failed: Counter = field(default_factory=Counter) + # Recording ------------------------------------------------------------- def record_unknown(self, raw_type: str) -> None: @@ -39,6 +48,19 @@ class LossReport: def record_filtered_role(self, role: str) -> None: self.filtered_roles[role or "?"] += 1 + def record_collapsed(self, origin: str, size_bytes: int = 0) -> None: + self.collapsed[origin or "?"] += 1 + if isinstance(size_bytes, int) and size_bytes > 0: + self.collapsed_bytes += size_bytes + + def record_media_downloaded(self, size_bytes: int = 0) -> None: + self.media_downloaded += 1 + if isinstance(size_bytes, int) and size_bytes > 0: + self.media_downloaded_bytes += size_bytes + + def record_media_failed(self, reason: str) -> None: + self.media_failed[reason or "?"] += 1 + def record_message(self) -> None: self.messages_rendered += 1 @@ -58,6 +80,22 @@ class LossReport: lines.append(f" messages rendered: {self.messages_rendered}") lines.extend(_format_section("unknown blocks: ", self.unknown_blocks)) lines.extend(_format_section("extraction failures: ", self.extraction_failures)) + lines.extend(_format_section("collapsed by policy: ", self.collapsed)) + if self.collapsed: + lines.append( + f" ≈{self.collapsed_bytes / 1024:.0f} KB omitted " + "(intentional — EXPORTER_HIDDEN_CONTENT=full to keep)" + ) + if self.media_downloaded or self.media_failed: + lines.append( + f" media downloaded: {self.media_downloaded} " + f"(≈{self.media_downloaded_bytes / 1024:.0f} KB)" + ) + if self.media_failed: + total_failed = sum(self.media_failed.values()) + lines.append(f" media failed: {total_failed}") + for reason, count in self.media_failed.most_common(_TOP_N_BREAKDOWN): + lines.append(f" {reason}={count}") lines.append( " filtered roles: " "(filter lifted in v0.4.0 — counter retained for future use, expected 0)" diff --git a/src/main.py b/src/main.py index db8aca4..6534c21 100644 --- a/src/main.py +++ b/src/main.py @@ -2,6 +2,7 @@ import importlib.metadata import logging +import os import platform import shutil import sys @@ -109,17 +110,37 @@ def cli(ctx: click.Context, verbose: bool, quiet: bool, debug: bool, no_log_file @cli.command() +@click.option( + "--from-browser", + "from_browser", + is_flag=False, + flag_value="brave", + default=None, + metavar="[BROWSER]", + help=( + "Extract tokens straight from a local browser's cookie store instead " + "of the manual DevTools flow. Defaults to brave when no browser is " + "named; also supports chrome, chromium, edge, firefox. The browser " + "must be installed and logged in on this machine." + ), +) @click.pass_context -def auth(ctx: click.Context) -> None: +def auth(ctx: click.Context, from_browser: str | None) -> None: """Interactive setup wizard for session tokens. Guides you through finding and saving your ChatGPT and Claude session - tokens. Tokens are never echoed to the terminal. + tokens. Tokens are never echoed to the terminal. With --from-browser, + tokens are pulled from the browser's cookie store and written to .env + without any copy-pasting. Token lifetimes: ChatGPT (__Secure-next-auth.session-token): ~7 days (JWT) Claude (sessionKey): ~30 days (opaque string) """ + if from_browser: + _auth_from_browser(from_browser.lower()) + return + os_name = platform.system() console.print("\n[bold cyan]AI Chat Exporter — Token Setup Wizard[/bold cyan]\n") @@ -279,38 +300,117 @@ def _auth_claude(os_name: str) -> None: _write_token_to_env("CLAUDE_SESSION_KEY", key) +def _auth_from_browser(browser: str) -> None: + """Extract ChatGPT + Claude tokens from a local browser and write .env. + + Each provider is validated against its live API before anything is + written, so a stale cookie (logged out since last visit) never replaces + a working token. Exits 1 only when nothing could be configured. + """ + from src.browser_tokens import ( + BrowserTokenError, + SUPPORTED_BROWSERS, + extract_chatgpt_tokens, + extract_claude_session, + ) + + if browser not in SUPPORTED_BROWSERS: + err_console.print( + f"[red]Unsupported browser '{browser}'. " + f"Supported: {', '.join(SUPPORTED_BROWSERS)}[/red]" + ) + sys.exit(1) + + console.print(f"\n[bold cyan]Extracting session tokens from {browser}…[/bold cyan]\n") + configured = 0 + + # ── ChatGPT ────────────────────────────────────────────────────────── + try: + token, token_1 = extract_chatgpt_tokens(browser) + with console.status("[dim]Validating ChatGPT token…[/dim]"): + from src.providers.chatgpt import ChatGPTProvider + prov = ChatGPTProvider(session_token=token, session_token_1=token_1) + prov._fetch_access_token() + _set_env_key("CHATGPT_SESSION_TOKEN", token) + _set_env_key("CHATGPT_SESSION_TOKEN_1", token_1 or "") + console.print("[green]ChatGPT: token extracted, validated, and saved.[/green]") + configured += 1 + except BrowserTokenError as e: + console.print(f"[yellow]ChatGPT: {e}[/yellow]") + except ProviderError as e: + console.print( + f"[yellow]ChatGPT: extracted cookie failed live validation " + f"({e.original}) — .env not changed. Log in to chatgpt.com in " + f"{browser} and retry.[/yellow]" + ) + + # ── Claude ─────────────────────────────────────────────────────────── + try: + session_key = extract_claude_session(browser) + with console.status("[dim]Validating Claude session key…[/dim]"): + from src.providers.claude import ClaudeProvider + prov = ClaudeProvider(session_key) + prov.list_conversations(offset=0, limit=1) + _set_env_key("CLAUDE_SESSION_KEY", session_key) + console.print("[green]Claude: session key extracted, validated, and saved.[/green]") + configured += 1 + except BrowserTokenError as e: + console.print(f"[yellow]Claude: {e}[/yellow]") + except ProviderError as e: + console.print( + f"[yellow]Claude: extracted cookie failed live validation " + f"({e.original}) — .env not changed. Log in to claude.ai in " + f"{browser} and retry.[/yellow]" + ) + + if configured: + console.print( + f"\n[green]Done — {configured} provider(s) configured. " + "Run 'ai-chat-exporter doctor' to verify.[/green]" + ) + else: + err_console.print( + "\n[red]No tokens could be extracted. Use the manual wizard " + "instead: ai-chat-exporter auth[/red]" + ) + sys.exit(1) + + def _write_token_to_env(key: str, value: str) -> None: """Write or update a key in .env, offering to create the file if it doesn't exist.""" - env_path = Path(".env") - if click.confirm(f"Write {key} to .env?", default=True): - if not env_path.exists(): - # Create from example if available - example = Path(".env.example") - if example.exists(): - import shutil as _shutil - _shutil.copy2(example, env_path) - console.print("[dim]Created .env from .env.example[/dim]") - else: - env_path.touch() + _set_env_key(key, value) - lines = env_path.read_text(encoding="utf-8").splitlines(keepends=True) - updated = False - new_lines = [] - for line in lines: - if line.startswith(f"{key}=") or line.startswith(f"{key} ="): - new_lines.append(f"{key}={value}\n") - updated = True - else: - new_lines.append(line) - if not updated: - new_lines.append(f"\n{key}={value}\n") +def _set_env_key(key: str, value: str) -> None: + """Write or update a key in .env without prompting (creates .env if needed).""" + env_path = Path(".env") + if not env_path.exists(): + # Create from example if available + example = Path(".env.example") + if example.exists(): + import shutil as _shutil + _shutil.copy2(example, env_path) + console.print("[dim]Created .env from .env.example[/dim]") + else: + env_path.touch() - env_path.write_text("".join(new_lines), encoding="utf-8") - import os - os.chmod(env_path, 0o600) - console.print(f"[green]{key} written to .env (permissions: 600)[/green]") + lines = env_path.read_text(encoding="utf-8").splitlines(keepends=True) + updated = False + new_lines = [] + for line in lines: + if line.startswith(f"{key}=") or line.startswith(f"{key} ="): + new_lines.append(f"{key}={value}\n") + updated = True + else: + new_lines.append(line) + + if not updated: + new_lines.append(f"\n{key}={value}\n") + + env_path.write_text("".join(new_lines), encoding="utf-8") + os.chmod(env_path, 0o600) + console.print(f"[green]{key} written to .env (permissions: 600)[/green]") # ────────────────────────────────────────────────────────────────────────────── @@ -326,15 +426,19 @@ def doctor(ctx: click.Context) -> None: Checks token presence, format, expiry, directory permissions, disk space, and live API reachability. Exits with code 1 if any checks fail. """ - checks = _run_doctor_checks() + checks = _run_doctor_checks(cache=ctx.obj["cache"]) _print_doctor_table(checks) if any(not c["pass"] for c in checks): sys.exit(1) -def _run_doctor_checks() -> list[dict]: - """Run all doctor checks and return results.""" +def _run_doctor_checks(cache: Cache | None = None) -> list[dict]: + """Run all doctor checks and return results. + + ``cache``: when provided, also verify manifest ↔ disk integrity (every + ``file_path`` recorded in the manifest exists on disk). + """ import os import jwt as pyjwt from datetime import timezone @@ -405,6 +509,18 @@ def _run_doctor_checks() -> list[dict]: except OSError as e: add("Disk space check", False, str(e)) + # Manifest ↔ disk integrity + if cache is not None: + referenced = cache.all_file_paths() + if referenced: + missing = sorted( + p for p in referenced if not Path(p).expanduser().exists() + ) + detail = f"{len(referenced) - len(missing)}/{len(referenced)} manifest files present" + if missing: + detail += f"; first missing: {missing[0]} — re-export to fix" + add("Manifest files on disk", not missing, detail) + # API reachability if chatgpt_token: try: @@ -453,7 +569,7 @@ def _print_doctor_table(checks: list[dict]) -> None: @cli.command() @click.option( "--provider", - type=click.Choice(["chatgpt", "claude", "all"], case_sensitive=False), + type=click.Choice(["chatgpt", "claude", "claude-code", "all"], case_sensitive=False), default="all", show_default=True, help="Which provider to export.", @@ -487,6 +603,37 @@ def _print_doctor_table(checks: list[dict]) -> None: "Use 'none' for conversations outside any project." ), ) +@click.option( + "--hidden-content", + type=click.Choice(["full", "placeholder", "omit"], case_sensitive=False), + default=None, + help=( + "What to do with content that was invisible in the provider's web UI " + "(file-retrieval tool dumps, hidden context): keep in full, collapse to " + "a one-line placeholder, or omit. Overrides EXPORTER_HIDDEN_CONTENT " + "(default: placeholder)." + ), +) +@click.option( + "--max-conversations", + type=click.IntRange(min=1), + default=None, + help=( + "Cap how many conversations are downloaded this run (per provider). " + "Runs are resumable, so re-running continues where the cap stopped. " + "Overrides MAX_CONVERSATIONS_PER_RUN (default: unlimited)." + ), +) +@click.option( + "--download-media", + type=click.Choice(["images", "all", "off"], case_sensitive=False), + default=None, + help=( + "Download conversation assets next to the exports: images only, " + "all (also audio/files), or off. Overrides EXPORTER_DOWNLOAD_MEDIA " + "(default: images)." + ), +) @click.option("--dry-run", is_flag=True, help="Show what would be exported without writing anything.") @click.pass_context def export( @@ -496,6 +643,9 @@ def export( output_dir: str | None, since: str | None, project_filter: str | None, + hidden_content: str | None, + max_conversations: int | None, + download_media: str | None, dry_run: bool, ) -> None: """Export new and updated conversations to Markdown or JSON. @@ -507,6 +657,13 @@ def export( debug = ctx.obj.get("debug", False) cache: Cache = ctx.obj["cache"] + # CLI flags win over .env: set before load_config (load_dotenv uses + # override=False, so an existing env var is not clobbered by .env). + if hidden_content: + os.environ["EXPORTER_HIDDEN_CONTENT"] = hidden_content.lower() + if download_media: + os.environ["EXPORTER_DOWNLOAD_MEDIA"] = download_media.lower() + # Load config (may raise ConfigError) try: from src.config import load_config @@ -517,7 +674,7 @@ def export( # First-run: auto-doctor if not cache.last_run(): console.print("[dim]First run — checking configuration…[/dim]") - checks = _run_doctor_checks() + checks = _run_doctor_checks(cache=cache) _print_doctor_table(checks) if any(not c["pass"] for c in checks): err_console.print( @@ -537,6 +694,9 @@ def export( err_console.print(f"[red]Invalid --since date: '{since}'. Use YYYY-MM-DD.[/red]") sys.exit(1) + # Session cap: CLI flag wins over MAX_CONVERSATIONS_PER_RUN env default. + session_cap = max_conversations if max_conversations is not None else cfg.max_conversations + # Determine which providers to run providers_to_run = _resolve_providers(provider, cfg) if not providers_to_run: @@ -580,15 +740,34 @@ def export( skipped = len(all_convs) - len(to_export) summary[prov_name]["skipped"] = skipped + # Apply the session cap. Resumability makes this safe: the manifest + # records each conversation immediately, so the next run picks up + # exactly where the cap stopped. + deferred = 0 + if session_cap is not None and len(to_export) > session_cap: + deferred = len(to_export) - session_cap + to_export = to_export[:session_cap] + summary[prov_name]["deferred"] = deferred + if dry_run: _print_dry_run_table(prov_name, to_export, prov_instance, export_base, structure, skipped) + if deferred: + console.print( + f" [yellow]Session cap {session_cap}: {deferred} more pending beyond this run.[/yellow]" + ) continue if not to_export: console.print(f" [dim]{skipped} conversations already up to date.[/dim]") continue - console.print(f" [dim]{len(to_export)} to export, {skipped} already up to date.[/dim]") + if deferred: + console.print( + f" [dim]{len(to_export)} to export (session cap; {deferred} deferred), " + f"{skipped} already up to date.[/dim]" + ) + else: + console.print(f" [dim]{len(to_export)} to export, {skipped} already up to date.[/dim]") from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskProgressColumn @@ -617,6 +796,19 @@ def export( full_raw[key] = val normalized = prov_instance.normalize_conversation(full_raw, loss_report) + # Download assets (images by default) so the renderers + # can inline them. Failures keep placeholders; never fatal. + if cfg.download_media != "off": + from src.media import resolve_media + resolve_media( + normalized, + prov_instance, + export_base, + structure, + cfg.download_media, + loss_report, + ) + exported_path: Path | None = None if md_exporter: exported_path = md_exporter.export(normalized) @@ -647,6 +839,12 @@ def export( if not dry_run: _print_export_summary(summary) + total_deferred = sum(s.get("deferred", 0) for s in summary.values()) + if total_deferred: + console.print( + f"[yellow]{total_deferred} conversation(s) deferred by the session cap " + f"({session_cap} per provider per run). Re-run the same command to continue.[/yellow]" + ) # Emit the data-loss summary at INFO level so it lands in the log file # AND the operator's console (default level is INFO). for line in loss_report.format_summary().split("\n"): @@ -683,6 +881,7 @@ def _resolve_providers(provider: str, cfg) -> list[tuple[str, object]]: session_token=cfg.chatgpt_session_token, session_token_1=cfg.chatgpt_session_token_1, project_ids=cfg.chatgpt_project_ids, + hidden_content=cfg.hidden_content, ), )) except ProviderError as e: @@ -694,6 +893,19 @@ def _resolve_providers(provider: str, cfg) -> list[tuple[str, object]]: if provider in ("claude", "all"): try_add("claude", cfg.claude_session_key, ClaudeProvider) + if provider in ("claude-code", "all"): + from src.providers.claude_code import ClaudeCodeProvider, DEFAULT_PROJECTS_DIR + cc_dir = Path(os.getenv("CLAUDE_CODE_DIR", DEFAULT_PROJECTS_DIR)).expanduser() + if cc_dir.is_dir(): + result.append(( + "claude-code", + ClaudeCodeProvider(projects_dir=cc_dir, hidden_content=cfg.hidden_content), + )) + elif provider == "claude-code": + logging.getLogger(__name__).warning( + "[claude-code] Skipping — %s not found (set CLAUDE_CODE_DIR).", cc_dir + ) + return result @@ -790,7 +1002,7 @@ def _print_export_summary(summary: dict[str, dict[str, int]]) -> None: @cli.command(name="list") @click.option( "--provider", - type=click.Choice(["chatgpt", "claude", "all"], case_sensitive=False), + type=click.Choice(["chatgpt", "claude", "claude-code", "all"], case_sensitive=False), default="all", show_default=True, ) @@ -856,7 +1068,7 @@ def list_conversations(ctx: click.Context, provider: str, project_filter: str | @click.option("--clear", is_flag=True, help="Clear cached entries.") @click.option( "--provider", - type=click.Choice(["chatgpt", "claude", "all"], case_sensitive=False), + type=click.Choice(["chatgpt", "claude", "claude-code", "all"], case_sensitive=False), default="all", help="Provider to target (used with --clear).", ) @@ -886,6 +1098,106 @@ def cache(ctx: click.Context, show: bool, clear: bool, provider: str) -> None: console.print("Specify --show or --clear. Use --help for options.") +# ────────────────────────────────────────────────────────────────────────────── +# prune command +# ────────────────────────────────────────────────────────────────────────────── + + +@cli.command() +@click.option("--dry-run", is_flag=True, help="List stale files without deleting anything.") +@click.option("--yes", "-y", is_flag=True, help="Delete without a confirmation prompt.") +@click.pass_context +def prune(ctx: click.Context, dry_run: bool, yes: bool) -> None: + """Delete export files no longer referenced by the cache manifest. + + Removes leftovers from old folder layouts and orphaned files from fixed + bugs (e.g. pre-v0.5.0 nested year folders, files with a missing + conversation ID). Only .md and .json files under EXPORT_DIR are + considered; the manifest is the source of truth. + """ + cache_obj: Cache = ctx.obj["cache"] + + from dotenv import load_dotenv + load_dotenv(override=False) + export_dir = Path(os.getenv("EXPORT_DIR", "./exports")).expanduser() + + if not export_dir.exists(): + console.print(f"[yellow]Export directory {export_dir} does not exist — nothing to prune.[/yellow]") + return + + referenced_raw = cache_obj.all_file_paths() + if not referenced_raw: + # Footgun guard: with an empty manifest (e.g. right after `cache + # --clear`) every export file would look stale and prune would wipe + # the entire archive. + err_console.print( + "[red]The manifest references no export files (did you just run " + "'cache --clear'?). Refusing to prune — run an export first.[/red]" + ) + sys.exit(1) + + referenced = {Path(p).expanduser().resolve() for p in referenced_raw} + on_disk = sorted( + f for f in export_dir.rglob("*") + if f.is_file() and f.suffix in (".md", ".json") + ) + stale = [f for f in on_disk if f.resolve() not in referenced] + + if not stale: + console.print( + f"[green]Nothing to prune — all {len(on_disk)} export files are referenced " + "by the manifest.[/green]" + ) + return + + total_bytes = sum(f.stat().st_size for f in stale) + console.print( + f"[bold]{len(stale)} stale file(s)[/bold] " + f"({total_bytes / (1024 * 1024):.1f} MB) not referenced by the manifest " + f"(of {len(on_disk)} total):\n" + ) + for f in stale: + try: + shown = f.relative_to(export_dir.resolve()) if f.is_absolute() else f + except ValueError: + shown = f + console.print(f" [dim]{shown}[/dim]") + + if dry_run: + console.print("\n[yellow]Dry run — nothing deleted.[/yellow]") + return + + if not yes and not click.confirm(f"\nDelete these {len(stale)} files?", default=False): + console.print("Aborted — nothing deleted.") + return + + deleted = 0 + for f in stale: + try: + f.unlink() + deleted += 1 + except OSError as e: + logger.error("[prune] Could not delete %s: %s", f, e) + + # Sweep now-empty directories (deepest first). + removed_dirs = 0 + for d in sorted( + (d for d in export_dir.rglob("*") if d.is_dir()), + key=lambda p: len(p.parts), + reverse=True, + ): + try: + d.rmdir() # only succeeds when empty + removed_dirs += 1 + except OSError: + pass + + console.print( + f"[green]Deleted {deleted} file(s) ({total_bytes / (1024 * 1024):.1f} MB) " + f"and {removed_dirs} empty director(ies).[/green]" + ) + + # ────────────────────────────────────────────────────────────────────────────── # joplin command # ────────────────────────────────────────────────────────────────────────────── @@ -894,7 +1206,7 @@ def cache(ctx: click.Context, show: bool, clear: bool, provider: str) -> None: @cli.command() @click.option( "--provider", - type=click.Choice(["chatgpt", "claude", "all"], case_sensitive=False), + type=click.Choice(["chatgpt", "claude", "claude-code", "all"], case_sensitive=False), default="all", show_default=True, help="Which provider's conversations to sync to Joplin.", @@ -967,10 +1279,9 @@ def joplin(ctx: click.Context, provider: str, project_filter: str | None, dry_ru # Determine which providers to process providers_to_sync: list[str] = [] - if provider in ("chatgpt", "all"): - providers_to_sync.append("chatgpt") - if provider in ("claude", "all"): - providers_to_sync.append("claude") + for prov in ("chatgpt", "claude", "claude-code"): + if provider in (prov, "all"): + providers_to_sync.append(prov) summary: dict[str, dict[str, int]] = {} @@ -1042,6 +1353,17 @@ def joplin(ctx: click.Context, provider: str, project_filter: str | None, dry_ru body = Path(file_path).read_text(encoding="utf-8") logger.debug("[joplin] Read %d chars from %s", len(body), file_path) + # Upload any downloaded media as Joplin resources and + # rewrite local media/ links to :/resourceId references. + # The cache stores the resource map so re-syncs reuse IDs. + from src.joplin import upload_media_and_rewrite + res_map = cache_obj.get_joplin_resources(prov_name, conv_id) + body, new_res_map = upload_media_and_rewrite( + body, Path(file_path).parent, client, res_map + ) + if new_res_map != res_map: + cache_obj.set_joplin_resources(prov_name, conv_id, new_res_map) + # Get or create the nested notebook nb_path = notebook_path(prov_name, project) notebook_id = client.get_or_create_notebook_path(list(nb_path)) diff --git a/src/media.py b/src/media.py new file mode 100644 index 0000000..bb41f88 --- /dev/null +++ b/src/media.py @@ -0,0 +1,202 @@ +"""Media resolution — download conversation assets next to the export files. + +Walks a normalized conversation's placeholder blocks, downloads each asset via +the provider, saves it under a ``media/`` directory beside the conversation's +Markdown file, and annotates the block with a relative ``local_path`` so the +renderer emits a real inline image / file link instead of a placeholder. + +Policy (``EXPORTER_DOWNLOAD_MEDIA``): + images (default) — image_placeholder blocks only + all — also file_placeholder blocks (voice-mode audio etc.; + measured 2026-06-12: 556 clips ≈ 162MB whose transcripts + are already in the exports as text) + off — leave placeholders untouched + +Failures (expired assets, network) keep the existing placeholder and are +counted in the run summary — never fatal to the export. +""" + +import logging +import os +import tempfile +from pathlib import Path + +from src.blocks import BLOCK_TYPE_FILE_PLACEHOLDER, BLOCK_TYPE_IMAGE_PLACEHOLDER +from src.loss_report import LossReport +from src.providers.base import ProviderError +from src.utils import build_export_path, generate_filename + +logger = logging.getLogger(__name__) + +MEDIA_IMAGES = "images" +MEDIA_ALL = "all" +MEDIA_OFF = "off" +VALID_MEDIA_POLICIES = {MEDIA_IMAGES, MEDIA_ALL, MEDIA_OFF} + +_EXT_BY_MIME = { + "image/png": ".png", + "image/jpeg": ".jpg", + "image/gif": ".gif", + "image/webp": ".webp", + "audio/wav": ".wav", + "audio/x-wav": ".wav", + "audio/mpeg": ".mp3", + "audio/mp4": ".m4a", + "audio/aac": ".aac", + "application/pdf": ".pdf", +} + + +def resolve_media_policy() -> str: + """Read EXPORTER_DOWNLOAD_MEDIA from the environment, defaulting to images.""" + value = os.getenv("EXPORTER_DOWNLOAD_MEDIA", "").strip().lower() + if not value: + return MEDIA_IMAGES + if value not in VALID_MEDIA_POLICIES: + logger.warning( + "EXPORTER_DOWNLOAD_MEDIA=%r is invalid (expected images|all|off) " + "— using 'images'.", + value, + ) + return MEDIA_IMAGES + return value + + +def resolve_media( + normalized: dict, + provider, + export_base: Path, + structure: str, + policy: str, + report: LossReport, +) -> int: + """Download assets for one normalized conversation. Returns download count. + + Mutates placeholder blocks in place (adds ``local_path``). Idempotent: + an asset already on disk is annotated without an API call. + """ + if policy == MEDIA_OFF: + return 0 + + download = getattr(provider, "download_asset", None) + if download is None: + # Provider has no remote assets (e.g. claude-code local transcripts). + return 0 + + wanted_types = {BLOCK_TYPE_IMAGE_PLACEHOLDER} + if policy == MEDIA_ALL: + wanted_types.add(BLOCK_TYPE_FILE_PLACEHOLDER) + + targets = [ + block + for message in normalized.get("messages", []) + for block in message.get("blocks", []) + if block.get("type") in wanted_types and block.get("ref") + ] + if not targets: + return 0 + + # Same path computation the exporter uses, so media/ lands beside the .md. + filename = generate_filename( + normalized.get("title", "Untitled"), + normalized.get("id", ""), + normalized.get("created_at") or "2000-01-01", + ) + conv_dir = build_export_path( + export_base, + normalized.get("provider", ""), + normalized.get("project"), + normalized.get("created_at") or "2000-01-01", + filename, + structure, + ).parent + media_dir = conv_dir / "media" + + downloaded = 0 + for block in targets: + ref = block["ref"] + file_id = _safe_asset_name(provider, ref) + if not file_id: + report.record_media_failed("unparseable-ref") + continue + + existing = _find_existing(media_dir, file_id) + if existing is not None: + block["local_path"] = f"media/{existing.name}" + continue + + try: + content, mime, file_name = download(ref) + except ProviderError as e: + logger.warning( + "[media] Could not download %s: %s", ref[:60], e.original + ) + reason = "expired-or-missing" if "404" in str(e.original) or "not found" in str( + e.original + ).lower() else "download-error" + report.record_media_failed(reason) + continue + + ext = _pick_extension(mime, file_name) + target = media_dir / f"{file_id}{ext}" + try: + _write_atomic(target, content) + except OSError as e: + logger.error("[media] Could not write %s: %s", target, e) + report.record_media_failed("write-error") + continue + + block["local_path"] = f"media/{target.name}" + report.record_media_downloaded(len(content)) + downloaded += 1 + + return downloaded + + +def _safe_asset_name(provider, ref: str) -> str | None: + """A stable, filesystem-safe name for the asset (the provider file ID).""" + parser = getattr(provider, "parse_asset_file_id", None) + if parser is None: + from src.providers.chatgpt import parse_asset_file_id as parser + file_id = parser(ref) + if not file_id: + return None + return "".join(c if c.isalnum() or c in "-_." else "_" for c in file_id) + + +def _find_existing(media_dir: Path, file_id: str) -> Path | None: + """Return an already-downloaded file for this asset, if present.""" + if not media_dir.is_dir(): + return None + for candidate in media_dir.glob(f"{file_id}.*"): + if candidate.is_file() and candidate.stat().st_size > 0: + return candidate + return None + + +def _pick_extension(mime: str | None, file_name: str | None) -> str: + if mime: + base_mime = mime.split(";")[0].strip().lower() + if base_mime in _EXT_BY_MIME: + return _EXT_BY_MIME[base_mime] + if file_name: + suffix = Path(file_name).suffix + if suffix and len(suffix) <= 8: + return suffix.lower() + return ".bin" + + +def _write_atomic(target: Path, content: bytes) -> None: + target.parent.mkdir(parents=True, exist_ok=True) + fd, tmp_name = tempfile.mkstemp(dir=target.parent, suffix=".tmp") + try: + with os.fdopen(fd, "wb") as fh: + fh.write(content) + os.chmod(tmp_name, 0o600) + os.replace(tmp_name, target) + except OSError: + try: + os.unlink(tmp_name) + except OSError: + pass + raise diff --git a/src/providers/base.py b/src/providers/base.py index b229fee..4bacf6d 100644 --- a/src/providers/base.py +++ b/src/providers/base.py @@ -1,6 +1,7 @@ """Abstract base class for AI chat providers.""" import logging +import os import random import time from abc import ABC, abstractmethod @@ -37,6 +38,62 @@ MAX_RETRIES = 3 BACKOFF_BASE = 2.0 BACKOFF_MAX = 60.0 +# EXPORTER_HIDDEN_CONTENT policy — what to do with content that was invisible +# in the provider's UI (retrieval dumps, hidden-flagged context, agent tool +# traffic). Shared by all providers. +HIDDEN_CONTENT_FULL = "full" +HIDDEN_CONTENT_PLACEHOLDER = "placeholder" +HIDDEN_CONTENT_OMIT = "omit" +VALID_HIDDEN_CONTENT_POLICIES = { + HIDDEN_CONTENT_FULL, + HIDDEN_CONTENT_PLACEHOLDER, + HIDDEN_CONTENT_OMIT, +} + + +def resolve_hidden_content_policy() -> str: + """Read EXPORTER_HIDDEN_CONTENT from the environment, defaulting to placeholder.""" + value = os.getenv("EXPORTER_HIDDEN_CONTENT", "").strip().lower() + if not value: + return HIDDEN_CONTENT_PLACEHOLDER + if value not in VALID_HIDDEN_CONTENT_POLICIES: + logger.warning( + "EXPORTER_HIDDEN_CONTENT=%r is invalid (expected full|placeholder|omit) " + "— using 'placeholder'.", + value, + ) + return HIDDEN_CONTENT_PLACEHOLDER + return value + + +# Default polite pacing between consecutive API requests (seconds). Keeps +# traffic human-paced instead of bursty. REQUEST_DELAY=0 disables. +DEFAULT_REQUEST_DELAY = 1.0 + + +def resolve_request_delay() -> float: + """Read REQUEST_DELAY from the environment, defaulting to DEFAULT_REQUEST_DELAY.""" + raw = os.getenv("REQUEST_DELAY", "").strip() + if not raw: + return DEFAULT_REQUEST_DELAY + try: + value = float(raw) + except ValueError: + logger.warning( + "REQUEST_DELAY=%r is not a number — using default %.1fs.", + raw, + DEFAULT_REQUEST_DELAY, + ) + return DEFAULT_REQUEST_DELAY + if value < 0: + logger.warning( + "REQUEST_DELAY=%r is negative — using default %.1fs.", + raw, + DEFAULT_REQUEST_DELAY, + ) + return DEFAULT_REQUEST_DELAY + return value + # Realistic Chrome User-Agent USER_AGENT = ( "Mozilla/5.0 (X11; Linux x86_64) " @@ -76,6 +133,9 @@ class BaseProvider(ABC): "Accept-Language": "en-US,en;q=0.9", } ) + # Polite pacing between consecutive requests (REQUEST_DELAY env). + self._request_delay = resolve_request_delay() + self._last_request_at: float | None = None # ------------------------------------------------------------------ # Abstract interface — subclasses must implement these @@ -103,6 +163,23 @@ class BaseProvider(ABC): # Concrete helpers # ------------------------------------------------------------------ + def _pace(self) -> None: + """Sleep so consecutive requests are at least REQUEST_DELAY apart (±25% jitter). + + getattr fallback: tests construct providers via ``__new__``, skipping + ``__init__`` — those run unpaced. + """ + delay = getattr(self, "_request_delay", 0) + if delay <= 0: + return + target = delay * random.uniform(0.75, 1.25) + last = getattr(self, "_last_request_at", None) + if last is not None: + wait = target - (time.monotonic() - last) + if wait > 0: + time.sleep(wait) + self._last_request_at = time.monotonic() + def fetch_all_conversations(self, since: datetime | None = None) -> list[dict]: """Fetch every conversation, handling pagination automatically. @@ -208,6 +285,7 @@ class BaseProvider(ABC): while attempt <= MAX_RETRIES: attempt += 1 + self._pace() start = time.monotonic() try: diff --git a/src/providers/chatgpt.py b/src/providers/chatgpt.py index 31d1022..02dd409 100644 --- a/src/providers/chatgpt.py +++ b/src/providers/chatgpt.py @@ -19,6 +19,7 @@ Response: {"items": [...], "cursor": ""} Pagination ends when cursor is null or an empty string. """ +import json import logging import os from typing import Any @@ -26,10 +27,13 @@ from typing import Any from curl_cffi import requests as curl_requests from src.blocks import ( + COLLAPSED_KIND_HIDDEN_CONTEXT, + COLLAPSED_KIND_TOOL_DUMP, UNKNOWN_REASON_EXTRACTION_FAILED, UNKNOWN_REASON_UNKNOWN_FIELD_IN_KNOWN_TYPE, UNKNOWN_REASON_UNKNOWN_TYPE, make_code_block, + make_collapsed_block, make_file_placeholder, make_hidden_context_marker, make_image_placeholder, @@ -39,7 +43,16 @@ from src.blocks import ( make_unknown_block, ) from src.loss_report import LossReport -from src.providers.base import BaseProvider, ProviderError, REQUEST_TIMEOUT +from src.providers.base import ( + BaseProvider, + HIDDEN_CONTENT_FULL, + HIDDEN_CONTENT_OMIT, + HIDDEN_CONTENT_PLACEHOLDER, + ProviderError, + REQUEST_TIMEOUT, + VALID_HIDDEN_CONTENT_POLICIES, + resolve_hidden_content_policy, +) logger = logging.getLogger(__name__) @@ -50,6 +63,34 @@ AUTH_SESSION_URL = "https://chatgpt.com/api/auth/session" # Run: python -c "from curl_cffi.requests import BrowserType; print(list(BrowserType))" IMPERSONATE = "chrome120" +def parse_asset_file_id(ref: str) -> str | None: + """Extract the file ID from a ChatGPT asset pointer. + + Observed forms (2026-06-12): + sediment://file_00000000245c71fda50034d7f1647791 → file_… + sediment://8456107fc383a53#file_…979c…#p_6.png (generated) → file_… + file-service://file-AbCdEf → file-AbCdEf + """ + if not isinstance(ref, str) or not ref: + return None + if ref.startswith("file-service://"): + return ref.removeprefix("file-service://") or None + if ref.startswith("sediment://"): + body = ref.removeprefix("sediment://") + for part in body.split("#"): + if part.startswith("file_") or part.startswith("file-"): + return part + return body or None + return None + + +# Tool-role authors whose messages are retrieval dumps — the full text of +# attached/project files re-injected by ChatGPT on every tool run. Measured +# 2026-06-12 across this user's three largest conversations: 86% of all +# content bytes, not flagged is_visually_hidden_from_conversation. +# myfiles_browser is the legacy name for the same retrieval tool. +_COLLAPSE_TOOL_AUTHORS = {"file_search", "myfiles_browser"} + class ChatGPTProvider(BaseProvider): """Provider for ChatGPT conversations via the internal web API. @@ -72,6 +113,7 @@ class ChatGPTProvider(BaseProvider): session_token: str | None = None, session_token_1: str | None = None, project_ids: list[str] | None = None, + hidden_content: str | None = None, ) -> None: # Pass a curl_cffi session to the base class instead of a requests.Session. # curl_cffi.requests.Session is API-compatible with requests.Session. @@ -112,6 +154,14 @@ class ChatGPTProvider(BaseProvider): # Cache of project_id → display name (avoids re-fetching gizmo details) self._project_name_cache: dict[str, str] = {} + # Policy for messages invisible in the web UI (retrieval dumps, + # hidden-flagged context): full | placeholder | omit. + self._hidden_content = ( + hidden_content + if hidden_content in VALID_HIDDEN_CONTENT_POLICIES + else resolve_hidden_content_policy() + ) + # ChatGPT now splits large session cookies into .0 / .1 chunks. # Always send both named chunks; the server reassembles them. self._session.cookies.set( @@ -561,6 +611,59 @@ class ChatGPTProvider(BaseProvider): ) return data + # ------------------------------------------------------------------ + # Asset downloads + # ------------------------------------------------------------------ + + def download_asset(self, ref: str) -> tuple[bytes, str | None, str | None]: + """Download a sediment:// / file-service:// asset. + + Two hops (verified live 2026-06-12): the files endpoint returns a + signed ``download_url``; fetching that yields the bytes. Expired + assets (e.g. old generated images) 404 on the first hop. + + Returns: + (content_bytes, mime_type_or_None, file_name_or_None) + + Raises: + ProviderError: unparseable ref, expired/missing asset, or + download failure. + """ + file_id = parse_asset_file_id(ref) + if not file_id: + raise ProviderError( + self.provider_name, + f"download_asset({ref[:40]})", + ValueError(f"Unrecognised asset reference: {ref[:80]}"), + ) + + meta = self._make_request("GET", f"{BASE_URL}/files/{file_id}/download") + download_url = meta.get("download_url") + if not download_url: + raise ProviderError( + self.provider_name, + f"download_asset({file_id})", + RuntimeError(f"No download_url in response: {meta.get('detail') or meta}"), + ) + + self._pace() + resp = self._session.request("GET", download_url, timeout=REQUEST_TIMEOUT) + if resp.status_code != 200: + raise ProviderError( + self.provider_name, + f"download_asset({file_id})", + RuntimeError(f"Signed URL returned HTTP {resp.status_code}"), + ) + mime = resp.headers.get("content-type") or None + file_name = meta.get("file_name") or None + logger.debug( + "[chatgpt] Downloaded asset %s: %d bytes (%s)", + file_id, + len(resp.content), + mime, + ) + return resp.content, mime, file_name + # ------------------------------------------------------------------ # Normalization # ------------------------------------------------------------------ @@ -598,7 +701,9 @@ class ChatGPTProvider(BaseProvider): ) mapping: dict = raw.get("mapping", {}) - messages = _extract_messages(mapping, raw, conv_id, report) + # getattr fallback: tests construct providers via __new__, skipping __init__. + policy = getattr(self, "_hidden_content", None) or resolve_hidden_content_policy() + messages = _extract_messages(mapping, raw, conv_id, report, policy) for _ in messages: report.record_message() report.record_conversation() @@ -631,7 +736,11 @@ def _ts_to_iso(ts: float | int | str | None) -> str: def _extract_messages( - mapping: dict[str, Any], raw: dict, conv_id: str, report: LossReport + mapping: dict[str, Any], + raw: dict, + conv_id: str, + report: LossReport, + policy: str = HIDDEN_CONTENT_FULL, ) -> list[dict]: """Walk the ChatGPT conversation mapping tree to produce an ordered message list. @@ -661,7 +770,7 @@ def _extract_messages( node = mapping.get(node_id, {}) msg_data = node.get("message") if msg_data: - built = _build_message(msg_data, conv_id, node_id, report) + built = _build_message(msg_data, conv_id, node_id, report, policy) if built is not None: messages.append(built) @@ -688,12 +797,17 @@ def _find_root(mapping: dict[str, Any]) -> str | None: def _build_message( - msg_data: dict, conv_id: str, node_id: str, report: LossReport + msg_data: dict, + conv_id: str, + node_id: str, + report: LossReport, + policy: str = HIDDEN_CONTENT_FULL, ) -> dict | None: """Construct a normalized message dict (with ``blocks``) for one ChatGPT node. - Returns None for messages that should be skipped (truly empty). Otherwise - returns a dict with ``role``, ``content_type``, ``timestamp``, ``blocks``. + Returns None for messages that should be skipped (truly empty, or omitted + by the EXPORTER_HIDDEN_CONTENT policy). Otherwise returns a dict with + ``role``, ``content_type``, ``timestamp``, ``blocks``. """ author = msg_data.get("author") or {} role = author.get("role", "") or "" @@ -727,9 +841,43 @@ def _build_message( ) return None - if is_hidden: - # Prepend a marker so the reader knows this message is hidden in the - # source UI. The marker is content-type-agnostic. + # Messages invisible in the web UI: tool retrieval dumps (identified by + # author.name — they are NOT hidden-flagged) and hidden-flagged context. + collapse_kind: str | None = None + if role == "tool" and author_name in _COLLAPSE_TOOL_AUTHORS: + collapse_kind = COLLAPSED_KIND_TOOL_DUMP + elif is_hidden: + collapse_kind = COLLAPSED_KIND_HIDDEN_CONTEXT + + if collapse_kind is not None and policy != HIDDEN_CONTENT_FULL: + origin = ( + author_name if collapse_kind == COLLAPSED_KIND_TOOL_DUMP else content_type + ) or "?" + size_bytes = len(json.dumps(content_obj, ensure_ascii=False, default=str)) + report.record_collapsed(origin, size_bytes) + logger.debug( + "[chatgpt] %s %s message (%s, %dB) in conversation %s " + "per EXPORTER_HIDDEN_CONTENT=%s", + "Omitted" if policy == HIDDEN_CONTENT_OMIT else "Collapsed", + collapse_kind, + origin, + size_bytes, + conv_id[:8], + policy, + ) + if policy == HIDDEN_CONTENT_OMIT: + return None + blocks = [ + make_collapsed_block( + origin=origin, + content_type=content_type, + size_bytes=size_bytes, + kind=collapse_kind, + ) + ] + elif is_hidden: + # Policy 'full': prepend a marker so the reader knows this message is + # hidden in the source UI. The marker is content-type-agnostic. blocks = [make_hidden_context_marker(content_type)] + blocks # Vestigial content_type: "code" for code-only messages, otherwise "text" diff --git a/src/providers/claude_code.py b/src/providers/claude_code.py new file mode 100644 index 0000000..d12ce76 --- /dev/null +++ b/src/providers/claude_code.py @@ -0,0 +1,476 @@ +"""Claude Code session provider — archives local agent transcripts. + +Reads JSONL session files from ``~/.claude/projects//.jsonl`` +(override with ``CLAUDE_CODE_DIR``). No tokens, no rate limits, no ToS risk — +but the data lives in single files Claude Code may clean up, and it contains +deliverables (reviews, plans, analyses) that exist nowhere else. + +Rendering follows the EXPORTER_HIDDEN_CONTENT policy (decided 2026-06-12): +prose-only by default. User prompts and assistant text are kept; tool_use / +tool_result traffic is collapsed to one grouped placeholder per activity run +(measured: dialogue prose is ~4% of session bytes); thinking blocks are +dropped (counted in the run summary, no placeholder). ``full`` keeps +everything. + +Record types in a session file: ``user`` / ``assistant`` carry the dialogue +(Anthropic-style ``message.content`` block arrays); ``ai-title`` carries the +evolving session title (last one wins); ``last-prompt``, +``file-history-snapshot``, ``attachment``, ``permission-mode``, ``system`` +are harness records and are skipped. Records flagged ``isSidechain`` are +subagent transcripts; ``isMeta`` are harness-generated user records — both +skipped. +""" + +import json +import logging +import os +import re +from collections import Counter +from datetime import datetime, timezone +from pathlib import Path + +from src.blocks import ( + COLLAPSED_KIND_TOOL_DUMP, + UNKNOWN_REASON_UNKNOWN_TYPE, + make_collapsed_block, + make_image_placeholder, + make_text_block, + make_thinking_block, + make_tool_result_block, + make_tool_use_block, + make_unknown_block, +) +from src.loss_report import LossReport +from src.providers.base import ( + BaseProvider, + HIDDEN_CONTENT_FULL, + ProviderError, + VALID_HIDDEN_CONTENT_POLICIES, + resolve_hidden_content_policy, +) + +logger = logging.getLogger(__name__) + +DEFAULT_PROJECTS_DIR = "~/.claude/projects" + +# Harness-injected tags inside user message text. Stripped so exports contain +# the dialogue, not the CLI plumbing. A record that is nothing but tags +# (e.g. a /model invocation) ends up empty and is skipped. +_HARNESS_TAG_RE = re.compile( + r".*?" + r"|.*?" + r"|.*?" + r"|.*?" + r"|.*?" + r"|.*?" + r"|.*?", + re.DOTALL, +) + +# How many distinct tool names to list in a collapsed-activity placeholder. +_TOOL_NAMES_SHOWN = 4 + + +def _strip_harness_noise(text: str) -> str: + if not isinstance(text, str): + return "" + return _HARNESS_TAG_RE.sub("", text).strip() + + +class ClaudeCodeProvider(BaseProvider): + """Local-file provider over Claude Code session transcripts.""" + + provider_name = "claude-code" + + def __init__( + self, + projects_dir: str | Path | None = None, + hidden_content: str | None = None, + ) -> None: + super().__init__() + self._projects_dir = Path( + projects_dir or os.getenv("CLAUDE_CODE_DIR", DEFAULT_PROJECTS_DIR) + ).expanduser() + self._hidden_content = ( + hidden_content + if hidden_content in VALID_HIDDEN_CONTENT_POLICIES + else resolve_hidden_content_policy() + ) + # conv_id → session file path, populated by _scan() + self._path_map: dict[str, Path] = {} + + # ------------------------------------------------------------------ + # BaseProvider interface + # ------------------------------------------------------------------ + + def list_conversations(self, offset: int = 0, limit: int = 100) -> list[dict]: + full = self._scan() + return full[offset : offset + limit] + + def fetch_all_conversations(self, since: datetime | None = None) -> list[dict]: + convs = self._scan() + if since is not None: + since_aware = since if since.tzinfo else since.replace(tzinfo=timezone.utc) + convs = [ + c for c in convs + if datetime.fromisoformat(c["updated_at"]) >= since_aware + ] + logger.info( + "[claude-code] Found %d session(s) under %s", len(convs), self._projects_dir + ) + return convs + + def get_conversation(self, conv_id: str) -> dict: + path = self._path_map.get(conv_id) + if path is None: + # Direct call without a prior listing (e.g. tests) — scan first. + self._scan() + path = self._path_map.get(conv_id) + if path is None or not path.exists(): + raise ProviderError( + self.provider_name, + f"get_conversation({conv_id[:8]})", + FileNotFoundError(f"No session file for id {conv_id}"), + ) + + records: list[dict] = [] + bad_lines = 0 + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + try: + records.append(json.loads(line)) + except json.JSONDecodeError: + bad_lines += 1 + if bad_lines: + logger.warning( + "[claude-code] %s: skipped %d unparseable line(s)", path.name, bad_lines + ) + + mtime = datetime.fromtimestamp(path.stat().st_mtime, tz=timezone.utc) + return { + "id": conv_id, + "_path": str(path), + "_records": records, + # Listing and normalized updated_at must match, or the cache + # staleness comparison would re-export every session every run. + "_mtime_iso": mtime.isoformat(), + } + + def normalize_conversation(self, raw: dict, loss_report: LossReport | None = None) -> dict: + report = loss_report if loss_report is not None else LossReport() + policy = getattr(self, "_hidden_content", None) or resolve_hidden_content_policy() + conv_id = raw.get("id") or "" + records: list[dict] = raw.get("_records") or [] + + title = _extract_title(records) + project = _extract_project(records, raw.get("_path")) + created_at = next( + (r.get("timestamp") for r in records if r.get("timestamp")), "" + ) + updated_at = raw.get("_mtime_iso") or next( + (r.get("timestamp") for r in reversed(records) if r.get("timestamp")), "" + ) + + messages = _extract_messages(records, conv_id, report, policy) + for _ in messages: + report.record_message() + report.record_conversation() + + return { + "id": conv_id, + "title": title, + "provider": self.provider_name, + "project": project, + "created_at": created_at or "", + "updated_at": updated_at or "", + "message_count": len(messages), + "messages": messages, + } + + # ------------------------------------------------------------------ + # Scanning + # ------------------------------------------------------------------ + + def _scan(self) -> list[dict]: + if not self._projects_dir.is_dir(): + logger.warning( + "[claude-code] Projects directory %s does not exist", self._projects_dir + ) + return [] + + convs: list[dict] = [] + for proj_dir in sorted(p for p in self._projects_dir.iterdir() if p.is_dir()): + for session_file in sorted(proj_dir.glob("*.jsonl")): + try: + stat = session_file.stat() + except OSError: + continue + if stat.st_size == 0: + continue + conv_id = session_file.stem + self._path_map[conv_id] = session_file + title, project, created = _read_session_meta(session_file) + convs.append( + { + "id": conv_id, + "title": title, + "project": project, + # The --project filter and dry-run table read the + # listing dict, not the normalized conversation. + "_project_name": project, + "created_at": created, + "updated_at": datetime.fromtimestamp( + stat.st_mtime, tz=timezone.utc + ).isoformat(), + "_path": str(session_file), + "_project_dir": proj_dir.name, + } + ) + return convs + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + + +def _read_session_meta(path: Path) -> tuple[str, str | None, str]: + """Light single-pass scan for listing metadata: (title, project, created_at). + + Substring guards keep this cheap — only candidate lines are JSON-parsed. + The full-fidelity extraction happens later in normalize_conversation. + """ + title = "" + project: str | None = None + created = "" + try: + with path.open(encoding="utf-8") as fh: + for line in fh: + if '"ai-title"' in line: + try: + rec = json.loads(line) + except json.JSONDecodeError: + continue + if rec.get("type") == "ai-title" and rec.get("aiTitle"): + title = str(rec["aiTitle"]) # last one wins + continue + if (not created or project is None) and ( + '"timestamp"' in line or '"cwd"' in line + ): + try: + rec = json.loads(line) + except json.JSONDecodeError: + continue + if not created and rec.get("timestamp"): + created = str(rec["timestamp"]) + if project is None and rec.get("cwd"): + project = Path(rec["cwd"]).name or None + except OSError as e: + logger.warning("[claude-code] Could not read %s: %s", path, e) + return title, project, created + + +def _extract_title(records: list[dict]) -> str: + """Last ai-title record wins; fall back to the first real user prompt.""" + title = "" + for rec in records: + if rec.get("type") == "ai-title" and rec.get("aiTitle"): + title = str(rec["aiTitle"]) + if title: + return title + + for rec in records: + if rec.get("type") != "user" or rec.get("isMeta") or rec.get("isSidechain"): + continue + content = (rec.get("message") or {}).get("content") + if isinstance(content, str): + text = _strip_harness_noise(content) + elif isinstance(content, list): + text = " ".join( + _strip_harness_noise(item.get("text", "")) + for item in content + if isinstance(item, dict) and item.get("type") == "text" + ).strip() + else: + text = "" + if text: + return text[:80] + return "Untitled session" + + +def _extract_project(records: list[dict], path: str | None) -> str | None: + """Project = basename of the session's working directory.""" + for rec in records: + cwd = rec.get("cwd") + if cwd: + name = Path(cwd).name + if name: + return name + # Fallback: the munged directory name (cannot be reliably de-munged + # because '-' is both the path separator and a legal name character). + if path: + return Path(path).parent.name.lstrip("-") or None + return None + + +def _extract_messages( + records: list[dict], conv_id: str, report: LossReport, policy: str +) -> list[dict]: + messages: list[dict] = [] + # Pending collapsed tool activity: name → call count, plus total bytes. + pending_tools: Counter = Counter() + pending_bytes = 0 + + def flush_pending() -> None: + nonlocal pending_bytes + if not pending_tools: + return + shown = ", ".join( + f"{name} ×{count}" for name, count in pending_tools.most_common(_TOOL_NAMES_SHOWN) + ) + if len(pending_tools) > _TOOL_NAMES_SHOWN: + shown += ", …" + calls = sum(pending_tools.values()) + block = make_collapsed_block( + origin=f"{calls} calls: {shown}", + content_type="tool_activity", + size_bytes=pending_bytes, + kind=COLLAPSED_KIND_TOOL_DUMP, + ) + if messages and messages[-1]["role"] == "assistant": + messages[-1]["blocks"].append(block) + else: + messages.append( + {"role": "tool", "content_type": "text", "timestamp": None, "blocks": [block]} + ) + pending_tools.clear() + pending_bytes = 0 + + for rec in records: + if rec.get("type") not in ("user", "assistant"): + continue + if rec.get("isSidechain") or rec.get("isMeta"): + continue + + msg = rec.get("message") or {} + role = msg.get("role") or rec["type"] + content = msg.get("content") + blocks: list[dict] = [] + # This record's own tool traffic — merged into pending only after the + # record's message is appended, so the placeholder lands on it (not on + # the previous message). + local_tools: Counter = Counter() + local_bytes = 0 + + if isinstance(content, str): + text = _strip_harness_noise(content) + block = make_text_block(text) + if block: + blocks.append(block) + elif isinstance(content, list): + for item in content: + if not isinstance(item, dict): + continue + item_type = item.get("type", "") + if item_type == "text": + block = make_text_block(_strip_harness_noise(item.get("text", ""))) + if block: + blocks.append(block) + elif item_type in ("thinking", "redacted_thinking"): + if policy == HIDDEN_CONTENT_FULL: + block = make_thinking_block( + item.get("thinking") or item.get("text") or "" + ) + if block: + blocks.append(block) + else: + # Decision 2026-06-12: thinking is dropped without a + # placeholder, but stays visible in the run summary. + report.record_collapsed( + "thinking", len(json.dumps(item, default=str)) + ) + elif item_type == "tool_use": + if policy == HIDDEN_CONTENT_FULL: + blocks.append( + make_tool_use_block( + item.get("name", ""), item.get("input"), item.get("id") + ) + ) + else: + name = item.get("name") or "tool" + size = len(json.dumps(item, default=str)) + local_tools[name] += 1 + local_bytes += size + report.record_collapsed(name, size) + elif item_type == "tool_result": + if policy == HIDDEN_CONTENT_FULL: + blocks.append( + make_tool_result_block( + _stringify_tool_result(item.get("content")), + is_error=bool(item.get("is_error")), + ) + ) + else: + size = len(json.dumps(item, default=str)) + local_bytes += size + report.record_collapsed("tool_result", size) + elif item_type == "image": + blocks.append( + make_image_placeholder(ref="embedded image", source="user_upload") + ) + else: + logger.warning( + "[claude-code] Unknown content block type %r in session %s", + item_type, + conv_id[:8], + ) + report.record_unknown(f"claude-code.{item_type or '?'}") + blocks.append( + make_unknown_block( + raw_type=f"claude-code.{item_type or '?'}", + observed_keys=list(item.keys()), + reason=UNKNOWN_REASON_UNKNOWN_TYPE, + ) + ) + + if not blocks: + # Tool-result-only records (and similar): traffic accumulates and + # is attached to the message that initiated it. + pending_tools.update(local_tools) + pending_bytes += local_bytes + continue + + # Attach previous records' tool activity to the previous message + # before starting a new one, so the placeholder lands between the + # dialogue turns it actually occurred between. + flush_pending() + messages.append( + { + "role": role, + "content_type": "text", + "timestamp": rec.get("timestamp"), + "blocks": blocks, + } + ) + pending_tools.update(local_tools) + pending_bytes += local_bytes + + flush_pending() + return messages + + +def _stringify_tool_result(content) -> str: + """tool_result content may be a string or a list of text blocks.""" + if isinstance(content, str): + return content + if isinstance(content, list): + parts = [] + for item in content: + if isinstance(item, dict) and item.get("type") == "text": + parts.append(item.get("text", "")) + else: + parts.append(json.dumps(item, default=str)) + return "\n".join(parts) + return json.dumps(content, default=str) if content is not None else "" diff --git a/tests/test_browser_tokens.py b/tests/test_browser_tokens.py new file mode 100644 index 0000000..7f2e7ef --- /dev/null +++ b/tests/test_browser_tokens.py @@ -0,0 +1,155 @@ +"""Unit tests for browser cookie extraction (browser_cookie3 mocked).""" + +import sys +import types +from types import SimpleNamespace + +import pytest + +from src.browser_tokens import ( + BrowserTokenError, + extract_chatgpt_tokens, + extract_claude_session, +) + + +def _fake_bc3(monkeypatch, cookies_by_domain, raises=None): + """Install a stub browser_cookie3 module whose ``brave`` loader returns + SimpleNamespace cookies for the requested domain (or raises).""" + module = types.ModuleType("browser_cookie3") + + def brave(domain_name=""): + if raises is not None: + raise raises + return [ + SimpleNamespace(name=name, value=value) + for name, value in cookies_by_domain.get(domain_name, {}).items() + ] + + module.brave = brave + monkeypatch.setitem(sys.modules, "browser_cookie3", module) + return module + + +class TestExtractChatGPT: + def test_chunked_cookies(self, monkeypatch): + _fake_bc3(monkeypatch, { + "chatgpt.com": { + "__Secure-next-auth.session-token.0": "eyJpart0", + "__Secure-next-auth.session-token.1": "part1rest", + }, + }) + assert extract_chatgpt_tokens("brave") == ("eyJpart0", "part1rest") + + def test_single_cookie_fallback(self, monkeypatch): + _fake_bc3(monkeypatch, { + "chatgpt.com": {"__Secure-next-auth.session-token": "eyJsingle"}, + }) + assert extract_chatgpt_tokens("brave") == ("eyJsingle", None) + + def test_missing_cookie_raises_with_login_hint(self, monkeypatch): + _fake_bc3(monkeypatch, {"chatgpt.com": {"unrelated": "x"}}) + with pytest.raises(BrowserTokenError, match="log in"): + extract_chatgpt_tokens("brave") + + def test_loader_failure_wrapped(self, monkeypatch): + _fake_bc3(monkeypatch, {}, raises=RuntimeError("database is locked")) + with pytest.raises(BrowserTokenError, match="Close the browser"): + extract_chatgpt_tokens("brave") + + def test_unsupported_browser_rejected(self): + with pytest.raises(BrowserTokenError, match="Unsupported browser"): + extract_chatgpt_tokens("netscape") + + +class TestExtractClaude: + def test_session_key(self, monkeypatch): + _fake_bc3(monkeypatch, {"claude.ai": {"sessionKey": "sk-ant-sid01-xyz"}}) + assert extract_claude_session("brave") == "sk-ant-sid01-xyz" + + def test_missing_raises(self, monkeypatch): + _fake_bc3(monkeypatch, {"claude.ai": {}}) + with pytest.raises(BrowserTokenError, match="log in"): + extract_claude_session("brave") + + +class TestAuthFromBrowserCLI: + def test_writes_env_after_validation(self, monkeypatch, tmp_path): + """--from-browser extracts, validates live (mocked), and writes .env.""" + from click.testing import CliRunner + from src.main import cli + from src.cache import Cache + + cache = Cache(tmp_path / "cache") + cache.acknowledge_tos() + + _fake_bc3(monkeypatch, { + "chatgpt.com": { + "__Secure-next-auth.session-token.0": "eyJtok0", + "__Secure-next-auth.session-token.1": "tok1", + }, + "claude.ai": {"sessionKey": "sk-ant-sid01-abc"}, + }) + + # Stub out the live validation calls. + import src.providers.chatgpt as chatgpt_mod + import src.providers.claude as claude_mod + monkeypatch.setattr( + chatgpt_mod.ChatGPTProvider, "_fetch_access_token", lambda self: "at" + ) + monkeypatch.setattr( + claude_mod.ClaudeProvider, + "list_conversations", + lambda self, offset=0, limit=100: [], + ) + + runner = CliRunner(mix_stderr=True) + with runner.isolated_filesystem(temp_dir=tmp_path): + result = runner.invoke( + cli, + ["--no-log-file", "auth", "--from-browser"], + env={"CACHE_DIR": str(tmp_path / "cache")}, + ) + assert result.exit_code == 0, result.output + from pathlib import Path + env_text = Path(".env").read_text() + + assert "CHATGPT_SESSION_TOKEN=eyJtok0" in env_text + assert "CHATGPT_SESSION_TOKEN_1=tok1" in env_text + assert "CLAUDE_SESSION_KEY=sk-ant-sid01-abc" in env_text + assert "2 provider(s) configured" in result.output + + def test_validation_failure_leaves_env_untouched(self, monkeypatch, tmp_path): + from click.testing import CliRunner + from src.main import cli + from src.cache import Cache + from src.providers.base import ProviderError + + cache = Cache(tmp_path / "cache") + cache.acknowledge_tos() + + _fake_bc3(monkeypatch, { + "chatgpt.com": {"__Secure-next-auth.session-token.0": "eyJstale"}, + "claude.ai": {}, + }) + + import src.providers.chatgpt as chatgpt_mod + + def _fail(self): + raise ProviderError("chatgpt", "auth", RuntimeError("401")) + + monkeypatch.setattr(chatgpt_mod.ChatGPTProvider, "_fetch_access_token", _fail) + + runner = CliRunner(mix_stderr=True) + with runner.isolated_filesystem(temp_dir=tmp_path): + result = runner.invoke( + cli, + ["--no-log-file", "auth", "--from-browser", "brave"], + env={"CACHE_DIR": str(tmp_path / "cache")}, + ) + import pathlib + env_exists = pathlib.Path(".env").exists() + + assert result.exit_code == 1 + assert "failed live validation" in result.output + assert not env_exists diff --git a/tests/test_claude_code.py b/tests/test_claude_code.py new file mode 100644 index 0000000..798cfca --- /dev/null +++ b/tests/test_claude_code.py @@ -0,0 +1,230 @@ +"""Unit tests for the Claude Code session provider.""" + +import json + +import pytest + +from src.blocks import ( + BLOCK_TYPE_COLLAPSED, + BLOCK_TYPE_TEXT, + BLOCK_TYPE_THINKING, + BLOCK_TYPE_TOOL_RESULT, + BLOCK_TYPE_TOOL_USE, + render_blocks_to_markdown, +) +from src.loss_report import LossReport +from src.providers.claude_code import ClaudeCodeProvider + + +def _write_session(tmp_path, records, project_dir="-home-jesse-myproj", name="abc-123"): + proj = tmp_path / project_dir + proj.mkdir(parents=True, exist_ok=True) + f = proj / f"{name}.jsonl" + f.write_text("\n".join(json.dumps(r) for r in records), encoding="utf-8") + return f + + +def _records(): + """A representative session: title, harness noise, dialogue, tool traffic.""" + return [ + {"type": "ai-title", "aiTitle": "First title", "sessionId": "abc-123"}, + { + "type": "user", + "cwd": "/home/jesse/myproj", + "timestamp": "2026-05-01T10:00:00.000Z", + "message": { + "role": "user", + "content": ( + "Caveat: ignore" + "/model" + "Set model" + "Please review my project." + ), + }, + }, + { + "type": "assistant", + "timestamp": "2026-05-01T10:00:05.000Z", + "message": { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "Let me think about this."}, + {"type": "text", "text": "I'll review it now."}, + {"type": "tool_use", "id": "t1", "name": "Read", "input": {"file_path": "/x"}}, + {"type": "tool_use", "id": "t2", "name": "Read", "input": {"file_path": "/y"}}, + {"type": "tool_use", "id": "t3", "name": "Bash", "input": {"command": "ls"}}, + ], + }, + }, + { + "type": "user", + "timestamp": "2026-05-01T10:00:08.000Z", + "message": { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": "file contents " * 50}, + ], + }, + }, + { + "type": "assistant", + "timestamp": "2026-05-01T10:00:20.000Z", + "message": { + "role": "assistant", + "content": [{"type": "text", "text": "Here is my review: all good."}], + }, + }, + # Harness records — skipped + {"type": "file-history-snapshot", "snapshot": {"big": "blob"}}, + {"type": "permission-mode", "mode": "plan"}, + # Subagent and meta records — skipped + { + "type": "assistant", + "isSidechain": True, + "message": {"role": "assistant", "content": [{"type": "text", "text": "subagent noise"}]}, + }, + { + "type": "user", + "isMeta": True, + "message": {"role": "user", "content": "meta noise"}, + }, + # Later title wins + {"type": "ai-title", "aiTitle": "Review my project", "sessionId": "abc-123"}, + ] + + +class TestClaudeCodeProvider: + def _provider(self, tmp_path, policy="placeholder"): + return ClaudeCodeProvider(projects_dir=tmp_path, hidden_content=policy) + + def test_listing_metadata(self, tmp_path): + _write_session(tmp_path, _records()) + p = self._provider(tmp_path) + convs = p.fetch_all_conversations() + assert len(convs) == 1 + c = convs[0] + assert c["id"] == "abc-123" + assert c["title"] == "Review my project" # last ai-title wins + assert c["_project_name"] == "myproj" # from cwd basename + assert c["created_at"] == "2026-05-01T10:00:00.000Z" + assert c["updated_at"] # file mtime + + def test_empty_files_skipped(self, tmp_path): + proj = tmp_path / "-home-x" + proj.mkdir() + (proj / "empty.jsonl").write_text("") + p = self._provider(tmp_path) + assert p.fetch_all_conversations() == [] + + def test_normalize_prose_only_default(self, tmp_path): + _write_session(tmp_path, _records()) + p = self._provider(tmp_path) + p.fetch_all_conversations() + raw = p.get_conversation("abc-123") + result = p.normalize_conversation(raw) + + assert result["provider"] == "claude-code" + assert result["title"] == "Review my project" + assert result["project"] == "myproj" + + roles = [m["role"] for m in result["messages"]] + assert roles == ["user", "assistant", "assistant"] + + # Harness tags stripped, dialogue kept + user_text = result["messages"][0]["blocks"][0]["text"] + assert user_text == "Please review my project." + assert "Caveat" not in user_text + + # Tool activity collapsed into one grouped placeholder on the + # assistant message that ran the tools (includes the tool_result + # bytes from the following user record). + first_assistant = result["messages"][1] + types = [b["type"] for b in first_assistant["blocks"]] + assert types == [BLOCK_TYPE_TEXT, BLOCK_TYPE_COLLAPSED] + collapsed = first_assistant["blocks"][1] + assert "3 calls" in collapsed["origin"] + assert "Read ×2" in collapsed["origin"] + assert "Bash ×1" in collapsed["origin"] + assert collapsed["size_bytes"] > 500 + + # Thinking dropped without placeholder; sidechain/meta absent + all_blocks = [b for m in result["messages"] for b in m["blocks"]] + assert not any(b["type"] == BLOCK_TYPE_THINKING for b in all_blocks) + assert not any( + "subagent noise" in (b.get("text") or "") or "meta noise" in (b.get("text") or "") + for b in all_blocks + ) + + def test_collapsed_counted_in_loss_report(self, tmp_path): + _write_session(tmp_path, _records()) + p = self._provider(tmp_path) + p.fetch_all_conversations() + report = LossReport() + p.normalize_conversation(p.get_conversation("abc-123"), report) + assert report.collapsed["Read"] == 2 + assert report.collapsed["Bash"] == 1 + assert report.collapsed["tool_result"] == 1 + assert report.collapsed["thinking"] == 1 + assert report.collapsed_bytes > 500 + + def test_full_policy_keeps_everything(self, tmp_path): + _write_session(tmp_path, _records()) + p = self._provider(tmp_path, policy="full") + p.fetch_all_conversations() + result = p.normalize_conversation(p.get_conversation("abc-123")) + all_blocks = [b for m in result["messages"] for b in m["blocks"]] + types = {b["type"] for b in all_blocks} + assert BLOCK_TYPE_THINKING in types + assert BLOCK_TYPE_TOOL_USE in types + assert BLOCK_TYPE_TOOL_RESULT in types + assert BLOCK_TYPE_COLLAPSED not in types + + def test_updated_at_consistent_with_listing(self, tmp_path): + """Listing and normalized updated_at must match or the cache would + consider every session stale on every run (mtime vs last timestamp).""" + _write_session(tmp_path, _records()) + p = self._provider(tmp_path) + listing = p.fetch_all_conversations()[0] + normalized = p.normalize_conversation(p.get_conversation("abc-123")) + assert normalized["updated_at"] == listing["updated_at"] + + def test_title_falls_back_to_first_prompt(self, tmp_path): + records = [r for r in _records() if r.get("type") != "ai-title"] + _write_session(tmp_path, records) + p = self._provider(tmp_path) + p.fetch_all_conversations() + result = p.normalize_conversation(p.get_conversation("abc-123")) + assert result["title"] == "Please review my project." + + def test_unknown_block_type_surfaces(self, tmp_path): + records = [ + { + "type": "assistant", + "timestamp": "2026-05-01T10:00:00.000Z", + "cwd": "/home/jesse/myproj", + "message": { + "role": "assistant", + "content": [ + {"type": "text", "text": "hello"}, + {"type": "future_block_xyz", "data": 1}, + ], + }, + }, + ] + _write_session(tmp_path, records) + p = self._provider(tmp_path) + p.fetch_all_conversations() + report = LossReport() + result = p.normalize_conversation(p.get_conversation("abc-123"), report) + assert report.unknown_blocks["claude-code.future_block_xyz"] == 1 + rendered = render_blocks_to_markdown(result["messages"][0]["blocks"]) + assert "Unsupported content" in rendered + + def test_collapsed_placeholder_renders_grouped_line(self, tmp_path): + _write_session(tmp_path, _records()) + p = self._provider(tmp_path) + p.fetch_all_conversations() + result = p.normalize_conversation(p.get_conversation("abc-123")) + rendered = render_blocks_to_markdown(result["messages"][1]["blocks"]) + assert "> 🔧 **Tool output** — `3 calls: Read ×2, Bash ×1`" in rendered + assert "omitted (EXPORTER_HIDDEN_CONTENT=full to keep)" in rendered diff --git a/tests/test_cli.py b/tests/test_cli.py index 17065cf..bcbbdf7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -124,10 +124,30 @@ class TestExportSinceValidation: "CHATGPT_SESSION_TOKEN": "eyJtesttoken", "CACHE_DIR": str(tmp_path), "EXPORT_DIR": str(tmp_path / "exports"), + # This test hits the real (failing) auth endpoint with retries; + # don't add politeness pacing on top of the backoff sleeps. + "REQUEST_DELAY": "0", }, ) assert "Invalid --since date" not in result.output + def test_max_conversations_zero_rejected(self, tmp_path): + """--max-conversations uses IntRange(min=1); 0 must be rejected by click.""" + self._pre_populated_cache(tmp_path) + + runner = CliRunner(mix_stderr=True) + result = runner.invoke( + cli, + ["--no-log-file", "export", "--max-conversations", "0"], + env={ + "CHATGPT_SESSION_TOKEN": "eyJtesttoken", + "CACHE_DIR": str(tmp_path), + "EXPORT_DIR": str(tmp_path / "exports"), + }, + ) + assert result.exit_code == 2 + assert "max-conversations" in result.output + # --------------------------------------------------------------------------- # LossReport summary @@ -145,8 +165,9 @@ class TestLossReportSummary: assert "[export] Run summary:" in out assert "conversations: 0" in out assert "messages rendered: 0" in out - # Both "(none)" sentinels present — never empty parens - assert out.count("(none)") == 2 + # All three "(none)" sentinels present — never empty parens + # (unknown blocks, extraction failures, collapsed by policy) + assert out.count("(none)") == 3 def test_top_5_breakdown(self): from src.loss_report import LossReport @@ -174,3 +195,83 @@ class TestLossReportSummary: out = report.format_summary() assert "conversations: 1" in out assert "messages rendered: 2" in out + + +# --------------------------------------------------------------------------- +# prune command +# --------------------------------------------------------------------------- + + +class TestPrune: + """prune deletes export files not referenced by the manifest.""" + + def _setup(self, tmp_path): + """Cache with one referenced file; one stale file + empty-dir candidate.""" + cache = Cache(tmp_path / "cache") + cache.acknowledge_tos() + export_dir = tmp_path / "exports" + keep = export_dir / "chatgpt" / "proj.2026" / "keep.md" + keep.parent.mkdir(parents=True) + keep.write_text("kept") + stale = export_dir / "chatgpt" / "proj" / "2026" / "old-layout.md" + stale.parent.mkdir(parents=True) + stale.write_text("stale") + cache.mark_exported("chatgpt", "conv-1", {"file_path": str(keep)}) + return export_dir, keep, stale + + def _invoke(self, tmp_path, *args): + runner = CliRunner(mix_stderr=True) + return runner.invoke( + cli, + ["--no-log-file", "prune", *args], + env={ + "CACHE_DIR": str(tmp_path / "cache"), + "EXPORT_DIR": str(tmp_path / "exports"), + }, + ) + + def test_dry_run_lists_but_keeps_files(self, tmp_path): + export_dir, keep, stale = self._setup(tmp_path) + result = self._invoke(tmp_path, "--dry-run") + assert result.exit_code == 0 + assert "old-layout.md" in result.output + assert "Dry run" in result.output + assert stale.exists() and keep.exists() + + def test_yes_deletes_stale_keeps_referenced_sweeps_dirs(self, tmp_path): + export_dir, keep, stale = self._setup(tmp_path) + result = self._invoke(tmp_path, "--yes") + assert result.exit_code == 0 + assert not stale.exists() + assert keep.exists() + # Old-layout dirs are now empty and swept + assert not (export_dir / "chatgpt" / "proj").exists() + + def test_refuses_with_empty_manifest(self, tmp_path): + """Footgun guard: after cache --clear, prune must not wipe the archive.""" + cache = Cache(tmp_path / "cache") + cache.acknowledge_tos() + export_dir = tmp_path / "exports" + f = export_dir / "chatgpt" / "a.md" + f.parent.mkdir(parents=True) + f.write_text("data") + result = self._invoke(tmp_path, "--yes") + assert result.exit_code == 1 + assert "Refusing to prune" in result.output + assert f.exists() + + def test_aborts_without_confirmation(self, tmp_path): + export_dir, keep, stale = self._setup(tmp_path) + runner = CliRunner(mix_stderr=True) + result = runner.invoke( + cli, + ["--no-log-file", "prune"], + input="n\n", + env={ + "CACHE_DIR": str(tmp_path / "cache"), + "EXPORT_DIR": str(tmp_path / "exports"), + }, + ) + assert result.exit_code == 0 + assert "Aborted" in result.output + assert stale.exists() diff --git a/tests/test_config.py b/tests/test_config.py index e2e8098..15ac29c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -54,3 +54,65 @@ class TestValidateChatGPTToken: result = _validate_chatgpt_token("notajwttoken") assert any("does not look like a JWT" in r.message for r in caplog.records) assert result is None + + +class TestSessionLimiterConfig: + """MAX_CONVERSATIONS_PER_RUN and REQUEST_DELAY parsing in load_config.""" + + def _load(self, monkeypatch, tmp_path, **env): + from src.config import load_config + monkeypatch.setenv("EXPORT_DIR", str(tmp_path / "exports")) + monkeypatch.setenv("CACHE_DIR", str(tmp_path / "cache")) + for key in ( + "MAX_CONVERSATIONS_PER_RUN", + "REQUEST_DELAY", + "EXPORTER_HIDDEN_CONTENT", + "EXPORTER_DOWNLOAD_MEDIA", + ): + monkeypatch.delenv(key, raising=False) + for key, value in env.items(): + monkeypatch.setenv(key, value) + return load_config() + + def test_defaults(self, monkeypatch, tmp_path): + cfg = self._load(monkeypatch, tmp_path) + assert cfg.max_conversations is None + assert cfg.request_delay == 1.0 + assert cfg.hidden_content == "placeholder" + assert cfg.download_media == "images" + + def test_download_media_valid(self, monkeypatch, tmp_path): + cfg = self._load(monkeypatch, tmp_path, EXPORTER_DOWNLOAD_MEDIA="all") + assert cfg.download_media == "all" + + def test_download_media_invalid_raises(self, monkeypatch, tmp_path): + from src.config import ConfigError + with pytest.raises(ConfigError, match="EXPORTER_DOWNLOAD_MEDIA"): + self._load(monkeypatch, tmp_path, EXPORTER_DOWNLOAD_MEDIA="sometimes") + + def test_valid_values(self, monkeypatch, tmp_path): + cfg = self._load( + monkeypatch, tmp_path, + MAX_CONVERSATIONS_PER_RUN="25", REQUEST_DELAY="0.5", + ) + assert cfg.max_conversations == 25 + assert cfg.request_delay == 0.5 + + def test_zero_delay_allowed(self, monkeypatch, tmp_path): + cfg = self._load(monkeypatch, tmp_path, REQUEST_DELAY="0") + assert cfg.request_delay == 0.0 + + def test_non_integer_cap_raises(self, monkeypatch, tmp_path): + from src.config import ConfigError + with pytest.raises(ConfigError, match="MAX_CONVERSATIONS_PER_RUN"): + self._load(monkeypatch, tmp_path, MAX_CONVERSATIONS_PER_RUN="lots") + + def test_zero_cap_raises(self, monkeypatch, tmp_path): + from src.config import ConfigError + with pytest.raises(ConfigError, match="at least 1"): + self._load(monkeypatch, tmp_path, MAX_CONVERSATIONS_PER_RUN="0") + + def test_negative_delay_raises(self, monkeypatch, tmp_path): + from src.config import ConfigError + with pytest.raises(ConfigError, match="REQUEST_DELAY"): + self._load(monkeypatch, tmp_path, REQUEST_DELAY="-1") diff --git a/tests/test_media.py b/tests/test_media.py new file mode 100644 index 0000000..939fbb5 --- /dev/null +++ b/tests/test_media.py @@ -0,0 +1,248 @@ +"""Tests for media downloads and Joplin resource rewriting.""" + +from pathlib import Path + +import pytest + +from src.blocks import ( + make_file_placeholder, + make_image_placeholder, + render_blocks_to_markdown, +) +from src.loss_report import LossReport +from src.media import resolve_media, resolve_media_policy +from src.providers.base import ProviderError +from src.providers.chatgpt import parse_asset_file_id + + +# --------------------------------------------------------------------------- +# Asset reference parsing +# --------------------------------------------------------------------------- + + +class TestParseAssetFileId: + def test_plain_sediment(self): + assert parse_asset_file_id("sediment://file_00000000245c71fda5") == "file_00000000245c71fda5" + + def test_generated_image_with_hash_and_page(self): + ref = "sediment://8456107fc383a53#file_00000000979c71f685#p_6.png" + assert parse_asset_file_id(ref) == "file_00000000979c71f685" + + def test_file_service_scheme(self): + assert parse_asset_file_id("file-service://file-AbCdEf") == "file-AbCdEf" + + def test_unrecognised(self): + assert parse_asset_file_id("https://example.com/x.png") is None + assert parse_asset_file_id("") is None + assert parse_asset_file_id(None) is None + + +# --------------------------------------------------------------------------- +# Media policy +# --------------------------------------------------------------------------- + + +class TestMediaPolicy: + def test_default(self, monkeypatch): + monkeypatch.delenv("EXPORTER_DOWNLOAD_MEDIA", raising=False) + assert resolve_media_policy() == "images" + + def test_valid(self, monkeypatch): + monkeypatch.setenv("EXPORTER_DOWNLOAD_MEDIA", "all") + assert resolve_media_policy() == "all" + + def test_invalid_falls_back(self, monkeypatch, caplog): + monkeypatch.setenv("EXPORTER_DOWNLOAD_MEDIA", "bogus") + assert resolve_media_policy() == "images" + + +# --------------------------------------------------------------------------- +# resolve_media +# --------------------------------------------------------------------------- + + +class _FakeProvider: + """Minimal provider exposing download_asset + the ref parser.""" + + def __init__(self, assets=None, fail_refs=None): + self._assets = assets or {} + self._fail_refs = fail_refs or {} + self.calls = [] + + def parse_asset_file_id(self, ref): + return parse_asset_file_id(ref) + + def download_asset(self, ref): + self.calls.append(ref) + if ref in self._fail_refs: + raise ProviderError("chatgpt", "download_asset", self._fail_refs[ref]) + return self._assets[ref] # (content, mime, file_name) + + +def _conv_with(blocks): + return { + "id": "conv-1", + "title": "Has Media", + "provider": "chatgpt", + "project": None, + "created_at": "2026-05-20T00:00:00+00:00", + "messages": [{"role": "user", "blocks": blocks}], + } + + +class TestResolveMedia: + def test_downloads_image_and_inlines(self, tmp_path): + ref = "sediment://file_img1" + provider = _FakeProvider({ref: (b"\x89PNG\r\n", "image/png", "x.png")}) + block = make_image_placeholder(ref=ref, source="user_upload") + conv = _conv_with([block]) + report = LossReport() + + n = resolve_media(conv, provider, tmp_path, "provider/project/year", "images", report) + + assert n == 1 + assert report.media_downloaded == 1 + assert block["local_path"] == "media/file_img1.png" + rendered = render_blocks_to_markdown([block]) + assert rendered == "![user_upload](media/file_img1.png)" + # File written under the conversation's media/ dir + written = list(tmp_path.rglob("media/file_img1.png")) + assert written and written[0].read_bytes() == b"\x89PNG\r\n" + + def test_images_policy_skips_files(self, tmp_path): + ref = "sediment://file_audio1" + provider = _FakeProvider({ref: (b"RIFF", "audio/wav", "a.wav")}) + block = make_file_placeholder(ref=ref, mime="audio/wav", size_bytes=1000) + conv = _conv_with([block]) + report = LossReport() + + n = resolve_media(conv, provider, tmp_path, "provider/project/year", "images", report) + assert n == 0 + assert "local_path" not in block + assert provider.calls == [] + + def test_all_policy_downloads_files(self, tmp_path): + ref = "sediment://file_audio1" + provider = _FakeProvider({ref: (b"RIFFdata", "audio/wav", "a.wav")}) + block = make_file_placeholder(ref=ref, mime="audio/wav", size_bytes=8) + conv = _conv_with([block]) + report = LossReport() + + n = resolve_media(conv, provider, tmp_path, "provider/project/year", "all", report) + assert n == 1 + assert block["local_path"] == "media/file_audio1.wav" + rendered = render_blocks_to_markdown([block]) + assert "media/file_audio1.wav" in rendered and rendered.startswith("> 📎") + + def test_off_policy_noop(self, tmp_path): + provider = _FakeProvider({"sediment://file_x": (b"x", "image/png", None)}) + block = make_image_placeholder(ref="sediment://file_x", source="user_upload") + conv = _conv_with([block]) + report = LossReport() + assert resolve_media(conv, provider, tmp_path, "provider/project/year", "off", report) == 0 + assert provider.calls == [] + + def test_idempotent_uses_disk(self, tmp_path): + ref = "sediment://file_img1" + provider = _FakeProvider({ref: (b"\x89PNG", "image/png", "x.png")}) + conv = _conv_with([make_image_placeholder(ref=ref, source="user_upload")]) + report = LossReport() + resolve_media(conv, provider, tmp_path, "provider/project/year", "images", report) + assert len(provider.calls) == 1 + + # Second run, fresh blocks: file already on disk → no new API call. + conv2 = _conv_with([make_image_placeholder(ref=ref, source="user_upload")]) + resolve_media(conv2, provider, tmp_path, "provider/project/year", "images", LossReport()) + assert len(provider.calls) == 1 # unchanged + assert conv2["messages"][0]["blocks"][0]["local_path"] == "media/file_img1.png" + + def test_failure_keeps_placeholder_and_counts(self, tmp_path): + ref = "sediment://file_gone" + provider = _FakeProvider(fail_refs={ref: RuntimeError("Signed URL returned HTTP 404")}) + block = make_image_placeholder(ref=ref, source="model_generated") + conv = _conv_with([block]) + report = LossReport() + + n = resolve_media(conv, provider, tmp_path, "provider/project/year", "images", report) + assert n == 0 + assert "local_path" not in block + assert report.media_failed["expired-or-missing"] == 1 + # Still renders as a placeholder, not a broken image link + assert render_blocks_to_markdown([block]).startswith("> 🖼️") + + def test_provider_without_download_asset(self, tmp_path): + """claude-code has no remote assets — resolve_media must no-op.""" + class NoDownload: + pass + block = make_image_placeholder(ref="sediment://file_x", source="user_upload") + conv = _conv_with([block]) + assert resolve_media(conv, NoDownload(), tmp_path, "provider/project/year", "images", LossReport()) == 0 + + +# --------------------------------------------------------------------------- +# Joplin media rewriting +# --------------------------------------------------------------------------- + + +class _FakeJoplin: + def __init__(self): + self.uploaded = [] + self._n = 0 + + def create_resource(self, file_path, title=None): + self.uploaded.append(Path(file_path).name) + self._n += 1 + return f"res{self._n}" + + +class TestUploadMediaAndRewrite: + def _note(self, tmp_path, body): + media = tmp_path / "media" + media.mkdir() + (media / "file_img1.png").write_bytes(b"\x89PNG") + (media / "clip.wav").write_bytes(b"RIFF") + return body + + def test_rewrites_image_and_file_links(self, tmp_path): + from src.joplin import upload_media_and_rewrite + body = self._note( + tmp_path, + "Look: ![user_upload](media/file_img1.png)\n" + "> 📎 **File attached** — [clip.wav](media/clip.wav) (audio/wav)", + ) + client = _FakeJoplin() + new_body, res_map = upload_media_and_rewrite(body, tmp_path, client, {}) + + assert "![user_upload](:/res1)" in new_body + assert "(:/res2)" in new_body + assert "media/" not in new_body + assert res_map == {"media/file_img1.png": "res1", "media/clip.wav": "res2"} + assert len(client.uploaded) == 2 + + def test_reuses_known_resource_ids(self, tmp_path): + from src.joplin import upload_media_and_rewrite + body = self._note(tmp_path, "![x](media/file_img1.png)") + client = _FakeJoplin() + existing = {"media/file_img1.png": "existing-res"} + new_body, res_map = upload_media_and_rewrite(body, tmp_path, client, existing) + assert "(:/existing-res)" in new_body + assert client.uploaded == [] # no re-upload + assert res_map == existing + + def test_missing_file_left_as_link(self, tmp_path): + from src.joplin import upload_media_and_rewrite + (tmp_path / "media").mkdir() + body = "![x](media/gone.png)" + client = _FakeJoplin() + new_body, res_map = upload_media_and_rewrite(body, tmp_path, client, {}) + assert new_body == body + assert res_map == {} + assert client.uploaded == [] + + def test_no_media_links_untouched(self, tmp_path): + from src.joplin import upload_media_and_rewrite + body = "Just text with [a link](https://example.com) and `media/foo` in code." + client = _FakeJoplin() + new_body, res_map = upload_media_and_rewrite(body, tmp_path, client, {}) + assert new_body == body + assert res_map == {} diff --git a/tests/test_providers.py b/tests/test_providers.py index 13454b7..7b3ccc3 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -171,6 +171,7 @@ class TestChatGPTNormalization: def test_user_editable_context_emits_blocks(self): raw = json.loads((FIXTURES / "chatgpt_conversation.json").read_text()) p = self._get_provider() + p._hidden_content = "full" result = p.normalize_conversation(raw) # The user_editable_context message has user_profile + user_instructions. @@ -194,6 +195,7 @@ class TestChatGPTNormalization: raw = json.loads((FIXTURES / "chatgpt_conversation.json").read_text()) p = self._get_provider() + p._hidden_content = "full" result = p.normalize_conversation(raw) uec_msgs = [ @@ -290,6 +292,236 @@ class TestChatGPTUnknownContent: assert report.unknown_blocks["future_unknown_type_xyz"] == 1 +class TestChatGPTHiddenContentPolicy: + """EXPORTER_HIDDEN_CONTENT: collapse retrieval dumps and hidden context. + + file_search dumps re-inject attached-file text on every tool run (measured + 86% of content bytes on real data) and are NOT hidden-flagged — author.name + is the discriminator. Hidden-flagged messages (Custom Instructions) are the + secondary case. + """ + + def _get_provider(self, policy=None): + from src.providers.chatgpt import ChatGPTProvider + p = ChatGPTProvider.__new__(ChatGPTProvider) + import requests + p._session = requests.Session() + p._org_id = None + p._project_ids = [] + p._project_map = {} + p._project_name_cache = {} + if policy is not None: + p._hidden_content = policy + return p + + def _make_conv(self): + return { + "id": "test-collapse", + "title": "T", + "create_time": 1700000000.0, + "update_time": 1700000001.0, + "mapping": { + "root": {"id": "root", "message": None, "parent": None, "children": ["dump"]}, + "dump": { + "id": "dump", + "parent": "root", + "children": ["answer"], + "message": { + "id": "dump", + "author": {"role": "tool", "name": "file_search"}, + "content": { + "content_type": "multimodal_text", + "parts": ["Full text of an attached file " * 100], + }, + }, + }, + "answer": { + "id": "answer", + "parent": "dump", + "children": ["uec"], + "message": { + "id": "answer", + "author": {"role": "assistant"}, + "content": {"content_type": "text", "parts": ["The answer."]}, + }, + }, + "uec": { + "id": "uec", + "parent": "answer", + "children": [], + "message": { + "id": "uec", + "author": {"role": "user"}, + "content": { + "content_type": "user_editable_context", + "user_profile": "Preferred name: Jesse", + "user_instructions": "Be concise.", + }, + "metadata": {"is_visually_hidden_from_conversation": True}, + }, + }, + }, + } + + def test_default_policy_is_placeholder(self, monkeypatch): + from src.blocks import BLOCK_TYPE_COLLAPSED + monkeypatch.delenv("EXPORTER_HIDDEN_CONTENT", raising=False) + # No _hidden_content attribute → env fallback → placeholder default. + result = self._get_provider().normalize_conversation(self._make_conv()) + collapsed = [ + b for m in result["messages"] for b in m["blocks"] + if b.get("type") == BLOCK_TYPE_COLLAPSED + ] + assert len(collapsed) == 2 + + def test_placeholder_collapses_file_search_dump(self): + from src.blocks import BLOCK_TYPE_COLLAPSED, COLLAPSED_KIND_TOOL_DUMP + result = self._get_provider("placeholder").normalize_conversation(self._make_conv()) + tool_msgs = [m for m in result["messages"] if m["role"] == "tool"] + assert len(tool_msgs) == 1 + (block,) = tool_msgs[0]["blocks"] + assert block["type"] == BLOCK_TYPE_COLLAPSED + assert block["kind"] == COLLAPSED_KIND_TOOL_DUMP + assert block["origin"] == "file_search" + assert block["size_bytes"] > 1000 + + def test_placeholder_collapses_hidden_flagged_message(self): + from src.blocks import BLOCK_TYPE_COLLAPSED, COLLAPSED_KIND_HIDDEN_CONTEXT + result = self._get_provider("placeholder").normalize_conversation(self._make_conv()) + user_msgs = [m for m in result["messages"] if m["role"] == "user"] + assert len(user_msgs) == 1 + (block,) = user_msgs[0]["blocks"] + assert block["type"] == BLOCK_TYPE_COLLAPSED + assert block["kind"] == COLLAPSED_KIND_HIDDEN_CONTEXT + assert block["origin"] == "user_editable_context" + + def test_placeholder_keeps_normal_messages(self): + result = self._get_provider("placeholder").normalize_conversation(self._make_conv()) + assistant_msgs = [m for m in result["messages"] if m["role"] == "assistant"] + assert assistant_msgs[0]["blocks"][0]["type"] == BLOCK_TYPE_TEXT + assert assistant_msgs[0]["blocks"][0]["text"] == "The answer." + + def test_non_retrieval_tool_authors_not_collapsed(self): + """web.run / browser / python tool messages are real content — keep.""" + from src.blocks import BLOCK_TYPE_COLLAPSED + conv = self._make_conv() + conv["mapping"]["dump"]["message"]["author"]["name"] = "web.run" + conv["mapping"]["dump"]["message"]["content"] = { + "content_type": "text", + "parts": ["Search results summary."], + } + result = self._get_provider("placeholder").normalize_conversation(conv) + tool_msgs = [m for m in result["messages"] if m["role"] == "tool"] + assert tool_msgs[0]["blocks"][0]["type"] != BLOCK_TYPE_COLLAPSED + + def test_omit_drops_messages_entirely(self): + result = self._get_provider("omit").normalize_conversation(self._make_conv()) + assert [m["role"] for m in result["messages"]] == ["assistant"] + + def test_full_preserves_v040_behavior(self): + result = self._get_provider("full").normalize_conversation(self._make_conv()) + tool_msgs = [m for m in result["messages"] if m["role"] == "tool"] + assert tool_msgs[0]["blocks"][0]["type"] == BLOCK_TYPE_TEXT + user_msgs = [m for m in result["messages"] if m["role"] == "user"] + assert user_msgs[0]["blocks"][0]["type"] == BLOCK_TYPE_HIDDEN_CONTEXT_MARKER + + def test_collapsed_tallied_in_loss_report(self): + report = LossReport() + self._get_provider("placeholder").normalize_conversation(self._make_conv(), report) + assert report.collapsed["file_search"] == 1 + assert report.collapsed["user_editable_context"] == 1 + assert report.collapsed_bytes > 1000 + summary = report.format_summary() + assert "collapsed by policy: 2" in summary + assert "EXPORTER_HIDDEN_CONTENT=full to keep" in summary + + def test_collapsed_block_renders_one_line_with_size(self): + from src.blocks import BLOCK_TYPE_COLLAPSED + result = self._get_provider("placeholder").normalize_conversation(self._make_conv()) + tool_msgs = [m for m in result["messages"] if m["role"] == "tool"] + rendered = render_blocks_to_markdown(tool_msgs[0]["blocks"]) + assert rendered.startswith("> 🔧 **Tool output** — `file_search` (") + assert "KB" in rendered + assert "omitted (EXPORTER_HIDDEN_CONTENT=full to keep)" in rendered + assert "\n" not in rendered + + def test_invalid_env_value_falls_back_to_placeholder(self, monkeypatch, caplog): + from src.providers.chatgpt import resolve_hidden_content_policy + monkeypatch.setenv("EXPORTER_HIDDEN_CONTENT", "bogus") + with caplog.at_level(logging.WARNING): + assert resolve_hidden_content_policy() == "placeholder" + assert any("EXPORTER_HIDDEN_CONTENT" in r.message for r in caplog.records) + + +class TestRequestPacing: + """REQUEST_DELAY politeness pacing between consecutive API requests.""" + + def _get_provider(self, delay): + from src.providers.chatgpt import ChatGPTProvider + p = ChatGPTProvider.__new__(ChatGPTProvider) + p._request_delay = delay + p._last_request_at = None + return p + + def test_first_request_not_delayed(self, monkeypatch): + sleeps = [] + monkeypatch.setattr("src.providers.base.time.sleep", lambda s: sleeps.append(s)) + p = self._get_provider(2.0) + p._pace() + assert sleeps == [] + + def test_consecutive_requests_paced_with_jitter_window(self, monkeypatch): + sleeps = [] + monkeypatch.setattr("src.providers.base.time.sleep", lambda s: sleeps.append(s)) + monkeypatch.setattr("src.providers.base.random.uniform", lambda a, b: 1.0) + clock = {"now": 100.0} + monkeypatch.setattr("src.providers.base.time.monotonic", lambda: clock["now"]) + p = self._get_provider(2.0) + p._pace() + clock["now"] = 100.5 # only 0.5s elapsed since last request + p._pace() + assert sleeps == [pytest.approx(1.5)] + + def test_no_sleep_when_enough_time_elapsed(self, monkeypatch): + sleeps = [] + monkeypatch.setattr("src.providers.base.time.sleep", lambda s: sleeps.append(s)) + monkeypatch.setattr("src.providers.base.random.uniform", lambda a, b: 1.0) + clock = {"now": 100.0} + monkeypatch.setattr("src.providers.base.time.monotonic", lambda: clock["now"]) + p = self._get_provider(1.0) + p._pace() + clock["now"] = 105.0 + p._pace() + assert sleeps == [] + + def test_zero_delay_disables_pacing(self, monkeypatch): + sleeps = [] + monkeypatch.setattr("src.providers.base.time.sleep", lambda s: sleeps.append(s)) + p = self._get_provider(0) + p._pace() + p._pace() + assert sleeps == [] + + def test_pace_noop_without_init_attrs(self): + """Providers built via __new__ in tests have no pacing attrs — must not crash.""" + from src.providers.chatgpt import ChatGPTProvider + p = ChatGPTProvider.__new__(ChatGPTProvider) + p._pace() # no exception + + def test_resolve_request_delay_default_and_overrides(self, monkeypatch, caplog): + from src.providers.base import resolve_request_delay, DEFAULT_REQUEST_DELAY + monkeypatch.delenv("REQUEST_DELAY", raising=False) + assert resolve_request_delay() == DEFAULT_REQUEST_DELAY + monkeypatch.setenv("REQUEST_DELAY", "2.5") + assert resolve_request_delay() == 2.5 + monkeypatch.setenv("REQUEST_DELAY", "0") + assert resolve_request_delay() == 0.0 + monkeypatch.setenv("REQUEST_DELAY", "abc") + with caplog.at_level(logging.WARNING): + assert resolve_request_delay() == DEFAULT_REQUEST_DELAY + assert any("REQUEST_DELAY" in r.message for r in caplog.records) + + # --------------------------------------------------------------------------- # Claude # ---------------------------------------------------------------------------