Compare commits
18
Commits
304cf4fde4
..
v0.8.0
+45
-5
@@ -6,9 +6,12 @@
|
|||||||
|
|
||||||
# --- ChatGPT ---
|
# --- ChatGPT ---
|
||||||
# How to get: open chatgpt.com in Chrome → F12 → Application tab
|
# How to get: open chatgpt.com in Chrome → F12 → Application tab
|
||||||
# → Cookies → https://chatgpt.com → find "__Secure-next-auth.session-token" → copy Value
|
# → Cookies → https://chatgpt.com → find the two cookie chunks:
|
||||||
# Token type: JWT (starts with "eyJ"). Typically valid for ~7 days.
|
# __Secure-next-auth.session-token.0 (starts with "eyJ") → CHATGPT_SESSION_TOKEN
|
||||||
|
# __Secure-next-auth.session-token.1 (the remainder) → CHATGPT_SESSION_TOKEN_1
|
||||||
|
# Token type: JWE. Typically valid for ~7 days.
|
||||||
CHATGPT_SESSION_TOKEN=
|
CHATGPT_SESSION_TOKEN=
|
||||||
|
CHATGPT_SESSION_TOKEN_1=
|
||||||
|
|
||||||
# ChatGPT Projects (optional): comma-separated list of project gizmo IDs.
|
# ChatGPT Projects (optional): comma-separated list of project gizmo IDs.
|
||||||
# Project conversations are NOT included in the default /conversations listing.
|
# Project conversations are NOT included in the default /conversations listing.
|
||||||
@@ -23,6 +26,18 @@ CHATGPT_PROJECT_IDS=
|
|||||||
# Token type: opaque string. Typically valid for ~30 days.
|
# Token type: opaque string. Typically valid for ~30 days.
|
||||||
CLAUDE_SESSION_KEY=
|
CLAUDE_SESSION_KEY=
|
||||||
|
|
||||||
|
# --- Claude Code (local agent sessions) ---
|
||||||
|
# The claude-code provider reads local Claude Code transcripts. By default it
|
||||||
|
# scans ~/.claude/projects/ (plus $CLAUDE_CONFIG_DIR/projects when that is set).
|
||||||
|
# To scan additional roots — e.g. other machines' sessions copied onto this box —
|
||||||
|
# set a ':'-separated list of projects roots. Sessions are merged by folder.
|
||||||
|
#CLAUDE_CODE_DIR=~/.claude/projects:/mnt/backup/laptop/.claude/projects
|
||||||
|
#
|
||||||
|
# Session titles are tagged with the git repos they touched, e.g.
|
||||||
|
# "Resume StartWRT work [start-technologies]". To never tag specific repos,
|
||||||
|
# list their names here (comma-separated).
|
||||||
|
#CLAUDE_CODE_REPO_TAG_IGNORE=some-repo,another-repo
|
||||||
|
|
||||||
# --- Output ---
|
# --- Output ---
|
||||||
# Where exported Markdown files are written (default: ./exports)
|
# Where exported Markdown files are written (default: ./exports)
|
||||||
EXPORT_DIR=./exports
|
EXPORT_DIR=./exports
|
||||||
@@ -33,6 +48,31 @@ EXPORT_DIR=./exports
|
|||||||
# provider/year → exports/claude/2024/file.md (ignores projects)
|
# provider/year → exports/claude/2024/file.md (ignores projects)
|
||||||
OUTPUT_STRUCTURE=provider/project/year
|
OUTPUT_STRUCTURE=provider/project/year
|
||||||
|
|
||||||
|
# What to do with content that was invisible in the provider's web UI
|
||||||
|
# (file-retrieval tool dumps, hidden context like Custom Instructions).
|
||||||
|
# These dumps can be 90% of a conversation's bytes. Options:
|
||||||
|
# placeholder (default) → one-line placeholder with tool name and size
|
||||||
|
# full → keep everything (pre-v0.6.0 behavior)
|
||||||
|
# omit → drop entirely (still counted in the run summary)
|
||||||
|
EXPORTER_HIDDEN_CONTENT=placeholder
|
||||||
|
|
||||||
|
# Download conversation assets (images, audio) next to the Markdown, into a
|
||||||
|
# media/ folder, and inline them. Options:
|
||||||
|
# images (default) → images only
|
||||||
|
# all → also audio/voice clips and other files
|
||||||
|
# off → keep text placeholders, download nothing
|
||||||
|
# Downloaded media is uploaded to Joplin as resources on the next `joplin` run.
|
||||||
|
EXPORTER_DOWNLOAD_MEDIA=images
|
||||||
|
|
||||||
|
# Cap how many conversations are downloaded per export run (per provider).
|
||||||
|
# Runs are resumable — a capped run continues where it stopped next time.
|
||||||
|
# Keeps big backfills from looking like scraper traffic. Unset = unlimited.
|
||||||
|
#MAX_CONVERSATIONS_PER_RUN=25
|
||||||
|
|
||||||
|
# Seconds between consecutive API requests (small random jitter is added).
|
||||||
|
# Default 1.0; set 0 to disable pacing.
|
||||||
|
#REQUEST_DELAY=1.0
|
||||||
|
|
||||||
# --- Joplin ---
|
# --- Joplin ---
|
||||||
# Automate importing exported conversations into Joplin as notes.
|
# Automate importing exported conversations into Joplin as notes.
|
||||||
# Requires Joplin desktop running with the Web Clipper service enabled.
|
# Requires Joplin desktop running with the Web Clipper service enabled.
|
||||||
@@ -46,9 +86,9 @@ JOPLIN_API_URL=http://localhost:41184
|
|||||||
# JOPLIN_REQUEST_TIMEOUT=30
|
# JOPLIN_REQUEST_TIMEOUT=30
|
||||||
|
|
||||||
# --- Cache ---
|
# --- Cache ---
|
||||||
# Where the sync manifest and logs are stored (default: ~/.ai-chat-exporter)
|
# Where the sync manifest is stored (default: ./cache, inside the install directory)
|
||||||
CACHE_DIR=~/.ai-chat-exporter
|
CACHE_DIR=./cache
|
||||||
|
|
||||||
# --- Logging ---
|
# --- Logging ---
|
||||||
# Log file path. Set to "none" to disable file logging.
|
# Log file path. Set to "none" to disable file logging.
|
||||||
LOG_FILE=~/.ai-chat-exporter/logs/exporter.log
|
LOG_FILE=./cache/logs/exporter.log
|
||||||
|
|||||||
@@ -25,10 +25,14 @@ exports/
|
|||||||
!CHANGELOG.md
|
!CHANGELOG.md
|
||||||
|
|
||||||
# Cache and logs
|
# Cache and logs
|
||||||
|
cache/
|
||||||
.ai-chat-exporter/
|
.ai-chat-exporter/
|
||||||
logs/
|
logs/
|
||||||
*.log
|
*.log
|
||||||
|
|
||||||
|
# Test tracking
|
||||||
|
test-plan.csv
|
||||||
|
|
||||||
# Editor / OS
|
# Editor / OS
|
||||||
.DS_Store
|
.DS_Store
|
||||||
.idea/
|
.idea/
|
||||||
|
|||||||
+94
-14
@@ -3,19 +3,99 @@
|
|||||||
All notable changes to this project will be documented here.
|
All notable changes to this project will be documented here.
|
||||||
Format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
|
Format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
|
||||||
|
|
||||||
## [0.2.0] - Unreleased
|
## [0.8.0] - 2026-07-06
|
||||||
### Added
|
|
||||||
- Joplin import automation: `joplin` command syncs exported Markdown files to Joplin as notes
|
Focused on the Claude Code provider after the tool's on-disk layout changed and
|
||||||
- Notebooks created automatically per provider+project (`ChatGPT - My Project`, etc.)
|
Claude Code sessions proved hard to find in Joplin.
|
||||||
- Re-running is safe: notes are updated, not duplicated (Joplin note ID stored in manifest)
|
|
||||||
- `JOPLIN_API_TOKEN`, `JOPLIN_API_URL`, `JOPLIN_REQUEST_TIMEOUT` config variables
|
|
||||||
- Configurable request timeout with clear error messages and actionable hints on timeout
|
|
||||||
- `--project` filter on `export` and `list` commands (case-insensitive substring or `none`)
|
|
||||||
- ChatGPT Projects support via `CHATGPT_PROJECT_IDS` env var
|
|
||||||
|
|
||||||
## [0.1.0] - Unreleased
|
|
||||||
### Added
|
### Added
|
||||||
- Initial implementation: ChatGPT and Claude export via internal web APIs
|
- **Subagent capture.** Claude Code now stores subagent (Task-tool) transcripts as separate `<session>/subagents/agent-*.jsonl` files (with an `agent-*.meta.json` sidecar). These were previously invisible to the exporter — all delegated work (reviews, research, plans) was lost. Each subagent is now folded into its parent session inline at the `Task`/`Agent` call that spawned it, rendered as a collapsible `<details>` block labeled with its `agentType`/`description`. The subagent's own tool traffic is collapsed under the same `EXPORTER_HIDDEN_CONTENT` policy as the main dialogue; nesting is handled at any depth via `toolUseId` matching.
|
||||||
- Markdown and JSON exporters
|
- **Repo tags in Claude Code titles.** Sessions launched from a workspace root all land in one folder-named notebook, so titles now carry the repos each session touched, e.g. `Resume StartWRT project work [start-technologies]` — visible in the note list and matched by Joplin search. A file's repo is the git repository it lives in (nearest ancestor with a `.git`), resolved from tool_use paths anywhere in the filesystem — so cross-workspace work is captured and non-repo noise (config dirs, one-off files, reference dirs) is excluded because it isn't a git repo. Frequency-ordered, capped at 3. Escape hatch: `CLAUDE_CODE_REPO_TAG_IGNORE` (comma-separated repo names). Tags reflect the current git layout, so a since-deleted/moved repo drops from the tag on re-export.
|
||||||
- Local cache/manifest for incremental sync
|
- **Multiple projects roots.** `CLAUDE_CODE_DIR` now accepts an `os.pathsep`-separated list of roots (a single path stays backward compatible), and `CLAUDE_CONFIG_DIR`'s `projects/` tree is scanned automatically when set. Sessions from all roots are merged by launch-folder; a session UUID present in two roots keeps the newer-mtime copy.
|
||||||
- CLI with export, list, cache, doctor, and auth commands
|
|
||||||
|
### Changed
|
||||||
|
- **Claude Code gets its own top-level Joplin notebook, `AI-ClaudeCode`** (was nested under `AI-Claude` alongside Claude web chats, which made dev sessions hard to find). Existing Claude Code notes self-heal into the new notebook on the next `joplin` run — `update_note` now sets `parent_id`, so a changed provider→notebook mapping relocates notes in place instead of duplicating them. After migrating, the emptied `AI-Claude/{Myworkspace,Services,…}` notebooks can be deleted by hand (Joplin does not auto-remove empty folders).
|
||||||
|
|
||||||
|
### Notes
|
||||||
|
- The sibling `<session>/tool-results/*.txt` sidecars (externalized large tool outputs) are intentionally not captured — tool_result content is collapsed under the default policy anyway.
|
||||||
|
|
||||||
|
## [0.7.0] - 2026-06-28
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- `canary` command — probes the ChatGPT/Claude web APIs for schema drift against the fields the parser actually depends on (one listing page + one conversation per provider), not the full response shape. Flags the silent-failure risks for a backup tool: a renamed retrieval-tool author bypassing the hidden-content collapse, a new `content_type`, drifted message fields, or non-empty Claude `attachments`/`files` the normalizer ignores. ERROR findings exit non-zero; WARN findings are surfaced but non-fatal. See `FUTURE.md` §10.
|
||||||
|
- `doctor` now reports a real ChatGPT token-health check ("ChatGPT token active") based on the `/api/auth/session` `error` field. ChatGPT session tokens are JWEs whose `exp` is encrypted and unreadable client-side; the previous decode path could never yield an expiry. The honest signal is `error == "RefreshAccessTokenError"` (verified live), which means the session token is dead even though `expires`/`accessToken` still echo stale values. See `FUTURE.md` §9.
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- `ChatGPTProvider._fetch_access_token` now checks the `/api/auth/session` `error` field and fails fast with a clear "refresh your token" message. Previously it returned the stale `accessToken` present on a dead session, producing a confusing downstream 401 instead of an actionable auth error.
|
||||||
|
|
||||||
|
### Removed
|
||||||
|
- `auth --from-browser` and the `browser-cookie3` dependency (`src/browser_tokens.py`). Browser cookie auto-extraction is not viable: modern Chromium App-Bound Encryption (Chrome 127+/current Brave) keys cookies off a SYSTEM-level layer that can't be decrypted off disk without Administrator rights and AV-flagged SYSTEM impersonation, and fails on Brave specifically (symptom: `Unable to get key for cookie decryption`). Auth is manual (DevTools wizard) only. See `FUTURE.md` §2 for the full rationale.
|
||||||
|
|
||||||
|
## [0.6.0] - 2026-06-12
|
||||||
|
|
||||||
|
First tagged release. The project was developed through several internal
|
||||||
|
milestones (0.1.0–0.5.0) that were never published; their changes are
|
||||||
|
consolidated here.
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
**Core export & sync**
|
||||||
|
- ChatGPT and Claude export via their internal web APIs, with a local cache/manifest for incremental, resumable sync (only new or updated conversations are fetched; each is written to the manifest immediately).
|
||||||
|
- Markdown and JSON exporters. Markdown rendering happens at exporter-write time (providers produce typed blocks; exporters render them), which keeps the door open for future Obsidian/HTML output.
|
||||||
|
- CLI: `export`, `list`, `cache`, `doctor`, `auth`, `joplin`, `prune`.
|
||||||
|
- `--project` filter on `export`/`list`/`joplin` (case-insensitive substring, or `none`).
|
||||||
|
- ChatGPT Projects support via `CHATGPT_PROJECT_IDS` (project conversations are fetched separately from the default listing).
|
||||||
|
|
||||||
|
**Joplin integration**
|
||||||
|
- `joplin` command syncs exported Markdown to Joplin as notes; notebooks are created automatically and nested per provider+project. Re-running is safe — the Joplin note ID is stored in the manifest, so notes are updated, not duplicated.
|
||||||
|
- `JOPLIN_API_TOKEN`, `JOPLIN_API_URL`, `JOPLIN_REQUEST_TIMEOUT` config, with actionable error messages on timeout/connection failure.
|
||||||
|
|
||||||
|
**Rich content (typed blocks)**
|
||||||
|
- Messages carry an ordered `blocks` list: text, code, thinking, tool_use, tool_result, citation, image_placeholder, file_placeholder, unknown.
|
||||||
|
- ChatGPT voice mode: `audio_transcription` parts render as text; audio asset pointers render as `📎 File attached` placeholders with size/duration. Custom Instructions (`user_editable_context`/`model_editable_context`) now appear (were silently dropped) with a `> ℹ️ Hidden context` marker.
|
||||||
|
- ChatGPT `execution_output`, `system_error`, and `tether_browsing_display` render as `tool_result` blocks (with `tool_name`, `is_error`, optional `summary`); transient browse spinners skip silently.
|
||||||
|
- Defensive Claude block extraction (text/thinking/tool_use/tool_result/image, including nested-block flattening).
|
||||||
|
- `_safe_fence` picks a backtick fence longer than any run in the content, so embedded triple-backticks can't corrupt rendering (verified live in Joplin).
|
||||||
|
- Data-loss visibility: a `LossReport` summary at the end of every `export` run breaks down `unknown blocks` and `extraction failures` by raw type; `unknown` blocks render as visible `> ⚠️ Unsupported content` with the raw type and observed keys.
|
||||||
|
|
||||||
|
**Invisible-content collapse (`EXPORTER_HIDDEN_CONTENT=full|placeholder|omit`, default `placeholder`; `export --hidden-content`)**
|
||||||
|
- Messages that were invisible in the ChatGPT web UI are collapsed to one-line placeholders instead of exported in full. Two triggers: (a) tool-role retrieval dumps identified by `author.name` (`file_search`, `myfiles_browser`) — ChatGPT re-injects the full text of attached/project files on every tool run, measured at 86% of all content bytes across the three largest conversations and **not** hidden-flagged; (b) messages flagged `is_visually_hidden_from_conversation` (Custom Instructions). Code-execution results, web-search output, and all dialogue are untouched.
|
||||||
|
- New `collapsed` block renders as `> 🔧 **Tool output** — file_search (15.0 KB) — omitted (EXPORTER_HIDDEN_CONTENT=full to keep)` (ℹ️ Hidden context variant for hidden-flagged messages). The `LossReport` gains a `collapsed by policy` section (count per origin + total KB) so the omission stays visible.
|
||||||
|
|
||||||
|
**Claude Code session provider (`--provider claude-code`)**
|
||||||
|
- Archives local agent transcripts from `~/.claude/projects/*/*.jsonl` (override with `CLAUDE_CODE_DIR`) — no tokens, no rate limits, no ToS exposure. Prose-only by default per the hidden-content policy: dialogue and deliverables kept; tool traffic grouped into one placeholder per activity run (`> 🔧 Tool output — 14 calls: Read ×9, Bash ×2 (86KB) — omitted`); thinking dropped (counted in the summary); subagent (`isSidechain`) and harness (`isMeta`, snapshots, command tags) records stripped. Titles from the last `ai-title` record; project from the working-directory basename. Joplin notebooks nest under the `AI-Claude` parent. Measured: 19MB of JSONL → 804KB of Markdown across 25 sessions.
|
||||||
|
|
||||||
|
**Binary/image downloads (`EXPORTER_DOWNLOAD_MEDIA=images|all|off`, default `images`; `export --download-media`)**
|
||||||
|
- Downloads ChatGPT conversation assets into a `media/` folder beside each export and inlines them — images become real `` embeds, files become local links. Two-hop fetch via `/backend-api/files/{id}/download` → signed URL (verified live). `all` also pulls voice-mode audio (≈162MB of clips in this archive, transcripts already in text — hence the images-only default). Idempotent (skips assets already on disk), atomic writes, `600` perms. Expired/missing assets (old generated images 404) keep their placeholder and are counted as `media failed` — never fatal.
|
||||||
|
- Joplin resource upload: the `joplin` command uploads downloaded media as resources and rewrites `media/…` links to `:/resourceId`, so images render inside notes (verified live: resources created with correct mime/size and linked to the note). Resource IDs are tracked per-conversation in the manifest, so re-syncs reuse them instead of duplicating.
|
||||||
|
|
||||||
|
**Token setup & throughput**
|
||||||
|
- `auth --from-browser [brave|chrome|chromium|edge|firefox]`: extracts ChatGPT session-token cookies and the Claude `sessionKey` straight from a local browser's cookie store (via `browser-cookie3`; defaults to Brave), validates each against the live API, and writes `.env`. Tokens that fail validation never overwrite a working entry; clear fallback when the browser is absent or the cookie DB is locked.
|
||||||
|
- Session limiter: `--max-conversations N` / `MAX_CONVERSATIONS_PER_RUN` cap downloads per run (per provider); capped-out conversations are reported as "deferred" and resumed next run.
|
||||||
|
- Polite pacing: `REQUEST_DELAY` (default 1.0s, ±25% jitter, `0` disables) between consecutive API requests, with the existing 429 backoff as the reactive net.
|
||||||
|
|
||||||
|
**Re-rendering**
|
||||||
|
- `export --force` re-exports every conversation even if cached and unchanged, so the whole archive can be re-rendered after a formatting/feature change without `cache --clear`. Runs as a tracked campaign (a stamped start time in the manifest): each run re-renders the least-recently-exported conversations, the "still to go" count shrinks toward zero, finished providers do no further work, and the campaign auto-completes (reporting "Force re-render complete"). Combines with `--max-conversations` to spread the work across runs.
|
||||||
|
- `mark_exported` now preserves `joplin_note_id` / `joplin_synced_at` / `joplin_resources` across re-exports (refreshing `exported_at`), so a re-export followed by `joplin` updates the existing notes instead of creating duplicates.
|
||||||
|
- Fix: `.env` is now loaded at the very start of every command, so `CACHE_DIR` and `LOG_FILE` are honored (previously the cache silently used `./cache` regardless of `CACHE_DIR`).
|
||||||
|
|
||||||
|
**Archive hygiene**
|
||||||
|
- `prune` deletes export files not referenced by the manifest (old-layout trees, no-ID orphans), with listing, confirmation, `--dry-run`, `-y`, and empty-directory sweep. Refuses to run when the manifest references no files (so `cache --clear` + `prune` can't wipe the archive). First live run removed 420 stale files (9.4 MB).
|
||||||
|
- `doctor` verifies manifest ↔ disk integrity: every recorded `file_path` must exist on disk.
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
- ChatGPT role filter that dropped `tool`/`system` messages is **lifted**; all roles route through normal extraction (truly empty messages skip via the empty-content guard).
|
||||||
|
- `BaseProvider.normalize_conversation` accepts an optional `LossReport` parameter.
|
||||||
|
- `ChatGPTProvider.normalize_conversation` reads `conversation_id` as a fallback for `id` (live detail responses use `conversation_id`; fixtures use `id`).
|
||||||
|
- `Config` validates `EXPORTER_HIDDEN_CONTENT`, `EXPORTER_DOWNLOAD_MEDIA`, `MAX_CONVERSATIONS_PER_RUN`, and `REQUEST_DELAY`, and logs the active values at startup.
|
||||||
|
- New dependency: `browser-cookie3==0.20.1`.
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Custom Instructions (`user_editable_context`/`model_editable_context`) were silently dropped from every conversation (parts-vs-direct-fields mismatch).
|
||||||
|
|
||||||
|
### Migration
|
||||||
|
- JSON exports: messages contain typed `blocks` and may omit the legacy `content` field — external consumers should prefer `blocks`.
|
||||||
|
- To re-render existing exports with the current rendering and collapse policy: `python -m src.main cache --clear` then `python -m src.main export`, followed by `joplin` to update notes (and upload media resources). Per-conversation message counts may increase as previously-dropped Custom Instructions, image-only turns, and tool-only turns now appear.
|
||||||
|
|
||||||
|
### Test suite
|
||||||
|
- 264 tests, all passing.
|
||||||
|
|||||||
@@ -1,27 +1,480 @@
|
|||||||
# Planned Future Work
|
# Planned Future Work
|
||||||
|
|
||||||
Items completed in each release are moved to the changelog. Items here are
|
> **Status 2026-07-06 (v0.8.0): Claude Code coverage reopened and shipped.**
|
||||||
designed for but not yet implemented. The codebase is structured to make each
|
> Claude Code changed its on-disk layout (subagent transcripts moved to separate
|
||||||
of these additions straightforward.
|
> `subagents/*.jsonl` files) and its sessions were hard to find in Joplin. v0.8.0
|
||||||
|
> addressed this: subagent capture (folded `<details>`), repo `[tags]` in titles,
|
||||||
|
> an own `AI-ClaudeCode` notebook with self-healing note moves, and multi-root
|
||||||
|
> scanning (`CLAUDE_CODE_DIR` list + `CLAUDE_CONFIG_DIR`). See the changelog.
|
||||||
|
>
|
||||||
|
> **Status 2026-06-28: feature-complete / done for now.** As of v0.7.0 the
|
||||||
|
> active roadmap is empty and the remaining backlog below has been **closed as
|
||||||
|
> not needed** — the tool does what it's needed to do as a local, manually-run
|
||||||
|
> backup CLI. Items are kept for reference only; revisit on demand if a real
|
||||||
|
> need shows up. Nothing here is planned work.
|
||||||
|
|
||||||
|
Items completed in each release are moved to the changelog. Items below the
|
||||||
|
roadmap were designed for but intentionally not implemented. The codebase is
|
||||||
|
structured to make each of these additions straightforward if ever revived.
|
||||||
|
|
||||||
**Completed:**
|
**Completed:**
|
||||||
- v0.1.0 — Core export: ChatGPT + Claude, incremental sync, Markdown + JSON output
|
- v0.1.0 — Core export: ChatGPT + Claude, incremental sync, Markdown + JSON output
|
||||||
- v0.2.0 — Joplin import automation (`joplin` command, create/update notes, notebook auto-creation)
|
- v0.2.0 — Joplin import automation (`joplin` command, create/update notes, notebook auto-creation)
|
||||||
|
- v0.4.0 — Rich content support: typed message blocks (text, code, thinking, tool_use, tool_result, image_placeholder, file_placeholder, unknown); ChatGPT voice transcripts as text + audio placeholders; Custom Instructions extraction; data-loss visibility via `LossReport` summary and visible `unknown` blocks
|
||||||
|
- v0.5.0 — Nested Joplin notebooks, date-prefixed note titles, flat year folders
|
||||||
|
- v0.6.0 — Collapse tool retrieval dumps & hidden context (`EXPORTER_HIDDEN_CONTENT` policy; roadmap item 1); session limiter + request pacing (`MAX_CONVERSATIONS_PER_RUN`, `REQUEST_DELAY`; roadmap item 6)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Export `--force` Flag (v0.2.x)
|
# Roadmap (decided 2026-06-12)
|
||||||
|
|
||||||
Add `--force` to the `export` command to re-export already-cached conversations
|
Priorities reflect the tool's primary purpose — a trustworthy backup so that
|
||||||
without permanently clearing the entire manifest. Useful for re-generating files
|
conversation data is not lost if a provider account is ever closed — plus the
|
||||||
after changing the Markdown template or output structure.
|
day-to-day friction of the weekly ChatGPT token refresh. The tool stays a
|
||||||
|
local, manually-run CLI; the headless/StartOS direction was dropped
|
||||||
|
2026-06-28 (see #7 and #8), which also retires the token-freshness problem
|
||||||
|
(manual refresh is sufficient).
|
||||||
|
|
||||||
Implementation: pass a `force=True` flag to `cache.get_new_or_updated()`, which
|
**Now (in order):**
|
||||||
returns all conversations regardless of cache state when force is True.
|
1. ~~Collapse tool retrieval dumps & hidden context~~ — **shipped in v0.6.0**
|
||||||
|
(full-archive `export --force` re-export completed 2026-06-13)
|
||||||
|
2. ~~Brave cookie auto-extraction~~ — **shipped in v0.6.0, removed afterward.**
|
||||||
|
Not viable: modern Chromium App-Bound Encryption (Chrome 127+/current Brave)
|
||||||
|
needs admin + SYSTEM impersonation that AV flags as credential theft, and
|
||||||
|
fails on Brave specifically. Auth is manual (DevTools) — see entry below.
|
||||||
|
3. ~~Claude Code session provider~~ — **shipped in v0.6.0**
|
||||||
|
4. ~~Archive hygiene — `prune` + `doctor` integrity check~~ — **shipped in v0.6.0**
|
||||||
|
|
||||||
Current workaround: `python -m src.main cache --clear` then re-run export.
|
**Soon, but later:**
|
||||||
|
|
||||||
## Joplin `--force` Flag (v0.2.x)
|
5. ~~Binary content downloads~~ — **shipped in v0.6.0**
|
||||||
|
6. ~~Per-session download limiter + polite pacing~~ — **shipped in v0.6.0**
|
||||||
|
7. ~~Scheduled / watch mode~~ — **dropped 2026-06-28**; the tool stays a
|
||||||
|
manually-run CLI, so no in-app polling loop is needed
|
||||||
|
8. ~~StartOS service packaging~~ — **dropped 2026-06-28**; the local CLI is
|
||||||
|
sufficient (source convos live in the cloud and can be re-downloaded;
|
||||||
|
Joplin already syncs encrypted to an offsite S3 provider). Dropping this
|
||||||
|
also retires the headless token-freshness problem — manual weekly refresh
|
||||||
|
is fine.
|
||||||
|
|
||||||
|
**Active:**
|
||||||
|
9. ~~Surface remaining token validity on `doctor`~~ — **IMPLEMENTED 2026-06-28**
|
||||||
|
via the `/api/auth/session` `error` field (not `expires`). See §9.
|
||||||
|
10. ~~Provider API-drift detection~~ — **IMPLEMENTED 2026-06-28** as the
|
||||||
|
`canary` command. See §10.
|
||||||
|
|
||||||
|
**Deprioritized** (entries kept at the bottom of this file; revisit on
|
||||||
|
demand): `--force` flags, per-conversation cache reset, official export-ZIP
|
||||||
|
fallback, o1/o3 reasoning reclassification, Obsidian output, search command.
|
||||||
|
Additional web providers (Gemini/Grok/Perplexity) are explicitly out of
|
||||||
|
scope — no significant usage to archive.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Collapse Tool Retrieval Dumps & Hidden Context — SHIPPED v0.6.0
|
||||||
|
|
||||||
|
**Implemented 2026-06-12** as designed below, with one scoping correction
|
||||||
|
from live recon: the retrieval dumps are NOT flagged
|
||||||
|
`is_visually_hidden_from_conversation` — `author.name == "file_search"` is
|
||||||
|
the discriminator (the hidden flag only marks Custom Instructions and small
|
||||||
|
system stubs). Verified live: worst files shrink 93% (524KB → 36KB).
|
||||||
|
Full-archive re-export (the `export --force` campaign) + Joplin re-sync
|
||||||
|
completed 2026-06-13.
|
||||||
|
|
||||||
|
**Problem (measured 2026-06-12 against a full fresh export):** 45% of the
|
||||||
|
entire 11.2MB archive (260 files) is tool-role messages; 29 files are
|
||||||
|
majority tool-dump. Worst case: a 524KB conversation where 495KB (94%, 64
|
||||||
|
messages) is ChatGPT's file-retrieval tool re-injecting the full text of
|
||||||
|
the user's own attached documents, the same files dumped dozens of times
|
||||||
|
per conversation. These messages were invisible in the ChatGPT web UI;
|
||||||
|
they appear in exports because v0.4.0 lifted the role filter to fix silent
|
||||||
|
data loss. Custom Instructions hidden-context blocks are a minor secondary
|
||||||
|
case (~2KB, once per conversation) — the originally planned
|
||||||
|
`EXPORTER_INCLUDE_HIDDEN_CONTEXT` toggle alone would not help: the worst
|
||||||
|
file contains zero hidden-context blocks.
|
||||||
|
|
||||||
|
**Fix: collapse, don't drop** (consistent with the no-silent-drop rule):
|
||||||
|
|
||||||
|
- `EXPORTER_HIDDEN_CONTENT=full|placeholder|omit` env var, default
|
||||||
|
`placeholder`, plus a `--hidden-content` CLI override on `export`.
|
||||||
|
- `placeholder` renders affected messages as one line with type and size:
|
||||||
|
`> 🔧 Tool output (file_search, 24KB) — omitted
|
||||||
|
(EXPORTER_HIDDEN_CONTENT=full to keep)`. Expected effect: archive
|
||||||
|
roughly halves; worst files shrink ~90%.
|
||||||
|
- **Scope — collapse:** (a) tool-role retrieval dumps, identified by raw
|
||||||
|
`author.name` (`file_search`, `myfiles_browser`, …) in the API response
|
||||||
|
(the rendered Markdown only shows a generic "🔧 Tool" label, so the
|
||||||
|
decision must happen in the provider, not the renderer); (b) messages
|
||||||
|
flagged `is_visually_hidden_from_conversation`, including
|
||||||
|
`user_editable_context` / `model_editable_context` (Custom
|
||||||
|
Instructions) — subsumes the old suppress-hidden-context idea.
|
||||||
|
- **Scope — keep at full size:** code-execution `tool_result` blocks and
|
||||||
|
web-search results; those are usually content the user wants.
|
||||||
|
- Count collapsed messages in the post-export summary so the omission
|
||||||
|
stays visible (mirror the LossReport presentation, but as intentional
|
||||||
|
policy, not loss).
|
||||||
|
|
||||||
|
Re-export workflow after shipping: `cache --clear` + `export` (same as
|
||||||
|
the v0.4.0 migration).
|
||||||
|
|
||||||
|
## 2. Brave Cookie Auto-Extraction — REMOVED (not viable)
|
||||||
|
|
||||||
|
**Shipped v0.6.0 (2026-06-12), removed 2026-06-27.** `auth --from-browser`
|
||||||
|
plus `src/browser_tokens.py` and the `browser-cookie3` dependency are gone.
|
||||||
|
Auth is manual (DevTools) only. Do not re-attempt without a fundamentally
|
||||||
|
different mechanism (see below).
|
||||||
|
|
||||||
|
**Why it doesn't work.** Modern Chromium browsers encrypt cookies on Windows
|
||||||
|
with **App-Bound Encryption** (Chrome 127+, July 2024; current Brave).
|
||||||
|
Cookies are written with a `v20` prefix and keyed off a secret wrapped in a
|
||||||
|
**SYSTEM-level** DPAPI layer plus app validation. `browser-cookie3` only
|
||||||
|
knows the legacy `v10`/DPAPI key, so its AES-GCM MAC check fails — the exact
|
||||||
|
symptom hit in the field:
|
||||||
|
|
||||||
|
```
|
||||||
|
ChatGPT: Could not read brave cookies for chatgpt.com: Unable to get key for cookie decryption.
|
||||||
|
```
|
||||||
|
|
||||||
|
Decrypting `v20` at all requires unwrapping the SYSTEM layer, which means
|
||||||
|
running as SYSTEM (e.g. a PsExec-style service) — i.e. **Administrator
|
||||||
|
rights** and behavior that AV/EDR flags as infostealer activity. The one
|
||||||
|
maintained Python option (`rookiepy`) needs admin from Chrome v130+, was
|
||||||
|
**archived 2026-06-07**, and has an unresolved bug where **Brave returns 0
|
||||||
|
cookies** even after the key is retrieved. ABE is *designed* to stop exactly
|
||||||
|
this, so no off-disk reader is a reliable, non-invasive fit.
|
||||||
|
|
||||||
|
**If ever revisited:** the only non-admin path is Chrome Remote Debugging
|
||||||
|
(launch the browser with `--remote-debugging-port`, read cookies via
|
||||||
|
`Network.getAllCookies` — the running browser decrypts for you). Heavier and
|
||||||
|
intrusive; not worth it for a weekly token refresh that takes 30 seconds by
|
||||||
|
hand. With the headless/StartOS direction dropped (#8), manual DevTools
|
||||||
|
refresh is the accepted approach — no automated extraction is needed.
|
||||||
|
|
||||||
|
## 3. Claude Code Session Provider — SHIPPED v0.6.0
|
||||||
|
|
||||||
|
**Implemented 2026-06-12** as designed below (`src/providers/claude_code.py`,
|
||||||
|
`--provider claude-code`, `CLAUDE_CODE_DIR` override). Additional findings
|
||||||
|
during implementation: `isSidechain` records are subagent transcripts (skipped),
|
||||||
|
`isMeta` marks harness-generated user records (skipped), and listing/normalized
|
||||||
|
`updated_at` must both use file mtime or the cache would re-export every
|
||||||
|
session every run. First export: 25 sessions, 19MB JSONL → 804KB Markdown.
|
||||||
|
|
||||||
|
Archive local Claude Code session transcripts. No tokens, no rate limits,
|
||||||
|
no ToS risk — the data is already on disk but lives in a single JSONL per
|
||||||
|
session that Claude Code may clean up, and it contains deliverables
|
||||||
|
(reviews, plans, analyses) that exist nowhere else.
|
||||||
|
|
||||||
|
Decisions (2026-06-12):
|
||||||
|
- **Rendering: prose-only.** Keep user prompts and assistant text
|
||||||
|
(including full deliverable write-ups); collapse tool activity to
|
||||||
|
one-line placeholders (`> 🔧 Tool activity — 14 calls (Read ×9, Bash ×2),
|
||||||
|
86KB — omitted`); **exclude thinking blocks**.
|
||||||
|
- **Joplin: sync enabled.** Each coding project becomes a notebook nested
|
||||||
|
under an **"AI-Claude"** parent notebook (nested-notebook support shipped
|
||||||
|
in v0.5.0).
|
||||||
|
|
||||||
|
Data facts (measured 2026-06-12):
|
||||||
|
- Source: `~/.claude/projects/<munged-cwd>/<session-uuid>.jsonl`.
|
||||||
|
Currently 29 sessions, 18.8MB total, largest 3.8MB.
|
||||||
|
- Representative 2.1MB session: tool_result 436KB, tool_use 122KB,
|
||||||
|
thinking 104KB, dialogue prose only ~24KB (~4%) — collapsing tool
|
||||||
|
activity is what makes these exports readable.
|
||||||
|
- Record types: `user` / `assistant` (Anthropic-style `message.content`
|
||||||
|
block arrays) plus harness records: `ai-title` (use for note title and
|
||||||
|
filename slug), `last-prompt`, `file-history-snapshot`, `attachment`,
|
||||||
|
`permission-mode`, `system` (skip). Strip harness noise from user
|
||||||
|
messages (`<local-command-caveat>`, `<command-name>` blocks).
|
||||||
|
|
||||||
|
Implementation shape: new `src/providers/claude_code.py` implementing the
|
||||||
|
`BaseProvider` interface — `list_conversations` scans project dirs,
|
||||||
|
`get_conversation` parses the JSONL, `normalize_conversation` maps onto the
|
||||||
|
existing block schema (content is already block-shaped: text / tool_use /
|
||||||
|
tool_result / thinking). Incremental sync via file mtime/size recorded in
|
||||||
|
the existing manifest. Project name derives from the munged cwd dirname.
|
||||||
|
|
||||||
|
## 4. Archive Hygiene: `prune` Command + Manifest Integrity — SHIPPED v0.6.0
|
||||||
|
|
||||||
|
**Implemented 2026-06-12** as designed below, plus an empty-manifest guard
|
||||||
|
(refuses to prune right after `cache --clear`). First live run removed 420
|
||||||
|
stale files (9.4 MB, old-layout trees + `_.md` orphans); doctor now reports
|
||||||
|
manifest↔disk integrity (293/293 after the run).
|
||||||
|
|
||||||
|
A backup is only trustworthy if the on-disk tree matches the manifest.
|
||||||
|
Observed 2026-06-12: pre-v0.5.0 layout trees (`tspc-expertcouncil/2025/`)
|
||||||
|
and `_.md` no-ID orphans (from the empty-conversation-id bug fixed in
|
||||||
|
v0.4.1) sit alongside current exports and would double-sync into Joplin.
|
||||||
|
|
||||||
|
- `prune` command: delete export files not referenced by the manifest.
|
||||||
|
`--dry-run` (default off, but always print the list before deleting)
|
||||||
|
shows what would be removed and why (old layout / orphan / unknown).
|
||||||
|
- `doctor` extension: verify every manifest entry's `file_path` exists on
|
||||||
|
disk; report missing files (re-export candidates) and unreferenced files
|
||||||
|
(prune candidates).
|
||||||
|
|
||||||
|
## 5. Binary Content Downloads — SHIPPED v0.6.0
|
||||||
|
|
||||||
|
**Implemented 2026-06-12** (`src/media.py`, `EXPORTER_DOWNLOAD_MEDIA`,
|
||||||
|
`download_asset`/`parse_asset_file_id` on the ChatGPT provider, Joplin
|
||||||
|
`create_resource` + `upload_media_and_rewrite`). Live recon settled the
|
||||||
|
download mechanism: `GET /backend-api/files/{id}/download` returns a signed
|
||||||
|
`download_url`; a second GET yields the bytes (works for user uploads;
|
||||||
|
older AI-generated images 404 — expired server-side, handled gracefully).
|
||||||
|
Asset refs come in three shapes — `sediment://file_…`,
|
||||||
|
`sediment://<hash>#file_…#p_N.png` (generated), `file-service://…`. Archive
|
||||||
|
scan: 14 images (4 uploads / 10 generated) + 556 audio clips ≈162MB, so
|
||||||
|
images-only is the default and audio is opt-in via `all`.
|
||||||
|
|
||||||
|
Original notes below.
|
||||||
|
|
||||||
|
**Priority note (2026-06-12): "later, but soon" — under the
|
||||||
|
backup-if-account-closes goal, embedded images are part of the data that
|
||||||
|
would be lost; placeholders alone don't preserve them.**
|
||||||
|
|
||||||
|
v0.4.0 ships placeholders for images and audio assets but does not download
|
||||||
|
the binary content. The `_safe_fence`-wrapped placeholders include the asset
|
||||||
|
reference (`sediment://...` or `file-service://...`), MIME type, size, and
|
||||||
|
duration where available; the actual bytes are not preserved.
|
||||||
|
|
||||||
|
Next steps:
|
||||||
|
- Download attached images alongside the Markdown export, save under a
|
||||||
|
`media/` sibling directory with a stable filename derived from the asset
|
||||||
|
reference.
|
||||||
|
- Replace `image_placeholder` rendering with an inline ``
|
||||||
|
reference once the file is on disk.
|
||||||
|
- Joplin integration: upload binaries as Joplin resources via `POST /resources`,
|
||||||
|
rewrite the rendered Markdown to use `:/resourceId` references, and track
|
||||||
|
the resource ID in the cache manifest so re-syncs stay idempotent.
|
||||||
|
- DALL-E images on the assistant side: not observed in this user's data; the
|
||||||
|
code path exists (`source = "model_generated"`) but is untested.
|
||||||
|
|
||||||
|
The block-level schema is already in place — only the file-fetch + rewrite
|
||||||
|
layer needs to be added. See the `image_placeholder` and `file_placeholder`
|
||||||
|
block definitions in `src/blocks.py`.
|
||||||
|
|
||||||
|
## 6. Per-Session Download Limiter — SHIPPED v0.6.0
|
||||||
|
|
||||||
|
**Implemented 2026-06-12** as designed below: `--max-conversations N` /
|
||||||
|
`MAX_CONVERSATIONS_PER_RUN` session cap with deferred-count reporting, and
|
||||||
|
`REQUEST_DELAY` pacing (default 1.0s ±25% jitter) in `BaseProvider._request`.
|
||||||
|
Verified live: a 2-pending run with cap 1 exported one, deferred one, and
|
||||||
|
the re-run picked it up.
|
||||||
|
|
||||||
|
Cap how many conversations are downloaded in a single `export` run so the
|
||||||
|
tool never hammers the ChatGPT/Claude internal APIs with a large burst —
|
||||||
|
most importantly on the very first export, which otherwise fetches the
|
||||||
|
entire conversation history in one session. Because every run is resumable
|
||||||
|
(the manifest records each conversation immediately), a capped run simply
|
||||||
|
exports the first N pending conversations and the next run picks up where
|
||||||
|
it left off. This keeps traffic looking like a human-paced session rather
|
||||||
|
than a scraper, reducing the risk of rate limiting or account flags.
|
||||||
|
|
||||||
|
Two complementary pieces:
|
||||||
|
|
||||||
|
1. **Session cap** — `--max-conversations N` flag (and
|
||||||
|
`MAX_CONVERSATIONS_PER_RUN` env default). Implementation: in the
|
||||||
|
`export` command, slice the pending list after the cache filter:
|
||||||
|
`to_export = to_export[:n]`. On exit, print exported-vs-remaining
|
||||||
|
counts (reuse the message format from the 429 early-exit path) and
|
||||||
|
remind the user to re-run to continue.
|
||||||
|
2. **Polite pacing** — `REQUEST_DELAY` env var (seconds, with small
|
||||||
|
random jitter) slept between per-conversation detail fetches in
|
||||||
|
`BaseProvider`, so even a capped run doesn't fire requests
|
||||||
|
back-to-back. The existing 429 backoff in `_request` stays as the
|
||||||
|
reactive safety net.
|
||||||
|
|
||||||
|
Note: the conversation *listing* (paginated, 100/page) still runs in full
|
||||||
|
each time so the cache comparison works — the cap applies to the heavy
|
||||||
|
per-conversation detail fetches, which dominate request volume.
|
||||||
|
|
||||||
|
This is a stepping stone to the StartOS service: a capped, politely-paced
|
||||||
|
export — scheduled by the host (cron/StartOS), not an in-app loop — is the
|
||||||
|
traffic profile a headless deployment needs.
|
||||||
|
|
||||||
|
## 7. Scheduled / Watch Mode — DROPPED (2026-06-28)
|
||||||
|
|
||||||
|
An in-app `watch`/scheduler loop is not worth building. Scheduling belongs to
|
||||||
|
whatever hosts the tool: a user cron line locally, and on the long-term
|
||||||
|
StartOS target the platform's own scheduling. Either way the tool only needs
|
||||||
|
to do one capped, politely-paced `export` + `joplin` run and exit — which it
|
||||||
|
already does. If cron ergonomics ever feel clunky, a thin `sync` subcommand
|
||||||
|
that chains `export` then `joplin` for a single cron line is a trivial
|
||||||
|
add-on, but the polling loop itself is off the roadmap.
|
||||||
|
|
||||||
|
## 8. StartOS Service Packaging — DROPPED (2026-06-28)
|
||||||
|
|
||||||
|
Not pursuing a headless StartOS service. The local, manually-run CLI is
|
||||||
|
sufficient: the source conversations live in the providers' clouds and can be
|
||||||
|
re-downloaded, and Joplin already syncs (encrypted) to an offsite S3 provider,
|
||||||
|
so durability is covered without a server in the loop.
|
||||||
|
|
||||||
|
Dropping this also retires the one genuinely hard sub-problem it carried —
|
||||||
|
session-token freshness without a browser. There is no headless context to
|
||||||
|
keep fresh; the weekly manual DevTools refresh is acceptable. (Local cookie
|
||||||
|
extraction remains a dead end regardless — see #2.)
|
||||||
|
|
||||||
|
## 9. Token Validity on `doctor` — IMPLEMENTED (2026-06-28)
|
||||||
|
|
||||||
|
Shipped: `doctor` now adds a "ChatGPT token active" check via
|
||||||
|
`ChatGPTProvider.session_health()` (reads `/api/auth/session`, passes iff
|
||||||
|
`error` is falsy and `accessToken` is present), the never-working JWE/`exp`
|
||||||
|
decode path was removed, and `_fetch_access_token` now fails fast on a set
|
||||||
|
`error` instead of returning a stale token. Tests in
|
||||||
|
`tests/test_providers.py::TestChatGPTSessionHealth`. Investigation trail
|
||||||
|
below for the record.
|
||||||
|
|
||||||
|
Goal: if it's cheap to tell how much longer a token will work, show it on
|
||||||
|
`doctor`. Findings from live recon:
|
||||||
|
|
||||||
|
- **Not readable from the token itself.** ChatGPT's `CHATGPT_SESSION_TOKEN`
|
||||||
|
is a **JWE** (header `{"alg":"dir","enc":"A256GCM"}`, `eyJ…` prefix is just
|
||||||
|
the encrypted protected header) — the `exp` claim is AES-256-GCM encrypted
|
||||||
|
with an OpenAI-only key, so it cannot be decoded client-side. Claude's
|
||||||
|
`sk-…` key is fully opaque. The existing `doctor` JWT-decode path therefore
|
||||||
|
never yields an expiry for the real tokens (falls to the "not decodable"
|
||||||
|
branch).
|
||||||
|
- **`/api/auth/session` exposes an `expires`** (the provider already calls
|
||||||
|
this endpoint in `_fetch_access_token`; the response includes `expires`
|
||||||
|
alongside `accessToken`). Live value observed 2026-06-28:
|
||||||
|
`2026-09-26` — **~90 days out**. This contradicts both the code's ~7-day
|
||||||
|
assumption and the lived weekly-refresh cadence, so it is almost certainly
|
||||||
|
the NextAuth **rolling session window** (re-extended on every call), not
|
||||||
|
the point at which the pasted token actually 401s. Displaying it verbatim
|
||||||
|
would give false confidence.
|
||||||
|
- **RESOLVED 2026-06-28 by a live 401 data point.** When the ChatGPT token
|
||||||
|
was actually dead (conversations API → 401), `/api/auth/session` still
|
||||||
|
returned **HTTP 200** with `expires: 2026-09-26` (~90 days out) — and that
|
||||||
|
`expires` *advanced* between two calls seconds apart (`04:20:08` → `04:30:37`).
|
||||||
|
So `expires` is a **rolling session window that rolls forward on every call
|
||||||
|
even for a dead token**; displaying it would actively lie. The same response
|
||||||
|
carried `error: "RefreshAccessTokenError"` and a stale `accessToken`.
|
||||||
|
- **The real signal is `error`, not `expires` or `accessToken`.** On a healthy
|
||||||
|
token `error` is absent/null; when the session token is dead NextAuth can't
|
||||||
|
refresh and sets `error: "RefreshAccessTokenError"` while still echoing a
|
||||||
|
rolling `expires` and a stale `accessToken`. This is an exact, free, binary
|
||||||
|
health check.
|
||||||
|
- **Design (ready to build):**
|
||||||
|
1. `doctor` ChatGPT check → read `/api/auth/session`; pass iff `error` is
|
||||||
|
falsy and `accessToken` present; on `RefreshAccessTokenError` report
|
||||||
|
"token expired — refresh". Drop the JWE/`exp` decode path (it can never
|
||||||
|
work) and do NOT surface `expires`.
|
||||||
|
2. Latent bug to fix alongside: `_fetch_access_token` reads `accessToken`
|
||||||
|
without checking `error`, so it proceeds with a stale token and yields a
|
||||||
|
confusing downstream 401 instead of a clear "refresh your token" message.
|
||||||
|
Check `error` there and fail fast.
|
||||||
|
3. Claude stays a 401-only signal (opaque `sk-`, no equivalent endpoint).
|
||||||
|
|
||||||
|
## 10. Provider API-Drift Detection — IMPLEMENTED (2026-06-28)
|
||||||
|
|
||||||
|
Shipped: the `canary` command + `BaseProvider.check_drift()` (overridden by
|
||||||
|
ChatGPT and Claude). It fetches one listing page + one conversation per
|
||||||
|
provider and asserts only the normalizer's load-bearing fields, emitting
|
||||||
|
`DRIFT_OK/WARN/ERROR` findings (`src/providers/base.py`). Severity badges
|
||||||
|
print as a Rich table; ERROR exits non-zero, WARN is non-fatal so a backup
|
||||||
|
run is never blocked. Drift vocabularies (`_KNOWN_TOOL_AUTHORS`,
|
||||||
|
`_HANDLED_CONTENT_TYPES`) live in `chatgpt.py` next to the collapse set they
|
||||||
|
guard. Tests: `TestChatGPTDriftCanary`, `TestClaudeDriftCanary`,
|
||||||
|
`TestCanaryCommand`. Verified live 2026-06-28 — both providers OK. Recon
|
||||||
|
trail below for the record.
|
||||||
|
|
||||||
|
## 10b. Provider API-Drift Detection — investigation (2026-06-28)
|
||||||
|
|
||||||
|
The export depends on undocumented internal web APIs (ChatGPT/Claude) that can
|
||||||
|
change shape without notice. The worst failure for a backup tool is *silent*:
|
||||||
|
a response-schema change that makes the exporter skip or mis-parse content
|
||||||
|
without erroring. `doctor` currently checks token validity, reachability, and
|
||||||
|
manifest↔disk integrity — but not "does the provider's response still look
|
||||||
|
like what the parser expects."
|
||||||
|
|
||||||
|
To investigate: a lightweight schema/shape assertion on a known-good sample
|
||||||
|
of each provider's listing + conversation-detail responses (presence and type
|
||||||
|
of the fields the normalizers rely on), surfaced as a `doctor` check or a
|
||||||
|
dedicated canary.
|
||||||
|
|
||||||
|
**Live recon — Claude captured 2026-06-28 (ChatGPT pending a token refresh):**
|
||||||
|
|
||||||
|
- **Dependency surface (assert ONLY these — see below for why):**
|
||||||
|
- listing item: `uuid`, `name`, `updated_at`/`created_at`, `project.name`.
|
||||||
|
- conversation detail: `uuid`/`id`, `name`, `created_at`, `updated_at`,
|
||||||
|
`project.name`, `chat_messages[]`.
|
||||||
|
- message: `sender` (`human`/`assistant`), `text` (string) or `content`
|
||||||
|
(list of typed blocks), `created_at`.
|
||||||
|
- **Key finding — full-shape diffing is the wrong design.** Claude's
|
||||||
|
`settings` object is full of volatile internal codenames that churn
|
||||||
|
constantly: `enabled_bananagrams`, `enabled_sourdough`, `enabled_foccacia`,
|
||||||
|
`enabled_saffron`, `enabled_turmeric`, `enabled_monkeys_in_a_barrel`,
|
||||||
|
`paprika_mode`, `enabled_megaminds`, … A "any new/removed key = drift"
|
||||||
|
canary would fire on every UI experiment. The canary MUST target the
|
||||||
|
normalizer's load-bearing fields only, not the whole response. (Aligns with
|
||||||
|
the drop-noise-don't-retain-it principle.)
|
||||||
|
- **Real Claude messages are flat `text`/`sender`** — in this archive every
|
||||||
|
message had a string `text` and NO `content` block list (0 rich blocks
|
||||||
|
observed). So `_extract_claude_blocks` / `_dispatch_claude_block` (tool_use,
|
||||||
|
thinking, image, …) is an **unexercised theoretical path**; drift there
|
||||||
|
can't be "caught" by a canary because it never runs on real data — it's a
|
||||||
|
safety net for if Claude ever switches to block content. The canary should
|
||||||
|
assert the flat shape and *warn if `content` ever appears as a list* (that
|
||||||
|
itself is the drift event that would activate the dormant code).
|
||||||
|
- **Possible silent-loss spot (separate from drift):** Claude messages carry
|
||||||
|
`attachments` and `files` arrays (empty in this sample) that the normalizer
|
||||||
|
ignores entirely. If a user ever attaches files in Claude, they'd be
|
||||||
|
dropped without a LossReport entry. Worth a follow-up check.
|
||||||
|
**Live recon — ChatGPT captured 2026-06-28:**
|
||||||
|
|
||||||
|
- **Dependency surface (assert ONLY these):**
|
||||||
|
- listing item: `id`, `title`, `update_time`/`create_time`.
|
||||||
|
- conversation detail: `conversation_id`/`id`, `title`, `create_time`,
|
||||||
|
`update_time`, `mapping` (non-empty).
|
||||||
|
- mapping node: `message`, `children` (the tree walk depends on both);
|
||||||
|
message: `author.role`, `author.name`, `content.content_type`,
|
||||||
|
`content.parts`, `metadata.is_visually_hidden_from_conversation`.
|
||||||
|
- **content_type vocabulary observed (all currently handled):** `text`,
|
||||||
|
`model_editable_context`, `multimodal_text`, `thoughts`, `code`,
|
||||||
|
`execution_output`, `reasoning_recap`, `user_editable_context`,
|
||||||
|
`tether_browsing_display`. A *new* content_type already degrades gracefully
|
||||||
|
(visible `unknown` block + WARNING + LossReport tally) — so content_type
|
||||||
|
drift is **already non-silent**. The canary just needs to confirm the known
|
||||||
|
set still parses to non-empty blocks.
|
||||||
|
- **The genuinely silent drift risk — `author.name` collapse keys.**
|
||||||
|
`_COLLAPSE_TOOL_AUTHORS = {"file_search", "myfiles_browser"}`. Recon
|
||||||
|
confirms `file_search` is live (and `web.run`/`python` are correctly left
|
||||||
|
un-collapsed). If OpenAI renames `file_search`, the collapse **silently
|
||||||
|
stops** and the archive re-bloats with no error or LossReport entry. This is
|
||||||
|
the top canary target: assert that retrieval-dump tool authors are still
|
||||||
|
recognized, or at least flag unfamiliar `(role="tool", author.name)` pairs.
|
||||||
|
- **Second silent risk — empty `content.parts`.** A `text` message whose
|
||||||
|
`parts` field is renamed/emptied yields zero blocks and is skipped with only
|
||||||
|
a debug log = silent loss. Canary should assert a sampled `text` message
|
||||||
|
produces a non-empty block.
|
||||||
|
|
||||||
|
**Recon complete for both providers. Canary design (ready to build):** a
|
||||||
|
`doctor` check (or dedicated `canary` command) that, per provider, fetches one
|
||||||
|
listing page + one conversation and asserts the dependency-surface fields
|
||||||
|
above by presence+type — NOT full shape (Claude `settings` codenames prove
|
||||||
|
full-shape diffing is pure noise). Specific tripwires: (ChatGPT) unfamiliar
|
||||||
|
`(tool, author.name)` pair and empty `parts` on a text message; (Claude)
|
||||||
|
`content` appearing as a list, and non-empty `attachments`/`files`. Failures
|
||||||
|
surface as a warning, never a hard error (a backup tool must still run).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Deprioritized — CLOSED as not needed (2026-06-28)
|
||||||
|
|
||||||
|
These were considered and intentionally **not** built. Closed, not planned —
|
||||||
|
the tool is feature-complete for its purpose. Kept for reference in case a
|
||||||
|
real need ever revives one: Joplin `--force`, per-conversation cache reset,
|
||||||
|
official export-ZIP fallback, o1/o3 reasoning reclassification, Obsidian
|
||||||
|
output, token-expiry notifications (also moot — see §9), and a search
|
||||||
|
command. (Also closed, outside this list: handling Claude `attachments`/
|
||||||
|
`files`, which the canary will flag if they ever appear in real data.)
|
||||||
|
|
||||||
|
## Export `--force` Flag — SHIPPED v0.6.0
|
||||||
|
|
||||||
|
Implemented 2026-06-12: `export --force` passes `force=True` to
|
||||||
|
`cache.get_new_or_updated()`. Shipped alongside a `mark_exported` fix that
|
||||||
|
preserves Joplin links across re-exports, so a forced re-render + `joplin`
|
||||||
|
updates existing notes instead of duplicating them.
|
||||||
|
|
||||||
|
## Joplin `--force` Flag
|
||||||
|
|
||||||
Similarly, add `--force` to the `joplin` command to re-sync all cached
|
Similarly, add `--force` to the `joplin` command to re-sync all cached
|
||||||
conversations to Joplin regardless of whether they've been synced before.
|
conversations to Joplin regardless of whether they've been synced before.
|
||||||
@@ -30,7 +483,7 @@ Useful after making formatting changes to the Markdown exporter.
|
|||||||
Implementation: in `get_joplin_pending()`, return all entries that have a
|
Implementation: in `get_joplin_pending()`, return all entries that have a
|
||||||
`file_path` when `force=True`, ignoring `joplin_synced_at`.
|
`file_path` when `force=True`, ignoring `joplin_synced_at`.
|
||||||
|
|
||||||
## Per-Conversation Cache Reset (v0.2.x)
|
## Per-Conversation Cache Reset
|
||||||
|
|
||||||
Add `cache --reset --conversation <id>` to force re-export or re-sync of a
|
Add `cache --reset --conversation <id>` to force re-export or re-sync of a
|
||||||
single conversation without clearing the entire provider cache.
|
single conversation without clearing the entire provider cache.
|
||||||
@@ -38,9 +491,7 @@ single conversation without clearing the entire provider cache.
|
|||||||
Current workaround: manually edit `~/.ai-chat-exporter/manifest.json` and
|
Current workaround: manually edit `~/.ai-chat-exporter/manifest.json` and
|
||||||
delete the entry, then re-run export.
|
delete the entry, then re-run export.
|
||||||
|
|
||||||
---
|
## Official API Fallback
|
||||||
|
|
||||||
## Official API Fallback (v0.3.0)
|
|
||||||
|
|
||||||
If the unofficial internal web API approach breaks, migrate to official export
|
If the unofficial internal web API approach breaks, migrate to official export
|
||||||
file parsing as a fallback:
|
file parsing as a fallback:
|
||||||
@@ -56,51 +507,17 @@ To add this: implement `src/providers/file_chatgpt.py` and
|
|||||||
`src/providers/file_claude.py`, then add `--input-file` flag to the
|
`src/providers/file_claude.py`, then add `--input-file` flag to the
|
||||||
export command to accept a pre-downloaded export ZIP or JSON.
|
export command to accept a pre-downloaded export ZIP or JSON.
|
||||||
|
|
||||||
---
|
Deprioritized 2026-06-12: the official ChatGPT export does not cover what
|
||||||
|
this user needs (project data), so it isn't a real fallback here.
|
||||||
|
|
||||||
## Rich Content Support (v0.4.0)
|
## Reclassify o1/o3 Reasoning Subparts
|
||||||
|
|
||||||
Currently only text content is exported. Future versions should handle:
|
v0.4.0 leaves dict parts inside `text` content_type messages with shape
|
||||||
|
`{"summary": ..., "content": ...}` rendered as plain text (defensive — the
|
||||||
|
shape was inferred from a code comment, not captured live). Once a real
|
||||||
|
reasoning conversation is captured, reclassify these as `thinking` blocks.
|
||||||
|
|
||||||
### Claude
|
## Obsidian Vault Output
|
||||||
- Artifacts (code, documents, HTML) — export as separate files, link from Markdown
|
|
||||||
- Uploaded images — download and embed or link
|
|
||||||
- Extended thinking/reasoning blocks — include as collapsible sections
|
|
||||||
- Tool call results and web search citations — include as footnotes or appendices
|
|
||||||
|
|
||||||
### ChatGPT
|
|
||||||
- DALL-E generated images — download and embed or link
|
|
||||||
- Code Interpreter outputs — export code and results
|
|
||||||
- File attachments — download and reference
|
|
||||||
- Voice transcripts — include as text
|
|
||||||
|
|
||||||
Implementation note: the normalized message schema already includes a
|
|
||||||
`content_type` field placeholder. When this work begins, extend the schema
|
|
||||||
rather than replacing it. Non-text content already logs a WARNING when
|
|
||||||
encountered so users can see what was skipped.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Scheduled / Watch Mode (v0.5.0)
|
|
||||||
|
|
||||||
Add a `watch` command (or cron integration helper) to run exports automatically
|
|
||||||
on a schedule:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python -m src.main watch --interval 6h # poll every 6 hours
|
|
||||||
```
|
|
||||||
|
|
||||||
This would run `export` + `joplin` in sequence, then sleep. Alternatively,
|
|
||||||
provide a `cron` command that prints the correct crontab line for the user's
|
|
||||||
setup.
|
|
||||||
|
|
||||||
Implementation: simple loop with `time.sleep()`, or emit a crontab entry
|
|
||||||
string that calls the export and joplin commands in sequence. A `--once`
|
|
||||||
flag would do a single run then exit (useful for cron itself).
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Obsidian Vault Output (v0.5.0)
|
|
||||||
|
|
||||||
Add an `obsidian` command (or `--target obsidian` flag) to sync exported
|
Add an `obsidian` command (or `--target obsidian` flag) to sync exported
|
||||||
conversations into an Obsidian vault directory. The current Markdown format
|
conversations into an Obsidian vault directory. The current Markdown format
|
||||||
@@ -116,40 +533,17 @@ Obsidian. An `ObsidianSyncer` class (mirroring `JoplinClient`) would simply
|
|||||||
copy files to the vault directory and maintain a flat or nested folder
|
copy files to the vault directory and maintain a flat or nested folder
|
||||||
structure matching the user's Obsidian setup. No API needed — just file I/O.
|
structure matching the user's Obsidian setup. No API needed — just file I/O.
|
||||||
|
|
||||||
---
|
## Token Expiry Notifications
|
||||||
|
|
||||||
## Joplin Nested Notebooks (future)
|
Moved to the active roadmap as §9 (Token Validity on `doctor`). The original
|
||||||
|
"proactively notify before expiry" idea is blocked by the same finding: a
|
||||||
|
reliable expiry time isn't available client-side (ChatGPT token is encrypted,
|
||||||
|
Claude's is opaque, and `/api/auth/session`'s `expires` looks like a rolling
|
||||||
|
window rather than the real refresh cadence). Any heads-up — a `doctor`
|
||||||
|
line, an `expiry` subcommand, or a `notify-send` nudge — depends on first
|
||||||
|
resolving the §9 open question of what signal is actually trustworthy.
|
||||||
|
|
||||||
Currently notebooks are flat: `ChatGPT - My Project`. Joplin supports nested
|
## Search Command
|
||||||
notebooks via `parent_id`. A future option (`JOPLIN_NESTED_NOTEBOOKS=true`)
|
|
||||||
could create a two-level hierarchy:
|
|
||||||
|
|
||||||
```
|
|
||||||
ChatGPT/
|
|
||||||
My Project/
|
|
||||||
No Project/
|
|
||||||
Claude/
|
|
||||||
Budget Tracker/
|
|
||||||
```
|
|
||||||
|
|
||||||
Implementation: `get_or_create_notebook` would first find/create the provider
|
|
||||||
notebook, then find/create the project notebook as a child.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Token Expiry Notifications (future)
|
|
||||||
|
|
||||||
Proactively warn when a token is close to expiry (within 48h for ChatGPT),
|
|
||||||
rather than only surfacing the warning at startup. Options:
|
|
||||||
|
|
||||||
- Add an `expiry` subcommand that prints token status and exits non-zero if
|
|
||||||
any token is expired or expiring soon (useful in scripts/cron)
|
|
||||||
- Send a desktop notification via `notify-send` (Linux) or `osascript` (macOS)
|
|
||||||
when a token is within 24h of expiry
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Search Command (future)
|
|
||||||
|
|
||||||
Add a `search` command to full-text search across all exported Markdown files:
|
Add a `search` command to full-text search across all exported Markdown files:
|
||||||
|
|
||||||
|
|||||||
@@ -28,6 +28,8 @@ This tool is designed for a single user backing up their own conversations. Do n
|
|||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
|
### Linux / macOS
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git clone <repo-url>
|
git clone <repo-url>
|
||||||
cd ai-chat-exporter
|
cd ai-chat-exporter
|
||||||
@@ -36,6 +38,37 @@ source .venv/bin/activate
|
|||||||
pip install -e ".[dev]"
|
pip install -e ".[dev]"
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Windows
|
||||||
|
|
||||||
|
No admin access required. Run these in **Command Prompt** (`cmd.exe`) — it's the simplest option on Windows because it doesn't have PowerShell's script execution policy restrictions.
|
||||||
|
|
||||||
|
```bat
|
||||||
|
git clone <repo-url>
|
||||||
|
cd ai-chat-exporter
|
||||||
|
python -m venv .venv
|
||||||
|
.venv\Scripts\activate
|
||||||
|
pip install -e ".[dev]"
|
||||||
|
```
|
||||||
|
|
||||||
|
All `ai-chat-exporter` commands work identically in Command Prompt.
|
||||||
|
|
||||||
|
**Using PowerShell instead?** If you prefer PowerShell, you may need to allow script execution first (one-time, current user only):
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
Set-ExecutionPolicy RemoteSigned -Scope CurrentUser
|
||||||
|
```
|
||||||
|
|
||||||
|
Then activate the venv and run commands the same way.
|
||||||
|
|
||||||
|
**Prerequisites:**
|
||||||
|
- Python 3.11 or later — install from [python.org](https://www.python.org/downloads/windows/). During installation, tick **"Add Python to PATH"**.
|
||||||
|
- Git — install from [git-scm.com](https://git-scm.com/) if not already present.
|
||||||
|
|
||||||
|
**Notes:**
|
||||||
|
- The cache manifest and logs are stored in `cache\` inside the install directory — the same as on Linux.
|
||||||
|
- File permission hardening (`chmod 600`) is silently ignored on Windows — not a concern for single-user desktop use.
|
||||||
|
- Joplin Web Clipper runs on `localhost:41184` on all platforms; no configuration changes needed.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## First Run: Run Doctor
|
## First Run: Run Doctor
|
||||||
@@ -43,10 +76,20 @@ pip install -e ".[dev]"
|
|||||||
Before anything else, validate your setup:
|
Before anything else, validate your setup:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
python -m src.main doctor
|
ai-chat-exporter doctor
|
||||||
```
|
```
|
||||||
|
|
||||||
This checks token presence, format, expiry, directory permissions, disk space, and live API connectivity. Fix any failures before proceeding.
|
This checks token presence, format, token health (via the `/api/auth/session` `error` field), directory permissions, disk space, and live API connectivity. Fix any failures before proceeding.
|
||||||
|
|
||||||
|
### Checking for API drift
|
||||||
|
|
||||||
|
The exporter reads ChatGPT's and Claude's undocumented internal web APIs, which can change shape without notice. The worst failure for a backup tool is *silent* — a response change that makes the exporter skip or mis-parse content without erroring. Run the canary to catch that early:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ai-chat-exporter canary
|
||||||
|
```
|
||||||
|
|
||||||
|
It probes one conversation per provider and checks only the fields the parser depends on (a renamed retrieval-tool author that would bypass the content-collapse, a new content type, drifted message fields, ignored attachments). `OK` means the live schema still matches; `WARN`/`ERROR` tells you exactly what changed. Worth running before a large export or on a schedule.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -54,12 +97,22 @@ This checks token presence, format, expiry, directory permissions, disk space, a
|
|||||||
|
|
||||||
Session tokens are how your browser stays logged in. This tool uses them to access your chat history on your behalf.
|
Session tokens are how your browser stays logged in. This tool uses them to access your chat history on your behalf.
|
||||||
|
|
||||||
|
Tokens are entered manually — copy them from your browser's DevTools and run the wizard:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ai-chat-exporter auth
|
||||||
|
```
|
||||||
|
|
||||||
|
The wizard detects your OS, shows the correct DevTools shortcut, and writes the values to `.env` without echoing them to the terminal.
|
||||||
|
|
||||||
|
> **Why no auto-extraction from the browser?** Modern Chromium browsers (Chrome 127+, current Brave) encrypt cookies on Windows with **App-Bound Encryption**: the key is bound to the browser through a SYSTEM-level service, so reading cookies off disk requires Administrator rights and a PsExec-style SYSTEM impersonation that antivirus flags as credential theft — and it still fails on Brave specifically. There is no reliable, non-invasive way to pull these cookies automatically, so the manual DevTools flow below is the supported path.
|
||||||
|
|
||||||
### Token Lifetimes
|
### Token Lifetimes
|
||||||
|
|
||||||
| Provider | Cookie Name | Lifetime | Expiry Detection |
|
| Provider | Cookie Name | Lifetime | Expiry Detection |
|
||||||
|----------|-------------|----------|-----------------|
|
|----------|-------------|----------|-----------------|
|
||||||
| ChatGPT | `__Secure-next-auth.session-token` | ~7 days | JWT `exp` claim (decoded automatically) |
|
| ChatGPT | `__Secure-next-auth.session-token.0` + `.1` | refresh ~weekly | `error` field of `/api/auth/session` — `doctor` reports "ChatGPT token active". The token is an encrypted JWE, so its `exp` is **not** readable client-side, and the `expires` field is a misleading rolling window; the `error` (`RefreshAccessTokenError` when dead) is the honest signal. |
|
||||||
| Claude | `sessionKey` | ~30 days | Only detectable via 401 response |
|
| Claude | `sessionKey` | ~30 days | Opaque token — only detectable via 401 response |
|
||||||
|
|
||||||
### Finding Tokens in Chrome DevTools
|
### Finding Tokens in Chrome DevTools
|
||||||
|
|
||||||
@@ -69,14 +122,18 @@ Session tokens are how your browser stays logged in. This tool uses them to acce
|
|||||||
4. In the left panel, expand **Cookies** and click the site URL
|
4. In the left panel, expand **Cookies** and click the site URL
|
||||||
5. Find the cookie by name and copy its **Value**
|
5. Find the cookie by name and copy its **Value**
|
||||||
|
|
||||||
**ChatGPT:** go to `https://chatgpt.com` → find `__Secure-next-auth.session-token` → copy Value (starts with `eyJ`)
|
**ChatGPT:** go to `https://chatgpt.com` → find **two** cookies:
|
||||||
|
- `__Secure-next-auth.session-token.0` — copy Value (starts with `eyJ`) → `CHATGPT_SESSION_TOKEN`
|
||||||
|
- `__Secure-next-auth.session-token.1` — copy Value → `CHATGPT_SESSION_TOKEN_1`
|
||||||
|
|
||||||
|
ChatGPT splits large session tokens across two cookies to stay under the browser's 4KB cookie limit. Both are required.
|
||||||
|
|
||||||
**Claude:** go to `https://claude.ai` → find `sessionKey` → copy Value
|
**Claude:** go to `https://claude.ai` → find `sessionKey` → copy Value
|
||||||
|
|
||||||
### When Tokens Expire
|
### When Tokens Expire
|
||||||
|
|
||||||
When a token expires you'll see a `401 Unauthorized` error. To refresh:
|
When a token expires you'll see a `401 Unauthorized` error. To refresh:
|
||||||
- Re-run the `auth` wizard: `python -m src.main auth`
|
- Re-run the `auth` wizard: `ai-chat-exporter auth`
|
||||||
- Or manually update the value in your `.env` file
|
- Or manually update the value in your `.env` file
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -86,7 +143,7 @@ When a token expires you'll see a `401 Unauthorized` error. To refresh:
|
|||||||
The easiest way to configure tokens is the interactive wizard:
|
The easiest way to configure tokens is the interactive wizard:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
python -m src.main auth
|
ai-chat-exporter auth
|
||||||
```
|
```
|
||||||
|
|
||||||
This walks you through finding your token, validates it, shows the expiry date (ChatGPT only), and offers to write it to your `.env` automatically. Tokens are never echoed to the terminal.
|
This walks you through finding your token, validates it, shows the expiry date (ChatGPT only), and offers to write it to your `.env` automatically. Tokens are never echoed to the terminal.
|
||||||
@@ -105,7 +162,8 @@ cp .env.example .env
|
|||||||
|
|
||||||
| Variable | Description |
|
| Variable | Description |
|
||||||
|----------|-------------|
|
|----------|-------------|
|
||||||
| `CHATGPT_SESSION_TOKEN` | Your ChatGPT JWT session token (`eyJ…`) |
|
| `CHATGPT_SESSION_TOKEN` | ChatGPT session token chunk `.0` (starts with `eyJ…`) |
|
||||||
|
| `CHATGPT_SESSION_TOKEN_1` | ChatGPT session token chunk `.1` (the remainder) |
|
||||||
| `CHATGPT_PROJECT_IDS` | Comma-separated ChatGPT project IDs (see below) |
|
| `CHATGPT_PROJECT_IDS` | Comma-separated ChatGPT project IDs (see below) |
|
||||||
| `CLAUDE_SESSION_KEY` | Your Claude session key |
|
| `CLAUDE_SESSION_KEY` | Your Claude session key |
|
||||||
|
|
||||||
@@ -115,6 +173,10 @@ cp .env.example .env
|
|||||||
|----------|---------|-------------|
|
|----------|---------|-------------|
|
||||||
| `EXPORT_DIR` | `./exports` | Where to write exported Markdown files |
|
| `EXPORT_DIR` | `./exports` | Where to write exported Markdown files |
|
||||||
| `OUTPUT_STRUCTURE` | `provider/project/year` | Folder structure (see below) |
|
| `OUTPUT_STRUCTURE` | `provider/project/year` | Folder structure (see below) |
|
||||||
|
| `EXPORTER_HIDDEN_CONTENT` | `placeholder` | What to do with content invisible in the provider's web UI (file-retrieval tool dumps, Custom Instructions): `placeholder` collapses to a one-line note with size, `full` keeps everything, `omit` drops it. Dumps can be 90% of a conversation's bytes. |
|
||||||
|
| `EXPORTER_DOWNLOAD_MEDIA` | `images` | Download conversation assets into a `media/` folder beside each export and inline them: `images` (images only), `all` (also audio/voice clips), `off` (text placeholders only). Downloaded media is uploaded to Joplin as resources on the next `joplin` run. |
|
||||||
|
| `MAX_CONVERSATIONS_PER_RUN` | unlimited | Cap downloads per export run (per provider). Runs are resumable, so re-running continues where the cap stopped — useful to spread a large first export across several sessions. |
|
||||||
|
| `REQUEST_DELAY` | `1.0` | Seconds between consecutive API requests, with small random jitter, so traffic stays human-paced. Set `0` to disable. |
|
||||||
|
|
||||||
### Joplin
|
### Joplin
|
||||||
|
|
||||||
@@ -128,8 +190,8 @@ cp .env.example .env
|
|||||||
|
|
||||||
| Variable | Default | Description |
|
| Variable | Default | Description |
|
||||||
|----------|---------|-------------|
|
|----------|---------|-------------|
|
||||||
| `CACHE_DIR` | `~/.ai-chat-exporter` | Where to store the sync manifest |
|
| `CACHE_DIR` | `./cache` | Where to store the sync manifest |
|
||||||
| `LOG_FILE` | `~/.ai-chat-exporter/logs/exporter.log` | Log file path (`none` to disable) |
|
| `LOG_FILE` | `./cache/logs/exporter.log` | Log file path (`none` to disable) |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -154,6 +216,31 @@ The `auth` wizard can also guide you through this step interactively.
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
## Claude Code Sessions
|
||||||
|
|
||||||
|
The `claude-code` provider archives your local [Claude Code](https://claude.com/claude-code) agent transcripts — no tokens, no API, no ToS exposure. Sessions are read from `~/.claude/projects/`.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ai-chat-exporter export --provider claude-code
|
||||||
|
ai-chat-exporter joplin --provider claude-code
|
||||||
|
```
|
||||||
|
|
||||||
|
Exports are prose-only by default: your prompts and Claude's write-ups are kept, tool activity is grouped into one-line placeholders (`> 🔧 Tool output — 14 calls: Read ×9, Bash ×2 (86KB) — omitted`), and internal reasoning is dropped (counted in the run summary). Set `EXPORTER_HIDDEN_CONTENT=full` to keep everything. Sessions become notes under their own top-level **`AI-ClaudeCode`** Joplin notebook, in a sub-notebook per launch folder. The provider is included in `--provider all` whenever a sessions directory exists.
|
||||||
|
|
||||||
|
**Subagents.** Claude Code stores subagent (Task-tool) transcripts as separate files under `<session>/subagents/`; each is folded into its parent session inline, as a collapsible `<details>` block labeled with the subagent's type and description (its own tool traffic is collapsed like the main dialogue).
|
||||||
|
|
||||||
|
**Repo tags.** Sessions launched from a workspace root all share one folder-named notebook, so titles carry the repos each session touched — `Resume StartWRT project work [start-technologies]` — for at-a-glance scanning and search. A file's repo is the git repository it lives in (nearest ancestor with a `.git`), resolved from the tool paths in the transcript, so work is tagged wherever it happened — even across workspaces — and config/one-off files are ignored (they aren't repos). Repos are frequency-ordered and capped at 3. To never tag specific repos, set `CLAUDE_CODE_REPO_TAG_IGNORE` (comma-separated). Note this reads your current git layout, so a repo you later delete or move drops from the tag on re-export.
|
||||||
|
|
||||||
|
**Multiple locations.** By default the provider scans `~/.claude/projects/` plus `$CLAUDE_CONFIG_DIR/projects` when `CLAUDE_CONFIG_DIR` is set. To scan additional roots (e.g. other machines' sessions copied onto this box), set `CLAUDE_CODE_DIR` to a `:`-separated list:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
CLAUDE_CODE_DIR="$HOME/.claude/projects:/mnt/backup/laptop/.claude/projects"
|
||||||
|
```
|
||||||
|
|
||||||
|
Sessions from all roots are merged by folder (no per-machine label); if the same session UUID appears in two roots, the newer copy wins. Note the exporter only sees this machine's disk and cannot recover sessions Claude Code has already pruned — run it regularly.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## Output Structure
|
## Output Structure
|
||||||
|
|
||||||
All exported files go under `EXPORT_DIR`. The folder structure maps directly to Joplin notebooks.
|
All exported files go under `EXPORT_DIR`. The folder structure maps directly to Joplin notebooks.
|
||||||
@@ -218,15 +305,15 @@ Each provider+project combination maps to a flat Joplin notebook created automat
|
|||||||
### `auth` — Interactive token setup
|
### `auth` — Interactive token setup
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
python -m src.main auth
|
ai-chat-exporter auth # manual wizard (DevTools flow)
|
||||||
```
|
```
|
||||||
|
|
||||||
Guided wizard to find and save session tokens and ChatGPT project IDs. Detects OS and shows the correct DevTools shortcut.
|
Guided wizard to find and save session tokens and ChatGPT project IDs. Detects OS and shows the correct DevTools shortcut, then writes the values to `.env`.
|
||||||
|
|
||||||
### `doctor` — Health check
|
### `doctor` — Health check
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
python -m src.main doctor
|
ai-chat-exporter doctor
|
||||||
```
|
```
|
||||||
|
|
||||||
Checks: token presence, JWT validity and expiry, directory permissions, disk space, live API reachability. Exits with code 0 if all pass, 1 if any fail.
|
Checks: token presence, JWT validity and expiry, directory permissions, disk space, live API reachability. Exits with code 0 if all pass, 1 if any fail.
|
||||||
@@ -235,49 +322,64 @@ Checks: token presence, JWT validity and expiry, directory permissions, disk spa
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Export everything (new/updated only)
|
# Export everything (new/updated only)
|
||||||
python -m src.main export
|
ai-chat-exporter export
|
||||||
|
|
||||||
# Single provider
|
# Single provider
|
||||||
python -m src.main export --provider claude
|
ai-chat-exporter export --provider claude
|
||||||
|
|
||||||
# JSON output
|
# JSON output
|
||||||
python -m src.main export --format json
|
ai-chat-exporter export --format json
|
||||||
|
|
||||||
# Both Markdown and JSON
|
# Both Markdown and JSON
|
||||||
python -m src.main export --format both
|
ai-chat-exporter export --format both
|
||||||
|
|
||||||
# Only conversations updated since a date
|
# Only conversations updated since a date
|
||||||
python -m src.main export --since 2024-06-01
|
ai-chat-exporter export --since 2024-06-01
|
||||||
|
|
||||||
# Only conversations in a specific project (case-insensitive substring)
|
# Only conversations in a specific project (case-insensitive substring)
|
||||||
python -m src.main export --project "learning python"
|
ai-chat-exporter export --project "learning python"
|
||||||
|
|
||||||
# Only conversations outside any project
|
# Only conversations outside any project
|
||||||
python -m src.main export --project none
|
ai-chat-exporter export --project none
|
||||||
|
|
||||||
# Write to a custom directory
|
# Write to a custom directory
|
||||||
python -m src.main export --output /path/to/my/notes
|
ai-chat-exporter export --output /path/to/my/notes
|
||||||
|
|
||||||
# Preview without writing anything
|
# Preview without writing anything
|
||||||
python -m src.main export --dry-run
|
ai-chat-exporter export --dry-run
|
||||||
```
|
```
|
||||||
|
|
||||||
Options: `--provider [chatgpt|claude|all]`, `--format [markdown|json|both]`, `--output PATH`, `--since YYYY-MM-DD`, `--project NAME`, `--dry-run`
|
Options: `--provider [chatgpt|claude|claude-code|all]`, `--format [markdown|json|both]`, `--output PATH`, `--since YYYY-MM-DD`, `--project NAME`, `--hidden-content [full|placeholder|omit]`, `--download-media [images|all|off]`, `--max-conversations N`, `--force`, `--dry-run`
|
||||||
|
|
||||||
|
**Re-rendering the whole archive after an upgrade.** New formatting or features (collapse policy, media downloads) only change conversations as they're re-exported. To re-render everything you already have, use `--force` — it re-exports every conversation even if unchanged, **without** `cache --clear`, so your Joplin note links are preserved (a later `joplin` run updates the existing notes instead of duplicating them).
|
||||||
|
|
||||||
|
A force re-render runs as a tracked "campaign": each run re-renders the least-recently-exported conversations, the "still to go" count shrinks each run, and finished providers do no further work. The simplest approach is one uncapped pass:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ai-chat-exporter export --force # re-renders everything in one paced pass
|
||||||
|
ai-chat-exporter joplin # update notes + upload media
|
||||||
|
```
|
||||||
|
|
||||||
|
Or spread the load with `--max-conversations`, re-running until it reports "Force re-render complete":
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ai-chat-exporter export --force --max-conversations 50 # repeat until complete
|
||||||
|
```
|
||||||
|
|
||||||
### `list` — List conversations
|
### `list` — List conversations
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# List all conversations for all providers
|
# List all conversations for all providers
|
||||||
python -m src.main list
|
ai-chat-exporter list
|
||||||
|
|
||||||
# Single provider
|
# Single provider
|
||||||
python -m src.main list --provider chatgpt
|
ai-chat-exporter list --provider chatgpt
|
||||||
|
|
||||||
# Filter by project
|
# Filter by project
|
||||||
python -m src.main list --project "learning python"
|
ai-chat-exporter list --project "learning python"
|
||||||
|
|
||||||
# Only conversations outside any project
|
# Only conversations outside any project
|
||||||
python -m src.main list --project none
|
ai-chat-exporter list --project none
|
||||||
```
|
```
|
||||||
|
|
||||||
Fetches and displays all conversations without exporting them. Useful for verifying what the tool can see before running an export.
|
Fetches and displays all conversations without exporting them. Useful for verifying what the tool can see before running an export.
|
||||||
@@ -286,19 +388,19 @@ Fetches and displays all conversations without exporting them. Useful for verify
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Sync all pending conversations to Joplin
|
# Sync all pending conversations to Joplin
|
||||||
python -m src.main joplin
|
ai-chat-exporter joplin
|
||||||
|
|
||||||
# Preview what would be synced without sending anything
|
# Preview what would be synced without sending anything
|
||||||
python -m src.main joplin --dry-run
|
ai-chat-exporter joplin --dry-run
|
||||||
|
|
||||||
# Sync a single provider
|
# Sync a single provider
|
||||||
python -m src.main joplin --provider chatgpt
|
ai-chat-exporter joplin --provider chatgpt
|
||||||
|
|
||||||
# Sync only conversations in a specific project
|
# Sync only conversations in a specific project
|
||||||
python -m src.main joplin --project "learning python"
|
ai-chat-exporter joplin --project "learning python"
|
||||||
|
|
||||||
# Sync only conversations outside any project
|
# Sync only conversations outside any project
|
||||||
python -m src.main joplin --project none
|
ai-chat-exporter joplin --project none
|
||||||
```
|
```
|
||||||
|
|
||||||
Reads the local export cache and pushes each exported Markdown file to Joplin as a note. Notebooks are created automatically. Re-running is safe — notes are updated (not duplicated).
|
Reads the local export cache and pushes each exported Markdown file to Joplin as a note. Notebooks are created automatically. Re-running is safe — notes are updated (not duplicated).
|
||||||
@@ -311,24 +413,40 @@ Reads the local export cache and pushes each exported Markdown file to Joplin as
|
|||||||
|
|
||||||
Options: `--provider [chatgpt|claude|all]`, `--project NAME`, `--dry-run`
|
Options: `--provider [chatgpt|claude|all]`, `--project NAME`, `--dry-run`
|
||||||
|
|
||||||
|
### `prune` — Delete stale export files
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Preview what would be deleted
|
||||||
|
ai-chat-exporter prune --dry-run
|
||||||
|
|
||||||
|
# Delete (asks for confirmation; -y skips the prompt)
|
||||||
|
ai-chat-exporter prune
|
||||||
|
```
|
||||||
|
|
||||||
|
Deletes export files no longer referenced by the cache manifest — leftovers
|
||||||
|
from old folder layouts or fixed bugs that would otherwise double-sync into
|
||||||
|
Joplin. Refuses to run when the manifest is empty (e.g. right after
|
||||||
|
`cache --clear`) so it can never wipe a freshly cleared archive. The `doctor`
|
||||||
|
command separately verifies that every manifest entry's file exists on disk.
|
||||||
|
|
||||||
### `cache` — Manage the sync manifest
|
### `cache` — Manage the sync manifest
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Show statistics
|
# Show statistics
|
||||||
python -m src.main cache --show
|
ai-chat-exporter cache --show
|
||||||
|
|
||||||
# Clear all cached entries (forces full re-export next run)
|
# Clear all cached entries (forces full re-export next run)
|
||||||
python -m src.main cache --clear
|
ai-chat-exporter cache --clear
|
||||||
|
|
||||||
# Clear a single provider
|
# Clear a single provider
|
||||||
python -m src.main cache --clear --provider claude
|
ai-chat-exporter cache --clear --provider claude
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## How the Cache Works
|
## How the Cache Works
|
||||||
|
|
||||||
The cache manifest lives at `~/.ai-chat-exporter/manifest.json` and records every exported conversation: its title, project, `updated_at` timestamp, output file path, and (after Joplin sync) the Joplin note ID.
|
The cache manifest lives at `cache/manifest.json` (inside the install directory) and records every exported conversation: its title, project, `updated_at` timestamp, output file path, and (after Joplin sync) the Joplin note ID.
|
||||||
|
|
||||||
On every `export` run:
|
On every `export` run:
|
||||||
1. Fetch the full conversation list from the provider
|
1. Fetch the full conversation list from the provider
|
||||||
@@ -343,7 +461,7 @@ On every `joplin` run:
|
|||||||
|
|
||||||
**This design makes every run inherently resumable.** If the tool is interrupted for any reason — rate limit, network drop, Ctrl+C, crash — simply re-run the same command. It will skip already-processed conversations and continue from where it stopped.
|
**This design makes every run inherently resumable.** If the tool is interrupted for any reason — rate limit, network drop, Ctrl+C, crash — simply re-run the same command. It will skip already-processed conversations and continue from where it stopped.
|
||||||
|
|
||||||
To force a full re-export: `python -m src.main cache --clear` then re-run export.
|
To force a full re-export: `ai-chat-exporter cache --clear` then re-run export.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -351,7 +469,7 @@ To force a full re-export: `python -m src.main cache --clear` then re-run export
|
|||||||
|
|
||||||
### `401 Unauthorized`
|
### `401 Unauthorized`
|
||||||
Your session token has expired.
|
Your session token has expired.
|
||||||
- Run `python -m src.main auth` to get a new token interactively
|
- Run `ai-chat-exporter auth` to get a new token interactively
|
||||||
- Or manually copy a fresh cookie value into your `.env` file
|
- Or manually copy a fresh cookie value into your `.env` file
|
||||||
|
|
||||||
Note: Claude's `sessionKey` is an opaque string — the only way to know it's expired is the 401 error. ChatGPT JWTs have an `exp` claim that the `doctor` command can decode and display.
|
Note: Claude's `sessionKey` is an opaque string — the only way to know it's expired is the 401 error. ChatGPT JWTs have an `exp` claim that the `doctor` command can decode and display.
|
||||||
@@ -388,13 +506,19 @@ Make sure you've added the project IDs to `CHATGPT_PROJECT_IDS` in your `.env`.
|
|||||||
The provider's internal API may have changed. Run with `--debug`, sanitize the output (remove any personal content), and check the project's GitHub Issues for known fixes.
|
The provider's internal API may have changed. Run with `--debug`, sanitize the output (remove any personal content), and check the project's GitHub Issues for known fixes.
|
||||||
|
|
||||||
### Non-text content warnings
|
### Non-text content warnings
|
||||||
Images, code interpreter outputs, DALL-E generations, and Claude artifacts are not exported in v0.2.0. A WARNING is logged for each skipped item. See `FUTURE.md` for the roadmap.
|
Since v0.4.0, rich content is preserved as typed blocks in the export. ChatGPT voice transcripts render as text and audio assets as `📎 File attached` placeholders with size and duration metadata. Anything the extractor doesn't recognise renders as a visible `> ⚠️ Unsupported content` block naming the type and observed keys, *and* increments a counter in the post-export summary so you can tell whether real content is being silently skipped.
|
||||||
|
|
||||||
|
### Embedded images and media
|
||||||
|
Since v0.6.0, image attachments are downloaded by default into a `media/` folder beside each export and inlined as ``. Audio and other files download only with `EXPORTER_DOWNLOAD_MEDIA=all`. On the next `joplin` run these are uploaded as Joplin resources so they render inside the note. Assets that have expired on the provider's side (older AI-generated images often do) keep their text placeholder and are tallied as `media failed` in the run summary — they're never fatal to the export. Set `EXPORTER_DOWNLOAD_MEDIA=off` to skip downloads entirely.
|
||||||
|
|
||||||
|
### Tool output / hidden context collapsed
|
||||||
|
Since v0.6.0, content that was invisible in the ChatGPT web UI is collapsed by default to one-line placeholders like `> 🔧 Tool output — file_search (15.0 KB) — omitted`. This is ChatGPT's file-retrieval tool re-injecting your attached files on every run — it can be 90% of a conversation's bytes while containing none of the dialogue. Custom Instructions collapse the same way (`> ℹ️ Hidden context`). The post-export summary lists every collapsed message by origin with the total KB omitted. To keep everything, set `EXPORTER_HIDDEN_CONTENT=full` in `.env` or pass `--hidden-content full`.
|
||||||
|
|
||||||
### Empty export / all conversations skipped
|
### Empty export / all conversations skipped
|
||||||
No new or updated conversations since your last run. To verify: `python -m src.main cache --show`. To force a full re-export: `python -m src.main cache --clear`.
|
No new or updated conversations since your last run. To verify: `ai-chat-exporter cache --show`. To force a full re-export: `ai-chat-exporter cache --clear`.
|
||||||
|
|
||||||
### Filing a bug report
|
### Filing a bug report
|
||||||
1. Run with `--debug`: `python -m src.main export --debug 2>&1 | tee debug.log`
|
1. Run with `--debug`: `ai-chat-exporter export --debug 2>&1 | tee debug.log`
|
||||||
2. Remove any personal conversation content from `debug.log`
|
2. Remove any personal conversation content from `debug.log`
|
||||||
3. Open a GitHub Issue with the sanitized log and the exact command you ran
|
3. Open a GitHub Issue with the sanitized log and the exact command you ran
|
||||||
|
|
||||||
@@ -402,12 +526,9 @@ No new or updated conversations since your last run. To verify: `python -m src.m
|
|||||||
|
|
||||||
## Future Work
|
## Future Work
|
||||||
|
|
||||||
See `FUTURE.md` for planned features:
|
See `FUTURE.md` for the full roadmap. Current priorities:
|
||||||
|
|
||||||
- **v0.2.x** — `export --force` flag; `joplin --force` flag; per-conversation cache reset
|
- **Watch/scheduled mode** on the way to a headless StartOS service
|
||||||
- **v0.3.0** — Official API fallback: parse export ZIP files from ChatGPT/Claude settings
|
|
||||||
- **v0.4.0** — Rich content: images, artifacts, code interpreter output, extended thinking
|
|
||||||
- **v0.5.0** — Watch/scheduled mode; Obsidian vault output
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "ai-chat-exporter"
|
name = "ai-chat-exporter"
|
||||||
version = "0.2.0"
|
version = "0.8.0"
|
||||||
description = "Export ChatGPT and Claude conversation history to Markdown for personal archival in Joplin"
|
description = "Export ChatGPT and Claude conversation history to Markdown for personal archival in Joplin"
|
||||||
requires-python = ">=3.11"
|
requires-python = ">=3.11"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
|||||||
+451
@@ -0,0 +1,451 @@
|
|||||||
|
"""Typed content blocks for normalized messages.
|
||||||
|
|
||||||
|
Providers produce ordered lists of blocks; exporters render them. Living outside
|
||||||
|
``src/providers/`` deliberately — blocks are a separate concern from extraction
|
||||||
|
or rendering, shared by both layers.
|
||||||
|
|
||||||
|
Block dicts always have ``type`` set to one of the BLOCK_TYPE_* constants.
|
||||||
|
Construct via the ``make_*`` helpers; never build dicts by hand. The ``unknown``
|
||||||
|
block constructor REQUIRES a corresponding WARNING log + ``LossReport`` tally
|
||||||
|
at the call site — see plan §Data-loss visibility.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
BLOCK_TYPE_TEXT = "text"
|
||||||
|
BLOCK_TYPE_CODE = "code"
|
||||||
|
BLOCK_TYPE_THINKING = "thinking"
|
||||||
|
BLOCK_TYPE_TOOL_USE = "tool_use"
|
||||||
|
BLOCK_TYPE_TOOL_RESULT = "tool_result"
|
||||||
|
BLOCK_TYPE_CITATION = "citation"
|
||||||
|
BLOCK_TYPE_IMAGE_PLACEHOLDER = "image_placeholder"
|
||||||
|
BLOCK_TYPE_FILE_PLACEHOLDER = "file_placeholder"
|
||||||
|
BLOCK_TYPE_UNKNOWN = "unknown"
|
||||||
|
BLOCK_TYPE_HIDDEN_CONTEXT_MARKER = "hidden_context_marker"
|
||||||
|
BLOCK_TYPE_COLLAPSED = "collapsed"
|
||||||
|
BLOCK_TYPE_SUBAGENT = "subagent"
|
||||||
|
|
||||||
|
COLLAPSED_KIND_TOOL_DUMP = "tool_dump"
|
||||||
|
COLLAPSED_KIND_HIDDEN_CONTEXT = "hidden_context"
|
||||||
|
|
||||||
|
UNKNOWN_REASON_UNKNOWN_TYPE = "unknown_type"
|
||||||
|
UNKNOWN_REASON_EXTRACTION_FAILED = "extraction_failed"
|
||||||
|
UNKNOWN_REASON_ALL_BLOCKS_FAILED = "all_blocks_failed"
|
||||||
|
UNKNOWN_REASON_UNKNOWN_FIELD_IN_KNOWN_TYPE = "unknown_field_in_known_type"
|
||||||
|
|
||||||
|
_OBSERVED_KEYS_LIMIT = 10
|
||||||
|
|
||||||
|
# Role labels for the turns rendered inside a folded subagent <details> block.
|
||||||
|
_SUBAGENT_ROLE_LABELS = {
|
||||||
|
"user": "🧑 Human",
|
||||||
|
"assistant": "🤖 Assistant",
|
||||||
|
"tool": "🔧 Tool",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Constructors
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def make_text_block(text: str) -> dict | None:
|
||||||
|
"""Return a text block, or None if the text is empty/whitespace-only.
|
||||||
|
|
||||||
|
Returning None lets callers do ``if block: blocks.append(block)`` and prune
|
||||||
|
empty blocks at construction time. See plan §Finalizing the message dict.
|
||||||
|
"""
|
||||||
|
if not isinstance(text, str) or not text.strip():
|
||||||
|
return None
|
||||||
|
return {"type": BLOCK_TYPE_TEXT, "text": text}
|
||||||
|
|
||||||
|
|
||||||
|
def make_code_block(code: str, language: str = "") -> dict | None:
|
||||||
|
"""Return a code block, or None if code is empty."""
|
||||||
|
if not isinstance(code, str) or not code.strip():
|
||||||
|
return None
|
||||||
|
return {"type": BLOCK_TYPE_CODE, "language": language or "", "code": code}
|
||||||
|
|
||||||
|
|
||||||
|
def make_thinking_block(text: str) -> dict | None:
|
||||||
|
"""Return a thinking block, or None if empty."""
|
||||||
|
if not isinstance(text, str) or not text.strip():
|
||||||
|
return None
|
||||||
|
return {"type": BLOCK_TYPE_THINKING, "text": text}
|
||||||
|
|
||||||
|
|
||||||
|
def make_tool_use_block(name: str, input_data: Any, tool_id: str | None = None) -> dict:
|
||||||
|
"""Return a tool_use block.
|
||||||
|
|
||||||
|
Always returns a block (no None) — tool calls are meaningful even with
|
||||||
|
empty inputs.
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"type": BLOCK_TYPE_TOOL_USE,
|
||||||
|
"name": name or "",
|
||||||
|
"input": input_data if input_data is not None else {},
|
||||||
|
"tool_id": tool_id,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def make_tool_result_block(
|
||||||
|
output: str,
|
||||||
|
tool_name: str | None = None,
|
||||||
|
is_error: bool = False,
|
||||||
|
summary: str | None = None,
|
||||||
|
) -> dict:
|
||||||
|
"""Return a tool_result block.
|
||||||
|
|
||||||
|
``summary`` is an optional short human label rendered between header and
|
||||||
|
fence (e.g. ChatGPT's ``metadata.reasoning_title`` for execution_output).
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"type": BLOCK_TYPE_TOOL_RESULT,
|
||||||
|
"tool_name": tool_name,
|
||||||
|
"output": output if isinstance(output, str) else str(output),
|
||||||
|
"is_error": bool(is_error),
|
||||||
|
"summary": summary,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def make_citation_block(
|
||||||
|
url: str,
|
||||||
|
title: str | None = None,
|
||||||
|
snippet: str | None = None,
|
||||||
|
) -> dict | None:
|
||||||
|
if not url:
|
||||||
|
return None
|
||||||
|
return {
|
||||||
|
"type": BLOCK_TYPE_CITATION,
|
||||||
|
"url": url,
|
||||||
|
"title": title,
|
||||||
|
"snippet": snippet,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def make_image_placeholder(
|
||||||
|
ref: str,
|
||||||
|
source: str = "unknown",
|
||||||
|
mime: str | None = None,
|
||||||
|
) -> dict:
|
||||||
|
"""source ∈ {'user_upload', 'model_generated', 'unknown'}."""
|
||||||
|
return {
|
||||||
|
"type": BLOCK_TYPE_IMAGE_PLACEHOLDER,
|
||||||
|
"ref": ref or "",
|
||||||
|
"source": source,
|
||||||
|
"mime": mime,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def make_file_placeholder(
|
||||||
|
ref: str,
|
||||||
|
filename: str | None = None,
|
||||||
|
mime: str | None = None,
|
||||||
|
size_bytes: int | None = None,
|
||||||
|
duration_seconds: float | None = None,
|
||||||
|
) -> dict:
|
||||||
|
return {
|
||||||
|
"type": BLOCK_TYPE_FILE_PLACEHOLDER,
|
||||||
|
"ref": ref or "",
|
||||||
|
"filename": filename,
|
||||||
|
"mime": mime,
|
||||||
|
"size_bytes": size_bytes,
|
||||||
|
"duration_seconds": duration_seconds,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def make_unknown_block(
|
||||||
|
raw_type: str,
|
||||||
|
observed_keys: list[str] | None = None,
|
||||||
|
reason: str = UNKNOWN_REASON_UNKNOWN_TYPE,
|
||||||
|
summary: str | None = None,
|
||||||
|
) -> dict:
|
||||||
|
"""Construct an unknown block.
|
||||||
|
|
||||||
|
Every call site MUST also emit a WARNING log and increment a LossReport
|
||||||
|
tally — see plan §Data-loss visibility. The block surfaces the loss at
|
||||||
|
read time; the WARNING surfaces it at run time. Both signals matter.
|
||||||
|
"""
|
||||||
|
keys = list(observed_keys or [])[:_OBSERVED_KEYS_LIMIT]
|
||||||
|
return {
|
||||||
|
"type": BLOCK_TYPE_UNKNOWN,
|
||||||
|
"raw_type": raw_type,
|
||||||
|
"observed_keys": keys,
|
||||||
|
"reason": reason,
|
||||||
|
"summary": summary,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def make_collapsed_block(
|
||||||
|
origin: str,
|
||||||
|
content_type: str,
|
||||||
|
size_bytes: int,
|
||||||
|
kind: str,
|
||||||
|
) -> dict:
|
||||||
|
"""One-line stand-in for a message omitted by the EXPORTER_HIDDEN_CONTENT policy.
|
||||||
|
|
||||||
|
``kind`` ∈ {COLLAPSED_KIND_TOOL_DUMP, COLLAPSED_KIND_HIDDEN_CONTEXT}.
|
||||||
|
Unlike ``unknown`` blocks (unexpected loss), a collapsed block records an
|
||||||
|
intentional policy decision — the content was invisible in the provider's
|
||||||
|
web UI and the user chose not to keep it. Every call site MUST tally via
|
||||||
|
``LossReport.record_collapsed`` so the omission stays visible in the
|
||||||
|
post-export summary.
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"type": BLOCK_TYPE_COLLAPSED,
|
||||||
|
"origin": origin or "",
|
||||||
|
"content_type": content_type or "",
|
||||||
|
"size_bytes": int(size_bytes or 0),
|
||||||
|
"kind": kind,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def make_subagent_block(
|
||||||
|
agent_type: str,
|
||||||
|
description: str,
|
||||||
|
messages: list[dict],
|
||||||
|
) -> dict:
|
||||||
|
"""A folded Claude Code subagent (Task-tool) transcript.
|
||||||
|
|
||||||
|
Claude Code stores subagent transcripts as separate ``subagents/*.jsonl``
|
||||||
|
files; the provider folds each one into its parent session at the point the
|
||||||
|
``Task``/``Agent`` tool spawned it. ``messages`` is a normal list of
|
||||||
|
``{role, blocks}`` dicts (the same shape as a conversation's messages),
|
||||||
|
extracted under the same hidden-content policy as the main dialogue — so the
|
||||||
|
subagent's own tool traffic is already collapsed inside these blocks.
|
||||||
|
|
||||||
|
Rendered as a collapsible ``<details>`` element so the main conversation
|
||||||
|
stays readable while the deliverable is preserved (and stays searchable).
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"type": BLOCK_TYPE_SUBAGENT,
|
||||||
|
"agent_type": agent_type or "",
|
||||||
|
"description": description or "",
|
||||||
|
"messages": messages or [],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def make_hidden_context_marker(content_type: str) -> dict:
|
||||||
|
"""A short prepend block that flags the surrounding message as hidden context.
|
||||||
|
|
||||||
|
Driven by the ``metadata.is_visually_hidden_from_conversation`` flag, not by
|
||||||
|
content_type matching. The marker tells the reader "this message is
|
||||||
|
hidden in the source UI; we're showing it here for archival fidelity."
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"type": BLOCK_TYPE_HIDDEN_CONTEXT_MARKER,
|
||||||
|
"content_type": content_type or "",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Rendering
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def render_blocks_to_markdown(blocks: list[dict]) -> str:
|
||||||
|
"""Render an ordered list of blocks to a single Markdown string.
|
||||||
|
|
||||||
|
Blocks are joined with one blank line between them. Pure function; no I/O.
|
||||||
|
"""
|
||||||
|
if not blocks:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
rendered: list[str] = []
|
||||||
|
for block in blocks:
|
||||||
|
chunk = _render_one(block)
|
||||||
|
if chunk:
|
||||||
|
rendered.append(chunk)
|
||||||
|
|
||||||
|
return "\n\n".join(rendered)
|
||||||
|
|
||||||
|
|
||||||
|
def _render_one(block: dict) -> str:
|
||||||
|
btype = block.get("type", "")
|
||||||
|
if btype == BLOCK_TYPE_TEXT:
|
||||||
|
return block.get("text", "")
|
||||||
|
if btype == BLOCK_TYPE_CODE:
|
||||||
|
lang = block.get("language") or ""
|
||||||
|
code = block.get("code", "")
|
||||||
|
fence = _safe_fence(code)
|
||||||
|
return f"{fence}{lang}\n{code}\n{fence}"
|
||||||
|
if btype == BLOCK_TYPE_THINKING:
|
||||||
|
text = block.get("text", "")
|
||||||
|
quoted = _blockquote_prefix(text)
|
||||||
|
return f"**💭 Reasoning**\n\n{quoted}"
|
||||||
|
if btype == BLOCK_TYPE_TOOL_USE:
|
||||||
|
name = block.get("name", "")
|
||||||
|
input_data = block.get("input", {})
|
||||||
|
body_json = json.dumps(input_data, indent=2, sort_keys=False, default=str, ensure_ascii=False)
|
||||||
|
fence = _safe_fence(body_json)
|
||||||
|
body = f"{fence}json\n{body_json}\n{fence}"
|
||||||
|
quoted = _blockquote_prefix(f"🔧 **Tool: {name}**\n{body}")
|
||||||
|
return quoted
|
||||||
|
if btype == BLOCK_TYPE_TOOL_RESULT:
|
||||||
|
output = block.get("output", "")
|
||||||
|
is_error = bool(block.get("is_error"))
|
||||||
|
tool_name = block.get("tool_name") or ""
|
||||||
|
summary = block.get("summary") or ""
|
||||||
|
icon = "❌" if is_error else "📤"
|
||||||
|
label = "Result (error)" if is_error else "Result"
|
||||||
|
if tool_name:
|
||||||
|
header = f"{icon} **{label}: {tool_name}**"
|
||||||
|
else:
|
||||||
|
header = f"{icon} **{label}**"
|
||||||
|
fence = _safe_fence(output)
|
||||||
|
body = f"{fence}\n{output}\n{fence}"
|
||||||
|
if summary:
|
||||||
|
inner = f"{header}\n*{summary}*\n{body}"
|
||||||
|
else:
|
||||||
|
inner = f"{header}\n{body}"
|
||||||
|
return _blockquote_prefix(inner)
|
||||||
|
if btype == BLOCK_TYPE_CITATION:
|
||||||
|
url = block.get("url", "")
|
||||||
|
title = block.get("title") or url
|
||||||
|
return f"[{title}]({url})"
|
||||||
|
if btype == BLOCK_TYPE_IMAGE_PLACEHOLDER:
|
||||||
|
ref = block.get("ref", "")
|
||||||
|
source = block.get("source", "unknown")
|
||||||
|
mime = block.get("mime")
|
||||||
|
local_path = block.get("local_path")
|
||||||
|
if local_path:
|
||||||
|
# Downloaded — render as a real inline image.
|
||||||
|
return f""
|
||||||
|
meta_parts = [source] if source else []
|
||||||
|
if mime:
|
||||||
|
meta_parts.append(mime)
|
||||||
|
meta_parts.append("content not preserved in this export")
|
||||||
|
meta = ", ".join(meta_parts)
|
||||||
|
return f"> 🖼️ **Image attached** — `{ref}` ({meta})"
|
||||||
|
if btype == BLOCK_TYPE_FILE_PLACEHOLDER:
|
||||||
|
ref = block.get("ref", "")
|
||||||
|
filename = block.get("filename")
|
||||||
|
label = filename or ref
|
||||||
|
mime = block.get("mime")
|
||||||
|
size_bytes = block.get("size_bytes")
|
||||||
|
duration = block.get("duration_seconds")
|
||||||
|
local_path = block.get("local_path")
|
||||||
|
meta_parts: list[str] = []
|
||||||
|
if mime:
|
||||||
|
meta_parts.append(mime)
|
||||||
|
size_label = _format_size(size_bytes)
|
||||||
|
if size_label:
|
||||||
|
meta_parts.append(size_label)
|
||||||
|
if isinstance(duration, (int, float)) and duration > 0:
|
||||||
|
meta_parts.append(f"{duration:.2f}s")
|
||||||
|
if local_path:
|
||||||
|
# Downloaded — render as a link to the local copy.
|
||||||
|
meta = ", ".join(meta_parts) if meta_parts else ""
|
||||||
|
meta_str = f" ({meta})" if meta else ""
|
||||||
|
return f"> 📎 **File attached** — [{Path(local_path).name}]({local_path}){meta_str}"
|
||||||
|
meta_parts.append("content not preserved in this export")
|
||||||
|
meta = ", ".join(meta_parts)
|
||||||
|
return f"> 📎 **File attached** — `{label}` ({meta})"
|
||||||
|
if btype == BLOCK_TYPE_UNKNOWN:
|
||||||
|
raw_type = block.get("raw_type", "?")
|
||||||
|
reason = block.get("reason", UNKNOWN_REASON_UNKNOWN_TYPE)
|
||||||
|
keys = block.get("observed_keys") or []
|
||||||
|
summary = block.get("summary")
|
||||||
|
first_line = f"⚠️ **Unsupported content** — type `{raw_type}` ({reason})"
|
||||||
|
lines = [first_line]
|
||||||
|
if summary:
|
||||||
|
lines.append(summary)
|
||||||
|
if keys:
|
||||||
|
keys_str = ", ".join(f"`{k}`" for k in keys)
|
||||||
|
lines.append(f"Keys observed: {keys_str}")
|
||||||
|
return _blockquote_prefix("\n".join(lines))
|
||||||
|
if btype == BLOCK_TYPE_SUBAGENT:
|
||||||
|
agent_type = block.get("agent_type") or "subagent"
|
||||||
|
description = block.get("description") or ""
|
||||||
|
summary = f"🤖 Subagent: {agent_type}"
|
||||||
|
if description:
|
||||||
|
summary += f" — {description}"
|
||||||
|
parts = ["<details>", f"<summary>{summary}</summary>", ""]
|
||||||
|
for msg in block.get("messages") or []:
|
||||||
|
body = render_blocks_to_markdown(msg.get("blocks") or [])
|
||||||
|
if not body.strip():
|
||||||
|
continue
|
||||||
|
role = msg.get("role", "assistant")
|
||||||
|
label = _SUBAGENT_ROLE_LABELS.get(role, f"💬 {role.capitalize()}")
|
||||||
|
parts.append(f"**{label}**")
|
||||||
|
parts.append("")
|
||||||
|
parts.append(body)
|
||||||
|
parts.append("")
|
||||||
|
parts.append("</details>")
|
||||||
|
return "\n".join(parts)
|
||||||
|
if btype == BLOCK_TYPE_HIDDEN_CONTEXT_MARKER:
|
||||||
|
ctype = block.get("content_type", "")
|
||||||
|
return f"> ℹ️ **Hidden context** — `{ctype}`"
|
||||||
|
if btype == BLOCK_TYPE_COLLAPSED:
|
||||||
|
origin = block.get("origin", "")
|
||||||
|
kind = block.get("kind", "")
|
||||||
|
if kind == COLLAPSED_KIND_HIDDEN_CONTEXT:
|
||||||
|
icon, label = "ℹ️", "Hidden context"
|
||||||
|
else:
|
||||||
|
icon, label = "🔧", "Tool output"
|
||||||
|
size_label = _format_size(block.get("size_bytes"))
|
||||||
|
size_part = f" ({size_label})" if size_label else ""
|
||||||
|
return (
|
||||||
|
f"> {icon} **{label}** — `{origin}`{size_part} — omitted "
|
||||||
|
"(EXPORTER_HIDDEN_CONTENT=full to keep)"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Defensive: a block of unrecognised local type (shouldn't happen if
|
||||||
|
# constructors are used). Render as visible warning rather than dropping.
|
||||||
|
return f"> ⚠️ **Internal: unrecognised block type** — `{btype}`"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _format_size(size_bytes: Any) -> str:
|
||||||
|
"""Human-readable size ('24.1 KB', '1.05 MB'), or '' when absent/zero."""
|
||||||
|
if not isinstance(size_bytes, int) or size_bytes <= 0:
|
||||||
|
return ""
|
||||||
|
kb = size_bytes / 1024
|
||||||
|
return f"{kb:.1f} KB" if kb < 1024 else f"{kb / 1024:.2f} MB"
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_fence(text: str) -> str:
|
||||||
|
"""Return a backtick fence longer than the longest run of backticks in ``text``.
|
||||||
|
|
||||||
|
CommonMark requires the closing fence to be at least as long as the opening
|
||||||
|
fence. Picking N+1 (where N = longest run in content) ensures the content's
|
||||||
|
own backticks are inert. Minimum is 3.
|
||||||
|
|
||||||
|
Verified live against Joplin during planning — see plan
|
||||||
|
§Backtick-corruption defense.
|
||||||
|
"""
|
||||||
|
if not isinstance(text, str):
|
||||||
|
return "```"
|
||||||
|
longest_run = 0
|
||||||
|
current_run = 0
|
||||||
|
for ch in text:
|
||||||
|
if ch == "`":
|
||||||
|
current_run += 1
|
||||||
|
if current_run > longest_run:
|
||||||
|
longest_run = current_run
|
||||||
|
else:
|
||||||
|
current_run = 0
|
||||||
|
fence_len = max(3, longest_run + 1)
|
||||||
|
return "`" * fence_len
|
||||||
|
|
||||||
|
|
||||||
|
def _blockquote_prefix(text: str) -> str:
|
||||||
|
"""Prefix every line of ``text`` with ``> `` so the whole block renders as a quote.
|
||||||
|
|
||||||
|
Empty source lines become ``>`` (no trailing space) so blockquote continuity
|
||||||
|
is preserved without trailing-whitespace noise.
|
||||||
|
"""
|
||||||
|
if not isinstance(text, str):
|
||||||
|
return ""
|
||||||
|
out_lines: list[str] = []
|
||||||
|
for line in text.split("\n"):
|
||||||
|
if line == "":
|
||||||
|
out_lines.append(">")
|
||||||
|
else:
|
||||||
|
out_lines.append(f"> {line}")
|
||||||
|
return "\n".join(out_lines)
|
||||||
+100
-3
@@ -84,27 +84,70 @@ class Cache:
|
|||||||
if provider not in self._data:
|
if provider not in self._data:
|
||||||
self._data[provider] = {}
|
self._data[provider] = {}
|
||||||
|
|
||||||
self._data[provider][conv_id] = {
|
# Preserve the Joplin link across re-exports. Without this, re-rendering
|
||||||
|
# a conversation (e.g. a forced re-export to pick up new formatting)
|
||||||
|
# would drop joplin_note_id, and the next sync would create a duplicate
|
||||||
|
# note instead of updating the existing one. exported_at is refreshed,
|
||||||
|
# so get_joplin_pending() still flags the note for an update.
|
||||||
|
prev = self._data[provider].get(conv_id) or {}
|
||||||
|
entry = {
|
||||||
"title": metadata.get("title", ""),
|
"title": metadata.get("title", ""),
|
||||||
"project": metadata.get("project"),
|
"project": metadata.get("project"),
|
||||||
|
"created_at": metadata.get("created_at", ""),
|
||||||
"updated_at": metadata.get("updated_at", ""),
|
"updated_at": metadata.get("updated_at", ""),
|
||||||
"exported_at": datetime.now(tz=timezone.utc).isoformat(),
|
"exported_at": datetime.now(tz=timezone.utc).isoformat(),
|
||||||
"file_path": metadata.get("file_path", ""),
|
"file_path": metadata.get("file_path", ""),
|
||||||
}
|
}
|
||||||
|
for carried in ("joplin_note_id", "joplin_synced_at", "joplin_resources"):
|
||||||
|
if carried in prev:
|
||||||
|
entry[carried] = prev[carried]
|
||||||
|
|
||||||
|
self._data[provider][conv_id] = entry
|
||||||
self._data["last_run"] = datetime.now(tz=timezone.utc).isoformat()
|
self._data["last_run"] = datetime.now(tz=timezone.utc).isoformat()
|
||||||
self._save()
|
self._save()
|
||||||
|
|
||||||
def get_new_or_updated(self, provider: str, conversations: list[dict]) -> list[dict]:
|
def get_new_or_updated(
|
||||||
|
self,
|
||||||
|
provider: str,
|
||||||
|
conversations: list[dict],
|
||||||
|
force: bool = False,
|
||||||
|
campaign_at: str | None = None,
|
||||||
|
) -> list[dict]:
|
||||||
"""Filter a conversation list to only new or updated conversations.
|
"""Filter a conversation list to only new or updated conversations.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
provider: "chatgpt" or "claude"
|
provider: "chatgpt" or "claude"
|
||||||
conversations: List of raw conversation dicts from the provider.
|
conversations: List of raw conversation dicts from the provider.
|
||||||
Each must have an ``id``/``uuid`` and ``updated_at``/``update_time``.
|
Each must have an ``id``/``uuid`` and ``updated_at``/``update_time``.
|
||||||
|
force: When True, return conversations to re-render regardless of
|
||||||
|
cache freshness — used by ``export --force``.
|
||||||
|
campaign_at: In force mode, exclude conversations already re-exported
|
||||||
|
at/after this ISO timestamp (the re-render campaign start), so
|
||||||
|
each run advances and finished providers do nothing.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Subset that needs to be exported.
|
Subset that needs to be exported, ordered oldest-export-first in
|
||||||
|
force mode.
|
||||||
"""
|
"""
|
||||||
|
if force:
|
||||||
|
# Re-export for a re-render campaign, ordered least-recently-exported
|
||||||
|
# first (never-exported sorts first). When ``campaign_at`` is given,
|
||||||
|
# conversations already re-exported during this campaign
|
||||||
|
# (exported_at >= campaign_at) are excluded — so finished providers
|
||||||
|
# do no work and the remaining count shrinks to zero. Each capped run
|
||||||
|
# re-renders the next-oldest batch. Conversations without an id drop.
|
||||||
|
entries = self._data.get(provider, {})
|
||||||
|
|
||||||
|
def _exported_at(conv: dict) -> str:
|
||||||
|
conv_id = conv.get("id") or conv.get("uuid", "")
|
||||||
|
entry = entries.get(conv_id) or {}
|
||||||
|
return entry.get("exported_at") or ""
|
||||||
|
|
||||||
|
candidates = [c for c in conversations if c.get("id") or c.get("uuid")]
|
||||||
|
if campaign_at is not None:
|
||||||
|
candidates = [c for c in candidates if _exported_at(c) < campaign_at]
|
||||||
|
return sorted(candidates, key=_exported_at)
|
||||||
|
|
||||||
result = []
|
result = []
|
||||||
for conv in conversations:
|
for conv in conversations:
|
||||||
conv_id = conv.get("id") or conv.get("uuid", "")
|
conv_id = conv.get("id") or conv.get("uuid", "")
|
||||||
@@ -172,6 +215,22 @@ class Cache:
|
|||||||
entry["joplin_synced_at"] = datetime.now(tz=timezone.utc).isoformat()
|
entry["joplin_synced_at"] = datetime.now(tz=timezone.utc).isoformat()
|
||||||
self._save()
|
self._save()
|
||||||
|
|
||||||
|
def get_joplin_resources(self, provider: str, conv_id: str) -> dict[str, str]:
|
||||||
|
"""Return the {relative_media_path: joplin_resource_id} map for a conversation."""
|
||||||
|
entry = self._data.get(provider, {}).get(conv_id)
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
return {}
|
||||||
|
resources = entry.get("joplin_resources")
|
||||||
|
return dict(resources) if isinstance(resources, dict) else {}
|
||||||
|
|
||||||
|
def set_joplin_resources(self, provider: str, conv_id: str, resources: dict[str, str]) -> None:
|
||||||
|
"""Persist the {relative_media_path: joplin_resource_id} map for a conversation."""
|
||||||
|
entry = self._data.get(provider, {}).get(conv_id)
|
||||||
|
if entry is None:
|
||||||
|
return
|
||||||
|
entry["joplin_resources"] = dict(resources)
|
||||||
|
self._save()
|
||||||
|
|
||||||
def get_joplin_pending(self, provider: str) -> list[tuple[str, dict]]:
|
def get_joplin_pending(self, provider: str) -> list[tuple[str, dict]]:
|
||||||
"""Return (conv_id, entry) pairs that need to be synced to Joplin.
|
"""Return (conv_id, entry) pairs that need to be synced to Joplin.
|
||||||
|
|
||||||
@@ -209,10 +268,48 @@ class Cache:
|
|||||||
|
|
||||||
return pending
|
return pending
|
||||||
|
|
||||||
|
def all_file_paths(self) -> set[str]:
|
||||||
|
"""Return every non-empty ``file_path`` in the manifest, across providers.
|
||||||
|
|
||||||
|
Used by ``prune`` (files on disk but not here are stale) and by the
|
||||||
|
doctor integrity check (files here but not on disk are missing).
|
||||||
|
"""
|
||||||
|
paths: set[str] = set()
|
||||||
|
for key, entries in self._data.items():
|
||||||
|
if not isinstance(entries, dict) or key in (
|
||||||
|
"version",
|
||||||
|
"last_run",
|
||||||
|
"tos_acknowledged_at",
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
for entry in entries.values():
|
||||||
|
if isinstance(entry, dict) and entry.get("file_path"):
|
||||||
|
paths.add(entry["file_path"])
|
||||||
|
return paths
|
||||||
|
|
||||||
def last_run(self) -> str | None:
|
def last_run(self) -> str | None:
|
||||||
"""Return the ISO8601 timestamp of the last export run, or None."""
|
"""Return the ISO8601 timestamp of the last export run, or None."""
|
||||||
return self._data.get("last_run")
|
return self._data.get("last_run")
|
||||||
|
|
||||||
|
def exported_at(self, provider: str, conv_id: str) -> str:
|
||||||
|
"""Return the recorded ``exported_at`` for a conversation, or '' if absent."""
|
||||||
|
entry = self._data.get(provider, {}).get(conv_id)
|
||||||
|
return entry.get("exported_at", "") if isinstance(entry, dict) else ""
|
||||||
|
|
||||||
|
# Force re-render campaign marker (top-level scalar; skipped by stats/clear,
|
||||||
|
# which only act on dict-valued provider keys).
|
||||||
|
def get_force_campaign(self) -> str | None:
|
||||||
|
"""Return the active force-re-render campaign start timestamp, or None."""
|
||||||
|
return self._data.get("force_campaign_at")
|
||||||
|
|
||||||
|
def set_force_campaign(self, started_at: str) -> None:
|
||||||
|
self._data["force_campaign_at"] = started_at
|
||||||
|
self._save()
|
||||||
|
|
||||||
|
def clear_force_campaign(self) -> None:
|
||||||
|
if self._data.pop("force_campaign_at", None) is not None:
|
||||||
|
self._save()
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
# Private helpers
|
# Private helpers
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
|
|||||||
+82
-5
@@ -20,6 +20,12 @@ _CLAUDE_PLACEHOLDER = ""
|
|||||||
# Valid OUTPUT_STRUCTURE values
|
# Valid OUTPUT_STRUCTURE values
|
||||||
VALID_STRUCTURES = {"provider/project/year", "provider/project", "provider/year"}
|
VALID_STRUCTURES = {"provider/project/year", "provider/project", "provider/year"}
|
||||||
|
|
||||||
|
# Valid EXPORTER_HIDDEN_CONTENT values (see src/providers/base.py)
|
||||||
|
VALID_HIDDEN_CONTENT = {"full", "placeholder", "omit"}
|
||||||
|
|
||||||
|
# Valid EXPORTER_DOWNLOAD_MEDIA values (see src/media.py)
|
||||||
|
VALID_DOWNLOAD_MEDIA = {"images", "all", "off"}
|
||||||
|
|
||||||
|
|
||||||
class ConfigError(Exception):
|
class ConfigError(Exception):
|
||||||
"""Raised when required configuration is missing or invalid."""
|
"""Raised when required configuration is missing or invalid."""
|
||||||
@@ -28,6 +34,7 @@ class ConfigError(Exception):
|
|||||||
@dataclass
|
@dataclass
|
||||||
class Config:
|
class Config:
|
||||||
chatgpt_session_token: str | None
|
chatgpt_session_token: str | None
|
||||||
|
chatgpt_session_token_1: str | None
|
||||||
claude_session_key: str | None
|
claude_session_key: str | None
|
||||||
export_dir: Path
|
export_dir: Path
|
||||||
output_structure: str
|
output_structure: str
|
||||||
@@ -42,6 +49,16 @@ class Config:
|
|||||||
# Joplin local REST API settings (Web Clipper service)
|
# Joplin local REST API settings (Web Clipper service)
|
||||||
joplin_api_token: str | None = None
|
joplin_api_token: str | None = None
|
||||||
joplin_api_url: str = "http://localhost:41184"
|
joplin_api_url: str = "http://localhost:41184"
|
||||||
|
# Policy for content invisible in the provider web UI (retrieval dumps,
|
||||||
|
# hidden context): full | placeholder | omit
|
||||||
|
hidden_content: str = "placeholder"
|
||||||
|
# Session cap: max conversations downloaded per export run (None = unlimited).
|
||||||
|
# Every run is resumable, so a capped run just continues next time.
|
||||||
|
max_conversations: int | None = None
|
||||||
|
# Seconds between consecutive API requests (politeness pacing; 0 disables)
|
||||||
|
request_delay: float = 1.0
|
||||||
|
# Asset download policy: images | all | off
|
||||||
|
download_media: str = "images"
|
||||||
|
|
||||||
|
|
||||||
def load_config() -> Config:
|
def load_config() -> Config:
|
||||||
@@ -55,16 +72,23 @@ def load_config() -> Config:
|
|||||||
load_dotenv(override=False)
|
load_dotenv(override=False)
|
||||||
|
|
||||||
chatgpt_token = os.getenv("CHATGPT_SESSION_TOKEN", "").strip() or None
|
chatgpt_token = os.getenv("CHATGPT_SESSION_TOKEN", "").strip() or None
|
||||||
|
chatgpt_token_1 = os.getenv("CHATGPT_SESSION_TOKEN_1", "").strip() or None
|
||||||
claude_key = os.getenv("CLAUDE_SESSION_KEY", "").strip() or None
|
claude_key = os.getenv("CLAUDE_SESSION_KEY", "").strip() or None
|
||||||
export_dir = Path(os.getenv("EXPORT_DIR", "./exports")).expanduser()
|
export_dir = Path(os.getenv("EXPORT_DIR", "./exports")).expanduser()
|
||||||
output_structure = os.getenv("OUTPUT_STRUCTURE", "provider/project/year").strip()
|
output_structure = os.getenv("OUTPUT_STRUCTURE", "provider/project/year").strip()
|
||||||
cache_dir = Path(os.getenv("CACHE_DIR", "~/.ai-chat-exporter")).expanduser()
|
cache_dir = Path(os.getenv("CACHE_DIR", "./cache")).expanduser()
|
||||||
log_file = os.getenv("LOG_FILE", "~/.ai-chat-exporter/logs/exporter.log").strip()
|
log_file = os.getenv("LOG_FILE", "./cache/logs/exporter.log").strip()
|
||||||
|
|
||||||
# Joplin
|
# Joplin
|
||||||
joplin_token = os.getenv("JOPLIN_API_TOKEN", "").strip() or None
|
joplin_token = os.getenv("JOPLIN_API_TOKEN", "").strip() or None
|
||||||
joplin_url = os.getenv("JOPLIN_API_URL", "http://localhost:41184").strip()
|
joplin_url = os.getenv("JOPLIN_API_URL", "http://localhost:41184").strip()
|
||||||
|
|
||||||
|
hidden_content = os.getenv("EXPORTER_HIDDEN_CONTENT", "").strip().lower() or "placeholder"
|
||||||
|
|
||||||
|
max_conversations_raw = os.getenv("MAX_CONVERSATIONS_PER_RUN", "").strip()
|
||||||
|
request_delay_raw = os.getenv("REQUEST_DELAY", "").strip()
|
||||||
|
download_media = os.getenv("EXPORTER_DOWNLOAD_MEDIA", "").strip().lower() or "images"
|
||||||
|
|
||||||
# Parse CHATGPT_PROJECT_IDS — comma-separated list of gizmo IDs (g-p-xxx)
|
# Parse CHATGPT_PROJECT_IDS — comma-separated list of gizmo IDs (g-p-xxx)
|
||||||
_project_ids_raw = os.getenv("CHATGPT_PROJECT_IDS", "").strip()
|
_project_ids_raw = os.getenv("CHATGPT_PROJECT_IDS", "").strip()
|
||||||
chatgpt_project_ids = [
|
chatgpt_project_ids = [
|
||||||
@@ -88,6 +112,46 @@ def load_config() -> Config:
|
|||||||
f"Must be one of: {', '.join(sorted(VALID_STRUCTURES))}"
|
f"Must be one of: {', '.join(sorted(VALID_STRUCTURES))}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Validate hidden-content policy
|
||||||
|
if hidden_content not in VALID_HIDDEN_CONTENT:
|
||||||
|
errors.append(
|
||||||
|
f"EXPORTER_HIDDEN_CONTENT '{hidden_content}' is invalid. "
|
||||||
|
f"Must be one of: {', '.join(sorted(VALID_HIDDEN_CONTENT))}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Validate media download policy
|
||||||
|
if download_media not in VALID_DOWNLOAD_MEDIA:
|
||||||
|
errors.append(
|
||||||
|
f"EXPORTER_DOWNLOAD_MEDIA '{download_media}' is invalid. "
|
||||||
|
f"Must be one of: {', '.join(sorted(VALID_DOWNLOAD_MEDIA))}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Validate session cap
|
||||||
|
max_conversations: int | None = None
|
||||||
|
if max_conversations_raw:
|
||||||
|
try:
|
||||||
|
max_conversations = int(max_conversations_raw)
|
||||||
|
except ValueError:
|
||||||
|
errors.append(
|
||||||
|
f"MAX_CONVERSATIONS_PER_RUN '{max_conversations_raw}' is not an integer."
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
if max_conversations < 1:
|
||||||
|
errors.append(
|
||||||
|
f"MAX_CONVERSATIONS_PER_RUN must be at least 1 (got {max_conversations})."
|
||||||
|
)
|
||||||
|
|
||||||
|
# Validate request pacing
|
||||||
|
request_delay = 1.0
|
||||||
|
if request_delay_raw:
|
||||||
|
try:
|
||||||
|
request_delay = float(request_delay_raw)
|
||||||
|
except ValueError:
|
||||||
|
errors.append(f"REQUEST_DELAY '{request_delay_raw}' is not a number.")
|
||||||
|
else:
|
||||||
|
if request_delay < 0:
|
||||||
|
errors.append(f"REQUEST_DELAY must be >= 0 (got {request_delay}).")
|
||||||
|
|
||||||
# Validate and decode ChatGPT JWT
|
# Validate and decode ChatGPT JWT
|
||||||
chatgpt_expiry: datetime | None = None
|
chatgpt_expiry: datetime | None = None
|
||||||
if chatgpt_token:
|
if chatgpt_token:
|
||||||
@@ -101,7 +165,7 @@ def load_config() -> Config:
|
|||||||
if not chatgpt_token and not claude_key:
|
if not chatgpt_token and not claude_key:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Neither CHATGPT_SESSION_TOKEN nor CLAUDE_SESSION_KEY is set. "
|
"Neither CHATGPT_SESSION_TOKEN nor CLAUDE_SESSION_KEY is set. "
|
||||||
"Run 'python -m src.main auth' to configure credentials."
|
"Run 'ai-chat-exporter auth' to configure credentials."
|
||||||
)
|
)
|
||||||
|
|
||||||
# Create and validate output directory
|
# Create and validate output directory
|
||||||
@@ -127,6 +191,7 @@ def load_config() -> Config:
|
|||||||
|
|
||||||
config = Config(
|
config = Config(
|
||||||
chatgpt_session_token=chatgpt_token,
|
chatgpt_session_token=chatgpt_token,
|
||||||
|
chatgpt_session_token_1=chatgpt_token_1,
|
||||||
claude_session_key=claude_key,
|
claude_session_key=claude_key,
|
||||||
export_dir=export_dir,
|
export_dir=export_dir,
|
||||||
output_structure=output_structure,
|
output_structure=output_structure,
|
||||||
@@ -136,6 +201,10 @@ def load_config() -> Config:
|
|||||||
chatgpt_project_ids=chatgpt_project_ids,
|
chatgpt_project_ids=chatgpt_project_ids,
|
||||||
joplin_api_token=joplin_token,
|
joplin_api_token=joplin_token,
|
||||||
joplin_api_url=joplin_url,
|
joplin_api_url=joplin_url,
|
||||||
|
hidden_content=hidden_content,
|
||||||
|
max_conversations=max_conversations,
|
||||||
|
request_delay=request_delay,
|
||||||
|
download_media=download_media,
|
||||||
)
|
)
|
||||||
|
|
||||||
_log_startup_summary(config)
|
_log_startup_summary(config)
|
||||||
@@ -173,7 +242,7 @@ def _validate_chatgpt_token(token: str) -> datetime | None:
|
|||||||
if delta.total_seconds() < 0:
|
if delta.total_seconds() < 0:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"CHATGPT_SESSION_TOKEN expired at %s. "
|
"CHATGPT_SESSION_TOKEN expired at %s. "
|
||||||
"Run 'python -m src.main auth' to refresh it.",
|
"Run 'ai-chat-exporter auth' to refresh it.",
|
||||||
expiry.strftime("%Y-%m-%d %H:%M UTC"),
|
expiry.strftime("%Y-%m-%d %H:%M UTC"),
|
||||||
)
|
)
|
||||||
elif delta.total_seconds() < 86400:
|
elif delta.total_seconds() < 86400:
|
||||||
@@ -220,7 +289,11 @@ def _log_startup_summary(cfg: Config) -> None:
|
|||||||
"Joplin: %s | "
|
"Joplin: %s | "
|
||||||
"export_dir=%s | "
|
"export_dir=%s | "
|
||||||
"structure=%s | "
|
"structure=%s | "
|
||||||
"cache_dir=%s",
|
"cache_dir=%s | "
|
||||||
|
"hidden_content=%s | "
|
||||||
|
"max_conversations=%s | "
|
||||||
|
"request_delay=%.1fs | "
|
||||||
|
"download_media=%s",
|
||||||
chatgpt_status,
|
chatgpt_status,
|
||||||
claude_status,
|
claude_status,
|
||||||
len(cfg.chatgpt_project_ids),
|
len(cfg.chatgpt_project_ids),
|
||||||
@@ -228,4 +301,8 @@ def _log_startup_summary(cfg: Config) -> None:
|
|||||||
cfg.export_dir,
|
cfg.export_dir,
|
||||||
cfg.output_structure,
|
cfg.output_structure,
|
||||||
cfg.cache_dir,
|
cfg.cache_dir,
|
||||||
|
cfg.hidden_content,
|
||||||
|
cfg.max_conversations if cfg.max_conversations is not None else "unlimited",
|
||||||
|
cfg.request_delay,
|
||||||
|
cfg.download_media,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ from datetime import datetime, timezone
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
from src.blocks import render_blocks_to_markdown
|
||||||
from src.utils import build_export_path, generate_filename
|
from src.utils import build_export_path, generate_filename
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -15,6 +16,7 @@ _ROLE_LABELS = {
|
|||||||
"user": ("🧑 Human", "user"),
|
"user": ("🧑 Human", "user"),
|
||||||
"assistant": ("🤖 Assistant", "assistant"),
|
"assistant": ("🤖 Assistant", "assistant"),
|
||||||
"system": ("⚙️ System", "system"),
|
"system": ("⚙️ System", "system"),
|
||||||
|
"tool": ("🔧 Tool", "tool"),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -125,10 +127,17 @@ class MarkdownExporter:
|
|||||||
# Messages
|
# Messages
|
||||||
for msg in messages:
|
for msg in messages:
|
||||||
role = msg.get("role", "user")
|
role = msg.get("role", "user")
|
||||||
content = msg.get("content", "")
|
blocks = msg.get("blocks") or []
|
||||||
timestamp = msg.get("timestamp")
|
timestamp = msg.get("timestamp")
|
||||||
|
|
||||||
if not content or not content.strip():
|
# Prefer rendering from blocks (v0.4.0+). Backward-compat fallback:
|
||||||
|
# if blocks is missing/empty AND content exists, render content as-is.
|
||||||
|
if blocks:
|
||||||
|
body = render_blocks_to_markdown(blocks)
|
||||||
|
else:
|
||||||
|
body = msg.get("content", "") or ""
|
||||||
|
|
||||||
|
if not body or not body.strip():
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"[markdown] Skipping empty/whitespace message in conversation %s",
|
"[markdown] Skipping empty/whitespace message in conversation %s",
|
||||||
conv_id[:8],
|
conv_id[:8],
|
||||||
@@ -143,7 +152,7 @@ class MarkdownExporter:
|
|||||||
else:
|
else:
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
||||||
lines.append(content)
|
lines.append(body)
|
||||||
lines.append("")
|
lines.append("")
|
||||||
lines.append("---")
|
lines.append("---")
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|||||||
+152
-28
@@ -1,7 +1,10 @@
|
|||||||
"""Joplin Data API client for importing notes into Joplin desktop."""
|
"""Joplin Data API client for importing notes into Joplin desktop."""
|
||||||
|
|
||||||
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
@@ -32,8 +35,8 @@ class JoplinClient:
|
|||||||
def __init__(self, base_url: str, token: str) -> None:
|
def __init__(self, base_url: str, token: str) -> None:
|
||||||
self._base_url = base_url.rstrip("/")
|
self._base_url = base_url.rstrip("/")
|
||||||
self._token = token
|
self._token = token
|
||||||
# In-memory cache of notebook title → ID to avoid repeated GET /folders
|
# In-memory cache: (parent_id | None, title) → folder ID
|
||||||
self._notebook_cache: dict[str, str] = {}
|
self._notebook_cache: dict[tuple[str | None, str], str] = {}
|
||||||
self._notebooks_loaded = False
|
self._notebooks_loaded = False
|
||||||
logger.debug("[joplin] Client initialised with base_url=%s", self._base_url)
|
logger.debug("[joplin] Client initialised with base_url=%s", self._base_url)
|
||||||
|
|
||||||
@@ -89,13 +92,13 @@ class JoplinClient:
|
|||||||
"""Return all Joplin notebooks (folders), handling pagination.
|
"""Return all Joplin notebooks (folders), handling pagination.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
List of folder dicts with at least ``id`` and ``title`` keys.
|
List of folder dicts with at least ``id``, ``title``, and ``parent_id`` keys.
|
||||||
"""
|
"""
|
||||||
results: list[dict] = []
|
results: list[dict] = []
|
||||||
page = 1
|
page = 1
|
||||||
while True:
|
while True:
|
||||||
logger.debug("[joplin] GET /folders page=%d", page)
|
logger.debug("[joplin] GET /folders page=%d", page)
|
||||||
resp = self._get("/folders", params={"page": page, "fields": "id,title"})
|
resp = self._get("/folders", params={"page": page, "fields": "id,title,parent_id"})
|
||||||
items = resp.get("items", [])
|
items = resp.get("items", [])
|
||||||
results.extend(items)
|
results.extend(items)
|
||||||
logger.debug("[joplin] /folders page=%d → %d items, has_more=%s", page, len(items), resp.get("has_more"))
|
logger.debug("[joplin] /folders page=%d → %d items, has_more=%s", page, len(items), resp.get("has_more"))
|
||||||
@@ -104,11 +107,12 @@ class JoplinClient:
|
|||||||
page += 1
|
page += 1
|
||||||
return results
|
return results
|
||||||
|
|
||||||
def get_or_create_notebook(self, title: str) -> str:
|
def get_or_create_notebook(self, title: str, parent_id: str | None = None) -> str:
|
||||||
"""Return the Joplin folder ID for ``title``, creating it if needed.
|
"""Return the Joplin folder ID for ``title`` under ``parent_id``, creating if needed.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
title: Notebook display name (e.g. "ChatGPT - My Project").
|
title: Notebook display name.
|
||||||
|
parent_id: ID of the parent folder, or None for a root notebook.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Joplin folder ID string.
|
Joplin folder ID string.
|
||||||
@@ -116,19 +120,40 @@ class JoplinClient:
|
|||||||
if not self._notebooks_loaded:
|
if not self._notebooks_loaded:
|
||||||
self._load_notebook_cache()
|
self._load_notebook_cache()
|
||||||
|
|
||||||
if title in self._notebook_cache:
|
key = (parent_id, title)
|
||||||
folder_id = self._notebook_cache[title]
|
if key in self._notebook_cache:
|
||||||
logger.debug("[joplin] Notebook cache hit: %r → %s", title, folder_id)
|
folder_id = self._notebook_cache[key]
|
||||||
|
logger.debug("[joplin] Notebook cache hit: %r (parent=%s) → %s", title, parent_id, folder_id)
|
||||||
return folder_id
|
return folder_id
|
||||||
|
|
||||||
# Not found — create it
|
# Not found — create it
|
||||||
logger.info("[joplin] Creating notebook: %r", title)
|
logger.info("[joplin] Creating notebook: %r (parent=%s)", title, parent_id)
|
||||||
resp = self._post("/folders", {"title": title})
|
data: dict = {"title": title}
|
||||||
|
if parent_id:
|
||||||
|
data["parent_id"] = parent_id
|
||||||
|
resp = self._post("/folders", data)
|
||||||
folder_id = resp["id"]
|
folder_id = resp["id"]
|
||||||
self._notebook_cache[title] = folder_id
|
self._notebook_cache[key] = folder_id
|
||||||
logger.debug("[joplin] Notebook created: %r → %s", title, folder_id)
|
logger.debug("[joplin] Notebook created: %r → %s", title, folder_id)
|
||||||
return folder_id
|
return folder_id
|
||||||
|
|
||||||
|
def get_or_create_notebook_path(self, path: list[str]) -> str:
|
||||||
|
"""Ensure a nested notebook path exists and return the leaf folder ID.
|
||||||
|
|
||||||
|
Creates intermediate notebooks as needed.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
path: Ordered list of notebook names, e.g. ["AI-ChatGPT", "No Project"].
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Joplin folder ID of the deepest (leaf) notebook.
|
||||||
|
"""
|
||||||
|
parent_id: str | None = None
|
||||||
|
for name in path:
|
||||||
|
parent_id = self.get_or_create_notebook(name, parent_id)
|
||||||
|
assert parent_id is not None
|
||||||
|
return parent_id
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
# Notes
|
# Notes
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
@@ -153,21 +178,70 @@ class JoplinClient:
|
|||||||
logger.info("[joplin] Note created: %r → %s", title, note_id)
|
logger.info("[joplin] Note created: %r → %s", title, note_id)
|
||||||
return note_id
|
return note_id
|
||||||
|
|
||||||
def update_note(self, note_id: str, title: str, body: str) -> None:
|
def update_note(
|
||||||
|
self, note_id: str, title: str, body: str, parent_id: str | None = None
|
||||||
|
) -> None:
|
||||||
"""Update the title and body of an existing note.
|
"""Update the title and body of an existing note.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
note_id: Joplin note ID.
|
note_id: Joplin note ID.
|
||||||
title: New note title.
|
title: New note title.
|
||||||
body: New note body (Markdown).
|
body: New note body (Markdown).
|
||||||
|
parent_id: If given, also move the note into this notebook. Joplin's
|
||||||
|
``PUT /notes/:id`` relocates a note when ``parent_id`` is set —
|
||||||
|
this is how notes self-heal to a new notebook mapping (e.g. the
|
||||||
|
Claude Code AI-Claude → AI-ClaudeCode migration) on re-sync.
|
||||||
"""
|
"""
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"[joplin] Updating note %s: %r (%d chars)",
|
"[joplin] Updating note %s: %r (%d chars)",
|
||||||
note_id, title, len(body),
|
note_id, title, len(body),
|
||||||
)
|
)
|
||||||
self._put(f"/notes/{note_id}", {"title": title, "body": body})
|
data: dict = {"title": title, "body": body}
|
||||||
|
if parent_id:
|
||||||
|
data["parent_id"] = parent_id
|
||||||
|
self._put(f"/notes/{note_id}", data)
|
||||||
logger.info("[joplin] Note updated: %r (%s)", title, note_id)
|
logger.info("[joplin] Note updated: %r (%s)", title, note_id)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Resources (embedded media)
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def create_resource(self, file_path: "Path", title: str | None = None) -> str:
|
||||||
|
"""Upload a local file as a Joplin resource and return its ID.
|
||||||
|
|
||||||
|
Joplin's ``POST /resources`` is multipart: a ``data`` file part plus a
|
||||||
|
``props`` JSON part. The returned ID is referenced from note bodies as
|
||||||
|
``:/<id>``.
|
||||||
|
"""
|
||||||
|
url = f"{self._base_url}/resources"
|
||||||
|
props = json.dumps({"title": title or file_path.name})
|
||||||
|
logger.debug("[joplin] POST /resources (%s)", file_path.name)
|
||||||
|
try:
|
||||||
|
with file_path.open("rb") as fh:
|
||||||
|
resp = requests.post(
|
||||||
|
url,
|
||||||
|
params={"token": self._token},
|
||||||
|
files={
|
||||||
|
"data": (file_path.name, fh),
|
||||||
|
"props": (None, props),
|
||||||
|
},
|
||||||
|
timeout=_REQUEST_TIMEOUT,
|
||||||
|
)
|
||||||
|
resp.raise_for_status()
|
||||||
|
resource_id = resp.json()["id"]
|
||||||
|
logger.info("[joplin] Resource created: %s → %s", file_path.name, resource_id)
|
||||||
|
return resource_id
|
||||||
|
except requests.exceptions.ConnectionError as e:
|
||||||
|
raise JoplinError(
|
||||||
|
"Cannot connect to Joplin. Is Joplin desktop running with Web Clipper enabled?"
|
||||||
|
) from e
|
||||||
|
except requests.exceptions.Timeout as e:
|
||||||
|
raise JoplinError(_timeout_message("POST", "/resources")) from e
|
||||||
|
except requests.exceptions.HTTPError as e:
|
||||||
|
raise JoplinError(_http_error_message("POST", "/resources", e)) from e
|
||||||
|
except (requests.exceptions.RequestException, KeyError, OSError) as e:
|
||||||
|
raise JoplinError(f"Joplin resource upload failed for {file_path.name}: {e}") from e
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
# HTTP helpers
|
# HTTP helpers
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
@@ -233,11 +307,14 @@ class JoplinClient:
|
|||||||
def _load_notebook_cache(self) -> None:
|
def _load_notebook_cache(self) -> None:
|
||||||
logger.debug("[joplin] Loading notebook list from Joplin…")
|
logger.debug("[joplin] Loading notebook list from Joplin…")
|
||||||
notebooks = self.list_notebooks()
|
notebooks = self.list_notebooks()
|
||||||
self._notebook_cache = {nb["title"]: nb["id"] for nb in notebooks}
|
self._notebook_cache = {
|
||||||
|
(nb.get("parent_id") or None, nb["title"]): nb["id"]
|
||||||
|
for nb in notebooks
|
||||||
|
}
|
||||||
self._notebooks_loaded = True
|
self._notebooks_loaded = True
|
||||||
logger.debug("[joplin] Notebook cache loaded: %d notebooks", len(self._notebook_cache))
|
logger.debug("[joplin] Notebook cache loaded: %d notebooks", len(self._notebook_cache))
|
||||||
for title, folder_id in self._notebook_cache.items():
|
for (parent_id, title), folder_id in self._notebook_cache.items():
|
||||||
logger.debug("[joplin] %r → %s", title, folder_id)
|
logger.debug("[joplin] (%s) %r → %s", parent_id or "root", title, folder_id)
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
@@ -279,25 +356,72 @@ def _http_error_message(method: str, path: str, e: requests.exceptions.HTTPError
|
|||||||
return f"Joplin {method} {path} failed: HTTP {status}{body_snippet}"
|
return f"Joplin {method} {path} failed: HTTP {status}{body_snippet}"
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Embedded-media rewriting
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
# Markdown image/link targets pointing at the local media/ sibling directory,
|
||||||
|
# e.g.  or [name](media/clip.wav).
|
||||||
|
_MEDIA_LINK_RE = re.compile(r"(!?\[[^\]]*\])\((media/[^)\s]+)\)")
|
||||||
|
|
||||||
|
|
||||||
|
def upload_media_and_rewrite(
|
||||||
|
body: str,
|
||||||
|
note_dir: Path,
|
||||||
|
client: "JoplinClient",
|
||||||
|
resource_map: dict[str, str],
|
||||||
|
) -> tuple[str, dict[str, str]]:
|
||||||
|
"""Rewrite local ``media/…`` links to Joplin ``:/resourceId`` references.
|
||||||
|
|
||||||
|
Uploads each referenced file as a Joplin resource (once), rewriting both
|
||||||
|
image embeds and file links. ``resource_map`` (relative-path → resource_id)
|
||||||
|
is the idempotency record from the cache: known paths are reused, new ones
|
||||||
|
are added and returned so the caller can persist them. Missing files are
|
||||||
|
left as-is so the link still points at the on-disk copy.
|
||||||
|
"""
|
||||||
|
new_map = dict(resource_map)
|
||||||
|
|
||||||
|
def replace(match: re.Match) -> str:
|
||||||
|
label, rel_path = match.group(1), match.group(2)
|
||||||
|
resource_id = new_map.get(rel_path)
|
||||||
|
if resource_id is None:
|
||||||
|
file_path = (note_dir / rel_path).resolve()
|
||||||
|
if not file_path.is_file():
|
||||||
|
logger.warning("[joplin] Media file missing, leaving link: %s", rel_path)
|
||||||
|
return match.group(0)
|
||||||
|
resource_id = client.create_resource(file_path)
|
||||||
|
new_map[rel_path] = resource_id
|
||||||
|
return f"{label}(:/{resource_id})"
|
||||||
|
|
||||||
|
return _MEDIA_LINK_RE.sub(replace, body), new_map
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
# Notebook naming helper
|
# Notebook naming helper
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
_PROVIDER_DISPLAY = {
|
_PROVIDER_DISPLAY = {
|
||||||
"chatgpt": "ChatGPT",
|
"chatgpt": "AI-ChatGPT",
|
||||||
"claude": "Claude",
|
"claude": "AI-Claude",
|
||||||
|
# Decision 2026-07-06: Claude Code coding sessions get their own top-level
|
||||||
|
# notebook (was nested under AI-Claude, which made them hard to find,
|
||||||
|
# intermixed with Claude web projects and named by dev folder). Existing
|
||||||
|
# notes self-heal into here on the next sync — update_note moves them.
|
||||||
|
"claude-code": "AI-ClaudeCode",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def notebook_title(provider: str, project: str | None) -> str:
|
def notebook_path(provider: str, project: str | None) -> tuple[str, str]:
|
||||||
"""Derive a flat Joplin notebook title from provider and project name.
|
"""Return (parent_notebook, child_notebook) for the given provider and project.
|
||||||
|
|
||||||
|
The parent is the top-level provider notebook; the child is the project name.
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
notebook_title("chatgpt", "no-project") → "ChatGPT - No Project"
|
notebook_path("chatgpt", None) → ("AI-ChatGPT", "No Project")
|
||||||
notebook_title("claude", "budget-tracker") → "Claude - Budget Tracker"
|
notebook_path("chatgpt", "no-project") → ("AI-ChatGPT", "No Project")
|
||||||
notebook_title("chatgpt", None) → "ChatGPT - No Project"
|
notebook_path("claude", "budget-tracker") → ("AI-Claude", "Budget Tracker")
|
||||||
"""
|
"""
|
||||||
prov_display = _PROVIDER_DISPLAY.get(provider, provider.capitalize())
|
parent = _PROVIDER_DISPLAY.get(provider, f"AI-{provider.capitalize()}")
|
||||||
proj = (project or "no-project").replace("-", " ").title()
|
child = (project or "no-project").replace("-", " ").title()
|
||||||
return f"{prov_display} - {proj}"
|
return parent, child
|
||||||
|
|||||||
@@ -0,0 +1,123 @@
|
|||||||
|
"""Per-export-run tally for content that was dropped or partially extracted.
|
||||||
|
|
||||||
|
Surfaces the loss visibility that the rest of the system promises in its
|
||||||
|
output (visible ``unknown`` blocks). The summary emitted at the end of
|
||||||
|
each export is the load-bearing operator-facing signal: if a real content
|
||||||
|
type starts being silently dropped, this is where it shows up.
|
||||||
|
|
||||||
|
Pass a single instance through ``BaseProvider.normalize_conversation`` and
|
||||||
|
read it back in ``src/main.py`` after the export loop. No global state.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections import Counter
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
|
||||||
|
_TOP_N_BREAKDOWN = 5
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class LossReport:
|
||||||
|
"""Counters for things that didn't render cleanly in an export run."""
|
||||||
|
|
||||||
|
# Type-keyed counters. Values are int counts.
|
||||||
|
unknown_blocks: Counter = field(default_factory=Counter)
|
||||||
|
extraction_failures: Counter = field(default_factory=Counter)
|
||||||
|
filtered_roles: Counter = field(default_factory=Counter)
|
||||||
|
# Messages collapsed/omitted by the EXPORTER_HIDDEN_CONTENT policy —
|
||||||
|
# intentional, but still surfaced so the omission is never invisible.
|
||||||
|
collapsed: Counter = field(default_factory=Counter)
|
||||||
|
collapsed_bytes: int = 0
|
||||||
|
|
||||||
|
# Aggregate counters
|
||||||
|
messages_rendered: int = 0
|
||||||
|
conversations: int = 0
|
||||||
|
|
||||||
|
# Media downloads (EXPORTER_DOWNLOAD_MEDIA): successes / skips / failures
|
||||||
|
media_downloaded: int = 0
|
||||||
|
media_downloaded_bytes: int = 0
|
||||||
|
media_failed: Counter = field(default_factory=Counter)
|
||||||
|
|
||||||
|
# Recording -------------------------------------------------------------
|
||||||
|
|
||||||
|
def record_unknown(self, raw_type: str) -> None:
|
||||||
|
self.unknown_blocks[raw_type or "?"] += 1
|
||||||
|
|
||||||
|
def record_extraction_failure(self, raw_type: str) -> None:
|
||||||
|
self.extraction_failures[raw_type or "?"] += 1
|
||||||
|
|
||||||
|
def record_filtered_role(self, role: str) -> None:
|
||||||
|
self.filtered_roles[role or "?"] += 1
|
||||||
|
|
||||||
|
def record_collapsed(self, origin: str, size_bytes: int = 0) -> None:
|
||||||
|
self.collapsed[origin or "?"] += 1
|
||||||
|
if isinstance(size_bytes, int) and size_bytes > 0:
|
||||||
|
self.collapsed_bytes += size_bytes
|
||||||
|
|
||||||
|
def record_media_downloaded(self, size_bytes: int = 0) -> None:
|
||||||
|
self.media_downloaded += 1
|
||||||
|
if isinstance(size_bytes, int) and size_bytes > 0:
|
||||||
|
self.media_downloaded_bytes += size_bytes
|
||||||
|
|
||||||
|
def record_media_failed(self, reason: str) -> None:
|
||||||
|
self.media_failed[reason or "?"] += 1
|
||||||
|
|
||||||
|
def record_message(self) -> None:
|
||||||
|
self.messages_rendered += 1
|
||||||
|
|
||||||
|
def record_conversation(self) -> None:
|
||||||
|
self.conversations += 1
|
||||||
|
|
||||||
|
# Summary ---------------------------------------------------------------
|
||||||
|
|
||||||
|
def format_summary(self) -> str:
|
||||||
|
"""Return a multi-line summary table suitable for INFO logging.
|
||||||
|
|
||||||
|
Format pinned by plan §Post-export summary — "(none)" sentinel when a
|
||||||
|
counter is empty, top-5 breakdown with "+ N more types" overflow.
|
||||||
|
"""
|
||||||
|
lines: list[str] = ["[export] Run summary:"]
|
||||||
|
lines.append(f" conversations: {self.conversations}")
|
||||||
|
lines.append(f" messages rendered: {self.messages_rendered}")
|
||||||
|
lines.extend(_format_section("unknown blocks: ", self.unknown_blocks))
|
||||||
|
lines.extend(_format_section("extraction failures: ", self.extraction_failures))
|
||||||
|
lines.extend(_format_section("collapsed by policy: ", self.collapsed))
|
||||||
|
if self.collapsed:
|
||||||
|
lines.append(
|
||||||
|
f" ≈{self.collapsed_bytes / 1024:.0f} KB omitted "
|
||||||
|
"(intentional — EXPORTER_HIDDEN_CONTENT=full to keep)"
|
||||||
|
)
|
||||||
|
if self.media_downloaded or self.media_failed:
|
||||||
|
lines.append(
|
||||||
|
f" media downloaded: {self.media_downloaded} "
|
||||||
|
f"(≈{self.media_downloaded_bytes / 1024:.0f} KB)"
|
||||||
|
)
|
||||||
|
if self.media_failed:
|
||||||
|
total_failed = sum(self.media_failed.values())
|
||||||
|
lines.append(f" media failed: {total_failed}")
|
||||||
|
for reason, count in self.media_failed.most_common(_TOP_N_BREAKDOWN):
|
||||||
|
lines.append(f" {reason}={count}")
|
||||||
|
lines.append(
|
||||||
|
" filtered roles: "
|
||||||
|
"(filter lifted in v0.4.0 — counter retained for future use, expected 0)"
|
||||||
|
)
|
||||||
|
if self.filtered_roles:
|
||||||
|
for role, count in self.filtered_roles.most_common(_TOP_N_BREAKDOWN):
|
||||||
|
lines.append(f" {role}={count}")
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
def _format_section(label: str, counter: Counter) -> list[str]:
|
||||||
|
"""Render one counter section: header line + indented breakdown lines."""
|
||||||
|
total = sum(counter.values())
|
||||||
|
header = f" {label} {total}"
|
||||||
|
if total == 0:
|
||||||
|
return [header, " (none)"]
|
||||||
|
|
||||||
|
lines = [header]
|
||||||
|
most_common = counter.most_common()
|
||||||
|
for raw_type, count in most_common[:_TOP_N_BREAKDOWN]:
|
||||||
|
lines.append(f" {raw_type}={count}")
|
||||||
|
if len(most_common) > _TOP_N_BREAKDOWN:
|
||||||
|
remainder = len(most_common) - _TOP_N_BREAKDOWN
|
||||||
|
lines.append(f" + {remainder} more types")
|
||||||
|
return lines
|
||||||
+518
-83
@@ -2,11 +2,12 @@
|
|||||||
|
|
||||||
import importlib.metadata
|
import importlib.metadata
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
import platform
|
import platform
|
||||||
import shutil
|
import shutil
|
||||||
import sys
|
import sys
|
||||||
import traceback
|
import traceback
|
||||||
from datetime import datetime
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import click
|
import click
|
||||||
@@ -16,7 +17,8 @@ from rich.table import Table
|
|||||||
from src.cache import Cache, CacheError
|
from src.cache import Cache, CacheError
|
||||||
from src.config import ConfigError
|
from src.config import ConfigError
|
||||||
from src.logging_config import setup_logging
|
from src.logging_config import setup_logging
|
||||||
from src.providers.base import ProviderError
|
from src.loss_report import LossReport
|
||||||
|
from src.providers.base import DRIFT_ERROR, DRIFT_OK, DRIFT_WARN, ProviderError
|
||||||
|
|
||||||
console = Console()
|
console = Console()
|
||||||
err_console = Console(stderr=True)
|
err_console = Console(stderr=True)
|
||||||
@@ -60,6 +62,14 @@ def cli(ctx: click.Context, verbose: bool, quiet: bool, debug: bool, no_log_file
|
|||||||
"""Export ChatGPT and Claude conversations to Markdown for personal archival."""
|
"""Export ChatGPT and Claude conversations to Markdown for personal archival."""
|
||||||
ctx.ensure_object(dict)
|
ctx.ensure_object(dict)
|
||||||
|
|
||||||
|
# Load .env BEFORE reading any env var (LOG_FILE, CACHE_DIR). Without this
|
||||||
|
# the cache/log paths fell back to defaults regardless of .env — the cache
|
||||||
|
# in particular silently lived at ./cache even when CACHE_DIR was set.
|
||||||
|
# override=False so a real shell env var still wins over .env.
|
||||||
|
import os
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
load_dotenv(override=False)
|
||||||
|
|
||||||
# Determine console log level
|
# Determine console log level
|
||||||
if debug or verbose:
|
if debug or verbose:
|
||||||
level = logging.DEBUG
|
level = logging.DEBUG
|
||||||
@@ -69,8 +79,7 @@ def cli(ctx: click.Context, verbose: bool, quiet: bool, debug: bool, no_log_file
|
|||||||
level = logging.INFO
|
level = logging.INFO
|
||||||
|
|
||||||
# Determine log file path from env (setup_logging handles "none")
|
# Determine log file path from env (setup_logging handles "none")
|
||||||
import os
|
log_file = os.getenv("LOG_FILE", "./cache/logs/exporter.log")
|
||||||
log_file = os.getenv("LOG_FILE", "~/.ai-chat-exporter/logs/exporter.log")
|
|
||||||
|
|
||||||
setup_logging(level=level, log_file=log_file, no_log_file=no_log_file)
|
setup_logging(level=level, log_file=log_file, no_log_file=no_log_file)
|
||||||
|
|
||||||
@@ -79,7 +88,7 @@ def cli(ctx: click.Context, verbose: bool, quiet: bool, debug: bool, no_log_file
|
|||||||
|
|
||||||
# Initialise cache (needed for ToS gate on every command)
|
# Initialise cache (needed for ToS gate on every command)
|
||||||
import os
|
import os
|
||||||
cache_dir = Path(os.getenv("CACHE_DIR", "~/.ai-chat-exporter")).expanduser()
|
cache_dir = Path(os.getenv("CACHE_DIR", "./cache")).expanduser()
|
||||||
try:
|
try:
|
||||||
cache = Cache(cache_dir)
|
cache = Cache(cache_dir)
|
||||||
except CacheError as e:
|
except CacheError as e:
|
||||||
@@ -140,12 +149,10 @@ def auth(ctx: click.Context) -> None:
|
|||||||
if configure_claude:
|
if configure_claude:
|
||||||
_auth_claude(os_name)
|
_auth_claude(os_name)
|
||||||
|
|
||||||
console.print("\n[green]Done! Run 'python -m src.main doctor' to verify your setup.[/green]")
|
console.print("\n[green]Done! Run 'ai-chat-exporter doctor' to verify your setup.[/green]")
|
||||||
|
|
||||||
|
|
||||||
def _auth_chatgpt(os_name: str) -> None:
|
def _auth_chatgpt(os_name: str) -> None:
|
||||||
import jwt as pyjwt
|
|
||||||
|
|
||||||
console.print("\n[bold]─── ChatGPT ───[/bold]")
|
console.print("\n[bold]─── ChatGPT ───[/bold]")
|
||||||
console.print("1. Open [link=https://chatgpt.com]https://chatgpt.com[/link] and log in.")
|
console.print("1. Open [link=https://chatgpt.com]https://chatgpt.com[/link] and log in.")
|
||||||
if os_name == "Darwin":
|
if os_name == "Darwin":
|
||||||
@@ -153,32 +160,47 @@ def _auth_chatgpt(os_name: str) -> None:
|
|||||||
else:
|
else:
|
||||||
console.print("2. Press [bold]F12[/bold] to open DevTools → Application tab.")
|
console.print("2. Press [bold]F12[/bold] to open DevTools → Application tab.")
|
||||||
console.print("3. Expand [bold]Cookies[/bold] → [bold]https://chatgpt.com[/bold]")
|
console.print("3. Expand [bold]Cookies[/bold] → [bold]https://chatgpt.com[/bold]")
|
||||||
console.print("4. Find [bold]__Secure-next-auth.session-token[/bold] → copy the Value.")
|
console.print("4. ChatGPT splits the session token across two cookies:")
|
||||||
console.print(" (Token starts with 'eyJ...' — it is a long JWT string)")
|
console.print(" [bold]__Secure-next-auth.session-token.0[/bold] (starts with 'eyJ')")
|
||||||
console.print("5. Paste it below (input is hidden).\n")
|
console.print(" [bold]__Secure-next-auth.session-token.1[/bold] (the remainder)")
|
||||||
|
console.print(" Copy each Value in turn and paste below.")
|
||||||
|
console.print(" (If you only see one cookie without a .0/.1 suffix, paste it for .0 and leave .1 blank.)\n")
|
||||||
|
|
||||||
token = click.prompt("ChatGPT session token", hide_input=True, default="", show_default=False).strip()
|
token = click.prompt("ChatGPT session token (.0)", hide_input=True, default="", show_default=False).strip()
|
||||||
if not token:
|
if not token:
|
||||||
console.print("[yellow]Skipped ChatGPT token.[/yellow]")
|
console.print("[yellow]Skipped ChatGPT token.[/yellow]")
|
||||||
return
|
return
|
||||||
|
|
||||||
# Validate
|
token_1 = click.prompt("ChatGPT session token (.1, leave blank if absent)", hide_input=True, default="", show_default=False).strip() or None
|
||||||
|
|
||||||
|
# Format hint only — the token is a JWE whose expiry can't be decoded
|
||||||
|
# client-side, so validity is judged by the live check below (the
|
||||||
|
# /api/auth/session `error` field), not by parsing the token. See §9.
|
||||||
if not token.startswith("eyJ"):
|
if not token.startswith("eyJ"):
|
||||||
console.print("[yellow]Warning: token doesn't look like a JWT (expected 'eyJ...').[/yellow]")
|
console.print("[yellow]Warning: token doesn't look like a JWT (expected 'eyJ...').[/yellow]")
|
||||||
|
|
||||||
expiry_str = ""
|
# Live validation — exchange session token for an access token
|
||||||
|
_valid = False
|
||||||
|
_error: str | None = None
|
||||||
|
with console.status("[dim]Validating token with ChatGPT API…[/dim]"):
|
||||||
try:
|
try:
|
||||||
payload = pyjwt.decode(token, options={"verify_signature": False})
|
from src.providers.chatgpt import ChatGPTProvider
|
||||||
exp = payload.get("exp")
|
_prov = ChatGPTProvider(session_token=token, session_token_1=token_1)
|
||||||
if exp:
|
_prov._fetch_access_token()
|
||||||
from datetime import timezone
|
_valid = True
|
||||||
expiry = datetime.fromtimestamp(exp, tz=timezone.utc)
|
except ProviderError as e:
|
||||||
expiry_str = expiry.strftime("%Y-%m-%d %H:%M UTC")
|
_error = str(e.original)
|
||||||
console.print(f"[green]Token decoded — expires: {expiry_str}[/green]")
|
except Exception as e:
|
||||||
except Exception:
|
_error = str(e)
|
||||||
console.print("[yellow]Could not decode token expiry.[/yellow]")
|
|
||||||
|
if _valid:
|
||||||
|
console.print("[green]✓ Token verified — connected to ChatGPT API.[/green]")
|
||||||
|
else:
|
||||||
|
console.print(f"[red]✗ Token validation failed: {_error}[/red]")
|
||||||
|
|
||||||
_write_token_to_env("CHATGPT_SESSION_TOKEN", token)
|
_write_token_to_env("CHATGPT_SESSION_TOKEN", token)
|
||||||
|
if token_1:
|
||||||
|
_write_token_to_env("CHATGPT_SESSION_TOKEN_1", token_1)
|
||||||
|
|
||||||
# --- ChatGPT Projects ---
|
# --- ChatGPT Projects ---
|
||||||
console.print("\n[bold]ChatGPT Projects (optional)[/bold]")
|
console.print("\n[bold]ChatGPT Projects (optional)[/bold]")
|
||||||
@@ -231,15 +253,37 @@ def _auth_claude(os_name: str) -> None:
|
|||||||
console.print("[yellow]Skipped Claude token.[/yellow]")
|
console.print("[yellow]Skipped Claude token.[/yellow]")
|
||||||
return
|
return
|
||||||
|
|
||||||
console.print("[green]Claude session key saved.[/green]")
|
# Live validation — fetch org ID (the first call any Claude operation makes)
|
||||||
|
_valid = False
|
||||||
|
_error: str | None = None
|
||||||
|
with console.status("[dim]Validating token with Claude API…[/dim]"):
|
||||||
|
try:
|
||||||
|
from src.providers.claude import ClaudeProvider
|
||||||
|
_prov = ClaudeProvider(session_key=key)
|
||||||
|
_prov._get_org_id()
|
||||||
|
_valid = True
|
||||||
|
except ProviderError as e:
|
||||||
|
_error = str(e.original)
|
||||||
|
except Exception as e:
|
||||||
|
_error = str(e)
|
||||||
|
|
||||||
|
if _valid:
|
||||||
|
console.print("[green]✓ Token verified — connected to Claude API.[/green]")
|
||||||
|
else:
|
||||||
|
console.print(f"[red]✗ Token validation failed: {_error}[/red]")
|
||||||
|
|
||||||
_write_token_to_env("CLAUDE_SESSION_KEY", key)
|
_write_token_to_env("CLAUDE_SESSION_KEY", key)
|
||||||
|
|
||||||
|
|
||||||
def _write_token_to_env(key: str, value: str) -> None:
|
def _write_token_to_env(key: str, value: str) -> None:
|
||||||
"""Write or update a key in .env, offering to create the file if it doesn't exist."""
|
"""Write or update a key in .env, offering to create the file if it doesn't exist."""
|
||||||
env_path = Path(".env")
|
|
||||||
|
|
||||||
if click.confirm(f"Write {key} to .env?", default=True):
|
if click.confirm(f"Write {key} to .env?", default=True):
|
||||||
|
_set_env_key(key, value)
|
||||||
|
|
||||||
|
|
||||||
|
def _set_env_key(key: str, value: str) -> None:
|
||||||
|
"""Write or update a key in .env without prompting (creates .env if needed)."""
|
||||||
|
env_path = Path(".env")
|
||||||
if not env_path.exists():
|
if not env_path.exists():
|
||||||
# Create from example if available
|
# Create from example if available
|
||||||
example = Path(".env.example")
|
example = Path(".env.example")
|
||||||
@@ -264,7 +308,6 @@ def _write_token_to_env(key: str, value: str) -> None:
|
|||||||
new_lines.append(f"\n{key}={value}\n")
|
new_lines.append(f"\n{key}={value}\n")
|
||||||
|
|
||||||
env_path.write_text("".join(new_lines), encoding="utf-8")
|
env_path.write_text("".join(new_lines), encoding="utf-8")
|
||||||
import os
|
|
||||||
os.chmod(env_path, 0o600)
|
os.chmod(env_path, 0o600)
|
||||||
console.print(f"[green]{key} written to .env (permissions: 600)[/green]")
|
console.print(f"[green]{key} written to .env (permissions: 600)[/green]")
|
||||||
|
|
||||||
@@ -282,18 +325,105 @@ def doctor(ctx: click.Context) -> None:
|
|||||||
Checks token presence, format, expiry, directory permissions, disk space,
|
Checks token presence, format, expiry, directory permissions, disk space,
|
||||||
and live API reachability. Exits with code 1 if any checks fail.
|
and live API reachability. Exits with code 1 if any checks fail.
|
||||||
"""
|
"""
|
||||||
checks = _run_doctor_checks()
|
checks = _run_doctor_checks(cache=ctx.obj["cache"])
|
||||||
_print_doctor_table(checks)
|
_print_doctor_table(checks)
|
||||||
|
|
||||||
if any(not c["pass"] for c in checks):
|
if any(not c["pass"] for c in checks):
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
def _run_doctor_checks() -> list[dict]:
|
@cli.command()
|
||||||
"""Run all doctor checks and return results."""
|
@click.option(
|
||||||
|
"--provider",
|
||||||
|
type=click.Choice(["chatgpt", "claude", "all"]),
|
||||||
|
default="all",
|
||||||
|
help="Which web-API provider(s) to probe.",
|
||||||
|
)
|
||||||
|
@click.pass_context
|
||||||
|
def canary(ctx: click.Context, provider: str) -> None:
|
||||||
|
"""Probe provider APIs for schema drift against the fields the parser needs.
|
||||||
|
|
||||||
|
Fetches one listing page + one conversation per provider and checks only
|
||||||
|
the normalizer's load-bearing fields (not the full response shape, which
|
||||||
|
churns harmlessly). Surfaces the silent-failure risks for a backup tool:
|
||||||
|
a renamed retrieval-tool author bypassing the hidden-content collapse, a
|
||||||
|
new content_type, drifted message fields, or ignored attachments.
|
||||||
|
|
||||||
|
Exits non-zero if any ERROR finding is reported (a load-bearing field is
|
||||||
|
missing/mistyped). WARN findings are surfaced but do not fail. See
|
||||||
|
FUTURE.md §10.
|
||||||
|
"""
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
|
||||||
|
load_dotenv(override=False)
|
||||||
|
|
||||||
|
chatgpt_token = os.getenv("CHATGPT_SESSION_TOKEN", "").strip() or None
|
||||||
|
claude_key = os.getenv("CLAUDE_SESSION_KEY", "").strip() or None
|
||||||
|
|
||||||
|
targets: list[tuple[str, object]] = []
|
||||||
|
if provider in ("chatgpt", "all") and chatgpt_token:
|
||||||
|
from src.providers.chatgpt import ChatGPTProvider
|
||||||
|
|
||||||
|
ct1 = os.getenv("CHATGPT_SESSION_TOKEN_1", "").strip() or None
|
||||||
|
targets.append(
|
||||||
|
("chatgpt", lambda: ChatGPTProvider(chatgpt_token, session_token_1=ct1))
|
||||||
|
)
|
||||||
|
if provider in ("claude", "all") and claude_key:
|
||||||
|
from src.providers.claude import ClaudeProvider
|
||||||
|
|
||||||
|
targets.append(("claude", lambda: ClaudeProvider(claude_key)))
|
||||||
|
|
||||||
|
if not targets:
|
||||||
|
err_console.print(
|
||||||
|
"[yellow]No web-API provider tokens configured "
|
||||||
|
"(set CHATGPT_SESSION_TOKEN / CLAUDE_SESSION_KEY).[/yellow]"
|
||||||
|
)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
rows: list[tuple[str, dict]] = []
|
||||||
|
for name, factory in targets:
|
||||||
|
try:
|
||||||
|
findings = factory().check_drift()
|
||||||
|
except ProviderError as e:
|
||||||
|
findings = [{"severity": DRIFT_ERROR, "check": "probe", "detail": str(e.original)[:100]}]
|
||||||
|
except Exception as e:
|
||||||
|
findings = [{"severity": DRIFT_ERROR, "check": "probe", "detail": str(e)[:100]}]
|
||||||
|
for f in findings:
|
||||||
|
rows.append((name, f))
|
||||||
|
|
||||||
|
_print_canary_table(rows)
|
||||||
|
if any(f["severity"] == DRIFT_ERROR for _, f in rows):
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
def _print_canary_table(rows: list[tuple[str, dict]]) -> None:
|
||||||
|
table = Table(title="API Drift Canary", show_header=True)
|
||||||
|
table.add_column("Provider", style="bold")
|
||||||
|
table.add_column("Severity", justify="center")
|
||||||
|
table.add_column("Check")
|
||||||
|
table.add_column("Detail")
|
||||||
|
badge = {
|
||||||
|
DRIFT_OK: "[green]OK[/green]",
|
||||||
|
DRIFT_WARN: "[yellow]WARN[/yellow]",
|
||||||
|
DRIFT_ERROR: "[red]ERROR[/red]",
|
||||||
|
}
|
||||||
|
for name, f in rows:
|
||||||
|
table.add_row(
|
||||||
|
name,
|
||||||
|
badge.get(f["severity"], f["severity"]),
|
||||||
|
f.get("check", ""),
|
||||||
|
f.get("detail", ""),
|
||||||
|
)
|
||||||
|
console.print(table)
|
||||||
|
|
||||||
|
|
||||||
|
def _run_doctor_checks(cache: Cache | None = None) -> list[dict]:
|
||||||
|
"""Run all doctor checks and return results.
|
||||||
|
|
||||||
|
``cache``: when provided, also verify manifest ↔ disk integrity (every
|
||||||
|
``file_path`` recorded in the manifest exists on disk).
|
||||||
|
"""
|
||||||
import os
|
import os
|
||||||
import jwt as pyjwt
|
|
||||||
from datetime import timezone
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
|
|
||||||
# Load .env so doctor works without the user having to export vars manually
|
# Load .env so doctor works without the user having to export vars manually
|
||||||
@@ -315,25 +445,21 @@ def _run_doctor_checks() -> list[dict]:
|
|||||||
if chatgpt_token:
|
if chatgpt_token:
|
||||||
is_jwt = chatgpt_token.startswith("eyJ")
|
is_jwt = chatgpt_token.startswith("eyJ")
|
||||||
add("ChatGPT token is valid JWT", is_jwt, "" if is_jwt else "Expected token starting with 'eyJ'")
|
add("ChatGPT token is valid JWT", is_jwt, "" if is_jwt else "Expected token starting with 'eyJ'")
|
||||||
|
# Expiry is NOT decodable from the token: ChatGPT's is a JWE (encrypted),
|
||||||
|
# so the `exp` claim is sealed server-side. The honest, free validity
|
||||||
|
# signal is the `error` field of /api/auth/session — `RefreshAccessTokenError`
|
||||||
|
# means the session token is dead while `expires`/`accessToken` still
|
||||||
|
# echo stale values. See FUTURE.md §9.
|
||||||
if is_jwt:
|
if is_jwt:
|
||||||
|
from src.providers.chatgpt import ChatGPTProvider
|
||||||
|
ct1 = os.getenv("CHATGPT_SESSION_TOKEN_1", "").strip() or None
|
||||||
try:
|
try:
|
||||||
payload = pyjwt.decode(chatgpt_token, options={"verify_signature": False})
|
ok, detail = ChatGPTProvider(
|
||||||
exp = payload.get("exp")
|
chatgpt_token, session_token_1=ct1
|
||||||
if exp:
|
).session_health()
|
||||||
expiry = datetime.fromtimestamp(exp, tz=timezone.utc)
|
add("ChatGPT token active", ok, detail)
|
||||||
now = datetime.now(tz=timezone.utc)
|
except Exception as e:
|
||||||
delta = expiry - now
|
add("ChatGPT token active", False, str(e)[:80])
|
||||||
detail = f"Expires {expiry.strftime('%Y-%m-%d %H:%M UTC')} ({delta.days}d)"
|
|
||||||
ok = delta.total_seconds() > 0
|
|
||||||
add("ChatGPT token not expired", ok, detail)
|
|
||||||
if ok and delta.total_seconds() < 86400:
|
|
||||||
add("ChatGPT token expiry warning", False, "Expires in < 24h — refresh soon")
|
|
||||||
else:
|
|
||||||
add("ChatGPT token expiry", False, "JWT has no 'exp' claim")
|
|
||||||
except pyjwt.exceptions.DecodeError:
|
|
||||||
# JWE (encrypted JWT) — cannot decode without the server key.
|
|
||||||
# This is normal for ChatGPT's current token format. Token is present and valid.
|
|
||||||
add("ChatGPT token expiry", True, "Encrypted token (JWE) — expiry not decodable client-side")
|
|
||||||
|
|
||||||
# Claude key
|
# Claude key
|
||||||
if claude_key:
|
if claude_key:
|
||||||
@@ -341,7 +467,7 @@ def _run_doctor_checks() -> list[dict]:
|
|||||||
|
|
||||||
# Directories
|
# Directories
|
||||||
export_dir = Path(os.getenv("EXPORT_DIR", "./exports")).expanduser()
|
export_dir = Path(os.getenv("EXPORT_DIR", "./exports")).expanduser()
|
||||||
cache_dir = Path(os.getenv("CACHE_DIR", "~/.ai-chat-exporter")).expanduser()
|
cache_dir = Path(os.getenv("CACHE_DIR", "./cache")).expanduser()
|
||||||
|
|
||||||
for label, dirpath in [("Export dir writable", export_dir), ("Cache dir writable", cache_dir)]:
|
for label, dirpath in [("Export dir writable", export_dir), ("Cache dir writable", cache_dir)]:
|
||||||
try:
|
try:
|
||||||
@@ -361,11 +487,24 @@ def _run_doctor_checks() -> list[dict]:
|
|||||||
except OSError as e:
|
except OSError as e:
|
||||||
add("Disk space check", False, str(e))
|
add("Disk space check", False, str(e))
|
||||||
|
|
||||||
|
# Manifest ↔ disk integrity
|
||||||
|
if cache is not None:
|
||||||
|
referenced = cache.all_file_paths()
|
||||||
|
if referenced:
|
||||||
|
missing = sorted(
|
||||||
|
p for p in referenced if not Path(p).expanduser().exists()
|
||||||
|
)
|
||||||
|
detail = f"{len(referenced) - len(missing)}/{len(referenced)} manifest files present"
|
||||||
|
if missing:
|
||||||
|
detail += f"; first missing: {missing[0]} — re-export to fix"
|
||||||
|
add("Manifest files on disk", not missing, detail)
|
||||||
|
|
||||||
# API reachability
|
# API reachability
|
||||||
if chatgpt_token:
|
if chatgpt_token:
|
||||||
try:
|
try:
|
||||||
from src.providers.chatgpt import ChatGPTProvider
|
from src.providers.chatgpt import ChatGPTProvider
|
||||||
p = ChatGPTProvider(chatgpt_token)
|
chatgpt_token_1 = os.getenv("CHATGPT_SESSION_TOKEN_1", "").strip() or None
|
||||||
|
p = ChatGPTProvider(chatgpt_token, session_token_1=chatgpt_token_1)
|
||||||
results = p.list_conversations(offset=0, limit=1)
|
results = p.list_conversations(offset=0, limit=1)
|
||||||
add("ChatGPT API reachable", True, f"Got {len(results)} result(s)")
|
add("ChatGPT API reachable", True, f"Got {len(results)} result(s)")
|
||||||
except ProviderError as e:
|
except ProviderError as e:
|
||||||
@@ -408,7 +547,7 @@ def _print_doctor_table(checks: list[dict]) -> None:
|
|||||||
@cli.command()
|
@cli.command()
|
||||||
@click.option(
|
@click.option(
|
||||||
"--provider",
|
"--provider",
|
||||||
type=click.Choice(["chatgpt", "claude", "all"], case_sensitive=False),
|
type=click.Choice(["chatgpt", "claude", "claude-code", "all"], case_sensitive=False),
|
||||||
default="all",
|
default="all",
|
||||||
show_default=True,
|
show_default=True,
|
||||||
help="Which provider to export.",
|
help="Which provider to export.",
|
||||||
@@ -442,6 +581,47 @@ def _print_doctor_table(checks: list[dict]) -> None:
|
|||||||
"Use 'none' for conversations outside any project."
|
"Use 'none' for conversations outside any project."
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
@click.option(
|
||||||
|
"--hidden-content",
|
||||||
|
type=click.Choice(["full", "placeholder", "omit"], case_sensitive=False),
|
||||||
|
default=None,
|
||||||
|
help=(
|
||||||
|
"What to do with content that was invisible in the provider's web UI "
|
||||||
|
"(file-retrieval tool dumps, hidden context): keep in full, collapse to "
|
||||||
|
"a one-line placeholder, or omit. Overrides EXPORTER_HIDDEN_CONTENT "
|
||||||
|
"(default: placeholder)."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--max-conversations",
|
||||||
|
type=click.IntRange(min=1),
|
||||||
|
default=None,
|
||||||
|
help=(
|
||||||
|
"Cap how many conversations are downloaded this run (per provider). "
|
||||||
|
"Runs are resumable, so re-running continues where the cap stopped. "
|
||||||
|
"Overrides MAX_CONVERSATIONS_PER_RUN (default: unlimited)."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--download-media",
|
||||||
|
type=click.Choice(["images", "all", "off"], case_sensitive=False),
|
||||||
|
default=None,
|
||||||
|
help=(
|
||||||
|
"Download conversation assets next to the exports: images only, "
|
||||||
|
"all (also audio/files), or off. Overrides EXPORTER_DOWNLOAD_MEDIA "
|
||||||
|
"(default: images)."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--force",
|
||||||
|
is_flag=True,
|
||||||
|
help=(
|
||||||
|
"Re-export every conversation even if already cached and unchanged. "
|
||||||
|
"Use to re-render the whole archive after a formatting/feature change "
|
||||||
|
"without 'cache --clear' (which would drop Joplin note links). Combine "
|
||||||
|
"with --max-conversations to spread the work over several runs."
|
||||||
|
),
|
||||||
|
)
|
||||||
@click.option("--dry-run", is_flag=True, help="Show what would be exported without writing anything.")
|
@click.option("--dry-run", is_flag=True, help="Show what would be exported without writing anything.")
|
||||||
@click.pass_context
|
@click.pass_context
|
||||||
def export(
|
def export(
|
||||||
@@ -451,6 +631,10 @@ def export(
|
|||||||
output_dir: str | None,
|
output_dir: str | None,
|
||||||
since: str | None,
|
since: str | None,
|
||||||
project_filter: str | None,
|
project_filter: str | None,
|
||||||
|
hidden_content: str | None,
|
||||||
|
max_conversations: int | None,
|
||||||
|
download_media: str | None,
|
||||||
|
force: bool,
|
||||||
dry_run: bool,
|
dry_run: bool,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Export new and updated conversations to Markdown or JSON.
|
"""Export new and updated conversations to Markdown or JSON.
|
||||||
@@ -462,6 +646,13 @@ def export(
|
|||||||
debug = ctx.obj.get("debug", False)
|
debug = ctx.obj.get("debug", False)
|
||||||
cache: Cache = ctx.obj["cache"]
|
cache: Cache = ctx.obj["cache"]
|
||||||
|
|
||||||
|
# CLI flags win over .env: set before load_config (load_dotenv uses
|
||||||
|
# override=False, so an existing env var is not clobbered by .env).
|
||||||
|
if hidden_content:
|
||||||
|
os.environ["EXPORTER_HIDDEN_CONTENT"] = hidden_content.lower()
|
||||||
|
if download_media:
|
||||||
|
os.environ["EXPORTER_DOWNLOAD_MEDIA"] = download_media.lower()
|
||||||
|
|
||||||
# Load config (may raise ConfigError)
|
# Load config (may raise ConfigError)
|
||||||
try:
|
try:
|
||||||
from src.config import load_config
|
from src.config import load_config
|
||||||
@@ -472,7 +663,7 @@ def export(
|
|||||||
# First-run: auto-doctor
|
# First-run: auto-doctor
|
||||||
if not cache.last_run():
|
if not cache.last_run():
|
||||||
console.print("[dim]First run — checking configuration…[/dim]")
|
console.print("[dim]First run — checking configuration…[/dim]")
|
||||||
checks = _run_doctor_checks()
|
checks = _run_doctor_checks(cache=cache)
|
||||||
_print_doctor_table(checks)
|
_print_doctor_table(checks)
|
||||||
if any(not c["pass"] for c in checks):
|
if any(not c["pass"] for c in checks):
|
||||||
err_console.print(
|
err_console.print(
|
||||||
@@ -492,11 +683,26 @@ def export(
|
|||||||
err_console.print(f"[red]Invalid --since date: '{since}'. Use YYYY-MM-DD.[/red]")
|
err_console.print(f"[red]Invalid --since date: '{since}'. Use YYYY-MM-DD.[/red]")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
|
# Session cap: CLI flag wins over MAX_CONVERSATIONS_PER_RUN env default.
|
||||||
|
session_cap = max_conversations if max_conversations is not None else cfg.max_conversations
|
||||||
|
|
||||||
|
# Force re-render campaign: stamp a start time so progress can be tracked
|
||||||
|
# across runs. A conversation is "done this campaign" once its exported_at
|
||||||
|
# passes the stamp; finished providers then export nothing, and the
|
||||||
|
# "still awaiting" count shrinks to zero. A fresh campaign starts whenever
|
||||||
|
# none is active (the previous one completed or was never started).
|
||||||
|
campaign_at: str | None = None
|
||||||
|
if force and not dry_run:
|
||||||
|
campaign_at = cache.get_force_campaign()
|
||||||
|
if campaign_at is None:
|
||||||
|
campaign_at = datetime.now(timezone.utc).isoformat()
|
||||||
|
cache.set_force_campaign(campaign_at)
|
||||||
|
|
||||||
# Determine which providers to run
|
# Determine which providers to run
|
||||||
providers_to_run = _resolve_providers(provider, cfg)
|
providers_to_run = _resolve_providers(provider, cfg)
|
||||||
if not providers_to_run:
|
if not providers_to_run:
|
||||||
err_console.print(
|
err_console.print(
|
||||||
"[red]No providers configured. Run 'python -m src.main auth' to set up tokens.[/red]"
|
"[red]No providers configured. Run 'ai-chat-exporter auth' to set up tokens.[/red]"
|
||||||
)
|
)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
@@ -510,6 +716,9 @@ def export(
|
|||||||
# Summary counters
|
# Summary counters
|
||||||
summary: dict[str, dict[str, int]] = {}
|
summary: dict[str, dict[str, int]] = {}
|
||||||
|
|
||||||
|
# Single LossReport tracks data-loss visibility across all providers in this run.
|
||||||
|
loss_report = LossReport()
|
||||||
|
|
||||||
for prov_name, prov_instance in providers_to_run:
|
for prov_name, prov_instance in providers_to_run:
|
||||||
summary[prov_name] = {"exported": 0, "skipped": 0, "failed": 0}
|
summary[prov_name] = {"exported": 0, "skipped": 0, "failed": 0}
|
||||||
|
|
||||||
@@ -528,18 +737,43 @@ def export(
|
|||||||
f" [dim]--project filter '{project_filter}': {len(all_convs)} matching conversations.[/dim]"
|
f" [dim]--project filter '{project_filter}': {len(all_convs)} matching conversations.[/dim]"
|
||||||
)
|
)
|
||||||
|
|
||||||
to_export = cache.get_new_or_updated(prov_name, all_convs)
|
to_export = cache.get_new_or_updated(
|
||||||
|
prov_name, all_convs, force=force, campaign_at=campaign_at
|
||||||
|
)
|
||||||
skipped = len(all_convs) - len(to_export)
|
skipped = len(all_convs) - len(to_export)
|
||||||
summary[prov_name]["skipped"] = skipped
|
summary[prov_name]["skipped"] = skipped
|
||||||
|
|
||||||
|
# Apply the session cap. Resumability makes this safe: the manifest
|
||||||
|
# records each conversation immediately, so the next run picks up
|
||||||
|
# exactly where the cap stopped.
|
||||||
|
deferred = 0
|
||||||
|
if session_cap is not None and len(to_export) > session_cap:
|
||||||
|
deferred = len(to_export) - session_cap
|
||||||
|
to_export = to_export[:session_cap]
|
||||||
|
summary[prov_name]["deferred"] = deferred
|
||||||
|
|
||||||
if dry_run:
|
if dry_run:
|
||||||
_print_dry_run_table(prov_name, to_export, prov_instance, export_base, structure, skipped)
|
_print_dry_run_table(prov_name, to_export, prov_instance, export_base, structure, skipped)
|
||||||
|
if deferred:
|
||||||
|
console.print(
|
||||||
|
f" [yellow]Session cap {session_cap}: {deferred} more pending beyond this run.[/yellow]"
|
||||||
|
)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if not to_export:
|
if not to_export:
|
||||||
console.print(f" [dim]{skipped} conversations already up to date.[/dim]")
|
done_msg = (
|
||||||
|
"already re-rendered this campaign."
|
||||||
|
if force else "conversations already up to date."
|
||||||
|
)
|
||||||
|
console.print(f" [dim]{skipped} {done_msg}[/dim]")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
if deferred:
|
||||||
|
console.print(
|
||||||
|
f" [dim]{len(to_export)} to export (session cap; {deferred} deferred), "
|
||||||
|
f"{skipped} already up to date.[/dim]"
|
||||||
|
)
|
||||||
|
else:
|
||||||
console.print(f" [dim]{len(to_export)} to export, {skipped} already up to date.[/dim]")
|
console.print(f" [dim]{len(to_export)} to export, {skipped} already up to date.[/dim]")
|
||||||
|
|
||||||
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskProgressColumn
|
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskProgressColumn
|
||||||
@@ -557,7 +791,30 @@ def export(
|
|||||||
conv_id = raw_conv.get("id") or raw_conv.get("uuid", "unknown")
|
conv_id = raw_conv.get("id") or raw_conv.get("uuid", "unknown")
|
||||||
try:
|
try:
|
||||||
full_raw = prov_instance.get_conversation(conv_id)
|
full_raw = prov_instance.get_conversation(conv_id)
|
||||||
normalized = prov_instance.normalize_conversation(full_raw)
|
# Propagate metadata from the listing summary into the full
|
||||||
|
# detail so normalize_conversation can use it.
|
||||||
|
# - Keys starting with "_" are provider annotations
|
||||||
|
# (e.g. _project_name injected by ChatGPT project fetching).
|
||||||
|
# - "project" is included explicitly because Claude's detail
|
||||||
|
# endpoint omits it even though the listing returns it.
|
||||||
|
_PROPAGATE_KEYS = {"project"}
|
||||||
|
for key, val in raw_conv.items():
|
||||||
|
if (key.startswith("_") or key in _PROPAGATE_KEYS) and key not in full_raw:
|
||||||
|
full_raw[key] = val
|
||||||
|
normalized = prov_instance.normalize_conversation(full_raw, loss_report)
|
||||||
|
|
||||||
|
# Download assets (images by default) so the renderers
|
||||||
|
# can inline them. Failures keep placeholders; never fatal.
|
||||||
|
if cfg.download_media != "off":
|
||||||
|
from src.media import resolve_media
|
||||||
|
resolve_media(
|
||||||
|
normalized,
|
||||||
|
prov_instance,
|
||||||
|
export_base,
|
||||||
|
structure,
|
||||||
|
cfg.download_media,
|
||||||
|
loss_report,
|
||||||
|
)
|
||||||
|
|
||||||
exported_path: Path | None = None
|
exported_path: Path | None = None
|
||||||
if md_exporter:
|
if md_exporter:
|
||||||
@@ -569,6 +826,7 @@ def export(
|
|||||||
cache.mark_exported(prov_name, conv_id, {
|
cache.mark_exported(prov_name, conv_id, {
|
||||||
"title": normalized.get("title", ""),
|
"title": normalized.get("title", ""),
|
||||||
"project": normalized.get("project"),
|
"project": normalized.get("project"),
|
||||||
|
"created_at": normalized.get("created_at", ""),
|
||||||
"updated_at": normalized.get("updated_at", ""),
|
"updated_at": normalized.get("updated_at", ""),
|
||||||
"file_path": str(exported_path) if exported_path else "",
|
"file_path": str(exported_path) if exported_path else "",
|
||||||
})
|
})
|
||||||
@@ -586,8 +844,42 @@ def export(
|
|||||||
progress.advance(task)
|
progress.advance(task)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
# Force campaign: how many of this provider's conversations still
|
||||||
|
# predate the campaign stamp (i.e. not yet re-rendered this campaign).
|
||||||
|
if force and campaign_at is not None:
|
||||||
|
summary[prov_name]["awaiting"] = sum(
|
||||||
|
1
|
||||||
|
for c in all_convs
|
||||||
|
if cache.exported_at(prov_name, c.get("id") or c.get("uuid", "")) < campaign_at
|
||||||
|
)
|
||||||
|
|
||||||
if not dry_run:
|
if not dry_run:
|
||||||
_print_export_summary(summary)
|
_print_export_summary(summary)
|
||||||
|
if force:
|
||||||
|
total_awaiting = sum(s.get("awaiting", 0) for s in summary.values())
|
||||||
|
exported_now = sum(s.get("exported", 0) for s in summary.values())
|
||||||
|
if total_awaiting:
|
||||||
|
console.print(
|
||||||
|
f"[yellow]Force re-render: refreshed {exported_now} this run, "
|
||||||
|
f"{total_awaiting} still to go. Re-run the same command to continue.[/yellow]"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
cache.clear_force_campaign()
|
||||||
|
console.print(
|
||||||
|
"[green]Force re-render complete — every conversation has been "
|
||||||
|
"re-rendered with the current code.[/green]"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
total_deferred = sum(s.get("deferred", 0) for s in summary.values())
|
||||||
|
if total_deferred:
|
||||||
|
console.print(
|
||||||
|
f"[yellow]{total_deferred} conversation(s) deferred by the session cap "
|
||||||
|
f"({session_cap} per provider per run). Re-run the same command to continue.[/yellow]"
|
||||||
|
)
|
||||||
|
# Emit the data-loss summary at INFO level so it lands in the log file
|
||||||
|
# AND the operator's console (default level is INFO).
|
||||||
|
for line in loss_report.format_summary().split("\n"):
|
||||||
|
logger.info(line)
|
||||||
|
|
||||||
|
|
||||||
def _resolve_providers(provider: str, cfg) -> list[tuple[str, object]]:
|
def _resolve_providers(provider: str, cfg) -> list[tuple[str, object]]:
|
||||||
@@ -618,7 +910,9 @@ def _resolve_providers(provider: str, cfg) -> list[tuple[str, object]]:
|
|||||||
"chatgpt",
|
"chatgpt",
|
||||||
ChatGPTProvider(
|
ChatGPTProvider(
|
||||||
session_token=cfg.chatgpt_session_token,
|
session_token=cfg.chatgpt_session_token,
|
||||||
|
session_token_1=cfg.chatgpt_session_token_1,
|
||||||
project_ids=cfg.chatgpt_project_ids,
|
project_ids=cfg.chatgpt_project_ids,
|
||||||
|
hidden_content=cfg.hidden_content,
|
||||||
),
|
),
|
||||||
))
|
))
|
||||||
except ProviderError as e:
|
except ProviderError as e:
|
||||||
@@ -630,6 +924,21 @@ def _resolve_providers(provider: str, cfg) -> list[tuple[str, object]]:
|
|||||||
if provider in ("claude", "all"):
|
if provider in ("claude", "all"):
|
||||||
try_add("claude", cfg.claude_session_key, ClaudeProvider)
|
try_add("claude", cfg.claude_session_key, ClaudeProvider)
|
||||||
|
|
||||||
|
if provider in ("claude-code", "all"):
|
||||||
|
from src.providers.claude_code import ClaudeCodeProvider, resolve_roots
|
||||||
|
cc_roots = resolve_roots()
|
||||||
|
if any(r.is_dir() for r in cc_roots):
|
||||||
|
result.append((
|
||||||
|
"claude-code",
|
||||||
|
ClaudeCodeProvider(hidden_content=cfg.hidden_content),
|
||||||
|
))
|
||||||
|
elif provider == "claude-code":
|
||||||
|
logging.getLogger(__name__).warning(
|
||||||
|
"[claude-code] Skipping — none of these roots found (set "
|
||||||
|
"CLAUDE_CODE_DIR, ':'-separated for multiple): %s",
|
||||||
|
", ".join(str(r) for r in cc_roots),
|
||||||
|
)
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
@@ -726,7 +1035,7 @@ def _print_export_summary(summary: dict[str, dict[str, int]]) -> None:
|
|||||||
@cli.command(name="list")
|
@cli.command(name="list")
|
||||||
@click.option(
|
@click.option(
|
||||||
"--provider",
|
"--provider",
|
||||||
type=click.Choice(["chatgpt", "claude", "all"], case_sensitive=False),
|
type=click.Choice(["chatgpt", "claude", "claude-code", "all"], case_sensitive=False),
|
||||||
default="all",
|
default="all",
|
||||||
show_default=True,
|
show_default=True,
|
||||||
)
|
)
|
||||||
@@ -757,18 +1066,26 @@ def list_conversations(ctx: click.Context, provider: str, project_filter: str |
|
|||||||
if project_filter is not None:
|
if project_filter is not None:
|
||||||
all_convs = _filter_by_project(all_convs, project_filter)
|
all_convs = _filter_by_project(all_convs, project_filter)
|
||||||
|
|
||||||
table = Table()
|
# no_wrap + overflow="ellipsis" prevents Rich from wrapping cells to
|
||||||
table.add_column("Title")
|
# multiple lines on narrow terminals (e.g. Windows Command Prompt),
|
||||||
table.add_column("Project")
|
# which can otherwise make the output look garbled. Widths are tuned
|
||||||
table.add_column("Updated")
|
# to fit within an 80-column terminal.
|
||||||
table.add_column("ID")
|
# Total width budget for 80-column terminals:
|
||||||
|
# borders (5) + padding (4 cols * 2) = 13 chars of overhead
|
||||||
|
# remaining 67 chars split: 34 title + 15 project + 10 date + 8 id
|
||||||
|
table = Table(show_lines=False, expand=False, padding=(0, 1))
|
||||||
|
table.add_column("Title", no_wrap=True, overflow="ellipsis", max_width=34)
|
||||||
|
table.add_column("Project", no_wrap=True, overflow="ellipsis", max_width=15)
|
||||||
|
table.add_column("Updated", no_wrap=True, min_width=10)
|
||||||
|
table.add_column("ID", no_wrap=True, min_width=8)
|
||||||
|
|
||||||
for conv in all_convs:
|
for conv in all_convs:
|
||||||
title = conv.get("title") or "Untitled"
|
# ChatGPT uses "title"; Claude uses "name".
|
||||||
|
title = conv.get("title") or conv.get("name") or "Untitled"
|
||||||
project = _raw_project_name(conv) or ""
|
project = _raw_project_name(conv) or ""
|
||||||
updated = (conv.get("updated_at") or conv.get("update_time") or "")[:10]
|
updated = (conv.get("updated_at") or conv.get("update_time") or "")[:10]
|
||||||
conv_id = (conv.get("id") or conv.get("uuid") or "")[:8]
|
conv_id = (conv.get("id") or conv.get("uuid") or "")[:8]
|
||||||
table.add_row(title[:60], project[:30], updated, conv_id)
|
table.add_row(title, project, updated, conv_id)
|
||||||
|
|
||||||
console.print(table)
|
console.print(table)
|
||||||
console.print(f"Total: {len(all_convs)} conversations")
|
console.print(f"Total: {len(all_convs)} conversations")
|
||||||
@@ -784,7 +1101,7 @@ def list_conversations(ctx: click.Context, provider: str, project_filter: str |
|
|||||||
@click.option("--clear", is_flag=True, help="Clear cached entries.")
|
@click.option("--clear", is_flag=True, help="Clear cached entries.")
|
||||||
@click.option(
|
@click.option(
|
||||||
"--provider",
|
"--provider",
|
||||||
type=click.Choice(["chatgpt", "claude", "all"], case_sensitive=False),
|
type=click.Choice(["chatgpt", "claude", "claude-code", "all"], case_sensitive=False),
|
||||||
default="all",
|
default="all",
|
||||||
help="Provider to target (used with --clear).",
|
help="Provider to target (used with --clear).",
|
||||||
)
|
)
|
||||||
@@ -814,6 +1131,106 @@ def cache(ctx: click.Context, show: bool, clear: bool, provider: str) -> None:
|
|||||||
console.print("Specify --show or --clear. Use --help for options.")
|
console.print("Specify --show or --clear. Use --help for options.")
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────────────────────────────────────────────────────────────
|
||||||
|
# prune command
|
||||||
|
# ──────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@cli.command()
|
||||||
|
@click.option("--dry-run", is_flag=True, help="List stale files without deleting anything.")
|
||||||
|
@click.option("--yes", "-y", is_flag=True, help="Delete without a confirmation prompt.")
|
||||||
|
@click.pass_context
|
||||||
|
def prune(ctx: click.Context, dry_run: bool, yes: bool) -> None:
|
||||||
|
"""Delete export files no longer referenced by the cache manifest.
|
||||||
|
|
||||||
|
Removes leftovers from old folder layouts and orphaned files from fixed
|
||||||
|
bugs (e.g. pre-v0.5.0 nested year folders, files with a missing
|
||||||
|
conversation ID). Only .md and .json files under EXPORT_DIR are
|
||||||
|
considered; the manifest is the source of truth.
|
||||||
|
"""
|
||||||
|
cache_obj: Cache = ctx.obj["cache"]
|
||||||
|
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
load_dotenv(override=False)
|
||||||
|
export_dir = Path(os.getenv("EXPORT_DIR", "./exports")).expanduser()
|
||||||
|
|
||||||
|
if not export_dir.exists():
|
||||||
|
console.print(f"[yellow]Export directory {export_dir} does not exist — nothing to prune.[/yellow]")
|
||||||
|
return
|
||||||
|
|
||||||
|
referenced_raw = cache_obj.all_file_paths()
|
||||||
|
if not referenced_raw:
|
||||||
|
# Footgun guard: with an empty manifest (e.g. right after `cache
|
||||||
|
# --clear`) every export file would look stale and prune would wipe
|
||||||
|
# the entire archive.
|
||||||
|
err_console.print(
|
||||||
|
"[red]The manifest references no export files (did you just run "
|
||||||
|
"'cache --clear'?). Refusing to prune — run an export first.[/red]"
|
||||||
|
)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
referenced = {Path(p).expanduser().resolve() for p in referenced_raw}
|
||||||
|
on_disk = sorted(
|
||||||
|
f for f in export_dir.rglob("*")
|
||||||
|
if f.is_file() and f.suffix in (".md", ".json")
|
||||||
|
)
|
||||||
|
stale = [f for f in on_disk if f.resolve() not in referenced]
|
||||||
|
|
||||||
|
if not stale:
|
||||||
|
console.print(
|
||||||
|
f"[green]Nothing to prune — all {len(on_disk)} export files are referenced "
|
||||||
|
"by the manifest.[/green]"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
total_bytes = sum(f.stat().st_size for f in stale)
|
||||||
|
console.print(
|
||||||
|
f"[bold]{len(stale)} stale file(s)[/bold] "
|
||||||
|
f"({total_bytes / (1024 * 1024):.1f} MB) not referenced by the manifest "
|
||||||
|
f"(of {len(on_disk)} total):\n"
|
||||||
|
)
|
||||||
|
for f in stale:
|
||||||
|
try:
|
||||||
|
shown = f.relative_to(export_dir.resolve()) if f.is_absolute() else f
|
||||||
|
except ValueError:
|
||||||
|
shown = f
|
||||||
|
console.print(f" [dim]{shown}[/dim]")
|
||||||
|
|
||||||
|
if dry_run:
|
||||||
|
console.print("\n[yellow]Dry run — nothing deleted.[/yellow]")
|
||||||
|
return
|
||||||
|
|
||||||
|
if not yes and not click.confirm(f"\nDelete these {len(stale)} files?", default=False):
|
||||||
|
console.print("Aborted — nothing deleted.")
|
||||||
|
return
|
||||||
|
|
||||||
|
deleted = 0
|
||||||
|
for f in stale:
|
||||||
|
try:
|
||||||
|
f.unlink()
|
||||||
|
deleted += 1
|
||||||
|
except OSError as e:
|
||||||
|
logger.error("[prune] Could not delete %s: %s", f, e)
|
||||||
|
|
||||||
|
# Sweep now-empty directories (deepest first).
|
||||||
|
removed_dirs = 0
|
||||||
|
for d in sorted(
|
||||||
|
(d for d in export_dir.rglob("*") if d.is_dir()),
|
||||||
|
key=lambda p: len(p.parts),
|
||||||
|
reverse=True,
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
d.rmdir() # only succeeds when empty
|
||||||
|
removed_dirs += 1
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
console.print(
|
||||||
|
f"[green]Deleted {deleted} file(s) ({total_bytes / (1024 * 1024):.1f} MB) "
|
||||||
|
f"and {removed_dirs} empty director(ies).[/green]"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# ──────────────────────────────────────────────────────────────────────────────
|
# ──────────────────────────────────────────────────────────────────────────────
|
||||||
# joplin command
|
# joplin command
|
||||||
# ──────────────────────────────────────────────────────────────────────────────
|
# ──────────────────────────────────────────────────────────────────────────────
|
||||||
@@ -822,7 +1239,7 @@ def cache(ctx: click.Context, show: bool, clear: bool, provider: str) -> None:
|
|||||||
@cli.command()
|
@cli.command()
|
||||||
@click.option(
|
@click.option(
|
||||||
"--provider",
|
"--provider",
|
||||||
type=click.Choice(["chatgpt", "claude", "all"], case_sensitive=False),
|
type=click.Choice(["chatgpt", "claude", "claude-code", "all"], case_sensitive=False),
|
||||||
default="all",
|
default="all",
|
||||||
show_default=True,
|
show_default=True,
|
||||||
help="Which provider's conversations to sync to Joplin.",
|
help="Which provider's conversations to sync to Joplin.",
|
||||||
@@ -845,9 +1262,9 @@ def joplin(ctx: click.Context, provider: str, project_filter: str | None, dry_ru
|
|||||||
via its local REST API. Requires Joplin desktop to be running with the
|
via its local REST API. Requires Joplin desktop to be running with the
|
||||||
Web Clipper service enabled.
|
Web Clipper service enabled.
|
||||||
|
|
||||||
Notebooks are created automatically based on provider and project:
|
Notebooks are created automatically as nested folders:
|
||||||
exports/chatgpt/my-project/ → "ChatGPT - My Project" notebook
|
chatgpt / my-project → AI-ChatGPT / My Project
|
||||||
exports/claude/no-project/ → "Claude - No Project" notebook
|
claude / no-project → AI-Claude / No Project
|
||||||
|
|
||||||
Re-running is safe: notes are updated (not duplicated) on subsequent runs.
|
Re-running is safe: notes are updated (not duplicated) on subsequent runs.
|
||||||
|
|
||||||
@@ -873,7 +1290,7 @@ def joplin(ctx: click.Context, provider: str, project_filter: str | None, dry_ru
|
|||||||
)
|
)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
from src.joplin import JoplinClient, JoplinError, notebook_title
|
from src.joplin import JoplinClient, JoplinError, notebook_path
|
||||||
|
|
||||||
client = JoplinClient(cfg.joplin_api_url, cfg.joplin_api_token)
|
client = JoplinClient(cfg.joplin_api_url, cfg.joplin_api_token)
|
||||||
|
|
||||||
@@ -895,10 +1312,9 @@ def joplin(ctx: click.Context, provider: str, project_filter: str | None, dry_ru
|
|||||||
|
|
||||||
# Determine which providers to process
|
# Determine which providers to process
|
||||||
providers_to_sync: list[str] = []
|
providers_to_sync: list[str] = []
|
||||||
if provider in ("chatgpt", "all"):
|
for prov in ("chatgpt", "claude", "claude-code"):
|
||||||
providers_to_sync.append("chatgpt")
|
if provider in (prov, "all"):
|
||||||
if provider in ("claude", "all"):
|
providers_to_sync.append(prov)
|
||||||
providers_to_sync.append("claude")
|
|
||||||
|
|
||||||
summary: dict[str, dict[str, int]] = {}
|
summary: dict[str, dict[str, int]] = {}
|
||||||
|
|
||||||
@@ -953,7 +1369,9 @@ def joplin(ctx: click.Context, provider: str, project_filter: str | None, dry_ru
|
|||||||
|
|
||||||
for conv_id, entry in pending:
|
for conv_id, entry in pending:
|
||||||
file_path = entry.get("file_path", "")
|
file_path = entry.get("file_path", "")
|
||||||
title = entry.get("title") or "Untitled"
|
raw_title = entry.get("title") or "Untitled"
|
||||||
|
created_date = (entry.get("created_at") or "")[:10]
|
||||||
|
title = f"{created_date} {raw_title}" if created_date else raw_title
|
||||||
project = entry.get("project") or None
|
project = entry.get("project") or None
|
||||||
existing_note_id = entry.get("joplin_note_id")
|
existing_note_id = entry.get("joplin_note_id")
|
||||||
action = "update" if existing_note_id else "create"
|
action = "update" if existing_note_id else "create"
|
||||||
@@ -968,12 +1386,26 @@ def joplin(ctx: click.Context, provider: str, project_filter: str | None, dry_ru
|
|||||||
body = Path(file_path).read_text(encoding="utf-8")
|
body = Path(file_path).read_text(encoding="utf-8")
|
||||||
logger.debug("[joplin] Read %d chars from %s", len(body), file_path)
|
logger.debug("[joplin] Read %d chars from %s", len(body), file_path)
|
||||||
|
|
||||||
# Get or create the notebook
|
# Upload any downloaded media as Joplin resources and
|
||||||
nb_title = notebook_title(prov_name, project)
|
# rewrite local media/ links to :/resourceId references.
|
||||||
notebook_id = client.get_or_create_notebook(nb_title)
|
# The cache stores the resource map so re-syncs reuse IDs.
|
||||||
|
from src.joplin import upload_media_and_rewrite
|
||||||
|
res_map = cache_obj.get_joplin_resources(prov_name, conv_id)
|
||||||
|
body, new_res_map = upload_media_and_rewrite(
|
||||||
|
body, Path(file_path).parent, client, res_map
|
||||||
|
)
|
||||||
|
if new_res_map != res_map:
|
||||||
|
cache_obj.set_joplin_resources(prov_name, conv_id, new_res_map)
|
||||||
|
|
||||||
|
# Get or create the nested notebook
|
||||||
|
nb_path = notebook_path(prov_name, project)
|
||||||
|
notebook_id = client.get_or_create_notebook_path(list(nb_path))
|
||||||
|
|
||||||
if existing_note_id:
|
if existing_note_id:
|
||||||
client.update_note(existing_note_id, title, body)
|
# Pass notebook_id so a changed provider→notebook mapping
|
||||||
|
# (e.g. Claude Code AI-Claude → AI-ClaudeCode) relocates
|
||||||
|
# the existing note instead of leaving a stale copy.
|
||||||
|
client.update_note(existing_note_id, title, body, parent_id=notebook_id)
|
||||||
cache_obj.mark_joplin_synced(prov_name, conv_id, existing_note_id)
|
cache_obj.mark_joplin_synced(prov_name, conv_id, existing_note_id)
|
||||||
summary[prov_name]["updated"] += 1
|
summary[prov_name]["updated"] += 1
|
||||||
else:
|
else:
|
||||||
@@ -1007,7 +1439,7 @@ def joplin(ctx: click.Context, provider: str, project_filter: str | None, dry_ru
|
|||||||
|
|
||||||
|
|
||||||
def _print_joplin_dry_run_table(prov_name: str, pending: list[tuple[str, dict]]) -> None:
|
def _print_joplin_dry_run_table(prov_name: str, pending: list[tuple[str, dict]]) -> None:
|
||||||
from src.joplin import notebook_title
|
from src.joplin import notebook_path
|
||||||
|
|
||||||
table = Table(title=f"[DRY RUN] {prov_name.upper()} — Would sync {len(pending)} conversation(s)")
|
table = Table(title=f"[DRY RUN] {prov_name.upper()} — Would sync {len(pending)} conversation(s)")
|
||||||
table.add_column("Title")
|
table.add_column("Title")
|
||||||
@@ -1016,9 +1448,12 @@ def _print_joplin_dry_run_table(prov_name: str, pending: list[tuple[str, dict]])
|
|||||||
table.add_column("Action")
|
table.add_column("Action")
|
||||||
|
|
||||||
for conv_id, entry in pending[:50]:
|
for conv_id, entry in pending[:50]:
|
||||||
title = entry.get("title") or "Untitled"
|
raw_title = entry.get("title") or "Untitled"
|
||||||
|
created_date = (entry.get("created_at") or "")[:10]
|
||||||
|
title = f"{created_date} {raw_title}" if created_date else raw_title
|
||||||
project = entry.get("project") or "no-project"
|
project = entry.get("project") or "no-project"
|
||||||
nb = notebook_title(prov_name, entry.get("project"))
|
parent, child = notebook_path(prov_name, entry.get("project"))
|
||||||
|
nb = f"{parent} / {child}"
|
||||||
action = "update" if entry.get("joplin_note_id") else "create"
|
action = "update" if entry.get("joplin_note_id") else "create"
|
||||||
table.add_row(title[:50], project[:30], nb, action)
|
table.add_row(title[:50], project[:30], nb, action)
|
||||||
|
|
||||||
|
|||||||
+202
@@ -0,0 +1,202 @@
|
|||||||
|
"""Media resolution — download conversation assets next to the export files.
|
||||||
|
|
||||||
|
Walks a normalized conversation's placeholder blocks, downloads each asset via
|
||||||
|
the provider, saves it under a ``media/`` directory beside the conversation's
|
||||||
|
Markdown file, and annotates the block with a relative ``local_path`` so the
|
||||||
|
renderer emits a real inline image / file link instead of a placeholder.
|
||||||
|
|
||||||
|
Policy (``EXPORTER_DOWNLOAD_MEDIA``):
|
||||||
|
images (default) — image_placeholder blocks only
|
||||||
|
all — also file_placeholder blocks (voice-mode audio etc.;
|
||||||
|
measured 2026-06-12: 556 clips ≈ 162MB whose transcripts
|
||||||
|
are already in the exports as text)
|
||||||
|
off — leave placeholders untouched
|
||||||
|
|
||||||
|
Failures (expired assets, network) keep the existing placeholder and are
|
||||||
|
counted in the run summary — never fatal to the export.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from src.blocks import BLOCK_TYPE_FILE_PLACEHOLDER, BLOCK_TYPE_IMAGE_PLACEHOLDER
|
||||||
|
from src.loss_report import LossReport
|
||||||
|
from src.providers.base import ProviderError
|
||||||
|
from src.utils import build_export_path, generate_filename
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
MEDIA_IMAGES = "images"
|
||||||
|
MEDIA_ALL = "all"
|
||||||
|
MEDIA_OFF = "off"
|
||||||
|
VALID_MEDIA_POLICIES = {MEDIA_IMAGES, MEDIA_ALL, MEDIA_OFF}
|
||||||
|
|
||||||
|
_EXT_BY_MIME = {
|
||||||
|
"image/png": ".png",
|
||||||
|
"image/jpeg": ".jpg",
|
||||||
|
"image/gif": ".gif",
|
||||||
|
"image/webp": ".webp",
|
||||||
|
"audio/wav": ".wav",
|
||||||
|
"audio/x-wav": ".wav",
|
||||||
|
"audio/mpeg": ".mp3",
|
||||||
|
"audio/mp4": ".m4a",
|
||||||
|
"audio/aac": ".aac",
|
||||||
|
"application/pdf": ".pdf",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_media_policy() -> str:
|
||||||
|
"""Read EXPORTER_DOWNLOAD_MEDIA from the environment, defaulting to images."""
|
||||||
|
value = os.getenv("EXPORTER_DOWNLOAD_MEDIA", "").strip().lower()
|
||||||
|
if not value:
|
||||||
|
return MEDIA_IMAGES
|
||||||
|
if value not in VALID_MEDIA_POLICIES:
|
||||||
|
logger.warning(
|
||||||
|
"EXPORTER_DOWNLOAD_MEDIA=%r is invalid (expected images|all|off) "
|
||||||
|
"— using 'images'.",
|
||||||
|
value,
|
||||||
|
)
|
||||||
|
return MEDIA_IMAGES
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_media(
|
||||||
|
normalized: dict,
|
||||||
|
provider,
|
||||||
|
export_base: Path,
|
||||||
|
structure: str,
|
||||||
|
policy: str,
|
||||||
|
report: LossReport,
|
||||||
|
) -> int:
|
||||||
|
"""Download assets for one normalized conversation. Returns download count.
|
||||||
|
|
||||||
|
Mutates placeholder blocks in place (adds ``local_path``). Idempotent:
|
||||||
|
an asset already on disk is annotated without an API call.
|
||||||
|
"""
|
||||||
|
if policy == MEDIA_OFF:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
download = getattr(provider, "download_asset", None)
|
||||||
|
if download is None:
|
||||||
|
# Provider has no remote assets (e.g. claude-code local transcripts).
|
||||||
|
return 0
|
||||||
|
|
||||||
|
wanted_types = {BLOCK_TYPE_IMAGE_PLACEHOLDER}
|
||||||
|
if policy == MEDIA_ALL:
|
||||||
|
wanted_types.add(BLOCK_TYPE_FILE_PLACEHOLDER)
|
||||||
|
|
||||||
|
targets = [
|
||||||
|
block
|
||||||
|
for message in normalized.get("messages", [])
|
||||||
|
for block in message.get("blocks", [])
|
||||||
|
if block.get("type") in wanted_types and block.get("ref")
|
||||||
|
]
|
||||||
|
if not targets:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
# Same path computation the exporter uses, so media/ lands beside the .md.
|
||||||
|
filename = generate_filename(
|
||||||
|
normalized.get("title", "Untitled"),
|
||||||
|
normalized.get("id", ""),
|
||||||
|
normalized.get("created_at") or "2000-01-01",
|
||||||
|
)
|
||||||
|
conv_dir = build_export_path(
|
||||||
|
export_base,
|
||||||
|
normalized.get("provider", ""),
|
||||||
|
normalized.get("project"),
|
||||||
|
normalized.get("created_at") or "2000-01-01",
|
||||||
|
filename,
|
||||||
|
structure,
|
||||||
|
).parent
|
||||||
|
media_dir = conv_dir / "media"
|
||||||
|
|
||||||
|
downloaded = 0
|
||||||
|
for block in targets:
|
||||||
|
ref = block["ref"]
|
||||||
|
file_id = _safe_asset_name(provider, ref)
|
||||||
|
if not file_id:
|
||||||
|
report.record_media_failed("unparseable-ref")
|
||||||
|
continue
|
||||||
|
|
||||||
|
existing = _find_existing(media_dir, file_id)
|
||||||
|
if existing is not None:
|
||||||
|
block["local_path"] = f"media/{existing.name}"
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
content, mime, file_name = download(ref)
|
||||||
|
except ProviderError as e:
|
||||||
|
logger.warning(
|
||||||
|
"[media] Could not download %s: %s", ref[:60], e.original
|
||||||
|
)
|
||||||
|
reason = "expired-or-missing" if "404" in str(e.original) or "not found" in str(
|
||||||
|
e.original
|
||||||
|
).lower() else "download-error"
|
||||||
|
report.record_media_failed(reason)
|
||||||
|
continue
|
||||||
|
|
||||||
|
ext = _pick_extension(mime, file_name)
|
||||||
|
target = media_dir / f"{file_id}{ext}"
|
||||||
|
try:
|
||||||
|
_write_atomic(target, content)
|
||||||
|
except OSError as e:
|
||||||
|
logger.error("[media] Could not write %s: %s", target, e)
|
||||||
|
report.record_media_failed("write-error")
|
||||||
|
continue
|
||||||
|
|
||||||
|
block["local_path"] = f"media/{target.name}"
|
||||||
|
report.record_media_downloaded(len(content))
|
||||||
|
downloaded += 1
|
||||||
|
|
||||||
|
return downloaded
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_asset_name(provider, ref: str) -> str | None:
|
||||||
|
"""A stable, filesystem-safe name for the asset (the provider file ID)."""
|
||||||
|
parser = getattr(provider, "parse_asset_file_id", None)
|
||||||
|
if parser is None:
|
||||||
|
from src.providers.chatgpt import parse_asset_file_id as parser
|
||||||
|
file_id = parser(ref)
|
||||||
|
if not file_id:
|
||||||
|
return None
|
||||||
|
return "".join(c if c.isalnum() or c in "-_." else "_" for c in file_id)
|
||||||
|
|
||||||
|
|
||||||
|
def _find_existing(media_dir: Path, file_id: str) -> Path | None:
|
||||||
|
"""Return an already-downloaded file for this asset, if present."""
|
||||||
|
if not media_dir.is_dir():
|
||||||
|
return None
|
||||||
|
for candidate in media_dir.glob(f"{file_id}.*"):
|
||||||
|
if candidate.is_file() and candidate.stat().st_size > 0:
|
||||||
|
return candidate
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _pick_extension(mime: str | None, file_name: str | None) -> str:
|
||||||
|
if mime:
|
||||||
|
base_mime = mime.split(";")[0].strip().lower()
|
||||||
|
if base_mime in _EXT_BY_MIME:
|
||||||
|
return _EXT_BY_MIME[base_mime]
|
||||||
|
if file_name:
|
||||||
|
suffix = Path(file_name).suffix
|
||||||
|
if suffix and len(suffix) <= 8:
|
||||||
|
return suffix.lower()
|
||||||
|
return ".bin"
|
||||||
|
|
||||||
|
|
||||||
|
def _write_atomic(target: Path, content: bytes) -> None:
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
fd, tmp_name = tempfile.mkstemp(dir=target.parent, suffix=".tmp")
|
||||||
|
try:
|
||||||
|
with os.fdopen(fd, "wb") as fh:
|
||||||
|
fh.write(content)
|
||||||
|
os.chmod(tmp_name, 0o600)
|
||||||
|
os.replace(tmp_name, target)
|
||||||
|
except OSError:
|
||||||
|
try:
|
||||||
|
os.unlink(tmp_name)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
raise
|
||||||
+118
-3
@@ -1,6 +1,7 @@
|
|||||||
"""Abstract base class for AI chat providers."""
|
"""Abstract base class for AI chat providers."""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
import random
|
import random
|
||||||
import time
|
import time
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
@@ -9,6 +10,7 @@ from typing import Any
|
|||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
|
from src.loss_report import LossReport
|
||||||
from src.utils import redact_secrets
|
from src.utils import redact_secrets
|
||||||
|
|
||||||
# curl_cffi has its own exception hierarchy (rooted at CurlError → OSError),
|
# curl_cffi has its own exception hierarchy (rooted at CurlError → OSError),
|
||||||
@@ -36,6 +38,83 @@ MAX_RETRIES = 3
|
|||||||
BACKOFF_BASE = 2.0
|
BACKOFF_BASE = 2.0
|
||||||
BACKOFF_MAX = 60.0
|
BACKOFF_MAX = 60.0
|
||||||
|
|
||||||
|
# EXPORTER_HIDDEN_CONTENT policy — what to do with content that was invisible
|
||||||
|
# in the provider's UI (retrieval dumps, hidden-flagged context, agent tool
|
||||||
|
# traffic). Shared by all providers.
|
||||||
|
HIDDEN_CONTENT_FULL = "full"
|
||||||
|
HIDDEN_CONTENT_PLACEHOLDER = "placeholder"
|
||||||
|
HIDDEN_CONTENT_OMIT = "omit"
|
||||||
|
VALID_HIDDEN_CONTENT_POLICIES = {
|
||||||
|
HIDDEN_CONTENT_FULL,
|
||||||
|
HIDDEN_CONTENT_PLACEHOLDER,
|
||||||
|
HIDDEN_CONTENT_OMIT,
|
||||||
|
}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# API-drift canary (FUTURE.md §10)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# The export depends on undocumented internal web APIs that can change shape
|
||||||
|
# without notice. The canary fetches a small live sample and asserts only the
|
||||||
|
# normalizer's load-bearing fields (NOT full shape — provider settings objects
|
||||||
|
# churn constantly and would be pure noise). Findings carry a severity:
|
||||||
|
# ERROR — a load-bearing field is missing/mistyped; the parser will break or
|
||||||
|
# silently lose data. The backup is no longer trustworthy.
|
||||||
|
# WARN — something unfamiliar appeared (new content_type, new tool author,
|
||||||
|
# a dormant field went live). Investigate; not necessarily broken.
|
||||||
|
# OK — the assertion held.
|
||||||
|
DRIFT_ERROR = "error"
|
||||||
|
DRIFT_WARN = "warn"
|
||||||
|
DRIFT_OK = "ok"
|
||||||
|
|
||||||
|
|
||||||
|
def drift_finding(severity: str, check: str, detail: str = "") -> dict:
|
||||||
|
"""Build one canary finding."""
|
||||||
|
return {"severity": severity, "check": check, "detail": detail}
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_hidden_content_policy() -> str:
|
||||||
|
"""Read EXPORTER_HIDDEN_CONTENT from the environment, defaulting to placeholder."""
|
||||||
|
value = os.getenv("EXPORTER_HIDDEN_CONTENT", "").strip().lower()
|
||||||
|
if not value:
|
||||||
|
return HIDDEN_CONTENT_PLACEHOLDER
|
||||||
|
if value not in VALID_HIDDEN_CONTENT_POLICIES:
|
||||||
|
logger.warning(
|
||||||
|
"EXPORTER_HIDDEN_CONTENT=%r is invalid (expected full|placeholder|omit) "
|
||||||
|
"— using 'placeholder'.",
|
||||||
|
value,
|
||||||
|
)
|
||||||
|
return HIDDEN_CONTENT_PLACEHOLDER
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
# Default polite pacing between consecutive API requests (seconds). Keeps
|
||||||
|
# traffic human-paced instead of bursty. REQUEST_DELAY=0 disables.
|
||||||
|
DEFAULT_REQUEST_DELAY = 1.0
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_request_delay() -> float:
|
||||||
|
"""Read REQUEST_DELAY from the environment, defaulting to DEFAULT_REQUEST_DELAY."""
|
||||||
|
raw = os.getenv("REQUEST_DELAY", "").strip()
|
||||||
|
if not raw:
|
||||||
|
return DEFAULT_REQUEST_DELAY
|
||||||
|
try:
|
||||||
|
value = float(raw)
|
||||||
|
except ValueError:
|
||||||
|
logger.warning(
|
||||||
|
"REQUEST_DELAY=%r is not a number — using default %.1fs.",
|
||||||
|
raw,
|
||||||
|
DEFAULT_REQUEST_DELAY,
|
||||||
|
)
|
||||||
|
return DEFAULT_REQUEST_DELAY
|
||||||
|
if value < 0:
|
||||||
|
logger.warning(
|
||||||
|
"REQUEST_DELAY=%r is negative — using default %.1fs.",
|
||||||
|
raw,
|
||||||
|
DEFAULT_REQUEST_DELAY,
|
||||||
|
)
|
||||||
|
return DEFAULT_REQUEST_DELAY
|
||||||
|
return value
|
||||||
|
|
||||||
# Realistic Chrome User-Agent
|
# Realistic Chrome User-Agent
|
||||||
USER_AGENT = (
|
USER_AGENT = (
|
||||||
"Mozilla/5.0 (X11; Linux x86_64) "
|
"Mozilla/5.0 (X11; Linux x86_64) "
|
||||||
@@ -75,6 +154,9 @@ class BaseProvider(ABC):
|
|||||||
"Accept-Language": "en-US,en;q=0.9",
|
"Accept-Language": "en-US,en;q=0.9",
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
# Polite pacing between consecutive requests (REQUEST_DELAY env).
|
||||||
|
self._request_delay = resolve_request_delay()
|
||||||
|
self._last_request_at: float | None = None
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
# Abstract interface — subclasses must implement these
|
# Abstract interface — subclasses must implement these
|
||||||
@@ -89,13 +171,45 @@ class BaseProvider(ABC):
|
|||||||
"""Return the full conversation detail for a single ID."""
|
"""Return the full conversation detail for a single ID."""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def normalize_conversation(self, raw: dict) -> dict:
|
def normalize_conversation(self, raw: dict, loss_report: LossReport | None = None) -> dict:
|
||||||
"""Transform provider-specific schema to the common normalized schema."""
|
"""Transform provider-specific schema to the common normalized schema.
|
||||||
|
|
||||||
|
``loss_report`` accumulates counts of dropped/unhandled content so the
|
||||||
|
export loop can surface a single summary at the end. When None, providers
|
||||||
|
construct a throwaway local report (so calling normalize_conversation in
|
||||||
|
isolation, e.g. from tests, doesn't crash).
|
||||||
|
"""
|
||||||
|
|
||||||
|
def check_drift(self) -> list[dict]:
|
||||||
|
"""Probe the live API and assert the normalizer's load-bearing fields.
|
||||||
|
|
||||||
|
Returns a list of ``drift_finding`` dicts. The default is a no-op
|
||||||
|
(used by providers with no remote schema risk, e.g. local Claude Code
|
||||||
|
transcripts); web-API providers override this. See FUTURE.md §10.
|
||||||
|
"""
|
||||||
|
return []
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
# Concrete helpers
|
# Concrete helpers
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _pace(self) -> None:
|
||||||
|
"""Sleep so consecutive requests are at least REQUEST_DELAY apart (±25% jitter).
|
||||||
|
|
||||||
|
getattr fallback: tests construct providers via ``__new__``, skipping
|
||||||
|
``__init__`` — those run unpaced.
|
||||||
|
"""
|
||||||
|
delay = getattr(self, "_request_delay", 0)
|
||||||
|
if delay <= 0:
|
||||||
|
return
|
||||||
|
target = delay * random.uniform(0.75, 1.25)
|
||||||
|
last = getattr(self, "_last_request_at", None)
|
||||||
|
if last is not None:
|
||||||
|
wait = target - (time.monotonic() - last)
|
||||||
|
if wait > 0:
|
||||||
|
time.sleep(wait)
|
||||||
|
self._last_request_at = time.monotonic()
|
||||||
|
|
||||||
def fetch_all_conversations(self, since: datetime | None = None) -> list[dict]:
|
def fetch_all_conversations(self, since: datetime | None = None) -> list[dict]:
|
||||||
"""Fetch every conversation, handling pagination automatically.
|
"""Fetch every conversation, handling pagination automatically.
|
||||||
|
|
||||||
@@ -201,6 +315,7 @@ class BaseProvider(ABC):
|
|||||||
|
|
||||||
while attempt <= MAX_RETRIES:
|
while attempt <= MAX_RETRIES:
|
||||||
attempt += 1
|
attempt += 1
|
||||||
|
self._pace()
|
||||||
start = time.monotonic()
|
start = time.monotonic()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -326,7 +441,7 @@ class BaseProvider(ABC):
|
|||||||
msg = (
|
msg = (
|
||||||
f"[{self.provider_name}] Authentication failed (401 Unauthorized). "
|
f"[{self.provider_name}] Authentication failed (401 Unauthorized). "
|
||||||
"Your session token has likely expired. "
|
"Your session token has likely expired. "
|
||||||
"Run 'python -m src.main auth' to refresh your token."
|
"Run 'ai-chat-exporter auth' to refresh your token."
|
||||||
)
|
)
|
||||||
logger.error(msg)
|
logger.error(msg)
|
||||||
raise ProviderError(
|
raise ProviderError(
|
||||||
|
|||||||
+883
-72
File diff suppressed because it is too large
Load Diff
+221
-50
@@ -5,7 +5,25 @@ import os
|
|||||||
|
|
||||||
from curl_cffi import requests as curl_requests
|
from curl_cffi import requests as curl_requests
|
||||||
|
|
||||||
from src.providers.base import BaseProvider, ProviderError
|
from src.blocks import (
|
||||||
|
UNKNOWN_REASON_EXTRACTION_FAILED,
|
||||||
|
UNKNOWN_REASON_UNKNOWN_TYPE,
|
||||||
|
make_image_placeholder,
|
||||||
|
make_text_block,
|
||||||
|
make_thinking_block,
|
||||||
|
make_tool_result_block,
|
||||||
|
make_tool_use_block,
|
||||||
|
make_unknown_block,
|
||||||
|
)
|
||||||
|
from src.loss_report import LossReport
|
||||||
|
from src.providers.base import (
|
||||||
|
BaseProvider,
|
||||||
|
DRIFT_ERROR,
|
||||||
|
DRIFT_OK,
|
||||||
|
DRIFT_WARN,
|
||||||
|
ProviderError,
|
||||||
|
drift_finding,
|
||||||
|
)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -39,7 +57,7 @@ class ClaudeProvider(BaseProvider):
|
|||||||
"init",
|
"init",
|
||||||
RuntimeError(
|
RuntimeError(
|
||||||
"CLAUDE_SESSION_KEY is not set. "
|
"CLAUDE_SESSION_KEY is not set. "
|
||||||
"Run 'python -m src.main auth' to configure it."
|
"Run 'ai-chat-exporter auth' to configure it."
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
# Set sessionKey in the cookie jar
|
# Set sessionKey in the cookie jar
|
||||||
@@ -60,7 +78,7 @@ class ClaudeProvider(BaseProvider):
|
|||||||
"Note: Claude session keys are opaque — a 401 is the only expiry signal. "
|
"Note: Claude session keys are opaque — a 401 is the only expiry signal. "
|
||||||
"To refresh: open claude.ai in Chrome → F12 → Application → Cookies "
|
"To refresh: open claude.ai in Chrome → F12 → Application → Cookies "
|
||||||
"→ find 'sessionKey' → copy the value. "
|
"→ find 'sessionKey' → copy the value. "
|
||||||
"Then run 'python -m src.main auth' or update CLAUDE_SESSION_KEY in .env."
|
"Then run 'ai-chat-exporter auth' or update CLAUDE_SESSION_KEY in .env."
|
||||||
)
|
)
|
||||||
logger.error(msg)
|
logger.error(msg)
|
||||||
raise ProviderError(
|
raise ProviderError(
|
||||||
@@ -161,8 +179,9 @@ class ClaudeProvider(BaseProvider):
|
|||||||
|
|
||||||
return data
|
return data
|
||||||
|
|
||||||
def normalize_conversation(self, raw: dict) -> dict:
|
def normalize_conversation(self, raw: dict, loss_report: LossReport | None = None) -> dict:
|
||||||
"""Transform Claude raw schema to the common normalized schema."""
|
"""Transform Claude raw schema to the common normalized schema."""
|
||||||
|
report = loss_report if loss_report is not None else LossReport()
|
||||||
conv_id = raw.get("uuid") or raw.get("id", "")
|
conv_id = raw.get("uuid") or raw.get("id", "")
|
||||||
title = raw.get("name") or raw.get("title") or "Untitled"
|
title = raw.get("name") or raw.get("title") or "Untitled"
|
||||||
created_at = raw.get("created_at") or raw.get("create_time") or ""
|
created_at = raw.get("created_at") or raw.get("create_time") or ""
|
||||||
@@ -178,40 +197,37 @@ class ClaudeProvider(BaseProvider):
|
|||||||
|
|
||||||
# Messages
|
# Messages
|
||||||
raw_messages = raw.get("chat_messages") or raw.get("messages") or []
|
raw_messages = raw.get("chat_messages") or raw.get("messages") or []
|
||||||
messages = []
|
messages: list[dict] = []
|
||||||
|
|
||||||
for msg in raw_messages:
|
for msg in raw_messages:
|
||||||
role = _map_role(msg.get("sender") or msg.get("role", ""))
|
role = _map_role(msg.get("sender") or msg.get("role", ""))
|
||||||
if not role:
|
if not role:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Content can be a string or a list of content blocks
|
content_raw = msg.get("content") if "content" in msg else msg.get("text", "")
|
||||||
content_raw = msg.get("content") or msg.get("text") or ""
|
blocks = _extract_claude_blocks(content_raw, conv_id, report)
|
||||||
content, skipped_types = _extract_claude_text(content_raw, conv_id)
|
|
||||||
|
|
||||||
for ctype in skipped_types:
|
|
||||||
logger.warning(
|
|
||||||
"[claude] Skipping %s content in conversation %s "
|
|
||||||
"— rich content not yet supported (see FUTURE.md)",
|
|
||||||
ctype,
|
|
||||||
conv_id[:8],
|
|
||||||
)
|
|
||||||
|
|
||||||
timestamp = msg.get("created_at") or msg.get("timestamp") or None
|
timestamp = msg.get("created_at") or msg.get("timestamp") or None
|
||||||
|
|
||||||
if content is None:
|
if not blocks:
|
||||||
logger.debug("[claude] Skipping empty message in conversation %s", conv_id[:8])
|
logger.debug("[claude] Skipping empty message in conversation %s", conv_id[:8])
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
content_type = "text"
|
||||||
|
|
||||||
messages.append(
|
messages.append(
|
||||||
{
|
{
|
||||||
"role": role,
|
"role": role,
|
||||||
"content": content,
|
"content_type": content_type,
|
||||||
"content_type": "text",
|
|
||||||
"timestamp": timestamp,
|
"timestamp": timestamp,
|
||||||
|
"blocks": blocks,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
for _ in messages:
|
||||||
|
report.record_message()
|
||||||
|
report.record_conversation()
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"id": conv_id,
|
"id": conv_id,
|
||||||
"title": title,
|
"title": title,
|
||||||
@@ -223,6 +239,70 @@ class ClaudeProvider(BaseProvider):
|
|||||||
"messages": messages,
|
"messages": messages,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def check_drift(self) -> list[dict]:
|
||||||
|
"""Probe one listing page + one conversation; assert load-bearing fields.
|
||||||
|
|
||||||
|
See FUTURE.md §10. Real Claude messages are flat ``text``/``sender``;
|
||||||
|
the rich ``content`` block path is dormant, so its activation (content
|
||||||
|
becoming a list) is itself a drift event worth flagging. ``attachments``
|
||||||
|
/ ``files`` are ignored by the normalizer — non-empty values are a
|
||||||
|
silent-loss risk.
|
||||||
|
"""
|
||||||
|
findings: list[dict] = []
|
||||||
|
page = self.list_conversations(offset=0, limit=5)
|
||||||
|
if not page:
|
||||||
|
return [drift_finding(DRIFT_WARN, "listing",
|
||||||
|
"empty listing — cannot verify shape")]
|
||||||
|
|
||||||
|
item = page[0]
|
||||||
|
if not (item.get("uuid") or item.get("id")):
|
||||||
|
findings.append(drift_finding(DRIFT_ERROR, "listing", "summary missing uuid/id"))
|
||||||
|
if not (item.get("name") or item.get("title")):
|
||||||
|
findings.append(drift_finding(DRIFT_WARN, "listing", "no name/title on summary"))
|
||||||
|
if not (item.get("updated_at") or item.get("update_time")):
|
||||||
|
findings.append(drift_finding(DRIFT_WARN, "listing", "no updated_at on summary"))
|
||||||
|
|
||||||
|
conv_id = item.get("uuid") or item.get("id")
|
||||||
|
if not conv_id:
|
||||||
|
return findings or [drift_finding(DRIFT_ERROR, "listing", "no id to fetch")]
|
||||||
|
raw = self.get_conversation(conv_id)
|
||||||
|
if not (raw.get("uuid") or raw.get("id")):
|
||||||
|
findings.append(drift_finding(DRIFT_ERROR, "detail", "no uuid/id on detail"))
|
||||||
|
msgs = raw.get("chat_messages") or raw.get("messages")
|
||||||
|
if not isinstance(msgs, list) or not msgs:
|
||||||
|
findings.append(drift_finding(
|
||||||
|
DRIFT_ERROR, "detail", "chat_messages missing/empty — 0 messages"))
|
||||||
|
return findings
|
||||||
|
|
||||||
|
saw_sender = saw_text = False
|
||||||
|
flagged_list = flagged_attach = False
|
||||||
|
for m in msgs:
|
||||||
|
if m.get("sender") or m.get("role"):
|
||||||
|
saw_sender = True
|
||||||
|
content = m.get("content")
|
||||||
|
if isinstance(content, list) and not flagged_list:
|
||||||
|
findings.append(drift_finding(
|
||||||
|
DRIFT_WARN, "content",
|
||||||
|
"message.content is now a LIST — Claude switched to typed blocks; "
|
||||||
|
"the dormant _dispatch_claude_block path is now live, verify rendering"))
|
||||||
|
flagged_list = True
|
||||||
|
if m.get("text") or content:
|
||||||
|
saw_text = True
|
||||||
|
if (m.get("attachments") or m.get("files")) and not flagged_attach:
|
||||||
|
findings.append(drift_finding(
|
||||||
|
DRIFT_WARN, "attachments",
|
||||||
|
"message has non-empty attachments/files — the normalizer ignores "
|
||||||
|
"these (silent loss); add handling"))
|
||||||
|
flagged_attach = True
|
||||||
|
|
||||||
|
if not saw_sender:
|
||||||
|
findings.append(drift_finding(DRIFT_ERROR, "messages", "no sender/role on any message"))
|
||||||
|
if not saw_text:
|
||||||
|
findings.append(drift_finding(DRIFT_WARN, "messages", "no text/content found"))
|
||||||
|
if not findings:
|
||||||
|
findings.append(drift_finding(DRIFT_OK, "schema", "all load-bearing fields present"))
|
||||||
|
return findings
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Internal helpers
|
# Internal helpers
|
||||||
@@ -242,43 +322,134 @@ def _map_role(sender: str) -> str | None:
|
|||||||
return mapping.get(sender.lower()) if sender else None
|
return mapping.get(sender.lower()) if sender else None
|
||||||
|
|
||||||
|
|
||||||
def _extract_claude_text(
|
def _extract_claude_blocks(
|
||||||
content: str | list | dict, conv_id: str
|
content: str | list | dict | None, conv_id: str, report: LossReport
|
||||||
) -> tuple[str | None, list[str]]:
|
) -> list[dict]:
|
||||||
"""Extract plain text from a Claude content field.
|
"""Extract typed blocks from a Claude content field.
|
||||||
|
|
||||||
Returns:
|
Defensive dispatch — zero observed cases of rich Claude content in the
|
||||||
(text_or_None, list_of_skipped_content_types)
|
user's archive at planning time, so this is theory-only. Real shapes
|
||||||
|
will be locked in v0.4.1 once captured. Any unrecognised block type
|
||||||
|
surfaces as an `unknown` block + WARNING + tally.
|
||||||
"""
|
"""
|
||||||
skipped: list[str] = []
|
if content is None:
|
||||||
|
return []
|
||||||
|
|
||||||
if isinstance(content, str):
|
if isinstance(content, str):
|
||||||
text = content.strip()
|
block = make_text_block(content)
|
||||||
return (text if text else None), skipped
|
return [block] if block else []
|
||||||
|
|
||||||
if isinstance(content, list):
|
if isinstance(content, list):
|
||||||
parts: list[str] = []
|
blocks: list[dict] = []
|
||||||
for block in content:
|
for item in content:
|
||||||
if isinstance(block, str):
|
if isinstance(item, str):
|
||||||
parts.append(block)
|
block = make_text_block(item)
|
||||||
elif isinstance(block, dict):
|
if block:
|
||||||
btype = block.get("type", "text")
|
blocks.append(block)
|
||||||
if btype == "text":
|
elif isinstance(item, dict):
|
||||||
t = block.get("text", "").strip()
|
blocks.extend(_dispatch_claude_block(item, conv_id, report))
|
||||||
if t:
|
return blocks
|
||||||
parts.append(t)
|
|
||||||
else:
|
|
||||||
skipped.append(btype)
|
|
||||||
text = "\n".join(parts).strip()
|
|
||||||
return (text if text else None), skipped
|
|
||||||
|
|
||||||
if isinstance(content, dict):
|
if isinstance(content, dict):
|
||||||
btype = content.get("type", "text")
|
return _dispatch_claude_block(content, conv_id, report)
|
||||||
if btype == "text":
|
|
||||||
text = content.get("text", "").strip()
|
|
||||||
return (text if text else None), skipped
|
|
||||||
else:
|
|
||||||
skipped.append(btype)
|
|
||||||
return None, skipped
|
|
||||||
|
|
||||||
return None, skipped
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def _dispatch_claude_block(block: dict, conv_id: str, report: LossReport) -> list[dict]:
|
||||||
|
"""Translate one raw Claude content block into normalized blocks."""
|
||||||
|
btype = block.get("type", "text")
|
||||||
|
|
||||||
|
if btype == "text":
|
||||||
|
block_obj = make_text_block(block.get("text", "") or "")
|
||||||
|
return [block_obj] if block_obj else []
|
||||||
|
|
||||||
|
if btype == "thinking":
|
||||||
|
# Claude extended-thinking blocks may use 'thinking' or 'text' field.
|
||||||
|
text = block.get("thinking") or block.get("text") or ""
|
||||||
|
block_obj = make_thinking_block(text)
|
||||||
|
return [block_obj] if block_obj else []
|
||||||
|
|
||||||
|
if btype == "tool_use":
|
||||||
|
return [
|
||||||
|
make_tool_use_block(
|
||||||
|
name=block.get("name", "") or "",
|
||||||
|
input_data=block.get("input"),
|
||||||
|
tool_id=block.get("id"),
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
if btype == "tool_result":
|
||||||
|
# ``content`` may be a string or a list of nested blocks (recursive).
|
||||||
|
nested = block.get("content")
|
||||||
|
output = _flatten_tool_result_content(nested, conv_id, report)
|
||||||
|
return [
|
||||||
|
make_tool_result_block(
|
||||||
|
output=output,
|
||||||
|
tool_name=None,
|
||||||
|
is_error=bool(block.get("is_error")),
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
if btype == "image":
|
||||||
|
# Source shape is unverified; try the most likely fields.
|
||||||
|
source = block.get("source") or {}
|
||||||
|
ref = ""
|
||||||
|
if isinstance(source, dict):
|
||||||
|
ref = (
|
||||||
|
source.get("file_uuid")
|
||||||
|
or source.get("media_type")
|
||||||
|
or source.get("url")
|
||||||
|
or ""
|
||||||
|
)
|
||||||
|
return [make_image_placeholder(ref=ref or "(unknown)", source="user_upload")]
|
||||||
|
|
||||||
|
# Unknown block type
|
||||||
|
keys = list(block.keys())
|
||||||
|
logger.warning(
|
||||||
|
"[claude] Unknown block type %r in conversation %s "
|
||||||
|
"— see plan §Data-loss visibility (rendering as unknown block)",
|
||||||
|
btype,
|
||||||
|
conv_id[:8],
|
||||||
|
)
|
||||||
|
report.record_unknown(btype or "?")
|
||||||
|
return [
|
||||||
|
make_unknown_block(
|
||||||
|
raw_type=btype or "?",
|
||||||
|
observed_keys=keys,
|
||||||
|
reason=UNKNOWN_REASON_UNKNOWN_TYPE,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _flatten_tool_result_content(
|
||||||
|
nested: object, conv_id: str, report: LossReport
|
||||||
|
) -> str:
|
||||||
|
"""Flatten Claude tool_result content (string OR list of nested blocks) to text.
|
||||||
|
|
||||||
|
Recurses into nested text blocks; any non-text nested block becomes a
|
||||||
|
visible inline marker so non-text content isn't silently dropped.
|
||||||
|
"""
|
||||||
|
if nested is None:
|
||||||
|
return ""
|
||||||
|
if isinstance(nested, str):
|
||||||
|
return nested
|
||||||
|
if isinstance(nested, list):
|
||||||
|
chunks: list[str] = []
|
||||||
|
for item in nested:
|
||||||
|
if isinstance(item, str):
|
||||||
|
chunks.append(item)
|
||||||
|
elif isinstance(item, dict):
|
||||||
|
btype = item.get("type", "text")
|
||||||
|
if btype == "text":
|
||||||
|
chunks.append(item.get("text", "") or "")
|
||||||
|
else:
|
||||||
|
keys = list(item.keys())[:10]
|
||||||
|
report.record_extraction_failure(f"tool_result.{btype}")
|
||||||
|
chunks.append(
|
||||||
|
f"[Unsupported nested {btype} block; keys={keys}]"
|
||||||
|
)
|
||||||
|
return "\n".join(c for c in chunks if c)
|
||||||
|
if isinstance(nested, dict):
|
||||||
|
return _flatten_tool_result_content([nested], conv_id, report)
|
||||||
|
return str(nested)
|
||||||
|
|||||||
@@ -0,0 +1,750 @@
|
|||||||
|
"""Claude Code session provider — archives local agent transcripts.
|
||||||
|
|
||||||
|
Reads JSONL session files from ``~/.claude/projects/<munged-cwd>/<uuid>.jsonl``
|
||||||
|
(override with ``CLAUDE_CODE_DIR``). No tokens, no rate limits, no ToS risk —
|
||||||
|
but the data lives in single files Claude Code may clean up, and it contains
|
||||||
|
deliverables (reviews, plans, analyses) that exist nowhere else.
|
||||||
|
|
||||||
|
Rendering follows the EXPORTER_HIDDEN_CONTENT policy (decided 2026-06-12):
|
||||||
|
prose-only by default. User prompts and assistant text are kept; tool_use /
|
||||||
|
tool_result traffic is collapsed to one grouped placeholder per activity run
|
||||||
|
(measured: dialogue prose is ~4% of session bytes); thinking blocks are
|
||||||
|
dropped (counted in the run summary, no placeholder). ``full`` keeps
|
||||||
|
everything.
|
||||||
|
|
||||||
|
Record types in a session file: ``user`` / ``assistant`` carry the dialogue
|
||||||
|
(Anthropic-style ``message.content`` block arrays); ``ai-title`` carries the
|
||||||
|
evolving session title (last one wins); ``last-prompt``,
|
||||||
|
``file-history-snapshot``, ``attachment``, ``permission-mode``, ``system``
|
||||||
|
are harness records and are skipped. ``isMeta`` records are harness-generated
|
||||||
|
user records and are skipped.
|
||||||
|
|
||||||
|
Subagents (Task tool): Claude Code stores each subagent's transcript as a
|
||||||
|
separate ``<session>/subagents/agent-*.jsonl`` file with an ``agent-*.meta.json``
|
||||||
|
sidecar (``agentType``, ``description``, ``toolUseId``). ``_load_subagents``
|
||||||
|
loads them keyed by ``toolUseId``; at the ``Task``/``Agent`` tool call that
|
||||||
|
spawned it, the subagent is folded inline as a collapsible ``<details>`` block
|
||||||
|
(the subagent's own records are flagged ``isSidechain`` and only processed on
|
||||||
|
this recursive pass — the top-level pass still skips sidechain records).
|
||||||
|
|
||||||
|
Scan scope: multiple ``projects/`` roots are supported — ``CLAUDE_CODE_DIR`` may
|
||||||
|
be an ``os.pathsep``-separated list and ``CLAUDE_CONFIG_DIR``'s ``projects/`` is
|
||||||
|
included when set. Sessions from all roots are merged by launch-folder; a session
|
||||||
|
UUID present in two roots keeps the newer-mtime copy. See ``resolve_roots``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from src.blocks import (
|
||||||
|
COLLAPSED_KIND_TOOL_DUMP,
|
||||||
|
UNKNOWN_REASON_UNKNOWN_TYPE,
|
||||||
|
make_collapsed_block,
|
||||||
|
make_image_placeholder,
|
||||||
|
make_subagent_block,
|
||||||
|
make_text_block,
|
||||||
|
make_thinking_block,
|
||||||
|
make_tool_result_block,
|
||||||
|
make_tool_use_block,
|
||||||
|
make_unknown_block,
|
||||||
|
)
|
||||||
|
from src.loss_report import LossReport
|
||||||
|
from src.providers.base import (
|
||||||
|
BaseProvider,
|
||||||
|
HIDDEN_CONTENT_FULL,
|
||||||
|
ProviderError,
|
||||||
|
VALID_HIDDEN_CONTENT_POLICIES,
|
||||||
|
resolve_hidden_content_policy,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
DEFAULT_PROJECTS_DIR = "~/.claude/projects"
|
||||||
|
|
||||||
|
# Tool names whose call spawns a subagent (Task tool) — its separate transcript
|
||||||
|
# is folded inline. Both spellings have appeared across Claude Code versions.
|
||||||
|
_SUBAGENT_TOOL_NAMES = {"Task", "Agent"}
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_roots(projects_dir=None) -> list[Path]:
|
||||||
|
"""Resolve the ordered list of Claude Code ``projects/`` roots to scan.
|
||||||
|
|
||||||
|
Precedence:
|
||||||
|
1. Explicit ``projects_dir`` (a single path or a list) — used by tests.
|
||||||
|
2. ``CLAUDE_CODE_DIR`` env, split on ``os.pathsep`` (``:``) so multiple
|
||||||
|
roots can be given; a single path (no separator) stays backward
|
||||||
|
compatible. Falls back to the default root when unset.
|
||||||
|
3. Additionally, if ``CLAUDE_CONFIG_DIR`` is set, its ``projects/`` subdir
|
||||||
|
is appended — this is the common "non-default config dir" case.
|
||||||
|
|
||||||
|
Roots are expanded, de-duplicated (order preserved), and returned as-is
|
||||||
|
(existence is checked by the caller / ``_scan``). Sessions from all roots are
|
||||||
|
merged by launch-folder; there is no per-source label.
|
||||||
|
"""
|
||||||
|
raw: list[str]
|
||||||
|
if projects_dir is not None:
|
||||||
|
raw = [str(p) for p in projects_dir] if isinstance(projects_dir, (list, tuple)) \
|
||||||
|
else [str(projects_dir)]
|
||||||
|
else:
|
||||||
|
env = os.getenv("CLAUDE_CODE_DIR")
|
||||||
|
raw = env.split(os.pathsep) if env else [DEFAULT_PROJECTS_DIR]
|
||||||
|
config_dir = os.getenv("CLAUDE_CONFIG_DIR")
|
||||||
|
if config_dir:
|
||||||
|
raw.append(str(Path(config_dir) / "projects"))
|
||||||
|
|
||||||
|
roots: list[Path] = []
|
||||||
|
for p in raw:
|
||||||
|
p = p.strip()
|
||||||
|
if not p:
|
||||||
|
continue
|
||||||
|
path = Path(p).expanduser()
|
||||||
|
if path not in roots:
|
||||||
|
roots.append(path)
|
||||||
|
return roots
|
||||||
|
|
||||||
|
# Harness-injected tags inside user message text. Stripped so exports contain
|
||||||
|
# the dialogue, not the CLI plumbing. A record that is nothing but tags
|
||||||
|
# (e.g. a /model invocation) ends up empty and is skipped.
|
||||||
|
_HARNESS_TAG_RE = re.compile(
|
||||||
|
r"<local-command-caveat>.*?</local-command-caveat>"
|
||||||
|
r"|<command-name>.*?</command-name>"
|
||||||
|
r"|<command-message>.*?</command-message>"
|
||||||
|
r"|<command-args>.*?</command-args>"
|
||||||
|
r"|<command-contents>.*?</command-contents>"
|
||||||
|
r"|<local-command-stdout>.*?</local-command-stdout>"
|
||||||
|
r"|<system-reminder>.*?</system-reminder>",
|
||||||
|
re.DOTALL,
|
||||||
|
)
|
||||||
|
|
||||||
|
# How many distinct tool names to list in a collapsed-activity placeholder.
|
||||||
|
_TOOL_NAMES_SHOWN = 4
|
||||||
|
|
||||||
|
|
||||||
|
def _strip_harness_noise(text: str) -> str:
|
||||||
|
if not isinstance(text, str):
|
||||||
|
return ""
|
||||||
|
return _HARNESS_TAG_RE.sub("", text).strip()
|
||||||
|
|
||||||
|
|
||||||
|
class ClaudeCodeProvider(BaseProvider):
|
||||||
|
"""Local-file provider over Claude Code session transcripts."""
|
||||||
|
|
||||||
|
provider_name = "claude-code"
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
projects_dir: str | Path | None = None,
|
||||||
|
hidden_content: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
super().__init__()
|
||||||
|
self._projects_dirs = resolve_roots(projects_dir)
|
||||||
|
self._hidden_content = (
|
||||||
|
hidden_content
|
||||||
|
if hidden_content in VALID_HIDDEN_CONTENT_POLICIES
|
||||||
|
else resolve_hidden_content_policy()
|
||||||
|
)
|
||||||
|
# conv_id → session file path, populated by _scan()
|
||||||
|
self._path_map: dict[str, Path] = {}
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# BaseProvider interface
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def list_conversations(self, offset: int = 0, limit: int = 100) -> list[dict]:
|
||||||
|
full = self._scan()
|
||||||
|
return full[offset : offset + limit]
|
||||||
|
|
||||||
|
def fetch_all_conversations(self, since: datetime | None = None) -> list[dict]:
|
||||||
|
convs = self._scan()
|
||||||
|
if since is not None:
|
||||||
|
since_aware = since if since.tzinfo else since.replace(tzinfo=timezone.utc)
|
||||||
|
convs = [
|
||||||
|
c for c in convs
|
||||||
|
if datetime.fromisoformat(c["updated_at"]) >= since_aware
|
||||||
|
]
|
||||||
|
logger.info(
|
||||||
|
"[claude-code] Found %d session(s) under %s",
|
||||||
|
len(convs),
|
||||||
|
", ".join(str(d) for d in self._projects_dirs),
|
||||||
|
)
|
||||||
|
return convs
|
||||||
|
|
||||||
|
def get_conversation(self, conv_id: str) -> dict:
|
||||||
|
path = self._path_map.get(conv_id)
|
||||||
|
if path is None:
|
||||||
|
# Direct call without a prior listing (e.g. tests) — scan first.
|
||||||
|
self._scan()
|
||||||
|
path = self._path_map.get(conv_id)
|
||||||
|
if path is None or not path.exists():
|
||||||
|
raise ProviderError(
|
||||||
|
self.provider_name,
|
||||||
|
f"get_conversation({conv_id[:8]})",
|
||||||
|
FileNotFoundError(f"No session file for id {conv_id}"),
|
||||||
|
)
|
||||||
|
|
||||||
|
records = _parse_jsonl(path)
|
||||||
|
|
||||||
|
mtime = datetime.fromtimestamp(path.stat().st_mtime, tz=timezone.utc)
|
||||||
|
return {
|
||||||
|
"id": conv_id,
|
||||||
|
"_path": str(path),
|
||||||
|
"_records": records,
|
||||||
|
# Subagent (Task-tool) transcripts, keyed by the parent tool_use id
|
||||||
|
# that spawned them, folded inline during normalization.
|
||||||
|
"_subagents": _load_subagents(path),
|
||||||
|
# Listing and normalized updated_at must match, or the cache
|
||||||
|
# staleness comparison would re-export every session every run.
|
||||||
|
"_mtime_iso": mtime.isoformat(),
|
||||||
|
}
|
||||||
|
|
||||||
|
def normalize_conversation(self, raw: dict, loss_report: LossReport | None = None) -> dict:
|
||||||
|
report = loss_report if loss_report is not None else LossReport()
|
||||||
|
policy = getattr(self, "_hidden_content", None) or resolve_hidden_content_policy()
|
||||||
|
conv_id = raw.get("id") or ""
|
||||||
|
records: list[dict] = raw.get("_records") or []
|
||||||
|
|
||||||
|
subagents: dict = raw.get("_subagents") or {}
|
||||||
|
title = _extract_title(records)
|
||||||
|
# Append the repos this session touched, e.g. "… [repo-a, repo-b]", so
|
||||||
|
# sessions launched from a workspace root (which all land in one
|
||||||
|
# folder-named notebook) stay scannable and searchable.
|
||||||
|
launch_cwd = _extract_launch_cwd(records)
|
||||||
|
if launch_cwd:
|
||||||
|
repos = _repos_touched(records, launch_cwd)
|
||||||
|
if repos:
|
||||||
|
title = f"{title} [{', '.join(repos)}]"
|
||||||
|
project = _extract_project(records, raw.get("_path"))
|
||||||
|
created_at = next(
|
||||||
|
(r.get("timestamp") for r in records if r.get("timestamp")), ""
|
||||||
|
)
|
||||||
|
updated_at = raw.get("_mtime_iso") or next(
|
||||||
|
(r.get("timestamp") for r in reversed(records) if r.get("timestamp")), ""
|
||||||
|
)
|
||||||
|
|
||||||
|
messages = _extract_messages(records, conv_id, report, policy, subagents)
|
||||||
|
for _ in messages:
|
||||||
|
report.record_message()
|
||||||
|
report.record_conversation()
|
||||||
|
|
||||||
|
return {
|
||||||
|
"id": conv_id,
|
||||||
|
"title": title,
|
||||||
|
"provider": self.provider_name,
|
||||||
|
"project": project,
|
||||||
|
"created_at": created_at or "",
|
||||||
|
"updated_at": updated_at or "",
|
||||||
|
"message_count": len(messages),
|
||||||
|
"messages": messages,
|
||||||
|
}
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Scanning
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _scan(self) -> list[dict]:
|
||||||
|
existing = [d for d in self._projects_dirs if d.is_dir()]
|
||||||
|
if not existing:
|
||||||
|
logger.warning(
|
||||||
|
"[claude-code] No projects directory exists: %s",
|
||||||
|
", ".join(str(d) for d in self._projects_dirs),
|
||||||
|
)
|
||||||
|
return []
|
||||||
|
|
||||||
|
# conv_id → (mtime, conv dict). Roots are merged by launch-folder; if the
|
||||||
|
# same session UUID appears in two roots (e.g. a live dir and a backup),
|
||||||
|
# the newer-mtime copy wins so we never emit two entries for one session.
|
||||||
|
by_id: dict[str, tuple[float, dict]] = {}
|
||||||
|
for root in existing:
|
||||||
|
for proj_dir in sorted(p for p in root.iterdir() if p.is_dir()):
|
||||||
|
for session_file in sorted(proj_dir.glob("*.jsonl")):
|
||||||
|
try:
|
||||||
|
stat = session_file.stat()
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
if stat.st_size == 0:
|
||||||
|
continue
|
||||||
|
conv_id = session_file.stem
|
||||||
|
prev = by_id.get(conv_id)
|
||||||
|
if prev is not None and prev[0] >= stat.st_mtime:
|
||||||
|
logger.debug(
|
||||||
|
"[claude-code] Duplicate session %s in %s; keeping newer copy",
|
||||||
|
conv_id[:8], root,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
self._path_map[conv_id] = session_file
|
||||||
|
title, project, created = _read_session_meta(session_file)
|
||||||
|
by_id[conv_id] = (
|
||||||
|
stat.st_mtime,
|
||||||
|
{
|
||||||
|
"id": conv_id,
|
||||||
|
"title": title,
|
||||||
|
"project": project,
|
||||||
|
# The --project filter and dry-run table read the
|
||||||
|
# listing dict, not the normalized conversation.
|
||||||
|
"_project_name": project,
|
||||||
|
"created_at": created,
|
||||||
|
"updated_at": datetime.fromtimestamp(
|
||||||
|
stat.st_mtime, tz=timezone.utc
|
||||||
|
).isoformat(),
|
||||||
|
"_path": str(session_file),
|
||||||
|
"_project_dir": proj_dir.name,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
# Deterministic order: by bucket dir, then session id.
|
||||||
|
return sorted(
|
||||||
|
(conv for _, conv in by_id.values()),
|
||||||
|
key=lambda c: (c["_project_dir"], c["id"]),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Internal helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _read_session_meta(path: Path) -> tuple[str, str | None, str]:
|
||||||
|
"""Light single-pass scan for listing metadata: (title, project, created_at).
|
||||||
|
|
||||||
|
Substring guards keep this cheap — only candidate lines are JSON-parsed.
|
||||||
|
The full-fidelity extraction happens later in normalize_conversation.
|
||||||
|
"""
|
||||||
|
title = ""
|
||||||
|
project: str | None = None
|
||||||
|
created = ""
|
||||||
|
try:
|
||||||
|
with path.open(encoding="utf-8") as fh:
|
||||||
|
for line in fh:
|
||||||
|
if '"ai-title"' in line:
|
||||||
|
try:
|
||||||
|
rec = json.loads(line)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
continue
|
||||||
|
if rec.get("type") == "ai-title" and rec.get("aiTitle"):
|
||||||
|
title = str(rec["aiTitle"]) # last one wins
|
||||||
|
continue
|
||||||
|
if (not created or project is None) and (
|
||||||
|
'"timestamp"' in line or '"cwd"' in line
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
rec = json.loads(line)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
continue
|
||||||
|
if not created and rec.get("timestamp"):
|
||||||
|
created = str(rec["timestamp"])
|
||||||
|
if project is None and rec.get("cwd"):
|
||||||
|
project = Path(rec["cwd"]).name or None
|
||||||
|
except OSError as e:
|
||||||
|
logger.warning("[claude-code] Could not read %s: %s", path, e)
|
||||||
|
return title, project, created
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_title(records: list[dict]) -> str:
|
||||||
|
"""Last ai-title record wins; fall back to the first real user prompt."""
|
||||||
|
title = ""
|
||||||
|
for rec in records:
|
||||||
|
if rec.get("type") == "ai-title" and rec.get("aiTitle"):
|
||||||
|
title = str(rec["aiTitle"])
|
||||||
|
if title:
|
||||||
|
return title
|
||||||
|
|
||||||
|
for rec in records:
|
||||||
|
if rec.get("type") != "user" or rec.get("isMeta") or rec.get("isSidechain"):
|
||||||
|
continue
|
||||||
|
content = (rec.get("message") or {}).get("content")
|
||||||
|
if isinstance(content, str):
|
||||||
|
text = _strip_harness_noise(content)
|
||||||
|
elif isinstance(content, list):
|
||||||
|
text = " ".join(
|
||||||
|
_strip_harness_noise(item.get("text", ""))
|
||||||
|
for item in content
|
||||||
|
if isinstance(item, dict) and item.get("type") == "text"
|
||||||
|
).strip()
|
||||||
|
else:
|
||||||
|
text = ""
|
||||||
|
if text:
|
||||||
|
return text[:80]
|
||||||
|
return "Untitled session"
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_project(records: list[dict], path: str | None) -> str | None:
|
||||||
|
"""Project = basename of the session's working directory."""
|
||||||
|
for rec in records:
|
||||||
|
cwd = rec.get("cwd")
|
||||||
|
if cwd:
|
||||||
|
name = Path(cwd).name
|
||||||
|
if name:
|
||||||
|
return name
|
||||||
|
# Fallback: the munged directory name (cannot be reliably de-munged
|
||||||
|
# because '-' is both the path separator and a legal name character).
|
||||||
|
if path:
|
||||||
|
return Path(path).parent.name.lstrip("-") or None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_jsonl(path: Path) -> list[dict]:
|
||||||
|
"""Read a JSONL file into records, tolerating (and logging) bad lines."""
|
||||||
|
records: list[dict] = []
|
||||||
|
bad_lines = 0
|
||||||
|
try:
|
||||||
|
text = path.read_text(encoding="utf-8")
|
||||||
|
except OSError as e:
|
||||||
|
logger.warning("[claude-code] Could not read %s: %s", path, e)
|
||||||
|
return records
|
||||||
|
for line in text.splitlines():
|
||||||
|
line = line.strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
records.append(json.loads(line))
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
bad_lines += 1
|
||||||
|
if bad_lines:
|
||||||
|
logger.warning(
|
||||||
|
"[claude-code] %s: skipped %d unparseable line(s)", path.name, bad_lines
|
||||||
|
)
|
||||||
|
return records
|
||||||
|
|
||||||
|
|
||||||
|
def _load_subagents(session_path: Path) -> dict:
|
||||||
|
"""Map spawning tool_use id → ``{"meta": {...}, "records": [...]}``.
|
||||||
|
|
||||||
|
Claude Code writes subagent transcripts to
|
||||||
|
``<session>/subagents/agent-*.jsonl`` with an ``agent-*.meta.json`` sidecar
|
||||||
|
carrying ``toolUseId`` (which parent ``Task``/``Agent`` call spawned it).
|
||||||
|
Files without a readable meta (no id to position them) are skipped.
|
||||||
|
"""
|
||||||
|
submap: dict[str, dict] = {}
|
||||||
|
subdir = session_path.parent / session_path.stem / "subagents"
|
||||||
|
if not subdir.is_dir():
|
||||||
|
return submap
|
||||||
|
for jf in sorted(subdir.glob("*.jsonl")):
|
||||||
|
meta_path = jf.with_suffix(".meta.json")
|
||||||
|
try:
|
||||||
|
meta = json.loads(meta_path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, json.JSONDecodeError):
|
||||||
|
logger.warning(
|
||||||
|
"[claude-code] Subagent %s has no readable meta; skipping", jf.name
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
tool_id = meta.get("toolUseId")
|
||||||
|
if not tool_id:
|
||||||
|
continue
|
||||||
|
submap[tool_id] = {"meta": meta, "records": _parse_jsonl(jf)}
|
||||||
|
return submap
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_launch_cwd(records: list[dict]) -> str | None:
|
||||||
|
"""The session's working directory (constant per session; first cwd wins)."""
|
||||||
|
for rec in records:
|
||||||
|
cwd = rec.get("cwd")
|
||||||
|
if cwd:
|
||||||
|
return cwd
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
# tool_use input keys that carry a file path.
|
||||||
|
_TOOL_PATH_KEYS = ("file_path", "path", "notebook_path")
|
||||||
|
|
||||||
|
# Optional ignore-list: git repos to never tag (comma-separated names). Rarely
|
||||||
|
# needed with git-root detection (config/reference dirs are already excluded
|
||||||
|
# because they aren't git repos), but kept as an escape hatch.
|
||||||
|
def _ignored_repos() -> set[str]:
|
||||||
|
env = os.getenv("CLAUDE_CODE_REPO_TAG_IGNORE", "")
|
||||||
|
return {s.strip() for s in env.split(",") if s.strip()}
|
||||||
|
|
||||||
|
|
||||||
|
# dir Path → git-repo name it belongs to (or None). Process-wide; the working
|
||||||
|
# tree doesn't change under us mid-run, so caching walked dirs is safe.
|
||||||
|
_GIT_ROOT_CACHE: dict[Path, str | None] = {}
|
||||||
|
_CACHE_MISS = object()
|
||||||
|
|
||||||
|
|
||||||
|
def _git_root_name(path: Path, max_steps: int = 25) -> str | None:
|
||||||
|
"""Name of the git repo ``path`` lives in — nearest ancestor with ``.git``.
|
||||||
|
|
||||||
|
Walks up from ``path`` until a ``.git`` entry is found (returns that dir's
|
||||||
|
basename) or the filesystem root is reached (returns ``None``). Disk-based:
|
||||||
|
a path in no git repo, or a repo no longer on disk, yields ``None``.
|
||||||
|
"""
|
||||||
|
cur = path
|
||||||
|
for _ in range(max_steps):
|
||||||
|
cached = _GIT_ROOT_CACHE.get(cur, _CACHE_MISS)
|
||||||
|
if cached is not _CACHE_MISS:
|
||||||
|
return cached
|
||||||
|
try:
|
||||||
|
if (cur / ".git").exists():
|
||||||
|
_GIT_ROOT_CACHE[cur] = cur.name
|
||||||
|
return cur.name
|
||||||
|
except OSError:
|
||||||
|
break
|
||||||
|
if cur.parent == cur: # filesystem root
|
||||||
|
break
|
||||||
|
cur = cur.parent
|
||||||
|
_GIT_ROOT_CACHE[path] = None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
# Absolute path-like tokens inside Bash command strings (file_path/path keys are
|
||||||
|
# matched directly). git-root resolution short-circuits on non-repo paths.
|
||||||
|
_ABS_PATH_RE = re.compile(r"/(?:[\w.\-]+/)*[\w.\-]+")
|
||||||
|
|
||||||
|
|
||||||
|
def _repos_touched(
|
||||||
|
records: list[dict], launch_cwd: str, cap: int = 3, ignore: set[str] | None = None
|
||||||
|
) -> list[str]:
|
||||||
|
"""Repos a session touched, for the title's ``[repo-a, repo-b]`` tag.
|
||||||
|
|
||||||
|
A file's repo is the **git repository it lives in** (nearest ancestor with a
|
||||||
|
``.git``), resolved for every absolute path seen in a tool_use input
|
||||||
|
(``file_path``/``path``/``notebook_path`` and absolute home paths inside Bash
|
||||||
|
``command`` strings). This works anywhere in the filesystem — not just under
|
||||||
|
the launch directory — so cross-workspace work is captured, and non-repo
|
||||||
|
noise (config dirs, one-off files, reference dirs) is excluded because it
|
||||||
|
isn't a git repo. Ordered by touch frequency, capped with a trailing ``…``.
|
||||||
|
"""
|
||||||
|
ignore = _ignored_repos() if ignore is None else ignore
|
||||||
|
counts: Counter = Counter()
|
||||||
|
seen: set[str] = set()
|
||||||
|
|
||||||
|
def note(p) -> None:
|
||||||
|
if not isinstance(p, str) or not p:
|
||||||
|
return
|
||||||
|
if not p.startswith("/"): # resolve relative paths against the launch cwd
|
||||||
|
if not launch_cwd:
|
||||||
|
return
|
||||||
|
p = str(Path(launch_cwd) / p)
|
||||||
|
if p in seen:
|
||||||
|
return
|
||||||
|
seen.add(p)
|
||||||
|
name = _git_root_name(Path(p))
|
||||||
|
if name and not name.startswith(".") and name not in ignore:
|
||||||
|
counts[name] += 1
|
||||||
|
|
||||||
|
for rec in records:
|
||||||
|
msg = rec.get("message") or {}
|
||||||
|
content = msg.get("content")
|
||||||
|
if not isinstance(content, list):
|
||||||
|
continue
|
||||||
|
for item in content:
|
||||||
|
if not isinstance(item, dict) or item.get("type") != "tool_use":
|
||||||
|
continue
|
||||||
|
inp = item.get("input")
|
||||||
|
if not isinstance(inp, dict):
|
||||||
|
continue
|
||||||
|
for key in _TOOL_PATH_KEYS:
|
||||||
|
note(inp.get(key))
|
||||||
|
cmd = inp.get("command")
|
||||||
|
if isinstance(cmd, str):
|
||||||
|
for m in _ABS_PATH_RE.finditer(cmd):
|
||||||
|
note(m.group(0))
|
||||||
|
|
||||||
|
ordered = [name for name, _ in counts.most_common()]
|
||||||
|
if len(ordered) > cap:
|
||||||
|
return ordered[:cap] + ["…"]
|
||||||
|
return ordered
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_messages(
|
||||||
|
records: list[dict],
|
||||||
|
conv_id: str,
|
||||||
|
report: LossReport,
|
||||||
|
policy: str,
|
||||||
|
subagents: dict | None = None,
|
||||||
|
include_sidechain: bool = False,
|
||||||
|
) -> list[dict]:
|
||||||
|
"""Normalize Claude Code records into messages.
|
||||||
|
|
||||||
|
``subagents`` maps a spawning ``Task``/``Agent`` tool_use id to its separate
|
||||||
|
transcript ``{"meta": ..., "records": ...}``; when a matching tool_use is
|
||||||
|
seen its subagent is folded inline as a subagent block (extracted
|
||||||
|
recursively under the same policy). ``include_sidechain`` is set True for
|
||||||
|
those recursive subagent passes (subagent records are flagged
|
||||||
|
``isSidechain``); the top-level pass keeps skipping sidechain records so a
|
||||||
|
subagent is never also emitted as a stray top-level turn.
|
||||||
|
"""
|
||||||
|
subagents = subagents or {}
|
||||||
|
messages: list[dict] = []
|
||||||
|
# Pending collapsed tool activity: name → call count, plus total bytes.
|
||||||
|
pending_tools: Counter = Counter()
|
||||||
|
pending_bytes = 0
|
||||||
|
|
||||||
|
def flush_pending() -> None:
|
||||||
|
nonlocal pending_bytes
|
||||||
|
if not pending_tools:
|
||||||
|
return
|
||||||
|
shown = ", ".join(
|
||||||
|
f"{name} ×{count}" for name, count in pending_tools.most_common(_TOOL_NAMES_SHOWN)
|
||||||
|
)
|
||||||
|
if len(pending_tools) > _TOOL_NAMES_SHOWN:
|
||||||
|
shown += ", …"
|
||||||
|
calls = sum(pending_tools.values())
|
||||||
|
block = make_collapsed_block(
|
||||||
|
origin=f"{calls} calls: {shown}",
|
||||||
|
content_type="tool_activity",
|
||||||
|
size_bytes=pending_bytes,
|
||||||
|
kind=COLLAPSED_KIND_TOOL_DUMP,
|
||||||
|
)
|
||||||
|
if messages and messages[-1]["role"] == "assistant":
|
||||||
|
messages[-1]["blocks"].append(block)
|
||||||
|
else:
|
||||||
|
messages.append(
|
||||||
|
{"role": "tool", "content_type": "text", "timestamp": None, "blocks": [block]}
|
||||||
|
)
|
||||||
|
pending_tools.clear()
|
||||||
|
pending_bytes = 0
|
||||||
|
|
||||||
|
for rec in records:
|
||||||
|
if rec.get("type") not in ("user", "assistant"):
|
||||||
|
continue
|
||||||
|
if rec.get("isMeta"):
|
||||||
|
continue
|
||||||
|
if rec.get("isSidechain") and not include_sidechain:
|
||||||
|
continue
|
||||||
|
|
||||||
|
msg = rec.get("message") or {}
|
||||||
|
role = msg.get("role") or rec["type"]
|
||||||
|
content = msg.get("content")
|
||||||
|
blocks: list[dict] = []
|
||||||
|
# This record's own tool traffic — merged into pending only after the
|
||||||
|
# record's message is appended, so the placeholder lands on it (not on
|
||||||
|
# the previous message).
|
||||||
|
local_tools: Counter = Counter()
|
||||||
|
local_bytes = 0
|
||||||
|
|
||||||
|
if isinstance(content, str):
|
||||||
|
text = _strip_harness_noise(content)
|
||||||
|
block = make_text_block(text)
|
||||||
|
if block:
|
||||||
|
blocks.append(block)
|
||||||
|
elif isinstance(content, list):
|
||||||
|
for item in content:
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
continue
|
||||||
|
item_type = item.get("type", "")
|
||||||
|
if item_type == "text":
|
||||||
|
block = make_text_block(_strip_harness_noise(item.get("text", "")))
|
||||||
|
if block:
|
||||||
|
blocks.append(block)
|
||||||
|
elif item_type in ("thinking", "redacted_thinking"):
|
||||||
|
if policy == HIDDEN_CONTENT_FULL:
|
||||||
|
block = make_thinking_block(
|
||||||
|
item.get("thinking") or item.get("text") or ""
|
||||||
|
)
|
||||||
|
if block:
|
||||||
|
blocks.append(block)
|
||||||
|
else:
|
||||||
|
# Decision 2026-06-12: thinking is dropped without a
|
||||||
|
# placeholder, but stays visible in the run summary.
|
||||||
|
report.record_collapsed(
|
||||||
|
"thinking", len(json.dumps(item, default=str))
|
||||||
|
)
|
||||||
|
elif item_type == "tool_use":
|
||||||
|
name = item.get("name") or "tool"
|
||||||
|
tool_id = item.get("id")
|
||||||
|
if name in _SUBAGENT_TOOL_NAMES and tool_id in subagents:
|
||||||
|
# A Task/Agent spawn: fold its separate transcript inline
|
||||||
|
# instead of collapsing it. Its own tool traffic is
|
||||||
|
# collapsed by the recursive pass under the same policy.
|
||||||
|
sub = subagents[tool_id]
|
||||||
|
meta = sub.get("meta") or {}
|
||||||
|
sub_msgs = _extract_messages(
|
||||||
|
sub.get("records") or [],
|
||||||
|
conv_id,
|
||||||
|
report,
|
||||||
|
policy,
|
||||||
|
subagents,
|
||||||
|
include_sidechain=True,
|
||||||
|
)
|
||||||
|
blocks.append(
|
||||||
|
make_subagent_block(
|
||||||
|
agent_type=meta.get("agentType") or name,
|
||||||
|
description=meta.get("description") or "",
|
||||||
|
messages=sub_msgs,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
elif policy == HIDDEN_CONTENT_FULL:
|
||||||
|
blocks.append(
|
||||||
|
make_tool_use_block(
|
||||||
|
item.get("name", ""), item.get("input"), item.get("id")
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
size = len(json.dumps(item, default=str))
|
||||||
|
local_tools[name] += 1
|
||||||
|
local_bytes += size
|
||||||
|
report.record_collapsed(name, size)
|
||||||
|
elif item_type == "tool_result":
|
||||||
|
if policy == HIDDEN_CONTENT_FULL:
|
||||||
|
blocks.append(
|
||||||
|
make_tool_result_block(
|
||||||
|
_stringify_tool_result(item.get("content")),
|
||||||
|
is_error=bool(item.get("is_error")),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
size = len(json.dumps(item, default=str))
|
||||||
|
local_bytes += size
|
||||||
|
report.record_collapsed("tool_result", size)
|
||||||
|
elif item_type == "image":
|
||||||
|
blocks.append(
|
||||||
|
make_image_placeholder(ref="embedded image", source="user_upload")
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.warning(
|
||||||
|
"[claude-code] Unknown content block type %r in session %s",
|
||||||
|
item_type,
|
||||||
|
conv_id[:8],
|
||||||
|
)
|
||||||
|
report.record_unknown(f"claude-code.{item_type or '?'}")
|
||||||
|
blocks.append(
|
||||||
|
make_unknown_block(
|
||||||
|
raw_type=f"claude-code.{item_type or '?'}",
|
||||||
|
observed_keys=list(item.keys()),
|
||||||
|
reason=UNKNOWN_REASON_UNKNOWN_TYPE,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not blocks:
|
||||||
|
# Tool-result-only records (and similar): traffic accumulates and
|
||||||
|
# is attached to the message that initiated it.
|
||||||
|
pending_tools.update(local_tools)
|
||||||
|
pending_bytes += local_bytes
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Attach previous records' tool activity to the previous message
|
||||||
|
# before starting a new one, so the placeholder lands between the
|
||||||
|
# dialogue turns it actually occurred between.
|
||||||
|
flush_pending()
|
||||||
|
messages.append(
|
||||||
|
{
|
||||||
|
"role": role,
|
||||||
|
"content_type": "text",
|
||||||
|
"timestamp": rec.get("timestamp"),
|
||||||
|
"blocks": blocks,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
pending_tools.update(local_tools)
|
||||||
|
pending_bytes += local_bytes
|
||||||
|
|
||||||
|
flush_pending()
|
||||||
|
return messages
|
||||||
|
|
||||||
|
|
||||||
|
def _stringify_tool_result(content) -> str:
|
||||||
|
"""tool_result content may be a string or a list of text blocks."""
|
||||||
|
if isinstance(content, str):
|
||||||
|
return content
|
||||||
|
if isinstance(content, list):
|
||||||
|
parts = []
|
||||||
|
for item in content:
|
||||||
|
if isinstance(item, dict) and item.get("type") == "text":
|
||||||
|
parts.append(item.get("text", ""))
|
||||||
|
else:
|
||||||
|
parts.append(json.dumps(item, default=str))
|
||||||
|
return "\n".join(parts)
|
||||||
|
return json.dumps(content, default=str) if content is not None else ""
|
||||||
+3
-3
@@ -50,7 +50,7 @@ def build_export_path(
|
|||||||
created_at: ISO8601 creation timestamp (used for year folder).
|
created_at: ISO8601 creation timestamp (used for year folder).
|
||||||
filename: Already-generated filename from generate_filename().
|
filename: Already-generated filename from generate_filename().
|
||||||
structure: OUTPUT_STRUCTURE value. One of:
|
structure: OUTPUT_STRUCTURE value. One of:
|
||||||
"provider/project/year" (default)
|
"provider/project/year" (default) — project and year combined, e.g. no-project.2025/
|
||||||
"provider/project"
|
"provider/project"
|
||||||
"provider/year"
|
"provider/year"
|
||||||
|
|
||||||
@@ -64,14 +64,14 @@ def build_export_path(
|
|||||||
parts: list[str] = [provider]
|
parts: list[str] = [provider]
|
||||||
|
|
||||||
if structure == "provider/project/year":
|
if structure == "provider/project/year":
|
||||||
parts += [project_slug, year]
|
parts += [f"{project_slug}.{year}"]
|
||||||
elif structure == "provider/project":
|
elif structure == "provider/project":
|
||||||
parts += [project_slug]
|
parts += [project_slug]
|
||||||
elif structure == "provider/year":
|
elif structure == "provider/year":
|
||||||
parts += [year]
|
parts += [year]
|
||||||
else:
|
else:
|
||||||
# Unknown structure — fall back to default
|
# Unknown structure — fall back to default
|
||||||
parts += [project_slug, year]
|
parts += [f"{project_slug}.{year}"]
|
||||||
|
|
||||||
return base_dir.joinpath(*parts) / filename
|
return base_dir.joinpath(*parts) / filename
|
||||||
|
|
||||||
|
|||||||
+148
-9
@@ -8,12 +8,30 @@
|
|||||||
"node-root": {
|
"node-root": {
|
||||||
"id": "node-root",
|
"id": "node-root",
|
||||||
"parent": null,
|
"parent": null,
|
||||||
"children": ["node-1"],
|
"children": ["node-uec"],
|
||||||
"message": null
|
"message": null
|
||||||
},
|
},
|
||||||
|
"node-uec": {
|
||||||
|
"id": "node-uec",
|
||||||
|
"parent": "node-root",
|
||||||
|
"children": ["node-1"],
|
||||||
|
"message": {
|
||||||
|
"id": "node-uec",
|
||||||
|
"author": {"role": "user"},
|
||||||
|
"create_time": null,
|
||||||
|
"content": {
|
||||||
|
"content_type": "user_editable_context",
|
||||||
|
"user_profile": "Preferred name: Jesse",
|
||||||
|
"user_instructions": "The user provided the additional info about how they would like you to respond:\n```Always cite sources.```"
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"is_visually_hidden_from_conversation": true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"node-1": {
|
"node-1": {
|
||||||
"id": "node-1",
|
"id": "node-1",
|
||||||
"parent": "node-root",
|
"parent": "node-uec",
|
||||||
"children": ["node-2"],
|
"children": ["node-2"],
|
||||||
"message": {
|
"message": {
|
||||||
"id": "node-1",
|
"id": "node-1",
|
||||||
@@ -28,7 +46,7 @@
|
|||||||
"node-2": {
|
"node-2": {
|
||||||
"id": "node-2",
|
"id": "node-2",
|
||||||
"parent": "node-1",
|
"parent": "node-1",
|
||||||
"children": ["node-3"],
|
"children": ["node-mm-user"],
|
||||||
"message": {
|
"message": {
|
||||||
"id": "node-2",
|
"id": "node-2",
|
||||||
"author": {"role": "assistant"},
|
"author": {"role": "assistant"},
|
||||||
@@ -39,18 +57,139 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node-3": {
|
"node-mm-user": {
|
||||||
"id": "node-3",
|
"id": "node-mm-user",
|
||||||
"parent": "node-2",
|
"parent": "node-2",
|
||||||
"children": [],
|
"children": ["node-mm-assistant"],
|
||||||
"message": {
|
"message": {
|
||||||
"id": "node-3",
|
"id": "node-mm-user",
|
||||||
"author": {"role": "user"},
|
"author": {"role": "user"},
|
||||||
"create_time": 1704067300.0,
|
"create_time": 1704067300.0,
|
||||||
"content": {
|
"content": {
|
||||||
"content_type": "image_asset_pointer",
|
"content_type": "multimodal_text",
|
||||||
"parts": [{"content_type": "image_asset_pointer", "asset_pointer": "file://some-image"}]
|
"parts": [
|
||||||
|
{"content_type": "audio_transcription", "text": "What is the capital of France?", "direction": "in", "decoding_id": null},
|
||||||
|
{"content_type": "real_time_user_audio_video_asset_pointer", "frames_asset_pointers": [], "video_container_asset_pointer": null, "audio_asset_pointer": {"content_type": "audio_asset_pointer", "asset_pointer": "sediment://file_user001", "size_bytes": 50000, "format": "wav", "metadata": {"start": 0.0, "end": 2.5}}, "audio_start_timestamp": 1.0}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"metadata": {"voice_mode_message": true}
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
"node-mm-assistant": {
|
||||||
|
"id": "node-mm-assistant",
|
||||||
|
"parent": "node-mm-user",
|
||||||
|
"children": ["node-mm-user-rev"],
|
||||||
|
"message": {
|
||||||
|
"id": "node-mm-assistant",
|
||||||
|
"author": {"role": "assistant"},
|
||||||
|
"create_time": 1704067305.0,
|
||||||
|
"content": {
|
||||||
|
"content_type": "multimodal_text",
|
||||||
|
"parts": [
|
||||||
|
{"content_type": "audio_transcription", "text": "The capital of France is Paris.", "direction": "out", "decoding_id": null},
|
||||||
|
{"content_type": "audio_asset_pointer", "asset_pointer": "sediment://file_assistant001", "size_bytes": 80000, "format": "wav", "metadata": {"start": 0.0, "end": 3.2}}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node-mm-user-rev": {
|
||||||
|
"id": "node-mm-user-rev",
|
||||||
|
"parent": "node-mm-assistant",
|
||||||
|
"children": ["node-image-only"],
|
||||||
|
"message": {
|
||||||
|
"id": "node-mm-user-rev",
|
||||||
|
"author": {"role": "user"},
|
||||||
|
"create_time": 1704067400.0,
|
||||||
|
"content": {
|
||||||
|
"content_type": "multimodal_text",
|
||||||
|
"parts": [
|
||||||
|
{"content_type": "real_time_user_audio_video_asset_pointer", "frames_asset_pointers": [], "video_container_asset_pointer": null, "audio_asset_pointer": {"content_type": "audio_asset_pointer", "asset_pointer": "sediment://file_user002", "size_bytes": 30000, "format": "wav", "metadata": {"start": 0.0, "end": 1.5}}, "audio_start_timestamp": 5.0},
|
||||||
|
{"content_type": "audio_transcription", "text": "Tell me more please.", "direction": "in", "decoding_id": null}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node-image-only": {
|
||||||
|
"id": "node-image-only",
|
||||||
|
"parent": "node-mm-user-rev",
|
||||||
|
"children": ["node-exec-output"],
|
||||||
|
"message": {
|
||||||
|
"id": "node-image-only",
|
||||||
|
"author": {"role": "user"},
|
||||||
|
"create_time": 1704067500.0,
|
||||||
|
"content": {
|
||||||
|
"content_type": "multimodal_text",
|
||||||
|
"parts": [
|
||||||
|
{"content_type": "image_asset_pointer", "asset_pointer": "file-service://image001"}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node-exec-output": {
|
||||||
|
"id": "node-exec-output",
|
||||||
|
"parent": "node-image-only",
|
||||||
|
"children": ["node-exec-output-empty"],
|
||||||
|
"message": {
|
||||||
|
"id": "node-exec-output",
|
||||||
|
"author": {"role": "tool", "name": "container.exec", "metadata": {}},
|
||||||
|
"create_time": 1704067600.0,
|
||||||
|
"content": {
|
||||||
|
"content_type": "execution_output",
|
||||||
|
"text": "Hello from container.exec\nLine 2 of output"
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"aggregate_result": {"status": "success", "messages": []},
|
||||||
|
"reasoning_title": "Reading skill documentation"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node-exec-output-empty": {
|
||||||
|
"id": "node-exec-output-empty",
|
||||||
|
"parent": "node-exec-output",
|
||||||
|
"children": ["node-system-error"],
|
||||||
|
"message": {
|
||||||
|
"id": "node-exec-output-empty",
|
||||||
|
"author": {"role": "tool", "name": "python", "metadata": {}},
|
||||||
|
"create_time": 1704067610.0,
|
||||||
|
"content": {
|
||||||
|
"content_type": "execution_output",
|
||||||
|
"text": ""
|
||||||
|
},
|
||||||
|
"metadata": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node-system-error": {
|
||||||
|
"id": "node-system-error",
|
||||||
|
"parent": "node-exec-output-empty",
|
||||||
|
"children": ["node-tether-spinner"],
|
||||||
|
"message": {
|
||||||
|
"id": "node-system-error",
|
||||||
|
"author": {"role": "tool", "name": "web", "metadata": {}},
|
||||||
|
"create_time": 1704067620.0,
|
||||||
|
"content": {
|
||||||
|
"content_type": "system_error",
|
||||||
|
"name": "tool_error",
|
||||||
|
"text": "Error: Error from browse service: Error calling browse service: 503"
|
||||||
|
},
|
||||||
|
"metadata": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node-tether-spinner": {
|
||||||
|
"id": "node-tether-spinner",
|
||||||
|
"parent": "node-system-error",
|
||||||
|
"children": [],
|
||||||
|
"message": {
|
||||||
|
"id": "node-tether-spinner",
|
||||||
|
"author": {"role": "tool", "name": "file_search", "metadata": {}},
|
||||||
|
"create_time": 1704067630.0,
|
||||||
|
"content": {
|
||||||
|
"content_type": "tether_browsing_display",
|
||||||
|
"result": "",
|
||||||
|
"summary": "",
|
||||||
|
"assets": null,
|
||||||
|
"tether_id": null
|
||||||
|
},
|
||||||
|
"metadata": {"command": "spinner", "status": "running"}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+9
@@ -30,6 +30,15 @@
|
|||||||
"sender": "human",
|
"sender": "human",
|
||||||
"created_at": "2024-06-10T14:45:00.000Z",
|
"created_at": "2024-06-10T14:45:00.000Z",
|
||||||
"content": "Thank you, that helped!"
|
"content": "Thank you, that helped!"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"uuid": "msg-004",
|
||||||
|
"sender": "human",
|
||||||
|
"created_at": "2024-06-10T14:50:00.000Z",
|
||||||
|
"content": [
|
||||||
|
{"type": "text", "text": "What about this image?"},
|
||||||
|
{"type": "image", "source": {"file_uuid": "claude-image-uuid-1", "media_type": "image/png"}}
|
||||||
|
]
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -151,3 +151,106 @@ class TestGetNewOrUpdated:
|
|||||||
convs = [{"id": "a", "updated_at": "2024-06-01T00:00:00Z"}]
|
convs = [{"id": "a", "updated_at": "2024-06-01T00:00:00Z"}]
|
||||||
result = tmp_cache.get_new_or_updated("claude", convs)
|
result = tmp_cache.get_new_or_updated("claude", convs)
|
||||||
assert len(result) == 1
|
assert len(result) == 1
|
||||||
|
|
||||||
|
def test_force_returns_all_cached_and_unchanged(self, tmp_cache):
|
||||||
|
tmp_cache.mark_exported("claude", "a", {"updated_at": "2024-01-01T00:00:00Z"})
|
||||||
|
convs = [
|
||||||
|
{"id": "a", "updated_at": "2024-01-01T00:00:00Z"}, # cached, unchanged
|
||||||
|
{"id": "b", "updated_at": "2024-01-02T00:00:00Z"}, # new
|
||||||
|
]
|
||||||
|
assert len(tmp_cache.get_new_or_updated("claude", convs)) == 1
|
||||||
|
assert len(tmp_cache.get_new_or_updated("claude", convs, force=True)) == 2
|
||||||
|
|
||||||
|
def test_force_orders_least_recently_exported_first(self, tmp_cache):
|
||||||
|
# 'a' was exported long ago; 'b' just now; 'c' never.
|
||||||
|
tmp_cache.mark_exported("claude", "a", {"updated_at": "2024-01-01T00:00:00Z"})
|
||||||
|
tmp_cache._data["claude"]["a"]["exported_at"] = "2024-01-01T00:00:00Z"
|
||||||
|
tmp_cache.mark_exported("claude", "b", {"updated_at": "2024-01-01T00:00:00Z"})
|
||||||
|
tmp_cache._data["claude"]["b"]["exported_at"] = "2026-06-12T00:00:00Z"
|
||||||
|
convs = [{"id": "a"}, {"id": "b"}, {"id": "c"}]
|
||||||
|
ordered = [c["id"] for c in tmp_cache.get_new_or_updated("claude", convs, force=True)]
|
||||||
|
# never-exported ('c', exported_at "") and oldest ('a') come before 'b'
|
||||||
|
assert ordered == ["c", "a", "b"]
|
||||||
|
|
||||||
|
def test_force_capped_run_makes_progress(self, tmp_cache):
|
||||||
|
"""The bug: force + cap repeated the same head every run. Now each
|
||||||
|
capped run re-exports the next-oldest batch and converges."""
|
||||||
|
convs = [{"id": f"c{i}", "updated_at": "2024-01-01T00:00:00Z"} for i in range(5)]
|
||||||
|
seen = set()
|
||||||
|
for _ in range(3): # cap=2 over 3 runs should cover all 5
|
||||||
|
batch = tmp_cache.get_new_or_updated("claude", convs, force=True)[:2]
|
||||||
|
for c in batch:
|
||||||
|
seen.add(c["id"])
|
||||||
|
tmp_cache.mark_exported("claude", c["id"], {"file_path": f"/{c['id']}.md"})
|
||||||
|
assert seen == {f"c{i}" for i in range(5)}
|
||||||
|
|
||||||
|
def test_campaign_excludes_already_refreshed(self, tmp_cache):
|
||||||
|
"""With a campaign stamp, conversations re-exported during the campaign
|
||||||
|
(exported_at >= stamp) drop out, so the remaining count shrinks to 0."""
|
||||||
|
convs = [{"id": f"c{i}"} for i in range(5)]
|
||||||
|
campaign = "2026-06-12T00:00:00+00:00"
|
||||||
|
# Nothing exported yet → all 5 are candidates.
|
||||||
|
assert len(tmp_cache.get_new_or_updated("claude", convs, force=True,
|
||||||
|
campaign_at=campaign)) == 5
|
||||||
|
# Re-export 2 "during" the campaign (after the stamp).
|
||||||
|
for cid in ("c0", "c1"):
|
||||||
|
tmp_cache.mark_exported("claude", cid, {"file_path": f"/{cid}.md"})
|
||||||
|
tmp_cache._data["claude"][cid]["exported_at"] = "2026-06-12T01:00:00+00:00"
|
||||||
|
remaining = tmp_cache.get_new_or_updated("claude", convs, force=True,
|
||||||
|
campaign_at=campaign)
|
||||||
|
assert {c["id"] for c in remaining} == {"c2", "c3", "c4"}
|
||||||
|
# Finish them → none left.
|
||||||
|
for cid in ("c2", "c3", "c4"):
|
||||||
|
tmp_cache.mark_exported("claude", cid, {"file_path": f"/{cid}.md"})
|
||||||
|
tmp_cache._data["claude"][cid]["exported_at"] = "2026-06-12T02:00:00+00:00"
|
||||||
|
assert tmp_cache.get_new_or_updated("claude", convs, force=True,
|
||||||
|
campaign_at=campaign) == []
|
||||||
|
|
||||||
|
def test_force_campaign_marker_roundtrip(self, tmp_cache):
|
||||||
|
assert tmp_cache.get_force_campaign() is None
|
||||||
|
tmp_cache.set_force_campaign("2026-06-12T00:00:00+00:00")
|
||||||
|
assert tmp_cache.get_force_campaign() == "2026-06-12T00:00:00+00:00"
|
||||||
|
# Survives reload, and is not mistaken for a provider by stats().
|
||||||
|
reopened = Cache(tmp_cache._dir)
|
||||||
|
assert reopened.get_force_campaign() == "2026-06-12T00:00:00+00:00"
|
||||||
|
assert "force_campaign_at" not in reopened.stats()
|
||||||
|
tmp_cache.clear_force_campaign()
|
||||||
|
assert tmp_cache.get_force_campaign() is None
|
||||||
|
|
||||||
|
|
||||||
|
class TestReexportPreservesJoplinLink:
|
||||||
|
"""A re-export must not drop the Joplin note link, or re-syncing would
|
||||||
|
create duplicate notes instead of updating the existing ones."""
|
||||||
|
|
||||||
|
def test_joplin_fields_carried_across_reexport(self, tmp_cache):
|
||||||
|
tmp_cache.mark_exported("chatgpt", "c1", {
|
||||||
|
"title": "T", "updated_at": "2024-01-01T00:00:00Z", "file_path": "/x.md",
|
||||||
|
})
|
||||||
|
tmp_cache.mark_joplin_synced("chatgpt", "c1", "note-123")
|
||||||
|
tmp_cache.set_joplin_resources("chatgpt", "c1", {"media/a.png": "res-1"})
|
||||||
|
|
||||||
|
# Re-export the same conversation (new render, new file_path).
|
||||||
|
tmp_cache.mark_exported("chatgpt", "c1", {
|
||||||
|
"title": "T", "updated_at": "2024-01-01T00:00:00Z", "file_path": "/x2.md",
|
||||||
|
})
|
||||||
|
|
||||||
|
entry = tmp_cache.get_all_entries("chatgpt")["c1"]
|
||||||
|
assert entry["joplin_note_id"] == "note-123"
|
||||||
|
assert entry["joplin_resources"] == {"media/a.png": "res-1"}
|
||||||
|
assert entry["file_path"] == "/x2.md"
|
||||||
|
|
||||||
|
def test_reexport_marks_note_for_update_not_create(self, tmp_cache):
|
||||||
|
tmp_cache.mark_exported("chatgpt", "c1", {
|
||||||
|
"updated_at": "2024-01-01T00:00:00Z", "file_path": "/x.md",
|
||||||
|
})
|
||||||
|
tmp_cache.mark_joplin_synced("chatgpt", "c1", "note-123")
|
||||||
|
# Nothing pending right after sync.
|
||||||
|
assert tmp_cache.get_joplin_pending("chatgpt") == []
|
||||||
|
# Re-export bumps exported_at past joplin_synced_at → pending for update,
|
||||||
|
# carrying the existing note id so the sync updates rather than creates.
|
||||||
|
tmp_cache.mark_exported("chatgpt", "c1", {
|
||||||
|
"updated_at": "2024-01-01T00:00:00Z", "file_path": "/x.md",
|
||||||
|
})
|
||||||
|
pending = tmp_cache.get_joplin_pending("chatgpt")
|
||||||
|
assert len(pending) == 1
|
||||||
|
assert pending[0][1]["joplin_note_id"] == "note-123"
|
||||||
|
|||||||
@@ -0,0 +1,503 @@
|
|||||||
|
"""Unit tests for the Claude Code session provider."""
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from src.blocks import (
|
||||||
|
BLOCK_TYPE_COLLAPSED,
|
||||||
|
BLOCK_TYPE_TEXT,
|
||||||
|
BLOCK_TYPE_THINKING,
|
||||||
|
BLOCK_TYPE_TOOL_RESULT,
|
||||||
|
BLOCK_TYPE_TOOL_USE,
|
||||||
|
render_blocks_to_markdown,
|
||||||
|
)
|
||||||
|
from src.loss_report import LossReport
|
||||||
|
from src.providers.claude_code import ClaudeCodeProvider
|
||||||
|
|
||||||
|
|
||||||
|
def _write_session(tmp_path, records, project_dir="-home-jesse-myproj", name="abc-123"):
|
||||||
|
proj = tmp_path / project_dir
|
||||||
|
proj.mkdir(parents=True, exist_ok=True)
|
||||||
|
f = proj / f"{name}.jsonl"
|
||||||
|
f.write_text("\n".join(json.dumps(r) for r in records), encoding="utf-8")
|
||||||
|
return f
|
||||||
|
|
||||||
|
|
||||||
|
def _records():
|
||||||
|
"""A representative session: title, harness noise, dialogue, tool traffic."""
|
||||||
|
return [
|
||||||
|
{"type": "ai-title", "aiTitle": "First title", "sessionId": "abc-123"},
|
||||||
|
{
|
||||||
|
"type": "user",
|
||||||
|
"cwd": "/home/jesse/myproj",
|
||||||
|
"timestamp": "2026-05-01T10:00:00.000Z",
|
||||||
|
"message": {
|
||||||
|
"role": "user",
|
||||||
|
"content": (
|
||||||
|
"<local-command-caveat>Caveat: ignore</local-command-caveat>"
|
||||||
|
"<command-name>/model</command-name>"
|
||||||
|
"<local-command-stdout>Set model</local-command-stdout>"
|
||||||
|
"Please review my project."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "assistant",
|
||||||
|
"timestamp": "2026-05-01T10:00:05.000Z",
|
||||||
|
"message": {
|
||||||
|
"role": "assistant",
|
||||||
|
"content": [
|
||||||
|
{"type": "thinking", "thinking": "Let me think about this."},
|
||||||
|
{"type": "text", "text": "I'll review it now."},
|
||||||
|
{"type": "tool_use", "id": "t1", "name": "Read", "input": {"file_path": "/x"}},
|
||||||
|
{"type": "tool_use", "id": "t2", "name": "Read", "input": {"file_path": "/y"}},
|
||||||
|
{"type": "tool_use", "id": "t3", "name": "Bash", "input": {"command": "ls"}},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "user",
|
||||||
|
"timestamp": "2026-05-01T10:00:08.000Z",
|
||||||
|
"message": {
|
||||||
|
"role": "user",
|
||||||
|
"content": [
|
||||||
|
{"type": "tool_result", "tool_use_id": "t1", "content": "file contents " * 50},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "assistant",
|
||||||
|
"timestamp": "2026-05-01T10:00:20.000Z",
|
||||||
|
"message": {
|
||||||
|
"role": "assistant",
|
||||||
|
"content": [{"type": "text", "text": "Here is my review: all good."}],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
# Harness records — skipped
|
||||||
|
{"type": "file-history-snapshot", "snapshot": {"big": "blob"}},
|
||||||
|
{"type": "permission-mode", "mode": "plan"},
|
||||||
|
# Subagent and meta records — skipped
|
||||||
|
{
|
||||||
|
"type": "assistant",
|
||||||
|
"isSidechain": True,
|
||||||
|
"message": {"role": "assistant", "content": [{"type": "text", "text": "subagent noise"}]},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "user",
|
||||||
|
"isMeta": True,
|
||||||
|
"message": {"role": "user", "content": "meta noise"},
|
||||||
|
},
|
||||||
|
# Later title wins
|
||||||
|
{"type": "ai-title", "aiTitle": "Review my project", "sessionId": "abc-123"},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class TestClaudeCodeProvider:
|
||||||
|
def _provider(self, tmp_path, policy="placeholder"):
|
||||||
|
return ClaudeCodeProvider(projects_dir=tmp_path, hidden_content=policy)
|
||||||
|
|
||||||
|
def test_listing_metadata(self, tmp_path):
|
||||||
|
_write_session(tmp_path, _records())
|
||||||
|
p = self._provider(tmp_path)
|
||||||
|
convs = p.fetch_all_conversations()
|
||||||
|
assert len(convs) == 1
|
||||||
|
c = convs[0]
|
||||||
|
assert c["id"] == "abc-123"
|
||||||
|
assert c["title"] == "Review my project" # last ai-title wins
|
||||||
|
assert c["_project_name"] == "myproj" # from cwd basename
|
||||||
|
assert c["created_at"] == "2026-05-01T10:00:00.000Z"
|
||||||
|
assert c["updated_at"] # file mtime
|
||||||
|
|
||||||
|
def test_empty_files_skipped(self, tmp_path):
|
||||||
|
proj = tmp_path / "-home-x"
|
||||||
|
proj.mkdir()
|
||||||
|
(proj / "empty.jsonl").write_text("")
|
||||||
|
p = self._provider(tmp_path)
|
||||||
|
assert p.fetch_all_conversations() == []
|
||||||
|
|
||||||
|
def test_normalize_prose_only_default(self, tmp_path):
|
||||||
|
_write_session(tmp_path, _records())
|
||||||
|
p = self._provider(tmp_path)
|
||||||
|
p.fetch_all_conversations()
|
||||||
|
raw = p.get_conversation("abc-123")
|
||||||
|
result = p.normalize_conversation(raw)
|
||||||
|
|
||||||
|
assert result["provider"] == "claude-code"
|
||||||
|
assert result["title"] == "Review my project"
|
||||||
|
assert result["project"] == "myproj"
|
||||||
|
|
||||||
|
roles = [m["role"] for m in result["messages"]]
|
||||||
|
assert roles == ["user", "assistant", "assistant"]
|
||||||
|
|
||||||
|
# Harness tags stripped, dialogue kept
|
||||||
|
user_text = result["messages"][0]["blocks"][0]["text"]
|
||||||
|
assert user_text == "Please review my project."
|
||||||
|
assert "Caveat" not in user_text
|
||||||
|
|
||||||
|
# Tool activity collapsed into one grouped placeholder on the
|
||||||
|
# assistant message that ran the tools (includes the tool_result
|
||||||
|
# bytes from the following user record).
|
||||||
|
first_assistant = result["messages"][1]
|
||||||
|
types = [b["type"] for b in first_assistant["blocks"]]
|
||||||
|
assert types == [BLOCK_TYPE_TEXT, BLOCK_TYPE_COLLAPSED]
|
||||||
|
collapsed = first_assistant["blocks"][1]
|
||||||
|
assert "3 calls" in collapsed["origin"]
|
||||||
|
assert "Read ×2" in collapsed["origin"]
|
||||||
|
assert "Bash ×1" in collapsed["origin"]
|
||||||
|
assert collapsed["size_bytes"] > 500
|
||||||
|
|
||||||
|
# Thinking dropped without placeholder; sidechain/meta absent
|
||||||
|
all_blocks = [b for m in result["messages"] for b in m["blocks"]]
|
||||||
|
assert not any(b["type"] == BLOCK_TYPE_THINKING for b in all_blocks)
|
||||||
|
assert not any(
|
||||||
|
"subagent noise" in (b.get("text") or "") or "meta noise" in (b.get("text") or "")
|
||||||
|
for b in all_blocks
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_collapsed_counted_in_loss_report(self, tmp_path):
|
||||||
|
_write_session(tmp_path, _records())
|
||||||
|
p = self._provider(tmp_path)
|
||||||
|
p.fetch_all_conversations()
|
||||||
|
report = LossReport()
|
||||||
|
p.normalize_conversation(p.get_conversation("abc-123"), report)
|
||||||
|
assert report.collapsed["Read"] == 2
|
||||||
|
assert report.collapsed["Bash"] == 1
|
||||||
|
assert report.collapsed["tool_result"] == 1
|
||||||
|
assert report.collapsed["thinking"] == 1
|
||||||
|
assert report.collapsed_bytes > 500
|
||||||
|
|
||||||
|
def test_full_policy_keeps_everything(self, tmp_path):
|
||||||
|
_write_session(tmp_path, _records())
|
||||||
|
p = self._provider(tmp_path, policy="full")
|
||||||
|
p.fetch_all_conversations()
|
||||||
|
result = p.normalize_conversation(p.get_conversation("abc-123"))
|
||||||
|
all_blocks = [b for m in result["messages"] for b in m["blocks"]]
|
||||||
|
types = {b["type"] for b in all_blocks}
|
||||||
|
assert BLOCK_TYPE_THINKING in types
|
||||||
|
assert BLOCK_TYPE_TOOL_USE in types
|
||||||
|
assert BLOCK_TYPE_TOOL_RESULT in types
|
||||||
|
assert BLOCK_TYPE_COLLAPSED not in types
|
||||||
|
|
||||||
|
def test_updated_at_consistent_with_listing(self, tmp_path):
|
||||||
|
"""Listing and normalized updated_at must match or the cache would
|
||||||
|
consider every session stale on every run (mtime vs last timestamp)."""
|
||||||
|
_write_session(tmp_path, _records())
|
||||||
|
p = self._provider(tmp_path)
|
||||||
|
listing = p.fetch_all_conversations()[0]
|
||||||
|
normalized = p.normalize_conversation(p.get_conversation("abc-123"))
|
||||||
|
assert normalized["updated_at"] == listing["updated_at"]
|
||||||
|
|
||||||
|
def test_title_falls_back_to_first_prompt(self, tmp_path):
|
||||||
|
records = [r for r in _records() if r.get("type") != "ai-title"]
|
||||||
|
_write_session(tmp_path, records)
|
||||||
|
p = self._provider(tmp_path)
|
||||||
|
p.fetch_all_conversations()
|
||||||
|
result = p.normalize_conversation(p.get_conversation("abc-123"))
|
||||||
|
assert result["title"] == "Please review my project."
|
||||||
|
|
||||||
|
def test_unknown_block_type_surfaces(self, tmp_path):
|
||||||
|
records = [
|
||||||
|
{
|
||||||
|
"type": "assistant",
|
||||||
|
"timestamp": "2026-05-01T10:00:00.000Z",
|
||||||
|
"cwd": "/home/jesse/myproj",
|
||||||
|
"message": {
|
||||||
|
"role": "assistant",
|
||||||
|
"content": [
|
||||||
|
{"type": "text", "text": "hello"},
|
||||||
|
{"type": "future_block_xyz", "data": 1},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
_write_session(tmp_path, records)
|
||||||
|
p = self._provider(tmp_path)
|
||||||
|
p.fetch_all_conversations()
|
||||||
|
report = LossReport()
|
||||||
|
result = p.normalize_conversation(p.get_conversation("abc-123"), report)
|
||||||
|
assert report.unknown_blocks["claude-code.future_block_xyz"] == 1
|
||||||
|
rendered = render_blocks_to_markdown(result["messages"][0]["blocks"])
|
||||||
|
assert "Unsupported content" in rendered
|
||||||
|
|
||||||
|
def test_collapsed_placeholder_renders_grouped_line(self, tmp_path):
|
||||||
|
_write_session(tmp_path, _records())
|
||||||
|
p = self._provider(tmp_path)
|
||||||
|
p.fetch_all_conversations()
|
||||||
|
result = p.normalize_conversation(p.get_conversation("abc-123"))
|
||||||
|
rendered = render_blocks_to_markdown(result["messages"][1]["blocks"])
|
||||||
|
assert "> 🔧 **Tool output** — `3 calls: Read ×2, Bash ×1`" in rendered
|
||||||
|
assert "omitted (EXPORTER_HIDDEN_CONTENT=full to keep)" in rendered
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Subagent folding (Part B)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _write_subagent(
|
||||||
|
session_file, tool_use_id, records, agent_type="Explore",
|
||||||
|
description="do a thing", name="agent-x",
|
||||||
|
):
|
||||||
|
"""Write a <session>/subagents/<name>.jsonl + .meta.json sidecar."""
|
||||||
|
subdir = session_file.parent / session_file.stem / "subagents"
|
||||||
|
subdir.mkdir(parents=True, exist_ok=True)
|
||||||
|
(subdir / f"{name}.jsonl").write_text(
|
||||||
|
"\n".join(json.dumps(r) for r in records), encoding="utf-8"
|
||||||
|
)
|
||||||
|
(subdir / f"{name}.meta.json").write_text(
|
||||||
|
json.dumps({
|
||||||
|
"agentType": agent_type,
|
||||||
|
"description": description,
|
||||||
|
"toolUseId": tool_use_id,
|
||||||
|
}),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _parent_with_task(tool_use_id="toolu_sub1"):
|
||||||
|
return [
|
||||||
|
{"type": "ai-title", "aiTitle": "Parent session"},
|
||||||
|
{
|
||||||
|
"type": "user", "cwd": "/home/jesse/myproj",
|
||||||
|
"timestamp": "2026-05-01T10:00:00.000Z",
|
||||||
|
"message": {"role": "user", "content": "delegate the research"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "assistant", "timestamp": "2026-05-01T10:00:05.000Z",
|
||||||
|
"message": {"role": "assistant", "content": [
|
||||||
|
{"type": "text", "text": "I'll delegate this."},
|
||||||
|
{"type": "tool_use", "id": tool_use_id, "name": "Task",
|
||||||
|
"input": {"description": "research"}},
|
||||||
|
]},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _subagent_records():
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"type": "user", "isSidechain": True,
|
||||||
|
"timestamp": "2026-05-01T10:00:06.000Z",
|
||||||
|
"message": {"role": "user", "content": "Go research X"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "assistant", "isSidechain": True,
|
||||||
|
"timestamp": "2026-05-01T10:00:07.000Z",
|
||||||
|
"message": {"role": "assistant", "content": [
|
||||||
|
{"type": "text", "text": "Here is the research result."},
|
||||||
|
{"type": "tool_use", "id": "st1", "name": "Grep",
|
||||||
|
"input": {"pattern": "x"}},
|
||||||
|
]},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class TestSubagentFold:
|
||||||
|
def _provider(self, tmp_path, policy="placeholder"):
|
||||||
|
return ClaudeCodeProvider(projects_dir=tmp_path, hidden_content=policy)
|
||||||
|
|
||||||
|
def _normalize(self, tmp_path, policy="placeholder"):
|
||||||
|
f = _write_session(tmp_path, _parent_with_task(), name="parent-1")
|
||||||
|
_write_subagent(
|
||||||
|
f, "toolu_sub1", _subagent_records(),
|
||||||
|
agent_type="Explore", description="research X",
|
||||||
|
)
|
||||||
|
p = self._provider(tmp_path, policy)
|
||||||
|
p.fetch_all_conversations()
|
||||||
|
return p.normalize_conversation(p.get_conversation("parent-1"))
|
||||||
|
|
||||||
|
def test_subagent_folded_into_parent(self, tmp_path):
|
||||||
|
conv = self._normalize(tmp_path)
|
||||||
|
blocks = [b for m in conv["messages"] for b in m["blocks"]]
|
||||||
|
subs = [b for b in blocks if b["type"] == "subagent"]
|
||||||
|
assert len(subs) == 1
|
||||||
|
assert subs[0]["agent_type"] == "Explore"
|
||||||
|
assert subs[0]["description"] == "research X"
|
||||||
|
inner_text = [
|
||||||
|
bb.get("text") for m in subs[0]["messages"] for bb in m["blocks"]
|
||||||
|
]
|
||||||
|
assert "Go research X" in inner_text
|
||||||
|
assert "Here is the research result." in inner_text
|
||||||
|
|
||||||
|
def test_subagent_tools_collapsed(self, tmp_path):
|
||||||
|
conv = self._normalize(tmp_path)
|
||||||
|
sub = next(
|
||||||
|
b for m in conv["messages"] for b in m["blocks"]
|
||||||
|
if b["type"] == "subagent"
|
||||||
|
)
|
||||||
|
inner_blocks = [bb for m in sub["messages"] for bb in m["blocks"]]
|
||||||
|
assert any(b["type"] == BLOCK_TYPE_COLLAPSED for b in inner_blocks)
|
||||||
|
assert not any(b["type"] == BLOCK_TYPE_TOOL_USE for b in inner_blocks)
|
||||||
|
collapsed = next(b for b in inner_blocks if b["type"] == BLOCK_TYPE_COLLAPSED)
|
||||||
|
assert "Grep" in collapsed["origin"]
|
||||||
|
|
||||||
|
def test_subagent_not_listed_as_conversation(self, tmp_path):
|
||||||
|
f = _write_session(tmp_path, _parent_with_task(), name="parent-1")
|
||||||
|
_write_subagent(f, "toolu_sub1", _subagent_records())
|
||||||
|
p = self._provider(tmp_path)
|
||||||
|
convs = p.fetch_all_conversations()
|
||||||
|
assert [c["id"] for c in convs] == ["parent-1"]
|
||||||
|
|
||||||
|
def test_renders_as_details_block(self, tmp_path):
|
||||||
|
conv = self._normalize(tmp_path)
|
||||||
|
spawning = conv["messages"][1] # the assistant turn with the Task call
|
||||||
|
rendered = render_blocks_to_markdown(spawning["blocks"])
|
||||||
|
assert "<details>" in rendered
|
||||||
|
assert "<summary>🤖 Subagent: Explore — research X</summary>" in rendered
|
||||||
|
assert "</details>" in rendered
|
||||||
|
assert "Here is the research result." in rendered
|
||||||
|
|
||||||
|
def test_main_pass_still_skips_stray_sidechain(self, tmp_path):
|
||||||
|
# A sidechain record with no matching subagent file must NOT leak into
|
||||||
|
# the top-level dialogue.
|
||||||
|
records = _parent_with_task() + [
|
||||||
|
{"type": "assistant", "isSidechain": True,
|
||||||
|
"message": {"role": "assistant",
|
||||||
|
"content": [{"type": "text", "text": "stray sidechain"}]}},
|
||||||
|
]
|
||||||
|
f = _write_session(tmp_path, records, name="parent-2")
|
||||||
|
_write_subagent(f, "toolu_sub1", _subagent_records())
|
||||||
|
p = self._provider(tmp_path)
|
||||||
|
p.fetch_all_conversations()
|
||||||
|
conv = p.normalize_conversation(p.get_conversation("parent-2"))
|
||||||
|
top_text = [
|
||||||
|
b.get("text") for m in conv["messages"] for b in m["blocks"]
|
||||||
|
]
|
||||||
|
assert "stray sidechain" not in top_text
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Repo tags in title (Part C)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestRepoTags:
|
||||||
|
@staticmethod
|
||||||
|
def _mkrepo(base, name):
|
||||||
|
"""Create a git repo dir (bare .git marker) and return its path."""
|
||||||
|
(base / name / ".git").mkdir(parents=True, exist_ok=True)
|
||||||
|
return base / name
|
||||||
|
|
||||||
|
def _title(self, tmp_path, content):
|
||||||
|
(tmp_path / "ws").mkdir(exist_ok=True)
|
||||||
|
records = [
|
||||||
|
{"type": "ai-title", "aiTitle": "Work session"},
|
||||||
|
{"type": "user", "cwd": str(tmp_path / "ws"),
|
||||||
|
"timestamp": "2026-05-01T10:00:00.000Z",
|
||||||
|
"message": {"role": "user", "content": "go"}},
|
||||||
|
{"type": "assistant",
|
||||||
|
"message": {"role": "assistant", "content": content}},
|
||||||
|
]
|
||||||
|
_write_session(tmp_path, records, name="ws-1")
|
||||||
|
p = ClaudeCodeProvider(projects_dir=tmp_path)
|
||||||
|
p.fetch_all_conversations()
|
||||||
|
return p.normalize_conversation(p.get_conversation("ws-1"))["title"]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _read(path):
|
||||||
|
return {"type": "tool_use", "id": str(path), "name": "Read",
|
||||||
|
"input": {"file_path": str(path)}}
|
||||||
|
|
||||||
|
def test_repos_by_git_root_ordered_by_frequency(self, tmp_path):
|
||||||
|
a = self._mkrepo(tmp_path, "repo-a")
|
||||||
|
b = self._mkrepo(tmp_path, "repo-b")
|
||||||
|
content = [
|
||||||
|
self._read(a / "src/x.py"), self._read(a / "src/y.py"),
|
||||||
|
self._read(b / "z.py"),
|
||||||
|
# a file not inside any git repo → excluded
|
||||||
|
self._read(tmp_path / "CLAUDE.md"),
|
||||||
|
]
|
||||||
|
assert self._title(tmp_path, content) == "Work session [repo-a, repo-b]"
|
||||||
|
|
||||||
|
def test_cross_workspace_repo_tagged(self, tmp_path):
|
||||||
|
# Launched in ws/, but touched a repo in a sibling tree — still tagged.
|
||||||
|
other = self._mkrepo(tmp_path / "elsewhere", "faraway-repo")
|
||||||
|
content = [self._read(other / "deep/nested/file.py")]
|
||||||
|
assert self._title(tmp_path, content) == "Work session [faraway-repo]"
|
||||||
|
|
||||||
|
def test_no_repos_no_bracket(self, tmp_path):
|
||||||
|
content = [self._read(tmp_path / "loose/file.txt")] # no .git anywhere
|
||||||
|
assert self._title(tmp_path, content) == "Work session"
|
||||||
|
|
||||||
|
def test_cap_three_with_ellipsis(self, tmp_path):
|
||||||
|
content = [
|
||||||
|
self._read(self._mkrepo(tmp_path, f"repo-{c}") / "f/x.py")
|
||||||
|
for c in "abcd"
|
||||||
|
]
|
||||||
|
title = self._title(tmp_path, content)
|
||||||
|
assert title.startswith("Work session [repo-a, repo-b, repo-c, …]")
|
||||||
|
assert "repo-d" not in title
|
||||||
|
|
||||||
|
def test_paths_inside_bash_commands_count(self, tmp_path):
|
||||||
|
x = self._mkrepo(tmp_path, "repo-x")
|
||||||
|
content = [
|
||||||
|
{"type": "tool_use", "id": "a", "name": "Bash",
|
||||||
|
"input": {"command": f"cat {x / 'main.go'}"}},
|
||||||
|
]
|
||||||
|
assert self._title(tmp_path, content) == "Work session [repo-x]"
|
||||||
|
|
||||||
|
def test_ignore_list_via_env(self, tmp_path, monkeypatch):
|
||||||
|
a = self._mkrepo(tmp_path, "repo-a")
|
||||||
|
b = self._mkrepo(tmp_path, "repo-b")
|
||||||
|
monkeypatch.setenv("CLAUDE_CODE_REPO_TAG_IGNORE", "repo-b, notes")
|
||||||
|
content = [self._read(a / "x.py"), self._read(b / "y.py")]
|
||||||
|
assert self._title(tmp_path, content) == "Work session [repo-a]"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Multiple projects roots (Part D)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestMultiRoot:
|
||||||
|
def _titled(self, aititle, cwd="/home/jesse/p"):
|
||||||
|
return [
|
||||||
|
{"type": "ai-title", "aiTitle": aititle},
|
||||||
|
{"type": "user", "cwd": cwd, "timestamp": "2026-05-01T10:00:00.000Z",
|
||||||
|
"message": {"role": "user", "content": "hi"}},
|
||||||
|
]
|
||||||
|
|
||||||
|
def test_two_roots_merged(self, tmp_path):
|
||||||
|
r1, r2 = tmp_path / "root1", tmp_path / "root2"
|
||||||
|
_write_session(r1, self._titled("From root1"), name="s1")
|
||||||
|
_write_session(r2, self._titled("From root2"), name="s2")
|
||||||
|
p = ClaudeCodeProvider(projects_dir=[r1, r2])
|
||||||
|
ids = sorted(c["id"] for c in p.fetch_all_conversations())
|
||||||
|
assert ids == ["s1", "s2"]
|
||||||
|
|
||||||
|
def test_duplicate_uuid_newer_wins(self, tmp_path):
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
r1, r2 = tmp_path / "root1", tmp_path / "root2"
|
||||||
|
_write_session(r1, self._titled("Old copy"), name="dup")
|
||||||
|
f2 = _write_session(r2, self._titled("New copy"), name="dup")
|
||||||
|
os.utime(f2, (time.time() + 10, time.time() + 10))
|
||||||
|
p = ClaudeCodeProvider(projects_dir=[r1, r2])
|
||||||
|
convs = p.fetch_all_conversations()
|
||||||
|
assert len(convs) == 1
|
||||||
|
assert convs[0]["title"] == "New copy"
|
||||||
|
assert p._path_map["dup"] == f2
|
||||||
|
|
||||||
|
def test_single_path_backward_compatible(self, tmp_path):
|
||||||
|
_write_session(tmp_path, _records())
|
||||||
|
p = ClaudeCodeProvider(projects_dir=tmp_path)
|
||||||
|
assert len(p.fetch_all_conversations()) == 1
|
||||||
|
|
||||||
|
def test_env_pathsep_list(self, tmp_path, monkeypatch):
|
||||||
|
import os
|
||||||
|
from src.providers.claude_code import resolve_roots
|
||||||
|
r1, r2 = tmp_path / "root1", tmp_path / "root2"
|
||||||
|
_write_session(r1, self._titled("From root1"), name="s1")
|
||||||
|
_write_session(r2, self._titled("From root2"), name="s2")
|
||||||
|
monkeypatch.setenv("CLAUDE_CODE_DIR", f"{r1}{os.pathsep}{r2}")
|
||||||
|
assert r1 in resolve_roots() and r2 in resolve_roots()
|
||||||
|
p = ClaudeCodeProvider()
|
||||||
|
assert len(p.fetch_all_conversations()) == 2
|
||||||
|
|
||||||
|
def test_config_dir_projects_included(self, tmp_path, monkeypatch):
|
||||||
|
from src.providers.claude_code import resolve_roots
|
||||||
|
monkeypatch.delenv("CLAUDE_CODE_DIR", raising=False)
|
||||||
|
monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(tmp_path / "cfg"))
|
||||||
|
roots = resolve_roots()
|
||||||
|
assert (tmp_path / "cfg" / "projects") in roots
|
||||||
@@ -0,0 +1,299 @@
|
|||||||
|
"""CLI-level tests using Click's CliRunner — no live API calls required."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from click.testing import CliRunner
|
||||||
|
|
||||||
|
from src.cache import Cache
|
||||||
|
from src.main import _filter_by_project, cli
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# _filter_by_project (T-27)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestFilterByProject:
|
||||||
|
"""Unit tests for the project filter logic used by export/list/joplin."""
|
||||||
|
|
||||||
|
# ChatGPT conversations use the _project_name annotation key
|
||||||
|
def _chatgpt(self, conv_id, project_name):
|
||||||
|
return {"id": conv_id, "_project_name": project_name}
|
||||||
|
|
||||||
|
# Claude conversations use the project dict key
|
||||||
|
def _claude(self, conv_id, project_name):
|
||||||
|
proj = {"name": project_name} if project_name else None
|
||||||
|
return {"id": conv_id, "project": proj}
|
||||||
|
|
||||||
|
def test_none_filter_keeps_no_project_chatgpt(self):
|
||||||
|
convs = [self._chatgpt("a", None), self._chatgpt("b", "Python Course")]
|
||||||
|
result = _filter_by_project(convs, "none")
|
||||||
|
assert len(result) == 1
|
||||||
|
assert result[0]["id"] == "a"
|
||||||
|
|
||||||
|
def test_none_filter_keeps_no_project_claude(self):
|
||||||
|
convs = [self._claude("a", None), self._claude("b", "Python Course")]
|
||||||
|
result = _filter_by_project(convs, "none")
|
||||||
|
assert len(result) == 1
|
||||||
|
assert result[0]["id"] == "a"
|
||||||
|
|
||||||
|
def test_name_filter_case_insensitive(self):
|
||||||
|
convs = [
|
||||||
|
self._chatgpt("a", "Python Course"),
|
||||||
|
self._chatgpt("b", "Java Course"),
|
||||||
|
self._chatgpt("c", None),
|
||||||
|
]
|
||||||
|
result = _filter_by_project(convs, "PYTHON")
|
||||||
|
assert len(result) == 1
|
||||||
|
assert result[0]["id"] == "a"
|
||||||
|
|
||||||
|
def test_name_filter_substring_match(self):
|
||||||
|
convs = [
|
||||||
|
self._chatgpt("a", "Python Advanced Course"),
|
||||||
|
self._chatgpt("b", "Python Basics"),
|
||||||
|
self._chatgpt("c", "JavaScript"),
|
||||||
|
]
|
||||||
|
result = _filter_by_project(convs, "python")
|
||||||
|
assert len(result) == 2
|
||||||
|
assert {c["id"] for c in result} == {"a", "b"}
|
||||||
|
|
||||||
|
def test_no_matches_returns_empty(self):
|
||||||
|
convs = [self._chatgpt("a", "Python Course"), self._chatgpt("b", None)]
|
||||||
|
result = _filter_by_project(convs, "ruby")
|
||||||
|
assert result == []
|
||||||
|
|
||||||
|
def test_none_filter_excludes_all_with_projects(self):
|
||||||
|
convs = [self._chatgpt("a", "Project A"), self._chatgpt("b", "Project B")]
|
||||||
|
result = _filter_by_project(convs, "none")
|
||||||
|
assert result == []
|
||||||
|
|
||||||
|
def test_empty_string_project_treated_as_no_project(self):
|
||||||
|
convs = [{"id": "a", "_project_name": ""}, {"id": "b", "_project_name": "Real"}]
|
||||||
|
result = _filter_by_project(convs, "none")
|
||||||
|
assert len(result) == 1
|
||||||
|
assert result[0]["id"] == "a"
|
||||||
|
|
||||||
|
def test_claude_project_string_matched(self):
|
||||||
|
# Claude can also have project as a plain string
|
||||||
|
convs = [{"id": "a", "project": "python-course"}, {"id": "b", "project": None}]
|
||||||
|
result = _filter_by_project(convs, "python")
|
||||||
|
assert len(result) == 1
|
||||||
|
assert result[0]["id"] == "a"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# export --since validation (T-25)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestExportSinceValidation:
|
||||||
|
"""Test that --since with an invalid date exits cleanly with an error message."""
|
||||||
|
|
||||||
|
def _pre_populated_cache(self, tmp_path) -> Cache:
|
||||||
|
"""Create a cache that passes the ToS gate and first-run doctor check."""
|
||||||
|
cache = Cache(tmp_path)
|
||||||
|
cache.acknowledge_tos()
|
||||||
|
cache.mark_exported("chatgpt", "dummy-conv", {"updated_at": "2024-01-01T00:00:00Z"})
|
||||||
|
return cache
|
||||||
|
|
||||||
|
def test_invalid_since_date_exits_with_error(self, tmp_path):
|
||||||
|
self._pre_populated_cache(tmp_path)
|
||||||
|
|
||||||
|
runner = CliRunner(mix_stderr=True)
|
||||||
|
result = runner.invoke(
|
||||||
|
cli,
|
||||||
|
["--no-log-file", "export", "--since", "notadate"],
|
||||||
|
env={
|
||||||
|
"CHATGPT_SESSION_TOKEN": "eyJtesttoken",
|
||||||
|
"CACHE_DIR": str(tmp_path),
|
||||||
|
"EXPORT_DIR": str(tmp_path / "exports"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert result.exit_code == 1
|
||||||
|
assert "Invalid --since date" in result.output
|
||||||
|
assert "YYYY-MM-DD" in result.output
|
||||||
|
|
||||||
|
def test_valid_since_date_does_not_error(self, tmp_path):
|
||||||
|
"""A valid date should not produce the invalid-date error (may fail later on API)."""
|
||||||
|
self._pre_populated_cache(tmp_path)
|
||||||
|
|
||||||
|
runner = CliRunner(mix_stderr=True)
|
||||||
|
result = runner.invoke(
|
||||||
|
cli,
|
||||||
|
["--no-log-file", "export", "--since", "2024-01-01"],
|
||||||
|
env={
|
||||||
|
"CHATGPT_SESSION_TOKEN": "eyJtesttoken",
|
||||||
|
"CACHE_DIR": str(tmp_path),
|
||||||
|
"EXPORT_DIR": str(tmp_path / "exports"),
|
||||||
|
# This test hits the real (failing) auth endpoint with retries;
|
||||||
|
# don't add politeness pacing on top of the backoff sleeps.
|
||||||
|
"REQUEST_DELAY": "0",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert "Invalid --since date" not in result.output
|
||||||
|
|
||||||
|
def test_max_conversations_zero_rejected(self, tmp_path):
|
||||||
|
"""--max-conversations uses IntRange(min=1); 0 must be rejected by click."""
|
||||||
|
self._pre_populated_cache(tmp_path)
|
||||||
|
|
||||||
|
runner = CliRunner(mix_stderr=True)
|
||||||
|
result = runner.invoke(
|
||||||
|
cli,
|
||||||
|
["--no-log-file", "export", "--max-conversations", "0"],
|
||||||
|
env={
|
||||||
|
"CHATGPT_SESSION_TOKEN": "eyJtesttoken",
|
||||||
|
"CACHE_DIR": str(tmp_path),
|
||||||
|
"EXPORT_DIR": str(tmp_path / "exports"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert result.exit_code == 2
|
||||||
|
assert "max-conversations" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# LossReport summary
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestLossReportSummary:
|
||||||
|
"""The LossReport's format_summary() pinned format covers zero, top-5, and overflow cases."""
|
||||||
|
|
||||||
|
def test_zero_summary_uses_none_sentinel(self):
|
||||||
|
from src.loss_report import LossReport
|
||||||
|
|
||||||
|
report = LossReport()
|
||||||
|
out = report.format_summary()
|
||||||
|
assert "[export] Run summary:" in out
|
||||||
|
assert "conversations: 0" in out
|
||||||
|
assert "messages rendered: 0" in out
|
||||||
|
# All three "(none)" sentinels present — never empty parens
|
||||||
|
# (unknown blocks, extraction failures, collapsed by policy)
|
||||||
|
assert out.count("(none)") == 3
|
||||||
|
|
||||||
|
def test_top_5_breakdown(self):
|
||||||
|
from src.loss_report import LossReport
|
||||||
|
|
||||||
|
report = LossReport()
|
||||||
|
for raw_type in ("a", "b", "c", "d", "e", "f", "g"):
|
||||||
|
report.record_unknown(raw_type)
|
||||||
|
if raw_type == "a":
|
||||||
|
# Make 'a' the most common
|
||||||
|
for _ in range(4):
|
||||||
|
report.record_unknown("a")
|
||||||
|
out = report.format_summary()
|
||||||
|
# Top entry shown
|
||||||
|
assert "a=5" in out
|
||||||
|
# Overflow line present (7 types, top 5 + 2 more)
|
||||||
|
assert "+ 2 more types" in out
|
||||||
|
|
||||||
|
def test_messages_and_conversations_recorded(self):
|
||||||
|
from src.loss_report import LossReport
|
||||||
|
|
||||||
|
report = LossReport()
|
||||||
|
report.record_conversation()
|
||||||
|
report.record_message()
|
||||||
|
report.record_message()
|
||||||
|
out = report.format_summary()
|
||||||
|
assert "conversations: 1" in out
|
||||||
|
assert "messages rendered: 2" in out
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# prune command
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestPrune:
|
||||||
|
"""prune deletes export files not referenced by the manifest."""
|
||||||
|
|
||||||
|
def _setup(self, tmp_path):
|
||||||
|
"""Cache with one referenced file; one stale file + empty-dir candidate."""
|
||||||
|
cache = Cache(tmp_path / "cache")
|
||||||
|
cache.acknowledge_tos()
|
||||||
|
export_dir = tmp_path / "exports"
|
||||||
|
keep = export_dir / "chatgpt" / "proj.2026" / "keep.md"
|
||||||
|
keep.parent.mkdir(parents=True)
|
||||||
|
keep.write_text("kept")
|
||||||
|
stale = export_dir / "chatgpt" / "proj" / "2026" / "old-layout.md"
|
||||||
|
stale.parent.mkdir(parents=True)
|
||||||
|
stale.write_text("stale")
|
||||||
|
cache.mark_exported("chatgpt", "conv-1", {"file_path": str(keep)})
|
||||||
|
return export_dir, keep, stale
|
||||||
|
|
||||||
|
def _invoke(self, tmp_path, *args):
|
||||||
|
runner = CliRunner(mix_stderr=True)
|
||||||
|
return runner.invoke(
|
||||||
|
cli,
|
||||||
|
["--no-log-file", "prune", *args],
|
||||||
|
env={
|
||||||
|
"CACHE_DIR": str(tmp_path / "cache"),
|
||||||
|
"EXPORT_DIR": str(tmp_path / "exports"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_dry_run_lists_but_keeps_files(self, tmp_path):
|
||||||
|
export_dir, keep, stale = self._setup(tmp_path)
|
||||||
|
result = self._invoke(tmp_path, "--dry-run")
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "old-layout.md" in result.output
|
||||||
|
assert "Dry run" in result.output
|
||||||
|
assert stale.exists() and keep.exists()
|
||||||
|
|
||||||
|
def test_yes_deletes_stale_keeps_referenced_sweeps_dirs(self, tmp_path):
|
||||||
|
export_dir, keep, stale = self._setup(tmp_path)
|
||||||
|
result = self._invoke(tmp_path, "--yes")
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert not stale.exists()
|
||||||
|
assert keep.exists()
|
||||||
|
# Old-layout dirs are now empty and swept
|
||||||
|
assert not (export_dir / "chatgpt" / "proj").exists()
|
||||||
|
|
||||||
|
def test_refuses_with_empty_manifest(self, tmp_path):
|
||||||
|
"""Footgun guard: after cache --clear, prune must not wipe the archive."""
|
||||||
|
cache = Cache(tmp_path / "cache")
|
||||||
|
cache.acknowledge_tos()
|
||||||
|
export_dir = tmp_path / "exports"
|
||||||
|
f = export_dir / "chatgpt" / "a.md"
|
||||||
|
f.parent.mkdir(parents=True)
|
||||||
|
f.write_text("data")
|
||||||
|
result = self._invoke(tmp_path, "--yes")
|
||||||
|
assert result.exit_code == 1
|
||||||
|
assert "Refusing to prune" in result.output
|
||||||
|
assert f.exists()
|
||||||
|
|
||||||
|
def test_aborts_without_confirmation(self, tmp_path):
|
||||||
|
export_dir, keep, stale = self._setup(tmp_path)
|
||||||
|
runner = CliRunner(mix_stderr=True)
|
||||||
|
result = runner.invoke(
|
||||||
|
cli,
|
||||||
|
["--no-log-file", "prune"],
|
||||||
|
input="n\n",
|
||||||
|
env={
|
||||||
|
"CACHE_DIR": str(tmp_path / "cache"),
|
||||||
|
"EXPORT_DIR": str(tmp_path / "exports"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "Aborted" in result.output
|
||||||
|
assert stale.exists()
|
||||||
|
|
||||||
|
|
||||||
|
class TestCanaryCommand:
|
||||||
|
"""`canary` wiring — no live API calls (no tokens configured)."""
|
||||||
|
|
||||||
|
def test_no_tokens_exits_nonzero(self, tmp_path):
|
||||||
|
cache = Cache(tmp_path / "cache")
|
||||||
|
cache.acknowledge_tos()
|
||||||
|
runner = CliRunner(mix_stderr=True)
|
||||||
|
result = runner.invoke(
|
||||||
|
cli,
|
||||||
|
["--no-log-file", "canary"],
|
||||||
|
env={
|
||||||
|
"CACHE_DIR": str(tmp_path / "cache"),
|
||||||
|
"EXPORT_DIR": str(tmp_path / "exports"),
|
||||||
|
# Empty so .env (override=False) can't repopulate real tokens.
|
||||||
|
"CHATGPT_SESSION_TOKEN": "",
|
||||||
|
"CLAUDE_SESSION_KEY": "",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert result.exit_code == 1
|
||||||
|
assert "No web-API provider tokens" in result.output
|
||||||
@@ -0,0 +1,118 @@
|
|||||||
|
"""Tests for src/config.py — token validation logic (T-14)."""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import time
|
||||||
|
|
||||||
|
import jwt
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from src.config import _validate_chatgpt_token
|
||||||
|
|
||||||
|
|
||||||
|
class TestValidateChatGPTToken:
|
||||||
|
def test_expired_token_logs_warning(self, caplog):
|
||||||
|
# T-14: expired JWT must produce a clear warning
|
||||||
|
payload = {"exp": int(time.time()) - 3600} # expired 1 hour ago
|
||||||
|
token = jwt.encode(payload, "secret", algorithm="HS256")
|
||||||
|
with caplog.at_level(logging.WARNING, logger="src.config"):
|
||||||
|
result = _validate_chatgpt_token(token)
|
||||||
|
assert any("expired" in r.message.lower() for r in caplog.records)
|
||||||
|
assert result is not None # still returns the expiry datetime
|
||||||
|
|
||||||
|
def test_expiring_within_24h_logs_warning(self, caplog):
|
||||||
|
payload = {"exp": int(time.time()) + 3600} # expires in 1 hour
|
||||||
|
token = jwt.encode(payload, "secret", algorithm="HS256")
|
||||||
|
with caplog.at_level(logging.WARNING, logger="src.config"):
|
||||||
|
_validate_chatgpt_token(token)
|
||||||
|
assert any("less than 24 hours" in r.message for r in caplog.records)
|
||||||
|
|
||||||
|
def test_valid_token_no_expiry_warning(self, caplog):
|
||||||
|
payload = {"exp": int(time.time()) + 86400 * 5} # valid for 5 days
|
||||||
|
token = jwt.encode(payload, "secret", algorithm="HS256")
|
||||||
|
with caplog.at_level(logging.WARNING, logger="src.config"):
|
||||||
|
result = _validate_chatgpt_token(token)
|
||||||
|
assert not any("expired" in r.message.lower() for r in caplog.records)
|
||||||
|
assert result is not None
|
||||||
|
|
||||||
|
def test_token_without_exp_claim_logs_warning(self, caplog):
|
||||||
|
payload = {"sub": "user123"} # no exp
|
||||||
|
token = jwt.encode(payload, "secret", algorithm="HS256")
|
||||||
|
with caplog.at_level(logging.WARNING, logger="src.config"):
|
||||||
|
result = _validate_chatgpt_token(token)
|
||||||
|
assert any("'exp'" in r.message or "no 'exp'" in r.message for r in caplog.records)
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_jwe_encrypted_token_returns_none(self, caplog):
|
||||||
|
# JWE tokens (alg=dir) cannot be decoded client-side — this is normal for ChatGPT
|
||||||
|
jwe_like = "eyJhbGciOiJkaXIiLCJlbmMiOiJBMjU2R0NNIn0.fake.token.data.here"
|
||||||
|
with caplog.at_level(logging.DEBUG, logger="src.config"):
|
||||||
|
result = _validate_chatgpt_token(jwe_like)
|
||||||
|
assert result is None # cannot decode, but not an error
|
||||||
|
|
||||||
|
def test_non_jwt_string_logs_warning(self, caplog):
|
||||||
|
with caplog.at_level(logging.WARNING, logger="src.config"):
|
||||||
|
result = _validate_chatgpt_token("notajwttoken")
|
||||||
|
assert any("does not look like a JWT" in r.message for r in caplog.records)
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
|
||||||
|
class TestSessionLimiterConfig:
|
||||||
|
"""MAX_CONVERSATIONS_PER_RUN and REQUEST_DELAY parsing in load_config."""
|
||||||
|
|
||||||
|
def _load(self, monkeypatch, tmp_path, **env):
|
||||||
|
from src.config import load_config
|
||||||
|
monkeypatch.setenv("EXPORT_DIR", str(tmp_path / "exports"))
|
||||||
|
monkeypatch.setenv("CACHE_DIR", str(tmp_path / "cache"))
|
||||||
|
for key in (
|
||||||
|
"MAX_CONVERSATIONS_PER_RUN",
|
||||||
|
"REQUEST_DELAY",
|
||||||
|
"EXPORTER_HIDDEN_CONTENT",
|
||||||
|
"EXPORTER_DOWNLOAD_MEDIA",
|
||||||
|
):
|
||||||
|
monkeypatch.delenv(key, raising=False)
|
||||||
|
for key, value in env.items():
|
||||||
|
monkeypatch.setenv(key, value)
|
||||||
|
return load_config()
|
||||||
|
|
||||||
|
def test_defaults(self, monkeypatch, tmp_path):
|
||||||
|
cfg = self._load(monkeypatch, tmp_path)
|
||||||
|
assert cfg.max_conversations is None
|
||||||
|
assert cfg.request_delay == 1.0
|
||||||
|
assert cfg.hidden_content == "placeholder"
|
||||||
|
assert cfg.download_media == "images"
|
||||||
|
|
||||||
|
def test_download_media_valid(self, monkeypatch, tmp_path):
|
||||||
|
cfg = self._load(monkeypatch, tmp_path, EXPORTER_DOWNLOAD_MEDIA="all")
|
||||||
|
assert cfg.download_media == "all"
|
||||||
|
|
||||||
|
def test_download_media_invalid_raises(self, monkeypatch, tmp_path):
|
||||||
|
from src.config import ConfigError
|
||||||
|
with pytest.raises(ConfigError, match="EXPORTER_DOWNLOAD_MEDIA"):
|
||||||
|
self._load(monkeypatch, tmp_path, EXPORTER_DOWNLOAD_MEDIA="sometimes")
|
||||||
|
|
||||||
|
def test_valid_values(self, monkeypatch, tmp_path):
|
||||||
|
cfg = self._load(
|
||||||
|
monkeypatch, tmp_path,
|
||||||
|
MAX_CONVERSATIONS_PER_RUN="25", REQUEST_DELAY="0.5",
|
||||||
|
)
|
||||||
|
assert cfg.max_conversations == 25
|
||||||
|
assert cfg.request_delay == 0.5
|
||||||
|
|
||||||
|
def test_zero_delay_allowed(self, monkeypatch, tmp_path):
|
||||||
|
cfg = self._load(monkeypatch, tmp_path, REQUEST_DELAY="0")
|
||||||
|
assert cfg.request_delay == 0.0
|
||||||
|
|
||||||
|
def test_non_integer_cap_raises(self, monkeypatch, tmp_path):
|
||||||
|
from src.config import ConfigError
|
||||||
|
with pytest.raises(ConfigError, match="MAX_CONVERSATIONS_PER_RUN"):
|
||||||
|
self._load(monkeypatch, tmp_path, MAX_CONVERSATIONS_PER_RUN="lots")
|
||||||
|
|
||||||
|
def test_zero_cap_raises(self, monkeypatch, tmp_path):
|
||||||
|
from src.config import ConfigError
|
||||||
|
with pytest.raises(ConfigError, match="at least 1"):
|
||||||
|
self._load(monkeypatch, tmp_path, MAX_CONVERSATIONS_PER_RUN="0")
|
||||||
|
|
||||||
|
def test_negative_delay_raises(self, monkeypatch, tmp_path):
|
||||||
|
from src.config import ConfigError
|
||||||
|
with pytest.raises(ConfigError, match="REQUEST_DELAY"):
|
||||||
|
self._load(monkeypatch, tmp_path, REQUEST_DELAY="-1")
|
||||||
+340
-2
@@ -1,4 +1,4 @@
|
|||||||
"""Unit tests for src/exporters/."""
|
"""Unit tests for src/exporters/ and src/blocks.py."""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
@@ -7,6 +7,24 @@ from pathlib import Path
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
from src.blocks import (
|
||||||
|
BLOCK_TYPE_TEXT,
|
||||||
|
UNKNOWN_REASON_EXTRACTION_FAILED,
|
||||||
|
UNKNOWN_REASON_UNKNOWN_TYPE,
|
||||||
|
_blockquote_prefix,
|
||||||
|
_safe_fence,
|
||||||
|
make_code_block,
|
||||||
|
make_file_placeholder,
|
||||||
|
make_hidden_context_marker,
|
||||||
|
make_image_placeholder,
|
||||||
|
make_subagent_block,
|
||||||
|
make_text_block,
|
||||||
|
make_thinking_block,
|
||||||
|
make_tool_result_block,
|
||||||
|
make_tool_use_block,
|
||||||
|
make_unknown_block,
|
||||||
|
render_blocks_to_markdown,
|
||||||
|
)
|
||||||
from src.exporters.markdown import MarkdownExporter, _yaml_escape, _format_timestamp
|
from src.exporters.markdown import MarkdownExporter, _yaml_escape, _format_timestamp
|
||||||
from src.exporters.json_export import JSONExporter
|
from src.exporters.json_export import JSONExporter
|
||||||
|
|
||||||
@@ -102,6 +120,30 @@ class TestMarkdownFrontmatter:
|
|||||||
assert "```python" in content
|
assert "```python" in content
|
||||||
assert "print('hello')" in content
|
assert "print('hello')" in content
|
||||||
|
|
||||||
|
def test_subagent_block_renders_as_details(self, tmp_path):
|
||||||
|
sub = make_subagent_block(
|
||||||
|
agent_type="Explore",
|
||||||
|
description="find the thing",
|
||||||
|
messages=[
|
||||||
|
{"role": "user", "blocks": [make_text_block("Go find it")]},
|
||||||
|
{"role": "assistant", "blocks": [make_text_block("Found it here.")]},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
conv = {
|
||||||
|
**SAMPLE_CONV,
|
||||||
|
"provider": "claude-code",
|
||||||
|
"messages": [
|
||||||
|
{"role": "assistant", "content_type": "text", "timestamp": None,
|
||||||
|
"blocks": [make_text_block("Delegating."), sub]},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
content = MarkdownExporter(tmp_path).export(conv).read_text()
|
||||||
|
assert "<details>" in content
|
||||||
|
assert "<summary>🤖 Subagent: Explore — find the thing</summary>" in content
|
||||||
|
assert "Go find it" in content
|
||||||
|
assert "Found it here." in content
|
||||||
|
assert "</details>" in content
|
||||||
|
|
||||||
|
|
||||||
class TestMarkdownFilenameGeneration:
|
class TestMarkdownFilenameGeneration:
|
||||||
def test_filename_format(self, tmp_path):
|
def test_filename_format(self, tmp_path):
|
||||||
@@ -122,7 +164,7 @@ class TestMarkdownFilenameGeneration:
|
|||||||
def test_year_in_path(self, tmp_path):
|
def test_year_in_path(self, tmp_path):
|
||||||
exp = MarkdownExporter(tmp_path)
|
exp = MarkdownExporter(tmp_path)
|
||||||
path = exp.export(SAMPLE_CONV)
|
path = exp.export(SAMPLE_CONV)
|
||||||
assert "/2024/" in str(path)
|
assert ".2024/" in str(path)
|
||||||
|
|
||||||
def test_output_structure_provider_project(self, tmp_path):
|
def test_output_structure_provider_project(self, tmp_path):
|
||||||
exp = MarkdownExporter(tmp_path, output_structure="provider/project")
|
exp = MarkdownExporter(tmp_path, output_structure="provider/project")
|
||||||
@@ -199,6 +241,34 @@ class TestJSONExporter:
|
|||||||
assert " " in raw
|
assert " " in raw
|
||||||
|
|
||||||
|
|
||||||
|
class TestBothFormats:
|
||||||
|
"""T-38: Markdown and JSON exporters produce matching filenames for the same conversation."""
|
||||||
|
|
||||||
|
def test_both_formats_produce_files(self, tmp_path):
|
||||||
|
md_exp = MarkdownExporter(tmp_path)
|
||||||
|
json_exp = JSONExporter(tmp_path)
|
||||||
|
md_path = md_exp.export(SAMPLE_CONV)
|
||||||
|
json_path = json_exp.export(SAMPLE_CONV)
|
||||||
|
assert md_path.exists()
|
||||||
|
assert json_path.exists()
|
||||||
|
|
||||||
|
def test_both_formats_have_matching_stems(self, tmp_path):
|
||||||
|
md_exp = MarkdownExporter(tmp_path)
|
||||||
|
json_exp = JSONExporter(tmp_path)
|
||||||
|
md_path = md_exp.export(SAMPLE_CONV)
|
||||||
|
json_path = json_exp.export(SAMPLE_CONV)
|
||||||
|
assert md_path.suffix == ".md"
|
||||||
|
assert json_path.suffix == ".json"
|
||||||
|
assert md_path.stem == json_path.stem
|
||||||
|
|
||||||
|
def test_both_formats_same_directory(self, tmp_path):
|
||||||
|
md_exp = MarkdownExporter(tmp_path)
|
||||||
|
json_exp = JSONExporter(tmp_path)
|
||||||
|
md_path = md_exp.export(SAMPLE_CONV)
|
||||||
|
json_path = json_exp.export(SAMPLE_CONV)
|
||||||
|
assert md_path.parent == json_path.parent
|
||||||
|
|
||||||
|
|
||||||
class TestYamlEscape:
|
class TestYamlEscape:
|
||||||
def test_escapes_double_quotes(self):
|
def test_escapes_double_quotes(self):
|
||||||
assert _yaml_escape('Say "hello"') == 'Say \\"hello\\"'
|
assert _yaml_escape('Say "hello"') == 'Say \\"hello\\"'
|
||||||
@@ -222,3 +292,271 @@ class TestFormatTimestamp:
|
|||||||
|
|
||||||
def test_empty_string(self):
|
def test_empty_string(self):
|
||||||
assert _format_timestamp("") == ""
|
assert _format_timestamp("") == ""
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Block helpers and rendering
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestSafeFence:
|
||||||
|
def test_minimum_three_backticks(self):
|
||||||
|
assert _safe_fence("plain text") == "```"
|
||||||
|
|
||||||
|
def test_four_backticks_when_three_in_content(self):
|
||||||
|
assert _safe_fence("here ``` is a fence") == "````"
|
||||||
|
|
||||||
|
def test_five_backticks_when_four_in_content(self):
|
||||||
|
assert _safe_fence("here ```` is four") == "`````"
|
||||||
|
|
||||||
|
def test_handles_empty_string(self):
|
||||||
|
assert _safe_fence("") == "```"
|
||||||
|
|
||||||
|
def test_handles_run_at_end(self):
|
||||||
|
# Trailing run still counted
|
||||||
|
assert _safe_fence("text ending in ```") == "````"
|
||||||
|
|
||||||
|
|
||||||
|
class TestBlockquotePrefix:
|
||||||
|
def test_single_line(self):
|
||||||
|
assert _blockquote_prefix("hello") == "> hello"
|
||||||
|
|
||||||
|
def test_multi_line(self):
|
||||||
|
assert _blockquote_prefix("a\nb\nc") == "> a\n> b\n> c"
|
||||||
|
|
||||||
|
def test_empty_lines_become_naked_quote_marker(self):
|
||||||
|
assert _blockquote_prefix("a\n\nb") == "> a\n>\n> b"
|
||||||
|
|
||||||
|
def test_empty_string(self):
|
||||||
|
assert _blockquote_prefix("") == ">"
|
||||||
|
|
||||||
|
|
||||||
|
class TestBlockConstructors:
|
||||||
|
def test_make_text_block_returns_none_for_empty(self):
|
||||||
|
assert make_text_block("") is None
|
||||||
|
assert make_text_block(" ") is None
|
||||||
|
|
||||||
|
def test_make_text_block_returns_dict(self):
|
||||||
|
b = make_text_block("hello")
|
||||||
|
assert b == {"type": "text", "text": "hello"}
|
||||||
|
|
||||||
|
def test_make_code_block_returns_none_for_empty(self):
|
||||||
|
assert make_code_block("") is None
|
||||||
|
|
||||||
|
def test_make_thinking_block_returns_none_for_empty(self):
|
||||||
|
assert make_thinking_block("") is None
|
||||||
|
|
||||||
|
|
||||||
|
class TestRenderBlocks:
|
||||||
|
def test_text_block_renders_as_paragraph(self):
|
||||||
|
out = render_blocks_to_markdown([make_text_block("Hello world")])
|
||||||
|
assert out == "Hello world"
|
||||||
|
|
||||||
|
def test_blocks_separated_by_blank_line(self):
|
||||||
|
out = render_blocks_to_markdown(
|
||||||
|
[make_text_block("first"), make_text_block("second")]
|
||||||
|
)
|
||||||
|
assert out == "first\n\nsecond"
|
||||||
|
|
||||||
|
def test_code_block_with_language(self):
|
||||||
|
out = render_blocks_to_markdown([make_code_block("print(1)", language="python")])
|
||||||
|
assert "```python" in out
|
||||||
|
assert "print(1)" in out
|
||||||
|
|
||||||
|
def test_thinking_block_uses_blockquote(self):
|
||||||
|
out = render_blocks_to_markdown([make_thinking_block("step 1\nstep 2")])
|
||||||
|
assert "**💭 Reasoning**" in out
|
||||||
|
assert "> step 1" in out
|
||||||
|
assert "> step 2" in out
|
||||||
|
|
||||||
|
def test_tool_use_renders_as_blockquote_with_safe_fence(self):
|
||||||
|
out = render_blocks_to_markdown(
|
||||||
|
[make_tool_use_block("search", {"query": "test"})]
|
||||||
|
)
|
||||||
|
assert "> 🔧 **Tool: search**" in out
|
||||||
|
# Every line of the body is blockquote-prefixed
|
||||||
|
assert "> ```json" in out
|
||||||
|
assert "> }" in out
|
||||||
|
|
||||||
|
def test_tool_use_with_multiline_input(self):
|
||||||
|
out = render_blocks_to_markdown(
|
||||||
|
[make_tool_use_block("complex", {"a": 1, "b": [{"x": "y"}]})]
|
||||||
|
)
|
||||||
|
# Prefix every line of multi-line JSON
|
||||||
|
for line in out.split("\n"):
|
||||||
|
assert line.startswith(">") or line == ""
|
||||||
|
|
||||||
|
def test_tool_result_success_uses_outbox_icon(self):
|
||||||
|
out = render_blocks_to_markdown([make_tool_result_block("OK")])
|
||||||
|
assert "📤 **Result**" in out
|
||||||
|
assert "❌" not in out
|
||||||
|
|
||||||
|
def test_tool_result_error_uses_x_icon(self):
|
||||||
|
out = render_blocks_to_markdown([make_tool_result_block("oops", is_error=True)])
|
||||||
|
assert "❌ **Result (error)**" in out
|
||||||
|
assert "📤" not in out
|
||||||
|
|
||||||
|
def test_tool_result_with_tool_name_in_header(self):
|
||||||
|
out = render_blocks_to_markdown(
|
||||||
|
[make_tool_result_block("done", tool_name="container.exec")]
|
||||||
|
)
|
||||||
|
assert "📤 **Result: container.exec**" in out
|
||||||
|
|
||||||
|
def test_tool_result_error_with_tool_name(self):
|
||||||
|
out = render_blocks_to_markdown(
|
||||||
|
[make_tool_result_block("503", tool_name="web", is_error=True)]
|
||||||
|
)
|
||||||
|
assert "❌ **Result (error): web**" in out
|
||||||
|
|
||||||
|
def test_tool_result_summary_renders_as_italic_line(self):
|
||||||
|
out = render_blocks_to_markdown(
|
||||||
|
[
|
||||||
|
make_tool_result_block(
|
||||||
|
"output",
|
||||||
|
tool_name="container.exec",
|
||||||
|
summary="Reading skill documentation",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
# Summary line is italic, lives between header and fence,
|
||||||
|
# all inside the blockquote prefix.
|
||||||
|
assert "> *Reading skill documentation*" in out
|
||||||
|
# Order: header before summary before fence
|
||||||
|
header_idx = out.index("Result: container.exec")
|
||||||
|
summary_idx = out.index("Reading skill documentation")
|
||||||
|
fence_idx = out.index("output")
|
||||||
|
assert header_idx < summary_idx < fence_idx
|
||||||
|
|
||||||
|
def test_image_placeholder_rendering(self):
|
||||||
|
out = render_blocks_to_markdown(
|
||||||
|
[make_image_placeholder(ref="file-123", source="user_upload")]
|
||||||
|
)
|
||||||
|
assert "🖼️ **Image attached**" in out
|
||||||
|
assert "`file-123`" in out
|
||||||
|
assert "user_upload" in out
|
||||||
|
assert "content not preserved" in out
|
||||||
|
|
||||||
|
def test_file_placeholder_with_metadata(self):
|
||||||
|
out = render_blocks_to_markdown(
|
||||||
|
[make_file_placeholder(ref="sediment://x", mime="audio/wav", size_bytes=10240, duration_seconds=2.5)]
|
||||||
|
)
|
||||||
|
assert "📎 **File attached**" in out
|
||||||
|
assert "audio/wav" in out
|
||||||
|
assert "KB" in out
|
||||||
|
assert "2.50s" in out
|
||||||
|
|
||||||
|
def test_unknown_block_renders_with_keys(self):
|
||||||
|
out = render_blocks_to_markdown(
|
||||||
|
[
|
||||||
|
make_unknown_block(
|
||||||
|
raw_type="future_x",
|
||||||
|
observed_keys=["foo", "bar"],
|
||||||
|
reason=UNKNOWN_REASON_UNKNOWN_TYPE,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
assert "⚠️ **Unsupported content**" in out
|
||||||
|
assert "future_x" in out
|
||||||
|
assert "`foo`" in out
|
||||||
|
assert "`bar`" in out
|
||||||
|
|
||||||
|
def test_unknown_extraction_failed_includes_summary(self):
|
||||||
|
out = render_blocks_to_markdown(
|
||||||
|
[
|
||||||
|
make_unknown_block(
|
||||||
|
raw_type="audio_transcription",
|
||||||
|
observed_keys=["asset_pointer"],
|
||||||
|
reason=UNKNOWN_REASON_EXTRACTION_FAILED,
|
||||||
|
summary="expected key 'text' not found",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
assert "extraction_failed" in out
|
||||||
|
assert "expected key 'text' not found" in out
|
||||||
|
|
||||||
|
def test_hidden_context_marker(self):
|
||||||
|
out = render_blocks_to_markdown(
|
||||||
|
[make_hidden_context_marker("user_editable_context")]
|
||||||
|
)
|
||||||
|
assert "ℹ️ **Hidden context**" in out
|
||||||
|
assert "`user_editable_context`" in out
|
||||||
|
|
||||||
|
def test_safe_fence_prevents_runaway_code_block(self):
|
||||||
|
# Content contains an unbalanced opening fence — without _safe_fence
|
||||||
|
# this would corrupt downstream rendering.
|
||||||
|
evil_content = "before\n```Follow\ntext\nraw is: \"```"
|
||||||
|
block = make_code_block(evil_content)
|
||||||
|
out = render_blocks_to_markdown([block, make_text_block("after")])
|
||||||
|
# The 4-backtick wrap should be present
|
||||||
|
assert "````" in out
|
||||||
|
# The "after" text should appear OUTSIDE any code block — it follows
|
||||||
|
# the closing ```` fence.
|
||||||
|
assert out.endswith("after")
|
||||||
|
|
||||||
|
def test_block_order_preserved(self):
|
||||||
|
blocks = [
|
||||||
|
make_text_block("a"),
|
||||||
|
make_image_placeholder(ref="r1", source="user_upload"),
|
||||||
|
make_text_block("b"),
|
||||||
|
]
|
||||||
|
out = render_blocks_to_markdown(blocks)
|
||||||
|
assert out.index("a") < out.index("Image attached")
|
||||||
|
assert out.index("Image attached") < out.index("b")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Markdown exporter with blocks
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
SAMPLE_CONV_BLOCKS = {
|
||||||
|
"id": "blocks12345",
|
||||||
|
"title": "Blocks Conversation",
|
||||||
|
"provider": "claude",
|
||||||
|
"project": None,
|
||||||
|
"created_at": "2024-06-10T14:32:00Z",
|
||||||
|
"updated_at": "2024-06-10T15:00:00Z",
|
||||||
|
"message_count": 1,
|
||||||
|
"messages": [
|
||||||
|
{
|
||||||
|
"role": "assistant",
|
||||||
|
"content_type": "text",
|
||||||
|
"timestamp": None,
|
||||||
|
"blocks": [
|
||||||
|
{"type": "text", "text": "Here is the answer."},
|
||||||
|
{"type": "tool_use", "name": "search", "input": {"q": "x"}, "tool_id": "t1"},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class TestMarkdownExporterWithBlocks:
|
||||||
|
def test_renders_blocks(self, tmp_path):
|
||||||
|
exp = MarkdownExporter(tmp_path)
|
||||||
|
path = exp.export(SAMPLE_CONV_BLOCKS)
|
||||||
|
body = path.read_text()
|
||||||
|
assert "Here is the answer." in body
|
||||||
|
assert "🔧 **Tool: search**" in body
|
||||||
|
|
||||||
|
def test_falls_back_to_content_when_blocks_missing(self, tmp_path):
|
||||||
|
# Backward-compat: messages with `content` only (no `blocks`) still render.
|
||||||
|
exp = MarkdownExporter(tmp_path)
|
||||||
|
path = exp.export(SAMPLE_CONV) # SAMPLE_CONV has content only, no blocks
|
||||||
|
body = path.read_text()
|
||||||
|
assert "Hello, how are you?" in body
|
||||||
|
|
||||||
|
def test_skips_messages_with_neither_blocks_nor_content(self, tmp_path):
|
||||||
|
conv = {
|
||||||
|
**SAMPLE_CONV_BLOCKS,
|
||||||
|
"messages": [
|
||||||
|
{"role": "user", "content_type": "text", "timestamp": None, "blocks": []},
|
||||||
|
{"role": "assistant", "content_type": "text", "timestamp": None, "blocks": [
|
||||||
|
{"type": "text", "text": "I am here."}
|
||||||
|
]},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
exp = MarkdownExporter(tmp_path)
|
||||||
|
path = exp.export(conv)
|
||||||
|
body = path.read_text()
|
||||||
|
assert "I am here." in body
|
||||||
|
|||||||
+130
-16
@@ -5,7 +5,7 @@ from unittest.mock import MagicMock, patch
|
|||||||
import pytest
|
import pytest
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
from src.joplin import JoplinClient, JoplinError, _http_error_message, _timeout_message, notebook_title
|
from src.joplin import JoplinClient, JoplinError, _http_error_message, _timeout_message, notebook_path
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -31,25 +31,32 @@ def _mock_response(json_data=None, text="", status_code=200):
|
|||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# notebook_title helper
|
# notebook_path helper
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
class TestNotebookTitle:
|
class TestNotebookPath:
|
||||||
def test_no_project(self):
|
def test_no_project(self):
|
||||||
assert notebook_title("chatgpt", None) == "ChatGPT - No Project"
|
assert notebook_path("chatgpt", None) == ("AI-ChatGPT", "No Project")
|
||||||
|
|
||||||
def test_no_project_string(self):
|
def test_no_project_string(self):
|
||||||
assert notebook_title("chatgpt", "no-project") == "ChatGPT - No Project"
|
assert notebook_path("chatgpt", "no-project") == ("AI-ChatGPT", "No Project")
|
||||||
|
|
||||||
def test_project_with_hyphens(self):
|
def test_project_with_hyphens(self):
|
||||||
assert notebook_title("chatgpt", "my-project") == "ChatGPT - My Project"
|
assert notebook_path("chatgpt", "my-project") == ("AI-ChatGPT", "My Project")
|
||||||
|
|
||||||
def test_claude_provider(self):
|
def test_claude_provider(self):
|
||||||
assert notebook_title("claude", "budget-tracker") == "Claude - Budget Tracker"
|
assert notebook_path("claude", "budget-tracker") == ("AI-Claude", "Budget Tracker")
|
||||||
|
|
||||||
|
def test_claude_code_gets_own_top_level_notebook(self):
|
||||||
|
assert notebook_path("claude-code", "services") == ("AI-ClaudeCode", "Services")
|
||||||
|
|
||||||
def test_multi_word_project(self):
|
def test_multi_word_project(self):
|
||||||
assert notebook_title("claude", "ai-research-notes") == "Claude - Ai Research Notes"
|
assert notebook_path("claude", "ai-research-notes") == ("AI-Claude", "Ai Research Notes")
|
||||||
|
|
||||||
|
def test_returns_tuple(self):
|
||||||
|
result = notebook_path("chatgpt", "some-project")
|
||||||
|
assert isinstance(result, tuple) and len(result) == 2
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -57,6 +64,24 @@ class TestNotebookTitle:
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestUpdateNote:
|
||||||
|
def test_parent_id_moves_note(self):
|
||||||
|
client = _make_client()
|
||||||
|
client._put = MagicMock()
|
||||||
|
client.update_note("nid", "Title", "body", parent_id="nb1")
|
||||||
|
args, _ = client._put.call_args
|
||||||
|
assert args[0] == "/notes/nid"
|
||||||
|
assert args[1]["parent_id"] == "nb1"
|
||||||
|
assert args[1]["title"] == "Title"
|
||||||
|
|
||||||
|
def test_no_parent_id_omits_field(self):
|
||||||
|
client = _make_client()
|
||||||
|
client._put = MagicMock()
|
||||||
|
client.update_note("nid", "Title", "body")
|
||||||
|
args, _ = client._put.call_args
|
||||||
|
assert "parent_id" not in args[1]
|
||||||
|
|
||||||
|
|
||||||
class TestPing:
|
class TestPing:
|
||||||
def test_ping_success(self):
|
def test_ping_success(self):
|
||||||
client = _make_client()
|
client = _make_client()
|
||||||
@@ -236,18 +261,30 @@ class TestListNotebooks:
|
|||||||
|
|
||||||
|
|
||||||
class TestGetOrCreateNotebook:
|
class TestGetOrCreateNotebook:
|
||||||
def test_returns_existing_notebook_id(self):
|
def test_returns_existing_root_notebook_id(self):
|
||||||
client = _make_client()
|
client = _make_client()
|
||||||
with patch("requests.get") as mock_get:
|
with patch("requests.get") as mock_get:
|
||||||
mock_get.return_value = _mock_response(
|
mock_get.return_value = _mock_response(
|
||||||
json_data={
|
json_data={
|
||||||
"items": [{"id": "nb-existing", "title": "ChatGPT - No Project"}],
|
"items": [{"id": "nb-existing", "title": "AI-ChatGPT", "parent_id": ""}],
|
||||||
"has_more": False,
|
"has_more": False,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
nb_id = client.get_or_create_notebook("ChatGPT - No Project")
|
nb_id = client.get_or_create_notebook("AI-ChatGPT")
|
||||||
assert nb_id == "nb-existing"
|
assert nb_id == "nb-existing"
|
||||||
|
|
||||||
|
def test_returns_existing_child_notebook_id(self):
|
||||||
|
client = _make_client()
|
||||||
|
with patch("requests.get") as mock_get:
|
||||||
|
mock_get.return_value = _mock_response(
|
||||||
|
json_data={
|
||||||
|
"items": [{"id": "nb-child", "title": "No Project", "parent_id": "nb-parent"}],
|
||||||
|
"has_more": False,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
nb_id = client.get_or_create_notebook("No Project", parent_id="nb-parent")
|
||||||
|
assert nb_id == "nb-child"
|
||||||
|
|
||||||
def test_creates_new_notebook_when_not_found(self):
|
def test_creates_new_notebook_when_not_found(self):
|
||||||
client = _make_client()
|
client = _make_client()
|
||||||
with patch("requests.get") as mock_get, patch("requests.post") as mock_post:
|
with patch("requests.get") as mock_get, patch("requests.post") as mock_post:
|
||||||
@@ -255,26 +292,103 @@ class TestGetOrCreateNotebook:
|
|||||||
json_data={"items": [], "has_more": False}
|
json_data={"items": [], "has_more": False}
|
||||||
)
|
)
|
||||||
mock_post.return_value = _mock_response(
|
mock_post.return_value = _mock_response(
|
||||||
json_data={"id": "nb-new", "title": "ChatGPT - New Project"}
|
json_data={"id": "nb-new", "title": "AI-ChatGPT"}
|
||||||
)
|
)
|
||||||
nb_id = client.get_or_create_notebook("ChatGPT - New Project")
|
nb_id = client.get_or_create_notebook("AI-ChatGPT")
|
||||||
assert nb_id == "nb-new"
|
assert nb_id == "nb-new"
|
||||||
mock_post.assert_called_once()
|
mock_post.assert_called_once()
|
||||||
|
|
||||||
|
def test_creates_child_notebook_with_parent_id(self):
|
||||||
|
client = _make_client()
|
||||||
|
with patch("requests.get") as mock_get, patch("requests.post") as mock_post:
|
||||||
|
mock_get.return_value = _mock_response(
|
||||||
|
json_data={"items": [], "has_more": False}
|
||||||
|
)
|
||||||
|
mock_post.return_value = _mock_response(
|
||||||
|
json_data={"id": "nb-child", "title": "My Project"}
|
||||||
|
)
|
||||||
|
nb_id = client.get_or_create_notebook("My Project", parent_id="nb-parent")
|
||||||
|
assert nb_id == "nb-child"
|
||||||
|
_, kwargs = mock_post.call_args
|
||||||
|
assert kwargs["json"]["parent_id"] == "nb-parent"
|
||||||
|
|
||||||
|
def test_does_not_include_parent_id_for_root(self):
|
||||||
|
client = _make_client()
|
||||||
|
with patch("requests.get") as mock_get, patch("requests.post") as mock_post:
|
||||||
|
mock_get.return_value = _mock_response(json_data={"items": [], "has_more": False})
|
||||||
|
mock_post.return_value = _mock_response(json_data={"id": "nb-root", "title": "AI-Claude"})
|
||||||
|
client.get_or_create_notebook("AI-Claude")
|
||||||
|
_, kwargs = mock_post.call_args
|
||||||
|
assert "parent_id" not in kwargs["json"]
|
||||||
|
|
||||||
def test_caches_notebook_after_first_load(self):
|
def test_caches_notebook_after_first_load(self):
|
||||||
client = _make_client()
|
client = _make_client()
|
||||||
with patch("requests.get") as mock_get:
|
with patch("requests.get") as mock_get:
|
||||||
mock_get.return_value = _mock_response(
|
mock_get.return_value = _mock_response(
|
||||||
json_data={
|
json_data={
|
||||||
"items": [{"id": "nb1", "title": "Claude - No Project"}],
|
"items": [{"id": "nb1", "title": "AI-Claude", "parent_id": ""}],
|
||||||
"has_more": False,
|
"has_more": False,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
# Call twice — GET /folders should only happen once
|
# Call twice — GET /folders should only happen once
|
||||||
client.get_or_create_notebook("Claude - No Project")
|
client.get_or_create_notebook("AI-Claude")
|
||||||
client.get_or_create_notebook("Claude - No Project")
|
client.get_or_create_notebook("AI-Claude")
|
||||||
assert mock_get.call_count == 1
|
assert mock_get.call_count == 1
|
||||||
|
|
||||||
|
def test_different_parent_ids_are_distinct_cache_entries(self):
|
||||||
|
"""Same title under different parents are different notebooks."""
|
||||||
|
client = _make_client()
|
||||||
|
with patch("requests.get") as mock_get:
|
||||||
|
mock_get.return_value = _mock_response(
|
||||||
|
json_data={
|
||||||
|
"items": [
|
||||||
|
{"id": "nb-a", "title": "No Project", "parent_id": "parent-chatgpt"},
|
||||||
|
{"id": "nb-b", "title": "No Project", "parent_id": "parent-claude"},
|
||||||
|
],
|
||||||
|
"has_more": False,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
id_a = client.get_or_create_notebook("No Project", parent_id="parent-chatgpt")
|
||||||
|
id_b = client.get_or_create_notebook("No Project", parent_id="parent-claude")
|
||||||
|
assert id_a == "nb-a"
|
||||||
|
assert id_b == "nb-b"
|
||||||
|
|
||||||
|
|
||||||
|
class TestGetOrCreateNotebookPath:
|
||||||
|
def test_creates_two_level_path(self):
|
||||||
|
client = _make_client()
|
||||||
|
with patch("requests.get") as mock_get, patch("requests.post") as mock_post:
|
||||||
|
mock_get.return_value = _mock_response(json_data={"items": [], "has_more": False})
|
||||||
|
mock_post.side_effect = [
|
||||||
|
_mock_response(json_data={"id": "nb-parent", "title": "AI-ChatGPT"}),
|
||||||
|
_mock_response(json_data={"id": "nb-child", "title": "No Project"}),
|
||||||
|
]
|
||||||
|
leaf_id = client.get_or_create_notebook_path(["AI-ChatGPT", "No Project"])
|
||||||
|
assert leaf_id == "nb-child"
|
||||||
|
assert mock_post.call_count == 2
|
||||||
|
# Second POST should use the parent's ID
|
||||||
|
_, kwargs = mock_post.call_args_list[1]
|
||||||
|
assert kwargs["json"]["parent_id"] == "nb-parent"
|
||||||
|
|
||||||
|
def test_reuses_existing_parent_for_new_child(self):
|
||||||
|
client = _make_client()
|
||||||
|
with patch("requests.get") as mock_get, patch("requests.post") as mock_post:
|
||||||
|
mock_get.return_value = _mock_response(
|
||||||
|
json_data={
|
||||||
|
"items": [{"id": "nb-parent", "title": "AI-Claude", "parent_id": ""}],
|
||||||
|
"has_more": False,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
mock_post.return_value = _mock_response(
|
||||||
|
json_data={"id": "nb-child", "title": "Budget Tracker"}
|
||||||
|
)
|
||||||
|
leaf_id = client.get_or_create_notebook_path(["AI-Claude", "Budget Tracker"])
|
||||||
|
assert leaf_id == "nb-child"
|
||||||
|
# Only one POST — the parent already existed
|
||||||
|
assert mock_post.call_count == 1
|
||||||
|
_, kwargs = mock_post.call_args
|
||||||
|
assert kwargs["json"]["parent_id"] == "nb-parent"
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# create_note
|
# create_note
|
||||||
|
|||||||
@@ -0,0 +1,248 @@
|
|||||||
|
"""Tests for media downloads and Joplin resource rewriting."""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from src.blocks import (
|
||||||
|
make_file_placeholder,
|
||||||
|
make_image_placeholder,
|
||||||
|
render_blocks_to_markdown,
|
||||||
|
)
|
||||||
|
from src.loss_report import LossReport
|
||||||
|
from src.media import resolve_media, resolve_media_policy
|
||||||
|
from src.providers.base import ProviderError
|
||||||
|
from src.providers.chatgpt import parse_asset_file_id
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Asset reference parsing
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestParseAssetFileId:
|
||||||
|
def test_plain_sediment(self):
|
||||||
|
assert parse_asset_file_id("sediment://file_00000000245c71fda5") == "file_00000000245c71fda5"
|
||||||
|
|
||||||
|
def test_generated_image_with_hash_and_page(self):
|
||||||
|
ref = "sediment://8456107fc383a53#file_00000000979c71f685#p_6.png"
|
||||||
|
assert parse_asset_file_id(ref) == "file_00000000979c71f685"
|
||||||
|
|
||||||
|
def test_file_service_scheme(self):
|
||||||
|
assert parse_asset_file_id("file-service://file-AbCdEf") == "file-AbCdEf"
|
||||||
|
|
||||||
|
def test_unrecognised(self):
|
||||||
|
assert parse_asset_file_id("https://example.com/x.png") is None
|
||||||
|
assert parse_asset_file_id("") is None
|
||||||
|
assert parse_asset_file_id(None) is None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Media policy
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestMediaPolicy:
|
||||||
|
def test_default(self, monkeypatch):
|
||||||
|
monkeypatch.delenv("EXPORTER_DOWNLOAD_MEDIA", raising=False)
|
||||||
|
assert resolve_media_policy() == "images"
|
||||||
|
|
||||||
|
def test_valid(self, monkeypatch):
|
||||||
|
monkeypatch.setenv("EXPORTER_DOWNLOAD_MEDIA", "all")
|
||||||
|
assert resolve_media_policy() == "all"
|
||||||
|
|
||||||
|
def test_invalid_falls_back(self, monkeypatch, caplog):
|
||||||
|
monkeypatch.setenv("EXPORTER_DOWNLOAD_MEDIA", "bogus")
|
||||||
|
assert resolve_media_policy() == "images"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# resolve_media
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeProvider:
|
||||||
|
"""Minimal provider exposing download_asset + the ref parser."""
|
||||||
|
|
||||||
|
def __init__(self, assets=None, fail_refs=None):
|
||||||
|
self._assets = assets or {}
|
||||||
|
self._fail_refs = fail_refs or {}
|
||||||
|
self.calls = []
|
||||||
|
|
||||||
|
def parse_asset_file_id(self, ref):
|
||||||
|
return parse_asset_file_id(ref)
|
||||||
|
|
||||||
|
def download_asset(self, ref):
|
||||||
|
self.calls.append(ref)
|
||||||
|
if ref in self._fail_refs:
|
||||||
|
raise ProviderError("chatgpt", "download_asset", self._fail_refs[ref])
|
||||||
|
return self._assets[ref] # (content, mime, file_name)
|
||||||
|
|
||||||
|
|
||||||
|
def _conv_with(blocks):
|
||||||
|
return {
|
||||||
|
"id": "conv-1",
|
||||||
|
"title": "Has Media",
|
||||||
|
"provider": "chatgpt",
|
||||||
|
"project": None,
|
||||||
|
"created_at": "2026-05-20T00:00:00+00:00",
|
||||||
|
"messages": [{"role": "user", "blocks": blocks}],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class TestResolveMedia:
|
||||||
|
def test_downloads_image_and_inlines(self, tmp_path):
|
||||||
|
ref = "sediment://file_img1"
|
||||||
|
provider = _FakeProvider({ref: (b"\x89PNG\r\n", "image/png", "x.png")})
|
||||||
|
block = make_image_placeholder(ref=ref, source="user_upload")
|
||||||
|
conv = _conv_with([block])
|
||||||
|
report = LossReport()
|
||||||
|
|
||||||
|
n = resolve_media(conv, provider, tmp_path, "provider/project/year", "images", report)
|
||||||
|
|
||||||
|
assert n == 1
|
||||||
|
assert report.media_downloaded == 1
|
||||||
|
assert block["local_path"] == "media/file_img1.png"
|
||||||
|
rendered = render_blocks_to_markdown([block])
|
||||||
|
assert rendered == ""
|
||||||
|
# File written under the conversation's media/ dir
|
||||||
|
written = list(tmp_path.rglob("media/file_img1.png"))
|
||||||
|
assert written and written[0].read_bytes() == b"\x89PNG\r\n"
|
||||||
|
|
||||||
|
def test_images_policy_skips_files(self, tmp_path):
|
||||||
|
ref = "sediment://file_audio1"
|
||||||
|
provider = _FakeProvider({ref: (b"RIFF", "audio/wav", "a.wav")})
|
||||||
|
block = make_file_placeholder(ref=ref, mime="audio/wav", size_bytes=1000)
|
||||||
|
conv = _conv_with([block])
|
||||||
|
report = LossReport()
|
||||||
|
|
||||||
|
n = resolve_media(conv, provider, tmp_path, "provider/project/year", "images", report)
|
||||||
|
assert n == 0
|
||||||
|
assert "local_path" not in block
|
||||||
|
assert provider.calls == []
|
||||||
|
|
||||||
|
def test_all_policy_downloads_files(self, tmp_path):
|
||||||
|
ref = "sediment://file_audio1"
|
||||||
|
provider = _FakeProvider({ref: (b"RIFFdata", "audio/wav", "a.wav")})
|
||||||
|
block = make_file_placeholder(ref=ref, mime="audio/wav", size_bytes=8)
|
||||||
|
conv = _conv_with([block])
|
||||||
|
report = LossReport()
|
||||||
|
|
||||||
|
n = resolve_media(conv, provider, tmp_path, "provider/project/year", "all", report)
|
||||||
|
assert n == 1
|
||||||
|
assert block["local_path"] == "media/file_audio1.wav"
|
||||||
|
rendered = render_blocks_to_markdown([block])
|
||||||
|
assert "media/file_audio1.wav" in rendered and rendered.startswith("> 📎")
|
||||||
|
|
||||||
|
def test_off_policy_noop(self, tmp_path):
|
||||||
|
provider = _FakeProvider({"sediment://file_x": (b"x", "image/png", None)})
|
||||||
|
block = make_image_placeholder(ref="sediment://file_x", source="user_upload")
|
||||||
|
conv = _conv_with([block])
|
||||||
|
report = LossReport()
|
||||||
|
assert resolve_media(conv, provider, tmp_path, "provider/project/year", "off", report) == 0
|
||||||
|
assert provider.calls == []
|
||||||
|
|
||||||
|
def test_idempotent_uses_disk(self, tmp_path):
|
||||||
|
ref = "sediment://file_img1"
|
||||||
|
provider = _FakeProvider({ref: (b"\x89PNG", "image/png", "x.png")})
|
||||||
|
conv = _conv_with([make_image_placeholder(ref=ref, source="user_upload")])
|
||||||
|
report = LossReport()
|
||||||
|
resolve_media(conv, provider, tmp_path, "provider/project/year", "images", report)
|
||||||
|
assert len(provider.calls) == 1
|
||||||
|
|
||||||
|
# Second run, fresh blocks: file already on disk → no new API call.
|
||||||
|
conv2 = _conv_with([make_image_placeholder(ref=ref, source="user_upload")])
|
||||||
|
resolve_media(conv2, provider, tmp_path, "provider/project/year", "images", LossReport())
|
||||||
|
assert len(provider.calls) == 1 # unchanged
|
||||||
|
assert conv2["messages"][0]["blocks"][0]["local_path"] == "media/file_img1.png"
|
||||||
|
|
||||||
|
def test_failure_keeps_placeholder_and_counts(self, tmp_path):
|
||||||
|
ref = "sediment://file_gone"
|
||||||
|
provider = _FakeProvider(fail_refs={ref: RuntimeError("Signed URL returned HTTP 404")})
|
||||||
|
block = make_image_placeholder(ref=ref, source="model_generated")
|
||||||
|
conv = _conv_with([block])
|
||||||
|
report = LossReport()
|
||||||
|
|
||||||
|
n = resolve_media(conv, provider, tmp_path, "provider/project/year", "images", report)
|
||||||
|
assert n == 0
|
||||||
|
assert "local_path" not in block
|
||||||
|
assert report.media_failed["expired-or-missing"] == 1
|
||||||
|
# Still renders as a placeholder, not a broken image link
|
||||||
|
assert render_blocks_to_markdown([block]).startswith("> 🖼️")
|
||||||
|
|
||||||
|
def test_provider_without_download_asset(self, tmp_path):
|
||||||
|
"""claude-code has no remote assets — resolve_media must no-op."""
|
||||||
|
class NoDownload:
|
||||||
|
pass
|
||||||
|
block = make_image_placeholder(ref="sediment://file_x", source="user_upload")
|
||||||
|
conv = _conv_with([block])
|
||||||
|
assert resolve_media(conv, NoDownload(), tmp_path, "provider/project/year", "images", LossReport()) == 0
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Joplin media rewriting
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeJoplin:
|
||||||
|
def __init__(self):
|
||||||
|
self.uploaded = []
|
||||||
|
self._n = 0
|
||||||
|
|
||||||
|
def create_resource(self, file_path, title=None):
|
||||||
|
self.uploaded.append(Path(file_path).name)
|
||||||
|
self._n += 1
|
||||||
|
return f"res{self._n}"
|
||||||
|
|
||||||
|
|
||||||
|
class TestUploadMediaAndRewrite:
|
||||||
|
def _note(self, tmp_path, body):
|
||||||
|
media = tmp_path / "media"
|
||||||
|
media.mkdir()
|
||||||
|
(media / "file_img1.png").write_bytes(b"\x89PNG")
|
||||||
|
(media / "clip.wav").write_bytes(b"RIFF")
|
||||||
|
return body
|
||||||
|
|
||||||
|
def test_rewrites_image_and_file_links(self, tmp_path):
|
||||||
|
from src.joplin import upload_media_and_rewrite
|
||||||
|
body = self._note(
|
||||||
|
tmp_path,
|
||||||
|
"Look: \n"
|
||||||
|
"> 📎 **File attached** — [clip.wav](media/clip.wav) (audio/wav)",
|
||||||
|
)
|
||||||
|
client = _FakeJoplin()
|
||||||
|
new_body, res_map = upload_media_and_rewrite(body, tmp_path, client, {})
|
||||||
|
|
||||||
|
assert "" in new_body
|
||||||
|
assert "(:/res2)" in new_body
|
||||||
|
assert "media/" not in new_body
|
||||||
|
assert res_map == {"media/file_img1.png": "res1", "media/clip.wav": "res2"}
|
||||||
|
assert len(client.uploaded) == 2
|
||||||
|
|
||||||
|
def test_reuses_known_resource_ids(self, tmp_path):
|
||||||
|
from src.joplin import upload_media_and_rewrite
|
||||||
|
body = self._note(tmp_path, "")
|
||||||
|
client = _FakeJoplin()
|
||||||
|
existing = {"media/file_img1.png": "existing-res"}
|
||||||
|
new_body, res_map = upload_media_and_rewrite(body, tmp_path, client, existing)
|
||||||
|
assert "(:/existing-res)" in new_body
|
||||||
|
assert client.uploaded == [] # no re-upload
|
||||||
|
assert res_map == existing
|
||||||
|
|
||||||
|
def test_missing_file_left_as_link(self, tmp_path):
|
||||||
|
from src.joplin import upload_media_and_rewrite
|
||||||
|
(tmp_path / "media").mkdir()
|
||||||
|
body = ""
|
||||||
|
client = _FakeJoplin()
|
||||||
|
new_body, res_map = upload_media_and_rewrite(body, tmp_path, client, {})
|
||||||
|
assert new_body == body
|
||||||
|
assert res_map == {}
|
||||||
|
assert client.uploaded == []
|
||||||
|
|
||||||
|
def test_no_media_links_untouched(self, tmp_path):
|
||||||
|
from src.joplin import upload_media_and_rewrite
|
||||||
|
body = "Just text with [a link](https://example.com) and `media/foo` in code."
|
||||||
|
client = _FakeJoplin()
|
||||||
|
new_body, res_map = upload_media_and_rewrite(body, tmp_path, client, {})
|
||||||
|
assert new_body == body
|
||||||
|
assert res_map == {}
|
||||||
+983
-41
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,147 @@
|
|||||||
|
"""Tests for src/utils.py — filename generation, path building, redaction."""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from src.utils import (
|
||||||
|
build_export_path,
|
||||||
|
format_token_status,
|
||||||
|
generate_filename,
|
||||||
|
redact_secrets,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestGenerateFilename:
|
||||||
|
def test_basic_format(self):
|
||||||
|
name = generate_filename("Hello World", "abc12345def", "2024-06-10T14:00:00Z")
|
||||||
|
assert name == "2024-06-10_hello-world_abc12345.md"
|
||||||
|
|
||||||
|
def test_special_chars_slugified(self):
|
||||||
|
# T-36: titles with punctuation must produce safe, OS-compatible filenames
|
||||||
|
name = generate_filename("What's this?! A test.", "abc12345", "2024-06-01T00:00:00Z")
|
||||||
|
assert "?" not in name
|
||||||
|
assert "!" not in name
|
||||||
|
assert "'" not in name
|
||||||
|
assert " " not in name
|
||||||
|
assert name.startswith("2024-06-01_")
|
||||||
|
assert name.endswith("_abc12345.md")
|
||||||
|
|
||||||
|
def test_unicode_chars_handled(self):
|
||||||
|
name = generate_filename("Héllo Wörld", "abc12345", "2024-06-01T00:00:00Z")
|
||||||
|
assert " " not in name
|
||||||
|
assert name.endswith("_abc12345.md")
|
||||||
|
|
||||||
|
def test_empty_title_becomes_untitled(self):
|
||||||
|
name = generate_filename("", "abc12345", "2024-06-01T00:00:00Z")
|
||||||
|
assert "untitled" in name
|
||||||
|
|
||||||
|
def test_id_truncated_to_8_chars(self):
|
||||||
|
name = generate_filename("Test", "abcdefghijklmnop", "2024-06-01T00:00:00Z")
|
||||||
|
assert name.endswith("_abcdefgh.md")
|
||||||
|
|
||||||
|
def test_long_title_truncated(self):
|
||||||
|
long_title = "a" * 200
|
||||||
|
name = generate_filename(long_title, "abc12345", "2024-06-01T00:00:00Z")
|
||||||
|
# Slug is capped at 60 chars by max_length
|
||||||
|
slug_part = name.split("_")[1]
|
||||||
|
assert len(slug_part) <= 60
|
||||||
|
|
||||||
|
def test_date_comes_from_created_at(self):
|
||||||
|
name = generate_filename("Test", "abc12345", "2023-11-25T00:00:00Z")
|
||||||
|
assert name.startswith("2023-11-25_")
|
||||||
|
|
||||||
|
|
||||||
|
class TestBuildExportPath:
|
||||||
|
def test_default_structure_provider_project_year(self):
|
||||||
|
path = build_export_path(
|
||||||
|
Path("/exports"), "claude", "my-project", "2024-06-01T00:00:00Z", "file.md"
|
||||||
|
)
|
||||||
|
assert str(path) == "/exports/claude/my-project.2024/file.md"
|
||||||
|
|
||||||
|
def test_no_project_uses_no_project_slug(self):
|
||||||
|
path = build_export_path(
|
||||||
|
Path("/exports"), "chatgpt", None, "2024-06-01T00:00:00Z", "file.md"
|
||||||
|
)
|
||||||
|
assert "no-project.2024" in str(path)
|
||||||
|
|
||||||
|
def test_provider_project_structure_omits_year(self):
|
||||||
|
path = build_export_path(
|
||||||
|
Path("/exports"), "claude", "proj", "2024-06-01T00:00:00Z", "file.md",
|
||||||
|
structure="provider/project",
|
||||||
|
)
|
||||||
|
assert "2024" not in str(path)
|
||||||
|
assert "proj" in str(path)
|
||||||
|
|
||||||
|
def test_provider_year_structure_omits_project(self):
|
||||||
|
path = build_export_path(
|
||||||
|
Path("/exports"), "claude", "proj", "2024-06-01T00:00:00Z", "file.md",
|
||||||
|
structure="provider/year",
|
||||||
|
)
|
||||||
|
assert "proj" not in str(path)
|
||||||
|
assert "2024" in str(path)
|
||||||
|
|
||||||
|
def test_project_name_with_spaces_is_slugified(self):
|
||||||
|
path = build_export_path(
|
||||||
|
Path("/exports"), "claude", "My Project Name!", "2024-06-01T00:00:00Z", "file.md"
|
||||||
|
)
|
||||||
|
assert " " not in str(path)
|
||||||
|
assert "!" not in str(path)
|
||||||
|
|
||||||
|
|
||||||
|
class TestRedactSecrets:
|
||||||
|
def test_token_value_redacted(self):
|
||||||
|
data = {"token": "supersecret"}
|
||||||
|
result = redact_secrets(data)
|
||||||
|
assert result["token"] == "[REDACTED]"
|
||||||
|
|
||||||
|
def test_session_key_redacted(self):
|
||||||
|
result = redact_secrets({"sessionKey": "abc123"})
|
||||||
|
assert result["sessionKey"] == "[REDACTED]"
|
||||||
|
|
||||||
|
def test_non_sensitive_key_unchanged(self):
|
||||||
|
result = redact_secrets({"title": "My Chat", "id": "abc123"})
|
||||||
|
assert result["title"] == "My Chat"
|
||||||
|
assert result["id"] == "abc123"
|
||||||
|
|
||||||
|
def test_nested_dict_redacted(self):
|
||||||
|
data = {"user": {"token": "secret", "name": "Alice"}}
|
||||||
|
result = redact_secrets(data)
|
||||||
|
assert result["user"]["token"] == "[REDACTED]"
|
||||||
|
assert result["user"]["name"] == "Alice"
|
||||||
|
|
||||||
|
def test_list_of_dicts(self):
|
||||||
|
data = [{"password": "p@ss"}, {"title": "chat"}]
|
||||||
|
result = redact_secrets(data)
|
||||||
|
assert result[0]["password"] == "[REDACTED]"
|
||||||
|
assert result[1]["title"] == "chat"
|
||||||
|
|
||||||
|
|
||||||
|
class TestFormatTokenStatus:
|
||||||
|
def test_none_token_returns_not_set(self):
|
||||||
|
assert format_token_status(None) == "[NOT SET]"
|
||||||
|
|
||||||
|
def test_empty_token_returns_not_set(self):
|
||||||
|
assert format_token_status("") == "[NOT SET]"
|
||||||
|
|
||||||
|
def test_set_token_no_expiry(self):
|
||||||
|
assert format_token_status("sometoken") == "[SET]"
|
||||||
|
|
||||||
|
def test_expired_token(self):
|
||||||
|
from datetime import datetime, timezone, timedelta
|
||||||
|
expiry = datetime.now(tz=timezone.utc) - timedelta(days=1)
|
||||||
|
result = format_token_status("tok", expiry)
|
||||||
|
assert "EXPIRED" in result
|
||||||
|
|
||||||
|
def test_expiring_today_shows_hours(self):
|
||||||
|
from datetime import datetime, timezone, timedelta
|
||||||
|
expiry = datetime.now(tz=timezone.utc) + timedelta(hours=3)
|
||||||
|
result = format_token_status("tok", expiry)
|
||||||
|
assert "expires in" in result
|
||||||
|
assert "h" in result
|
||||||
|
|
||||||
|
def test_expiring_in_days(self):
|
||||||
|
from datetime import datetime, timezone, timedelta
|
||||||
|
expiry = datetime.now(tz=timezone.utc) + timedelta(days=10, hours=12)
|
||||||
|
result = format_token_status("tok", expiry)
|
||||||
|
assert "10 days" in result
|
||||||
Reference in New Issue
Block a user