From 1f5a445ada3960a168ff90ac70a42d5871a6a7ad Mon Sep 17 00:00:00 2001 From: JesseMarkowitz Date: Mon, 6 Jul 2026 05:04:41 -0400 Subject: [PATCH] =?UTF-8?q?feat:=20v0.8.0=20=E2=80=94=20Claude=20Code=20su?= =?UTF-8?q?bagent=20capture,=20own=20Joplin=20notebook,=20git-root=20repo?= =?UTF-8?q?=20tags,=20multi-root=20scanning?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Subagents: fold Task-tool transcripts (subagents/*.jsonl) inline as collapsible
blocks at the spawn point; their own tool traffic collapses under the same policy; nesting handled via toolUseId matching - Joplin: Claude Code gets its own top-level AI-ClaudeCode notebook; update_note sets parent_id so notes self-heal/relocate on re-sync - Repo [tags] in titles via git-root detection (nearest .git ancestor), home-wide and cross-workspace; CLAUDE_CODE_REPO_TAG_IGNORE escape hatch - Multi-root scanning: CLAUDE_CODE_DIR as os.pathsep list + CLAUDE_CONFIG_DIR; merged by launch-folder, newer-mtime wins on UUID collision - 298 tests passing Co-Authored-By: Claude Opus 4.8 --- .env.example | 12 ++ CHANGELOG.md | 16 ++ FUTURE.md | 7 + README.md | 16 +- pyproject.toml | 2 +- src/blocks.py | 52 +++++ src/joplin.py | 21 +- src/main.py | 17 +- src/providers/claude_code.py | 388 ++++++++++++++++++++++++++++++----- tests/test_claude_code.py | 273 ++++++++++++++++++++++++ tests/test_exporters.py | 25 +++ tests/test_joplin.py | 21 ++ 12 files changed, 779 insertions(+), 71 deletions(-) diff --git a/.env.example b/.env.example index 164b4f4..8a11b24 100644 --- a/.env.example +++ b/.env.example @@ -26,6 +26,18 @@ CHATGPT_PROJECT_IDS= # Token type: opaque string. Typically valid for ~30 days. CLAUDE_SESSION_KEY= +# --- Claude Code (local agent sessions) --- +# The claude-code provider reads local Claude Code transcripts. By default it +# scans ~/.claude/projects/ (plus $CLAUDE_CONFIG_DIR/projects when that is set). +# To scan additional roots — e.g. other machines' sessions copied onto this box — +# set a ':'-separated list of projects roots. Sessions are merged by folder. +#CLAUDE_CODE_DIR=~/.claude/projects:/mnt/backup/laptop/.claude/projects +# +# Session titles are tagged with the git repos they touched, e.g. +# "Resume StartWRT work [start-technologies]". To never tag specific repos, +# list their names here (comma-separated). +#CLAUDE_CODE_REPO_TAG_IGNORE=some-repo,another-repo + # --- Output --- # Where exported Markdown files are written (default: ./exports) EXPORT_DIR=./exports diff --git a/CHANGELOG.md b/CHANGELOG.md index 147dac4..5cead8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,22 @@ All notable changes to this project will be documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [0.8.0] - 2026-07-06 + +Focused on the Claude Code provider after the tool's on-disk layout changed and +Claude Code sessions proved hard to find in Joplin. + +### Added +- **Subagent capture.** Claude Code now stores subagent (Task-tool) transcripts as separate `/subagents/agent-*.jsonl` files (with an `agent-*.meta.json` sidecar). These were previously invisible to the exporter — all delegated work (reviews, research, plans) was lost. Each subagent is now folded into its parent session inline at the `Task`/`Agent` call that spawned it, rendered as a collapsible `
` block labeled with its `agentType`/`description`. The subagent's own tool traffic is collapsed under the same `EXPORTER_HIDDEN_CONTENT` policy as the main dialogue; nesting is handled at any depth via `toolUseId` matching. +- **Repo tags in Claude Code titles.** Sessions launched from a workspace root all land in one folder-named notebook, so titles now carry the repos each session touched, e.g. `Resume StartWRT project work [start-technologies]` — visible in the note list and matched by Joplin search. A file's repo is the git repository it lives in (nearest ancestor with a `.git`), resolved from tool_use paths anywhere in the filesystem — so cross-workspace work is captured and non-repo noise (config dirs, one-off files, reference dirs) is excluded because it isn't a git repo. Frequency-ordered, capped at 3. Escape hatch: `CLAUDE_CODE_REPO_TAG_IGNORE` (comma-separated repo names). Tags reflect the current git layout, so a since-deleted/moved repo drops from the tag on re-export. +- **Multiple projects roots.** `CLAUDE_CODE_DIR` now accepts an `os.pathsep`-separated list of roots (a single path stays backward compatible), and `CLAUDE_CONFIG_DIR`'s `projects/` tree is scanned automatically when set. Sessions from all roots are merged by launch-folder; a session UUID present in two roots keeps the newer-mtime copy. + +### Changed +- **Claude Code gets its own top-level Joplin notebook, `AI-ClaudeCode`** (was nested under `AI-Claude` alongside Claude web chats, which made dev sessions hard to find). Existing Claude Code notes self-heal into the new notebook on the next `joplin` run — `update_note` now sets `parent_id`, so a changed provider→notebook mapping relocates notes in place instead of duplicating them. After migrating, the emptied `AI-Claude/{Myworkspace,Services,…}` notebooks can be deleted by hand (Joplin does not auto-remove empty folders). + +### Notes +- The sibling `/tool-results/*.txt` sidecars (externalized large tool outputs) are intentionally not captured — tool_result content is collapsed under the default policy anyway. + ## [0.7.0] - 2026-06-28 ### Added diff --git a/FUTURE.md b/FUTURE.md index d48a859..8aba5fb 100644 --- a/FUTURE.md +++ b/FUTURE.md @@ -1,5 +1,12 @@ # Planned Future Work +> **Status 2026-07-06 (v0.8.0): Claude Code coverage reopened and shipped.** +> Claude Code changed its on-disk layout (subagent transcripts moved to separate +> `subagents/*.jsonl` files) and its sessions were hard to find in Joplin. v0.8.0 +> addressed this: subagent capture (folded `
`), repo `[tags]` in titles, +> an own `AI-ClaudeCode` notebook with self-healing note moves, and multi-root +> scanning (`CLAUDE_CODE_DIR` list + `CLAUDE_CONFIG_DIR`). See the changelog. +> > **Status 2026-06-28: feature-complete / done for now.** As of v0.7.0 the > active roadmap is empty and the remaining backlog below has been **closed as > not needed** — the tool does what it's needed to do as a local, manually-run diff --git a/README.md b/README.md index c21aad0..1d2884b 100644 --- a/README.md +++ b/README.md @@ -218,14 +218,26 @@ The `auth` wizard can also guide you through this step interactively. ## Claude Code Sessions -The `claude-code` provider archives your local [Claude Code](https://claude.com/claude-code) agent transcripts — no tokens, no API, no ToS exposure. Sessions are read from `~/.claude/projects/` (override with `CLAUDE_CODE_DIR` in `.env`). +The `claude-code` provider archives your local [Claude Code](https://claude.com/claude-code) agent transcripts — no tokens, no API, no ToS exposure. Sessions are read from `~/.claude/projects/`. ```bash ai-chat-exporter export --provider claude-code ai-chat-exporter joplin --provider claude-code ``` -Exports are prose-only by default: your prompts and Claude's write-ups are kept, tool activity is grouped into one-line placeholders (`> 🔧 Tool output — 14 calls: Read ×9, Bash ×2 (86KB) — omitted`), and internal reasoning is dropped (counted in the run summary). Set `EXPORTER_HIDDEN_CONTENT=full` to keep everything. Each coding project becomes a Joplin notebook under the `AI-Claude` parent. The provider is included in `--provider all` whenever the sessions directory exists. +Exports are prose-only by default: your prompts and Claude's write-ups are kept, tool activity is grouped into one-line placeholders (`> 🔧 Tool output — 14 calls: Read ×9, Bash ×2 (86KB) — omitted`), and internal reasoning is dropped (counted in the run summary). Set `EXPORTER_HIDDEN_CONTENT=full` to keep everything. Sessions become notes under their own top-level **`AI-ClaudeCode`** Joplin notebook, in a sub-notebook per launch folder. The provider is included in `--provider all` whenever a sessions directory exists. + +**Subagents.** Claude Code stores subagent (Task-tool) transcripts as separate files under `/subagents/`; each is folded into its parent session inline, as a collapsible `
` block labeled with the subagent's type and description (its own tool traffic is collapsed like the main dialogue). + +**Repo tags.** Sessions launched from a workspace root all share one folder-named notebook, so titles carry the repos each session touched — `Resume StartWRT project work [start-technologies]` — for at-a-glance scanning and search. A file's repo is the git repository it lives in (nearest ancestor with a `.git`), resolved from the tool paths in the transcript, so work is tagged wherever it happened — even across workspaces — and config/one-off files are ignored (they aren't repos). Repos are frequency-ordered and capped at 3. To never tag specific repos, set `CLAUDE_CODE_REPO_TAG_IGNORE` (comma-separated). Note this reads your current git layout, so a repo you later delete or move drops from the tag on re-export. + +**Multiple locations.** By default the provider scans `~/.claude/projects/` plus `$CLAUDE_CONFIG_DIR/projects` when `CLAUDE_CONFIG_DIR` is set. To scan additional roots (e.g. other machines' sessions copied onto this box), set `CLAUDE_CODE_DIR` to a `:`-separated list: + +```bash +CLAUDE_CODE_DIR="$HOME/.claude/projects:/mnt/backup/laptop/.claude/projects" +``` + +Sessions from all roots are merged by folder (no per-machine label); if the same session UUID appears in two roots, the newer copy wins. Note the exporter only sees this machine's disk and cannot recover sessions Claude Code has already pruned — run it regularly. --- diff --git a/pyproject.toml b/pyproject.toml index 77d8081..7eec696 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "ai-chat-exporter" -version = "0.7.0" +version = "0.8.0" description = "Export ChatGPT and Claude conversation history to Markdown for personal archival in Joplin" requires-python = ">=3.11" dependencies = [ diff --git a/src/blocks.py b/src/blocks.py index 1399cf7..6d0590b 100644 --- a/src/blocks.py +++ b/src/blocks.py @@ -25,6 +25,7 @@ BLOCK_TYPE_FILE_PLACEHOLDER = "file_placeholder" BLOCK_TYPE_UNKNOWN = "unknown" BLOCK_TYPE_HIDDEN_CONTEXT_MARKER = "hidden_context_marker" BLOCK_TYPE_COLLAPSED = "collapsed" +BLOCK_TYPE_SUBAGENT = "subagent" COLLAPSED_KIND_TOOL_DUMP = "tool_dump" COLLAPSED_KIND_HIDDEN_CONTEXT = "hidden_context" @@ -36,6 +37,13 @@ UNKNOWN_REASON_UNKNOWN_FIELD_IN_KNOWN_TYPE = "unknown_field_in_known_type" _OBSERVED_KEYS_LIMIT = 10 +# Role labels for the turns rendered inside a folded subagent
block. +_SUBAGENT_ROLE_LABELS = { + "user": "🧑 Human", + "assistant": "🤖 Assistant", + "tool": "🔧 Tool", +} + # --------------------------------------------------------------------------- # Constructors @@ -193,6 +201,31 @@ def make_collapsed_block( } +def make_subagent_block( + agent_type: str, + description: str, + messages: list[dict], +) -> dict: + """A folded Claude Code subagent (Task-tool) transcript. + + Claude Code stores subagent transcripts as separate ``subagents/*.jsonl`` + files; the provider folds each one into its parent session at the point the + ``Task``/``Agent`` tool spawned it. ``messages`` is a normal list of + ``{role, blocks}`` dicts (the same shape as a conversation's messages), + extracted under the same hidden-content policy as the main dialogue — so the + subagent's own tool traffic is already collapsed inside these blocks. + + Rendered as a collapsible ``
`` element so the main conversation + stays readable while the deliverable is preserved (and stays searchable). + """ + return { + "type": BLOCK_TYPE_SUBAGENT, + "agent_type": agent_type or "", + "description": description or "", + "messages": messages or [], + } + + def make_hidden_context_marker(content_type: str) -> dict: """A short prepend block that flags the surrounding message as hidden context. @@ -322,6 +355,25 @@ def _render_one(block: dict) -> str: keys_str = ", ".join(f"`{k}`" for k in keys) lines.append(f"Keys observed: {keys_str}") return _blockquote_prefix("\n".join(lines)) + if btype == BLOCK_TYPE_SUBAGENT: + agent_type = block.get("agent_type") or "subagent" + description = block.get("description") or "" + summary = f"🤖 Subagent: {agent_type}" + if description: + summary += f" — {description}" + parts = ["
", f"{summary}", ""] + for msg in block.get("messages") or []: + body = render_blocks_to_markdown(msg.get("blocks") or []) + if not body.strip(): + continue + role = msg.get("role", "assistant") + label = _SUBAGENT_ROLE_LABELS.get(role, f"💬 {role.capitalize()}") + parts.append(f"**{label}**") + parts.append("") + parts.append(body) + parts.append("") + parts.append("
") + return "\n".join(parts) if btype == BLOCK_TYPE_HIDDEN_CONTEXT_MARKER: ctype = block.get("content_type", "") return f"> ℹ️ **Hidden context** — `{ctype}`" diff --git a/src/joplin.py b/src/joplin.py index ce9225d..32f97d3 100644 --- a/src/joplin.py +++ b/src/joplin.py @@ -178,19 +178,28 @@ class JoplinClient: logger.info("[joplin] Note created: %r → %s", title, note_id) return note_id - def update_note(self, note_id: str, title: str, body: str) -> None: + def update_note( + self, note_id: str, title: str, body: str, parent_id: str | None = None + ) -> None: """Update the title and body of an existing note. Args: note_id: Joplin note ID. title: New note title. body: New note body (Markdown). + parent_id: If given, also move the note into this notebook. Joplin's + ``PUT /notes/:id`` relocates a note when ``parent_id`` is set — + this is how notes self-heal to a new notebook mapping (e.g. the + Claude Code AI-Claude → AI-ClaudeCode migration) on re-sync. """ logger.debug( "[joplin] Updating note %s: %r (%d chars)", note_id, title, len(body), ) - self._put(f"/notes/{note_id}", {"title": title, "body": body}) + data: dict = {"title": title, "body": body} + if parent_id: + data["parent_id"] = parent_id + self._put(f"/notes/{note_id}", data) logger.info("[joplin] Note updated: %r (%s)", title, note_id) # ------------------------------------------------------------------ @@ -395,9 +404,11 @@ def upload_media_and_rewrite( _PROVIDER_DISPLAY = { "chatgpt": "AI-ChatGPT", "claude": "AI-Claude", - # Decision 2026-06-12: Claude Code coding projects nest under the same - # AI-Claude parent, alongside Claude web projects. - "claude-code": "AI-Claude", + # Decision 2026-07-06: Claude Code coding sessions get their own top-level + # notebook (was nested under AI-Claude, which made them hard to find, + # intermixed with Claude web projects and named by dev folder). Existing + # notes self-heal into here on the next sync — update_note moves them. + "claude-code": "AI-ClaudeCode", } diff --git a/src/main.py b/src/main.py index e387b22..e718972 100644 --- a/src/main.py +++ b/src/main.py @@ -925,16 +925,18 @@ def _resolve_providers(provider: str, cfg) -> list[tuple[str, object]]: try_add("claude", cfg.claude_session_key, ClaudeProvider) if provider in ("claude-code", "all"): - from src.providers.claude_code import ClaudeCodeProvider, DEFAULT_PROJECTS_DIR - cc_dir = Path(os.getenv("CLAUDE_CODE_DIR", DEFAULT_PROJECTS_DIR)).expanduser() - if cc_dir.is_dir(): + from src.providers.claude_code import ClaudeCodeProvider, resolve_roots + cc_roots = resolve_roots() + if any(r.is_dir() for r in cc_roots): result.append(( "claude-code", - ClaudeCodeProvider(projects_dir=cc_dir, hidden_content=cfg.hidden_content), + ClaudeCodeProvider(hidden_content=cfg.hidden_content), )) elif provider == "claude-code": logging.getLogger(__name__).warning( - "[claude-code] Skipping — %s not found (set CLAUDE_CODE_DIR).", cc_dir + "[claude-code] Skipping — none of these roots found (set " + "CLAUDE_CODE_DIR, ':'-separated for multiple): %s", + ", ".join(str(r) for r in cc_roots), ) return result @@ -1400,7 +1402,10 @@ def joplin(ctx: click.Context, provider: str, project_filter: str | None, dry_ru notebook_id = client.get_or_create_notebook_path(list(nb_path)) if existing_note_id: - client.update_note(existing_note_id, title, body) + # Pass notebook_id so a changed provider→notebook mapping + # (e.g. Claude Code AI-Claude → AI-ClaudeCode) relocates + # the existing note instead of leaving a stale copy. + client.update_note(existing_note_id, title, body, parent_id=notebook_id) cache_obj.mark_joplin_synced(prov_name, conv_id, existing_note_id) summary[prov_name]["updated"] += 1 else: diff --git a/src/providers/claude_code.py b/src/providers/claude_code.py index d12ce76..ef0636d 100644 --- a/src/providers/claude_code.py +++ b/src/providers/claude_code.py @@ -16,9 +16,21 @@ Record types in a session file: ``user`` / ``assistant`` carry the dialogue (Anthropic-style ``message.content`` block arrays); ``ai-title`` carries the evolving session title (last one wins); ``last-prompt``, ``file-history-snapshot``, ``attachment``, ``permission-mode``, ``system`` -are harness records and are skipped. Records flagged ``isSidechain`` are -subagent transcripts; ``isMeta`` are harness-generated user records — both -skipped. +are harness records and are skipped. ``isMeta`` records are harness-generated +user records and are skipped. + +Subagents (Task tool): Claude Code stores each subagent's transcript as a +separate ``/subagents/agent-*.jsonl`` file with an ``agent-*.meta.json`` +sidecar (``agentType``, ``description``, ``toolUseId``). ``_load_subagents`` +loads them keyed by ``toolUseId``; at the ``Task``/``Agent`` tool call that +spawned it, the subagent is folded inline as a collapsible ``
`` block +(the subagent's own records are flagged ``isSidechain`` and only processed on +this recursive pass — the top-level pass still skips sidechain records). + +Scan scope: multiple ``projects/`` roots are supported — ``CLAUDE_CODE_DIR`` may +be an ``os.pathsep``-separated list and ``CLAUDE_CONFIG_DIR``'s ``projects/`` is +included when set. Sessions from all roots are merged by launch-folder; a session +UUID present in two roots keeps the newer-mtime copy. See ``resolve_roots``. """ import json @@ -34,6 +46,7 @@ from src.blocks import ( UNKNOWN_REASON_UNKNOWN_TYPE, make_collapsed_block, make_image_placeholder, + make_subagent_block, make_text_block, make_thinking_block, make_tool_result_block, @@ -53,6 +66,47 @@ logger = logging.getLogger(__name__) DEFAULT_PROJECTS_DIR = "~/.claude/projects" +# Tool names whose call spawns a subagent (Task tool) — its separate transcript +# is folded inline. Both spellings have appeared across Claude Code versions. +_SUBAGENT_TOOL_NAMES = {"Task", "Agent"} + + +def resolve_roots(projects_dir=None) -> list[Path]: + """Resolve the ordered list of Claude Code ``projects/`` roots to scan. + + Precedence: + 1. Explicit ``projects_dir`` (a single path or a list) — used by tests. + 2. ``CLAUDE_CODE_DIR`` env, split on ``os.pathsep`` (``:``) so multiple + roots can be given; a single path (no separator) stays backward + compatible. Falls back to the default root when unset. + 3. Additionally, if ``CLAUDE_CONFIG_DIR`` is set, its ``projects/`` subdir + is appended — this is the common "non-default config dir" case. + + Roots are expanded, de-duplicated (order preserved), and returned as-is + (existence is checked by the caller / ``_scan``). Sessions from all roots are + merged by launch-folder; there is no per-source label. + """ + raw: list[str] + if projects_dir is not None: + raw = [str(p) for p in projects_dir] if isinstance(projects_dir, (list, tuple)) \ + else [str(projects_dir)] + else: + env = os.getenv("CLAUDE_CODE_DIR") + raw = env.split(os.pathsep) if env else [DEFAULT_PROJECTS_DIR] + config_dir = os.getenv("CLAUDE_CONFIG_DIR") + if config_dir: + raw.append(str(Path(config_dir) / "projects")) + + roots: list[Path] = [] + for p in raw: + p = p.strip() + if not p: + continue + path = Path(p).expanduser() + if path not in roots: + roots.append(path) + return roots + # Harness-injected tags inside user message text. Stripped so exports contain # the dialogue, not the CLI plumbing. A record that is nothing but tags # (e.g. a /model invocation) ends up empty and is skipped. @@ -88,9 +142,7 @@ class ClaudeCodeProvider(BaseProvider): hidden_content: str | None = None, ) -> None: super().__init__() - self._projects_dir = Path( - projects_dir or os.getenv("CLAUDE_CODE_DIR", DEFAULT_PROJECTS_DIR) - ).expanduser() + self._projects_dirs = resolve_roots(projects_dir) self._hidden_content = ( hidden_content if hidden_content in VALID_HIDDEN_CONTENT_POLICIES @@ -116,7 +168,9 @@ class ClaudeCodeProvider(BaseProvider): if datetime.fromisoformat(c["updated_at"]) >= since_aware ] logger.info( - "[claude-code] Found %d session(s) under %s", len(convs), self._projects_dir + "[claude-code] Found %d session(s) under %s", + len(convs), + ", ".join(str(d) for d in self._projects_dirs), ) return convs @@ -133,26 +187,16 @@ class ClaudeCodeProvider(BaseProvider): FileNotFoundError(f"No session file for id {conv_id}"), ) - records: list[dict] = [] - bad_lines = 0 - for line in path.read_text(encoding="utf-8").splitlines(): - line = line.strip() - if not line: - continue - try: - records.append(json.loads(line)) - except json.JSONDecodeError: - bad_lines += 1 - if bad_lines: - logger.warning( - "[claude-code] %s: skipped %d unparseable line(s)", path.name, bad_lines - ) + records = _parse_jsonl(path) mtime = datetime.fromtimestamp(path.stat().st_mtime, tz=timezone.utc) return { "id": conv_id, "_path": str(path), "_records": records, + # Subagent (Task-tool) transcripts, keyed by the parent tool_use id + # that spawned them, folded inline during normalization. + "_subagents": _load_subagents(path), # Listing and normalized updated_at must match, or the cache # staleness comparison would re-export every session every run. "_mtime_iso": mtime.isoformat(), @@ -164,7 +208,16 @@ class ClaudeCodeProvider(BaseProvider): conv_id = raw.get("id") or "" records: list[dict] = raw.get("_records") or [] + subagents: dict = raw.get("_subagents") or {} title = _extract_title(records) + # Append the repos this session touched, e.g. "… [repo-a, repo-b]", so + # sessions launched from a workspace root (which all land in one + # folder-named notebook) stay scannable and searchable. + launch_cwd = _extract_launch_cwd(records) + if launch_cwd: + repos = _repos_touched(records, launch_cwd) + if repos: + title = f"{title} [{', '.join(repos)}]" project = _extract_project(records, raw.get("_path")) created_at = next( (r.get("timestamp") for r in records if r.get("timestamp")), "" @@ -173,7 +226,7 @@ class ClaudeCodeProvider(BaseProvider): (r.get("timestamp") for r in reversed(records) if r.get("timestamp")), "" ) - messages = _extract_messages(records, conv_id, report, policy) + messages = _extract_messages(records, conv_id, report, policy, subagents) for _ in messages: report.record_message() report.record_conversation() @@ -194,41 +247,59 @@ class ClaudeCodeProvider(BaseProvider): # ------------------------------------------------------------------ def _scan(self) -> list[dict]: - if not self._projects_dir.is_dir(): + existing = [d for d in self._projects_dirs if d.is_dir()] + if not existing: logger.warning( - "[claude-code] Projects directory %s does not exist", self._projects_dir + "[claude-code] No projects directory exists: %s", + ", ".join(str(d) for d in self._projects_dirs), ) return [] - convs: list[dict] = [] - for proj_dir in sorted(p for p in self._projects_dir.iterdir() if p.is_dir()): - for session_file in sorted(proj_dir.glob("*.jsonl")): - try: - stat = session_file.stat() - except OSError: - continue - if stat.st_size == 0: - continue - conv_id = session_file.stem - self._path_map[conv_id] = session_file - title, project, created = _read_session_meta(session_file) - convs.append( - { - "id": conv_id, - "title": title, - "project": project, - # The --project filter and dry-run table read the - # listing dict, not the normalized conversation. - "_project_name": project, - "created_at": created, - "updated_at": datetime.fromtimestamp( - stat.st_mtime, tz=timezone.utc - ).isoformat(), - "_path": str(session_file), - "_project_dir": proj_dir.name, - } - ) - return convs + # conv_id → (mtime, conv dict). Roots are merged by launch-folder; if the + # same session UUID appears in two roots (e.g. a live dir and a backup), + # the newer-mtime copy wins so we never emit two entries for one session. + by_id: dict[str, tuple[float, dict]] = {} + for root in existing: + for proj_dir in sorted(p for p in root.iterdir() if p.is_dir()): + for session_file in sorted(proj_dir.glob("*.jsonl")): + try: + stat = session_file.stat() + except OSError: + continue + if stat.st_size == 0: + continue + conv_id = session_file.stem + prev = by_id.get(conv_id) + if prev is not None and prev[0] >= stat.st_mtime: + logger.debug( + "[claude-code] Duplicate session %s in %s; keeping newer copy", + conv_id[:8], root, + ) + continue + self._path_map[conv_id] = session_file + title, project, created = _read_session_meta(session_file) + by_id[conv_id] = ( + stat.st_mtime, + { + "id": conv_id, + "title": title, + "project": project, + # The --project filter and dry-run table read the + # listing dict, not the normalized conversation. + "_project_name": project, + "created_at": created, + "updated_at": datetime.fromtimestamp( + stat.st_mtime, tz=timezone.utc + ).isoformat(), + "_path": str(session_file), + "_project_dir": proj_dir.name, + }, + ) + # Deterministic order: by bucket dir, then session id. + return sorted( + (conv for _, conv in by_id.values()), + key=lambda c: (c["_project_dir"], c["id"]), + ) # --------------------------------------------------------------------------- @@ -315,9 +386,188 @@ def _extract_project(records: list[dict], path: str | None) -> str | None: return None +def _parse_jsonl(path: Path) -> list[dict]: + """Read a JSONL file into records, tolerating (and logging) bad lines.""" + records: list[dict] = [] + bad_lines = 0 + try: + text = path.read_text(encoding="utf-8") + except OSError as e: + logger.warning("[claude-code] Could not read %s: %s", path, e) + return records + for line in text.splitlines(): + line = line.strip() + if not line: + continue + try: + records.append(json.loads(line)) + except json.JSONDecodeError: + bad_lines += 1 + if bad_lines: + logger.warning( + "[claude-code] %s: skipped %d unparseable line(s)", path.name, bad_lines + ) + return records + + +def _load_subagents(session_path: Path) -> dict: + """Map spawning tool_use id → ``{"meta": {...}, "records": [...]}``. + + Claude Code writes subagent transcripts to + ``/subagents/agent-*.jsonl`` with an ``agent-*.meta.json`` sidecar + carrying ``toolUseId`` (which parent ``Task``/``Agent`` call spawned it). + Files without a readable meta (no id to position them) are skipped. + """ + submap: dict[str, dict] = {} + subdir = session_path.parent / session_path.stem / "subagents" + if not subdir.is_dir(): + return submap + for jf in sorted(subdir.glob("*.jsonl")): + meta_path = jf.with_suffix(".meta.json") + try: + meta = json.loads(meta_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + logger.warning( + "[claude-code] Subagent %s has no readable meta; skipping", jf.name + ) + continue + tool_id = meta.get("toolUseId") + if not tool_id: + continue + submap[tool_id] = {"meta": meta, "records": _parse_jsonl(jf)} + return submap + + +def _extract_launch_cwd(records: list[dict]) -> str | None: + """The session's working directory (constant per session; first cwd wins).""" + for rec in records: + cwd = rec.get("cwd") + if cwd: + return cwd + return None + + +# tool_use input keys that carry a file path. +_TOOL_PATH_KEYS = ("file_path", "path", "notebook_path") + +# Optional ignore-list: git repos to never tag (comma-separated names). Rarely +# needed with git-root detection (config/reference dirs are already excluded +# because they aren't git repos), but kept as an escape hatch. +def _ignored_repos() -> set[str]: + env = os.getenv("CLAUDE_CODE_REPO_TAG_IGNORE", "") + return {s.strip() for s in env.split(",") if s.strip()} + + +# dir Path → git-repo name it belongs to (or None). Process-wide; the working +# tree doesn't change under us mid-run, so caching walked dirs is safe. +_GIT_ROOT_CACHE: dict[Path, str | None] = {} +_CACHE_MISS = object() + + +def _git_root_name(path: Path, max_steps: int = 25) -> str | None: + """Name of the git repo ``path`` lives in — nearest ancestor with ``.git``. + + Walks up from ``path`` until a ``.git`` entry is found (returns that dir's + basename) or the filesystem root is reached (returns ``None``). Disk-based: + a path in no git repo, or a repo no longer on disk, yields ``None``. + """ + cur = path + for _ in range(max_steps): + cached = _GIT_ROOT_CACHE.get(cur, _CACHE_MISS) + if cached is not _CACHE_MISS: + return cached + try: + if (cur / ".git").exists(): + _GIT_ROOT_CACHE[cur] = cur.name + return cur.name + except OSError: + break + if cur.parent == cur: # filesystem root + break + cur = cur.parent + _GIT_ROOT_CACHE[path] = None + return None + + +# Absolute path-like tokens inside Bash command strings (file_path/path keys are +# matched directly). git-root resolution short-circuits on non-repo paths. +_ABS_PATH_RE = re.compile(r"/(?:[\w.\-]+/)*[\w.\-]+") + + +def _repos_touched( + records: list[dict], launch_cwd: str, cap: int = 3, ignore: set[str] | None = None +) -> list[str]: + """Repos a session touched, for the title's ``[repo-a, repo-b]`` tag. + + A file's repo is the **git repository it lives in** (nearest ancestor with a + ``.git``), resolved for every absolute path seen in a tool_use input + (``file_path``/``path``/``notebook_path`` and absolute home paths inside Bash + ``command`` strings). This works anywhere in the filesystem — not just under + the launch directory — so cross-workspace work is captured, and non-repo + noise (config dirs, one-off files, reference dirs) is excluded because it + isn't a git repo. Ordered by touch frequency, capped with a trailing ``…``. + """ + ignore = _ignored_repos() if ignore is None else ignore + counts: Counter = Counter() + seen: set[str] = set() + + def note(p) -> None: + if not isinstance(p, str) or not p: + return + if not p.startswith("/"): # resolve relative paths against the launch cwd + if not launch_cwd: + return + p = str(Path(launch_cwd) / p) + if p in seen: + return + seen.add(p) + name = _git_root_name(Path(p)) + if name and not name.startswith(".") and name not in ignore: + counts[name] += 1 + + for rec in records: + msg = rec.get("message") or {} + content = msg.get("content") + if not isinstance(content, list): + continue + for item in content: + if not isinstance(item, dict) or item.get("type") != "tool_use": + continue + inp = item.get("input") + if not isinstance(inp, dict): + continue + for key in _TOOL_PATH_KEYS: + note(inp.get(key)) + cmd = inp.get("command") + if isinstance(cmd, str): + for m in _ABS_PATH_RE.finditer(cmd): + note(m.group(0)) + + ordered = [name for name, _ in counts.most_common()] + if len(ordered) > cap: + return ordered[:cap] + ["…"] + return ordered + + def _extract_messages( - records: list[dict], conv_id: str, report: LossReport, policy: str + records: list[dict], + conv_id: str, + report: LossReport, + policy: str, + subagents: dict | None = None, + include_sidechain: bool = False, ) -> list[dict]: + """Normalize Claude Code records into messages. + + ``subagents`` maps a spawning ``Task``/``Agent`` tool_use id to its separate + transcript ``{"meta": ..., "records": ...}``; when a matching tool_use is + seen its subagent is folded inline as a subagent block (extracted + recursively under the same policy). ``include_sidechain`` is set True for + those recursive subagent passes (subagent records are flagged + ``isSidechain``); the top-level pass keeps skipping sidechain records so a + subagent is never also emitted as a stray top-level turn. + """ + subagents = subagents or {} messages: list[dict] = [] # Pending collapsed tool activity: name → call count, plus total bytes. pending_tools: Counter = Counter() @@ -351,7 +601,9 @@ def _extract_messages( for rec in records: if rec.get("type") not in ("user", "assistant"): continue - if rec.get("isSidechain") or rec.get("isMeta"): + if rec.get("isMeta"): + continue + if rec.get("isSidechain") and not include_sidechain: continue msg = rec.get("message") or {} @@ -392,14 +644,36 @@ def _extract_messages( "thinking", len(json.dumps(item, default=str)) ) elif item_type == "tool_use": - if policy == HIDDEN_CONTENT_FULL: + name = item.get("name") or "tool" + tool_id = item.get("id") + if name in _SUBAGENT_TOOL_NAMES and tool_id in subagents: + # A Task/Agent spawn: fold its separate transcript inline + # instead of collapsing it. Its own tool traffic is + # collapsed by the recursive pass under the same policy. + sub = subagents[tool_id] + meta = sub.get("meta") or {} + sub_msgs = _extract_messages( + sub.get("records") or [], + conv_id, + report, + policy, + subagents, + include_sidechain=True, + ) + blocks.append( + make_subagent_block( + agent_type=meta.get("agentType") or name, + description=meta.get("description") or "", + messages=sub_msgs, + ) + ) + elif policy == HIDDEN_CONTENT_FULL: blocks.append( make_tool_use_block( item.get("name", ""), item.get("input"), item.get("id") ) ) else: - name = item.get("name") or "tool" size = len(json.dumps(item, default=str)) local_tools[name] += 1 local_bytes += size diff --git a/tests/test_claude_code.py b/tests/test_claude_code.py index 798cfca..58f2a6e 100644 --- a/tests/test_claude_code.py +++ b/tests/test_claude_code.py @@ -228,3 +228,276 @@ class TestClaudeCodeProvider: rendered = render_blocks_to_markdown(result["messages"][1]["blocks"]) assert "> 🔧 **Tool output** — `3 calls: Read ×2, Bash ×1`" in rendered assert "omitted (EXPORTER_HIDDEN_CONTENT=full to keep)" in rendered + + +# --------------------------------------------------------------------------- +# Subagent folding (Part B) +# --------------------------------------------------------------------------- + + +def _write_subagent( + session_file, tool_use_id, records, agent_type="Explore", + description="do a thing", name="agent-x", +): + """Write a /subagents/.jsonl + .meta.json sidecar.""" + subdir = session_file.parent / session_file.stem / "subagents" + subdir.mkdir(parents=True, exist_ok=True) + (subdir / f"{name}.jsonl").write_text( + "\n".join(json.dumps(r) for r in records), encoding="utf-8" + ) + (subdir / f"{name}.meta.json").write_text( + json.dumps({ + "agentType": agent_type, + "description": description, + "toolUseId": tool_use_id, + }), + encoding="utf-8", + ) + + +def _parent_with_task(tool_use_id="toolu_sub1"): + return [ + {"type": "ai-title", "aiTitle": "Parent session"}, + { + "type": "user", "cwd": "/home/jesse/myproj", + "timestamp": "2026-05-01T10:00:00.000Z", + "message": {"role": "user", "content": "delegate the research"}, + }, + { + "type": "assistant", "timestamp": "2026-05-01T10:00:05.000Z", + "message": {"role": "assistant", "content": [ + {"type": "text", "text": "I'll delegate this."}, + {"type": "tool_use", "id": tool_use_id, "name": "Task", + "input": {"description": "research"}}, + ]}, + }, + ] + + +def _subagent_records(): + return [ + { + "type": "user", "isSidechain": True, + "timestamp": "2026-05-01T10:00:06.000Z", + "message": {"role": "user", "content": "Go research X"}, + }, + { + "type": "assistant", "isSidechain": True, + "timestamp": "2026-05-01T10:00:07.000Z", + "message": {"role": "assistant", "content": [ + {"type": "text", "text": "Here is the research result."}, + {"type": "tool_use", "id": "st1", "name": "Grep", + "input": {"pattern": "x"}}, + ]}, + }, + ] + + +class TestSubagentFold: + def _provider(self, tmp_path, policy="placeholder"): + return ClaudeCodeProvider(projects_dir=tmp_path, hidden_content=policy) + + def _normalize(self, tmp_path, policy="placeholder"): + f = _write_session(tmp_path, _parent_with_task(), name="parent-1") + _write_subagent( + f, "toolu_sub1", _subagent_records(), + agent_type="Explore", description="research X", + ) + p = self._provider(tmp_path, policy) + p.fetch_all_conversations() + return p.normalize_conversation(p.get_conversation("parent-1")) + + def test_subagent_folded_into_parent(self, tmp_path): + conv = self._normalize(tmp_path) + blocks = [b for m in conv["messages"] for b in m["blocks"]] + subs = [b for b in blocks if b["type"] == "subagent"] + assert len(subs) == 1 + assert subs[0]["agent_type"] == "Explore" + assert subs[0]["description"] == "research X" + inner_text = [ + bb.get("text") for m in subs[0]["messages"] for bb in m["blocks"] + ] + assert "Go research X" in inner_text + assert "Here is the research result." in inner_text + + def test_subagent_tools_collapsed(self, tmp_path): + conv = self._normalize(tmp_path) + sub = next( + b for m in conv["messages"] for b in m["blocks"] + if b["type"] == "subagent" + ) + inner_blocks = [bb for m in sub["messages"] for bb in m["blocks"]] + assert any(b["type"] == BLOCK_TYPE_COLLAPSED for b in inner_blocks) + assert not any(b["type"] == BLOCK_TYPE_TOOL_USE for b in inner_blocks) + collapsed = next(b for b in inner_blocks if b["type"] == BLOCK_TYPE_COLLAPSED) + assert "Grep" in collapsed["origin"] + + def test_subagent_not_listed_as_conversation(self, tmp_path): + f = _write_session(tmp_path, _parent_with_task(), name="parent-1") + _write_subagent(f, "toolu_sub1", _subagent_records()) + p = self._provider(tmp_path) + convs = p.fetch_all_conversations() + assert [c["id"] for c in convs] == ["parent-1"] + + def test_renders_as_details_block(self, tmp_path): + conv = self._normalize(tmp_path) + spawning = conv["messages"][1] # the assistant turn with the Task call + rendered = render_blocks_to_markdown(spawning["blocks"]) + assert "
" in rendered + assert "🤖 Subagent: Explore — research X" in rendered + assert "
" in rendered + assert "Here is the research result." in rendered + + def test_main_pass_still_skips_stray_sidechain(self, tmp_path): + # A sidechain record with no matching subagent file must NOT leak into + # the top-level dialogue. + records = _parent_with_task() + [ + {"type": "assistant", "isSidechain": True, + "message": {"role": "assistant", + "content": [{"type": "text", "text": "stray sidechain"}]}}, + ] + f = _write_session(tmp_path, records, name="parent-2") + _write_subagent(f, "toolu_sub1", _subagent_records()) + p = self._provider(tmp_path) + p.fetch_all_conversations() + conv = p.normalize_conversation(p.get_conversation("parent-2")) + top_text = [ + b.get("text") for m in conv["messages"] for b in m["blocks"] + ] + assert "stray sidechain" not in top_text + + +# --------------------------------------------------------------------------- +# Repo tags in title (Part C) +# --------------------------------------------------------------------------- + + +class TestRepoTags: + @staticmethod + def _mkrepo(base, name): + """Create a git repo dir (bare .git marker) and return its path.""" + (base / name / ".git").mkdir(parents=True, exist_ok=True) + return base / name + + def _title(self, tmp_path, content): + (tmp_path / "ws").mkdir(exist_ok=True) + records = [ + {"type": "ai-title", "aiTitle": "Work session"}, + {"type": "user", "cwd": str(tmp_path / "ws"), + "timestamp": "2026-05-01T10:00:00.000Z", + "message": {"role": "user", "content": "go"}}, + {"type": "assistant", + "message": {"role": "assistant", "content": content}}, + ] + _write_session(tmp_path, records, name="ws-1") + p = ClaudeCodeProvider(projects_dir=tmp_path) + p.fetch_all_conversations() + return p.normalize_conversation(p.get_conversation("ws-1"))["title"] + + @staticmethod + def _read(path): + return {"type": "tool_use", "id": str(path), "name": "Read", + "input": {"file_path": str(path)}} + + def test_repos_by_git_root_ordered_by_frequency(self, tmp_path): + a = self._mkrepo(tmp_path, "repo-a") + b = self._mkrepo(tmp_path, "repo-b") + content = [ + self._read(a / "src/x.py"), self._read(a / "src/y.py"), + self._read(b / "z.py"), + # a file not inside any git repo → excluded + self._read(tmp_path / "CLAUDE.md"), + ] + assert self._title(tmp_path, content) == "Work session [repo-a, repo-b]" + + def test_cross_workspace_repo_tagged(self, tmp_path): + # Launched in ws/, but touched a repo in a sibling tree — still tagged. + other = self._mkrepo(tmp_path / "elsewhere", "faraway-repo") + content = [self._read(other / "deep/nested/file.py")] + assert self._title(tmp_path, content) == "Work session [faraway-repo]" + + def test_no_repos_no_bracket(self, tmp_path): + content = [self._read(tmp_path / "loose/file.txt")] # no .git anywhere + assert self._title(tmp_path, content) == "Work session" + + def test_cap_three_with_ellipsis(self, tmp_path): + content = [ + self._read(self._mkrepo(tmp_path, f"repo-{c}") / "f/x.py") + for c in "abcd" + ] + title = self._title(tmp_path, content) + assert title.startswith("Work session [repo-a, repo-b, repo-c, …]") + assert "repo-d" not in title + + def test_paths_inside_bash_commands_count(self, tmp_path): + x = self._mkrepo(tmp_path, "repo-x") + content = [ + {"type": "tool_use", "id": "a", "name": "Bash", + "input": {"command": f"cat {x / 'main.go'}"}}, + ] + assert self._title(tmp_path, content) == "Work session [repo-x]" + + def test_ignore_list_via_env(self, tmp_path, monkeypatch): + a = self._mkrepo(tmp_path, "repo-a") + b = self._mkrepo(tmp_path, "repo-b") + monkeypatch.setenv("CLAUDE_CODE_REPO_TAG_IGNORE", "repo-b, notes") + content = [self._read(a / "x.py"), self._read(b / "y.py")] + assert self._title(tmp_path, content) == "Work session [repo-a]" + + +# --------------------------------------------------------------------------- +# Multiple projects roots (Part D) +# --------------------------------------------------------------------------- + + +class TestMultiRoot: + def _titled(self, aititle, cwd="/home/jesse/p"): + return [ + {"type": "ai-title", "aiTitle": aititle}, + {"type": "user", "cwd": cwd, "timestamp": "2026-05-01T10:00:00.000Z", + "message": {"role": "user", "content": "hi"}}, + ] + + def test_two_roots_merged(self, tmp_path): + r1, r2 = tmp_path / "root1", tmp_path / "root2" + _write_session(r1, self._titled("From root1"), name="s1") + _write_session(r2, self._titled("From root2"), name="s2") + p = ClaudeCodeProvider(projects_dir=[r1, r2]) + ids = sorted(c["id"] for c in p.fetch_all_conversations()) + assert ids == ["s1", "s2"] + + def test_duplicate_uuid_newer_wins(self, tmp_path): + import os + import time + r1, r2 = tmp_path / "root1", tmp_path / "root2" + _write_session(r1, self._titled("Old copy"), name="dup") + f2 = _write_session(r2, self._titled("New copy"), name="dup") + os.utime(f2, (time.time() + 10, time.time() + 10)) + p = ClaudeCodeProvider(projects_dir=[r1, r2]) + convs = p.fetch_all_conversations() + assert len(convs) == 1 + assert convs[0]["title"] == "New copy" + assert p._path_map["dup"] == f2 + + def test_single_path_backward_compatible(self, tmp_path): + _write_session(tmp_path, _records()) + p = ClaudeCodeProvider(projects_dir=tmp_path) + assert len(p.fetch_all_conversations()) == 1 + + def test_env_pathsep_list(self, tmp_path, monkeypatch): + import os + from src.providers.claude_code import resolve_roots + r1, r2 = tmp_path / "root1", tmp_path / "root2" + _write_session(r1, self._titled("From root1"), name="s1") + _write_session(r2, self._titled("From root2"), name="s2") + monkeypatch.setenv("CLAUDE_CODE_DIR", f"{r1}{os.pathsep}{r2}") + assert r1 in resolve_roots() and r2 in resolve_roots() + p = ClaudeCodeProvider() + assert len(p.fetch_all_conversations()) == 2 + + def test_config_dir_projects_included(self, tmp_path, monkeypatch): + from src.providers.claude_code import resolve_roots + monkeypatch.delenv("CLAUDE_CODE_DIR", raising=False) + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(tmp_path / "cfg")) + roots = resolve_roots() + assert (tmp_path / "cfg" / "projects") in roots diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 8708bc2..d906885 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -17,6 +17,7 @@ from src.blocks import ( make_file_placeholder, make_hidden_context_marker, make_image_placeholder, + make_subagent_block, make_text_block, make_thinking_block, make_tool_result_block, @@ -119,6 +120,30 @@ class TestMarkdownFrontmatter: assert "```python" in content assert "print('hello')" in content + def test_subagent_block_renders_as_details(self, tmp_path): + sub = make_subagent_block( + agent_type="Explore", + description="find the thing", + messages=[ + {"role": "user", "blocks": [make_text_block("Go find it")]}, + {"role": "assistant", "blocks": [make_text_block("Found it here.")]}, + ], + ) + conv = { + **SAMPLE_CONV, + "provider": "claude-code", + "messages": [ + {"role": "assistant", "content_type": "text", "timestamp": None, + "blocks": [make_text_block("Delegating."), sub]}, + ], + } + content = MarkdownExporter(tmp_path).export(conv).read_text() + assert "
" in content + assert "🤖 Subagent: Explore — find the thing" in content + assert "Go find it" in content + assert "Found it here." in content + assert "
" in content + class TestMarkdownFilenameGeneration: def test_filename_format(self, tmp_path): diff --git a/tests/test_joplin.py b/tests/test_joplin.py index c480f12..2ee9c76 100644 --- a/tests/test_joplin.py +++ b/tests/test_joplin.py @@ -48,6 +48,9 @@ class TestNotebookPath: def test_claude_provider(self): assert notebook_path("claude", "budget-tracker") == ("AI-Claude", "Budget Tracker") + def test_claude_code_gets_own_top_level_notebook(self): + assert notebook_path("claude-code", "services") == ("AI-ClaudeCode", "Services") + def test_multi_word_project(self): assert notebook_path("claude", "ai-research-notes") == ("AI-Claude", "Ai Research Notes") @@ -61,6 +64,24 @@ class TestNotebookPath: # --------------------------------------------------------------------------- +class TestUpdateNote: + def test_parent_id_moves_note(self): + client = _make_client() + client._put = MagicMock() + client.update_note("nid", "Title", "body", parent_id="nb1") + args, _ = client._put.call_args + assert args[0] == "/notes/nid" + assert args[1]["parent_id"] == "nb1" + assert args[1]["title"] == "Title" + + def test_no_parent_id_omits_field(self): + client = _make_client() + client._put = MagicMock() + client.update_note("nid", "Title", "body") + args, _ = client._put.call_args + assert "parent_id" not in args[1] + + class TestPing: def test_ping_success(self): client = _make_client()