Jesse.Markowitz/AIChatExporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: v0.8.0 — Claude Code subagent capture, own Joplin notebook, git-root repo tags, multi-root scanning

Browse Source 
- Subagents: fold Task-tool transcripts (subagents/*.jsonl) inline as
  collapsible <details> blocks at the spawn point; their own tool traffic
  collapses under the same policy; nesting handled via toolUseId matching
- Joplin: Claude Code gets its own top-level AI-ClaudeCode notebook;
  update_note sets parent_id so notes self-heal/relocate on re-sync
- Repo [tags] in titles via git-root detection (nearest .git ancestor),
  home-wide and cross-workspace; CLAUDE_CODE_REPO_TAG_IGNORE escape hatch
- Multi-root scanning: CLAUDE_CODE_DIR as os.pathsep list + CLAUDE_CONFIG_DIR;
  merged by launch-folder, newer-mtime wins on UUID collision
- 298 tests passing

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
JesseMarkowitz
2026-07-06 05:04:41 -04:00
co-authored by Claude Opus 4.8
parent bbcb29c856
commit 1f5a445ada
12 changed files with 779 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/providers/claude_code.py
+331 -57
View File
@@ -16,9 +16,21 @@ Record types in a session file: ``user`` / ``assistant`` carry the dialogue
(Anthropic-style ``message.content`` block arrays); ``ai-title`` carries the
evolving session title (last one wins); ``last-prompt``,
``file-history-snapshot``, ``attachment``, ``permission-mode``, ``system``
are harness records and are skipped. Records flagged ``isSidechain`` are
subagent transcripts; ``isMeta`` are harness-generated user records — both
skipped.
are harness records and are skipped. ``isMeta`` records are harness-generated
user records and are skipped.
Subagents (Task tool): Claude Code stores each subagent's transcript as a
separate ``<session>/subagents/agent-*.jsonl`` file with an ``agent-*.meta.json``
sidecar (``agentType``, ``description``, ``toolUseId``). ``_load_subagents``
loads them keyed by ``toolUseId``; at the ``Task``/``Agent`` tool call that
spawned it, the subagent is folded inline as a collapsible ``<details>`` block
(the subagent's own records are flagged ``isSidechain`` and only processed on
this recursive pass — the top-level pass still skips sidechain records).
Scan scope: multiple ``projects/`` roots are supported — ``CLAUDE_CODE_DIR`` may
be an ``os.pathsep``-separated list and ``CLAUDE_CONFIG_DIR``'s ``projects/`` is
included when set. Sessions from all roots are merged by launch-folder; a session
UUID present in two roots keeps the newer-mtime copy. See ``resolve_roots``.
"""
import json
@@ -34,6 +46,7 @@ from src.blocks import (
UNKNOWN_REASON_UNKNOWN_TYPE,
make_collapsed_block,
make_image_placeholder,
make_subagent_block,
make_text_block,
make_thinking_block,
make_tool_result_block,
@@ -53,6 +66,47 @@ logger = logging.getLogger(__name__)
DEFAULT_PROJECTS_DIR = "~/.claude/projects"
# Tool names whose call spawns a subagent (Task tool) — its separate transcript
# is folded inline. Both spellings have appeared across Claude Code versions.
_SUBAGENT_TOOL_NAMES = {"Task", "Agent"}
def resolve_roots(projects_dir=None) -> list[Path]:
"""Resolve the ordered list of Claude Code ``projects/`` roots to scan.
Precedence:
1. Explicit ``projects_dir`` (a single path or a list) — used by tests.
2. ``CLAUDE_CODE_DIR`` env, split on ``os.pathsep`` (``:``) so multiple
roots can be given; a single path (no separator) stays backward
compatible. Falls back to the default root when unset.
3. Additionally, if ``CLAUDE_CONFIG_DIR`` is set, its ``projects/`` subdir
is appended — this is the common "non-default config dir" case.
Roots are expanded, de-duplicated (order preserved), and returned as-is
(existence is checked by the caller / ``_scan``). Sessions from all roots are
merged by launch-folder; there is no per-source label.
"""
raw: list[str]
if projects_dir is not None:
raw = [str(p) for p in projects_dir] if isinstance(projects_dir, (list, tuple)) \
else [str(projects_dir)]
else:
env = os.getenv("CLAUDE_CODE_DIR")
raw = env.split(os.pathsep) if env else [DEFAULT_PROJECTS_DIR]
config_dir = os.getenv("CLAUDE_CONFIG_DIR")
if config_dir:
raw.append(str(Path(config_dir) / "projects"))
roots: list[Path] = []
for p in raw:
p = p.strip()
if not p:
continue
path = Path(p).expanduser()
if path not in roots:
roots.append(path)
return roots
# Harness-injected tags inside user message text. Stripped so exports contain
# the dialogue, not the CLI plumbing. A record that is nothing but tags
# (e.g. a /model invocation) ends up empty and is skipped.
@@ -88,9 +142,7 @@ class ClaudeCodeProvider(BaseProvider):
hidden_content: str | None = None,
) -> None:
super().__init__()
self._projects_dir = Path(
projects_dir or os.getenv("CLAUDE_CODE_DIR", DEFAULT_PROJECTS_DIR)
).expanduser()
self._projects_dirs = resolve_roots(projects_dir)
self._hidden_content = (
hidden_content
if hidden_content in VALID_HIDDEN_CONTENT_POLICIES
@@ -116,7 +168,9 @@ class ClaudeCodeProvider(BaseProvider):
if datetime.fromisoformat(c["updated_at"]) >= since_aware
]
logger.info(
"[claude-code] Found %d session(s) under %s", len(convs), self._projects_dir
"[claude-code] Found %d session(s) under %s",
len(convs),
", ".join(str(d) for d in self._projects_dirs),
)
return convs
@@ -133,26 +187,16 @@ class ClaudeCodeProvider(BaseProvider):
FileNotFoundError(f"No session file for id {conv_id}"),
)
records: list[dict] = []
bad_lines = 0
for line in path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line:
continue
try:
records.append(json.loads(line))
except json.JSONDecodeError:
bad_lines += 1
if bad_lines:
logger.warning(
"[claude-code] %s: skipped %d unparseable line(s)", path.name, bad_lines
)
records = _parse_jsonl(path)
mtime = datetime.fromtimestamp(path.stat().st_mtime, tz=timezone.utc)
return {
"id": conv_id,
"_path": str(path),
"_records": records,
# Subagent (Task-tool) transcripts, keyed by the parent tool_use id
# that spawned them, folded inline during normalization.
"_subagents": _load_subagents(path),
# Listing and normalized updated_at must match, or the cache
# staleness comparison would re-export every session every run.
"_mtime_iso": mtime.isoformat(),
@@ -164,7 +208,16 @@ class ClaudeCodeProvider(BaseProvider):
conv_id = raw.get("id") or ""
records: list[dict] = raw.get("_records") or []
subagents: dict = raw.get("_subagents") or {}
title = _extract_title(records)
# Append the repos this session touched, e.g. "… [repo-a, repo-b]", so
# sessions launched from a workspace root (which all land in one
# folder-named notebook) stay scannable and searchable.
launch_cwd = _extract_launch_cwd(records)
if launch_cwd:
repos = _repos_touched(records, launch_cwd)
if repos:
title = f"{title} [{', '.join(repos)}]"
project = _extract_project(records, raw.get("_path"))
created_at = next(
(r.get("timestamp") for r in records if r.get("timestamp")), ""
@@ -173,7 +226,7 @@ class ClaudeCodeProvider(BaseProvider):
(r.get("timestamp") for r in reversed(records) if r.get("timestamp")), ""
)
messages = _extract_messages(records, conv_id, report, policy)
messages = _extract_messages(records, conv_id, report, policy, subagents)
for _ in messages:
report.record_message()
report.record_conversation()
@@ -194,41 +247,59 @@ class ClaudeCodeProvider(BaseProvider):
# ------------------------------------------------------------------
def _scan(self) -> list[dict]:
if not self._projects_dir.is_dir():
existing = [d for d in self._projects_dirs if d.is_dir()]
if not existing:
logger.warning(
"[claude-code] Projects directory %s does not exist", self._projects_dir
"[claude-code] No projects directory exists: %s",
", ".join(str(d) for d in self._projects_dirs),
)
return []
convs: list[dict] = []
for proj_dir in sorted(p for p in self._projects_dir.iterdir() if p.is_dir()):
for session_file in sorted(proj_dir.glob("*.jsonl")):
try:
stat = session_file.stat()
except OSError:
continue
if stat.st_size == 0:
continue
conv_id = session_file.stem
self._path_map[conv_id] = session_file
title, project, created = _read_session_meta(session_file)
convs.append(
{
"id": conv_id,
"title": title,
"project": project,
# The --project filter and dry-run table read the
# listing dict, not the normalized conversation.
"_project_name": project,
"created_at": created,
"updated_at": datetime.fromtimestamp(
stat.st_mtime, tz=timezone.utc
).isoformat(),
"_path": str(session_file),
"_project_dir": proj_dir.name,
}
)
return convs
# conv_id → (mtime, conv dict). Roots are merged by launch-folder; if the
# same session UUID appears in two roots (e.g. a live dir and a backup),
# the newer-mtime copy wins so we never emit two entries for one session.
by_id: dict[str, tuple[float, dict]] = {}
for root in existing:
for proj_dir in sorted(p for p in root.iterdir() if p.is_dir()):
for session_file in sorted(proj_dir.glob("*.jsonl")):
try:
stat = session_file.stat()
except OSError:
continue
if stat.st_size == 0:
continue
conv_id = session_file.stem
prev = by_id.get(conv_id)
if prev is not None and prev[0] >= stat.st_mtime:
logger.debug(
"[claude-code] Duplicate session %s in %s; keeping newer copy",
conv_id[:8], root,
)
continue
self._path_map[conv_id] = session_file
title, project, created = _read_session_meta(session_file)
by_id[conv_id] = (
stat.st_mtime,
{
"id": conv_id,
"title": title,
"project": project,
# The --project filter and dry-run table read the
# listing dict, not the normalized conversation.
"_project_name": project,
"created_at": created,
"updated_at": datetime.fromtimestamp(
stat.st_mtime, tz=timezone.utc
).isoformat(),
"_path": str(session_file),
"_project_dir": proj_dir.name,
},
)
# Deterministic order: by bucket dir, then session id.
return sorted(
(conv for _, conv in by_id.values()),
key=lambda c: (c["_project_dir"], c["id"]),
)
# ---------------------------------------------------------------------------
@@ -315,9 +386,188 @@ def _extract_project(records: list[dict], path: str | None) -> str | None:
return None
def _parse_jsonl(path: Path) -> list[dict]:
"""Read a JSONL file into records, tolerating (and logging) bad lines."""
records: list[dict] = []
bad_lines = 0
try:
text = path.read_text(encoding="utf-8")
except OSError as e:
logger.warning("[claude-code] Could not read %s: %s", path, e)
return records
for line in text.splitlines():
line = line.strip()
if not line:
continue
try:
records.append(json.loads(line))
except json.JSONDecodeError:
bad_lines += 1
if bad_lines:
logger.warning(
"[claude-code] %s: skipped %d unparseable line(s)", path.name, bad_lines
)
return records
def _load_subagents(session_path: Path) -> dict:
"""Map spawning tool_use id → ``{"meta": {...}, "records": [...]}``.
Claude Code writes subagent transcripts to
``<session>/subagents/agent-*.jsonl`` with an ``agent-*.meta.json`` sidecar
carrying ``toolUseId`` (which parent ``Task``/``Agent`` call spawned it).
Files without a readable meta (no id to position them) are skipped.
"""
submap: dict[str, dict] = {}
subdir = session_path.parent / session_path.stem / "subagents"
if not subdir.is_dir():
return submap
for jf in sorted(subdir.glob("*.jsonl")):
meta_path = jf.with_suffix(".meta.json")
try:
meta = json.loads(meta_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
logger.warning(
"[claude-code] Subagent %s has no readable meta; skipping", jf.name
)
continue
tool_id = meta.get("toolUseId")
if not tool_id:
continue
submap[tool_id] = {"meta": meta, "records": _parse_jsonl(jf)}
return submap
def _extract_launch_cwd(records: list[dict]) -> str | None:
"""The session's working directory (constant per session; first cwd wins)."""
for rec in records:
cwd = rec.get("cwd")
if cwd:
return cwd
return None
# tool_use input keys that carry a file path.
_TOOL_PATH_KEYS = ("file_path", "path", "notebook_path")
# Optional ignore-list: git repos to never tag (comma-separated names). Rarely
# needed with git-root detection (config/reference dirs are already excluded
# because they aren't git repos), but kept as an escape hatch.
def _ignored_repos() -> set[str]:
env = os.getenv("CLAUDE_CODE_REPO_TAG_IGNORE", "")
return {s.strip() for s in env.split(",") if s.strip()}
# dir Path → git-repo name it belongs to (or None). Process-wide; the working
# tree doesn't change under us mid-run, so caching walked dirs is safe.
_GIT_ROOT_CACHE: dict[Path, str | None] = {}
_CACHE_MISS = object()
def _git_root_name(path: Path, max_steps: int = 25) -> str | None:
"""Name of the git repo ``path`` lives in — nearest ancestor with ``.git``.
Walks up from ``path`` until a ``.git`` entry is found (returns that dir's
basename) or the filesystem root is reached (returns ``None``). Disk-based:
a path in no git repo, or a repo no longer on disk, yields ``None``.
"""
cur = path
for _ in range(max_steps):
cached = _GIT_ROOT_CACHE.get(cur, _CACHE_MISS)
if cached is not _CACHE_MISS:
return cached
try:
if (cur / ".git").exists():
_GIT_ROOT_CACHE[cur] = cur.name
return cur.name
except OSError:
break
if cur.parent == cur: # filesystem root
break
cur = cur.parent
_GIT_ROOT_CACHE[path] = None
return None
# Absolute path-like tokens inside Bash command strings (file_path/path keys are
# matched directly). git-root resolution short-circuits on non-repo paths.
_ABS_PATH_RE = re.compile(r"/(?:[\w.\-]+/)*[\w.\-]+")
def _repos_touched(
records: list[dict], launch_cwd: str, cap: int = 3, ignore: set[str] | None = None
) -> list[str]:
"""Repos a session touched, for the title's ``[repo-a, repo-b]`` tag.
A file's repo is the **git repository it lives in** (nearest ancestor with a
``.git``), resolved for every absolute path seen in a tool_use input
(``file_path``/``path``/``notebook_path`` and absolute home paths inside Bash
``command`` strings). This works anywhere in the filesystem — not just under
the launch directory — so cross-workspace work is captured, and non-repo
noise (config dirs, one-off files, reference dirs) is excluded because it
isn't a git repo. Ordered by touch frequency, capped with a trailing ``…``.
"""
ignore = _ignored_repos() if ignore is None else ignore
counts: Counter = Counter()
seen: set[str] = set()
def note(p) -> None:
if not isinstance(p, str) or not p:
return
if not p.startswith("/"): # resolve relative paths against the launch cwd
if not launch_cwd:
return
p = str(Path(launch_cwd) / p)
if p in seen:
return
seen.add(p)
name = _git_root_name(Path(p))
if name and not name.startswith(".") and name not in ignore:
counts[name] += 1
for rec in records:
msg = rec.get("message") or {}
content = msg.get("content")
if not isinstance(content, list):
continue
for item in content:
if not isinstance(item, dict) or item.get("type") != "tool_use":
continue
inp = item.get("input")
if not isinstance(inp, dict):
continue
for key in _TOOL_PATH_KEYS:
note(inp.get(key))
cmd = inp.get("command")
if isinstance(cmd, str):
for m in _ABS_PATH_RE.finditer(cmd):
note(m.group(0))
ordered = [name for name, _ in counts.most_common()]
if len(ordered) > cap:
return ordered[:cap] + ["…"]
return ordered
def _extract_messages(
records: list[dict], conv_id: str, report: LossReport, policy: str
records: list[dict],
conv_id: str,
report: LossReport,
policy: str,
subagents: dict | None = None,
include_sidechain: bool = False,
) -> list[dict]:
"""Normalize Claude Code records into messages.
``subagents`` maps a spawning ``Task``/``Agent`` tool_use id to its separate
transcript ``{"meta": ..., "records": ...}``; when a matching tool_use is
seen its subagent is folded inline as a subagent block (extracted
recursively under the same policy). ``include_sidechain`` is set True for
those recursive subagent passes (subagent records are flagged
``isSidechain``); the top-level pass keeps skipping sidechain records so a
subagent is never also emitted as a stray top-level turn.
"""
subagents = subagents or {}
messages: list[dict] = []
# Pending collapsed tool activity: name → call count, plus total bytes.
pending_tools: Counter = Counter()
@@ -351,7 +601,9 @@ def _extract_messages(
for rec in records:
if rec.get("type") not in ("user", "assistant"):
continue
if rec.get("isSidechain") or rec.get("isMeta"):
if rec.get("isMeta"):
continue
if rec.get("isSidechain") and not include_sidechain:
continue
msg = rec.get("message") or {}
@@ -392,14 +644,36 @@ def _extract_messages(
"thinking", len(json.dumps(item, default=str))
)
elif item_type == "tool_use":
if policy == HIDDEN_CONTENT_FULL:
name = item.get("name") or "tool"
tool_id = item.get("id")
if name in _SUBAGENT_TOOL_NAMES and tool_id in subagents:
# A Task/Agent spawn: fold its separate transcript inline
# instead of collapsing it. Its own tool traffic is
# collapsed by the recursive pass under the same policy.
sub = subagents[tool_id]
meta = sub.get("meta") or {}
sub_msgs = _extract_messages(
sub.get("records") or [],
conv_id,
report,
policy,
subagents,
include_sidechain=True,
)
blocks.append(
make_subagent_block(
agent_type=meta.get("agentType") or name,
description=meta.get("description") or "",
messages=sub_msgs,
)
)
elif policy == HIDDEN_CONTENT_FULL:
blocks.append(
make_tool_use_block(
item.get("name", ""), item.get("input"), item.get("id")
)
)
else:
name = item.get("name") or "tool"
size = len(json.dumps(item, default=str))
local_tools[name] += 1
local_bytes += size