feat: v0.6.0 — collapse policy, session limiter, Claude Code provider, prune, browser auth, media downloads
This commit is contained in:
JesseMarkowitz
@@ -0,0 +1,476 @@
|
||||
"""Claude Code session provider — archives local agent transcripts.
|
||||
|
||||
Reads JSONL session files from ``~/.claude/projects/<munged-cwd>/<uuid>.jsonl``
|
||||
(override with ``CLAUDE_CODE_DIR``). No tokens, no rate limits, no ToS risk —
|
||||
but the data lives in single files Claude Code may clean up, and it contains
|
||||
deliverables (reviews, plans, analyses) that exist nowhere else.
|
||||
|
||||
Rendering follows the EXPORTER_HIDDEN_CONTENT policy (decided 2026-06-12):
|
||||
prose-only by default. User prompts and assistant text are kept; tool_use /
|
||||
tool_result traffic is collapsed to one grouped placeholder per activity run
|
||||
(measured: dialogue prose is ~4% of session bytes); thinking blocks are
|
||||
dropped (counted in the run summary, no placeholder). ``full`` keeps
|
||||
everything.
|
||||
|
||||
Record types in a session file: ``user`` / ``assistant`` carry the dialogue
|
||||
(Anthropic-style ``message.content`` block arrays); ``ai-title`` carries the
|
||||
evolving session title (last one wins); ``last-prompt``,
|
||||
``file-history-snapshot``, ``attachment``, ``permission-mode``, ``system``
|
||||
are harness records and are skipped. Records flagged ``isSidechain`` are
|
||||
subagent transcripts; ``isMeta`` are harness-generated user records — both
|
||||
skipped.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from collections import Counter
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from src.blocks import (
|
||||
COLLAPSED_KIND_TOOL_DUMP,
|
||||
UNKNOWN_REASON_UNKNOWN_TYPE,
|
||||
make_collapsed_block,
|
||||
make_image_placeholder,
|
||||
make_text_block,
|
||||
make_thinking_block,
|
||||
make_tool_result_block,
|
||||
make_tool_use_block,
|
||||
make_unknown_block,
|
||||
)
|
||||
from src.loss_report import LossReport
|
||||
from src.providers.base import (
|
||||
BaseProvider,
|
||||
HIDDEN_CONTENT_FULL,
|
||||
ProviderError,
|
||||
VALID_HIDDEN_CONTENT_POLICIES,
|
||||
resolve_hidden_content_policy,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_PROJECTS_DIR = "~/.claude/projects"
|
||||
|
||||
# Harness-injected tags inside user message text. Stripped so exports contain
|
||||
# the dialogue, not the CLI plumbing. A record that is nothing but tags
|
||||
# (e.g. a /model invocation) ends up empty and is skipped.
|
||||
_HARNESS_TAG_RE = re.compile(
|
||||
r"<local-command-caveat>.*?</local-command-caveat>"
|
||||
r"|<command-name>.*?</command-name>"
|
||||
r"|<command-message>.*?</command-message>"
|
||||
r"|<command-args>.*?</command-args>"
|
||||
r"|<command-contents>.*?</command-contents>"
|
||||
r"|<local-command-stdout>.*?</local-command-stdout>"
|
||||
r"|<system-reminder>.*?</system-reminder>",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
# How many distinct tool names to list in a collapsed-activity placeholder.
|
||||
_TOOL_NAMES_SHOWN = 4
|
||||
|
||||
|
||||
def _strip_harness_noise(text: str) -> str:
|
||||
if not isinstance(text, str):
|
||||
return ""
|
||||
return _HARNESS_TAG_RE.sub("", text).strip()
|
||||
|
||||
|
||||
class ClaudeCodeProvider(BaseProvider):
|
||||
"""Local-file provider over Claude Code session transcripts."""
|
||||
|
||||
provider_name = "claude-code"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
projects_dir: str | Path | None = None,
|
||||
hidden_content: str | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self._projects_dir = Path(
|
||||
projects_dir or os.getenv("CLAUDE_CODE_DIR", DEFAULT_PROJECTS_DIR)
|
||||
).expanduser()
|
||||
self._hidden_content = (
|
||||
hidden_content
|
||||
if hidden_content in VALID_HIDDEN_CONTENT_POLICIES
|
||||
else resolve_hidden_content_policy()
|
||||
)
|
||||
# conv_id → session file path, populated by _scan()
|
||||
self._path_map: dict[str, Path] = {}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# BaseProvider interface
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def list_conversations(self, offset: int = 0, limit: int = 100) -> list[dict]:
|
||||
full = self._scan()
|
||||
return full[offset : offset + limit]
|
||||
|
||||
def fetch_all_conversations(self, since: datetime | None = None) -> list[dict]:
|
||||
convs = self._scan()
|
||||
if since is not None:
|
||||
since_aware = since if since.tzinfo else since.replace(tzinfo=timezone.utc)
|
||||
convs = [
|
||||
c for c in convs
|
||||
if datetime.fromisoformat(c["updated_at"]) >= since_aware
|
||||
]
|
||||
logger.info(
|
||||
"[claude-code] Found %d session(s) under %s", len(convs), self._projects_dir
|
||||
)
|
||||
return convs
|
||||
|
||||
def get_conversation(self, conv_id: str) -> dict:
|
||||
path = self._path_map.get(conv_id)
|
||||
if path is None:
|
||||
# Direct call without a prior listing (e.g. tests) — scan first.
|
||||
self._scan()
|
||||
path = self._path_map.get(conv_id)
|
||||
if path is None or not path.exists():
|
||||
raise ProviderError(
|
||||
self.provider_name,
|
||||
f"get_conversation({conv_id[:8]})",
|
||||
FileNotFoundError(f"No session file for id {conv_id}"),
|
||||
)
|
||||
|
||||
records: list[dict] = []
|
||||
bad_lines = 0
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
records.append(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
bad_lines += 1
|
||||
if bad_lines:
|
||||
logger.warning(
|
||||
"[claude-code] %s: skipped %d unparseable line(s)", path.name, bad_lines
|
||||
)
|
||||
|
||||
mtime = datetime.fromtimestamp(path.stat().st_mtime, tz=timezone.utc)
|
||||
return {
|
||||
"id": conv_id,
|
||||
"_path": str(path),
|
||||
"_records": records,
|
||||
# Listing and normalized updated_at must match, or the cache
|
||||
# staleness comparison would re-export every session every run.
|
||||
"_mtime_iso": mtime.isoformat(),
|
||||
}
|
||||
|
||||
def normalize_conversation(self, raw: dict, loss_report: LossReport | None = None) -> dict:
|
||||
report = loss_report if loss_report is not None else LossReport()
|
||||
policy = getattr(self, "_hidden_content", None) or resolve_hidden_content_policy()
|
||||
conv_id = raw.get("id") or ""
|
||||
records: list[dict] = raw.get("_records") or []
|
||||
|
||||
title = _extract_title(records)
|
||||
project = _extract_project(records, raw.get("_path"))
|
||||
created_at = next(
|
||||
(r.get("timestamp") for r in records if r.get("timestamp")), ""
|
||||
)
|
||||
updated_at = raw.get("_mtime_iso") or next(
|
||||
(r.get("timestamp") for r in reversed(records) if r.get("timestamp")), ""
|
||||
)
|
||||
|
||||
messages = _extract_messages(records, conv_id, report, policy)
|
||||
for _ in messages:
|
||||
report.record_message()
|
||||
report.record_conversation()
|
||||
|
||||
return {
|
||||
"id": conv_id,
|
||||
"title": title,
|
||||
"provider": self.provider_name,
|
||||
"project": project,
|
||||
"created_at": created_at or "",
|
||||
"updated_at": updated_at or "",
|
||||
"message_count": len(messages),
|
||||
"messages": messages,
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Scanning
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _scan(self) -> list[dict]:
|
||||
if not self._projects_dir.is_dir():
|
||||
logger.warning(
|
||||
"[claude-code] Projects directory %s does not exist", self._projects_dir
|
||||
)
|
||||
return []
|
||||
|
||||
convs: list[dict] = []
|
||||
for proj_dir in sorted(p for p in self._projects_dir.iterdir() if p.is_dir()):
|
||||
for session_file in sorted(proj_dir.glob("*.jsonl")):
|
||||
try:
|
||||
stat = session_file.stat()
|
||||
except OSError:
|
||||
continue
|
||||
if stat.st_size == 0:
|
||||
continue
|
||||
conv_id = session_file.stem
|
||||
self._path_map[conv_id] = session_file
|
||||
title, project, created = _read_session_meta(session_file)
|
||||
convs.append(
|
||||
{
|
||||
"id": conv_id,
|
||||
"title": title,
|
||||
"project": project,
|
||||
# The --project filter and dry-run table read the
|
||||
# listing dict, not the normalized conversation.
|
||||
"_project_name": project,
|
||||
"created_at": created,
|
||||
"updated_at": datetime.fromtimestamp(
|
||||
stat.st_mtime, tz=timezone.utc
|
||||
).isoformat(),
|
||||
"_path": str(session_file),
|
||||
"_project_dir": proj_dir.name,
|
||||
}
|
||||
)
|
||||
return convs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _read_session_meta(path: Path) -> tuple[str, str | None, str]:
|
||||
"""Light single-pass scan for listing metadata: (title, project, created_at).
|
||||
|
||||
Substring guards keep this cheap — only candidate lines are JSON-parsed.
|
||||
The full-fidelity extraction happens later in normalize_conversation.
|
||||
"""
|
||||
title = ""
|
||||
project: str | None = None
|
||||
created = ""
|
||||
try:
|
||||
with path.open(encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
if '"ai-title"' in line:
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if rec.get("type") == "ai-title" and rec.get("aiTitle"):
|
||||
title = str(rec["aiTitle"]) # last one wins
|
||||
continue
|
||||
if (not created or project is None) and (
|
||||
'"timestamp"' in line or '"cwd"' in line
|
||||
):
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if not created and rec.get("timestamp"):
|
||||
created = str(rec["timestamp"])
|
||||
if project is None and rec.get("cwd"):
|
||||
project = Path(rec["cwd"]).name or None
|
||||
except OSError as e:
|
||||
logger.warning("[claude-code] Could not read %s: %s", path, e)
|
||||
return title, project, created
|
||||
|
||||
|
||||
def _extract_title(records: list[dict]) -> str:
|
||||
"""Last ai-title record wins; fall back to the first real user prompt."""
|
||||
title = ""
|
||||
for rec in records:
|
||||
if rec.get("type") == "ai-title" and rec.get("aiTitle"):
|
||||
title = str(rec["aiTitle"])
|
||||
if title:
|
||||
return title
|
||||
|
||||
for rec in records:
|
||||
if rec.get("type") != "user" or rec.get("isMeta") or rec.get("isSidechain"):
|
||||
continue
|
||||
content = (rec.get("message") or {}).get("content")
|
||||
if isinstance(content, str):
|
||||
text = _strip_harness_noise(content)
|
||||
elif isinstance(content, list):
|
||||
text = " ".join(
|
||||
_strip_harness_noise(item.get("text", ""))
|
||||
for item in content
|
||||
if isinstance(item, dict) and item.get("type") == "text"
|
||||
).strip()
|
||||
else:
|
||||
text = ""
|
||||
if text:
|
||||
return text[:80]
|
||||
return "Untitled session"
|
||||
|
||||
|
||||
def _extract_project(records: list[dict], path: str | None) -> str | None:
|
||||
"""Project = basename of the session's working directory."""
|
||||
for rec in records:
|
||||
cwd = rec.get("cwd")
|
||||
if cwd:
|
||||
name = Path(cwd).name
|
||||
if name:
|
||||
return name
|
||||
# Fallback: the munged directory name (cannot be reliably de-munged
|
||||
# because '-' is both the path separator and a legal name character).
|
||||
if path:
|
||||
return Path(path).parent.name.lstrip("-") or None
|
||||
return None
|
||||
|
||||
|
||||
def _extract_messages(
|
||||
records: list[dict], conv_id: str, report: LossReport, policy: str
|
||||
) -> list[dict]:
|
||||
messages: list[dict] = []
|
||||
# Pending collapsed tool activity: name → call count, plus total bytes.
|
||||
pending_tools: Counter = Counter()
|
||||
pending_bytes = 0
|
||||
|
||||
def flush_pending() -> None:
|
||||
nonlocal pending_bytes
|
||||
if not pending_tools:
|
||||
return
|
||||
shown = ", ".join(
|
||||
f"{name} ×{count}" for name, count in pending_tools.most_common(_TOOL_NAMES_SHOWN)
|
||||
)
|
||||
if len(pending_tools) > _TOOL_NAMES_SHOWN:
|
||||
shown += ", …"
|
||||
calls = sum(pending_tools.values())
|
||||
block = make_collapsed_block(
|
||||
origin=f"{calls} calls: {shown}",
|
||||
content_type="tool_activity",
|
||||
size_bytes=pending_bytes,
|
||||
kind=COLLAPSED_KIND_TOOL_DUMP,
|
||||
)
|
||||
if messages and messages[-1]["role"] == "assistant":
|
||||
messages[-1]["blocks"].append(block)
|
||||
else:
|
||||
messages.append(
|
||||
{"role": "tool", "content_type": "text", "timestamp": None, "blocks": [block]}
|
||||
)
|
||||
pending_tools.clear()
|
||||
pending_bytes = 0
|
||||
|
||||
for rec in records:
|
||||
if rec.get("type") not in ("user", "assistant"):
|
||||
continue
|
||||
if rec.get("isSidechain") or rec.get("isMeta"):
|
||||
continue
|
||||
|
||||
msg = rec.get("message") or {}
|
||||
role = msg.get("role") or rec["type"]
|
||||
content = msg.get("content")
|
||||
blocks: list[dict] = []
|
||||
# This record's own tool traffic — merged into pending only after the
|
||||
# record's message is appended, so the placeholder lands on it (not on
|
||||
# the previous message).
|
||||
local_tools: Counter = Counter()
|
||||
local_bytes = 0
|
||||
|
||||
if isinstance(content, str):
|
||||
text = _strip_harness_noise(content)
|
||||
block = make_text_block(text)
|
||||
if block:
|
||||
blocks.append(block)
|
||||
elif isinstance(content, list):
|
||||
for item in content:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
item_type = item.get("type", "")
|
||||
if item_type == "text":
|
||||
block = make_text_block(_strip_harness_noise(item.get("text", "")))
|
||||
if block:
|
||||
blocks.append(block)
|
||||
elif item_type in ("thinking", "redacted_thinking"):
|
||||
if policy == HIDDEN_CONTENT_FULL:
|
||||
block = make_thinking_block(
|
||||
item.get("thinking") or item.get("text") or ""
|
||||
)
|
||||
if block:
|
||||
blocks.append(block)
|
||||
else:
|
||||
# Decision 2026-06-12: thinking is dropped without a
|
||||
# placeholder, but stays visible in the run summary.
|
||||
report.record_collapsed(
|
||||
"thinking", len(json.dumps(item, default=str))
|
||||
)
|
||||
elif item_type == "tool_use":
|
||||
if policy == HIDDEN_CONTENT_FULL:
|
||||
blocks.append(
|
||||
make_tool_use_block(
|
||||
item.get("name", ""), item.get("input"), item.get("id")
|
||||
)
|
||||
)
|
||||
else:
|
||||
name = item.get("name") or "tool"
|
||||
size = len(json.dumps(item, default=str))
|
||||
local_tools[name] += 1
|
||||
local_bytes += size
|
||||
report.record_collapsed(name, size)
|
||||
elif item_type == "tool_result":
|
||||
if policy == HIDDEN_CONTENT_FULL:
|
||||
blocks.append(
|
||||
make_tool_result_block(
|
||||
_stringify_tool_result(item.get("content")),
|
||||
is_error=bool(item.get("is_error")),
|
||||
)
|
||||
)
|
||||
else:
|
||||
size = len(json.dumps(item, default=str))
|
||||
local_bytes += size
|
||||
report.record_collapsed("tool_result", size)
|
||||
elif item_type == "image":
|
||||
blocks.append(
|
||||
make_image_placeholder(ref="embedded image", source="user_upload")
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"[claude-code] Unknown content block type %r in session %s",
|
||||
item_type,
|
||||
conv_id[:8],
|
||||
)
|
||||
report.record_unknown(f"claude-code.{item_type or '?'}")
|
||||
blocks.append(
|
||||
make_unknown_block(
|
||||
raw_type=f"claude-code.{item_type or '?'}",
|
||||
observed_keys=list(item.keys()),
|
||||
reason=UNKNOWN_REASON_UNKNOWN_TYPE,
|
||||
)
|
||||
)
|
||||
|
||||
if not blocks:
|
||||
# Tool-result-only records (and similar): traffic accumulates and
|
||||
# is attached to the message that initiated it.
|
||||
pending_tools.update(local_tools)
|
||||
pending_bytes += local_bytes
|
||||
continue
|
||||
|
||||
# Attach previous records' tool activity to the previous message
|
||||
# before starting a new one, so the placeholder lands between the
|
||||
# dialogue turns it actually occurred between.
|
||||
flush_pending()
|
||||
messages.append(
|
||||
{
|
||||
"role": role,
|
||||
"content_type": "text",
|
||||
"timestamp": rec.get("timestamp"),
|
||||
"blocks": blocks,
|
||||
}
|
||||
)
|
||||
pending_tools.update(local_tools)
|
||||
pending_bytes += local_bytes
|
||||
|
||||
flush_pending()
|
||||
return messages
|
||||
|
||||
|
||||
def _stringify_tool_result(content) -> str:
|
||||
"""tool_result content may be a string or a list of text blocks."""
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts = []
|
||||
for item in content:
|
||||
if isinstance(item, dict) and item.get("type") == "text":
|
||||
parts.append(item.get("text", ""))
|
||||
else:
|
||||
parts.append(json.dumps(item, default=str))
|
||||
return "\n".join(parts)
|
||||
return json.dumps(content, default=str) if content is not None else ""
|
||||
Reference in New Issue
Block a user