385 lines
15 KiB
Python
385 lines
15 KiB
Python
"""Local cache manifest for tracking exported and Joplin-synced conversations."""
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import shutil
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
MANIFEST_VERSION = 1
|
|
|
|
|
|
class CacheError(Exception):
|
|
"""Raised when the cache manifest cannot be loaded or written."""
|
|
|
|
|
|
class Cache:
|
|
"""Manages the local JSON manifest of exported and Joplin-synced conversations.
|
|
|
|
The manifest is the single source of truth for what has been exported and
|
|
synced. Every export run compares the provider's full conversation list
|
|
against this manifest to determine what is new or updated. The Joplin sync
|
|
run reads it to find conversations not yet pushed to Joplin (or re-exported
|
|
since the last sync).
|
|
|
|
Each entry tracks:
|
|
title, project, updated_at, exported_at, file_path,
|
|
joplin_note_id (after first sync), joplin_synced_at (after first sync)
|
|
|
|
File security:
|
|
- Permissions: 600 (owner read/write only)
|
|
- Atomic writes: .tmp → os.replace()
|
|
- chmod happens BEFORE os.replace() to prevent permission race windows
|
|
"""
|
|
|
|
def __init__(self, cache_dir: Path | str) -> None:
|
|
self._dir = Path(cache_dir).expanduser()
|
|
self._dir.mkdir(parents=True, exist_ok=True)
|
|
self._path = self._dir / "manifest.json"
|
|
self._data: dict[str, Any] = self._load()
|
|
|
|
# ------------------------------------------------------------------
|
|
# Public API
|
|
# ------------------------------------------------------------------
|
|
|
|
def is_cached(self, provider: str, conv_id: str, updated_at: str) -> bool:
|
|
"""Return True if this conversation is already exported and up to date.
|
|
|
|
A conversation is considered stale if the provider's ``updated_at``
|
|
is newer than the cached ``exported_at``.
|
|
"""
|
|
entry = self._data.get(provider, {}).get(conv_id)
|
|
if entry is None:
|
|
logger.debug("[cache] MISS %s/%s", provider, conv_id[:8])
|
|
return False
|
|
|
|
cached_updated = entry.get("updated_at", "")
|
|
if not cached_updated or not updated_at:
|
|
logger.debug("[cache] HIT (no date comparison) %s/%s", provider, conv_id[:8])
|
|
return True
|
|
|
|
try:
|
|
from src.utils import _parse_dt
|
|
cached_dt = _parse_dt(cached_updated)
|
|
provider_dt = _parse_dt(updated_at)
|
|
if provider_dt > cached_dt:
|
|
logger.debug("[cache] STALE %s/%s", provider, conv_id[:8])
|
|
return False
|
|
except Exception:
|
|
pass
|
|
|
|
logger.debug("[cache] HIT %s/%s", provider, conv_id[:8])
|
|
return True
|
|
|
|
def mark_exported(self, provider: str, conv_id: str, metadata: dict) -> None:
|
|
"""Record a successfully exported conversation in the manifest.
|
|
|
|
Writes to disk immediately (not batched) so progress is preserved
|
|
if the process is interrupted.
|
|
"""
|
|
if provider not in self._data:
|
|
self._data[provider] = {}
|
|
|
|
# Preserve the Joplin link across re-exports. Without this, re-rendering
|
|
# a conversation (e.g. a forced re-export to pick up new formatting)
|
|
# would drop joplin_note_id, and the next sync would create a duplicate
|
|
# note instead of updating the existing one. exported_at is refreshed,
|
|
# so get_joplin_pending() still flags the note for an update.
|
|
prev = self._data[provider].get(conv_id) or {}
|
|
entry = {
|
|
"title": metadata.get("title", ""),
|
|
"project": metadata.get("project"),
|
|
"created_at": metadata.get("created_at", ""),
|
|
"updated_at": metadata.get("updated_at", ""),
|
|
"exported_at": datetime.now(tz=timezone.utc).isoformat(),
|
|
"file_path": metadata.get("file_path", ""),
|
|
}
|
|
for carried in ("joplin_note_id", "joplin_synced_at", "joplin_resources"):
|
|
if carried in prev:
|
|
entry[carried] = prev[carried]
|
|
|
|
self._data[provider][conv_id] = entry
|
|
self._data["last_run"] = datetime.now(tz=timezone.utc).isoformat()
|
|
self._save()
|
|
|
|
def get_new_or_updated(
|
|
self,
|
|
provider: str,
|
|
conversations: list[dict],
|
|
force: bool = False,
|
|
campaign_at: str | None = None,
|
|
) -> list[dict]:
|
|
"""Filter a conversation list to only new or updated conversations.
|
|
|
|
Args:
|
|
provider: "chatgpt" or "claude"
|
|
conversations: List of raw conversation dicts from the provider.
|
|
Each must have an ``id``/``uuid`` and ``updated_at``/``update_time``.
|
|
force: When True, return conversations to re-render regardless of
|
|
cache freshness — used by ``export --force``.
|
|
campaign_at: In force mode, exclude conversations already re-exported
|
|
at/after this ISO timestamp (the re-render campaign start), so
|
|
each run advances and finished providers do nothing.
|
|
|
|
Returns:
|
|
Subset that needs to be exported, ordered oldest-export-first in
|
|
force mode.
|
|
"""
|
|
if force:
|
|
# Re-export for a re-render campaign, ordered least-recently-exported
|
|
# first (never-exported sorts first). When ``campaign_at`` is given,
|
|
# conversations already re-exported during this campaign
|
|
# (exported_at >= campaign_at) are excluded — so finished providers
|
|
# do no work and the remaining count shrinks to zero. Each capped run
|
|
# re-renders the next-oldest batch. Conversations without an id drop.
|
|
entries = self._data.get(provider, {})
|
|
|
|
def _exported_at(conv: dict) -> str:
|
|
conv_id = conv.get("id") or conv.get("uuid", "")
|
|
entry = entries.get(conv_id) or {}
|
|
return entry.get("exported_at") or ""
|
|
|
|
candidates = [c for c in conversations if c.get("id") or c.get("uuid")]
|
|
if campaign_at is not None:
|
|
candidates = [c for c in candidates if _exported_at(c) < campaign_at]
|
|
return sorted(candidates, key=_exported_at)
|
|
|
|
result = []
|
|
for conv in conversations:
|
|
conv_id = conv.get("id") or conv.get("uuid", "")
|
|
updated_at = conv.get("updated_at") or conv.get("update_time") or ""
|
|
if conv_id and not self.is_cached(provider, conv_id, updated_at):
|
|
result.append(conv)
|
|
return result
|
|
|
|
def stats(self) -> dict[str, int]:
|
|
"""Return count of cached conversations per provider."""
|
|
return {
|
|
provider: len(entries)
|
|
for provider, entries in self._data.items()
|
|
if isinstance(entries, dict) and provider not in ("version", "last_run", "tos_acknowledged_at")
|
|
}
|
|
|
|
def clear(self, provider: str | None = None) -> None:
|
|
"""Clear cached entries.
|
|
|
|
Args:
|
|
provider: If given, clear only that provider. If None, clear all.
|
|
"""
|
|
if provider:
|
|
cleared = len(self._data.get(provider, {}))
|
|
self._data[provider] = {}
|
|
logger.info("[cache] Cleared %d entries for provider '%s'", cleared, provider)
|
|
else:
|
|
for key in list(self._data.keys()):
|
|
if isinstance(self._data[key], dict) and key not in (
|
|
"version",
|
|
"last_run",
|
|
"tos_acknowledged_at",
|
|
):
|
|
self._data[key] = {}
|
|
logger.info("[cache] Cleared all provider entries")
|
|
self._save()
|
|
|
|
def is_tos_acknowledged(self) -> bool:
|
|
"""Return True if the user has acknowledged the ToS notice."""
|
|
return bool(self._data.get("tos_acknowledged_at"))
|
|
|
|
def acknowledge_tos(self) -> None:
|
|
"""Record ToS acknowledgement with a timestamp."""
|
|
self._data["tos_acknowledged_at"] = datetime.now(tz=timezone.utc).isoformat()
|
|
self._save()
|
|
logger.info("[cache] ToS acknowledged at %s", self._data["tos_acknowledged_at"])
|
|
|
|
def get_all_entries(self, provider: str) -> dict:
|
|
"""Return all cached entries for a provider (for --cache --show)."""
|
|
return dict(self._data.get(provider, {}))
|
|
|
|
def mark_joplin_synced(self, provider: str, conv_id: str, note_id: str) -> None:
|
|
"""Record a successful Joplin sync for a conversation.
|
|
|
|
Adds ``joplin_note_id`` and ``joplin_synced_at`` to the manifest entry
|
|
and writes atomically to disk.
|
|
"""
|
|
entry = self._data.get(provider, {}).get(conv_id)
|
|
if entry is None:
|
|
logger.warning(
|
|
"[cache] mark_joplin_synced: no cache entry for %s/%s", provider, conv_id[:8]
|
|
)
|
|
return
|
|
entry["joplin_note_id"] = note_id
|
|
entry["joplin_synced_at"] = datetime.now(tz=timezone.utc).isoformat()
|
|
self._save()
|
|
|
|
def get_joplin_resources(self, provider: str, conv_id: str) -> dict[str, str]:
|
|
"""Return the {relative_media_path: joplin_resource_id} map for a conversation."""
|
|
entry = self._data.get(provider, {}).get(conv_id)
|
|
if not isinstance(entry, dict):
|
|
return {}
|
|
resources = entry.get("joplin_resources")
|
|
return dict(resources) if isinstance(resources, dict) else {}
|
|
|
|
def set_joplin_resources(self, provider: str, conv_id: str, resources: dict[str, str]) -> None:
|
|
"""Persist the {relative_media_path: joplin_resource_id} map for a conversation."""
|
|
entry = self._data.get(provider, {}).get(conv_id)
|
|
if entry is None:
|
|
return
|
|
entry["joplin_resources"] = dict(resources)
|
|
self._save()
|
|
|
|
def get_joplin_pending(self, provider: str) -> list[tuple[str, dict]]:
|
|
"""Return (conv_id, entry) pairs that need to be synced to Joplin.
|
|
|
|
A conversation is pending when:
|
|
- It has never been synced (no ``joplin_note_id``), OR
|
|
- It was re-exported after the last Joplin sync
|
|
(``exported_at`` > ``joplin_synced_at``).
|
|
|
|
Returns:
|
|
List of (conv_id, entry_dict) tuples, where entry_dict includes
|
|
``file_path``, ``title``, ``project``, and optionally ``joplin_note_id``.
|
|
"""
|
|
pending = []
|
|
for conv_id, entry in self._data.get(provider, {}).items():
|
|
if not isinstance(entry, dict):
|
|
continue
|
|
if not entry.get("file_path"):
|
|
continue
|
|
|
|
note_id = entry.get("joplin_note_id")
|
|
if not note_id:
|
|
pending.append((conv_id, entry))
|
|
continue
|
|
|
|
# Re-sync if the file was re-exported after the last Joplin sync
|
|
exported_at = entry.get("exported_at", "")
|
|
synced_at = entry.get("joplin_synced_at", "")
|
|
if exported_at and synced_at:
|
|
try:
|
|
from src.utils import _parse_dt
|
|
if _parse_dt(exported_at) > _parse_dt(synced_at):
|
|
pending.append((conv_id, entry))
|
|
except Exception:
|
|
pass
|
|
|
|
return pending
|
|
|
|
def all_file_paths(self) -> set[str]:
|
|
"""Return every non-empty ``file_path`` in the manifest, across providers.
|
|
|
|
Used by ``prune`` (files on disk but not here are stale) and by the
|
|
doctor integrity check (files here but not on disk are missing).
|
|
"""
|
|
paths: set[str] = set()
|
|
for key, entries in self._data.items():
|
|
if not isinstance(entries, dict) or key in (
|
|
"version",
|
|
"last_run",
|
|
"tos_acknowledged_at",
|
|
):
|
|
continue
|
|
for entry in entries.values():
|
|
if isinstance(entry, dict) and entry.get("file_path"):
|
|
paths.add(entry["file_path"])
|
|
return paths
|
|
|
|
def last_run(self) -> str | None:
|
|
"""Return the ISO8601 timestamp of the last export run, or None."""
|
|
return self._data.get("last_run")
|
|
|
|
def exported_at(self, provider: str, conv_id: str) -> str:
|
|
"""Return the recorded ``exported_at`` for a conversation, or '' if absent."""
|
|
entry = self._data.get(provider, {}).get(conv_id)
|
|
return entry.get("exported_at", "") if isinstance(entry, dict) else ""
|
|
|
|
# Force re-render campaign marker (top-level scalar; skipped by stats/clear,
|
|
# which only act on dict-valued provider keys).
|
|
def get_force_campaign(self) -> str | None:
|
|
"""Return the active force-re-render campaign start timestamp, or None."""
|
|
return self._data.get("force_campaign_at")
|
|
|
|
def set_force_campaign(self, started_at: str) -> None:
|
|
self._data["force_campaign_at"] = started_at
|
|
self._save()
|
|
|
|
def clear_force_campaign(self) -> None:
|
|
if self._data.pop("force_campaign_at", None) is not None:
|
|
self._save()
|
|
|
|
# ------------------------------------------------------------------
|
|
# Private helpers
|
|
# ------------------------------------------------------------------
|
|
|
|
def _load(self) -> dict[str, Any]:
|
|
"""Load the manifest from disk. Returns a fresh manifest if absent."""
|
|
if not self._path.exists():
|
|
return self._fresh_manifest()
|
|
|
|
try:
|
|
text = self._path.read_text(encoding="utf-8")
|
|
data = json.loads(text)
|
|
except (json.JSONDecodeError, OSError) as e:
|
|
backup = self._path.with_suffix(".json.bak")
|
|
logger.warning(
|
|
"[cache] Manifest file is invalid (%s). "
|
|
"Backing up to %s and starting fresh.",
|
|
e,
|
|
backup,
|
|
)
|
|
try:
|
|
shutil.copy2(self._path, backup)
|
|
except OSError:
|
|
pass
|
|
return self._fresh_manifest()
|
|
|
|
# Version check
|
|
version = data.get("version")
|
|
if version is None:
|
|
logger.warning(
|
|
"[cache] Manifest has no 'version' field — treating as v1."
|
|
)
|
|
data["version"] = MANIFEST_VERSION
|
|
elif version > MANIFEST_VERSION:
|
|
logger.critical(
|
|
"[cache] Manifest version %d is newer than supported version %d. "
|
|
"Please update ai-chat-exporter, or delete %s to start fresh.",
|
|
version,
|
|
MANIFEST_VERSION,
|
|
self._path,
|
|
)
|
|
raise CacheError(
|
|
f"Unsupported manifest version {version}. "
|
|
"Update the tool or delete the manifest to start fresh."
|
|
)
|
|
|
|
return data
|
|
|
|
def _save(self) -> None:
|
|
"""Atomically write the manifest to disk with 600 permissions."""
|
|
tmp = self._path.with_suffix(".json.tmp")
|
|
try:
|
|
tmp.write_text(
|
|
json.dumps(self._data, indent=2, default=str),
|
|
encoding="utf-8",
|
|
)
|
|
# chmod BEFORE os.replace() to prevent permission race window
|
|
os.chmod(tmp, 0o600)
|
|
os.replace(tmp, self._path)
|
|
except OSError as e:
|
|
logger.error("[cache] Failed to write manifest: %s", e)
|
|
raise CacheError(f"Cannot write cache manifest to {self._path}: {e}") from e
|
|
|
|
@staticmethod
|
|
def _fresh_manifest() -> dict[str, Any]:
|
|
return {
|
|
"version": MANIFEST_VERSION,
|
|
"tos_acknowledged_at": None,
|
|
"last_run": None,
|
|
"chatgpt": {},
|
|
"claude": {},
|
|
}
|