diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b176fb..7d96b05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,10 @@ consolidated here. - Session limiter: `--max-conversations N` / `MAX_CONVERSATIONS_PER_RUN` cap downloads per run (per provider); capped-out conversations are reported as "deferred" and resumed next run. - Polite pacing: `REQUEST_DELAY` (default 1.0s, ±25% jitter, `0` disables) between consecutive API requests, with the existing 429 backoff as the reactive net. +**Re-rendering** +- `export --force` re-exports every conversation even if cached and unchanged, so the whole archive can be re-rendered after a formatting/feature change without `cache --clear`. Combines with `--max-conversations` to spread the work across runs. +- `mark_exported` now preserves `joplin_note_id` / `joplin_synced_at` / `joplin_resources` across re-exports (refreshing `exported_at`), so a re-export followed by `joplin` updates the existing notes instead of creating duplicates. + **Archive hygiene** - `prune` deletes export files not referenced by the manifest (old-layout trees, no-ID orphans), with listing, confirmation, `--dry-run`, `-y`, and empty-directory sweep. Refuses to run when the manifest references no files (so `cache --clear` + `prune` can't wipe the archive). First live run removed 420 stale files (9.4 MB). - `doctor` verifies manifest ↔ disk integrity: every recorded `file_path` must exist on disk. diff --git a/FUTURE.md b/FUTURE.md index e95dc52..d86c8b2 100644 --- a/FUTURE.md +++ b/FUTURE.md @@ -301,16 +301,12 @@ loop. Kept for reference; not on the active roadmap. -## Export `--force` Flag +## Export `--force` Flag — SHIPPED v0.6.0 -Add `--force` to the `export` command to re-export already-cached conversations -without permanently clearing the entire manifest. Useful for re-generating files -after changing the Markdown template or output structure. - -Implementation: pass a `force=True` flag to `cache.get_new_or_updated()`, which -returns all conversations regardless of cache state when force is True. - -Current workaround: `python -m src.main cache --clear` then re-run export. +Implemented 2026-06-12: `export --force` passes `force=True` to +`cache.get_new_or_updated()`. Shipped alongside a `mark_exported` fix that +preserves Joplin links across re-exports, so a forced re-render + `joplin` +updates existing notes instead of duplicating them. ## Joplin `--force` Flag diff --git a/README.md b/README.md index b421b15..3bb37e4 100644 --- a/README.md +++ b/README.md @@ -329,7 +329,14 @@ ai-chat-exporter export --output /path/to/my/notes ai-chat-exporter export --dry-run ``` -Options: `--provider [chatgpt|claude|claude-code|all]`, `--format [markdown|json|both]`, `--output PATH`, `--since YYYY-MM-DD`, `--project NAME`, `--hidden-content [full|placeholder|omit]`, `--download-media [images|all|off]`, `--max-conversations N`, `--dry-run` +Options: `--provider [chatgpt|claude|claude-code|all]`, `--format [markdown|json|both]`, `--output PATH`, `--since YYYY-MM-DD`, `--project NAME`, `--hidden-content [full|placeholder|omit]`, `--download-media [images|all|off]`, `--max-conversations N`, `--force`, `--dry-run` + +**Re-rendering the whole archive after an upgrade.** New formatting or features (collapse policy, media downloads) only change conversations as they're re-exported. To re-render everything you already have, use `--force` — it re-exports every conversation even if unchanged, **without** `cache --clear`, so your Joplin note links are preserved (a later `joplin` run updates the existing notes instead of duplicating them). Spread the load with `--max-conversations`: + +```bash +ai-chat-exporter export --force --max-conversations 50 # run repeatedly until 0 deferred +ai-chat-exporter joplin # update notes + upload media +``` ### `list` — List conversations diff --git a/src/cache.py b/src/cache.py index 5d6ca6c..d461e63 100644 --- a/src/cache.py +++ b/src/cache.py @@ -84,7 +84,13 @@ class Cache: if provider not in self._data: self._data[provider] = {} - self._data[provider][conv_id] = { + # Preserve the Joplin link across re-exports. Without this, re-rendering + # a conversation (e.g. a forced re-export to pick up new formatting) + # would drop joplin_note_id, and the next sync would create a duplicate + # note instead of updating the existing one. exported_at is refreshed, + # so get_joplin_pending() still flags the note for an update. + prev = self._data[provider].get(conv_id) or {} + entry = { "title": metadata.get("title", ""), "project": metadata.get("project"), "created_at": metadata.get("created_at", ""), @@ -92,20 +98,48 @@ class Cache: "exported_at": datetime.now(tz=timezone.utc).isoformat(), "file_path": metadata.get("file_path", ""), } + for carried in ("joplin_note_id", "joplin_synced_at", "joplin_resources"): + if carried in prev: + entry[carried] = prev[carried] + + self._data[provider][conv_id] = entry self._data["last_run"] = datetime.now(tz=timezone.utc).isoformat() self._save() - def get_new_or_updated(self, provider: str, conversations: list[dict]) -> list[dict]: + def get_new_or_updated( + self, provider: str, conversations: list[dict], force: bool = False + ) -> list[dict]: """Filter a conversation list to only new or updated conversations. Args: provider: "chatgpt" or "claude" conversations: List of raw conversation dicts from the provider. Each must have an ``id``/``uuid`` and ``updated_at``/``update_time``. + force: When True, return every conversation regardless of cache + state — used by ``export --force`` to re-render everything + without clearing the manifest (which would drop Joplin links). Returns: Subset that needs to be exported. """ + if force: + # Re-export everything, but order by least-recently-exported first + # (never-exported sorts first). With a per-run cap, the batch just + # re-rendered gets a fresh exported_at and sinks to the back, so + # each subsequent run advances through the archive instead of + # repeating the same head. Conversations without an id are dropped. + entries = self._data.get(provider, {}) + + def _exported_at(conv: dict) -> str: + conv_id = conv.get("id") or conv.get("uuid", "") + entry = entries.get(conv_id) or {} + return entry.get("exported_at") or "" + + return sorted( + (c for c in conversations if c.get("id") or c.get("uuid")), + key=_exported_at, + ) + result = [] for conv in conversations: conv_id = conv.get("id") or conv.get("uuid", "") diff --git a/src/main.py b/src/main.py index 6534c21..603dda9 100644 --- a/src/main.py +++ b/src/main.py @@ -62,6 +62,14 @@ def cli(ctx: click.Context, verbose: bool, quiet: bool, debug: bool, no_log_file """Export ChatGPT and Claude conversations to Markdown for personal archival.""" ctx.ensure_object(dict) + # Load .env BEFORE reading any env var (LOG_FILE, CACHE_DIR). Without this + # the cache/log paths fell back to defaults regardless of .env — the cache + # in particular silently lived at ./cache even when CACHE_DIR was set. + # override=False so a real shell env var still wins over .env. + import os + from dotenv import load_dotenv + load_dotenv(override=False) + # Determine console log level if debug or verbose: level = logging.DEBUG @@ -71,7 +79,6 @@ def cli(ctx: click.Context, verbose: bool, quiet: bool, debug: bool, no_log_file level = logging.INFO # Determine log file path from env (setup_logging handles "none") - import os log_file = os.getenv("LOG_FILE", "./cache/logs/exporter.log") setup_logging(level=level, log_file=log_file, no_log_file=no_log_file) @@ -634,6 +641,16 @@ def _print_doctor_table(checks: list[dict]) -> None: "(default: images)." ), ) +@click.option( + "--force", + is_flag=True, + help=( + "Re-export every conversation even if already cached and unchanged. " + "Use to re-render the whole archive after a formatting/feature change " + "without 'cache --clear' (which would drop Joplin note links). Combine " + "with --max-conversations to spread the work over several runs." + ), +) @click.option("--dry-run", is_flag=True, help="Show what would be exported without writing anything.") @click.pass_context def export( @@ -646,6 +663,7 @@ def export( hidden_content: str | None, max_conversations: int | None, download_media: str | None, + force: bool, dry_run: bool, ) -> None: """Export new and updated conversations to Markdown or JSON. @@ -736,7 +754,7 @@ def export( f" [dim]--project filter '{project_filter}': {len(all_convs)} matching conversations.[/dim]" ) - to_export = cache.get_new_or_updated(prov_name, all_convs) + to_export = cache.get_new_or_updated(prov_name, all_convs, force=force) skipped = len(all_convs) - len(to_export) summary[prov_name]["skipped"] = skipped @@ -841,10 +859,22 @@ def export( _print_export_summary(summary) total_deferred = sum(s.get("deferred", 0) for s in summary.values()) if total_deferred: - console.print( - f"[yellow]{total_deferred} conversation(s) deferred by the session cap " - f"({session_cap} per provider per run). Re-run the same command to continue.[/yellow]" - ) + if force: + # In force mode every conversation is "to_export", so the + # deferred count is a rolling remainder, not a shrinking + # backlog: each run re-renders the oldest batch and advances. + exported_now = sum(s.get("exported", 0) for s in summary.values()) + console.print( + f"[yellow]Force re-render: did the {exported_now} least-recently-exported " + f"this run; {total_deferred} still awaiting a refresh. Re-run to continue " + f"(the count stays high until the last pass — each run advances through " + f"the archive).[/yellow]" + ) + else: + console.print( + f"[yellow]{total_deferred} conversation(s) deferred by the session cap " + f"({session_cap} per provider per run). Re-run the same command to continue.[/yellow]" + ) # Emit the data-loss summary at INFO level so it lands in the log file # AND the operator's console (default level is INFO). for line in loss_report.format_summary().split("\n"): diff --git a/tests/test_cache.py b/tests/test_cache.py index 6ce3fc3..e7a3141 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -151,3 +151,73 @@ class TestGetNewOrUpdated: convs = [{"id": "a", "updated_at": "2024-06-01T00:00:00Z"}] result = tmp_cache.get_new_or_updated("claude", convs) assert len(result) == 1 + + def test_force_returns_all_cached_and_unchanged(self, tmp_cache): + tmp_cache.mark_exported("claude", "a", {"updated_at": "2024-01-01T00:00:00Z"}) + convs = [ + {"id": "a", "updated_at": "2024-01-01T00:00:00Z"}, # cached, unchanged + {"id": "b", "updated_at": "2024-01-02T00:00:00Z"}, # new + ] + assert len(tmp_cache.get_new_or_updated("claude", convs)) == 1 + assert len(tmp_cache.get_new_or_updated("claude", convs, force=True)) == 2 + + def test_force_orders_least_recently_exported_first(self, tmp_cache): + # 'a' was exported long ago; 'b' just now; 'c' never. + tmp_cache.mark_exported("claude", "a", {"updated_at": "2024-01-01T00:00:00Z"}) + tmp_cache._data["claude"]["a"]["exported_at"] = "2024-01-01T00:00:00Z" + tmp_cache.mark_exported("claude", "b", {"updated_at": "2024-01-01T00:00:00Z"}) + tmp_cache._data["claude"]["b"]["exported_at"] = "2026-06-12T00:00:00Z" + convs = [{"id": "a"}, {"id": "b"}, {"id": "c"}] + ordered = [c["id"] for c in tmp_cache.get_new_or_updated("claude", convs, force=True)] + # never-exported ('c', exported_at "") and oldest ('a') come before 'b' + assert ordered == ["c", "a", "b"] + + def test_force_capped_run_makes_progress(self, tmp_cache): + """The bug: force + cap repeated the same head every run. Now each + capped run re-exports the next-oldest batch and converges.""" + convs = [{"id": f"c{i}", "updated_at": "2024-01-01T00:00:00Z"} for i in range(5)] + seen = set() + for _ in range(3): # cap=2 over 3 runs should cover all 5 + batch = tmp_cache.get_new_or_updated("claude", convs, force=True)[:2] + for c in batch: + seen.add(c["id"]) + tmp_cache.mark_exported("claude", c["id"], {"file_path": f"/{c['id']}.md"}) + assert seen == {f"c{i}" for i in range(5)} + + +class TestReexportPreservesJoplinLink: + """A re-export must not drop the Joplin note link, or re-syncing would + create duplicate notes instead of updating the existing ones.""" + + def test_joplin_fields_carried_across_reexport(self, tmp_cache): + tmp_cache.mark_exported("chatgpt", "c1", { + "title": "T", "updated_at": "2024-01-01T00:00:00Z", "file_path": "/x.md", + }) + tmp_cache.mark_joplin_synced("chatgpt", "c1", "note-123") + tmp_cache.set_joplin_resources("chatgpt", "c1", {"media/a.png": "res-1"}) + + # Re-export the same conversation (new render, new file_path). + tmp_cache.mark_exported("chatgpt", "c1", { + "title": "T", "updated_at": "2024-01-01T00:00:00Z", "file_path": "/x2.md", + }) + + entry = tmp_cache.get_all_entries("chatgpt")["c1"] + assert entry["joplin_note_id"] == "note-123" + assert entry["joplin_resources"] == {"media/a.png": "res-1"} + assert entry["file_path"] == "/x2.md" + + def test_reexport_marks_note_for_update_not_create(self, tmp_cache): + tmp_cache.mark_exported("chatgpt", "c1", { + "updated_at": "2024-01-01T00:00:00Z", "file_path": "/x.md", + }) + tmp_cache.mark_joplin_synced("chatgpt", "c1", "note-123") + # Nothing pending right after sync. + assert tmp_cache.get_joplin_pending("chatgpt") == [] + # Re-export bumps exported_at past joplin_synced_at → pending for update, + # carrying the existing note id so the sync updates rather than creates. + tmp_cache.mark_exported("chatgpt", "c1", { + "updated_at": "2024-01-01T00:00:00Z", "file_path": "/x.md", + }) + pending = tmp_cache.get_joplin_pending("chatgpt") + assert len(pending) == 1 + assert pending[0][1]["joplin_note_id"] == "note-123"