From 3cf0b1eaa881a664bfe4f1deb550a8ffd26fcc52 Mon Sep 17 00:00:00 2001 From: JesseMarkowitz Date: Fri, 12 Jun 2026 22:39:12 -0400 Subject: [PATCH] fix: force re-render campaign tracking, cache dir loading, Joplin link preservation. Co-Authored-By: Fable 5 --- CHANGELOG.md | 3 ++- README.md | 14 ++++++++--- src/cache.py | 55 +++++++++++++++++++++++++++++++----------- src/main.py | 58 ++++++++++++++++++++++++++++++++++----------- tests/test_cache.py | 33 ++++++++++++++++++++++++++ 5 files changed, 131 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d96b05..0a6bd05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,8 +47,9 @@ consolidated here. - Polite pacing: `REQUEST_DELAY` (default 1.0s, ±25% jitter, `0` disables) between consecutive API requests, with the existing 429 backoff as the reactive net. **Re-rendering** -- `export --force` re-exports every conversation even if cached and unchanged, so the whole archive can be re-rendered after a formatting/feature change without `cache --clear`. Combines with `--max-conversations` to spread the work across runs. +- `export --force` re-exports every conversation even if cached and unchanged, so the whole archive can be re-rendered after a formatting/feature change without `cache --clear`. Runs as a tracked campaign (a stamped start time in the manifest): each run re-renders the least-recently-exported conversations, the "still to go" count shrinks toward zero, finished providers do no further work, and the campaign auto-completes (reporting "Force re-render complete"). Combines with `--max-conversations` to spread the work across runs. - `mark_exported` now preserves `joplin_note_id` / `joplin_synced_at` / `joplin_resources` across re-exports (refreshing `exported_at`), so a re-export followed by `joplin` updates the existing notes instead of creating duplicates. +- Fix: `.env` is now loaded at the very start of every command, so `CACHE_DIR` and `LOG_FILE` are honored (previously the cache silently used `./cache` regardless of `CACHE_DIR`). **Archive hygiene** - `prune` deletes export files not referenced by the manifest (old-layout trees, no-ID orphans), with listing, confirmation, `--dry-run`, `-y`, and empty-directory sweep. Refuses to run when the manifest references no files (so `cache --clear` + `prune` can't wipe the archive). First live run removed 420 stale files (9.4 MB). diff --git a/README.md b/README.md index 3bb37e4..94619c4 100644 --- a/README.md +++ b/README.md @@ -331,11 +331,19 @@ ai-chat-exporter export --dry-run Options: `--provider [chatgpt|claude|claude-code|all]`, `--format [markdown|json|both]`, `--output PATH`, `--since YYYY-MM-DD`, `--project NAME`, `--hidden-content [full|placeholder|omit]`, `--download-media [images|all|off]`, `--max-conversations N`, `--force`, `--dry-run` -**Re-rendering the whole archive after an upgrade.** New formatting or features (collapse policy, media downloads) only change conversations as they're re-exported. To re-render everything you already have, use `--force` — it re-exports every conversation even if unchanged, **without** `cache --clear`, so your Joplin note links are preserved (a later `joplin` run updates the existing notes instead of duplicating them). Spread the load with `--max-conversations`: +**Re-rendering the whole archive after an upgrade.** New formatting or features (collapse policy, media downloads) only change conversations as they're re-exported. To re-render everything you already have, use `--force` — it re-exports every conversation even if unchanged, **without** `cache --clear`, so your Joplin note links are preserved (a later `joplin` run updates the existing notes instead of duplicating them). + +A force re-render runs as a tracked "campaign": each run re-renders the least-recently-exported conversations, the "still to go" count shrinks each run, and finished providers do no further work. The simplest approach is one uncapped pass: ```bash -ai-chat-exporter export --force --max-conversations 50 # run repeatedly until 0 deferred -ai-chat-exporter joplin # update notes + upload media +ai-chat-exporter export --force # re-renders everything in one paced pass +ai-chat-exporter joplin # update notes + upload media +``` + +Or spread the load with `--max-conversations`, re-running until it reports "Force re-render complete": + +```bash +ai-chat-exporter export --force --max-conversations 50 # repeat until complete ``` ### `list` — List conversations diff --git a/src/cache.py b/src/cache.py index d461e63..387a2b3 100644 --- a/src/cache.py +++ b/src/cache.py @@ -107,7 +107,11 @@ class Cache: self._save() def get_new_or_updated( - self, provider: str, conversations: list[dict], force: bool = False + self, + provider: str, + conversations: list[dict], + force: bool = False, + campaign_at: str | None = None, ) -> list[dict]: """Filter a conversation list to only new or updated conversations. @@ -115,19 +119,23 @@ class Cache: provider: "chatgpt" or "claude" conversations: List of raw conversation dicts from the provider. Each must have an ``id``/``uuid`` and ``updated_at``/``update_time``. - force: When True, return every conversation regardless of cache - state — used by ``export --force`` to re-render everything - without clearing the manifest (which would drop Joplin links). + force: When True, return conversations to re-render regardless of + cache freshness — used by ``export --force``. + campaign_at: In force mode, exclude conversations already re-exported + at/after this ISO timestamp (the re-render campaign start), so + each run advances and finished providers do nothing. Returns: - Subset that needs to be exported. + Subset that needs to be exported, ordered oldest-export-first in + force mode. """ if force: - # Re-export everything, but order by least-recently-exported first - # (never-exported sorts first). With a per-run cap, the batch just - # re-rendered gets a fresh exported_at and sinks to the back, so - # each subsequent run advances through the archive instead of - # repeating the same head. Conversations without an id are dropped. + # Re-export for a re-render campaign, ordered least-recently-exported + # first (never-exported sorts first). When ``campaign_at`` is given, + # conversations already re-exported during this campaign + # (exported_at >= campaign_at) are excluded — so finished providers + # do no work and the remaining count shrinks to zero. Each capped run + # re-renders the next-oldest batch. Conversations without an id drop. entries = self._data.get(provider, {}) def _exported_at(conv: dict) -> str: @@ -135,10 +143,10 @@ class Cache: entry = entries.get(conv_id) or {} return entry.get("exported_at") or "" - return sorted( - (c for c in conversations if c.get("id") or c.get("uuid")), - key=_exported_at, - ) + candidates = [c for c in conversations if c.get("id") or c.get("uuid")] + if campaign_at is not None: + candidates = [c for c in candidates if _exported_at(c) < campaign_at] + return sorted(candidates, key=_exported_at) result = [] for conv in conversations: @@ -283,6 +291,25 @@ class Cache: """Return the ISO8601 timestamp of the last export run, or None.""" return self._data.get("last_run") + def exported_at(self, provider: str, conv_id: str) -> str: + """Return the recorded ``exported_at`` for a conversation, or '' if absent.""" + entry = self._data.get(provider, {}).get(conv_id) + return entry.get("exported_at", "") if isinstance(entry, dict) else "" + + # Force re-render campaign marker (top-level scalar; skipped by stats/clear, + # which only act on dict-valued provider keys). + def get_force_campaign(self) -> str | None: + """Return the active force-re-render campaign start timestamp, or None.""" + return self._data.get("force_campaign_at") + + def set_force_campaign(self, started_at: str) -> None: + self._data["force_campaign_at"] = started_at + self._save() + + def clear_force_campaign(self) -> None: + if self._data.pop("force_campaign_at", None) is not None: + self._save() + # ------------------------------------------------------------------ # Private helpers # ------------------------------------------------------------------ diff --git a/src/main.py b/src/main.py index 603dda9..aa12ab0 100644 --- a/src/main.py +++ b/src/main.py @@ -7,7 +7,7 @@ import platform import shutil import sys import traceback -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path import click @@ -715,6 +715,18 @@ def export( # Session cap: CLI flag wins over MAX_CONVERSATIONS_PER_RUN env default. session_cap = max_conversations if max_conversations is not None else cfg.max_conversations + # Force re-render campaign: stamp a start time so progress can be tracked + # across runs. A conversation is "done this campaign" once its exported_at + # passes the stamp; finished providers then export nothing, and the + # "still awaiting" count shrinks to zero. A fresh campaign starts whenever + # none is active (the previous one completed or was never started). + campaign_at: str | None = None + if force and not dry_run: + campaign_at = cache.get_force_campaign() + if campaign_at is None: + campaign_at = datetime.now(timezone.utc).isoformat() + cache.set_force_campaign(campaign_at) + # Determine which providers to run providers_to_run = _resolve_providers(provider, cfg) if not providers_to_run: @@ -754,7 +766,9 @@ def export( f" [dim]--project filter '{project_filter}': {len(all_convs)} matching conversations.[/dim]" ) - to_export = cache.get_new_or_updated(prov_name, all_convs, force=force) + to_export = cache.get_new_or_updated( + prov_name, all_convs, force=force, campaign_at=campaign_at + ) skipped = len(all_convs) - len(to_export) summary[prov_name]["skipped"] = skipped @@ -776,7 +790,11 @@ def export( continue if not to_export: - console.print(f" [dim]{skipped} conversations already up to date.[/dim]") + done_msg = ( + "already re-rendered this campaign." + if force else "conversations already up to date." + ) + console.print(f" [dim]{skipped} {done_msg}[/dim]") continue if deferred: @@ -855,22 +873,34 @@ def export( progress.advance(task) continue + # Force campaign: how many of this provider's conversations still + # predate the campaign stamp (i.e. not yet re-rendered this campaign). + if force and campaign_at is not None: + summary[prov_name]["awaiting"] = sum( + 1 + for c in all_convs + if cache.exported_at(prov_name, c.get("id") or c.get("uuid", "")) < campaign_at + ) + if not dry_run: _print_export_summary(summary) - total_deferred = sum(s.get("deferred", 0) for s in summary.values()) - if total_deferred: - if force: - # In force mode every conversation is "to_export", so the - # deferred count is a rolling remainder, not a shrinking - # backlog: each run re-renders the oldest batch and advances. - exported_now = sum(s.get("exported", 0) for s in summary.values()) + if force: + total_awaiting = sum(s.get("awaiting", 0) for s in summary.values()) + exported_now = sum(s.get("exported", 0) for s in summary.values()) + if total_awaiting: console.print( - f"[yellow]Force re-render: did the {exported_now} least-recently-exported " - f"this run; {total_deferred} still awaiting a refresh. Re-run to continue " - f"(the count stays high until the last pass — each run advances through " - f"the archive).[/yellow]" + f"[yellow]Force re-render: refreshed {exported_now} this run, " + f"{total_awaiting} still to go. Re-run the same command to continue.[/yellow]" ) else: + cache.clear_force_campaign() + console.print( + "[green]Force re-render complete — every conversation has been " + "re-rendered with the current code.[/green]" + ) + else: + total_deferred = sum(s.get("deferred", 0) for s in summary.values()) + if total_deferred: console.print( f"[yellow]{total_deferred} conversation(s) deferred by the session cap " f"({session_cap} per provider per run). Re-run the same command to continue.[/yellow]" diff --git a/tests/test_cache.py b/tests/test_cache.py index e7a3141..4a11507 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -184,6 +184,39 @@ class TestGetNewOrUpdated: tmp_cache.mark_exported("claude", c["id"], {"file_path": f"/{c['id']}.md"}) assert seen == {f"c{i}" for i in range(5)} + def test_campaign_excludes_already_refreshed(self, tmp_cache): + """With a campaign stamp, conversations re-exported during the campaign + (exported_at >= stamp) drop out, so the remaining count shrinks to 0.""" + convs = [{"id": f"c{i}"} for i in range(5)] + campaign = "2026-06-12T00:00:00+00:00" + # Nothing exported yet → all 5 are candidates. + assert len(tmp_cache.get_new_or_updated("claude", convs, force=True, + campaign_at=campaign)) == 5 + # Re-export 2 "during" the campaign (after the stamp). + for cid in ("c0", "c1"): + tmp_cache.mark_exported("claude", cid, {"file_path": f"/{cid}.md"}) + tmp_cache._data["claude"][cid]["exported_at"] = "2026-06-12T01:00:00+00:00" + remaining = tmp_cache.get_new_or_updated("claude", convs, force=True, + campaign_at=campaign) + assert {c["id"] for c in remaining} == {"c2", "c3", "c4"} + # Finish them → none left. + for cid in ("c2", "c3", "c4"): + tmp_cache.mark_exported("claude", cid, {"file_path": f"/{cid}.md"}) + tmp_cache._data["claude"][cid]["exported_at"] = "2026-06-12T02:00:00+00:00" + assert tmp_cache.get_new_or_updated("claude", convs, force=True, + campaign_at=campaign) == [] + + def test_force_campaign_marker_roundtrip(self, tmp_cache): + assert tmp_cache.get_force_campaign() is None + tmp_cache.set_force_campaign("2026-06-12T00:00:00+00:00") + assert tmp_cache.get_force_campaign() == "2026-06-12T00:00:00+00:00" + # Survives reload, and is not mistaken for a provider by stats(). + reopened = Cache(tmp_cache._dir) + assert reopened.get_force_campaign() == "2026-06-12T00:00:00+00:00" + assert "force_campaign_at" not in reopened.stats() + tmp_cache.clear_force_campaign() + assert tmp_cache.get_force_campaign() is None + class TestReexportPreservesJoplinLink: """A re-export must not drop the Joplin note link, or re-syncing would