Jesse.Markowitz/AIChatExporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: force re-render campaign tracking, cache dir loading, Joplin link preservation. Co-Authored-By: Fable 5

Browse Source
This commit is contained in:
JesseMarkowitz
2026-06-12 22:39:12 -04:00
parent 456975ad50
commit 3cf0b1eaa8
5 changed files with 131 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/cache.py
+41 -14
View File
@@ -107,7 +107,11 @@ class Cache:
self._save()
def get_new_or_updated(
self, provider: str, conversations: list[dict], force: bool = False
self,
provider: str,
conversations: list[dict],
force: bool = False,
campaign_at: str | None = None,
) -> list[dict]:
"""Filter a conversation list to only new or updated conversations.
@@ -115,19 +119,23 @@ class Cache:
provider: "chatgpt" or "claude"
conversations: List of raw conversation dicts from the provider.
Each must have an ``id``/``uuid`` and ``updated_at``/``update_time``.
force: When True, return every conversation regardless of cache
state — used by ``export --force`` to re-render everything
without clearing the manifest (which would drop Joplin links).
force: When True, return conversations to re-render regardless of
cache freshness — used by ``export --force``.
campaign_at: In force mode, exclude conversations already re-exported
at/after this ISO timestamp (the re-render campaign start), so
each run advances and finished providers do nothing.
Returns:
Subset that needs to be exported.
Subset that needs to be exported, ordered oldest-export-first in
force mode.
"""
if force:
# Re-export everything, but order by least-recently-exported first
# (never-exported sorts first). With a per-run cap, the batch just
# re-rendered gets a fresh exported_at and sinks to the back, so
# each subsequent run advances through the archive instead of
# repeating the same head. Conversations without an id are dropped.
# Re-export for a re-render campaign, ordered least-recently-exported
# first (never-exported sorts first). When ``campaign_at`` is given,
# conversations already re-exported during this campaign
# (exported_at >= campaign_at) are excluded — so finished providers
# do no work and the remaining count shrinks to zero. Each capped run
# re-renders the next-oldest batch. Conversations without an id drop.
entries = self._data.get(provider, {})
def _exported_at(conv: dict) -> str:
@@ -135,10 +143,10 @@ class Cache:
entry = entries.get(conv_id) or {}
return entry.get("exported_at") or ""
return sorted(
(c for c in conversations if c.get("id") or c.get("uuid")),
key=_exported_at,
)
candidates = [c for c in conversations if c.get("id") or c.get("uuid")]
if campaign_at is not None:
candidates = [c for c in candidates if _exported_at(c) < campaign_at]
return sorted(candidates, key=_exported_at)
result = []
for conv in conversations:
@@ -283,6 +291,25 @@ class Cache:
"""Return the ISO8601 timestamp of the last export run, or None."""
return self._data.get("last_run")
def exported_at(self, provider: str, conv_id: str) -> str:
"""Return the recorded ``exported_at`` for a conversation, or '' if absent."""
entry = self._data.get(provider, {}).get(conv_id)
return entry.get("exported_at", "") if isinstance(entry, dict) else ""
# Force re-render campaign marker (top-level scalar; skipped by stats/clear,
# which only act on dict-valued provider keys).
def get_force_campaign(self) -> str | None:
"""Return the active force-re-render campaign start timestamp, or None."""
return self._data.get("force_campaign_at")
def set_force_campaign(self, started_at: str) -> None:
self._data["force_campaign_at"] = started_at
self._save()
def clear_force_campaign(self) -> None:
if self._data.pop("force_campaign_at", None) is not None:
self._save()
# ------------------------------------------------------------------
# Private helpers
# ------------------------------------------------------------------
src/main.py
+44 -14
View File
@@ -7,7 +7,7 @@ import platform
import shutil
import sys
import traceback
from datetime import datetime
from datetime import datetime, timezone
from pathlib import Path
import click
@@ -715,6 +715,18 @@ def export(
# Session cap: CLI flag wins over MAX_CONVERSATIONS_PER_RUN env default.
session_cap = max_conversations if max_conversations is not None else cfg.max_conversations
# Force re-render campaign: stamp a start time so progress can be tracked
# across runs. A conversation is "done this campaign" once its exported_at
# passes the stamp; finished providers then export nothing, and the
# "still awaiting" count shrinks to zero. A fresh campaign starts whenever
# none is active (the previous one completed or was never started).
campaign_at: str | None = None
if force and not dry_run:
campaign_at = cache.get_force_campaign()
if campaign_at is None:
campaign_at = datetime.now(timezone.utc).isoformat()
cache.set_force_campaign(campaign_at)
# Determine which providers to run
providers_to_run = _resolve_providers(provider, cfg)
if not providers_to_run:
@@ -754,7 +766,9 @@ def export(
f" [dim]--project filter '{project_filter}': {len(all_convs)} matching conversations.[/dim]"
)
to_export = cache.get_new_or_updated(prov_name, all_convs, force=force)
to_export = cache.get_new_or_updated(
prov_name, all_convs, force=force, campaign_at=campaign_at
)
skipped = len(all_convs) - len(to_export)
summary[prov_name]["skipped"] = skipped
@@ -776,7 +790,11 @@ def export(
continue
if not to_export:
console.print(f" [dim]{skipped} conversations already up to date.[/dim]")
done_msg = (
"already re-rendered this campaign."
if force else "conversations already up to date."
)
console.print(f" [dim]{skipped} {done_msg}[/dim]")
continue
if deferred:
@@ -855,22 +873,34 @@ def export(
progress.advance(task)
continue
# Force campaign: how many of this provider's conversations still
# predate the campaign stamp (i.e. not yet re-rendered this campaign).
if force and campaign_at is not None:
summary[prov_name]["awaiting"] = sum(
1
for c in all_convs
if cache.exported_at(prov_name, c.get("id") or c.get("uuid", "")) < campaign_at
)
if not dry_run:
_print_export_summary(summary)
total_deferred = sum(s.get("deferred", 0) for s in summary.values())
if total_deferred:
if force:
# In force mode every conversation is "to_export", so the
# deferred count is a rolling remainder, not a shrinking
# backlog: each run re-renders the oldest batch and advances.
exported_now = sum(s.get("exported", 0) for s in summary.values())
if force:
total_awaiting = sum(s.get("awaiting", 0) for s in summary.values())
exported_now = sum(s.get("exported", 0) for s in summary.values())
if total_awaiting:
console.print(
f"[yellow]Force re-render: did the {exported_now} least-recently-exported "
f"this run; {total_deferred} still awaiting a refresh. Re-run to continue "
f"(the count stays high until the last pass — each run advances through "
f"the archive).[/yellow]"
f"[yellow]Force re-render: refreshed {exported_now} this run, "
f"{total_awaiting} still to go. Re-run the same command to continue.[/yellow]"
)
else:
cache.clear_force_campaign()
console.print(
"[green]Force re-render complete — every conversation has been "
"re-rendered with the current code.[/green]"
)
else:
total_deferred = sum(s.get("deferred", 0) for s in summary.values())
if total_deferred:
console.print(
f"[yellow]{total_deferred} conversation(s) deferred by the session cap "
f"({session_cap} per provider per run). Re-run the same command to continue.[/yellow]"