JesseMarkowitzandClaude Opus 5
8c3e00a520
An asset download is two hops — mint a signed URL, then fetch the bytes — and only the first went through _make_request. The second was a bare session.request with no 5xx handling, which lost two images in the 2026-10-20 09:00 run: one to a 500 the retried first hop had already recovered from, one to a 503. Both re-downloaded cleanly an hour later (106 KB and 1.3 MB), so the bytes were there all along. A third asset timed out and aborted its whole conversation. curl_cffi's RequestException subclasses OSError, so the un-wrapped timeout slipped past resolve_media's except ProviderError into the export loop's except OSError and was reported as "File write failed for conversation 6a945155" — a conversation that had already downloaded ~20 images. Nothing was wrong with any file write. The retry loop is factored out of _make_request into _send_with_retries, which returns the raw response so a bytes caller gets the same handling as a JSON one. Hop 2 goes through it: 5xx retried, Retry-After honoured, timeouts surfacing as ProviderError for resolve_media to placeholder. The hop is now logged, which it never was. Assets get their own ladder — an immediate retry, then ~2/4/8/16s with jitter kept, since a storage incident fails many assets at once — bounded by a per-asset wall-clock budget, because the timeout case costs 40s a go. Beyond the retry: - Transient failures (5xx, timeout, network) are recorded as pending_media so a later run re-fetches just that asset; capped at 3 runs. 404/403 stay permanent — settled 2026-08-17 for 19 assets. - A circuit breaker stops downloads after 5 consecutive transient failures, recording the rest as retryable. Permanent failures never trip it; any success resets the streak. - Media failures now appear in the set ids. The run that lost two assets pushed a clean "OK". Keeps FUTURE.md's per-conversation cache reset correctly closed. The README split (FUTURE.md §2) stays otial enough to justify breaking every anchor link. 37 new tests (414 passing). Verified live: both lost assets recovered, doctor green, a read-only chatgpt dns. The cross-run retry and breaker are unit-tested only. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/cwa7qnGp Claude-Session: https://claude.ai/code/session_012gCWSknsWvhLnMFwa7qnGp
351 lines
15 KiB
Python
351 lines
15 KiB
Python
"""Unit tests for src/cache.py."""
|
|
|
|
import json
|
|
import os
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from src.cache import Cache, CacheError, MANIFEST_VERSION
|
|
|
|
|
|
@pytest.fixture
|
|
def tmp_cache(tmp_path):
|
|
return Cache(tmp_path)
|
|
|
|
|
|
class TestIsCached:
|
|
def test_miss_when_no_entry(self, tmp_cache):
|
|
assert tmp_cache.is_cached("claude", "conv-abc", "2024-01-01T00:00:00Z") is False
|
|
|
|
def test_hit_after_mark_exported(self, tmp_cache):
|
|
tmp_cache.mark_exported("claude", "conv-abc", {"updated_at": "2024-01-01T00:00:00Z"})
|
|
assert tmp_cache.is_cached("claude", "conv-abc", "2024-01-01T00:00:00Z") is True
|
|
|
|
def test_stale_when_provider_has_newer_date(self, tmp_cache):
|
|
tmp_cache.mark_exported("claude", "conv-abc", {"updated_at": "2024-01-01T00:00:00Z"})
|
|
assert tmp_cache.is_cached("claude", "conv-abc", "2024-06-01T00:00:00Z") is False
|
|
|
|
def test_hit_when_provider_has_same_date(self, tmp_cache):
|
|
tmp_cache.mark_exported("chatgpt", "conv-xyz", {"updated_at": "2024-06-01T00:00:00Z"})
|
|
assert tmp_cache.is_cached("chatgpt", "conv-xyz", "2024-06-01T00:00:00Z") is True
|
|
|
|
def test_miss_for_different_provider(self, tmp_cache):
|
|
tmp_cache.mark_exported("claude", "conv-abc", {"updated_at": "2024-01-01T00:00:00Z"})
|
|
assert tmp_cache.is_cached("chatgpt", "conv-abc", "2024-01-01T00:00:00Z") is False
|
|
|
|
|
|
class TestAtomicWrite:
|
|
def test_manifest_has_600_permissions(self, tmp_path):
|
|
c = Cache(tmp_path)
|
|
c.mark_exported("claude", "x", {"updated_at": "2024-01-01"})
|
|
manifest = tmp_path / "manifest.json"
|
|
mode = oct(os.stat(manifest).st_mode)[-3:]
|
|
assert mode == "600"
|
|
|
|
def test_no_tmp_file_left_after_write(self, tmp_path):
|
|
c = Cache(tmp_path)
|
|
c.mark_exported("claude", "x", {"updated_at": "2024-01-01"})
|
|
tmp_files = list(tmp_path.glob("*.tmp"))
|
|
assert tmp_files == []
|
|
|
|
def test_manifest_is_valid_json(self, tmp_path):
|
|
c = Cache(tmp_path)
|
|
c.mark_exported("claude", "x", {})
|
|
manifest = tmp_path / "manifest.json"
|
|
data = json.loads(manifest.read_text())
|
|
assert isinstance(data, dict)
|
|
assert "claude" in data
|
|
|
|
|
|
class TestStats:
|
|
def test_empty_stats(self, tmp_cache):
|
|
stats = tmp_cache.stats()
|
|
assert stats["chatgpt"] == 0
|
|
assert stats["claude"] == 0
|
|
|
|
def test_stats_after_exports(self, tmp_cache):
|
|
tmp_cache.mark_exported("claude", "c1", {})
|
|
tmp_cache.mark_exported("claude", "c2", {})
|
|
tmp_cache.mark_exported("chatgpt", "g1", {})
|
|
stats = tmp_cache.stats()
|
|
assert stats["claude"] == 2
|
|
assert stats["chatgpt"] == 1
|
|
|
|
|
|
class TestClear:
|
|
def test_clear_single_provider(self, tmp_cache):
|
|
tmp_cache.mark_exported("claude", "c1", {})
|
|
tmp_cache.mark_exported("chatgpt", "g1", {})
|
|
tmp_cache.clear("claude")
|
|
assert tmp_cache.stats()["claude"] == 0
|
|
assert tmp_cache.stats()["chatgpt"] == 1
|
|
|
|
def test_clear_all(self, tmp_cache):
|
|
tmp_cache.mark_exported("claude", "c1", {})
|
|
tmp_cache.mark_exported("chatgpt", "g1", {})
|
|
tmp_cache.clear()
|
|
assert tmp_cache.stats()["claude"] == 0
|
|
assert tmp_cache.stats()["chatgpt"] == 0
|
|
|
|
|
|
class TestCorruptManifestRecovery:
|
|
def test_recovers_from_invalid_json(self, tmp_path):
|
|
manifest = tmp_path / "manifest.json"
|
|
manifest.write_text("{invalid json!!!", encoding="utf-8")
|
|
# Should not raise, should start fresh
|
|
c = Cache(tmp_path)
|
|
assert c.stats()["claude"] == 0
|
|
# Backup should exist
|
|
backup = tmp_path / "manifest.json.bak"
|
|
assert backup.exists()
|
|
assert backup.read_text() == "{invalid json!!!"
|
|
|
|
def test_raises_on_future_version(self, tmp_path):
|
|
manifest = tmp_path / "manifest.json"
|
|
manifest.write_text(
|
|
json.dumps({"version": MANIFEST_VERSION + 99, "chatgpt": {}, "claude": {}}),
|
|
encoding="utf-8",
|
|
)
|
|
with pytest.raises(CacheError, match="Unsupported manifest version"):
|
|
Cache(tmp_path)
|
|
|
|
|
|
class TestTosAcknowledgement:
|
|
def test_not_acknowledged_by_default(self, tmp_cache):
|
|
assert tmp_cache.is_tos_acknowledged() is False
|
|
|
|
def test_acknowledged_after_call(self, tmp_cache):
|
|
tmp_cache.acknowledge_tos()
|
|
assert tmp_cache.is_tos_acknowledged() is True
|
|
|
|
def test_acknowledgement_persists_across_instances(self, tmp_path):
|
|
c1 = Cache(tmp_path)
|
|
c1.acknowledge_tos()
|
|
c2 = Cache(tmp_path)
|
|
assert c2.is_tos_acknowledged() is True
|
|
|
|
|
|
class TestGetNewOrUpdated:
|
|
def test_returns_all_when_cache_empty(self, tmp_cache):
|
|
convs = [
|
|
{"id": "a", "updated_at": "2024-01-01T00:00:00Z"},
|
|
{"id": "b", "updated_at": "2024-01-02T00:00:00Z"},
|
|
]
|
|
result = tmp_cache.get_new_or_updated("claude", convs)
|
|
assert len(result) == 2
|
|
|
|
def test_skips_cached_unchanged(self, tmp_cache):
|
|
tmp_cache.mark_exported("claude", "a", {"updated_at": "2024-01-01T00:00:00Z"})
|
|
convs = [
|
|
{"id": "a", "updated_at": "2024-01-01T00:00:00Z"},
|
|
{"id": "b", "updated_at": "2024-01-02T00:00:00Z"},
|
|
]
|
|
result = tmp_cache.get_new_or_updated("claude", convs)
|
|
assert len(result) == 1
|
|
assert result[0]["id"] == "b"
|
|
|
|
def test_includes_stale_conversations(self, tmp_cache):
|
|
tmp_cache.mark_exported("claude", "a", {"updated_at": "2024-01-01T00:00:00Z"})
|
|
convs = [{"id": "a", "updated_at": "2024-06-01T00:00:00Z"}]
|
|
result = tmp_cache.get_new_or_updated("claude", convs)
|
|
assert len(result) == 1
|
|
|
|
def test_force_returns_all_cached_and_unchanged(self, tmp_cache):
|
|
tmp_cache.mark_exported("claude", "a", {"updated_at": "2024-01-01T00:00:00Z"})
|
|
convs = [
|
|
{"id": "a", "updated_at": "2024-01-01T00:00:00Z"}, # cached, unchanged
|
|
{"id": "b", "updated_at": "2024-01-02T00:00:00Z"}, # new
|
|
]
|
|
assert len(tmp_cache.get_new_or_updated("claude", convs)) == 1
|
|
assert len(tmp_cache.get_new_or_updated("claude", convs, force=True)) == 2
|
|
|
|
def test_force_orders_least_recently_exported_first(self, tmp_cache):
|
|
# 'a' was exported long ago; 'b' just now; 'c' never.
|
|
tmp_cache.mark_exported("claude", "a", {"updated_at": "2024-01-01T00:00:00Z"})
|
|
tmp_cache._data["claude"]["a"]["exported_at"] = "2024-01-01T00:00:00Z"
|
|
tmp_cache.mark_exported("claude", "b", {"updated_at": "2024-01-01T00:00:00Z"})
|
|
tmp_cache._data["claude"]["b"]["exported_at"] = "2026-06-12T00:00:00Z"
|
|
convs = [{"id": "a"}, {"id": "b"}, {"id": "c"}]
|
|
ordered = [c["id"] for c in tmp_cache.get_new_or_updated("claude", convs, force=True)]
|
|
# never-exported ('c', exported_at "") and oldest ('a') come before 'b'
|
|
assert ordered == ["c", "a", "b"]
|
|
|
|
def test_force_capped_run_makes_progress(self, tmp_cache):
|
|
"""The bug: force + cap repeated the same head every run. Now each
|
|
capped run re-exports the next-oldest batch and converges."""
|
|
convs = [{"id": f"c{i}", "updated_at": "2024-01-01T00:00:00Z"} for i in range(5)]
|
|
seen = set()
|
|
for _ in range(3): # cap=2 over 3 runs should cover all 5
|
|
batch = tmp_cache.get_new_or_updated("claude", convs, force=True)[:2]
|
|
for c in batch:
|
|
seen.add(c["id"])
|
|
tmp_cache.mark_exported("claude", c["id"], {"file_path": f"/{c['id']}.md"})
|
|
assert seen == {f"c{i}" for i in range(5)}
|
|
|
|
def test_campaign_excludes_already_refreshed(self, tmp_cache):
|
|
"""With a campaign stamp, conversations re-exported during the campaign
|
|
(exported_at >= stamp) drop out, so the remaining count shrinks to 0."""
|
|
convs = [{"id": f"c{i}"} for i in range(5)]
|
|
campaign = "2026-06-12T00:00:00+00:00"
|
|
# Nothing exported yet → all 5 are candidates.
|
|
assert len(tmp_cache.get_new_or_updated("claude", convs, force=True,
|
|
campaign_at=campaign)) == 5
|
|
# Re-export 2 "during" the campaign (after the stamp).
|
|
for cid in ("c0", "c1"):
|
|
tmp_cache.mark_exported("claude", cid, {"file_path": f"/{cid}.md"})
|
|
tmp_cache._data["claude"][cid]["exported_at"] = "2026-06-12T01:00:00+00:00"
|
|
remaining = tmp_cache.get_new_or_updated("claude", convs, force=True,
|
|
campaign_at=campaign)
|
|
assert {c["id"] for c in remaining} == {"c2", "c3", "c4"}
|
|
# Finish them → none left.
|
|
for cid in ("c2", "c3", "c4"):
|
|
tmp_cache.mark_exported("claude", cid, {"file_path": f"/{cid}.md"})
|
|
tmp_cache._data["claude"][cid]["exported_at"] = "2026-06-12T02:00:00+00:00"
|
|
assert tmp_cache.get_new_or_updated("claude", convs, force=True,
|
|
campaign_at=campaign) == []
|
|
|
|
def test_force_campaign_marker_roundtrip(self, tmp_cache):
|
|
assert tmp_cache.get_force_campaign() is None
|
|
tmp_cache.set_force_campaign("2026-06-12T00:00:00+00:00")
|
|
assert tmp_cache.get_force_campaign() == "2026-06-12T00:00:00+00:00"
|
|
# Survives reload, and is not mistaken for a provider by stats().
|
|
reopened = Cache(tmp_cache._dir)
|
|
assert reopened.get_force_campaign() == "2026-06-12T00:00:00+00:00"
|
|
assert "force_campaign_at" not in reopened.stats()
|
|
tmp_cache.clear_force_campaign()
|
|
assert tmp_cache.get_force_campaign() is None
|
|
|
|
|
|
class TestReexportPreservesJoplinLink:
|
|
"""A re-export must not drop the Joplin note link, or re-syncing would
|
|
create duplicate notes instead of updating the existing ones."""
|
|
|
|
def test_joplin_fields_carried_across_reexport(self, tmp_cache):
|
|
tmp_cache.mark_exported("chatgpt", "c1", {
|
|
"title": "T", "updated_at": "2024-01-01T00:00:00Z", "file_path": "/x.md",
|
|
})
|
|
tmp_cache.mark_joplin_synced("chatgpt", "c1", "note-123")
|
|
tmp_cache.set_joplin_resources("chatgpt", "c1", {"media/a.png": "res-1"})
|
|
|
|
# Re-export the same conversation (new render, new file_path).
|
|
tmp_cache.mark_exported("chatgpt", "c1", {
|
|
"title": "T", "updated_at": "2024-01-01T00:00:00Z", "file_path": "/x2.md",
|
|
})
|
|
|
|
entry = tmp_cache.get_all_entries("chatgpt")["c1"]
|
|
assert entry["joplin_note_id"] == "note-123"
|
|
assert entry["joplin_resources"] == {"media/a.png": "res-1"}
|
|
assert entry["file_path"] == "/x2.md"
|
|
|
|
def test_reexport_marks_note_for_update_not_create(self, tmp_cache):
|
|
tmp_cache.mark_exported("chatgpt", "c1", {
|
|
"updated_at": "2024-01-01T00:00:00Z", "file_path": "/x.md",
|
|
})
|
|
tmp_cache.mark_joplin_synced("chatgpt", "c1", "note-123")
|
|
# Nothing pending right after sync.
|
|
assert tmp_cache.get_joplin_pending("chatgpt") == []
|
|
# Re-export bumps exported_at past joplin_synced_at → pending for update,
|
|
# carrying the existing note id so the sync updates rather than creates.
|
|
tmp_cache.mark_exported("chatgpt", "c1", {
|
|
"updated_at": "2024-01-01T00:00:00Z", "file_path": "/x.md",
|
|
})
|
|
pending = tmp_cache.get_joplin_pending("chatgpt")
|
|
assert len(pending) == 1
|
|
assert pending[0][1]["joplin_note_id"] == "note-123"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cross-run media retry.
|
|
#
|
|
# Before this, a transient asset failure was permanent: resolve_media kept a
|
|
# placeholder, mark_exported ran anyway, and the conversation was never looked
|
|
# at again. Two assets were lost that way on 2026-10-20 to a 500 and a 503 —
|
|
# both recoverable. The manifest now remembers which refs are still missing.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestPendingMediaRetry:
|
|
UPDATED = "2026-10-20T09:00:00Z"
|
|
|
|
def test_pending_media_forces_a_re_export(self, tmp_cache):
|
|
tmp_cache.mark_exported(
|
|
"chatgpt", "conv-1",
|
|
{"updated_at": self.UPDATED, "pending_media": ["sediment://file_x"]},
|
|
)
|
|
assert tmp_cache.is_cached("chatgpt", "conv-1", self.UPDATED) is False
|
|
|
|
def test_no_pending_media_is_an_ordinary_hit(self, tmp_cache):
|
|
tmp_cache.mark_exported("chatgpt", "conv-1", {"updated_at": self.UPDATED})
|
|
assert tmp_cache.is_cached("chatgpt", "conv-1", self.UPDATED) is True
|
|
|
|
def test_a_resolved_asset_clears_the_flag(self, tmp_cache):
|
|
"""The run that recovers the asset must stop the daily re-export."""
|
|
tmp_cache.mark_exported(
|
|
"chatgpt", "conv-1",
|
|
{"updated_at": self.UPDATED, "pending_media": ["sediment://file_x"]},
|
|
)
|
|
assert tmp_cache.is_cached("chatgpt", "conv-1", self.UPDATED) is False
|
|
|
|
tmp_cache.mark_exported("chatgpt", "conv-1", {"updated_at": self.UPDATED})
|
|
assert tmp_cache.is_cached("chatgpt", "conv-1", self.UPDATED) is True
|
|
entry = tmp_cache.get_all_entries("chatgpt")["conv-1"]
|
|
assert "pending_media" not in entry
|
|
|
|
def test_retries_are_capped_so_it_does_not_re_export_forever(self, tmp_cache):
|
|
"""An asset that 5xxs every morning must stop costing a re-export."""
|
|
from src.cache import MAX_MEDIA_RETRY_RUNS
|
|
|
|
for run in range(MAX_MEDIA_RETRY_RUNS):
|
|
tmp_cache.mark_exported(
|
|
"chatgpt", "conv-1",
|
|
{"updated_at": self.UPDATED, "pending_media": ["sediment://file_x"]},
|
|
)
|
|
assert tmp_cache.is_cached("chatgpt", "conv-1", self.UPDATED) is False, (
|
|
f"run {run + 1} should still retry"
|
|
)
|
|
|
|
tmp_cache.mark_exported(
|
|
"chatgpt", "conv-1",
|
|
{"updated_at": self.UPDATED, "pending_media": ["sediment://file_x"]},
|
|
)
|
|
assert tmp_cache.is_cached("chatgpt", "conv-1", self.UPDATED) is True
|
|
|
|
def test_the_lost_refs_are_kept_as_a_record(self, tmp_cache):
|
|
from src.cache import MAX_MEDIA_RETRY_RUNS
|
|
|
|
for _ in range(MAX_MEDIA_RETRY_RUNS + 1):
|
|
tmp_cache.mark_exported(
|
|
"chatgpt", "conv-1",
|
|
{"updated_at": self.UPDATED, "pending_media": ["sediment://file_x"]},
|
|
)
|
|
entry = tmp_cache.get_all_entries("chatgpt")["conv-1"]
|
|
assert entry["pending_media"] == ["sediment://file_x"]
|
|
|
|
def test_joplin_link_survives_a_media_re_export(self, tmp_cache):
|
|
"""Re-exporting to recover an asset must not orphan the Joplin note."""
|
|
tmp_cache.mark_exported("chatgpt", "conv-1", {"updated_at": self.UPDATED})
|
|
tmp_cache.mark_joplin_synced("chatgpt", "conv-1", "note-123")
|
|
tmp_cache.set_joplin_resources("chatgpt", "conv-1", {"media/a.png": "res-1"})
|
|
|
|
tmp_cache.mark_exported(
|
|
"chatgpt", "conv-1",
|
|
{"updated_at": self.UPDATED, "pending_media": ["sediment://file_x"]},
|
|
)
|
|
entry = tmp_cache.get_all_entries("chatgpt")["conv-1"]
|
|
assert entry["joplin_note_id"] == "note-123"
|
|
# Resource ids are the idempotency record: without them the re-sync
|
|
# would re-upload every image in the note, not just the new one.
|
|
assert entry["joplin_resources"] == {"media/a.png": "res-1"}
|
|
|
|
def test_pending_media_survives_a_manifest_round_trip(self, tmp_path):
|
|
from src.cache import Cache
|
|
|
|
c = Cache(tmp_path)
|
|
c.mark_exported(
|
|
"chatgpt", "conv-1",
|
|
{"updated_at": self.UPDATED, "pending_media": ["sediment://file_x"]},
|
|
)
|
|
assert Cache(tmp_path).is_cached("chatgpt", "conv-1", self.UPDATED) is False
|