feat: v0.6.0 — collapse policy, session limiter, Claude Code provider, prune, browser auth, media downloads

tests/test_providers.py

@@ -171,6 +171,7 @@ class TestChatGPTNormalization:
     def test_user_editable_context_emits_blocks(self):
         raw = json.loads((FIXTURES / "chatgpt_conversation.json").read_text())
         p = self._get_provider()
+        p._hidden_content = "full"
         result = p.normalize_conversation(raw)
 
         # The user_editable_context message has user_profile + user_instructions.
@@ -194,6 +195,7 @@ class TestChatGPTNormalization:
 
         raw = json.loads((FIXTURES / "chatgpt_conversation.json").read_text())
         p = self._get_provider()
+        p._hidden_content = "full"
         result = p.normalize_conversation(raw)
 
         uec_msgs = [
@@ -290,6 +292,236 @@ class TestChatGPTUnknownContent:
         assert report.unknown_blocks["future_unknown_type_xyz"] == 1
 
 
+class TestChatGPTHiddenContentPolicy:
+    """EXPORTER_HIDDEN_CONTENT: collapse retrieval dumps and hidden context.
+
+    file_search dumps re-inject attached-file text on every tool run (measured
+    86% of content bytes on real data) and are NOT hidden-flagged — author.name
+    is the discriminator. Hidden-flagged messages (Custom Instructions) are the
+    secondary case.
+    """
+
+    def _get_provider(self, policy=None):
+        from src.providers.chatgpt import ChatGPTProvider
+        p = ChatGPTProvider.__new__(ChatGPTProvider)
+        import requests
+        p._session = requests.Session()
+        p._org_id = None
+        p._project_ids = []
+        p._project_map = {}
+        p._project_name_cache = {}
+        if policy is not None:
+            p._hidden_content = policy
+        return p
+
+    def _make_conv(self):
+        return {
+            "id": "test-collapse",
+            "title": "T",
+            "create_time": 1700000000.0,
+            "update_time": 1700000001.0,
+            "mapping": {
+                "root": {"id": "root", "message": None, "parent": None, "children": ["dump"]},
+                "dump": {
+                    "id": "dump",
+                    "parent": "root",
+                    "children": ["answer"],
+                    "message": {
+                        "id": "dump",
+                        "author": {"role": "tool", "name": "file_search"},
+                        "content": {
+                            "content_type": "multimodal_text",
+                            "parts": ["Full text of an attached file " * 100],
+                        },
+                    },
+                },
+                "answer": {
+                    "id": "answer",
+                    "parent": "dump",
+                    "children": ["uec"],
+                    "message": {
+                        "id": "answer",
+                        "author": {"role": "assistant"},
+                        "content": {"content_type": "text", "parts": ["The answer."]},
+                    },
+                },
+                "uec": {
+                    "id": "uec",
+                    "parent": "answer",
+                    "children": [],
+                    "message": {
+                        "id": "uec",
+                        "author": {"role": "user"},
+                        "content": {
+                            "content_type": "user_editable_context",
+                            "user_profile": "Preferred name: Jesse",
+                            "user_instructions": "Be concise.",
+                        },
+                        "metadata": {"is_visually_hidden_from_conversation": True},
+                    },
+                },
+            },
+        }
+
+    def test_default_policy_is_placeholder(self, monkeypatch):
+        from src.blocks import BLOCK_TYPE_COLLAPSED
+        monkeypatch.delenv("EXPORTER_HIDDEN_CONTENT", raising=False)
+        # No _hidden_content attribute → env fallback → placeholder default.
+        result = self._get_provider().normalize_conversation(self._make_conv())
+        collapsed = [
+            b for m in result["messages"] for b in m["blocks"]
+            if b.get("type") == BLOCK_TYPE_COLLAPSED
+        ]
+        assert len(collapsed) == 2
+
+    def test_placeholder_collapses_file_search_dump(self):
+        from src.blocks import BLOCK_TYPE_COLLAPSED, COLLAPSED_KIND_TOOL_DUMP
+        result = self._get_provider("placeholder").normalize_conversation(self._make_conv())
+        tool_msgs = [m for m in result["messages"] if m["role"] == "tool"]
+        assert len(tool_msgs) == 1
+        (block,) = tool_msgs[0]["blocks"]
+        assert block["type"] == BLOCK_TYPE_COLLAPSED
+        assert block["kind"] == COLLAPSED_KIND_TOOL_DUMP
+        assert block["origin"] == "file_search"
+        assert block["size_bytes"] > 1000
+
+    def test_placeholder_collapses_hidden_flagged_message(self):
+        from src.blocks import BLOCK_TYPE_COLLAPSED, COLLAPSED_KIND_HIDDEN_CONTEXT
+        result = self._get_provider("placeholder").normalize_conversation(self._make_conv())
+        user_msgs = [m for m in result["messages"] if m["role"] == "user"]
+        assert len(user_msgs) == 1
+        (block,) = user_msgs[0]["blocks"]
+        assert block["type"] == BLOCK_TYPE_COLLAPSED
+        assert block["kind"] == COLLAPSED_KIND_HIDDEN_CONTEXT
+        assert block["origin"] == "user_editable_context"
+
+    def test_placeholder_keeps_normal_messages(self):
+        result = self._get_provider("placeholder").normalize_conversation(self._make_conv())
+        assistant_msgs = [m for m in result["messages"] if m["role"] == "assistant"]
+        assert assistant_msgs[0]["blocks"][0]["type"] == BLOCK_TYPE_TEXT
+        assert assistant_msgs[0]["blocks"][0]["text"] == "The answer."
+
+    def test_non_retrieval_tool_authors_not_collapsed(self):
+        """web.run / browser / python tool messages are real content — keep."""
+        from src.blocks import BLOCK_TYPE_COLLAPSED
+        conv = self._make_conv()
+        conv["mapping"]["dump"]["message"]["author"]["name"] = "web.run"
+        conv["mapping"]["dump"]["message"]["content"] = {
+            "content_type": "text",
+            "parts": ["Search results summary."],
+        }
+        result = self._get_provider("placeholder").normalize_conversation(conv)
+        tool_msgs = [m for m in result["messages"] if m["role"] == "tool"]
+        assert tool_msgs[0]["blocks"][0]["type"] != BLOCK_TYPE_COLLAPSED
+
+    def test_omit_drops_messages_entirely(self):
+        result = self._get_provider("omit").normalize_conversation(self._make_conv())
+        assert [m["role"] for m in result["messages"]] == ["assistant"]
+
+    def test_full_preserves_v040_behavior(self):
+        result = self._get_provider("full").normalize_conversation(self._make_conv())
+        tool_msgs = [m for m in result["messages"] if m["role"] == "tool"]
+        assert tool_msgs[0]["blocks"][0]["type"] == BLOCK_TYPE_TEXT
+        user_msgs = [m for m in result["messages"] if m["role"] == "user"]
+        assert user_msgs[0]["blocks"][0]["type"] == BLOCK_TYPE_HIDDEN_CONTEXT_MARKER
+
+    def test_collapsed_tallied_in_loss_report(self):
+        report = LossReport()
+        self._get_provider("placeholder").normalize_conversation(self._make_conv(), report)
+        assert report.collapsed["file_search"] == 1
+        assert report.collapsed["user_editable_context"] == 1
+        assert report.collapsed_bytes > 1000
+        summary = report.format_summary()
+        assert "collapsed by policy:  2" in summary
+        assert "EXPORTER_HIDDEN_CONTENT=full to keep" in summary
+
+    def test_collapsed_block_renders_one_line_with_size(self):
+        from src.blocks import BLOCK_TYPE_COLLAPSED
+        result = self._get_provider("placeholder").normalize_conversation(self._make_conv())
+        tool_msgs = [m for m in result["messages"] if m["role"] == "tool"]
+        rendered = render_blocks_to_markdown(tool_msgs[0]["blocks"])
+        assert rendered.startswith("> 🔧 **Tool output** — `file_search` (")
+        assert "KB" in rendered
+        assert "omitted (EXPORTER_HIDDEN_CONTENT=full to keep)" in rendered
+        assert "\n" not in rendered
+
+    def test_invalid_env_value_falls_back_to_placeholder(self, monkeypatch, caplog):
+        from src.providers.chatgpt import resolve_hidden_content_policy
+        monkeypatch.setenv("EXPORTER_HIDDEN_CONTENT", "bogus")
+        with caplog.at_level(logging.WARNING):
+            assert resolve_hidden_content_policy() == "placeholder"
+        assert any("EXPORTER_HIDDEN_CONTENT" in r.message for r in caplog.records)
+
+
+class TestRequestPacing:
+    """REQUEST_DELAY politeness pacing between consecutive API requests."""
+
+    def _get_provider(self, delay):
+        from src.providers.chatgpt import ChatGPTProvider
+        p = ChatGPTProvider.__new__(ChatGPTProvider)
+        p._request_delay = delay
+        p._last_request_at = None
+        return p
+
+    def test_first_request_not_delayed(self, monkeypatch):
+        sleeps = []
+        monkeypatch.setattr("src.providers.base.time.sleep", lambda s: sleeps.append(s))
+        p = self._get_provider(2.0)
+        p._pace()
+        assert sleeps == []
+
+    def test_consecutive_requests_paced_with_jitter_window(self, monkeypatch):
+        sleeps = []
+        monkeypatch.setattr("src.providers.base.time.sleep", lambda s: sleeps.append(s))
+        monkeypatch.setattr("src.providers.base.random.uniform", lambda a, b: 1.0)
+        clock = {"now": 100.0}
+        monkeypatch.setattr("src.providers.base.time.monotonic", lambda: clock["now"])
+        p = self._get_provider(2.0)
+        p._pace()
+        clock["now"] = 100.5  # only 0.5s elapsed since last request
+        p._pace()
+        assert sleeps == [pytest.approx(1.5)]
+
+    def test_no_sleep_when_enough_time_elapsed(self, monkeypatch):
+        sleeps = []
+        monkeypatch.setattr("src.providers.base.time.sleep", lambda s: sleeps.append(s))
+        monkeypatch.setattr("src.providers.base.random.uniform", lambda a, b: 1.0)
+        clock = {"now": 100.0}
+        monkeypatch.setattr("src.providers.base.time.monotonic", lambda: clock["now"])
+        p = self._get_provider(1.0)
+        p._pace()
+        clock["now"] = 105.0
+        p._pace()
+        assert sleeps == []
+
+    def test_zero_delay_disables_pacing(self, monkeypatch):
+        sleeps = []
+        monkeypatch.setattr("src.providers.base.time.sleep", lambda s: sleeps.append(s))
+        p = self._get_provider(0)
+        p._pace()
+        p._pace()
+        assert sleeps == []
+
+    def test_pace_noop_without_init_attrs(self):
+        """Providers built via __new__ in tests have no pacing attrs — must not crash."""
+        from src.providers.chatgpt import ChatGPTProvider
+        p = ChatGPTProvider.__new__(ChatGPTProvider)
+        p._pace()  # no exception
+
+    def test_resolve_request_delay_default_and_overrides(self, monkeypatch, caplog):
+        from src.providers.base import resolve_request_delay, DEFAULT_REQUEST_DELAY
+        monkeypatch.delenv("REQUEST_DELAY", raising=False)
+        assert resolve_request_delay() == DEFAULT_REQUEST_DELAY
+        monkeypatch.setenv("REQUEST_DELAY", "2.5")
+        assert resolve_request_delay() == 2.5
+        monkeypatch.setenv("REQUEST_DELAY", "0")
+        assert resolve_request_delay() == 0.0
+        monkeypatch.setenv("REQUEST_DELAY", "abc")
+        with caplog.at_level(logging.WARNING):
+            assert resolve_request_delay() == DEFAULT_REQUEST_DELAY
+        assert any("REQUEST_DELAY" in r.message for r in caplog.records)
+
+
 # ---------------------------------------------------------------------------
 # Claude
 # ---------------------------------------------------------------------------