diff --git a/.env.example b/.env.example index c5db335..72d87b3 100644 --- a/.env.example +++ b/.env.example @@ -6,10 +6,18 @@ # --- ChatGPT --- # How to get: open chatgpt.com in Chrome → F12 → Application tab -# → Cookies → https://chatgpt.com → find the two cookie chunks: +# → Cookies → https://chatgpt.com → find the session token cookie. Chrome splits a +# cookie only above ~4KB, so you will see ONE of these two layouts: +# +# __Secure-next-auth.session-token (the whole value) → CHATGPT_SESSION_TOKEN +# (leave _1 empty) +# or, when the token was large enough to be split: # __Secure-next-auth.session-token.0 (starts with "eyJ") → CHATGPT_SESSION_TOKEN # __Secure-next-auth.session-token.1 (the remainder) → CHATGPT_SESSION_TOKEN_1 -# Token type: JWE. Typically valid for ~7 days. +# +# CHATGPT_SESSION_TOKEN_1 is OPTIONAL — leave it empty when there is no .1 cookie. +# But if a .1 cookie does exist, you must copy it: a partial .0 fails silently +# (HTTP 200 with no accessToken). Token type: JWE. Typically valid for ~7 days. CHATGPT_SESSION_TOKEN= CHATGPT_SESSION_TOKEN_1= diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ac3880..9e896e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,20 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] ### Fixed +- **An expired Claude session key reported a raw JSON dump instead of how to fix it.** `_make_request` routed only **401** to the auth handler (`src/providers/base.py`), and claude.ai does not use 401 — an invalid or expired `sessionKey` comes back as `403 permission_error` with `details.error_code = account_session_invalid`. So the one message that names the cookie, its ~30-day lifetime and the DevTools path to refresh it could never fire for Claude. What the user got instead was the generic 4xx path: `HTTP 403 — error: {'type': 'permission_error', 'message': 'Invalid authorization'…}`, which reads like a permissions problem with the account and not like "your key expired, here is how to replace it." + + Measured live 2026-09-20 against `GET /api/organizations`: a valid key returns 200, while an expired key, a deliberately malformed key and **no cookie at all** return byte-identical 403s carrying that code — i.e. the API treats a dead session as an absent one. This is the same mistake as the ChatGPT media 403s below: assuming 403 means "forbidden" when the service uses it for "unauthenticated." + + Auth detection is now a provider decision rather than a hardcoded status. `BaseProvider._is_auth_failure(response)` defaults to 401 and `ClaudeProvider` overrides it to add 403 **matched on `account_session_invalid`**, not on the bare status — so a genuine permission error, which carries a different code, is still reported as itself rather than being mislabelled an expired key. `_handle_401` is renamed `_handle_auth_failure` and takes the response, because a handler named for one status that must handle two is how this stayed hidden; its messages now state the status actually observed instead of asserting "401 Unauthorized". ChatGPT is unaffected: it does not override the default, so the deleted-asset 403 path is untouched. + + Seven regression tests cover the split (`TestAuthFailureDetection`), including the two that matter most: a Claude 403 with a *different* error code must **not** be treated as an auth failure, and a ChatGPT 403 must not either. The docs that repeated the wrong premise — `README.md`'s expiry table and "When Tokens Expire" section, the `auth` wizard's on-screen note, and the `ClaudeProvider` docstring — are corrected in the same change. + +- **The docs claimed both ChatGPT cookie chunks were required; they are not.** `README.md` stated flatly that "ChatGPT splits large session tokens across two cookies to stay under the browser's 4KB cookie limit. Both are required," and `.env.example` documented only the chunked layout — so a machine whose session token happens to fit in a single `__Secure-next-auth.session-token` cookie looked broken, with the user hunting for a `.1` that does not exist. Chrome splits a cookie only above ~4KB, so the layout varies by session size and the *same account* can be chunked on one machine and not on another. + + The code was already correct: `CHATGPT_SESSION_TOKEN_1` is optional (`src/providers/chatgpt.py:162`) and the `auth` wizard already told you to paste a lone cookie into `.0` and leave `.1` blank. Only the reference docs were wrong, and they are the ones read when setting up a new machine. + + Measured 2026-09-20 against `/api/auth/session`, reassembling a real 4089-byte token to test each naming: chunked `.0`+`.1` → 200 with an `accessToken`; the whole value under the unchunked name → 200 with an `accessToken`; the whole value under `.0` alone → 200 with an `accessToken`. The server reassembles a complete value sent under `.0`, so both layouts authenticate as the code already assumed. A *partial* `.0` with its `.1` omitted is the one combination that fails, and it fails **silently** — HTTP 200 with no `accessToken` rather than an error — which is now documented in both files alongside the correction. + - **The test suite sent real push notifications to the developer's phone.** `TestSyncCommand` invokes the actual `sync` command, which calls `load_config()`, which calls `load_dotenv()` — so the real `.env` was loaded and its live `NTFY_TOPIC` used for the POST. Every `pytest` run fired three or four pushes, including a fabricated "codex: 3 conversation(s) failed to export" straight out of a fixture, which is worse than noise: it reports a failure that never happened. Nothing appeared in `cache/logs/exporter.log` to explain it, because every test invocation passes `--no-log-file`. A `tests/conftest.py` autouse fixture now neutralises the environment for every test: `NTFY_TOPIC`/`NTFY_TOKEN` are emptied, and `NTFY_SERVER` and `JOPLIN_API_URL` are pointed at a closed local port, so a stray topic cannot reach the internet and a test cannot write notes into a real Joplin instance. The values are **emptied rather than deleted** — `load_dotenv(override=False)` skips only keys already present, so deleting one lets `.env` put it back. Verified by instrumenting `requests` across a full run: zero outbound requests, where the same instrumentation without the fixture records POSTs to the live ntfy topic. diff --git a/README.md b/README.md index 7b52285..c529d80 100644 --- a/README.md +++ b/README.md @@ -169,8 +169,8 @@ The wizard detects your OS, shows the correct DevTools shortcut, and writes the | Provider | Cookie Name | Lifetime | Expiry Detection | |----------|-------------|----------|-----------------| -| ChatGPT | `__Secure-next-auth.session-token.0` + `.1` | refresh ~weekly | `error` field of `/api/auth/session` — `doctor` reports "ChatGPT token active". The token is an encrypted JWE, so its `exp` is **not** readable client-side, and the `expires` field is a misleading rolling window; the `error` (`RefreshAccessTokenError` when dead) is the honest signal. | -| Claude | `sessionKey` | ~30 days | Opaque token — only detectable via 401 response | +| ChatGPT | `__Secure-next-auth.session-token` (split into `.0` + `.1` when over ~4KB) | refresh ~weekly | `error` field of `/api/auth/session` — `doctor` reports "ChatGPT token active". The token is an encrypted JWE, so its `exp` is **not** readable client-side, and the `expires` field is a misleading rolling window; the `error` (`RefreshAccessTokenError` when dead) is the honest signal. | +| Claude | `sessionKey` | ~30 days | Opaque token — only detectable from an API rejection. claude.ai answers an invalid session with **403** `permission_error` / `account_session_invalid`, **not** 401; `doctor` reports it on the "Claude API reachable" row. | ### Finding Tokens in Chrome DevTools @@ -180,20 +180,38 @@ The wizard detects your OS, shows the correct DevTools shortcut, and writes the 4. In the left panel, expand **Cookies** and click the site URL 5. Find the cookie by name and copy its **Value** -**ChatGPT:** go to `https://chatgpt.com` → find **two** cookies: -- `__Secure-next-auth.session-token.0` — copy Value (starts with `eyJ`) → `CHATGPT_SESSION_TOKEN` -- `__Secure-next-auth.session-token.1` — copy Value → `CHATGPT_SESSION_TOKEN_1` +**ChatGPT:** go to `https://chatgpt.com` → find the session token cookie. You will +see **one of two layouts**, depending on how large your session token is: -ChatGPT splits large session tokens across two cookies to stay under the browser's 4KB cookie limit. Both are required. +- **One cookie**, `__Secure-next-auth.session-token` — copy Value → `CHATGPT_SESSION_TOKEN`, and leave `CHATGPT_SESSION_TOKEN_1` empty. +- **Two cookies**, `__Secure-next-auth.session-token.0` and `.1` — copy `.0` (starts with `eyJ`) → `CHATGPT_SESSION_TOKEN`, and `.1` → `CHATGPT_SESSION_TOKEN_1`. + +Chrome splits a cookie only when it exceeds ~4KB, so a larger session is chunked +and a smaller one is not — the same account can differ from machine to machine. +`CHATGPT_SESSION_TOKEN_1` is optional; both layouts authenticate, because the +server reassembles a complete value sent under the `.0` name. + +What does *not* work is sending a **partial** chunk — `.0` on its own when a `.1` +exists. That fails silently: `/api/auth/session` answers HTTP 200 with no +`accessToken` rather than an error. If you see two cookies, copy both. **Claude:** go to `https://claude.ai` → find `sessionKey` → copy Value ### When Tokens Expire -When a token expires you'll see a `401 Unauthorized` error. To refresh: +An expired token shows up as an authentication error naming the cookie to +refresh and how. The status differs by provider — ChatGPT reports 401, while +claude.ai reports **403 "Invalid authorization"** (`account_session_invalid`) — +so don't read a 403 from Claude as a permissions problem with your account. + +To refresh: - Re-run the `auth` wizard: `ai-chat-exporter auth` - Or manually update the value in your `.env` file +`ai-chat-exporter doctor` is the quickest check: the "token set" rows only test +that a value is present, so an expired credential passes those and fails on the +"API reachable" row. + --- ## The `auth` Command @@ -776,12 +794,12 @@ To force a full re-export: `ai-chat-exporter cache --clear` then re-run export. ## Troubleshooting -### `401 Unauthorized` +### `Authentication failed` (401, or 403 from Claude) Your session token has expired. - Run `ai-chat-exporter auth` to get a new token interactively - Or manually copy a fresh cookie value into your `.env` file -Note: Claude's `sessionKey` is an opaque string — the only way to know it's expired is the 401 error. ChatGPT JWTs have an `exp` claim that the `doctor` command can decode and display. +Note: neither token's expiry can be read client-side. Claude's `sessionKey` is an opaque string, and claude.ai reports an invalid one as **403** "Invalid authorization" (`account_session_invalid`), not 401. ChatGPT's token is an encrypted JWE; `doctor` reads the `error` field of `/api/auth/session` instead. See [When Tokens Expire](#when-tokens-expire). ### `429 Rate Limited` The tool automatically pauses, saves progress, and exits with a clear message showing how many conversations were exported vs remaining. Just re-run the same export command to resume — the cache picks up exactly where it left off. diff --git a/src/main.py b/src/main.py index 51327be..62cc3f9 100644 --- a/src/main.py +++ b/src/main.py @@ -258,7 +258,10 @@ def _auth_claude(os_name: str) -> None: console.print("2. Press [bold]F12[/bold] to open DevTools → Application tab.") console.print("3. Expand [bold]Cookies[/bold] → [bold]https://claude.ai[/bold]") console.print("4. Find [bold]sessionKey[/bold] → copy the Value.") - console.print(" (Note: Claude tokens expire after ~30 days; a 401 error is the only signal.)") + console.print( + " (Note: Claude tokens expire after ~30 days; an API rejection — 403 " + "'Invalid authorization' — is the only signal.)" + ) console.print("5. Paste it below (input is hidden).\n") key = click.prompt("Claude session key", hide_input=True, default="", show_default=False).strip() diff --git a/src/providers/base.py b/src/providers/base.py index 7e66b34..820880b 100644 --- a/src/providers/base.py +++ b/src/providers/base.py @@ -335,7 +335,8 @@ class BaseProvider(ABC): Parsed JSON response body. Raises: - ProviderError: On 401, exhausted retries, or unrecoverable errors. + ProviderError: On an authentication failure, exhausted retries, or + unrecoverable errors. """ kwargs.setdefault("timeout", REQUEST_TIMEOUT) @@ -361,14 +362,19 @@ class BaseProvider(ABC): elapsed_ms, ) - # ── 401: token expired / invalid ────────────────────────── - if response.status_code == 401: - self._handle_401() - # _handle_401 raises ProviderError — this line never runs + # ── Auth failure: token expired / invalid ───────────────── + # Not keyed on 401 alone: a provider is free to answer an + # invalid session with some other status, and claude.ai does + # (403). Ask the provider rather than assuming. + if self._is_auth_failure(response): + self._handle_auth_failure(response) + # _handle_auth_failure raises — this line never runs raise ProviderError( self.provider_name, f"{method} {url}", - RuntimeError("401 Unauthorized"), + RuntimeError( + f"HTTP {response.status_code} — not authenticated" + ), ) # ── 429: rate limited ────────────────────────────────────── @@ -476,11 +482,22 @@ class BaseProvider(ABC): last_exc or RuntimeError("Unknown error"), ) - def _handle_401(self) -> None: - """Log a clear human-readable message for a 401 and raise ProviderError.""" + def _is_auth_failure(self, response: Any) -> bool: + """Whether this response means the stored credential is not valid. + + Defaults to 401. Providers whose API reports an invalid session with a + different status override this — claude.ai returns 403, so keying auth + handling on 401 alone reports an expired key as a generic permission + error and never tells the user to refresh it. + """ + return bool(response.status_code == 401) + + def _handle_auth_failure(self, response: Any) -> None: + """Log a clear human-readable message for an auth failure and raise.""" # Subclasses override to include provider-specific cookie name msg = ( - f"[{self.provider_name}] Authentication failed (401 Unauthorized). " + f"[{self.provider_name}] Authentication failed " + f"(HTTP {response.status_code}). " "Your session token has likely expired. " "Run 'ai-chat-exporter auth' to refresh your token." ) @@ -488,7 +505,9 @@ class BaseProvider(ABC): raise ProviderError( self.provider_name, "authentication", - RuntimeError("401 Unauthorized — token expired"), + RuntimeError( + f"HTTP {response.status_code} — session token expired or invalid" + ), ) @staticmethod diff --git a/src/providers/chatgpt.py b/src/providers/chatgpt.py index fbd3ffa..8222a94 100644 --- a/src/providers/chatgpt.py +++ b/src/providers/chatgpt.py @@ -307,9 +307,9 @@ class ChatGPTProvider(BaseProvider): ) return access_token - def _handle_401(self) -> None: + def _handle_auth_failure(self, response: Any) -> None: msg = ( - "[chatgpt] Authentication failed (401 Unauthorized). " + f"[chatgpt] Authentication failed (HTTP {response.status_code}). " "Your __Secure-next-auth.session-token has likely expired (~7 day lifetime). " "The session token is used to obtain a short-lived access token via /api/auth/session. " "To refresh: open chatgpt.com in Chrome → F12 → Application → Cookies " @@ -320,7 +320,9 @@ class ChatGPTProvider(BaseProvider): raise ProviderError( self.provider_name, "authentication", - RuntimeError("401 Unauthorized — ChatGPT token expired"), + RuntimeError( + f"HTTP {response.status_code} — ChatGPT session token expired" + ), ) # ------------------------------------------------------------------ diff --git a/src/providers/claude.py b/src/providers/claude.py index a96b48f..a0ee891 100644 --- a/src/providers/claude.py +++ b/src/providers/claude.py @@ -2,6 +2,7 @@ import logging import os +from typing import Any from curl_cffi import requests as curl_requests @@ -38,7 +39,9 @@ class ClaudeProvider(BaseProvider): Cloudflare's bot detection (same issue as chatgpt.com). Authentication: sessionKey cookie (~30 day lifetime, opaque string). - Expiry cannot be decoded client-side — a 401 is the only signal. + Expiry cannot be decoded client-side, so an API rejection is the only + signal — and claude.ai sends 403 permission_error / account_session_invalid + for an invalid session, not 401. See ``_is_auth_failure``. """ provider_name = "claude" @@ -71,11 +74,41 @@ class ClaudeProvider(BaseProvider): self._org_id: str | None = None # cached per session logger.debug("[claude] Session initialised with Chrome TLS impersonation (key: [REDACTED])") - def _handle_401(self) -> None: + # claude.ai answers an invalid or expired sessionKey with 403 + # permission_error / account_session_invalid — never 401. Verified live + # 2026-09-20 against GET /api/organizations: a valid key returns 200, while + # an expired key, a garbage key and no cookie at all return byte-identical + # 403s carrying this code. Matching on the code rather than on the bare + # status keeps a genuine permission problem (which would carry a different + # code) reported as itself. + _SESSION_INVALID_CODE = "account_session_invalid" + + def _is_auth_failure(self, response: Any) -> bool: + if response.status_code == 401: + return True + if response.status_code != 403: + return False + try: + body = response.json() + except Exception: + return False + if not isinstance(body, dict): + return False + error = body.get("error") + if not isinstance(error, dict): + return False + details = error.get("details") + if not isinstance(details, dict): + return False + return details.get("error_code") == self._SESSION_INVALID_CODE + + def _handle_auth_failure(self, response: Any) -> None: msg = ( - "[claude] Authentication failed (401 Unauthorized). " + f"[claude] Authentication failed (HTTP {response.status_code}). " "Your sessionKey has likely expired (~30 day lifetime). " - "Note: Claude session keys are opaque — a 401 is the only expiry signal. " + "Note: Claude session keys are opaque, and claude.ai reports an " + "invalid one as 403 'Invalid authorization', not 401 — an API " + "rejection is the only expiry signal. " "To refresh: open claude.ai in Chrome → F12 → Application → Cookies " "→ find 'sessionKey' → copy the value. " "Then run 'ai-chat-exporter auth' or update CLAUDE_SESSION_KEY in .env." @@ -84,7 +117,9 @@ class ClaudeProvider(BaseProvider): raise ProviderError( self.provider_name, "authentication", - RuntimeError("401 Unauthorized — Claude session key expired"), + RuntimeError( + f"HTTP {response.status_code} — Claude session key expired or invalid" + ), ) def _get_org_id(self) -> str: diff --git a/tests/test_providers.py b/tests/test_providers.py index 3ba3722..acb9614 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -1184,6 +1184,133 @@ class TestErrorBodyDiagnostics: assert "[REDACTED]" in described +# --------------------------------------------------------------------------- +# Auth failure detection: claude.ai reports an invalid/expired sessionKey as +# 403 permission_error / account_session_invalid, never 401. Verified live +# 2026-09-20 against GET /api/organizations — a valid key returned 200, while +# an expired key, a garbage key and no cookie at all returned byte-identical +# 403s. Keying auth handling on 401 alone hid the refresh instructions behind +# a raw JSON dump. +# --------------------------------------------------------------------------- + + +class TestAuthFailureDetection: + class _Resp: + reason = "" + headers: dict = {} + + def __init__(self, status, payload=None): + self.status_code = status + self.ok = 200 <= status < 400 + self._payload = payload + self.text = "" + + def json(self): + if self._payload is None: + raise ValueError("not json") + return self._payload + + @staticmethod + def _bare(cls, response): + p = cls.__new__(cls) + p._request_delay = 0 + p._last_request_at = None + p._session = type("S", (), {"request": lambda *a, **k: response})() + return p + + @staticmethod + def _session_invalid(message="Invalid authorization"): + return { + "type": "error", + "error": { + "type": "permission_error", + "message": message, + "details": { + "error_code": "account_session_invalid", + "error_visibility": "user_facing", + }, + }, + } + + def test_claude_403_session_invalid_is_an_auth_failure(self): + from src.providers.base import ProviderError + from src.providers.claude import ClaudeProvider + + resp = self._Resp(403, self._session_invalid()) + prov = self._bare(ClaudeProvider, resp) + assert prov._is_auth_failure(resp) is True + + with pytest.raises(ProviderError) as exc: + prov._make_request("GET", "https://claude.ai/api/organizations") + # The actionable message, not a dump of the response body. + assert exc.value.operation == "authentication" + assert "session key expired or invalid" in str(exc.value.original) + + def test_claude_403_auth_failure_tells_the_user_how_to_refresh(self, caplog): + from src.providers.base import ProviderError + from src.providers.claude import ClaudeProvider + + resp = self._Resp(403, self._session_invalid()) + with caplog.at_level(logging.ERROR): + with pytest.raises(ProviderError): + self._bare(ClaudeProvider, resp)._make_request( + "GET", "https://claude.ai/api/organizations" + ) + logged = caplog.text + assert "sessionKey" in logged + assert "CLAUDE_SESSION_KEY" in logged + # States the status it actually saw, rather than claiming 401. + assert "403" in logged + assert "401 Unauthorized" not in logged + + def test_claude_403_with_another_error_code_is_not_an_auth_failure(self): + """A genuine permission problem must stay reported as itself.""" + from src.providers.base import ProviderError + from src.providers.claude import ClaudeProvider + + payload = { + "type": "error", + "error": { + "type": "permission_error", + "message": "Organization access denied", + "details": {"error_code": "org_access_denied"}, + }, + } + resp = self._Resp(403, payload) + prov = self._bare(ClaudeProvider, resp) + assert prov._is_auth_failure(resp) is False + + with pytest.raises(ProviderError) as exc: + prov._make_request("GET", "https://claude.ai/api/organizations") + assert exc.value.operation != "authentication" + assert "Organization access denied" in str(exc.value.original) + + def test_claude_403_with_unparseable_body_is_not_an_auth_failure(self): + from src.providers.claude import ClaudeProvider + + resp = self._Resp(403, None) + assert self._bare(ClaudeProvider, resp)._is_auth_failure(resp) is False + + def test_claude_401_is_still_an_auth_failure(self): + from src.providers.claude import ClaudeProvider + + resp = self._Resp(401, {}) + assert self._bare(ClaudeProvider, resp)._is_auth_failure(resp) is True + + def test_chatgpt_403_is_not_an_auth_failure(self): + """Guards the deleted-asset path: a media 403 is not an expired token.""" + from src.providers.chatgpt import ChatGPTProvider + + resp = self._Resp(403, {"detail": "Forbidden"}) + assert self._bare(ChatGPTProvider, resp)._is_auth_failure(resp) is False + + def test_chatgpt_401_is_an_auth_failure(self): + from src.providers.chatgpt import ChatGPTProvider + + resp = self._Resp(401, {}) + assert self._bare(ChatGPTProvider, resp)._is_auth_failure(resp) is True + + # --------------------------------------------------------------------------- # Deleted assets: ChatGPT's download endpoint answers a missing upload with # 403 Forbidden, not 404. Measured 2026-08-17 over 18 such assets — every one