Jesse.Markowitz/AIChatExporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: v0.2.0 — Joplin import, ChatGPT Projects, --project filter

Browse Source 
Core features:
- Add `joplin` command: syncs exported Markdown to Joplin via local REST API
- Notebooks auto-created per provider+project (e.g. "ChatGPT - My Project")
- Idempotent: notes updated (not duplicated) on re-run; note ID tracked in manifest
- Add `--project` filter to `export` and `list` commands (substring or 'none')
- Add ChatGPT Projects support via CHATGPT_PROJECT_IDS env var

Config:
- Add JOPLIN_API_TOKEN, JOPLIN_API_URL, JOPLIN_REQUEST_TIMEOUT
- Version now read from importlib.metadata (single source of truth: pyproject.toml)
- Bump version to 0.2.0

Quality:
- Explicit Timeout handling in JoplinClient with actionable error messages
- token validation (validate_token) separate from connectivity (ping)
- Remove debug_auth.py, debug_claude.py, and untracked .har file
- Add *.har to .gitignore (may contain auth cookies/session tokens)
- Update README, CHANGELOG, FUTURE.md to reflect v0.2.0

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Jesse.Markowitz
2026-03-01 06:04:03 -05:00
co-authored by Claude Sonnet 4.6
parent 23d7c17255
commit 304cf4fde4
16 changed files with 1795 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/providers/chatgpt.py
+381 -14
View File
@@ -1,4 +1,23 @@
"""ChatGPT provider — accesses chat.openai.com internal web API."""
"""ChatGPT provider — accesses chat.openai.com internal web API.
ChatGPT Projects discovery
--------------------------
ChatGPT Projects are internally implemented as "snorlax"-type gizmos with IDs
starting with "g-p-". They are *not* returned by any gizmo listing endpoint
(/gizmos/mine, /gizmos/pinned, /gizmos/discovery, /gizmos/search). The
frontend appears to load project IDs from page-level state, not a dedicated
listing API.
Therefore, project IDs must be supplied by the user via CHATGPT_PROJECT_IDS.
Each project gizmo ID looks like "g-p-68c2b2b3037c8191890036fb4ae3ed9f" and
can be read from the browser URL when viewing a project:
https://chatgpt.com/g/{project-gizmo-id}-{slug}/project
Project conversations are fetched via cursor-based pagination at:
GET /backend-api/gizmos/{project_gizmo_id}/conversations?cursor=0
Response: {"items": [...], "cursor": "<opaque_base64_or_null>"}
Pagination ends when cursor is null or an empty string.
"""
import logging
import os
@@ -34,17 +53,22 @@ class ChatGPTProvider(BaseProvider):
provider_name = "chatgpt"
def __init__(self, session_token: str | None = None) -> None:
def __init__(
self,
session_token: str | None = None,
project_ids: list[str] | None = None,
) -> None:
# Pass a curl_cffi session to the base class instead of a requests.Session.
# curl_cffi.requests.Session is API-compatible with requests.Session.
cf_session = curl_requests.Session(impersonate=IMPERSONATE)
super().__init__(session=cf_session) # type: ignore[arg-type]
# Remove the User-Agent set by BaseProvider. curl_cffi sets a UA that is
# consistent with its TLS JA3 fingerprint for chrome120. If we leave a
# mismatched UA (e.g. Chrome/121 header with Chrome/120 TLS), Cloudflare's
# bot detection flags it. Removing it lets curl_cffi manage its own UA.
# Remove headers that curl_cffi manages as part of its Chrome fingerprint.
# Overriding User-Agent, Accept, or Accept-Language with non-Chrome values
# creates header/TLS inconsistencies that Cloudflare's bot detection flags.
self._session.headers.pop("User-Agent", None)
self._session.headers.pop("Accept", None)
self._session.headers.pop("Accept-Language", None)
token = session_token or os.getenv("CHATGPT_SESSION_TOKEN", "").strip()
if not token:
@@ -58,6 +82,17 @@ class ChatGPTProvider(BaseProvider):
)
self._session_token = token
# Project gizmo IDs (g-p-xxx) whose conversations we'll fetch.
# ChatGPT project conversations do not appear in the default
# /conversations listing — they require explicit project IDs.
self._project_ids: list[str] = project_ids or []
# Maps conv_id → project_name; populated by fetch_all_conversations()
self._project_map: dict[str, str] = {}
# Cache of project_id → display name (avoids re-fetching gizmo details)
self._project_name_cache: dict[str, str] = {}
# Set the session cookie in the cookie jar
self._session.cookies.set(
"__Secure-next-auth.session-token",
@@ -66,10 +101,13 @@ class ChatGPTProvider(BaseProvider):
path="/",
)
# Set only Referer and sec-fetch-* headers for the auth exchange.
# Origin is intentionally omitted: Chrome does not send Origin on
# same-origin GET requests, and its presence alongside
# sec-fetch-site: same-origin contradicts the browser fingerprint.
self._session.headers.update(
{
"Referer": "https://chatgpt.com/",
"Origin": "https://chatgpt.com",
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-origin",
@@ -78,8 +116,16 @@ class ChatGPTProvider(BaseProvider):
# Exchange the session cookie for an access token
self._access_token: str = self._fetch_access_token()
# Now set backend-api headers (after auth, so they don't interfere with
# the auth exchange which expects a browser-style request).
self._session.headers["Authorization"] = f"Bearer {self._access_token}"
logger.debug("[chatgpt] Session initialised with Chrome TLS impersonation (token: [REDACTED])")
self._session.headers["Accept"] = "application/json"
self._session.headers["Origin"] = "https://chatgpt.com"
logger.debug(
"[chatgpt] Session initialised (Chrome TLS impersonation, %d project ID(s) configured)",
len(self._project_ids),
)
def _fetch_access_token(self) -> str:
"""Exchange the session cookie for a Bearer access token.
@@ -132,14 +178,22 @@ class ChatGPTProvider(BaseProvider):
RuntimeError("401 Unauthorized — ChatGPT token expired"),
)
# ------------------------------------------------------------------
# Default workspace conversations (offset-based pagination)
# ------------------------------------------------------------------
def list_conversations(self, offset: int = 0, limit: int = 100) -> list[dict]:
"""Fetch one page of conversations.
"""Fetch one page of conversations from the default workspace.
Note: Project conversations are NOT included here. They require
separate fetching via list_project_conversations().
Returns:
List of conversation summary dicts.
"""
url = f"{BASE_URL}/conversations"
params = {"offset": offset, "limit": limit, "order": "updated"}
logger.debug("[chatgpt] list_conversations: GET %s params=%s", url, params)
try:
data = self._make_request("GET", url, params=params)
except ProviderError:
@@ -149,18 +203,315 @@ class ChatGPTProvider(BaseProvider):
if not isinstance(data, dict):
self._warn_unexpected_schema("list_conversations", "root")
logger.debug("[chatgpt] list_conversations: unexpected root type %s", type(data))
return []
items = data.get("items")
if items is None:
self._warn_unexpected_schema("list_conversations", "items")
logger.debug("[chatgpt] list_conversations: response keys = %s", list(data.keys()))
return []
logger.debug("[chatgpt] list_conversations: got %d items (offset=%d)", len(items), offset)
return items
# ------------------------------------------------------------------
# Project conversations (cursor-based pagination)
# ------------------------------------------------------------------
def _fetch_project_name(self, project_id: str) -> str:
"""Fetch the display name for a project gizmo.
Calls GET /backend-api/gizmos/{project_id} and returns the display
name from gizmo.display.name. Falls back to the project_id itself
if the fetch fails or the name is missing.
Result is cached in self._project_name_cache.
"""
if project_id in self._project_name_cache:
return self._project_name_cache[project_id]
url = f"{BASE_URL}/gizmos/{project_id}"
logger.debug("[chatgpt] _fetch_project_name: GET %s", url)
try:
data = self._make_request("GET", url)
gizmo = data.get("gizmo", {}) if isinstance(data, dict) else {}
name = (gizmo.get("display") or {}).get("name") or gizmo.get("name") or ""
name = name.strip() or project_id
gizmo_type = gizmo.get("gizmo_type", "?")
logger.debug(
"[chatgpt] _fetch_project_name[%s]: name=%r gizmo_type=%r",
project_id[:12],
name,
gizmo_type,
)
except ProviderError as e:
logger.warning(
"[chatgpt] Could not fetch project name for %s: %s — using ID as name",
project_id,
e,
)
name = project_id
self._project_name_cache[project_id] = name
return name
def list_project_conversations(
self, project_id: str, cursor: str = "0"
) -> tuple[list[dict], str | None]:
"""Fetch one page of conversations for a project gizmo.
Uses cursor-based pagination (not offset). The initial cursor is "0".
Subsequent cursors come from the response's "cursor" field.
Endpoint: GET /backend-api/gizmos/{project_id}/conversations?cursor=<cursor>
Returns:
(items, next_cursor) — next_cursor is None or "" when exhausted.
"""
url = f"{BASE_URL}/gizmos/{project_id}/conversations"
params = {"cursor": cursor}
logger.debug(
"[chatgpt] list_project_conversations[%s]: GET %s cursor=%r",
project_id[:12],
url,
cursor,
)
try:
data = self._make_request("GET", url, params=params)
except ProviderError:
raise
except Exception as e:
raise ProviderError(self.provider_name, "list_project_conversations", e) from e
logger.debug(
"[chatgpt] list_project_conversations[%s]: response type=%s",
project_id[:12],
type(data).__name__,
)
if isinstance(data, list):
# Bare list — no next cursor available
logger.debug(
"[chatgpt] list_project_conversations[%s]: bare list with %d items",
project_id[:12],
len(data),
)
return data, None
if not isinstance(data, dict):
self._warn_unexpected_schema("list_project_conversations", "root")
logger.debug(
"[chatgpt] list_project_conversations[%s]: unexpected type %s value=%r",
project_id[:12],
type(data),
data,
)
return [], None
logger.debug(
"[chatgpt] list_project_conversations[%s]: response keys=%s",
project_id[:12],
list(data.keys()),
)
items = data.get("items") or data.get("conversations") or []
next_cursor = data.get("cursor") or None # empty string → treat as None
if not items and data:
logger.debug(
"[chatgpt] list_project_conversations[%s]: no items found; full response=%r",
project_id[:12],
data,
)
logger.debug(
"[chatgpt] list_project_conversations[%s]: %d items, next_cursor=%r",
project_id[:12],
len(items),
next_cursor[:20] + "…" if next_cursor and len(next_cursor) > 20 else next_cursor,
)
return items, next_cursor
# ------------------------------------------------------------------
# Combined fetch (default workspace + all configured projects)
# ------------------------------------------------------------------
def fetch_all_conversations(self, since=None) -> list[dict]:
"""Fetch all conversations: default workspace + every configured project.
ChatGPT project conversations are not included in the default
/conversations listing. They must be fetched separately via the
gizmos conversations endpoint using project IDs from CHATGPT_PROJECT_IDS.
Builds self._project_map (conv_id → project_name) as a side effect so
that normalize_conversation() can attach the project name without an
additional API call.
Args:
since: Optional datetime — only return conversations updated at or
after this time (client-side filter, same as base class).
Returns:
Combined list of raw conversation summary dicts.
"""
# Reset maps so a fresh fetch always rebuilds them cleanly
self._project_map = {}
# --- Default workspace (base class handles offset-based pagination) ---
logger.info("[chatgpt] Fetching default workspace conversations…")
default_convs = super().fetch_all_conversations(since=None)
logger.info("[chatgpt] Default workspace: %d conversations", len(default_convs))
# --- Project conversations ---
if not self._project_ids:
logger.info(
"[chatgpt] No project IDs configured — skipping project conversations. "
"To include projects, set CHATGPT_PROJECT_IDS in .env "
"(see 'python -m src.main auth' for instructions)."
)
return self._apply_since_filter(default_convs, since)
logger.info(
"[chatgpt] Fetching conversations for %d project(s): %s",
len(self._project_ids),
self._project_ids,
)
project_convs: list[dict] = []
for project_id in self._project_ids:
project_name = self._fetch_project_name(project_id)
logger.info(
"[chatgpt] Project '%s' (%s): fetching conversations…",
project_name,
project_id,
)
cursor: str = "0"
page = 0
project_total = 0
while True:
page += 1
logger.debug(
"[chatgpt] Project '%s': page %d cursor=%r",
project_name,
page,
cursor[:20] + "…" if len(cursor) > 20 else cursor,
)
try:
batch, next_cursor = self.list_project_conversations(
project_id, cursor=cursor
)
except ProviderError as e:
logger.warning(
"[chatgpt] Project '%s': failed to fetch page %d: %s — stopping pagination",
project_name,
page,
e,
)
break
if not batch:
logger.debug(
"[chatgpt] Project '%s': empty batch on page %d — done",
project_name,
page,
)
break
for conv in batch:
conv_id = conv.get("id")
if conv_id:
self._project_map[conv_id] = project_name
else:
logger.debug(
"[chatgpt] Project '%s': conversation with no id: %r",
project_name,
conv,
)
# Annotate so callers can filter by project without the map
conv["_project_name"] = project_name
project_convs.extend(batch)
project_total += len(batch)
logger.debug(
"[chatgpt] Project '%s': page %d → %d items (project total: %d)",
project_name,
page,
len(batch),
project_total,
)
if not next_cursor:
logger.debug(
"[chatgpt] Project '%s': no next cursor — pagination complete",
project_name,
)
break
cursor = next_cursor
logger.info(
"[chatgpt] Project '%s': %d conversations fetched",
project_name,
project_total,
)
all_convs = default_convs + project_convs
logger.info(
"[chatgpt] Total: %d conversations (%d default + %d from %d project(s))",
len(all_convs),
len(default_convs),
len(project_convs),
len(self._project_ids),
)
logger.debug(
"[chatgpt] _project_map: %d entries → %s",
len(self._project_map),
{k[:8]: v for k, v in self._project_map.items()},
)
return self._apply_since_filter(all_convs, since)
def _apply_since_filter(self, convs: list[dict], since) -> list[dict]:
"""Filter conversations to those updated at or after `since`."""
if since is None:
return convs
since_naive = since.replace(tzinfo=None)
filtered = []
for c in convs:
raw_ts = c.get("updated_at") or c.get("update_time") or ""
if raw_ts:
try:
from src.utils import _parse_dt
updated = _parse_dt(str(raw_ts)).replace(tzinfo=None)
if updated >= since_naive:
filtered.append(c)
except Exception:
filtered.append(c) # include if date unparseable
else:
filtered.append(c)
logger.info(
"[chatgpt] After --since filter: %d/%d conversations",
len(filtered),
len(convs),
)
return filtered
# ------------------------------------------------------------------
# Single conversation detail
# ------------------------------------------------------------------
def get_conversation(self, conv_id: str) -> dict:
"""Fetch full conversation detail for a single ID."""
url = f"{BASE_URL}/conversation/{conv_id}"
logger.debug("[chatgpt] get_conversation: GET %s", url)
try:
data = self._make_request("GET", url)
except ProviderError:
@@ -172,25 +523,41 @@ class ChatGPTProvider(BaseProvider):
self._warn_unexpected_schema("get_conversation", "root")
return {}
logger.debug(
"[chatgpt] get_conversation[%s]: keys=%s mapping_size=%d",
conv_id[:8],
list(data.keys()),
len(data.get("mapping", {})),
)
return data
# ------------------------------------------------------------------
# Normalization
# ------------------------------------------------------------------
def normalize_conversation(self, raw: dict) -> dict:
"""Transform ChatGPT raw schema to the common normalized schema.
ChatGPT stores messages in a nested ``mapping`` dict where each node
has an ``id``, ``message``, and ``children`` list. We walk the tree
from the root node to build a flat ordered message list.
Project name is looked up from self._project_map (populated by
fetch_all_conversations). The conversation detail endpoint does not
include project information.
"""
conv_id = raw.get("id", "")
title = raw.get("title") or "Untitled"
created_at = _ts_to_iso(raw.get("create_time"))
updated_at = _ts_to_iso(raw.get("update_time"))
# Project info — ChatGPT calls it "gizmo_id" or stores project info differently.
# As of 2024, personal projects appear as a separate projects API; conversations
# linked to a project have a non-null `workspace_id` or similar field.
# We use `project_title` if present, else None.
project: str | None = raw.get("project_title") or raw.get("workspace_title") or None
# Look up project name from the map built during fetch_all_conversations.
project = self._project_map.get(conv_id) if conv_id else None
logger.debug(
"[chatgpt] normalize_conversation[%s]: project_map lookup → %r",
conv_id[:8] if conv_id else "?",
project,
)
mapping: dict = raw.get("mapping", {})
messages = _extract_messages(mapping, raw, conv_id)