From a4b7def6e1afdcf29b6bda25e1c9f2e517fe8490 Mon Sep 17 00:00:00 2001
From: Anton Daitche <a.daitche@gmail.com>
Date: Sun, 13 Sep 2026 08:23:58 +0200
Subject: [PATCH] Paginate publication chat threads

Previously, get_threads() returned only the first page. Fetch older pages with the before cursor and stop once the requested limit is reached.

Tested against the live Substack API with single-page and multi-page limits.
---
 substack_api/chat.py | 39 ++++++++++++++++++++++++++++++++-------
 tests/test_chat.py   | 21 +++++++++++++++++++++
 2 files changed, 53 insertions(+), 7 deletions(-)

diff --git a/substack_api/chat.py b/substack_api/chat.py
index 34a1cd7..cb900f0 100644
--- a/substack_api/chat.py
+++ b/substack_api/chat.py
@@ -531,7 +531,9 @@ def publication_id(self) -> int:
         """The publication ID for this chat."""
         return self._publication_id
 
-    def _fetch_threads_data(self, force_refresh: bool = False) -> Dict[str, Any]:
+    def _fetch_threads_data(
+        self, force_refresh: bool = False, limit: Optional[int] = None
+    ) -> Dict[str, Any]:
         """
         Fetch threads data from the API.
 
@@ -539,6 +541,8 @@ def _fetch_threads_data(self, force_refresh: bool = False) -> Dict[str, Any]:
         ----------
         force_refresh : bool
             If True, fetch fresh data even if cached data exists.
+        limit : Optional[int]
+            Stop after fetching this many threads.
 
         Returns
         -------
@@ -557,6 +561,9 @@ def _fetch_threads_data(self, force_refresh: bool = False) -> Dict[str, Any]:
         if self._threads_data is not None and not force_refresh:
             return self._threads_data
 
+        if force_refresh:
+            self._threads_data = None
+
         if not self.auth or not self.auth.authenticated:
             raise ChatAuthenticationRequired(
                 "Authentication is required to access publication chats."
@@ -584,8 +591,28 @@ def _fetch_threads_data(self, force_refresh: bool = False) -> Dict[str, Any]:
             )
 
         response.raise_for_status()
-        self._threads_data = response.json()
-        return self._threads_data
+        data = response.json()
+
+        while data.get("moreBefore") and data.get("threads"):
+            if limit is not None and len(data["threads"]) >= limit:
+                break
+
+            before = data["threads"][-1]["communityPost"]["created_at"]
+            response = self.auth.get(url, params={"before": before}, timeout=30)
+            response.raise_for_status()
+            page = response.json()
+            page_threads = page.get("threads", [])
+
+            if not page_threads:
+                break
+
+            data["threads"].extend(page_threads)
+            data["moreBefore"] = page.get("moreBefore", False)
+
+        if limit is None:
+            self._threads_data = data
+
+        return data
 
     def get_threads(
         self, limit: Optional[int] = None, force_refresh: bool = False
@@ -596,9 +623,7 @@ def get_threads(
         Parameters
         ----------
         limit : Optional[int]
-            Client-side truncation of the first page of results returned by the
-            API. The full page is always fetched; this just slices the list.
-            If None, returns all threads from the page.
+            Maximum threads returned.
         force_refresh : bool
             If True, fetch fresh data from the API.
 
@@ -614,7 +639,7 @@ def get_threads(
         ChatNotFound
             If the publication is not found.
         """
-        data = self._fetch_threads_data(force_refresh=force_refresh)
+        data = self._fetch_threads_data(force_refresh=force_refresh, limit=limit)
         threads = [
             ChatThread(
                 publication_id=self._publication_id,
diff --git a/tests/test_chat.py b/tests/test_chat.py
index c352570..9ef8b73 100644
--- a/tests/test_chat.py
+++ b/tests/test_chat.py
@@ -522,6 +522,27 @@ def test_get_threads_caching(self, mock_auth, sample_threads_data):
         chat.get_threads(force_refresh=True)
         assert mock_auth.get.call_count == 2
 
+    def test_get_threads_paginates_before(self, mock_auth):
+        def response(threads):
+            result = MagicMock(status_code=200)
+            result.json.return_value = {"threads": threads, "moreBefore": True}
+            return result
+
+        newer = {"communityPost": {"id": "newer", "created_at": "2026-01-20"}}
+        older = {"communityPost": {"id": "older", "created_at": "2026-01-10"}}
+        chat = Chat(publication_id=4906951, auth=mock_auth)
+
+        mock_auth.get.side_effect = [response([newer])]
+        assert [thread.id for thread in chat.get_threads(limit=1)] == ["newer"]
+        assert mock_auth.get.call_count == 1
+
+        mock_auth.get.reset_mock()
+        mock_auth.get.side_effect = [response([newer]), response([older]), response([])]
+
+        threads = chat.get_threads()
+        assert [thread.id for thread in threads] == ["newer", "older"]
+        assert mock_auth.get.call_args.kwargs["params"] == {"before": "2026-01-10"}
+
     def test_get_threads_unauthenticated(self, mock_unauth):
         """Test Chat.get_threads raises error when not authenticated."""
         chat = Chat(publication_id=4906951, auth=mock_unauth)