Refactor fetchItems for pagination (#1143)

* Refactor base fetchItems for pagination

* Use base fetchItems for LibrarySection methods

* Use base fetchItems for MyPlexAccount watchlist

* Use base fetchItems for PlexServer history

* Fix imports

plexapi/base.py

@@ -4,7 +4,7 @@ import weakref
 from urllib.parse import urlencode
 from xml.etree import ElementTree
 
-from plexapi import log, utils
+from plexapi import X_PLEX_CONTAINER_SIZE, log, utils
 from plexapi.exceptions import BadRequest, NotFound, UnknownType, Unsupported
 from plexapi.utils import cached_property
 
@@ -147,42 +147,7 @@ class PlexObject:
         elem = ElementTree.fromstring(xml)
         return self._buildItemOrNone(elem, cls)
 
-    def fetchItem(self, ekey, cls=None, **kwargs):
+    def fetchItems(self, ekey, cls=None, container_start=None, container_size=None, maxresults=None, **kwargs):
-        """ Load the specified key to find and build the first item with the
-            specified tag and attrs. If no tag or attrs are specified then
-            the first item in the result set is returned.
-
-            Parameters:
-                ekey (str or int): Path in Plex to fetch items from. If an int is passed
-                    in, the key will be translated to /library/metadata/<key>. This allows
-                    fetching an item only knowing its key-id.
-                cls (:class:`~plexapi.base.PlexObject`): If you know the class of the
-                    items to be fetched, passing this in will help the parser ensure
-                    it only returns those items. By default we convert the xml elements
-                    with the best guess PlexObjects based on tag and type attrs.
-                etag (str): Only fetch items with the specified tag.
-                **kwargs (dict): Optionally add XML attribute to filter the items.
-                    See :func:`~plexapi.base.PlexObject.fetchItems` for more details
-                    on how this is used.
-        """
-        if ekey is None:
-            raise BadRequest('ekey was not provided')
-        if isinstance(ekey, int):
-            ekey = f'/library/metadata/{ekey}'
-
-        data = self._server.query(ekey)
-        item = self.findItem(data, cls, ekey, **kwargs)
-
-        if item:
-            librarySectionID = utils.cast(int, data.attrib.get('librarySectionID'))
-            if librarySectionID:
-                item.librarySectionID = librarySectionID
-            return item
-
-        clsname = cls.__name__ if cls else 'None'
-        raise NotFound(f'Unable to find elem: cls={clsname}, attrs={kwargs}')
-
-    def fetchItems(self, ekey, cls=None, container_start=None, container_size=None, **kwargs):
         """ Load the specified key to find and build all items with the specified tag
             and attrs.
 
@@ -195,6 +160,7 @@ class PlexObject:
                 etag (str): Only fetch items with the specified tag.
                 container_start (None, int): offset to get a subset of the data
                 container_size (None, int): How many items in data
+                maxresults (int, optional): Only return the specified number of results.
                 **kwargs (dict): Optionally add XML attribute to filter the items.
                     See the details below for more info.
 
@@ -259,39 +225,77 @@ class PlexObject:
         if ekey is None:
             raise BadRequest('ekey was not provided')
 
-        params = {}
+        container_start = container_start or 0
-        if container_start is not None:
+        container_size = container_size or X_PLEX_CONTAINER_SIZE
-            params["X-Plex-Container-Start"] = container_start
+        offset = container_start
-        if container_size is not None:
-            params["X-Plex-Container-Size"] = container_size
 
-        data = self._server.query(ekey, params=params)
+        if maxresults is not None:
-        items = self.findItems(data, cls, ekey, **kwargs)
+            container_size = min(container_size, maxresults)
 
-        librarySectionID = utils.cast(int, data.attrib.get('librarySectionID'))
+        results = []
-        if librarySectionID:
+        subresults = []
-            for item in items:
+        headers = {}
-                item.librarySectionID = librarySectionID
-        return items
 
-    def findItem(self, data, cls=None, initpath=None, rtag=None, **kwargs):
+        while True:
-        """ Load the specified data to find and build the first items with the specified tag
+            headers['X-Plex-Container-Start'] = str(container_start)
-            and attrs. See :func:`~plexapi.base.PlexObject.fetchItem` for more details
+            headers['X-Plex-Container-Size'] = str(container_size)
-            on how this is used.
+
+            data = self._server.query(ekey, headers=headers)
+            subresults = self.findItems(data, cls, ekey, **kwargs)
+            total_size = utils.cast(int, data.attrib.get('totalSize') or data.attrib.get('size')) or len(subresults)
+
+            if not subresults:
+                if offset > total_size:
+                    log.info('container_start is greater than the number of items')
+
+            librarySectionID = utils.cast(int, data.attrib.get('librarySectionID'))
+            if librarySectionID:
+                for item in subresults:
+                    item.librarySectionID = librarySectionID
+
+            results.extend(subresults)
+
+            wanted_number_of_items = total_size - offset
+            if maxresults is not None:
+                wanted_number_of_items = min(maxresults, wanted_number_of_items)
+                container_size = min(container_size, wanted_number_of_items - len(results))
+
+            if wanted_number_of_items <= len(results):
+                break
+
+            container_start += container_size
+
+            if container_start > total_size:
+                break
+
+        return results
+
+    def fetchItem(self, ekey, cls=None, **kwargs):
+        """ Load the specified key to find and build the first item with the
+            specified tag and attrs. If no tag or attrs are specified then
+            the first item in the result set is returned.
+
+            Parameters:
+                ekey (str or int): Path in Plex to fetch items from. If an int is passed
+                    in, the key will be translated to /library/metadata/<key>. This allows
+                    fetching an item only knowing its key-id.
+                cls (:class:`~plexapi.base.PlexObject`): If you know the class of the
+                    items to be fetched, passing this in will help the parser ensure
+                    it only returns those items. By default we convert the xml elements
+                    with the best guess PlexObjects based on tag and type attrs.
+                etag (str): Only fetch items with the specified tag.
+                **kwargs (dict): Optionally add XML attribute to filter the items.
+                    See :func:`~plexapi.base.PlexObject.fetchItems` for more details
+                    on how this is used.
         """
-        # filter on cls attrs if specified
+        if isinstance(ekey, int):
-        if cls and cls.TAG and 'tag' not in kwargs:
+            ekey = f'/library/metadata/{ekey}'
-            kwargs['etag'] = cls.TAG
+
-        if cls and cls.TYPE and 'type' not in kwargs:
+        try:
-            kwargs['type'] = cls.TYPE
+            return self.fetchItems(ekey, cls, **kwargs)[0]
-        # rtag to iter on a specific root tag
+        except IndexError:
-        if rtag:
+            clsname = cls.__name__ if cls else 'None'
-            data = next(data.iter(rtag), [])
+            raise NotFound(f'Unable to find elem: cls={clsname}, attrs={kwargs}') from None
-        # loop through all data elements to find matches
-        for elem in data:
-            if self._checkAttrs(elem, **kwargs):
-                item = self._buildItemOrNone(elem, cls, initpath)
-                return item
 
     def findItems(self, data, cls=None, initpath=None, rtag=None, **kwargs):
         """ Load the specified data to find and build all items with the specified tag
@@ -315,6 +319,16 @@ class PlexObject:
                     items.append(item)
         return items
 
+    def findItem(self, data, cls=None, initpath=None, rtag=None, **kwargs):
+        """ Load the specified data to find and build the first items with the specified tag
+            and attrs. See :func:`~plexapi.base.PlexObject.fetchItem` for more details
+            on how this is used.
+        """
+        try:
+            return self.findItems(data, cls, initpath, rtag, **kwargs)[0]
+        except IndexError:
+            return None
+
     def firstAttr(self, *attrs):
         """ Return the first attribute in attrs that is not None. """
         for attr in attrs:
@@ -643,7 +657,7 @@ class PlexPartialObject(PlexObject):
                 'have not allowed items to be deleted', self.key)
             raise
 
-    def history(self, maxresults=9999999, mindate=None):
+    def history(self, maxresults=None, mindate=None):
         """ Get Play History for a media item.
 
             Parameters:

plexapi/client.py

@@ -3,6 +3,7 @@ import time
 from xml.etree import ElementTree
 
 import requests
+
 from plexapi import BASE_HEADERS, CONFIG, TIMEOUT, log, logfilter, utils
 from plexapi.base import PlexObject
 from plexapi.exceptions import BadRequest, NotFound, Unauthorized, Unsupported

plexapi/library.py

@@ -3,7 +3,7 @@ import re
 from datetime import datetime
 from urllib.parse import quote_plus, urlencode
 
-from plexapi import X_PLEX_CONTAINER_SIZE, log, media, utils
+from plexapi import log, media, utils
 from plexapi.base import OPERATORS, PlexObject
 from plexapi.exceptions import BadRequest, NotFound
 from plexapi.settings import Setting
@@ -352,7 +352,7 @@ class Library(PlexObject):
             part += urlencode(kwargs)
         return self._server.query(part, method=self._server._session.post)
 
-    def history(self, maxresults=9999999, mindate=None):
+    def history(self, maxresults=None, mindate=None):
         """ Get Play History for all library Sections for the owner.
             Parameters:
                 maxresults (int): Only return the specified number of results (optional).
@@ -421,40 +421,6 @@ class LibrarySection(PlexObject):
         self._totalDuration = None
         self._totalStorage = None
 
-    def fetchItems(self, ekey, cls=None, container_start=None, container_size=None, **kwargs):
-        """ Load the specified key to find and build all items with the specified tag
-            and attrs. See :func:`~plexapi.base.PlexObject.fetchItem` for more details
-            on how this is used.
-
-            Parameters:
-                container_start (None, int): offset to get a subset of the data
-                container_size (None, int): How many items in data
-
-        """
-        url_kw = {}
-        if container_start is not None:
-            url_kw["X-Plex-Container-Start"] = container_start
-        if container_size is not None:
-            url_kw["X-Plex-Container-Size"] = container_size
-
-        if ekey is None:
-            raise BadRequest('ekey was not provided')
-        data = self._server.query(ekey, params=url_kw)
-
-        if '/all' in ekey:
-            # totalSize is only included in the xml response
-            # if container size is used.
-            total_size = data.attrib.get("totalSize") or data.attrib.get("size")
-            self._totalViewSize = utils.cast(int, total_size)
-
-        items = self.findItems(data, cls, ekey, **kwargs)
-
-        librarySectionID = utils.cast(int, data.attrib.get('librarySectionID'))
-        if librarySectionID:
-            for item in items:
-                item.librarySectionID = librarySectionID
-        return items
-
     @cached_property
     def totalSize(self):
         """ Returns the total number of items in the library for the default library type. """
@@ -1268,7 +1234,7 @@ class LibrarySection(PlexObject):
         return self._server.search(query, mediatype, limit, sectionId=self.key)
 
     def search(self, title=None, sort=None, maxresults=None, libtype=None,
-               container_start=0, container_size=X_PLEX_CONTAINER_SIZE, limit=None, filters=None, **kwargs):
+               container_start=None, container_size=None, limit=None, filters=None, **kwargs):
         """ Search the library. The http requests will be batched in container_size. If you are only looking for the
             first <num> results, it would be wise to set the maxresults option to that amount so the search doesn't iterate
             over all results on the server.
@@ -1524,43 +1490,8 @@ class LibrarySection(PlexObject):
         """
         key, kwargs = self._buildSearchKey(
             title=title, sort=sort, libtype=libtype, limit=limit, filters=filters, returnKwargs=True, **kwargs)
-        return self._search(key, maxresults, container_start, container_size, **kwargs)
+        return self.fetchItems(
-
+            key, container_start=container_start, container_size=container_size, maxresults=maxresults, **kwargs)
-    def _search(self, key, maxresults, container_start, container_size, **kwargs):
-        """ Perform the actual library search and return the results. """
-        results = []
-        subresults = []
-        offset = container_start
-
-        if maxresults is not None:
-            container_size = min(container_size, maxresults)
-
-        while True:
-            subresults = self.fetchItems(key, container_start=container_start,
-                                         container_size=container_size, **kwargs)
-            if not len(subresults):
-                if offset > self._totalViewSize:
-                    log.info("container_start is higher than the number of items in the library")
-
-            results.extend(subresults)
-
-            # self._totalViewSize is not used as a condition in the while loop as
-            # this require a additional http request.
-            # self._totalViewSize is updated from self.fetchItems
-            wanted_number_of_items = self._totalViewSize - offset
-            if maxresults is not None:
-                wanted_number_of_items = min(maxresults, wanted_number_of_items)
-                container_size = min(container_size, maxresults - len(results))
-
-            if wanted_number_of_items <= len(results):
-                break
-
-            container_start += container_size
-
-            if container_start > self._totalViewSize:
-                break
-
-        return results
 
     def _locations(self):
         """ Returns a list of :class:`~plexapi.library.Location` objects
@@ -1637,7 +1568,7 @@ class LibrarySection(PlexObject):
 
         return myplex.sync(client=client, clientId=clientId, sync_item=sync_item)
 
-    def history(self, maxresults=9999999, mindate=None):
+    def history(self, maxresults=None, mindate=None):
         """ Get Play History for this library Section for the owner.
             Parameters:
                 maxresults (int): Only return the specified number of results (optional).

plexapi/mixins.py

@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
 from datetime import datetime
-
 from urllib.parse import parse_qsl, quote, quote_plus, unquote, urlencode, urlsplit
 
 from plexapi import media, settings, utils

plexapi/myplex.py

@@ -7,8 +7,9 @@ from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
 from xml.etree import ElementTree
 
 import requests
-from plexapi import (BASE_HEADERS, CONFIG, TIMEOUT, X_PLEX_CONTAINER_SIZE,
+
-                     X_PLEX_ENABLE_FAST_CONNECT, X_PLEX_IDENTIFIER, log, logfilter, utils)
+from plexapi import (BASE_HEADERS, CONFIG, TIMEOUT, X_PLEX_ENABLE_FAST_CONNECT, X_PLEX_IDENTIFIER,
+                     log, logfilter, utils)
 from plexapi.base import PlexObject
 from plexapi.client import PlexClient
 from plexapi.exceptions import BadRequest, NotFound, Unauthorized
@@ -786,7 +787,7 @@ class MyPlexAccount(PlexObject):
             raise BadRequest(f'({response.status_code}) {codename} {response.url}; {errtext}')
         return response.json()['token']
 
-    def history(self, maxresults=9999999, mindate=None):
+    def history(self, maxresults=None, mindate=None):
         """ Get Play History for all library sections on all servers for the owner.
 
             Parameters:
@@ -819,7 +820,7 @@ class MyPlexAccount(PlexObject):
         data = self.query(f'{self.MUSIC}/hubs')
         return self.findItems(data)
 
-    def watchlist(self, filter=None, sort=None, libtype=None, maxresults=9999999, **kwargs):
+    def watchlist(self, filter=None, sort=None, libtype=None, maxresults=None, **kwargs):
         """ Returns a list of :class:`~plexapi.video.Movie` and :class:`~plexapi.video.Show` items in the user's watchlist.
             Note: The objects returned are from Plex's online metadata. To get the matching item on a Plex server,
             search for the media using the guid.
@@ -859,23 +860,10 @@ class MyPlexAccount(PlexObject):
         if libtype:
             params['type'] = utils.searchType(libtype)
 
-        params['X-Plex-Container-Start'] = 0
-        params['X-Plex-Container-Size'] = min(X_PLEX_CONTAINER_SIZE, maxresults)
         params.update(kwargs)
 
-        results, subresults = [], '_init'
+        key = f'{self.METADATA}/library/sections/watchlist/{filter}{utils.joinArgs(params)}'
-        while subresults and maxresults > len(results):
+        return self._toOnlineMetadata(self.fetchItems(key, maxresults=maxresults), **kwargs)
-            data = self.query(f'{self.METADATA}/library/sections/watchlist/{filter}', params=params)
-            subresults = self.findItems(data)
-            results += subresults[:maxresults - len(results)]
-            params['X-Plex-Container-Start'] += params['X-Plex-Container-Size']
-
-            # totalSize is available in first response, update maxresults from it
-            totalSize = utils.cast(int, data.attrib.get('totalSize'))
-            if maxresults > totalSize:
-                maxresults = totalSize
-
-        return self._toOnlineMetadata(results, **kwargs)
 
     def onWatchlist(self, item):
         """ Returns True if the item is on the user's watchlist.
@@ -1161,7 +1149,7 @@ class MyPlexUser(PlexObject):
 
         raise NotFound(f'Unable to find server {name}')
 
-    def history(self, maxresults=9999999, mindate=None):
+    def history(self, maxresults=None, mindate=None):
         """ Get all Play History for a user in all shared servers.
             Parameters:
                 maxresults (int): Only return the specified number of results (optional).
@@ -1235,7 +1223,7 @@ class Section(PlexObject):
         self.sectionId = self.id  # For backwards compatibility
         self.sectionKey = self.key  # For backwards compatibility
 
-    def history(self, maxresults=9999999, mindate=None):
+    def history(self, maxresults=None, mindate=None):
         """ Get all Play History for a user for this section in this shared server.
             Parameters:
                 maxresults (int): Only return the specified number of results (optional).

plexapi/server.py

@@ -1,11 +1,11 @@
 # -*- coding: utf-8 -*-
+import os
 from urllib.parse import urlencode
 from xml.etree import ElementTree
 
 import requests
-import os
+
-from plexapi import (BASE_HEADERS, CONFIG, TIMEOUT, X_PLEX_CONTAINER_SIZE, log,
+from plexapi import BASE_HEADERS, CONFIG, TIMEOUT, log, logfilter
-                     logfilter)
 from plexapi import utils
 from plexapi.alert import AlertListener
 from plexapi.base import PlexObject
@@ -637,7 +637,7 @@ class PlexServer(PlexObject):
             # figure out what method this is..
             return self.query(part, method=self._session.put)
 
-    def history(self, maxresults=9999999, mindate=None, ratingKey=None, accountID=None, librarySectionID=None):
+    def history(self, maxresults=None, mindate=None, ratingKey=None, accountID=None, librarySectionID=None):
         """ Returns a list of media items from watched history. If there are many results, they will
             be fetched from the server in batches of X_PLEX_CONTAINER_SIZE amounts. If you're only
             looking for the first <num> results, it would be wise to set the maxresults option to that
@@ -651,7 +651,6 @@ class PlexServer(PlexObject):
                 accountID (int/str) Request history for a specific account ID.
                 librarySectionID (int/str) Request history for a specific library section ID.
         """
-        results, subresults = [], '_init'
         args = {'sort': 'viewedAt:desc'}
         if ratingKey:
             args['metadataItemID'] = ratingKey
@@ -661,14 +660,9 @@ class PlexServer(PlexObject):
             args['librarySectionID'] = librarySectionID
         if mindate:
             args['viewedAt>'] = int(mindate.timestamp())
-        args['X-Plex-Container-Start'] = 0
+        
-        args['X-Plex-Container-Size'] = min(X_PLEX_CONTAINER_SIZE, maxresults)
+        key = f'/status/sessions/history/all{utils.joinArgs(args)}'
-        while subresults and maxresults > len(results):
+        return self.fetchItems(key, maxresults=maxresults)
-            key = f'/status/sessions/history/all{utils.joinArgs(args)}'
-            subresults = self.fetchItems(key)
-            results += subresults[:maxresults - len(results)]
-            args['X-Plex-Container-Start'] += args['X-Plex-Container-Size']
-        return results
 
     def playlists(self, playlistType=None, sectionId=None, title=None, sort=None, **kwargs):
         """ Returns a list of all :class:`~plexapi.playlist.Playlist` objects on the server.

plexapi/sonos.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 import requests
+
 from plexapi import CONFIG, X_PLEX_IDENTIFIER
 from plexapi.client import PlexClient
 from plexapi.exceptions import BadRequest

plexapi/utils.py

@@ -18,6 +18,7 @@ from urllib.parse import quote
 from requests.status_codes import _codes as codes
 
 import requests
+
 from plexapi.exceptions import BadRequest, NotFound, Unauthorized
 
 try: