Repository: TheFriendlyCoder/friendlypins

Issue:
lazy load boards and pins
To simplify and optimize the interactions with boards and pins we should lazy-load the data from these two API calls using iterators:

    User.boards
    Board.all_pins

Discussion:

Patch:
diff --git a/src/friendlypins/board.py b/src/friendlypins/board.py
index 045655f..7f51a3b 100644
--- a/src/friendlypins/board.py
+++ b/src/friendlypins/board.py
@@ -64,16 +64,12 @@ class Board(object):
         return int(self._data['counts']['pins'])
 
     @property
-    def all_pins(self):
-        """Gets a list of all pins from this board
+    def pins(self):
+        """Generator for iterating over the pins linked to this board
 
-        NOTE: This process may take a long time to complete and require
-        a lot of memory for boards that contain large numbers of pins
-
-        :rtype: :class:`list` of :class:`friendlypins.pin.Pin`
+        :rtype: Generator of :class:`friendlypins.pin.Pin`
         """
-        self._log.debug('Gettings all pins for board %s...', self.name)
-        retval = list()
+        self._log.debug('Loading pins for board %s...', self.name)
 
         properties = {
             "fields": ','.join([
@@ -93,19 +89,20 @@ class Board(object):
             ])
         }
 
+        page = 0
         while True:
+            self._log.debug("Loading pins page %s", page)
             result = self._io.get(
                 "boards/{0}/pins".format(self.unique_id),
                 properties)
             assert 'data' in result
 
             for cur_item in result['data']:
-                retval.append(Pin(cur_item, self._io))
+                yield Pin(cur_item, self._io)
             if not result["page"]["cursor"]:
                 break
             properties["cursor"] = result["page"]["cursor"]
-
-        return retval
+            page += 1
 
 
 if __name__ == "__main__":
diff --git a/src/friendlypins/user.py b/src/friendlypins/user.py
index de3306d..3d69138 100644
--- a/src/friendlypins/user.py
+++ b/src/friendlypins/user.py
@@ -94,21 +94,39 @@ class User(object):
 
     @property
     def boards(self):
-        """Gets a list of boards owned by this user
+        """Generator for iterating over the boards owned by this user
 
-        :rtype: :class:`list` of :class:`friendlypins.board.Board`
+        :rtype: Generator of :class:`friendlypins.board.Board`
         """
-        self._log.debug("Loading boards for user %s...", self.name)
-
-        fields = "id,name,url,description,creator,created_at,counts,image"
-        result = self._io.get('me/boards', {"fields": fields})
-
-        assert 'data' in result
-
-        retval = []
-        for cur_item in result['data']:
-            retval.append(Board(cur_item, self._io))
-        return retval
+        self._log.debug('Loading boards for user %s...', self.name)
+
+        properties = {
+            "fields": ','.join([
+                "id",
+                "name",
+                "url",
+                "description",
+                "creator",
+                "created_at",
+                "counts",
+                "image"
+            ])
+        }
+
+        page = 0
+        while True:
+            self._log.debug("Loading boards page %s", page)
+            result = self._io.get("me/boards", properties)
+            assert 'data' in result
+
+            for cur_item in result['data']:
+                yield Board(cur_item, self._io)
+
+            if not result["page"]["cursor"]:
+                break
+
+            properties["cursor"] = result["page"]["cursor"]
+            page += 1
 
 
 if __name__ == "__main__":
diff --git a/src/friendlypins/utils/console_actions.py b/src/friendlypins/utils/console_actions.py
index 1ca66ef..da40dfc 100644
--- a/src/friendlypins/utils/console_actions.py
+++ b/src/friendlypins/utils/console_actions.py
@@ -71,13 +71,12 @@ def download_thumbnails(api_token, board_name, output_folder, delete):
         log.error("Could not find selected board: %s", board_name)
         return 1
 
-    all_pins = selected_board.all_pins
     log.info('Downloading thumbnails...')
     if not os.path.exists(output_folder):
         os.makedirs(output_folder)
 
     with tqdm(total=selected_board.num_pins, unit='b', ncols=80) as pbar:
-        for cur_pin in all_pins:
+        for cur_pin in selected_board.pins:
             retval = _download_pin(cur_pin, output_folder)
             if retval:
                 return retval


Test patch:
diff --git a/unit_tests/test_board.py b/unit_tests/test_board.py
index 5daac85..2314b19 100644
--- a/unit_tests/test_board.py
+++ b/unit_tests/test_board.py
@@ -24,7 +24,7 @@ def test_board_properties():
     assert obj.num_pins == expected_pin_count
 
 
-def test_get_all_pins():
+def test_get_pins():
     data = {
         'id': '987654321',
         'name': 'MyBoard'
@@ -54,7 +54,9 @@ def test_get_all_pins():
     mock_io.get.return_value = expected_data
     obj = Board(data, mock_io)
 
-    result = obj.all_pins
+    result = list()
+    for item in obj.pins:
+        result.append(item)
 
     assert len(result) == 1
     assert expected_url == result[0].url
diff --git a/unit_tests/test_console_actions.py b/unit_tests/test_console_actions.py
index 170f224..92138cc 100644
--- a/unit_tests/test_console_actions.py
+++ b/unit_tests/test_console_actions.py
@@ -267,7 +267,10 @@ def test_download_thumbnails_missing_board(rest_io, action_requests, mock_open,
             "counts": {
                 "pins": 1
             }
-        }]
+        }],
+        "page": {
+            "cursor": None
+        }
     }
 
     # Fake pin data for the fake board, with fake thumbnail metadata
diff --git a/unit_tests/test_user.py b/unit_tests/test_user.py
index 9184835..0512b58 100644
--- a/unit_tests/test_user.py
+++ b/unit_tests/test_user.py
@@ -44,13 +44,19 @@ def test_get_boards():
             "id": str(expected_id),
             "name": expected_name,
             "url": expected_url
-        }]
+        }],
+        "page": {
+            "cursor": None
+        }
     }
 
     mock_io = mock.MagicMock()
     mock_io.get.return_value = expected_data
     obj = User(data, mock_io)
-    result = obj.boards
+
+    result = list()
+    for item in obj.boards:
+        result.append(item)
 
     assert len(result) == 1
     assert expected_url == result[0].url