Repository: planetlabs/planet-client-python

Issue:
change polling output to a progress bar
Polling can fill up the stdout with all its status messages. Change the status to an updating progress bar. Move status messages to logging.

Discussion:

Patch:
diff --git a/docs/api.md b/docs/api.md
index 4266658..147ef1c 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -16,4 +16,8 @@
     rendering:
       show_root_full_path: false
 
+## ::: planet.reporting
+    rendering:
+      show_root_full_path: false
+
 
diff --git a/docs/guide.md b/docs/guide.md
index e0aaf2e..757874d 100644
--- a/docs/guide.md
+++ b/docs/guide.md
@@ -64,7 +64,7 @@ For example, to obtain and store authentication information:
 
 When a `Session` is created, by default, authentication is read from the secret
 file created with `Auth.write()`. This behavior can be modified by specifying
-`Auth` explicitely using the methods `Auth.from_file()` and `Auth.from_env()`.
+`Auth` explicitly using the methods `Auth.from_file()` and `Auth.from_env()`.
 While `Auth.from_key()` and `Auth.from_login` can be used, it is recommended
 that those functions be used in authentication initialization and the
 authentication information be stored using `Auth.write()`.
@@ -189,6 +189,38 @@ the context of a `Session` with the `OrdersClient`:
 
 ```
 
+#### Polling and Downloading an Order
+
+Once an order is created, the Orders API takes some time to create the order
+and thus we must wait a while before downloading the order.
+We can use polling to watch the order creation process and find out when the
+order is created successfully and ready to download.
+
+With polling and download, it is often desired to track progress as these
+processes can take a long time. Therefore, in this example, we use a progress
+bar from the `reporting` module to report poll status. `download_order` has
+reporting built in.
+
+```python
+from planet import reporting
+
+>>> async def create_poll_and_download():
+...     async with Session() as sess:
+...         cl = OrdersClient(sess)
+...         with reporting.StateBar(state='creating') as bar:
+...             # create order
+...             order = await cl.create_order(request)
+...             bar.update(state='created', order_id=order.id)
+...
+...             # poll
+...             await cl.poll(order.id, report=bar.update)
+...
+...         # download
+...         await cl.download_order(order.id)
+...
+>>> asyncio.run(create_poll_and_download())
+```
+
 ## CLI
 
 ### Authentication
diff --git a/examples/orders_create_and_download_multiple_orders.py b/examples/orders_create_and_download_multiple_orders.py
index e2d958f..ecaa79a 100644
--- a/examples/orders_create_and_download_multiple_orders.py
+++ b/examples/orders_create_and_download_multiple_orders.py
@@ -26,7 +26,6 @@ import os
 
 import planet
 
-API_KEY = os.getenv('PL_API_KEY')
 DOWNLOAD_DIR = os.getenv('TEST_DOWNLOAD_DIR', '.')
 
 iowa_aoi = {
@@ -43,10 +42,10 @@ iowa_images = [
     '20200925_161029_69_2223',
     '20200925_161027_48_2223'
 ]
-iowa_order = planet.OrderDetails(
+iowa_order = planet.order_request.build_request(
     'iowa_order',
-    [planet.Product(iowa_images, 'analytic', 'PSScene4Band')],
-    tools=[planet.Tool('clip', {'aoi': iowa_aoi})]
+    [planet.order_request.product(iowa_images, 'analytic', 'PSScene4Band')],
+    tools=[planet.order_request.clip_tool(iowa_aoi)]
 )
 
 oregon_aoi = {
@@ -63,37 +62,33 @@ oregon_images = [
     '20200909_182525_1014',
     '20200909_182524_1014'
 ]
-oregon_order = planet.OrderDetails(
+oregon_order = planet.order_request.build_request(
     'oregon_order',
-    [planet.Product(oregon_images, 'analytic', 'PSScene4Band')],
-    tools=[planet.Tool('clip', {'aoi': oregon_aoi})]
+    [planet.order_request.product(oregon_images, 'analytic', 'PSScene4Band')],
+    tools=[planet.order_request.clip_tool(oregon_aoi)]
 )
 
 
 async def create_and_download(order_detail, directory, client):
-    # create
-    print('Creating order')
-    oid = await client.create_order(order_detail)
-    print(f'Order created: {oid}')
+    with planet.reporting.StateBar(state='creating') as reporter:
+        # create
+        order = await client.create_order(order_detail)
+        reporter.update(state='created', order_id=order.id)
 
-    # poll
-    state = await client.poll(oid, verbose=True)
-    print(f'Order {oid} final state: {state}')
+        # poll
+        await client.poll(order.id, report=reporter.update)
 
     # download
-    print(f'Downloading {oid} to {directory}.')
-    filenames = await client.download_order(oid, directory, progress_bar=True)
-    print(f'Downloaded {oid}: '
-          f'{len(filenames)} files downloaded to {directory}.')
+    await client.download_order(order.id, progress_bar=True)
 
 
 async def main():
-    async with planet.Session(auth=(API_KEY, '')) as ps:
+    async with planet.Session() as ps:
         client = planet.OrdersClient(ps)
 
         await asyncio.gather(
             create_and_download(iowa_order, DOWNLOAD_DIR, client),
-            create_and_download(oregon_order, DOWNLOAD_DIR, client)
+            create_and_download(oregon_order, DOWNLOAD_DIR, client),
         )
 
 
diff --git a/planet/__init__.py b/planet/__init__.py
index 57ea767..fe05042 100644
--- a/planet/__init__.py
+++ b/planet/__init__.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 from .http import Session
 from .models import Order
-from . import order_request
+from . import order_request, reporting
 from .__version__ import __version__  # NOQA
 from .auth import Auth
 from .clients import OrdersClient
@@ -24,5 +24,6 @@ __all__ = [
     OrdersClient,
     Order,
     order_request,
+    reporting,
     Auth,
 ]
diff --git a/planet/clients/orders.py b/planet/clients/orders.py
index 878d2e9..0284734 100644
--- a/planet/clients/orders.py
+++ b/planet/clients/orders.py
@@ -161,10 +161,13 @@ class OrdersClient():
         except exceptions.BadQuery as ex:
             msg_json = json.loads(ex.message)
 
-            # get first error field
-            field = next(iter(msg_json['field'].keys()))
-            msg = (msg_json['general'][0]['message'] + ' - ' +
-                   msg_json['field'][field][0]['message'])
+            msg = msg_json['general'][0]['message']
+            try:
+                # get first error field
+                field = next(iter(msg_json['field'].keys()))
+                msg += ' - ' + msg_json['field'][field][0]['message']
+            except AttributeError:
+                pass
             raise exceptions.BadQuery(msg)
 
         order = Order(resp.json())
@@ -343,28 +346,43 @@ class OrdersClient():
         self,
         order_id: str,
         state: str = None,
-        wait: int = 10,
-        verbose: bool = False
+        wait: int = 1,
+        report=None
     ) -> str:
-        """Poll for order status until order reaches desired state.
+        """Poll for order status until order reaches desired state, optionally
+        reporting status.
+
+        By default, the Orders API is polled every 1 second for status updates.
+        The API rate limit for this endpoint is 10 requests per second.
+        If many orders are being polled asynchronously, consider
+        increasing the wait to avoid throttling.
+
+        Example:
+            ```python
+            from planet.reporting import StateBar
+
+            with StateBar() as bar:
+                await poll(order_id, report=bar.update)
+            ```
 
         Parameters:
             order_id: The ID of the order
             state: State to poll until. If multiple, use list. Defaults to
                 any completed state.
             wait: Time (in seconds) between polls.
-            verbose: Print current state at each poll
+            report: Callback function for progress updates. Invoked with
+                keyword arguments `state` (poll state) and `logger`
+                (callback for logging progress bar status). Recommended
+                value is `reporting.StateBar.update`.
 
         Returns
-            Completed state of the order
+            Completed state of the order.
 
         Raises:
             planet.exceptions.APIException: On API error.
             OrdersClientException: If order_id is not valid or state is not
                 supported.
         """
-        completed = False
-
         if state:
             if state not in ORDERS_STATES:
                 raise OrdersClientException(
@@ -374,19 +392,22 @@ class OrdersClient():
         else:
             states = ORDERS_STATES_COMPLETE
 
+        completed = False
         while not completed:
             t = time.time()
+
             order = await self.get_order(order_id)
             state = order.state
-            msg = f'order {order_id} state: {state}'
-            LOGGER.info(msg)
-            if verbose:
-                print(msg)
+
+            if report:
+                report(state=order.state)
+            else:
+                LOGGER.debug(state)
 
             completed = state in states
             if not completed:
                 sleep_time = max(wait-(time.time()-t), 0)
-                LOGGER.info(f'sleeping {sleep_time}s')
+                LOGGER.debug(f'sleeping {sleep_time}s')
                 await asyncio.sleep(sleep_time)
         return state
 
diff --git a/planet/reporting.py b/planet/reporting.py
new file mode 100644
index 0000000..0664ea6
--- /dev/null
+++ b/planet/reporting.py
@@ -0,0 +1,103 @@
+# Copyright 2021 Planet Labs, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Functionality for reporting progress."""
+import logging
+
+from tqdm.asyncio import tqdm
+
+LOGGER = logging.getLogger(__name__)
+
+
+class ProgressBar():
+    """Abstract base class for progress bar reporters."""
+    def __init__(
+        self,
+        disable: bool = False
+    ):
+        self.bar = None
+        self.disable = disable
+
+    def __str__(self):
+        return str(self.bar)
+
+    def __enter__(self):
+        self.open_bar()
+        return self
+
+    def __exit__(self, *args):
+        self.bar.close()
+
+    def open_bar(self):
+        """Initialize and start the progress bar."""
+        return NotImplementedError
+
+
+class StateBar(ProgressBar):
+    """Bar reporter of order state.
+
+    Example:
+        ```python
+        from planet import reporting
+
+        with reporting.StateBar(state='creating') as bar:
+            bar.update(state='created', order_id='oid')
+            ...
+        ```
+    """
+    def __init__(
+        self,
+        order_id: str = None,
+        state: str = None,
+        disable: bool = False,
+    ):
+        """Initialize the object.
+
+        Parameters:
+            order_id: Id of the order.
+            state: State of the order.
+        """
+
+        self.state = state or ''
+        self.order_id = order_id or ''
+        super().__init__(disable=disable)
+
+    def open_bar(self):
+        """Initialize and start the progress bar."""
+        self.bar = tqdm(
+            bar_format="{elapsed} - {desc} - {postfix[0]}: {postfix[1]}",
+            desc=self.desc, postfix=["state", self.state],
+            disable=self.disable)
+
+    @property
+    def desc(self):
+        return f'order {self.order_id}'
+
+    def update(
+        self,
+        state: str = None,
+        order_id: str = None
+    ):
+        if state:
+            if state != self.state:
+                LOGGER.info('{order_id} state: {state}')
+            self.state = state
+            self.bar.postfix[1] = self.state
+
+        if order_id:
+            self.order_id = order_id
+            self.bar.set_description_str(self.desc, refresh=False)
+
+        self.bar.refresh()
+
+        LOGGER.debug(str(self.bar))
diff --git a/planet/scripts/cli.py b/planet/scripts/cli.py
index 3065ed1..5cb1118 100644
--- a/planet/scripts/cli.py
+++ b/planet/scripts/cli.py
@@ -209,12 +209,14 @@ async def cancel(ctx, order_id):
 async def download(ctx, order_id, quiet, overwrite, dest):
     '''Download order by order ID.'''
     async with orders_client(ctx) as cl:
-        await cl.poll(str(order_id), verbose=True)
-        filenames = await cl.download_order(
-                str(order_id),
-                directory=dest,
-                overwrite=overwrite,
-                progress_bar=not quiet)
+        with planet.reporting.StateBar(order_id=order_id,
+                                       disable=quiet) as bar:
+            await cl.poll(str(order_id), report=bar.update)
+            filenames = await cl.download_order(
+                    str(order_id),
+                    directory=dest,
+                    overwrite=overwrite,
+                    progress_bar=not quiet)
     click.echo(f'Downloaded {len(filenames)} files.')
 
 


Test patch:
diff --git a/tests/integration/test_orders_api.py b/tests/integration/test_orders_api.py
index f837e1a..bdbf763 100644
--- a/tests/integration/test_orders_api.py
+++ b/tests/integration/test_orders_api.py
@@ -18,13 +18,14 @@ import logging
 import math
 import os
 from pathlib import Path
+from unittest.mock import create_autospec
 
 
 import httpx
 import pytest
 import respx
 
-from planet import OrdersClient, clients, exceptions
+from planet import OrdersClient, clients, exceptions, reporting
 
 
 TEST_URL = 'http://MockNotRealURL/'
@@ -436,9 +437,17 @@ async def test_poll(oid, order_description, session):
         httpx.Response(HTTPStatus.OK, json=order_description2),
         httpx.Response(HTTPStatus.OK, json=order_description3)
     ]
-    state = await cl.poll(oid, wait=0)
+
+    mock_bar = create_autospec(reporting.StateBar)
+    mock_report = mock_bar.update
+    state = await cl.poll(oid, wait=0, report=mock_report)
     assert state == 'success'
 
+    # check state was reported as expected
+    assert mock_report.call_count == 3
+    states = [c[2]['state'] for c in mock_report.mock_calls]
+    assert ['queued', 'running', 'success'] == states
+
     route = respx.get(get_url)
     route.side_effect = [
         httpx.Response(HTTPStatus.OK, json=order_description),
diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py
index 4ae039d..3da5dbf 100644
--- a/tests/unit/test_cli.py
+++ b/tests/unit/test_cli.py
@@ -145,8 +145,15 @@ def test_cli_orders_download(runner, monkeypatch, oid):
     result = runner.invoke(
         cli, ['orders', 'download', oid])
     assert not result.exception
-    # assert "file1" in result.output
-    assert 'Downloaded 1 files.\n' == result.output
+    expected = 'Downloaded 1 files.\n'
+    # allow for some progress reporting
+    assert expected in result.output
+
+    # test quiet option, should be no progress reporting
+    result = runner.invoke(
+        cli, ['orders', 'download', '-q', oid])
+    assert not result.exception
+    assert expected == result.output
 
 
 class AsyncMock(Mock):
diff --git a/tests/unit/test_reporting.py b/tests/unit/test_reporting.py
new file mode 100644
index 0000000..c5264ac
--- /dev/null
+++ b/tests/unit/test_reporting.py
@@ -0,0 +1,34 @@
+# Copyright 2020 Planet Labs, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+import logging
+import re
+
+from planet import reporting
+
+LOGGER = logging.getLogger(__name__)
+
+
+def test_StateBar_enabled():
+    with reporting.StateBar() as bar:
+        expected_init = '..:.. - order  - state: '
+        assert(re.fullmatch(expected_init, str(bar)))
+
+        expected_update = '..:.. - order 1 - state: init'
+        bar.update(order_id='1', state='init')
+        assert(re.fullmatch(expected_update, str(bar)))
+
+
+def test_StateBar_disabled():
+    with reporting.StateBar(disable=True) as bar:
+        assert bar.bar.disable