Repository: dask/dask
Pull request #4788 (2019-05-10 04:05:13)

Add CLI API docs
This PR adds auto-generated API documentation for the dask CLI using [sphinx-click](https://github.com/click-contrib/sphinx-click). Here's a local screenshot:

<img width="1439" alt="Screen Shot 2019-05-09 at 10 57 58 PM" src="https://user-images.githubusercontent.com/11656932/57501818-96b79380-72ae-11e9-9ae2-eff3995baab7.png">

Fixes https://github.com/dask/distributed/issues/1975

- [ ] Tests added / passed
- [ ] Passes `flake8 dask`


jcrist: Thanks @jrbourbeau, overall this looks good to me. If you're happy with the way things are here, please go ahead and merge.

Patch:
diff --git a/dask/utils.py b/dask/utils.py
index da1a3ffdb..6cacae859 100644
--- a/dask/utils.py
+++ b/dask/utils.py
@@ -1,5 +1,6 @@
 from __future__ import absolute_import, division, print_function
 
+from datetime import timedelta
 import functools
 import inspect
 import os
@@ -10,7 +11,7 @@ import re
 from errno import ENOENT
 from contextlib import contextmanager
 from importlib import import_module
-from numbers import Integral
+from numbers import Integral, Number
 from threading import Lock
 import uuid
 from weakref import WeakValueDictionary
@@ -1091,6 +1092,119 @@ byte_sizes.update({k[0]: v for k, v in byte_sizes.items() if k and 'i' not in k}
 byte_sizes.update({k[:-1]: v for k, v in byte_sizes.items() if k and 'i' in k})
 
 
+def format_time(n):
+    """ format integers as time
+
+    >>> format_time(1)
+    '1.00 s'
+    >>> format_time(0.001234)
+    '1.23 ms'
+    >>> format_time(0.00012345)
+    '123.45 us'
+    >>> format_time(123.456)
+    '123.46 s'
+    """
+    if n >= 1:
+        return "%.2f s" % n
+    if n >= 1e-3:
+        return "%.2f ms" % (n * 1e3)
+    return "%.2f us" % (n * 1e6)
+
+
+def format_bytes(n):
+    """ Format bytes as text
+
+    >>> format_bytes(1)
+    '1 B'
+    >>> format_bytes(1234)
+    '1.23 kB'
+    >>> format_bytes(12345678)
+    '12.35 MB'
+    >>> format_bytes(1234567890)
+    '1.23 GB'
+    >>> format_bytes(1234567890000)
+    '1.23 TB'
+    >>> format_bytes(1234567890000000)
+    '1.23 PB'
+    """
+    if n > 1e15:
+        return "%0.2f PB" % (n / 1e15)
+    if n > 1e12:
+        return "%0.2f TB" % (n / 1e12)
+    if n > 1e9:
+        return "%0.2f GB" % (n / 1e9)
+    if n > 1e6:
+        return "%0.2f MB" % (n / 1e6)
+    if n > 1e3:
+        return "%0.2f kB" % (n / 1000)
+    return "%d B" % n
+
+
+timedelta_sizes = {
+    "s": 1,
+    "ms": 1e-3,
+    "us": 1e-6,
+    "ns": 1e-9,
+    "m": 60,
+    "h": 3600,
+    "d": 3600 * 24,
+}
+
+tds2 = {
+    "second": 1,
+    "minute": 60,
+    "hour": 60 * 60,
+    "day": 60 * 60 * 24,
+    "millisecond": 1e-3,
+    "microsecond": 1e-6,
+    "nanosecond": 1e-9,
+}
+tds2.update({k + "s": v for k, v in tds2.items()})
+timedelta_sizes.update(tds2)
+timedelta_sizes.update({k.upper(): v for k, v in timedelta_sizes.items()})
+
+
+def parse_timedelta(s, default="seconds"):
+    """ Parse timedelta string to number of seconds
+
+    Examples
+    --------
+    >>> parse_timedelta('3s')
+    3
+    >>> parse_timedelta('3.5 seconds')
+    3.5
+    >>> parse_timedelta('300ms')
+    0.3
+    >>> parse_timedelta(timedelta(seconds=3))  # also supports timedeltas
+    3
+    """
+    if isinstance(s, timedelta):
+        s = s.total_seconds()
+        return int(s) if int(s) == s else s
+    if isinstance(s, Number):
+        s = str(s)
+    s = s.replace(" ", "")
+    if not s[0].isdigit():
+        s = "1" + s
+
+    for i in range(len(s) - 1, -1, -1):
+        if not s[i].isalpha():
+            break
+    index = i + 1
+
+    prefix = s[:index]
+    suffix = s[index:] or default
+
+    n = float(prefix)
+
+    multiplier = timedelta_sizes[suffix.lower()]
+
+    result = n * multiplier
+    if int(result) == result:
+        result = int(result)
+    return result
+
+
 def has_keyword(func, keyword):
     try:
         if PY3:
diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt
index 7d43c8c37..3070f13b9 100644
--- a/docs/requirements-docs.txt
+++ b/docs/requirements-docs.txt
@@ -1,6 +1,7 @@
 numpydoc==0.8
 sphinx
 dask_sphinx_theme>=1.1.0
+sphinx-click
 toolz
 cloudpickle
 pandas>=0.19.0
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 3508e784b..7ef37d354 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -27,7 +27,8 @@ from shutil import copyfile
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = ['sphinx.ext.autodoc', 'sphinx.ext.mathjax', 'sphinx.ext.intersphinx',
-              'sphinx.ext.autosummary', 'sphinx.ext.extlinks', 'numpydoc']
+              'sphinx.ext.autosummary', 'sphinx.ext.extlinks', 'numpydoc',
+              'sphinx_click.ext']
 
 numpydoc_show_class_members = False
 
diff --git a/docs/source/setup/cli.rst b/docs/source/setup/cli.rst
index 50f44d70f..8e5892676 100644
--- a/docs/source/setup/cli.rst
+++ b/docs/source/setup/cli.rst
@@ -88,87 +88,25 @@ API
 
 .. warning::
 
-   These may be out-dated.  We recommend referring to the ``--help`` text of your
-   installed version.
-
-dask-scheduler
-~~~~~~~~~~~~~~
-
-.. code-block:: bash
-
-   $ dask-scheduler --help
-   Usage: dask-scheduler [OPTIONS]
-
-   Options:
-     --host TEXT             URI, IP or hostname of this server
-     --port INTEGER          Serving port
-     --interface TEXT        Preferred network interface like 'eth0' or 'ib0'
-     --tls-ca-file PATH      CA cert(s) file for TLS (in PEM format)
-     --tls-cert PATH         certificate file for TLS (in PEM format)
-     --tls-key PATH          private key file for TLS (in PEM format)
-     --bokeh-port INTEGER    Bokeh port for visual diagnostics
-     --bokeh / --no-bokeh    Launch Bokeh Web UI  [default: True]
-     --show / --no-show      Show web UI
-     --bokeh-whitelist TEXT  IP addresses to whitelist for bokeh
-     --bokeh-prefix TEXT     Prefix for the bokeh app
-     --use-xheaders BOOLEAN  User xheaders in bokeh app for ssl termination in
-                             header  [default: False]
-     --pid-file TEXT         File to write the process PID
-     --scheduler-file TEXT   File to write connection information. This may be a
-                             good way to share connection information if your
-                             cluster is on a shared network file system
-     --local-directory TEXT  Directory to place scheduler files
-     --preload TEXT          Module that should be loaded by each worker process
-                             like "foo.bar" or "/path/to/foo.py"
-     --help                  Show this message and exit
-
-
-dask-worker
-~~~~~~~~~~~
-
-.. code-block:: bash
-
-   $ dask-worker --help
-   Usage: dask-worker [OPTIONS] [SCHEDULER]
-
-   Options:
-     --tls-ca-file PATH            CA cert(s) file for TLS (in PEM format)
-     --tls-cert PATH               certificate file for TLS (in PEM format)
-     --tls-key PATH                private key file for TLS (in PEM format)
-     --worker-port INTEGER         Serving computation port, defaults to random
-     --nanny-port INTEGER          Serving nanny port, defaults to random
-     --bokeh-port INTEGER          Bokeh port, defaults to 8789
-     --bokeh / --no-bokeh          Launch Bokeh Web UI  [default: True]
-     --listen-address TEXT         The address to which the worker binds.
-                                   Example: tcp://0.0.0.0:9000
-     --contact-address TEXT        The address the worker advertises to the
-                                   scheduler for communication with it and other
-                                   workers. Example: tcp://127.0.0.1:9000
-     --host TEXT                   Serving host. Should be an ip address that is
-                                   visible to the scheduler and other workers.
-                                   See --listen-address and --contact-address if
-                                   you need different listen and contact
-                                   addresses. See --interface
-     --interface TEXT              Network interface like 'eth0' or 'ib0'
-     --nthreads INTEGER            Number of threads per process
-     --nprocs INTEGER              Number of worker processes.  Defaults to one
-     --name TEXT                   A unique name for this worker like 'worker-1'
-     --memory-limit TEXT           Bytes of memory that the worker can use. This
-                                   can be an integer (bytes), float (fraction of
-                                   total system memory), string (like 5GB or
-                                   5000M), 'auto', or zero for no memory
-                                   management
-     --reconnect / --no-reconnect  Reconnect to scheduler if disconnected
-     --nanny / --no-nanny          Start workers in nanny process for management
-     --pid-file TEXT               File to write the process PID
-     --local-directory TEXT        Directory to place worker files
-     --resources TEXT              Resources for task constraints like "GPU=2
-                                   MEM=10e9"
-     --scheduler-file TEXT         Filename to JSON encoded scheduler information. 
-                                   Use with dask-scheduler --scheduler-file
-     --death-timeout FLOAT         Seconds to wait for a scheduler before closing
-     --bokeh-prefix TEXT           Prefix for the bokeh app
-     --preload TEXT                Module that should be loaded by each worker
-                                   process like "foo.bar" or "/path/to/foo.py"
-     --help                        Show this message and exit
+   The command line documentation here may differ depending on your installed
+   version. We recommend referring to the output of ``<command> --help``.
 
+.. click:: distributed.cli.dask_scheduler:main
+   :prog: dask-scheduler
+   :show-nested:
+
+.. click:: distributed.cli.dask_worker:main
+   :prog: dask-worker
+   :show-nested:
+
+.. click:: distributed.cli.dask_ssh:main
+   :prog: dask-ssh
+   :show-nested:
+
+.. click:: distributed.cli.dask_submit:main
+   :prog: dask-submit
+   :show-nested:
+
+.. click:: distributed.cli.dask_remote:main
+   :prog: dask-remote
+   :show-nested:


Test patch:
diff --git a/dask/tests/test_utils.py b/dask/tests/test_utils.py
index cfa94ace7..eb747781d 100644
--- a/dask/tests/test_utils.py
+++ b/dask/tests/test_utils.py
@@ -1,3 +1,4 @@
+import datetime
 import functools
 import operator
 import pickle
@@ -10,7 +11,8 @@ from dask.utils import (takes_multiple_arguments, Dispatch, random_state_data,
                         memory_repr, methodcaller, M, skip_doctest,
                         SerializableLock, funcname, ndeepmap, ensure_dict,
                         extra_titles, asciitable, itemgetter, partial_by_order,
-                        has_keyword, derived_from)
+                        has_keyword, derived_from, parse_timedelta,
+                        parse_bytes)
 from dask.utils_test import inc
 from dask.highlevelgraph import HighLevelGraph
 
@@ -427,3 +429,42 @@ def test_derived_from_dask_dataframe():
     [axis_arg] = [line for line in dd.DataFrame.dropna.__doc__.split('\n') if 'axis :' in line]
     assert "not supported" in axis_arg.lower()
     assert "dask" in axis_arg.lower()
+
+
+def test_parse_bytes():
+    assert parse_bytes("100") == 100
+    assert parse_bytes("100 MB") == 100000000
+    assert parse_bytes("100M") == 100000000
+    assert parse_bytes("5kB") == 5000
+    assert parse_bytes("5.4 kB") == 5400
+    assert parse_bytes("1kiB") == 1024
+    assert parse_bytes("1Mi") == 2 ** 20
+    assert parse_bytes("1e6") == 1000000
+    assert parse_bytes("1e6 kB") == 1000000000
+    assert parse_bytes("MB") == 1000000
+
+
+def test_parse_timedelta():
+    for text, value in [
+        ("1s", 1),
+        ("100ms", 0.1),
+        ("5S", 5),
+        ("5.5s", 5.5),
+        ("5.5 s", 5.5),
+        ("1 second", 1),
+        ("3.3 seconds", 3.3),
+        ("3.3 milliseconds", 0.0033),
+        ("3500 us", 0.0035),
+        ("1 ns", 1e-9),
+        ("2m", 120),
+        ("2 minutes", 120),
+        (datetime.timedelta(seconds=2), 2),
+        (datetime.timedelta(milliseconds=100), 0.1),
+    ]:
+        result = parse_timedelta(text)
+        assert abs(result - value) < 1e-14
+
+    assert parse_timedelta("1ms", default="seconds") == 0.001
+    assert parse_timedelta("1", default="seconds") == 1
+    assert parse_timedelta("1", default="ms") == 0.001
+    assert parse_timedelta(1, default="ms") == 0.001