Repository: dask/dask
Pull request #7484 (2021-03-29 19:44:28)

Use powers-of-two when displaying RAM
Closes https://github.com/dask/distributed/issues/3818
Twin of https://github.com/dask/distributed/pull/4649

Memory is ubiquitously expressed in power of twos.
The one and only exception that I'm aware of is when selling hard drives.

This change impacts:
- dask.array.Array.\_repr_html\_
- distributed.Client.\_\_repr\_\_
- distributed.Client.\_repr_html\_
- distributed.cluster.Cluster._widget_status
- distributed.cluster.Cluster.\_\_repr\_\_
- distributed.Scheduler.performance_report
- distributed worker logs
- distributed bokeh GUI




Patch:
diff --git a/.github/workflows/additional.yml b/.github/workflows/additional.yml
index fcc813821..bb075714e 100644
--- a/.github/workflows/additional.yml
+++ b/.github/workflows/additional.yml
@@ -17,7 +17,10 @@ jobs:
       - name: Setup Conda Environment
         uses: conda-incubator/setup-miniconda@v2
         with:
-          miniconda-version: "latest"
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          use-mamba: true
+          channel-priority: true
           python-version: "3.7"
           environment-file: continuous_integration/environment-${{ matrix.environment }}.yaml
           activate-environment: test-environment
@@ -77,7 +80,10 @@ jobs:
       - name: Setup Conda Environment
         uses: conda-incubator/setup-miniconda@v2
         with:
-          miniconda-version: "latest"
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          use-mamba: true
+          channel-priority: true
           python-version: "3.8"
           environment-file: continuous_integration/environment-latest.yaml
           activate-environment: test-environment
@@ -104,7 +110,10 @@ jobs:
       - name: Setup Conda
         uses: conda-incubator/setup-miniconda@v2
         with:
-          miniconda-version: "latest"
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          use-mamba: true
+          channel-priority: true
           python-version: "3.7"
           activate-environment: test-environment
           auto-activate-base: false
diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml
index 18e28da9f..e7063d961 100644
--- a/.github/workflows/upstream.yml
+++ b/.github/workflows/upstream.yml
@@ -36,8 +36,11 @@ jobs:
       - name: Setup Conda Environment
         uses: conda-incubator/setup-miniconda@v2
         with:
-          miniconda-version: "latest"
-          python-version: 3.8
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          use-mamba: true
+          channel-priority: true
+          python-version: "3.8"
           environment-file: continuous_integration/environment-3.8-dev.yaml
           activate-environment: test-environment
           auto-activate-base: false
diff --git a/continuous_integration/scripts/install.sh b/continuous_integration/scripts/install.sh
index 34f429343..cf610101a 100644
--- a/continuous_integration/scripts/install.sh
+++ b/continuous_integration/scripts/install.sh
@@ -7,9 +7,12 @@ set -xe
 # python -m pip install --no-deps cityhash
 
 if [[ ${UPSTREAM_DEV} ]]; then
-    conda update -y -c arrow-nightlies pyarrow
+    mamba update -y -c arrow-nightlies pyarrow
 
+    # FIXME https://github.com/mamba-org/mamba/issues/412
+    # mamba uninstall --force numpy pandas
     conda uninstall --force numpy pandas
+
     python -m pip install --no-deps --pre \
         -i https://pypi.anaconda.org/scipy-wheels-nightly/simple \
         numpy \
@@ -29,11 +32,11 @@ fi
 
 # Install dask
 python -m pip install --quiet --no-deps -e .[complete]
-echo conda list
-conda list
+echo mamba list
+mamba list
 
 # For debugging
 echo -e "--\n--Conda Environment (re-create this with \`conda env create --name <name> -f <output_file>\`)\n--"
-conda env export | grep -E -v '^prefix:.*$'
+mamba env export | grep -E -v '^prefix:.*$'
 
 set +xe
diff --git a/dask/utils.py b/dask/utils.py
index 2ded33748..bbe15493f 100644
--- a/dask/utils.py
+++ b/dask/utils.py
@@ -1387,34 +1387,35 @@ def format_time_ago(n: datetime) -> str:
     return "Just now"
 
 
-def format_bytes(n):
+def format_bytes(n: int) -> str:
     """Format bytes as text
 
     >>> from dask.utils import format_bytes
     >>> format_bytes(1)
     '1 B'
     >>> format_bytes(1234)
-    '1.23 kB'
+    '1.21 kiB'
     >>> format_bytes(12345678)
-    '12.35 MB'
+    '11.77 MiB'
     >>> format_bytes(1234567890)
-    '1.23 GB'
+    '1.15 GiB'
     >>> format_bytes(1234567890000)
-    '1.23 TB'
+    '1.12 TiB'
     >>> format_bytes(1234567890000000)
-    '1.23 PB'
+    '1.10 PiB'
+
+    For all values < 2**60, the output is always <= 10 characters.
     """
-    if n > 1e15:
-        return "%0.2f PB" % (n / 1e15)
-    if n > 1e12:
-        return "%0.2f TB" % (n / 1e12)
-    if n > 1e9:
-        return "%0.2f GB" % (n / 1e9)
-    if n > 1e6:
-        return "%0.2f MB" % (n / 1e6)
-    if n > 1e3:
-        return "%0.2f kB" % (n / 1000)
-    return "%d B" % n
+    for prefix, k in (
+        ("Pi", 2 ** 50),
+        ("Ti", 2 ** 40),
+        ("Gi", 2 ** 30),
+        ("Mi", 2 ** 20),
+        ("ki", 2 ** 10),
+    ):
+        if n >= k * 0.9:
+            return f"{n / k:.2f} {prefix}B"
+    return f"{n} B"
 
 
 timedelta_sizes = {
diff --git a/setup.py b/setup.py
index 316798636..1728b0a3a 100755
--- a/setup.py
+++ b/setup.py
@@ -61,6 +61,7 @@ setup(
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "License :: OSI Approved :: BSD License",
     ],
     packages=packages + tests,
     long_description=open("README.rst").read() if exists("README.rst") else "",


Test patch:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 04a545e25..18f9985fb 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -25,7 +25,10 @@ jobs:
       - name: Setup Conda Environment
         uses: conda-incubator/setup-miniconda@v2
         with:
-          miniconda-version: "latest"
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          use-mamba: true
+          channel-priority: true
           python-version: ${{ matrix.python-version }}
           environment-file: continuous_integration/environment-${{ matrix.python-version }}.yaml
           activate-environment: test-environment
@@ -35,6 +38,8 @@ jobs:
         shell: bash -l {0}
         if: ${{ matrix.os == 'macos-latest' && matrix.python-version == '3.8' }}
         run: |
+          # FIXME https://github.com/mamba-org/mamba/issues/412
+          # mamba uninstall --force numpy
           conda uninstall --force numpy
           python -m pip install --no-deps numpy==1.20
 
@@ -43,7 +48,7 @@ jobs:
         # test_base.py::test_tokenize_object_with_recursion_error_returns_uuid
         shell: bash -l {0}
         if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '3.9' }}
-        run: conda uninstall ipython
+        run: mamba uninstall ipython
 
       - name: Install
         shell: bash -l {0}
diff --git a/continuous_integration/scripts/test_imports.sh b/continuous_integration/scripts/test_imports.sh
index 1a57a0015..547d0f762 100644
--- a/continuous_integration/scripts/test_imports.sh
+++ b/continuous_integration/scripts/test_imports.sh
@@ -5,13 +5,13 @@ set -o errexit
 test_import () {
     echo "Create environment: python=$PYTHON_VERSION $1"
     # Create an empty environment
-    conda create -q -y -n test-imports -c conda-forge python=$PYTHON_VERSION pyyaml fsspec toolz partd cloudpickle $1
+    mamba create -q -y -n test-imports -c conda-forge python=$PYTHON_VERSION pyyaml fsspec toolz partd cloudpickle $1
     conda activate test-imports
     pip install -e .
     echo "python -c '$2'"
     python -c "$2"
     conda deactivate
-    conda env remove -n test-imports
+    mamba env remove -n test-imports
 }
 
 test_import ""                                "import dask, dask.base, dask.multiprocessing, dask.threaded, dask.optimization, dask.bag, dask.delayed, dask.graph_manipulation, dask.layers"
diff --git a/dask/tests/test_utils.py b/dask/tests/test_utils.py
index 36f125104..d53a22206 100644
--- a/dask/tests/test_utils.py
+++ b/dask/tests/test_utils.py
@@ -32,6 +32,7 @@ from dask.utils import (
     iter_chunks,
     stringify,
     stringify_collection_keys,
+    format_bytes,
 )
 from dask.utils_test import inc
 from dask.highlevelgraph import HighLevelGraph
@@ -637,3 +638,24 @@ def test_stringify_collection_keys():
     assert res[0] == str(obj[0])
     assert res[1] == str(obj[1])
     assert res[2] == obj[2]
+
+
+@pytest.mark.parametrize(
+    "n,expect",
+    [
+        (0, "0 B"),
+        (920, "920 B"),
+        (930, "0.91 kiB"),
+        (921.23 * 2 ** 10, "921.23 kiB"),
+        (931.23 * 2 ** 10, "0.91 MiB"),
+        (921.23 * 2 ** 20, "921.23 MiB"),
+        (931.23 * 2 ** 20, "0.91 GiB"),
+        (921.23 * 2 ** 30, "921.23 GiB"),
+        (931.23 * 2 ** 30, "0.91 TiB"),
+        (921.23 * 2 ** 40, "921.23 TiB"),
+        (931.23 * 2 ** 40, "0.91 PiB"),
+        (2 ** 60, "1024.00 PiB"),
+    ],
+)
+def test_format_bytes(n, expect):
+    assert format_bytes(int(n)) == expect