Repository: NVIDIA-Merlin/NVTabular

Issue:
[DOC] Add dask-cudf instructions to documentation
We should add documentation on how to use the multi-gpu version of nvtabular.

We have a [benchmark](https://www.reviewnb.com/) using a dask client for multi-gpu already. We should export a simplified view of this as an example notebook, and link that notebook into the sphinx docs.


Discussion:

Resolving patch:
diff --git a/nvtabular/io.py b/nvtabular/io.py
--- a/nvtabular/io.py
+++ b/nvtabular/io.py
@@ -131,10 +131,16 @@ def _merge_general_metadata(meta_list):
     meta = None
     for md in meta_list:
         if meta:
-            meta["data_paths"] += md["data_paths"]
-            meta["file_stats"] += md["file_stats"]
+            if "data_paths" in md:
+                meta["data_paths"] += md["data_paths"]
+            if "file_stats" in md:
+                meta["file_stats"] += md["file_stats"]
         else:
             meta = md.copy()
+            if "data_paths" not in meta:
+                meta["data_paths"] = []
+            if "file_stats" not in meta:
+                meta["file_stats"] = []
     return meta
 
 


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,4 +1,5 @@
 import glob
+import os
 import random
 
 import cudf
@@ -41,6 +42,7 @@
 ]
 
 _CLIENT = None
+_CUDA_CLUSTER = None
 
 
 @pytest.fixture(scope="session")
@@ -51,6 +53,18 @@ def client():
     return _CLIENT
 
 
+@pytest.fixture(scope="session")
+def cuda_cluster():
+    from dask_cuda import LocalCUDACluster
+
+    global _CUDA_CLUSTER
+    if _CUDA_CLUSTER is None:
+        CUDA_VISIBLE_DEVICES = os.environ.get("CUDA_VISIBLE_DEVICES", "0")
+        n_workers = min(2, len(CUDA_VISIBLE_DEVICES.split(",")))
+        _CUDA_CLUSTER = LocalCUDACluster(n_workers=n_workers)
+    return _CUDA_CLUSTER
+
+
 @pytest.fixture(scope="session")
 def datasets(tmpdir_factory):
     df = cudf.datasets.timeseries(
diff --git a/tests/unit/test_notebooks.py b/tests/unit/test_notebooks.py
--- a/tests/unit/test_notebooks.py
+++ b/tests/unit/test_notebooks.py
@@ -49,6 +49,27 @@ def test_rossman_example(tmpdir):
     _run_notebook(tmpdir, notebook_path, lambda line: line.replace("EPOCHS = 25", "EPOCHS = 1"))
 
 
+def test_multigpu_dask_example(tmpdir, cuda_cluster):
+    pytest.importorskip("dask_cuda")
+    os.environ["BASE_DIR"] = str(tmpdir)
+    scheduler_port = cuda_cluster.scheduler_address
+
+    def _nb_modify(line):
+        # Use cuda_cluster "fixture" port rather than allowing notebook
+        # to deploy a LocalCUDACluster within the subprocess
+        line = line.replace("cluster = None", f"cluster = '{scheduler_port}'")
+        # Use a much smaller "toy" dataset
+        line = line.replace("write_count = 25", "write_count = 4")
+        line = line.replace('freq = "1s"', 'freq = "1h"')
+        # Use smaller partitions for smaller dataset
+        line = line.replace("part_mem_fraction=0.1", "part_size=1_000_000")
+        line = line.replace("out_files_per_proc=8", "out_files_per_proc=1")
+        return line
+
+    notebook_path = os.path.join(dirname(TEST_PATH), "examples", "multi-gpu_dask.ipynb")
+    _run_notebook(tmpdir, notebook_path, _nb_modify)
+
+
 def _run_notebook(tmpdir, notebook_path, transform=None):
     # read in the notebook as JSON, and extract a python script from it
     notebook = json.load(open(notebook_path))