Repository: modin-project/modin

Issue:
Add jupyter tutorials for Modin on Unidist



Discussion:

Resolving patch:
diff --git a/examples/tutorial/jupyter/execution/pandas_on_unidist/setup_kernel.py b/examples/tutorial/jupyter/execution/pandas_on_unidist/setup_kernel.py
new file mode 100644
--- /dev/null
+++ b/examples/tutorial/jupyter/execution/pandas_on_unidist/setup_kernel.py
@@ -0,0 +1,54 @@
+# Licensed to Modin Development Team under one or more contributor license agreements.
+# See the NOTICE file distributed with this work for additional information regarding
+# copyright ownership.  The Modin Development Team licenses this file to you under the
+# Apache License, Version 2.0 (the "License"); you may not use this file except in
+# compliance with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific language
+# governing permissions and limitations under the License.
+
+import sys
+from ipykernel import kernelspec
+
+
+default_make_ipkernel_cmd = kernelspec.make_ipkernel_cmd
+
+
+def custom_make_ipkernel_cmd(*args, **kwargs):
+    """
+    Build modified Popen command list for launching an IPython kernel with MPI.
+
+    Parameters
+    ----------
+    *args : iterable
+        Additional positional arguments to be passed in `default_make_ipkernel_cmd`.
+    **kwargs : dict
+        Additional keyword arguments to be passed in `default_make_ipkernel_cmd`.
+
+    Returns
+    -------
+    array
+        A Popen command list.
+
+    Notes
+    -----
+    The parameters of the function should be kept in sync with the ones of the original function.
+    """
+    mpi_arguments = ["mpiexec", "-n", "1"]
+    arguments = default_make_ipkernel_cmd(*args, **kwargs)
+    return mpi_arguments + arguments
+
+
+kernelspec.make_ipkernel_cmd = custom_make_ipkernel_cmd
+
+if __name__ == "__main__":
+    kernel_name = "python3mpi"
+    display_name = "Python 3 (ipykernel) with MPI"
+    dest = kernelspec.install(
+        kernel_name=kernel_name, display_name=display_name, prefix=sys.prefix
+    )
+    print(f"Installed kernelspec {kernel_name} in {dest}")  # noqa: T201


Test patch:
diff --git a/examples/tutorial/jupyter/execution/pandas_on_unidist/test/test_notebooks.py b/examples/tutorial/jupyter/execution/pandas_on_unidist/test/test_notebooks.py
new file mode 100644
--- /dev/null
+++ b/examples/tutorial/jupyter/execution/pandas_on_unidist/test/test_notebooks.py
@@ -0,0 +1,123 @@
+# Licensed to Modin Development Team under one or more contributor license agreements.
+# See the NOTICE file distributed with this work for additional information regarding
+# copyright ownership.  The Modin Development Team licenses this file to you under the
+# Apache License, Version 2.0 (the "License"); you may not use this file except in
+# compliance with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific language
+# governing permissions and limitations under the License.
+
+import os
+import sys
+
+import nbformat
+
+MODIN_DIR = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), *[".." for _ in range(6)])
+)
+sys.path.insert(0, MODIN_DIR)
+from examples.tutorial.jupyter.execution.test.utils import (  # noqa: E402
+    _replace_str,
+    _execute_notebook,
+    test_dataset_path,
+    download_taxi_dataset,
+    set_kernel,
+)
+
+# the kernel name "python3mpi" must match the one
+# that is set up in `examples/tutorial/jupyter/execution/pandas_on_unidist/setup_kernel.py`
+# for `Unidist` engine
+set_kernel(kernel_name="python3mpi")
+
+local_notebooks_dir = "examples/tutorial/jupyter/execution/pandas_on_unidist/local"
+
+
+# in this notebook user should replace 'import pandas as pd' with
+# 'import modin.pandas as pd' to make notebook work
+def test_exercise_1():
+    modified_notebook_path = os.path.join(local_notebooks_dir, "exercise_1_test.ipynb")
+    nb = nbformat.read(
+        os.path.join(local_notebooks_dir, "exercise_1.ipynb"),
+        as_version=nbformat.NO_CONVERT,
+    )
+
+    _replace_str(nb, "import pandas as pd", "import modin.pandas as pd")
+
+    nbformat.write(nb, modified_notebook_path)
+    _execute_notebook(modified_notebook_path)
+
+
+# this notebook works "as is" but for testing purposes we can use smaller dataset
+def test_exercise_2():
+    modified_notebook_path = os.path.join(local_notebooks_dir, "exercise_2_test.ipynb")
+    nb = nbformat.read(
+        os.path.join(local_notebooks_dir, "exercise_2.ipynb"),
+        as_version=nbformat.NO_CONVERT,
+    )
+
+    new_cell = f'path = "{test_dataset_path}"\n' + download_taxi_dataset
+
+    _replace_str(
+        nb,
+        'path = "s3://dask-data/nyc-taxi/2015/yellow_tripdata_2015-01.csv"',
+        new_cell,
+    )
+
+    nbformat.write(nb, modified_notebook_path)
+    _execute_notebook(modified_notebook_path)
+
+
+# in this notebook user should add custom mad implementation
+# to make notebook work
+def test_exercise_3():
+    modified_notebook_path = os.path.join(local_notebooks_dir, "exercise_3_test.ipynb")
+    nb = nbformat.read(
+        os.path.join(local_notebooks_dir, "exercise_3.ipynb"),
+        as_version=nbformat.NO_CONVERT,
+    )
+
+    user_mad_implementation = """PandasQueryCompiler.sq_mad_custom = TreeReduce.register(lambda cell_value, **kwargs: cell_value ** 2,
+                                                             pandas.DataFrame.mad)
+
+def sq_mad_func(self, axis=None, skipna=True, level=None, **kwargs):
+    if axis is None:
+        axis = 0
+
+    return self._reduce_dimension(
+        self._query_compiler.sq_mad_custom(
+            axis=axis, skipna=skipna, level=level, **kwargs
+        )
+    )
+
+pd.DataFrame.sq_mad_custom = sq_mad_func
+
+modin_mad_custom = df.sq_mad_custom()
+    """
+
+    _replace_str(nb, "modin_mad_custom = ...", user_mad_implementation)
+
+    nbformat.write(nb, modified_notebook_path)
+    _execute_notebook(modified_notebook_path)
+
+
+# this notebook works "as is" but for testing purposes we can use smaller dataset
+def test_exercise_4():
+    modified_notebook_path = os.path.join(local_notebooks_dir, "exercise_4_test.ipynb")
+    nb = nbformat.read(
+        os.path.join(local_notebooks_dir, "exercise_4.ipynb"),
+        as_version=nbformat.NO_CONVERT,
+    )
+
+    s3_path_cell = f's3_path = "{test_dataset_path}"\n' + download_taxi_dataset
+    _replace_str(
+        nb,
+        's3_path = "s3://dask-data/nyc-taxi/2015/yellow_tripdata_2015-01.csv"',
+        s3_path_cell,
+    )
+
+    nbformat.write(nb, modified_notebook_path)
+    _execute_notebook(modified_notebook_path)
diff --git a/examples/tutorial/jupyter/execution/test/utils.py b/examples/tutorial/jupyter/execution/test/utils.py
--- a/examples/tutorial/jupyter/execution/test/utils.py
+++ b/examples/tutorial/jupyter/execution/test/utils.py
@@ -15,8 +15,6 @@
 from nbconvert.preprocessors import ExecutePreprocessor
 
 test_dataset_path = "taxi.csv"
-ep = ExecutePreprocessor(timeout=600, kernel_name="python3")
-
 download_taxi_dataset = f"""import os
 import urllib.request
 if not os.path.exists("{test_dataset_path}"):
@@ -25,6 +23,40 @@
     """
 
 
+# Default kernel name for ``ExecutePreprocessor`` to be created
+_default_kernel_name = "python3"
+
+
+def set_kernel(kernel_name):
+    """
+    Set custom kernel for ``ExecutePreprocessor`` to be created.
+
+    Parameters
+    ----------
+    kernel_name : str
+        Kernel name.
+    """
+    global _default_kernel_name
+    _default_kernel_name = kernel_name
+
+
+def make_execute_preprocessor():
+    """
+    Make ``ExecutePreprocessor`` with the `_default_kernel_name`.
+
+    Returns
+    -------
+    nbconvert.preprocessors.ExecutePreprocessor
+        Execute processor entity.
+
+    Notes
+    -----
+    Note that `_default_kernel_name` can be changed for the concrete executions
+    (e.g., ``PandasOnUnidist`` with MPI backend).
+    """
+    return ExecutePreprocessor(timeout=600, kernel_name=_default_kernel_name)
+
+
 def _execute_notebook(notebook):
     """
     Execute a jupyter notebook.
@@ -35,6 +67,7 @@ def _execute_notebook(notebook):
         File-like object or path to the notebook to execute.
     """
     nb = nbformat.read(notebook, as_version=nbformat.NO_CONVERT)
+    ep = make_execute_preprocessor()
     ep.preprocess(nb)