Repository: narwhals-dev/narwhals

Issue:
enh: `how='diagonal'` in `concat`
This would also be useful in Prophet...not essential, but it would reduce the diff

check the description here https://docs.pola.rs/api/python/stable/reference/api/polars.concat.html

Discussion:

Patch:
diff --git a/README.md b/README.md
index a7a08188..6133a0ed 100644
--- a/README.md
+++ b/README.md
@@ -42,6 +42,7 @@ Get started!
 Join the party!
 
 - [altair](https://github.com/vega/altair/)
+- [hierarchicalforecast](https://github.com/Nixtla/hierarchicalforecast)
 - [marimo](https://github.com/marimo-team/marimo)
 - [panel-graphic-walker](https://github.com/panel-extensions/panel-graphic-walker)
 - [plotly](https://plotly.com)
diff --git a/narwhals/_arrow/namespace.py b/narwhals/_arrow/namespace.py
index 8a39ff09..d6737ce9 100644
--- a/narwhals/_arrow/namespace.py
+++ b/narwhals/_arrow/namespace.py
@@ -12,6 +12,7 @@ from narwhals._arrow.expr import ArrowExpr
 from narwhals._arrow.selectors import ArrowSelectorNamespace
 from narwhals._arrow.series import ArrowSeries
 from narwhals._arrow.utils import broadcast_series
+from narwhals._arrow.utils import diagonal_concat
 from narwhals._arrow.utils import horizontal_concat
 from narwhals._arrow.utils import vertical_concat
 from narwhals._expression_parsing import combine_root_names
@@ -299,7 +300,7 @@ class ArrowNamespace:
         self,
         items: Iterable[ArrowDataFrame],
         *,
-        how: Literal["horizontal", "vertical"],
+        how: Literal["horizontal", "vertical", "diagonal"],
     ) -> ArrowDataFrame:
         dfs: list[Any] = [item._native_frame for item in items]
 
@@ -315,6 +316,12 @@ class ArrowNamespace:
                 backend_version=self._backend_version,
                 dtypes=self._dtypes,
             )
+        if how == "diagonal":
+            return ArrowDataFrame(
+                diagonal_concat(dfs, self._backend_version),
+                backend_version=self._backend_version,
+                dtypes=self._dtypes,
+            )
         raise NotImplementedError
 
     def sum(self, *column_names: str) -> ArrowExpr:
diff --git a/narwhals/_arrow/utils.py b/narwhals/_arrow/utils.py
index 2ad2b334..45377847 100644
--- a/narwhals/_arrow/utils.py
+++ b/narwhals/_arrow/utils.py
@@ -237,6 +237,26 @@ def vertical_concat(dfs: list[Any]) -> Any:
     return pa.concat_tables(dfs).combine_chunks()
 
 
+def diagonal_concat(dfs: list[Any], backend_version: tuple[int, ...]) -> Any:
+    """Concatenate (native) DataFrames diagonally.
+
+    Should be in namespace.
+    """
+    if not dfs:
+        msg = "No dataframes to concatenate"  # pragma: no cover
+        raise AssertionError(msg)
+
+    import pyarrow as pa  # ignore-banned-import
+
+    kwargs = (
+        {"promote": True}
+        if backend_version < (14, 0, 0)
+        else {"promote_options": "default"}  # type: ignore[dict-item]
+    )
+
+    return pa.concat_tables(dfs, **kwargs).combine_chunks()
+
+
 def floordiv_compat(left: Any, right: Any) -> Any:
     # The following lines are adapted from pandas' pyarrow implementation.
     # Ref: https://github.com/pandas-dev/pandas/blob/262fcfbffcee5c3116e86a951d8b693f90411e68/pandas/core/arrays/arrow/array.py#L124-L154
diff --git a/narwhals/_dask/namespace.py b/narwhals/_dask/namespace.py
index 9aca1d8d..b3dc5840 100644
--- a/narwhals/_dask/namespace.py
+++ b/narwhals/_dask/namespace.py
@@ -198,7 +198,7 @@ class DaskNamespace:
         self,
         items: Iterable[DaskLazyFrame],
         *,
-        how: Literal["horizontal", "vertical"],
+        how: Literal["horizontal", "vertical", "diagonal"],
     ) -> DaskLazyFrame:
         import dask.dataframe as dd  # ignore-banned-import
 
@@ -242,6 +242,13 @@ class DaskNamespace:
                 backend_version=self._backend_version,
                 dtypes=self._dtypes,
             )
+        if how == "diagonal":
+            return DaskLazyFrame(
+                dd.concat(dfs, axis=0, join="outer"),
+                backend_version=self._backend_version,
+                dtypes=self._dtypes,
+            )
+
         raise NotImplementedError
 
     def mean_horizontal(self, *exprs: IntoDaskExpr) -> DaskExpr:
diff --git a/narwhals/_pandas_like/namespace.py b/narwhals/_pandas_like/namespace.py
index 45ba1d85..b001f19d 100644
--- a/narwhals/_pandas_like/namespace.py
+++ b/narwhals/_pandas_like/namespace.py
@@ -16,6 +16,7 @@ from narwhals._pandas_like.expr import PandasLikeExpr
 from narwhals._pandas_like.selectors import PandasSelectorNamespace
 from narwhals._pandas_like.series import PandasLikeSeries
 from narwhals._pandas_like.utils import create_compliant_series
+from narwhals._pandas_like.utils import diagonal_concat
 from narwhals._pandas_like.utils import horizontal_concat
 from narwhals._pandas_like.utils import vertical_concat
 
@@ -344,7 +345,7 @@ class PandasLikeNamespace:
         self,
         items: Iterable[PandasLikeDataFrame],
         *,
-        how: Literal["horizontal", "vertical"],
+        how: Literal["horizontal", "vertical", "diagonal"],
     ) -> PandasLikeDataFrame:
         dfs: list[Any] = [item._native_frame for item in items]
         if how == "horizontal":
@@ -369,6 +370,18 @@ class PandasLikeNamespace:
                 backend_version=self._backend_version,
                 dtypes=self._dtypes,
             )
+
+        if how == "diagonal":
+            return PandasLikeDataFrame(
+                diagonal_concat(
+                    dfs,
+                    implementation=self._implementation,
+                    backend_version=self._backend_version,
+                ),
+                implementation=self._implementation,
+                backend_version=self._backend_version,
+                dtypes=self._dtypes,
+            )
         raise NotImplementedError
 
     def when(
diff --git a/narwhals/_pandas_like/utils.py b/narwhals/_pandas_like/utils.py
index b9f29360..44bfc34b 100644
--- a/narwhals/_pandas_like/utils.py
+++ b/narwhals/_pandas_like/utils.py
@@ -229,6 +229,32 @@ def vertical_concat(
         raise TypeError(msg)
 
 
+def diagonal_concat(
+    dfs: list[Any], *, implementation: Implementation, backend_version: tuple[int, ...]
+) -> Any:
+    """Concatenate (native) DataFrames diagonally.
+
+    Should be in namespace.
+    """
+    if not dfs:
+        msg = "No dataframes to concatenate"  # pragma: no cover
+        raise AssertionError(msg)
+
+    if implementation in PANDAS_LIKE_IMPLEMENTATION:
+        extra_kwargs = (
+            {"copy": False, "sort": False}
+            if implementation is Implementation.PANDAS and backend_version < (1,)
+            else {"copy": False}
+            if implementation is Implementation.PANDAS and backend_version < (3,)
+            else {}
+        )
+        return implementation.to_native_namespace().concat(dfs, axis=0, **extra_kwargs)
+
+    else:  # pragma: no cover
+        msg = f"Expected pandas-like implementation ({PANDAS_LIKE_IMPLEMENTATION}), found {implementation}"
+        raise TypeError(msg)
+
+
 def native_series_from_iterable(
     data: Iterable[Any],
     name: str,
diff --git a/narwhals/_polars/namespace.py b/narwhals/_polars/namespace.py
index 0a9f53c8..19d2693d 100644
--- a/narwhals/_polars/namespace.py
+++ b/narwhals/_polars/namespace.py
@@ -5,6 +5,7 @@ from typing import Any
 from typing import Iterable
 from typing import Literal
 from typing import Sequence
+from typing import overload
 
 from narwhals._expression_parsing import parse_into_exprs
 from narwhals._polars.utils import extract_args_kwargs
@@ -68,11 +69,27 @@ class PolarsNamespace:
             pl.len(), dtypes=self._dtypes, backend_version=self._backend_version
         )
 
+    @overload
     def concat(
         self,
-        items: Sequence[PolarsDataFrame | PolarsLazyFrame],
+        items: Sequence[PolarsDataFrame],
         *,
-        how: Literal["vertical", "horizontal"],
+        how: Literal["vertical", "horizontal", "diagonal"],
+    ) -> PolarsDataFrame: ...
+
+    @overload
+    def concat(
+        self,
+        items: Sequence[PolarsLazyFrame],
+        *,
+        how: Literal["vertical", "horizontal", "diagonal"],
+    ) -> PolarsLazyFrame: ...
+
+    def concat(
+        self,
+        items: Sequence[PolarsDataFrame] | Sequence[PolarsLazyFrame],
+        *,
+        how: Literal["vertical", "horizontal", "diagonal"],
     ) -> PolarsDataFrame | PolarsLazyFrame:
         import polars as pl  # ignore-banned-import()
 
diff --git a/narwhals/functions.py b/narwhals/functions.py
index 1459d352..42b235b5 100644
--- a/narwhals/functions.py
+++ b/narwhals/functions.py
@@ -40,17 +40,19 @@ if TYPE_CHECKING:
 def concat(
     items: Iterable[FrameT],
     *,
-    how: Literal["horizontal", "vertical"] = "vertical",
+    how: Literal["horizontal", "vertical", "diagonal"] = "vertical",
 ) -> FrameT:
     """Concatenate multiple DataFrames, LazyFrames into a single entity.
 
     Arguments:
         items: DataFrames, LazyFrames to concatenate.
-        how: {'vertical', 'horizontal'}:
+        how: concatenating strategy:
 
             - vertical: Concatenate vertically. Column names must match.
             - horizontal: Concatenate horizontally. If lengths don't match, then
                 missing rows are filled with null values.
+            - diagonal: Finds a union between the column schemas and fills missing column
+                values with null.
 
     Returns:
         A new DataFrame, Lazyframe resulting from the concatenation.
@@ -75,17 +77,17 @@ def concat(
         Let's define a dataframe-agnostic function:
 
         >>> @nw.narwhalify
-        ... def func(df1, df2):
+        ... def agnostic_vertical_concat(df1, df2):
         ...     return nw.concat([df1, df2], how="vertical")
 
-        >>> func(df_pd_1, df_pd_2)
+        >>> agnostic_vertical_concat(df_pd_1, df_pd_2)
            a  b
         0  1  4
         1  2  5
         2  3  6
         0  5  1
         1  2  4
-        >>> func(df_pl_1, df_pl_2)
+        >>> agnostic_vertical_concat(df_pl_1, df_pl_2)
         shape: (5, 2)
         ┌─────┬─────┐
         │ a   ┆ b   │
@@ -115,16 +117,16 @@ def concat(
         Defining a dataframe-agnostic function:
 
         >>> @nw.narwhalify
-        ... def func(df1, df2):
+        ... def agnostic_horizontal_concat(df1, df2):
         ...     return nw.concat([df1, df2], how="horizontal")
 
-        >>> func(df_pd_1, df_pd_2)
+        >>> agnostic_horizontal_concat(df_pd_1, df_pd_2)
            a  b    c    d
         0  1  4  5.0  1.0
         1  2  5  2.0  4.0
         2  3  6  NaN  NaN
 
-        >>> func(df_pl_1, df_pl_2)
+        >>> agnostic_horizontal_concat(df_pl_1, df_pl_2)
         shape: (3, 4)
         ┌─────┬─────┬──────┬──────┐
         │ a   ┆ b   ┆ c    ┆ d    │
@@ -136,9 +138,47 @@ def concat(
         │ 3   ┆ 6   ┆ null ┆ null │
         └─────┴─────┴──────┴──────┘
 
+        Let's look at case a for diagonal concatenation:
+
+        >>> import pandas as pd
+        >>> import polars as pl
+        >>> import narwhals as nw
+        >>> data_1 = {"a": [1, 2], "b": [3.5, 4.5]}
+        >>> data_2 = {"a": [3, 4], "z": ["x", "y"]}
+
+        >>> df_pd_1 = pd.DataFrame(data_1)
+        >>> df_pd_2 = pd.DataFrame(data_2)
+        >>> df_pl_1 = pl.DataFrame(data_1)
+        >>> df_pl_2 = pl.DataFrame(data_2)
+
+        Defining a dataframe-agnostic function:
+
+        >>> @nw.narwhalify
+        ... def agnostic_diagonal_concat(df1, df2):
+        ...     return nw.concat([df1, df2], how="diagonal")
+
+        >>> agnostic_diagonal_concat(df_pd_1, df_pd_2)
+           a    b    z
+        0  1  3.5  NaN
+        1  2  4.5  NaN
+        0  3  NaN    x
+        1  4  NaN    y
+
+        >>> agnostic_diagonal_concat(df_pl_1, df_pl_2)
+        shape: (4, 3)
+        ┌─────┬──────┬──────┐
+        │ a   ┆ b    ┆ z    │
+        │ --- ┆ ---  ┆ ---  │
+        │ i64 ┆ f64  ┆ str  │
+        ╞═════╪══════╪══════╡
+        │ 1   ┆ 3.5  ┆ null │
+        │ 2   ┆ 4.5  ┆ null │
+        │ 3   ┆ null ┆ x    │
+        │ 4   ┆ null ┆ y    │
+        └─────┴──────┴──────┘
     """
-    if how not in ("horizontal", "vertical"):  # pragma: no cover
-        msg = "Only horizontal and vertical concatenations are supported"
+    if how not in {"horizontal", "vertical", "diagonal"}:  # pragma: no cover
+        msg = "Only vertical, horizontal and diagonal concatenations are supported."
         raise NotImplementedError(msg)
     if not items:
         msg = "No items to concatenate"
diff --git a/narwhals/stable/v1/__init__.py b/narwhals/stable/v1/__init__.py
index 2a94b268..53f4cd0c 100644
--- a/narwhals/stable/v1/__init__.py
+++ b/narwhals/stable/v1/__init__.py
@@ -2726,7 +2726,7 @@ def max_horizontal(*exprs: IntoExpr | Iterable[IntoExpr]) -> Expr:
 def concat(
     items: Iterable[DataFrame[Any]],
     *,
-    how: Literal["horizontal", "vertical"] = "vertical",
+    how: Literal["horizontal", "vertical", "diagonal"] = "vertical",
 ) -> DataFrame[Any]: ...
 
 
@@ -2734,24 +2734,26 @@ def concat(
 def concat(
     items: Iterable[LazyFrame[Any]],
     *,
-    how: Literal["horizontal", "vertical"] = "vertical",
+    how: Literal["horizontal", "vertical", "diagonal"] = "vertical",
 ) -> LazyFrame[Any]: ...
 
 
 def concat(
     items: Iterable[DataFrame[Any] | LazyFrame[Any]],
     *,
-    how: Literal["horizontal", "vertical"] = "vertical",
+    how: Literal["horizontal", "vertical", "diagonal"] = "vertical",
 ) -> DataFrame[Any] | LazyFrame[Any]:
     """Concatenate multiple DataFrames, LazyFrames into a single entity.
 
     Arguments:
         items: DataFrames, LazyFrames to concatenate.
-        how: {'vertical', 'horizontal'}:
+        how: concatenating strategy:
 
             - vertical: Concatenate vertically. Column names must match.
             - horizontal: Concatenate horizontally. If lengths don't match, then
                 missing rows are filled with null values.
+            - diagonal: Finds a union between the column schemas and fills missing column
+                values with null.
 
     Returns:
         A new DataFrame, Lazyframe resulting from the concatenation.
@@ -2776,17 +2778,17 @@ def concat(
         Let's define a dataframe-agnostic function:
 
         >>> @nw.narwhalify
-        ... def func(df1, df2):
+        ... def agnostic_vertical_concat(df1, df2):
         ...     return nw.concat([df1, df2], how="vertical")
 
-        >>> func(df_pd_1, df_pd_2)
+        >>> agnostic_vertical_concat(df_pd_1, df_pd_2)
            a  b
         0  1  4
         1  2  5
         2  3  6
         0  5  1
         1  2  4
-        >>> func(df_pl_1, df_pl_2)
+        >>> agnostic_vertical_concat(df_pl_1, df_pl_2)
         shape: (5, 2)
         ┌─────┬─────┐
         │ a   ┆ b   │
@@ -2816,16 +2818,16 @@ def concat(
         Defining a dataframe-agnostic function:
 
         >>> @nw.narwhalify
-        ... def func(df1, df2):
+        ... def agnostic_horizontal_concat(df1, df2):
         ...     return nw.concat([df1, df2], how="horizontal")
 
-        >>> func(df_pd_1, df_pd_2)
+        >>> agnostic_horizontal_concat(df_pd_1, df_pd_2)
            a  b    c    d
         0  1  4  5.0  1.0
         1  2  5  2.0  4.0
         2  3  6  NaN  NaN
 
-        >>> func(df_pl_1, df_pl_2)
+        >>> agnostic_horizontal_concat(df_pl_1, df_pl_2)
         shape: (3, 4)
         ┌─────┬─────┬──────┬──────┐
         │ a   ┆ b   ┆ c    ┆ d    │
@@ -2837,6 +2839,44 @@ def concat(
         │ 3   ┆ 6   ┆ null ┆ null │
         └─────┴─────┴──────┴──────┘
 
+        Let's look at case a for diagonal concatenation:
+
+        >>> import pandas as pd
+        >>> import polars as pl
+        >>> import narwhals as nw
+        >>> data_1 = {"a": [1, 2], "b": [3.5, 4.5]}
+        >>> data_2 = {"a": [3, 4], "z": ["x", "y"]}
+
+        >>> df_pd_1 = pd.DataFrame(data_1)
+        >>> df_pd_2 = pd.DataFrame(data_2)
+        >>> df_pl_1 = pl.DataFrame(data_1)
+        >>> df_pl_2 = pl.DataFrame(data_2)
+
+        Defining a dataframe-agnostic function:
+
+        >>> @nw.narwhalify
+        ... def agnostic_diagonal_concat(df1, df2):
+        ...     return nw.concat([df1, df2], how="diagonal")
+
+        >>> agnostic_diagonal_concat(df_pd_1, df_pd_2)
+           a    b    z
+        0  1  3.5  NaN
+        1  2  4.5  NaN
+        0  3  NaN    x
+        1  4  NaN    y
+
+        >>> agnostic_diagonal_concat(df_pl_1, df_pl_2)
+        shape: (4, 3)
+        ┌─────┬──────┬──────┐
+        │ a   ┆ b    ┆ z    │
+        │ --- ┆ ---  ┆ ---  │
+        │ i64 ┆ f64  ┆ str  │
+        ╞═════╪══════╪══════╡
+        │ 1   ┆ 3.5  ┆ null │
+        │ 2   ┆ 4.5  ┆ null │
+        │ 3   ┆ null ┆ x    │
+        │ 4   ┆ null ┆ y    │
+        └─────┴──────┴──────┘
     """
     return _stableify(nw.concat(items, how=how))  # type: ignore[no-any-return]
 


Test patch:
diff --git a/.github/workflows/downstream_tests.yml b/.github/workflows/downstream_tests.yml
index f9b8a42f..c8a1cbcc 100644
--- a/.github/workflows/downstream_tests.yml
+++ b/.github/workflows/downstream_tests.yml
@@ -272,3 +272,43 @@ jobs:
         run: |
             cd plotly.py
             pytest packages/python/plotly/plotly/tests/test_optional/test_px
+
+  hierarchicalforecast:
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+        os: [ubuntu-latest]
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: "true"
+          cache-suffix: ${{ matrix.python-version }}
+          cache-dependency-glob: "**requirements*.txt"
+      - name: clone-hierarchicalforecast
+        run: |
+            git clone https://github.com/Nixtla/hierarchicalforecast --depth=1
+            cd hierarchicalforecast
+            git log
+      - name: install-basics
+        run: uv pip install --upgrade tox virtualenv setuptools --system
+      - name: install-deps
+        run: |
+            cd hierarchicalforecast
+            uv pip install --system ".[dev]"
+      - name: install-narwhals-dev
+        run: |
+            uv pip uninstall narwhals --system
+            uv pip install -e . --system
+      - name: show-deps
+        run: uv pip freeze
+      - name: Run tests
+        run: |
+            cd hierarchicalforecast
+            nbdev_test --do_print --timing --n_workers 0
diff --git a/tests/frame/concat_test.py b/tests/frame/concat_test.py
index 876cec03..567cb4ca 100644
--- a/tests/frame/concat_test.py
+++ b/tests/frame/concat_test.py
@@ -48,3 +48,23 @@ def test_concat_vertical(constructor: Constructor) -> None:
         nw.concat([df_left, df_right.rename({"d": "i"})], how="vertical").collect()
     with pytest.raises((Exception, TypeError), match="unable to vstack|unable to append"):
         nw.concat([df_left, df_left.select("d")], how="vertical").collect()
+
+
+def test_concat_diagonal(constructor: Constructor) -> None:
+    data_1 = {"a": [1, 3], "b": [4, 6]}
+    data_2 = {"a": [100, 200], "z": ["x", "y"]}
+    expected = {
+        "a": [1, 3, 100, 200],
+        "b": [4, 6, None, None],
+        "z": [None, None, "x", "y"],
+    }
+
+    df_1 = nw.from_native(constructor(data_1)).lazy()
+    df_2 = nw.from_native(constructor(data_2)).lazy()
+
+    result = nw.concat([df_1, df_2], how="diagonal")
+
+    assert_equal_data(result, expected)
+
+    with pytest.raises(ValueError, match="No items"):
+        nw.concat([], how="diagonal")