Repository: python-graphblas/python-graphblas

Issue:
Expose `GxB_pack_HyperHash` and `GxB_unpack_HyperHash`
as e.g. two new methods on `matrix.ss`.

Discussion:
coveralls: 
[![Coverage Status](https://coveralls.io/builds/53647815/badge)](https://coveralls.io/builds/53647815)

Coverage decreased (-0.06%) to 98.609% when pulling **119993a794b0a3b4186d00255694e55b72a1b007 on eriknw:pack_hyperhash** into **2fc650eafa79e9d59414c6faf4c302a37cab45fe on python-graphblas:main**.


Patch:
diff --git a/graphblas/core/matrix.py b/graphblas/core/matrix.py
index d250e79..e95cd3c 100644
--- a/graphblas/core/matrix.py
+++ b/graphblas/core/matrix.py
@@ -6,7 +6,7 @@ import numpy as np
 from .. import backend, binary, monoid, select, semiring
 from ..dtypes import _INDEX, FP64, lookup_dtype, unify
 from ..exceptions import DimensionMismatch, NoValue, check_status
-from . import NULL, automethods, ffi, lib, utils
+from . import automethods, ffi, lib, utils
 from .base import BaseExpression, BaseType, _check_mask, call
 from .expr import AmbiguousAssignOrExtract, IndexerResolver, Updater
 from .mask import Mask, StructuralMask, ValueMask
@@ -2248,7 +2248,7 @@ class Matrix(BaseType):
         import pygraphblas as pg
 
         matrix = pg.Matrix(self.gb_obj, pg.types._gb_type_to_type(self.dtype.gb_obj))
-        self.gb_obj = NULL
+        self.gb_obj = ffi_new("GrB_Matrix*")
         return matrix
 
     @classmethod
@@ -2274,7 +2274,7 @@ class Matrix(BaseType):
         rv = cls(matrix._matrix, dtype)
         rv._nrows = matrix.nrows
         rv._ncols = matrix.ncols
-        matrix._matrix = NULL
+        matrix._matrix = ffi_new("GrB_Matrix*")
         return rv
 
 
diff --git a/graphblas/core/ss/matrix.py b/graphblas/core/ss/matrix.py
index 245c6c2..728e6a0 100644
--- a/graphblas/core/ss/matrix.py
+++ b/graphblas/core/ss/matrix.py
@@ -1,6 +1,5 @@
 import itertools
 import warnings
-from numbers import Integral, Number
 
 import numba
 import numpy as np
@@ -13,21 +12,23 @@ from ... import monoid
 from ...dtypes import _INDEX, BOOL, INT64, _string_to_dtype, lookup_dtype
 from ...exceptions import _error_code_lookup, check_status, check_status_carg
 from .. import NULL, ffi, lib
-from ..base import call, record_raw
+from ..base import call
 from ..scalar import Scalar, _as_scalar, _scalar_index
 from ..utils import (
     _CArray,
+    _MatrixArray,
     _Pointer,
+    get_order,
     get_shape,
     ints_to_numpy_buffer,
     libget,
+    normalize_chunks,
     output_type,
     values_to_numpy_buffer,
     wrapdoc,
 )
 from .config import BaseConfig
 from .descriptor import get_compression_descriptor, get_nthreads_descriptor
-from .utils import get_order
 
 ffi_new = ffi.new
 
@@ -177,96 +178,6 @@ def head(matrix, n=10, dtype=None, *, sort=False):
     return rows, cols, vals
 
 
-def normalize_chunks(chunks, shape):
-    """Normalize chunks argument for use by `Matrix.ss.split`.
-
-    Examples
-    --------
-    >>> shape = (10, 20)
-    >>> normalize_chunks(10, shape)
-    [(10,), (10, 10)]
-    >>> normalize_chunks((10, 10), shape)
-    [(10,), (10, 10)]
-    >>> normalize_chunks([None, (5, 15)], shape)
-    [(10,), (5, 15)]
-    >>> normalize_chunks((5, (5, None)), shape)
-    [(5, 5), (5, 15)]
-    """
-    if isinstance(chunks, (list, tuple)):
-        pass
-    elif isinstance(chunks, Number):
-        chunks = (chunks,) * len(shape)
-    elif isinstance(chunks, np.ndarray):
-        chunks = chunks.tolist()
-    else:
-        raise TypeError(
-            f"chunks argument must be a list, tuple, or numpy array; got: {type(chunks)}"
-        )
-    if len(chunks) != len(shape):
-        typ = "Vector" if len(shape) == 1 else "Matrix"
-        raise ValueError(
-            f"chunks argument must be of length {len(shape)} (one for each dimension of a {typ})"
-        )
-    chunksizes = []
-    for size, chunk in zip(shape, chunks):
-        if chunk is None:
-            cur_chunks = [size]
-        elif isinstance(chunk, Integral) or isinstance(chunk, float) and chunk.is_integer():
-            chunk = int(chunk)
-            if chunk < 0:
-                raise ValueError(f"Chunksize must be greater than 0; got: {chunk}")
-            div, mod = divmod(size, chunk)
-            cur_chunks = [chunk] * div
-            if mod:
-                cur_chunks.append(mod)
-        elif isinstance(chunk, (list, tuple)):
-            cur_chunks = []
-            none_index = None
-            for c in chunk:
-                if isinstance(c, Integral) or isinstance(c, float) and c.is_integer():
-                    c = int(c)
-                    if c < 0:
-                        raise ValueError(f"Chunksize must be greater than 0; got: {c}")
-                elif c is None:
-                    if none_index is not None:
-                        raise TypeError(
-                            'None value in chunks for "the rest" can only appear once per dimension'
-                        )
-                    none_index = len(cur_chunks)
-                    c = 0
-                else:
-                    raise TypeError(
-                        "Bad type for element in chunks; expected int or None, but got: "
-                        f"{type(chunks)}"
-                    )
-                cur_chunks.append(c)
-            if none_index is not None:
-                fill = size - sum(cur_chunks)
-                if fill < 0:
-                    raise ValueError(
-                        "Chunks are too large; None value in chunks would need to be negative "
-                        "to match size of input"
-                    )
-                cur_chunks[none_index] = fill
-        elif isinstance(chunk, np.ndarray):
-            if not np.issubdtype(chunk.dtype, np.integer):
-                raise TypeError(f"numpy array for chunks must be integer dtype; got {chunk.dtype}")
-            if chunk.ndim != 1:
-                raise TypeError(
-                    f"numpy array for chunks must be 1-dimension; got ndim={chunk.ndim}"
-                )
-            if (chunk < 0).any():
-                raise ValueError(f"Chunksize must be greater than 0; got: {chunk[chunk < 0]}")
-            cur_chunks = chunk.tolist()
-        else:
-            raise TypeError(
-                "Chunks for a dimension must be an integer, a list or tuple of integers, or None."
-                f"  Got: {type(chunk)}"
-            )
-        chunksizes.append(cur_chunks)
-    return chunksizes
-
-
 def _concat_mn(tiles, *, is_matrix=None):
     """Argument checking for `Matrix.ss.concat` and returns number of tiles in each dimension"""
     from ..matrix import Matrix, TransposedMatrix
@@ -326,16 +237,6 @@ def _as_matrix(x):
     return x._as_matrix() if hasattr(x, "_as_matrix") else x
 
 
-class MatrixArray:
-    __slots__ = "_carg", "_exc_arg", "name"
-
-    def __init__(self, matrices, exc_arg=None, *, name):
-        self._carg = matrices
-        self._exc_arg = exc_arg
-        self.name = name
-        record_raw(f"GrB_Matrix {name}[{len(matrices)}];")
-
-
 class MatrixConfig(BaseConfig):
     """Get and set configuration options for this Matrix.
 
@@ -539,7 +440,7 @@ class ss:
         call(
             "GxB_Matrix_split",
             [
-                MatrixArray(tiles, self._parent, name="tiles"),
+                _MatrixArray(tiles, self._parent, name="tiles"),
                 _as_scalar(m, _INDEX, is_cscalar=True),
                 _as_scalar(n, _INDEX, is_cscalar=True),
                 _CArray(tile_nrows),
@@ -580,7 +481,7 @@ class ss:
             "GxB_Matrix_concat",
             [
                 self._parent,
-                MatrixArray(ctiles, name="tiles"),
+                _MatrixArray(ctiles, name="tiles"),
                 _as_scalar(m, _INDEX, is_cscalar=True),
                 _as_scalar(n, _INDEX, is_cscalar=True),
                 None,
@@ -3644,6 +3545,39 @@ class ss:
         else:
             raise ValueError(f"Invalid format: {format}")
 
+    def unpack_hyperhash(self, *, compute=False, name=None):
+        """Unpacks the hyper_hash of a hypersparse matrix if possible.
+
+        Will return None if the matrix is not hypersparse or if the hash is not computed.
+        Use ``compute=True`` to compute the hyper_hash if the input is hypersparse.
+
+        Use ``pack_hyperhash`` to move a hyper_hash matrix that was previously unpacked
+        back into a matrix.
+
+        This may be used before unpacking a HyperCSR or HyperCSC matrix to preserve the
+        full underlying data structure that can be packed back into an empty matrix.
+        """
+        from ..matrix import Matrix
+
+        if compute and self.format.startswith("hypercs"):
+            self._parent.wait()
+        rv = Matrix._from_obj(ffi_new("GrB_Matrix*"), INT64, 0, 0, name=name)
+        call("GxB_unpack_HyperHash", [self._parent, _Pointer(rv), None])
+        if rv.gb_obj[0] == NULL:
+            return
+        rv._nrows = rv.nrows
+        rv._ncols = rv.ncols
+        return rv
+
+    def pack_hyperhash(self, Y):
+        """Pack a hyper_hash matrix Y into the current hypersparse matrix.
+
+        The hyper_hash matrix Y should be from ``unpack_hyperhash`` and unmodified.
+
+        This uses move semantics. Y will become an invalid matrix.
+        """
+        call("GxB_pack_HyperHash", [self._parent, _Pointer(Y), None])
+
     @wrapdoc(head)
     def head(self, n=10, dtype=None, *, sort=False):
         return head(self._parent, n, dtype, sort=sort)
diff --git a/graphblas/core/ss/utils.py b/graphblas/core/ss/utils.py
deleted file mode 100644
index b9f8798..0000000
--- a/graphblas/core/ss/utils.py
+++ /dev/null
@@ -1,11 +0,0 @@
-def get_order(order):
-    val = order.lower()
-    if val in {"c", "row", "rows", "rowwise"}:
-        return "rowwise"
-    elif val in {"f", "col", "cols", "column", "columns", "colwise", "columnwise"}:
-        return "columnwise"
-    else:
-        raise ValueError(
-            f"Bad value for order: {order!r}.  "
-            'Expected "rowwise", "columnwise", "rows", "columns", "C", or "F"'
-        )
diff --git a/graphblas/core/ss/vector.py b/graphblas/core/ss/vector.py
index 79da58c..870246d 100644
--- a/graphblas/core/ss/vector.py
+++ b/graphblas/core/ss/vector.py
@@ -13,12 +13,20 @@ from ...exceptions import _error_code_lookup, check_status, check_status_carg
 from .. import NULL, ffi, lib
 from ..base import call
 from ..scalar import Scalar, _as_scalar
-from ..utils import _CArray, ints_to_numpy_buffer, libget, values_to_numpy_buffer, wrapdoc
+from ..utils import (
+    _CArray,
+    _MatrixArray,
+    get_order,
+    ints_to_numpy_buffer,
+    libget,
+    normalize_chunks,
+    values_to_numpy_buffer,
+    wrapdoc,
+)
 from .config import BaseConfig
 from .descriptor import get_compression_descriptor, get_nthreads_descriptor
-from .matrix import MatrixArray, _concat_mn, normalize_chunks
+from .matrix import _concat_mn
 from .prefix_scan import prefix_scan
-from .utils import get_order
 
 ffi_new = ffi.new
 
@@ -257,7 +265,7 @@ class ss:
         call(
             "GxB_Matrix_split",
             [
-                MatrixArray(tiles, parent, name="tiles"),
+                _MatrixArray(tiles, parent, name="tiles"),
                 _as_scalar(m, _INDEX, is_cscalar=True),
                 _as_scalar(1, _INDEX, is_cscalar=True),
                 _CArray(tile_nrows),
@@ -286,7 +294,7 @@ class ss:
             "GxB_Matrix_concat",
             [
                 self._parent._as_matrix(),
-                MatrixArray(ctiles, name="tiles"),
+                _MatrixArray(ctiles, name="tiles"),
                 _as_scalar(m, _INDEX, is_cscalar=True),
                 _as_scalar(1, _INDEX, is_cscalar=True),
                 None,
diff --git a/graphblas/core/utils.py b/graphblas/core/utils.py
index 48831a3..8a236bb 100644
--- a/graphblas/core/utils.py
+++ b/graphblas/core/utils.py
@@ -1,3 +1,5 @@
+from numbers import Integral, Number
+
 import numpy as np
 
 from ..dtypes import _INDEX, lookup_dtype
@@ -109,6 +111,109 @@ def get_shape(nrows, ncols, dtype=None, **arrays):
     return nrows, ncols
 
 
+def get_order(order):
+    val = order.lower()
+    if val in {"c", "row", "rows", "rowwise"}:
+        return "rowwise"
+    elif val in {"f", "col", "cols", "column", "columns", "colwise", "columnwise"}:
+        return "columnwise"
+    else:
+        raise ValueError(
+            f"Bad value for order: {order!r}.  "
+            'Expected "rowwise", "columnwise", "rows", "columns", "C", or "F"'
+        )
+
+
+def normalize_chunks(chunks, shape):
+    """Normalize chunks argument for use by `Matrix.ss.split`.
+
+    Examples
+    --------
+    >>> shape = (10, 20)
+    >>> normalize_chunks(10, shape)
+    [(10,), (10, 10)]
+    >>> normalize_chunks((10, 10), shape)
+    [(10,), (10, 10)]
+    >>> normalize_chunks([None, (5, 15)], shape)
+    [(10,), (5, 15)]
+    >>> normalize_chunks((5, (5, None)), shape)
+    [(5, 5), (5, 15)]
+    """
+    if isinstance(chunks, (list, tuple)):
+        pass
+    elif isinstance(chunks, Number):
+        chunks = (chunks,) * len(shape)
+    elif isinstance(chunks, np.ndarray):
+        chunks = chunks.tolist()
+    else:
+        raise TypeError(
+            f"chunks argument must be a list, tuple, or numpy array; got: {type(chunks)}"
+        )
+    if len(chunks) != len(shape):
+        typ = "Vector" if len(shape) == 1 else "Matrix"
+        raise ValueError(
+            f"chunks argument must be of length {len(shape)} (one for each dimension of a {typ})"
+        )
+    chunksizes = []
+    for size, chunk in zip(shape, chunks):
+        if chunk is None:
+            cur_chunks = [size]
+        elif isinstance(chunk, Integral) or isinstance(chunk, float) and chunk.is_integer():
+            chunk = int(chunk)
+            if chunk < 0:
+                raise ValueError(f"Chunksize must be greater than 0; got: {chunk}")
+            div, mod = divmod(size, chunk)
+            cur_chunks = [chunk] * div
+            if mod:
+                cur_chunks.append(mod)
+        elif isinstance(chunk, (list, tuple)):
+            cur_chunks = []
+            none_index = None
+            for c in chunk:
+                if isinstance(c, Integral) or isinstance(c, float) and c.is_integer():
+                    c = int(c)
+                    if c < 0:
+                        raise ValueError(f"Chunksize must be greater than 0; got: {c}")
+                elif c is None:
+                    if none_index is not None:
+                        raise TypeError(
+                            'None value in chunks for "the rest" can only appear once per dimension'
+                        )
+                    none_index = len(cur_chunks)
+                    c = 0
+                else:
+                    raise TypeError(
+                        "Bad type for element in chunks; expected int or None, but got: "
+                        f"{type(chunks)}"
+                    )
+                cur_chunks.append(c)
+            if none_index is not None:
+                fill = size - sum(cur_chunks)
+                if fill < 0:
+                    raise ValueError(
+                        "Chunks are too large; None value in chunks would need to be negative "
+                        "to match size of input"
+                    )
+                cur_chunks[none_index] = fill
+        elif isinstance(chunk, np.ndarray):
+            if not np.issubdtype(chunk.dtype, np.integer):
+                raise TypeError(f"numpy array for chunks must be integer dtype; got {chunk.dtype}")
+            if chunk.ndim != 1:
+                raise TypeError(
+                    f"numpy array for chunks must be 1-dimension; got ndim={chunk.ndim}"
+                )
+            if (chunk < 0).any():
+                raise ValueError(f"Chunksize must be greater than 0; got: {chunk[chunk < 0]}")
+            cur_chunks = chunk.tolist()
+        else:
+            raise TypeError(
+                "Chunks for a dimension must be an integer, a list or tuple of integers, or None."
+                f"  Got: {type(chunk)}"
+            )
+        chunksizes.append(cur_chunks)
+    return chunksizes
+
+
 class class_property:
     __slots__ = "classval", "member_property"
 
@@ -175,6 +280,18 @@ class _Pointer:
         return f"&{name}"
 
 
+class _MatrixArray:
+    __slots__ = "_carg", "_exc_arg", "name"
+
+    def __init__(self, matrices, exc_arg=None, *, name):
+        from .base import record_raw
+
+        self._carg = matrices
+        self._exc_arg = exc_arg
+        self.name = name
+        record_raw(f"GrB_Matrix {name}[{len(matrices)}];")
+
+
 def _autogenerate_code(
     filename,
     text,
diff --git a/graphblas/core/vector.py b/graphblas/core/vector.py
index da60153..90c40ad 100644
--- a/graphblas/core/vector.py
+++ b/graphblas/core/vector.py
@@ -6,7 +6,7 @@ import numpy as np
 from .. import backend, binary, monoid, select, semiring
 from ..dtypes import _INDEX, FP64, INT64, lookup_dtype, unify
 from ..exceptions import DimensionMismatch, NoValue, check_status
-from . import NULL, automethods, ffi, lib, utils
+from . import automethods, ffi, lib, utils
 from .base import BaseExpression, BaseType, _check_mask, call
 from .expr import AmbiguousAssignOrExtract, IndexerResolver, Updater
 from .mask import Mask, StructuralMask, ValueMask
@@ -1582,7 +1582,7 @@ class Vector(BaseType):
         import pygraphblas as pg
 
         vector = pg.Vector(self.gb_obj, pg.types._gb_type_to_type(self.dtype.gb_obj))
-        self.gb_obj = NULL
+        self.gb_obj = ffi_new("GrB_Vector*")
         return vector
 
     @classmethod
@@ -1607,7 +1607,7 @@ class Vector(BaseType):
         dtype = lookup_dtype(vector.gb_type)
         rv = cls(vector._vector, dtype)
         rv._size = vector.size
-        vector._vector = NULL
+        vector._vector = ffi_new("GrB_Vector*")
         return rv
 
 
diff --git a/setup.py b/setup.py
index c997c84..aa85e5c 100644
--- a/setup.py
+++ b/setup.py
@@ -25,7 +25,7 @@ setup(
     url="https://github.com/python-graphblas/python-graphblas",
     packages=find_packages(),
     python_requires=">=3.8",
-    install_requires=["suitesparse-graphblas >=7.2.0.0, <7.4", "numba", "donfig", "pyyaml"],
+    install_requires=["suitesparse-graphblas >=7.3.0.0, <7.4", "numba", "donfig", "pyyaml"],
     extras_require=extras_require,
     include_package_data=True,
     license="Apache License 2.0",


Test patch:
diff --git a/graphblas/tests/test_matrix.py b/graphblas/tests/test_matrix.py
index da9cbdf..026f9f7 100644
--- a/graphblas/tests/test_matrix.py
+++ b/graphblas/tests/test_matrix.py
@@ -2527,7 +2527,7 @@ def test_diag(A, params):
 
 
 def test_normalize_chunks():
-    from graphblas.core.ss.matrix import normalize_chunks
+    from graphblas.core.utils import normalize_chunks
 
     shape = (20, 20)
     assert normalize_chunks(10, shape) == [[10, 10], [10, 10]]
@@ -3746,3 +3746,20 @@ def test_to_csr_from_csc(A):
     assert expected.isequal(B, check_dtype=True)
     assert Matrix.from_csr(*B.to_csr(), ncols=2).isequal(B)
     assert Matrix.from_csr(*B.to_csr()).isequal(B[:, 0:0].new())
+
+
+def test_ss_pack_hyperhash(A):
+    A.ss.config["sparsity_control"] = "sparse"
+    assert A.ss.unpack_hyperhash() is None
+
+    C = Matrix(int, 20000, 200000)
+    C.ss.config["sparsity_control"] = "hypersparse"
+    C[100, 2000] = 2
+    C[10, 20] = 1
+    Y = C.ss.unpack_hyperhash()
+    Y = C.ss.unpack_hyperhash(compute=True)
+    assert C.ss.unpack_hyperhash() is None
+    assert Y.nrows == C.nrows
+    C.ss.pack_hyperhash(Y)
+    assert Y.gb_obj[0] == graphblas.core.NULL
+    assert C.ss.unpack_hyperhash() is not None