Repository: scipy/scipy

Issue:
BUG: Memory usage in griddata function in version 1.12
### Describe your issue.

I'm working on interpolating big sparse matrices, and i noticed that upgrading from version 1.11.3 to version 1.12 my code gives me a memory error using the function griddata from scipy.interpolate, saying that is trying to allocate 45 GB.
The exact same code works fine in 1.11.3 allocating 16GB of memory in total.
The matrix in question has a shape of (14790, 12816) and the dtype is float32.
My machine has 32GB of RAM.

### Reproducing Code Example

```python
from scipy.interpolate import griddata
import numpy as np
from scipy.sparse import csr_matrix

shape = (14790, 12816)
num_nonzero = 488686

random_rows = np.random.randint(0, shape[0], num_nonzero)
random_cols = np.random.randint(0, shape[1], num_nonzero)

random_values = np.random.rand(num_nonzero).astype(np.float32)

sparse_matrix = csr_matrix((random_values, (random_rows, random_cols)), shape=shape, dtype=np.float32)
sparse_matrix = sparse_matrix.toarray()

coords = np.column_stack(np.nonzero(sparse_matrix))
values = sparse_matrix[coords[:, 0], coords[:, 1]]
grid_x, grid_y = np.mgrid[0:sparse_matrix.shape[0], 0:sparse_matrix.shape[1]]
depth_mask_cut = griddata(coords, values, (grid_x, grid_y), method='cubic')
```


### Error message

```shell
Traceback (most recent call last):
  File "C:\Users\aleal\Documents\Geoin\Aerial instance segmentation\building-detection\rooftops_detection_from_aerial_images\preprocess_data.py", line 51, in <module>
    main()
  File "C:\Users\aleal\Documents\Geoin\Aerial instance segmentation\building-detection\rooftops_detection_from_aerial_images\preprocess_data.py", line 47, in main
    preprocess_data(config)
  File "C:\Users\aleal\Documents\Geoin\Aerial instance segmentation\building-detection\rooftops_detection_from_aerial_images\utils\dataset_handler.py", line 74, in preprocess_data
    cutted_mask, cutted_depth_mask, cutted_image = preprocess_mask_image(mask, image, depth_mask, config)
  File "C:\Users\aleal\Documents\Geoin\Aerial instance segmentation\building-detection\rooftops_detection_from_aerial_images\utils\utils.py", line 517, in preprocess_mask_image
    depth_mask_cut = griddata(coords, values, (grid_x, grid_y), method='cubic')
  File "C:\Users\aleal\Documents\Geoin\Aerial instance segmentation\building-detection\rooftops_detection_from_aerial_images\building_detection_enviroment\lib\site-packages\scipy\interpolate\_ndgriddata.py", line 329, in griddata
    return ip(xi)
  File "interpnd.pyx", line 192, in scipy.interpolate.interpnd.NDInterpolatorBase.__call__
  File "interpnd.pyx", line 969, in scipy.interpolate.interpnd.CloughTocher2DInterpolator._evaluate_double
  File "interpnd.pyx", line 990, in scipy.interpolate.interpnd.CloughTocher2DInterpolator._do_evaluate
  File "interpnd.pyx", line 172, in scipy.interpolate.interpnd.NDInterpolatorBase._find_simplicies
  File "C:\Users\aleal\Documents\Geoin\Aerial instance segmentation\building-detection\rooftops_detection_from_aerial_images\building_detection_enviroment\lib\site-packages\numpy\lib\function_base.py", line 962, in copy
    return array(a, order=order, subok=subok, copy=True)
numpy.core._exceptions._ArrayMemoryError: Unable to allocate 45.2 GiB for an array with shape (189548640, 32) and data type float64
```


### SciPy/NumPy/Python version and system information

```shell
1.12.0 1.26.4 sys.version_info(major=3, minor=10, micro=11, releaselevel='final', serial=0)
{
  "Compilers": {
    "c": {
      "name": "gcc",
      "linker": "ld.bfd",
      "version": "10.3.0",
      "commands": "cc"
    },
    "cython": {
      "name": "cython",
      "linker": "cython",
      "version": "3.0.8",
      "commands": "cython"
    },
    "c++": {
      "name": "gcc",
      "linker": "ld.bfd",
      "version": "10.3.0",
      "commands": "c++"
    },
    "fortran": {
      "name": "gcc",
      "linker": "ld.bfd",
      "version": "10.3.0",
      "commands": "gfortran"
    },
    "pythran": {
      "version": "0.15.0",
      "include directory": "C:\\Users\\runneradmin\\AppData\\Local\\Temp\\pip-build-env-bidi1l_7\\overlay\\Lib\\site-packages/pythran"   
    }
  },
  "Machine Information": {
    "host": {
      "cpu": "x86_64",
      "family": "x86_64",
      "endian": "little",
      "system": "windows"
    },
    "build": {
      "cpu": "x86_64",
      "family": "x86_64",
      "endian": "little",
      "system": "windows"
    },
    "cross-compiled": false
  },
  "Build Dependencies": {
    "blas": {
      "name": "openblas",
      "found": true,
      "version": "0.3.21.dev",
      "detection method": "pkgconfig",
      "include directory": "/c/opt/64/include",
      "lib directory": "/c/opt/64/lib",
      "openblas configuration": "USE_64BITINT= DYNAMIC_ARCH=1 DYNAMIC_OLDER= NO_CBLAS= NO_LAPACK= NO_LAPACKE= NO_AFFINITY=1 USE_OPENMP= SKYLAKEX MAX_THREADS=2",
      "pc file directory": "c:/opt/64/lib/pkgconfig"
    },
    "lapack": {
      "name": "openblas",
      "found": true,
      "version": "0.3.21.dev",
      "detection method": "pkgconfig",
      "include directory": "/c/opt/64/include",
      "lib directory": "/c/opt/64/lib",
      "openblas configuration": "USE_64BITINT= DYNAMIC_ARCH=1 DYNAMIC_OLDER= NO_CBLAS= NO_LAPACK= NO_LAPACKE= NO_AFFINITY=1 USE_OPENMP= SKYLAKEX MAX_THREADS=2",
      "pc file directory": "c:/opt/64/lib/pkgconfig"
    },
    "pybind11": {
      "name": "pybind11",
      "version": "2.11.1",
      "detection method": "config-tool",
      "include directory": "unknown"
    }
  },
  "Python Information": {
    "path": "C:\\Users\\runneradmin\\AppData\\Local\\Temp\\cibw-run-u773bfjp\\cp310-win_amd64\\build\\venv\\Scripts\\python.exe",        
    "version": "3.10"
  }
}
```



Discussion:
Can you add a reproducible example --- something we can run as a standalone script, which defines all input arrays etc.
I have just updated the issue code section.
Thanks. So sparse matrices here is only a means to construct (rather large) dense inputs to `griddata`: it only receives dense ndarrays and never sees any csr_matrices. Good -- because it never supported and is unlikely to ever support sparse arrays directly.

The only PR which touched CloughTocher2DInterpolator in the 1.12.x time frame was https://github.com/scipy/scipy/pull/18376, any ideas @hovavalon ?


After inspecting my changes, I think the source of the defect is that after my changes indeed more memory is used, as the data about the location and simplicies of all the input points is kept in the memory at once.
In the `_find_simplicies` function, I use `np.copy` in order to return the locally defined cython arrays as numpy arrays.
I think that a possible solution is to replace the `_find_simplicies` function with `_find_simplex`, which will find a single simplex, and will be used in the `_do_evaluate` function.
A class may override this function so that it computes all the simplicies and saves them, and then just queries the saved results on call.

A possible temporary (and very ugly) solution is to split the data and call  `griddata` a few times. You might need to call the garbage collector between subsequent calls. I am sorry, I should have checked my changes on larger inputs.
If you have no special need in version `1.12` of course that it might be easier for you do downgrade to `1.11.x`.

What do you think?
Unfortunately, I won't have the time to create a PR in the next few weeks, but I will try to assist, and might fix it later myself if no one else will.

Going back to `_find_simplex` sounds right indeed. This would get us back to the `c` array being `double[NPY_MAXARGS]` instead of O(number_of_data_points times NPY_MAXARGS). 


I'm trying to solve this bug but I have a question: before @hovavalon's changes, in the `LinearNDInterpolator` class, to find the simplex we would call `qhull._find_simplex(&info, c, &xi[0,0] + i*ndim, &start, eps, eps_broad)` and in the `CloughTocher2DInterpolator` class we would call `qhull._find_simplex(&info, c, &xi[i,0], &start, eps, eps_broad)`. After @hovavalon's changes, the `_find_simplicies` function uses the second version of this call in both classes. Which version of this call should I use in the `LinearNDInterpolator` class?

Resolving patch:
diff --git a/benchmarks/benchmarks/interpolate.py b/benchmarks/benchmarks/interpolate.py
--- a/benchmarks/benchmarks/interpolate.py
+++ b/benchmarks/benchmarks/interpolate.py
@@ -8,6 +8,8 @@
 with safe_import():
     import scipy.interpolate as interpolate
 
+with safe_import():
+    from scipy.sparse import csr_matrix
 
 class Leaks(Benchmark):
     unit = "relative increase with repeats"
@@ -83,7 +85,34 @@ def setup(self, n_grids, method):
     def time_evaluation(self, n_grids, method):
         interpolate.griddata(self.points, self.values, (self.grid_x, self.grid_y),
                              method=method)
+        
+class GridDataPeakMem(Benchmark):
+    """
+    Benchmark based on https://github.com/scipy/scipy/issues/20357
+    """
+    def setup(self):
+        shape = (7395, 6408)
+        num_nonzero = 488686
+
+        rng = np.random.default_rng(1234)
 
+        random_rows = rng.integers(0, shape[0], num_nonzero)
+        random_cols = rng.integers(0, shape[1], num_nonzero)
+
+        random_values = rng.random(num_nonzero, dtype=np.float32)
+
+        sparse_matrix = csr_matrix((random_values, (random_rows, random_cols)), 
+                                   shape=shape, dtype=np.float32)
+        sparse_matrix = sparse_matrix.toarray()
+
+        self.coords = np.column_stack(np.nonzero(sparse_matrix))
+        self.values = sparse_matrix[self.coords[:, 0], self.coords[:, 1]]
+        self.grid_x, self.grid_y = np.mgrid[0:sparse_matrix.shape[0], 
+                                            0:sparse_matrix.shape[1]]
+
+    def peakmem_griddata(self):
+        interpolate.griddata(self.coords, self.values, (self.grid_x, self.grid_y), 
+                             method='cubic')
 
 class Interpolate1d(Benchmark):
     param_names = ['n_samples', 'method']
@@ -424,9 +453,6 @@ def __init__(self, points, xi, tol=1e-6, maxiter=400, **kwargs):
                                                         tol=tol, maxiter=maxiter)
         self.xi = None
         self._preprocess_xi(*xi)
-        self.simplices, self.c = (
-            interpolate.CloughTocher2DInterpolator._find_simplicies(self, self.xi)
-        )
 
     def _preprocess_xi(self, *args):
         if self.xi is None:
@@ -435,9 +461,6 @@ def _preprocess_xi(self, *args):
             )
         return self.xi, self.interpolation_points_shape
     
-    def _find_simplicies(self, xi):
-        return self.simplices, self.c
-
     def __call__(self, values):
         self._set_values(values)
         return super().__call__(self.xi)


Test patch: