Repository: pydap/pydap
Pull request #473 (2025-04-03 04:07:57)

enables caching of multiple dap4 urls and custom cache key for dimensions dap responses
<!-- Thank you very much for your hard work and contribution! -->
~DRAFT PR~

This PR will enable

- [x] Caching of dap4 urls (dmrs and dap responses). These urls must be defined as a list object and passed to the new function `datacube_urls`. 
- [x] Caching multiple dap responses of all dimensions within a datacube (say an entire Collection consisting of 100 granules/urls that share same dimensions) by a single key (per dimension). 
For example:
Consider dimensions: 'time', 'lat', 'lon', 'Z' all appear in each of the `N` urls that make up the collection. When requesting the `dap` response of the array `time`, it will no longer be downloaded N times. It will instead it is only downloaded once. Same for each of the other dimensions.
- [x] Test are added and all pass locally.


### New behavior
The following is an example of how it works:
```python
fom pydap.net import create_session
# use requests-cache by setting use_cache=True.
new_session = create_session(use_cache=True)

# Consider a list of dap4 urls
urls[:5]
>>>
['dap4://oceandata.sci.gsfc.nasa.gov/opendap/PACE_OCI/L3SMI/2024/0305/PACE_OCI.20240305.L3m.DAY.CHL.V3_0.chlor_a.4km.nc?dap4.ce=/lat;/lon;/chlor_a',
 'dap4://oceandata.sci.gsfc.nasa.gov/opendap/PACE_OCI/L3SMI/2024/0306/PACE_OCI.20240306.L3m.DAY.CHL.V3_0.chlor_a.4km.nc?dap4.ce=/lat;/lon;/chlor_a',
 'dap4://oceandata.sci.gsfc.nasa.gov/opendap/PACE_OCI/L3SMI/2024/0307/PACE_OCI.20240307.L3m.DAY.CHL.V3_0.chlor_a.4km.nc?dap4.ce=/lat;/lon;/chlor_a',
 'dap4://oceandata.sci.gsfc.nasa.gov/opendap/PACE_OCI/L3SMI/2024/0308/PACE_OCI.20240308.L3m.DAY.CHL.V3_0.chlor_a.4km.nc?dap4.ce=/lat;/lon;/chlor_a',
 'dap4://oceandata.sci.gsfc.nasa.gov/opendap/PACE_OCI/L3SMI/2024/0309/PACE_OCI.20240309.L3m.DAY.CHL.V3_0.chlor_a.4km.nc?dap4.ce=/lat;/lon;/chlor_a']
 
 
# now cache the datacube dmrs and their dimensions
# this line may take O(10) seconds, and only needs to be run once
datacube_urls(urls, cached_session)

# the cached-session now contains reference to O(100) OPeNDAP urls. Internally it uses 
# pydap so the metadata/dmr has been verified to be consistent with creating a pydap dataset
# and therefore an xarray dataset.

## open a virtually aggregated xarray dataset using the cached session
ds = xr.open_mfdataset(urls, engine='pydap', session=cached_session, parallel=True, combine='nested', concat_dim='time')

# That last line may take 1-5 seconds, depending on N-URLS and ncores.
```
Because it uses requests-cache - it can handle two types of authentication: token based and username/password based via netrc file (case above).



Mikejmnez: closing this

Patch:
diff --git a/src/pydap/client.py b/src/pydap/client.py
index 1a8be0a..eb89f7c 100644
--- a/src/pydap/client.py
+++ b/src/pydap/client.py
@@ -44,21 +44,25 @@ lazy mechanism for function call, supporting any function. Eg, to call the
 
 """
 
+import os
+import re
+import warnings
+from concurrent.futures import ThreadPoolExecutor
 from io import BytesIO, open
+from os.path import commonprefix
+from urllib.parse import parse_qs, unquote, urlencode
 
 from requests.utils import urlparse, urlunparse
+from requests_cache import CachedSession
 
-import pydap.handlers.dap
-import pydap.lib
-import pydap.model
-import pydap.net
-import pydap.parsers.das
-import pydap.parsers.dds
-import pydap.parsers.dmr
-from pydap.handlers.dap import UNPACKDAP4DATA, DAPHandler
-from pydap.lib import DEFAULT_TIMEOUT as DEFAULT_TIMEOUT
-
-from .net import create_session
+from .handlers.dap import UNPACKDAP4DATA, DAPHandler, StreamReader, unpack_dap2_data
+from .lib import DEFAULT_TIMEOUT as DEFAULT_TIMEOUT
+from .lib import encode
+from .model import DapType
+from .net import GET, create_session
+from .parsers.das import add_attributes, parse_das
+from .parsers.dds import dds_to_dataset
+from .parsers.dmr import DMRParser, dmr_to_dataset
 
 
 def open_url(
@@ -146,6 +150,97 @@ def open_url(
     return dataset
 
 
+def datacube_urls(urls, session=None):
+    """caches metadata for multiple OPeNDAP DAP4 URL, dap responses for
+    each dimension (once) on the datacube.
+    Parameters
+    ----------
+    urls : list
+        The URLs of the datasets that define a datacube. Each URL must begin
+        with the same base URL, and begin with `dap4://`.
+        session : requests.Session or requests-cache.CachedSession
+            A requests session object.
+
+    """
+
+    if not session:
+        session = create_session()
+    if not isinstance(urls, list) or len(urls) == 1:
+        raise TypeError("urls must be a list of `len` >= 2. Try again!")
+
+    # check elements in urls are strings
+    if not all(isinstance(url, str) for url in urls):
+        raise TypeError("`urls` must be a list of string urls")
+
+    schemes = [urlparse(urls[n]).scheme for n in range(len(urls))]
+    # check if all urls have the same scheme
+    scheme = set(schemes)
+    if len(scheme) > 1:
+        raise ValueError(
+            "URLs must have the same scheme. Try again with the same protocol."
+        )
+    if not scheme.pop() == "dap4":
+        warnings.warn(
+            "URLs must be a dap4 URL and begin with `dap4://`. To "
+            " learn about da4p2 urls, please visit the following link: \n "
+            " https://pydap.github.io/pydap/en/faqs/dap2_or_dap4_url.html"
+        )
+        return None
+    # All URLs begin with dap4 - to make sure DAP4 compliant
+    URLs = ["http" + urls[i][4:] for i in range(len(urls))]
+    ncores = min(len(urls), os.cpu_count() * 4)
+    dmr_urls = [
+        url + ".dmr" if "?" not in url else url.replace("?", ".dmr?") for url in URLs
+    ]
+
+    # this caches the dmr
+    with session as Session:  # Authenticate once
+        with ThreadPoolExecutor(max_workers=ncores) as executor:
+            results = list(
+                executor.map(lambda url: open_dmr(url, session=Session), dmr_urls)
+            )
+
+    # Download dimensions once and construct cache key their dap responses
+    base_url = URLs[0].split("?")[0]
+    # identify dimensions that repeat across the urls
+    nested = [
+        [val for val in results[i].dimensions.keys()] for i in range(len(results))
+    ]
+    dims = set([item for sublist in nested for item in sublist])
+    if not dims:
+        warnings.warn("No dimensions found in the dataset.")
+        return None
+
+    # TODO: make sure count of dimensions is the same as the number of urls
+
+    new_urls = [
+        base_url
+        + ".dap?dap4.ce="
+        + dim
+        + "%5B0:1:"
+        + str(results[0].dimensions[dim] - 1)
+        + "%5D"
+        for dim in list(dims)
+    ]
+    # xarray does not escaped CE
+    dim_ces = set(
+        [
+            dim + "[0:1:" + str(results[0].dimensions[dim] - 1) + "]"
+            for dim in list(dims)
+        ]
+    )
+    print("datacube has dimensions", dim_ces)
+    # create custom cache keys
+    if isinstance(session, CachedSession):
+        patch_session_for_shared_dap_cache(
+            session, shared_vars=dim_ces, known_url_list=URLs
+        )
+        with session as Session:  # Authenticate once / download dap for each dim
+            with ThreadPoolExecutor(max_workers=ncores) as executor:
+                results = list(executor.map(lambda url: Session.get(url), new_urls))
+    return None
+
+
 def open_file(file_path, das_path=None):
     extension = file_path.split(".")[-1]
     if extension == "dods":
@@ -158,6 +253,28 @@ def open_file(file_path, das_path=None):
         return open_dmr_file(file_path=file_path)
 
 
+def open_dmr(path, session=None):
+    """Retrieves a remote dmr from a server and returns an empty dataset.
+    If the path is to a local file, then it is opened directly.
+    """
+    if path is None:
+        return None
+    if path.startswith("http") and "dmr" in path:
+        # Open a remote dmr
+        if session is None:
+            session = create_session()
+        r = session.get(path, stream=True)
+        dmr = r.text
+        dataset = DMRParser(dmr).init_dataset()
+        return dataset
+    else:
+        try:
+            return open_dmr_file(path)
+        except FileNotFoundError:
+            print(f"File {path} not found.")
+            return None
+
+
 def open_dmr_file(file_path):
     """
     Opens a DMR. This differs from a dap response, since it is a single chunk,
@@ -166,7 +283,7 @@ def open_dmr_file(file_path):
     with open(file_path, "rb") as f:
         dmr = f.read()
     dmr = dmr.decode("ascii")
-    dataset = pydap.parsers.dmr.dmr_to_dataset(dmr)
+    dataset = dmr_to_dataset(dmr)
     return dataset
 
 
@@ -200,17 +317,17 @@ def open_dods_file(file_path, das_path=None):
             if line.strip() == "Data:":
                 break
             dds += line
-    dataset = pydap.parsers.dds.dds_to_dataset(dds)
+    dataset = dds_to_dataset(dds)
     pos = len(dds) + len("Data:\n")
 
     with open(file_path, "rb") as f:
         f.seek(pos)
-        dataset.data = pydap.handlers.dap.unpack_dap2_data(f, dataset)
+        dataset.data = unpack_dap2_data(f, dataset)
 
     if das_path is not None:
         with open(das_path) as f:
-            das = pydap.parsers.das.parse_das(f.read())
-            pydap.parsers.das.add_attributes(dataset, das)
+            das = parse_das(f.read())
+            add_attributes(dataset, das)
 
     return dataset
 
@@ -225,23 +342,23 @@ def open_dods_url(
 ):
     """Open a `.dods` response directly, returning a dataset."""
 
-    r = pydap.net.GET(url, application, session, timeout=timeout)
+    r = GET(url, application, session, timeout=timeout)
 
     dds, data = r.body.split(b"\nData:\n", 1)
     dds = dds.decode(r.content_encoding or "ascii")
-    dataset = pydap.parsers.dds.dds_to_dataset(dds)
-    stream = pydap.handlers.dap.StreamReader(BytesIO(data))
-    dataset.data = pydap.handlers.dap.unpack_dap2_data(stream, dataset)
+    dataset = dds_to_dataset(dds)
+    stream = StreamReader(BytesIO(data))
+    dataset.data = unpack_dap2_data(stream, dataset)
 
     if metadata:
         scheme, netloc, path, params, query, fragment = urlparse(url)
         dasurl = urlunparse(
             (scheme, netloc, path[:-4] + "das", params, query, fragment)
         )
-        r = pydap.net.GET(dasurl, application, session, timeout=timeout, verify=verify)
+        r = GET(dasurl, application, session, timeout=timeout, verify=verify)
 
-        das = pydap.parsers.das.parse_das(r.text)
-        pydap.parsers.das.add_attributes(dataset, das)
+        das = parse_das(r.text)
+        add_attributes(dataset, das)
 
     return dataset
 
@@ -288,10 +405,10 @@ class ServerFunction(object):
     def __call__(self, *args):
         params = []
         for arg in args:
-            if isinstance(arg, (pydap.model.DapType, ServerFunctionResult)):
+            if isinstance(arg, (DapType, ServerFunctionResult)):
                 params.append(arg.id)
             else:
-                params.append(pydap.lib.encode(arg))
+                params.append(encode(arg))
         id_ = self.name + "(" + ",".join(params) + ")"
         return ServerFunctionResult(
             self.baseurl, id_, self.application, self.session, timeout=self.timeout
@@ -329,6 +446,70 @@ class ServerFunctionResult(object):
         return self[name]
 
 
+def compute_base_url_prefix(url_list):
+    """
+    Compute the longest common base path across a list of URLs.
+    Returns the common prefix as a normalized base URL.
+    """
+    if not isinstance(url_list, list) or len(url_list) < 2:
+        raise ValueError("url_list must be a list of at least two URLs.")
+    if not all(isinstance(url, str) for url in url_list):
+        raise ValueError("url_list must contain only strings.")
+    if not all(url.startswith("http") for url in url_list):
+        raise ValueError("url_list must contain valid HTTP URLs.")
+    parsed_paths = [urlparse(url).path.split("?")[0] for url in url_list]
+    parsed_paths = [("/").join(path.split("/")[:-1]) + "/" for path in parsed_paths]
+    common_path = os.path.dirname(commonprefix(parsed_paths))
+    if common_path == "/":
+        raise ValueError("No common path found in the provided URLs.")
+    # Normalize the common path
+    parsed_example = urlparse(url_list[0])
+    return f"{parsed_example.scheme}://{parsed_example.netloc}{common_path}"
+
+
+def patch_session_for_shared_dap_cache(session, shared_vars, known_url_list=None):
+    """
+    Patch CachedSession to normalize cache keys for:
+    - Earthdata URLs: group by DAAC + collection_id
+    - General POSIX-style URLs: group by shared base path (computed from known_url_list)
+    """
+    original_create_key = session.cache.create_key
+    general_base = compute_base_url_prefix(known_url_list) if known_url_list else None
+
+    def custom_create_key(request, **kwargs):
+        parsed = urlparse(request.url)
+        path = unquote(parsed.path)
+        query = parse_qs(parsed.query)
+        dap4_ce = query.get("dap4.ce", [None])[0]
+        if dap4_ce:
+            dap4_ce = unquote(dap4_ce)
+
+        if dap4_ce in shared_vars:
+            # Handle Earthdata-style URLs
+            if parsed.netloc == "opendap.earthdata.nasa.gov":
+                match = re.search(r"(/providers/[^/]+/collections/[^/]+)", path)
+                if match:
+                    dataset_path = match.group(1)
+                    base_url = (
+                        f"{parsed.scheme}://{parsed.netloc}{dataset_path}/shared.dap"
+                    )
+                    normalized_url = f"{base_url}?{urlencode({'dap4.ce': dap4_ce})}"
+                    return normalized_url
+
+            # Handle general POSIX-style URLs
+            base_path = urlparse(general_base).path
+            if general_base and path.startswith(base_path):
+                # Use the computed shared base + virtual shared filename
+                # url_path=urlparse(general_base).path
+                base_url = f"{parsed.scheme}://{parsed.netloc}{base_path}/shared.nc"
+                normalized_url = f"{base_url}?{urlencode({'dap4.ce': dap4_ce})}"
+                return normalized_url
+
+        return original_create_key(request, **kwargs)
+
+    session.cache.create_key = custom_create_key
+
+
 if __name__ == "__main__":
     fname = "/home/griessbaum/Dropbox/UCSB/pydap_cpt/pydap_notebooks/data/"
     ds = open_file(fname + "coads_climatology.nc.dmr")
diff --git a/src/pydap/handlers/dap.py b/src/pydap/handlers/dap.py
index 2d3c7cd..a93e9ce 100644
--- a/src/pydap/handlers/dap.py
+++ b/src/pydap/handlers/dap.py
@@ -17,7 +17,6 @@ import pprint
 import re
 import sys
 import warnings
-import warnings as _warnings
 from concurrent.futures import ThreadPoolExecutor
 from io import BufferedReader, BytesIO
 from itertools import chain
@@ -132,7 +131,7 @@ class DAPHandler(BaseHandler):
         if self.query[:4] == "dap4":
             return "dap4"
         else:
-            _warnings.warn(
+            warnings.warn(
                 "PyDAP was unable to determine the DAP protocol defaulting "
                 "to DAP2. DAP2 is consider legacy and may result in slower "
                 "responses. \nConsider replacing `http` in your `url` with "
@@ -152,11 +151,15 @@ class DAPHandler(BaseHandler):
             self.attach_das()
 
     def dataset_from_dap4(self):
+        if not self.path.endswith(".dmr"):
+            path = self.path + ".dmr"
+        else:
+            path = self.path
         dmr_url = urlunparse(
             (
                 self.scheme,
                 self.netloc,
-                self.path + ".dmr",
+                path,
                 "",
                 _quote(self.query),
                 self.fragment,


Test patch:
diff --git a/src/pydap/tests/test_client.py b/src/pydap/tests/test_client.py
index 8e6030a..d143e1a 100644
--- a/src/pydap/tests/test_client.py
+++ b/src/pydap/tests/test_client.py
@@ -6,13 +6,23 @@ import numpy as np
 import pytest
 import requests
 
-from pydap.client import open_dods_url, open_file, open_url
-from pydap.handlers.lib import BaseHandler
-from pydap.model import BaseType, DatasetType, GridType
-from pydap.tests.datasets import SimpleGrid, SimpleSequence, SimpleStructure
+from ..client import (
+    compute_base_url_prefix,
+    datacube_urls,
+    open_dmr,
+    open_dods_url,
+    open_file,
+    open_url,
+    patch_session_for_shared_dap_cache,
+)
+from ..handlers.lib import BaseHandler
+from ..model import BaseType, DatasetType, GridType
+from ..net import create_session
+from .datasets import SimpleGrid, SimpleSequence, SimpleStructure
 
 DODS = os.path.join(os.path.dirname(__file__), "data/test.01.dods")
 DAS = os.path.join(os.path.dirname(__file__), "data/test.01.das")
+SimpleGroupdmr = os.path.join(os.path.dirname(__file__), "data/dmrs/SimpleGroup.dmr")
 
 
 @pytest.fixture
@@ -362,3 +372,230 @@ def test_cache(use_cache):
             url, protocol="dap2", use_cache=use_cache, cache_kwargs=cache_kwargs
         )
         assert isinstance(ds, DatasetType)
+
+
+@pytest.mark.parametrize(
+    "urls",
+    ["not a list", ["A", "B", "C", 1], ["http://localhost:8001/"]],
+)
+def test_typerror_datacube_urls(urls):
+    """Test that TypeError is raised when datacube_urls takes an argument that
+    is not a list, or a list of a single element.
+    """
+    with pytest.raises(TypeError):
+        datacube_urls(urls)
+
+
+@pytest.mark.parametrize(
+    "urls",
+    [
+        ["dap4://localhost:8001/", "dap4://localhost:8002/", "http://localhost:8003/"],
+        ["dap2://localhost:8001/", "dap4://localhost:8001/"],
+    ],
+)
+def test_valueerror_datacube_urls(urls):
+    """Test that ValueError is raised when datacube_urls takes a list of
+    urls that are not all the same type.
+    """
+    with pytest.raises(ValueError):
+        datacube_urls(urls)
+
+
+@pytest.mark.parametrize(
+    "urls",
+    [
+        ["http://localhost:8001/", "http://localhost:8002/", "http://localhost:8003/"],
+        ["dap2://localhost:8001/", "dap2://localhost:8002/", "dap2://localhost:8003/"],
+    ],
+)
+def test_warning_datacube_urls(urls):
+    """Test that a warning is raised when datacube_urls takes a list of urls
+    that are all the same type. Nothing is returned
+    """
+    with pytest.warns(Warning):
+        returns = datacube_urls(urls)
+    # Check that return is `None`
+    assert returns is None
+
+
+ce1 = "?dap4.ce=/i[0:1:1];/j[0:1:2];/bears[0:1:1][0:1:2];/l[0:1:2]"
+
+
+@pytest.mark.parametrize(
+    "urls",
+    [
+        [
+            "dap4://test.opendap.org/opendap/data/nc/123bears.nc",
+            "dap4://test.opendap.org/opendap/data/nc/123bears.nc" + ce1,
+        ],
+    ],
+)
+def test_cached_datacube_urls(urls):
+    """Test that datacube_urls effectively caches the dmr of the urls, along
+    with the dap4 urls of the dimensions
+    """
+    pyds = open_dmr(urls[0].replace("dap4", "http") + ".dmr")
+    dims = list(pyds.dimensions)  # dimensions of full dataset
+
+    cached_session = create_session(use_cache=True)
+    cached_session.cache.clear()  # clears any existing cache
+    datacube_urls(urls, cached_session)
+    # check that the cached session has all the dmr urls and
+    # caches the dap response of the dimensions only once
+    assert len(cached_session.cache.urls()) == len(urls) + len(dims)
+    # THE FOLLOWING IS AN IMPORTANT CHECK. THE EXTRA CACHED URLS
+    # ARE THE DAP RESPONSES OF EACH DIMENSION. AND THESE ARE THE LAST
+    # TO CACHE. MEANING THE FIRST ELEMENTS OF THE LIST OF CACHED URLS.
+    dap_urls = [url.replace("dap4", "http") for url in urls]
+    dim_dap_urls = [dap_urls[0] + ".dap?dap4.ce=" + dim for dim in dims]
+    N = len(dims)  # should be 3 for this dataset.
+    for n in range(N):
+        assert cached_session.cache.urls()[n].split("%")[0] == dim_dap_urls[n]
+
+
+def tests_no_dims_cache(remote_url):
+    base_url = remote_url + "nc/123bears.nc"
+    ce1 = "?dap4.ce=/bears[0:1:1][0:1:2]"
+    ce2 = "?dap4.ce=/order[0:1:1][0:1:2]"
+    ce3 = "?dap4.ce=/shot[0:1:1][0:1:2]"
+    CE = [ce1, ce2, ce3]
+    dap_urls = [
+        base_url.replace("http", "dap4") + ce for ce in CE
+    ]  # dap urls with constraints expressions
+    with pytest.warns(Warning):
+        # no dimensions in the urls
+        # so the dap urls are not cached
+        datacube_urls(dap_urls)
+
+
+@pytest.mark.parametrize(
+    "urls",
+    [
+        ["http://localhost:8001/", 1, "http://localhost:8003/"],
+        [
+            "dap2://localhost:8001/",
+            "dap2://localhost:8002/",
+            "dap2://localhost:8003/",
+        ],
+        ["http://localhost:8001/"],
+        [
+            "http://localhost:8001/common/path/data.nc",
+            "http://localhost:8001/common/path/data.nc",
+            "http://localhost:8001/NO/COMMON/PATH/HERE/data.nc",
+        ],
+    ],
+)
+def test_ValueErrors_compute_base_url_prefix(urls):
+    """Tests that ValueError is raised wuen urls
+    is not a `uniform` list of https urls belonging to the same
+    data cube. That is, URLS MUST be valid, and have a common path
+    """
+    with pytest.raises(ValueError):
+        compute_base_url_prefix(urls)
+
+
+cloud_common = "/providers/POCLOUD/collections/granules"
+cloud_urls = "https://opendap.earthdata.nasa.gov"
+posix_urls = "http://localhost:8001"
+posix_common = "/common/path"
+
+
+@pytest.mark.parametrize(
+    "urls, common_path",
+    [
+        (
+            [
+                posix_urls + posix_common + "/data.nc",
+                posix_urls + posix_common + "/data.nc",
+                posix_urls + posix_common + "/data.nc",
+            ],
+            posix_urls + posix_common,
+        ),
+        (
+            [
+                cloud_urls + cloud_common + "/fileA.nc",
+                cloud_urls + cloud_common + "/fileC.nc",
+                cloud_urls + cloud_common + "/fileB.nc",
+            ],
+            cloud_urls + cloud_common,
+        ),
+    ],
+)
+def test_compute_base_url_prefix(urls, common_path):
+    """Tests that compute_base_url_prefix returns the correct common path
+    for a list of urls. The urls must be valid and have a common path.
+    """
+    base_url = compute_base_url_prefix(urls)
+    assert base_url == common_path
+
+
+@pytest.mark.parametrize(
+    "url, expected",
+    [
+        [None, None],
+        ["FileNotFound", None],
+        [
+            "http://test.opendap.org/opendap/data/nc/coads_climatology.nc.dmr",
+            DatasetType,
+        ],
+        [
+            SimpleGroupdmr,
+            DatasetType,
+        ],
+    ],
+)
+def test_open_dmr(url, expected):
+    """Test `open_dmr` for various urls"""
+    pyds = open_dmr(url)
+    if expected:
+        assert isinstance(pyds, expected)
+    else:
+        assert pyds == expected
+
+
+@pytest.mark.parametrize(
+    "urls",
+    [
+        [
+            "http://test.opendap.org/opendap/data/nc/123bears.nc",
+            "http://test.opendap.org/opendap/data/nc/124bears.nc",
+            "http://test.opendap.org/opendap/data/nc/125bears.nc",
+        ],
+    ],
+)
+def test_patch_session_for_shared_dap_cache(urls):
+    """Test that the session is patched correctly for shared dap cache."""
+    # Create a session using requests-cache
+    cached_session = create_session(use_cache=True)
+    # Clear any existing cache
+    cached_session.cache.clear()
+    # Create custom cache key for each of the dimensions
+    dimensions = ["i[0:1:1]", "j[0:1:2]", "l[0:1:2]"]
+
+    patch_session_for_shared_dap_cache(
+        cached_session, shared_vars=dimensions, known_url_list=urls
+    )
+    assert len(cached_session.cache.urls()) == 0
+
+    # construct urls to create cache keys
+    test_urls = [urls[0] + ".dap?dap4.ce=" + dim for dim in dimensions]
+    # create cache keys for the urls - discard the list
+    _ = [cached_session.get(url) for url in test_urls]
+
+    # make sure that the urls are cached for each dimension
+    assert len(cached_session.cache.urls()) == len(dimensions)
+
+    for dim in dimensions:
+        test_url2 = urls[1] + ".dap?dap4.ce=" + dim
+        test_url3 = urls[2] + ".dap?dap4.ce=" + dim
+
+        # test that the urls are being cached
+        r2 = cached_session.get(test_url2)
+        r3 = cached_session.get(test_url3)
+
+        # assert that data was cached - otherwise 404 (Non-existent URLS!)
+        assert r2.from_cache
+        assert r3.from_cache
+
+    # assert that there is no new cached key
+    assert len(cached_session.cache.urls()) == len(dimensions)