Repository: British-Oceanographic-Data-Centre/COAsT (language: python)

Issue:
Remote access to dataset. E.g. loading copernicus data remotely

fix citations. v2.0.3



Discussion / hints:

Fix patch:
diff --git a/CITATION.cff b/CITATION.cff
index 7f0dce23..62545e2c 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -24,6 +24,7 @@ authors:
    given-names: "Thomas"
  - family-names: "Cazaly"
    given-names: "Matthew"
+   orcid: "https://orcid.org/0000-0002-0679-4054"
  - family-names: "Hearn"
    given-names: "Malcolm"
  - family-names: "Jennings"
diff --git a/base.Dockerfile b/base.Dockerfile
new file mode 100644
index 00000000..491d00fc
--- /dev/null
+++ b/base.Dockerfile
@@ -0,0 +1,6 @@
+FROM docker-repo.bodc.me/bodc/conda:latest
+
+RUN /home/bodc/miniconda3/bin/conda install -c conda-forge gcc
+
+ADD requirements.txt requirements.txt
+RUN pip install -r requirements.txt
diff --git a/coast/__init__.py b/coast/__init__.py
index 7f60ddab..c8955bf0 100644
--- a/coast/__init__.py
+++ b/coast/__init__.py
@@ -22,6 +22,8 @@
 from .data.config_parser import ConfigParser
 from ._utils.xesmf_convert import xesmf_convert
 from ._utils.process_data import Process_data
+from .data.opendap import OpendapInfo
+from .data.copernicus import Copernicus, Product
 
 # Set default for logging level when coast is imported
 import logging
diff --git a/coast/data/coast.py b/coast/data/coast.py
index 737717ab..9b1db871 100644
--- a/coast/data/coast.py
+++ b/coast/data/coast.py
@@ -4,6 +4,7 @@
 from dask.distributed import Client
 import copy
 from .._utils.logging_util import get_slug, debug, info, warn, warning
+from .opendap import OpendapInfo
 
 
 def setup_dask_client(workers: int = 2, threads: int = 2, memory_limit_per_worker: str = "2GB"):
@@ -15,7 +16,7 @@ def __init__(
         self,
         file: str = None,
         chunks: dict = None,
-        multiple=False,
+        multiple: bool = False,
         workers: int = 2,  # TODO Do something with this unused parameter or delete it
         threads: int = 2,  # TODO Do something with this unused parameter or delete it
         memory_limit_per_worker: str = "2GB",  # TODO Do something with this unused parameter or delete it
@@ -44,12 +45,16 @@ def load(self, file_or_dir, chunks: dict = None, multiple=False):
         Loads a file into a COAsT object's dataset variable using xarray
 
         Args:
-            file_or_dir (str)     : file name or directory to multiple files.
-            chunks (dict)  : Chunks to use in Dask [default None]
+            file_or_dir (str): file name, OPeNDAP accessor, or directory to multiple files.
+            chunks (dict): Chunks to use in Dask [default None]
             multiple (bool): If true, load in multiple files from directory.
                              If false load a single file [default False]
         """
-        if multiple:
+        if (opendap := isinstance(file_or_dir, OpendapInfo)) and multiple:
+            raise NotImplementedError("Loading multiple OPeNDAP datasets is not supported")
+        elif opendap:
+            self.load_dataset(file_or_dir.open_dataset(chunks=chunks))
+        elif multiple:
             self.load_multiple(file_or_dir, chunks)
         else:
             self.load_single(file_or_dir, chunks)
diff --git a/coast/data/copernicus.py b/coast/data/copernicus.py
new file mode 100644
index 00000000..3b7121d5
--- /dev/null
+++ b/coast/data/copernicus.py
@@ -0,0 +1,64 @@
+"""Functionality for accessing Copernicus datasets via OPeNDAP."""
+
+from dataclasses import dataclass
+from .opendap import OpendapInfo
+
+
+COPERNICUS_CAS = "https://cmems-cas.cls.fr/cas/login"
+COPERNICUS_URL = "https://{}.cmems-du.eu/thredds/dodsC/{}"
+
+
+@dataclass
+class CopernicusBase:
+    """Information required for accessing Copernicus datasets via OPeNDAP."""
+
+    username: str
+    password: str
+    database: str
+    cas_url: str = COPERNICUS_CAS
+    url_template: str = COPERNICUS_URL
+
+    def get_url(self, product_id: str):
+        """Get the URL required to access a Copernicus OPeNDAP dataset.
+
+        Args:
+            product_id: The product ID belonging to the chosen dataset.
+
+        Returns:
+            The constructed URL.
+        """
+        return self.url_template.format(self.database, product_id)
+
+
+class Product(OpendapInfo):
+    """Information required to access and stream data from a Copernicus product."""
+
+    @classmethod
+    def from_copernicus(cls, product_id: str, copernicus: CopernicusBase) -> "Product":
+        """Instantiate a Product using Copernicus information and a specific product ID.
+
+        Args:
+            product_id: The product ID of the chosen Copernicus OPeNDAP dataset.
+            copernicus: A previously instantiated Copernicus info object.
+
+        Returns:
+            An instantiated Product accessor.
+        """
+        return cls.from_cas(
+            copernicus.get_url(product_id), copernicus.cas_url, copernicus.username, copernicus.password
+        )
+
+
+class Copernicus(CopernicusBase):
+    """An object for accessing Copernicus products via OPeNDAP."""
+
+    def get_product(self, product_id: str) -> Product:
+        """Instantiate a Product related to a specific product ID.
+
+        Args:
+            product_id: The product ID of the chosen Copernicus OPeNDAP dataset.
+
+        Returns:
+            The instantiated Product accessor.
+        """
+        return Product.from_copernicus(product_id, self)
diff --git a/coast/data/opendap.py b/coast/data/opendap.py
new file mode 100644
index 00000000..ac8138b5
--- /dev/null
+++ b/coast/data/opendap.py
@@ -0,0 +1,57 @@
+"""Functionality for accessing OPeNDAP datasets."""
+
+from dataclasses import dataclass
+from typing import Optional
+from xarray.backends import PydapDataStore
+from xarray import Dataset, open_dataset
+from pydap.client import open_url
+from pydap.cas.get_cookies import setup_session
+import requests
+
+
+CASTGC = "CASTGC"
+
+
+@dataclass
+class OpendapInfo:
+    """A class for accessing streamable OPeNDAP data."""
+
+    url: str
+    session: Optional[requests.Session] = None
+
+    def get_store(self) -> PydapDataStore:
+        """Access an OPeNDAP data store.
+
+        Returns:
+            The OPeNDAP data store accessed from the instance's URL.
+        """
+        return PydapDataStore(open_url(self.url, session=self.session))
+
+    def open_dataset(self, chunks: Optional[dict] = None) -> Dataset:
+        """Open the remote XArray dataset for streaming.
+
+        Args:
+            chunks: Chunks to use in Dask.
+
+        Returns:
+            The opened XArray dataset.
+        """
+        with open_dataset(self.get_store(), chunks=chunks) as dataset:
+            return dataset
+
+    @classmethod
+    def from_cas(cls, url: str, cas_url: str, username: str, password: str) -> "OpendapInfo":
+        """Instantiate OpendapInfo with a session authenticated against CAS.
+
+        Args:
+            url: The OPeNDAP dataset URL.
+            cas_url: The CAS login URL.
+            username: The username to authenticate with.
+            password: The password to authenticate with.
+
+        Returns:
+            The instantiated OPeNDAP accessor.
+        """
+        session = setup_session(cas_url, username, password)
+        session.cookies.set(CASTGC, session.cookies.get_dict()[CASTGC])
+        return cls(url, session)
diff --git a/conda_dev_requirements.txt b/conda_dev_requirements.txt
index cb9be384..1c68fcab 100644
--- a/conda_dev_requirements.txt
+++ b/conda_dev_requirements.txt
@@ -10,5 +10,8 @@ utide>=0.2.6
 scikit-learn>=1.0.2
 scikit-image>=0.19.2
 statsmodels>=0.13.2
+pydap>=3.2.2
+lxml>=4.9.0
+requests>=2.27.1
 cartopy>=0.20.2
 spyder>=5.1.5
diff --git a/config/example_cmems_grid_t.json b/config/example_cmems_grid_t.json
index 6fe0363e..e9c11771 100644
--- a/config/example_cmems_grid_t.json
+++ b/config/example_cmems_grid_t.json
@@ -9,13 +9,13 @@
 		"dimension_map": {
 			"time": "t_dim",
 			"depth": "z_dim",
-			"lon": "x_dim",
-			"lat": "y_dim"
+			"longitude": "x_dim",
+			"latitude": "y_dim"
 		},
 		"variable_map": {
 			"time": "time",
-			"lon": "longitude",
-			"lat": "latitude",
+			"longitude": "longitude",
+			"latitude": "latitude",
 			"thetao": "temperature",
 			"so": "salinity",
                         "depth": "z_dim",
diff --git a/config/example_cmems_grid_uv.json b/config/example_cmems_grid_uv.json
index 52f25a3b..8f8d39d3 100644
--- a/config/example_cmems_grid_uv.json
+++ b/config/example_cmems_grid_uv.json
@@ -9,13 +9,13 @@
 		"dimension_map": {
 			"time": "t_dim",
 			"depth": "z_dim",
-			"lon": "x_dim",
-			"lat": "y_dim"
+			"longitude": "x_dim",
+			"latitude": "y_dim"
 		},
 		"variable_map": {
 			"time": "time",
-			"lon": "longitude",
-			"lat": "latitude",
+			"longitude": "longitude",
+			"latitude": "latitude",
 			"uo": "u_velocity",
 			"vo": "v_velocity"
 		},
diff --git a/requirements.txt b/requirements.txt
index 3d17093d..845d005a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,7 @@
-.
 PyYAML==6.0
 oyaml==1.0
 pytest==7.1.1
+pytest-mock==3.7.0
 numpy>=1.22.3
 dask>=2022.3.0
 dask[complete]>=2022.3.0
@@ -14,3 +14,6 @@ utide>=0.2.6
 scikit-learn>=1.0.2
 scikit-image>=0.19.2
 statsmodels>=0.13.2
+pydap>=3.2.2
+lxml>=4.9.0  # Required for pydap CAS parsing
+requests>=2.27.1
diff --git a/setup.py b/setup.py
index 7fea0d31..8ecf1dfa 100644
--- a/setup.py
+++ b/setup.py
@@ -25,6 +25,10 @@
         "keywords": ["NEMO", "shallow water", "ocean assessment"],
         "project_urls": {"documentation": "https://british-oceanographic-data-centre.github.io/COAsT/"},
         "install_requires": [
+            "PyYAML==6.0",
+            "oyaml==1.0",
+            "pytest==7.1.1",
+            "pytest-mock==3.7.0",
             "numpy>=1.22.3",
             "dask>=2022.3.0",
             "dask[complete]>=2022.3.0",
@@ -37,8 +41,11 @@
             "scikit-learn>=1.0.2",
             "scikit-image>=0.19.2",
             "statsmodels>=0.13.2",
+            "pydap>=3.2.2",
+            "lxml>=4.9.0",  # Required for pydap CAS parsing,
+            "requests>=2.27.1",
         ],
-        "python_requires": ">=3.8",
+        "python_requires": ">=3.8,<3.10",  # Pydap not currently compatible with 3.10
         "packages": ["coast"],
         "include_package_data": True,
         "github": "British-Oceanographic-Data-Centre",


Test patch:
diff --git a/tests/test_copernicus.py b/tests/test_copernicus.py
new file mode 100644
index 00000000..ed58938c
--- /dev/null
+++ b/tests/test_copernicus.py
@@ -0,0 +1,50 @@
+"""Unit tests for Copernicus functionality."""
+
+import pytest
+from coast import Copernicus, Product
+
+
+@pytest.fixture(name="copernicus")
+def copernicus_fixture(mocker) -> Copernicus:
+    """Return a Copernicus accessor with mocked parameters."""
+    mocker.sentinel.template = mocker.Mock()
+    mocker.sentinel.template.format.return_value = mocker.sentinel.url
+    return Copernicus(
+        mocker.sentinel.username,
+        mocker.sentinel.password,
+        mocker.sentinel.database,
+        cas_url=mocker.sentinel.cas_url,
+        url_template=mocker.sentinel.template,
+    )
+
+
+def test_get_url(copernicus, mocker):
+    """Test that a URL is generated with the correct parameters."""
+    url = copernicus.get_url(mocker.sentinel.product)
+
+    assert url == mocker.sentinel.url
+    assert mocker.sentinel.template.format.called_with(mocker.sentinel.database, mocker.sentinel.product)
+
+
+def test_from_copernicus(copernicus, mocker):
+    """Test that a Product is instantiated with the correct parameters."""
+    from_cas = mocker.patch("coast.data.opendap.OpendapInfo.from_cas", return_value=mocker.sentinel.product)
+
+    product = Product.from_copernicus(mocker.sentinel.product_id, copernicus)
+
+    assert product == mocker.sentinel.product
+    from_cas.assert_called_with(
+        mocker.sentinel.url, mocker.sentinel.cas_url, mocker.sentinel.username, mocker.sentinel.password
+    )
+
+
+def test_get_product(copernicus, mocker):
+    """Test that a Product is instantiated with the correct parameters."""
+    from_copernicus = mocker.patch(
+        "coast.data.copernicus.Product.from_copernicus", return_value=mocker.sentinel.product
+    )
+
+    product = copernicus.get_product(mocker.sentinel.product_id)
+
+    assert product == mocker.sentinel.product
+    from_copernicus.assert_called_once_with(mocker.sentinel.product_id, copernicus)
diff --git a/tests/test_copernicus_integration.py b/tests/test_copernicus_integration.py
new file mode 100644
index 00000000..4e6ece3f
--- /dev/null
+++ b/tests/test_copernicus_integration.py
@@ -0,0 +1,52 @@
+"""Integration tests for Copernicus functionality."""
+
+import logging
+from pathlib import Path
+from os import environ
+import pytest
+import numpy
+from xarray import Dataset
+from coast import Copernicus, Gridded
+
+
+DATABASE = "nrt"
+PRODUCT_ID = "global-analysis-forecast-phy-001-024"
+CONFIG = (Path(__file__).parent.parent / "config" / "example_cmems_grid_t.json").resolve(strict=True)
+
+USERNAME = environ.get("COPERNICUS_USERNAME")
+PASSWORD = environ.get("COPERNICUS_PASSWORD")
+
+CREDENTIALS = USERNAME is not None and PASSWORD is not None
+
+if not CREDENTIALS:
+    logging.warning("https://marine.copernicus.eu/ credentials not set, integration tests will not be run!")
+
+
+@pytest.fixture(name="copernicus")
+def copernicus_fixture() -> Copernicus:
+    """Return a functional Copernicus data accessor."""
+    return Copernicus(USERNAME, PASSWORD, DATABASE)
+
+
+@pytest.mark.skipif(condition=not CREDENTIALS, reason="Copernicus credentials are not set.")
+def test_get_product(copernicus):
+    """Connect to Copernicus, access some metadata and data values and ingest into a Gridded object."""
+    product = copernicus.get_product(PRODUCT_ID)
+    nemo_t = Gridded(fn_data=product, config=str(CONFIG))
+
+    # Check that the dataset has been initialised as expected
+    assert isinstance(nemo_t.dataset, Dataset)
+    # Check for some expected CMEMS metadata
+    assert nemo_t.dataset.attrs["comment"] == "CMEMS product"
+    assert nemo_t.dataset.attrs["easting"] == "longitude"
+    assert nemo_t.dataset.attrs["northing"] == "latitude"
+    assert nemo_t.dataset.attrs["institution"] == "MERCATOR OCEAN"
+    # Verify that dimensions have been mapped correctly
+    dimensions = nemo_t.dataset.dims
+    assert "x_dim" in dimensions
+    assert "y_dim" in dimensions
+    assert "z_dim" in dimensions
+    assert "t_dim" in dimensions
+    # Download and inspect some values
+    latitude = nemo_t.dataset.latitude[:-100].values
+    assert all(isinstance(value, numpy.float32) and not numpy.isnan(value) for value in latitude)
diff --git a/tests/test_opendap.py b/tests/test_opendap.py
new file mode 100644
index 00000000..dbc57d45
--- /dev/null
+++ b/tests/test_opendap.py
@@ -0,0 +1,59 @@
+"""Unit tests for OPeNDAP functionality."""
+
+import pytest
+from coast import OpendapInfo
+from coast.data.opendap import CASTGC
+
+
+@pytest.fixture(name="opendap_info")
+def opendap_info_fixture(mocker) -> OpendapInfo:
+    """Return an OPeNDAP accessor with mocked parameters."""
+    return OpendapInfo(mocker.sentinel.url, session=mocker.sentinel.session)
+
+
+def test_get_store(opendap_info, mocker) -> None:
+    """Test that a store is instantiated with the correct parameters."""
+    open_url = mocker.patch("coast.data.opendap.open_url", return_value=mocker.sentinel.dataset)
+    pydap_data_store = mocker.patch("coast.data.opendap.PydapDataStore", return_value=mocker.sentinel.returned_store)
+
+    store = opendap_info.get_store()
+
+    open_url.assert_called_with(mocker.sentinel.url, session=mocker.sentinel.session)
+    pydap_data_store.assert_called_with(mocker.sentinel.dataset)
+    assert store == mocker.sentinel.returned_store
+
+
+@pytest.mark.parametrize(["chunking"], [(True,), (False,)])
+def test_open_dataset(opendap_info, mocker, chunking):
+    """Test that a dataset is initialised with the correct parameters."""
+    get_store = mocker.patch("coast.data.opendap.OpendapInfo.get_store", return_value=mocker.sentinel.store)
+    dataset = mocker.Mock()
+    dataset.__enter__ = mocker.Mock()  # Requirement of context manager
+    dataset.__exit__ = mocker.Mock()  # Requirement of context manager
+    open_dataset = mocker.patch("coast.data.opendap.open_dataset", return_value=dataset)
+    chunks = mocker.Mock() if chunking else None
+
+    dataset = opendap_info.open_dataset(chunks=chunks)
+
+    assert dataset == dataset
+    get_store.assert_called_once_with()
+    open_dataset.assert_called_with(mocker.sentinel.store, chunks=chunks)
+
+
+def test_from_cas(mocker):
+    """Test that an OPeNDAP accessor is instantiated with the correct parameters."""
+    mocker.sentinel.session = mocker.Mock()
+    mocker.sentinel.session.cookies.get_dict.return_value = {"CASTGC": mocker.sentinel.castgc}
+
+    setup_session = mocker.patch("coast.data.opendap.setup_session", return_value=mocker.sentinel.session)
+
+    opendap_info = OpendapInfo.from_cas(
+        mocker.sentinel.url, mocker.sentinel.cas_url, mocker.sentinel.username, mocker.sentinel.password
+    )
+
+    assert isinstance(opendap_info, OpendapInfo)
+    assert opendap_info.url == mocker.sentinel.url
+    assert opendap_info.session == mocker.sentinel.session
+    assert setup_session.called_with(mocker.sentinel.cas_url, mocker.sentinel.username, mocker.sentinel.password)
+    assert mocker.sentinel.session.cookies.set.called_with(CASTGC, mocker.sentinel.castgc)
+    assert opendap_info.session == mocker.sentinel.session