Repository: solo-spice/sospice (language: python)

Issue:
Enhance documentation



Discussion / hints:

Fix patch:
diff --git a/.flake8 b/.flake8
index bfa185d..304aa38 100644
--- a/.flake8
+++ b/.flake8
@@ -2,3 +2,4 @@
 max-line-length = 120
 max-complexity = 18
 select = B,C,E,F,W,T4,B9
+per-file-ignores = __init__.py:F401
diff --git a/.gitignore b/.gitignore
index cae20b2..bd468ea 100644
--- a/.gitignore
+++ b/.gitignore
@@ -70,6 +70,7 @@ instance/
 
 # Sphinx documentation
 docs/_build/
+docs/source/_autosummary/
 
 # PyBuilder
 .pybuilder/
diff --git a/Makefile b/Makefile
index fcb917b..ea46918 100644
--- a/Makefile
+++ b/Makefile
@@ -1,3 +1,5 @@
+build:
+	python3 -m build
 install:
 	python -m pip install .
 test:
@@ -8,3 +10,10 @@ pre-commit:
 	pre-commit
 show-version:
 	python -m setuptools_scm
+doc-html:
+	make -C docs html
+publish:
+	@echo "Please make sure that changes have been merged to the main branch, with correct git tag in Github and version information in pyproject.toml"
+	@echo "Please press <Enter> to confirm"
+	@read foo
+	python3 -m twine upload --verbose dist/*
diff --git a/README.rst b/README.rst
index 2e3cf9f..29fd93a 100644
--- a/README.rst
+++ b/README.rst
@@ -23,6 +23,11 @@ wiki <https://spice-wiki.ias.u-psud.fr/doku.php/data:data_analysis_manual>`__.
    -  ``spice_error``: Computation of uncertainties on data, coming from
       different noise components.
 
+- Catalog: ``catalog``
+
+   -  ``Catalog``: access and read catalog, find files in catalog.
+   -  ``Release```: find and access releases.
+
 -  Instrument modelling: ``instrument_modelling``
 
    -  ``Spice``: instrument calibration parameters, effective area,
@@ -37,7 +42,7 @@ Package philosophy
 
 We want ``sospice`` to be:
 
--  Convenient to install. It is installable by ``pip`` and it will be
+-  Convenient to install. It is installable by ``pip`` and it is
    published on `PyPI <https://pypi.org/>`__
 -  Useful, providing a single package for all SPICE-specific steps of
    your data analysis.
diff --git a/docs/source/_templates/autosummary/module.rst b/docs/source/_templates/autosummary/module.rst
new file mode 100644
index 0000000..6090b5e
--- /dev/null
+++ b/docs/source/_templates/autosummary/module.rst
@@ -0,0 +1,5 @@
+{{ fullname }}
+{{ underline }}
+
+.. automodule:: {{ fullname }}
+    :members:
diff --git a/docs/source/api.rst b/docs/source/api.rst
index cb1338e..ceed4a6 100644
--- a/docs/source/api.rst
+++ b/docs/source/api.rst
@@ -1,7 +1,7 @@
 API
 ===
 
-There should be an API documentation (generated from the code docstrings) here, but it seems that we have to update the Sphinx configuration for this.
+This API documentation is generated from the Python code docstrings (we still have to finalize the Sphinx configuration for this documentation to reflect all code documentation).
 
 .. autosummary::
    :toctree: _autosummary
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 3bb7766..122f0c6 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -36,6 +36,7 @@
 # ones.
 extensions = [
     "matplotlib.sphinxext.plot_directive",
+    "numpydoc",
     # "sphinx_automodapi.automodapi",
     # "sphinx_automodapi.smart_resolver",
     # "sphinx_changelog",
@@ -55,6 +56,7 @@
     # "sphinx_copybutton",
     # "hoverxref.extension",
 ]
+
 autosummary_generate = True  # Turn on sphinx.ext.autosummary
 
 # Add any paths that contain templates here, relative to this directory.
@@ -63,7 +65,7 @@
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ["build", "htmlcov", "sospice.egg-info", "venv"]
+exclude_patterns = ["build", "htmlcov", "sospice.egg-info", "venv", "tests"]
 
 # -- Options for intersphinx extension ---------------------------------------
 intersphinx_mapping = {
diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst
index 8e665ab..d9f3df7 100644
--- a/docs/source/getting_started.rst
+++ b/docs/source/getting_started.rst
@@ -1,7 +1,7 @@
 Getting Started
 ===============
 
-The stable version of ``sospice`` will be available through PyPI via ``pip``,
+The stable version of ``sospice`` is available through PyPI via ``pip``,
 
 .. code-block:: shell
 
diff --git a/pyproject.toml b/pyproject.toml
index 3626a2f..3eba870 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -8,14 +8,19 @@ authors = [
   { name="SPICE consortium", email="spice-github.ias@universite-paris-saclay.fr" },
 ]
 description = "Python data analysis tools for Solar Orbiter/SPICE"
-readme = "README.md"
-requires-python = ">=3.7"
+readme = "README.rst"
+requires-python = ">=3.8"
 classifiers = [
     "Programming Language :: Python :: 3",
-    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "License :: OSI Approved :: BSD License",
     "Operating System :: OS Independent",
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Science/Research",
+    "Topic :: Scientific/Engineering :: Astronomy",
 ]
-
 dynamic = ["version", "dependencies"]
 
 [tool.setuptools.dynamic]
@@ -46,3 +51,10 @@ exclude = '''
   | venv
 )/
 '''
+
+[tool.coverage.run]
+omit = [
+    "*/tests/*",
+    "tests/*",
+    "__init__.py",
+]
diff --git a/requirements.txt b/requirements.txt
index 038ba9d..63da470 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,14 +1,18 @@
-astropy
-black
-flake8
-matplotlib
-numpy
-pandas
-pre-commit
-pytest
-pytest-cov
-scipy
+astropy>=4.0
+black>=23.0
+flake8>=6.0
+matplotlib>=3.7.0
+numpy>=1.24.0
+numpydoc>=1.4
+pandas>=2.0
+parfive>=2.0
+pre-commit>=3.0
+pytest>=7.0
+pytest-cov>=4.0
+requests>=2.30
+scipy>=1.10
 setuptools>=45
-setuptools_scm[toml]>=6.2
-sphinx
-sunpy
+setuptools_scm[toml]>=7.0
+Sphinx>=7.0
+sunpy>=4.0
+twine>=4.0
diff --git a/sospice/catalog/__init__.py b/sospice/catalog/__init__.py
index e69de29..ce04369 100644
--- a/sospice/catalog/__init__.py
+++ b/sospice/catalog/__init__.py
@@ -0,0 +1,2 @@
+from .release import Release
+from .catalog import Catalog
diff --git a/sospice/catalog/catalog.py b/sospice/catalog/catalog.py
new file mode 100644
index 0000000..12f0602
--- /dev/null
+++ b/sospice/catalog/catalog.py
@@ -0,0 +1,191 @@
+from dataclasses import dataclass
+
+import pandas as pd
+from pathlib import Path
+from astropy.utils.data import download_file
+
+from .release import Release
+
+
+def get_file_relative_path(cat_row):
+    """
+    Get file relative path for a given catalog entry
+
+    Parameters
+    ----------
+    cat_row: pandas.Series
+        SPICE catalog entry
+
+    Return
+    ------
+    pathlib.PosixPath
+        File path, relative to the "fits" directory of the file archive:
+        leveln/yyyy/mm/dd
+
+    Requires a DATE-BEG header (so does not work with L0 files).
+    """
+    date = cat_row["DATE-BEG"]
+    return (
+        Path(f"level{cat_row.LEVEL[1]}")  # noqa: W503
+        / f"{date.year}"  # noqa: W503
+        / f"{date.month:02}"  # noqa: W503
+        / f"{date.day:02}"  # noqa: W503
+    )
+
+
+@dataclass
+class Catalog(pd.DataFrame):
+    """
+    A SPICE catalog, initialized (in that order) either from a filename, a release tag, or a pandas.DataFrame.
+
+    Parameters
+    ----------
+    filename: str
+        A file name (or URL) for the catalog
+    release_tag: str
+        A release tag. The catalog is fetched online and dowloaded to the astropy cache.
+    data_frame: pandas.DataFrame
+        A pandas DataFrame to be used as SPICE catalog. Some basic checks are made to ensure
+        that is can be used as a SPICE catalog.
+    update_cache: bool
+        Update cached catalog for the given release tag
+    """
+
+    filename: str = None
+    release_tag: str = None
+    data_frame: str = None
+    update_cache: bool = False
+
+    def __post_init__(self):
+        """
+        Read catalog and update object
+        """
+        self._normalize_arguments()
+        if self.release_tag is not None:
+            self._cache_release_catalog()
+        if self.filename is not None:
+            super().__init__(self.read_catalog())
+        else:
+            if self.data_frame is None:
+                self.data_frame = pd.DataFrame()
+            self._validate_data_frame()
+            super().__init__(self.data_frame)
+            del self.data_frame  # needed for memory usage?
+            self.data_frame = None
+
+    def _normalize_arguments(self):
+        """
+        Prioritize filename then release tag then data frame
+        """
+        if self.filename is not None:
+            self.release_tag = None
+            self.data_frame = None
+        elif self.release_tag is not None:
+            self.data_frame = None
+
+    def _cache_release_catalog(self):
+        """
+        Used cached catalog or download release catalog to astropy cache
+        """
+        print("Enter cache release catalog")
+        if self.release_tag is None:
+            return
+        if self.release_tag == "latest":
+            self.release_tag = None
+        release = Release(self.release_tag)
+        assert release.exists
+        self.filename = download_file(release.catalog_url, cache=True)
+        self.release_tag = None
+
+    def _validate_data_frame(self):
+        """
+        Check that the data_frame argument can be considered a valid SPICE catalog (or raise an exception)
+        """
+        print(f"enter validate data frame {self.data_frame}")
+        assert self.data_frame is not None
+        if self.data_frame.empty:
+            return True  # an empty data frame is valid
+        required_columns = {
+            "NAXIS1",
+            "NAXIS2",
+            "NAXIS3",
+            "NAXIS4",
+            "OBT_BEG",
+            "LEVEL",
+            "FILENAME",
+            "DATE-BEG",
+            "SPIOBSID",
+            "RASTERNO",
+            "STUDYTYP",
+            "MISOSTUD",
+            "XPOSURE",
+            "CRVAL1",
+            "CDELT1",
+            "CRVAL2",
+            "CDELT2",
+            "STP",
+            "DSUN_AU",
+            "CROTA",
+            "OBS_ID",
+            "SOOPNAME",
+            "SOOPTYPE",
+            "NWIN",
+            "DARKMAP",
+            "COMPLETE",
+            "SLIT_WID",
+            "DATE",
+            "PARENT",
+            "HGLT_OBS",
+            "HGLN_OBS",
+            "PRSTEP1",
+            "PRPROC1",
+            "PRPVER1",
+            "PRPARA1",
+        }
+        assert required_columns.issubset(self.data_frame.columns)
+
+    def read_catalog(self):
+        """
+        Read SPICE FITS files catalog
+
+        Return
+        ------
+        pandas.DataFrame
+            Catalog
+        """
+        if not Path(self.filename).exists():
+            raise RuntimeError(f"File {self.filename} does not exist")
+        df = pd.read_csv(
+            self.filename,
+            low_memory=False,
+        )
+        date_columns = ["DATE-BEG", "DATE", "TIMAQUTC"]
+        for date_column in date_columns:
+            df[date_column] = pd.to_datetime(df[date_column], format="ISO8601")
+        return df
+
+    def find_file(self, date=None, level="L2"):
+        """
+        Find closest file to some observation data (DATE-BEG)
+
+        Parameters
+        ----------
+        date: datetime.datetime, pandas.Timestamp...
+            Target date
+        level: str
+            LEVEL to select (≥L1)
+
+        Return
+        ------
+        pandas.Series
+            File with closest date
+        """
+        if self.empty or date is None:
+            return None
+        if type(date) is str:
+            date = pd.Timestamp(date)
+        df = self[self.LEVEL == level]
+        df.set_index("DATE-BEG", inplace=True)
+        index = df.index.get_indexer([date], method="nearest")
+        df.reset_index(inplace=True)
+        return df.iloc[index[0]]
diff --git a/sospice/catalog/release.py b/sospice/catalog/release.py
new file mode 100644
index 0000000..5e94711
--- /dev/null
+++ b/sospice/catalog/release.py
@@ -0,0 +1,86 @@
+from dataclasses import dataclass
+import requests
+
+
+@dataclass
+class Release:
+    tag: str = None
+    base_url: str = "https://spice.osups.universite-paris-saclay.fr/spice-data/"
+    _latest_tag: str = None
+
+    def __post_init__(self):
+        """
+        Initialize object to latest release if no release tag set
+        """
+        if self.tag is None:
+            self.tag = self.latest_tag
+
+    @property
+    def url(self):
+        """
+        Return
+        ------
+        str
+            Release URL
+        """
+        return f"{self.base_url}release-{self.tag}/"
+
+    @property
+    def catalog_url(self):
+        """
+        Return
+        ------
+        str
+            Catalog URL for release
+        """
+        return self.url + "catalog.csv"
+
+    @property
+    def latest_tag(self):
+        """
+        Return
+        ------
+        str
+            Tag of latest release
+        """
+        if self._latest_tag is None:
+            url = f"{self.base_url}metadata/latest-release.txt"
+            result = requests.get(url)
+            if not result.ok:
+                raise RuntimeError(
+                    "Could not access URL for file with latest release tag"
+                )
+            self._latest_tag = result.text.split("\n")[0]
+        return self._latest_tag
+
+    @property
+    def is_latest(self):
+        """
+        Return
+        ------
+        bool
+            True is object corresponds to latest release
+        """
+        return self.tag == self.latest_tag
+
+    @property
+    def exists(self):
+        """
+        Return
+        ------
+        bool
+            True if release exists (is accessible online)
+        """
+        result = requests.get(self.url)
+        return result.ok
+
+
+def get_latest_release_tag():
+    """
+    Return
+    ------
+    str
+        Latest available release tag
+    """
+    release = Release()
+    return release.latest_tag


Test patch:
diff --git a/sospice/catalog/tests/test_catalog.py b/sospice/catalog/tests/test_catalog.py
new file mode 100644
index 0000000..48ba4cc
--- /dev/null
+++ b/sospice/catalog/tests/test_catalog.py
@@ -0,0 +1,119 @@
+import pytest
+from datetime import datetime
+import pandas as pd
+
+
+from ..catalog import Catalog, get_file_relative_path
+
+
+@pytest.fixture
+def catalog2():
+    return Catalog(release_tag="2.0")
+
+
+@pytest.fixture
+def catalog_latest():
+    return Catalog(release_tag="latest")
+
+
+@pytest.fixture
+def catalog_empty():
+    return Catalog()
+
+
+@pytest.fixture
+def catalog_df():
+    df = pd.DataFrame(
+        {
+            "NAXIS1": [12, 15, 20],
+            "NAXIS2": [100, 101, 102],
+            "NAXIS3": [41, 44, 47],
+            "NAXIS4": [1, 1, 1],
+            "OBT_BEG": [0, 0, 0],
+            "LEVEL": ["L2", 0, 0],
+            "FILENAME": [0, 0, 0],
+            "DATE-BEG": [pd.Timestamp("2023-02-01T12:34"), 0, 0],
+            "SPIOBSID": [0, 0, 0],
+            "RASTERNO": [0, 0, 0],
+            "STUDYTYP": [0, 0, 0],
+            "MISOSTUD": [0, 0, 0],
+            "XPOSURE": [0, 0, 0],
+            "CRVAL1": [0, 0, 0],
+            "CDELT1": [0, 0, 0],
+            "CRVAL2": [0, 0, 0],
+            "CDELT2": [0, 0, 0],
+            "STP": [0, 0, 0],
+            "DSUN_AU": [0, 0, 0],
+            "CROTA": [0, 0, 0],
+            "OBS_ID": [0, 0, 0],
+            "SOOPNAME": [0, 0, 0],
+            "SOOPTYPE": [0, 0, 0],
+            "NWIN": [0, 0, 0],
+            "DARKMAP": [0, 0, 0],
+            "COMPLETE": [0, 0, 0],
+            "SLIT_WID": [0, 0, 0],
+            "DATE": [0, 0, 0],
+            "PARENT": [0, 0, 0],
+            "HGLT_OBS": [0, 0, 0],
+            "HGLN_OBS": [0, 0, 0],
+            "PRSTEP1": [0, 0, 0],
+            "PRPROC1": [0, 0, 0],
+            "PRPVER1": [0, 0, 0],
+            "PRPARA1": [0, 0, 0],
+            "proc_steps": [
+                '[{"PRPROC": "JPEG Compression (On-board)"},'
+                ' {"PRPROC": "spice_prep_dark_offset_correction.pro",'
+                ' "PRPVER": "1.3", "PRPARA": "dark_spiobsid=117440828"}]',
+                0,
+                0,
+            ],
+        }
+    )
+    return Catalog(data_frame=df)
+
+
+def test_get_file_relative_path(catalog_df):
+    path = get_file_relative_path(catalog_df.iloc[0])
+    assert path.as_posix() == "level2/2023/02/01"
+
+
+class TestCatalog:
+    def test_init_filename(self):
+        filename = "wepocmwkx.fts"
+        with pytest.raises(RuntimeError, match="does not exist"):
+            catalog = Catalog(filename)  # noqa: F841
+
+    def test_init_release(self, catalog2, catalog_latest):
+        columns = {
+            "NAXIS1",
+            "NAXIS2",
+            "NAXIS3",
+            "NAXIS4",
+            "OBT_BEG",
+            "LEVEL",
+        }
+        assert columns.issubset(catalog2.columns)
+        assert len(catalog2) > 10000
+        assert columns.issubset(catalog_latest.columns)
+        assert len(catalog_latest) > 10000
+        assert catalog_latest._cache_release_catalog() is None
+
+    def test_init_empty(self, catalog_empty):
+        assert len(catalog_empty) == 0
+        assert len(catalog_empty.columns) == 0
+
+    def test_init_dataframe(self, catalog_df):
+        assert len(catalog_df) == 3
+        assert len(catalog_df.columns) == 36
+        assert catalog_df.iloc[1].NAXIS2 == 101
+
+    def test_find_file(self, catalog2):
+        expected_filename = "solo_L2_spice-n-exp_20211204T120022_V02_83886365-000.fits"
+        result = catalog2.find_file(pd.Timestamp("2021-10-10"))
+        assert result.FILENAME == expected_filename
+        result = catalog2.find_file(datetime(2021, 10, 10))
+        assert result.FILENAME == expected_filename
+        result = catalog2.find_file("2021-10-10")
+        assert result.FILENAME == expected_filename
+        result = catalog2.find_file(None)
+        assert result is None
diff --git a/sospice/catalog/tests/test_release.py b/sospice/catalog/tests/test_release.py
new file mode 100644
index 0000000..5555534
--- /dev/null
+++ b/sospice/catalog/tests/test_release.py
@@ -0,0 +1,33 @@
+import pytest
+
+from ..release import Release, get_latest_release_tag
+
+
+@pytest.fixture
+def release2():
+    return Release("2.0")
+
+
+class TestRelease:
+    def test_init(self, release2):
+        assert release2.tag == "2.0"
+        assert release2.base_url.startswith(
+            "https://spice.osups.universite-paris-saclay.fr"
+        )
+        assert release2.url.endswith("2.0/")
+        assert release2.catalog_url.endswith("/catalog.csv")
+        assert release2.exists  # online access
+        assert not release2.is_latest  # online access
+
+    def test_latest(self):  # online access
+        release = Release()
+        assert release.tag != "2.0"
+        assert release.base_url.startswith(
+            "https://spice.osups.universite-paris-saclay.fr"
+        )
+        assert release.catalog_url.endswith("/catalog.csv")
+        assert release.exists
+        assert release.is_latest
+
+    def test_get_latest(self):
+        assert get_latest_release_tag() != "2.0"