Repository: pastas/pastas

Issue:
Add cache=True to ps.load_dataset
A caching option should be added to the load-dataset method, because GH doesn't like it this way. GH Actions sometime fails now, so the tests should also be removed and only be executes when the master branch is updated.

Discussion:

Patch:
diff --git a/pastas/dataset.py b/pastas/dataset.py
index 2f1d579c..ec8b68b8 100644
--- a/pastas/dataset.py
+++ b/pastas/dataset.py
@@ -5,6 +5,7 @@ a subfolder in the pastas-data repository.
 
 """
 
+from functools import lru_cache
 from typing import Dict, List, Union
 
 from pandas import DataFrame, read_csv
@@ -12,6 +13,7 @@ from pandas import DataFrame, read_csv
 GITHUB_URL = "https://api.github.com/repos/pastas/pastas-data/contents/"
 
 
+@lru_cache
 def load_dataset(name: str) -> Union[DataFrame, Dict[str, DataFrame]]:
     """Load csv-files from a subfolder in the pastas dataset repository on GitHub.
 
@@ -65,11 +67,15 @@ def load_dataset(name: str) -> Union[DataFrame, Dict[str, DataFrame]]:
     data = {}
 
     # Loop over the files in the folder
-    for file in r.json():
-        if file["name"].endswith(".csv"):
-            # Read file
-            df = read_csv(file["download_url"], index_col=0, parse_dates=True)
-            data[file["name"].split(".")[0]] = df
+    rjson = r.json()
+    read_csv_kwargs = requests.get(
+        [x for x in rjson if x["name"] == "settings.json"][0]["download_url"]
+    ).json()
+    for file in rjson:
+        fname = file["name"]
+        if fname.endswith(".csv"):
+            df = read_csv(file["download_url"], **read_csv_kwargs[fname])
+            data[fname.split(".")[0]] = df
 
     # Return the data, if only one file is found return the dataframe, otherwise return
     # a dictionary with the dataframes
@@ -84,7 +90,8 @@ def load_dataset(name: str) -> Union[DataFrame, Dict[str, DataFrame]]:
         )
 
 
-def list_datasets() -> List[str]:
+@lru_cache
+def list_datasets(silent: bool = True) -> List[str]:
     """Print a list of available datasets in the pastas-data repository on GitHub.
 
     Returns
@@ -124,8 +131,11 @@ def list_datasets() -> List[str]:
             data.append(file["name"])
 
     # Print the list of datasets
-    print("Available datasets in the pastas-data repository on GitHub:")
-    for folder in data:
-        print(f" - {folder}")
-    print(f"Use ps.load_dataset('folder_name') to load a dataset from the repository.")
+    if not silent:
+        print("Available datasets in the pastas-data repository on GitHub:")
+        for folder in data:
+            print(f" - {folder}")
+        print(
+            f"Use ps.load_dataset('folder_name') to load a dataset from the repository."
+        )
     return data
diff --git a/pyproject.toml b/pyproject.toml
index 21db4ba2..8ed5ccac 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -48,7 +48,7 @@ documentation = "https://pastas.readthedocs.io"
 [project.optional-dependencies]
 solvers = ["lmfit >= 1.0.0", "emcee >= 3.0"]
 latexify = ["latexify-py"]
-full = ["pastas[latexify,solvers]"]
+full = ["pastas[latexify,solvers]", "requests"]
 formatting = ["isort", "black[jupyter]"]
 linting = ["flake8"]
 pytesting = ["pytest>=7", "pytest-cov", "pytest-sugar"]


Test patch:
diff --git a/tests/test_dataset.py b/tests/test_dataset.py
index 6b2d4849..9bd3c029 100644
--- a/tests/test_dataset.py
+++ b/tests/test_dataset.py
@@ -1,7 +1,8 @@
 import pytest
-from pastas.dataset import list_datasets, load_dataset
 from pandas import DataFrame
 
+from pastas.dataset import list_datasets, load_dataset
+
 
 def test_load_single_csv():
     # Test loading a single csv file
@@ -26,7 +27,7 @@ def test_invalid_folder_name():
 
 def test_list_datasets():
     # Test listing available datasets
-    list_datasets()
+    list_datasets(silent=False)
     # Add assertions here to verify the output of the function
     # For example, you can check if the output contains certain dataset names
     # assert "collenteur_2021" in list_datasets()