Repository: PEtab-dev/libpetab-python

Issue:
Validation fails for remote files
`validate_yaml_semantics` uses `os.path.isfile` to check whether referenced files are available and fails for remote files.

Discussion:
codecov-commenter: ## [Codecov](https://app.codecov.io/gh/PEtab-dev/libpetab-python/pull/287?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=PEtab-dev) Report
Attention: Patch coverage is `69.23077%` with `4 lines` in your changes missing coverage. Please review.
> Project coverage is 74.37%. Comparing base [(`6299139`)](https://app.codecov.io/gh/PEtab-dev/libpetab-python/commit/62991395aeebb5b60a02ba39d6dc9a0347db4957?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=PEtab-dev) to head [(`aa2c215`)](https://app.codecov.io/gh/PEtab-dev/libpetab-python/commit/aa2c215b0f9361ce646ef605ba511fa4ce41ceb2?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=PEtab-dev).

| [Files](https://app.codecov.io/gh/PEtab-dev/libpetab-python/pull/287?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=PEtab-dev) | Patch % | Lines |
|---|---|---|
| [petab/v1/yaml.py](https://app.codecov.io/gh/PEtab-dev/libpetab-python/pull/287?src=pr&el=tree&filepath=petab%2Fv1%2Fyaml.py&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=PEtab-dev#diff-cGV0YWIvdjEveWFtbC5weQ==) | 69.23% | [3 Missing and 1 partial :warning: ](https://app.codecov.io/gh/PEtab-dev/libpetab-python/pull/287?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=PEtab-dev) |

<details><summary>Additional details and impacted files</summary>


```diff
@@           Coverage Diff            @@
##           develop     #287   +/-   ##
========================================
  Coverage    74.37%   74.37%           
========================================
  Files           70       70           
  Lines         4448     4460   +12     
  Branches       952      953    +1     
========================================
+ Hits          3308     3317    +9     
- Misses         869      872    +3     
  Partials       271      271           
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/PEtab-dev/libpetab-python/pull/287?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=PEtab-dev).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=PEtab-dev).


Patch:
diff --git a/petab/v1/problem.py b/petab/v1/problem.py
index 203e8d9..e49e4d7 100644
--- a/petab/v1/problem.py
+++ b/petab/v1/problem.py
@@ -5,7 +5,7 @@ import os
 import tempfile
 from collections.abc import Iterable
 from math import nan
-from pathlib import Path
+from pathlib import Path, PurePosixPath
 from typing import TYPE_CHECKING
 from warnings import warn
 
@@ -485,7 +485,7 @@ class Problem:
 
         if prefix_path is None:
             return filenames["yaml_file"]
-        return str(prefix_path / filenames["yaml_file"])
+        return str(PurePosixPath(prefix_path, filenames["yaml_file"]))
 
     def to_files(
         self,
diff --git a/petab/v1/yaml.py b/petab/v1/yaml.py
index 42ea732..94dfbcb 100644
--- a/petab/v1/yaml.py
+++ b/petab/v1/yaml.py
@@ -8,6 +8,7 @@ from urllib.parse import unquote, urlparse, urlunparse
 
 import jsonschema
 import numpy as np
+import pandas as pd
 import yaml
 from pandas.io.common import get_handle
 
@@ -110,17 +111,36 @@ def validate_yaml_semantics(
     """
     if not path_prefix:
         if isinstance(yaml_config, str | Path):
-            path_prefix = os.path.dirname(str(yaml_config))
+            path_prefix = get_path_prefix(yaml_config)
         else:
             path_prefix = ""
 
     yaml_config = load_yaml(yaml_config)
 
     def _check_file(_filename: str, _field: str):
-        if not os.path.isfile(_filename):
+        # this could be a regular path or some local or remote URL
+        # the simplest check is just trying to load the respective table or
+        # sbml model
+        if _field == SBML_FILES:
+            from ..models.sbml_model import SbmlModel
+
+            try:
+                SbmlModel.from_file(_filename)
+            except Exception as e:
+                raise AssertionError(
+                    f"Failed to read '{_filename}' provided as '{_field}'."
+                ) from e
+            return
+
+        try:
+            pd.read_csv(_filename, sep="\t")
+        except pd.errors.EmptyDataError:
+            # at this stage, we don't care about the content
+            pass
+        except Exception as e:
             raise AssertionError(
-                f"File '{_filename}' provided as '{_field}' " "does not exist."
-            )
+                f"Failed to read '{_filename}' provided as '{_field}'."
+            ) from e
 
     # Handles both a single parameter file, and a parameter file that has been
     # split into multiple subset files.
@@ -128,7 +148,9 @@ def validate_yaml_semantics(
         np.array(yaml_config[PARAMETER_FILE]).flat
     ):
         _check_file(
-            os.path.join(path_prefix, parameter_subset_file),
+            f"{path_prefix}/{parameter_subset_file}"
+            if path_prefix
+            else parameter_subset_file,
             parameter_subset_file,
         )
 
@@ -142,7 +164,12 @@ def validate_yaml_semantics(
         ]:
             if field in problem_config:
                 for filename in problem_config[field]:
-                    _check_file(os.path.join(path_prefix, filename), field)
+                    _check_file(
+                        f"{path_prefix}/{filename}"
+                        if path_prefix
+                        else filename,
+                        field,
+                    )
 
 
 def load_yaml(yaml_config: dict | Path | str) -> dict:


Test patch:
diff --git a/tests/v1/test_yaml.py b/tests/v1/test_yaml.py
index 42e1072..82ab242 100644
--- a/tests/v1/test_yaml.py
+++ b/tests/v1/test_yaml.py
@@ -37,8 +37,10 @@ def test_create_problem_yaml():
         observable_file = Path(outdir, "observables.tsv")
         yaml_file = Path(outdir, "problem.yaml")
         visualization_file = Path(outdir, "visualization.tsv")
+
+        _create_dummy_sbml_model(sbml_file)
+
         for file in (
-            sbml_file,
             condition_file,
             measurement_file,
             parameter_file,
@@ -65,13 +67,14 @@ def test_create_problem_yaml():
         observable_file2 = Path(outdir, "observables2.tsv")
         yaml_file2 = Path(outdir, "problem2.yaml")
         for file in (
-            sbml_file2,
             condition_file2,
             measurement_file2,
             observable_file2,
         ):
             file.touch()
 
+        _create_dummy_sbml_model(sbml_file2)
+
         sbml_files = [sbml_file, sbml_file2]
         condition_files = [condition_file, condition_file2]
         measurement_files = [measurement_file, measurement_file2]
@@ -87,10 +90,27 @@ def test_create_problem_yaml():
         validate(yaml_file2)
 
 
-def test_get_path_prefix_local():
+def test_get_path_prefix():
     assert get_path_prefix("/some/dir/file.yaml") == str(Path("/some/dir"))
     assert get_path_prefix("some/dir/file.yaml") == str(Path("some/dir"))
     assert (
         get_path_prefix("https://petab.rocks/dir/file.yaml")
         == "https://petab.rocks/dir"
     )
+
+
+def test_validate_remote():
+    yaml_url = (
+        "https://raw.githubusercontent.com/PEtab-dev/petab_test_suite"
+        "/main/petabtests/cases/v1.0.0/sbml/0001/_0001.yaml"
+    )
+
+    validate(yaml_url)
+
+
+def _create_dummy_sbml_model(sbml_file: Path | str):
+    import libsbml
+
+    sbml_doc = libsbml.SBMLDocument()
+    sbml_doc.createModel()
+    libsbml.writeSBMLToFile(sbml_doc, str(sbml_file))