Repository: NeurodataWithoutBorders/nwbinspector (language: python)

Issue:
[Read Helpers IIa] Add read_nwbfile with TestCase tests
Introducing the latest version of the `read_nwbfile` function as a stand-alone PR with its own unit tests; the integration of it into the full NWB Inspector workflow and subsequent integration testing +/- debugging will occur in a follow-up

I also reorganized the `tools` sub-module here to separate the division of each helper function (some of which may be able to be deprecated soon), which I will also splinter in a smaller PR (which should be merged first, or I'll just rebase this one...) to make this PR specific to only the introduction of the `read_nwbfile` function and its tests

This PR serves as an first example of the convenience functions that will and should eventually make their way up to PyNWB once confirmed to operate as intended to the best of our knowledge

Depends on [HDMF #882](https://github.com/hdmf-dev/hdmf/pull/882)



Discussion / hints:

Fix patch:
diff --git a/src/nwbinspector/tools/__init__.py b/src/nwbinspector/tools/__init__.py
new file mode 100644
index 000000000..a13c67c0f
--- /dev/null
+++ b/src/nwbinspector/tools/__init__.py
@@ -0,0 +1,3 @@
+from .dandi import get_s3_urls_and_dandi_paths
+from .nwb import all_of_type, get_nwbfile_path_from_internal_object
+from ..testing import check_streaming_enabled, make_minimal_nwbfile  # To maintain back-compatability
diff --git a/src/nwbinspector/tools.py b/src/nwbinspector/tools/dandi.py
similarity index 57%
rename from src/nwbinspector/tools.py
rename to src/nwbinspector/tools/dandi.py
index 0c15e9760..b3dd890b1 100644
--- a/src/nwbinspector/tools.py
+++ b/src/nwbinspector/tools/dandi.py
@@ -1,35 +1,10 @@
-"""Helper functions for internal use that rely on external dependencies (i.e., pynwb)."""
+"""Helper functions related to DANDI for internal use that rely on external dependencies (i.e., dandi)."""
 import re
-from uuid import uuid4
-from datetime import datetime
-from typing import Optional, Dict, Tuple
+from typing import Optional, Dict
 from concurrent.futures import ProcessPoolExecutor, as_completed
-from urllib import request
-from warnings import warn
 
-import h5py
-from pynwb import NWBFile
 
-from .utils import is_module_installed, calculate_number_of_cpu
-
-
-def make_minimal_nwbfile():
-    """Most basic NWBFile that can exist."""
-    return NWBFile(session_description="", identifier=str(uuid4()), session_start_time=datetime.now().astimezone())
-
-
-def all_of_type(nwbfile: NWBFile, neurodata_type):
-    """Iterate over all objects inside an NWBFile object and return those that match the given neurodata_type."""
-    for obj in nwbfile.objects.values():
-        if isinstance(obj, neurodata_type):
-            yield obj
-
-
-def get_nwbfile_path_from_internal_object(obj):
-    """Determine the file path on disk for a NWBFile given only an internal object of that file."""
-    if isinstance(obj, NWBFile):
-        return obj.container_source
-    return obj.get_ancestor("NWBFile").container_source
+from ..utils import is_module_installed, calculate_number_of_cpu
 
 
 def get_s3_urls_and_dandi_paths(dandiset_id: str, version_id: Optional[str] = None, n_jobs: int = 1) -> Dict[str, str]:
@@ -78,19 +53,3 @@ def _get_content_url_and_path(asset, follow_redirects: int = 1, strip_query: boo
     Must be globally defined (not as a part of get_s3_urls..) in order to be pickled.
     """
     return {asset.get_content_url(follow_redirects=1, strip_query=True): asset.path}
-
-
-def check_streaming_enabled() -> Tuple[bool, Optional[str]]:
-    """
-    General purpose helper for determining if the environment can support S3 DANDI streaming.
-
-    Returns the boolean status of the check and, if False, provides a string reason for the failure for the user to
-    utilize as they please (raise an error or warning with that message, print it, or ignore it).
-    """
-    try:
-        request.urlopen("https://dandiarchive.s3.amazonaws.com/ros3test.nwb", timeout=1)
-    except request.URLError:
-        return False, "Internet access to DANDI failed."
-    if "ros3" not in h5py.registered_drivers():
-        return False, "ROS3 driver not installed."
-    return True, None
diff --git a/src/nwbinspector/tools/nwb.py b/src/nwbinspector/tools/nwb.py
new file mode 100644
index 000000000..86f7bc130
--- /dev/null
+++ b/src/nwbinspector/tools/nwb.py
@@ -0,0 +1,16 @@
+"""Helper functions related to NWB for internal use that rely on external dependencies (i.e., pynwb)."""
+from pynwb import NWBFile
+
+
+def all_of_type(nwbfile: NWBFile, neurodata_type):
+    """Iterate over all objects inside an NWBFile object and return those that match the given neurodata_type."""
+    for obj in nwbfile.objects.values():
+        if isinstance(obj, neurodata_type):
+            yield obj
+
+
+def get_nwbfile_path_from_internal_object(obj):
+    """Determine the file path on disk for a NWBFile given only an internal object of that file."""
+    if isinstance(obj, NWBFile):
+        return obj.container_source
+    return obj.get_ancestor("NWBFile").container_source


Test patch:
diff --git a/.github/workflows/update-testing-files.yml b/.github/workflows/update-testing-files.yml
index c3ee466fd..2753c1b1d 100644
--- a/.github/workflows/update-testing-files.yml
+++ b/.github/workflows/update-testing-files.yml
@@ -25,6 +25,10 @@ jobs:
 
       - name: Install DANDI
         run: pip install -e .[dandi]
+      - name: Install older PyNWB version to generate example invalid files
+        run: |
+          pip uninstall pynwb
+          pip install pynwb==2.1.0
       - name: Update testing configuration path
         run: python -c "from nwbinspector.testing import update_testing_config; update_testing_config(key='LOCAL_PATH', value='./testing_files/')"
       - name: Generate testing files
diff --git a/src/nwbinspector/testing.py b/src/nwbinspector/testing.py
index 0a9bf1e45..dc23cbfb8 100644
--- a/src/nwbinspector/testing.py
+++ b/src/nwbinspector/testing.py
@@ -1,15 +1,22 @@
 """Helper functions for internal use across the testing suite."""
 import os
 import json
+from uuid import uuid4
 from distutils.util import strtobool
 from pathlib import Path
+from datetime import datetime
 from typing import Tuple, Optional
+from urllib import request
+from tempfile import mkdtemp
+from shutil import rmtree
+from warnings import warn
 
+import h5py
 from packaging.version import Version
-from pynwb import NWBHDF5IO
+from pynwb import NWBHDF5IO, NWBFile
 from pynwb.image import ImageSeries
+from hdmf.testing import TestCase as HDMFTestCase
 
-from .tools import check_streaming_enabled, make_minimal_nwbfile
 from .utils import is_module_installed, get_package_version
 
 
@@ -118,3 +125,28 @@ def generate_image_series_testing_files():  # pragma: no cover
     )
     with NWBHDF5IO(path=local_path / "image_series_testing_file.nwb", mode="w") as io:
         io.write(nwbfile)
+
+
+def make_minimal_nwbfile():
+    """
+    Most basic NWBFile that can exist.
+
+    TODO: replace with pynwb.mock if we can require minimal PyNWB version (or perhaps just for a testing)
+    """
+    return NWBFile(session_description="", identifier=str(uuid4()), session_start_time=datetime.now().astimezone())
+
+
+def check_streaming_enabled() -> Tuple[bool, Optional[str]]:
+    """
+    General purpose helper for determining if the environment can support S3 DANDI streaming.
+
+    Returns the boolean status of the check and, if False, provides a string reason for the failure for the user to
+    utilize as they please (raise an error or warning with that message, print it, or ignore it).
+    """
+    try:
+        request.urlopen("https://dandiarchive.s3.amazonaws.com/ros3test.nwb", timeout=1)
+    except request.URLError:
+        return False, "Internet access to DANDI failed."
+    if "ros3" not in h5py.registered_drivers():
+        return False, "ROS3 driver not installed."
+    return True, None