Repository: NCAS-CMS/cfdm

Issue:
Reading of `http://` remote files does not work
In cfdm `1.10.1.0` we get:
```python
>>> remote_file = 'http://psl.noaa.gov/thredds/dodsC/Datasets/cru/crutem5/Monthlies/air.mon.anom.nobs.nc'
>>> import netCDF4
>>> nc = netCDF4.Dataset(remote_file, 'r')  # works
>>> import cfdm
>>> j = cfdm.read(remote_file)  # doesn't work
<snip>
packages/cfdm/read_write/netcdf/netcdfread.py:810, in NetCDFRead.read(self, filename, extra, default_version, external, extra_read_vars, _scan_only, verbose, mask, warnings, warn_valid, domain)
    807     raise IOError(f"Can't read directory {filename}")
    809 if not os.path.isfile(filename):
--> 810     raise IOError(f"Can't read non-existent file {filename}")
    812 g["filename"] = filename
    814 # ------------------------------------------------------------
    815 # Open the netCDF file to be read
    816 # ------------------------------------------------------------

OSError: Can't read non-existent file http://psl.noaa.gov/thredds/dodsC/Datasets/cru/crutem5/Monthlies/air.mon.anom.nobs.nc
```
The problem is that cfdm is testing `remote_file` for being a file with `os.path.isfile`, which returns `False` for remote URLs.

PR to follow. 

Discussion:

Patch:
diff --git a/Changelog.rst b/Changelog.rst
index 73d3efdea..ff7f4a697 100644
--- a/Changelog.rst
+++ b/Changelog.rst
@@ -6,6 +6,8 @@ Version 1.10.1.1
 * Fix bug that caused very slow reads of datasets with compession by
   gathering or DSG ragged arrays
   (https://github.com/NCAS-CMS/cfdm/issues/260)
+* Fix bug that prevented `cfdm.read` from accessing remote files in URL
+  locations (https://github.com/NCAS-CMS/cfdm/issues/262)
 
 ----
 
diff --git a/cfdm/read_write/netcdf/netcdfread.py b/cfdm/read_write/netcdf/netcdfread.py
index ad6e1baf9..c3c3c8c75 100644
--- a/cfdm/read_write/netcdf/netcdfread.py
+++ b/cfdm/read_write/netcdf/netcdfread.py
@@ -8,6 +8,7 @@ import tempfile
 from ast import literal_eval
 from copy import deepcopy
 from functools import reduce
+from urllib.parse import urlparse
 
 import netCDF4
 import netcdf_flattener
@@ -584,6 +585,66 @@ class NetCDFRead(IORead):
 
         return cdl
 
+    @classmethod
+    def is_file(cls, filename):
+        """Return `True` if *filename* is a file.
+
+        Note that a remote URL starting with ``http://`` or
+        ``https://`` is always considered as a file.
+
+        .. versionadded:: (cfdm) 1.10.1.1
+
+        :Parameters:
+
+            filename: `str`
+                The name of the file.
+
+        :Returns:
+
+            `bool`
+                Whether or not *filename* is a file.
+
+        **Examples**
+
+        >>> {{package}}.{{class}}.is_file('file.nc')
+        True
+        >>> {{package}}.{{class}}.is_file('http://file.nc')
+        True
+        >>> {{package}}.{{class}}.is_file('https://file.nc')
+        True
+
+        """
+        # Assume that URLs are files
+        u = urlparse(filename)
+        if u.scheme in ("http", "https"):
+            return True
+
+        return os.path.isfile(filename)
+
+    @classmethod
+    def is_dir(cls, filename):
+        """Return `True` if *filename* is a directory.
+
+        .. versionadded:: (cfdm) 1.10.1.1
+
+        :Parameters:
+
+            filename: `str`
+                The name of the file.
+
+        :Returns:
+
+            `bool`
+                Whether or not *filename* is a directory.
+
+        **Examples**
+
+        >>> {{package}}.{{class}}.is_dir('file.nc')
+        False
+
+        """
+        return os.path.isdir(filename)
+
     def default_netCDF_fill_value(self, ncvar):
         """The default netCDF fill value for a variable.
 
@@ -803,10 +864,10 @@ class NetCDFRead(IORead):
 
         filename = os.path.expanduser(os.path.expandvars(filename))
 
-        if os.path.isdir(filename):
+        if self.is_dir(filename):
             raise IOError(f"Can't read directory {filename}")
 
-        if not os.path.isfile(filename):
+        if not self.is_file(filename):
             raise IOError(f"Can't read non-existent file {filename}")
 
         g["filename"] = filename
diff --git a/cfdm/read_write/read.py b/cfdm/read_write/read.py
index 4d8ec8311..7207e6323 100644
--- a/cfdm/read_write/read.py
+++ b/cfdm/read_write/read.py
@@ -294,6 +294,9 @@ def read(
     >>> j = cfdm.read('parent.nc', external=['external1.nc', 'external2.nc'])
 
     """
+    # Initialise a netCDF read object
+    netcdf = NetCDFRead(_implementation)
+
     # Parse the field parameter
     if extra is None:
         extra = ()
@@ -302,19 +305,15 @@ def read(
 
     filename = os.path.expanduser(os.path.expandvars(filename))
 
-    if os.path.isdir(filename):
+    if netcdf.is_dir(filename):
         raise IOError(f"Can't read directory {filename}")
 
-    if not os.path.isfile(filename):
+    if not netcdf.is_file(filename):
         raise IOError(f"Can't read non-existent file {filename}")
 
     # ----------------------------------------------------------------
     # Read the file into field/domain contructs
     # ----------------------------------------------------------------
-
-    # Initialise a netCDF read object
-    netcdf = NetCDFRead(_implementation)
-
     cdl = False
     if netcdf.is_cdl_file(filename):
         # Create a temporary netCDF file from input CDL


Test patch:
diff --git a/cfdm/test/test_read_write.py b/cfdm/test/test_read_write.py
index 069522b35..ec27b0207 100644
--- a/cfdm/test/test_read_write.py
+++ b/cfdm/test/test_read_write.py
@@ -983,6 +983,15 @@ class read_writeTest(unittest.TestCase):
         self.assertEqual(len(h), 1)
         self.assertTrue(f.equals(h[0]))
 
+    def test_read_url(self):
+        """Test reading urls."""
+        remote = "http://psl.noaa.gov/thredds/dodsC/Datasets/cru/crutem5/Monthlies/air.mon.anom.nobs.nc"
+        # Check that the file remote is indeed accesible
+        _ = netCDF4.Dataset(remote, "r")
+        # Check that cfdm can access it
+        f = cfdm.read(remote)
+        self.assertEqual(len(f), 1)
+
 
 if __name__ == "__main__":
     print("Run date:", datetime.datetime.now())