Repository: fsspec/filesystem_spec (Python)

Issue:
Error reading pyarrow parquet dataset with HTTPFilesystem and AWS presigned urls
When trying out [databricks delta sharing](https://databricks.com/blog/2021/05/26/introducing-delta-sharing-an-open-protocol-for-secure-data-sharing.html) I encountered a bug when reading the delta file into a pyarrow pandas dataset.

Investigation lead to discovering a problem in the determination of the file size within the [fsspec](https://pypi.org/project/fsspec/) python library.

In https://github.com/intake/filesystem_spec/blob/master/fsspec/implementations/http.py#L673-L681 there is no error handling present. 
And with the presigned url a HEAD request might return a 403 is the signing was done based on the GET method.

Working on making a proper PR. 

for reproducing the error I create a smal repo: https://github.com/krisgeus/filesystem_spec_issue_reproduced


Discussion:

Patch:
diff --git a/fsspec/implementations/http.py b/fsspec/implementations/http.py
index f3ae3deca..229ef9a7e 100644
--- a/fsspec/implementations/http.py
+++ b/fsspec/implementations/http.py
@@ -573,13 +573,16 @@ def close(self):
         pass
 
     def __reduce__(self):
-        return reopen, (
-            self.fs,
-            self.url,
-            self.mode,
-            self.blocksize,
-            self.cache.name,
-            self.size,
+        return (
+            reopen,
+            (
+                self.fs,
+                self.url,
+                self.mode,
+                self.blocksize,
+                self.cache.name,
+                self.size,
+            ),
         )
 
 
@@ -671,13 +674,20 @@ async def _file_size(url, session=None, size_policy="head", **kwargs):
     else:
         raise TypeError('size_policy must be "head" or "get", got %s' "" % size_policy)
     async with r:
-        # TODO:
-        #  recognise lack of 'Accept-Ranges', or  'Accept-Ranges': 'none' (not 'bytes')
-        #  to mean streaming only, no random access => return None
-        if "Content-Length" in r.headers:
-            return int(r.headers["Content-Length"])
-        elif "Content-Range" in r.headers:
-            return int(r.headers["Content-Range"].split("/")[1])
+        try:
+            r.raise_for_status()
+
+            # TODO:
+            #  recognise lack of 'Accept-Ranges',
+            #                 or 'Accept-Ranges': 'none' (not 'bytes')
+            #  to mean streaming only, no random access => return None
+            if "Content-Length" in r.headers:
+                return int(r.headers["Content-Length"])
+            elif "Content-Range" in r.headers:
+                return int(r.headers["Content-Range"].split("/")[1])
+        except aiohttp.ClientResponseError:
+            logger.debug("Error retrieving file size")
+            return None
         r.close()
 
 


Test patch:
diff --git a/fsspec/implementations/tests/test_http.py b/fsspec/implementations/tests/test_http.py
index 6b526ed43..eb90691d4 100644
--- a/fsspec/implementations/tests/test_http.py
+++ b/fsspec/implementations/tests/test_http.py
@@ -66,10 +66,15 @@ def do_GET(self):
             self._respond(200, data=d)
 
     def do_HEAD(self):
+        d = data if self.path == "/index/realfile" else index
+        if "head_not_auth" in self.headers:
+            self._respond(
+                403, {"Content-Length": 123}, b"not authorized for HEAD request"
+            )
+            return
         if "head_ok" not in self.headers:
             self._respond(405)
             return
-        d = data if self.path == "/index/realfile" else index
         if self.path.rstrip("/") not in ["/index/realfile", "/index"]:
             self._respond(404)
         elif "give_length" in self.headers:
@@ -286,6 +291,8 @@ def test_methods(server):
         {"give_length": "true"},
         {"give_length": "true", "head_ok": "true"},
         {"give_range": "true"},
+        {"give_length": "true", "head_not_auth": "true"},
+        {"give_range": "true", "head_not_auth": "true"},
     ],
 )
 def test_random_access(server, headers):