Repository: ResonantGeoData/ResonantGeoData

Issue:
`yield_checksumfiles` function doesn't handle `ChecksumFiles` with file paths as their `name` correctly
To reproduce, run [this management script](https://gist.github.com/mvandenburgh/6c9b284b6bf2b684b43236fe0da35a93) and then try to view the `Raster` that it creates. The tile requests will fail with 500 errors with errors about wrong file paths in the server log.

I came across this bug when working on Danesfield. On that project we encode the file path of a given `ChecksumFile` inside the `name` column - for example, `foo/bar/foobar.txt` represents a file hierarchy of
```
foo/
  bar/
    foobar.txt
````

[This line](https://github.com/ResonantGeoData/ResonantGeoData/blob/main/django-rgd/rgd/models/file.py#L290) in RGD is incompatible with this approach, since it assumes the ChecksumFile name is a flat file name instead of a file path. For example, if we have a ChecksumFile with the name `foo.tiff`, the current code would work:
```
path = '/tmp/rgd/file_cache/foo.tiff'
with yield_checksumfiles([self], path.parent):
    yield path
```
Note that `path.parent` would be `/tmp/rgd/file_cache/foo.tiff`, which is correct. But if the file name is `a/b/c/foo.tiff`:
```
path = '/tmp/rgd/file_cache/a/b/c/foo.tiff'
with yield_checksumfiles([self], path.parent):
    yield path
```
the call to `path.parent` would evaluate to `/tmp/rgd/file_cache/a/b/c`, which is incorrect and leads to 500 errors since the server can't find the files in that location.


Discussion:
I have this fixed on a local branch, once i add a regression test I'll open a PR for it.

Resolving patch:
diff --git a/django-rgd/rgd/models/file.py b/django-rgd/rgd/models/file.py
--- a/django-rgd/rgd/models/file.py
+++ b/django-rgd/rgd/models/file.py
@@ -230,7 +230,7 @@ def download_to_local_path(self, directory: str = None):
                 elif self.type == FileSourceType.URL:
                     return download_url_file_to_local_path(self.url, dest_path)
 
-    def get_cache_path(self):
+    def get_cache_path(self, root: bool = False):
         """Generate a predetermined path in the cache directory.
 
         This will use the associated FileSet's cache path if this resource
@@ -244,6 +244,8 @@ def get_cache_path(self):
             directory.mkdir(parents=True, exist_ok=True)
         else:
             directory = self.file_set.get_cache_path()
+        if root:
+            return directory
         return directory / f'{self.name}'
 
     @contextmanager
@@ -278,6 +280,7 @@ def yield_local_path(self, try_fuse: bool = True, yield_file_set: bool = False):
             return
         # Fallback to loading entire file locally - this uses `get_temp_path`
         logger.debug('`yield_local_path` falling back to downloading entire file to local storage.')
+        root = self.get_cache_path(root=True)
         path = self.get_cache_path()
         if yield_file_set and self.file_set:
             # NOTE: This is messy and should be improved but it ensures the directory remains locked
@@ -287,7 +290,7 @@ def yield_local_path(self, try_fuse: bool = True, yield_file_set: bool = False):
         # Not in file_set. Download to cache dir
         from .utils import yield_checksumfiles
 
-        with yield_checksumfiles([self], path.parent):
+        with yield_checksumfiles([self], root):
             yield path
 
     def get_url(self, internal: bool = False):


Test patch:
diff --git a/django-rgd/tests/test_download.py b/django-rgd/tests/test_download.py
--- a/django-rgd/tests/test_download.py
+++ b/django-rgd/tests/test_download.py
@@ -15,12 +15,17 @@ def file_path():
     return datastore.fetch(FILENAME)
 
 
+@pytest.mark.parametrize(
+    'filename',
+    (FILENAME, f'a/b/c/{FILENAME}'),  # edge case - file name that represents a file path
+)
 @pytest.mark.django_db(transaction=True)
-def test_yield_local_path_file(file_path):
+def test_yield_local_path_file(file_path, filename):
     model = ChecksumFile()
     model.type = FileSourceType.FILE_FIELD
     with open(file_path, 'rb') as f:
-        model.file.save(FILENAME, f)
+        model.file.save(filename, f)
+        model.name = filename
     model.file_set = FileSet.objects.create()
     model.save()
     path = model.yield_local_path()
@@ -28,21 +33,31 @@ def test_yield_local_path_file(file_path):
         assert os.path.exists(path)
 
 
+@pytest.mark.parametrize(
+    'filename',
+    (FILENAME, f'a/b/c/{FILENAME}'),  # edge case - file name that represents a file path
+)
 @pytest.mark.django_db(transaction=True)
-def test_yield_local_path_url_http():
+def test_yield_local_path_url_http(filename):
     model = ChecksumFile()
     model.type = FileSourceType.URL
     model.url = datastore.get_url(FILENAME)
+    model.name = filename
     model.save()
     with model.yield_local_path() as path:
         assert os.path.exists(path)
 
 
+@pytest.mark.parametrize(
+    'filename',
+    (FILENAME, f'a/b/c/{FILENAME}'),  # edge case - file name that represents a file path
+)
 @pytest.mark.django_db(transaction=True)
-def test_yield_local_path_url_s3(s3_url):
+def test_yield_local_path_url_s3(s3_url, filename):
     model = ChecksumFile()
     model.type = FileSourceType.URL
     model.url = s3_url
+    model.name = filename
     model.save()
     with model.yield_local_path() as path:
         assert os.path.exists(path)