Repository: ResonantGeoData/ResonantGeoData

Issue:
Move lock files to separate, predetermined path
After #551 lands, we should move all of the file locks to another predetermined path in the cache such that they do not get in the way of ETL routines or downstream projects that checkout files and use all adjacent files for some task

cc @AlmightyYakob 


Discussion:

Resolving patch:
diff --git a/django-rgd/rgd/models/file.py b/django-rgd/rgd/models/file.py
--- a/django-rgd/rgd/models/file.py
+++ b/django-rgd/rgd/models/file.py
@@ -9,7 +9,6 @@
 from django.conf import settings
 from django.contrib.gis.db import models
 from django_extensions.db.models import TimeStampedModel
-from filelock import FileLock
 from rgd.utility import (
     _link_url,
     clean_file_cache,
@@ -19,6 +18,7 @@
     download_field_file_to_local_path,
     download_url_file_to_local_path,
     get_cache_dir,
+    get_file_lock,
     patch_internal_presign,
     precheck_fuse,
     safe_urlopen,
@@ -197,9 +197,7 @@ def download_to_local_path(self, directory: str = None):
             dest_path = Path(directory, self.name)
         dest_path.parent.mkdir(parents=True, exist_ok=True)
         # Thread/process safe locking for file access
-        lock = FileLock(
-            f'{dest_path}.lock'
-        )  # timeout=getattr(settings, 'RGD_FILE_LOCK_TIMEOUT', 10000))
+        lock = get_file_lock(dest_path)
 
         with lock:  # TODO: handle timeouts in condition
             if dest_path.exists() and dest_path.stat().st_size > 0:
@@ -274,7 +272,7 @@ def yield_local_path(self, try_fuse: bool = True, yield_file_set: bool = False):
         # Not in file_set. Download to cache dir
         path = self.download_to_local_path()
         # provide a lock on this file while yielding
-        lock = FileLock(f'{path}.lock')
+        lock = get_file_lock(path)
         with lock:
             yield path
 
diff --git a/django-rgd/rgd/models/utils.py b/django-rgd/rgd/models/utils.py
--- a/django-rgd/rgd/models/utils.py
+++ b/django-rgd/rgd/models/utils.py
@@ -7,9 +7,8 @@
 
 from django.core.exceptions import ValidationError
 from django.db.models import QuerySet
-from filelock import FileLock
 
-from ..utility import compute_checksum_url, compute_hash, get_or_create_no_commit
+from ..utility import compute_checksum_url, compute_hash, get_file_lock, get_or_create_no_commit
 from .collection import Collection
 from .file import ChecksumFile, FileSourceType
 from .mixins import Status
@@ -139,8 +138,7 @@ def yield_checksumfiles(queryset: Union[QuerySet, List[ChecksumFile]], directory
     directory = Path(directory)
     directory.touch()
     # Acquire a lock on the directory so that it isn't cleaned up
-    lock_file_path = f'{directory}.lock'
-    lock = FileLock(lock_file_path)
+    lock = get_file_lock(directory)
     lock.acquire()
     # Download each file to the directory and yeild it so that the lock can be released when done
     try:
diff --git a/django-rgd/rgd/utility.py b/django-rgd/rgd/utility.py
--- a/django-rgd/rgd/utility.py
+++ b/django-rgd/rgd/utility.py
@@ -45,6 +45,21 @@ def get_cache_dir():
     return path
 
 
+def get_lock_dir():
+    path = Path(get_temp_dir(), 'file_locks')
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def get_file_lock(path: Path):
+    """Create a file lock under the lock directory."""
+    # Computes the hash using Pathlib's hash implementation on absolute path
+    sha = hash(path.absolute())
+    lock_path = Path(get_lock_dir(), f'{sha}.lock')
+    lock = FileLock(lock_path)
+    return lock
+
+
 @contextmanager
 def safe_urlopen(url: str, *args, **kwargs):
     with contextlib.closing(urlopen(url, *args, **kwargs)) as remote:
@@ -279,9 +294,7 @@ def clean_file_cache(override_target=None):
     """
     cache = get_cache_dir()
     # Sort each directory by mtime
-    paths = sorted(
-        [f for f in Path(cache).iterdir() if not str(f).endswith('.lock')], key=os.path.getmtime
-    )  # This sorts oldest to latest
+    paths = sorted(Path(cache).iterdir(), key=os.path.getmtime)  # This sorts oldest to latest
     # While free space is not enough, remove directories until all ar gone
     initial = psutil.disk_usage(cache).free
     target = override_target or getattr(settings, 'RGD_TARGET_AVAILABLE_CACHE', 2)
@@ -295,8 +308,7 @@ def clean_file_cache(override_target=None):
             break
         path = paths.pop(0)
         # Check if resource is locked and skip if so
-        lock_path = Path(f'{path}.lock')
-        lock = FileLock(lock_path)
+        lock = get_file_lock(path)
         try:
             with lock.acquire(timeout=0):
                 if os.path.isdir(path):
@@ -304,7 +316,7 @@ def clean_file_cache(override_target=None):
                 else:
                     os.remove(path)
             # Remove the lockfile as well
-            os.remove(lock_path)
+            os.remove(lock.lock_file)
             logger.info(f'removed: {path}')
         except Timeout:
             # Another task holds the lock on this file/directory: do not delete


Test patch: