Repository: DataBiosphere/toil

Issue:
Lock file manipulation can stop the workflow if a transient generic "Input/output error" is ever encountered
Related to  #4654, the Singularity cache mutex is likely to be on a networked filesystem, and possibly on Ceph.

When going to unlock (or lock?) a file on such a filesystem, it's possible to get an "Input/output error" (errno 5). Usually this would indicate that the hard disk is broken or some other terrible thing is happening, but on distributed filesystems it might just be packet loss or a dropped connection, and trying again might fix the problem.

We should retry attempts to interact with the lock files if we get generic IO errors.

┆Issue is synchronized with this [Jira Story](https://ucsc-cgl.atlassian.net/browse/TOIL-1541)
┆Issue Number: TOIL-1541



Discussion:

Resolving patch:
diff --git a/src/toil/deferred.py b/src/toil/deferred.py
--- a/src/toil/deferred.py
+++ b/src/toil/deferred.py
@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import fcntl
+import errno
 import logging
 import os
 import tempfile
@@ -20,9 +20,11 @@
 
 import dill
 
+
 from toil.lib.io import robust_rmtree
 from toil.realtimeLogger import RealtimeLogger
 from toil.resource import ModuleDescriptor
+from toil.lib.threading import safe_lock, safe_unlock_and_close
 
 logger = logging.getLogger(__name__)
 
@@ -128,18 +130,22 @@ def __init__(self, stateDirBase: str) -> None:
 
         # Lock the state file. The lock will automatically go away if our process does.
         try:
-            fcntl.lockf(self.stateFD, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            safe_lock(self.stateFD, block=False)
         except OSError as e:
-            # Someone else might have locked it even though they should not have.
-            raise RuntimeError(f"Could not lock deferred function state file {self.stateFileName}: {str(e)}")
+            if e.errno in (errno.EACCES, errno.EAGAIN):
+                # Someone else locked it even though they should not have.
+                raise RuntimeError(f"Could not lock deferred function state file {self.stateFileName}") from e
+            else:
+                # Something else went wrong
+                raise
 
         # Rename it to remove the suffix
         os.rename(self.stateFileName, self.stateFileName[:-len(self.WIP_SUFFIX)])
         self.stateFileName = self.stateFileName[:-len(self.WIP_SUFFIX)]
 
-        # Wrap the FD in a Python file object, which we will use to actually use it.
+        # Get a Python file object for the file, which we will use to actually use it.
         # Problem: we can't be readable and writable at the same time. So we need two file objects.
-        self.stateFileOut = os.fdopen(self.stateFD, 'wb')
+        self.stateFileOut = open(self.stateFileName, 'wb')
         self.stateFileIn = open(self.stateFileName, 'rb')
 
         logger.debug("Opened with own state file %s" % self.stateFileName)
@@ -157,10 +163,7 @@ def __del__(self):
             os.unlink(self.stateFileName)
 
         # Unlock it
-        fcntl.lockf(self.stateFD, fcntl.LOCK_UN)
-
-        # Don't bother with close, destroying will close and it seems to maybe
-        # have been GC'd already anyway.
+        safe_unlock_and_close(self.stateFD)
 
     @contextmanager
     def open(self):
@@ -318,11 +321,16 @@ def _runOrphanedDeferredFunctions(self):
                     continue
 
                 try:
-                    fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
-                except OSError:
-                    # File is still locked by someone else.
-                    # Look at the next file instead
-                    continue
+                    safe_lock(fd, block=False)
+                except OSError as e:
+                    os.close(fd)
+                    if e.errno in (errno.EACCES, errno.EAGAIN):
+                        # File is still locked by someone else.
+                        # Look at the next file instead
+                        continue
+                    else:
+                        # Something else went wrong
+                        raise
 
                 logger.debug("Locked file %s" % fullFilename)
 
@@ -330,7 +338,7 @@ def _runOrphanedDeferredFunctions(self):
                 foundFiles = True
 
                 # Actually run all the stored deferred functions
-                fileObj = os.fdopen(fd, 'rb')
+                fileObj = open(fullFilename, 'rb')
                 self._runAllDeferredFunctions(fileObj)
                 states_handled += 1
 
@@ -342,10 +350,6 @@ def _runOrphanedDeferredFunctions(self):
                     pass
 
                 # Unlock it
-                fcntl.lockf(fd, fcntl.LOCK_UN)
-
-                # Now close it. This closes the backing file descriptor. See
-                # <https://stackoverflow.com/a/24984929>
-                fileObj.close()
+                safe_unlock_and_close(fd)
 
         logger.debug("Ran orphaned deferred functions from %d abandoned state files", states_handled)
diff --git a/src/toil/fileStores/nonCachingFileStore.py b/src/toil/fileStores/nonCachingFileStore.py
--- a/src/toil/fileStores/nonCachingFileStore.py
+++ b/src/toil/fileStores/nonCachingFileStore.py
@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import errno
-import fcntl
 import logging
 import os
 import tempfile
@@ -43,7 +42,7 @@
 from toil.lib.compatibility import deprecated
 from toil.lib.io import make_public_dir, robust_rmtree
 from toil.lib.retry import ErrorCondition, retry
-from toil.lib.threading import get_process_name, process_name_exists
+from toil.lib.threading import get_process_name, process_name_exists, safe_lock, safe_unlock_and_close
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -258,18 +257,23 @@ def _removeDeadJobs(cls, coordination_dir: str, batchSystemShutdown: bool=False)
                 # We need to have a race to pick someone to clean up.
 
                 try:
-                    # Open the directory
+                    # Open the directory.
+                    # We can't open a directory for write, only for read.
                     dirFD = os.open(jobState['jobDir'], os.O_RDONLY)
                 except FileNotFoundError:
                     # The cleanup has happened and we can't contest for it
                     continue
 
                 try:
-                    # Try and lock it
-                    fcntl.lockf(dirFD, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                    # Try and lock it non-blocking
+                    safe_lock(dirFD, block=False)
                 except OSError as e:
-                    # We lost the race. Someone else is alive and has it locked.
                     os.close(dirFD)
+                    if e.errno not in (errno.EACCES, errno.EAGAIN):
+                        # Something went wrong
+                        raise
+                    # Otherwise, we lost the race. Someone else is alive and
+                    # has it locked. So loop around again.
                 else:
                     # We got it
                     logger.warning('Detected that job (%s) prematurely terminated.  Fixing the '
@@ -283,8 +287,7 @@ def _removeDeadJobs(cls, coordination_dir: str, batchSystemShutdown: bool=False)
                             # cleanup. Leave that to the batch system cleanup code.
                             robust_rmtree(jobState['jobDir'])
                     finally:
-                        fcntl.lockf(dirFD, fcntl.LOCK_UN)
-                        os.close(dirFD)
+                        safe_unlock_and_close(dirFD)
 
     @classmethod
     def _getAllJobStates(cls, coordination_dir: str) -> Iterator[Dict[str, str]]:
diff --git a/src/toil/lib/threading.py b/src/toil/lib/threading.py
--- a/src/toil/lib/threading.py
+++ b/src/toil/lib/threading.py
@@ -23,6 +23,7 @@
 import os
 import sys
 import tempfile
+import time
 import threading
 import traceback
 from contextlib import contextmanager
@@ -35,6 +36,51 @@
 
 logger = logging.getLogger(__name__)
 
+def safe_lock(fd: int, block: bool = True, shared: bool = False) -> None:
+    """
+    Get an fcntl lock, while retrying on IO errors.
+
+    Raises OSError with EACCES or EAGAIN when a nonblocking lock is not
+    immediately available.
+    """
+
+    error_backoff = 1
+
+    while True:
+        try:
+            # Wait until we can exclusively lock it.
+            lock_mode = (fcntl.LOCK_SH if shared else fcntl.LOCK_EX) | (fcntl.LOCK_NB if not block else 0)
+            fcntl.flock(fd, lock_mode)
+            return
+        except OSError as e:
+            if e.errno in (errno.EACCES, errno.EAGAIN):
+                # Nonblocking lock not available.
+                raise
+            elif e.errno == errno.EIO:
+                # Sometimes Ceph produces IO errors when talking to lock files.
+                # Back off and try again.
+                # TODO: Should we eventually give up if the disk really is
+                # broken? If so we should use the retry system.
+                logger.error("IO error talking to lock file. Retrying after %s seconds.", error_backoff)
+                time.sleep(error_backoff)
+                error_backoff = min(60, error_backoff * 2)
+                continue
+            else:
+                raise
+
+def safe_unlock_and_close(fd: int) -> None:
+    """
+    Release an fcntl lock and close the file descriptor, while handling fcntl IO errors.
+    """
+    try:
+        fcntl.flock(fd, fcntl.LOCK_UN)
+    except OSError as e:
+        if e.errno != errno.EIO:
+            raise
+        # Sometimes Ceph produces EIO. We don't need to retry then because
+        # we're going to close the FD and after that the file can't remain
+        # locked by us.
+    os.close(fd)
 
 class ExceptionalThread(threading.Thread):
     """
@@ -280,10 +326,14 @@ def get_process_name(base_dir: str) -> str:
 
         # Lock the file. The lock will automatically go away if our process does.
         try:
-            fcntl.lockf(nameFD, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            safe_lock(nameFD, block=False)
         except OSError as e:
-            # Someone else might have locked it even though they should not have.
-            raise RuntimeError(f"Could not lock process name file {nameFileName}: {str(e)}")
+            if e.errno in (errno.EACCES, errno.EAGAIN):
+                # Someone else locked it even though they should not have.
+                raise RuntimeError(f"Could not lock process name file {nameFileName}") from e
+            else:
+                # Something else is wrong
+                raise
 
         # Save the basename
         current_process_name_for[base_dir] = os.path.basename(nameFileName)
@@ -328,13 +378,18 @@ def process_name_exists(base_dir: str, name: str) -> bool:
             # Otherwise see if we can lock it shared, for which we need an FD, but
             # only for reading.
             nameFD = os.open(nameFileName, os.O_RDONLY)
-            fcntl.lockf(nameFD, fcntl.LOCK_SH | fcntl.LOCK_NB)
         except FileNotFoundError as e:
             # File has vanished
             return False
+        try:
+            safe_lock(nameFD, block=False, shared=True)
         except OSError as e:
-            # Could not lock. Process is alive.
-            return True
+            if e.errno in (errno.EACCES, errno.EAGAIN):
+                # Could not lock. Process is alive.
+                return True
+            else:
+                # Something else went wrong
+                raise
         else:
             # Could lock. Process is dead.
             # Remove the file. We race to be the first to do so.
@@ -342,8 +397,8 @@ def process_name_exists(base_dir: str, name: str) -> bool:
                 os.remove(nameFileName)
             except:
                 pass
-            # Unlock
-            fcntl.lockf(nameFD, fcntl.LOCK_UN)
+            safe_unlock_and_close(nameFD)
+            nameFD = None
             # Report process death
             return False
     finally:
@@ -381,12 +436,28 @@ def global_mutex(base_dir: str, mutex: str) -> Iterator[None]:
     # there's a race where someone can open the file before we unlink it and
     # get a lock on the deleted file.
 
+    error_backoff = 1
+
     while True:
         # Try to create the file, ignoring if it exists or not.
         fd = os.open(lock_filename, os.O_CREAT | os.O_WRONLY)
 
-        # Wait until we can exclusively lock it.
-        fcntl.lockf(fd, fcntl.LOCK_EX)
+        try:
+            # Wait until we can exclusively lock it.
+            safe_lock(fd)
+        except OSError as e:
+            if e.errno == errno.EIO:
+                # Sometimes Ceph produces IO errors when talking to lock files.
+                os.close(fd)
+                # Back off and try again
+                logger.error("IO error talking to lock file %s. Retrying after %s seconds.", lock_filename, error_backoff)
+                time.sleep(error_backoff)
+                error_backoff = min(60, error_backoff * 2)
+                continue
+            else:
+                # Something unexpected went wrong
+                os.close(fd)
+                raise
 
         # Holding the lock, make sure we are looking at the same file on disk still.
         try:
@@ -394,16 +465,14 @@ def global_mutex(base_dir: str, mutex: str) -> Iterator[None]:
             fd_stats = os.fstat(fd)
         except OSError as e:
             if e.errno == errno.ESTALE:
-                # The file handle has gone stale, because somebody removed the file.
+                # The file handle has gone stale, because somebody removed the
+                # file.
                 # Try again.
-                try:
-                    fcntl.lockf(fd, fcntl.LOCK_UN)
-                except OSError:
-                    pass
-                os.close(fd)
+                safe_unlock_and_close(fd)
                 continue
             else:
                 # Something else broke
+                os.close(fd)
                 raise
 
         try:
@@ -417,8 +486,7 @@ def global_mutex(base_dir: str, mutex: str) -> Iterator[None]:
             # any). This usually happens, because before someone releases a
             # lock, they delete the file. Go back and contend again. TODO: This
             # allows a lot of queue jumping on our mutex.
-            fcntl.lockf(fd, fcntl.LOCK_UN)
-            os.close(fd)
+            safe_unlock_and_close(fd)
             continue
         else:
             # We have a lock on the file that the name points to. Since we
@@ -466,8 +534,7 @@ def global_mutex(base_dir: str, mutex: str) -> Iterator[None]:
         # might have opened it before we unlinked it and will wake up when they
         # get the worthless lock on the now-unlinked file. We have to do some
         # stat gymnastics above to work around this.
-        fcntl.lockf(fd, fcntl.LOCK_UN)
-        os.close(fd)
+        safe_unlock_and_close(fd)
 
 
 class LastProcessStandingArena:
@@ -548,7 +615,7 @@ def enter(self) -> None:
             except Exception as e:
                 raise RuntimeError("Could not make lock file in " + self.lockfileDir) from e
             # Nobody can see it yet, so lock it right away
-            fcntl.lockf(self.lockfileFD, fcntl.LOCK_EX) # type: ignore
+            safe_lock(self.lockfileFD) # type: ignore
 
             # Now we're properly in, so release the global mutex
 
@@ -583,8 +650,7 @@ def leave(self) -> Iterator[bool]:
             except:
                 pass
             self.lockfileName = None
-            fcntl.lockf(self.lockfileFD, fcntl.LOCK_UN)
-            os.close(self.lockfileFD)
+            safe_unlock_and_close(self.lockfileFD)
             self.lockfileFD = None
 
             for item in os.listdir(self.lockfileDir):
@@ -598,17 +664,22 @@ def leave(self) -> Iterator[bool]:
                     continue
 
                 try:
-                    fcntl.lockf(fd, fcntl.LOCK_SH | fcntl.LOCK_NB)
+                    safe_lock(fd, block=False, shared=True)
                 except OSError as e:
-                    # Could not lock. It's alive!
-                    break
+                    if e.errno in (errno.EACCES, errno.EAGAIN):
+                        # Could not lock. It's alive!
+                        break
+                    else:
+                        # Something else is wrong
+                        os.close(fd)
+                        raise
                 else:
                     # Could lock. Process is dead.
                     try:
                         os.remove(full_path)
                     except:
                         pass
-                    fcntl.lockf(fd, fcntl.LOCK_UN)
+                    safe_unlock_and_close(fd)
                     # Continue with the loop normally.
             else:
                 # Nothing alive was found. Nobody will come in while we hold


Test patch: