Repository: cvmfs/cvmfs
Pull request #3796: Avoid use of fusermount3 by premounting & remove fuse3 dependency
Author: login: DrDaveD
Created: 2025-03-24T18:39:39Z | Merged: 2025-05-05T21:37:11Z

- Fixes #3795

Linked issue #3795: Remove dependency on fusermount3 and the fuse3 package
As noted in https://github.com/cvmfs/cvmfs/pull/3791#issuecomment-2744094524, when cvmfs2 is running as root with a uid= option, it unnecessarily invokes fusermount3.  This can be avoided by doing the mount before changing the effective uid and then invoking the fuse library with a "/dev/fd/N" string in place of the mountpoint, where N is the number of a "/dev/fuse" file descriptor that was passed to a mount of fuse at the mountpoint.

This should also remove the need to depend on the fuse3 package.  It requires fuse3-libs >= 3.3.0, which is available in all currently supported common operating systems. It will also remove the need for a fusermount3 apparmor profile including /cvmfs for Ubuntu 24.10+ as done in #3791.
Linked issue comments: user: login: vvolkl
created_at: 2025-03-24T14:36:53Z
body: This is a very nice catch and sounds like a good plan! It might take me a bit, but I'll try it out and put in a patch.
author_association: CONTRIBUTOR
user: login: DrDaveD
created_at: 2025-03-24T14:55:01Z
body: I'll submit a PR for that because I have example programs on how it's done.  I already wrote some code for it and need to compile & test.
author_association: CONTRIBUTOR

Diff:
diff --git a/cvmfs/loader.cc b/cvmfs/loader.cc
index a3205e8788..916b851753 100644
--- a/cvmfs/loader.cc
+++ b/cvmfs/loader.cc
@@ -439,6 +439,24 @@ static fuse_args *ParseCmdLine(int argc, char *argv[]) {
 }
 
 
+static bool MatchFuseOption(const fuse_args *mount_options, const char *opt) {
+  for (int i = 0; i < mount_options->argc; i++) {
+    char *arg = mount_options->argv[i];
+    char *p = strstr(arg, opt);
+    if (p != NULL) {
+      if (p == arg)
+        return true;
+      char c = *(p-1);
+      if ((c == ',') || (c == ' '))
+        return true;
+      if ((c == 'o') && (p >= arg+2) && (*(p-2) == '-'))
+        return true;
+    }
+  }
+  return false;
+}
+
+
 static void SetFuseOperations(struct fuse_lowlevel_ops *loader_operations) {
   memset(loader_operations, 0, sizeof(*loader_operations));
 
@@ -881,6 +899,47 @@ int FuseMain(int argc, char *argv[]) {
     }
   }
 
+#if CVMFS_USE_LIBFUSE != 2
+  int premount_fd = -1;
+  if (!premounted_ && !suid_mode_ && getuid() == 0) {
+    // If not already premounted or using suid mode, premount the fuse
+    // mountpoint before dropping privileges to avoid the need for fusermount.
+    // Requires libfuse >= 3.3.0.
+    platform_stat64 info;
+    // Need to know if it is a directory or not
+    if (platform_stat(mount_point_->c_str(), &info) != 0) { LogCvmfs(kLogCvmfs, kLogStderr | kLogSyslogErr,
+               "Failed to stat mountpoint %s (%d)",
+               mount_point_->c_str(), errno);
+      return kFailPermission;
+    }
+    premount_fd = open("/dev/fuse", O_RDWR);
+    if (premount_fd == -1) {
+      LogCvmfs(kLogCvmfs, kLogStderr | kLogSyslogErr,
+               "Failed to open /dev/fuse (%d)", errno);
+      return kFailPermission;
+    }
+    char opts[128];
+    snprintf(opts, sizeof(opts),
+      "fd=%i,rootmode=%o,user_id=0,group_id=0%s%s",
+      premount_fd, info.st_mode & S_IFMT,
+      MatchFuseOption(mount_options, "default_permissions") ?
+                                     ",default_permissions" : "",
+      MatchFuseOption(mount_options, "allow_other") ? ",allow_other" : "");
+    unsigned long flags = MS_NOSUID | MS_NODEV | MS_RELATIME;
+    if (!MatchFuseOption(mount_options, "rw")) {
+      flags |= MS_RDONLY;
+    }
+    if (mount("cvmfs2", mount_point_->c_str(), "fuse", flags, opts) == -1) {
+      LogCvmfs(kLogCvmfs, kLogStderr | kLogSyslogErr,
+               "Failed to mount -t fuse -o %s cvmfs2 %s (%d)",
+               opts, mount_point_->c_str(), errno);
+      return kFailPermission;
+    }
+  }
+#endif
+
+  int fd_mountinfo = -1; // needs to be declared before start using goto
+
   // Drop credentials
   if ((uid_ != 0) || (gid_ != 0)) {
     LogCvmfs(kLogCvmfs, kLogStdout, "CernVM-FS: running with credentials %d:%d",
@@ -889,7 +948,8 @@ int FuseMain(int argc, char *argv[]) {
     if (!SwitchCredentials(uid_, gid_, retrievable)) {
       LogCvmfs(kLogCvmfs, kLogStderr | kLogSyslogErr,
                "Failed to drop credentials");
-      return kFailPermission;
+      retval = kFailPermission;
+      goto cleanup;
     }
   }
   if (disable_watchdog_) {
@@ -919,7 +979,8 @@ int FuseMain(int argc, char *argv[]) {
     LogCvmfs(kLogCvmfs, kLogStderr | kLogSyslogErr,
              "CernVM-FS: ACL support requested but not available in this "
              "version of libfuse");
-    return kFailPermission;
+    retval = kFailPermission;
+    goto cleanup;
   }
 #endif
 
@@ -932,7 +993,8 @@ int FuseMain(int argc, char *argv[]) {
   if (!retval) {
     LogCvmfs(kLogCvmfs, kLogStderr | kLogSyslogErr,
              "Failed to initialize loader socket");
-    return kFailLoaderTalk;
+    retval = kFailLoaderTalk;
+    goto cleanup;
   }
 
   // TODO(jblomer): we probably want to apply a default setting related to the
@@ -978,7 +1040,8 @@ int FuseMain(int argc, char *argv[]) {
            "CernVM-FS: loading Fuse module... ");
   cvmfs_exports_ = LoadLibrary(debug_mode_, loader_exports_);
   if (!cvmfs_exports_) {
-    return kFailLoadLibrary;
+    retval = kFailLoadLibrary;
+    goto cleanup;
   }
   retval = cvmfs_exports_->fnInit(loader_exports_);
   if (retval != kFailOk) {
@@ -993,7 +1056,7 @@ int FuseMain(int argc, char *argv[]) {
              cvmfs_exports_->fnGetErrorMsg().c_str(),
              retval, Code2Ascii((Failures)retval));
     cvmfs_exports_->fnFini();
-    return retval;
+    goto cleanup;
   }
   LogCvmfs(kLogCvmfs, kLogStdout, "done");
 
@@ -1006,7 +1069,8 @@ int FuseMain(int argc, char *argv[]) {
       LogCvmfs(kLogCvmfs, kLogStderr | kLogSyslogErr,
                "failed to re-gain root permissions for mounting");
       cvmfs_exports_->fnFini();
-      return kFailPermission;
+      retval = kFailPermission;
+      goto cleanup;
     }
   }
 
@@ -1024,7 +1088,8 @@ int FuseMain(int argc, char *argv[]) {
     LogCvmfs(kLogCvmfs, kLogStderr | kLogSyslogErr,
              "failed to create Fuse channel");
     cvmfs_exports_->fnFini();
-    return kFailMount;
+    retval = kFailMount;
+    goto cleanup;
   }
 
   session = fuse_lowlevel_new(mount_options, &loader_operations,
@@ -1034,7 +1099,8 @@ int FuseMain(int argc, char *argv[]) {
              "failed to create Fuse session");
     fuse_unmount(mount_point_->c_str(), channel);
     cvmfs_exports_->fnFini();
-    return kFailMount;
+    retval = kFailMount;
+    goto cleanup;
   }
 #else
   // libfuse3
@@ -1044,14 +1110,22 @@ int FuseMain(int argc, char *argv[]) {
     LogCvmfs(kLogCvmfs, kLogStderr | kLogSyslogErr,
              "failed to create Fuse session");
     cvmfs_exports_->fnFini();
-    return kFailMount;
+    retval = kFailMount;
+    goto cleanup;
+  }
+  if (premount_fd >= 0) {
+    char premount_str[64];
+    snprintf(premount_str, sizeof(premount_str), "/dev/fd/%d", premount_fd);
+    retval = fuse_session_mount(session, premount_str);
+  } else {
+    retval = fuse_session_mount(session, mount_point_->c_str());
   }
-  retval = fuse_session_mount(session, mount_point_->c_str());
   if (retval != 0) {
     LogCvmfs(kLogCvmfs, kLogStderr | kLogSyslogErr,
              "failed to mount file system");
     cvmfs_exports_->fnFini();
-    return kFailMount;
+    retval = kFailMount;
+    goto cleanup;
   }
 #endif
 
@@ -1062,12 +1136,13 @@ int FuseMain(int argc, char *argv[]) {
       LogCvmfs(kLogCvmfs, kLogStderr | kLogSyslogErr,
                "failed to drop permissions after mounting");
       cvmfs_exports_->fnFini();
-      return kFailPermission;
+      retval = kFailPermission;
+      goto cleanup;
     }
   }
 
   // Determine device id
-  int fd_mountinfo = open("/proc/self/mountinfo", O_RDONLY);
+  fd_mountinfo = open("/proc/self/mountinfo", O_RDONLY);
   if (fd_mountinfo > 0) {
     std::string line;
     while (GetLineFd(fd_mountinfo, &line)) {
@@ -1155,9 +1230,6 @@ int FuseMain(int argc, char *argv[]) {
 
   CloseLibrary();
 
-  LogCvmfs(kLogCvmfs, kLogSyslog, "CernVM-FS: unmounted %s (%s)",
-           mount_point_->c_str(), repository_name_->c_str());
-
   delete fence_reload_;
   delete loader_exports_;
   delete config_files_;
@@ -1167,13 +1239,50 @@ int FuseMain(int argc, char *argv[]) {
   fence_reload_ = NULL;
   loader_exports_ = NULL;
   config_files_ = NULL;
+  socket_path_ = NULL;
+
+    if (retval != 0)
+      retval = kFailFuseLoop;
+    else
+      retval = kFailOk;
+
+#if CVMFS_USE_LIBFUSE != 2
+  if (premount_fd >= 0) {
+    goto cleanup;
+  }
+#endif
+
+  LogCvmfs(kLogCvmfs, kLogSyslog, "CernVM-FS: unmounted %s (%s)",
+           mount_point_->c_str(), repository_name_->c_str());
+
   repository_name_ = NULL;
   mount_point_ = NULL;
-  socket_path_ = NULL;
 
-  if (retval != 0)
-    return kFailFuseLoop;
-  return kFailOk;
+  return retval;
+
+cleanup:
+#if CVMFS_USE_LIBFUSE != 2
+  if (premount_fd >= 0) {
+    if (!SwitchCredentials(0, getgid(), true)) {
+      LogCvmfs(kLogCvmfs, kLogStderr | kLogSyslogErr,
+               "failed to re-gain root permissions for umounting");
+      retval = kFailPermission;
+    } else if (umount(mount_point_->c_str()) < 0) {
+      LogCvmfs(kLogCvmfs, kLogStderr | kLogSyslogErr,
+               "failed to umount %s (%d)", mount_point_->c_str(), errno);
+      retval = kFailPermission;
+    } else {
+      LogCvmfs(kLogCvmfs, kLogSyslog, "CernVM-FS: unmounted %s (%s)",
+               mount_point_->c_str(), repository_name_->c_str());
+    }
+    close(premount_fd);
+  }
+#endif
+
+  repository_name_ = NULL;
+  mount_point_ = NULL;
+
+  return retval;
 }
 
 
diff --git a/packaging/debian/cvmfs/control b/packaging/debian/cvmfs/control
index 5cfaba4e89..93357824d0 100644
--- a/packaging/debian/cvmfs/control
+++ b/packaging/debian/cvmfs/control
@@ -4,7 +4,7 @@ Source: cvmfs
 Section: utils
 Priority: extra
 Maintainer: Jakob Blomer <jblomer@cern.ch>
-Build-Depends: libfuse3-dev, debhelper (>= 9), autotools-dev, cmake, cpio, libcap-dev, libssl-dev, libfuse-dev, pkg-config, libattr1-dev, patch, python3-dev, python3-setuptools, unzip, uuid-dev, valgrind, libz-dev
+Build-Depends: libfuse3-dev (>= 3.3.0), debhelper (>= 9), autotools-dev, cmake, cpio, libcap-dev, libssl-dev, libfuse-dev, pkg-config, libattr1-dev, patch, python3-dev, python3-setuptools, unzip, uuid-dev, valgrind, libz-dev
 Standards-Version: 3.9.6.1
 Homepage: http://cernvm.cern.ch/portal/filesystem
 
@@ -68,7 +68,7 @@ Description: CernVM File System Unit Tests
 Package: cvmfs-fuse3
 Architecture: i386 amd64 armhf arm64
 #Pre-Depends: ${misc:Pre-Depends}   (preparation for multiarch support)
-Depends: fuse3, libfuse3-3, cvmfs (= ${binary:Version})
+Depends: libfuse3-3 (>= 3.3.0), cvmfs (= ${binary:Version})
 #Multi-Arch: same   (preparation for multiarch support)
 Homepage: http://cernvm.cern.ch
 Description: CernVM File System fuse3 libraries
diff --git a/packaging/rpm/cvmfs-universal.spec b/packaging/rpm/cvmfs-universal.spec
index ce9d42a79c..260b4d3000 100644
--- a/packaging/rpm/cvmfs-universal.spec
+++ b/packaging/rpm/cvmfs-universal.spec
@@ -103,7 +103,7 @@ BuildRequires: valgrind-devel
 BuildRequires: cmake
 BuildRequires: fuse-devel
 %if 0%{?build_fuse3}
-BuildRequires: fuse3-devel
+BuildRequires: fuse3-devel >= 3.3.0
 %endif
 BuildRequires: libattr-devel
 BuildRequires: openssl-devel
@@ -197,8 +197,7 @@ Common utility libraries for CernVM-FS packages
 Summary: additional libraries to enable libfuse3 support
 Group: Applications/System
 Requires: cvmfs = %{version}-%{release}
-Requires: fuse3
-Requires: fuse3-libs
+Requires: fuse3-libs >= 3.3.0
 %description fuse3
 Shared libraries implementing the CernVM-FS fuse module based on libfuse3
 %endif
@@ -759,6 +758,7 @@ systemctl daemon-reload
 %endif
 
 %changelog
+# - When using fuse3, require at least version 3.3.0 (for premounting).
 * Mon Mar 3 2025 Dave Dykstra <dwd@cern.ch>> - 2.12.7-2
 - Apply %pretrans transaction check to all subpackages that need to
   have their version stay in sync


Reviews:
user: login: vvolkl
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-05T07:37:01Z
user: login: vvolkl
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-05T07:38:36Z
user: login: vvolkl
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-05T07:38:52Z

Review comments:
diff_hunk: @@ -801,11 +820,11 @@ int FuseMain(int argc, char *argv[]) {
 
   // Logging
   if (options_manager->GetValue("CVMFS_SYSLOG_LEVEL", &parameter))
-    SetLogSyslogLevel(String2Uint64(parameter));
+    SetLogSyslogLevel(atoi(parameter.c_str()));
path: cvmfs/loader.cc
user: login: vvolkl
body: ```suggestion
    SetLogSyslogLevel(String2Uint64(parameter));
```
change unrelated to this pr
created_at: 2025-05-05T07:36:14Z
author_association: CONTRIBUTOR
diff_hunk: @@ -801,11 +820,11 @@ int FuseMain(int argc, char *argv[]) {
 
   // Logging
   if (options_manager->GetValue("CVMFS_SYSLOG_LEVEL", &parameter))
-    SetLogSyslogLevel(String2Uint64(parameter));
+    SetLogSyslogLevel(atoi(parameter.c_str()));
   else
     SetLogSyslogLevel(3);
   if (options_manager->GetValue("CVMFS_SYSLOG_FACILITY", &parameter))
-    SetLogSyslogFacility(String2Int64(parameter));
+    SetLogSyslogFacility(atoi(parameter.c_str()));
path: cvmfs/loader.cc
user: login: vvolkl
body: ```suggestion
    SetLogSyslogFacility(String2Int64(parameter));
```
change unrelated to this pr
created_at: 2025-05-05T07:36:57Z
author_association: CONTRIBUTOR
diff_hunk: @@ -824,7 +843,7 @@ int FuseMain(int argc, char *argv[]) {
 
   // Number of file descriptors
   if (options_manager->GetValue("CVMFS_NFILES", &parameter)) {
-    int retval = SetLimitNoFile(String2Uint64(parameter));
+    int retval = SetLimitNoFile(atoi(parameter.c_str()));
path: cvmfs/loader.cc
user: login: vvolkl
body: ```suggestion
    int retval = SetLimitNoFile(String2Uint64(parameter));
```
revert unrelated linter changes
created_at: 2025-05-05T07:38:36Z
author_association: CONTRIBUTOR
diff_hunk: @@ -402,6 +402,7 @@ static int ParseFuseOptions(void *data __attribute__((unused)), const char *arg,
   }
 }
 
+
path: cvmfs/loader.cc
user: login: vvolkl
body: ```suggestion
```
remove whitespace change
created_at: 2025-05-05T07:38:52Z
author_association: CONTRIBUTOR

PR comments:
user: login: DrDaveD
created_at: 2025-03-24T19:21:47Z
body: @cernvm-bot cloudtest
author_association: CONTRIBUTOR
user: login: cernvm-bot
created_at: 2025-03-24T19:33:03Z
body: building for cloudtests finished: SUCCESS
https://lcgapp-services.cern.ch/cvmfs-jenkins/job/CvmfsFullBuildDocker/4828/
author_association: COLLABORATOR
user: login: cernvm-bot
created_at: 2025-03-24T23:21:17Z
body: cloudtests finished: FAILURE
https://lcgapp-services.cern.ch/cvmfs-jenkins/job/CvmfsCloudTesting/2415/
author_association: COLLABORATOR
user: login: DrDaveD
created_at: 2025-03-25T18:45:33Z
body: @cernvm-bot cloudtest
author_association: CONTRIBUTOR
user: login: cernvm-bot
created_at: 2025-03-25T18:56:40Z
body: building for cloudtests finished: SUCCESS
https://lcgapp-services.cern.ch/cvmfs-jenkins/job/CvmfsFullBuildDocker/4830/
author_association: COLLABORATOR
user: login: DrDaveD
created_at: 2025-03-26T15:04:06Z
body: @vvolkl The linter complains about something that I didn't change.  The fedora40 cloudtest failed two tests for a seemingly unrelated reason, and the debian12 test is stuck.  Please review.
author_association: CONTRIBUTOR
user: login: DrDaveD
created_at: 2025-03-26T15:54:29Z
body: Oh, I did change the line that the linter is complaining about, but I only changed the return variable's scope and it is complaining about the parameter.  I didn't find a good precedent to convert a C++ string to an int instead of int64_t.  The way I will try is `atoi(parameter.c_str())`.  Please let me know if there's a better way.
author_association: CONTRIBUTOR
user: login: DrDaveD
created_at: 2025-03-26T17:54:24Z
body: This time the linter is failing earlier, unable to find some packages at security.ubuntu.com
author_association: CONTRIBUTOR
user: login: DrDaveD
created_at: 2025-03-26T18:00:16Z
body: Closing & reopening to retry cpp-linter and CI / Cloudtests both of which seem to have temporary problems.
author_association: CONTRIBUTOR
user: login: DrDaveD
created_at: 2025-03-26T20:07:01Z
body: @cernvm-bot cloudtest
author_association: CONTRIBUTOR
user: login: cernvm-bot
created_at: 2025-03-26T20:18:06Z
body: building for cloudtests finished: SUCCESS
https://lcgapp-services.cern.ch/cvmfs-jenkins/job/CvmfsFullBuildDocker/4832/
author_association: COLLABORATOR
user: login: cernvm-bot
created_at: 2025-03-27T18:56:48Z
body: cloudtests finished: FAILURE
https://lcgapp-services.cern.ch/cvmfs-jenkins/job/CvmfsCloudTesting/2416/
author_association: COLLABORATOR
user: login: cernvm-bot
created_at: 2025-03-28T20:18:13Z
body: cloudtests finished: FAILURE
https://lcgapp-services.cern.ch/cvmfs-jenkins/job/CvmfsCloudTesting/2418/
author_association: COLLABORATOR
user: login: DrDaveD
created_at: 2025-04-02T21:40:29Z
body: Closing  & reopening to re-run linter
author_association: CONTRIBUTOR
user: login: DrDaveD
created_at: 2025-04-02T21:40:49Z
body: @cernvm-bot cloudtest
author_association: CONTRIBUTOR
user: login: cernvm-bot
created_at: 2025-04-02T21:53:55Z
body: building for cloudtests finished: SUCCESS
https://lcgapp-services.cern.ch/cvmfs-jenkins/job/CvmfsFullBuildDocker/4838/
author_association: COLLABORATOR
user: login: DrDaveD
created_at: 2025-04-03T19:03:02Z
body: @cernvm-bot cloudtest
author_association: CONTRIBUTOR
user: login: cernvm-bot
created_at: 2025-04-03T19:14:23Z
body: building for cloudtests finished: SUCCESS
https://lcgapp-services.cern.ch/cvmfs-jenkins/job/CvmfsFullBuildDocker/4840/
author_association: COLLABORATOR
user: login: DrDaveD
created_at: 2025-04-03T20:48:32Z
body: Closing & reopening to retry failed cloudtest 098-telemetry (the test works on my test machine).
author_association: CONTRIBUTOR
user: login: DrDaveD
created_at: 2025-04-04T19:56:34Z
body: @jblomer The CI / Cloudtests are failing in a very obscure way, can you give any insight?  I tried reproducing the github actions environment as closely as possible, using an Ubuntu 24.04 VM and executing the same docker commands as github actions, and yet I cannot get the 098-telemetry test to fail.  The actions log shows
```
-- Testing Telemetry Aggregator: Influx (Thu Apr  3 23:16:27 CEST 2025 / test number 098)
# Created by CVMFS integration tests
CVMFS_SERVER_APACHE_RELOAD_IS_RESTART=true
create temporary directory
tmpfile is ./test098.log.chwE7hLz
Check detection of counter change
   Mount lhcb.cern.ch
   UDP Listener: wait for 1 package from lhcb.cern.ch
   Do something to change counters
   UDP Listener: Wait for another package from lhcb.cern.ch
   Umount lhcb.cern.ch
   Check output in file ./test098.log.chwE7hLz
      Right number of lines?
      Changes registered?
Check results
Static value
   1 unique value in absolute
   delta value must be 0
Dynamic value
   2 unique values in absolute
   delta value must NOT be 0
cleanup Telemetry Aggregator: Influx


Log content:
cvmfs_client_counters_absolute,repo=lhcb.cern.ch cache.n_certificate_misses=1,catalog_mgr.n_listing=2,catalog_mgr.n_lookup_path=14,catalog_mgr.n_nested_listing=2,catalog_mgr.n_write_lock=1,catalog_mgr.ns_write_lock=351,catalog_revision=102899,cvmfs.n_fs_dir_open=2,cvmfs.n_fs_stat=2,cvmfs.n_fs_statfs=1,download.n_requests=5,download.sz_transfer_time=7998,download.sz_transferred_bytes=16333,fetch.n_downloads=1,fetch.n_invocations=1,inode_cache.n_hit=3,inode_cache.n_insert=1,inode_cache.n_miss=1,inode_cache.sz_allocated=1663672,inode_cache.sz_size=5568,inode_tracker.n_hit_inode=26,md5_path_cache.n_hit=13,md5_path_cache.n_insert=13,md5_path_cache.n_miss=13,md5_path_cache.sz_allocated=13186784,md5_path_cache.sz_size=39232,path_cache.n_miss=2,path_cache.sz_allocated=1901240,path_cache.sz_size=5568,sqlite.n_read=30,sqlite.no_open=1,sqlite.sz_read=29796 1743715001873499904

cvmfs_client_counters_absolute,repo=lhcb.cern.ch cache.n_certificate_misses=1,catalog_mgr.n_listing=3,catalog_mgr.n_lookup_path=20,catalog_mgr.n_nested_listing=7,catalog_mgr.n_write_lock=2,catalog_mgr.ns_write_lock=802,catalog_revision=102899,cvmfs.n_fs_dir_open=3,cvmfs.n_fs_lookup=4,cvmfs.n_fs_open=1,cvmfs.n_fs_read=1,cvmfs.n_fs_readlink=1,cvmfs.n_fs_stat=2,cvmfs.n_fs_statfs=1,download.n_requests=7,download.sz_transfer_time=18239,download.sz_transferred_bytes=21271,fetch.n_downloads=3,fetch.n_invocations=4,inode_cache.n_hit=4,inode_cache.n_insert=3,inode_cache.n_miss=3,inode_cache.sz_allocated=1663672,inode_cache.sz_size=5568,inode_tracker.n_hit_inode=43,inode_tracker.n_hit_path=5,inode_tracker.n_insert=4,inode_tracker.no_reference=4,md5_path_cache.n_hit=28,md5_path_cache.n_insert=15,md5_path_cache.n_miss=15,md5_path_cache.sz_allocated=13186784,md5_path_cache.sz_size=39232,page_cache_tracker.n_insert=1,page_cache_tracker.n_open_cached=1,path_cache.n_insert=3,path_cache.n_miss=8,path_cache.sz_allocated=1901240,path_cache.sz_size=5568,sqlite.n_read=43,sqlite.no_open=2,sqlite.sz_read=42184 1743715016879390976
cvmfs_client_counters_delta,repo=lhcb.cern.ch cache.n_certificate_misses=0,catalog_mgr.n_listing=0,catalog_mgr.n_lookup_path=3,catalog_mgr.n_nested_listing=2,catalog_mgr.n_write_lock=1,catalog_mgr.ns_write_lock=451,catalog_revision=0,cvmfs.n_fs_dir_open=0,cvmfs.n_fs_lookup=0,cvmfs.n_fs_open=1,cvmfs.n_fs_read=1,cvmfs.n_fs_readlink=0,cvmfs.n_fs_stat=0,cvmfs.n_fs_statfs=0,download.n_requests=2,download.sz_transfer_time=10241,download.sz_transferred_bytes=4938,fetch.n_downloads=1,fetch.n_invocations=2,inode_cache.n_hit=0,inode_cache.n_insert=1,inode_cache.n_miss=1,inode_cache.sz_allocated=0,inode_cache.sz_size=0,inode_tracker.n_hit_inode=0,inode_tracker.n_hit_path=2,inode_tracker.n_insert=1,inode_tracker.no_reference=1,md5_path_cache.n_hit=0,md5_path_cache.n_insert=1,md5_path_cache.n_miss=0,md5_path_cache.sz_allocated=0,md5_path_cache.sz_size=0,page_cache_tracker.n_insert=1,page_cache_tracker.n_open_cached=1,path_cache.n_insert=1,path_cache.n_miss=1,path_cache.sz_allocated=0,path_cache.sz_size=0,sqlite.n_read=13,sqlite.no_open=1,sqlite.sz_read=12388 1743715016879390976

execution took 29.284 seconds
Testcase failed with RETVAL 22
```
and test 22 is
```
  local check_delta_dynamic=$(grep $CVMFS_TEST_098_TMPFILE -e "_delta"\
                      | awk '{split($0,a,"md5_path_cache.n_miss="); split(a[2],b,","); print b[1]}')
...
  [ "$check_delta_dynamic" -eq "0" ] && return 22
```
So indeed, `cvmfs_client_counters_delta...md5_path_cache.n_miss=0` and the test says it must not be.  What does that mean, how important is it, and how could what I changed here affect that?

It failed 3 times with this error, action run [550](https://github.com/cvmfs/cvmfs/actions/runs/14230155517), [554](https://github.com/cvmfs/cvmfs/actions/runs/14251062939), and [555](https://github.com/cvmfs/cvmfs/actions/runs/14252501902), but succeeded in [551](https://github.com/cvmfs/cvmfs/actions/runs/14230385662) with a value of 2 instead of 0.
author_association: CONTRIBUTOR
user: login: cernvm-bot
created_at: 2025-04-04T21:54:02Z
body: cloudtests finished: FAILURE
https://lcgapp-services.cern.ch/cvmfs-jenkins/job/CvmfsCloudTesting/2422/
author_association: COLLABORATOR
user: login: cernvm-bot
created_at: 2025-04-05T19:14:30Z
body: cloudtests finished: FAILURE
https://lcgapp-services.cern.ch/cvmfs-jenkins/job/CvmfsCloudTesting/2423/
author_association: COLLABORATOR
user: login: DrDaveD
created_at: 2025-04-08T21:50:32Z
body: Closing & reopening to try ci / cloudtests again.
author_association: CONTRIBUTOR
user: login: DrDaveD
created_at: 2025-04-08T21:54:44Z
body: @cernvm-bot cloudtest
author_association: CONTRIBUTOR
user: login: cernvm-bot
created_at: 2025-04-08T22:06:19Z
body: building for cloudtests finished: SUCCESS
https://lcgapp-services.cern.ch/cvmfs-jenkins/job/CvmfsFullBuildDocker/4842/
author_association: COLLABORATOR
user: login: cernvm-bot
created_at: 2025-04-10T22:06:29Z
body: cloudtests finished: FAILURE
https://lcgapp-services.cern.ch/cvmfs-jenkins/job/CvmfsCloudTesting/2424/
author_association: COLLABORATOR
user: login: vvolkl
created_at: 2025-04-22T14:42:42Z
body: @cernvm-bot cloudtest
author_association: CONTRIBUTOR
user: login: cernvm-bot
created_at: 2025-04-22T14:54:04Z
body: building for cloudtests finished: SUCCESS
https://lcgapp-services.cern.ch/cvmfs-jenkins/job/CvmfsFullBuildDocker/4845/
author_association: COLLABORATOR
user: login: vvolkl
created_at: 2025-04-23T07:42:53Z
body: @cernvm-bot cloudtest
author_association: CONTRIBUTOR
user: login: cernvm-bot
created_at: 2025-04-23T07:54:17Z
body: building for cloudtests finished: SUCCESS
https://lcgapp-services.cern.ch/cvmfs-jenkins/job/CvmfsFullBuildDocker/4847/
author_association: COLLABORATOR
user: login: cernvm-bot
created_at: 2025-04-23T09:54:09Z
body: cloudtests finished: FAILURE
https://lcgapp-services.cern.ch/cvmfs-jenkins/job/CvmfsCloudTesting/2425/
author_association: COLLABORATOR
user: login: cernvm-bot
created_at: 2025-04-25T07:54:24Z
body: cloudtests finished: FAILURE
https://lcgapp-services.cern.ch/cvmfs-jenkins/job/CvmfsCloudTesting/2427/
author_association: COLLABORATOR
user: login: vvolkl
created_at: 2025-05-05T07:39:51Z
body: @cernvm-bot cloudtest
author_association: CONTRIBUTOR
user: login: cernvm-bot
created_at: 2025-05-05T07:51:01Z
body: building for cloudtests finished: SUCCESS
https://lcgapp-services.cern.ch/cvmfs-jenkins/job/CvmfsFullBuildDocker/4880/
author_association: COLLABORATOR
user: login: vvolkl
created_at: 2025-05-05T12:56:33Z
body: @cernvm-bot cloudtest
author_association: CONTRIBUTOR
user: login: cernvm-bot
created_at: 2025-05-05T13:14:40Z
body: building for cloudtests finished: SUCCESS
https://lcgapp-services.cern.ch/cvmfs-jenkins/job/CvmfsFullBuildDocker/4884/
author_association: COLLABORATOR
user: login: cernvm-bot
created_at: 2025-05-05T13:15:14Z
body: cloudtests finished: FAILURE
https://lcgapp-services.cern.ch/cvmfs-jenkins/job/CvmfsCloudTesting/2448/
author_association: COLLABORATOR
user: login: vvolkl
created_at: 2025-05-05T15:55:45Z
body: @cernvm-bot cloudtest
author_association: CONTRIBUTOR
user: login: cernvm-bot
created_at: 2025-05-05T16:05:10Z
body: building for cloudtests finished: SUCCESS
https://lcgapp-services.cern.ch/cvmfs-jenkins/job/CvmfsFullBuildDocker/4885/
author_association: COLLABORATOR
user: login: cernvm-bot
created_at: 2025-05-05T18:09:44Z
body: cloudtests finished: FAILURE
https://lcgapp-services.cern.ch/cvmfs-jenkins/job/CvmfsCloudTesting/2444/
author_association: COLLABORATOR
user: login: vvolkl
created_at: 2025-05-05T18:26:28Z
body: @cernvm-bot cloudtest
author_association: CONTRIBUTOR
user: login: cernvm-bot
created_at: 2025-05-05T18:35:52Z
body: building for cloudtests finished: SUCCESS
https://lcgapp-services.cern.ch/cvmfs-jenkins/job/CvmfsFullBuildDocker/4886/
author_association: COLLABORATOR
user: login: vvolkl
created_at: 2025-05-05T21:35:52Z
body: Testfailures unrelated, LGTM
author_association: CONTRIBUTOR
user: login: cernvm-bot
created_at: 2025-05-06T09:31:37Z
body: cloudtests finished: FAILURE
https://lcgapp-services.cern.ch/cvmfs-jenkins/job/CvmfsCloudTesting/2450/
author_association: COLLABORATOR
user: login: cernvm-bot
created_at: 2025-05-06T09:33:45Z
body: cloudtests finished: FAILURE
https://lcgapp-services.cern.ch/cvmfs-jenkins/job/CvmfsCloudTesting/2449/
author_association: COLLABORATOR