Repository: systemd/systemd
Pull request #35708: Ratelimit attempts to open watchdog, increase logging
Author: login: keszybz
Labels: name: util-lib
name: tests
name: ci-failure-appears-unrelated
Created: 2024-12-20T18:31:02Z | Merged: 2025-03-24T20:06:58Z



Diff:
diff --git a/src/core/main.c b/src/core/main.c
index bca6c96b4e115..23ce9aa92cfd3 100644
--- a/src/core/main.c
+++ b/src/core/main.c
@@ -1985,9 +1985,9 @@ static int do_reexecute(
         assert(saved_rlimit_memlock);
         assert(ret_error_message);
 
-        /* Close and disarm the watchdog, so that the new instance can reinitialize it, but doesn't get
-         * rebooted while we do that */
-        watchdog_close(true);
+        /* Close and disarm the watchdog, so that the new instance can reinitialize it, but the machine
+         * doesn't get rebooted while we do that. */
+        watchdog_close(/* disarm= */ true);
 
         if (!switch_root_dir && objective == MANAGER_SOFT_REBOOT) {
                 /* If no switch root dir is specified, then check if /run/nextroot/ qualifies and use that */
diff --git a/src/core/manager.c b/src/core/manager.c
index c6c9779976375..afccb48d7edda 100644
--- a/src/core/manager.c
+++ b/src/core/manager.c
@@ -3352,14 +3352,14 @@ int manager_loop(Manager *m) {
 
         while (m->objective == MANAGER_OK) {
 
-                (void) watchdog_ping();
-
                 if (!ratelimit_below(&rl)) {
                         /* Yay, something is going seriously wrong, pause a little */
                         log_warning("Looping too fast. Throttling execution a little.");
                         sleep(1);
                 }
 
+                (void) watchdog_ping();
+
                 if (manager_dispatch_load_queue(m) > 0)
                         continue;
 
@@ -3391,7 +3391,7 @@ int manager_loop(Manager *m) {
                         continue;
 
                 /* Sleep for watchdog runtime wait time */
-                r = sd_event_run(m->event, watchdog_runtime_wait());
+                r = sd_event_run(m->event, watchdog_runtime_wait(/* divisor= */ 2));
                 if (r < 0)
                         return log_error_errno(r, "Failed to run event loop: %m");
         }
@@ -3582,9 +3582,6 @@ void manager_set_watchdog(Manager *m, WatchdogType t, usec_t timeout) {
         if (MANAGER_IS_USER(m))
                 return;
 
-        if (m->watchdog[t] == timeout)
-                return;
-
         if (m->watchdog_overridden[t] == USEC_INFINITY) {
                 if (t == WATCHDOG_RUNTIME)
                         (void) watchdog_setup(timeout);
@@ -3603,9 +3600,6 @@ void manager_override_watchdog(Manager *m, WatchdogType t, usec_t timeout) {
         if (MANAGER_IS_USER(m))
                 return;
 
-        if (m->watchdog_overridden[t] == timeout)
-                return;
-
         usec = timeout == USEC_INFINITY ? m->watchdog[t] : timeout;
         if (t == WATCHDOG_RUNTIME)
                 (void) watchdog_setup(usec);
@@ -3877,6 +3871,9 @@ static void manager_notify_finished(Manager *m) {
 
         bus_manager_send_finished(m, firmware_usec, loader_usec, kernel_usec, initrd_usec, userspace_usec, total_usec);
 
+        if (MANAGER_IS_SYSTEM(m) && detect_container() <= 0)
+                watchdog_report_if_missing();
+
         log_taint_string(m);
 }
 
diff --git a/src/shared/watchdog.c b/src/shared/watchdog.c
index edf20edcf9661..5b3f950b80e56 100644
--- a/src/shared/watchdog.c
+++ b/src/shared/watchdog.c
@@ -7,6 +7,8 @@
 #include <unistd.h>
 #include <linux/watchdog.h>
 
+#include "sd-messages.h"
+
 #include "devnum-util.h"
 #include "errno-util.h"
 #include "fd-util.h"
@@ -21,7 +23,9 @@ static int watchdog_fd = -EBADF;
 static char *watchdog_device = NULL;
 static usec_t watchdog_timeout = 0; /* 0 → close device and USEC_INFINITY → don't change timeout */
 static usec_t watchdog_pretimeout = 0; /* 0 → disable pretimeout and USEC_INFINITY → don't change pretimeout */
-static usec_t watchdog_last_ping = USEC_INFINITY;
+static usec_t watchdog_last_good_ping = USEC_INFINITY;
+static usec_t watchdog_last_bad_ping = USEC_INFINITY;
+static unsigned watchdog_bad_pings = 0;
 static bool watchdog_supports_pretimeout = false; /* Depends on kernel state that might change at runtime */
 static char *watchdog_pretimeout_governor = NULL;
 
@@ -34,6 +38,11 @@ static char *watchdog_pretimeout_governor = NULL;
  */
 #define WATCHDOG_TIMEOUT_MAX_SEC (CONST_MIN(UINT_MAX/1000U, (unsigned)INT_MAX))
 
+/* How many times to ping the watchdog in a single burst. */
+#define WATCHDOG_PING_BURST 3
+/* How many times to ping the watchdog in total before giving up on it. */
+#define WATCHDOG_MAX_FAILED_PINGS 15
+
 #define WATCHDOG_GOV_NAME_MAXLEN 20 /* From the kernel watchdog driver */
 
 static int saturated_usec_to_sec(usec_t val) {
@@ -191,18 +200,44 @@ static int watchdog_set_pretimeout(void) {
 }
 
 usec_t watchdog_get_last_ping(clockid_t clock) {
-        return map_clock_usec(watchdog_last_ping, CLOCK_BOOTTIME, clock);
+        return map_clock_usec(watchdog_last_good_ping, CLOCK_BOOTTIME, clock);
 }
 
 static int watchdog_ping_now(void) {
-        assert(watchdog_fd >= 0);
+        int r;
 
-        if (ioctl(watchdog_fd, WDIOC_KEEPALIVE, 0) < 0)
-                return log_warning_errno(errno, "Failed to ping hardware watchdog, ignoring: %m");
+        assert(watchdog_fd >= 0);
 
-        watchdog_last_ping = now(CLOCK_BOOTTIME);
+        r = RET_NERRNO(ioctl(watchdog_fd, WDIOC_KEEPALIVE, 0));
+        if (r < 0) {
+                watchdog_last_bad_ping = now(CLOCK_BOOTTIME);
+                if (++ watchdog_bad_pings >= WATCHDOG_MAX_FAILED_PINGS) {
+                        log_struct_errno(LOG_ERR, r,
+                                         LOG_MESSAGE("Failed to ping hardware watchdog %s, closing watchdog after %u attempts: %m",
+                                                     watchdog_device, watchdog_bad_pings),
+                                         "MESSAGE_ID=" SD_MESSAGE_WATCHDOG_PING_FAILED_STR,
+                                         "WATCHDOG_DEVICE=%s", watchdog_device);
+                        watchdog_timeout = USEC_INFINITY;
+                        watchdog_close(/* disarm= */ true);
+                } else
+                        log_struct_errno(LOG_WARNING, r,
+                                         LOG_MESSAGE("Failed to ping hardware watchdog %s: %m",
+                                                     watchdog_device),
+                                         "MESSAGE_ID=" SD_MESSAGE_WATCHDOG_PING_FAILED_STR,
+                                         "WATCHDOG_DEVICE=%s", watchdog_device);
+        } else {
+                watchdog_last_good_ping = now(CLOCK_BOOTTIME);
+                if (watchdog_bad_pings > 0) {
+                        log_struct(LOG_INFO,
+                                   LOG_MESSAGE("Succesfullly pinged hardware watchdog %s after %u attempts: %m",
+                                               watchdog_device, watchdog_bad_pings + 1),
+                                   "WATCHDOG_DEVICE=%s", watchdog_device);
+                        watchdog_bad_pings = 0;
+                        watchdog_last_bad_ping = USEC_INFINITY;
+                }
+        }
 
-        return 0;
+        return r;
 }
 
 static int watchdog_update_pretimeout(void) {
@@ -307,13 +342,20 @@ static int watchdog_update_timeout(void) {
         return watchdog_ping_now();
 }
 
-static int watchdog_open(void) {
+static int watchdog_open(bool ignore_ratelimit) {
+        static RateLimit watchdog_open_ratelimit = { 0, 1 }; /* The interval is initialized dynamically below. */
         struct watchdog_info ident;
         char **try_order;
         int r;
 
-        if (watchdog_fd >= 0)
-                return 0;
+        assert(watchdog_fd < 0);
+        assert(watchdog_timeout > 0);
+
+        watchdog_open_ratelimit.interval = MIN(watchdog_timeout / 2, 5 * USEC_PER_SEC);
+
+        if (!ratelimit_below(&watchdog_open_ratelimit))
+                if (!ignore_ratelimit)  /* If ignore_ratelimit, we update the timestamp, but continue. */
+                        return -EWOULDBLOCK;
 
         /* Let's prefer new-style /dev/watchdog0 (i.e. kernel 3.5+) over classic /dev/watchdog. The former
          * has the benefit that we can easily find the matching directory in sysfs from it, as the relevant
@@ -323,7 +365,7 @@ static int watchdog_open(void) {
                 STRV_MAKE("/dev/watchdog0", "/dev/watchdog") : STRV_MAKE(watchdog_device);
 
         STRV_FOREACH(wd, try_order) {
-                watchdog_fd = open(*wd, O_WRONLY|O_CLOEXEC);
+                watchdog_fd = RET_NERRNO(open(*wd, O_WRONLY|O_CLOEXEC));
                 if (watchdog_fd >= 0) {
                         if (free_and_strdup(&watchdog_device, *wd) < 0) {
                                 r = log_oom_debug();
@@ -333,22 +375,37 @@ static int watchdog_open(void) {
                         break;
                 }
 
-                if (errno != ENOENT)
-                        return log_debug_errno(errno, "Failed to open watchdog device %s: %m", *wd);
+                if (watchdog_fd != -ENOENT)
+                        return log_struct_errno(LOG_WARNING, watchdog_fd,
+                                                LOG_MESSAGE("Failed to open watchdog device %s: %m", *wd),
+                                                "MESSAGE_ID=" SD_MESSAGE_WATCHDOG_OPEN_FAILED_STR,
+                                                "WATCHDOG_DEVICE=%s", *wd);
         }
 
         if (watchdog_fd < 0)
-                return log_debug_errno(SYNTHETIC_ERRNO(ENOENT), "Failed to open watchdog device %s.", watchdog_device ?: "auto");
+                return log_debug_errno(watchdog_fd, "Failed to open %swatchdog device%s%s.",
+                                       watchdog_device ? "" : "any ",
+                                       watchdog_device ? " " : "",
+                                       strempty(watchdog_device));
 
-        watchdog_last_ping = USEC_INFINITY;
+        watchdog_last_good_ping = watchdog_last_bad_ping = USEC_INFINITY;
+        watchdog_bad_pings = 0;
 
-        if (ioctl(watchdog_fd, WDIOC_GETSUPPORT, &ident) < 0)
-                log_debug_errno(errno, "Hardware watchdog %s does not support WDIOC_GETSUPPORT ioctl, ignoring: %m", watchdog_device);
+        r = RET_NERRNO(ioctl(watchdog_fd, WDIOC_GETSUPPORT, &ident));
+        if (r < 0)
+                log_struct_errno(LOG_INFO, r,
+                                 LOG_MESSAGE("Using hardware watchdog %s, no support for WDIOC_GETSUPPORT ioctl: %m",
+                                             watchdog_device),
+                                 "MESSAGE_ID=" SD_MESSAGE_WATCHDOG_OPENED_STR,
+                                 "WATCHDOG_DEVICE=%s", watchdog_device);
         else
-                log_info("Using hardware watchdog '%s', version %x, device %s",
-                         ident.identity,
-                         ident.firmware_version,
-                         watchdog_device);
+                log_struct(LOG_INFO,
+                           LOG_MESSAGE("Using hardware watchdog %s: '%s', version %x.",
+                                       watchdog_device,
+                                       ident.identity,
+                                       ident.firmware_version),
+                           "MESSAGE_ID=" SD_MESSAGE_WATCHDOG_OPENED_STR,
+                           "WATCHDOG_DEVICE=%s", watchdog_device);
 
         r = watchdog_update_timeout();
         if (r < 0)
@@ -384,7 +441,7 @@ int watchdog_setup(usec_t timeout) {
          * it's a nop if the device is already opened). */
 
         if (timeout == 0) {
-                watchdog_close(true);
+                watchdog_close(/* disarm= */ true);
                 return 0;
         }
 
@@ -398,7 +455,7 @@ int watchdog_setup(usec_t timeout) {
         watchdog_timeout = timeout;
 
         if (watchdog_fd < 0)
-                return watchdog_open();
+                return watchdog_open(/* ignore_ratelimit= */ false);
 
         r = watchdog_update_timeout();
         if (r < 0)
@@ -428,6 +485,10 @@ int watchdog_setup_pretimeout_governor(const char *governor) {
 }
 
 static usec_t watchdog_calc_timeout(void) {
+        /* If we failed to many times, don't schedule the next attempt at all. */
+        if (watchdog_bad_pings >= WATCHDOG_MAX_FAILED_PINGS)
+                return 0;
+
         /* Calculate the effective timeout which accounts for the watchdog
          * pretimeout if configured and supported. */
         if (watchdog_supports_pretimeout && timestamp_is_set(watchdog_pretimeout) && watchdog_timeout >= watchdog_pretimeout)
@@ -436,46 +497,71 @@ static usec_t watchdog_calc_timeout(void) {
                 return watchdog_timeout;
 }
 
-usec_t watchdog_runtime_wait(void) {
+usec_t watchdog_runtime_wait(unsigned divisor) {
         usec_t timeout = watchdog_calc_timeout();
         if (!timestamp_is_set(timeout))
                 return USEC_INFINITY;
 
-        /* Sleep half the watchdog timeout since the last successful ping at most */
-        if (timestamp_is_set(watchdog_last_ping)) {
+        /* Sleep watchdog timeout / divisor since the last successful ping attempt at most,
+         * or the fifth of that if the ping failed. */
+
+        usec_t ts = MAX(timestamp_is_set(watchdog_last_good_ping) ? watchdog_last_good_ping : 0u,
+                        timestamp_is_set(watchdog_last_bad_ping) ? watchdog_last_bad_ping : 0u);
+        if (ts > 0) {
                 usec_t ntime = now(CLOCK_BOOTTIME);
+                if (ts == watchdog_last_bad_ping)
+                        divisor *= 5;
 
-                assert(ntime >= watchdog_last_ping);
-                return usec_sub_unsigned(watchdog_last_ping + (timeout / 2), ntime);
+                assert(ntime >= ts);
+                return usec_sub_unsigned(ts + timeout / divisor, ntime);
         }
 
-        return timeout / 2;
+        return timeout / divisor;
 }
 
 int watchdog_ping(void) {
-        usec_t ntime, timeout;
-
         if (watchdog_timeout == 0)
                 return 0;
 
         if (watchdog_fd < 0)
                 /* open_watchdog() will automatically ping the device for us if necessary */
-                return watchdog_open();
+                return watchdog_open(/* ignore_ratelimit= */ false);
 
-        ntime = now(CLOCK_BOOTTIME);
-        timeout = watchdog_calc_timeout();
-
-        /* Never ping earlier than watchdog_timeout/4 and try to ping
-         * by watchdog_timeout/2 plus scheduling latencies at the latest */
-        if (timestamp_is_set(watchdog_last_ping)) {
-                assert(ntime >= watchdog_last_ping);
-                if ((ntime - watchdog_last_ping) < (timeout / 4))
-                        return 0;
-        }
+        /* Ping approximately watchdog_timeout/4 after a successful ping, or even less than that
+         * after an unsuccessful ping. */
+        if (watchdog_runtime_wait(/* divisor= */ 4) > 0)
+                return 0;
 
         return watchdog_ping_now();
 }
 
+void watchdog_report_if_missing(void) {
+        /* If we failed to open the watchdog because the device doesn't exist, report why. If we successfully
+         * opened a device, or opening failed for a different reason, we logged then. But ENOENT failure may
+         * be transient, for example because modules being loaded or the hardware being detected. This
+         * function can be called to log after things have settled down.
+         *
+         * If a device was specified explicitly, raise level. */
+
+        if (watchdog_fd != -ENOENT)
+                return;
+
+        /* We attempt to open the watchdog one last time here, so that if we log, we log accurately. */
+        if (watchdog_open(/* ignore_ratelimit= */ true) >= 0)
+                return;
+
+        if (watchdog_device)
+                log_struct_errno(LOG_WARNING, watchdog_fd,
+                                 LOG_MESSAGE("Failed to open watchdog device %s before the initial transaction completed: %m",
+                                             watchdog_device),
+                                 "MESSAGE_ID=" SD_MESSAGE_WATCHDOG_OPEN_FAILED_STR,
+                                 "WATCHDOG_DEVICE=%s", watchdog_device);
+        else
+                log_struct_errno(LOG_WARNING, watchdog_fd,
+                                 LOG_MESSAGE("Failed to open any watchdog device before the initial transaction completed: %m"),
+                                 "MESSAGE_ID=" SD_MESSAGE_WATCHDOG_OPEN_FAILED_STR);
+}
+
 void watchdog_close(bool disarm) {
 
         /* Once closed, pinging the device becomes a NOP and we request a new
diff --git a/src/shared/watchdog.h b/src/shared/watchdog.h
index bf9212a8650b9..c0fcccbd0735b 100644
--- a/src/shared/watchdog.h
+++ b/src/shared/watchdog.h
@@ -13,8 +13,9 @@ int watchdog_setup(usec_t timeout);
 int watchdog_setup_pretimeout(usec_t usec);
 int watchdog_setup_pretimeout_governor(const char *governor);
 int watchdog_ping(void);
+void watchdog_report_if_missing(void);
 void watchdog_close(bool disarm);
-usec_t watchdog_runtime_wait(void);
+usec_t watchdog_runtime_wait(unsigned divisor);
 
 static inline void watchdog_free_device(void) {
         (void) watchdog_set_device(NULL);
diff --git a/src/systemd/sd-messages.h b/src/systemd/sd-messages.h
index 60b28087f6f0a..502a2e5d5aee4 100644
--- a/src/systemd/sd-messages.h
+++ b/src/systemd/sd-messages.h
@@ -256,6 +256,15 @@ _SD_BEGIN_DECLARATIONS;
 #define SD_MESSAGE_TIME_BUMP                          SD_ID128_MAKE(7d,b7,3c,8a,f0,d9,4e,eb,82,2a,e0,43,23,fe,6a,b6)
 #define SD_MESSAGE_TIME_BUMP_STR                      SD_ID128_MAKE_STR(7d,b7,3c,8a,f0,d9,4e,eb,82,2a,e0,43,23,fe,6a,b6)
 
+#define SD_MESSAGE_WATCHDOG_OPENED                    SD_ID128_MAKE(21,66,8d,bd,3d,7a,4a,32,a2,67,6d,53,da,da,b0,22)
+#define SD_MESSAGE_WATCHDOG_OPENED_STR                SD_ID128_MAKE_STR(21,66,8d,bd,3d,7a,4a,32,a2,67,6d,53,da,da,b0,22)
+
+#define SD_MESSAGE_WATCHDOG_OPEN_FAILED               SD_ID128_MAKE(37,5a,c1,51,ef,9d,4d,e3,90,68,b3,ef,bf,ed,0c,ee)
+#define SD_MESSAGE_WATCHDOG_OPEN_FAILED_STR           SD_ID128_MAKE_STR(37,5a,c1,51,ef,9d,4d,e3,90,68,b3,ef,bf,ed,0c,ee)
+
+#define SD_MESSAGE_WATCHDOG_PING_FAILED               SD_ID128_MAKE(87,39,78,9e,ca,06,43,25,af,15,a8,ed,0e,cf,c5,56)
+#define SD_MESSAGE_WATCHDOG_PING_FAILED_STR           SD_ID128_MAKE_STR(87,39,78,9e,ca,06,43,25,af,15,a8,ed,0e,cf,c5,56)
+
 #define SD_MESSAGE_SHUTDOWN_SCHEDULED                 SD_ID128_MAKE(9e,70,66,27,9d,c8,40,3d,a7,9c,e4,b1,a6,90,64,b2)
 #define SD_MESSAGE_SHUTDOWN_SCHEDULED_STR             SD_ID128_MAKE_STR(9e,70,66,27,9d,c8,40,3d,a7,9c,e4,b1,a6,90,64,b2)
 
diff --git a/src/test/test-watchdog.c b/src/test/test-watchdog.c
index 70d069c08b53b..67e6581cfb02f 100644
--- a/src/test/test-watchdog.c
+++ b/src/test/test-watchdog.c
@@ -8,32 +8,30 @@
 #include "watchdog.h"
 
 int main(int argc, char *argv[]) {
-        usec_t t;
-        unsigned i, count;
         int r;
-        bool slow;
 
         test_setup_logging(LOG_DEBUG);
 
-        slow = slow_tests_enabled();
+        bool slow = slow_tests_enabled();
 
-        t = slow ? 10 * USEC_PER_SEC : 2 * USEC_PER_SEC;
-        count = slow ? 5 : 3;
+        usec_t timeout = slow ? 10 * USEC_PER_SEC : 2 * USEC_PER_SEC;
+        unsigned count = slow ? 5 : 3;
 
-        r = watchdog_setup(t);
+        log_info("Initializing watchdog with timeout of %s", FORMAT_TIMESPAN(timeout, USEC_PER_SEC));
+        r = watchdog_setup(timeout);
         if (r < 0)
                 log_warning_errno(r, "Failed to open watchdog: %m");
 
-        for (i = 0; i < count; i++) {
-                t = watchdog_runtime_wait();
-                log_info("Sleeping " USEC_FMT " microseconds...", t);
-                usleep_safe(t);
+        for (unsigned i = 0; i < count; i++) {
+                timeout = watchdog_runtime_wait(/* divisor= */ 2);
+                log_info("Sleeping %s…", FORMAT_TIMESPAN(timeout, USEC_PER_SEC));
+                usleep_safe(timeout);
                 log_info("Pinging...");
                 r = watchdog_ping();
                 if (r < 0)
                         log_warning_errno(r, "Failed to ping watchdog: %m");
         }
 
-        watchdog_close(true);
+        watchdog_close(/* disarm= */ true);
         return 0;
 }


Reviews:
user: login: yuwata
state: CHANGES_REQUESTED
author_association: MEMBER
submitted_at: 2024-12-20T19:11:17Z
user: login: keszybz
state: COMMENTED
author_association: MEMBER
submitted_at: 2024-12-21T09:08:30Z
user: login: yuwata
state: APPROVED
author_association: MEMBER
submitted_at: 2024-12-23T05:37:44Z
user: login: poettering
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-01-02T13:45:13Z
user: login: poettering
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-01-02T13:47:08Z
user: login: poettering
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-01-02T13:48:04Z
user: login: poettering
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-01-02T13:49:38Z
user: login: poettering
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-01-02T13:51:07Z
user: login: poettering
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-01-09T12:35:54Z
user: login: keszybz
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-21T09:33:07Z
user: login: keszybz
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-21T10:06:02Z
user: login: keszybz
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-21T14:58:25Z
user: login: poettering
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-23T10:33:49Z
user: login: poettering
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-23T10:35:02Z
user: login: poettering
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-23T10:39:48Z
user: login: poettering
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-23T10:40:32Z
user: login: poettering
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-23T10:40:57Z
user: login: poettering
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-23T10:41:06Z
user: login: poettering
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-23T10:48:10Z
user: login: poettering
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-23T10:51:46Z
user: login: keszybz
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T09:35:44Z
user: login: keszybz
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T09:38:12Z
user: login: keszybz
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T09:41:44Z
user: login: keszybz
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T09:45:36Z
user: login: bluca
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T13:30:30Z
user: login: bluca
body: the new message IDs need to be listed in the catalog
state: CHANGES_REQUESTED
author_association: MEMBER
submitted_at: 2025-03-24T13:30:47Z
user: login: keszybz
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T13:50:39Z
user: login: bluca
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T14:10:33Z
user: login: keszybz
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T14:35:01Z
user: login: bluca
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T20:07:39Z
user: login: keszybz
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T21:51:57Z

Review comments:
diff_hunk: @@ -482,6 +482,18 @@ int watchdog_ping(void) {
         return watchdog_ping_now();
 }
 
+void watchdog_report_missing(void) {
+        /* If we failed to open the watchdog because the device doesn't exist, report why. If we successfully
+         * opened a device, or opening failed for a different reason, we logged then. But ENOENT failure may
+         * be transient, for example because modules being loaded or the hardware is being detected. This
+         * function can be called to log after things have settled down. */
+        if (watchdog_fd == -ENOENT)
+                log_debug_errno(watchdog_fd, "Failed to open %swatchdog device%s%s: %m",
path: src/shared/watchdog.c
user: login: yuwata
body: Hm? This is already logged in `watchdog_open()`.

Maybe better to raise the log level, at least when `watchdog_device` is explicitly specified ?
created_at: 2024-12-20T19:05:20Z
author_association: MEMBER
diff_hunk: @@ -482,6 +482,18 @@ int watchdog_ping(void) {
         return watchdog_ping_now();
 }
 
+void watchdog_report_missing(void) {
+        /* If we failed to open the watchdog because the device doesn't exist, report why. If we successfully
+         * opened a device, or opening failed for a different reason, we logged then. But ENOENT failure may
+         * be transient, for example because modules being loaded or the hardware is being detected. This
+         * function can be called to log after things have settled down. */
+        if (watchdog_fd == -ENOENT)
+                log_debug_errno(watchdog_fd, "Failed to open %swatchdog device%s%s: %m",
path: src/shared/watchdog.c
user: login: keszybz
body: Oh, indeed. This was supposed to be at a higher level but I forgot to update after copying the code.
created_at: 2024-12-21T09:08:30Z
author_association: MEMBER
diff_hunk: @@ -482,6 +482,23 @@ int watchdog_ping(void) {
         return watchdog_ping_now();
 }
 
+void watchdog_report_missing(void) {
+        /* If we failed to open the watchdog because the device doesn't exist, report why. If we successfully
+         * opened a device, or opening failed for a different reason, we logged then. But ENOENT failure may
+         * be transient, for example because modules being loaded or the hardware is being detected. This
+         * function can be called to log after things have settled down.
+         *
+         * If a device was specified explicitly, raise level. */
+
path: src/shared/watchdog.c
user: login: poettering
body: i think if watchdog stuff is turned off explicitly we shouldn't log here
created_at: 2025-01-02T13:45:13Z
author_association: MEMBER
diff_hunk: @@ -482,6 +482,23 @@ int watchdog_ping(void) {
         return watchdog_ping_now();
 }
 
+void watchdog_report_missing(void) {
+        /* If we failed to open the watchdog because the device doesn't exist, report why. If we successfully
+         * opened a device, or opening failed for a different reason, we logged then. But ENOENT failure may
+         * be transient, for example because modules being loaded or the hardware is being detected. This
+         * function can be called to log after things have settled down.
+         *
+         * If a device was specified explicitly, raise level. */
+
+        if (watchdog_fd == -ENOENT)
+                log_full_errno(watchdog_device ? LOG_WARNING : LOG_NOTICE,
+                               watchdog_fd,
+                               "Failed to open %swatchdog device%s%s: %m",
path: src/shared/watchdog.c
user: login: poettering
body: i'd clarify in the message when/why we are logging this, i.e. include "even after initial transaction completed"

maybe even make this a structured log message...
created_at: 2025-01-02T13:47:08Z
author_association: MEMBER
diff_hunk: @@ -482,6 +482,23 @@ int watchdog_ping(void) {
         return watchdog_ping_now();
 }
 
+void watchdog_report_missing(void) {
+        /* If we failed to open the watchdog because the device doesn't exist, report why. If we successfully
+         * opened a device, or opening failed for a different reason, we logged then. But ENOENT failure may
+         * be transient, for example because modules being loaded or the hardware is being detected. This
+         * function can be called to log after things have settled down.
+         *
+         * If a device was specified explicitly, raise level. */
+
+        if (watchdog_fd == -ENOENT)
+                log_full_errno(watchdog_device ? LOG_WARNING : LOG_NOTICE,
+                               watchdog_fd,
+                               "Failed to open %swatchdog device%s%s: %m",
+                               watchdog_device ? "" : "any ",
+                               watchdog_device ? " " : "",
+                               strempty(watchdog_device));
+}
path: src/shared/watchdog.c
user: login: poettering
body: this should probably be suppressed entirely unless MANAGER_IS_SYSTEM() holds. 

and also in case detect_container() holds...
created_at: 2025-01-02T13:48:04Z
author_association: MEMBER
diff_hunk: @@ -311,12 +311,16 @@ static int watchdog_update_timeout(void) {
 }
 
 static int watchdog_open(void) {
+        static RateLimit watchdog_open_ratelimit = { 5 * USEC_PER_SEC, 1 };
path: src/shared/watchdog.c
user: login: poettering
body: i think we should lower this to half of the configured ping interval if that's lower, because in that case people apparently want quicker reaction to failures...
created_at: 2025-01-02T13:49:38Z
author_association: MEMBER
diff_hunk: @@ -311,12 +311,16 @@ static int watchdog_update_timeout(void) {
 }
 
 static int watchdog_open(void) {
+        static RateLimit watchdog_open_ratelimit = { 5 * USEC_PER_SEC, 1 };
         struct watchdog_info ident;
         char **try_order;
         int r;
 
         assert(watchdog_fd < 0);
 
+        if (!ratelimit_below(&watchdog_open_ratelimit))
+                return -EWOULDBLOCK;
path: src/shared/watchdog.c
user: login: poettering
body: i think it would make sense to explicit reset the ratelimit right before you write that new warning message of the previous commit. and then maybe also do an explicit watchdog open request right after, so that the warning definitely is "correct" at the point in time where we write it.
created_at: 2025-01-02T13:51:06Z
author_association: MEMBER
diff_hunk: @@ -195,10 +195,13 @@ usec_t watchdog_get_last_ping(clockid_t clock) {
 }
 
 static int watchdog_ping_now(void) {
+        int r;
+
         assert(watchdog_fd >= 0);
 
-        if (ioctl(watchdog_fd, WDIOC_KEEPALIVE, 0) < 0)
-                return log_warning_errno(errno, "Failed to ping hardware watchdog, ignoring: %m");
+        r = RET_NERRNO(ioctl(watchdog_fd, WDIOC_KEEPALIVE, 0));
+        if (r < 0)
+                return log_warning_errno(r, "Failed to ping hardware watchdog, ignoring: %m");
path: src/shared/watchdog.c
user: login: poettering
body: btw, any chance you can add ratelimiting to this message too? in order to address #35405. as it appears there are broken watchdogs around that can be opened but where pinging them fails. we should probably warn about that a couple of times, but then give up. 

(also, we might want to consider closing the fd here again on failure. (either here or in watchdog_ping() or watchdog_open() after calling watchdog_ping_now()). but that's another story...)
created_at: 2025-01-09T12:35:54Z
author_association: MEMBER
diff_hunk: @@ -482,6 +482,23 @@ int watchdog_ping(void) {
         return watchdog_ping_now();
 }
 
+void watchdog_report_missing(void) {
+        /* If we failed to open the watchdog because the device doesn't exist, report why. If we successfully
+         * opened a device, or opening failed for a different reason, we logged then. But ENOENT failure may
+         * be transient, for example because modules being loaded or the hardware is being detected. This
+         * function can be called to log after things have settled down.
+         *
+         * If a device was specified explicitly, raise level. */
+
path: src/shared/watchdog.c
user: login: keszybz
body: We will get here only if we attempted to open the watchdog, which we should only do if it was configured. `watchdog_fd` starts out as `-EBADF`, and here we log iff `watchdog_fd == -ENOENT`.
created_at: 2025-03-21T09:33:06Z
author_association: MEMBER
diff_hunk: @@ -482,6 +482,23 @@ int watchdog_ping(void) {
         return watchdog_ping_now();
 }
 
+void watchdog_report_missing(void) {
+        /* If we failed to open the watchdog because the device doesn't exist, report why. If we successfully
+         * opened a device, or opening failed for a different reason, we logged then. But ENOENT failure may
+         * be transient, for example because modules being loaded or the hardware is being detected. This
+         * function can be called to log after things have settled down.
+         *
+         * If a device was specified explicitly, raise level. */
+
+        if (watchdog_fd == -ENOENT)
+                log_full_errno(watchdog_device ? LOG_WARNING : LOG_NOTICE,
+                               watchdog_fd,
+                               "Failed to open %swatchdog device%s%s: %m",
+                               watchdog_device ? "" : "any ",
+                               watchdog_device ? " " : "",
+                               strempty(watchdog_device));
+}
path: src/shared/watchdog.c
user: login: keszybz
body: Indeed. But we can't do it here, because this is the shared code that doesn't know what Manager is. I changed the caller.
created_at: 2025-03-21T10:06:02Z
author_association: MEMBER
diff_hunk: @@ -195,10 +195,13 @@ usec_t watchdog_get_last_ping(clockid_t clock) {
 }
 
 static int watchdog_ping_now(void) {
+        int r;
+
         assert(watchdog_fd >= 0);
 
-        if (ioctl(watchdog_fd, WDIOC_KEEPALIVE, 0) < 0)
-                return log_warning_errno(errno, "Failed to ping hardware watchdog, ignoring: %m");
+        r = RET_NERRNO(ioctl(watchdog_fd, WDIOC_KEEPALIVE, 0));
+        if (r < 0)
+                return log_warning_errno(r, "Failed to ping hardware watchdog, ignoring: %m");
path: src/shared/watchdog.c
user: login: keszybz
body: Yeah, I think we should close the fd. If the watchdog works, and we don't close it, we'd get rebooted. So we should at least try to close it.
created_at: 2025-03-21T14:58:24Z
author_association: MEMBER
diff_hunk: @@ -3871,6 +3871,9 @@ static void manager_notify_finished(Manager *m) {
 
         bus_manager_send_finished(m, firmware_usec, loader_usec, kernel_usec, initrd_usec, userspace_usec, total_usec);
 
+        if (MANAGER_IS_SYSTEM(m) && detect_container() <= 0)
+                watchdog_report_if_missing();
path: src/core/manager.c
user: login: poettering
body: this is racy of course. the initial transaction might be over very quickly, and the watchdog drivers might take quite long to initialize, if they are kmods and connected via slow wiring..

That might not be a reason not to do this, but it does make me concerned about making this anything beyond LOG_NOTICE
created_at: 2025-03-23T10:33:48Z
author_association: MEMBER
diff_hunk: @@ -3871,6 +3871,9 @@ static void manager_notify_finished(Manager *m) {
 
         bus_manager_send_finished(m, firmware_usec, loader_usec, kernel_usec, initrd_usec, userspace_usec, total_usec);
 
+        if (MANAGER_IS_SYSTEM(m) && detect_container() <= 0)
path: src/core/manager.c
user: login: poettering
body: the 2nd check really should be moved inside of watchdog_report_if_missing()
created_at: 2025-03-23T10:35:02Z
author_association: MEMBER
diff_hunk: @@ -3391,7 +3391,7 @@ int manager_loop(Manager *m) {
                         continue;
 
                 /* Sleep for watchdog runtime wait time */
-                r = sd_event_run(m->event, watchdog_runtime_wait());
+                r = sd_event_run(m->event, watchdog_runtime_wait(2));
path: src/core/manager.c
user: login: poettering
body: can you prefix the argument with `/* divisor= */`, this is otherwise too opaque. 

(here and all other calls)
created_at: 2025-03-23T10:39:48Z
author_association: MEMBER
diff_hunk: @@ -462,20 +496,26 @@ static usec_t watchdog_calc_timeout(void) {
                 return watchdog_timeout;
 }
 
-usec_t watchdog_runtime_wait(void) {
+usec_t watchdog_runtime_wait(unsigned divisor) {
path: src/shared/watchdog.c
user: login: poettering
body: maybe add `assert(divisor > 0);` here, to catch division by zero?
created_at: 2025-03-23T10:40:32Z
author_association: MEMBER
diff_hunk: @@ -23,7 +23,7 @@ int main(int argc, char *argv[]) {
                 log_warning_errno(r, "Failed to open watchdog: %m");
 
         for (unsigned i = 0; i < count; i++) {
-                timeout = watchdog_runtime_wait();
+                timeout = watchdog_runtime_wait(2);
path: src/test/test-watchdog.c
user: login: poettering
body: as above `/* divisor= */`
created_at: 2025-03-23T10:40:57Z
author_association: MEMBER
diff_hunk: @@ -486,17 +526,10 @@ int watchdog_ping(void) {
                 /* open_watchdog() will automatically ping the device for us if necessary */
                 return watchdog_open(/* ignore_ratelimit= */ false);
 
-        /* Never ping earlier than watchdog_timeout/4 and try to ping
-         * by watchdog_timeout/2 plus scheduling latencies at the latest */
-        if (timestamp_is_set(watchdog_last_ping)) {
-                usec_t ntime = now(CLOCK_BOOTTIME),
-                       timeout = watchdog_calc_timeout();
-
-                assert(ntime >= watchdog_last_ping);
-
-                if (ntime - watchdog_last_ping < timeout/4)
-                        return 0;
-        }
+        /* Ping approximately watchdog_timeout/4 after a successful ping, or even less than that
+         * after an unsuccessful ping. */
+        if (watchdog_runtime_wait(4) > 0)
path: src/shared/watchdog.c
user: login: poettering
body: as above
created_at: 2025-03-23T10:41:06Z
author_association: MEMBER
diff_hunk: @@ -202,12 +209,34 @@ static int watchdog_ping_now(void) {
         assert(watchdog_fd >= 0);
 
         r = RET_NERRNO(ioctl(watchdog_fd, WDIOC_KEEPALIVE, 0));
-        if (r < 0)
-                return log_warning_errno(r, "Failed to ping hardware watchdog, ignoring: %m");
-
-        watchdog_last_ping = now(CLOCK_BOOTTIME);
+        if (r < 0) {
+                watchdog_last_bad_ping = now(CLOCK_BOOTTIME);
+                if (++ watchdog_bad_pings >= WATCHDOG_MAX_FAILED_PINGS) {
+                        log_struct_errno(LOG_ERR, r,
+                                         LOG_MESSAGE("Failed to ping hardware watchdog %s, closing watchdog after %u attempts: %m",
+                                                     watchdog_device, watchdog_bad_pings),
+                                         "MESSAGE_ID=" SD_MESSAGE_WATCHDOG_PING_FAILED_STR,
+                                         "WATCHDOG_DEVICE=%s", watchdog_device);
+                        watchdog_timeout = USEC_INFINITY;
+                        watchdog_close(/* disarm= */ true);
+                } else
+                        log_struct_errno(LOG_WARNING, r,
+                                         LOG_MESSAGE("Failed to ping hardware watchdog %s: %m",
+                                                     watchdog_device),
+                                         "MESSAGE_ID=" SD_MESSAGE_WATCHDOG_PING_FAILED_STR,
+                                         "WATCHDOG_DEVICE=%s", watchdog_device);
+        } else {
+                watchdog_last_good_ping = now(CLOCK_BOOTTIME);
+                if (watchdog_bad_pings > 0) {
+                        log_struct(LOG_INFO,
+                                   LOG_MESSAGE("Succesfullly pinged hardware watchdog %s after %u attempts: %m",
+                                               watchdog_device, watchdog_bad_pings + 1),
+                                   "WATCHDOG_DEVICE=%s", watchdog_device);
+                        watchdog_bad_pings = 0;
path: src/shared/watchdog.c
user: login: poettering
body: maybe also reset watchdog_last_bad_ping, just in case?
created_at: 2025-03-23T10:48:09Z
author_association: MEMBER
diff_hunk: @@ -462,20 +496,26 @@ static usec_t watchdog_calc_timeout(void) {
                 return watchdog_timeout;
 }
 
-usec_t watchdog_runtime_wait(void) {
+usec_t watchdog_runtime_wait(unsigned divisor) {
         usec_t timeout = watchdog_calc_timeout();
         if (!timestamp_is_set(timeout))
                 return USEC_INFINITY;
 
-        /* Sleep half the watchdog timeout since the last successful ping at most */
-        if (timestamp_is_set(watchdog_last_ping)) {
+        /* Sleep watchdog timeout / divisor since the last successful ping attempt at most,
+         * or the fifth of that if the ping failed. */
+
+        usec_t ts = MAX(timestamp_is_set(watchdog_last_good_ping) ? watchdog_last_good_ping : 0u,
+                        timestamp_is_set(watchdog_last_bad_ping) ? watchdog_last_bad_ping : 0u);
+        if (ts > 0) {
                 usec_t ntime = now(CLOCK_BOOTTIME);
+                if (watchdog_last_bad_ping == ts)
path: src/shared/watchdog.c
user: login: poettering
body: not loving this kind of test that works from the results backwards...
created_at: 2025-03-23T10:51:45Z
author_association: MEMBER
diff_hunk: @@ -3871,6 +3871,9 @@ static void manager_notify_finished(Manager *m) {
 
         bus_manager_send_finished(m, firmware_usec, loader_usec, kernel_usec, initrd_usec, userspace_usec, total_usec);
 
+        if (MANAGER_IS_SYSTEM(m) && detect_container() <= 0)
+                watchdog_report_if_missing();
path: src/core/manager.c
user: login: keszybz
body: I think that if the system is *configured* to use a watchdog, then it's iffy it it doesn't get detected. Let's make it a warning for now. If poeple report that they have systems where this doesn't hold, we can always downgrade.
created_at: 2025-03-24T09:35:44Z
author_association: MEMBER
diff_hunk: @@ -3871,6 +3871,9 @@ static void manager_notify_finished(Manager *m) {
 
         bus_manager_send_finished(m, firmware_usec, loader_usec, kernel_usec, initrd_usec, userspace_usec, total_usec);
 
+        if (MANAGER_IS_SYSTEM(m) && detect_container() <= 0)
path: src/core/manager.c
user: login: keszybz
body: A check in this form appears a few lines above… I considered moving this call up into that block, but it doesn't fit exactly. But moving half of this check here to a different place would introduce unnecessary difference from the rest of the logic in this function.
created_at: 2025-03-24T09:38:12Z
author_association: MEMBER
diff_hunk: @@ -462,20 +496,26 @@ static usec_t watchdog_calc_timeout(void) {
                 return watchdog_timeout;
 }
 
-usec_t watchdog_runtime_wait(void) {
+usec_t watchdog_runtime_wait(unsigned divisor) {
path: src/shared/watchdog.c
user: login: keszybz
body: It's called in exactly thee places, one of which is test code. I added the `/* divisor= */` comments as requested, so I *hope* nobody will change on of those integers to 0. This is not public API. 
created_at: 2025-03-24T09:41:43Z
author_association: MEMBER
diff_hunk: @@ -462,20 +496,26 @@ static usec_t watchdog_calc_timeout(void) {
                 return watchdog_timeout;
 }
 
-usec_t watchdog_runtime_wait(void) {
+usec_t watchdog_runtime_wait(unsigned divisor) {
         usec_t timeout = watchdog_calc_timeout();
         if (!timestamp_is_set(timeout))
                 return USEC_INFINITY;
 
-        /* Sleep half the watchdog timeout since the last successful ping at most */
-        if (timestamp_is_set(watchdog_last_ping)) {
+        /* Sleep watchdog timeout / divisor since the last successful ping attempt at most,
+         * or the fifth of that if the ping failed. */
+
+        usec_t ts = MAX(timestamp_is_set(watchdog_last_good_ping) ? watchdog_last_good_ping : 0u,
+                        timestamp_is_set(watchdog_last_bad_ping) ? watchdog_last_bad_ping : 0u);
+        if (ts > 0) {
                 usec_t ntime = now(CLOCK_BOOTTIME);
+                if (watchdog_last_bad_ping == ts)
path: src/shared/watchdog.c
user: login: keszybz
body: Yeah. But the alternative would be to either repeat the logic or have a lot more verbose code… I reversed the order in the check, I hope that that makes it more readable.
created_at: 2025-03-24T09:45:36Z
author_association: MEMBER
diff_hunk: @@ -3352,14 +3352,14 @@ int manager_loop(Manager *m) {
 
         while (m->objective == MANAGER_OK) {
 
-                (void) watchdog_ping();
path: src/core/manager.c
user: login: bluca
body: Not a huge fan, as this could be the difference between a board getting reset or recovering, given the other issue has been fixed I'd rather leave this here. It's not just the sleep of one sec, but yielding the CPU, etc can also add up
created_at: 2025-03-24T13:30:30Z
author_association: MEMBER
diff_hunk: @@ -3352,14 +3352,14 @@ int manager_loop(Manager *m) {
 
         while (m->objective == MANAGER_OK) {
 
-                (void) watchdog_ping();
path: src/core/manager.c
user: login: keszybz
body: Board recovering from what? Sorry, I don't understand what you mean here.

This change takes effect if we loop faster than the ratelimit at line 3340, which has an burst of 50k. We quite certainly shouldn't have to ping the watchdog more often then every millisecond.
created_at: 2025-03-24T13:50:39Z
author_association: MEMBER
diff_hunk: @@ -3352,14 +3352,14 @@ int manager_loop(Manager *m) {
 
         while (m->objective == MANAGER_OK) {
 
-                (void) watchdog_ping();
path: src/core/manager.c
user: login: bluca
body: There might be other reason to go around this loop, other than the watchdog itself, so in general I think it's safer to ping before going to sleep, rather than after. I've definitely had cases where the timeouts where on the knife's edge
created_at: 2025-03-24T14:10:33Z
author_association: MEMBER
diff_hunk: @@ -3352,14 +3352,14 @@ int manager_loop(Manager *m) {
 
         while (m->objective == MANAGER_OK) {
 
-                (void) watchdog_ping();
path: src/core/manager.c
user: login: keszybz
body: This is just a safety net. If we're looping, something is seriously broken in our code. The watchdog stuff is important, but so is the handling of jobs and responding to dbus messages, etc. Since clearly it is possible for us mess up the timeouts in various places, incl. the watchdog code, I think the patch makes sense. Also, one could argue that if something is off, and PID1 is busy-looping, then this makes real failure more likely.


created_at: 2025-03-24T14:35:00Z
author_association: MEMBER
diff_hunk: @@ -3352,14 +3352,14 @@ int manager_loop(Manager *m) {
 
         while (m->objective == MANAGER_OK) {
 
-                (void) watchdog_ping();
path: src/core/manager.c
user: login: bluca
body: Yeah job handling etc are important, but none of that will cause a hardware reset. Having been bitten by this round trip being delayed just enough to trigger it and break production nodes, I would really prefer to be more conservative and ping as the very first thing that happens in the main loop, always
created_at: 2025-03-24T20:07:39Z
author_association: MEMBER
diff_hunk: @@ -3352,14 +3352,14 @@ int manager_loop(Manager *m) {
 
         while (m->objective == MANAGER_OK) {
 
-                (void) watchdog_ping();
path: src/core/manager.c
user: login: keszybz
body: But if the hardware watchdog ping happens to rarely, then there are four possible reasons: one is that the ping is attempted to late, e.g. by not taking latency into account. But considering that we use half of the interval as the time, I'm doubtful whether this is the case. But if you have some logs or scenarios which show that this might be the case, we can always lower it to 1/5th or some other heuristic formula. The second reason might be if the schedule loop and the timeout logic in watchdog.c are somehow misaligned and the ping is delayed because of that. The third reason would be that we do some processing which takes a lot of time, and we hit the watchdog timeout because we're not looping for a long time. This is something that we should avoid too, since it'd also create a bad user experience in other regards, incl. slow starting of jobs, delayed timers, slow responses to systemctl, possibly blocking of the graphical env if it tries to do a synchronous call and expects it to be quick. The fourth reason would be that watchdog just might be set to fast for the given CPU capabilities and the amount of work done by systemd. But either way, let's really figure this out and apply a real fix, not attempt to microoptimize by rearranging order of operations.
created_at: 2025-03-24T21:51:57Z
author_association: MEMBER

PR comments:
user: login: keszybz
created_at: 2025-03-21T15:00:02Z
body: Reworked this for all the comments:
- structured logging is added for the important messages
- we ratelimit open attempts, with the interval based on the watchdog timeout
- we ratelimit ping attempts and close the watchdog after a fixed number of failures

author_association: MEMBER
user: login: keszybz
created_at: 2025-03-24T13:52:39Z
body: > the new message IDs need to be listed in the catalog

I don't think so. The catalog is used to provide extra explanations. Here the message is sufficient. The structured log message is added to make it easy to find the message and to provide the watchdog device name. 

author_association: MEMBER
user: login: bluca
created_at: 2025-03-24T14:15:10Z
body: Ok, I thought we wanted to have all message_id defined as a generic rule
author_association: MEMBER
user: login: keszybz
created_at: 2025-03-24T20:04:01Z
body: 22/95 systemd:integration-tests / TEST-85-NETWORK-NetworkdNetworkTests                          FAIL            346.46s   exit status 1
86/95 systemd:integration-tests / TEST-70-TPM2                                                  FAIL            247.99s   exit status 1

 53/95 systemd:integration-tests / TEST-24-CRYPTSETUP                                            FAIL            131.12s   exit status 1

62/93 systemd:integration-tests / TEST-46-HOMED                                                 FAIL             36.23s   exit status 1

13407s 13/95 systemd:integration-tests / TEST-64-UDEV-STORAGE-virtio_scsi_identically_named_partitions FAIL             21.86s   exit status 1
13407s 51/95 systemd:integration-tests / TEST-24-CRYPTSETUP                                            TIMEOUT        1800.34s   killed by signal 15 SIGTERM

2711s socket.timeout: The read operation timed out

Failed to discover tests, TMT metadata are absent or corrupted. Search for '[E] [test-schedule-tmt' on the artifacts page for details. WAT?

None of those failures look related to this patchset.

EDIT: https://github.com/systemd/systemd/pull/36831 has a similar set of failures.
author_association: MEMBER