Repository: winstonallo/taskmaster
Pull request #34: Dynamic healthchecks
Author: login: winstonallo
Created: 2025-04-19T14:05:23Z | Merged: 2025-04-20T11:33:43Z

Closes: #8
Closes: #30

Add:
* New optional `healthcheck` field in the config, allows configuration of a command to run for healthchecks instead of waiting `starttime` to determine whether a process is healthy
* ~~New state `Starting` to differentiate between dynamic and simple healthchecks~~
* HealthCheckRunner struct to encapsulate the more complex healthcheck logic

HealthChecks are run in a tokio task with time out, their results are communicated via one-shot channels.


Diff:
diff --git a/.clippy.toml b/.clippy.toml
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/.clippy.toml
@@ -0,0 +1 @@
+
diff --git a/.gitignore b/.gitignore
index 37dcd1d..cd7f1f0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,4 +3,5 @@ writetofile.sh
 file.txt
 history.txt
 a.out
+main.c
 test.c
\ No newline at end of file
diff --git a/.rustfmt.toml b/.rustfmt.toml
new file mode 100644
index 0000000..3798388
--- /dev/null
+++ b/.rustfmt.toml
@@ -0,0 +1,5 @@
+chain_width = 120
+fn_call_width = 120
+max_width = 160
+array_width = 160
+edition = "2024"
diff --git a/config/example.toml b/config/example.toml
index 30dd5c0..1a18747 100644
--- a/config/example.toml
+++ b/config/example.toml
@@ -10,16 +10,16 @@ socketpath = "/tmp/.taskmaster.sock"
 authgroup = "winstonallo"
 
 # Process name.
-[processes.ls]
+[processes.sleep]
 
 # Command to run in order to start this process.
 # Must be an absolute path.
 # Required.
-cmd = "/home/winstonallo/mastery/taskmaster/writetofile.sh"
+cmd = "/usr/bin/sleep"
 
 # Arguments to pass to 'cmd'.
 # Defaults to [].
-args = []
+args = ["1"]
 
 # Number of processes to start.
 # Defaults to 1, max 255.
@@ -44,25 +44,41 @@ autostart = true
 #   <max-retries> times when it exits unexpectedly. Maximum value for 
 #   max-retries is 255.
 # - "always": Always restart when exiting, no matter the status.
-autorestart = "on-failure[:5]"
-
-# Backoff time (in seconds) for retries.
-# Defaults to 5, max 255.
-backoff = 1
+autorestart = "always"
 
 # List of exit codes to be interpreted as success.
 # Defaults to [0].
 exitcodes = [0, 2]
 
-# Number of attempts to start the program before giving up.
-# Defaults to 3, max 255.
-startretries = 3
+# Optional: configure a specific command to run to determine whether a 
+# process is healthy. Overrides `startttime`.
+[processes.sleep.healthcheck]
 
 # Time (in seconds) given for the program to startup - if the program 
-# does not start within this time, it will be considered a failure.
+# runs for this time, it will be considered healthy.
 # Defaults to 5, max 65536.
 starttime = 1
 
+# Command to run in order to start this process.
+# Must be an absolute path.
+# Required.
+cmd = "/usr/bin/sleep"
+
+# Arguments to pass to 'cmd'.
+# Defaults to [].
+args = ["2"]
+
+# Time, in seconds, given for the healthcheck to run through. If it runs for longer,
+# it will be considered failed.
+timeout = 1
+
+
+# How many times to retry the healthcheck.
+retries = 3
+
+# How much time to wait after a failed healthcheck before retrying.
+backoff = 5
+
 # List of signals triggering a stop of the process, options are any 
 # FreeBSD signal (https://www.math.stonybrook.edu/~ccc/dfc/dfc/signals.html).
 # Defaults to ["TERM"].
@@ -87,16 +103,16 @@ stderr = "/tmp/ls.stderr"
 env = [["STARTED_BY", "abied-ch"], ["ANSWER", "42"]]
 
 # Process name.
-[processes.lsls]
+[processes.ls]
 
 # Command to run in order to start this process.
 # Must be an absolute path.
 # Required.
-cmd = "/home/winstonallo/mastery/taskmaster/writetofile.sh"
+cmd = "/usr/bin/ls"
 
 # Arguments to pass to 'cmd'.
 # Defaults to [].
-args = []
+args = ["-laR"]
 
 # Number of processes to start.
 # Defaults to 1, max 255.
@@ -109,7 +125,7 @@ umask = "022"
 # Working directory for the process.
 # Must be an absolute path.
 # Required.
-workingdir = "/tmp"
+workingdir = "/home/winstonallo/mastery/taskmaster/src"
 
 # Start the program automatically when starting taskmaster. 
 # Defaults to false.
@@ -121,24 +137,12 @@ autostart = true
 #   <max-retries> times when it exits unexpectedly. Maximum value for 
 #   max-retries is 255.
 # - "always": Always restart when exiting, no matter the status.
-autorestart = "on-failure[:5]"
-
-# Backoff time (in seconds) for retries.
-# Defaults to 5, max 255.
-backoff = 2
+autorestart = "on-failure[:1]"
 
 # List of exit codes to be interpreted as success.
 # Defaults to [0].
-exitcodes = [0, 2]
-
-# Number of attempts to start the program before giving up.
-# Defaults to 3, max 255.
-startretries = 3
+exitcodes = [0]
 
-# Time (in seconds) given for the program to startup - if the program 
-# does not start within this time, it will be considered a failure.
-# Defaults to 5, max 65536.
-starttime = 2
 
 # List of signals triggering a stop of the process, options are any 
 # FreeBSD signal (https://www.math.stonybrook.edu/~ccc/dfc/dfc/signals.html).
diff --git a/config/felix.toml b/config/felix.toml
index d13b6be..841aafb 100644
--- a/config/felix.toml
+++ b/config/felix.toml
@@ -7,7 +7,7 @@ socketpath = "/tmp/.taskmaster.sock"
 # Name of the group to be created for authenticating client (similarly to 
 # the docker group.
 # Defaults to "taskmaster".
-authgroup = "everyone"
+authgroup = "winstonallo"
 
 # Process name.
 [processes.ls]
@@ -15,7 +15,7 @@ authgroup = "everyone"
 # Command to run in order to start this process.
 # Must be an absolute path.
 # Required.
-cmd = "/Users/code/source/taskmaster/a.out"
+cmd = "/home/winstonallo/mastery/taskmaster/a.out"
 
 # Arguments to pass to 'cmd'.
 # Defaults to [].
diff --git a/config/healthcheck.toml b/config/healthcheck.toml
new file mode 100644
index 0000000..5b4065b
--- /dev/null
+++ b/config/healthcheck.toml
@@ -0,0 +1,26 @@
+socketpath = "/tmp/.taskmaster.sock"
+
+authgroup = "winstonallo"
+
+[processes.sleep]
+
+cmd = "/usr/bin/sleep"
+args = ["60"]
+processes = 1
+umask = "022"
+workingdir = "/tmp"
+autostart = true
+autorestart = "always"
+exitcodes = [0, 2]
+[processes.sleep.healthcheck]
+cmd = "/usr/bin/sleep"
+args = ["2"]
+backoff = 1
+timeout = 1
+retries = 1
+
+stopsignals = ["TERM", "USR1"]
+stoptime = 5
+stdout = "/tmp/ls.stdout"
+stderr = "/tmp/ls.stderr"
+env = [["STARTED_BY", "abied-ch"], ["ANSWER", "42"]]
diff --git a/rustfmt.toml b/rustfmt.toml
deleted file mode 100644
index b28d8cf..0000000
--- a/rustfmt.toml
+++ /dev/null
@@ -1 +0,0 @@
-max_width = 160
diff --git a/src/bin/taskmaster.rs b/src/bin/taskmaster.rs
index a84f450..436ef8f 100644
--- a/src/bin/taskmaster.rs
+++ b/src/bin/taskmaster.rs
@@ -28,9 +28,4 @@ async fn main() -> Result<(), Box<dyn Error>> {
 
     log_info!("starting taskmaster..");
     daemon.run().await
-    // let res = tasklib::run::daemon::run(&mut daemon.processes, conf.socketpath().to_string(), conf.authgroup().to_string()).await;
-    // if let Ok(()) = res {
-    //     return Ok(());
-    // }
-    // tokio::signal::ctrl_c().await?;
 }
diff --git a/src/bin/taskshell.rs b/src/bin/taskshell.rs
index c3f8715..aaa69bc 100644
--- a/src/bin/taskshell.rs
+++ b/src/bin/taskshell.rs
@@ -44,10 +44,7 @@ fn build_request_status() -> Request {
 }
 
 fn build_request_status_single(name: &str) -> Request {
-    Request::new(
-        ID_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed),
-        RequestType::new_status_single(name),
-    )
+    Request::new(ID_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed), RequestType::new_status_single(name))
 }
 
 fn build_request_start(name: &str) -> Request {
diff --git a/src/conf/proc.rs b/src/conf/proc.rs
index 5bd12b3..09df5b9 100644
--- a/src/conf/proc.rs
+++ b/src/conf/proc.rs
@@ -26,6 +26,7 @@ mod defaults;
 pub mod types;
 
 use serde::Deserialize;
+use types::HealthCheck;
 
 /// # ProcessConfig
 /// `src/conf/proc/mod.rs`
@@ -55,17 +56,11 @@ pub struct ProcessConfig {
     #[serde(default = "defaults::dflt_autorestart")]
     autorestart: types::AutoRestart,
 
-    #[serde(default = "defaults::dflt_backoff")]
-    backoff: u8,
-
     #[serde(default = "defaults::dflt_exitcodes")]
     exitcodes: Vec<i32>,
 
-    #[serde(default = "defaults::dflt_startretries")]
-    startretries: u8,
-
-    #[serde(default = "defaults::dflt_startttime")]
-    starttime: u16,
+    #[serde(default)]
+    healthcheck: HealthCheck,
 
     #[serde(default = "defaults::dflt_stopsignals")]
     stopsignals: Vec<types::StopSignal>,
@@ -112,22 +107,12 @@ impl ProcessConfig {
         &self.autorestart
     }
 
-    pub fn backoff(&self) -> u8 {
-        assert_eq!(self.autorestart.mode(), "on-failure");
-
-        self.backoff
-    }
-
     pub fn exitcodes(&self) -> &Vec<i32> {
         &self.exitcodes
     }
 
-    pub fn startretries(&self) -> u8 {
-        self.startretries
-    }
-
-    pub fn starttime(&self) -> u16 {
-        self.starttime
+    pub fn healthcheck(&self) -> &HealthCheck {
+        &self.healthcheck
     }
 
     pub fn stopsignals(&self) -> &Vec<types::StopSignal> {
@@ -170,10 +155,8 @@ impl ProcessConfig {
             workingdir: types::AccessibleDirectory::default(),
             autostart: true,
             autorestart: types::AutoRestart::default(),
-            backoff: 5,
             exitcodes: vec![0],
-            startretries: 1,
-            starttime: 5,
+            healthcheck: HealthCheck::default(),
             stopsignals: vec![types::StopSignal(SIGTERM)],
             stoptime: 5,
             stdout: types::WritableFile::from_path("/tmp/taskmaster_test.stdout"),
diff --git a/src/conf/proc/defaults.rs b/src/conf/proc/defaults.rs
index 0bd4a86..3684c24 100644
--- a/src/conf/proc/defaults.rs
+++ b/src/conf/proc/defaults.rs
@@ -22,22 +22,10 @@ pub fn dflt_autorestart() -> types::AutoRestart {
     types::AutoRestart::default()
 }
 
-pub fn dflt_backoff() -> u8 {
-    5
-}
-
 pub fn dflt_exitcodes() -> Vec<i32> {
     vec![0i32]
 }
 
-pub fn dflt_startretries() -> u8 {
-    3
-}
-
-pub fn dflt_startttime() -> u16 {
-    5
-}
-
 pub fn dflt_stopsignals() -> Vec<types::StopSignal> {
     vec![types::StopSignal(SIGTERM)]
 }
diff --git a/src/conf/proc/types.rs b/src/conf/proc/types.rs
index 674a9fe..5fc9b9f 100644
--- a/src/conf/proc/types.rs
+++ b/src/conf/proc/types.rs
@@ -1,10 +1,12 @@
 mod autorestart;
+mod healthcheck;
 mod path;
 mod stopsignal;
 mod umask;
 
 pub use self::{
     autorestart::AutoRestart,
+    healthcheck::{HealthCheck, HealthCheckType},
     path::{AccessibleDirectory, ExecutableFile, WritableFile},
     stopsignal::StopSignal,
     umask::Umask,
diff --git a/src/conf/proc/types/autorestart.rs b/src/conf/proc/types/autorestart.rs
index c21478a..e821464 100644
--- a/src/conf/proc/types/autorestart.rs
+++ b/src/conf/proc/types/autorestart.rs
@@ -51,9 +51,7 @@ impl<'de> Deserialize<'de> for AutoRestart {
         }
 
         if !s.starts_with("on-failure[:") || !s.ends_with("]") {
-            return Err(serde::de::Error::custom(format!(
-                "invalid value for on-failure: {s}, expected 'on-failure[:max-retries]'"
-            )));
+            return Err(serde::de::Error::custom(format!("invalid value for on-failure: {s}, expected 'on-failure[:max-retries]'")));
         }
 
         let max_retries_str = &s[12..s.len() - 1];
@@ -61,9 +59,7 @@ impl<'de> Deserialize<'de> for AutoRestart {
         let max_retries = match max_retries_str.parse::<u8>() {
             Ok(n) => n,
             Err(e) => {
-                return Err(serde::de::Error::custom(format!(
-                    "invalid max-retries value for on-failure: {max_retries_str}: {e}, expected u8"
-                )));
+                return Err(serde::de::Error::custom(format!("invalid max-retries value for on-failure: {max_retries_str}: {e}, expected u8")));
             }
         };
 
diff --git a/src/conf/proc/types/healthcheck.rs b/src/conf/proc/types/healthcheck.rs
new file mode 100644
index 0000000..2bc283a
--- /dev/null
+++ b/src/conf/proc/types/healthcheck.rs
@@ -0,0 +1,89 @@
+use serde::Deserialize;
+
+fn dflt_timeout() -> usize {
+    10
+}
+
+fn dflt_backoff() -> usize {
+    5
+}
+
+fn dflt_retries() -> usize {
+    5
+}
+
+#[derive(Deserialize, Debug, Clone)]
+#[serde(untagged)]
+pub enum HealthCheckType {
+    Command {
+        cmd: String,
+        #[serde(default)]
+        args: Vec<String>,
+    },
+    Uptime {
+        starttime: u8,
+    },
+}
+
+#[allow(unused)]
+#[derive(Deserialize, Debug, Clone)]
+pub struct HealthCheck {
+    #[serde(flatten)]
+    check: HealthCheckType,
+    #[serde(default = "dflt_timeout")]
+    timeout: usize,
+    #[serde(default = "dflt_retries")]
+    retries: usize,
+    #[serde(default = "dflt_backoff")]
+    backoff: usize,
+}
+
+impl Default for HealthCheck {
+    fn default() -> Self {
+        Self {
+            check: HealthCheckType::Uptime { starttime: 5 },
+            timeout: dflt_timeout(),
+            retries: dflt_retries(),
+            backoff: dflt_backoff(),
+        }
+    }
+}
+
+impl HealthCheck {
+    pub fn healthcheck(&self) -> &HealthCheckType {
+        &self.check
+    }
+
+    pub fn cmd(&self) -> &str {
+        match &self.check {
+            HealthCheckType::Command { cmd, .. } => cmd,
+            _ => panic!("cmd() called on an Uptime HealthCheck"),
+        }
+    }
+
+    pub fn args(&self) -> &[String] {
+        match &self.check {
+            HealthCheckType::Command { cmd: _, args } => args,
+            _ => panic!("args() called on an Uptime HealthCheck"),
+        }
+    }
+
+    pub fn starttime(&self) -> u8 {
+        match &self.check {
+            HealthCheckType::Uptime { starttime } => *starttime,
+            _ => panic!("starttime() called on a Command HealthCheck"),
+        }
+    }
+
+    pub fn timeout(&self) -> usize {
+        self.timeout
+    }
+
+    pub fn retries(&self) -> usize {
+        self.retries
+    }
+
+    pub fn backoff(&self) -> usize {
+        self.backoff
+    }
+}
diff --git a/src/conf/proc/types/umask.rs b/src/conf/proc/types/umask.rs
index 145b478..56c6db7 100644
--- a/src/conf/proc/types/umask.rs
+++ b/src/conf/proc/types/umask.rs
@@ -35,9 +35,7 @@ impl<'de> Deserialize<'de> for Umask {
         let mask = match u32::from_str_radix(&s, 8) {
             Ok(mask) => mask,
             Err(err) => {
-                return Err(serde::de::Error::custom(format!(
-                    "invalid value for umask: '{s}', expected 3 octal digits: {err}"
-                )));
+                return Err(serde::de::Error::custom(format!("invalid value for umask: '{s}', expected 3 octal digits: {err}")));
             }
         };
 
diff --git a/src/conf/tests.rs b/src/conf/tests.rs
index 3517ffd..f258a68 100644
--- a/src/conf/tests.rs
+++ b/src/conf/tests.rs
@@ -18,8 +18,6 @@ mod from_str {
         assert_eq!(conf.processes()["nginx"].autostart(), defaults::dflt_autostart());
         assert_eq!(conf.processes()["nginx"].autorestart().mode(), defaults::dflt_autorestart().mode());
         assert_eq!(conf.processes()["nginx"].exitcodes(), &defaults::dflt_exitcodes());
-        assert_eq!(conf.processes()["nginx"].startretries(), defaults::dflt_startretries());
-        assert_eq!(conf.processes()["nginx"].starttime(), defaults::dflt_startttime());
         assert_eq!(conf.processes()["nginx"].stopsignals(), &defaults::dflt_stopsignals());
         assert_eq!(conf.processes()["nginx"].stoptime(), defaults::dflt_stoptime());
         assert_eq!(conf.processes()["nginx"].stdout(), "/tmp/nginx.stdout");
@@ -30,10 +28,7 @@ mod from_str {
     #[test]
     fn socketpath_default() {
         let conf_str = "[processes.nginx]\ncmd = \"/usr/sbin/nginx\"\nworkingdir = \"/tmp\"\n";
-        assert_eq!(
-            Config::from_str(conf_str).expect("could not parse config").socketpath(),
-            "/tmp/.taskmaster.sock".to_string()
-        )
+        assert_eq!(Config::from_str(conf_str).expect("could not parse config").socketpath(), "/tmp/.taskmaster.sock".to_string())
     }
 
     #[test]
@@ -70,10 +65,7 @@ mod from_str {
     fn args_default() {
         let conf_str = "[processes.nginx]\ncmd = \"/usr/sbin/nginx\"\nworkingdir = \"/tmp\"\n";
         let expected_vec: Vec<String> = vec![];
-        assert_eq!(
-            Config::from_str(conf_str).expect("could not parse config").processes()["nginx"].args(),
-            &expected_vec
-        )
+        assert_eq!(Config::from_str(conf_str).expect("could not parse config").processes()["nginx"].args(), &expected_vec)
     }
 
     #[test]
@@ -193,10 +185,7 @@ mod from_str {
     #[test]
     fn exitcodes_default() {
         let conf_str = "[processes.nginx]\ncmd = \"/usr/sbin/nginx\"\nworkingdir = \"/tmp\"\n";
-        assert_eq!(
-            *Config::from_str(conf_str).expect("could not parse config").processes()["nginx"].exitcodes(),
-            vec![0]
-        );
+        assert_eq!(*Config::from_str(conf_str).expect("could not parse config").processes()["nginx"].exitcodes(), vec![0]);
     }
 
     #[test]
@@ -205,27 +194,12 @@ mod from_str {
         assert!(Config::from_str(conf_str).is_err());
     }
 
-    #[test]
-    fn startretries_default() {
-        let conf_str = "[processes.nginx]\ncmd = \"/usr/sbin/nginx\"\nworkingdir = \"/tmp\"\n";
-        assert_eq!(
-            Config::from_str(conf_str).expect("could not parse config").processes()["nginx"].startretries(),
-            3
-        );
-    }
-
     #[test]
     fn starttime_out_of_range() {
         let conf_str = "[processes.nginx]\ncmd = \"/usr/sbin/nginx\"\nworkingdir = \"/tmp\"\nstarttime = 70000";
         assert!(Config::from_str(conf_str).is_err());
     }
 
-    #[test]
-    fn starttime_default() {
-        let conf_str = "[processes.nginx]\ncmd = \"/usr/sbin/nginx\"\nworkingdir = \"/tmp\"\n";
-        assert_eq!(Config::from_str(conf_str).expect("could not parse config").processes()["nginx"].starttime(), 5);
-    }
-
     #[test]
     fn stopsignals_non_existing_freebsd_signal() {
         let conf_str = "[processes.nginx]\ncmd = \"/usr/sbin/nginx\"\nworkingdir = \"/tmp\"\nstopsignals = [\"ARTHUR\"]";
@@ -386,27 +360,23 @@ mod from_file {
     fn valid_config_all_fields_set() {
         let conf = Config::from_file("./tests/configs/example.toml").expect("could not parse config");
 
-        assert_eq!(conf.processes().keys().cloned().collect::<Vec<String>>(), vec!["nginx".to_string()]);
-        assert_eq!(conf.processes()["nginx"].cmd().path(), "/usr/sbin/nginx");
-        assert_eq!(conf.processes()["nginx"].processes(), 1);
-        assert_eq!(conf.processes()["nginx"].umask(), 0o022);
-        assert_eq!(conf.processes()["nginx"].workingdir(), &AccessibleDirectory::default());
-        assert!(conf.processes()["nginx"].autostart());
-        assert_eq!(conf.processes()["nginx"].autorestart().mode(), "on-failure");
-        assert_eq!(conf.processes()["nginx"].autorestart().max_retries(), 5);
-        assert_eq!(conf.processes()["nginx"].exitcodes(), &vec![0, 2]);
-        assert_eq!(conf.processes()["nginx"].startretries(), 3);
-        assert_eq!(conf.processes()["nginx"].starttime(), 5);
-        assert_eq!(conf.processes()["nginx"].stopsignals(), &vec![StopSignal(SIGTERM), StopSignal(SIGUSR1)]);
-        assert_eq!(conf.processes()["nginx"].stoptime(), 5);
-        assert_eq!(conf.processes()["nginx"].stdout(), ("/tmp/nginx.stdout".to_string()));
-        assert_eq!(conf.processes()["nginx"].stderr(), ("/tmp/nginx.stderr".to_string()));
+        assert!(conf.processes().keys().cloned().collect::<Vec<String>>().contains(&"sleep".to_string()));
+        assert!(conf.processes().keys().cloned().collect::<Vec<String>>().contains(&"ls".to_string()));
+        assert_eq!(conf.processes()["sleep"].cmd().path(), "/usr/bin/sleep");
+        assert_eq!(conf.processes()["sleep"].processes(), 1);
+        assert_eq!(conf.processes()["sleep"].umask(), 0o022);
+        assert_eq!(conf.processes()["sleep"].workingdir(), &AccessibleDirectory::default());
+        assert!(conf.processes()["sleep"].autostart());
+        assert_eq!(conf.processes()["sleep"].autorestart().mode(), "on-failure");
+        assert_eq!(conf.processes()["sleep"].autorestart().max_retries(), 5);
+        assert_eq!(conf.processes()["sleep"].exitcodes(), &vec![0, 2]);
+        assert_eq!(conf.processes()["sleep"].stopsignals(), &vec![StopSignal(SIGTERM), StopSignal(SIGUSR1)]);
+        assert_eq!(conf.processes()["sleep"].stoptime(), 5);
+        assert_eq!(conf.processes()["sleep"].stdout(), ("/tmp/sleep.stdout".to_string()));
+        assert_eq!(conf.processes()["sleep"].stderr(), ("/tmp/sleep.stderr".to_string()));
         assert_eq!(
-            conf.processes()["nginx"].env(),
-            &Some(vec![
-                (("STARTED_BY".to_string()), ("abied-ch".to_string())),
-                (("ANSWER".to_string()), ("42".to_string()))
-            ])
+            conf.processes()["sleep"].env(),
+            &Some(vec![(("STARTED_BY".to_string()), ("abied-ch".to_string())), (("ANSWER".to_string()), ("42".to_string()))])
         );
     }
 }
diff --git a/src/jsonrpc/short_process.rs b/src/jsonrpc/short_process.rs
index 828df73..f8c4c93 100644
--- a/src/jsonrpc/short_process.rs
+++ b/src/jsonrpc/short_process.rs
@@ -30,6 +30,7 @@ impl ShortProcess {
 pub enum State {
     Idle,
     Ready,
+    Starting(u64),
     HealthCheck(u64),
     Healthy,
     Failed,
@@ -61,6 +62,7 @@ impl fmt::Display for State {
         let s = match self {
             Idle => "idle".to_owned(),
             Ready => "ready".to_owned(),
+            Starting(s) => format!("starting since {} seconds", s),
             HealthCheck(s) => format!("healthcheck since {} seconds", s),
             Healthy => "healthy".to_owned(),
             Failed => "failed".to_owned(),
diff --git a/src/log.rs b/src/log.rs
index 02702ac..fe2f8e4 100644
--- a/src/log.rs
+++ b/src/log.rs
@@ -127,14 +127,14 @@ macro_rules! log_fatal {
 #[macro_export]
 macro_rules! proc_info {
     ($proc:expr, $($arg:tt)*) => {
-        $crate::log::prefix_info($proc, format_args!($($arg)*))
+        $crate::log::prefix_info($proc.name(), format_args!($($arg)*))
     };
 }
 
 #[macro_export]
 macro_rules! proc_warning {
     ($proc:expr, $($arg:tt)*) => {
-        $crate::log::prefix_warning($proc, format_args!($($arg)*))
+        $crate::log::prefix_warning($proc.name(), format_args!($($arg)*))
     };
 }
 
diff --git a/src/run/proc.rs b/src/run/proc.rs
index 7d91de8..eea3138 100644
--- a/src/run/proc.rs
+++ b/src/run/proc.rs
@@ -13,7 +13,7 @@ use crate::{
 pub use error::ProcessError;
 use libc::{c_int, signal, umask};
 
-use super::statemachine::states::ProcessState;
+use super::statemachine::{healthcheck::HealthCheckRunner, states::ProcessState};
 
 mod error;
 
@@ -24,8 +24,8 @@ pub struct Process {
     name: String,
     child: Option<Child>,
     conf: ProcessConfig,
-    startup_failures: u8,
-    runtime_failures: u8,
+    healthcheck: HealthCheckRunner,
+    runtime_failures: usize,
     state: ProcessState,
     desired_states: VecDeque<ProcessState>,
 }
@@ -33,12 +33,13 @@ pub struct Process {
 impl Process {
     pub fn from_process_config(conf: conf::proc::ProcessConfig, proc_name: &str) -> Self {
         let is_autostart = conf.autostart();
+        let healthcheck = conf.healthcheck().clone();
         Self {
             id: None,
             name: proc_name.to_string(),
             child: None,
             conf,
-            startup_failures: 0,
+            healthcheck: HealthCheckRunner::from_healthcheck_config(&healthcheck),
             runtime_failures: 0,
             state: ProcessState::Idle,
             desired_states: match is_autostart {
@@ -97,10 +98,6 @@ impl Process {
         &mut self.conf
     }
 
-    pub fn runtime_failures(&self) -> u8 {
-        self.runtime_failures
-    }
-
     pub fn desired_states(&self) -> &VecDeque<ProcessState> {
         &self.desired_states
     }
@@ -109,29 +106,47 @@ impl Process {
         &mut self.desired_states
     }
 
+    pub fn runtime_failures(&self) -> usize {
+        self.runtime_failures
+    }
+
     pub fn increment_runtime_failures(&mut self) {
         self.runtime_failures = self.runtime_failures.saturating_add(1);
     }
 
-    pub fn startup_failures(&self) -> u8 {
-        self.startup_failures
+    pub fn healthcheck_failures(&self) -> usize {
+        assert!(self.healthcheck.has_command_healthcheck());
+        self.healthcheck.failures()
+    }
+
+    pub fn increment_healthcheck_failures(&mut self) {
+        self.healthcheck.increment_failures();
+    }
+
+    /// Returns `true` if a dynamic healthcheck is configured for this process.
+    pub fn has_command_healthcheck(&self) -> bool {
+        self.healthcheck.has_command_healthcheck()
+    }
+
+    pub fn healthcheck(&self) -> &HealthCheckRunner {
+        &self.healthcheck
     }
 
-    pub fn increment_startup_failures(&mut self) {
-        self.startup_failures = self.startup_failures.saturating_add(1);
+    pub fn healthcheck_mut(&mut self) -> &mut HealthCheckRunner {
+        &mut self.healthcheck
     }
 
-    /// Returns a `time::Instant` representing the next time the process should
-    /// be retried according to its configured backoff, assuming the failure
-    /// happened at the time of calling this method.
     pub fn retry_at(&self) -> time::Instant {
-        Instant::now() + Duration::from_secs(self.conf.backoff() as u64)
+        Instant::now() + Duration::from_secs(self.healthcheck().backoff() as u64)
     }
 
-    /// Checks whether the process is healthy, according to `started_at` and its
-    /// configured healthcheck time.
-    pub fn healthy(&self, started_at: time::Instant) -> bool {
-        Instant::now().duration_since(started_at).as_secs() >= self.conf.starttime() as u64
+    pub fn start_healthcheck(&mut self) {
+        let mut healthcheck = self.healthcheck_mut();
+        healthcheck.start();
+    }
+
+    pub fn passed_starttime(&self, started_at: time::Instant) -> bool {
+        Instant::now().duration_since(started_at).as_secs() >= self.healthcheck.starttime() as u64
     }
 
     fn spawn(&self) -> Result<Child, ProcessError> {
@@ -166,6 +181,10 @@ impl Process {
     }
 
     pub fn start(&mut self) -> Result<(), ProcessError> {
+        if self.child.is_some() {
+            return Err(ProcessError::AlreadyRunning);
+        }
+
         assert_ne!(self.state(), ProcessState::Healthy);
 
         self.child = match self.spawn() {
@@ -215,7 +234,7 @@ impl Process {
     pub fn kill_gracefully(&mut self) -> Result<(), &str> {
         use ProcessState::*;
         match self.state() {
-            HealthCheck(_) | Healthy => {}
+            HealthCheck(_) | Healthy | Failed(_) => {}
             _ => return Err("process not running"),
         }
 
@@ -227,7 +246,7 @@ impl Process {
         unsafe {
             libc::kill(child.id() as i32, libc::SIGTERM);
         }
-        proc_info!(self.name(), "shutting down, PID {} gracefully", child.id());
+        proc_info!(self, "shutting down, PID {} gracefully", child.id());
 
         Ok(())
     }
@@ -245,29 +264,9 @@ impl Process {
         };
 
         child.kill();
-        proc_info!(self.name(), "killed, PID {}", child.id());
+        proc_info!(self, "killed, PID {}", child.id());
         self.id.take();
 
         Ok(())
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn state() {
-        let proc = Process {
-            id: Some(1),
-            name: ("".to_string()),
-            child: None,
-            conf: conf::proc::ProcessConfig::testconfig(),
-            startup_failures: 0,
-            runtime_failures: 0,
-            state: ProcessState::Idle,
-            desired_states: VecDeque::new(),
-        };
-        assert_eq!(proc.state(), ProcessState::Idle)
-    }
-}
diff --git a/src/run/proc/error.rs b/src/run/proc/error.rs
index ba717f8..bcc2f9b 100644
--- a/src/run/proc/error.rs
+++ b/src/run/proc/error.rs
@@ -3,6 +3,7 @@ use std::{error::Error as StdError, fmt, io};
 pub enum ProcessError {
     Internal(String),
     CouldNotSpawn(String),
+    AlreadyRunning,
 }
 
 impl fmt::Display for ProcessError {
@@ -10,6 +11,7 @@ impl fmt::Display for ProcessError {
         match self {
             ProcessError::Internal(msg) => write!(f, "internal error: {}", msg),
             ProcessError::CouldNotSpawn(msg) => write!(f, "could not spawn child process: {}", msg),
+            ProcessError::AlreadyRunning => write!(f, "process is already running"),
         }
     }
 }
diff --git a/src/run/statemachine.rs b/src/run/statemachine.rs
index 4e9b794..758b7b5 100644
--- a/src/run/statemachine.rs
+++ b/src/run/statemachine.rs
@@ -1,3 +1,4 @@
 mod desired;
+pub mod healthcheck;
 mod monitor;
 pub mod states;
diff --git a/src/run/statemachine/healthcheck.rs b/src/run/statemachine/healthcheck.rs
new file mode 100644
index 0000000..f8df246
--- /dev/null
+++ b/src/run/statemachine/healthcheck.rs
@@ -0,0 +1,120 @@
+#![allow(unused)]
+use std::{
+    time::{Duration, Instant},
+    vec,
+};
+
+use crate::conf::proc::types::{HealthCheck, HealthCheckType};
+
+#[derive(Debug)]
+pub struct HealthCheckRunner {
+    failures: usize,
+    task: Option<tokio::task::JoinHandle<()>>,
+    receiver: Option<tokio::sync::oneshot::Receiver<HealthCheckEvent>>,
+    check: HealthCheckType,
+    timeout: usize,
+    retries: usize,
+    backoff: usize,
+}
+
+#[derive(Debug)]
+pub enum HealthCheckEvent {
+    Passed,
+    Failed(String),
+}
+
+impl HealthCheckRunner {
+    pub fn from_healthcheck_config(hc: &HealthCheck) -> Self {
+        Self {
+            failures: 0,
+            task: None,
+            receiver: None,
+            check: hc.healthcheck().clone(),
+            timeout: hc.timeout(),
+            retries: hc.retries(),
+            backoff: hc.backoff(),
+        }
+    }
+
+    pub fn has_command_healthcheck(&self) -> bool {
+        matches!(self.check, HealthCheckType::Command { .. })
+    }
+
+    pub fn cmd(&self) -> String {
+        match &self.check {
+            HealthCheckType::Command { cmd, .. } => cmd.to_string(),
+            _ => panic!("cmd() called on an Uptime HealthCheck"),
+        }
+    }
+
+    pub fn args(&self) -> Vec<String> {
+        match &self.check {
+            HealthCheckType::Command { cmd: _, args } => args.clone(),
+            _ => panic!("args() called on an Uptime HealthCheck"),
+        }
+    }
+
+    pub fn starttime(&self) -> u8 {
+        match &self.check {
+            HealthCheckType::Uptime { starttime } => *starttime,
+            _ => panic!("starttime() called on a Command HealthCheck"),
+        }
+    }
+
+    pub fn failures(&self) -> usize {
+        self.failures
+    }
+
+    pub fn retries(&self) -> usize {
+        self.retries
+    }
+
+    pub fn backoff(&self) -> usize {
+        self.backoff
+    }
+
+    pub fn increment_failures(&mut self) {
+        self.failures = self.failures.saturating_add(1);
+    }
+
+    pub fn task(&self) -> &Option<tokio::task::JoinHandle<()>> {
+        &self.task
+    }
+
+    pub fn receiver(&mut self) -> &mut Option<tokio::sync::oneshot::Receiver<HealthCheckEvent>> {
+        &mut self.receiver
+    }
+
+    pub fn clear(&mut self) {
+        self.task = None;
+        self.receiver = None;
+    }
+
+    async fn spawn(cmd: &str, args: &Vec<String>, timeout: Duration) -> HealthCheckEvent {
+        let mut command = tokio::process::Command::new(cmd);
+        command.args(args);
+
+        match tokio::time::timeout(timeout, command.output()).await {
+            Ok(Ok(output)) if output.status.success() => HealthCheckEvent::Passed,
+            Ok(Ok(output)) => HealthCheckEvent::Failed(format!("exit code: {}, stderr: {}", output.status, String::from_utf8_lossy(&output.stderr))),
+            Ok(Err(e)) => HealthCheckEvent::Failed(e.to_string()),
+            Err(_) => HealthCheckEvent::Failed(format!("timed out after {} seconds", timeout.as_secs())),
+        }
+    }
+
+    pub fn start(&mut self) {
+        let (sender, receiver) = tokio::sync::oneshot::channel::<HealthCheckEvent>();
+        self.receiver = Some(receiver);
+
+        let cmd = self.cmd().clone();
+        let args = self.args().to_vec();
+        let timeout = Duration::from_secs(self.timeout as u64);
+
+        let handle = tokio::task::spawn(async move {
+            let result = HealthCheckRunner::spawn(&cmd, &args, timeout).await;
+            let _ = sender.send(result);
+        });
+
+        self.task = Some(handle);
+    }
+}
diff --git a/src/run/statemachine/monitor.rs b/src/run/statemachine/monitor.rs
index b484b60..88afc3d 100644
--- a/src/run/statemachine/monitor.rs
+++ b/src/run/statemachine/monitor.rs
@@ -1,6 +1,12 @@
 use std::time::Instant;
 
-use crate::{proc_info, proc_warning, run::proc::Process};
+use crate::{
+    proc_info, proc_warning,
+    run::{
+        proc::{Process, ProcessError},
+        statemachine::healthcheck::HealthCheckEvent,
+    },
+};
 
 use super::states::ProcessState;
 
@@ -12,63 +18,126 @@ pub fn monitor_ready(p: &mut Process) -> Option<ProcessState> {
     match p.start() {
         Ok(()) => {
             let pid = p.id().expect("id should always be set if the process is running");
-            proc_info!(&p.name(), "spawned, PID {}", pid);
+            proc_info!(&p, "spawned, PID {}", pid);
             Some(ProcessState::HealthCheck(Instant::now()))
         }
         Err(err) => {
-            proc_warning!(&p.name(), "failed to start: {}", err);
-            p.increment_startup_failures();
+            proc_warning!(&p, "failed to start: {}", err);
+            p.increment_healthcheck_failures();
             Some(ProcessState::Failed(Box::new(ProcessState::HealthCheck(Instant::now()))))
         }
     }
 }
 
-pub fn monitor_health_check(started_at: &Instant, p: &mut Process) -> Option<ProcessState> {
-    if p.healthy(*started_at) {
-        proc_info!(&p.name(), "has been running for {} seconds, marking as healthy", p.config().starttime());
-
-        return Some(ProcessState::Healthy);
-    }
+/// Check whether `p` exited and return the appropriate state based on its
+/// exit code.
+///
+/// Returns `None` if `p` is running.
+fn exited_state(p: &mut Process) -> Option<ProcessState> {
+    assert!(matches!(p.state(), ProcessState::HealthCheck(_) | ProcessState::Healthy));
 
     if let Some(code) = p.exited() {
+        let msg = match p.state() {
+            ProcessState::Healthy => "while healthy".to_string(),
+            ProcessState::HealthCheck(t) => format!("in healthcheck after {}s", Instant::now().duration_since(t).as_secs()),
+            _ => String::new(),
+        };
+
         if !p.config().exitcodes().contains(&code) {
-            proc_warning!(&p.name(), "exited during healthcheck with unexpected code ({})", code);
-            return Some(ProcessState::Failed(Box::new(ProcessState::HealthCheck(*started_at))));
+            proc_warning!(&p, "exited {} with unexpected code ({})", msg, code);
+            return Some(ProcessState::Failed(Box::new(p.state())));
         } else {
-            proc_info!(&p.name(), "exited with healthy code ({})", code);
+            proc_info!(&p, "exited {} with healthy code ({})", msg, code);
             return Some(ProcessState::Completed);
         }
     }
+
     None
 }
 
-pub fn monitor_healthy(p: &mut Process) -> Option<ProcessState> {
-    if let Some(code) = p.exited() {
-        if !p.config().exitcodes().contains(&code) {
-            proc_warning!(&p.name(), "exited with unexpected code ({})", code);
-            return Some(ProcessState::Failed(Box::new(ProcessState::Healthy)));
-        } else {
-            proc_info!(&p.name(), "exited with healthy code ({})", code);
-            return Some(ProcessState::Completed);
+fn healthcheck_command(started_at: &Instant, p: &mut Process) -> Option<ProcessState> {
+    assert!(p.has_command_healthcheck(), "a process without configured healthcheck should never be used here");
+
+    if let Some(exited_state) = exited_state(p) {
+        p.healthcheck_mut().clear();
+        return Some(exited_state);
+    }
+
+    let receiver = match p.healthcheck_mut().receiver() {
+        Some(receiver) => receiver,
+        None => {
+            proc_info!(p, "starting healthcheck - cmd: {}, args: {:?}", p.healthcheck().cmd(), p.healthcheck().args());
+            p.start_healthcheck();
+            return None;
         }
+    };
+
+    match receiver.try_recv() {
+        Ok(result) => {
+            p.healthcheck_mut().clear();
+            match result {
+                HealthCheckEvent::Passed => {
+                    proc_info!(&p, "healthcheck successful");
+                    Some(ProcessState::Healthy)
+                }
+                HealthCheckEvent::Failed(reason) => {
+                    proc_warning!(&p, "healthcheck failed: {}", reason);
+                    Some(ProcessState::Failed(Box::new(ProcessState::HealthCheck(*started_at))))
+                }
+            }
+        }
+        Err(e) => match e {
+            tokio::sync::oneshot::error::TryRecvError::Empty => None,
+            tokio::sync::oneshot::error::TryRecvError::Closed => {
+                proc_warning!(&p, "healthcheck channel closed unexpectedly");
+                p.healthcheck_mut().clear();
+                Some(ProcessState::Failed(Box::new(ProcessState::HealthCheck(*started_at))))
+            }
+        },
     }
+}
+
+fn healthcheck_starttime(started_at: &Instant, p: &mut Process) -> Option<ProcessState> {
+    assert!(!p.has_command_healthcheck(), "a process with a configured healthcheck should never be used here");
+
+    if let Some(exited_state) = exited_state(p) {
+        return Some(exited_state);
+    }
+
+    if p.passed_starttime(*started_at) {
+        proc_info!(&p, "has been running for {} seconds, marking as healthy", p.healthcheck().starttime());
+
+        return Some(ProcessState::Healthy);
+    }
+
     None
 }
 
-pub fn failed_runtime(p: &mut Process) -> Option<ProcessState> {
+pub fn monitor_healthcheck(started_at: &Instant, p: &mut Process) -> Option<ProcessState> {
+    match p.has_command_healthcheck() {
+        true => healthcheck_command(started_at, p),
+        false => healthcheck_starttime(started_at, p),
+    }
+}
+
+pub fn monitor_healthy(p: &mut Process) -> Option<ProcessState> {
+    exited_state(p)
+}
+
+pub fn failed_healthy(p: &mut Process) -> Option<ProcessState> {
     match p.config().autorestart().mode() {
         "always" => Some(ProcessState::HealthCheck(Instant::now())),
         "on-failure" => {
             let max_retries = p.config().autorestart().max_retries();
 
-            if p.runtime_failures() == max_retries {
-                proc_warning!(&p.name(), "exited unexpectedly {} times, giving up", p.runtime_failures());
+            if p.runtime_failures() == max_retries as usize {
+                proc_warning!(&p, "exited unexpectedly {} times, giving up", p.runtime_failures());
                 return Some(ProcessState::Stopped);
             }
 
-            let rem_attempts = max_retries - p.runtime_failures();
-            let backoff = p.config().backoff();
-            proc_warning!(&p.name(), "retrying in {} second(s) ({} attempt(s) left)", backoff, rem_attempts);
+            let rem_attempts = max_retries as usize - p.runtime_failures();
+            let backoff = p.healthcheck().backoff();
+            proc_warning!(&p, "retrying in {} second(s) ({} attempt(s) left)", backoff, rem_attempts);
 
             p.increment_runtime_failures();
             Some(ProcessState::WaitingForRetry(p.retry_at()))
@@ -78,22 +147,24 @@ pub fn failed_runtime(p: &mut Process) -> Option<ProcessState> {
 }
 
 pub fn failed_healthcheck(p: &mut Process) -> Option<ProcessState> {
-    if p.startup_failures() == p.config().startretries() {
-        proc_warning!(&p.name(), "reached max startretries, giving up");
-        Some(ProcessState::Stopped)
+    p.increment_healthcheck_failures();
+
+    if p.healthcheck_failures() == p.healthcheck().retries() {
+        proc_warning!(p, "not healthy after {} attempts, giving up", p.healthcheck().retries());
+        let _ = p.kill_gracefully();
+        Some(ProcessState::Stopping(Instant::now()))
     } else {
-        proc_warning!(&p.name(), "restarting in {} seconds", p.config().backoff());
-        p.increment_startup_failures();
+        proc_info!(p, "retrying healthcheck in {} seconds", p.healthcheck().backoff());
         Some(ProcessState::WaitingForRetry(p.retry_at()))
     }
 }
 
 pub fn monitor_failed(p: &mut Process) -> Option<ProcessState> {
     if let ProcessState::Failed(prev_state) = p.state().clone() {
-        assert!(matches!(*prev_state, ProcessState::HealthCheck(_)) || *prev_state == ProcessState::Healthy);
+        assert!(matches!(*prev_state, ProcessState::HealthCheck(_) | ProcessState::Healthy));
 
         match *prev_state {
-            ProcessState::Healthy => failed_runtime(p),
+            ProcessState::Healthy => failed_healthy(p),
             ProcessState::HealthCheck(_) => failed_healthcheck(p),
             _ => None,
         }
@@ -109,13 +180,16 @@ pub fn monitor_waiting_for_retry(retry_at: &Instant, p: &mut Process) -> Option<
 
     match p.start() {
         Ok(()) => {
-            proc_info!(&p.name(), "spawned, PID {}", p.id().expect("if the process started, its id should be set"));
+            proc_info!(&p, "spawned, PID {}", p.id().expect("if the process started, its id should be set"));
             Some(ProcessState::HealthCheck(Instant::now()))
         }
-        Err(err) => {
-            proc_warning!(&p.name(), "failed to start: {}", err);
-            Some(ProcessState::Failed(Box::new(ProcessState::HealthCheck(Instant::now()))))
-        }
+        Err(e) => match e {
+            ProcessError::AlreadyRunning => Some(ProcessState::HealthCheck(Instant::now())),
+            _ => {
+                proc_warning!(&p, "failed to start: {}", e);
+                Some(ProcessState::Failed(Box::new(ProcessState::HealthCheck(Instant::now()))))
+            }
+        },
     }
 }
 
@@ -126,11 +200,11 @@ pub fn monitor_completed(p: &mut Process) -> Option<ProcessState> {
 
     match p.start() {
         Ok(()) => {
-            proc_info!(p.name(), "spawned, PID {}", p.id().expect("if the process started, its id should be set"));
+            proc_info!(p, "spawned, PID {}", p.id().expect("if the process started, its id should be set"));
             Some(ProcessState::HealthCheck(Instant::now()))
         }
         Err(err) => {
-            proc_warning!(p.name(), "failed to start: {}", err);
+            proc_warning!(p, "failed to start: {}", err);
             Some(ProcessState::Failed(Box::new(ProcessState::HealthCheck(Instant::now()))))
         }
     }
@@ -139,16 +213,19 @@ pub fn monitor_completed(p: &mut Process) -> Option<ProcessState> {
 pub fn monitor_stopping(p: &mut Process) -> Option<ProcessState> {
     if let Some(code) = p.exited() {
         if !p.config().exitcodes().contains(&code) {
-            proc_warning!(&p.name(), "exited with unexpected code ({})", code);
+            proc_warning!(&p, "exited with unexpected code ({})", code);
             return Some(ProcessState::Failed(Box::new(ProcessState::Healthy)));
         } else {
-            proc_info!(&p.name(), "exited with healthy code ({})", code);
+            proc_info!(&p, "exited with healthy code ({})", code);
             return Some(ProcessState::Completed);
         }
     }
     None
 }
 
-pub fn monitor_stopped(_p: &mut Process) -> Option<ProcessState> {
+pub fn monitor_stopped(p: &mut Process) -> Option<ProcessState> {
+    // When a process is killed, its entry in the process table is kept
+    // until the parent either exits or calls wait() on it.
+    let _ = p.exited();
     None
 }
diff --git a/src/run/statemachine/states.rs b/src/run/statemachine/states.rs
index b4ce807..084bad6 100644
--- a/src/run/statemachine/states.rs
+++ b/src/run/statemachine/states.rs
@@ -8,7 +8,7 @@ use crate::run::{
     statemachine::{
         desired::{desire_healthy, desire_idle, desire_ready},
         monitor::{
-            monitor_completed, monitor_failed, monitor_health_check, monitor_healthy, monitor_idle, monitor_ready, monitor_stopped, monitor_stopping,
+            monitor_completed, monitor_failed, monitor_healthcheck, monitor_healthy, monitor_idle, monitor_ready, monitor_stopped, monitor_stopping,
             monitor_waiting_for_retry,
         },
     },
@@ -30,13 +30,14 @@ pub enum ProcessState {
 
 impl Display for ProcessState {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let now = Instant::now();
         match self {
             ProcessState::Idle => write!(f, "idle"),
             ProcessState::Ready => write!(f, "ready"),
-            ProcessState::HealthCheck(started_at) => write!(f, "in healthcheck since {} seconds", Instant::now().duration_since(*started_at).as_secs()),
+            ProcessState::HealthCheck(started_at) => write!(f, "in healthcheck since {} seconds", now.duration_since(*started_at).as_secs()),
             ProcessState::Healthy => write!(f, "healthy"),
             ProcessState::Failed(prev_state) => write!(f, "failed while in state: {}", *prev_state),
-            ProcessState::WaitingForRetry(retry_at) => write!(f, "retrying in {} seconds", retry_at.duration_since(Instant::now()).as_secs()),
+            ProcessState::WaitingForRetry(retry_at) => write!(f, "retrying in {} seconds", retry_at.duration_since(now).as_secs()),
             ProcessState::Completed => write!(f, "completed successfully"),
             ProcessState::Stopping(_) => write!(f, "stopping"),
             ProcessState::Stopped => write!(f, "stopped"),
@@ -50,7 +51,7 @@ impl ProcessState {
         match self {
             Idle => monitor_idle(),
             Ready => monitor_ready(proc),
-            HealthCheck(started_at) => monitor_health_check(started_at, proc),
+            HealthCheck(started_at) => monitor_healthcheck(started_at, proc),
             Healthy => monitor_healthy(proc),
             Failed(_process_state) => monitor_failed(proc),
             WaitingForRetry(retry_at) => monitor_waiting_for_retry(retry_at, proc),
diff --git a/tests/configs/example.toml b/tests/configs/example.toml
index ae4db34..50a1415 100644
--- a/tests/configs/example.toml
+++ b/tests/configs/example.toml
@@ -1,12 +1,25 @@
 # Example config for taskmaster.
 
+# Path to the socket used for connections between taskmaster and its client.
+# Defaults to "/tmp/.taskmaster.sock".
+socketpath = "/tmp/.taskmaster.sock"
+
+# Name of the group to be created for authenticating client (similarly to 
+# the docker group.
+# Defaults to "taskmaster".
+authgroup = "winstonallo"
+
 # Process name.
-[processes.nginx]
+[processes.sleep]
 
 # Command to run in order to start this process.
 # Must be an absolute path.
 # Required.
-cmd = "/usr/sbin/nginx"
+cmd = "/usr/bin/sleep"
+
+# Arguments to pass to 'cmd'.
+# Defaults to [].
+args = ["1"]
 
 # Number of processes to start.
 # Defaults to 1, max 255.
@@ -37,14 +50,99 @@ autorestart = "on-failure[:5]"
 # Defaults to [0].
 exitcodes = [0, 2]
 
-# Number of attempts to start the program before giving up.
-# Defaults to 3, max 255.
-startretries = 3
+# List of signals triggering a stop of the process, options are any 
+# FreeBSD signal (https://www.math.stonybrook.edu/~ccc/dfc/dfc/signals.html).
+# Defaults to ["TERM"].
+stopsignals = ["TERM", "USR1"]
+
+# Time (in seconds) to wait for the process to stop. If it does not stop 
+# within this time, it will be forcibly killed.
+# Defaults to 5, max 255.
+stoptime = 5
+
+# File the standard output of the process should be redirected to.
+# Defaults to "/tmp/<process name>.stdout".
+stdout = "/tmp/sleep.stdout"
+
+# File the standard error of the process should be redirected to.
+# Defaults to "/tmp/<process name>.stderr".
+stderr = "/tmp/sleep.stderr"
+
+# Key value pairs of environment variables to be injected into the process
+# at startup.
+# Defaults to [[]].
+env = [["STARTED_BY", "abied-ch"], ["ANSWER", "42"]]
+
+# Optional: configure a specific command to run to determine whether a 
+# process is healthy. Overrides `startttime`.
+[processes.sleep.healthcheck]
 
 # Time (in seconds) given for the program to startup - if the program 
-# does not start within this time, it will be considered a failure.
+# runs for this time, it will be considered healthy.
 # Defaults to 5, max 65536.
-starttime = 5
+starttime = 1
+
+# Command to run in order to start this process.
+# Must be an absolute path.
+# Required.
+cmd = "/usr/bin/sleep"
+
+# Arguments to pass to 'cmd'.
+# Defaults to [].
+args = ["2"]
+
+# Time, in seconds, given for the healthcheck to run through. If it runs for longer,
+# it will be considered failed.
+timeout = 1
+
+
+# How many times to retry the healthcheck.
+retries = 3
+
+# How much time to wait after a failed healthcheck before retrying.
+backoff = 5
+
+# Process name.
+[processes.ls]
+
+# Command to run in order to start this process.
+# Must be an absolute path.
+# Required.
+cmd = "/usr/bin/ls"
+
+# Arguments to pass to 'cmd'.
+# Defaults to [].
+args = ["-laR"]
+
+# Number of processes to start.
+# Defaults to 1, max 255.
+processes = 1
+
+# Mask for files created by the process.
+# Defaults to "022".
+umask = "022"
+
+# Working directory for the process.
+# Must be an absolute path.
+# Required.
+workingdir = "/tmp"
+
+# Start the program automatically when starting taskmaster. 
+# Defaults to false.
+autostart = true
+
+# Restart the process when it quits, options are:
+# - "no" (default): Never restart the process automatically.
+# - "on-failure[:max-retries]": Try to restart the process 
+#   <max-retries> times when it exits unexpectedly. Maximum value for 
+#   max-retries is 255.
+# - "always": Always restart when exiting, no matter the status.
+autorestart = "on-failure[:1]"
+
+# List of exit codes to be interpreted as success.
+# Defaults to [0].
+exitcodes = [0]
+
 
 # List of signals triggering a stop of the process, options are any 
 # FreeBSD signal (https://www.math.stonybrook.edu/~ccc/dfc/dfc/signals.html).
@@ -58,11 +156,11 @@ stoptime = 5
 
 # File the standard output of the process should be redirected to.
 # Defaults to "/tmp/<process name>.stdout".
-stdout = "/tmp/nginx.stdout"
+stdout = "/tmp/ls.stdout"
 
 # File the standard error of the process should be redirected to.
 # Defaults to "/tmp/<process name>.stderr".
-stderr = "/tmp/nginx.stderr"
+stderr = "/tmp/ls.stderr"
 
 # Key value pairs of environment variables to be injected into the process
 # at startup.


Reviews:
user: login: copilot-pull-request-reviewer[bot]
body: ## Pull Request Overview

This pull request introduces dynamic healthchecks by adding an optional healthcheck command in the configuration and a new Starting state to differentiate between processes using dynamic healthchecks and the simple starttime-based approach. Key changes include:
- Introducing the Starting state in the state machine (including display and monitor logic updates).
- Adding a HealthCheckRunner module and integrating healthcheck support in Process management.
- Updating configuration files and tests to support the new healthcheck functionality.

### Reviewed Changes

Copilot reviewed 20 out of 20 changed files in this pull request and generated 1 comment.

<details>
<summary>Show a summary per file</summary>

| File                                            | Description                                                                               |
| ----------------------------------------------- | ----------------------------------------------------------------------------------------- |
| src/run/statemachine/states.rs                   | Added a new Starting state and updated state transitions to account for dynamic healthchecks.  |
| src/run/statemachine/monitor.rs                  | Modified monitoring functions to handle the Starting state and updated healthcheck monitoring. |
| src/run/statemachine/healthcheck.rs              | Added a new HealthCheckRunner module to spawn and manage healthcheck processes.              |
| src/run/statemachine.rs                          | Exported the new healthcheck module.                                                      |
| src/run/proc.rs                                 | Integrated healthcheck support into the process lifecycle and updated type conversions.      |
| src/log.rs                                      | Updated logging macros to use process names.                                               |
| Other config and test files                     | Updated configuration and tests to allow specifying a healthcheck configuration.            |
</details>





state: COMMENTED
author_association: NONE
submitted_at: 2025-04-19T14:06:05Z
user: login: copilot-pull-request-reviewer[bot]
body: ## Pull Request Overview

This PR introduces dynamic healthchecks by adding an optional healthcheck field in the configuration, a new Starting state, and a HealthCheckRunner to encapsulate the complex healthcheck logic. Key changes include:
- The addition of the Starting state in the process state machine to differentiate dynamic and simple healthchecks.
- Refactoring of monitor functions to handle the new dynamic healthcheck logic.
- Updates in configuration types, tests, and logging macros to support healthcheck configuration.

### Reviewed Changes

Copilot reviewed 19 out of 19 changed files in this pull request and generated no comments.

<details>
<summary>Show a summary per file</summary>

| File                                   | Description                                                                |
| -------------------------------------- | -------------------------------------------------------------------------- |
| src/run/statemachine/states.rs          | Added the Starting state and updated Display and desired state logic       |
| src/run/statemachine/monitor.rs         | Refactored monitoring functions to support healthcheck vs. starting flows    |
| src/run/statemachine/healthcheck.rs     | Introduced HealthCheckRunner and its supporting methods for dynamic checks  |
| src/run/statemachine.rs                 | Exported the new healthcheck module                                        |
| src/run/proc.rs                         | Updated Process struct and methods to support healthcheck configuration     |
| src/log.rs                              | Modified logging macros to use process name instead of a raw identifier       |
| src/jsonrpc/short_process.rs            | Mapped the new Starting state for RPC output formatting                    |
| src/conf/*                              | Updated configuration types, tests, and examples to include healthcheck fields|
| config/*                                | Added and updated config examples to illustrate dynamic healthcheck support   |
| .rustfmt.toml, .clippy.toml             | Updated formatting and linting settings                                    |
</details>





state: COMMENTED
author_association: NONE
submitted_at: 2025-04-19T14:09:31Z
user: login: fbruggem
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-19T16:44:11Z
user: login: fbruggem
body: LGTM - nicely done with the OneShotChannel!

TL;DR I'm fine with 2 States if we rename them to clearer names but still wanted to share one perspective
`Starting` =>`HealthCheckTime`
`HealthCheck` => `HealthCheckDynamic`

### Perspective
About the `Starting` state. I understand that having both in one makes the function more complex - this problem can be "solved/dirty fixed" by 

```rust
match p.is_dynamic_health() {
    Some(_) => dynamic(),
    None => timed(),
}
```

I would rather have a monitor function branched out with this match than another state which basically has the same "outside" behaviour. 
Because `Starting` and `HealthCheck` have the same communication to the "outside".
Either - failed try again until max tries
Or - succeeded NICE go into healthy

Also when we change something about how health-check works we might get code duplication or forget to change it in both places.
state: CHANGES_REQUESTED
author_association: COLLABORATOR
submitted_at: 2025-04-19T17:31:36Z
user: login: copilot-pull-request-reviewer[bot]
body: ## Pull Request Overview

This PR introduces dynamic healthchecks to determine process health by allowing a configurable command instead of relying solely on a fixed start time. Key changes include adding a new optional healthcheck command field to the configuration, a new ProcessState variant (Starting), and refactoring related monitor and process logic to support dynamic healthchecks.

### Reviewed Changes

Copilot reviewed 25 out of 25 changed files in this pull request and generated 1 comment.

<details>
<summary>Show a summary per file</summary>

| File                                   | Description                                                                      |
| -------------------------------------- | -------------------------------------------------------------------------------- |
| src/run/statemachine/monitor.rs        | Refactored monitor functions to handle dynamic healthchecks and updated error handling. |
| src/run/statemachine/healthcheck.rs    | Introduced HealthCheckRunner to encapsulate healthcheck logic and asynchronous execution. |
| src/run/proc/error.rs                  | Added a new error variant (AlreadyRunning) for process start conflicts.          |
| src/run/proc.rs                        | Updated process lifecycle logic to work with dynamic healthchecks and new failure counters. |
| src/log.rs                             | Updated logging macros to extract process names consistently from Process objects. |
| src/jsonrpc/short_process.rs           | Added a new Starting variant to support dynamic healthcheck states in JSON output. |
| src/conf/...                           | Updated configuration parsing and tests to remove deprecated startup parameters. |
</details>



<details>
<summary>Comments suppressed due to low confidence (1)</summary>

**src/run/proc.rs:184**
* Review the 'AlreadyRunning' check in start(); ensure that returning ProcessError::AlreadyRunning correctly reflects the process lifecycle and that no necessary state resets are omitted.
```
if self.child.is_some() {
```
</details>


state: COMMENTED
author_association: NONE
submitted_at: 2025-04-20T08:57:40Z
user: login: copilot-pull-request-reviewer[bot]
body: ## Pull Request Overview

This PR implements dynamic healthchecks by adding an optional healthcheck field in the configuration, a new Starting state, and a dedicated HealthCheckRunner to encapsulate the dynamic healthcheck logic.  
- Adds support for command-based healthchecks instead of a fixed start time  
- Introduces a new Starting state and adjusts error handling and logging for healthcheck outcomes  
- Updates configuration and tests to reflect the new healthcheck approach

### Reviewed Changes

Copilot reviewed 25 out of 25 changed files in this pull request and generated no comments.

<details>
<summary>Show a summary per file</summary>

| File                                     | Description |
| ---------------------------------------- | ----------- |
| src/run/statemachine/monitor.rs          | Refactored healthcheck functions and updated logging calls to use process objects directly |
| src/run/statemachine/healthcheck.rs        | Introduced HealthCheckRunner to manage dynamic healthcheck execution and timeout handling |
| src/run/statemachine.rs                   | Added module visibility for healthcheck support |
| src/run/proc/error.rs                     | Added AlreadyRunning error variant to prevent duplicate process starts |
| src/run/proc.rs                           | Updated process logic to use HealthCheckRunner and revised failure counter logic |
| src/log.rs                                | Adjusted logging macros to derive process names from Process objects |
| config/*.toml, src/conf/*, tests          | Updated configuration and tests to work with dynamic healthcheck configuration |
</details>



<details>
<summary>Comments suppressed due to low confidence (1)</summary>

**src/run/statemachine/healthcheck.rs:60**
* There is a spelling error in the panic message: "startime()" should be "starttime()" for consistency with the rest of the codebase.
```
            _ => panic!("startime() called on an Uptime HealthCheck"),
```
</details>


state: COMMENTED
author_association: NONE
submitted_at: 2025-04-20T08:59:50Z
user: login: copilot-pull-request-reviewer[bot]
body: ## Pull Request Overview

This PR introduces dynamic healthchecks, adding a new optional healthcheck command configuration and a Starting state, while refactoring process state management and logging.  
- Implemented healthcheck command logic and corresponding state transitions.  
- Updated logging to pass process objects instead of just their names.  
- Refactored configuration and defaults for healthcheck-related settings.

### Reviewed Changes

Copilot reviewed 25 out of 25 changed files in this pull request and generated 1 comment.

<details>
<summary>Show a summary per file</summary>

| File                                      | Description                                                             |
| ----------------------------------------- | ----------------------------------------------------------------------- |
| src/run/statemachine/monitor.rs           | Refactored process state transitions and logging for dynamic healthchecks.|
| src/run/statemachine/healthcheck.rs         | Added HealthCheckRunner and related logic to run healthcheck commands.    |
| src/run/proc.rs                            | Updated process start and failure logic to support the new healthcheck mode.|
| src/log.rs                                 | Modified logging macros to extract process names from process objects.     |
| Various config and test files             | Adjusted configuration fields to use the new healthcheck settings.         |
</details>



<details>
<summary>Comments suppressed due to low confidence (1)</summary>

**src/run/statemachine/monitor.rs:21**
* [nitpick] For consistency in logging macro usage, consider standardizing the argument passed (either p or &p) across the module so that it uniformly calls p.name().
```
proc_info!(&p, "spawned, PID {}", pid);
```
</details>


state: COMMENTED
author_association: NONE
submitted_at: 2025-04-20T09:01:54Z

Review comments:
diff_hunk: @@ -33,6 +34,7 @@ impl Display for ProcessState {
         match self {
             ProcessState::Idle => write!(f, "idle"),
             ProcessState::Ready => write!(f, "ready"),
+            ProcessState::Starting(started_at) => write!(f, "starting up since {} seconds", Instant::now().duration_since(*started_at).as_secs()),
path: src/run/statemachine/states.rs
user: login: Copilot
body: Consider caching Instant::now() in a variable before calculating the elapsed time to avoid minor discrepancies caused by multiple calls within the same match arm.
created_at: 2025-04-19T14:06:05Z
author_association: NONE
diff_hunk: @@ -24,21 +24,24 @@ pub struct Process {
     name: String,
     child: Option<Child>,
     conf: ProcessConfig,
-    startup_failures: u8,
-    runtime_failures: u8,
+    startup_failures: usize,
+    healthcheck: Option<HealthCheckRunner>,
+    runtime_failures: usize,
     state: ProcessState,
     desired_states: VecDeque<ProcessState>,
 }
 
 impl Process {
     pub fn from_process_config(conf: conf::proc::ProcessConfig, proc_name: &str) -> Self {
         let is_autostart = conf.autostart();
+        let healthcheck = conf.healthcheck();
path: src/run/proc.rs
user: login: fbruggem
body: NIT PICK: cloning here and not on `conf: conf.clone()` should be less expensive because you only clone a subset of the struct and not all of it.
created_at: 2025-04-19T16:44:11Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,89 @@
+use serde::Deserialize;
+
+fn dflt_timeout() -> usize {
+    10
+}
+
+fn dflt_backoff() -> usize {
+    5
+}
+
+fn dflt_retries() -> usize {
+    5
+}
+
+#[derive(Deserialize, Debug, Clone)]
+#[serde(untagged)]
+pub enum HealthCheckType {
+    Command {
+        cmd: String,
+        #[serde(default)]
+        args: Vec<String>,
+    },
+    Uptime {
+        starttime: u8,
+    },
+}
+
+#[allow(unused)]
+#[derive(Deserialize, Debug, Clone)]
+pub struct HealthCheck {
+    #[serde(flatten)]
+    check: HealthCheckType,
+    #[serde(default = "dflt_timeout")]
+    timeout: usize,
+    #[serde(default = "dflt_retries")]
+    retries: usize,
+    #[serde(default = "dflt_backoff")]
+    backoff: usize,
+}
+
+impl Default for HealthCheck {
+    fn default() -> Self {
+        Self {
+            check: HealthCheckType::Uptime { starttime: 5 },
+            timeout: dflt_timeout(),
+            retries: dflt_retries(),
+            backoff: dflt_backoff(),
+        }
+    }
+}
+
+impl HealthCheck {
+    pub fn healthcheck(&self) -> &HealthCheckType {
+        &self.check
+    }
+
+    pub fn cmd(&self) -> &str {
+        match &self.check {
+            HealthCheckType::Command { cmd, .. } => cmd,
+            _ => panic!("cmd() called on an Uptime HealthCheck"),
+        }
+    }
+
+    pub fn args(&self) -> &[String] {
+        match &self.check {
+            HealthCheckType::Command { cmd: _, args } => args,
+            _ => panic!("args() called on an Uptime HealthCheck"),
+        }
+    }
+
+    pub fn starttime(&self) -> u8 {
+        match &self.check {
+            HealthCheckType::Uptime { starttime } => *starttime,
+            _ => panic!("startime() called on a Command HealthCheck"),
+        }
path: src/conf/proc/types/healthcheck.rs
user: login: Copilot
body: Typo in panic message: 'startime' should be 'starttime' to maintain consistency in naming.
```suggestion
            _ => panic!("starttime() called on a Command HealthCheck"),
```
created_at: 2025-04-20T08:57:39Z
author_association: NONE
diff_hunk: @@ -12,63 +18,125 @@ pub fn monitor_ready(p: &mut Process) -> Option<ProcessState> {
     match p.start() {
         Ok(()) => {
             let pid = p.id().expect("id should always be set if the process is running");
-            proc_info!(&p.name(), "spawned, PID {}", pid);
+            proc_info!(&p, "spawned, PID {}", pid);
             Some(ProcessState::HealthCheck(Instant::now()))
         }
         Err(err) => {
-            proc_warning!(&p.name(), "failed to start: {}", err);
-            p.increment_startup_failures();
+            proc_warning!(&p, "failed to start: {}", err);
+            p.increment_healthcheck_failures();
             Some(ProcessState::Failed(Box::new(ProcessState::HealthCheck(Instant::now()))))
         }
     }
 }
 
-pub fn monitor_health_check(started_at: &Instant, p: &mut Process) -> Option<ProcessState> {
-    if p.healthy(*started_at) {
-        proc_info!(&p.name(), "has been running for {} seconds, marking as healthy", p.config().starttime());
-
-        return Some(ProcessState::Healthy);
-    }
+/// Check whether `p` exited and return the appropriate state based on its
+/// exit code.
+///
+/// Returns `None` if `p` is running.
+fn exited_state(p: &mut Process) -> Option<ProcessState> {
+    assert!(matches!(p.state(), ProcessState::HealthCheck(_) | ProcessState::Healthy));
 
     if let Some(code) = p.exited() {
+        let msg = match p.state() {
+            ProcessState::Healthy => "while healthy".to_string(),
+            ProcessState::HealthCheck(t) => format!("in healthcheck after {}s", Instant::now().duration_since(t).as_secs()),
+            _ => String::new(),
+        };
+
         if !p.config().exitcodes().contains(&code) {
-            proc_warning!(&p.name(), "exited during healthcheck with unexpected code ({})", code);
-            return Some(ProcessState::Failed(Box::new(ProcessState::HealthCheck(*started_at))));
+            proc_warning!(&p, "exited {} with unexpected code ({})", msg, code);
+            return Some(ProcessState::Failed(Box::new(p.state())));
path: src/run/statemachine/monitor.rs
user: login: Copilot
body: Wrapping the current process state in Failed may propagate an unintended state (e.g. Healthy) when an unexpected exit code occurs; consider explicitly using the healthcheck start time or a dedicated healthcheck state for clarity.
```suggestion
            match p.state() {
                ProcessState::Healthy => {
                    return Some(ProcessState::Failed(Box::new(ProcessState::Healthy)));
                }
                ProcessState::HealthCheck(t) => {
                    return Some(ProcessState::Failed(Box::new(ProcessState::HealthCheck(t))));
                }
                _ => unreachable!("Unexpected state in exited_state"),
            }
```
created_at: 2025-04-20T09:01:54Z
author_association: NONE

PR comments:
user: login: winstonallo
created_at: 2025-04-20T05:43:58Z
body: > LGTM - nicely done with the OneShotChannel!
> 
> 
> 
> TL;DR I'm fine with 2 States if we rename them to clearer names but still wanted to share one perspective
> 
> `Starting` =>`HealthCheckTime`
> 
> `HealthCheck` => `HealthCheckDynamic`
> 
> 
> 
> ### Perspective
> 
> About the `Starting` state. I understand that having both in one makes the function more complex - this problem can be "solved/dirty fixed" by 
> 
> 
> 
> ```rust
> 
> match p.is_dynamic_health() {
> 
>     Some(_) => dynamic(),
> 
>     None => timed(),
> 
> }
> 
> ```
> 
> 
> 
> I would rather have a monitor function branched out with this match than another state which basically has the same "outside" behaviour. 
> 
> Because `Starting` and `HealthCheck` have the same communication to the "outside".
> 
> Either - failed try again until max tries
> 
> Or - succeeded NICE go into healthy
> 
> 
> 
> Also when we change something about how health-check works we might get code duplication or forget to change it in both places.

Sure we can do that, just having a hard time accepting starttime as a **health**check, a process stuck in an infinite loop would pass it. It's probably just me being pedantic though, I'll leave it as one state for now.
author_association: OWNER
user: login: winstonallo
created_at: 2025-04-20T08:13:20Z
body: @FelixBrgm fixed 2 other things:
1. when a dynamic healthcheck fails, the process was always being restarted even if it was still running
2. fixed #30 - entries in the process table are kept until the parent calls wait() on the process, fixed by adding `let _ = p.exited();` in `monitor_stopped`
author_association: OWNER
user: login: winstonallo
created_at: 2025-04-20T08:54:32Z
body: Also moved the entire healthcheck config to the HealthCheckRunner struct
author_association: OWNER
user: login: winstonallo
created_at: 2025-04-20T11:35:35Z
body: merged by accident -.-
author_association: OWNER
user: login: fbruggem
created_at: 2025-04-20T13:27:13Z
body: If its now one state it should be fine^*
author_association: COLLABORATOR