Repository: code-ready/crc
Pull request #1721 (2020-11-24 15:21:27)

Improve/fix systemd state parsing
The current code does not handle correctly stopped/not found units, and it also does not support systemd socket units.
This PR will fix this. It starts with preparatory work so that local command runners and ssh command runners behave the same
(separate stdout and stderr), since at the moment local command runners use separate output streams, while ssh command runners
combine stdout and stderr.

**Fixes:** Issue #1705

## Solution/Idea

This fixes the 'libvirtd is running' preflight check on systems using socket activation for libvirtd (libvirtd.socket is `active (listening`) and libvirtd.service is stopped after a few minutes of inactivity).

## Testing

This should be testable on rhel 8.3 by enabling/starting libvirtd.socket, stopping libvirtd.service, and running `crc setup`, the 'check-libvirt-running' test should succeed without any attempt at fixing it up.

openshift-ci-robot: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/code-ready/crc/pull/1721#" title="Author self-approved">cfergeau</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=code-ready%2Fcrc).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/code-ready/crc/blob/master/OWNERS)~~ [cfergeau]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
praveenkumar: /lgtm

https://github.com/code-ready/crc/pull/1721#discussion_r530353550 @cfergeau Are you going to force push that?
openshift-ci-robot: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/code-ready/crc/pull/1721#" title="Author self-approved">cfergeau</a>*, *<a href="https://github.com/code-ready/crc/pull/1721#pullrequestreview-539069974" title="Approved">guillaumerose</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=code-ready%2Fcrc).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/code-ready/crc/blob/master/OWNERS)~~ [cfergeau,guillaumerose]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
cfergeau: > /lgtm
> 
> [#1721 (comment)](https://github.com/code-ready/crc/pull/1721#discussion_r530353550) @cfergeau Are you going to force push that?

Yes I will, I was expecting to need some changes, and to push everything at once :)
openshift-ci-robot: New changes are detected. LGTM label has been removed.
openshift-ci-robot: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/code-ready/crc/pull/1721#" title="Author self-approved">cfergeau</a>*, *<a href="https://github.com/code-ready/crc/pull/1721#pullrequestreview-539069974" title="Approved">guillaumerose</a>*, *<a href="https://github.com/code-ready/crc/pull/1721#pullrequestreview-539718754" title="Approved">praveenkumar</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=code-ready%2Fcrc).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/code-ready/crc/blob/master/OWNERS)~~ [cfergeau,guillaumerose,praveenkumar]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->

Patch:
diff --git a/.travis.yml b/.travis.yml
index 61ee2a35f..b5de06940 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,10 +12,5 @@ before_script:
   - sudo apt-get -qq -y install podman
 
 script:
-- make
-- make vendorcheck
-- make cross-lint
-- make cross
-- make test
-- make build_integration
+- make check
 - make BUNDLE_DIR=/tmp MOCK_BUNDLE=true release
diff --git a/Makefile b/Makefile
index 857cdf33f..0595b0b0e 100644
--- a/Makefile
+++ b/Makefile
@@ -74,6 +74,9 @@ vendor:
 vendorcheck:
 	./verify-vendor.sh
 
+.PHONY: check
+check: cross build_integration $(HOST_BUILD_DIR)/crc-embedder test cross-lint vendorcheck
+
 # Start of the actual build targets
 
 .PHONY: install
diff --git a/pkg/crc/cluster/cluster.go b/pkg/crc/cluster/cluster.go
index efdf99a7f..49686ba09 100644
--- a/pkg/crc/cluster/cluster.go
+++ b/pkg/crc/cluster/cluster.go
@@ -22,7 +22,7 @@ const vmPullSecretPath = "/var/lib/kubelet/config.json"
 
 func WaitForSSH(sshRunner *ssh.Runner) error {
 	checkSSHConnectivity := func() error {
-		_, err := sshRunner.Run("exit 0")
+		_, _, err := sshRunner.Run("exit 0")
 		if err != nil {
 			return &errors.RetriableError{Err: err}
 		}
@@ -52,7 +52,7 @@ func CheckCertsValidity(sshRunner *ssh.Runner) (bool, bool, error) {
 }
 
 func checkCertValidity(sshRunner *ssh.Runner, cert string) (bool, error) {
-	output, err := sshRunner.Run(fmt.Sprintf(`date --date="$(sudo openssl x509 -in %s -noout -enddate | cut -d= -f 2)" --iso-8601=seconds`, cert))
+	output, _, err := sshRunner.Run(fmt.Sprintf(`date --date="$(sudo openssl x509 -in %s -noout -enddate | cut -d= -f 2)" --iso-8601=seconds`, cert))
 	if err != nil {
 		return false, err
 	}
@@ -71,7 +71,7 @@ func checkCertValidity(sshRunner *ssh.Runner, cert string) (bool, error) {
 func GetRootPartitionUsage(sshRunner *ssh.Runner) (int64, int64, error) {
 	cmd := "df -B1 --output=size,used,target /sysroot | tail -1"
 
-	out, err := sshRunner.Run(cmd)
+	out, _, err := sshRunner.Run(cmd)
 
 	if err != nil {
 		return 0, 0, err
@@ -257,7 +257,7 @@ func addProxyCACertToInstance(sshRunner *ssh.Runner, proxy *network.ProxyConfig)
 	if err := sshRunner.CopyData([]byte(proxy.ProxyCACert), "/etc/pki/ca-trust/source/anchors/openshift-config-user-ca-bundle.crt", 0600); err != nil {
 		return err
 	}
-	if _, err := sshRunner.Run("sudo update-ca-trust"); err != nil {
+	if _, _, err := sshRunner.Run("sudo update-ca-trust"); err != nil {
 		return err
 	}
 	return nil
@@ -280,7 +280,7 @@ func (p *PullSecret) Value() (string, error) {
 }
 
 func EnsurePullSecretPresentOnInstanceDisk(sshRunner *ssh.Runner, pullSecret *PullSecret) error {
-	if _, err := sshRunner.Run(fmt.Sprintf("test -e %s", vmPullSecretPath)); err == nil {
+	if _, _, err := sshRunner.Run(fmt.Sprintf("test -e %s", vmPullSecretPath)); err == nil {
 		return nil
 	}
 	logging.Info("Adding user's pull secret to instance disk...")
diff --git a/pkg/crc/machine/machine.go b/pkg/crc/machine/machine.go
index ab7e51455..b11d30cfc 100644
--- a/pkg/crc/machine/machine.go
+++ b/pkg/crc/machine/machine.go
@@ -288,14 +288,14 @@ func (client *client) Start(startConfig StartConfig) (*StartResult, error) {
 	}
 
 	// Trigger disk resize, this will be a no-op if no disk size change is needed
-	if _, err = sshRunner.Run("sudo xfs_growfs / >/dev/null"); err != nil {
+	if _, _, err = sshRunner.Run("sudo xfs_growfs / >/dev/null"); err != nil {
 		return nil, errors.Wrap(err, "Error updating filesystem size")
 	}
 
 	// Start network time synchronization if `CRC_DEBUG_ENABLE_STOP_NTP` is not set
 	if stopNtp, _ := strconv.ParseBool(os.Getenv("CRC_DEBUG_ENABLE_STOP_NTP")); !stopNtp {
 		logging.Info("Starting network time synchronization in CodeReady Containers VM")
-		if _, err := sshRunner.Run("sudo timedatectl set-ntp on"); err != nil {
+		if _, _, err := sshRunner.Run("sudo timedatectl set-ntp on"); err != nil {
 			return nil, errors.Wrap(err, "Failed to start network time synchronization")
 		}
 	}
@@ -755,14 +755,14 @@ func updateSSHKeyPair(sshRunner *crcssh.Runner) error {
 		return err
 	}
 
-	authorizedKeys, err := sshRunner.Run("cat /home/core/.ssh/authorized_keys")
+	authorizedKeys, _, err := sshRunner.Run("cat /home/core/.ssh/authorized_keys")
 	if err == nil && strings.TrimSpace(authorizedKeys) == strings.TrimSpace(string(publicKey)) {
 		return nil
 	}
 
 	logging.Info("Updating authorized keys ...")
 	cmd := fmt.Sprintf("echo '%s' > /home/core/.ssh/authorized_keys; chmod 644 /home/core/.ssh/authorized_keys", publicKey)
-	_, err = sshRunner.Run(cmd)
+	_, _, err = sshRunner.Run(cmd)
 	if err != nil {
 		return err
 	}
diff --git a/pkg/crc/network/nameservers.go b/pkg/crc/network/nameservers.go
index 8bdb6bb79..5fb0acfe7 100644
--- a/pkg/crc/network/nameservers.go
+++ b/pkg/crc/network/nameservers.go
@@ -11,7 +11,7 @@ import (
 // HasGivenNameserversConfigured returns true if the instance uses a provided nameserver.
 func HasGivenNameserversConfigured(sshRunner *ssh.Runner, nameserver NameServer) (bool, error) {
 	cmd := "cat /etc/resolv.conf"
-	out, err := sshRunner.Run(cmd)
+	out, _, err := sshRunner.Run(cmd)
 
 	if err != nil {
 		return false, err
@@ -22,7 +22,7 @@ func HasGivenNameserversConfigured(sshRunner *ssh.Runner, nameserver NameServer)
 
 func GetResolvValuesFromInstance(sshRunner *ssh.Runner) (*ResolvFileValues, error) {
 	cmd := "cat /etc/resolv.conf"
-	out, err := sshRunner.Run(cmd)
+	out, _, err := sshRunner.Run(cmd)
 
 	if err != nil {
 		return nil, err
@@ -54,7 +54,7 @@ func AddNameserversToInstance(sshRunner *ssh.Runner, nameservers []NameServer) e
 
 // writes nameserver to the /etc/resolv.conf inside the instance
 func addNameserverToInstance(sshRunner *ssh.Runner, nameserver NameServer) error {
-	_, err := sshRunner.Run(fmt.Sprintf("NS=%s; cat /etc/resolv.conf |grep -i \"^nameserver $NS\" || echo \"nameserver $NS\" | sudo tee -a /etc/resolv.conf", nameserver.IPAddress))
+	_, _, err := sshRunner.Run(fmt.Sprintf("NS=%s; cat /etc/resolv.conf |grep -i \"^nameserver $NS\" || echo \"nameserver $NS\" | sudo tee -a /etc/resolv.conf", nameserver.IPAddress))
 	if err != nil {
 		return fmt.Errorf("%s: %s", "Error adding nameserver", err.Error())
 	}
diff --git a/pkg/crc/oc/oc.go b/pkg/crc/oc/oc.go
index 8d5cb88cd..4161926d1 100644
--- a/pkg/crc/oc/oc.go
+++ b/pkg/crc/oc/oc.go
@@ -59,7 +59,7 @@ type SSHRunner struct {
 
 func UseOCWithSSH(sshRunner *ssh.Runner) Config {
 	return Config{
-		Runner:           ssh.NewRemoteCommandRunner(sshRunner),
+		Runner:           sshRunner,
 		OcExecutablePath: "oc",
 		KubeconfigPath:   "/opt/kubeconfig",
 		Context:          constants.DefaultContext,
diff --git a/pkg/crc/preflight/preflight_checks_linux.go b/pkg/crc/preflight/preflight_checks_linux.go
index d7ac48aa7..48cb4a963 100644
--- a/pkg/crc/preflight/preflight_checks_linux.go
+++ b/pkg/crc/preflight/preflight_checks_linux.go
@@ -223,12 +223,19 @@ func checkLibvirtServiceRunning() error {
 	libvirtSystemdUnits := []string{"virtqemud.socket", "libvirtd.socket", "virtqemud.service", "libvirtd.service"}
 	for _, unit := range libvirtSystemdUnits {
 		status, err := sd.Status(unit)
-		if err == nil && status == states.Running {
-			logging.Debugf("%s is running", unit)
-			return nil
+		if err == nil {
+			switch status {
+			case states.Running:
+				logging.Debugf("%s is running", unit)
+				return nil
+			case states.Listening:
+				logging.Debugf("%s is listening", unit)
+				return nil
+			default:
+				logging.Debugf("%s is neither running nor listening", unit)
+			}
 		}
 
-		logging.Debugf("%s is not running", unit)
 	}
 
 	logging.Warnf("No active (running) libvirtd systemd unit could be found - make sure one of libvirt systemd units is enabled so that it's autostarted at boot time.")
diff --git a/pkg/crc/services/dns/dns.go b/pkg/crc/services/dns/dns.go
index 091019786..b3e41396b 100644
--- a/pkg/crc/services/dns/dns.go
+++ b/pkg/crc/services/dns/dns.go
@@ -49,17 +49,17 @@ func setupDnsmasq(serviceConfig services.ServicePostStartConfig) error {
 	}
 
 	// Remove the dnsmasq container if it exists during the VM stop cycle
-	_, _ = serviceConfig.SSHRunner.Run("sudo podman rm -f dnsmasq")
+	_, _, _ = serviceConfig.SSHRunner.Run("sudo podman rm -f dnsmasq")
 
 	// Remove the CNI network definition forcefully
 	// https://github.com/containers/libpod/issues/2767
 	// TODO: We need to revisit it once podman update the CNI plugins.
-	_, _ = serviceConfig.SSHRunner.Run(fmt.Sprintf("sudo rm -f /var/lib/cni/networks/podman/%s", dnsContainerIP))
+	_, _, _ = serviceConfig.SSHRunner.Run(fmt.Sprintf("sudo rm -f /var/lib/cni/networks/podman/%s", dnsContainerIP))
 
 	// Start the dnsmasq container
 	dnsServerRunCmd := fmt.Sprintf("sudo podman run  --ip %s --name dnsmasq -v %s:/etc/dnsmasq.conf -p 53:%d/udp --privileged -d %s",
 		dnsContainerIP, dnsConfigFilePathInInstance, dnsServicePort, dnsContainerImage)
-	if _, err := serviceConfig.SSHRunner.Run(dnsServerRunCmd); err != nil {
+	if _, _, err := serviceConfig.SSHRunner.Run(dnsServerRunCmd); err != nil {
 		return err
 	}
 	return nil
@@ -102,7 +102,7 @@ func CheckCRCLocalDNSReachable(serviceConfig services.ServicePostStartConfig) (s
 	var queryOutput string
 	var err error
 	checkLocalDNSReach := func() error {
-		queryOutput, err = serviceConfig.SSHRunner.Run(fmt.Sprintf("host -R 3 %s", appsURI))
+		queryOutput, _, err = serviceConfig.SSHRunner.Run(fmt.Sprintf("host -R 3 %s", appsURI))
 		if err != nil {
 			return &errors.RetriableError{Err: err}
 		}
@@ -116,7 +116,8 @@ func CheckCRCLocalDNSReachable(serviceConfig services.ServicePostStartConfig) (s
 }
 
 func CheckCRCPublicDNSReachable(serviceConfig services.ServicePostStartConfig) (string, error) {
-	return serviceConfig.SSHRunner.Run(fmt.Sprintf("host -R 3 %s", publicDNSQueryURI))
+	stdout, _, err := serviceConfig.SSHRunner.Run(fmt.Sprintf("host -R 3 %s", publicDNSQueryURI))
+	return stdout, err
 }
 
 func addOpenShiftHosts(serviceConfig services.ServicePostStartConfig) error {
diff --git a/pkg/crc/ssh/client.go b/pkg/crc/ssh/client.go
index 1cd5c6023..f3865ee1c 100644
--- a/pkg/crc/ssh/client.go
+++ b/pkg/crc/ssh/client.go
@@ -1,6 +1,7 @@
 package ssh
 
 import (
+	"bytes"
 	"fmt"
 	"io/ioutil"
 	"net"
@@ -12,7 +13,7 @@ import (
 )
 
 type Client interface {
-	Output(command string) (string, error)
+	Run(command string) ([]byte, []byte, error)
 	Close()
 }
 
@@ -87,18 +88,23 @@ func (client *NativeClient) session() (*ssh.Session, error) {
 	return session, err
 }
 
-func (client *NativeClient) Output(command string) (string, error) {
+func (client *NativeClient) Run(command string) ([]byte, []byte, error) {
 	session, err := client.session()
 	if err != nil {
-		return "", err
+		return nil, nil, err
 	}
 	defer session.Close()
 
-	output, err := session.CombinedOutput(command)
-	if err != nil {
-		return "", err
-	}
-	return string(output), nil
+	var (
+		stdout bytes.Buffer
+		stderr bytes.Buffer
+	)
+	session.Stdout = &stdout
+	session.Stderr = &stderr
+
+	err = session.Run(command)
+
+	return stdout.Bytes(), stderr.Bytes(), err
 }
 
 func (client *NativeClient) Close() {
diff --git a/pkg/crc/ssh/ssh.go b/pkg/crc/ssh/ssh.go
index 170ae4400..d6dfb9e9f 100644
--- a/pkg/crc/ssh/ssh.go
+++ b/pkg/crc/ssh/ssh.go
@@ -9,7 +9,6 @@ import (
 
 	"github.com/code-ready/crc/pkg/crc/constants"
 	"github.com/code-ready/crc/pkg/crc/logging"
-	crcos "github.com/code-ready/crc/pkg/os"
 )
 
 type Runner struct {
@@ -30,20 +29,30 @@ func (runner *Runner) Close() {
 	runner.client.Close()
 }
 
-// Create a host using the driver's config
-func (runner *Runner) Run(command string) (string, error) {
-	return runner.runSSHCommandFromDriver(command, false)
+func (runner *Runner) Run(cmd string, args ...string) (string, string, error) {
+	if len(args) != 0 {
+		cmd = fmt.Sprintf("%s %s", cmd, strings.Join(args, " "))
+	}
+	return runner.runSSHCommand(cmd, false)
 }
 
-func (runner *Runner) RunPrivate(command string) (string, error) {
-	return runner.runSSHCommandFromDriver(command, true)
+func (runner *Runner) RunPrivate(cmd string, args ...string) (string, string, error) {
+	if len(args) != 0 {
+		cmd = fmt.Sprintf("%s %s", cmd, strings.Join(args, " "))
+	}
+	return runner.runSSHCommand(cmd, true)
+}
+
+func (runner *Runner) RunPrivileged(reason string, cmdAndArgs ...string) (string, string, error) {
+	commandline := fmt.Sprintf("sudo %s", strings.Join(cmdAndArgs, " "))
+	return runner.runSSHCommand(commandline, false)
 }
 
 func (runner *Runner) CopyData(data []byte, destFilename string, mode os.FileMode) error {
 	logging.Debugf("Creating %s with permissions 0%o in the CRC VM", destFilename, mode)
 	base64Data := base64.StdEncoding.EncodeToString(data)
 	command := fmt.Sprintf("sudo install -m 0%o /dev/null %s && cat <<EOF | base64 --decode | sudo tee %s\n%s\nEOF", mode, destFilename, destFilename, base64Data)
-	_, err := runner.RunPrivate(command)
+	_, _, err := runner.RunPrivate(command)
 
 	return err
 }
@@ -56,14 +65,14 @@ func (runner *Runner) CopyFile(srcFilename string, destFilename string, mode os.
 	return runner.CopyData(data, destFilename, mode)
 }
 
-func (runner *Runner) runSSHCommandFromDriver(command string, runPrivate bool) (string, error) {
+func (runner *Runner) runSSHCommand(command string, runPrivate bool) (string, string, error) {
 	if runPrivate {
 		logging.Debugf("About to run SSH command with hidden output")
 	} else {
 		logging.Debugf("About to run SSH command:\n%s", command)
 	}
 
-	output, err := runner.client.Output(command)
+	stdout, stderr, err := runner.client.Run(command)
 	if runPrivate {
 		if err != nil {
 			logging.Debugf("SSH command failed")
@@ -71,45 +80,15 @@ func (runner *Runner) runSSHCommandFromDriver(command string, runPrivate bool) (
 			logging.Debugf("SSH command succeeded")
 		}
 	} else {
-		logging.Debugf("SSH command results: err: %v, output: %s", err, output)
+		logging.Debugf("SSH command results: err: %v, output: %s", err, string(stdout))
 	}
 
 	if err != nil {
-		return "", fmt.Errorf(`ssh command error:
+		return "", "", fmt.Errorf(`ssh command error:
 command : %s
 err     : %v
-output  : %s`, command, err, output)
+output  : %s`, command, err, string(stdout))
 	}
 
-	return output, nil
-}
-
-type remoteCommandRunner struct {
-	sshRunner *Runner
-}
-
-func (cmdRunner *remoteCommandRunner) Run(cmd string, args ...string) (string, string, error) {
-	commandline := fmt.Sprintf("%s %s", cmd, strings.Join(args, " "))
-	out, err := cmdRunner.sshRunner.Run(commandline)
-	return out, "", err
-}
-
-func (cmdRunner *remoteCommandRunner) RunPrivate(cmd string, args ...string) (string, string, error) {
-	commandline := fmt.Sprintf("%s %s", cmd, strings.Join(args, " "))
-	out, err := cmdRunner.sshRunner.RunPrivate(commandline)
-	return out, "", err
-}
-
-func (cmdRunner *remoteCommandRunner) RunPrivileged(reason string, cmdAndArgs ...string) (string, string, error) {
-	commandline := fmt.Sprintf("sudo %s", strings.Join(cmdAndArgs, " "))
-
-	out, err := cmdRunner.sshRunner.Run(commandline)
-
-	return out, "", err
-}
-
-func NewRemoteCommandRunner(runner *Runner) crcos.CommandRunner {
-	return &remoteCommandRunner{
-		sshRunner: runner,
-	}
+	return string(stdout), string(stderr), nil
 }
diff --git a/pkg/crc/systemd/states/state.go b/pkg/crc/systemd/states/state.go
index 19907e30b..4b5609adb 100644
--- a/pkg/crc/systemd/states/state.go
+++ b/pkg/crc/systemd/states/state.go
@@ -9,6 +9,7 @@ type State int
 const (
 	Unknown State = iota
 	Running
+	Listening
 	Stopped
 	NotFound
 	Error
@@ -17,6 +18,7 @@ const (
 var states = []string{
 	"unknown",
 	"active (running)",
+	"active (listening)",
 	"inactive (dead)",
 	"could not be found",
 	"error",
@@ -33,6 +35,9 @@ func Compare(input string) State {
 	if strings.Contains(input, states[Running]) {
 		return Running
 	}
+	if strings.Contains(input, states[Listening]) {
+		return Listening
+	}
 	if strings.Contains(input, states[Stopped]) {
 		return Stopped
 	}
diff --git a/pkg/crc/systemd/systemd.go b/pkg/crc/systemd/systemd.go
index e7b69ed9e..825f26b25 100644
--- a/pkg/crc/systemd/systemd.go
+++ b/pkg/crc/systemd/systemd.go
@@ -15,7 +15,7 @@ type Commander struct {
 
 func NewInstanceSystemdCommander(sshRunner *ssh.Runner) *Commander {
 	return &Commander{
-		commandRunner: ssh.NewRemoteCommandRunner(sshRunner),
+		commandRunner: sshRunner,
 	}
 }
 
@@ -78,6 +78,15 @@ func (c Commander) service(name string, action actions.Action) (states.State, er
 	}
 
 	if err != nil {
+		state := states.Compare(stdOut)
+		if state != states.Unknown {
+			return state, nil
+		}
+		state = states.Compare(stdErr)
+		if state == states.NotFound {
+			return state, nil
+		}
+
 		return states.Error, fmt.Errorf("Executing systemctl action failed: %s %v: %s", stdOut, err, stdErr)
 	}
 


Test patch:
diff --git a/pkg/crc/ssh/ssh_test.go b/pkg/crc/ssh/ssh_test.go
index 96bf698cc..cbfb2292b 100644
--- a/pkg/crc/ssh/ssh_test.go
+++ b/pkg/crc/ssh/ssh_test.go
@@ -52,17 +52,11 @@ func TestRunner(t *testing.T) {
 	assert.NoError(t, err)
 	defer runner.Close()
 
-	bin, err := runner.Run("echo hello")
+	bin, _, err := runner.Run("echo hello")
 	assert.NoError(t, err)
 	assert.Equal(t, "hello", bin)
 	assert.NoError(t, runner.CopyData([]byte(`hello world`), "/hello", 0644))
 
-	cmdRunner := NewRemoteCommandRunner(runner)
-	stdout, stderr, err := cmdRunner.Run("echo", "hello")
-	assert.NoError(t, err)
-	assert.Equal(t, "hello", stdout)
-	assert.Empty(t, stderr)
-
 	assert.Equal(t, 1, *totalConn)
 }
 
diff --git a/pkg/crc/systemd/systemd_linux_test.go b/pkg/crc/systemd/systemd_linux_test.go
index 4622e5381..43dd0349f 100644
--- a/pkg/crc/systemd/systemd_linux_test.go
+++ b/pkg/crc/systemd/systemd_linux_test.go
@@ -1,6 +1,7 @@
 package systemd
 
 import (
+	"errors"
 	"fmt"
 	"testing"
 
@@ -34,6 +35,26 @@ func TestSystemd(t *testing.T) {
 	assert.Error(t, err)
 }
 
+func TestSystemdStatuses(t *testing.T) {
+	systemctl, _ := newMockCommander(t)
+
+	status, err := systemctl.Status("running.service")
+	assert.NoError(t, err)
+	assert.Equal(t, states.Running.String(), status.String())
+
+	status, err = systemctl.Status("listening.socket")
+	assert.NoError(t, err)
+	assert.Equal(t, states.Listening.String(), status.String())
+
+	status, err = systemctl.Status("stopped.service")
+	assert.NoError(t, err)
+	assert.Equal(t, states.Stopped.String(), status.String())
+
+	status, err = systemctl.Status("notfound.service")
+	assert.NoError(t, err)
+	assert.Equal(t, states.NotFound.String(), status.String())
+}
+
 type mockSystemdRunner struct {
 	test    *testing.T
 	failing bool
@@ -41,7 +62,21 @@ type mockSystemdRunner struct {
 
 func (r *mockSystemdRunner) Run(command string, args ...string) (string, string, error) {
 	assertSystemCtlCommand(r.test, "status", command, args)
-	return r.status(states.Running)
+	assert.GreaterOrEqual(r.test, len(args), 2)
+
+	unitName := args[1]
+	switch unitName {
+	case "running.service":
+		return r.status(states.Running)
+	case "listening.socket":
+		return r.status(states.Listening)
+	case "stopped.service":
+		return r.status(states.Stopped)
+	case "notfound.service":
+		return r.status(states.NotFound)
+	default:
+		return r.status(states.Running)
+	}
 }
 
 func (r *mockSystemdRunner) RunPrivate(command string, args ...string) (string, string, error) {
@@ -69,20 +104,69 @@ func (r *mockSystemdRunner) setFailing(failing bool) {
 	r.failing = failing
 }
 
+const (
+	statusRunning string = `● running.service - running service
+     Loaded: loaded (/usr/lib/systemd/system/running.service; enabled; vendor preset: enabled)
+     Active: active (running) since Tue 2020-11-24 14:52:20 CET; 15s ago
+TriggeredBy: ● listening.socket
+       Docs: man:running(8)
+             https://running.example.com
+   Main PID: 224516 (running)
+      Tasks: 19 (limit: 32768)
+     Memory: 36.9M
+        CPU: 414ms
+     CGroup: /system.slice/running.service
+             ├─ 19588 /usr/sbin/true
+             ├─ 19589 /usr/sbin/true
+             └─224516 /usr/sbin/running
+`
+	statusListening string = `● listening.socket - listening socket
+     Loaded: loaded (/usr/lib/systemd/system/listening.socket; enabled; vendor preset: disabled)
+     Active: active (listening) since Mon 2020-11-16 17:02:29 CET; 1 weeks 0 days ago
+   Triggers: ● running.service
+     Listen: /run/listening/listening-sock (Stream)
+      Tasks: 0 (limit: 38160)
+     Memory: 0B
+        CPU: 0
+     CGroup: /system.slice/listening.socket
+`
+	statusStopped string = `● stopped.service - stopped service
+     Loaded: loaded (/usr/lib/systemd/system/stopped.service; enabled; vendor preset: enabled)
+     Active: inactive (dead) since Tue 2020-11-17 12:22:15 CET; 1 weeks 0 days ago
+TriggeredBy: ● listening.socket
+       Docs: man:stopped(8)
+             https://stopped.example.com
+    Process: 64922 ExecStart=/usr/sbin/true (code=exited, status=0/SUCCESS)
+   Main PID: 64922 (code=exited, status=0/SUCCESS)
+        CPU: 327ms
+`
+	statusNotFound string = "Unit notfound.service could not be found."
+)
+
 func (r *mockSystemdRunner) status(s states.State) (string, string, error) {
 	var (
 		err    error
 		stdout string
+		stderr string
 	)
 
 	if r.failing {
-		stdout = "error"
-		err = fmt.Errorf("Failed to run systemd command")
-	} else {
-		stdout = s.String()
+		return "error", "", fmt.Errorf("Failed to run systemd command")
+	}
+	switch s {
+	case states.Running:
+		stdout = statusRunning
+	case states.Listening:
+		stdout = statusListening
+	case states.Stopped:
+		stdout = statusStopped
+		err = errors.New("exit code: 3 - see EXIT STATUS in man systemctl")
+	case states.NotFound:
+		stderr = statusNotFound
+		err = errors.New("exit code: 4 - see EXIT STATUS in man systemctl")
 	}
 
-	return stdout, "", err
+	return stdout, stderr, err
 }
 
 func assertSystemCtlCommands(t *testing.T, systemctlCommands []string, cmd string, args []string) {
diff --git a/test/integration/crcsuite/collect.go b/test/integration/crcsuite/collect.go
index 38e7050cb..230e7382e 100644
--- a/test/integration/crcsuite/collect.go
+++ b/test/integration/crcsuite/collect.go
@@ -186,7 +186,7 @@ func (collector *VMCommandCollector) Collect(w Writer) error {
 	if err != nil {
 		return err
 	}
-	out, err := ssh.Output(collector.Command)
+	out, _, err := ssh.Run(collector.Command)
 	if err != nil {
 		return errors.Wrapf(err, "collecting %s", collector.Target)
 	}
@@ -207,12 +207,12 @@ func (collector *ContainerLogCollector) Collect(w Writer) error {
 	if err != nil {
 		return err
 	}
-	out, err := ssh.Output(fmt.Sprintf("sudo %s ps -a -q", collector.Process))
+	out, _, err := ssh.Run(fmt.Sprintf("sudo %s ps -a -q", collector.Process))
 	if err != nil {
 		return err
 	}
-	for _, id := range strings.Split(strings.TrimSpace(out), "\n") {
-		inspect, err := ssh.Output(fmt.Sprintf("sudo %s inspect %s", collector.Process, id))
+	for _, id := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+		inspect, _, err := ssh.Run(fmt.Sprintf("sudo %s inspect %s", collector.Process, id))
 		if err != nil {
 			logging.Errorf("error while inspecting %s: %v", id, err)
 			continue
@@ -221,7 +221,7 @@ func (collector *ContainerLogCollector) Collect(w Writer) error {
 			logging.Errorf("error while inspecting %s: %v", id, err)
 			continue
 		}
-		logs, err := ssh.Output(fmt.Sprintf("sudo %s logs --tail 200 %s", collector.Process, id))
+		logs, _, err := ssh.Run(fmt.Sprintf("sudo %s logs --tail 200 %s", collector.Process, id))
 		if err != nil {
 			logging.Errorf("error while getting logs %s: %v", id, err)
 			continue