Repository: kolide/launcher

Issue:
Cannot build for ubuntu 20.04 using ubuntu-latest runner image
The GitHub ubuntu runner image moved to point to ubuntu 22.04 instead of 20.04. When this happened, we discovered that new ubuntu builds were broken for 20.04:

```
$ sudo ./launcher interactive
./launcher: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.32' not found (required by ./launcher)
./launcher: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.34' not found (required by ./launcher)
```

For the time being, we locked our build image to ubuntu 20.04 [here](https://github.com/kolide/launcher/pull/949). In the future, however, we will eventually want to use `ubuntu-latest` again; this issue is for producing builds that are compatible for the versions of ubuntu we want to support.

Hints from issue comments:
@directionless: Unlike macOS, I'm not sure we can specify the SDK target. I think we just have to build on an older OS. I don't know if we need to build on an old OS, or just using an old libc. 

osquery does this by building inside a docker container with a fairly customized toolchain. I don't know what kind of options we have@directionless: I don't think there's any work to be done here. When we need to upgrade the build machines, we'll figure out what the oldest supported platform is@directionless: Turns out this does matter...@RebeccaMahany: (https://github.com/kolide/launcher/pull/2131 broke Ubuntu 20.04 builds)

Solution patch:
diff --git a/ee/tuf/ci/valid_executable.go b/ee/tuf/ci/valid_executable.go
index 22710006..d50dbde6 100644
--- a/ee/tuf/ci/valid_executable.go
+++ b/ee/tuf/ci/valid_executable.go
@@ -1,15 +1,73 @@
 package tufci
 
 import (
+	"context"
+	"fmt"
 	"os"
 	"path/filepath"
+	"runtime"
+	"sync"
 	"testing"
+	"time"
 
+	"github.com/kolide/kit/fsutil"
+	"github.com/kolide/launcher/pkg/packaging"
 	"github.com/stretchr/testify/require"
 )
 
+var testOsqueryBinary string
+
+// downloadOnceFunc downloads a real osquery binary for use in tests. This function
+// can be called multiple times but will only execute once -- the osquery binary is
+// stored at path `testOsqueryBinary` and can be reused by all subsequent tests.
+var downloadOnceFunc = sync.OnceFunc(func() {
+	downloadDir, err := os.MkdirTemp("", "tufci")
+	if err != nil {
+		fmt.Printf("failed to make temp dir for test osquery binary: %v", err)
+		os.Exit(1) //nolint:forbidigo // Fine to use os.Exit inside tests
+	}
+
+	target := packaging.Target{}
+	if err := target.PlatformFromString(runtime.GOOS); err != nil {
+		fmt.Printf("error parsing platform %s: %v", runtime.GOOS, err)
+		os.RemoveAll(downloadDir) // explicit removal as defer will not run when os.Exit is called
+		os.Exit(1)                //nolint:forbidigo // Fine to use os.Exit inside tests
+	}
+	target.Arch = packaging.ArchFlavor(runtime.GOARCH)
+	if runtime.GOOS == "darwin" {
+		target.Arch = packaging.Universal
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	dlPath, err := packaging.FetchBinary(ctx, downloadDir, "osqueryd", target.PlatformBinaryName("osqueryd"), "nightly", target)
+	if err != nil {
+		fmt.Printf("error fetching binary osqueryd binary: %v", err)
+		cancel()                  // explicit cancel as defer will not run when os.Exit is called
+		os.RemoveAll(downloadDir) // explicit removal as defer will not run when os.Exit is called
+		os.Exit(1)                //nolint:forbidigo // Fine to use os.Exit inside tests
+	}
+
+	testOsqueryBinary = filepath.Join(downloadDir, filepath.Base(dlPath))
+	if runtime.GOOS == "windows" {
+		testOsqueryBinary += ".exe"
+	}
+
+	if err := fsutil.CopyFile(dlPath, testOsqueryBinary); err != nil {
+		fmt.Printf("error copying osqueryd binary: %v", err)
+		cancel()                  // explicit cancel as defer will not run when os.Exit is called
+		os.RemoveAll(downloadDir) // explicit removal as defer will not run when os.Exit is called
+		os.Exit(1)                //nolint:forbidigo // Fine to use os.Exit inside tests
+	}
+})
+
+// CopyBinary ensures we've downloaded a test osquery binary, then creates a symlink
+// between the real binary and the expected `executablePath` location. (We used to
+// actually copy the entire binary, but found that was very slow.)
 func CopyBinary(t *testing.T, executablePath string) {
-	require.NoError(t, os.MkdirAll(filepath.Dir(executablePath), 0755))
+	downloadOnceFunc()
 
-	require.NoError(t, os.Symlink(os.Args[0], executablePath))
+	require.NoError(t, os.MkdirAll(filepath.Dir(executablePath), 0755))
+	require.NoError(t, os.Symlink(testOsqueryBinary, executablePath))
 }
diff --git a/ee/tuf/util.go b/ee/tuf/util.go
index f5b91d76..bdf5e3e7 100644
--- a/ee/tuf/util.go
+++ b/ee/tuf/util.go
@@ -48,23 +48,25 @@ func CheckExecutable(ctx context.Context, slogger *slog.Logger, potentialBinary
 	// See: https://github.com/golang/go/issues/22315
 	for i := 0; i < 3; i += 1 {
 		out, execErr := runExecutableCheck(ctx, potentialBinary, args...)
-		if execErr != nil && errors.Is(execErr, syscall.ETXTBSY) {
-			continue
+		if execErr == nil {
+			slogger.Log(ctx, slog.LevelInfo,
+				"successfully checked executable",
+			)
+			return nil
 		}
 
-		if err := supressRoutineErrors(ctx, execErr, out); err != nil {
-			slogger.Log(ctx, slog.LevelWarn,
-				"executable check returned error",
-				"err", err,
-				"exec_err", execErr,
-				"command_output", string(out),
-			)
-			return err
+		// Check to see if we should retry
+		if errors.Is(execErr, syscall.ETXTBSY) {
+			continue
 		}
-		slogger.Log(ctx, slog.LevelInfo,
-			"successfully checked executable",
+
+		// Non-retryable error
+		slogger.Log(ctx, slog.LevelWarn,
+			"executable check returned error",
+			"exec_err", execErr,
+			"command_output", string(out),
 		)
-		return nil
+		return fmt.Errorf("running executable check: got output `%s` and error: %w", string(out), execErr)
 	}
 
 	slogger.Log(ctx, slog.LevelWarn,
@@ -94,27 +96,3 @@ func runExecutableCheck(ctx context.Context, potentialBinary string, args ...str
 
 	return out, err
 }
-
-// supressRoutineErrors attempts to tell whether the error was a
-// program that has executed, and then exited, vs one that's execution
-// was entirely unsuccessful. This differentiation allows us to
-// detect, and recover, from corrupt updates vs something in-app.
-func supressRoutineErrors(ctx context.Context, err error, combinedOutput []byte) error {
-	_, span := traces.StartSpan(ctx)
-	defer span.End()
-
-	if err == nil {
-		return nil
-	}
-
-	// Suppress exit codes of 1 or 2. These are generally indicative of
-	// an unknown command line flag, _not_ a corrupt download. (exit
-	// code 0 will be nil, and never trigger this block)
-	if exitError, ok := err.(*exec.ExitError); ok {
-		if exitError.ExitCode() == 1 || exitError.ExitCode() == 2 {
-			// suppress these
-			return nil
-		}
-	}
-	return fmt.Errorf("exec error: output: `%s`, err: %w", string(combinedOutput), err)
-}



Test patch:
diff --git a/ee/tuf/util_test.go b/ee/tuf/util_test.go
index f82a74b7..222865e9 100644
--- a/ee/tuf/util_test.go
+++ b/ee/tuf/util_test.go
@@ -9,7 +9,6 @@ import (
 	"testing"
 	"time"
 
-	tufci "github.com/kolide/launcher/ee/tuf/ci"
 	"github.com/kolide/launcher/pkg/log/multislogger"
 	"github.com/stretchr/testify/require"
 )
@@ -71,7 +70,10 @@ func TestCheckExecutable(t *testing.T) {
 	tmpDir := t.TempDir()
 	binaryName := windowsAddExe("testbinary")
 	targetExe := filepath.Join(tmpDir, binaryName)
-	tufci.CopyBinary(t, targetExe)
+	require.NoError(t, os.MkdirAll(filepath.Dir(tmpDir), 0755))
+
+	// We use the golang test binary here so we can run `TestHelperProcess` with the desired outcome
+	require.NoError(t, os.Symlink(os.Args[0], targetExe))
 	require.NoError(t, os.Chmod(targetExe, 0755))
 
 	var tests = []struct {
@@ -84,11 +86,11 @@ func TestCheckExecutable(t *testing.T) {
 		},
 		{
 			testName:    "exit1",
-			expectedErr: false,
+			expectedErr: true,
 		},
 		{
 			testName:    "exit2",
-			expectedErr: false,
+			expectedErr: true,
 		},
 		{
 			testName:    "sleep",