Repository: hashicorp/terraform-exec

Issue:
Does this library provide a way to gracefully shutdown
If I use terraform cli and kill a running terraform task, it will gracefully shutdown.
If I use this library to invoke terraform and cancel it, will it behave the same way? If not, is there any trick?

Discussion:
@scallisterdd: I just looked into this and discovered that this library does _not_ shutdown gracefully if a context gets cancelled. [Here](https://github.com/hashicorp/terraform-exec/blob/main/tfexec/cmd_default.go#L17) is the relevant code in the `runTerraformCmd` function and [here](https://github.com/hashicorp/terraform-exec/blob/main/tfexec/cmd.go#L186) in the `buildTerraformCmd` function. `exec.CommandContext` will issue a `SIGKILL` instead of a `SIGINT`, which immediately kills the underlying terraform process without allowing it a chance to clean-up.

Solution patch:
diff --git a/tfexec/cmd.go b/tfexec/cmd.go
index 5e16032..c8b18ff 100644
--- a/tfexec/cmd.go
+++ b/tfexec/cmd.go
@@ -14,6 +14,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"runtime"
 	"strings"
 
 	"github.com/hashicorp/terraform-exec/internal/version"
@@ -187,6 +188,14 @@ func (tf *Terraform) buildTerraformCmd(ctx context.Context, mergeEnv map[string]
 
 	cmd.Env = tf.buildEnv(mergeEnv)
 	cmd.Dir = tf.workingDir
+	if runtime.GOOS != "windows" {
+		// Windows does not support SIGINT so we cannot do graceful cancellation
+		// see https://pkg.go.dev/os#Signal (os.Interrupt)
+		cmd.Cancel = func() error {
+			return cmd.Process.Signal(os.Interrupt)
+		}
+		cmd.WaitDelay = tf.waitDelay
+	}
 
 	tf.logger.Printf("[INFO] running Terraform command: %s", cmd.String())
 
diff --git a/tfexec/errors.go b/tfexec/errors.go
index c6645e8..14a4772 100644
--- a/tfexec/errors.go
+++ b/tfexec/errors.go
@@ -62,6 +62,10 @@ func (e cmdErr) Is(target error) bool {
 	return false
 }
 
+func (e cmdErr) Unwrap() error {
+	return e.err
+}
+
 func (e cmdErr) Error() string {
 	return e.err.Error()
 }
diff --git a/tfexec/terraform.go b/tfexec/terraform.go
index 628b733..9088616 100644
--- a/tfexec/terraform.go
+++ b/tfexec/terraform.go
@@ -5,12 +5,15 @@ package tfexec
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
 	"os"
+	"runtime"
 	"sync"
+	"time"
 
 	"github.com/hashicorp/go-version"
 )
@@ -67,6 +70,9 @@ type Terraform struct {
 	// TF_LOG_PROVIDER environment variable
 	logProvider string
 
+	// waitDelay represents the WaitDelay field of the [exec.Cmd] of Terraform
+	waitDelay time.Duration
+
 	versionLock  sync.Mutex
 	execVersion  *version.Version
 	provVersions map[string]*version.Version
@@ -95,6 +101,7 @@ func NewTerraform(workingDir string, execPath string) (*Terraform, error) {
 		workingDir: workingDir,
 		env:        nil, // explicit nil means copy os.Environ
 		logger:     log.New(ioutil.Discard, "", 0),
+		waitDelay:  60 * time.Second,
 	}
 
 	return &tf, nil
@@ -216,6 +223,15 @@ func (tf *Terraform) SetSkipProviderVerify(skip bool) error {
 	return nil
 }
 
+// SetWaitDelay sets the WaitDelay of running Terraform process as [exec.Cmd]
+func (tf *Terraform) SetWaitDelay(delay time.Duration) error {
+	if runtime.GOOS == "windows" {
+		return errors.New("cannot set WaitDelay, graceful cancellation not supported on windows")
+	}
+	tf.waitDelay = delay
+	return nil
+}
+
 // WorkingDir returns the working directory for Terraform.
 func (tf *Terraform) WorkingDir() string {
 	return tf.workingDir



Test patch:
diff --git a/tfexec/internal/e2etest/errors_test.go b/tfexec/internal/e2etest/errors_test.go
index 7bef86c..c99aad9 100644
--- a/tfexec/internal/e2etest/errors_test.go
+++ b/tfexec/internal/e2etest/errors_test.go
@@ -168,6 +168,10 @@ func TestContext_sleepTimeoutExpired(t *testing.T) {
 			t.Skip("the ability to interrupt an apply was added in protocol 5.0 in Terraform 0.12, so test is not valid")
 		}
 
+		// sleep will not react to SIGINT
+		// This ensures that process is killed within the expected time limit.
+		tf.SetWaitDelay(500 * time.Millisecond)
+
 		err := tf.Init(context.Background())
 		if err != nil {
 			t.Fatalf("err during init: %s", err)
diff --git a/tfexec/sleepmock_test.go b/tfexec/sleepmock_test.go
new file mode 100644
index 0000000..57f4089
--- /dev/null
+++ b/tfexec/sleepmock_test.go
@@ -0,0 +1,25 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package tfexec
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"time"
+)
+
+func sleepMock(rawDuration string) {
+	signal.Ignore(os.Interrupt)
+
+	d, err := time.ParseDuration(rawDuration)
+	if err != nil {
+		log.Fatalf("invalid duration format: %s", err)
+	}
+
+	fmt.Printf("sleeping for %s\n", d)
+
+	time.Sleep(d)
+}
diff --git a/tfexec/terraform_test.go b/tfexec/terraform_test.go
index 93a6de4..e20da21 100644
--- a/tfexec/terraform_test.go
+++ b/tfexec/terraform_test.go
@@ -8,9 +8,11 @@ import (
 	"errors"
 	"io/ioutil"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"runtime"
 	"testing"
+	"time"
 
 	"github.com/hashicorp/terraform-exec/tfexec/internal/testutil"
 )
@@ -18,6 +20,11 @@ import (
 var tfCache *testutil.TFCache
 
 func TestMain(m *testing.M) {
+	if rawDuration := os.Getenv("MOCK_SLEEP_DURATION"); rawDuration != "" {
+		sleepMock(rawDuration)
+		return
+	}
+
 	os.Exit(func() int {
 		var err error
 		installDir, err := ioutil.TempDir("", "tfinstall")
@@ -813,6 +820,78 @@ func TestCheckpointDisablePropagation_v1(t *testing.T) {
 	})
 }
 
+func TestGracefulCancellation_interruption(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("graceful cancellation not supported on windows")
+	}
+	mockExecPath, err := os.Executable()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	td := t.TempDir()
+
+	tf, err := NewTerraform(td, mockExecPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ctx := context.Background()
+	ctx, cancelFunc := context.WithTimeout(ctx, 100*time.Millisecond)
+	t.Cleanup(cancelFunc)
+
+	_, _, err = tf.version(ctx)
+	if err != nil {
+		var exitErr *exec.ExitError
+		isExitErr := errors.As(err, &exitErr)
+		if isExitErr && exitErr.ProcessState.String() == "signal: interrupt" {
+			return
+		}
+		if isExitErr {
+			t.Fatalf("expected interrupt signal, received %q", exitErr)
+		}
+		t.Fatalf("unexpected command error: %s", err)
+	}
+}
+
+func TestGracefulCancellation_withDelay(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("graceful cancellation not supported on windows")
+	}
+	mockExecPath, err := os.Executable()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	td := t.TempDir()
+	tf, err := NewTerraform(td, mockExecPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tf.SetEnv(map[string]string{
+		"MOCK_SLEEP_DURATION": "5s",
+	})
+	tf.SetLogger(testutil.TestLogger())
+	tf.SetWaitDelay(100 * time.Millisecond)
+
+	ctx := context.Background()
+	ctx, cancelFunc := context.WithTimeout(ctx, 100*time.Millisecond)
+	t.Cleanup(cancelFunc)
+
+	_, _, err = tf.version(ctx)
+	if err != nil {
+		var exitErr *exec.ExitError
+		isExitErr := errors.As(err, &exitErr)
+		if isExitErr && exitErr.ProcessState.String() == "signal: killed" {
+			return
+		}
+		if isExitErr {
+			t.Fatalf("expected kill signal, received %q", exitErr)
+		}
+		t.Fatalf("unexpected command error: %s", err)
+	}
+}
+
 // test that a suitable error is returned if NewTerraform is called without a valid
 // executable path
 func TestNoTerraformBinary(t *testing.T) {