Repository: GoogleCloudPlatform/k8s-config-connector
Pull request #3864: Conductor: Add support for retries for failed commands
Author: login: barney-s
Labels: name: lgtm
name: approved
Created: 2025-03-05T20:24:43Z | Merged: 2025-03-06T03:06:02Z

Add support for retries for failed commands.
- each command can cap the retry count using MaxRetries. This is useful for deterministic commands.
- global retry flag (default: 3) is used for all commands.




Diff:
diff --git a/experiments/conductor/cmd/runner/crd_generator_commands.go b/experiments/conductor/cmd/runner/crd_generator_commands.go
index 302ffe40a72..05fd534587a 100644
--- a/experiments/conductor/cmd/runner/crd_generator_commands.go
+++ b/experiments/conductor/cmd/runner/crd_generator_commands.go
@@ -187,7 +187,8 @@ func generateTypesAndMapper(opts *RunnerOptions, branch Branch) {
 				"--api-version", fmt.Sprintf("%s.cnrm.cloud.google.com/v1alpha1", branch.Group),
 				"--resource", fmt.Sprintf("%s:%s", branch.Kind, branch.Proto),
 			},
-			WorkDir: controllerBuilderDir,
+			WorkDir:    controllerBuilderDir,
+			MaxRetries: 2,
 		}
 		_, _, err := executeCommand(opts, cfg)
 		if err != nil {
@@ -211,7 +212,8 @@ func generateTypesAndMapper(opts *RunnerOptions, branch Branch) {
 				"--service", branch.Package,
 				"--api-version", fmt.Sprintf("%s.cnrm.cloud.google.com/v1alpha1", branch.Group),
 			},
-			WorkDir: controllerBuilderDir,
+			WorkDir:    controllerBuilderDir,
+			MaxRetries: 2,
 		}
 		_, _, err := executeCommand(opts, cfg)
 		if err != nil {
@@ -228,7 +230,8 @@ func generateTypesAndMapper(opts *RunnerOptions, branch Branch) {
 				"-w",
 				mapperDir,
 			},
-			WorkDir: workDir,
+			WorkDir:    workDir,
+			MaxRetries: 2,
 		}
 		_, _, err = executeCommand(opts, cfg)
 		if err != nil {
@@ -258,9 +261,10 @@ func generateCRD(opts *RunnerOptions, branch Branch) {
 
 	// Generate CRDs
 	cfg := CommandConfig{
-		Name:    "Generate CRDs",
-		Cmd:     filepath.Join(workDir, "dev", "tasks", "generate-crds"),
-		WorkDir: workDir,
+		Name:       "Generate CRDs",
+		Cmd:        filepath.Join(workDir, "dev", "tasks", "generate-crds"),
+		WorkDir:    workDir,
+		MaxRetries: 2,
 	}
 	_, _, err := executeCommand(opts, cfg)
 	if err != nil {
@@ -287,11 +291,12 @@ func generateSpecStatus(opts *RunnerOptions, branch Branch) {
 	stdinInput := fmt.Sprintf("// +kcc:proto=%s.%s\n", branch.ProtoSvc, branch.Proto)
 
 	cfg := CommandConfig{
-		Name:    "Spec/Status generation",
-		Cmd:     "controllerbuilder",
-		Args:    []string{"prompt", "--src-dir", workDir, "--proto-dir", filepath.Join(workDir, ".build", "third_party", "googleapis")},
-		WorkDir: workDir,
-		Stdin:   strings.NewReader(stdinInput),
+		Name:         "Spec/Status generation",
+		Cmd:          "controllerbuilder",
+		Args:         []string{"prompt", "--src-dir", workDir, "--proto-dir", filepath.Join(workDir, ".build", "third_party", "googleapis")},
+		WorkDir:      workDir,
+		Stdin:        strings.NewReader(stdinInput),
+		RetryBackoff: GenerativeCommandRetryBackoff,
 	}
 	_, _, err := executeCommand(opts, cfg)
 	if err != nil {
@@ -325,11 +330,12 @@ func generateFuzzer(opts *RunnerOptions, branch Branch) {
 `, branch.ProtoMsg)
 
 	cfg := CommandConfig{
-		Name:    "Fuzzer generation",
-		Cmd:     "controllerbuilder",
-		Args:    []string{"prompt", "--src-dir", workDir, "--proto-dir", filepath.Join(workDir, ".build", "third_party", "googleapis")},
-		WorkDir: workDir,
-		Stdin:   strings.NewReader(stdinInput),
+		Name:         "Fuzzer generation",
+		Cmd:          "controllerbuilder",
+		Args:         []string{"prompt", "--src-dir", workDir, "--proto-dir", filepath.Join(workDir, ".build", "third_party", "googleapis")},
+		WorkDir:      workDir,
+		Stdin:        strings.NewReader(stdinInput),
+		RetryBackoff: GenerativeCommandRetryBackoff,
 	}
 	output, _, err := executeCommand(opts, cfg)
 	if err != nil {
@@ -346,11 +352,12 @@ func generateFuzzer(opts *RunnerOptions, branch Branch) {
 	stdinInput = fmt.Sprintf("Add an unnamed (_) go import for %s to the imports in %s", importLine, registerPath)
 
 	cfg = CommandConfig{
-		Name:    "Import addition",
-		Cmd:     "codebot",
-		Args:    []string{"--prompt=/dev/stdin"},
-		WorkDir: workDir,
-		Stdin:   strings.NewReader(stdinInput),
+		Name:         "Import addition",
+		Cmd:          "codebot",
+		Args:         []string{"--prompt=/dev/stdin"},
+		WorkDir:      workDir,
+		Stdin:        strings.NewReader(stdinInput),
+		RetryBackoff: GenerativeCommandRetryBackoff,
 	}
 	_, _, err = executeCommand(opts, cfg)
 	if err != nil {
diff --git a/experiments/conductor/cmd/runner/mock_commands.go b/experiments/conductor/cmd/runner/mock_commands.go
index b3a44bd6970..73d600fe4b7 100644
--- a/experiments/conductor/cmd/runner/mock_commands.go
+++ b/experiments/conductor/cmd/runner/mock_commands.go
@@ -21,6 +21,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 )
 
 const SCRIPT_YAML_PROMPT string = `I am trying to create a test case for mockgcp.
@@ -86,10 +87,11 @@ func createScriptYaml(opts *RunnerOptions, branch Branch) {
 
 	// Run the LLM to generate the file.
 	cfg := CommandConfig{
-		Name:    "Generate script",
-		Cmd:     "codebot",
-		Args:    []string{"--ui-type=prompt", "--prompt=prompt.txt"},
-		WorkDir: workDir,
+		Name:         "Generate script",
+		Cmd:          "codebot",
+		Args:         []string{"--ui-type=prompt", "--prompt=prompt.txt"},
+		WorkDir:      workDir,
+		RetryBackoff: GenerativeCommandRetryBackoff,
 	}
 	_, _, err := executeCommand(opts, cfg)
 	if err != nil {
@@ -121,10 +123,12 @@ func enableAPIs(opts *RunnerOptions, branch Branch) {
 	for _, api := range branch.ApisEnabled {
 		log.Printf("[Enable APIs] Enabling API %s", api)
 		cfg := CommandConfig{
-			Name:    fmt.Sprintf("Enable API %s", api),
-			Cmd:     "gcloud",
-			Args:    []string{"services", "enable", api},
-			WorkDir: workDir,
+			Name:         fmt.Sprintf("Enable API %s", api),
+			Cmd:          "gcloud",
+			Args:         []string{"services", "enable", api},
+			WorkDir:      workDir,
+			MaxRetries:   3,
+			RetryBackoff: 10 * time.Second,
 		}
 		_, _, err := executeCommand(opts, cfg)
 		if err != nil {
@@ -234,8 +238,9 @@ func captureHttpLog(opts *RunnerOptions, branch Branch) {
 			"test", "./mockgcptests",
 			"-run", fmt.Sprintf("TestScripts/mock%s/testdata/%s/crud", branch.Group, branch.Resource),
 		},
-		WorkDir: workDir,
-		Env:     map[string]string{"WRITE_GOLDEN_OUTPUT": "1", "E2E_GCP_TARGET": "real"},
+		WorkDir:    workDir,
+		Env:        map[string]string{"WRITE_GOLDEN_OUTPUT": "1", "E2E_GCP_TARGET": "real"},
+		MaxRetries: 2,
 	}
 	_, _, err := executeCommand(opts, cfg)
 	if err != nil {
@@ -327,7 +332,8 @@ func generateMockGo(opts *RunnerOptions, branch Branch) {
 				"--proto-dir", fmt.Sprintf("%s/.build/third_party/googleapis/", opts.branchRepoDir),
 				"--input-file", "service_prompt.txt",
 			},
-			WorkDir: workDir,
+			WorkDir:      workDir,
+			RetryBackoff: GenerativeCommandRetryBackoff,
 		}
 		output, _, err := executeCommand(opts, cfg)
 		if err != nil {
@@ -366,7 +372,8 @@ func generateMockGo(opts *RunnerOptions, branch Branch) {
 				"--proto-dir", fmt.Sprintf("%s/.build/third_party/googleapis/", opts.branchRepoDir),
 				"--input-file", "resource_prompt.txt",
 			},
-			WorkDir: workDir,
+			WorkDir:      workDir,
+			RetryBackoff: GenerativeCommandRetryBackoff,
 		}
 		output, _, err := executeCommand(opts, cfg)
 		if err != nil {
@@ -459,10 +466,11 @@ func addServiceToRoundTrip(opts *RunnerOptions, branch Branch) {
 
 	// Run the LLM to add the service to roundtrip file.
 	cfg := CommandConfig{
-		Name:    "Add service to roundtrip",
-		Cmd:     "codebot",
-		Args:    []string{"--ui-type=prompt", "--prompt=roundtrip_prompt.txt"},
-		WorkDir: workDir,
+		Name:         "Add service to roundtrip",
+		Cmd:          "codebot",
+		Args:         []string{"--ui-type=prompt", "--prompt=roundtrip_prompt.txt"},
+		WorkDir:      workDir,
+		RetryBackoff: GenerativeCommandRetryBackoff,
 	}
 	_, _, err := executeCommand(opts, cfg)
 	if err != nil {
@@ -510,10 +518,11 @@ func addProtoToMakfile(opts *RunnerOptions, branch Branch) {
 
 	// Run the LLM to add the service to roundtrip file.
 	cfg := CommandConfig{
-		Name:    "Add proto to makefile",
-		Cmd:     "codebot",
-		Args:    []string{"--ui-type=prompt", "--prompt=makefile_prompt.txt"},
-		WorkDir: workDir,
+		Name:         "Add proto to makefile",
+		Cmd:          "codebot",
+		Args:         []string{"--ui-type=prompt", "--prompt=makefile_prompt.txt"},
+		WorkDir:      workDir,
+		RetryBackoff: GenerativeCommandRetryBackoff,
 	}
 	_, _, err := executeCommand(opts, cfg)
 	if err != nil {
@@ -547,8 +556,9 @@ func runMockgcpTests(opts *RunnerOptions, branch Branch) {
 			"test", "./mockgcptests", "-v",
 			"-run", fmt.Sprintf("TestScripts/mock%s/testdata/%s/crud", branch.Group, branch.Resource),
 		},
-		WorkDir: workDir,
-		Env:     map[string]string{"WRITE_GOLDEN_OUTPUT": "1", "E2E_GCP_TARGET": "mock"},
+		WorkDir:    workDir,
+		Env:        map[string]string{"WRITE_GOLDEN_OUTPUT": "1", "E2E_GCP_TARGET": "mock"},
+		MaxRetries: 2,
 	}
 	_, _, err := executeCommand(opts, cfg)
 	if err != nil {
diff --git a/experiments/conductor/cmd/runner/runner.go b/experiments/conductor/cmd/runner/runner.go
index 543cd4db155..e0023079f7d 100644
--- a/experiments/conductor/cmd/runner/runner.go
+++ b/experiments/conductor/cmd/runner/runner.go
@@ -92,6 +92,8 @@ func BuildRunnerCmd() *cobra.Command {
 		"", "", "dedicated directory for logging, empty for stdout.")
 	cmd.Flags().DurationVarP(&opts.timeout, "timeout",
 		"t", 5*time.Minute, "Global timeout for commands.")
+	cmd.Flags().IntVarP(&opts.defaultRetries, "retries",
+		"r", 10, "Default number of retries for failed commands.")
 
 	return cmd
 }
@@ -103,9 +105,13 @@ type RunnerOptions struct {
 	loggingDir     string
 	timeout        time.Duration
 	readFileType   string
+	defaultRetries int // Default number of retries for commands
 }
 
 func (opts *RunnerOptions) validateFlags() error {
+	if opts.defaultRetries < 0 {
+		return fmt.Errorf("retries flag cannot be negative, got %d", opts.defaultRetries)
+	}
 	return nil
 }
 
@@ -559,6 +565,7 @@ If you are using gcloud to create something, make sure you are deleting it after
 Print the list of APIs, one on each line in this format api-required: <api-name>
 Only include APIs that are directly needed by this command.
 `, branch.Command)),
+		RetryBackoff: GenerativeCommandRetryBackoff,
 	}
 
 	output, _, err := executeCommand(opts, cfg)
@@ -645,10 +652,11 @@ func inferProtoPath(opts *RunnerOptions, branch Branch, workDir string) string {
 	}
 
 	cfg := CommandConfig{
-		Name:    "Search for service",
-		Cmd:     "egrep",
-		Args:    args,
-		WorkDir: workDir,
+		Name:       "Search for service",
+		Cmd:        "egrep",
+		Args:       args,
+		WorkDir:    workDir,
+		MaxRetries: 2,
 	}
 	output, errOutput, err := executeCommand(opts, cfg)
 	if err != nil {
@@ -667,10 +675,11 @@ func inferProtoPath(opts *RunnerOptions, branch Branch, workDir string) string {
 			}
 
 			cfg = CommandConfig{
-				Name:    "Search for any service",
-				Cmd:     "egrep",
-				Args:    args,
-				WorkDir: workDir,
+				Name:       "Search for any service",
+				Cmd:        "egrep",
+				Args:       args,
+				WorkDir:    workDir,
+				MaxRetries: 2,
 			}
 			output, errOutput, err = executeCommand(opts, cfg)
 			if err != nil {
diff --git a/experiments/conductor/cmd/runner/utilities.go b/experiments/conductor/cmd/runner/utilities.go
index ebe2c7e9f99..bccd45e2ed0 100644
--- a/experiments/conductor/cmd/runner/utilities.go
+++ b/experiments/conductor/cmd/runner/utilities.go
@@ -27,6 +27,10 @@ import (
 	"time"
 )
 
+const (
+	GenerativeCommandRetryBackoff = 60 * time.Second
+)
+
 var commandMap = map[int64]string{
 	cmdEnableGCPAPIs:       "enablegcpapis",
 	cmdCreateScriptYaml:    "createscriptyaml",
@@ -277,13 +281,15 @@ func formatCommandOutput(output string) string {
 
 // CommandConfig holds the configuration for executing a command
 type CommandConfig struct {
-	Name    string            // Name of the command for logging
-	Cmd     string            // The command to run
-	Args    []string          // Command arguments
-	WorkDir string            // Working directory
-	Stdin   io.Reader         // Optional stdin
-	Env     map[string]string // Optional environment variables
-	Timeout time.Duration     // Timeout duration (default 5m)
+	Name         string            // Name of the command for logging
+	Cmd          string            // The command to run
+	Args         []string          // Command arguments
+	WorkDir      string            // Working directory
+	Stdin        io.Reader         // Optional stdin
+	Env          map[string]string // Optional environment variables
+	Timeout      time.Duration     // Timeout duration (default 5m)
+	MaxRetries   int               // Maximum number of retries allowed for this command
+	RetryBackoff time.Duration     // Time to wait between retries (default 1s)
 }
 
 // Helper function to execute a command with timing and logging
@@ -296,61 +302,90 @@ func executeCommand(opts *RunnerOptions, cfg CommandConfig) (string, string, err
 		}
 	}
 
-	log.Printf("Starting command step: %s", cfg.Name)
-	log.Printf("[%s] working directory: %s", cfg.Name, cfg.WorkDir)
-	log.Printf("[%s] command: %s %s", cfg.Name, cfg.Cmd, strings.Join(cfg.Args, " "))
-	if cfg.Stdin != nil {
-		log.Printf("[%s] stdin: %s", cfg.Name, cfg.Stdin)
+	if cfg.RetryBackoff == 0 {
+		cfg.RetryBackoff = time.Second
 	}
-	if len(cfg.Env) > 0 {
-		log.Printf("[%s] environment:", cfg.Name)
-		for k, v := range cfg.Env {
-			log.Printf("  %s=%s", k, v)
-		}
+
+	maxRetries := opts.defaultRetries
+	// If MaxRetries is set in config, cap it at that
+	if cfg.MaxRetries > 0 && cfg.MaxRetries < maxRetries {
+		maxRetries = cfg.MaxRetries
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), cfg.Timeout)
-	defer cancel()
+	var lastErr error
+	var output, errOutput string
+	retryCount := 0
 
-	cmd := exec.CommandContext(ctx, cfg.Cmd, cfg.Args...)
-	cmd.Dir = cfg.WorkDir
+	for {
+		log.Printf("Starting command step: %s (attempt %d/%d)", cfg.Name, retryCount+1, maxRetries+1)
+		log.Printf("[%s] working directory: %s", cfg.Name, cfg.WorkDir)
+		log.Printf("[%s] command: %s %s", cfg.Name, cfg.Cmd, strings.Join(cfg.Args, " "))
+		if cfg.Stdin != nil {
+			log.Printf("[%s] stdin: %s", cfg.Name, cfg.Stdin)
+		}
+		if len(cfg.Env) > 0 {
+			log.Printf("[%s] environment:", cfg.Name)
+			for k, v := range cfg.Env {
+				log.Printf("  %s=%s", k, v)
+			}
+		}
 
-	var outBuf, errBuf strings.Builder
-	cmd.Stdout = &outBuf
-	cmd.Stderr = &errBuf
+		ctx, cancel := context.WithTimeout(context.Background(), cfg.Timeout)
+		cmd := exec.CommandContext(ctx, cfg.Cmd, cfg.Args...)
+		cmd.Dir = cfg.WorkDir
 
-	if cfg.Stdin != nil {
-		cmd.Stdin = cfg.Stdin
-	}
+		var outBuf, errBuf strings.Builder
+		cmd.Stdout = &outBuf
+		cmd.Stderr = &errBuf
 
-	// Set up environment variables
-	if len(cfg.Env) > 0 {
-		cmd.Env = os.Environ() // Start with current environment
-		for k, v := range cfg.Env {
-			cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
+		if cfg.Stdin != nil {
+			cmd.Stdin = cfg.Stdin
 		}
-	}
 
-	start := time.Now()
+		// Set up environment variables
+		if len(cfg.Env) > 0 {
+			cmd.Env = os.Environ() // Start with current environment
+			for k, v := range cfg.Env {
+				cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
+			}
+		}
 
-	err := cmd.Run()
-	stop := time.Now()
-	diff := stop.Sub(start)
-	if err != nil {
-		log.Printf("[%s] ERROR (%v): \n", cfg.Name, diff)
-		log.Printf("[%s] err: %q\n", cfg.Name, err)
-	} else {
-		log.Printf("[%s] SUCCESS (%v): \n", cfg.Name, diff)
-	}
+		start := time.Now()
+		err := cmd.Run()
+		stop := time.Now()
+		diff := stop.Sub(start)
+		cancel()
+
+		output = outBuf.String()
+		errOutput = errBuf.String()
+
+		if err == nil {
+			log.Printf("[%s] SUCCESS (%v)", cfg.Name, diff)
+			printCommandOutput(output)
+			if errBuf.Len() > 0 {
+				log.Printf("[%s] stderr output:", cfg.Name)
+				printCommandOutput(errOutput)
+			}
+			return output, errOutput, nil
+		}
+
+		lastErr = err
+		log.Printf("[%s] ERROR (%v): %v", cfg.Name, diff, err)
+		printCommandOutput(output)
+		if errBuf.Len() > 0 {
+			log.Printf("[%s] stderr output:", cfg.Name)
+			printCommandOutput(errOutput)
+		}
 
-	output := outBuf.String()
-	errOutput := errBuf.String()
+		retryCount++
+		if maxRetries >= 0 && retryCount > maxRetries {
+			log.Printf("[%s] Exceeded maximum retries (%d), giving up", cfg.Name, maxRetries)
+			break
+		}
 
-	printCommandOutput(output)
-	if errBuf.Len() > 0 {
-		log.Printf("[%s] stderr output:\n", cfg.Name)
-		printCommandOutput(errOutput)
+		log.Printf("[%s] Retrying in %v...", cfg.Name, cfg.RetryBackoff)
+		time.Sleep(cfg.RetryBackoff)
 	}
 
-	return output, errOutput, err
+	return output, errOutput, fmt.Errorf("command failed after %d attempts: %w", retryCount, lastErr)
 }


Reviews:
user: login: maqiuyujoyce
body: Thank you for adding the retries, @barney-s !

I have a few thoughts/questions:

1. The place where I found retries are needed is when the communication with LLM hangs/terminates unexpectedly. We talk to codebot for steps like script generation, API identification (right?), mockgcp generation. So I'm thinking we may want to focus on retries about these steps. And in this PR, I didn't find the retry for creating scripts. As I shared, I had to go through 12 retries to finish generating scripts for 40 resources.
2. I think with max retries of 2, the retry may not be that useful, especially you set the default retry to be 3. I'd probably start the max retries to be at least 10 considering we are doing batch work.
3. Have you tried to verify the retry works for steps involving using codebot for more than 50 resources?
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-05T22:22:11Z
user: login: maqiuyujoyce
body: /lgtm
/approve
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-06T01:07:53Z
user: login: barney-s
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-06T02:11:19Z
user: login: barney-s
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-06T02:16:38Z
user: login: barney-s
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-06T02:20:47Z

Review comments:
diff_hunk: @@ -296,61 +298,90 @@ func executeCommand(opts *RunnerOptions, cfg CommandConfig) (string, string, err
 		}
 	}
 
-	log.Printf("Starting command step: %s", cfg.Name)
-	log.Printf("[%s] working directory: %s", cfg.Name, cfg.WorkDir)
-	log.Printf("[%s] command: %s %s", cfg.Name, cfg.Cmd, strings.Join(cfg.Args, " "))
-	if cfg.Stdin != nil {
-		log.Printf("[%s] stdin: %s", cfg.Name, cfg.Stdin)
+	if cfg.RetryBackoff == 0 {
+		cfg.RetryBackoff = time.Second
 	}
-	if len(cfg.Env) > 0 {
-		log.Printf("[%s] environment:", cfg.Name)
-		for k, v := range cfg.Env {
-			log.Printf("  %s=%s", k, v)
-		}
+
+	maxRetries := opts.defaultRetries
+	// If MaxRetries is set in config, cap it at that
+	if cfg.MaxRetries > 0 && cfg.MaxRetries < maxRetries {
+		maxRetries = cfg.MaxRetries
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), cfg.Timeout)
-	defer cancel()
+	var lastErr error
+	var output, errOutput string
+	retryCount := 0
 
-	cmd := exec.CommandContext(ctx, cfg.Cmd, cfg.Args...)
-	cmd.Dir = cfg.WorkDir
+	for {
+		log.Printf("Starting command step: %s (attempt %d/%d)", cfg.Name, retryCount+1, maxRetries+1)
+		log.Printf("[%s] working directory: %s", cfg.Name, cfg.WorkDir)
+		log.Printf("[%s] command: %s %s", cfg.Name, cfg.Cmd, strings.Join(cfg.Args, " "))
+		if cfg.Stdin != nil {
+			log.Printf("[%s] stdin: %s", cfg.Name, cfg.Stdin)
+		}
+		if len(cfg.Env) > 0 {
+			log.Printf("[%s] environment:", cfg.Name)
+			for k, v := range cfg.Env {
+				log.Printf("  %s=%s", k, v)
+			}
+		}
 
-	var outBuf, errBuf strings.Builder
-	cmd.Stdout = &outBuf
-	cmd.Stderr = &errBuf
+		ctx, cancel := context.WithTimeout(context.Background(), cfg.Timeout)
+		cmd := exec.CommandContext(ctx, cfg.Cmd, cfg.Args...)
+		cmd.Dir = cfg.WorkDir
 
-	if cfg.Stdin != nil {
-		cmd.Stdin = cfg.Stdin
-	}
+		var outBuf, errBuf strings.Builder
+		cmd.Stdout = &outBuf
+		cmd.Stderr = &errBuf
 
-	// Set up environment variables
-	if len(cfg.Env) > 0 {
-		cmd.Env = os.Environ() // Start with current environment
-		for k, v := range cfg.Env {
-			cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
+		if cfg.Stdin != nil {
+			cmd.Stdin = cfg.Stdin
+		}
+
+		// Set up environment variables
+		if len(cfg.Env) > 0 {
+			cmd.Env = os.Environ() // Start with current environment
+			for k, v := range cfg.Env {
+				cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
+			}
 		}
-	}
 
-	start := time.Now()
+		start := time.Now()
+		err := cmd.Run()
+		stop := time.Now()
+		diff := stop.Sub(start)
+		cancel()
+
+		output = outBuf.String()
+		errOutput = errBuf.String()
+
+		if err == nil {
+			log.Printf("[%s] SUCCESS (%v)", cfg.Name, diff)
+			printCommandOutput(output)
+			if errBuf.Len() > 0 {
+				log.Printf("[%s] stderr output:", cfg.Name)
+				printCommandOutput(errOutput)
+			}
+			return output, errOutput, nil
+		}
 
-	err := cmd.Run()
-	stop := time.Now()
-	diff := stop.Sub(start)
-	if err != nil {
-		log.Printf("[%s] ERROR (%v): \n", cfg.Name, diff)
-		log.Printf("[%s] err: %q\n", cfg.Name, err)
-	} else {
-		log.Printf("[%s] SUCCESS (%v): \n", cfg.Name, diff)
-	}
+		lastErr = err
+		log.Printf("[%s] ERROR (%v): %v", cfg.Name, diff, err)
+		printCommandOutput(output)
+		if errBuf.Len() > 0 {
+			log.Printf("[%s] stderr output:", cfg.Name)
+			printCommandOutput(errOutput)
+		}
 
-	output := outBuf.String()
-	errOutput := errBuf.String()
+		retryCount++
+		if maxRetries >= 0 && retryCount > maxRetries {
path: experiments/conductor/cmd/runner/utilities.go
user: login: maqiuyujoyce
body: Nit: Looks like the maxRetries is either opts.defaultRetries or the hard coded maxRetries, so `if retryCount > maxRetries {` seems to be sufficient. Otherwise when opts.defaultRetries is set to a negative value, then this command will never stop by itself.
created_at: 2025-03-06T01:03:55Z
author_association: COLLABORATOR
diff_hunk: @@ -296,61 +298,90 @@ func executeCommand(opts *RunnerOptions, cfg CommandConfig) (string, string, err
 		}
 	}
 
-	log.Printf("Starting command step: %s", cfg.Name)
-	log.Printf("[%s] working directory: %s", cfg.Name, cfg.WorkDir)
-	log.Printf("[%s] command: %s %s", cfg.Name, cfg.Cmd, strings.Join(cfg.Args, " "))
-	if cfg.Stdin != nil {
-		log.Printf("[%s] stdin: %s", cfg.Name, cfg.Stdin)
+	if cfg.RetryBackoff == 0 {
+		cfg.RetryBackoff = time.Second
path: experiments/conductor/cmd/runner/utilities.go
user: login: maqiuyujoyce
body: Nit: If this is related to quota then I guess one second of backoff time may not be sufficient. But we can improve it later.
created_at: 2025-03-06T01:06:45Z
author_association: COLLABORATOR
diff_hunk: @@ -82,6 +82,8 @@ func BuildRunnerCmd() *cobra.Command {
 		"", "", "dedicated directory for logging, empty for stdout.")
 	cmd.Flags().DurationVarP(&opts.timeout, "timeout",
 		"t", 5*time.Minute, "Global timeout for commands.")
+	cmd.Flags().IntVarP(&opts.defaultRetries, "retries",
+		"r", 10, "Default number of retries for failed commands.")
path: experiments/conductor/cmd/runner/runner.go
user: login: maqiuyujoyce
body: Nit: We should suggest increasing timeout when the number of retries here is large.
created_at: 2025-03-06T01:07:39Z
author_association: COLLABORATOR
diff_hunk: @@ -82,6 +82,8 @@ func BuildRunnerCmd() *cobra.Command {
 		"", "", "dedicated directory for logging, empty for stdout.")
 	cmd.Flags().DurationVarP(&opts.timeout, "timeout",
 		"t", 5*time.Minute, "Global timeout for commands.")
+	cmd.Flags().IntVarP(&opts.defaultRetries, "retries",
+		"r", 10, "Default number of retries for failed commands.")
path: experiments/conductor/cmd/runner/runner.go
user: login: barney-s
body: duration is per retry
created_at: 2025-03-06T02:11:19Z
author_association: COLLABORATOR
diff_hunk: @@ -296,61 +298,90 @@ func executeCommand(opts *RunnerOptions, cfg CommandConfig) (string, string, err
 		}
 	}
 
-	log.Printf("Starting command step: %s", cfg.Name)
-	log.Printf("[%s] working directory: %s", cfg.Name, cfg.WorkDir)
-	log.Printf("[%s] command: %s %s", cfg.Name, cfg.Cmd, strings.Join(cfg.Args, " "))
-	if cfg.Stdin != nil {
-		log.Printf("[%s] stdin: %s", cfg.Name, cfg.Stdin)
+	if cfg.RetryBackoff == 0 {
+		cfg.RetryBackoff = time.Second
path: experiments/conductor/cmd/runner/utilities.go
user: login: barney-s
body: changed it to 60s for generative commands.
created_at: 2025-03-06T02:16:37Z
author_association: COLLABORATOR
diff_hunk: @@ -296,61 +298,90 @@ func executeCommand(opts *RunnerOptions, cfg CommandConfig) (string, string, err
 		}
 	}
 
-	log.Printf("Starting command step: %s", cfg.Name)
-	log.Printf("[%s] working directory: %s", cfg.Name, cfg.WorkDir)
-	log.Printf("[%s] command: %s %s", cfg.Name, cfg.Cmd, strings.Join(cfg.Args, " "))
-	if cfg.Stdin != nil {
-		log.Printf("[%s] stdin: %s", cfg.Name, cfg.Stdin)
+	if cfg.RetryBackoff == 0 {
+		cfg.RetryBackoff = time.Second
 	}
-	if len(cfg.Env) > 0 {
-		log.Printf("[%s] environment:", cfg.Name)
-		for k, v := range cfg.Env {
-			log.Printf("  %s=%s", k, v)
-		}
+
+	maxRetries := opts.defaultRetries
+	// If MaxRetries is set in config, cap it at that
+	if cfg.MaxRetries > 0 && cfg.MaxRetries < maxRetries {
+		maxRetries = cfg.MaxRetries
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), cfg.Timeout)
-	defer cancel()
+	var lastErr error
+	var output, errOutput string
+	retryCount := 0
 
-	cmd := exec.CommandContext(ctx, cfg.Cmd, cfg.Args...)
-	cmd.Dir = cfg.WorkDir
+	for {
+		log.Printf("Starting command step: %s (attempt %d/%d)", cfg.Name, retryCount+1, maxRetries+1)
+		log.Printf("[%s] working directory: %s", cfg.Name, cfg.WorkDir)
+		log.Printf("[%s] command: %s %s", cfg.Name, cfg.Cmd, strings.Join(cfg.Args, " "))
+		if cfg.Stdin != nil {
+			log.Printf("[%s] stdin: %s", cfg.Name, cfg.Stdin)
+		}
+		if len(cfg.Env) > 0 {
+			log.Printf("[%s] environment:", cfg.Name)
+			for k, v := range cfg.Env {
+				log.Printf("  %s=%s", k, v)
+			}
+		}
 
-	var outBuf, errBuf strings.Builder
-	cmd.Stdout = &outBuf
-	cmd.Stderr = &errBuf
+		ctx, cancel := context.WithTimeout(context.Background(), cfg.Timeout)
+		cmd := exec.CommandContext(ctx, cfg.Cmd, cfg.Args...)
+		cmd.Dir = cfg.WorkDir
 
-	if cfg.Stdin != nil {
-		cmd.Stdin = cfg.Stdin
-	}
+		var outBuf, errBuf strings.Builder
+		cmd.Stdout = &outBuf
+		cmd.Stderr = &errBuf
 
-	// Set up environment variables
-	if len(cfg.Env) > 0 {
-		cmd.Env = os.Environ() // Start with current environment
-		for k, v := range cfg.Env {
-			cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
+		if cfg.Stdin != nil {
+			cmd.Stdin = cfg.Stdin
+		}
+
+		// Set up environment variables
+		if len(cfg.Env) > 0 {
+			cmd.Env = os.Environ() // Start with current environment
+			for k, v := range cfg.Env {
+				cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
+			}
 		}
-	}
 
-	start := time.Now()
+		start := time.Now()
+		err := cmd.Run()
+		stop := time.Now()
+		diff := stop.Sub(start)
+		cancel()
+
+		output = outBuf.String()
+		errOutput = errBuf.String()
+
+		if err == nil {
+			log.Printf("[%s] SUCCESS (%v)", cfg.Name, diff)
+			printCommandOutput(output)
+			if errBuf.Len() > 0 {
+				log.Printf("[%s] stderr output:", cfg.Name)
+				printCommandOutput(errOutput)
+			}
+			return output, errOutput, nil
+		}
 
-	err := cmd.Run()
-	stop := time.Now()
-	diff := stop.Sub(start)
-	if err != nil {
-		log.Printf("[%s] ERROR (%v): \n", cfg.Name, diff)
-		log.Printf("[%s] err: %q\n", cfg.Name, err)
-	} else {
-		log.Printf("[%s] SUCCESS (%v): \n", cfg.Name, diff)
-	}
+		lastErr = err
+		log.Printf("[%s] ERROR (%v): %v", cfg.Name, diff, err)
+		printCommandOutput(output)
+		if errBuf.Len() > 0 {
+			log.Printf("[%s] stderr output:", cfg.Name)
+			printCommandOutput(errOutput)
+		}
 
-	output := outBuf.String()
-	errOutput := errBuf.String()
+		retryCount++
+		if maxRetries >= 0 && retryCount > maxRetries {
path: experiments/conductor/cmd/runner/utilities.go
user: login: barney-s
body: added a check to make sure defaultretries is not -ve
created_at: 2025-03-06T02:20:47Z
author_association: COLLABORATOR

PR comments:
user: login: barney-s
created_at: 2025-03-05T23:46:35Z
body:  > 1. The place where I found retries are needed is when the communication with LLM hangs/terminates unexpectedly. We talk to codebot for steps like script generation, API identification (right?), mockgcp generation. So I'm thinking we may want to focus on retries about these steps. And in this PR, I didn't find the retry for creating scripts. As I shared, I had to go through 12 retries to finish generating scripts for 40 resources.

Bumped the default retries to 10. For LLM steps i have not set maxRetries. So it would take the user passed/default retries count for those.

> 2. I think with max retries of 2, the retry may not be that useful, especially you set the default retry to be 3. I'd probably start the max retries to be at least 10 considering we are doing batch work.

maxRetries is an optional per command max that is hardcoded. The goal is not to retry certain deterministic commands N times and instead cap them to maxRetries. Re your suggestion 2 is as good as 1. I tend to agree. Just that it tries once more. 

> 3. Have you tried to verify the retry works for steps involving using codebot for more than 50 resources?

Have not yet. Will do and update here. 
author_association: COLLABORATOR
user: login: google-oss-prow[bot]
created_at: 2025-03-06T01:08:11Z
body: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/GoogleCloudPlatform/k8s-config-connector/pull/3864#pullrequestreview-2662836964" title="Approved">maqiuyujoyce</a>*

The full list of commands accepted by this bot can be found [here](https://oss.gprow.dev/command-help?repo=GoogleCloudPlatform%2Fk8s-config-connector).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[experiments/OWNERS](https://github.com/GoogleCloudPlatform/k8s-config-connector/blob/master/experiments/OWNERS)~~ [maqiuyujoyce]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
author_association: CONTRIBUTOR
user: login: anhdle-sso
created_at: 2025-03-06T02:33:04Z
body: /lgtm
author_association: COLLABORATOR