Change:
restartCommand, []string{kubeletHealthCheckURL}, "kubelet.log", <ins> e.monitorParent, </ins> true /* restartOnExit */) return server, server.start() } <del> // A server manages a separate server process started and killed with // commands. type server struct { // name is the name of the server, it is only used for logging. name string // startCommand is the command used to start the server startCommand *exec.Cmd // killCommand is the command used to stop the server. It is not required. If it // is not specified, `sudo kill` will be used to stop the server. killCommand *exec.Cmd // restartCommand is the command used to restart the server. If provided, it will be used // instead of startCommand when restarting the server. restartCommand *exec.Cmd // healthCheckUrls is the urls used to check whether the server is ready. healthCheckUrls []string // outFilename is the name of the log file. The stdout and stderr of the server // will be redirected to this file. outFilename string // restartOnExit determines whether a restart loop is launched with the server restartOnExit bool // Writing to this channel, if it is not nil, stops the restart loop. // When tearing down a server, you should check for this channel and write to it if it exists. stopRestartingCh chan<- bool // Read from this to confirm that the restart loop has stopped. ackStopRestartingCh <-chan bool } // newServer returns a new server with the given name, commands, health check // URLs, etc. func newServer(name string, start, kill, restart *exec.Cmd, urls []string, outputFileName string, restartOnExit bool) *server { return &server{ name:            name, startCommand:    start, killCommand:     kill, restartCommand:  restart, healthCheckUrls: urls, outFilename:     outputFileName, restartOnExit:   restartOnExit, } } // commandToString format command to string. func commandToString(c *exec.Cmd) string { if c == nil { return "" } return strings.Join(append([]string{c.Path}, c.Args[1:]...), " ") } func (s *server) String() string { return fmt.Sprintf("server %q start-command: `%s`, kill-command: `%s`, restart-command: `%s`, health-check: %v, output-file: %q", s.name, commandToString(s.startCommand), commandToString(s.killCommand), commandToString(s.restartCommand), s.healthCheckUrls, s.outFilename) </del> <ins> func adjustConfigForSystemd(config string) string { return strings.Replace(config, "%", "%%", -1) </ins> } <del> // start starts the server by running its commands, monitors it with a health // check, and ensures that it is restarted if applicable. // // Note: restartOnExit == true requires len(s.healthCheckUrls) > 0 to work properly. func (s *server) start() error { glog.Infof("Starting server %q with command %q", s.name, commandToString(s.startCommand)) errCh := make(chan error) // Set up restart channels if the server is configured for restart on exit. var stopRestartingCh, ackStopRestartingCh chan bool if s.restartOnExit { if len(s.healthCheckUrls) == 0 { return fmt.Errorf("Tried to start %s which has s.restartOnExit == true, but no health check urls provided.", s) } stopRestartingCh = make(chan bool) ackStopRestartingCh = make(chan bool) s.stopRestartingCh = stopRestartingCh s.ackStopRestartingCh = ackStopRestartingCh </del> <ins> // getLogFiles gets logs of interest either via journalctl or by creating sym // links. Since we scp files from the remote directory, symlinks will be // treated as normal files and file contents will be copied over. func (e *E2EServices) getLogFiles() { // Nothing to do if report dir is not specified. if framework.TestContext.ReportDir == "" { return </ins> } <del> // This goroutine actually runs the start command for the server. go func() { defer close(errCh) // Create the output filename outPath := path.Join(framework.TestContext.ReportDir, s.outFilename) outfile, err := os.Create(outPath) if err != nil { errCh <- fmt.Errorf("failed to create file %q for `%s` %v.", outPath, s, err) return } else { glog.Infof("Output file for server %q: %v", s.name, outfile.Name()) } defer outfile.Close() defer outfile.Sync() // Set the command to write the output file s.startCommand.Stdout = outfile s.startCommand.Stderr = outfile // Death of this test process should kill the server as well. attrs := &syscall.SysProcAttr{} // Hack to set linux-only field without build tags. deathSigField := reflect.ValueOf(attrs).Elem().FieldByName("Pdeathsig") if deathSigField.IsValid() { deathSigField.Set(reflect.ValueOf(syscall.SIGTERM)) } else { errCh <- fmt.Errorf("failed to set Pdeathsig field (non-linux build)") return } s.startCommand.SysProcAttr = attrs // Start the command err = s.startCommand.Start() if err != nil { errCh <- fmt.Errorf("failed to run %s: %v", s, err) return } if !s.restartOnExit { glog.Infof("Waiting for server %q start command to complete", s.name) // If we aren't planning on restarting, ok to Wait() here to release resources. // Otherwise, we Wait() in the restart loop. err = s.startCommand.Wait() if err != nil { errCh <- fmt.Errorf("failed to run start command for server %q: %v", s.name, err) return </del> <ins> glog.Info("Fetching log files...") journaldFound := isJournaldAvailable() for targetFileName, logFileData := range e.logFiles { targetLink := path.Join(framework.TestContext.ReportDir, targetFileName) if journaldFound { // Skip log files that do not have an equivalent in journald-based machines. if len(logFileData.journalctlCommand) == 0 { continue </ins> } <del> } else { usedStartCmd := true for { glog.Infof("Running health check for service %q", s.name) // Wait for an initial health check to pass, so that we are sure the server started. err := readinessCheck(s.name, s.healthCheckUrls, nil) if err != nil { if usedStartCmd { glog.Infof("Waiting for server %q start command to complete after initial health check failed", s.name) s.startCommand.Wait() // Release resources if necessary. } // This should not happen, immediately stop the e2eService process. glog.Fatalf("restart loop readinessCheck failed for %s", s) } else { glog.Infof("Initial health check passed for service %q", s.name) } // Initial health check passed, wait until a health check fails again. stillAlive: for { select { case <-stopRestartingCh: ackStopRestartingCh <- true return case <-time.After(time.Second): for _, url := range s.healthCheckUrls { resp, err := http.Head(url) if err != nil || resp.StatusCode != http.StatusOK { break stillAlive } } } } if usedStartCmd { s.startCommand.Wait() // Release resources from last cmd usedStartCmd = false } if s.restartCommand != nil { // Always make a fresh copy of restartCommand before // running, we may have to restart multiple times s.restartCommand = &exec.Cmd{ Path:        s.restartCommand.Path, Args:        s.restartCommand.Args, Env:         s.restartCommand.Env, Dir:         s.restartCommand.Dir, Stdin:       s.restartCommand.Stdin, Stdout:      s.restartCommand.Stdout, Stderr:      s.restartCommand.Stderr, ExtraFiles:  s.restartCommand.ExtraFiles, SysProcAttr: s.restartCommand.SysProcAttr, } // Run and wait for exit. This command is assumed to have // short duration, e.g. systemctl restart glog.Infof("Restarting server %q with restart command", s.name) err = s.restartCommand.Run() if err != nil { // This should not happen, immediately stop the e2eService process. glog.Fatalf("restarting server %s with restartCommand failed. Error: %v.", s, err) } } else { s.startCommand = &exec.Cmd{ Path:        s.startCommand.Path, Args:        s.startCommand.Args, Env:         s.startCommand.Env, Dir:         s.startCommand.Dir, Stdin:       s.startCommand.Stdin, Stdout:      s.startCommand.Stdout, Stderr:      s.startCommand.Stderr, ExtraFiles:  s.startCommand.ExtraFiles, SysProcAttr: s.startCommand.SysProcAttr, } glog.Infof("Restarting server %q with start command", s.name) err = s.startCommand.Start() usedStartCmd = true if err != nil { // This should not happen, immediately stop the e2eService process. glog.Fatalf("Restarting %s with startCommand failed. Error: %v.", s, err) } </del> <ins> glog.Infof("Get log file %q with journalctl command %v.", targetFileName, logFileData.journalctlCommand) out, err := exec.Command("sudo", append([]string{"journalctl"}, logFileData.journalctlCommand...)...).CombinedOutput() if err != nil { glog.Errorf("failed to get %q from journald: %v, %v", targetFileName, string(out), err) } else { if err = ioutil.WriteFile(targetLink, out, 0644); err != nil { glog.Errorf("failed to write logs to %q: %v", targetLink, err) </ins> } } <del> } }() return readinessCheck(s.name, s.healthCheckUrls, errCh) } // kill runs the server's kill command. func (s *server) kill() error { glog.Infof("Kill server %q", s.name) name := s.name cmd := s.startCommand // If s has a restart loop, turn it off. if s.restartOnExit { s.stopRestartingCh <- true <-s.ackStopRestartingCh } if s.killCommand != nil { return s.killCommand.Run() } if cmd == nil { return fmt.Errorf("could not kill %q because both `killCommand` and `startCommand` are nil", name) } if cmd.Process == nil { glog.V(2).Infof("%q not running", name) return nil } pid := cmd.Process.Pid if pid <= 1 { return fmt.Errorf("invalid PID %d for %q", pid, name) } // Attempt to shut down the process in a friendly manner before forcing it. waitChan := make(chan error) go func() { _, err := cmd.Process.Wait() waitChan <- err close(waitChan) }() const timeout = 10 * time.Second for _, signal := range []string{"-TERM", "-KILL"} { glog.V(2).Infof("Killing process %d (%s) with %s", pid, name, signal) cmd := exec.Command("sudo", "kill", signal, strconv.Itoa(pid)) // Run the 'kill' command in a separate process group so sudo doesn't ignore it attrs := &syscall.SysProcAttr{} // Hack to set unix-only field without build tags. setpgidField := reflect.ValueOf(attrs).Elem().FieldByName("Setpgid") if setpgidField.IsValid() { setpgidField.Set(reflect.ValueOf(true)) } else { return fmt.Errorf("Failed to set Setpgid field (non-unix build)") } cmd.SysProcAttr = attrs _, err := cmd.Output() if err != nil { glog.Errorf("Error signaling process %d (%s) with %s: %v", pid, name, signal, err) </del> continue } <del> select { case err := <-waitChan: if err != nil { return fmt.Errorf("error stopping %q: %v", name, err) </del> <ins> for _, file := range logFileData.files { if _, err := os.Stat(file); err != nil { // Expected file not found on this distro. continue } if err := copyLogFile(file, targetLink); err != nil { glog.Error(err) } else { break </ins> } <del> // Success! return nil case <-time.After(timeout): // Continue. </del> } } <del> return fmt.Errorf("unable to stop %q", name) </del> } <del> func adjustConfigForSystemd(config string) string { return strings.Replace(config, "%", "%%", -1) </del> <ins> // isJournaldAvailable returns whether the system executing the tests uses // journald. func isJournaldAvailable() bool { _, err := exec.LookPath("journalctl") return err == nil </ins> } func copyLogFile(src, target string) error {

Discussion:
In node e2e, we have the option to keep the services running after test for further debug. However, we set to for the services, which means that once the parent exits, they will receive and exit with the parent if they don't trap the signal. With this, option will never work. If we still need this feature, we should not set if is false. /cc
We still need this feature. It helps with debugging test failures by keeping all the daemons running upon failure. , Lantao Liu wrote: