Repository: hashicorp/nomad
File: drivers/shared/executor/executor_linux.go (Go)

Code before:
func NewExecutorWithIsolation(logger hclog.Logger, compute cpustats.Compute) Executor {
	sigch := make(chan os.Signal, 4)

	le := &LibcontainerExecutor{
		id:             strings.ReplaceAll(uuid.Generate(), "-", "_"),
		logger:         logger.Named("isolated_executor"),
		compute:        compute,
		totalCpuStats:  cpustats.New(compute),
		userCpuStats:   cpustats.New(compute),
		systemCpuStats: cpustats.New(compute),
		sigChan:        sigch,
	}

	go le.catchSignals()

	le.processStats = procstats.New(compute, le)
	return le
}

func (l *LibcontainerExecutor) ListProcesses() set.Collection[int] {
	return procstats.List(l.command)
}

// cleanOldProcessesInCGroup kills processes that might ended up orphans when
// the executor was unexpectedly killed and nomad can't reconnect to them.
func (l *LibcontainerExecutor) cleanOldProcessesInCGroup(nomadRelativePath string) error {
	l.logger.Debug("looking for old processes", "path", nomadRelativePath)

	root := cgroupslib.GetDefaultRoot()
	orphanedPIDs, err := cgroups.GetAllPids(filepath.Join(root, nomadRelativePath))
	if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("unable to get orphaned task PIDs: %v", err)
	}

	for _, pid := range orphanedPIDs {
		l.logger.Info("killing orphaned process", "pid", pid)

		// Avoid bringing down the whole node by mistake, very unlikely case,
		// but it's better to be sure.
		if pid == 1 {
			continue
		}

		if err := syscall.Kill(pid, syscall.SIGKILL); err != nil {
			return fmt.Errorf("unable to send signal to process %d: %v", pid, err)
		}
	}

	// Make sure the PID was removed from the cgroup file, otherwise
	// libcontainer will not be able to launch. Five retries every 100 ms should be
	// more than enough.

Review comment (bug):
Note to reviewers: I changed two things about this method. For one, it returns an error if it fails, because libcontainer 1.2+ will not launch a process in a cgroup that already has a process in it; not sure what exactly was the behavior before, but an error from this method can not be ignored. Secondly, new libcontainer is slightly faster in launching processes, so we end up having a race condition between the time the process gets a SIGKILL and the time its PID gets removed from the cgroups pidfile. Thus this method now only returns successfully once it makes sure the PID got removed. 

Code after:
func NewExecutorWithIsolation(logger hclog.Logger, compute cpustats.Compute) Executor {
	sigch := make(chan os.Signal, 4)

	le := &LibcontainerExecutor{
		id:             strings.ReplaceAll(uuid.Generate(), "-", "_"),
		logger:         logger.Named("isolated_executor"),
		compute:        compute,
		totalCpuStats:  cpustats.New(compute),
		userCpuStats:   cpustats.New(compute),
		systemCpuStats: cpustats.New(compute),
		sigChan:        sigch,
	}

	go le.catchSignals()

	le.processStats = procstats.New(compute, le)
	return le
}

func (l *LibcontainerExecutor) ListProcesses() set.Collection[int] {
	return procstats.List(l.command)
}

// cleanOldProcessesInCGroup kills processes that might ended up orphans when
// the executor was unexpectedly killed and nomad can't reconnect to them.
func (l *LibcontainerExecutor) cleanOldProcessesInCGroup(nomadRelativePath string) error {
	l.logger.Debug("looking for old processes", "path", nomadRelativePath)

	root := cgroupslib.GetDefaultRoot()
	orphanedPIDs, err := cgroups.GetAllPids(filepath.Join(root, nomadRelativePath))
	if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("unable to get orphaned task PIDs: %v", err)
	}

	for _, pid := range orphanedPIDs {
		l.logger.Info("killing orphaned process", "pid", pid)

		// Avoid bringing down the whole node by mistake, very unlikely case,
		// but it's better to be sure.
		if pid == 1 {
			continue
		}

		if err := syscall.Kill(pid, syscall.SIGKILL); err != nil {
			return fmt.Errorf("unable to send signal to process %d: %v", pid, err)
		}
	}

	if len(orphanedPIDs) == 0 {
		return nil
	}