func NewAgentListener(sess *Session) (net.Listener, string, error) {
	dir, err := ioutil.TempDir("", "auth-agent")
	if err != nil {
		return nil, dir, err
	}
	if err := os.Chown(dir, sess.Uid, sess.Gid); err != nil {
		return nil, dir, err
	}
	l, err := net.Listen("unix", path.Join(dir, "listener.sock"))
	if err != nil {
		os.RemoveAll(dir)
		return nil, dir, err
	}
	if err := os.Chown(l.Addr().String(), sess.Uid, sess.Gid); err != nil {
		return nil, dir, err
	}
	return l, dir, nil
}
I borrowed this code from https://github.com/gliderlabs/ssh/blob/47df570d18ad49f77cf66f76bc3fce3e92400768/agent.go#L38
And modify code to remove temporary directory after closing
package agent
import (
	"bufio"
	"bytes"
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"github.com/gliderlabs/ssh"
	"github.com/kohkimakimoto/cofu/cofu"
	"github.com/kohkimakimoto/cofu/ext/agent/envfile"
	"github.com/kr/pty"
	"github.com/pkg/errors"
	"io"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"os/user"
	"path"
	"strconv"
	"strings"
	"syscall"
	"time"
	"unsafe"
)
func startSSHServer(a *Agent) error {
	logger := a.Logger
	ssh.Handle(func(sess ssh.Session) {
		defer func() {
			if err := recover(); err != nil {
				logger.Error(err)
				if _, err := io.WriteString(sess.Stderr(), fmt.Sprintf("%v\r\n", err)); err != nil {
					logger.Error(err)
				}
				if err := sess.Exit(1); err != nil {
					logger.Error(err)
				}
			}
		}()
		logger.Infof("Connected from: %s", sess.RemoteAddr().String())
		err := handleSSHSession(a, sess)
		if err != nil {
			if _, err := io.WriteString(sess.Stderr(), fmt.Sprintf("ERROR: %v\r\n", err)); err != nil {
				logger.Error(err)
			}
			logger.Error(err)
		}
		status := exitStatus(err)
		if err := sess.Exit(status); err != nil {
			logger.Error(err)
		}
		logger.Infof("SSH connection exited with status %d", status)
	})
	var options []ssh.Option
	publicKeyOption := ssh.PublicKeyAuth(func(ctx ssh.Context, key ssh.PublicKey) bool {
		if a.Config.HotReload {
			config, err := a.Config.Reload()
			if err != nil {
				logger.Errorf("failed to reload config: %v", err)
			}
			a.Config = config
		}
		remoteAddr := ctx.RemoteAddr().String()
		remoteHost := remoteAddr[:strings.LastIndex(remoteAddr, ":")]
		logger.Debugf("remoteAddr: %s", remoteAddr)
		logger.Debugf("remoteHost: %s", remoteHost)
		if a.Config.DisableLocalAuth && (remoteHost == "127.0.0.1" || remoteHost == "[::1]") {
			logger.Debugf("passed public key auth because the config disables local auth")
			return true
		}
		if checkAuthKey(a, ctx, key) {
			return true
		}
		return false
	})
	options = append(options, publicKeyOption)
	if a.Config.HostKeyFile != "" {
		if _, err := os.Stat(a.Config.HostKeyFile); os.IsNotExist(err) {
			b, err := generateNewKey()
			if err != nil {
				return err
			}
			if err := ioutil.WriteFile(a.Config.HostKeyFile, b, 0600); err != nil {
				return err
			}
			logger.Infof("%s is not existed. generated it automatically.", a.Config.HostKeyFile)
		}
		hostKeyOption := ssh.HostKeyFile(a.Config.HostKeyFile)
		options = append(options, hostKeyOption)
		logger.Infof("Using host key file %s", a.Config.HostKeyFile)
	} else if a.Config.HostKey != "" {
		hostKeyOption := ssh.HostKeyPEM([]byte(a.Config.HostKey))
		options = append(options, hostKeyOption)
		logger.Info("Using host key from the config file")
	} else {
		hostKeyFile := "/tmp/cofu-agent-host.key"
		if _, err := os.Stat(hostKeyFile); os.IsNotExist(err) {
			b, err := generateNewKey()
			if err != nil {
				return err
			}
			if err := ioutil.WriteFile(hostKeyFile, b, 0600); err != nil {
				return err
			}
		}
		hostKeyOption := ssh.HostKeyFile(hostKeyFile)
		options = append(options, hostKeyOption)
		logger.Infof("Using host key file %s", hostKeyFile)
	}
	logger.Infof("Starting SSH protocol server on %s", a.Config.Addr)
	return ssh.ListenAndServe(a.Config.Addr, nil, options...)
}
func handleSSHSession(a *Agent, sshSession ssh.Session) error {
	logger := a.Logger
	sess := NewSession(a, sshSession)
	if err := a.SessionManager.SetSession(sess); err != nil {
		return err
	}
	defer sess.Terminate()
	logger.Infof("Allocated session: %s sandbox: %s", sess.ID, sess.Sandbox)
	ptyReq, winCh, isPty := sess.Pty()
	logger.Debugf("cofu-agent got a ssh command: %v", sess.Command())
	logger.Debugf("The session user is %s", sess.User())
	currentEnviron := append(sess.Environ(),
		fmt.Sprintf("COFU_AGENT_VERSION=%s", cofu.Version),
		fmt.Sprintf("COFU_AGENT_SESSION=%s", sess.ID),
		fmt.Sprintf("COFU_AGENT_SESSION_ID=%s", sess.ID),
		fmt.Sprintf("COFU_AGENT_SANDBOX=%s", sess.Sandbox),
		fmt.Sprintf("COFU_COMMAND=%s", cofu.BinPath),
	)
	if sess.Fn != nil {
		logger.Infof("fetched function: %s", sess.Fn.Name)
		currentEnviron = append(currentEnviron, fmt.Sprintf("COFU_AGENT_FUNCTION=%s", sess.Fn.Name))
	}
	sessCommand := []string{}
	if len(sess.Command()) > 0 {
		sessCommand = sess.Command()
	} else {
		if sess.Fn != nil {
			if sess.Fn.Command != nil && len(sess.Fn.Command) > 0 {
				sessCommand = sess.Fn.Command
			}
		} else {
			sessCommand = []string{"/bin/bash"}
		}
	}
	if sess.Fn != nil {
		currentEnviron = append(currentEnviron, fmt.Sprintf("COFU_AGENT_FUNCTION_SESSION_COMMAND=%s", strings.Join(sessCommand, " ")))
	}
	if isPty {
		currentEnviron = append(currentEnviron, "COFU_AGENT_PTY=1")
	}
	uid, gid, u, err := getUidAndGid(sess)
	if err != nil {
		return err
	}
	logger.Debugf("This session runs by user: %d, group: %d", uid, gid)
	if os.Getuid() != 0 && os.Getuid() != uid {
		return errors.New("The Cofu Agent is running non root user. The connection can be accepted only by same user who runs Cofu Agent.")
	}
	sess.Uid = uid
	sess.Gid = gid
	sess.UserStruct = u
	currentEnviron = append(currentEnviron,
		fmt.Sprintf("USER=%s", u.Name),
		fmt.Sprintf("LOGNAME=%s", u.Name),
		fmt.Sprintf("HOME=%s", u.HomeDir),
	)
	currentEnviron, err = takeForwardAgentIfRequested(sess, currentEnviron)
	if err != nil {
		return err
	}
	sandBoxDir, err := a.CreateSandBoxDirIfNotExist(sess)
	if err != nil {
		return err
	}
	currentEnviron = append(currentEnviron, fmt.Sprintf("COFU_AGENT_SANDBOX_DIR=%s", sandBoxDir))
	if a.Config.EnvironmentFile != "" {
		if _, err := os.Stat(a.Config.EnvironmentFile); err == nil {
			appendedEnv, err := envfile.Load(a.Config.EnvironmentFile, currentEnviron)
			if err != nil {
				logger.Error(err)
			}
			if appendedEnv != nil {
				currentEnviron = append(currentEnviron, appendedEnv...)
			}
		}
	}
	for _, v := range a.Config.Environment {
		currentEnviron = append(currentEnviron, expandEnvironToString(v, currentEnviron))
	}
	commandAndArgs := []string{}
	if sess.Fn != nil && sess.Fn.Entrypoint != nil && len(sess.Fn.Entrypoint) > 0 {
		commandAndArgs = sess.Fn.Entrypoint
	} else {
		commandAndArgs = []string{}
	}
	commandAndArgs = append(commandAndArgs, sessCommand...)
	if len(commandAndArgs) == 0 {
		return errors.New("You've successfully authenticated, but it has no command to run.")
	}
	command := commandAndArgs[0]
	args := []string{}
	if len(commandAndArgs) >= 2 {
		args = commandAndArgs[1:]
	}
	logger.Debugf("command: %v", commandAndArgs)
	cmd := exec.Command(command, args...)
	cmd.Env = append(os.Environ(), currentEnviron...)
	if os.Getuid() == 0 {
		cmd.SysProcAttr = &syscall.SysProcAttr{}
		cmd.SysProcAttr.Credential = &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid)}
	}
	cmd.Dir = sandBoxDir
	if isPty {
		cmd.Env = append(cmd.Env, fmt.Sprintf("TERM=%s", ptyReq.Term))
		f, err := pty.Start(cmd)
		if err != nil {
			return err
		}
		go func() {
			for win := range winCh {
				setWinsize(f, win.Width, win.Height)
			}
		}()
		go func() {
			io.Copy(f, sess)  
		}()
		io.Copy(sess, f)  
		return cmd.Wait()
	} else {
		cmd.Stdout = sess
		cmd.Stderr = sess.Stderr()
		stdin, err := cmd.StdinPipe()
		if err != nil {
			return err
		}
		go func() {
			io.Copy(stdin, sess)
			stdin.Close()
		}()
		if err := cmd.Start(); err != nil {
			return err
		}
		if sess.Fn != nil && sess.Fn.Timeout > 0 {
			done := make(chan error)
			go func() {
				done <- cmd.Wait()
			}()
			select {
			case <-time.After(time.Duration(sess.Fn.Timeout) * time.Second):
				if err := cmd.Process.Kill(); err != nil {
					return errors.Wrapf(err, "failed to kill: "+err.Error())
				}
				return fmt.Errorf("Cofu Agent function timeout. it took time over %d sec.", sess.Fn.Timeout)
			case err := <-done:
				defer close(done)
				if err != nil {
					return err
				}
				return nil
			}
		} else {
			return cmd.Wait()
		}
	}
}
func exitStatus(err error) int {
	var exitStatus int
	if err != nil {
		if e2, ok := err.(*exec.ExitError); ok {
			if status, ok := e2.Sys().(syscall.WaitStatus); ok {
				exitStatus = status.ExitStatus()
			} else {
				exitStatus = 1
			}
		} else {
			exitStatus = 1
		}
	} else {
		exitStatus = 0
	}
	return exitStatus
}
func setWinsize(f *os.File, w, h int) {
	syscall.Syscall(syscall.SYS_IOCTL, f.Fd(), uintptr(syscall.TIOCSWINSZ),
		uintptr(unsafe.Pointer(&struct{ h, w, x, y uint16 }{uint16(h), uint16(w), 0, 0})))
}
func getUidAndGid(sess *Session) (int, int, *user.User, error) {
	if sess.Fn != nil {
		return getUidAndGidFromFunctionConfig(sess.Fn)
	} else {
		return getUidAndGidFromUsername(sess.User())
	}
}
func getUidAndGidFromFunctionConfig(fn *FunctionConfig) (int, int, *user.User, error) {
	var uid, gid int
	var u *user.User
	var err error
	if fn.User != "" {
		uid, gid, u, err = getUidAndGidFromUsername(fn.User)
		if err != nil {
			return -1, -1, nil, err
		}
	} else {
		uid = os.Getuid()
		gid = os.Getgid()
		u, err = LookupUserStruct(strconv.Itoa(uid))
		if err != nil {
			return -1, -1, nil, err
		}
	}
	groupName := fn.Group
	if groupName != "" {
		id, err := LookupGroup(groupName)
		if err != nil {
			return -1, -1, nil, err
		}
		gid = id
	}
	return uid, gid, u, nil
}
func getUidAndGidFromUsername(userName string) (int, int, *user.User, error) {
	u, err := LookupUserStruct(userName)
	if err != nil {
		return -1, -1, nil, err
	}
	uid, err := strconv.Atoi(u.Uid)
	if err != nil {
		return -1, -1, nil, err
	}
	gid, err := strconv.Atoi(u.Gid)
	if err != nil {
		return -1, -1, nil, err
	}
	return uid, gid, u, nil
}
func takeForwardAgentIfRequested(sess *Session, env []string) ([]string, error) {
	logger := sess.Agt.Logger
	if ssh.AgentRequested(sess) {
		l, tmpDir, err := NewAgentListener(sess)
		if err != nil {
			return env, err
		}
		logger.Debugf("Created agent listener to forward agent: %s", l.Addr().String())
		sess.ForwardAgentListener = l
		go func() {
			ssh.ForwardAgentConnections(l, sess)
			logger.Debugf("Closed agent listener: %s", l.Addr().String())
			os.RemoveAll(tmpDir)
			logger.Debugf("Removed: %s", tmpDir)
		}()
		env = append(env, fmt.Sprintf("%s=%s", "SSH_AUTH_SOCK", l.Addr().String()))
		return env, nil
	}
	return env, nil
}