Repository: etcd-io/etcd
File: server/embed/etcd.go (Go)

Code before:

		zap.String("v2-deprecation", string(ec.V2Deprecation)),
	)
}

// Config returns the current configuration.
func (e *Etcd) Config() Config {
	return e.cfg
}

// Close gracefully shuts down all servers/listeners.
// Client requests will be terminated with request timeout.
// After timeout, enforce remaning requests be closed immediately.
//
// The rough workflow to shut down etcd:
//  1. close the `stopc` channel, so that all error handlers (child
//     goroutines) won't send back any errors anymore;
//  2. close all client and metrics listeners, so that etcd server
//     stops receiving any new connection immediately;
//  3. stop the http and grpc servers gracefully, within request timeout;
//  4. call the cancel function to close the gateway context, so that
//     all gateway connections are closed.
//  5. stop etcd server gracefully, and ensure the main raft loop
//     goroutine is stopped;
//  6. stop all peer listeners, so that it stops receives peer connections
//     and messages (wait up to 1-second);
//  7. wait for all child goroutines (i.e. client handlers, peer handlers
//     and metrics handlers) to exit;
//  8. close the `errc` channel to release the resource. Note that it's only
//     safe to close the `errc` after step 7 above is done, otherwise the
//     child goroutines may send errors back to already closed `errc` channel.
func (e *Etcd) Close() {
	fields := []zap.Field{
		zap.String("name", e.cfg.Name),
		zap.String("data-dir", e.cfg.Dir),
		zap.Strings("advertise-peer-urls", e.cfg.getAdvertisePeerURLs()),
		zap.Strings("advertise-client-urls", e.cfg.getAdvertiseClientURLs()),
	}
	lg := e.GetLogger()
	lg.Info("closing etcd server", fields...)
	defer func() {
		lg.Info("closed etcd server", fields...)
		verify.MustVerifyIfEnabled(verify.Config{
			Logger:     lg,
			DataDir:    e.cfg.Dir,
			ExactIndex: false,
		})
		lg.Sync()
	}()

	//  1. close the `stopc` channel, so that all error handlers (child

Review comment (bug):
Thanks for comments, however it's not what I had in mind. Sorry for not being clearer, having a comments for Close function is good, however I'm more worried about interactions of multiple concurrency primitives like `errc  chan error`, `serversC chan *servers`, `closeOnce sync.Once`, `	wg sync.WaitGroup`, `stopc chan struct{}`. There is zero documentation for those variables. It's not even clear for me what they are responsible, how they are uses by sub what are different states?

Some things that I noticed:
* All sub-goroutines are expected to add to `wg sync.WaitGroup` and write errors to `errc  chan error`.
* Before closing the `errc  chan error` channel, we need to wait for `wg sync.WaitGroup` to ensure all goroutines that write to `errc` has exited (writing to closed channel causes panic).
* To avoid deadlock on subroutines writing errors to `errc` they will give up as writing error when `stopc chan struct{}` is closed. 

If we tracked all of those dependencies, maybe we can untangle the web.

Maybe we can add this information to those variables.
```
type Etcd struct {
    ...
	stopc chan struct{}
	
    // errc includes error from subroutines. 
    // Closed after all subroutines exit (checked via `wg`).
    // Writers should avoid writing after `stopc` is closed by select on reading from `stopc`.
	errc  chan error

	closeOnce sync.Once
    // wg tracks lifecycle of all subroutines. 
	wg        sync.WaitGroup
}
```




Code after:
		zap.String("discovery-keepalive-time", sc.DiscoveryCfg.KeepAliveTime.String()),
		zap.String("discovery-keepalive-timeout", sc.DiscoveryCfg.KeepAliveTimeout.String()),
		zap.Bool("discovery-insecure-transport", sc.DiscoveryCfg.Secure.InsecureTransport),
		zap.Bool("discovery-insecure-skip-tls-verify", sc.DiscoveryCfg.Secure.InsecureSkipVerify),
		zap.String("discovery-cert", sc.DiscoveryCfg.Secure.Cert),
		zap.String("discovery-key", sc.DiscoveryCfg.Secure.Key),
		zap.String("discovery-cacert", sc.DiscoveryCfg.Secure.Cacert),
		zap.String("discovery-user", sc.DiscoveryCfg.Auth.Username),

		zap.String("downgrade-check-interval", sc.DowngradeCheckTime.String()),
		zap.Int("max-learners", sc.ExperimentalMaxLearners),

		zap.String("v2-deprecation", string(ec.V2Deprecation)),
	)
}

// Config returns the current configuration.
func (e *Etcd) Config() Config {
	return e.cfg
}

// Close gracefully shuts down all servers/listeners.
// Client requests will be terminated with request timeout.
// After timeout, enforce remaning requests be closed immediately.
//
// The rough workflow to shut down etcd:
//  1. close the `stopc` channel, so that all error handlers (child
//     goroutines) won't send back any errors anymore;
//  2. stop the http and grpc servers gracefully, within request timeout;
//  3. close all client and metrics listeners, so that etcd server
//     stops receiving any new connection;
//  4. call the cancel function to close the gateway context, so that
//     all gateway connections are closed.
//  5. stop etcd server gracefully, and ensure the main raft loop
//     goroutine is stopped;
//  6. stop all peer listeners, so that it stops receiving peer connections
//     and messages (wait up to 1-second);
//  7. wait for all child goroutines (i.e. client handlers, peer handlers
//     and metrics handlers) to exit;
//  8. close the `errc` channel to release the resource. Note that it's only
//     safe to close the `errc` after step 7 above is done, otherwise the
//     child goroutines may send errors back to already closed `errc` channel.
func (e *Etcd) Close() {
	fields := []zap.Field{
		zap.String("name", e.cfg.Name),
		zap.String("data-dir", e.cfg.Dir),
		zap.Strings("advertise-peer-urls", e.cfg.getAdvertisePeerURLs()),
		zap.Strings("advertise-client-urls", e.cfg.getAdvertiseClientURLs()),
	}
	lg := e.GetLogger()
	lg.Info("closing etcd server", fields...)