Repository: canonical/snapd
Pull request #15291: daemon: only cancel notices requests on Stop/shutdown
Author: login: pedronis
Labels: name: cherry-picked
name: Needs Documentation -auto-
description: Label automatically added which indicates the change needs documentation
Created: 2025-04-04T11:35:09Z | Merged: 2025-04-16T09:01:32Z

We still want to wait for most requests when shutting down/restarting snapd except for notices which are long polling up to a timeout chosen by the clients.

Related: [SNAPDENG-34668](https://warthogs.atlassian.net/browse/SNAPDENG-34668) https://bugs.launchpad.net/snapd/+bug/2104066

[SNAPDENG-34668]: https://warthogs.atlassian.net/browse/SNAPDENG-34668?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ


Diff:
diff --git a/daemon/api_notices.go b/daemon/api_notices.go
index 097f95e8deb..a6f32c21aac 100644
--- a/daemon/api_notices.go
+++ b/daemon/api_notices.go
@@ -132,7 +132,9 @@ func getNotices(c *Command, r *http.Request, user *auth.UserState) Response {
 
 	if timeout != 0 {
 		// Wait up to timeout for notices matching given filter to occur
-		ctx, cancel := context.WithTimeout(r.Context(), timeout)
+		// Use daemon's tomb context so that the request will get canceled as well
+		// when the tomb gets killed when shutting down the daemon
+		ctx, cancel := context.WithTimeout(c.d.tomb.Context(r.Context()), timeout)
 		defer cancel()
 
 		notices, err = st.WaitNotices(ctx, filter)
diff --git a/daemon/daemon.go b/daemon/daemon.go
index 14d5cf6dd8c..531844f1635 100644
--- a/daemon/daemon.go
+++ b/daemon/daemon.go
@@ -90,8 +90,7 @@ type Daemon struct {
 
 	expectedRebootDidNotHappen bool
 
-	mu     sync.Mutex
-	cancel func()
+	mu sync.Mutex
 }
 
 // A ResponseFunc handles one of the individual verbs for a method
@@ -333,15 +332,6 @@ func (d *Daemon) Start(ctx context.Context) (err error) {
 		panic("internal error: no Overlord")
 	}
 
-	ctx, cancel := context.WithCancel(ctx)
-	d.cancel = cancel
-	defer func() {
-		// cancel the context on any errors
-		if err != nil {
-			cancel()
-		}
-	}()
-
 	to, reasoning, err := d.overlord.StartupTimeout()
 	if err != nil {
 		return err
@@ -365,13 +355,6 @@ func (d *Daemon) Start(ctx context.Context) (err error) {
 	d.serve = &http.Server{
 		Handler:   logit(d.router),
 		ConnState: d.connTracker.trackConn,
-		BaseContext: func(net.Listener) context.Context {
-			// requests will use the context provided to Start, as
-			// the caller will likely cancel it when appropriate
-			// thus canceling any outstanding requests to the snapd
-			// API
-			return ctx
-		},
 	}
 
 	// enable standby handling
@@ -499,10 +482,6 @@ func (d *Daemon) Stop(sigCh chan<- os.Signal) error {
 		return fmt.Errorf("internal error: no Overlord")
 	}
 
-	if d.cancel != nil {
-		d.cancel()
-	}
-
 	d.tomb.Kill(nil)
 
 	// check the state associated with a potential restart with the lock to
diff --git a/daemon/daemon_test.go b/daemon/daemon_test.go
index c5de71e24f0..fcd09c5131e 100644
--- a/daemon/daemon_test.go
+++ b/daemon/daemon_test.go
@@ -22,7 +22,6 @@ package daemon
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"net"
@@ -1492,7 +1491,7 @@ func clientForSnapdSocket() *http.Client {
 	}
 }
 
-func (s *daemonSuite) TestRequestContextCanceledOnStop(c *check.C) {
+func (s *daemonSuite) TestNoticesRequestCanceledOnStop(c *check.C) {
 	d, err := New()
 	c.Assert(err, check.IsNil)
 	// don't talk to the store, needs to be called after daemon.New()
@@ -1503,24 +1502,23 @@ func (s *daemonSuite) TestRequestContextCanceledOnStop(c *check.C) {
 
 	c.Assert(d.Init(), check.IsNil)
 
-	gotReqC := make(chan struct{})
-	reqErrC := make(chan error, 1)
-	d.router.HandleFunc("/test-call", func(w http.ResponseWriter, r *http.Request) {
-		close(gotReqC)
-		// since Stop() is called in the test, the request will get
-		// canceled
-		<-r.Context().Done()
-		reqErrC <- r.Context().Err()
-		w.WriteHeader(500)
-	})
-
 	client := clientForSnapdSocket()
 
-	req, err := http.NewRequest("GET", "http://localhost/test-call", nil)
+	req, err := http.NewRequest("GET", "http://localhost/v2/notices?timeout=10s", nil)
 	c.Assert(err, check.IsNil)
 
+	gotReqC := make(chan struct{})
+	d.router.Use(func(h http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			c.Assert(r.URL.String(), check.Equals, "/v2/notices?timeout=10s")
+			close(gotReqC)
+			h.ServeHTTP(w, r)
+		})
+	})
+
 	c.Assert(d.Start(context.Background()), check.IsNil)
 
+	var resp map[string]any
 	clientC := make(chan struct{})
 	go func() {
 		// this will block until we call stop
@@ -1529,51 +1527,19 @@ func (s *daemonSuite) TestRequestContextCanceledOnStop(c *check.C) {
 			defer r.Body.Close()
 		}
 		c.Check(err, check.IsNil)
+		err = json.NewDecoder(r.Body).Decode(&resp)
+		c.Assert(err, check.IsNil)
 		close(clientC)
 	}()
 
 	<-gotReqC
 	d.Stop(nil)
-	reqErr := <-reqErrC
-	c.Check(errors.Is(reqErr, context.Canceled), check.Equals, true,
-		check.Commentf("unexpected error %v", reqErr))
 	<-clientC
-}
-
-func (s *daemonSuite) TestRequestContextPropagated(c *check.C) {
-	d, err := New()
-	c.Assert(err, check.IsNil)
-	// don't talk to the store, needs to be called after daemon.New()
-	snapstate.CanAutoRefresh = nil
-
-	// mark as already seeded
-	s.markSeeded(d)
-
-	c.Assert(d.Init(), check.IsNil)
-
-	type testKey struct{}
-
-	reqC := make(chan any, 1)
-	d.router.HandleFunc("/test-call", func(w http.ResponseWriter, r *http.Request) {
-		defer close(reqC)
-		reqC <- r.Context().Value(testKey{})
-	})
-
-	client := clientForSnapdSocket()
-
-	req, err := http.NewRequest("GET", "http://localhost/test-call", nil)
-	c.Assert(err, check.IsNil)
-
-	ctx := context.WithValue(context.Background(), testKey{}, "hello")
-	c.Assert(d.Start(ctx), check.IsNil)
-
-	r, err := client.Do(req)
-	if r != nil {
-		defer r.Body.Close()
-	}
-	c.Check(err, check.IsNil)
-
-	v := <-reqC
-	c.Assert(v, check.DeepEquals, "hello")
-	d.Stop(nil)
+	c.Check(resp, check.DeepEquals, map[string]interface{}{
+		"status-code": 500.,
+		"status":      "Internal Server Error",
+		"result": map[string]interface{}{
+			"message": "request canceled",
+		},
+		"type": "error"})
 }


Reviews:
user: login: olivercalder
body: This looks like a nice improvement, thanks. Should it potentially be backported to pebble as well?
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-04T13:48:13Z
user: login: bboozzoo
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-08T14:18:00Z
user: login: pedronis
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-10T07:27:35Z
user: login: pedronis
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-10T07:30:18Z
user: login: bboozzoo
body: LGTM
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-10T08:07:47Z
user: login: olivercalder
body: LGTM, thanks!
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-10T14:40:03Z

Review comments:
diff_hunk: @@ -1503,24 +1502,23 @@ func (s *daemonSuite) TestRequestContextCanceledOnStop(c *check.C) {
 
 	c.Assert(d.Init(), check.IsNil)
 
-	gotReqC := make(chan struct{})
-	reqErrC := make(chan error, 1)
-	d.router.HandleFunc("/test-call", func(w http.ResponseWriter, r *http.Request) {
-		close(gotReqC)
-		// since Stop() is called in the test, the request will get
-		// canceled
-		<-r.Context().Done()
-		reqErrC <- r.Context().Err()
-		w.WriteHeader(500)
-	})
-
 	client := clientForSnapdSocket()
 
-	req, err := http.NewRequest("GET", "http://localhost/test-call", nil)
+	req, err := http.NewRequest("GET", "http://localhost/v2/notices?timeout=10s", nil)
path: daemon/daemon_test.go
user: login: olivercalder
body: If the timeout is 10s in the request, should there be a check below that it takes less than 10s for clientC to be closed?
created_at: 2025-04-04T13:44:55Z
author_association: MEMBER
diff_hunk: @@ -1529,51 +1527,19 @@ func (s *daemonSuite) TestRequestContextCanceledOnStop(c *check.C) {
 			defer r.Body.Close()
 		}
 		c.Check(err, check.IsNil)
+		err = json.NewDecoder(r.Body).Decode(&resp)
+		c.Assert(err, check.IsNil)
 		close(clientC)
 	}()
 
 	<-gotReqC
 	d.Stop(nil)
-	reqErr := <-reqErrC
-	c.Check(errors.Is(reqErr, context.Canceled), check.Equals, true,
-		check.Commentf("unexpected error %v", reqErr))
 	<-clientC
path: daemon/daemon_test.go
user: login: olivercalder
body: Something like this:

```suggestion
	select {
	case <-clientC:
		// request successfully cancelled
	case <-time.NewTimer(time.Second).C:
		check.Error("request was not cancelled by the daemon stopping")
	}
```
created_at: 2025-04-04T13:47:14Z
author_association: MEMBER
diff_hunk: @@ -132,7 +132,7 @@ func getNotices(c *Command, r *http.Request, user *auth.UserState) Response {
 
 	if timeout != 0 {
 		// Wait up to timeout for notices matching given filter to occur
-		ctx, cancel := context.WithTimeout(r.Context(), timeout)
+		ctx, cancel := context.WithTimeout(c.d.tomb.Context(r.Context()), timeout)
path: daemon/api_notices.go
user: login: bboozzoo
body: I think I'd add a comment that we're using daemon's tomb context so that the request will get canceled when the tomb gets killed when shutting down the daemon
created_at: 2025-04-08T14:18:00Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1503,24 +1502,23 @@ func (s *daemonSuite) TestRequestContextCanceledOnStop(c *check.C) {
 
 	c.Assert(d.Init(), check.IsNil)
 
-	gotReqC := make(chan struct{})
-	reqErrC := make(chan error, 1)
-	d.router.HandleFunc("/test-call", func(w http.ResponseWriter, r *http.Request) {
-		close(gotReqC)
-		// since Stop() is called in the test, the request will get
-		// canceled
-		<-r.Context().Done()
-		reqErrC <- r.Context().Err()
-		w.WriteHeader(500)
-	})
-
 	client := clientForSnapdSocket()
 
-	req, err := http.NewRequest("GET", "http://localhost/test-call", nil)
+	req, err := http.NewRequest("GET", "http://localhost/v2/notices?timeout=10s", nil)
path: daemon/daemon_test.go
user: login: pedronis
body: the check for the error will fail if this succeed
created_at: 2025-04-10T07:27:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -132,7 +132,7 @@ func getNotices(c *Command, r *http.Request, user *auth.UserState) Response {
 
 	if timeout != 0 {
 		// Wait up to timeout for notices matching given filter to occur
-		ctx, cancel := context.WithTimeout(r.Context(), timeout)
+		ctx, cancel := context.WithTimeout(c.d.tomb.Context(r.Context()), timeout)
path: daemon/api_notices.go
user: login: pedronis
body: done
created_at: 2025-04-10T07:30:18Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1503,24 +1502,23 @@ func (s *daemonSuite) TestRequestContextCanceledOnStop(c *check.C) {
 
 	c.Assert(d.Init(), check.IsNil)
 
-	gotReqC := make(chan struct{})
-	reqErrC := make(chan error, 1)
-	d.router.HandleFunc("/test-call", func(w http.ResponseWriter, r *http.Request) {
-		close(gotReqC)
-		// since Stop() is called in the test, the request will get
-		// canceled
-		<-r.Context().Done()
-		reqErrC <- r.Context().Err()
-		w.WriteHeader(500)
-	})
-
 	client := clientForSnapdSocket()
 
-	req, err := http.NewRequest("GET", "http://localhost/test-call", nil)
+	req, err := http.NewRequest("GET", "http://localhost/v2/notices?timeout=10s", nil)
path: daemon/daemon_test.go
user: login: olivercalder
body: Ah I see, right, a timeout based on the timeout in the path will just return no notices, while a context cancellation will be a real error. So the test is good as-is.
created_at: 2025-04-10T14:39:52Z
author_association: MEMBER

PR comments:
user: login: github-actions[bot]
created_at: 2025-04-04T13:24:27Z
body: Tue Apr 15 15:33:57 UTC 2025
The following results are from: https://github.com/canonical/snapd/actions/runs/14462493810
## Failures:
### Preparing:
- openstack:opensuse-tumbleweed-64
- openstack:opensuse-tumbleweed-64
- openstack:opensuse-tumbleweed-64
- openstack:opensuse-tumbleweed-64
- openstack:opensuse-tumbleweed-64
- openstack:opensuse-15.6-64:tests/main/snap-run-inhibition-flow
### Executing:
- openstack:centos-9-64:tests/main/degraded
- openstack:opensuse-15.6-64:tests/main/auto-refresh-pre-download:close_mid_restart
### Restoring:
- openstack:opensuse-tumbleweed-64
- openstack:opensuse-tumbleweed-64
- openstack:opensuse-tumbleweed-64
- openstack:opensuse-tumbleweed-64
- openstack:opensuse-tumbleweed-64
- openstack:opensuse-15.6-64:tests/main/snap-run-inhibition-flow
- openstack:opensuse-15.6-64:tests/main/
- openstack:opensuse-15.6-64
author_association: NONE
user: login: olivercalder
created_at: 2025-04-10T18:17:06Z
body: I'm interested to see how this might interact with the locks around `Stop` in #15302
author_association: MEMBER
user: login: codecov[bot]
created_at: 2025-04-16T09:02:03Z
body: ## [Codecov](https://app.codecov.io/gh/canonical/snapd/pull/15291?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=canonical) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 0.00%. Comparing base [(`5e81c9d`)](https://app.codecov.io/gh/canonical/snapd/commit/5e81c9d8af4b52f0519b1a087b706dd530217497?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=canonical) to head [(`5dd5529`)](https://app.codecov.io/gh/canonical/snapd/commit/5dd55296a619a2732949eccdcd4f8e999c3708a6?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=canonical).
> Report is 111 commits behind head on master.

<details><summary>Additional details and impacted files</summary>


```diff
@@       Coverage Diff       @@
##   master   #15291   +/-   ##
===============================
===============================

```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/canonical/snapd/pull/15291?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=canonical).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=canonical).

<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>
author_association: NONE