Repository: moby/moby
Pull request #49770: Add registry error handling for push and pull
Author: login: dmcgowan
Labels: name: area/distribution
description: Image Distribution
name: status/2-code-review
name: kind/enhancement
description: Enhancements are not bugs or new features but can improve usability or performance.
name: impact/changelog
name: containerd-integration
description: Issues and PRs related to containerd integration
Created: 2025-04-08T06:39:01Z | Merged: 2025-04-10T19:32:57Z

Adds handling of the unexpected status errors to look for error codes and messages provided by the registry. Currently nothing from the body is returned, only the HTTP status code.

```markdown changelog
containerd image store: Improve `docker push/pull` handling of remote registry errors
```


Diff:
diff --git a/daemon/containerd/image_pull.go b/daemon/containerd/image_pull.go
index 8947d3489a737..6f4113d1caa08 100644
--- a/daemon/containerd/image_pull.go
+++ b/daemon/containerd/image_pull.go
@@ -228,7 +228,7 @@ func (i *ImageService) pullTag(ctx context.Context, ref reference.Named, platfor
 				return errdefs.NotFound(fmt.Errorf("no matching manifest for %s in the manifest list entries: %w", platformStr, err))
 			}
 		}
-		return err
+		return translateRegistryError(ctx, err)
 	}
 
 	logger := log.G(ctx).WithFields(log.Fields{
diff --git a/daemon/containerd/image_push.go b/daemon/containerd/image_push.go
index a4d73c22e0540..d54113218a8b3 100644
--- a/daemon/containerd/image_push.go
+++ b/daemon/containerd/image_push.go
@@ -191,7 +191,7 @@ func (i *ImageService) pushRef(ctx context.Context, targetRef reference.Named, p
 
 		if err != nil {
 			if !cerrdefs.IsNotFound(err) {
-				return errdefs.System(err)
+				return translateRegistryError(ctx, err)
 			}
 			progress.Aux(out, auxprogress.ContentMissing{
 				ContentMissing: true,
diff --git a/daemon/containerd/registry_errors.go b/daemon/containerd/registry_errors.go
new file mode 100644
index 0000000000000..a961d9ca693d6
--- /dev/null
+++ b/daemon/containerd/registry_errors.go
@@ -0,0 +1,78 @@
+package containerd
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/containerd/containerd/v2/core/remotes/docker"
+	remoteerrors "github.com/containerd/containerd/v2/core/remotes/errors"
+	cerrdefs "github.com/containerd/errdefs"
+	"github.com/containerd/log"
+)
+
+func translateRegistryError(ctx context.Context, err error) error {
+	// Check for registry specific error
+	var derrs docker.Errors
+	if !errors.As(err, &derrs) {
+		var remoteErr remoteerrors.ErrUnexpectedStatus
+		if errors.As(err, &remoteErr) {
+			if jerr := json.Unmarshal(remoteErr.Body, &derrs); jerr != nil {
+				log.G(ctx).WithError(derrs).Debug("unable to unmarshal registry error")
+				return fmt.Errorf("%w: %w", cerrdefs.ErrUnknown, err)
+			}
+		} else {
+			var derr docker.Error
+			if errors.As(err, &derr) {
+				derrs = append(derrs, derr)
+			} else {
+				return err
+			}
+		}
+	}
+	var errs []error
+	for _, err := range derrs {
+		var derr docker.Error
+		if errors.As(err, &derr) {
+			var message string
+
+			if derr.Message != "" {
+				message = derr.Message
+			} else {
+				message = derr.Code.Message()
+			}
+
+			if detail, ok := derr.Detail.(string); ok {
+				message = fmt.Sprintf("%s - %s", message, detail)
+			}
+
+			switch derr.Code {
+			case docker.ErrorCodeUnsupported:
+				err = cerrdefs.ErrNotImplemented.WithMessage(message)
+			case docker.ErrorCodeUnauthorized:
+				err = cerrdefs.ErrUnauthenticated.WithMessage(message)
+			case docker.ErrorCodeDenied:
+				err = cerrdefs.ErrPermissionDenied.WithMessage(message)
+			case docker.ErrorCodeUnavailable:
+				err = cerrdefs.ErrUnavailable.WithMessage(message)
+			case docker.ErrorCodeTooManyRequests:
+				err = cerrdefs.ErrResourceExhausted.WithMessage(message)
+			default:
+				err = cerrdefs.ErrUnknown.WithMessage(message)
+			}
+		} else {
+			errs = append(errs, cerrdefs.ErrUnknown.WithMessage(err.Error()))
+		}
+		errs = append(errs, err)
+	}
+	switch len(errs) {
+	case 0:
+		err = cerrdefs.ErrUnknown.WithMessage(err.Error())
+	case 1:
+		err = errs[0]
+	default:
+		err = errors.Join(errs...)
+	}
+	return fmt.Errorf("error from registry: %w", err)
+}
diff --git a/integration-cli/docker_cli_push_test.go b/integration-cli/docker_cli_push_test.go
index 77933a57e7a89..9d2dc6e60d6f9 100644
--- a/integration-cli/docker_cli_push_test.go
+++ b/integration-cli/docker_cli_push_test.go
@@ -255,7 +255,7 @@ func getTestTokenService(status int, body string, retries int) *httptest.Server
 }
 
 func (s *DockerRegistryAuthTokenSuite) TestPushTokenServiceUnauthResponse(c *testing.T) {
-	ts := getTestTokenService(http.StatusUnauthorized, `{"errors": [{"Code":"UNAUTHORIZED", "message": "a message", "detail": null}]}`, 0)
+	ts := getTestTokenService(http.StatusUnauthorized, `{"errors": [{"Code":"UNAUTHORIZED", "message": "a message about not being authorized", "detail": null}]}`, 0)
 	defer ts.Close()
 	s.setupRegistryWithTokenService(c, ts.URL)
 
@@ -268,10 +268,9 @@ func (s *DockerRegistryAuthTokenSuite) TestPushTokenServiceUnauthResponse(c *tes
 
 	// Auth service errors are not part of the spec and containerd doesn't parse them.
 	if testEnv.UsingSnapshotter() {
-		assert.Check(c, is.Contains(out, "failed to authorize: failed to fetch anonymous token"))
-		assert.Check(c, is.Contains(out, "401 Unauthorized"))
+		assert.Check(c, is.Contains(out, "a message about not being authorized"))
 	} else {
-		assert.Check(c, is.Contains(out, "unauthorized: a message"))
+		assert.Check(c, is.Contains(out, "unauthorized: a message about not being authorized"))
 	}
 }
 
@@ -297,7 +296,7 @@ func (s *DockerRegistryAuthTokenSuite) TestPushMisconfiguredTokenServiceResponse
 }
 
 func (s *DockerRegistryAuthTokenSuite) TestPushMisconfiguredTokenServiceResponseError(c *testing.T) {
-	ts := getTestTokenService(http.StatusTooManyRequests, `{"errors": [{"code":"TOOMANYREQUESTS","message":"out of tokens"}]}`, 3)
+	ts := getTestTokenService(http.StatusTooManyRequests, `{"errors": [{"code":"TOOMANYREQUESTS","message":"out of tokens"}]}`, 0)
 	defer ts.Close()
 	s.setupRegistryWithTokenService(c, ts.URL)
 
@@ -311,8 +310,7 @@ func (s *DockerRegistryAuthTokenSuite) TestPushMisconfiguredTokenServiceResponse
 
 	// Auth service errors are not part of the spec and containerd doesn't parse them.
 	if testEnv.UsingSnapshotter() {
-		assert.Check(c, is.Contains(out, "failed to authorize: failed to fetch anonymous token"))
-		assert.Check(c, is.Contains(out, "503 Service Unavailable"))
+		assert.Check(c, is.Contains(out, "out of tokens"))
 	} else {
 		split := strings.Split(out, "\n")
 		assert.Check(c, is.Equal(split[len(split)-2], "toomanyrequests: out of tokens"))


Reviews:
user: login: vvoland
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-10T15:21:35Z
user: login: thaJeztah
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-10T16:27:27Z
user: login: dmcgowan
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-10T17:06:40Z
user: login: thaJeztah
body: LGTM
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-10T19:32:19Z

Review comments:
diff_hunk: @@ -0,0 +1,78 @@
+package containerd
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/containerd/containerd/v2/core/remotes/docker"
+	remoteerrors "github.com/containerd/containerd/v2/core/remotes/errors"
+	cerrdefs "github.com/containerd/errdefs"
+	"github.com/containerd/log"
+)
+
+func translateRegistryError(ctx context.Context, err error) error {
+	// Check for registry specific error
+	var derrs docker.Errors
+	if !errors.As(err, &derrs) {
+		var remoteErr remoteerrors.ErrUnexpectedStatus
+		if errors.As(err, &remoteErr) {
+			if jerr := json.Unmarshal(remoteErr.Body, &derrs); jerr != nil {
+				log.G(ctx).WithError(derrs).Debug("unable to unmarshal registry error")
+				return fmt.Errorf("%w: %w", cerrdefs.ErrUnknown, err)
+			}
+		} else {
+			var derr docker.Error
+			if errors.As(err, &derr) {
+				derrs = append(derrs, derr)
+			} else {
+				return err
+			}
+		}
+	}
+	var errs []error
+	for _, err := range derrs {
+		var derr docker.Error
+		if errors.As(err, &derr) {
+			var message string
+
+			if derr.Message != "" {
+				message = derr.Message
+			} else {
+				message = derr.Code.Message()
+			}
+
+			if detail, ok := derr.Detail.(string); ok {
+				message = fmt.Sprintf("%s - %s", message, detail)
+			}
+
+			switch derr.Code {
+			case docker.ErrorCodeUnsupported:
+				err = cerrdefs.ErrNotImplemented.WithMessage(message)
+			case docker.ErrorCodeUnauthorized:
+				err = cerrdefs.ErrUnauthenticated.WithMessage(message)
+			case docker.ErrorCodeDenied:
+				err = cerrdefs.ErrPermissionDenied.WithMessage(message)
+			case docker.ErrorCodeUnavailable:
+				err = cerrdefs.ErrUnavailable.WithMessage(message)
+			case docker.ErrorCodeTooManyRequests:
+				err = cerrdefs.ErrResourceExhausted.WithMessage(message)
+			default:
+				err = cerrdefs.ErrUnknown.WithMessage(message)
+			}
+		} else {
+			errs = append(errs, cerrdefs.ErrUnknown.WithMessage(err.Error()))
+		}
+		errs = append(errs, err)
+	}
+	switch len(errs) {
+	case 0:
+		err = cerrdefs.ErrUnknown.WithMessage(err.Error())
+	case 1:
+		err = errs[0]
+	default:
+		err = errors.Join(errs...)
+	}
+	return fmt.Errorf("error from registry: %w", err)
path: daemon/containerd/registry_errors.go
user: login: thaJeztah
body: I was curious if we have a test that verifies we return the correct HTTP status. Although not sure if these result in a status on the request as they may be after the initial response?
created_at: 2025-04-10T16:27:27Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,78 @@
+package containerd
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/containerd/containerd/v2/core/remotes/docker"
+	remoteerrors "github.com/containerd/containerd/v2/core/remotes/errors"
+	cerrdefs "github.com/containerd/errdefs"
+	"github.com/containerd/log"
+)
+
+func translateRegistryError(ctx context.Context, err error) error {
+	// Check for registry specific error
+	var derrs docker.Errors
+	if !errors.As(err, &derrs) {
+		var remoteErr remoteerrors.ErrUnexpectedStatus
+		if errors.As(err, &remoteErr) {
+			if jerr := json.Unmarshal(remoteErr.Body, &derrs); jerr != nil {
+				log.G(ctx).WithError(derrs).Debug("unable to unmarshal registry error")
+				return fmt.Errorf("%w: %w", cerrdefs.ErrUnknown, err)
+			}
+		} else {
+			var derr docker.Error
+			if errors.As(err, &derr) {
+				derrs = append(derrs, derr)
+			} else {
+				return err
+			}
+		}
+	}
+	var errs []error
+	for _, err := range derrs {
+		var derr docker.Error
+		if errors.As(err, &derr) {
+			var message string
+
+			if derr.Message != "" {
+				message = derr.Message
+			} else {
+				message = derr.Code.Message()
+			}
+
+			if detail, ok := derr.Detail.(string); ok {
+				message = fmt.Sprintf("%s - %s", message, detail)
+			}
+
+			switch derr.Code {
+			case docker.ErrorCodeUnsupported:
+				err = cerrdefs.ErrNotImplemented.WithMessage(message)
+			case docker.ErrorCodeUnauthorized:
+				err = cerrdefs.ErrUnauthenticated.WithMessage(message)
+			case docker.ErrorCodeDenied:
+				err = cerrdefs.ErrPermissionDenied.WithMessage(message)
+			case docker.ErrorCodeUnavailable:
+				err = cerrdefs.ErrUnavailable.WithMessage(message)
+			case docker.ErrorCodeTooManyRequests:
+				err = cerrdefs.ErrResourceExhausted.WithMessage(message)
+			default:
+				err = cerrdefs.ErrUnknown.WithMessage(message)
+			}
+		} else {
+			errs = append(errs, cerrdefs.ErrUnknown.WithMessage(err.Error()))
+		}
+		errs = append(errs, err)
+	}
+	switch len(errs) {
+	case 0:
+		err = cerrdefs.ErrUnknown.WithMessage(err.Error())
+	case 1:
+		err = errs[0]
+	default:
+		err = errors.Join(errs...)
+	}
+	return fmt.Errorf("error from registry: %w", err)
path: daemon/containerd/registry_errors.go
user: login: dmcgowan
body: Correct status from the Docker API? I only saw tests checking the user output
created_at: 2025-04-10T17:06:40Z
author_association: MEMBER

PR comments:
user: login: thaJeztah
created_at: 2025-04-10T16:29:26Z
body: Not sure if it's the correct way to test, but I was looking to test against docker hub 😅 


With curl;

Without containerd;

```bash
curl -XPOST --unix-socket /var/run/docker.sock 'http://localhost/v1.48/images/create?fromImage=thajeztah%2Fprivate-test-image&tag=latest'
{"message":"pull access denied for thajeztah/private-test-image, repository does not exist or may require 'docker login': denied: requested access to the resource is denied"}
```

With containerd;


```bash
curl -XPOST --unix-socket /var/run/docker.sock 'http://localhost/v1.48/images/create?fromImage=thajeztah%2Fprivate-test-image&tag=latest'
{"message":"pull access denied for thajeztah/private-test-image, repository does not exist or may require 'docker login'"}
```


docker 28.0.4 without containerd;

```bash
docker pull thajeztah/private-test-image
Using default tag: latest
Error response from daemon: pull access denied for thajeztah/private-test-image, repository does not exist or may require 'docker login': denied: requested access to the resource is denied

docker pull busybox
docker tag thajeztah/private-test-image
docker push thajeztah/private-test-image
Using default tag: latest
The push refers to repository [docker.io/thajeztah/private-test-image]
265dab34035e: Unavailable
server message: insufficient_scope: authorization failed
```

docker 28.0.4 with containerd;


```bash
docker pull thajeztah/private-test-image
Using default tag: latest
Error response from daemon: pull access denied for thajeztah/private-test-image, repository does not exist or may require 'docker login'

docker tag busybox thajeztah/private-test-image
docker push thajeztah/private-test-image
Using default tag: latest
The push refers to repository [docker.io/thajeztah/private-test-image]
be632cf9bbb6: Preparing
denied: requested access to the resource is denied
```

this PR with containerd;

```bash
docker pull thajeztah/private-test-image
Using default tag: latest
Error response from daemon: pull access denied for thajeztah/private-test-image, repository does not exist or may require 'docker login'

docker tag busybox thajeztah/private-test-image
docker push thajeztah/private-test-image
Using default tag: latest
The push refers to repository [docker.io/thajeztah/private-test-image]
265dab34035e: Unavailable
server message: insufficient_scope: authorization failed
```

author_association: MEMBER
user: login: dmcgowan
created_at: 2025-04-10T18:29:36Z
body: > Not sure if it's the correct way to test, but I was looking to test against docker hub 😅

I had to manually test as well, although there were a couple tests that handle the response code but the retry behavior with containerd is different today.

This should be good to get in, the tests you ran look like that is expected
author_association: MEMBER