Repository: google/go-containerregistry
Pull request #1088 (2021-07-26 15:04:51)

Add recipe to calculate image size
```
$ crane manifest gcr.io/buildpacks/builder:v1 | jq '.config.size + ([.layers[].size] | add)'
238034322
$ crane manifest gcr.io/buildpacks/builder:v1 | jq '.config.size + ([.layers[].size] | add)' | numfmt --to=iec
228M
```

jonjohnsonjr: This won't work for an index, how should we handle that?

Patch:
diff --git a/cmd/crane/recipes.md b/cmd/crane/recipes.md
index 6cc25c5b..c172d1b5 100644
--- a/cmd/crane/recipes.md
+++ b/cmd/crane/recipes.md
@@ -51,3 +51,19 @@ diff <(crane config busybox:1.32 | jq) <(crane config busybox:1.33 | jq)
 ```
 diff <(crane manifest busybox:1.32 | jq) <(crane manifest busybox:1.33 | jq)
 ```
+
+### Get total image size
+
+Given an image manifest, you can calculate the total size of all layer blobs and the image's config blob using `jq`:
+
+```
+crane manifest gcr.io/buildpacks/builder:v1 | jq '.config.size + ([.layers[].size] | add)'
+```
+
+This will produce a number of bytes, which you can make human-readable by passing to [`numfmt`](https://www.gnu.org/software/coreutils/manual/html_node/numfmt-invocation.html)
+
+```
+crane manifest gcr.io/buildpacks/builder:v1 | jq '.config.size + ([.layers[].size] | add)' | numfmt --to=iec
+```
+
+For image indexes, you can pass the `--platform` flag to `crane` to get a platform-specific image.
diff --git a/internal/verify/verify.go b/internal/verify/verify.go
index a26f9958..9d62214f 100644
--- a/internal/verify/verify.go
+++ b/internal/verify/verify.go
@@ -17,7 +17,9 @@
 package verify
 
 import (
+	"bytes"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"hash"
 	"io"
@@ -81,3 +83,25 @@ func ReadCloser(r io.ReadCloser, size int64, h v1.Hash) (io.ReadCloser, error) {
 		CloseFunc: r.Close,
 	}, nil
 }
+
+// Descriptor verifies that the embedded Data field matches the Size and Digest
+// fields of the given v1.Descriptor, returning an error if the Data field is
+// missing or if it contains incorrect data.
+func Descriptor(d v1.Descriptor) error {
+	if d.Data == nil {
+		return errors.New("error verifying descriptor; Data == nil")
+	}
+
+	h, sz, err := v1.SHA256(bytes.NewReader(d.Data))
+	if err != nil {
+		return err
+	}
+	if h != d.Digest {
+		return fmt.Errorf("error verifying Digest; got %q, want %q", h, d.Digest)
+	}
+	if sz != d.Size {
+		return fmt.Errorf("error verifying Size; got %d, want %d", sz, d.Size)
+	}
+
+	return nil
+}
diff --git a/pkg/v1/manifest.go b/pkg/v1/manifest.go
index 51a46704..8ded1ee5 100644
--- a/pkg/v1/manifest.go
+++ b/pkg/v1/manifest.go
@@ -43,6 +43,7 @@ type Descriptor struct {
 	MediaType   types.MediaType   `json:"mediaType"`
 	Size        int64             `json:"size"`
 	Digest      Hash              `json:"digest"`
+	Data        []byte            `json:"data,omitempty"`
 	URLs        []string          `json:"urls,omitempty"`
 	Annotations map[string]string `json:"annotations,omitempty"`
 	Platform    *Platform         `json:"platform,omitempty"`
diff --git a/pkg/v1/mutate/image.go b/pkg/v1/mutate/image.go
index bbdaf1c8..de284c36 100644
--- a/pkg/v1/mutate/image.go
+++ b/pkg/v1/mutate/image.go
@@ -130,6 +130,11 @@ func (i *image) compute() error {
 	manifest.Config.Digest = d
 	manifest.Config.Size = sz
 
+	// If Data was set in the base image, we need to update it in the mutated image.
+	if m.Config.Data != nil {
+		manifest.Config.Data = rcfg
+	}
+
 	// With OCI media types, this should not be set, see discussion:
 	// https://github.com/opencontainers/image-spec/pull/795
 	if i.mediaType != nil {
diff --git a/pkg/v1/mutate/index.go b/pkg/v1/mutate/index.go
index 767b1b74..e1f4937c 100644
--- a/pkg/v1/mutate/index.go
+++ b/pkg/v1/mutate/index.go
@@ -51,6 +51,9 @@ func computeDescriptor(ia IndexAddendum) (*v1.Descriptor, error) {
 	if len(ia.Descriptor.Annotations) != 0 {
 		desc.Annotations = ia.Descriptor.Annotations
 	}
+	if ia.Descriptor.Data != nil {
+		desc.Data = ia.Descriptor.Data
+	}
 
 	return desc, nil
 }
diff --git a/pkg/v1/remote/catalog.go b/pkg/v1/remote/catalog.go
index 21b5dbba..2f9ee11d 100644
--- a/pkg/v1/remote/catalog.go
+++ b/pkg/v1/remote/catalog.go
@@ -88,10 +88,12 @@ func Catalog(ctx context.Context, target name.Registry, options ...Option) ([]st
 	}
 
 	uri := &url.URL{
-		Scheme:   target.Scheme(),
-		Host:     target.RegistryStr(),
-		Path:     "/v2/_catalog",
-		RawQuery: "n=10000",
+		Scheme: target.Scheme(),
+		Host:   target.RegistryStr(),
+		Path:   "/v2/_catalog",
+		// ECR returns an error if n > 1000:
+		// https://github.com/google/go-containerregistry/issues/1091
+		RawQuery: "n=1000",
 	}
 
 	client := http.Client{Transport: tr}
diff --git a/pkg/v1/remote/image.go b/pkg/v1/remote/image.go
index dbffeeb8..a36416d8 100644
--- a/pkg/v1/remote/image.go
+++ b/pkg/v1/remote/image.go
@@ -15,6 +15,7 @@
 package remote
 
 import (
+	"bytes"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -100,6 +101,14 @@ func (r *remoteImage) RawConfigFile() ([]byte, error) {
 		return nil, err
 	}
 
+	if m.Config.Data != nil {
+		if err := verify.Descriptor(m.Config); err != nil {
+			return nil, err
+		}
+		r.config = m.Config.Data
+		return r.config, nil
+	}
+
 	body, err := r.fetchBlob(r.context, m.Config.Size, m.Config.Digest)
 	if err != nil {
 		return nil, err
@@ -143,6 +152,10 @@ func (rl *remoteImageLayer) Compressed() (io.ReadCloser, error) {
 		return nil, err
 	}
 
+	if d.Data != nil {
+		return verify.ReadCloser(ioutil.NopCloser(bytes.NewReader(d.Data)), d.Size, d.Digest)
+	}
+
 	// We don't want to log binary layers -- this can break terminals.
 	ctx := redact.NewContext(rl.ri.context, "omitting binary blobs from logs")
 
diff --git a/pkg/v1/remote/index.go b/pkg/v1/remote/index.go
index c1393435..7a542b38 100644
--- a/pkg/v1/remote/index.go
+++ b/pkg/v1/remote/index.go
@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/google/go-containerregistry/internal/verify"
 	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/partial"
@@ -198,9 +199,20 @@ func (r *remoteIndex) childByHash(h v1.Hash) (*Descriptor, error) {
 // Convert one of this index's child's v1.Descriptor into a remote.Descriptor, with the given platform option.
 func (r *remoteIndex) childDescriptor(child v1.Descriptor, platform v1.Platform) (*Descriptor, error) {
 	ref := r.Ref.Context().Digest(child.Digest.String())
-	manifest, _, err := r.fetchManifest(ref, []types.MediaType{child.MediaType})
-	if err != nil {
-		return nil, err
+	var (
+		manifest []byte
+		err      error
+	)
+	if child.Data != nil {
+		if err := verify.Descriptor(child); err != nil {
+			return nil, err
+		}
+		manifest = child.Data
+	} else {
+		manifest, _, err = r.fetchManifest(ref, []types.MediaType{child.MediaType})
+		if err != nil {
+			return nil, err
+		}
 	}
 	return &Descriptor{
 		fetcher: fetcher{
diff --git a/pkg/v1/remote/list.go b/pkg/v1/remote/list.go
index e4a005aa..433ccd1e 100644
--- a/pkg/v1/remote/list.go
+++ b/pkg/v1/remote/list.go
@@ -31,14 +31,16 @@ type tags struct {
 	Tags []string `json:"tags"`
 }
 
-// List wraps ListWithContext using the background context.
-func List(repo name.Repository, options ...Option) ([]string, error) {
-	return ListWithContext(context.Background(), repo, options...)
+// ListWithContext calls List with the given context.
+//
+// Deprecated: Use List and WithContext. This will be removed in a future release.
+func ListWithContext(ctx context.Context, repo name.Repository, options ...Option) ([]string, error) {
+	return List(repo, append(options, WithContext(ctx))...)
 }
 
-// ListWithContext calls /tags/list for the given repository, returning the list of tags
+// List calls /tags/list for the given repository, returning the list of tags
 // in the "tags" property.
-func ListWithContext(ctx context.Context, repo name.Repository, options ...Option) ([]string, error) {
+func List(repo name.Repository, options ...Option) ([]string, error) {
 	o, err := makeOptions(repo, options...)
 	if err != nil {
 		return nil, err
@@ -58,13 +60,6 @@ func ListWithContext(ctx context.Context, repo name.Repository, options ...Optio
 		RawQuery: "n=1000",
 	}
 
-	// This is lazy, but I want to make sure List(..., WithContext(ctx)) works
-	// without calling makeOptions() twice (which can have side effects).
-	// This means ListWithContext(ctx, ..., WithContext(ctx2)) prefers ctx2.
-	if o.context != context.Background() {
-		ctx = o.context
-	}
-
 	client := http.Client{Transport: tr}
 	tagList := []string{}
 	parsed := tags{}
@@ -72,16 +67,15 @@ func ListWithContext(ctx context.Context, repo name.Repository, options ...Optio
 	// get responses until there is no next page
 	for {
 		select {
-		case <-ctx.Done():
-			return nil, ctx.Err()
+		case <-o.context.Done():
+			return nil, o.context.Err()
 		default:
 		}
 
-		req, err := http.NewRequest("GET", uri.String(), nil)
+		req, err := http.NewRequestWithContext(o.context, "GET", uri.String(), nil)
 		if err != nil {
 			return nil, err
 		}
-		req = req.WithContext(ctx)
 
 		resp, err := client.Do(req)
 		if err != nil {
diff --git a/pkg/v1/remote/transport/ping.go b/pkg/v1/remote/transport/ping.go
index 396d4e03..c25d1b24 100644
--- a/pkg/v1/remote/transport/ping.go
+++ b/pkg/v1/remote/transport/ping.go
@@ -108,8 +108,8 @@ func ping(ctx context.Context, reg name.Registry, t http.RoundTripper) (*pingRes
 			}, nil
 		case http.StatusUnauthorized:
 			if challenges := authchallenge.ResponseChallenges(resp); len(challenges) != 0 {
-				// If we hit more than one, I'm not even sure what to do.
-				wac := challenges[0]
+				// If we hit more than one, let's try to find one that we know how to handle.
+				wac := pickFromMultipleChallenges(challenges)
 				return &pingResp{
 					challenge:  challenge(wac.Scheme).Canonical(),
 					parameters: wac.Parameters,
@@ -127,3 +127,22 @@ func ping(ctx context.Context, reg name.Registry, t http.RoundTripper) (*pingRes
 	}
 	return nil, errors.New(strings.Join(errs, "; "))
 }
+
+func pickFromMultipleChallenges(challenges []authchallenge.Challenge) authchallenge.Challenge {
+
+	// It might happen there are multiple www-authenticate headers, e.g. `Negotiate` and `Basic`.
+	// Picking simply the first one could result eventually in `unrecognized challenge` error,
+	// that's why we're looping through the challenges in search for one that can be handled.
+	allowedSchemes := []string{"basic", "bearer"}
+
+	for _, wac := range challenges {
+		currentScheme := strings.ToLower(wac.Scheme)
+		for _, allowed := range allowedSchemes {
+			if allowed == currentScheme {
+				return wac
+			}
+		}
+	}
+
+	return challenges[0]
+}
diff --git a/pkg/v1/zz_deepcopy_generated.go b/pkg/v1/zz_deepcopy_generated.go
index 3f92f091..b32b8b77 100644
--- a/pkg/v1/zz_deepcopy_generated.go
+++ b/pkg/v1/zz_deepcopy_generated.go
@@ -115,6 +115,11 @@ func (in *ConfigFile) DeepCopy() *ConfigFile {
 func (in *Descriptor) DeepCopyInto(out *Descriptor) {
 	*out = *in
 	out.Digest = in.Digest
+	if in.Data != nil {
+		in, out := &in.Data, &out.Data
+		*out = make([]byte, len(*in))
+		copy(*out, *in)
+	}
 	if in.URLs != nil {
 		in, out := &in.URLs, &out.URLs
 		*out = make([]string, len(*in))


Test patch:
diff --git a/internal/verify/verify_test.go b/internal/verify/verify_test.go
index f4087f83..9de625db 100644
--- a/internal/verify/verify_test.go
+++ b/internal/verify/verify_test.go
@@ -16,6 +16,7 @@ package verify
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"strings"
@@ -86,3 +87,47 @@ func TestBadSize(t *testing.T) {
 		})
 	}
 }
+
+func TestDescriptor(t *testing.T) {
+	for _, tc := range []struct {
+		err  error
+		desc v1.Descriptor
+	}{{
+		err: errors.New("error verifying descriptor; Data == nil"),
+	}, {
+		err: errors.New(`error verifying Digest; got "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", want ":"`),
+		desc: v1.Descriptor{
+			Data: []byte("abc"),
+		},
+	}, {
+		err: errors.New("error verifying Size; got 3, want 0"),
+		desc: v1.Descriptor{
+			Data: []byte("abc"),
+			Digest: v1.Hash{
+				Algorithm: "sha256",
+				Hex:       "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
+			},
+		},
+	}, {
+		desc: v1.Descriptor{
+			Data: []byte("abc"),
+			Size: 3,
+			Digest: v1.Hash{
+				Algorithm: "sha256",
+				Hex:       "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
+			},
+		},
+	}} {
+		got, want := Descriptor(tc.desc), tc.err
+
+		if got == nil {
+			if want != nil {
+				t.Errorf("Descriptor(): got nil, want %v", want)
+			}
+		} else if want == nil {
+			t.Errorf("Descriptor(): got %v, want nil", got)
+		} else if got, want := got.Error(), want.Error(); got != want {
+			t.Errorf("Descriptor(): got %q, want %q", got, want)
+		}
+	}
+}
diff --git a/pkg/v1/remote/image_test.go b/pkg/v1/remote/image_test.go
index 9e7de066..338eef4a 100644
--- a/pkg/v1/remote/image_test.go
+++ b/pkg/v1/remote/image_test.go
@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -403,6 +404,8 @@ func TestPullingManifestList(t *testing.T) {
 	manifest.Manifests[0].Platform = &fakePlatform
 	// Make sure the second manifest does.
 	manifest.Manifests[1].Platform = &defaultPlatform
+	// Do short-circuiting via Data.
+	manifest.Manifests[1].Data = mustRawManifest(t, child)
 	rawManifest, err := json.Marshal(manifest)
 	if err != nil {
 		t.Fatal(err)
@@ -674,3 +677,65 @@ func TestPullingForeignLayer(t *testing.T) {
 		t.Errorf("failed to Write: %v", err)
 	}
 }
+
+func TestData(t *testing.T) {
+	img := randomImage(t)
+	manifest, err := img.Manifest()
+	if err != nil {
+		t.Fatal(err)
+	}
+	layers, err := img.Layers()
+	if err != nil {
+		t.Fatal(err)
+	}
+	cb, err := img.RawConfigFile()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	manifest.Config.Data = cb
+	rc, err := layers[0].Compressed()
+	if err != nil {
+		t.Fatal(err)
+	}
+	lb, err := ioutil.ReadAll(rc)
+	if err != nil {
+		t.Fatal(err)
+	}
+	manifest.Layers[0].Data = lb
+	rawManifest, err := json.Marshal(manifest)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/v2/":
+			w.WriteHeader(http.StatusOK)
+		case "/v2/test/manifests/latest":
+			if r.Method != http.MethodGet {
+				t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet)
+			}
+			w.Write(rawManifest)
+		default:
+			// explode if we try to read blob or config
+			t.Fatalf("Unexpected path: %v", r.URL.Path)
+		}
+	}))
+	defer server.Close()
+	u, err := url.Parse(server.URL)
+	if err != nil {
+		t.Fatalf("url.Parse(%v) = %v", server.URL, err)
+	}
+	ref, err := newReference(u.Host, "test", "latest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	rmt, err := Image(ref)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := validate.Image(rmt); err != nil {
+		t.Fatal(err)
+	}
+}
diff --git a/pkg/v1/remote/list_test.go b/pkg/v1/remote/list_test.go
index 8f7dcac1..89700b83 100644
--- a/pkg/v1/remote/list_test.go
+++ b/pkg/v1/remote/list_test.go
@@ -20,6 +20,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"strings"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -110,8 +111,8 @@ func TestCancelledList(t *testing.T) {
 	}
 
 	_, err = ListWithContext(ctx, repo)
-	if want, got := context.Canceled, err; got != want {
-		t.Errorf("wanted %v got %v", want, got)
+	if err == nil || !strings.Contains(err.Error(), "context canceled") {
+		t.Errorf(`unexpected error; want "context canceled", got %v`, err)
 	}
 }
 
diff --git a/pkg/v1/remote/transport/ping_test.go b/pkg/v1/remote/transport/ping_test.go
index aa3a42f3..afddd011 100644
--- a/pkg/v1/remote/transport/ping_test.go
+++ b/pkg/v1/remote/transport/ping_test.go
@@ -147,6 +147,55 @@ func TestPingBearerChallengeWithParams(t *testing.T) {
 	}
 }
 
+func TestPingMultipleChallenges(t *testing.T) {
+	server := httptest.NewServer(
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Add("WWW-Authenticate", "Negotiate")
+			w.Header().Add("WWW-Authenticate", `Basic realm="http://auth.example.com/token"`)
+			http.Error(w, "Unauthorized", http.StatusUnauthorized)
+		}))
+	defer server.Close()
+	tprt := &http.Transport{
+		Proxy: func(req *http.Request) (*url.URL, error) {
+			return url.Parse(server.URL)
+		},
+	}
+
+	pr, err := ping(context.Background(), testRegistry, tprt)
+	if err != nil {
+		t.Errorf("ping() = %v", err)
+	}
+	if pr.challenge != basic {
+		t.Errorf("ping(); got %v, want %v", pr.challenge, basic)
+	}
+	if got, want := len(pr.parameters), 1; got != want {
+		t.Errorf("ping(); got %v, want %v", got, want)
+	}
+}
+
+func TestPingMultipleNotSupportedChallenges(t *testing.T) {
+	server := httptest.NewServer(
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Add("WWW-Authenticate", "Negotiate")
+			w.Header().Add("WWW-Authenticate", "Digest")
+			http.Error(w, "Unauthorized", http.StatusUnauthorized)
+		}))
+	defer server.Close()
+	tprt := &http.Transport{
+		Proxy: func(req *http.Request) (*url.URL, error) {
+			return url.Parse(server.URL)
+		},
+	}
+
+	pr, err := ping(context.Background(), testRegistry, tprt)
+	if err != nil {
+		t.Errorf("ping() = %v", err)
+	}
+	if pr.challenge != "negotiate" {
+		t.Errorf("ping(); got %v, want %v", pr.challenge, "negotiate")
+	}
+}
+
 func TestUnsupportedStatus(t *testing.T) {
 	server := httptest.NewServer(
 		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {