Change:
<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: What happened: We have a private registry configured (where all images including the sandbox one), are hosted. The registry credentials are being pulled by one of k8s credentials providers () With Kubernetes 1.7 sandbox image failed to be pulled with the following error message: Looks like with this only creds from docker config are being evaluated on the sandbox image pull What you expected to happen: Sandbox image pull should be attempted using credentials returned by not just the basic docker credential provider, but other credentials providers registered in the system. How to reproduce it (as minimally and precisely as possible): 1) Register private repo credentials with one of the k8s credential providers (we've tried with rancher). Configure the to use the image from the the private repo. 2) Make sure you are not logged in to this private repo on the kubelet host 3) Try to create a pod. Sandbox image pull will fail with an error mentioned above. Anything else we need to know?: Non-sandbox images from the same private repo were able to download correctly using private repo credentials provider. Environment: Kubernetes version (use ): 1.7.2 Cloud provider or hardware configuration**: OS (e.g. from /etc/os-release): Ubuntu 16.04 Kernel (e.g. ): Install tools: Others:
There are no sig labels on this issue. Please by: a sig: e.g., to notify the contributor experience sig, OR the label manually: e.g., to apply the label Note: Method 1 will trigger an email to the group. You can find the group list and label list . The in the method 1 has to be replaced with one of these: bugs, feature-requests, pr-reviews, test-failures, proposals
/sig node Just tested the same use case against k8s 1.5.x, it worked fine there and sandbox image was pulled using the credentials passed by the credential provider, not from docker config file. So it might have regressed in 1.7.x (haven't tested it on 1.6, so can't tell for sure)
Confirmed a bug introduced in dockershim. Will file a PR to fix this.
Could you also verify whether fixed your problem?
tested against your commits cherry-picked on top of 1.8 beta, the fix seems to work
thanks.
[MILESTONENOTIFIER] Milestone Labels Complete Issue label settings: sig/node: Issue will be escalated to these SIGs if needed. priority/important-soon: Escalate to the issue owners and SIG owner; move out of milestone after several unsuccessful escalation attempts. kind/bug: Fixes a bug discovered during the current release. <detailsAdditional instructions available <a href=""here</aThe commands available for adding these labels are documented <a href=""here</a</details

Discussion:
"//pkg/kubelet/util/ioutils:go_default_library", "//pkg/security/apparmor:go_default_library", "//pkg/util/hash:go_default_library", <ins> "//pkg/util/parsers:go_default_library", </ins> "//pkg/util/term:go_default_library", "//vendor/github.com/blang/semver:go_default_library", "//vendor/github.com/docker/docker/api/types:go_default_library",

import ( "fmt" <del> "os" "path/filepath" </del> "regexp" "strconv" "strings"

dockernat "github.com/docker/go-connections/nat" "github.com/golang/glog" <ins> utilerrors "k8s.io/apimachinery/pkg/util/errors" </ins> "k8s.io/kubernetes/pkg/credentialprovider" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime" <ins> "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" </ins> "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/security/apparmor" <del> "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" </del> <ins> "k8s.io/kubernetes/pkg/util/parsers" </ins> ) const (

// ensureSandboxImageExists pulls the sandbox image when it's not present. func ensureSandboxImageExists(client libdocker.Interface, image string) error { <del> dockerCfgSearchPath := []string{"/.docker", filepath.Join(os.Getenv("HOME"), ".docker")} return ensureSandboxImageExistsDockerCfg(client, image, dockerCfgSearchPath) } func ensureSandboxImageExistsDockerCfg(client libdocker.Interface, image string, dockerCfgSearchPath []string) error { </del> _, err := client.InspectImageByRef(image) if err == nil { return nil

return fmt.Errorf("failed to inspect sandbox image %q: %v", image, err) } <del> // To support images in private registries, try to read docker config authConfig := dockertypes.AuthConfig{} keyring := &credentialprovider.BasicDockerKeyring{} var cfgLoadErr error if cfg, err := credentialprovider.ReadDockerConfigJSONFile(dockerCfgSearchPath); err == nil { keyring.Add(cfg) } else if cfg, err := credentialprovider.ReadDockercfgFile(dockerCfgSearchPath); err == nil { keyring.Add(cfg) } else { cfgLoadErr = err </del> <ins> repoToPull, _, _, err := parsers.ParseImageName(image) if err != nil { return err </ins> } <del> if creds, withCredentials := keyring.Lookup(image); withCredentials { // Use the first one that matched our image for _, cred := range creds { authConfig.Username = cred.Username authConfig.Password = cred.Password break </del> <ins> keyring := credentialprovider.NewDockerKeyring() creds, withCredentials := keyring.Lookup(repoToPull) if !withCredentials { glog.V(3).Infof("Pulling image %q without credentials", image) err := client.PullImage(image, dockertypes.AuthConfig{}, dockertypes.ImagePullOptions{}) if err != nil { return fmt.Errorf("failed pulling image %q: %v", image, err) </ins> } <ins> return nil </ins> } <del> err = client.PullImage(image, authConfig, dockertypes.ImagePullOptions{}) if err != nil { if cfgLoadErr != nil { glog.Warningf("Couldn't load Docker cofig. If sandbox image %q is in a private registry, this will cause further errors. Error: %v", image, cfgLoadErr) </del> <ins> var pullErrs []error for _, currentCreds := range creds { authConfig := credentialprovider.LazyProvide(currentCreds) err := client.PullImage(image, authConfig, dockertypes.ImagePullOptions{}) // If there was no error, return success if err == nil { return nil </ins> } <del> return fmt.Errorf("unable to pull sandbox image %q: %v", image, err) </del> <ins> pullErrs = append(pullErrs, err) </ins> } <del> return nil </del> <ins> return utilerrors.NewAggregate(pullErrs) </ins> } func getAppArmorOpts(profile string) ([]dockerOpt, error) {

package dockershim import ( <del> "encoding/base64" </del> "fmt" "io/ioutil" "os"

"github.com/stretchr/testify/require" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime" <del> "k8s.io/kubernetes/pkg/security/apparmor" </del> "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" <ins> "k8s.io/kubernetes/pkg/security/apparmor" </ins> ) func TestLabelsAndAnnotationsRoundTrip(t *testing.T) {

func TestEnsureSandboxImageExists(t *testing.T) { sandboxImage := "gcr.io/test/image" <del> registryHost := "https://gcr.io/" </del> authConfig := dockertypes.AuthConfig{Username: "user", Password: "pass"} <del> authB64 := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", authConfig.Username, authConfig.Password))) authJSON := fmt.Sprintf("{"auths": {"%s": {"auth": "%s"} } }", registryHost, authB64) </del> for desc, test := range map[string]struct { injectImage  bool imgNeedsAuth bool

calls:        []string{"inspect_image", "pull"}, err:          true, }, <del> "should pull private image using dockerauth if image doesn't exist": { injectImage:  true, imgNeedsAuth: true, injectErr:    libdocker.ImageNotFoundError{ID: "image_id"}, calls:        []string{"inspect_image", "pull"}, configJSON:   authJSON, err:          false, }, </del> } { t.Logf("TestCase: %q", desc) _, fakeDocker, _ := newTestDockerService()

} fakeDocker.InjectError("inspect_image", test.injectErr) <del> var dockerCfgSearchPath []string if test.configJSON != "" { tmpdir, err := writeDockerConfig(test.configJSON) require.NoError(t, err, "could not create a temp docker config file") dockerCfgSearchPath = append(dockerCfgSearchPath, filepath.Join(tmpdir, ".docker")) defer os.RemoveAll(tmpdir) } err := ensureSandboxImageExistsDockerCfg(fakeDocker, sandboxImage, dockerCfgSearchPath) </del> <ins> err := ensureSandboxImageExists(fakeDocker, sandboxImage) </ins> assert.NoError(t, fakeDocker.AssertCalls(test.calls)) assert.Equal(t, test.err, err != nil) }