Change:
This bug affects any pod that has a container with a nested volume (any type) beneath an API data volume, such as a secret, configMap, downwardAPI or projected volume. In the description and reproduction details below I use a secret volume, but any of the four will demonstrate this behavior. Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: When creating a pod with nested secrets volumes (e.g. one volume mounted at and one mounted at ), the container fails to start with an error like: What you expected to happen: The container should start with both and mounted. How to reproduce it (as minimally and precisely as possible): out master, any version since a file with 2 secrets and a pod that makes volumes for the 2 secrets, with one of them mounted within the other: the 2 secrets and the pod. Assuming you named the above file : Anything else we need to know?: This may work for other runtimes if they create mountpoints outside of the sandbox. But it's definitely broken with Docker as the runtime.
This should be fixed by /sig storage /sig node /assign

Discussion:
if err := wrapped.SetUpAt(dir, fsGroup); err != nil { return err } <ins> if err := volumeutil.MakeNestedMountpoints(b.volName, dir, b.pod); err != nil { return err } </ins> optional := b.source.Optional != nil && *b.source.Optional configMap, err := b.getConfigMap(b.pod.Namespace, b.source.Name)

glog.Errorf("Unable to setup downwardAPI volume %v for pod %v/%v: %s", b.volName, b.pod.Namespace, b.pod.Name, err.Error()) return err } <ins> if err := volumeutil.MakeNestedMountpoints(b.volName, dir, *b.pod); err != nil { return err } </ins> data, err := CollectData(b.source.Items, b.pod, b.plugin.host, b.source.DefaultMode) if err != nil {

if err := wrapped.SetUpAt(dir, fsGroup); err != nil { return err } <ins> if err := volumeutil.MakeNestedMountpoints(s.volName, dir, *s.pod); err != nil { return err } </ins> data, err := s.collectData() if err != nil {

if err := wrapped.SetUpAt(dir, fsGroup); err != nil { return err } <ins> if err := volumeutil.MakeNestedMountpoints(b.volName, dir, b.pod); err != nil { return err } </ins> optional := b.source.Optional != nil && *b.source.Optional secret, err := b.getSecret(b.pod.Namespace, b.source.SecretName)

"finalizer.go", "io_util.go", "metrics.go", <ins> "nested_volumes.go", </ins> "resize_util.go", "util.go", ] + select({

go_test( name = "go_default_test", srcs = [ <ins> "nested_volumes_test.go", </ins> "resize_util_test.go", "util_test.go", ] + select({

"//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", <ins> "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", </ins> "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/client-go/util/testing:go_default_library", ],

<ins> /* Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package util import ( "fmt" "k8s.io/api/core/v1" "os" "path" "path/filepath" "sort" "strings" ) // getNestedMountpoints returns a list of mountpoint directories that should be created // for the volume indicated by name. // note: the returned list is relative to baseDir func getNestedMountpoints(name, baseDir string, pod v1.Pod) ([]string, error) { var retval []string checkContainer := func(container *v1.Container) error { var allMountPoints []string // all mount points in this container var myMountPoints []string  // mount points that match name for _, vol := range container.VolumeMounts { cleaned := filepath.Clean(vol.MountPath) allMountPoints = append(allMountPoints, cleaned) if vol.Name == name { myMountPoints = append(myMountPoints, cleaned) } } sort.Strings(allMountPoints) parentPrefix := ".." + string(os.PathSeparator) // Examine each place where this volume is mounted for _, myMountPoint := range myMountPoints { if strings.HasPrefix(myMountPoint, parentPrefix) { // Don't let a container trick us into creating directories outside of its rootfs return fmt.Errorf("Invalid container mount point %v", myMountPoint) } myMPSlash := myMountPoint + string(os.PathSeparator) // The previously found nested mountpoint (or "" if none found yet) prevNestedMP := "" // examine each mount point to see if it's nested beneath this volume // (but skip any that are double-nested beneath this volume) // For example, if this volume is mounted as /dir and other volumes are mounted //              as /dir/nested and /dir/nested/other, only create /dir/nested. for _, mp := range allMountPoints { if !strings.HasPrefix(mp, myMPSlash) { continue // skip -- not nested beneath myMountPoint } if prevNestedMP != "" && strings.HasPrefix(mp, prevNestedMP) { continue // skip -- double nested beneath myMountPoint } // since this mount point is nested, remember it so that we can check that following ones aren't nested beneath this one prevNestedMP = mp + string(os.PathSeparator) retval = append(retval, mp[len(myMPSlash):]) } } return nil } for _, container := range pod.Spec.InitContainers { if err := checkContainer(&container); err != nil { return nil, err } } for _, container := range pod.Spec.Containers { if err := checkContainer(&container); err != nil { return nil, err } } return retval, nil } // MakeNestedMountpoints creates mount points in baseDir for volumes mounted beneath name func MakeNestedMountpoints(name, baseDir string, pod v1.Pod) error { dirs, err := getNestedMountpoints(name, baseDir, pod) if err != nil { return err } for _, dir := range dirs { err := os.MkdirAll(path.Join(baseDir, dir), 0755) if err != nil { return fmt.Errorf("Unable to create nested volume mountpoints: %v", err) } } return nil } </ins>

<ins> /* Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package util import ( "io/ioutil" "os" "path" "testing" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" ) type testCases struct { name     string err      bool expected sets.String volname  string pod      v1.Pod } func TestGetNestedMountpoints(t *testing.T) { var ( testNamespace = "test_namespace" testPodUID    = types.UID("test_pod_uid") ) tc := []testCases{ { name:     "Simple Pod", err:      false, expected: sets.NewString(), volname:  "vol1", pod: v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: testNamespace, UID:       testPodUID, }, Spec: v1.PodSpec{ Containers: []v1.Container{ { VolumeMounts: []v1.VolumeMount{ {MountPath: "/dir", Name: "vol1"}, }, }, }, }, }, }, { name:     "Simple Nested Pod", err:      false, expected: sets.NewString("nested"), volname:  "vol1", pod: v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: testNamespace, UID:       testPodUID, }, Spec: v1.PodSpec{ Containers: []v1.Container{ { VolumeMounts: []v1.VolumeMount{ {MountPath: "/dir", Name: "vol1"}, {MountPath: "/dir/nested", Name: "vol2"}, }, }, }, }, }, }, { name:     "Unsorted Nested Pod", err:      false, expected: sets.NewString("nested", "nested2"), volname:  "vol1", pod: v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: testNamespace, UID:       testPodUID, }, Spec: v1.PodSpec{ Containers: []v1.Container{ { VolumeMounts: []v1.VolumeMount{ {MountPath: "/dir/nested/double", Name: "vol3"}, {MountPath: "/ignore", Name: "vol4"}, {MountPath: "/dir/nested", Name: "vol2"}, {MountPath: "/ignore2", Name: "vol5"}, {MountPath: "/dir", Name: "vol1"}, {MountPath: "/dir/nested2", Name: "vol3"}, }, }, }, }, }, }, { name:     "Multiple vol1 mounts Pod", err:      false, expected: sets.NewString("nested", "nested2"), volname:  "vol1", pod: v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: testNamespace, UID:       testPodUID, }, Spec: v1.PodSpec{ Containers: []v1.Container{ { VolumeMounts: []v1.VolumeMount{ {MountPath: "/dir", Name: "vol1"}, {MountPath: "/dir/nested", Name: "vol2"}, {MountPath: "/ignore", Name: "vol4"}, {MountPath: "/other", Name: "vol1"}, {MountPath: "/other/nested2", Name: "vol3"}, }, }, }, }, }, }, { name:     "Big Pod", err:      false, volname:  "vol1", expected: sets.NewString("sub1/sub2/sub3", "sub1/sub2/sub4", "sub1/sub2/sub6", "sub"), pod: v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: testNamespace, UID:       testPodUID, }, Spec: v1.PodSpec{ Containers: []v1.Container{ { VolumeMounts: []v1.VolumeMount{ {MountPath: "/mnt", Name: "vol1"}, {MountPath: "/ignore", Name: "vol2"}, {MountPath: "/mnt/sub1/sub2/sub3", Name: "vol3"}, {MountPath: "/mnt/sub1/sub2/sub4", Name: "vol4"}, {MountPath: "/mnt/sub1/sub2/sub4/skip", Name: "vol5"}, {MountPath: "/mnt/sub1/sub2/sub4/skip2", Name: "vol5a"}, {MountPath: "/mnt/sub1/sub2/sub6", Name: "vol6"}, {MountPath: "/mnt7", Name: "vol7"}, }, }, }, InitContainers: []v1.Container{ { VolumeMounts: []v1.VolumeMount{ {MountPath: "/mnt/dir", Name: "vol1"}, {MountPath: "/mnt/dir_ignore", Name: "vol8"}, {MountPath: "/ignore", Name: "vol9"}, {MountPath: "/mnt/dir/sub", Name: "vol11"}, }, }, }, }, }, }, { name:     "Naughty Pod", err:      true, expected: nil, volname:  "vol1", pod: v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: testNamespace, UID:       testPodUID, }, Spec: v1.PodSpec{ Containers: []v1.Container{ { VolumeMounts: []v1.VolumeMount{ {MountPath: "foo/../../dir", Name: "vol1"}, {MountPath: "foo/../../dir/skip", Name: "vol10"}, }, }, }, }, }, }, } for _, test := range tc { dir, err := ioutil.TempDir("", "TestMakeNestedMountpoints.") if err != nil { t.Errorf("Unexpected error trying to create temp directory: %v", err) return } defer os.RemoveAll(dir) rootdir := path.Join(dir, "vol") err = os.Mkdir(rootdir, 0755) if err != nil { t.Errorf("Unexpected error trying to create temp root directory: %v", err) return } dirs, err := getNestedMountpoints(test.volname, rootdir, test.pod) if test.err { if err == nil { t.Errorf("%v: expected error, got nil", test.name) } continue } else { if err != nil { t.Errorf("%v: expected no error, got %v", test.name, err) continue } } actual := sets.NewString(dirs...) if !test.expected.Equal(actual) { t.Errorf("%v: unexpected nested directories created:nexpected: %vn     got: %v", test.name, test.expected, actual) } } } </ins>