Change:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Deletion of a Pod with an ISCSI volume in mode results in an error during , leaving an ISCSI target logged into on the kubelet node despite the target being unused. In my case, the ISCSI Target server enforces a single session per LUN, which blocks the Pod from starting on another Node in the cluster. What you expected to happen: For the kubelet to issue a logout request for the given target once the refcounter hits zero. How to reproduce it (as minimally and precisely as possible): Have an ISCSI Target server Create a target/LUN that contains an ext4 filesystem. Have a Node configured as an ISCSI Initiator capable of logging into the those targets. Create a Pod on the Node that includes an volume referring the LUN, using (rather than ) mode. minimally: Delete the Pod Wait for the Pod to be destroyed and the related ISCSI block device to be removed. on the Node will show the target is still logged in Kubelet error: Anything else we need to know?: Environment: Kubernetes version (use ): 1.16.2 Cloud provider or hardware configuration: Reproducible on a bare metal (Intel NUC) lab and DigitalOcean DOKS. OS (e.g: ): Ubuntu 18.04 Kernel (e.g. ): 4.15 (NUC) / 4.19 (DigitalOcean) Install tools: Ubuntu package /sig storage

Discussion:
], embed = [":go_default_library"], deps = [ <ins> "//pkg/kubelet/config:go_default_library", </ins> "//pkg/volume:go_default_library", "//pkg/volume/testing:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library",

// getVolCount returns the number of volumes in use by the kubelet. // It does so by counting the number of directories prefixed by the given portal and IQN. func getVolCount(dir, portal, iqn string) (int, error) { <del> // The topmost dirs are named after the ifaces, e.g., iface-default or iface-127.0.0.1:3260:pv0 </del> <ins> // For FileSystem volumes, the topmost dirs are named after the ifaces, e.g., iface-default or iface-127.0.0.1:3260:pv0. // For Block volumes, the default topmost dir is volumeDevices. </ins> contents, err := ioutil.ReadDir(dir) if err != nil { <ins> if os.IsNotExist(err) { return 0, nil } </ins> return 0, err }

"reflect" "testing" <del> "k8s.io/utils/exec/testing" </del> <ins> testingexec "k8s.io/utils/exec/testing" "k8s.io/kubernetes/pkg/kubelet/config" </ins> "k8s.io/kubernetes/pkg/volume" volumetest "k8s.io/kubernetes/pkg/volume/testing" )

} func TestGetVolCount(t *testing.T) { <ins> // This will create a dir structure like this: // /tmp/refcounter555814673 // ├── iface-127.0.0.1:3260:pv1 // │   └── 127.0.0.1:3260-iqn.2003-01.io.k8s:e2e.volume-1-lun-3 // └── iface-127.0.0.1:3260:pv2 // │   ├── 127.0.0.1:3260-iqn.2003-01.io.k8s:e2e.volume-1-lun-2 // │   └── 192.168.0.1:3260-iqn.2003-01.io.k8s:e2e.volume-1-lun-1 // └── volumeDevices //     └── 192.168.0.2:3260-iqn.2003-01.io.k8s:e2e.volume-1-lun-4 //     └── 192.168.0.3:3260-iqn.2003-01.io.k8s:e2e.volume-1-lun-5 baseDir, err := createFakePluginDirs() if err != nil { t.Errorf("error creating fake plugin dir: %v", err) } defer os.RemoveAll(baseDir) </ins> testCases := []struct { <del> name   string portal string iqn    string count  int </del> <ins> name    string baseDir string portal  string iqn     string count   int </ins> }{ { <del> name:   "wrong portal, no volumes", portal: "192.168.0.2:3260", // incorrect IP address iqn:    "iqn.2003-01.io.k8s:e2e.volume-1", count:  0, </del> <ins> name:    "wrong portal, no volumes", baseDir: baseDir, portal:  "192.168.0.2:3260", // incorrect IP address iqn:     "iqn.2003-01.io.k8s:e2e.volume-1", count:   0, </ins> }, { <del> name:   "wrong iqn, no volumes", portal: "127.0.0.1:3260", iqn:    "iqn.2003-01.io.k8s:e2e.volume-3", // incorrect volume count:  0, </del> <ins> name:    "wrong iqn, no volumes", baseDir: baseDir, portal:  "127.0.0.1:3260", iqn:     "iqn.2003-01.io.k8s:e2e.volume-3", // incorrect volume count:   0, </ins> }, { <del> name:   "single volume", portal: "192.168.0.1:3260", iqn:    "iqn.2003-01.io.k8s:e2e.volume-1", count:  1, </del> <ins> name:    "single volume", baseDir: baseDir, portal:  "192.168.0.1:3260", iqn:     "iqn.2003-01.io.k8s:e2e.volume-1", count:   1, </ins> }, { <del> name:   "two volumes", portal: "127.0.0.1:3260", iqn:    "iqn.2003-01.io.k8s:e2e.volume-1", count:  2, </del> <ins> name:    "two volumes", baseDir: baseDir, portal:  "127.0.0.1:3260", iqn:     "iqn.2003-01.io.k8s:e2e.volume-1", count:   2, }, { name:    "volumeDevices (block) volume", baseDir: filepath.Join(baseDir, config.DefaultKubeletVolumeDevicesDirName), portal:  "192.168.0.2:3260", iqn:     "iqn.2003-01.io.k8s:e2e.volume-1-lun-4", count:   1, }, { name:    "nonexistent path", baseDir: filepath.Join(baseDir, "this_path_should_not_exist"), portal:  "127.0.0.1:3260", iqn:     "iqn.2003-01.io.k8s:e2e.unknown", count:   0, </ins> }, } <del> // This will create a dir structure like this: // /tmp/refcounter555814673 // ├── iface-127.0.0.1:3260:pv1 // │   └── 127.0.0.1:3260-iqn.2003-01.io.k8s:e2e.volume-1-lun-3 // └── iface-127.0.0.1:3260:pv2 //     ├── 127.0.0.1:3260-iqn.2003-01.io.k8s:e2e.volume-1-lun-2 //     └── 192.168.0.1:3260-iqn.2003-01.io.k8s:e2e.volume-1-lun-1 baseDir, err := createFakePluginDir() if err != nil { t.Errorf("error creating fake plugin dir: %v", err) } defer os.RemoveAll(baseDir) </del> for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { <del> count, err := getVolCount(baseDir, tc.portal, tc.iqn) </del> <ins> count, err := getVolCount(tc.baseDir, tc.portal, tc.iqn) </ins> if err != nil { t.Errorf("expected no error, got %v", err) }

} } <del> func createFakePluginDir() (string, error) { </del> <ins> func createFakePluginDirs() (string, error) { </ins> dir, err := ioutil.TempDir("", "refcounter") if err != nil { return "", err

"iface-127.0.0.1:3260:pv1/127.0.0.1:3260-iqn.2003-01.io.k8s:e2e.volume-1-lun-3", "iface-127.0.0.1:3260:pv2/127.0.0.1:3260-iqn.2003-01.io.k8s:e2e.volume-1-lun-2", "iface-127.0.0.1:3260:pv2/192.168.0.1:3260-iqn.2003-01.io.k8s:e2e.volume-1-lun-1", <ins> filepath.Join(config.DefaultKubeletVolumeDevicesDirName, "iface-127.0.0.1:3260/192.168.0.2:3260-iqn.2003-01.io.k8s:e2e.volume-1-lun-4"), filepath.Join(config.DefaultKubeletVolumeDevicesDirName, "iface-127.0.0.1:3260/192.168.0.3:3260-iqn.2003-01.io.k8s:e2e.volume-1-lun-5"), </ins> } for _, d := range subdirs {