Change:
BUG REPORT Kubernetes version (use ): Client Version: version.Info{Major:"1", Minor:"5", GitVersion:"v1.5.2", GitCommit:"", GitTreeState:"clean", BuildDate:"2017-01-12T04:57:25Z", GoVersion:"go1.7.4", Compiler:"gc", Platform:"linux/amd64"} Server Version: version.Info{Major:"1", Minor:"5", GitVersion:"v1.5.2", GitCommit:"", GitTreeState:"clean", BuildDate:"2017-01-12T04:52:34Z", GoVersion:"go1.7.4", Compiler:"gc", Platform:"linux/amd64"} Environment: Cloud provider or hardware configuration: Bare metal OS (e.g. from /etc/os-release): CentOS 7; reproduced in Ubuntu 14.0.4.2. Kernel (e.g. ): Linux centos7 3.10.0-514.6.1.el7.x8664 SMP Wed Jan 18 13:06:36 UTC 2017 x8664 x8664 x8664 GNU/Linux; kernel 3.16.0-77-generic What happened: On a host where an iSCSI LUN from a given target is mounted in a pod, a user who creates a new LUN on the same target and attempts to create a pod that mounts the new LUN will fail to create the pod. Specifically, depending on the operating system, the pod will hang in the ContainerCreating stage with either a "Could not attach disk: Timeout after 10s" error (CentOS 7) or an error about establishing a session when one already exists (Ubuntu 14). What you expected to happen: The pod should successfully mount the LUN. How to reproduce it (as minimally and precisely as possible): Given a working iSCSI target and a single node Kubernetes cluster: a LUN on the iSCSI target a pod that uses the LUN from step 1 (either natively or through a PV/PVC) a new LUN on the iSCSI target (note that the target portal and IQN must be identical). a pod that uses the LUN from step 3. The pod created in step 4 will fail to start. Anything else we need to know: As far as I can tell, this is due to a bug with the Kubernetes iSCSI implementation. The AttachDisk() function in first checks if the LUN exists in /dev/disk/by-path/; if it doesn't, it attempts to log in to the target and recheck for the LUN's existence (lines 123-138 in the 1.5 branch). In this case, though, the session already exists, as it was created with the pod in step 2. In Ubuntu, attempting to log in a second time triggers an error, which gets reported in the pod description. In CentOS 7, this gets treated as a NOP; however, the new LUN will never become visible, and thus, the pod will fail to start. This can be solved manually by logging into the host and running . It should be possible to fix it in Kubernetes by adding a check in AttachDisk() to see if a session already exists for the target portal and IQN and, if so, doing a rescan of the session.
Could someone from please help with triage? Thanks!

Discussion:
return refCount, nil } <del> // make a directory like /var/lib/kubelet/plugins/kubernetes.io/iscsi/portal-some_iqn-lun-lun_id func makePDNameInternal(host volume.VolumeHost, portal string, iqn string, lun string) string { return path.Join(host.GetPluginDir(iscsiPluginName), portal+"-"+iqn+"-lun-"+lun) </del> <ins> // make a directory like /var/lib/kubelet/plugins/kubernetes.io/iscsi/iface_name/portal-some_iqn-lun-lun_id func makePDNameInternal(host volume.VolumeHost, portal string, iqn string, lun string, iface string) string { return path.Join(host.GetPluginDir(iscsiPluginName), "iface-"+iface, portal+"-"+iqn+"-lun-"+lun) </ins> } type ISCSIUtil struct{} func (util *ISCSIUtil) MakeGlobalPDName(iscsi iscsiDisk) string { <del> return makePDNameInternal(iscsi.plugin.host, iscsi.portals[0], iscsi.iqn, iscsi.lun) </del> <ins> return makePDNameInternal(iscsi.plugin.host, iscsi.portals[0], iscsi.iqn, iscsi.lun, iscsi.iface) </ins> } func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) error { var devicePath string var devicePaths []string var iscsiTransport string out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "iface", "-I", b.iface, "-o", "show"}) if err != nil { glog.Errorf("iscsi: could not read iface %s error: %s", b.iface, string(out))

} iscsiTransport = extractTransportname(string(out)) bkpPortal := b.portals for _, tp := range bkpPortal { <ins> // Rescan sessions to discover newly mapped LUNs. Do not specify the interface when rescanning // to avoid establishing additional sessions to the same target. out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", tp, "-T", b.iqn, "-R"}) if err != nil { glog.Errorf("iscsi: failed to rescan session with error: %s (%v)", string(out), err) } </ins> if iscsiTransport == "" { glog.Errorf("iscsi: could not find transport name in iface %s", b.iface) return errors.New(fmt.Sprintf("Could not parse iface file for %s", b.iface))

return err } refCount, err := getDevicePrefixRefCount(c.mounter, prefix) if err == nil && refCount == 0 { <del> // this portal/iqn are no longer referenced, log out // extract portal and iqn from device path </del> <ins> // This portal/iqn/iface is no longer referenced, log out. // Extract the portal and iqn from device path. </ins> portal, iqn, err := extractPortalAndIqn(device) if err != nil { return err } <del> glog.Infof("iscsi: log out target %s iqn %s", portal, iqn) out, err := c.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", portal, "-T", iqn, "--logout"}) if err != nil { glog.Errorf("iscsi: failed to detach disk Error: %s", string(out)) </del> <ins> // Extract the iface from the mountPath and use it to log out. If the iface // is not found, maintain the previous behavior to facilitate kubelet upgrade. // Logout may fail as no session may exist for the portal/IQN on the specified interface. iface, found := extractIface(mntPath) if found { glog.Infof("iscsi: log out target %s iqn %s iface %s", portal, iqn, iface) out, err := c.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", portal, "-T", iqn, "-I", iface, "--logout"}) if err != nil { glog.Errorf("iscsi: failed to detach disk Error: %s", string(out)) } } else { glog.Infof("iscsi: log out target %s iqn %s", portal, iqn) out, err := c.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", portal, "-T", iqn, "--logout"}) if err != nil { glog.Errorf("iscsi: failed to detach disk Error: %s", string(out)) } </ins> } } }

return "", "", fmt.Errorf("iscsi detach disk: malformatted mnt path: %s", mntPath) } device := mntPath[(ind + 1):] <del> // strip -lun- from device path ind = strings.LastIndex(device, "-lun-") </del> <ins> // strip -lun- from mount path ind = strings.LastIndex(mntPath, "-lun-") </ins> if ind < 0 { return "", "", fmt.Errorf("iscsi detach disk: malformatted mnt path: %s", mntPath) } <del> prefix := device[:ind] </del> <ins> prefix := mntPath[:ind] </ins> return device, prefix, nil } <ins> func extractIface(mntPath string) (string, bool) { re := regexp.MustCompile(`.+/iface-([^/]+)/.+`) re_output := re.FindStringSubmatch(mntPath) if re_output != nil { return re_output[1], true } return "", false } </ins> func extractPortalAndIqn(device string) (string, string, error) { ind1 := strings.Index(device, "-") if ind1 < 0 {

func TestExtractDeviceAndPrefix(t *testing.T) { devicePath := "127.0.0.1:3260-iqn.2014-12.com.example:test.tgt00" <ins> mountPrefix := "/var/lib/kubelet/plugins/kubernetes.io/iscsi/iface-default/" + devicePath </ins> lun := "-lun-0" <del> device, prefix, err := extractDeviceAndPrefix("/var/lib/kubelet/plugins/kubernetes.io/iscsi/" + devicePath + lun) if err != nil || device != (devicePath+lun) || prefix != devicePath { t.Errorf("extractDeviceAndPrefix: expected %s and %s, got %v %s and %s", devicePath+lun, devicePath, err, device, prefix) </del> <ins> device, prefix, err := extractDeviceAndPrefix(mountPrefix + lun) if err != nil || device != (devicePath+lun) || prefix != mountPrefix { t.Errorf("extractDeviceAndPrefix: expected %s and %s, got %v %s and %s", devicePath+lun, mountPrefix, err, device, prefix) } } func TestExtractIface(t *testing.T) { ifaceName := "default" devicePath := "127.0.0.1:3260-iqn.2014-12.com.example:test.tgt00-lun-0" iface, found := extractIface("/var/lib/kubelet/plugins/kubernetes.io/iscsi/iface-" + ifaceName + "/" + devicePath) if !found || iface != ifaceName { t.Errorf("extractIface: expected %s and %t, got %s and %t", ifaceName, true, iface, found) } iface, found = extractIface("/var/lib/kubelet/plugins/kubernetes.io/iscsi/" + devicePath) if found || iface != "" { t.Errorf("extractIface: expected %s and %t, got %s and %t", "", false, iface, found) </ins> } }