Change:
What happened: Azure Disks get randomly mounted in a way that makes them unusable, returning for every disk I/O operation. Affected Pods remain in until someone manually recreates them. What you expected to happen: Azure Disks are consistently usable by Pods inside which they are mounted. How to reproduce it: Create the following StatefulSet. Adjust the replicas and delete the associated Pods () until the problem occurs (i.e. some Pod ends up in ). <details</summary</details</brDefault StorageClass (as created by ACS-Engine): <details</summary</details</brAnything else we need to know?: Nodes were updated from 1.10.4 to 1.10.10, then from 1.10.10 to without a reboot. According to written by the issue is fixed, but I can still observe it quite often. Ref Azure/acs-engine Environment: Kubernetes version: v1.10.10 + Cloud provider or hardware configuration: Azure (westeurope) OS: Ubuntu 16.04.4 LTS Kernel: 4.13.0-1018-azure Install tools: ACS-Engine Others: /kind bug /sig storage /sig azure
In my current test environment: (pod uid is -f23b-11e8-bf38-) on the node Notice the mismatch between the disk referenced by lun0 () and the actual mount (). on the node
After deleting the failing Pod and waiting for its successful recreation: (pod uid is now -f247-11e8-bf38-) on the node This time disks are matching (). on the node
/assign

Discussion:
ctx, cancel := getContextWithCancel() defer cancel() <ins> // Invalidate the cache right after updating defer as.cloud.vmCache.Delete(vmName) </ins> _, err = as.VirtualMachinesClient.CreateOrUpdate(ctx, nodeResourceGroup, vmName, newVM) if err != nil { klog.Errorf("azureDisk - attach disk(%s) failed, err: %v", diskName, err)

} } else { klog.V(2).Infof("azureDisk - attach disk(%s) succeeded", diskName) <del> // Invalidate the cache right after updating as.cloud.vmCache.Delete(vmName) </del> } return err }

klog.V(2).Infof("azureDisk - update(%s): vm(%s) - detach disk(%s)", nodeResourceGroup, vmName, diskName) ctx, cancel := getContextWithCancel() defer cancel() <ins> // Invalidate the cache right after updating defer as.cloud.vmCache.Delete(vmName) </ins> _, err = as.VirtualMachinesClient.CreateOrUpdate(ctx, nodeResourceGroup, vmName, newVM) if err != nil { klog.Errorf("azureDisk - detach disk(%s) failed, err: %v", diskName, err) } else { klog.V(2).Infof("azureDisk - detach disk(%s) succeeded", diskName) <del> // Invalidate the cache right after updating as.cloud.vmCache.Delete(vmName) </del> } return err }

ctx, cancel := getContextWithCancel() defer cancel() <del> klog.V(2).Infof("azureDisk - update(%s): vm(%s) - attach disk(%s)", nodeResourceGroup, nodeName, diskName) </del> <ins> // Invalidate the cache right after updating key := buildVmssCacheKey(nodeResourceGroup, ss.makeVmssVMName(ssName, instanceID)) defer ss.vmssVMCache.Delete(key) klog.V(2).Infof("azureDisk - update(%s): vm(%s) - attach disk(%s)", nodeResourceGroup, nodeName, diskName) </ins> _, err = ss.VirtualMachineScaleSetVMsClient.Update(ctx, nodeResourceGroup, ssName, instanceID, vm) if err != nil { detail := err.Error()

} } else { klog.V(2).Infof("azureDisk - attach disk(%s) succeeded", diskName) <del> // Invalidate the cache right after updating key := buildVmssCacheKey(nodeResourceGroup, ss.makeVmssVMName(ssName, instanceID)) ss.vmssVMCache.Delete(key) </del> } return err }

vm.StorageProfile.DataDisks = &disks ctx, cancel := getContextWithCancel() defer cancel() <ins> // Invalidate the cache right after updating key := buildVmssCacheKey(nodeResourceGroup, ss.makeVmssVMName(ssName, instanceID)) defer ss.vmssVMCache.Delete(key) </ins> klog.V(2).Infof("azureDisk - update(%s): vm(%s) - detach disk(%s)", nodeResourceGroup, nodeName, diskName) _, err = ss.VirtualMachineScaleSetVMsClient.Update(ctx, nodeResourceGroup, ssName, instanceID, vm) if err != nil { klog.Errorf("azureDisk - detach disk(%s) from %s failed, err: %v", diskName, nodeName, err) } else { klog.V(2).Infof("azureDisk - detach disk(%s) succeeded", diskName) <del> // Invalidate the cache right after updating key := buildVmssCacheKey(nodeResourceGroup, ss.makeVmssVMName(ssName, instanceID)) ss.vmssVMCache.Delete(key) </del> } return err