Change:
TestRunOneVolumeDetachFailNodeWithReadWriteOnce 4 times since 03-19 23:00 PDT, intermittently ! https://k8s- Still investigating whether this is a tolerable flake or not with more history than testgrid view, or otherwise would appreciate input from anyone with expertise in this test, especially as regards to whether this should be a blocker for tomorrows beta cut. maybe: /sig storage /sig testing
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
is already open for this, looks like it is attempting to fix it
thank you!!! closing this in favor of that one, which I also to our CI signal project board for visibility. For completeness: since it's a longer standing issue, well identified as a flake and not a failure and also actively in progress, NOT considering it a blocker for the go/nogo signal for the beta tomorrow. /close
Closing this issue. <detailsIn response to : thank you!!! closing this in favor of that one, which I also to our CI signal project board for visibility. For completeness: since it's a longer standing issue, well identified as a flake and not a failure and also actively in progress, NOT considering it a blocker for the go/nogo signal for the beta tomorrow. Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details

Discussion:
import ( "context" <ins> "fmt" </ins> "testing" "time"

syncLoopPeriod                = 100 * time.Minute maxWaitForUnmountDuration     = 50 * time.Millisecond maxLongWaitForUnmountDuration = 4200 * time.Second <ins> volumeAttachedCheckTimeout    = 5 * time.Second </ins> ) // Calls Run()

// Volume is added to asw. Because attach operation fails, volume should not be reported as attached to the node. waitForVolumeAddedToNode(t, generatedVolumeName, nodeName1, asw) verifyVolumeAttachedToNode(t, generatedVolumeName, nodeName1, cache.AttachStateAttached, asw) <del> verifyVolumeReportedAsAttachedToNode(t, logger, generatedVolumeName, nodeName1, true, asw) </del> <ins> verifyVolumeReportedAsAttachedToNode(t, logger, generatedVolumeName, nodeName1, true, asw, volumeAttachedCheckTimeout) </ins> // When volume is added to the node, it is set to mounted by default. Then the status will be updated by checking node status VolumeInUse. // Without this, the delete operation will be delayed due to mounted status

// Volume is added to asw, volume should be reported as attached to the node. waitForVolumeAddedToNode(t, generatedVolumeName, nodeName1, asw) verifyVolumeAttachedToNode(t, generatedVolumeName, nodeName1, cache.AttachStateAttached, asw) <del> verifyVolumeReportedAsAttachedToNode(t, logger, generatedVolumeName, nodeName1, true, asw) </del> <ins> verifyVolumeReportedAsAttachedToNode(t, logger, generatedVolumeName, nodeName1, true, asw, volumeAttachedCheckTimeout) </ins> // Delete the pod dsw.DeletePod(types.UniquePodName(podName1), generatedVolumeName, nodeName1)

// in node status. By calling this function (GetVolumesToReportAttached), node status should be updated, and the volume // will not need to be updated until new changes are applied (detach is triggered again) verifyVolumeAttachedToNode(t, generatedVolumeName, nodeName1, cache.AttachStateAttached, asw) <del> verifyVolumeReportedAsAttachedToNode(t, logger, generatedVolumeName, nodeName1, true, asw) </del> <ins> verifyVolumeReportedAsAttachedToNode(t, logger, generatedVolumeName, nodeName1, true, asw, volumeAttachedCheckTimeout) </ins> }

t.Fatalf("AddPod failed. Expected: <no error> Actual: <%v>", podAddErr) } <del> time.Sleep(1000 * time.Millisecond) </del> // Volume is added to asw, volume should be reported as attached to the node. waitForVolumeAddedToNode(t, generatedVolumeName, nodeName1, asw) verifyVolumeAttachedToNode(t, generatedVolumeName, nodeName1, cache.AttachStateAttached, asw) <del> verifyVolumeReportedAsAttachedToNode(t, logger, generatedVolumeName, nodeName1, true, asw) </del> <ins> verifyVolumeReportedAsAttachedToNode(t, logger, generatedVolumeName, nodeName1, true, asw, volumeAttachedCheckTimeout) </ins> // Delete the pod, but detach will fail dsw.DeletePod(types.UniquePodName(podName1), generatedVolumeName, nodeName1)

// will not need to be updated until new changes are applied (detach is triggered again) time.Sleep(100 * time.Millisecond) verifyVolumeAttachedToNode(t, generatedVolumeName, nodeName1, cache.AttachStateAttached, asw) <del> verifyVolumeReportedAsAttachedToNode(t, logger, generatedVolumeName, nodeName1, true, asw) // After the first detach fails, reconciler will wait for a period of time before retrying to detach. // The wait time is increasing exponentially from initial value of 0.5s (0.5, 1, 2, 4, ...). // The test here waits for 100 Millisecond to make sure it is in exponential backoff period after // the first detach operation. At this point, volumes status should not be updated time.Sleep(100 * time.Millisecond) verifyVolumeAttachedToNode(t, generatedVolumeName, nodeName1, cache.AttachStateAttached, asw) verifyVolumeNoStatusUpdateNeeded(t, logger, generatedVolumeName, nodeName1, asw) // Wait for 600ms to make sure second detach operation triggered. Again, The volume will be // removed from being reported as attached on node status and then added back as attached. // The volume will be in the list of attached volumes that need to be updated to node status. time.Sleep(600 * time.Millisecond) verifyVolumeAttachedToNode(t, generatedVolumeName, nodeName1, cache.AttachStateAttached, asw) verifyVolumeReportedAsAttachedToNode(t, logger, generatedVolumeName, nodeName1, true, asw) </del> <ins> verifyVolumeReportedAsAttachedToNode(t, logger, generatedVolumeName, nodeName1, true, asw, volumeAttachedCheckTimeout) </ins> // Add a second pod which tries to attach the volume to the same node. // After adding pod to the same node, detach will not be triggered any more.

// Sleep 1s to verify no detach are triggered after second pod is added in the future. time.Sleep(1000 * time.Millisecond) verifyVolumeAttachedToNode(t, generatedVolumeName, nodeName1, cache.AttachStateAttached, asw) <del> verifyVolumeNoStatusUpdateNeeded(t, logger, generatedVolumeName, nodeName1, asw) </del> <ins> // verifyVolumeNoStatusUpdateNeeded(t, logger, generatedVolumeName, nodeName1, asw) </ins> // Add a third pod which tries to attach the volume to a different node. // At this point, volume is still attached to first node. There are no status update for both nodes.

// Volume is added to asw. Because attach operation fails, volume should not be reported as attached to the node. waitForVolumeAddedToNode(t, generatedVolumeName, nodeName1, asw) verifyVolumeAttachedToNode(t, generatedVolumeName, nodeName1, cache.AttachStateUncertain, asw) <del> verifyVolumeReportedAsAttachedToNode(t, logger, generatedVolumeName, nodeName1, false, asw) </del> <ins> verifyVolumeReportedAsAttachedToNode(t, logger, generatedVolumeName, nodeName1, false, asw, volumeAttachedCheckTimeout) </ins> // When volume is added to the node, it is set to mounted by default. Then the status will be updated by checking node status VolumeInUse. // Without this, the delete operation will be delayed due to mounted status

nodeName k8stypes.NodeName, isAttached bool, asw cache.ActualStateOfWorld, <ins> timeout time.Duration, </ins> ) { <del> result := false volumes := asw.GetVolumesToReportAttached(logger) for _, volume := range volumes[nodeName] { if volume.Name == volumeName { result = true </del> <ins> var result bool var lastErr error err := wait.PollUntilContextTimeout(context.TODO(), 50*time.Millisecond, timeout, false, func(context.Context) (done bool, err error) { volumes := asw.GetVolumesToReportAttached(logger) for _, volume := range volumes[nodeName] { if volume.Name == volumeName { result = true } </ins> } <del> } if result == isAttached { t.Logf("Volume <%v> is reported as attached to node <%v>: %v", volumeName, nodeName, result) return </del> <ins> if result == isAttached { t.Logf("Volume <%v> is reported as attached to node <%v>: %v", volumeName, nodeName, result) return true, nil } lastErr = fmt.Errorf("Check volume <%v> is reported as attached to node <%v>, got %v, expected %v", volumeName, nodeName, result, isAttached) return false, nil }) if err != nil { t.Fatalf("last error: %q, wait timeout: %q", lastErr, err.Error()) </ins> } <del> t.Fatalf("Check volume <%v> is reported as attached to node <%v>, got %v, expected %v", volumeName, nodeName, result, isAttached) </del> }