Repository: kubernetes-csi/external-snapshotter (go)

Issue:
Failed to delete volumesnapshot if the volumesnapshotclass is deleted
In case of PVC deletion, nothing is blocking the PVC deletion even if the storageclass is deleted. external provisioner will send a request to the CSI driver with volumeID and no secrets will be sent as it cannot get the storageclass name. but in the case of volume snapshot, the volumesnapshot deletion will never complete as the volumesnapshotclass is already deleted. As per the CSI spec the secrets are optional parameters why external snapshotter is not sending requests to the CSI driver for delete snasphot. is this expected behavior?

````logs
I1029 10:13:02.926462       1 snapshot_controller.go:308] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent: set DeletionTimeStamp on content [snapcontent-f4e2764a-7f9f-4619-a89e-9d7d1318bcd4].
I1029 10:13:02.934162       1 snapshot_controller.go:316] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent: Remove Finalizer for VolumeSnapshot[default/rbd-pvc-snapshot]
I1029 10:13:02.934192       1 snapshot_controller.go:901] checkandRemovePVCFinalizer for snapshot [rbd-pvc-snapshot]: snapshot status [&v1beta1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc000285150), CreationTime:(*v1.Time)(0xc0004c2d80), ReadyToUse:(*bool)(0xc0002a8308), RestoreSize:(*resource.Quantity)(0xc00029f300), Error:(*v1beta1.VolumeSnapshotError)(0xc0002851b0)}]
I1029 10:13:02.943296       1 reflector.go:369] github.com/kubernetes-csi/external-snapshotter/client/v3/informers/externalversions/factory.go:117: forcing resync
I1029 10:13:02.943400       1 snapshot_controller_base.go:158] enqueued "default/rbd-pvc-snapshot" for sync
I1029 10:13:02.951105       1 util.go:264] storeObjectUpdate updating snapshot "default/rbd-pvc-snapshot" with version 1567
I1029 10:13:02.951304       1 snapshot_controller.go:1344] Removed protection finalizer from volume snapshot default/rbd-pvc-snapshot
I1029 10:13:02.951460       1 snapshot_controller_base.go:202] syncSnapshotByKey[default/rbd-pvc-snapshot]
I1029 10:13:02.951525       1 snapshot_controller_base.go:205] snapshotWorker: snapshot namespace [default] name [rbd-pvc-snapshot]
I1029 10:13:02.951545       1 snapshot_controller_base.go:328] checkAndUpdateSnapshotClass [rbd-pvc-snapshot]: VolumeSnapshotClassName [csi-rbdplugin-snapclass]
I1029 10:13:02.951555       1 snapshot_controller.go:1176] getSnapshotClass: VolumeSnapshotClassName [csi-rbdplugin-snapclass]
E1029 10:13:02.951573       1 snapshot_controller.go:1180] failed to retrieve snapshot class csi-rbdplugin-snapclass from the informer: "volumesnapshotclass.snapshot.storage.k8s.io \"csi-rbdplugin-snapclass\" not found"
E1029 10:13:02.951597       1 snapshot_controller_base.go:331] checkAndUpdateSnapshotClass failed to getSnapshotClass volumesnapshotclass.snapshot.storage.k8s.io "csi-rbdplugin-snapclass" not found
I1029 10:13:02.951609       1 snapshot_controller.go:721] updateSnapshotStatusWithEvent[default/rbd-pvc-snapshot]
I1029 10:13:02.951618       1 snapshot_controller.go:724] updateSnapshotStatusWithEvent[rbd-pvc-snapshot]: the same error &{2020-10-29 10:11:39 +0000 UTC 0xc0002851d0} is already set
I1029 10:13:02.951694       1 snapshot_controller_base.go:220] Snapshot "default/rbd-pvc-snapshot" is being deleted. SnapshotClass has already been removed
I1029 10:13:02.951715       1 snapshot_controller_base.go:222] Updating snapshot "default/rbd-pvc-snapshot"
I1029 10:13:02.951728       1 snapshot_controller_base.go:358] updateSnapshot "default/rbd-pvc-snapshot"
I1029 10:13:02.951751       1 util.go:264] storeObjectUpdate updating snapshot "default/rbd-pvc-snapshot" with version 1567
I1029 10:13:02.951778       1 snapshot_controller.go:180] synchronizing VolumeSnapshot[default/rbd-pvc-snapshot]: bound to: "snapcontent-f4e2764a-7f9f-4619-a89e-9d7d1318bcd4", Completed: false
I1029 10:13:02.951792       1 snapshot_controller.go:182] syncSnapshot [default/rbd-pvc-snapshot]: check if we should remove finalizer on snapshot PVC source and remove it if we can
I1029 10:13:02.951813       1 snapshot_controller.go:901] checkandRemovePVCFinalizer for snapshot [rbd-pvc-snapshot]: snapshot status [&v1beta1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc000285150), CreationTime:(*v1.Time)(0xc0004c2d80), ReadyToUse:(*bool)(0xc0002a8308), RestoreSize:(*resource.Quantity)(0xc00029f300), Error:(*v1beta1.VolumeSnapshotError)(0xc0002851b0)}]
I1029 10:13:02.951893       1 snapshot_controller.go:191] syncSnapshot[default/rbd-pvc-snapshot]: check if we should add invalid label on snapshot
I1029 10:13:02.951919       1 snapshot_controller.go:238] processSnapshotWithDeletionTimestamp VolumeSnapshot[default/rbd-pvc-snapshot]: bound to: "snapcontent-f4e2764a-7f9f-4619-a89e-9d7d1318bcd4", Completed: false
I1029 10:13:02.951936       1 snapshot_controller.go:272] processSnapshotWithDeletionTimestamp[default/rbd-pvc-snapshot]: delete snapshot content and remove finalizer from snapshot if needed
I1029 10:13:02.951951       1 snapshot_controller.go:278] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent VolumeSnapshot[default/rbd-pvc-snapshot]: bound to: "snapcontent-f4e2764a-7f9f-4619-a89e-9d7d1318bcd4", Completed: false
I1029 10:13:02.951991       1 snapshot_controller.go:796] isVolumeBeingCreatedFromSnapshot: no volume is being created from snapshot default/rbd-pvc-snapshot
I1029 10:13:02.952021       1 snapshot_controller.go:297] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent[default/rbd-pvc-snapshot]: Set VolumeSnapshotBeingDeleted annotation on the content [snapcontent-f4e2764a-7f9f-4619-a89e-9d7d1318bcd4]
I1029 10:13:02.952036       1 snapshot_controller.go:308] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent: set DeletionTimeStamp on content [snapcontent-f4e2764a-7f9f-4619-a89e-9d7d1318bcd4].
I1029 10:13:02.964786       1 snapshot_controller.go:316] checkandRemoveSnapshotFinalizersAndCheckandDeleteContent: Remove Finalizer for VolumeSnapshot[default/rbd-pvc-snapshot]
I1029 10:13:02.964811       1 snapshot_controller.go:901] checkandRemovePVCFinalizer for snapshot [rbd-pvc-snapshot]: snapshot status [&v1beta1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName:(*string)(0xc000285150), CreationTime:(*v1.Time)(0xc0004c2d80), ReadyToUse:(*bool)(0xc0002a8308), RestoreSize:(*resource.Quantity)(0xc00029f300), Error:(*v1beta1.VolumeSnapshotError)(0xc0002851b0)}]
I1029 10:13:02.974612       1 util.go:264] storeObjectUpdate updating snapshot "default/rbd-pvc-snapshot" with version 1567
I1029 10:13:02.974651       1 snapshot_controller.go:1344] Removed protection finalizer from volume snapshot default/rbd-pvc-snapshot
```

```
E1029 10:16:55.679819       1 snapshot_controller.go:224] getCSISnapshotInput failed to getClassFromVolumeSnapshot failed to retrieve snapshot class csi-rbdplugin-snapclass from the informer: "volumesnapshotclass.snapshot.storage.k8s.io \"csi-rbdplugin-snapclass\" not found"
E1029 10:16:55.685015       1 goroutinemap.go:150] Operation for "delete-snapcontent-f4e2764a-7f9f-4619-a89e-9d7d1318bcd4" failed. No retries permitted until 2020-10-29 10:17:27.679951762 +0000 UTC m=+442.637480008 (durationBeforeRetry 32s). Error: "failed to get input parameters to delete snapshot for content snapcontent-f4e2764a-7f9f-4619-a89e-9d7d1318bcd4: \"failed to retrieve snapshot class csi-rbdplugin-snapclass from the informer: \\\"volumesnapshotclass.snapshot.storage.k8s.io \\\\\\\"csi-rbdplugin-snapclass\\\\\\\" not found\\\"\""
I1029 10:16:55.684904       1 event.go:281] Event(v1.ObjectReference{Kind:"VolumeSnapshotContent", Namespace:"", Name:"snapcontent-f4e2764a-7f9f-4619-a89e-9d7d1318bcd4", UID:"ffc95922-3b31-494e-a40a-abc11f97c648", APIVersion:"snapshot.storage.k8s.io/v1beta1", ResourceVersion:"1565", FieldPath:""}): type: 'Warning' reason: 'SnapshotDeleteError' Failed to get snapshot class or credentials
E1029 10:17:55.679735       1 snapshot_controller.go:481] failed to retrieve snapshot class csi-rbdplugin-snapclass from the informer: "volumesnapshotclass.snapshot.storage.k8s.io \"csi-rbdplugin-snapclass\" not found"
E1029 10:17:55.679764       1 snapshot_controller.go:224] getCSISnapshotInput failed to getClassFromVolumeSnapshot failed to retrieve snapshot class csi-rbdplugin-snapclass from the informer: "volumesnapshotclass.snapshot.storage.k8s.io \"csi-rbdplugin-snapclass\" not found"
E1029 10:17:55.679819       1 goroutinemap.go:150] Operation for "delete-snapcontent-f4e2764a-7f9f-4619-a89e-9d7d1318bcd4" failed. No retries permitted until 2020-10-29 10:18:59.679780946 +0000 UTC m=+534.637309142 (durationBeforeRetry 1m4s). Error: "failed to get input parameters to delete snapshot for content snapcontent-f4e2764a-7f9f-4619-a89e-9d7d1318bcd4: \"failed to retrieve snapshot class csi-rbdplugin-snapclass from the informer: \\\"volumesnapshotclass.snapshot.storage.k8s.io \\\\\\\"csi-rbdplugin-snapclass\\\\\\\" not found\\\"\""
I1029 10:17:55.681554       1 event.go:281] Event(v1.ObjectReference{Kind:"VolumeSnapshotContent", Namespace:"", Name:"snapcontent-f4e2764a-7f9f-4619-a89e-9d7d1318bcd4", UID:"ffc95922-3b31-494e-a40a-abc11f97c648", APIVersion:"snapshot.storage.k8s.io/v1beta1", ResourceVersion:"1565", FieldPath:""}): type: 'Warning' reason: 'SnapshotDeleteError' Failed to get snapshot class or credentials
E1029 10:19:55.680458       1 snapshot_controller.go:481] failed to retrieve snapshot class csi-rbdplugin-snapclass from the informer: "volumesnapshotclass.snapshot.storage.k8s.io \"csi-rbdplugin-snapclass\" not found"
E1029 10:19:55.680653       1 snapshot_controller.go:224] getCSISnapshotInput failed to getClassFromVolumeSnapshot failed to retrieve snapshot class csi-rbdplugin-snapclass from the informer: "volumesnapshotclass.snapshot.storage.k8s.io \"csi-rbdplugin-snapclass\" not found"
E1029 10:19:55.680896       1 goroutinemap.go:150] Operation for "delete-snapcontent-f4e2764a-7f9f-4619-a89e-9d7d1318bcd4" failed. No retries permitted until 2020-10-29 10:21:57.68076332 +0000 UTC m=+712.638291701 (durationBeforeRetry 2m2s). Error: "failed to get input parameters to delete snapshot for content snapcontent-f4e2764a-7f9f-4619-a89e-9d7d1318bcd4: \"failed to retrieve snapshot class csi-rbdplugin-snapclass from the informer: \\\"volumesnapshotclass.snapshot.storage.k8s.io \\\\\\\"csi-rbdplugin-snapclass\\\\\\\" not found\\\"\""
I1029 10:19:55.687959       1 event.go:281] Event(v1.ObjectReference{Kind:"VolumeSnapshotContent", Namespace:"", Name:"snapcontent-f4e2764a-7f9f-4619-a89e-9d7d1318bcd4", UID:"ffc95922-3b31-494e-a40a-abc11f97c648", APIVersion:"snapshot.storage.k8s.io/v1beta1", ResourceVersion:"1565", FieldPath:""}): type: 'Warning' reason: 'SnapshotDeleteError' Failed to get snapshot class or credentials
```

cc @xing-yang 

Pull request description:
Get credentials before checking if the SnapshotClass exists
**What type of PR is this?**
/kind bug

**What this PR does / why we need it**:
If a CSIDriver requires credentials, but the VolumeSnapshotClass has been deleted, then the VolumeSnapshotContent cannot be deleted even though it contains a reference to the credentials in its own annotations.

This PR grabs the credentials instead of checking to see if the VolumeSnapshotClass exists, so that we can delete VolumeSnapshotContent objects if the secret annotation is on the VolumeSnapshotContent.

**Which issue(s) this PR fixes**:
Fixes #412

**Special notes for your reviewer**:

**Does this PR introduce a user-facing change?**:
```release-note
NONE
```


Patch:
diff --git a/pkg/sidecar-controller/snapshot_controller.go b/pkg/sidecar-controller/snapshot_controller.go
index 51ad10415..7b6ea128b 100644
--- a/pkg/sidecar-controller/snapshot_controller.go
+++ b/pkg/sidecar-controller/snapshot_controller.go
@@ -27,7 +27,6 @@ import (
 	codes "google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	v1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	klog "k8s.io/klog/v2"
 )
@@ -338,9 +337,9 @@ func (ctrl *csiSnapshotSideCarController) createSnapshotWrapper(content *crdv1.V
 func (ctrl *csiSnapshotSideCarController) deleteCSISnapshotOperation(content *crdv1.VolumeSnapshotContent) error {
 	klog.V(5).Infof("deleteCSISnapshotOperation [%s] started", content.Name)
 
-	_, snapshotterCredentials, err := ctrl.getCSISnapshotInput(content)
-	if err != nil && !errors.IsNotFound(err) {
-		ctrl.eventRecorder.Event(content, v1.EventTypeWarning, "SnapshotDeleteError", "Failed to get snapshot class or credentials")
+	snapshotterCredentials, err := ctrl.GetCredentialsFromAnnotation(content)
+	if err != nil {
+		ctrl.eventRecorder.Event(content, v1.EventTypeWarning, "SnapshotDeleteError", "Failed to get snapshot credentials")
 		return fmt.Errorf("failed to get input parameters to delete snapshot for content %s: %q", content.Name, err)
 	}
 


Test patch:
diff --git a/pkg/sidecar-controller/snapshot_delete_test.go b/pkg/sidecar-controller/snapshot_delete_test.go
index d0604fabe..7f18abc0d 100644
--- a/pkg/sidecar-controller/snapshot_delete_test.go
+++ b/pkg/sidecar-controller/snapshot_delete_test.go
@@ -339,7 +339,7 @@ func TestDeleteSync(t *testing.T) {
 		{
 			name:                "1-15 - (dynamic)deletion of content with no snapshotclass should succeed",
 			initialContents:     newContentArrayWithDeletionTimestamp("content1-15", "sid1-15", "snap1-15", "sid1-15", "", "", "snap1-15-volumehandle", deletePolicy, nil, &defaultSize, true, &timeNowMetav1),
-			expectedContents:    newContentArrayWithDeletionTimestamp("content1-15", "sid1-15", "snap1-15", "sid1-15", "", "", "snap1-15-volumehandle", deletePolicy, nil, &defaultSize, true, &timeNowMetav1),
+			expectedContents:    newContentArrayWithDeletionTimestamp("content1-15", "sid1-15", "snap1-15", "", "", "", "snap1-15-volumehandle", deletePolicy, nil, &defaultSize, false, &timeNowMetav1),
 			errors:              noerrors,
 			expectedDeleteCalls: []deleteCall{{"sid1-15", nil, nil}},
 			test:                testSyncContent,