Repository: dell/csi-powermax
Pull request #488: MultiAZ NodeGetInfo changes
Author: login: EvgenyUglov
Created: 2025-03-14T03:42:54Z | Merged: 2025-03-18T19:55:21Z

# Description
This PR adds changes to `NodeGetInfo` to update node topology according to zone labels from the secret

# GitHub Issues
List the GitHub issues impacted by this PR:

| GitHub Issue # |
| -------------- |
|https://github.com/dell/csm/issues/1748|

# Checklist:

- [x] Have you run format,vet & lint checks against your submission?
- [x] Have you made sure that the code compiles?
- [x] Did you run the unit & integration tests successfully?
- [ ] Have you maintained at least 90% code coverage?
- [ ] Have you commented your code, particularly in hard-to-understand areas
- [ ] Have you done corresponding changes to the documentation
- [x] Did you run tests in a real Kubernetes cluster?
- [x] Backward compatibility is not broken

# How Has This Been Tested?
Please describe the tests that you ran to verify your changes. Please also list any relevant details for your test configuration



Diff:
diff --git a/service/node.go b/service/node.go
index 212ffd5b..ab89bfdf 100644
--- a/service/node.go
+++ b/service/node.go
@@ -96,6 +96,8 @@ var symToMaskingViewTargets sync.Map
 // Map to store if sym has fc connectivity or not
 var isSymConnFC sync.Map
 
+var symmetrixIDs []string
+
 // InvalidateSymToMaskingViewTargets - invalidates the cache
 // Only used for testing
 func (s *service) InvalidateSymToMaskingViewTargets() {
@@ -1323,6 +1325,13 @@ func (s *service) NodeGetInfo(
 		}
 		maxPowerMaxVolumesPerNode = s.opts.MaxVolumesPerNode
 	}
+	for _, array := range symmetrixIDs {
+		arrayLabels := s.opts.StorageArrays[array].Labels
+		for arrayLabelKey, arrayLabelVal := range arrayLabels {
+			log.Infof("adding label '%s' with value '%s' to the topology map", arrayLabelKey, arrayLabelVal)
+			topology[arrayLabelKey] = arrayLabelVal.(string)
+		}
+	}
 	return &csi.NodeGetInfoResponse{
 		NodeId: s.opts.NodeName,
 		AccessibleTopology: &csi.Topology{
@@ -1567,8 +1576,7 @@ func (s *service) nodeStartup(ctx context.Context) error {
 		log.Debug("vmHost created successfully")
 	}
 
-	arrays := s.retryableGetSymmetrixIDList()
-	symmetrixIDs := arrays.SymmetrixIDs
+	symmetrixIDs = s.filterArraysByZoneInfo(s.opts.StorageArrays)
 	log.Debug(fmt.Sprintf("GetSymmetrixIDList returned: %v", symmetrixIDs))
 
 	err = s.nodeHostSetup(ctx, portWWNs, IQNs, hostNQN, symmetrixIDs)
diff --git a/service/node_test.go b/service/node_test.go
index 03ab260b..27072a47 100644
--- a/service/node_test.go
+++ b/service/node_test.go
@@ -1666,8 +1666,11 @@ func TestNodeGetInfo(t *testing.T) {
 		mockUtilsInterface        *k8smock.MockUtilsInterface
 		maxVolumesPerNode         int64
 		csiNodeGetInfoRequest     *csi.NodeGetInfoRequest
+		storageArrays             map[string]StorageArrayConfig
 		want                      map[string]string
+		expectedZoneInfo          map[string]string
 		wantErr                   bool
+		wantResp                  bool
 	}
 
 	testCases := []test{
@@ -1906,6 +1909,40 @@ func TestNodeGetInfo(t *testing.T) {
 			portGroups: []string{"portgroup1"},
 			wantErr:    false,
 		},
+		{
+			name:     "Zone labels added to topology",
+			nodeName: "node1",
+			getClient: func() *mocks.MockPmaxClient {
+				c := mocks.NewMockPmaxClient(gmock.NewController(t))
+				c.EXPECT().WithSymmetrixID("array1").AnyTimes().Return(c)
+				symmetrix.Initialize([]string{"array1"}, c)
+				return c
+			},
+			managedArrays: []string{"array1"},
+			arrayTransportProtocolMap: map[string]string{
+				"array1": NvmeTCPTransportProtocol,
+			},
+			nvmeTCPClient: gonvme.NewMockNVMe(map[string]string{}),
+			initFunc: func() *k8smock.MockUtilsInterface {
+				symmetrixIDs = []string{"array1"}
+				mockUtilsInterface := k8smock.NewMockUtilsInterface(gomock.NewController(t))
+				mockUtilsInterface.EXPECT().GetNodeLabels("node1").Return(map[string]string{"regionlabel": "R1", "zonelabel": "Z1"}, nil).AnyTimes()
+				return mockUtilsInterface
+			},
+			loggedInArrays: map[string]bool{},
+			loggedInNVMeArrays: map[string]bool{
+				"array1": true,
+			},
+			portGroups: []string{"portgroup1"},
+			storageArrays: map[string]StorageArrayConfig{
+				"array1": {
+					Labels: map[string]interface{}{"regionlabel": "R1", "zonelabel": "Z1"},
+				},
+			},
+			expectedZoneInfo: map[string]string{"regionlabel": "R1", "zonelabel": "Z1"},
+			wantErr:          false,
+			wantResp:         true,
+		},
 	}
 	// Run the tests
 	for _, tc := range testCases {
@@ -1919,6 +1956,7 @@ func TestNodeGetInfo(t *testing.T) {
 					PortGroups:        tc.portGroups,
 					MaxVolumesPerNode: tc.maxVolumesPerNode,
 					IsVsphereEnabled:  tc.isVsphereEnabled,
+					StorageArrays:     tc.storageArrays,
 				},
 				arrayTransportProtocolMap: tc.arrayTransportProtocolMap,
 				nvmetcpClient:             tc.nvmeTCPClient,
@@ -1929,10 +1967,17 @@ func TestNodeGetInfo(t *testing.T) {
 			}
 			tc.pmaxClient = tc.getClient()
 			// Call the function and check the results
-			_, err := s.NodeGetInfo(context.Background(), tc.csiNodeGetInfoRequest)
+			resp, err := s.NodeGetInfo(context.Background(), tc.csiNodeGetInfoRequest)
 			if tc.wantErr && err == nil {
 				t.Errorf("Expected error but got none")
 			}
+			if tc.wantResp {
+				topology := resp.AccessibleTopology.Segments
+				if topology["regionlabel"] != tc.expectedZoneInfo["regionlabel"] ||
+					topology["zonelabel"] != tc.expectedZoneInfo["zonelabel"] {
+					t.Errorf("Expected zone info to be %v and %v but got %v and %v", tc.expectedZoneInfo["regionlabel"], tc.expectedZoneInfo["zonelabel"], topology["regionlabel"], topology["zonelabel"])
+				}
+			}
 		})
 	}
 }
diff --git a/service/service.go b/service/service.go
index 5abf9c9e..068d7aa9 100644
--- a/service/service.go
+++ b/service/service.go
@@ -1056,3 +1056,35 @@ func setArrayConfigEnvs(ctx context.Context) error {
 
 	return nil
 }
+
+func (s *service) filterArraysByZoneInfo(storageArrays map[string]StorageArrayConfig) []string {
+	results := make([]string, 0)
+
+	for arrayID, arrayConfig := range storageArrays {
+		arrayLabels := arrayConfig.Labels
+		nodeLabels, err := s.k8sUtils.GetNodeLabels(s.opts.NodeFullName)
+		if err != nil {
+			log.Infof("failed to get Node Labels with error '%s'", err.Error())
+		}
+
+		keepArray := true
+		if len(arrayLabels) != 0 {
+			for arrayLabelKey, arrayLabelVal := range arrayLabels {
+				if nodeLabelVal, ok := nodeLabels[arrayLabelKey]; !ok || nodeLabelVal != arrayLabelVal.(string) {
+					keepArray = false
+					break
+				}
+			}
+		}
+
+		if keepArray {
+			results = append(results, arrayID)
+		}
+	}
+
+	if len(results) == 0 {
+		log.Warn("No arrays match the node labels")
+	}
+
+	return results
+}
diff --git a/service/service_test.go b/service/service_test.go
index 95459256..bc28b1a5 100644
--- a/service/service_test.go
+++ b/service/service_test.go
@@ -20,11 +20,19 @@ import (
 	"net/http"
 	_ "net/http/pprof" // #nosec G108
 	"os"
+	"reflect"
 	"testing"
 	"time"
 
 	"github.com/cucumber/godog"
+	"github.com/dell/csi-powermax/v2/k8smock"
+	"github.com/dell/csi-powermax/v2/pkg/symmetrix"
+	"github.com/dell/csi-powermax/v2/pkg/symmetrix/mocks"
+	pmax "github.com/dell/gopowermax/v2"
+	gomock "github.com/golang/mock/gomock"
+	log "github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
+	gmock "go.uber.org/mock/gomock"
 )
 
 var (
@@ -158,3 +166,97 @@ func TestGetStorageArrays(t *testing.T) {
 		})
 	}
 }
+
+func TestFilterArraysByZoneInfo(t *testing.T) {
+	testCases := []struct {
+		name           string
+		pmaxClient     pmax.Pmax
+		secretParams   *viper.Viper
+		getClient      func() *mocks.MockPmaxClient
+		opts           Opts
+		expectedArrays []string
+		storageArrays  map[string]StorageArrayConfig
+		initFunc       func() *k8smock.MockUtilsInterface
+	}{
+		{
+			name: "Storage array and node labels match",
+			getClient: func() *mocks.MockPmaxClient {
+				c := mocks.NewMockPmaxClient(gmock.NewController(t))
+				c.EXPECT().WithSymmetrixID("array1").AnyTimes().Return(c)
+				symmetrix.Initialize([]string{"array1"}, c)
+				return c
+			},
+			initFunc: func() *k8smock.MockUtilsInterface {
+				mockUtilsInterface := k8smock.NewMockUtilsInterface(gomock.NewController(t))
+				mockUtilsInterface.EXPECT().GetNodeLabels("node1").Return(map[string]string{"topology.kubernetes.io/zone": "Z1"}, nil)
+				return mockUtilsInterface
+			},
+			storageArrays: map[string]StorageArrayConfig{
+				"array1": {
+					Labels: map[string]interface{}{"topology.kubernetes.io/zone": "Z1"},
+				},
+			},
+			expectedArrays: []string{"array1"},
+		},
+		{
+			name: "Storage array and node labels do not match",
+			getClient: func() *mocks.MockPmaxClient {
+				c := mocks.NewMockPmaxClient(gmock.NewController(t))
+				c.EXPECT().WithSymmetrixID("array1").AnyTimes().Return(c)
+				symmetrix.Initialize([]string{"array1"}, c)
+				return c
+			},
+			initFunc: func() *k8smock.MockUtilsInterface {
+				mockUtilsInterface := k8smock.NewMockUtilsInterface(gomock.NewController(t))
+				mockUtilsInterface.EXPECT().GetNodeLabels("node1").Return(map[string]string{"topology.kubernetes.io/region": "R1"}, nil)
+				return mockUtilsInterface
+			},
+			storageArrays: map[string]StorageArrayConfig{
+				"array1": {
+					Labels: map[string]interface{}{"differentlabel1": "differentvalue1"},
+				},
+			},
+			expectedArrays: []string{},
+		},
+		{
+			name: "Storage array and node do not have zone info",
+			getClient: func() *mocks.MockPmaxClient {
+				c := mocks.NewMockPmaxClient(gmock.NewController(t))
+				c.EXPECT().WithSymmetrixID("array1").AnyTimes().Return(c)
+				symmetrix.Initialize([]string{"array1"}, c)
+				return c
+			},
+			initFunc: func() *k8smock.MockUtilsInterface {
+				mockUtilsInterface := k8smock.NewMockUtilsInterface(gomock.NewController(t))
+				mockUtilsInterface.EXPECT().GetNodeLabels("node1").Return(map[string]string{}, nil)
+				return mockUtilsInterface
+			},
+			storageArrays: map[string]StorageArrayConfig{
+				"array1": {
+					Labels: map[string]interface{}{},
+				},
+			},
+			expectedArrays: []string{"array1"},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			// Create a new service instance for testing
+			s := &service{
+				opts: Opts{
+					NodeName:     "node1",
+					NodeFullName: "node1",
+				},
+				k8sUtils: tc.initFunc(),
+			}
+			tc.pmaxClient = tc.getClient()
+			// Call the function and check the results
+			filteredArrays := s.filterArraysByZoneInfo(tc.storageArrays)
+			log.Infof("Filtered arrays: %v", filteredArrays)
+			if !reflect.DeepEqual(filteredArrays, tc.expectedArrays) {
+				t.Errorf("Expected %v, got %v", tc.expectedArrays, filteredArrays)
+			}
+		})
+	}
+}


Reviews:
user: login: donatwork
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-17T11:17:00Z
user: login: EvgenyUglov
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T17:44:34Z
user: login: donatwork
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T17:49:48Z
user: login: EvgenyUglov
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T17:52:17Z
user: login: anathoodell
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T17:57:54Z
user: login: EvgenyUglov
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T18:02:46Z
user: login: anathoodell
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T18:12:20Z
user: login: donatwork
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T19:54:17Z

Review comments:
diff_hunk: @@ -1056,3 +1056,48 @@ func setArrayConfigEnvs(ctx context.Context) error {
 
 	return nil
 }
+
+func (s *service) filterArraysByZoneInfo(storageArrays map[string]StorageArrayConfig) ([]string, error) {
+	results := make([]string, 0)
+
+	for arrayID, arrayConfig := range storageArrays {
+		arrayLabels := arrayConfig.Labels
+		nodeLabels, err := s.k8sUtils.GetNodeLabels(s.opts.NodeFullName)
+		if err != nil {
+			log.Infof("failed to get Node Labels with error '%s'", err.Error())
+		}
+
+		keepArray := true
+		if hasZoneInfo(nodeLabels) || len(arrayLabels) != 0 {
+			for arrayLabelKey := range arrayLabels {
+				if _, ok := nodeLabels[arrayLabelKey]; !ok {
+					keepArray = false
+					break
+				}
+			}
+		}
+
+		if keepArray {
+			results = append(results, arrayID)
+		}
+	}
+
+	if len(results) == 0 {
+		log.Warn("No arrays match the node labels")
+	}
+
+	return results, nil
+}
+
+// hasZoneInfo checks if the given labels contain the zone information.
+func hasZoneInfo(labels map[string]string) bool {
+	hasZoneInfo := false
+	for labelKey := range labels {
+		if strings.EqualFold(labelKey, "topology.kubernetes.io/region") || strings.EqualFold(labelKey, "topology.kubernetes.io/zone") {
path: service/service.go
user: login: donatwork
body: This is not correct. The keys are user defined so may not necessarily be what you are checking for. I do think that the topology.kubernetes.io prefix should be standardized for topology setting in order to separate from other labels. However that needs to be reviewed and agreed upon so better to check for labels.
created_at: 2025-03-17T11:17:00Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1056,3 +1056,48 @@ func setArrayConfigEnvs(ctx context.Context) error {
 
 	return nil
 }
+
+func (s *service) filterArraysByZoneInfo(storageArrays map[string]StorageArrayConfig) ([]string, error) {
+	results := make([]string, 0)
+
+	for arrayID, arrayConfig := range storageArrays {
+		arrayLabels := arrayConfig.Labels
+		nodeLabels, err := s.k8sUtils.GetNodeLabels(s.opts.NodeFullName)
+		if err != nil {
+			log.Infof("failed to get Node Labels with error '%s'", err.Error())
+		}
+
+		keepArray := true
+		if hasZoneInfo(nodeLabels) || len(arrayLabels) != 0 {
+			for arrayLabelKey := range arrayLabels {
+				if _, ok := nodeLabels[arrayLabelKey]; !ok {
+					keepArray = false
+					break
+				}
+			}
+		}
+
+		if keepArray {
+			results = append(results, arrayID)
+		}
+	}
+
+	if len(results) == 0 {
+		log.Warn("No arrays match the node labels")
+	}
+
+	return results, nil
+}
+
+// hasZoneInfo checks if the given labels contain the zone information.
+func hasZoneInfo(labels map[string]string) bool {
+	hasZoneInfo := false
+	for labelKey := range labels {
+		if strings.EqualFold(labelKey, "topology.kubernetes.io/region") || strings.EqualFold(labelKey, "topology.kubernetes.io/zone") {
path: service/service.go
user: login: EvgenyUglov
body: Agreed. This part is no longer needed after discussion the labels naming with Trevor. Removed in newer commits
created_at: 2025-03-18T17:44:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -96,6 +96,8 @@ var symToMaskingViewTargets sync.Map
 // Map to store if sym has fc connectivity or not
 var isSymConnFC sync.Map
 
+var symmetrixIDs []string
path: service/node.go
user: login: donatwork
body: Do we really need another global to store the IDs?
created_at: 2025-03-18T17:49:48Z
author_association: CONTRIBUTOR
diff_hunk: @@ -96,6 +96,8 @@ var symToMaskingViewTargets sync.Map
 // Map to store if sym has fc connectivity or not
 var isSymConnFC sync.Map
 
+var symmetrixIDs []string
path: service/node.go
user: login: EvgenyUglov
body: I didn't want to call filter function twice. The first time we call it in `nodeStartup`, then we need to go through the same list of arrays in `NodeGetInfo`. We either call filter function or use already filtered arrays. I thought storing them after the node startup would be better
created_at: 2025-03-18T17:52:17Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1056,3 +1056,35 @@ func setArrayConfigEnvs(ctx context.Context) error {
 
 	return nil
 }
+
+func (s *service) filterArraysByZoneInfo(storageArrays map[string]StorageArrayConfig) []string {
+	results := make([]string, 0)
+
+	for arrayID, arrayConfig := range storageArrays {
+		arrayLabels := arrayConfig.Labels
+		nodeLabels, err := s.k8sUtils.GetNodeLabels(s.opts.NodeFullName)
+		if err != nil {
+			log.Infof("failed to get Node Labels with error '%s'", err.Error())
path: service/service.go
user: login: anathoodell
body: Should this be an Error log?
created_at: 2025-03-18T17:57:54Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1056,3 +1056,35 @@ func setArrayConfigEnvs(ctx context.Context) error {
 
 	return nil
 }
+
+func (s *service) filterArraysByZoneInfo(storageArrays map[string]StorageArrayConfig) []string {
+	results := make([]string, 0)
+
+	for arrayID, arrayConfig := range storageArrays {
+		arrayLabels := arrayConfig.Labels
+		nodeLabels, err := s.k8sUtils.GetNodeLabels(s.opts.NodeFullName)
+		if err != nil {
+			log.Infof("failed to get Node Labels with error '%s'", err.Error())
path: service/service.go
user: login: EvgenyUglov
body: Good question. I followed the same way as it is implemented in NodeGetInfo. It is an Info log there. Do we want both places to show an error? We don't interrupt after this, so for me it looks more like an Info, but logically it is an error
created_at: 2025-03-18T18:02:46Z
author_association: CONTRIBUTOR

PR comments:
user: login: github-actions[bot]
created_at: 2025-03-18T17:46:23Z
body: ### Merging this branch will **not change** overall coverage

| Impacted Packages | Coverage Δ | :robot: |
|-------------------|------------|---------|
| github.com/dell/csi-powermax/service | 0.00% (ø) |  |

---

<details>

<summary>Coverage by file</summary>

### Changed files (no unit tests)

| Changed File | Coverage Δ | Total | Covered | Missed | :robot: |
|--------------|------------|-------|---------|--------|---------|
| github.com/dell/csi-powermax/service/node.go | 0.00% (ø) | 0 | 0 | 0 |  |
| github.com/dell/csi-powermax/service/service.go | 0.00% (ø) | 0 | 0 | 0 |  |

_Please note that the "Total", "Covered", and "Missed" counts above refer to ***code statements*** instead of lines of code. The value in brackets refers to the test coverage of that file in the old version of the code._

### Changed unit test files

- github.com/dell/csi-powermax/service/node_test.go
- github.com/dell/csi-powermax/service/service_test.go

</details>

author_association: CONTRIBUTOR