func (s *service) getCSIVolume(vol *siotypes.Volume, systemID string) *csi.Volume {
	storagePoolName := s.getStoragePoolNameFromID(systemID, vol.StoragePoolID)
	attributes := map[string]string{
		"Name":            vol.Name,
		"StoragePoolID":   vol.StoragePoolID,
		"StoragePoolName": storagePoolName,
		"StorageSystem":   systemID,
		"CreationTime":    time.Unix(int64(vol.CreationTime), 0).String(),
	}
	dash := "-"
	vi := &csi.Volume{
		VolumeId:      systemID + dash + vol.ID,
		CapacityBytes: int64(vol.SizeInKb * bytesInKiB),
		VolumeContext: attributes,
	}
	return vi
}
getCSIVolume converts the given siotypes.Volume to a CSI volume
package service
import (
	"context"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
	"github.com/container-storage-interface/spec/lib/go/csi"
	"github.com/dell/csi-vxflexos/v2/core"
	"github.com/dell/csi-vxflexos/v2/k8sutils"
	"github.com/dell/dell-csi-extensions/podmon"
	volumeGroupSnapshot "github.com/dell/dell-csi-extensions/volumeGroupSnapshot"
	"github.com/dell/gocsi"
	csictx "github.com/dell/gocsi/context"
	sio "github.com/dell/goscaleio"
	siotypes "github.com/dell/goscaleio/types/v1"
	"github.com/fsnotify/fsnotify"
	"github.com/golang/protobuf/ptypes"
	"github.com/sirupsen/logrus"
	"github.com/spf13/viper"
	"k8s.io/client-go/kubernetes"
	"sigs.k8s.io/yaml"
	"google.golang.org/grpc"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
	Name = "csi-vxflexos.dellemc.com"
	KeyThickProvisioning = "thickprovisioning"
	thinProvisioned  = "ThinProvisioned"
	thickProvisioned = "ThickProvisioned"
	defaultPrivDir   = "/dev/disk/csi-vxflexos"
	SystemTopologySystemValue string = "csi-vxflexos.dellemc.com"
	DefaultLogLevel = logrus.DebugLevel
	ParamCSILogLevel = "CSI_LOG_LEVEL"
)
var ArrayConfigFile string
var DriverConfigParamsFile string
var KubeConfig string
var K8sClientset kubernetes.Interface
var Log = logrus.New()
type ArrayConnectionData struct {
	SystemID                  string `json:"systemID"`
	Username                  string `json:"username"`
	Password                  string `json:"password"`
	Endpoint                  string `json:"endpoint"`
	SkipCertificateValidation bool   `json:"skipCertificateValidation,omitempty"`
	Insecure                  bool   `json:"insecure,omitempty"`
	IsDefault                 bool   `json:"isDefault,omitempty"`
	AllSystemNames            string `json:"allSystemNames"`
}
var Manifest = map[string]string{
	"url":    "http://github.com/dell/csi-vxflexos",
	"semver": core.SemVer,
	"commit": core.CommitSha32,
	"formed": core.CommitTime.Format(time.RFC1123),
}
type Service interface {
	csi.ControllerServer
	csi.IdentityServer
	csi.NodeServer
	BeforeServe(context.Context, *gocsi.StoragePlugin, net.Listener) error
	RegisterAdditionalServers(server *grpc.Server)
	ProcessMapSecretChange() error
}
type Opts struct {
	arrays                     map[string]*ArrayConnectionData
	defaultSystemID            string  
	SdcGUID                    string
	Thick                      bool
	AutoProbe                  bool
	DisableCerts               bool    
	Lsmod                      string  
	drvCfgQueryMDM             string  
	EnableSnapshotCGDelete     bool    
	EnableListVolumesSnapshots bool    
	AllowRWOMultiPodAccess     bool    
}
type service struct {
	opts                Opts
	adminClients        map[string]*sio.Client
	systems             map[string]*sio.System
	mode                string
	volCache            []*siotypes.Volume
	volCacheRWL         sync.RWMutex
	volCacheSystemID    string  
	snapCache           []*siotypes.Volume
	snapCacheRWL        sync.RWMutex
	snapCacheSystemID   string  
	privDir             string
	storagePoolIDToName map[string]string
	statisticsCounter   int
	volumePrefixToSystems   map[string][]string
	connectedSystemNameToID map[string]string
}
func (s *service) ProcessMapSecretChange() error {
	vc := viper.New()
	vc.AutomaticEnv()
	Log.WithField("file", DriverConfigParamsFile).Info("driver configuration file ")
	vc.SetConfigFile(DriverConfigParamsFile)
	if err := vc.ReadInConfig(); err != nil {
		Log.WithError(err).Error("unable to read config file, using default values")
	}
	if err := s.updateDriverConfigParams(Log, vc); err != nil {
		return err
	}
	vc.WatchConfig()
	vc.OnConfigChange(func(e fsnotify.Event) {
		Log.WithField("file", DriverConfigParamsFile).Info("log configuration file changed")
		if err := s.updateDriverConfigParams(Log, vc); err != nil {
			Log.Warn(err)
		}
	})
	va := viper.New()
	va.SetConfigFile(ArrayConfigFile)
	va.SetConfigType("ini")
	Log.WithField("file", ArrayConfigFile).Info("array configuration file")
	va.WatchConfig()
	va.OnConfigChange(func(e fsnotify.Event) {
		Log.WithField("file", ArrayConfigFile).Info("array configuration file changed")
		var err error
		s.opts.arrays, err = getArrayConfig(context.Background())
		if err != nil {
			Log.WithError(err).Error("unable to reload multi array config file")
		}
		err = s.doProbe(context.Background())
		if err != nil {
			Log.WithError(err).Error("unable to probe array in multi array config")
		}
		if err = s.logCsiNodeTopologyKeys(); err != nil {
			Log.WithError(err).Error("unable to log csiNode topology keys")
		}
	})
	return nil
}
func (s *service) logCsiNodeTopologyKeys() error {
	if K8sClientset == nil {
		err := k8sutils.CreateKubeClientSet(KubeConfig)
		if err != nil {
			Log.WithError(err).Error("unable to create k8s clientset for query")
			return err
		}
		K8sClientset = k8sutils.Clientset
	}
	csiNodes, err := K8sClientset.StorageV1().CSINodes().List(context.TODO(), metav1.ListOptions{})
	node, err := s.NodeGetInfo(context.Background(), nil)
	fmt.Printf("debug get csinode info")
	if node != nil {
		Log.WithField("node info", node.NodeId).Info("NodeInfo ID")
		segMap := node.AccessibleTopology.Segments
		for key := range segMap {
			Log.WithField("node info key", key).Info("NodeInfo topologykeys")
		}
		if err == nil {
			for i, csiNode := range csiNodes.Items {
				if len(csiNode.Spec.Drivers) > 0 {
					csinodeID := csiNode.Spec.Drivers[i].NodeID
					csiNodeName := csiNode.Spec.Drivers[i].Name
					if csinodeID == node.NodeId && csiNodeName == Name {
						csinodeID := csiNode.Spec.Drivers[i].NodeID
						Log.WithField("csinode", csiNode.Name).Info("csiNode name")
						Log.WithField("csinode ID", csinodeID).Info("csiNode id")
						tkeys := csiNode.Spec.Drivers[i].TopologyKeys
						if tkeys != nil {
							Log.WithField("csinode topologykeys", len(tkeys)).Info("count")
							needMap := make(map[string]string)
							for key := range segMap {
								for _, tkey := range tkeys {
									if tkey != key {
										needMap[key] = "missing"
									} else {
										Log.WithField("csinode topologykeys", "ok").Info("found")
									}
								}
							}
							for akey := range needMap {
								Log.WithField("csinode missing topology key", akey).Info("node key")
							}
						}
					}
				}
			}
		} else {
			Log.WithError(err).Error("unable to list csiNodes in cluster")
		}
	}
	return nil
}
func New() Service {
	return &service{
		storagePoolIDToName:     map[string]string{},
		connectedSystemNameToID: map[string]string{},
		volumePrefixToSystems:   map[string][]string{},
	}
}
func (s *service) updateDriverConfigParams(logger *logrus.Logger, v *viper.Viper) error {
	logFormat := v.GetString("CSI_LOG_FORMAT")
	logFormat = strings.ToLower(logFormat)
	logger.WithField("format", logFormat).Info("Read CSI_LOG_FORMAT from log configuration file")
	if strings.EqualFold(logFormat, "json") {
		logger.SetFormatter(&logrus.JSONFormatter{})
	} else {
		if logFormat != "text" {
			logger.WithField("format", logFormat).Info("CSI_LOG_FORMAT value not recognized, setting to text")
		}
		logger.SetFormatter(&logrus.TextFormatter{})
	}
	level := DefaultLogLevel
	if v.IsSet(ParamCSILogLevel) {
		logLevel := v.GetString(ParamCSILogLevel)
		if logLevel != "" {
			logLevel = strings.ToLower(logLevel)
			logger.WithField("level", logLevel).Info("Read CSI_LOG_LEVEL from log configuration file")
			var err error
			level, err = logrus.ParseLevel(logLevel)
			if err != nil {
				Log.WithError(err).Errorf("CSI_LOG_LEVEL %s value not recognized, setting to debug error: %s ", logLevel, err.Error())
				logger.SetLevel(DefaultLogLevel)
				return fmt.Errorf("input log level %q is not valid", logLevel)
			}
		}
	}
	logger.SetLevel(level)
	_ = os.Setenv(gocsi.EnvVarLogLevel, level.String())
	return nil
}
func (s *service) BeforeServe(
	ctx context.Context, sp *gocsi.StoragePlugin, lis net.Listener) error {
	defer func() {
		fields := map[string]interface{}{
			"sdcGUID":                s.opts.SdcGUID,
			"thickprovision":         s.opts.Thick,
			"privatedir":             s.privDir,
			"autoprobe":              s.opts.AutoProbe,
			"mode":                   s.mode,
			"allowRWOMultiPodAccess": s.opts.AllowRWOMultiPodAccess,
		}
		Log.WithFields(fields).Infof("configured %s", Name)
	}()
	s.mode = csictx.Getenv(ctx, gocsi.EnvVarMode)
	opts := Opts{}
	var err error
	opts.arrays, err = getArrayConfig(ctx)
	if err != nil {
		Log.Warnf("unable to get arrays from config: %s", err.Error())
		return err
	}
	if err = s.ProcessMapSecretChange(); err != nil {
		Log.Warnf("unable to configure dynamic configMap secret change detection : %s", err.Error())
		return err
	}
	if guid, ok := csictx.LookupEnv(ctx, EnvSDCGUID); ok {
		opts.SdcGUID = guid
	}
	if pd, ok := csictx.LookupEnv(ctx, "X_CSI_PRIVATE_MOUNT_DIR"); ok {
		s.privDir = pd
	}
	if snapshotCGDelete, ok := csictx.LookupEnv(ctx, "X_CSI_VXFLEXOS_ENABLESNAPSHOTCGDELETE"); ok {
		if snapshotCGDelete == "true" {
			opts.EnableSnapshotCGDelete = true
		}
	}
	if listVolumesSnapshots, ok := csictx.LookupEnv(ctx, "X_CSI_VXFLEXOS_ENABLELISTVOLUMESNAPSHOTS"); ok {
		if listVolumesSnapshots == "true" {
			opts.EnableListVolumesSnapshots = true
		}
	}
	if allowRWOMultiPodAccess, ok := csictx.LookupEnv(ctx, EnvAllowRWOMultiPodAccess); ok {
		if allowRWOMultiPodAccess == "true" {
			opts.AllowRWOMultiPodAccess = true
			mountAllowRWOMultiPodAccess = true
		}
	}
	if s.privDir == "" {
		s.privDir = defaultPrivDir
	}
	if err = s.logCsiNodeTopologyKeys(); err != nil {
		Log.WithError(err).Error("unable to log csiNode topology keys")
	}
	pb := func(n string) bool {
		if v, ok := csictx.LookupEnv(ctx, n); ok {
			b, err := strconv.ParseBool(v)
			if err != nil {
				Log.WithField(n, v).Debug(
					"invalid boolean value. defaulting to false")
				return false
			}
			return b
		}
		return false
	}
	opts.Thick = pb(EnvThick)
	opts.AutoProbe = true
	s.opts = opts
	s.adminClients = make(map[string]*sio.Client)
	s.systems = make(map[string]*sio.System)
	if _, ok := csictx.LookupEnv(ctx, "X_CSI_VXFLEXOS_NO_PROBE_ON_START"); !ok {
		return s.doProbe(ctx)
	}
	return nil
}
func (s *service) doProbe(ctx context.Context) error {
	if !strings.EqualFold(s.mode, "node") {
		if err := s.systemProbeAll(ctx); err != nil {
			return err
		}
	}
	if !strings.EqualFold(s.mode, "controller") {
		if err := s.systemProbeAll(ctx); err != nil {
			return err
		}
		if err := s.nodeProbe(ctx); err != nil {
			return err
		}
	}
	return nil
}
func (s *service) RegisterAdditionalServers(server *grpc.Server) {
	Log.Info("Registering additional GRPC servers")
	podmon.RegisterPodmonServer(server, s)
	volumeGroupSnapshot.RegisterVolumeGroupSnapshotServer(server, s)
}
func (s *service) getVolProvisionType(params map[string]string) string {
	volType := thinProvisioned
	if s.opts.Thick {
		volType = thickProvisioned
	}
	if tp, ok := params[KeyThickProvisioning]; ok {
		tpb, err := strconv.ParseBool(tp)
		if err != nil {
			Log.Warnf("invalid boolean received %s=(%#v) in params",
				KeyThickProvisioning, tp)
		} else if tpb {
			volType = thickProvisioned
		} else {
			volType = thinProvisioned
		}
	}
	return volType
}
func (s *service) getVolByID(id string, systemID string) (*siotypes.Volume, error) {
	adminClient := s.adminClients[systemID]
	if adminClient == nil {
		return nil, fmt.Errorf("can't find adminClient by id %s", systemID)
	}
	vols, err := adminClient.GetVolume("", strings.TrimSpace(id), "", "", false)
	if err != nil {
		return nil, err
	}
	return vols[0], nil
}
func (s *service) getSDCID(sdcGUID string, systemID string) (string, error) {
	sdcGUID = strings.ToUpper(sdcGUID)
	if s.systems[systemID] == nil {
		return "", fmt.Errorf("getSDCID error systemID not found: %s", systemID)
	}
	id, err := s.systems[systemID].FindSdc("SdcGuid", sdcGUID)
	if err != nil {
		return "", fmt.Errorf("error finding SDC from GUID: %s, err: %s",
			sdcGUID, err.Error())
	}
	return id.Sdc.ID, nil
}
func (s *service) getStoragePoolID(name string, systemID string) (string, error) {
	pool, err := s.adminClients[systemID].FindStoragePool("", name, "")
	if err != nil {
		return "", err
	}
	return pool.ID, nil
}