Repository: kubecost/disk-autoscaler (language: go)

Issue:
[Feature] Audit mode
### Problem Statement

Users would like to see the action DAS will take to ensure it will work according to expectations and also to ensure the risks are known.

### Solution Description

Enable an audit mode in DAS so users can see, perhaps via log messages, the actions DAS would take without having it actually perform any changes.

### Alternatives

_No response_

### Additional Context

_No response_

### Troubleshooting

- [x] I have searched other issues in this repository and mine is not recorded.


Discussion / hints:
Would be really cool to see expected savings as part of this!

Fix patch:
diff --git a/README.md b/README.md
index c29a0c2..5ffbcaf 100644
--- a/README.md
+++ b/README.md
@@ -49,6 +49,16 @@ See [here](https://docs.kubecost.com/install-and-configure/install) for the full
 
 4. Disk Auto-Scaler will automatically scan the cluster for the annotations above and scale the disks accordingly. Disk Auto-Scaler will re-check every hour for changes in utilization and scale if needed.
 
+### Audit Mode
+
+By default, Disk Auto-Scaler runs in audit mode which allows you to assess the changes it would make rather than having it proceed with any such changes. In this mode, it will print the pertinent information about a potential resizing action in the logs along with the savings achieved as a result as shown in the log snippet below.
+
+```log
+2024-05-16T22:44:21Z INF Namespace: thomasn-nightly-dev, Deployment: thomasn-nightly-dev-cost-analyzer, PVC: thomasn-nightly-dev-cost-analyzer, PV: pvc-691a99a8-ad61-4432-b125-fd136e8de168, Target Utilization: 70%, current size is: 32Gi, recommended size is: 2Gi, and expected monthly savings is: $3.00
+```
+
+To disable the default audit mode, change the value of the `DAS_AUDIT_MODE` environment variable to `"false"` and Disk Auto-Scaler will perform the resizing operations automatically.
+
 For an example StorageClass resource which is supported by Disk Autoscaler, please see [here](aws/gp3-storageClass.yaml).
 
 ### Scaling Up
@@ -69,9 +79,9 @@ When scaling down, Disk Auto-Scaler decreases the size of a given PVC. To do thi
 * Only `WaitForFirstConsumer` is supported leveraging Kubernetes deploying PersistentVolume on the same node as the original PV backing the PVC.
 * Storage class with driver `ebs.csi.aws.com` only supports the `ReadWriteOnce` access mode.
 
-### Environment Variables
+## Environment Variables
 
-The following are the environment variables which may be passed to the disk auto-scaler container along with a description and an example value.
+The following are the environment variables which may be passed to the Disk Auto-Scaler container along with a description and an example value.
 
 | Env Var               | Description | Example(s) |
 | --------------------- | ----------- | ------- |
@@ -79,6 +89,9 @@ The following are the environment variables which may be passed to the disk auto
 | `DAS_KUBECONFIG`      | Path to the Kubeconfig to be used by the disk auto-scaler. | `/foo/bar` |
 | `DAS_LOG_LEVEL`       | Set the desired logging level of the disk auto-scaler. Defaults to `info` if not specified. | `debug` |
 | `DAS_EXCLUDE_NAMESPACES`| The namespaces are excluded from disk auto-scaling. It is recommended to include the kube-system namespace and the namespace where Kubecost is installed. This supports regular expressions. | `"kubecost,kube-*,openshift-*"`|
+| `DAS_AUDIT_MODE`| Read-only execution of the Disk Auto Scaler, which offers recommended Persistent Volume (PV) sizes for deployments using the Kubecost PV right-sizing API, along with a list of cost savings predicted by Kubecost.| `"true"`|
+
+## Annotations
 
 ### User-Configurable Annotations
 
diff --git a/cmd/diskautoscaler/main.go b/cmd/diskautoscaler/main.go
index 3ae9ab2..a240a7c 100644
--- a/cmd/diskautoscaler/main.go
+++ b/cmd/diskautoscaler/main.go
@@ -23,6 +23,8 @@ import (
 const (
 	envPrefix    = "das"
 	logLevelConf = "log_level"
+	dasBurst     = 10
+	dasQPS       = 20
 )
 
 func initK8sRest() (*rest.Config, error) {
@@ -64,6 +66,8 @@ func initK8sRest() (*rest.Config, error) {
 				Str("path", kubeconfigPath).
 				Msgf("Built K8s config from local config")
 		}
+		config.QPS = dasQPS
+		config.Burst = dasBurst
 
 		return config, nil
 	}
diff --git a/manifests/install.yaml b/manifests/install.yaml
index 5745c55..baeba22 100644
--- a/manifests/install.yaml
+++ b/manifests/install.yaml
@@ -68,6 +68,8 @@ spec:
               value: http://kubecost-cost-analyzer.kubecost:9090/model
             - name: DAS_EXCLUDE_NAMESPACES
               value: "kubecost,kube-*,openshift-*"
+            - name: DAS_AUDIT_MODE
+              value: "true"
           image: gcr.io/kubecost1/disk-autoscaler:latest
           imagePullPolicy: IfNotPresent
           name: disk-autoscaler
diff --git a/pkg/diskscaler/diskscaler.go b/pkg/diskscaler/diskscaler.go
index 39a2d05..0055511 100644
--- a/pkg/diskscaler/diskscaler.go
+++ b/pkg/diskscaler/diskscaler.go
@@ -45,6 +45,7 @@ type DiskScaler struct {
 	dynamicK8sClient *dynamic.DynamicClient
 	clusterID        string
 	kubecostsvc      *pvsizingrecommendation.KubecostService
+	auditMode        bool
 }
 
 type pvcDetails struct {
@@ -60,7 +61,12 @@ type pvcDetails struct {
 	isSkippedForDeletion bool
 }
 
-func NewDiskScaler(clientConfig *rest.Config, basicK8sClient kubernetes.Interface, dynamicK8sClient *dynamic.DynamicClient, clusterID string, kubecostsvc *pvsizingrecommendation.KubecostService) (*DiskScaler, error) {
+func NewDiskScaler(clientConfig *rest.Config,
+	basicK8sClient kubernetes.Interface,
+	dynamicK8sClient *dynamic.DynamicClient,
+	clusterID string,
+	kubecostsvc *pvsizingrecommendation.KubecostService,
+	auditMode bool) (*DiskScaler, error) {
 	if basicK8sClient == nil {
 		return nil, fmt.Errorf("must have a Kubernetes client")
 	}
@@ -75,6 +81,7 @@ func NewDiskScaler(clientConfig *rest.Config, basicK8sClient kubernetes.Interfac
 		dynamicK8sClient: dynamicK8sClient,
 		clusterID:        clusterID,
 		kubecostsvc:      kubecostsvc,
+		auditMode:        auditMode,
 	}, nil
 }
 
@@ -85,7 +92,10 @@ func (ds *DiskScaler) runDiskScalingWorkflow(ctx context.Context, namespace, dep
 		return fmt.Errorf("disk scaling failed : %w", err)
 	}
 
-	log.Debug().Msgf("ctx: %s, volume map is: %+v", ctx.Value(diskScalerRunContextKey), volMap)
+	// No further action is needed if audit mode is enabled
+	if ds.auditMode {
+		return nil
+	}
 
 	originalScale, err := ds.retryscaleDeployment(ctx, deployment, namespace, 0)
 	if err != nil {
@@ -280,10 +290,10 @@ func (ds *DiskScaler) updateDeploymentWithSmallerSizePV(ctx context.Context, dep
 
 // getKubecostRecommendationForPV is used to get the recommendation from
 // kubecost service for a particular pvName.
-func (ds *DiskScaler) getKubecostRecommendationForPV(ctx context.Context, pvName string, targetUtilization int, interval string) (resource.Quantity, error) {
+func (ds *DiskScaler) getKubecostRecommendationForPV(ctx context.Context, pvName string, targetUtilization int, interval string) (pvsizingrecommendation.RecommendationSizeWithSavings, error) {
 	recommendedStorageQuantity, err := ds.kubecostsvc.GetRecommendation(ctx, pvName, targetUtilization, interval)
 	if err != nil {
-		return resource.Quantity{}, fmt.Errorf("failed to get recommendation from kubecost err: %w", err)
+		return recommendedStorageQuantity, fmt.Errorf("failed to get recommendation from kubecost err: %w", err)
 	}
 	return recommendedStorageQuantity, nil
 }
@@ -357,26 +367,69 @@ func (ds *DiskScaler) getPVCMap(ctx context.Context, namespace string, deploymen
 	volumes := v1Dep.Spec.Template.Spec.Volumes
 	for _, vol := range volumes {
 		if vol.PersistentVolumeClaim == nil {
+			// The PVC name is required even when in audit mode so we continue and don't provide any recommendation or err logs
+			// this condition is typically encountered when we have ephemeral storage, i.e storage tied to pod lifecyle.
+			if ds.auditMode {
+				continue
+			}
 			log.Error().Msgf("ctx: %s, deployment %s contains non PV claim volume source", ctx.Value(diskScalerRunContextKey), deploymentName)
-			return volumeMap, fmt.Errorf("deployment %s contains non PV claim volume source", deploymentName)
+			return map[string]*pvcDetails{}, fmt.Errorf("deployment %s contains non PV claim volume source", deploymentName)
 		}
 		pvcName := vol.PersistentVolumeClaim.ClaimName
 		k8sPVCInfo, err := ds.getPVCInfo(ctx, namespace, pvcName)
 		if err != nil {
+			// The PVC information is required even when in audit mode so we continue and don't provide any recommendations or err logs
+			if ds.auditMode {
+				continue
+			}
 			return map[string]*pvcDetails{}, fmt.Errorf("failed to get volume map for pvc: %s with err: %w", pvcName, err)
 		}
 
-		newPVCName, err := ds.newPVCName(ctx, namespace, pvcName)
+		pvName := k8sPVCInfo.Spec.VolumeName
+
+		// Check to see if PV is not hostPath mounted. Kubecost doesn't provide recommendations for hostPath mounts at this time.
+		// i.e volume mounted on node itself rather than a Physical volume.
+		pvInfo, err := ds.getPVInfo(ctx, pvName)
 		if err != nil {
-			return map[string]*pvcDetails{}, fmt.Errorf("failed to create a new PVC Name: %w", err)
+			// checking pvInfo before asking for recommendation from kubecost is important. In case of audit mode error log is ignore
+			// for this pv if there's any failure.
+			if ds.auditMode {
+				continue
+			}
+			return map[string]*pvcDetails{}, fmt.Errorf("unable to get kubernetes pv information for pv %s with err:%w", pvName, err)
 		}
 
-		pvName := k8sPVCInfo.Spec.VolumeName
-		log.Debug().Msgf("ctx: %s, backing volume name is: %s for pvc: %s", ctx.Value(diskScalerRunContextKey), pvName, pvcName)
 		spec := k8sPVCInfo.Spec
 		storageCapacity := k8sPVCInfo.Status.Capacity[v1.ResourceStorage]
 		storageClassName := k8sPVCInfo.Spec.StorageClassName
 
+		isValid := ds.isPVValidForDiskScaling(pvInfo)
+		if !isValid {
+			if ds.auditMode {
+				// hostpath mounted pvs are ingnored by disk scaler at this time. In case of audit mode we dont provide any error
+				continue
+			}
+			return map[string]*pvcDetails{}, fmt.Errorf("pv %s is invalid for disk autoscaling", pvName)
+		}
+
+		log.Debug().Msgf("ctx: %s, backing volume name is: %s for pvc: %s", ctx.Value(diskScalerRunContextKey), pvName, pvcName)
+		recommendation, err := ds.getKubecostRecommendationForPV(ctx, pvName, intTargetUtilization, interval)
+		if err == nil {
+			log.Info().Msgf("Namespace: %s, Deployment: %s, PVC: %s, PV: %s, Target Utilization: %d%%, current size is: %s, recommended size is: %s, and expected monthly savings is: $%.2f", namespace, deploymentName, pvcName, pvName, intTargetUtilization, storageCapacity.String(), recommendation.RecommendedResourceSize.String(), recommendation.Savings)
+		}
+		if ds.auditMode {
+			continue
+		}
+		resizeTo := recommendation.RecommendedResourceSize
+		if err != nil {
+			return map[string]*pvcDetails{}, fmt.Errorf("unable to get recommendation from kubecost %w", err)
+		}
+
+		newPVCName, err := ds.newPVCName(ctx, namespace, pvcName)
+		if err != nil {
+			return map[string]*pvcDetails{}, fmt.Errorf("failed to create a new PVC Name: %w", err)
+		}
+
 		scClass, err := ds.getStorageClassInfo(ctx, k8sPVCInfo.GetName(), *storageClassName)
 		if err != nil {
 			return map[string]*pvcDetails{}, fmt.Errorf("failed to get storage class info: %w", err)
@@ -398,22 +451,6 @@ func (ds *DiskScaler) getPVCMap(ctx context.Context, namespace string, deploymen
 			return map[string]*pvcDetails{}, fmt.Errorf("cannot support volume binding mode %s for storage class %s", volumeBindingMode, *storageClassName)
 		}
 
-		// CHeck to see if pv is not hostpath mounted
-		pvInfo, err := ds.getPVInfo(ctx, pvName)
-		if err != nil {
-			return map[string]*pvcDetails{}, fmt.Errorf("unable to get kubernetes pv information for pv %s with err:%w", pvName, err)
-		}
-
-		isValid := ds.isPVValidForDiskScaling(pvInfo)
-		if !isValid {
-			return map[string]*pvcDetails{}, fmt.Errorf("pv %s is invalid for disk autoscaling", pvName)
-		}
-
-		resizeTo, err := ds.getKubecostRecommendationForPV(ctx, pvName, intTargetUtilization, interval)
-		if err != nil {
-			return map[string]*pvcDetails{}, fmt.Errorf("unable to get recommendation from kubecost %w", err)
-		}
-
 		pvcDetails := &pvcDetails{
 			currentSize:          storageCapacity,
 			storageClass:         *storageClassName,
diff --git a/pkg/diskscaler/error.go b/pkg/diskscaler/error.go
index ae6a90c..9721b5a 100644
--- a/pkg/diskscaler/error.go
+++ b/pkg/diskscaler/error.go
@@ -14,7 +14,7 @@ type DiskScalingAllFailedError struct {
 }
 
 func (e *DiskScalingAllFailedError) Error() string {
-	return fmt.Sprintf("failed to scale all the persistent volume claims in deployment %s belonging to namespace %s", e.namespace, e.deployment)
+	return fmt.Sprintf("failed to scale all the persistent volume claims in deployment %s belonging to namespace %s", e.deployment, e.namespace)
 }
 
 // Custom error to return to the service calling the
@@ -26,5 +26,5 @@ type DiskScalingPartialFailedError struct {
 }
 
 func (e *DiskScalingPartialFailedError) Error() string {
-	return fmt.Sprintf("failed to scale persistent volume claims %s in deployment %s belonging to namespace %s", strings.Join(e.pvc, ","), e.namespace, e.deployment)
+	return fmt.Sprintf("failed to scale persistent volume claims %s in deployment %s belonging to namespace %s", strings.Join(e.pvc, ","), e.deployment, e.namespace)
 }
diff --git a/pkg/diskscaler/service.go b/pkg/diskscaler/service.go
index 00d7290..1346ca7 100644
--- a/pkg/diskscaler/service.go
+++ b/pkg/diskscaler/service.go
@@ -58,12 +58,19 @@ type DiskScalerService struct {
 	ds                     *DiskScaler
 	resizeAll              bool
 	excludedNamespaceRegex *regexp.Regexp
+	auditMode              bool
 }
 
-func NewDiskScalerService(clientConfig *rest.Config, k8sClient kubernetes.Interface, dynamicK8sClient *dynamic.DynamicClient, resizeAll bool, kubecostSvc *pvsizingrecommendation.KubecostService, excludedNamespaces []string) (*DiskScalerService, error) {
+func NewDiskScalerService(clientConfig *rest.Config,
+	k8sClient kubernetes.Interface,
+	dynamicK8sClient *dynamic.DynamicClient,
+	resizeAll bool,
+	auditMode bool,
+	kubecostSvc *pvsizingrecommendation.KubecostService,
+	excludedNamespaces []string) (*DiskScalerService, error) {
 	// To-DO :fill it via kubecost API
 	clusterID := "localCluster"
-	ds, err := NewDiskScaler(clientConfig, k8sClient, dynamicK8sClient, clusterID, kubecostSvc)
+	ds, err := NewDiskScaler(clientConfig, k8sClient, dynamicK8sClient, clusterID, kubecostSvc, auditMode)
 	if err != nil {
 		return nil, fmt.Errorf("unable to create NewDiskScaler: %w", err)
 	}
@@ -79,6 +86,7 @@ func NewDiskScalerService(clientConfig *rest.Config, k8sClient kubernetes.Interf
 		ds:                     ds,
 		resizeAll:              resizeAll,
 		excludedNamespaceRegex: regex,
+		auditMode:              auditMode,
 	}
 	return dss, nil
 }
@@ -158,6 +166,11 @@ func (dss *DiskScalerService) run(diskAutoScalerRun string) (RunStatus, error) {
 
 	}
 	wg.Wait()
+
+	if dss.auditMode {
+		log.Info().Msgf("disk autoscaling audit run at : %s", diskAutoScalerRun)
+		return status, nil
+	}
 	log.Info().Msgf("disk autoscaling run at : %s had success: %d failed: %d", diskAutoScalerRun, status.SuccessRun, status.FailedRun)
 	if result != nil {
 		log.Error().Msgf("disk autoscaling run at : %s errors: %s", diskAutoScalerRun, result.Error())
@@ -188,13 +201,15 @@ func (dss *DiskScalerService) startAutomatedScaling() error {
 				continue
 			}
 			lastRunFailed = false
-
+			if dss.auditMode {
+				return
+			}
 			log.Debug().Msgf("status at %s :%+v, triggered the disk scaling", diskAutoScalerRun, status)
 			if status.NumEnabled == 0 {
 				log.Debug().Msgf("No workloads have autoscaling enabled at %s", diskAutoScalerRun)
 			}
 			if status.NumEligible == 0 {
-				log.Debug().Msgf("No workload with autoscaling enabled can be resized again yet at %s", diskAutoScalerRun)
+				log.Debug().Msgf("No workload with autoscaling eligible at %s", diskAutoScalerRun)
 			}
 		}
 	}()
@@ -214,6 +229,11 @@ func (dss *DiskScalerService) workloadIsEnabled(meta metav1.ObjectMeta) bool {
 		return false
 	}
 
+	// For audit mode we provide recommendation for all deployment workloads
+	if dss.auditMode {
+		return true
+	}
+
 	if dss.resizeAll {
 		if val := meta.Annotations[AnnotationExcluded]; val != "true" {
 			return true
@@ -245,6 +265,11 @@ func (dss *DiskScalerService) workloadIsEligible(meta metav1.ObjectMeta, current
 		return false
 	}
 
+	// For audit mode we provide recommendation for all deployment workloads
+	if dss.auditMode {
+		return true
+	}
+
 	currentTime, err := time.Parse(time.RFC3339, currentRun)
 	if err != nil {
 		return false
diff --git a/pkg/diskscaler/setup.go b/pkg/diskscaler/setup.go
index 113e3cb..a7067bf 100644
--- a/pkg/diskscaler/setup.go
+++ b/pkg/diskscaler/setup.go
@@ -21,9 +21,11 @@ const (
 func Setup(mux *http.ServeMux, clientConfig *rest.Config, k8sClient kubernetes.Interface, dynamicK8sClient *dynamic.DynamicClient) error {
 	costModelPath, err := getDiskScalerCostModelPath()
 	if len(costModelPath) == 0 {
-		return err
+		return fmt.Errorf("setup of Disk Auto Scaler failed: %w", err)
 	}
 
+	auditMode := viper.GetBool("audit-mode")
+
 	resizeAll := viper.GetBool("resize-all")
 	if resizeAll {
 		log.Warn().Msg("disk auto scaler is experimental and at this time resize-all will be overridden to false")
@@ -45,7 +47,7 @@ func Setup(mux *http.ServeMux, clientConfig *rest.Config, k8sClient kubernetes.I
 	}
 
 	recommendationSvc := pvsizingrecommendation.NewKubecostService(costModelPath)
-	dss, err := NewDiskScalerService(clientConfig, k8sClient, dynamicK8sClient, resizeAll, recommendationSvc, excludedNamespaces)
+	dss, err := NewDiskScalerService(clientConfig, k8sClient, dynamicK8sClient, resizeAll, auditMode, recommendationSvc, excludedNamespaces)
 	if err != nil {
 		return fmt.Errorf("failed to create disk scaler service: %w", err)
 	}
@@ -54,6 +56,7 @@ func Setup(mux *http.ServeMux, clientConfig *rest.Config, k8sClient kubernetes.I
 	if err != nil {
 		return fmt.Errorf("unable to start disk scaler service loop: %w", err)
 	}
+
 	mux.HandleFunc("/diskAutoScaler/enable", dss.enableDiskAutoScaling)
 	mux.HandleFunc("/diskAutoScaler/exclude", dss.excludeDiskAutoScaling)
 	return nil
diff --git a/pkg/pvsizingrecommendation/query.go b/pkg/pvsizingrecommendation/query.go
index ac93cb5..43f5b40 100644
--- a/pkg/pvsizingrecommendation/query.go
+++ b/pkg/pvsizingrecommendation/query.go
@@ -10,32 +10,55 @@ import (
 	"path"
 	"regexp"
 	"strings"
+	"sync"
+	"time"
 
 	"github.com/rs/zerolog/log"
 	"k8s.io/apimachinery/pkg/api/resource"
 )
 
 const (
-	oneGiBytes            = 1024.0 * 1024.0 * 1024.0
-	supportedStorageRegex = "^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$"
-	equalityThreshold     = 0.00001
+	oneGiBytes                 = 1024.0 * 1024.0 * 1024.0
+	supportedStorageRegex      = "^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$"
+	equalityThreshold          = 0.00001
+	cacheRefresh               = 10 * time.Minute
+	overwriteTargetUtilization = 70
 )
 
 type KubecostService struct {
 	clusterInfoApiPath    string
 	recommendationApiPath string
+	cache                 map[string][]byte
+	mu                    sync.Mutex
 }
 
 func NewKubecostService(modelPath string) *KubecostService {
 	modelPath = strings.TrimSuffix(modelPath, "/")
-
+	cache := make(map[string][]byte)
 	svc := &KubecostService{
 		clusterInfoApiPath:    fmt.Sprintf("%s/%s", modelPath, path.Join("clusterInfo")),
 		recommendationApiPath: fmt.Sprintf("%s/%s", modelPath, path.Join("savings", "persistentVolumeSizing")),
+		cache:                 cache,
 	}
+	ticker := time.NewTicker(cacheRefresh)
+	go func() {
+		for {
+			for t := time.Now(); ; t = <-ticker.C {
+				log.Trace().Msgf("refreshing cache storing kubecost response at %s", t.String())
+				svc.refreshCache()
+			}
+		}
+	}()
 	return svc
 }
 
+// refreshCache refreshes the cache storing kubecost pv savings recommendation
+func (krs *KubecostService) refreshCache() {
+	krs.mu.Lock()
+	defer krs.mu.Unlock()
+	krs.cache = make(map[string][]byte, 0)
+}
+
 // CheckAvailable returns nil if the service is available to handle requests.
 func (krs *KubecostService) CheckAvailable(ctx context.Context) error {
 	request, err := http.NewRequestWithContext(ctx, "GET", krs.recommendationApiPath, nil)
@@ -67,19 +90,83 @@ type RecResponse struct {
 }
 
 type PVRecommendation struct {
-	VolumeName           string  `json:"volumeName"`
-	AverageUsageBytes    float64 `json:"averageUsageBytes"`
-	CurrentCapacityBytes float64 `json:"currentCapacityBytes"`
+	VolumeName               string  `json:"volumeName"`
+	AverageUsageBytes        float64 `json:"averageUsageBytes"`
+	CurrentCapacityBytes     float64 `json:"currentCapacityBytes"`
+	RecommendedCapacityBytes float64 `json:"recommendedCapacityBytes"`
+	SavingsMonthly           float64 `json:"savingsMonthly"`
+}
+
+type RecommendationSizeWithSavings struct {
+	RecommendedResourceSize resource.Quantity
+	Savings                 float64
 }
 
-func (krs *KubecostService) GetRecommendation(ctx context.Context, pvName string, targetUtilization int, interval string) (resource.Quantity, error) {
+func (krs *KubecostService) GetRecommendation(ctx context.Context, pvName string, targetUtilization int, interval string) (RecommendationSizeWithSavings, error) {
+	ohPercentage := computeOverHeadPercentForTargetUtilization(targetUtilization)
+	recommendation := RecommendationSizeWithSavings{}
+	respBody, err := krs.getFromCacheOrFetch(interval, ohPercentage)
+	if err != nil {
+		return recommendation, fmt.Errorf("failed to fetch pv recommendation from kubecost: %w", err)
+	}
+
+	var recResp RecResponse
+	err = json.Unmarshal(respBody, &recResp)
+	if err != nil {
+		return recommendation, fmt.Errorf("unable to parse the response from kubecost")
+	}
+	var recommendedBytes float64
+	var savingsMonthly float64
+	for _, pvRecommendation := range recResp.Recommendation {
+		// Make sure that the recommended bytes is 70% utilization of current average usage byte compute from kubecost
+		if pvRecommendation.VolumeName == pvName {
+			recommendedBytes = pvRecommendation.RecommendedCapacityBytes
+			savingsMonthly = pvRecommendation.SavingsMonthly
+		}
+	}
+
+	// Happens when the storage provisioned but kubecost hasnt recieved data
+	if almostEqual(recommendedBytes, 0.0) {
+		return recommendation, fmt.Errorf("unable to find accurate utilization from kubecost at this time")
+	}
+
+	// 1 Gi is the smallest storage size in AWS EBS
+	// ref : https://kubernetes.io/docs/tasks/administer-cluster/limit-storage-consumption/#limitrange-to-limit-requests-for-storage
+	if recommendedBytes < oneGiBytes {
+		log.Debug().Msgf("recommendedBytes from kubecost %f[in Bytes] is less than 1 Gi, defaulting to 1 Gi, since minimal storage provision is 1Gi", recommendedBytes)
+		recommendation.RecommendedResourceSize = resource.MustParse("1Gi")
+		recommendation.Savings = savingsMonthly
+		return recommendation, nil
+	}
+
+	recommendedSize, err := convertKubecostBytesToStorageRecommendation(recommendedBytes / oneGiBytes)
+	if err != nil {
+		return recommendation, fmt.Errorf("failed to convert kubecost bytes recommendation to storage request: %w", err)
+	}
+	recommendation.RecommendedResourceSize = recommendedSize
+	recommendation.Savings = savingsMonthly
+
+	return recommendation, nil
+}
+
+// getFromCacheOrFetch fetches from cache instead of repeated calling kubecost pvsizing recommendation end point
+func (krs *KubecostService) getFromCacheOrFetch(window string, overheadPercent string) ([]byte, error) {
+	krs.mu.Lock()
+	defer krs.mu.Unlock()
+
+	cacheKey := window
+	if content, ok := krs.cache[window]; ok {
+		return content, nil
+	}
+
 	queryParams := map[string]string{
-		"window": interval,
+		"window":          window,
+		"overheadPercent": overheadPercent,
 	}
 
 	req, err := http.NewRequest("GET", krs.recommendationApiPath, nil)
 	if err != nil {
-		return resource.Quantity{}, fmt.Errorf("making request: %s", err)
+		return []byte{}, fmt.Errorf("making request: %s", err)
 	}
 	q := req.URL.Query()
 	for k, v := range queryParams {
@@ -89,45 +176,24 @@ func (krs *KubecostService) GetRecommendation(ctx context.Context, pvName string
 	log.Debug().
 		Str("url", req.URL.String()).
 		Msgf("Request recommendation")
+
 	client := &http.Client{}
 	resp, err := client.Do(req)
 	if err != nil {
-		return resource.Quantity{}, fmt.Errorf("executing query: %w", err)
+		return []byte{}, fmt.Errorf("executing query: %w", err)
 	}
 	defer resp.Body.Close()
 	respBody, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return resource.Quantity{}, fmt.Errorf("reading response body: %s", err)
-	}
-	if resp.StatusCode != http.StatusOK {
-		return resource.Quantity{}, fmt.Errorf("non-OK response status (%d), body: %s", resp.StatusCode, string(respBody))
-	}
-	var recResp RecResponse
-	err = json.Unmarshal(respBody, &recResp)
-	if err != nil {
-		return resource.Quantity{}, fmt.Errorf("unable to parse the response from kubecost")
-	}
-	var recommendedBytes float64
-	for _, pvRecommendation := range recResp.Recommendation {
-		// Make sure that the recommended bytes is 70% utilization of current average usage byte compute from kubecost
-		if pvRecommendation.VolumeName == pvName {
-			recommendedBytes = pvRecommendation.AverageUsageBytes / (float64(targetUtilization) / 100.0)
-		}
-	}
-
-	// Happens when the storage provisioned but kubecost hasnt recieved data
-	if almostEqual(recommendedBytes, 0.0) {
-		return resource.Quantity{}, fmt.Errorf("unable to find accurate utilization from kubecost at this time")
+		return []byte{}, fmt.Errorf("reading response body: %s", err)
 	}
 
-	// 1 Gi is the smallest storage size in AWS EBS
-	// ref : https://kubernetes.io/docs/tasks/administer-cluster/limit-storage-consumption/#limitrange-to-limit-requests-for-storage
-	if recommendedBytes < oneGiBytes {
-		log.Warn().Msgf("recommendedBytes from kubecost %f[in Bytes] is less than 1 Gi, defaulting to 1 Gi, since minimal storage provision is 1Gi", recommendedBytes)
-		return resource.MustParse("1Gi"), nil
+	if resp.StatusCode != http.StatusOK {
+		return []byte{}, fmt.Errorf("non-OK response status (%d), body: %s", resp.StatusCode, string(respBody))
 	}
 
-	return convertKubecostBytesToStorageRecommendation(recommendedBytes / oneGiBytes)
+	krs.cache[cacheKey] = respBody
+	return respBody, nil
 }
 
 // convertKubecostBytesToStorageRecommendation converts the byte recommendation
@@ -153,6 +219,18 @@ func convertKubecostBytesToStorageRecommendation(bf float64) (resource.Quantity,
 	}
 	return resource.MustParse(recommendedStorageStr), nil
 }
+
+// Kubecost PV right sizing API takes overhead percentage, the 70% utilization i.e 70 in 100 is
+// 100 in 100+[30/70] equivalent. This function perform the conversion to parameter taken by kubecost.
+func computeOverHeadPercentForTargetUtilization(tu int) string {
+	// If user provides 0 in target utilization overwrite to 70% target utilization
+	if tu == 0 {
+		tu = overwriteTargetUtilization
+	}
+	overheadPercent := (float64(100-tu) / float64(tu) * 100)
+	return fmt.Sprintf("%.2f", overheadPercent)
+}
+
 func almostEqual(val1, val2 float64) bool {
 	return math.Abs(val1-val2) <= equalityThreshold
 }


Test patch:
diff --git a/pkg/pvsizingrecommendation/query_test.go b/pkg/pvsizingrecommendation/query_test.go
index 8ff2c78..26ecde5 100644
--- a/pkg/pvsizingrecommendation/query_test.go
+++ b/pkg/pvsizingrecommendation/query_test.go
@@ -102,3 +102,40 @@ func Test_convertKubecostBytesToStorageRecommendation(t *testing.T) {
 		}
 	}
 }
+
+func Test_computeOverHeadPercentForTargetUtilization(t *testing.T) {
+	type testCase struct {
+		name                    string
+		targetUtilization       int
+		expectedOverHeadpercent string
+	}
+
+	testCases := []testCase{
+		{
+			name:                    "when target utilization is 70, equivalent overhead percentage in kubecost",
+			targetUtilization:       70,
+			expectedOverHeadpercent: "42.86",
+		},
+		{
+			name:                    "when target utilization is 80, equivalent overhead percentage in kubecost",
+			targetUtilization:       80,
+			expectedOverHeadpercent: "25.00",
+		},
+		{
+			name:                    "when target utilization is 100, equivalent overhead percentage in kubecost",
+			targetUtilization:       100,
+			expectedOverHeadpercent: "0.00",
+		},
+		{
+			name:                    "when target utilization is 0, equivalent overhead percentage in kubecost",
+			targetUtilization:       0,
+			expectedOverHeadpercent: "42.86",
+		},
+	}
+	for _, tc := range testCases {
+		returnOverHeadpercent := computeOverHeadPercentForTargetUtilization(tc.targetUtilization)
+		if returnOverHeadpercent != tc.expectedOverHeadpercent {
+			t.Fatalf("test case %s: expectedOverHeadpercent %s but received %s", tc.name, tc.expectedOverHeadpercent, returnOverHeadpercent)
+		}
+	}
+}