Repository: mercari/tortoise
Pull request #374 (2024-02-29 14:00:43)

handle phase correctly for better mutating
<!--  
Thanks for sending a pull request!  
Please read the CLA carefully before submitting your contribution to Mercari. 
Under any circumstances, by submitting your contribution, you are deemed to accept and agree to be bound by the terms and conditions of the CLA.
https://www.mercari.com/cla/

Also, please read the contributor guide, which contains some general rules and suggestions:
https://github.com/mercari/tortoise/blob/main/docs/contributor-guide.md
-->

#### What this PR does / why we need it:

- handle phase better and document the flow
- mutate Pods and HPAs only when tortoise is ready.

#### Which issue(s) this PR fixes:

<!--
Before implementing any feature changes, please raise the issue and discuss what you propose with maintainers.
-->

Fixes #

#### Special notes for your reviewer:



Patch:
diff --git a/api/v1beta3/tortoise_types.go b/api/v1beta3/tortoise_types.go
index fefc654..f1daa41 100644
--- a/api/v1beta3/tortoise_types.go
+++ b/api/v1beta3/tortoise_types.go
@@ -217,20 +217,34 @@ type TortoisePhase string
 const (
 	// TortoisePhaseInitializing means tortoise is just created and initializing some components (HPA and VPA),
 	// and wait for those components to be ready.
+	// Possible flow: (none) → Initializing
 	TortoisePhaseInitializing TortoisePhase = "Initializing"
-	// TortoisePhaseGatheringData means tortoise is now gathering data and cannot make the accurate recommendations.
+	// TortoisePhaseGatheringData means tortoise is now gathering data for MinReplicas/MaxReplicas
+	// and cannot make the accurate recommendations.
+	// Possible flow: Initializing → GatheringData
 	TortoisePhaseGatheringData TortoisePhase = "GatheringData"
 	// TortoisePhaseWorking means tortoise is making the recommendations,
 	// and applying the recommendation values.
+	// Possible flow:
+	//  - GatheringData → Working (when all the data is ready)
+	//  - PartlyWorking → Working (when all the data is ready)
+	//  - BackToNormal → Working (minReplica goes back to the normal number)
 	TortoisePhaseWorking TortoisePhase = "Working"
 	// TortoisePhasePartlyWorking means tortoise has maxReplicas and minReplicas recommendations ready,
 	// and applying the recommendation values.
 	// But, some of the resources are not scaled due to some reasons. (probably still gathering data)
+	// Possible flow:
+	//  - GatheringData → PartlyWorking (only some of resources are ready)
+	//  - Working → PartlyWorking (autoscaling policy is changed)
 	TortoisePhasePartlyWorking TortoisePhase = "PartlyWorking"
 	// TortoisePhaseEmergency means tortoise is in the emergency mode.
+	//
+	// Possible flow:
+	//  - Working → Emergency
 	TortoisePhaseEmergency TortoisePhase = "Emergency"
 	// TortoisePhaseBackToNormal means tortoise was in the emergency mode, and now it's coming back to the normal operation.
 	// During TortoisePhaseBackToNormal, the number of replicas of workloads are gradually reduced to the usual value.
+	//  - Emergency → BackToNormal
 	TortoisePhaseBackToNormal TortoisePhase = "BackToNormal"
 )
 
diff --git a/pkg/hpa/service.go b/pkg/hpa/service.go
index 2448376..b60e16d 100644
--- a/pkg/hpa/service.go
+++ b/pkg/hpa/service.go
@@ -190,7 +190,7 @@ func (c *Service) syncHPAMetricsWithTortoiseAutoscalingPolicy(ctx context.Contex
 		}
 		currenthpa.Spec.Metrics = append(currenthpa.Spec.Metrics, m)
 		hpaEdited = true
-		tortoise = utils.ChangeTortoiseResourcePhase(tortoise, d.containerName, d.rn, now, v1beta3.ContainerResourcePhaseGatheringData)
+		tortoise = utils.ChangeTortoiseContainerResourcePhase(tortoise, d.containerName, d.rn, now, v1beta3.ContainerResourcePhaseGatheringData)
 	}
 
 	// remove metrics
@@ -336,6 +336,11 @@ func (s *Service) RecordHPATargetUtilizationUpdate(tortoise *autoscalingv1beta3.
 }
 
 func (c *Service) ChangeHPAFromTortoiseRecommendation(tortoise *autoscalingv1beta3.Tortoise, hpa *v2.HorizontalPodAutoscaler, now time.Time, recordMetrics bool) (*v2.HorizontalPodAutoscaler, *autoscalingv1beta3.Tortoise, error) {
+	if tortoise.Status.TortoisePhase == v1beta3.TortoisePhaseInitializing || tortoise.Status.TortoisePhase == "" || tortoise.Spec.UpdateMode == autoscalingv1beta3.UpdateModeOff {
+		// Tortoise is not ready, don't update HPA
+		return hpa, tortoise, nil
+	}
+
 	readyHorizontalResourceAndContainer := sets.New[resourceNameAndContainerName]()
 	for _, p := range tortoise.Status.AutoscalingPolicy {
 		for rn, ap := range p.Policy {
@@ -351,6 +356,10 @@ func (c *Service) ChangeHPAFromTortoiseRecommendation(tortoise *autoscalingv1bet
 			}
 		}
 	}
+	if readyHorizontalResourceAndContainer.Len() == 0 {
+		// all horizontal are not ready, don't update HPA
+		return hpa, tortoise, nil
+	}
 
 	var allowed bool
 	tortoise, allowed = c.UpdatingHPATargetUtilizationAllowed(tortoise, now)
@@ -595,8 +604,6 @@ func (c *Service) UpdateHPAFromTortoiseRecommendation(ctx context.Context, torto
 		}
 
 		hpa = c.excludeExternalMetric(ctx, hpa)
-		// If HPA has a deprecated annotation, we replace it with a new one.
-		// It allows us to remove a deprecated annotation completely later
 		retHPA = hpa
 		return c.c.Update(ctx, hpa)
 	}
diff --git a/pkg/pod/pod.go b/pkg/pod/pod.go
index 7c82fba..bece830 100644
--- a/pkg/pod/pod.go
+++ b/pkg/pod/pod.go
@@ -43,8 +43,10 @@ func New(
 }
 
 func (s *Service) ModifyPodResource(pod *v1.Pod, t *v1beta3.Tortoise) {
-	if t.Spec.UpdateMode == v1beta3.UpdateModeOff {
-		// DryRun, don't update Pod
+	if t.Spec.UpdateMode == v1beta3.UpdateModeOff ||
+		t.Status.TortoisePhase == "" ||
+		t.Status.TortoisePhase == v1beta3.TortoisePhaseInitializing ||
+		t.Status.TortoisePhase == v1beta3.TortoisePhaseGatheringData {
 		return
 	}
 
diff --git a/pkg/tortoise/tortoise.go b/pkg/tortoise/tortoise.go
index 5a765e4..a49189e 100644
--- a/pkg/tortoise/tortoise.go
+++ b/pkg/tortoise/tortoise.go
@@ -85,6 +85,17 @@ func (s *Service) ShouldReconcileTortoiseNow(tortoise *v1beta3.Tortoise, now tim
 	return false, lastTime.Add(s.tortoiseUpdateInterval).Sub(now)
 }
 
+func initializeContainerResourcePhase(tortoise *v1beta3.Tortoise, now time.Time) *v1beta3.Tortoise {
+	for _, c := range tortoise.Status.AutoscalingPolicy {
+		for rn := range c.Policy {
+			// set all to gathering data
+			utils.ChangeTortoiseContainerResourcePhase(tortoise, c.ContainerName, rn, now, v1beta3.ContainerResourcePhaseGatheringData)
+		}
+	}
+
+	return tortoise
+}
+
 func (s *Service) UpdateTortoisePhase(tortoise *v1beta3.Tortoise, now time.Time) *v1beta3.Tortoise {
 	switch tortoise.Status.TortoisePhase {
 	case "":
@@ -98,6 +109,7 @@ func (s *Service) UpdateTortoisePhase(tortoise *v1beta3.Tortoise, now time.Time)
 	case v1beta3.TortoisePhaseInitializing:
 		// change it to GatheringData anyway. Later the controller may change it back to initialize if VPA isn't ready.
 		tortoise.Status.TortoisePhase = v1beta3.TortoisePhaseGatheringData
+		tortoise = initializeContainerResourcePhase(tortoise, now)
 	case v1beta3.TortoisePhaseGatheringData:
 		tortoise = s.changeTortoisePhaseWorkingIfTortoiseFinishedGatheringData(tortoise, now)
 		if tortoise.Status.TortoisePhase == v1beta3.TortoisePhaseWorking {
@@ -106,6 +118,12 @@ func (s *Service) UpdateTortoisePhase(tortoise *v1beta3.Tortoise, now time.Time)
 		if tortoise.Status.TortoisePhase == v1beta3.TortoisePhasePartlyWorking {
 			s.recorder.Event(tortoise, corev1.EventTypeNormal, event.PartlyWorking, "Tortoise finishes gathering data in some metrics and it starts to work on autoscaling for those metrics. But some metrics are still gathering data")
 		}
+	case v1beta3.TortoisePhaseWorking:
+		// Some resource may be changed to gathering data, if autoscaling policy is changed.
+		tortoise = s.changeTortoisePhaseWorkingIfTortoiseFinishedGatheringData(tortoise, now)
+		if tortoise.Status.TortoisePhase == v1beta3.TortoisePhasePartlyWorking {
+			s.recorder.Event(tortoise, corev1.EventTypeNormal, event.PartlyWorking, "Tortoise was fully working, but needs to collect some date for some resources")
+		}
 	case v1beta3.TortoisePhasePartlyWorking:
 		tortoise = s.changeTortoisePhaseWorkingIfTortoiseFinishedGatheringData(tortoise, now)
 		if tortoise.Status.TortoisePhase == v1beta3.TortoisePhaseWorking {
@@ -147,19 +165,28 @@ func hasHorizontal(tortoise *v1beta3.Tortoise) bool {
 }
 
 func (s *Service) changeTortoisePhaseWorkingIfTortoiseFinishedGatheringData(tortoise *v1beta3.Tortoise, now time.Time) *v1beta3.Tortoise {
+	// If recommendation of maxReplicas or minReplicas is 0, it means horizontal autoscaling is not ready yet.
+	horizontalUnready := false
 	for _, r := range tortoise.Status.Recommendations.Horizontal.MinReplicas {
 		if r.Value == 0 {
-			return tortoise
+			horizontalUnready = true
 		}
 	}
 	for _, r := range tortoise.Status.Recommendations.Horizontal.MaxReplicas {
 		if r.Value == 0 {
-			return tortoise
+			horizontalUnready = true
 		}
 	}
 
-	// Until MaxReplicas/MinReplicas recommendation are ready,
-	// we never set TortoisePhase to Working or PartlyWorking.
+	if horizontalUnready {
+		for _, c := range tortoise.Status.AutoscalingPolicy {
+			for rn, p := range c.Policy {
+				if p == v1beta3.AutoscalingTypeHorizontal {
+					utils.ChangeTortoiseContainerResourcePhase(tortoise, c.ContainerName, rn, now, v1beta3.ContainerResourcePhaseGatheringData)
+				}
+			}
+		}
+	}
 
 	someAreGathering := false
 	someAreWorking := false
@@ -197,6 +224,9 @@ func (s *Service) changeTortoisePhaseWorkingIfTortoiseFinishedGatheringData(tort
 	} else if !someAreGathering && someAreWorking {
 		// All are working.
 		tortoise.Status.TortoisePhase = v1beta3.TortoisePhaseWorking
+	} else if someAreGathering && !someAreWorking {
+		// All are gathering data.
+		tortoise.Status.TortoisePhase = v1beta3.TortoisePhaseGatheringData
 	}
 
 	return tortoise
@@ -265,11 +295,11 @@ func (s *Service) initializeTortoise(tortoise *v1beta3.Tortoise, now time.Time)
 	for _, p := range tortoise.Status.AutoscalingPolicy {
 		for rn, policy := range p.Policy {
 			if policy == v1beta3.AutoscalingTypeOff {
-				utils.ChangeTortoiseResourcePhase(tortoise, p.ContainerName, rn, now, v1beta3.ContainerResourcePhaseOff)
+				utils.ChangeTortoiseContainerResourcePhase(tortoise, p.ContainerName, rn, now, v1beta3.ContainerResourcePhaseOff)
 				continue
 			}
 
-			utils.ChangeTortoiseResourcePhase(tortoise, p.ContainerName, rn, now, v1beta3.ContainerResourcePhaseGatheringData)
+			utils.ChangeTortoiseContainerResourcePhase(tortoise, p.ContainerName, rn, now, v1beta3.ContainerResourcePhaseGatheringData)
 		}
 	}
 
@@ -541,8 +571,8 @@ func UpdateTortoiseAutoscalingPolicyInStatus(tortoise *v1beta3.Tortoise, hpa *v2
 				corev1.ResourceMemory: v1beta3.AutoscalingTypeVertical,
 			},
 		})
-		tortoise = utils.ChangeTortoiseResourcePhase(tortoise, p, corev1.ResourceCPU, now, v1beta3.ContainerResourcePhaseGatheringData)
-		tortoise = utils.ChangeTortoiseResourcePhase(tortoise, p, corev1.ResourceMemory, now, v1beta3.ContainerResourcePhaseGatheringData)
+		tortoise = utils.ChangeTortoiseContainerResourcePhase(tortoise, p, corev1.ResourceCPU, now, v1beta3.ContainerResourcePhaseGatheringData)
+		tortoise = utils.ChangeTortoiseContainerResourcePhase(tortoise, p, corev1.ResourceMemory, now, v1beta3.ContainerResourcePhaseGatheringData)
 	}
 
 	// And, if the existing HPA is attached, we modify the policy for resources managed by the HPA to Horizontal.
@@ -561,26 +591,26 @@ func UpdateTortoiseAutoscalingPolicyInStatus(tortoise *v1beta3.Tortoise, hpa *v2
 				if tortoise.Status.AutoscalingPolicy[i].Policy[corev1.ResourceCPU] != v1beta3.AutoscalingTypeHorizontal {
 					// If HPA has the metrics for this container's CPU, we should set Horizontal.
 					tortoise.Status.AutoscalingPolicy[i].Policy[corev1.ResourceCPU] = v1beta3.AutoscalingTypeHorizontal
-					tortoise = utils.ChangeTortoiseResourcePhase(tortoise, p.ContainerName, corev1.ResourceCPU, now, v1beta3.ContainerResourcePhaseGatheringData)
+					tortoise = utils.ChangeTortoiseContainerResourcePhase(tortoise, p.ContainerName, corev1.ResourceCPU, now, v1beta3.ContainerResourcePhaseGatheringData)
 				}
 			} else {
 				if tortoise.Status.AutoscalingPolicy[i].Policy[corev1.ResourceCPU] != v1beta3.AutoscalingTypeVertical {
 					// Otherwise, set Vertical.
 					tortoise.Status.AutoscalingPolicy[i].Policy[corev1.ResourceCPU] = v1beta3.AutoscalingTypeVertical
-					tortoise = utils.ChangeTortoiseResourcePhase(tortoise, p.ContainerName, corev1.ResourceCPU, now, v1beta3.ContainerResourcePhaseGatheringData)
+					tortoise = utils.ChangeTortoiseContainerResourcePhase(tortoise, p.ContainerName, corev1.ResourceCPU, now, v1beta3.ContainerResourcePhaseGatheringData)
 				}
 			}
 			if hpaManagedResourceAndContainer.Has(resourceNameAndContainerName{corev1.ResourceMemory, tortoise.Status.AutoscalingPolicy[i].ContainerName}) {
 				if tortoise.Status.AutoscalingPolicy[i].Policy[corev1.ResourceMemory] != v1beta3.AutoscalingTypeHorizontal {
 					// If HPA has the metrics for this container's memory, we should set Horizontal.
 					tortoise.Status.AutoscalingPolicy[i].Policy[corev1.ResourceMemory] = v1beta3.AutoscalingTypeHorizontal
-					tortoise = utils.ChangeTortoiseResourcePhase(tortoise, p.ContainerName, corev1.ResourceMemory, now, v1beta3.ContainerResourcePhaseGatheringData)
+					tortoise = utils.ChangeTortoiseContainerResourcePhase(tortoise, p.ContainerName, corev1.ResourceMemory, now, v1beta3.ContainerResourcePhaseGatheringData)
 				}
 			} else {
 				if tortoise.Status.AutoscalingPolicy[i].Policy[corev1.ResourceMemory] != v1beta3.AutoscalingTypeVertical {
 					// Otherwise, set Vertical.
 					tortoise.Status.AutoscalingPolicy[i].Policy[corev1.ResourceMemory] = v1beta3.AutoscalingTypeVertical
-					tortoise = utils.ChangeTortoiseResourcePhase(tortoise, p.ContainerName, corev1.ResourceMemory, now, v1beta3.ContainerResourcePhaseGatheringData)
+					tortoise = utils.ChangeTortoiseContainerResourcePhase(tortoise, p.ContainerName, corev1.ResourceMemory, now, v1beta3.ContainerResourcePhaseGatheringData)
 				}
 			}
 		}
@@ -590,11 +620,11 @@ func UpdateTortoiseAutoscalingPolicyInStatus(tortoise *v1beta3.Tortoise, hpa *v2
 	for i, p := range tortoise.Status.AutoscalingPolicy {
 		if containerWOResourceRequest.Has(resourceNameAndContainerName{corev1.ResourceCPU, tortoise.Status.AutoscalingPolicy[i].ContainerName}) {
 			tortoise.Status.AutoscalingPolicy[i].Policy[corev1.ResourceCPU] = v1beta3.AutoscalingTypeOff
-			tortoise = utils.ChangeTortoiseResourcePhase(tortoise, p.ContainerName, corev1.ResourceCPU, now, v1beta3.ContainerResourcePhaseOff)
+			tortoise = utils.ChangeTortoiseContainerResourcePhase(tortoise, p.ContainerName, corev1.ResourceCPU, now, v1beta3.ContainerResourcePhaseOff)
 		}
 		if containerWOResourceRequest.Has(resourceNameAndContainerName{corev1.ResourceMemory, tortoise.Status.AutoscalingPolicy[i].ContainerName}) {
 			tortoise.Status.AutoscalingPolicy[i].Policy[corev1.ResourceMemory] = v1beta3.AutoscalingTypeOff
-			tortoise = utils.ChangeTortoiseResourcePhase(tortoise, p.ContainerName, corev1.ResourceMemory, now, v1beta3.ContainerResourcePhaseOff)
+			tortoise = utils.ChangeTortoiseContainerResourcePhase(tortoise, p.ContainerName, corev1.ResourceMemory, now, v1beta3.ContainerResourcePhaseOff)
 		}
 	}
 
diff --git a/pkg/utils/tortoise.go b/pkg/utils/tortoise.go
index fe14c9d..538560e 100644
--- a/pkg/utils/tortoise.go
+++ b/pkg/utils/tortoise.go
@@ -35,7 +35,7 @@ func ChangeTortoiseCondition(t *v1beta3.Tortoise, conditionType v1beta3.Tortoise
 	return t
 }
 
-func ChangeTortoiseResourcePhase(tortoise *v1beta3.Tortoise, containerName string, rn corev1.ResourceName, now time.Time, phase v1beta3.ContainerResourcePhase) *v1beta3.Tortoise {
+func ChangeTortoiseContainerResourcePhase(tortoise *v1beta3.Tortoise, containerName string, rn corev1.ResourceName, now time.Time, phase v1beta3.ContainerResourcePhase) *v1beta3.Tortoise {
 	found := false
 	for i, p := range tortoise.Status.ContainerResourcePhases {
 		if p.ContainerName == containerName {
diff --git a/pkg/vpa/service.go b/pkg/vpa/service.go
index 96ce98d..e208a16 100644
--- a/pkg/vpa/service.go
+++ b/pkg/vpa/service.go
@@ -18,6 +18,7 @@ import (
 
 	autoscalingv1beta3 "github.com/mercari/tortoise/api/v1beta3"
 	"github.com/mercari/tortoise/pkg/event"
+	"github.com/mercari/tortoise/pkg/utils"
 )
 
 type Service struct {
@@ -248,44 +249,25 @@ func isMonitorVPAReady(vpa *v1.VerticalPodAutoscaler, tortoise *autoscalingv1bet
 }
 
 func SetAllVerticalContainerResourcePhaseWorking(tortoise *autoscalingv1beta3.Tortoise, now time.Time) *autoscalingv1beta3.Tortoise {
-	verticalResourceAndContainer := sets.New[resourceNameAndContainerName]()
+	someChanged := false
 	for _, p := range tortoise.Status.AutoscalingPolicy {
 		for rn, ap := range p.Policy {
 			if ap == autoscalingv1beta3.AutoscalingTypeVertical {
-				verticalResourceAndContainer.Insert(resourceNameAndContainerName{rn, p.ContainerName})
+				someChanged = true
+				utils.ChangeTortoiseContainerResourcePhase(
+					tortoise,
+					p.ContainerName,
+					rn,
+					now,
+					autoscalingv1beta3.ContainerResourcePhaseWorking,
+				)
 			}
 		}
 	}
 
-	found := false
-	for _, d := range verticalResourceAndContainer.UnsortedList() {
-		for i, p := range tortoise.Status.ContainerResourcePhases {
-			if p.ContainerName == d.containerName {
-				tortoise.Status.ContainerResourcePhases[i].ResourcePhases[d.rn] = autoscalingv1beta3.ResourcePhase{
-					Phase:              autoscalingv1beta3.ContainerResourcePhaseWorking,
-					LastTransitionTime: metav1.NewTime(now),
-				}
-				found = true
-				break
-			}
-		}
-		if !found {
-			tortoise.Status.ContainerResourcePhases = append(tortoise.Status.ContainerResourcePhases, autoscalingv1beta3.ContainerResourcePhases{
-				ContainerName: d.containerName,
-				ResourcePhases: map[corev1.ResourceName]autoscalingv1beta3.ResourcePhase{
-					d.rn: {
-						Phase:              autoscalingv1beta3.ContainerResourcePhaseWorking,
-						LastTransitionTime: metav1.NewTime(now),
-					},
-				},
-			})
-		}
+	if someChanged && tortoise.Status.TortoisePhase == autoscalingv1beta3.TortoisePhaseGatheringData {
+		tortoise.Status.TortoisePhase = autoscalingv1beta3.TortoisePhasePartlyWorking
 	}
 
 	return tortoise
 }
-
-type resourceNameAndContainerName struct {
-	rn            corev1.ResourceName
-	containerName string
-}


Test patch:
diff --git a/controllers/testdata/mutable-autoscalingpolicy-no-hpa-and-add-horizontal/after/hpa.yaml b/controllers/testdata/mutable-autoscalingpolicy-no-hpa-and-add-horizontal/after/hpa.yaml
index c2ed1ad..2d9ce3a 100644
--- a/controllers/testdata/mutable-autoscalingpolicy-no-hpa-and-add-horizontal/after/hpa.yaml
+++ b/controllers/testdata/mutable-autoscalingpolicy-no-hpa-and-add-horizontal/after/hpa.yaml
@@ -16,7 +16,7 @@ spec:
         value: 100
       selectPolicy: Max
       stabilizationWindowSeconds: 0
-  maxReplicas: 20
+  maxReplicas: 10000
   metrics:
   - containerResource:
       container: app
@@ -25,7 +25,7 @@ spec:
         averageUtilization: 70
         type: Utilization
     type: ContainerResource
-  minReplicas: 5
+  minReplicas: 3
   scaleTargetRef:
     apiVersion: apps/v1
     kind: Deployment
diff --git a/controllers/testdata/mutable-autoscalingpolicy-no-hpa-and-add-horizontal/after/tortoise.yaml b/controllers/testdata/mutable-autoscalingpolicy-no-hpa-and-add-horizontal/after/tortoise.yaml
index b187df3..ece6ec1 100644
--- a/controllers/testdata/mutable-autoscalingpolicy-no-hpa-and-add-horizontal/after/tortoise.yaml
+++ b/controllers/testdata/mutable-autoscalingpolicy-no-hpa-and-add-horizontal/after/tortoise.yaml
@@ -68,12 +68,6 @@ status:
       reason: ScaledUpBasedOnPreferredMaxReplicas
       status: "False"
       type: ScaledUpBasedOnPreferredMaxReplicas
-    - lastTransitionTime: "2023-01-01T00:00:00Z"
-      lastUpdateTime: "2023-01-01T00:00:00Z"
-      message: HPA target utilization is updated
-      reason: HPATargetUtilizationUpdated
-      status: "True"
-      type: HPATargetUtilizationUpdated
     - lastTransitionTime: "2023-01-01T00:00:00Z"
       lastUpdateTime: "2023-01-01T00:00:00Z"
       message: The recommendation is provided
diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-gathering-data/after/deployment.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-gathering-data/after/deployment.yaml
index a8a97e3..1f8bac7 100644
--- a/controllers/testdata/reconcile-for-the-single-container-pod-gathering-data/after/deployment.yaml
+++ b/controllers/testdata/reconcile-for-the-single-container-pod-gathering-data/after/deployment.yaml
@@ -8,6 +8,8 @@ spec:
   strategy: {}
   template:
     metadata:
+      annotations:
+        kubectl.kubernetes.io/restartedAt: "2023-01-01T00:00:00Z"
       creationTimestamp: null
       labels:
         app: mercari
diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-gathering-data/after/tortoise.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-gathering-data/after/tortoise.yaml
index b9ee8b2..1e2d07c 100644
--- a/controllers/testdata/reconcile-for-the-single-container-pod-gathering-data/after/tortoise.yaml
+++ b/controllers/testdata/reconcile-for-the-single-container-pod-gathering-data/after/tortoise.yaml
@@ -36,7 +36,7 @@ status:
     - containerName: app
       resource:
         cpu: "4"
-        memory: 4Gi
+        memory: 3Gi
     tortoiseConditions:
     - lastTransitionTime: "2023-01-01T00:00:00Z"
       lastUpdateTime: "2023-01-01T00:00:00Z"
@@ -45,6 +45,11 @@ status:
       reason: ScaledUpBasedOnPreferredMaxReplicas
       status: "False"
       type: ScaledUpBasedOnPreferredMaxReplicas
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      message: The recommendation is provided
+      status: "True"
+      type: VerticalRecommendationUpdated
     - lastTransitionTime: "2023-01-01T00:00:00Z"
       lastUpdateTime: "2023-01-01T00:00:00Z"
       status: "False"
@@ -90,4 +95,4 @@ status:
     verticalPodAutoscalers:
     - name: tortoise-monitor-mercari
       role: Monitor
-  tortoisePhase: GatheringData
+  tortoisePhase: PartlyWorking
diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/hpa.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/hpa.yaml
index fb6604e..d4accc3 100644
--- a/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/hpa.yaml
+++ b/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/hpa.yaml
@@ -18,7 +18,7 @@ spec:
         value: 100
       selectPolicy: Max
       stabilizationWindowSeconds: 0
-  maxReplicas: 20
+  maxReplicas: 100
   metrics:
   - external:
       metric:
@@ -34,7 +34,7 @@ spec:
         averageUtilization: 50
         type: Utilization
     type: ContainerResource
-  minReplicas: 5
+  minReplicas: 1
   scaleTargetRef:
     apiVersion: apps/v1
     kind: Deployment
diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/tortoise.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/tortoise.yaml
index 86def1c..e4e3536 100644
--- a/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/tortoise.yaml
+++ b/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/tortoise.yaml
@@ -46,12 +46,6 @@ status:
       reason: ScaledUpBasedOnPreferredMaxReplicas
       status: "False"
       type: ScaledUpBasedOnPreferredMaxReplicas
-    - lastTransitionTime: "2023-01-01T00:00:00Z"
-      lastUpdateTime: "2023-01-01T00:00:00Z"
-      message: HPA target utilization is updated
-      reason: HPATargetUtilizationUpdated
-      status: "True"
-      type: HPATargetUtilizationUpdated
     - lastTransitionTime: "2023-01-01T00:00:00Z"
       lastUpdateTime: "2023-01-01T00:00:00Z"
       message: The recommendation is provided
@@ -102,4 +96,4 @@ status:
     verticalPodAutoscalers:
     - name: tortoise-monitor-mercari
       role: Monitor
-  tortoisePhase: Working
+  tortoisePhase: PartlyWorking
diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-partly-working/after/hpa.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-partly-working/after/hpa.yaml
index 0eb6b7e..2924abb 100644
--- a/controllers/testdata/reconcile-for-the-single-container-pod-partly-working/after/hpa.yaml
+++ b/controllers/testdata/reconcile-for-the-single-container-pod-partly-working/after/hpa.yaml
@@ -18,7 +18,7 @@ spec:
         value: 100
       selectPolicy: Max
       stabilizationWindowSeconds: 0
-  maxReplicas: 20
+  maxReplicas: 100
   metrics:
   - containerResource:
       container: app
@@ -27,7 +27,7 @@ spec:
         averageUtilization: 50
         type: Utilization
     type: ContainerResource
-  minReplicas: 5
+  minReplicas: 1
   scaleTargetRef:
     apiVersion: apps/v1
     kind: Deployment
diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-partly-working/after/tortoise.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-partly-working/after/tortoise.yaml
index 27449e1..1e2d07c 100644
--- a/controllers/testdata/reconcile-for-the-single-container-pod-partly-working/after/tortoise.yaml
+++ b/controllers/testdata/reconcile-for-the-single-container-pod-partly-working/after/tortoise.yaml
@@ -45,12 +45,6 @@ status:
       reason: ScaledUpBasedOnPreferredMaxReplicas
       status: "False"
       type: ScaledUpBasedOnPreferredMaxReplicas
-    - lastTransitionTime: "2023-01-01T00:00:00Z"
-      lastUpdateTime: "2023-01-01T00:00:00Z"
-      message: HPA target utilization is updated
-      reason: HPATargetUtilizationUpdated
-      status: "True"
-      type: HPATargetUtilizationUpdated
     - lastTransitionTime: "2023-01-01T00:00:00Z"
       lastUpdateTime: "2023-01-01T00:00:00Z"
       message: The recommendation is provided
diff --git a/pkg/hpa/service_test.go b/pkg/hpa/service_test.go
index ee83990..7a975ed 100644
--- a/pkg/hpa/service_test.go
+++ b/pkg/hpa/service_test.go
@@ -42,7 +42,11 @@ func TestClient_UpdateHPAFromTortoiseRecommendation(t *testing.T) {
 			args: args{
 				ctx: context.Background(),
 				tortoise: &autoscalingv1beta3.Tortoise{
+					Spec: autoscalingv1beta3.TortoiseSpec{
+						UpdateMode: autoscalingv1beta3.UpdateModeAuto,
+					},
 					Status: autoscalingv1beta3.TortoiseStatus{
+						TortoisePhase: autoscalingv1beta3.TortoisePhaseWorking,
 						AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{
 							{
 								ContainerName: "app",
@@ -212,7 +216,11 @@ func TestClient_UpdateHPAFromTortoiseRecommendation(t *testing.T) {
 				},
 			},
 			wantTortoise: &autoscalingv1beta3.Tortoise{
+				Spec: autoscalingv1beta3.TortoiseSpec{
+					UpdateMode: autoscalingv1beta3.UpdateModeAuto,
+				},
 				Status: autoscalingv1beta3.TortoiseStatus{
+					TortoisePhase: autoscalingv1beta3.TortoisePhaseWorking,
 					AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{
 						{
 							ContainerName: "app",
@@ -305,7 +313,11 @@ func TestClient_UpdateHPAFromTortoiseRecommendation(t *testing.T) {
 			args: args{
 				ctx: context.Background(),
 				tortoise: &autoscalingv1beta3.Tortoise{
+					Spec: autoscalingv1beta3.TortoiseSpec{
+						UpdateMode: autoscalingv1beta3.UpdateModeAuto,
+					},
 					Status: autoscalingv1beta3.TortoiseStatus{
+						TortoisePhase: autoscalingv1beta3.TortoisePhaseWorking,
 						AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{
 							{
 								ContainerName: "app",
@@ -485,7 +497,11 @@ func TestClient_UpdateHPAFromTortoiseRecommendation(t *testing.T) {
 				},
 			},
 			wantTortoise: &autoscalingv1beta3.Tortoise{
+				Spec: autoscalingv1beta3.TortoiseSpec{
+					UpdateMode: autoscalingv1beta3.UpdateModeAuto,
+				},
 				Status: autoscalingv1beta3.TortoiseStatus{
+					TortoisePhase: autoscalingv1beta3.TortoisePhaseWorking,
 					AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{
 						{
 							ContainerName: "app",
@@ -578,7 +594,11 @@ func TestClient_UpdateHPAFromTortoiseRecommendation(t *testing.T) {
 			args: args{
 				ctx: context.Background(),
 				tortoise: &autoscalingv1beta3.Tortoise{
+					Spec: autoscalingv1beta3.TortoiseSpec{
+						UpdateMode: autoscalingv1beta3.UpdateModeAuto,
+					},
 					Status: autoscalingv1beta3.TortoiseStatus{
+						TortoisePhase: autoscalingv1beta3.TortoisePhaseWorking,
 						AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{
 							{
 								ContainerName: "app",
@@ -738,7 +758,11 @@ func TestClient_UpdateHPAFromTortoiseRecommendation(t *testing.T) {
 				},
 			},
 			wantTortoise: &autoscalingv1beta3.Tortoise{
+				Spec: autoscalingv1beta3.TortoiseSpec{
+					UpdateMode: autoscalingv1beta3.UpdateModeAuto,
+				},
 				Status: autoscalingv1beta3.TortoiseStatus{
+					TortoisePhase: autoscalingv1beta3.TortoisePhaseWorking,
 					AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{
 						{
 							ContainerName: "app",
@@ -831,7 +855,11 @@ func TestClient_UpdateHPAFromTortoiseRecommendation(t *testing.T) {
 			args: args{
 				ctx: context.Background(),
 				tortoise: &autoscalingv1beta3.Tortoise{
+					Spec: autoscalingv1beta3.TortoiseSpec{
+						UpdateMode: autoscalingv1beta3.UpdateModeAuto,
+					},
 					Status: autoscalingv1beta3.TortoiseStatus{
+						TortoisePhase: autoscalingv1beta3.TortoisePhaseWorking,
 						AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{
 							{
 								ContainerName: "app",
@@ -991,7 +1019,11 @@ func TestClient_UpdateHPAFromTortoiseRecommendation(t *testing.T) {
 				},
 			},
 			wantTortoise: &autoscalingv1beta3.Tortoise{
+				Spec: autoscalingv1beta3.TortoiseSpec{
+					UpdateMode: autoscalingv1beta3.UpdateModeAuto,
+				},
 				Status: autoscalingv1beta3.TortoiseStatus{
+					TortoisePhase: autoscalingv1beta3.TortoisePhaseWorking,
 					AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{
 						{
 							ContainerName: "app",
@@ -1084,7 +1116,11 @@ func TestClient_UpdateHPAFromTortoiseRecommendation(t *testing.T) {
 			args: args{
 				ctx: context.Background(),
 				tortoise: &autoscalingv1beta3.Tortoise{
+					Spec: autoscalingv1beta3.TortoiseSpec{
+						UpdateMode: autoscalingv1beta3.UpdateModeAuto,
+					},
 					Status: autoscalingv1beta3.TortoiseStatus{
+						TortoisePhase: autoscalingv1beta3.TortoisePhaseWorking,
 						AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{
 							{
 								ContainerName: "app",
@@ -1242,6 +1278,7 @@ func TestClient_UpdateHPAFromTortoiseRecommendation(t *testing.T) {
 						UpdateMode: autoscalingv1beta3.UpdateModeAuto,
 					},
 					Status: autoscalingv1beta3.TortoiseStatus{
+						TortoisePhase: autoscalingv1beta3.TortoisePhaseWorking,
 						AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{
 							{
 								ContainerName: "app",
@@ -1405,6 +1442,7 @@ func TestClient_UpdateHPAFromTortoiseRecommendation(t *testing.T) {
 					UpdateMode: autoscalingv1beta3.UpdateModeAuto,
 				},
 				Status: autoscalingv1beta3.TortoiseStatus{
+					TortoisePhase: autoscalingv1beta3.TortoisePhaseWorking,
 					AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{
 						{
 							ContainerName: "app",
@@ -1501,6 +1539,7 @@ func TestClient_UpdateHPAFromTortoiseRecommendation(t *testing.T) {
 						UpdateMode: autoscalingv1beta3.UpdateModeOff,
 					},
 					Status: autoscalingv1beta3.TortoiseStatus{
+						TortoisePhase: autoscalingv1beta3.TortoisePhaseWorking,
 						AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{
 							{
 								ContainerName: "app",
@@ -1648,6 +1687,163 @@ func TestClient_UpdateHPAFromTortoiseRecommendation(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "no update preformed when all horizontal is unready",
+			args: args{
+				ctx: context.Background(),
+				tortoise: &autoscalingv1beta3.Tortoise{
+					Spec: autoscalingv1beta3.TortoiseSpec{
+						UpdateMode: autoscalingv1beta3.UpdateModeOff,
+					},
+					Status: autoscalingv1beta3.TortoiseStatus{
+						TortoisePhase: autoscalingv1beta3.TortoisePhaseGatheringData,
+						AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{
+							{
+								ContainerName: "app",
+								Policy: map[v1.ResourceName]v1beta3.AutoscalingType{
+									v1.ResourceMemory: v1beta3.AutoscalingTypeHorizontal,
+								},
+							},
+							{
+								ContainerName: "istio-proxy",
+								Policy: map[v1.ResourceName]v1beta3.AutoscalingType{
+									v1.ResourceCPU: v1beta3.AutoscalingTypeHorizontal,
+								},
+							},
+						},
+						ContainerResourcePhases: []autoscalingv1beta3.ContainerResourcePhases{
+							{
+								ContainerName: "app",
+								ResourcePhases: map[v1.ResourceName]autoscalingv1beta3.ResourcePhase{
+									v1.ResourceMemory: {
+										Phase: autoscalingv1beta3.ContainerResourcePhaseGatheringData,
+									},
+								},
+							},
+							{
+								ContainerName: "istio-proxy",
+								ResourcePhases: map[v1.ResourceName]autoscalingv1beta3.ResourcePhase{
+									v1.ResourceCPU: {
+										Phase: autoscalingv1beta3.ContainerResourcePhaseGatheringData,
+									},
+								},
+							},
+						},
+						Targets: autoscalingv1beta3.TargetsStatus{
+							HorizontalPodAutoscaler: "hpa",
+						},
+						Recommendations: autoscalingv1beta3.Recommendations{
+							Horizontal: autoscalingv1beta3.HorizontalRecommendations{
+								TargetUtilizations: []autoscalingv1beta3.HPATargetUtilizationRecommendationPerContainer{
+									{
+										ContainerName: "app",
+										TargetUtilization: map[v1.ResourceName]int32{
+											v1.ResourceMemory: 90,
+										},
+									},
+									{
+										ContainerName: "istio-proxy",
+										TargetUtilization: map[v1.ResourceName]int32{
+											v1.ResourceCPU: 80,
+										},
+									},
+								},
+								MaxReplicas: []autoscalingv1beta3.ReplicasRecommendation{
+									{
+										From:      0,
+										To:        2,
+										Value:     6,
+										UpdatedAt: now,
+										WeekDay:   ptr.To(now.Weekday().String()),
+									},
+								},
+								MinReplicas: []autoscalingv1beta3.ReplicasRecommendation{
+									{
+										From:      0,
+										To:        2,
+										Value:     3,
+										UpdatedAt: now,
+										WeekDay:   ptr.To(now.Weekday().String()),
+									},
+								},
+							},
+						},
+					},
+				},
+				now: now.Time,
+			},
+			initialHPA: &v2.HorizontalPodAutoscaler{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "hpa",
+				},
+				Spec: v2.HorizontalPodAutoscalerSpec{
+					MinReplicas: ptrInt32(1),
+					MaxReplicas: 2,
+					Metrics: []v2.MetricSpec{
+						{
+							Type: v2.ObjectMetricSourceType,
+							// should be ignored
+						},
+						{
+							Type: v2.ContainerResourceMetricSourceType,
+							ContainerResource: &v2.ContainerResourceMetricSource{
+								Name: v1.ResourceMemory,
+								Target: v2.MetricTarget{
+									AverageUtilization: ptr.To[int32](60),
+								},
+								Container: "app",
+							},
+						},
+						{
+							Type: v2.ContainerResourceMetricSourceType,
+							ContainerResource: &v2.ContainerResourceMetricSource{
+								Name: v1.ResourceCPU,
+								Target: v2.MetricTarget{
+									AverageUtilization: ptr.To[int32](50),
+								},
+								Container: "istio-proxy",
+							},
+						},
+					},
+				},
+			},
+			want: &v2.HorizontalPodAutoscaler{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "hpa",
+				},
+				Spec: v2.HorizontalPodAutoscalerSpec{
+					MinReplicas: ptrInt32(1),
+					MaxReplicas: 2,
+					Metrics: []v2.MetricSpec{
+						{
+							Type: v2.ObjectMetricSourceType,
+							// should be ignored
+						},
+						{
+							Type: v2.ContainerResourceMetricSourceType,
+							ContainerResource: &v2.ContainerResourceMetricSource{
+								Name: v1.ResourceMemory,
+								Target: v2.MetricTarget{
+									AverageUtilization: ptr.To[int32](60),
+								},
+								Container: "app",
+							},
+						},
+						{
+							Type: v2.ContainerResourceMetricSourceType,
+							ContainerResource: &v2.ContainerResourceMetricSource{
+								Name: v1.ResourceCPU,
+								Target: v2.MetricTarget{
+									AverageUtilization: ptr.To[int32](50),
+								},
+								Container: "istio-proxy",
+							},
+						},
+					},
+				},
+			},
+			wantErr: false,
+		},
 		{
 			name: "no update preformed when ContainerResourcePhases isn't working",
 			args: args{
@@ -1657,6 +1853,7 @@ func TestClient_UpdateHPAFromTortoiseRecommendation(t *testing.T) {
 						UpdateMode: autoscalingv1beta3.UpdateModeAuto,
 					},
 					Status: autoscalingv1beta3.TortoiseStatus{
+						TortoisePhase: autoscalingv1beta3.TortoisePhasePartlyWorking,
 						AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{
 							{
 								ContainerName: "app",
@@ -1811,6 +2008,7 @@ func TestClient_UpdateHPAFromTortoiseRecommendation(t *testing.T) {
 				ctx: context.Background(),
 				tortoise: &autoscalingv1beta3.Tortoise{
 					Status: autoscalingv1beta3.TortoiseStatus{
+						TortoisePhase: autoscalingv1beta3.TortoisePhaseEmergency,
 						AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{
 							{
 								ContainerName: "app",
@@ -1828,7 +2026,6 @@ func TestClient_UpdateHPAFromTortoiseRecommendation(t *testing.T) {
 						Targets: autoscalingv1beta3.TargetsStatus{
 							HorizontalPodAutoscaler: "hpa",
 						},
-						TortoisePhase: autoscalingv1beta3.TortoisePhaseEmergency,
 						Recommendations: autoscalingv1beta3.Recommendations{
 							Horizontal: autoscalingv1beta3.HorizontalRecommendations{
 								TargetUtilizations: []autoscalingv1beta3.HPATargetUtilizationRecommendationPerContainer{
@@ -1948,6 +2145,7 @@ func TestClient_UpdateHPAFromTortoiseRecommendation(t *testing.T) {
 				ctx: context.Background(),
 				tortoise: &autoscalingv1beta3.Tortoise{
 					Status: autoscalingv1beta3.TortoiseStatus{
+						TortoisePhase: autoscalingv1beta3.TortoisePhaseBackToNormal,
 						AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{
 							{
 								ContainerName: "app",
@@ -1965,7 +2163,6 @@ func TestClient_UpdateHPAFromTortoiseRecommendation(t *testing.T) {
 						Targets: autoscalingv1beta3.TargetsStatus{
 							HorizontalPodAutoscaler: "hpa",
 						},
-						TortoisePhase: autoscalingv1beta3.TortoisePhaseBackToNormal,
 						Recommendations: autoscalingv1beta3.Recommendations{
 							Horizontal: autoscalingv1beta3.HorizontalRecommendations{
 								TargetUtilizations: []autoscalingv1beta3.HPATargetUtilizationRecommendationPerContainer{
@@ -2085,6 +2282,7 @@ func TestClient_UpdateHPAFromTortoiseRecommendation(t *testing.T) {
 				ctx: context.Background(),
 				tortoise: &autoscalingv1beta3.Tortoise{
 					Status: autoscalingv1beta3.TortoiseStatus{
+						TortoisePhase: autoscalingv1beta3.TortoisePhaseBackToNormal,
 						AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{
 							{
 								ContainerName: "app",
@@ -2102,7 +2300,6 @@ func TestClient_UpdateHPAFromTortoiseRecommendation(t *testing.T) {
 						Targets: autoscalingv1beta3.TargetsStatus{
 							HorizontalPodAutoscaler: "hpa",
 						},
-						TortoisePhase: autoscalingv1beta3.TortoisePhaseBackToNormal,
 						Recommendations: autoscalingv1beta3.Recommendations{
 							Horizontal: autoscalingv1beta3.HorizontalRecommendations{
 								TargetUtilizations: []autoscalingv1beta3.HPATargetUtilizationRecommendationPerContainer{
@@ -2216,6 +2413,7 @@ func TestClient_UpdateHPAFromTortoiseRecommendation(t *testing.T) {
 			},
 			wantTortoise: &autoscalingv1beta3.Tortoise{
 				Status: autoscalingv1beta3.TortoiseStatus{
+					TortoisePhase: autoscalingv1beta3.TortoisePhaseWorking,
 					AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{
 						{
 							ContainerName: "app",
@@ -2245,7 +2443,6 @@ func TestClient_UpdateHPAFromTortoiseRecommendation(t *testing.T) {
 					Targets: autoscalingv1beta3.TargetsStatus{
 						HorizontalPodAutoscaler: "hpa",
 					},
-					TortoisePhase: autoscalingv1beta3.TortoisePhaseWorking,
 					Recommendations: autoscalingv1beta3.Recommendations{
 						Horizontal: autoscalingv1beta3.HorizontalRecommendations{
 							TargetUtilizations: []autoscalingv1beta3.HPATargetUtilizationRecommendationPerContainer{
diff --git a/pkg/pod/pod_test.go b/pkg/pod/pod_test.go
index ee09d0d..6d44edf 100644
--- a/pkg/pod/pod_test.go
+++ b/pkg/pod/pod_test.go
@@ -55,6 +55,162 @@ func TestService_ModifyPodResource(t *testing.T) {
 					Spec: v1beta3.TortoiseSpec{
 						UpdateMode: v1beta3.UpdateModeOff,
 					},
+					Status: v1beta3.TortoiseStatus{
+						TortoisePhase: v1beta3.TortoisePhaseWorking,
+					},
+				},
+			},
+			want: &v1.Pod{
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+						{
+							Name: "container",
+							Resources: v1.ResourceRequirements{
+								Requests: v1.ResourceList{
+									v1.ResourceCPU:    resource.MustParse("100m"),
+									v1.ResourceMemory: resource.MustParse("100Mi"),
+								},
+								Limits: v1.ResourceList{
+									v1.ResourceCPU:    resource.MustParse("200m"),
+									v1.ResourceMemory: resource.MustParse("200Mi"),
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "Tortoise is just created",
+			args: args{
+				pod: &v1.Pod{
+					Spec: v1.PodSpec{
+						Containers: []v1.Container{
+							{
+								Name: "container",
+								Resources: v1.ResourceRequirements{
+									Requests: v1.ResourceList{
+										v1.ResourceCPU:    resource.MustParse("100m"),
+										v1.ResourceMemory: resource.MustParse("100Mi"),
+									},
+									Limits: v1.ResourceList{
+										v1.ResourceCPU:    resource.MustParse("200m"),
+										v1.ResourceMemory: resource.MustParse("200Mi"),
+									},
+								},
+							},
+						},
+					},
+				},
+				tortoise: &v1beta3.Tortoise{
+					Spec: v1beta3.TortoiseSpec{
+						UpdateMode: v1beta3.UpdateModeAuto,
+					},
+					Status: v1beta3.TortoiseStatus{
+						// TortoisePhase is empty
+					},
+				},
+			},
+			want: &v1.Pod{
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+						{
+							Name: "container",
+							Resources: v1.ResourceRequirements{
+								Requests: v1.ResourceList{
+									v1.ResourceCPU:    resource.MustParse("100m"),
+									v1.ResourceMemory: resource.MustParse("100Mi"),
+								},
+								Limits: v1.ResourceList{
+									v1.ResourceCPU:    resource.MustParse("200m"),
+									v1.ResourceMemory: resource.MustParse("200Mi"),
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "Tortoise is Initializing",
+			args: args{
+				pod: &v1.Pod{
+					Spec: v1.PodSpec{
+						Containers: []v1.Container{
+							{
+								Name: "container",
+								Resources: v1.ResourceRequirements{
+									Requests: v1.ResourceList{
+										v1.ResourceCPU:    resource.MustParse("100m"),
+										v1.ResourceMemory: resource.MustParse("100Mi"),
+									},
+									Limits: v1.ResourceList{
+										v1.ResourceCPU:    resource.MustParse("200m"),
+										v1.ResourceMemory: resource.MustParse("200Mi"),
+									},
+								},
+							},
+						},
+					},
+				},
+				tortoise: &v1beta3.Tortoise{
+					Spec: v1beta3.TortoiseSpec{
+						UpdateMode: v1beta3.UpdateModeAuto,
+					},
+					Status: v1beta3.TortoiseStatus{
+						TortoisePhase: v1beta3.TortoisePhaseInitializing,
+					},
+				},
+			},
+			want: &v1.Pod{
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+						{
+							Name: "container",
+							Resources: v1.ResourceRequirements{
+								Requests: v1.ResourceList{
+									v1.ResourceCPU:    resource.MustParse("100m"),
+									v1.ResourceMemory: resource.MustParse("100Mi"),
+								},
+								Limits: v1.ResourceList{
+									v1.ResourceCPU:    resource.MustParse("200m"),
+									v1.ResourceMemory: resource.MustParse("200Mi"),
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "Tortoise is Gatheringdata",
+			args: args{
+				pod: &v1.Pod{
+					Spec: v1.PodSpec{
+						Containers: []v1.Container{
+							{
+								Name: "container",
+								Resources: v1.ResourceRequirements{
+									Requests: v1.ResourceList{
+										v1.ResourceCPU:    resource.MustParse("100m"),
+										v1.ResourceMemory: resource.MustParse("100Mi"),
+									},
+									Limits: v1.ResourceList{
+										v1.ResourceCPU:    resource.MustParse("200m"),
+										v1.ResourceMemory: resource.MustParse("200Mi"),
+									},
+								},
+							},
+						},
+					},
+				},
+				tortoise: &v1beta3.Tortoise{
+					Spec: v1beta3.TortoiseSpec{
+						UpdateMode: v1beta3.UpdateModeAuto,
+					},
+					Status: v1beta3.TortoiseStatus{
+						TortoisePhase: v1beta3.TortoisePhaseGatheringData,
+					},
 				},
 			},
 			want: &v1.Pod{
@@ -117,6 +273,7 @@ func TestService_ModifyPodResource(t *testing.T) {
 						UpdateMode: v1beta3.UpdateModeAuto,
 					},
 					Status: v1beta3.TortoiseStatus{
+						TortoisePhase: v1beta3.TortoisePhaseWorking,
 						Conditions: v1beta3.Conditions{
 							ContainerResourceRequests: []v1beta3.ContainerResourceRequests{
 								{
@@ -211,6 +368,7 @@ func TestService_ModifyPodResource(t *testing.T) {
 						UpdateMode: v1beta3.UpdateModeAuto,
 					},
 					Status: v1beta3.TortoiseStatus{
+						TortoisePhase: v1beta3.TortoisePhaseWorking,
 						Conditions: v1beta3.Conditions{
 							ContainerResourceRequests: []v1beta3.ContainerResourceRequests{
 								{
@@ -303,6 +461,7 @@ func TestService_ModifyPodResource(t *testing.T) {
 						UpdateMode: v1beta3.UpdateModeAuto,
 					},
 					Status: v1beta3.TortoiseStatus{
+						TortoisePhase: v1beta3.TortoisePhaseWorking,
 						Conditions: v1beta3.Conditions{
 							ContainerResourceRequests: []v1beta3.ContainerResourceRequests{
 								{
@@ -400,6 +559,7 @@ func TestService_ModifyPodResource(t *testing.T) {
 						UpdateMode: v1beta3.UpdateModeAuto,
 					},
 					Status: v1beta3.TortoiseStatus{
+						TortoisePhase: v1beta3.TortoisePhaseWorking,
 						Conditions: v1beta3.Conditions{
 							ContainerResourceRequests: []v1beta3.ContainerResourceRequests{
 								{
@@ -517,6 +677,7 @@ func TestService_ModifyPodResource(t *testing.T) {
 						UpdateMode: v1beta3.UpdateModeAuto,
 					},
 					Status: v1beta3.TortoiseStatus{
+						TortoisePhase: v1beta3.TortoisePhaseWorking,
 						Conditions: v1beta3.Conditions{
 							ContainerResourceRequests: []v1beta3.ContainerResourceRequests{
 								{
@@ -654,6 +815,7 @@ func TestService_ModifyPodResource(t *testing.T) {
 						UpdateMode: v1beta3.UpdateModeAuto,
 					},
 					Status: v1beta3.TortoiseStatus{
+						TortoisePhase: v1beta3.TortoisePhaseWorking,
 						Conditions: v1beta3.Conditions{
 							ContainerResourceRequests: []v1beta3.ContainerResourceRequests{
 								{
diff --git a/pkg/tortoise/tortoise_test.go b/pkg/tortoise/tortoise_test.go
index ecc6584..2a514f2 100644
--- a/pkg/tortoise/tortoise_test.go
+++ b/pkg/tortoise/tortoise_test.go
@@ -1346,7 +1346,51 @@ func TestService_changeTortoisePhaseWorkingIfTortoiseFinishedGatheringData(t *te
 			name: "minReplicas/maxReplicas recommendation is not yet gathered",
 			tortoise: &v1beta3.Tortoise{
 				Status: v1beta3.TortoiseStatus{
-					TortoisePhase: v1beta3.TortoisePhaseGatheringData,
+					TortoisePhase: v1beta3.TortoisePhasePartlyWorking,
+					AutoscalingPolicy: []v1beta3.ContainerAutoscalingPolicy{
+						{
+							ContainerName: "app",
+							Policy: map[corev1.ResourceName]v1beta3.AutoscalingType{
+								corev1.ResourceMemory: v1beta3.AutoscalingTypeVertical,
+								corev1.ResourceCPU:    v1beta3.AutoscalingTypeHorizontal,
+							},
+						},
+						{
+							ContainerName: "app2",
+							Policy: map[corev1.ResourceName]v1beta3.AutoscalingType{
+								corev1.ResourceMemory: v1beta3.AutoscalingTypeVertical,
+								corev1.ResourceCPU:    v1beta3.AutoscalingTypeHorizontal,
+							},
+						},
+					},
+					ContainerResourcePhases: []v1beta3.ContainerResourcePhases{
+						{
+							ContainerName: "app",
+							ResourcePhases: map[corev1.ResourceName]v1beta3.ResourcePhase{
+								corev1.ResourceMemory: {
+									Phase:              v1beta3.ContainerResourcePhaseWorking,
+									LastTransitionTime: metav1.NewTime(now),
+								},
+								corev1.ResourceCPU: {
+									Phase:              v1beta3.ContainerResourcePhaseGatheringData,
+									LastTransitionTime: metav1.NewTime(now),
+								},
+							},
+						},
+						{
+							ContainerName: "app2",
+							ResourcePhases: map[corev1.ResourceName]v1beta3.ResourcePhase{
+								corev1.ResourceMemory: {
+									Phase:              v1beta3.ContainerResourcePhaseWorking,
+									LastTransitionTime: metav1.NewTime(now),
+								},
+								corev1.ResourceCPU: {
+									Phase:              v1beta3.ContainerResourcePhaseGatheringData,
+									LastTransitionTime: metav1.NewTime(now),
+								},
+							},
+						},
+					},
 					Recommendations: v1beta3.Recommendations{
 						Horizontal: v1beta3.HorizontalRecommendations{
 							MinReplicas: []v1beta3.ReplicasRecommendation{
@@ -1395,7 +1439,51 @@ func TestService_changeTortoisePhaseWorkingIfTortoiseFinishedGatheringData(t *te
 			},
 			want: &v1beta3.Tortoise{
 				Status: v1beta3.TortoiseStatus{
-					TortoisePhase: v1beta3.TortoisePhaseGatheringData,
+					TortoisePhase: v1beta3.TortoisePhasePartlyWorking,
+					AutoscalingPolicy: []v1beta3.ContainerAutoscalingPolicy{
+						{
+							ContainerName: "app",
+							Policy: map[corev1.ResourceName]v1beta3.AutoscalingType{
+								corev1.ResourceMemory: v1beta3.AutoscalingTypeVertical,
+								corev1.ResourceCPU:    v1beta3.AutoscalingTypeHorizontal,
+							},
+						},
+						{
+							ContainerName: "app2",
+							Policy: map[corev1.ResourceName]v1beta3.AutoscalingType{
+								corev1.ResourceMemory: v1beta3.AutoscalingTypeVertical,
+								corev1.ResourceCPU:    v1beta3.AutoscalingTypeHorizontal,
+							},
+						},
+					},
+					ContainerResourcePhases: []v1beta3.ContainerResourcePhases{
+						{
+							ContainerName: "app",
+							ResourcePhases: map[corev1.ResourceName]v1beta3.ResourcePhase{
+								corev1.ResourceMemory: {
+									Phase:              v1beta3.ContainerResourcePhaseWorking, // unchanged because it's vertical.
+									LastTransitionTime: metav1.NewTime(now),
+								},
+								corev1.ResourceCPU: {
+									Phase:              v1beta3.ContainerResourcePhaseGatheringData,
+									LastTransitionTime: metav1.NewTime(now),
+								},
+							},
+						},
+						{
+							ContainerName: "app2",
+							ResourcePhases: map[corev1.ResourceName]v1beta3.ResourcePhase{
+								corev1.ResourceMemory: {
+									Phase:              v1beta3.ContainerResourcePhaseWorking, // unchanged
+									LastTransitionTime: metav1.NewTime(now),
+								},
+								corev1.ResourceCPU: {
+									Phase:              v1beta3.ContainerResourcePhaseGatheringData,
+									LastTransitionTime: metav1.NewTime(now),
+								},
+							},
+						},
+					},
 					Recommendations: v1beta3.Recommendations{
 						Horizontal: v1beta3.HorizontalRecommendations{
 							MinReplicas: []v1beta3.ReplicasRecommendation{
diff --git a/pkg/vpa/service_test.go b/pkg/vpa/service_test.go
index 9709433..36904a1 100644
--- a/pkg/vpa/service_test.go
+++ b/pkg/vpa/service_test.go
@@ -128,6 +128,109 @@ func TestMakeAllVerticalContainerResourcePhaseRunning(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "modified correctly (gathringdata)",
+			args: args{
+				tortoise: &autoscalingv1beta3.Tortoise{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "tortoise",
+						Namespace: "default",
+					},
+					Status: autoscalingv1beta3.TortoiseStatus{
+						TortoisePhase: autoscalingv1beta3.TortoisePhaseGatheringData,
+						AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{
+							{
+								ContainerName: "app",
+								Policy: map[v1.ResourceName]v1beta3.AutoscalingType{
+									v1.ResourceMemory: v1beta3.AutoscalingTypeVertical,
+									v1.ResourceCPU:    v1beta3.AutoscalingTypeHorizontal,
+								},
+							},
+							{
+								ContainerName: "istio-proxy",
+								Policy: map[v1.ResourceName]v1beta3.AutoscalingType{
+									v1.ResourceMemory: v1beta3.AutoscalingTypeVertical,
+									v1.ResourceCPU:    v1beta3.AutoscalingTypeHorizontal,
+								},
+							},
+						},
+						ContainerResourcePhases: []autoscalingv1beta3.ContainerResourcePhases{
+							{
+								ContainerName: "app",
+								ResourcePhases: map[v1.ResourceName]autoscalingv1beta3.ResourcePhase{
+									v1.ResourceCPU: {
+										Phase: autoscalingv1beta3.ContainerResourcePhaseGatheringData,
+									},
+									v1.ResourceMemory: {
+										Phase: autoscalingv1beta3.ContainerResourcePhaseGatheringData,
+									},
+								},
+							},
+							{
+								ContainerName: "istio-proxy",
+								ResourcePhases: map[v1.ResourceName]autoscalingv1beta3.ResourcePhase{
+									v1.ResourceCPU: {
+										Phase: autoscalingv1beta3.ContainerResourcePhaseGatheringData,
+									},
+									v1.ResourceMemory: {
+										Phase: autoscalingv1beta3.ContainerResourcePhaseGatheringData,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			want: &autoscalingv1beta3.Tortoise{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "tortoise",
+					Namespace: "default",
+				},
+				Status: autoscalingv1beta3.TortoiseStatus{
+					TortoisePhase: autoscalingv1beta3.TortoisePhasePartlyWorking,
+					AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{
+						{
+							ContainerName: "app",
+							Policy: map[v1.ResourceName]v1beta3.AutoscalingType{
+								v1.ResourceMemory: v1beta3.AutoscalingTypeVertical,
+								v1.ResourceCPU:    v1beta3.AutoscalingTypeHorizontal,
+							},
+						},
+						{
+							ContainerName: "istio-proxy",
+							Policy: map[v1.ResourceName]v1beta3.AutoscalingType{
+								v1.ResourceMemory: v1beta3.AutoscalingTypeVertical,
+								v1.ResourceCPU:    v1beta3.AutoscalingTypeHorizontal,
+							},
+						},
+					},
+					ContainerResourcePhases: []autoscalingv1beta3.ContainerResourcePhases{
+						{
+							ContainerName: "app",
+							ResourcePhases: map[v1.ResourceName]autoscalingv1beta3.ResourcePhase{
+								v1.ResourceCPU: {
+									Phase: autoscalingv1beta3.ContainerResourcePhaseGatheringData,
+								},
+								v1.ResourceMemory: {
+									Phase: autoscalingv1beta3.ContainerResourcePhaseWorking,
+								},
+							},
+						},
+						{
+							ContainerName: "istio-proxy",
+							ResourcePhases: map[v1.ResourceName]autoscalingv1beta3.ResourcePhase{
+								v1.ResourceCPU: {
+									Phase: autoscalingv1beta3.ContainerResourcePhaseGatheringData,
+								},
+								v1.ResourceMemory: {
+									Phase: autoscalingv1beta3.ContainerResourcePhaseWorking,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {