Repository: tokamak-network/trh-sdk
Pull request #43: Issue/pvc not bound
Author: login: hazardvictor
Created: 2025-04-28T09:57:33Z | Merged: 2025-04-28T11:16:31Z

Summary of changes.
https://github.com/tokamak-network/trh-sdk/issues/42
- Check if the k8s cluster is ready before helm chart installation.
  - api health
  - pvc binding


Diff:
diff --git a/pkg/stacks/thanos/thanos_stack.go b/pkg/stacks/thanos/thanos_stack.go
index 7be80614..73247a41 100644
--- a/pkg/stacks/thanos/thanos_stack.go
+++ b/pkg/stacks/thanos/thanos_stack.go
@@ -510,6 +510,15 @@ func (t *ThanosStack) deployNetworkToAWS(ctx context.Context, deployConfig *type
 
 	fmt.Println("EKS configuration updated:", eksSetup)
 
+	// Step 7.1. Check if K8s cluster is ready
+	fmt.Println("Checking if K8s cluster is ready...")
+	k8sReady, err := utils.CheckK8sReady(namespace)
+	if err != nil {
+		fmt.Println("❌ Error checking K8s cluster readiness:", err)
+		return err
+	}
+	fmt.Printf("✅ K8s cluster is ready: %t\n", k8sReady)
+
 	// ---------------------------------------- Deploy chain --------------------------//
 	// Step 8. Add Helm repository
 	helmAddOuput, err := utils.ExecuteCommand("helm", []string{
diff --git a/pkg/utils/helm.go b/pkg/utils/helm.go
index 12925d98..3feef51f 100644
--- a/pkg/utils/helm.go
+++ b/pkg/utils/helm.go
@@ -1,7 +1,9 @@
 package utils
 
 import (
+	"fmt"
 	"strings"
+	"time"
 )
 
 // GetHelmReleases fetches the list of Helm releases in the given namespace
@@ -35,3 +37,54 @@ func FilterHelmReleases(namespace string, releaseName string) ([]string, error)
 	}
 	return helmReleases, nil
 }
+
+func CheckK8sReady(namespace string) (bool, error) {
+	// TODO: these values can be adjusted if it is not enough
+	maxRetries := 10
+	retryInterval := 20 * time.Second
+
+	var isPVCReady, isAPiReady bool
+	var err error
+
+	for i := 0; i < maxRetries; i++ {
+		isPVCReady, err = CheckPVCStatus(namespace)
+		if err != nil {
+			fmt.Printf("PVC status check failed (attempt %d/%d): %v\n", i+1, maxRetries, err)
+			time.Sleep(retryInterval)
+			continue
+		}
+
+		isAPiReady, err = CheckK8sApiHealth(namespace)
+		if err != nil {
+			fmt.Printf("K8s API health check failed (attempt %d/%d): %v\n", i+1, maxRetries, err)
+			time.Sleep(retryInterval)
+			continue
+		}
+
+		if isPVCReady && isAPiReady {
+			return true, nil
+		}
+
+		fmt.Printf("K8s not ready yet. Retrying in %v... (attempt %d/%d)\n", retryInterval, i+1, maxRetries)
+		time.Sleep(retryInterval)
+	}
+
+	return false, fmt.Errorf("K8s not ready after %d attempts", maxRetries)
+}
+
+func CheckK8sApiHealth(namespace string) (bool, error) {
+	apiHealth, err := ExecuteCommand("kubectl", "get", "--raw=/readyz")
+	if err != nil {
+		return false, err
+	}
+	return apiHealth == "ok", nil
+}
+
+func CheckPVCStatus(namespace string) (bool, error) {
+	pvcStatus, err := ExecuteCommand("bash", "-c", fmt.Sprintf("if kubectl get pvc -n %s -o jsonpath='{range .items[*]}{.status.phase}{\"\\n\"}{end}' | grep -qv Bound; then echo 'There are unbound PVCs'; else echo 'Bound'; fi", namespace))
+	fmt.Println("PVC status:", pvcStatus)
+	if err != nil {
+		return false, err
+	}
+	return pvcStatus == "Bound", nil
+}


Reviews:
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-28T10:14:26Z
state: APPROVED
author_association: NONE
submitted_at: 2025-04-28T11:13:07Z

Review comments:
diff_hunk: @@ -35,3 +37,61 @@ func FilterHelmReleases(namespace string, releaseName string) ([]string, error)
 	}
 	return helmReleases, nil
 }
+
+func CheckK8sReady(namespace string) (bool, error) {
+	maxRetries := 10
+	retryInterval := 20 * time.Second
+
+	var pvcStatus, apiHealth bool
+	var err error
+
+	for i := 0; i < maxRetries; i++ {
+		pvcStatus, err = CheckPVCStatus(namespace)
+		if err != nil {
+			if i == maxRetries-1 {
+				return false, err
+			}
+			fmt.Printf("PVC status check failed (attempt %d/%d): %v\n", i+1, maxRetries, err)
+			time.Sleep(retryInterval)
+			continue
+		}
+
+		apiHealth, err = CheckK8sApiHealth(namespace)
+		if err != nil {
+			if i == maxRetries-1 {
+				return false, err
+			}
+			fmt.Printf("K8s API health check failed (attempt %d/%d): %v\n", i+1, maxRetries, err)
+			time.Sleep(retryInterval)
+			continue
+		}
+
+		if pvcStatus && apiHealth {
+			return true, nil
+		}
+
+		if i < maxRetries-1 {
+			fmt.Printf("K8s not ready yet. Retrying in %v... (attempt %d/%d)\n", retryInterval, i+1, maxRetries)
+			time.Sleep(retryInterval)
+		}
+	}
+
+	return false, fmt.Errorf("K8s not ready after %d attempts", maxRetries)
+}
+
+func CheckK8sApiHealth(namespace string) (bool, error) {
+	apiHealth, err := ExecuteCommand("kubectl", "get", "--raw=/readyz?verbose")
+	if err != nil {
+		return false, err
+	}
+	return apiHealth != "", nil
+}
path: pkg/utils/helm.go
body: ```suggestion
func CheckK8sApiHealth(namespace string) (bool, error) {
	apiHealth, err := ExecuteCommand("kubectl", "get", "--raw=/readyz")
	if err != nil {
		return false, err
	}
	return apiHealth != "ok", nil
}
```
created_at: 2025-04-28T10:09:23Z
author_association: NONE
diff_hunk: @@ -35,3 +37,61 @@ func FilterHelmReleases(namespace string, releaseName string) ([]string, error)
 	}
 	return helmReleases, nil
 }
+
+func CheckK8sReady(namespace string) (bool, error) {
+	maxRetries := 10
+	retryInterval := 20 * time.Second
+
+	var pvcStatus, apiHealth bool
+	var err error
+
+	for i := 0; i < maxRetries; i++ {
+		pvcStatus, err = CheckPVCStatus(namespace)
path: pkg/utils/helm.go
body: ```suggestion
		isPVCReady, err = CheckPVCStatus(namespace)
```
created_at: 2025-04-28T10:11:01Z
author_association: NONE
diff_hunk: @@ -35,3 +37,61 @@ func FilterHelmReleases(namespace string, releaseName string) ([]string, error)
 	}
 	return helmReleases, nil
 }
+
+func CheckK8sReady(namespace string) (bool, error) {
+	maxRetries := 10
+	retryInterval := 20 * time.Second
+
+	var pvcStatus, apiHealth bool
+	var err error
+
+	for i := 0; i < maxRetries; i++ {
+		pvcStatus, err = CheckPVCStatus(namespace)
+		if err != nil {
+			if i == maxRetries-1 {
+				return false, err
+			}
+			fmt.Printf("PVC status check failed (attempt %d/%d): %v\n", i+1, maxRetries, err)
+			time.Sleep(retryInterval)
+			continue
+		}
path: pkg/utils/helm.go
body: ```suggestion
		if err != nil {
			fmt.Printf("PVC status check failed (attempt %d/%d): %v\n", i+1, maxRetries, err)
			time.Sleep(retryInterval)
			continue
		}
```
created_at: 2025-04-28T10:12:06Z
author_association: NONE
diff_hunk: @@ -35,3 +37,61 @@ func FilterHelmReleases(namespace string, releaseName string) ([]string, error)
 	}
 	return helmReleases, nil
 }
+
+func CheckK8sReady(namespace string) (bool, error) {
+	maxRetries := 10
+	retryInterval := 20 * time.Second
+
+	var pvcStatus, apiHealth bool
+	var err error
+
+	for i := 0; i < maxRetries; i++ {
+		pvcStatus, err = CheckPVCStatus(namespace)
+		if err != nil {
+			if i == maxRetries-1 {
+				return false, err
+			}
+			fmt.Printf("PVC status check failed (attempt %d/%d): %v\n", i+1, maxRetries, err)
+			time.Sleep(retryInterval)
+			continue
+		}
+
+		apiHealth, err = CheckK8sApiHealth(namespace)
+		if err != nil {
+			if i == maxRetries-1 {
+				return false, err
+			}
+			fmt.Printf("K8s API health check failed (attempt %d/%d): %v\n", i+1, maxRetries, err)
+			time.Sleep(retryInterval)
+			continue
path: pkg/utils/helm.go
body: Same above
created_at: 2025-04-28T10:12:30Z
author_association: NONE
diff_hunk: @@ -35,3 +37,61 @@ func FilterHelmReleases(namespace string, releaseName string) ([]string, error)
 	}
 	return helmReleases, nil
 }
+
+func CheckK8sReady(namespace string) (bool, error) {
+	maxRetries := 10
+	retryInterval := 20 * time.Second
+
+	var pvcStatus, apiHealth bool
+	var err error
+
+	for i := 0; i < maxRetries; i++ {
+		pvcStatus, err = CheckPVCStatus(namespace)
+		if err != nil {
+			if i == maxRetries-1 {
+				return false, err
+			}
+			fmt.Printf("PVC status check failed (attempt %d/%d): %v\n", i+1, maxRetries, err)
+			time.Sleep(retryInterval)
+			continue
+		}
+
+		apiHealth, err = CheckK8sApiHealth(namespace)
+		if err != nil {
+			if i == maxRetries-1 {
+				return false, err
+			}
+			fmt.Printf("K8s API health check failed (attempt %d/%d): %v\n", i+1, maxRetries, err)
+			time.Sleep(retryInterval)
+			continue
+		}
+
+		if pvcStatus && apiHealth {
+			return true, nil
+		}
+
+		if i < maxRetries-1 {
+			fmt.Printf("K8s not ready yet. Retrying in %v... (attempt %d/%d)\n", retryInterval, i+1, maxRetries)
+			time.Sleep(retryInterval)
+		}
path: pkg/utils/helm.go
body: Dont need this condition because we're in the for...loop condition

```suggestion
		fmt.Printf("K8s not ready yet. Retrying in %v... (attempt %d/%d)\n", retryInterval, i+1, maxRetries)
		time.Sleep(retryInterval)
```
created_at: 2025-04-28T10:13:19Z
author_association: NONE
diff_hunk: @@ -35,3 +37,61 @@ func FilterHelmReleases(namespace string, releaseName string) ([]string, error)
 	}
 	return helmReleases, nil
 }
+
+func CheckK8sReady(namespace string) (bool, error) {
+	maxRetries := 10
+	retryInterval := 20 * time.Second
path: pkg/utils/helm.go
body: Just for notes: 

// TODO: these values can be adjusted if not enough
created_at: 2025-04-28T10:14:22Z
author_association: NONE

PR comments:
user: login: hazardvictor
created_at: 2025-04-28T11:05:23Z
body: ![image](https://github.com/user-attachments/assets/ac08f56c-e1ac-4dae-b719-4f684630c2f4)

author_association: CONTRIBUTOR
user: login: hazardvictor
created_at: 2025-04-28T11:06:08Z
body: ![image](https://github.com/user-attachments/assets/afb4fe26-11b4-4cc2-8558-e179892af57d)

author_association: CONTRIBUTOR