Repository: istio/istio (go)

Issue:
Add support for gRPC probe
Our probe rewrite needs to handle https://github.com/kubernetes/enhancements/issues/2727

Pull request description:
Support rewriting gRPC probes
**Please provide a description of this PR:**
Support rewriting gRPC probes for Kubernetes v1.23
Fix #36314
 
I'm not very familiar with the integration test, but I've tested it in my local environment, and here's the result:

#### 1. Apply strict mtls:
```sh
kubectl apply -f - <<EOF
apiVersion: security.istio.io/v1beta1
kind: PeerAuthentication
metadata:
  name: "default"
  namespace: "istio-system"
spec:
  mtls:
    mode: STRICT
EOF
```
#### 2. Create a pod with a grpc probe

Before rewriting([an example in k8s doc](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#define-a-grpc-liveness-probe "example in k8s doc")):
```yaml
apiVersion: v1
kind: Pod
metadata:
  name: etcd-with-grpc
spec:
  containers:
  - name: etcd
    image: k8s.gcr.io/etcd:3.5.1-0
    command: [ "/usr/local/bin/etcd", "--data-dir",  "/var/lib/etcd", "--listen-client-urls", "http://0.0.0.0:2379", "--advertise-client-urls", "http://127.0.0.1:2379", "--log-level", "debug"]
    ports:
    - containerPort: 2379
    readinessProbe:
      grpc:
        port: 2379
      initialDelaySeconds: 10
```

After rewriting:
```yaml
...........
      readinessProbe:
        failureThreshold: 3
        httpGet:
          path: /app-health/etcd/readyz
          port: 15020
          scheme: HTTP
        initialDelaySeconds: 10
        periodSeconds: 10
        successThreshold: 1
        timeoutSeconds: 1
...........
      - name: ISTIO_KUBE_APP_PROBERS
        value: '{"/app-health/etcd/readyz":{"grpc":{"port":2379,"service":""},"timeoutSeconds":1}}'
      image: docker.io/istio/proxyv2:1.12.1
...........
```

And the container stays ready
```sh
[root@new grpc-health]# kubectl get po
NAME             READY   STATUS    RESTARTS   AGE
etcd-with-grpc   2/2     Running   0          5m3s
```

#### 3. Set rewrite to `false` and recreate the pod
```yaml
......
metadata:
  annotations: 
    sidecar.istio.io/rewriteAppHTTPProbers: "false"
......
```

Turns out the probe would not be rewrited:
```yaml
      readinessProbe:
        failureThreshold: 3
        grpc:
          port: 2379
          service: ""
        initialDelaySeconds: 10
        periodSeconds: 10
        successThreshold: 1
        timeoutSeconds: 1
```

The container would become unready
```sh
[root@new grpc-health]# kubectl get po
NAME                             READY   STATUS    RESTARTS   AGE
etcd-7c7d558878-bv8dd            1/2     Running   0          4m22s
```

Patch:
diff --git a/pilot/cmd/pilot-agent/status/server.go b/pilot/cmd/pilot-agent/status/server.go
index 6220c07ea4..993ba56e02 100644
--- a/pilot/cmd/pilot-agent/status/server.go
+++ b/pilot/cmd/pilot-agent/status/server.go
@@ -40,6 +40,11 @@ import (
 	"github.com/prometheus/client_golang/prometheus/collectors"
 	"github.com/prometheus/common/expfmt"
 	"go.opencensus.io/stats/view"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	grpcHealth "google.golang.org/grpc/health/grpc_health_v1"
+	grpcStatus "google.golang.org/grpc/status"
 	"k8s.io/apimachinery/pkg/util/intstr"
 
 	"istio.io/istio/pilot/cmd/pilot-agent/metrics"
@@ -99,6 +104,7 @@ type KubeAppProbers map[string]*Prober
 type Prober struct {
 	HTTPGet        *apimirror.HTTPGetAction   `json:"httpGet,omitempty"`
 	TCPSocket      *apimirror.TCPSocketAction `json:"tcpSocket,omitempty"`
+	GRPC           *apimirror.GRPCAction      `json:"grpc,omitempty"`
 	TimeoutSeconds int32                      `json:"timeoutSeconds,omitempty"`
 }
 
@@ -278,11 +284,18 @@ func validateAppKubeProber(path string, prober *Prober) error {
 	if !appProberPattern.Match([]byte(path)) {
 		return fmt.Errorf(`invalid path, must be in form of regex pattern %v`, appProberPattern)
 	}
-	if prober.HTTPGet == nil && prober.TCPSocket == nil {
-		return fmt.Errorf(`invalid prober type, must be of type httpGet or tcpSocket`)
+	count := 0
+	if prober.HTTPGet != nil {
+		count++
+	}
+	if prober.TCPSocket != nil {
+		count++
 	}
-	if prober.HTTPGet != nil && prober.TCPSocket != nil {
-		return fmt.Errorf(`invalid prober, type must be either httpGet or tcpSocket`)
+	if prober.GRPC != nil {
+		count++
+	}
+	if count != 1 {
+		return fmt.Errorf(`invalid prober type, must be one of type httpGet, tcpSocket or gRPC`)
 	}
 	if prober.HTTPGet != nil && prober.HTTPGet.Port.Type != intstr.Int {
 		return fmt.Errorf("invalid prober config for %v, the port must be int type", path)
@@ -621,11 +634,13 @@ func (s *Server) handleAppProbe(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	if prober.HTTPGet != nil {
+	switch {
+	case prober.HTTPGet != nil:
 		s.handleAppProbeHTTPGet(w, req, prober, path)
-	}
-	if prober.TCPSocket != nil {
+	case prober.TCPSocket != nil:
 		s.handleAppProbeTCPSocket(w, prober)
+	case prober.GRPC != nil:
+		s.handleAppProbeGRPC(w, req, prober)
 	}
 }
 
@@ -714,6 +729,74 @@ func (s *Server) handleAppProbeTCPSocket(w http.ResponseWriter, prober *Prober)
 	}
 }
 
+func (s *Server) handleAppProbeGRPC(w http.ResponseWriter, req *http.Request, prober *Prober) {
+	timeout := time.Duration(prober.TimeoutSeconds) * time.Second
+	// the DialOptions are referenced from https://github.com/kubernetes/kubernetes/blob/v1.23.1/pkg/probe/grpc/grpc.go#L55-L59
+	opts := []grpc.DialOption{
+		grpc.WithBlock(),
+		grpc.WithTransportCredentials(insecure.NewCredentials()), // credentials are currently not supported
+		grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {
+			d := net.Dialer{
+				LocalAddr: s.upstreamLocalAddress,
+				Timeout:   timeout,
+			}
+			return d.DialContext(ctx, "tcp", addr)
+		}),
+	}
+	if userAgent := req.Header["User-Agent"]; len(userAgent) > 0 {
+		// simulate kubelet
+		// please refer to:
+		// https://github.com/kubernetes/kubernetes/blob/v1.23.1/pkg/probe/grpc/grpc.go#L56
+		// https://github.com/kubernetes/kubernetes/blob/v1.23.1/pkg/probe/http/http.go#L103
+		opts = append(opts, grpc.WithUserAgent(userAgent[0]))
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	addr := net.JoinHostPort(s.appProbersDestination, fmt.Sprintf("%d", prober.GRPC.Port))
+	conn, err := grpc.DialContext(ctx, addr, opts...)
+	if err != nil {
+		log.Errorf("Failed to create grpc connection to probe app: %v", err)
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	defer conn.Close()
+
+	var svc string
+	if prober.GRPC.Service != nil {
+		svc = *prober.GRPC.Service
+	}
+	grpcClient := grpcHealth.NewHealthClient(conn)
+	resp, err := grpcClient.Check(ctx, &grpcHealth.HealthCheckRequest{
+		Service: svc,
+	})
+	// the error handling is referenced from https://github.com/kubernetes/kubernetes/blob/v1.23.1/pkg/probe/grpc/grpc.go#L88-L106
+	if err != nil {
+		status, ok := grpcStatus.FromError(err)
+		if ok {
+			switch status.Code() {
+			case codes.Unimplemented:
+				log.Errorf("server does not implement the grpc health protocol (grpc.health.v1.Health): %v", err)
+			case codes.DeadlineExceeded:
+				log.Errorf("grpc request not finished within timeout: %v", err)
+			default:
+				log.Errorf("grpc probe failed: %v", err)
+			}
+		} else {
+			log.Errorf("grpc probe failed: %v", err)
+		}
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	if resp.GetStatus() == grpcHealth.HealthCheckResponse_SERVING {
+		w.WriteHeader(http.StatusOK)
+		return
+	}
+	w.WriteHeader(http.StatusInternalServerError)
+}
+
 func (s *Server) handleNdsz(w http.ResponseWriter, r *http.Request) {
 	if !isRequestFromLocalhost(r) {
 		http.Error(w, "Only requests from localhost are allowed", http.StatusForbidden)
diff --git a/pkg/kube/apimirror/probe.go b/pkg/kube/apimirror/probe.go
index b06f55718b..0ea5a6f5f3 100644
--- a/pkg/kube/apimirror/probe.go
+++ b/pkg/kube/apimirror/probe.go
@@ -70,3 +70,17 @@ type TCPSocketAction struct {
 	// +optional
 	Host string `json:"host,omitempty" protobuf:"bytes,2,opt,name=host"`
 }
+
+// GRPCAction describes an action based on GRPC health check
+type GRPCAction struct {
+	// Port number of the gRPC service. Number must be in the range 1 to 65535.
+	Port int32 `json:"port" protobuf:"bytes,1,opt,name=port"`
+
+	// Service is the name of the service to place in the gRPC HealthCheckRequest
+	// (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md).
+	//
+	// If this is not specified, the default behavior is defined by gRPC.
+	// +optional
+	// +default=""
+	Service *string `json:"service" protobuf:"bytes,2,opt,name=service"`
+}
diff --git a/pkg/kube/inject/app_probe.go b/pkg/kube/inject/app_probe.go
index d6528b9acf..3729ede444 100644
--- a/pkg/kube/inject/app_probe.go
+++ b/pkg/kube/inject/app_probe.go
@@ -62,10 +62,15 @@ func FindContainer(name string, containers []corev1.Container) *corev1.Container
 
 // convertAppProber returns an overwritten `Probe` for pilot agent to take over.
 func convertAppProber(probe *corev1.Probe, newURL string, statusPort int) *corev1.Probe {
-	if probe != nil && probe.HTTPGet != nil {
+	if probe == nil {
+		return nil
+	}
+	if probe.HTTPGet != nil {
 		return convertAppProberHTTPGet(probe, newURL, statusPort)
-	} else if probe != nil && probe.TCPSocket != nil && features.RewriteTCPProbes {
+	} else if probe.TCPSocket != nil && features.RewriteTCPProbes {
 		return convertAppProberTCPSocket(probe, newURL, statusPort)
+	} else if probe.GRPC != nil {
+		return convertAppProberGRPC(probe, newURL, statusPort)
 	}
 
 	return nil
@@ -98,21 +103,43 @@ func convertAppProberTCPSocket(probe *corev1.Probe, newURL string, statusPort in
 	return p
 }
 
+// convertAppProberGRPC returns an overwritten `Probe` (gRPC) for pilot agent to take over.
+func convertAppProberGRPC(probe *corev1.Probe, newURL string, statusPort int) *corev1.Probe {
+	p := probe.DeepCopy()
+	// the sidecar intercepts all gRPC connections, so we change it to a HTTP probe and the sidecar will check gRPC
+	p.HTTPGet = &corev1.HTTPGetAction{}
+	p.HTTPGet.Port = intstr.FromInt(statusPort)
+	p.HTTPGet.Path = newURL
+	// For gRPC prober, we change to HTTP,
+	// and pilot agent uses gRPC to request application prober endpoint.
+	// Kubelet -> HTTP -> Pilot Agent -> gRPC -> Application
+	p.GRPC = nil
+	return p
+}
+
 type KubeAppProbers map[string]*Prober
 
 // Prober represents a single container prober
 type Prober struct {
 	HTTPGet        *corev1.HTTPGetAction   `json:"httpGet,omitempty"`
 	TCPSocket      *corev1.TCPSocketAction `json:"tcpSocket,omitempty"`
+	GRPC           *corev1.GRPCAction      `json:"grpc,omitempty"`
 	TimeoutSeconds int32                   `json:"timeoutSeconds,omitempty"`
 }
 
 // DumpAppProbers returns a json encoded string as `status.KubeAppProbers`.
 // Also update the probers so that all usages of named port will be resolved to integer.
-func DumpAppProbers(podspec *corev1.PodSpec, targetPort int32) string {
+func DumpAppProbers(podSpec *corev1.PodSpec, targetPort int32) string {
 	out := KubeAppProbers{}
 	updateNamedPort := func(p *Prober, portMap map[string]int32) *Prober {
-		if p == nil || (p.HTTPGet == nil && p.TCPSocket == nil) {
+		if p == nil {
+			return nil
+		}
+		if p.GRPC != nil {
+			// don't need to update for gRPC probe port as it only supports integer
+			return p
+		}
+		if p.HTTPGet == nil && p.TCPSocket == nil {
 			return nil
 		}
 
@@ -135,7 +162,7 @@ func DumpAppProbers(podspec *corev1.PodSpec, targetPort int32) string {
 		}
 		return p
 	}
-	for _, c := range podspec.Containers {
+	for _, c := range podSpec.Containers {
 		if c.Name == ProxyContainerName {
 			continue
 		}
@@ -171,10 +198,6 @@ func DumpAppProbers(podspec *corev1.PodSpec, targetPort int32) string {
 
 // patchRewriteProbe generates the patch for webhook.
 func patchRewriteProbe(annotations map[string]string, pod *corev1.Pod, defaultPort int32) {
-	sidecar := FindSidecar(pod.Spec.Containers)
-	if sidecar == nil {
-		return
-	}
 	statusPort := int(defaultPort)
 	if v, f := annotations[annotation.SidecarStatusPort.Name]; f {
 		p, err := strconv.Atoi(v)
@@ -188,10 +211,6 @@ func patchRewriteProbe(annotations map[string]string, pod *corev1.Pod, defaultPo
 		if c.Name == ProxyContainerName {
 			continue
 		}
-		portMap := map[string]int32{}
-		for _, p := range c.Ports {
-			portMap[p.Name] = p.ContainerPort
-		}
 		readyz, livez, startupz := status.FormatProberURL(c.Name)
 		if probePatch := convertAppProber(c.ReadinessProbe, readyz, statusPort); probePatch != nil {
 			c.ReadinessProbe = probePatch
@@ -226,5 +245,12 @@ func kubeProbeToInternalProber(probe *corev1.Probe) *Prober {
 		}
 	}
 
+	if probe.GRPC != nil {
+		return &Prober{
+			GRPC:           probe.GRPC,
+			TimeoutSeconds: probe.TimeoutSeconds,
+		}
+	}
+
 	return nil
 }
diff --git a/pkg/kube/inject/webhook.go b/pkg/kube/inject/webhook.go
index 2322f1f0dd..d92e576a32 100644
--- a/pkg/kube/inject/webhook.go
+++ b/pkg/kube/inject/webhook.go
@@ -590,6 +590,10 @@ func reorderPod(pod *corev1.Pod, req InjectionParameters) error {
 }
 
 func applyRewrite(pod *corev1.Pod, req InjectionParameters) error {
+	sidecar := FindSidecar(pod.Spec.Containers)
+	if sidecar == nil {
+		return nil
+	}
 	valuesStruct := &opconfig.Values{}
 	if err := gogoprotomarshal.ApplyYAML(req.valuesConfig, valuesStruct); err != nil {
 		log.Infof("Failed to parse values config: %v [%v]\n", err, req.valuesConfig)
@@ -597,10 +601,8 @@ func applyRewrite(pod *corev1.Pod, req InjectionParameters) error {
 	}
 
 	rewrite := ShouldRewriteAppHTTPProbers(pod.Annotations, valuesStruct.GetSidecarInjectorWebhook().GetRewriteAppHTTPProbe())
-	sidecar := FindSidecar(pod.Spec.Containers)
-
 	// We don't have to escape json encoding here when using golang libraries.
-	if rewrite && sidecar != nil {
+	if rewrite {
 		if prober := DumpAppProbers(&pod.Spec, req.meshConfig.GetDefaultConfig().GetStatusPort()); prober != "" {
 			sidecar.Env = append(sidecar.Env, corev1.EnvVar{Name: status.KubeAppProberEnvName, Value: prober})
 		}
diff --git a/prow/config/mixedlb-service.yaml b/prow/config/mixedlb-service.yaml
index d105456ddd..850fbd5d6e 100644
--- a/prow/config/mixedlb-service.yaml
+++ b/prow/config/mixedlb-service.yaml
@@ -5,6 +5,7 @@ apiVersion: kind.x-k8s.io/v1alpha4
 featureGates:
   MixedProtocolLBService: true
   EndpointSlice: true
+  GRPCContainerProbe: true
 kubeadmConfigPatches:
   - |
     apiVersion: kubeadm.k8s.io/v1beta2
diff --git a/releasenotes/notes/grpc-probe.yaml b/releasenotes/notes/grpc-probe.yaml
new file mode 100644
index 0000000000..fdac969889
--- /dev/null
+++ b/releasenotes/notes/grpc-probe.yaml
@@ -0,0 +1,6 @@
+apiVersion: release-notes/v2
+kind: feature
+area: traffic-management
+releaseNotes:
+- |
+  **Added** support rewriting gRPC probes
\ No newline at end of file


Test patch:
diff --git a/pilot/cmd/pilot-agent/status/server_test.go b/pilot/cmd/pilot-agent/status/server_test.go
index 91d5135aa6..1ba45b68db 100644
--- a/pilot/cmd/pilot-agent/status/server_test.go
+++ b/pilot/cmd/pilot-agent/status/server_test.go
@@ -35,6 +35,9 @@ import (
 
 	"github.com/prometheus/common/expfmt"
 	"github.com/prometheus/prometheus/pkg/textparse"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/health"
+	grpcHealth "google.golang.org/grpc/health/grpc_health_v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 
 	"istio.io/istio/pilot/cmd/pilot-agent/status/ready"
@@ -108,10 +111,15 @@ func TestNewServer(t *testing.T) {
 		{
 			probe: `{"/app-health/hello-world/readyz": {"tcpSocket": {"port": 8888}}}`,
 		},
-		// probes must be tcp or http, not both
+		// probes must be one of tcp, http or gRPC
 		{
 			probe: `{"/app-health/hello-world/readyz": {"tcpSocket": {"port": 8888}, "httpGet": {"path": "/", "port": 7777}}}`,
-			err:   "must be either httpGet or tcpSocket",
+			err:   "must be one of type httpGet, tcpSocket or gRPC",
+		},
+		// probes must be one of tcp, http or gRPC
+		{
+			probe: `{"/app-health/hello-world/readyz": {"grpc": {"port": 8888}, "httpGet": {"path": "/", "port": 7777}}}`,
+			err:   "must be one of type httpGet, tcpSocket or gRPC",
 		},
 		// Port is not Int typed (tcpSocket).
 		{
@@ -147,6 +155,22 @@ func TestNewServer(t *testing.T) {
 			probe: `{"/app-health/hello-world/readyz": {"httpGet": {"path": "hello/sunnyvale", "port": 8080}},
 "/app-health/business/livez": {"httpGet": {"port": 9090}}}`,
 		},
+		// A valid gRPC probe.
+		{
+			probe: `{"/app-health/hello-world/readyz": {"gRPC": {"port": 8080}}}`,
+		},
+		// A valid gRPC probe with null service.
+		{
+			probe: `{"/app-health/hello-world/readyz": {"gRPC": {"port": 8080, "service": null}}}`,
+		},
+		// A valid gRPC probe with service.
+		{
+			probe: `{"/app-health/hello-world/readyz": {"gRPC": {"port": 8080, "service": "foo"}}}`,
+		},
+		// A valid gRPC probe with service and timeout.
+		{
+			probe: `{"/app-health/hello-world/readyz": {"gRPC": {"port": 8080, "service": "foo"}, "timeoutSeconds": 10}}`,
+		},
 	}
 	for _, tc := range testCases {
 		_, err := NewServer(Options{
@@ -840,6 +864,119 @@ func TestHttpsAppProbe(t *testing.T) {
 	}
 }
 
+func TestGRPCAppProbe(t *testing.T) {
+	appServer := grpc.NewServer()
+	healthServer := health.NewServer()
+	healthServer.SetServingStatus("serving-svc", grpcHealth.HealthCheckResponse_SERVING)
+	healthServer.SetServingStatus("unknown-svc", grpcHealth.HealthCheckResponse_UNKNOWN)
+	healthServer.SetServingStatus("not-serving-svc", grpcHealth.HealthCheckResponse_NOT_SERVING)
+	grpcHealth.RegisterHealthServer(appServer, healthServer)
+
+	listener, err := net.Listen("tcp", ":0")
+	if err != nil {
+		t.Errorf("failed to allocate unused port %v", err)
+	}
+	go appServer.Serve(listener)
+	defer appServer.GracefulStop()
+
+	appPort := listener.Addr().(*net.TCPAddr).Port
+	// Starts the pilot agent status server.
+	server, err := NewServer(Options{
+		StatusPort: 0,
+		KubeAppProbers: fmt.Sprintf(`
+{
+    "/app-health/foo/livez": {
+        "grpc": {
+            "port": %v, 
+            "service": null
+        }, 
+        "timeoutSeconds": 1
+    }, 
+    "/app-health/foo/readyz": {
+        "grpc": {
+            "port": %v, 
+            "service": "not-serving-svc"
+        }, 
+        "timeoutSeconds": 1
+    }, 
+    "/app-health/bar/livez": {
+        "grpc": {
+            "port": %v, 
+            "service": "serving-svc"
+        }, 
+        "timeoutSeconds": 10
+    }, 
+    "/app-health/bar/readyz": {
+        "grpc": {
+            "port": %v, 
+            "service": "unknown-svc"
+        }, 
+        "timeoutSeconds": 10
+    }
+}`, appPort, appPort, appPort, appPort),
+	})
+	if err != nil {
+		t.Errorf("failed to create status server %v", err)
+		return
+	}
+	go server.Run(context.Background())
+
+	var statusPort uint16
+	if err := retry.UntilSuccess(func() error {
+		server.mutex.RLock()
+		statusPort = server.statusPort
+		server.mutex.RUnlock()
+		if statusPort == 0 {
+			return fmt.Errorf("no port allocated")
+		}
+		return nil
+	}); err != nil {
+		t.Fatalf("failed to getport: %v", err)
+	}
+	t.Logf("status server starts at port %v, app starts at port %v", statusPort, appPort)
+
+	testCases := []struct {
+		probePath  string
+		statusCode int
+	}{
+		{
+			probePath:  fmt.Sprintf(":%v/bad-path-should-be-disallowed", statusPort),
+			statusCode: http.StatusNotFound,
+		},
+		{
+			probePath:  fmt.Sprintf(":%v/app-health/foo/livez", statusPort),
+			statusCode: http.StatusOK,
+		},
+		{
+			probePath:  fmt.Sprintf(":%v/app-health/foo/readyz", statusPort),
+			statusCode: http.StatusInternalServerError,
+		},
+		{
+			probePath:  fmt.Sprintf(":%v/app-health/bar/livez", statusPort),
+			statusCode: http.StatusOK,
+		},
+		{
+			probePath:  fmt.Sprintf(":%v/app-health/bar/readyz", statusPort),
+			statusCode: http.StatusInternalServerError,
+		},
+	}
+	for _, tc := range testCases {
+		client := http.Client{}
+		req, err := http.NewRequest("GET", fmt.Sprintf("http://localhost%s", tc.probePath), nil)
+		if err != nil {
+			t.Errorf("[%v] failed to create request", tc.probePath)
+		}
+		resp, err := client.Do(req)
+		if err != nil {
+			t.Fatal("request failed")
+		}
+		defer resp.Body.Close()
+		if resp.StatusCode != tc.statusCode {
+			t.Errorf("[%v] unexpected status code, want = %v, got = %v", tc.probePath, tc.statusCode, resp.StatusCode)
+		}
+	}
+}
+
 func TestProbeHeader(t *testing.T) {
 	headerChecker := func(t *testing.T, header http.Header) net.Listener {
 		listener, err := net.Listen("tcp", ":0")
diff --git a/pkg/kube/inject/app_probe_test.go b/pkg/kube/inject/app_probe_test.go
index 35b94e949c..4031f2fa70 100644
--- a/pkg/kube/inject/app_probe_test.go
+++ b/pkg/kube/inject/app_probe_test.go
@@ -14,12 +14,15 @@
 package inject
 
 import (
+	"reflect"
 	"testing"
 
 	"github.com/gogo/protobuf/types"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 
 	"istio.io/api/annotation"
+	"istio.io/istio/pkg/test"
 )
 
 func TestFindSidecar(t *testing.T) {
@@ -110,3 +113,255 @@ func TestShouldRewriteAppHTTPProbers(t *testing.T) {
 		}
 	}
 }
+
+func TestDumpAppGRPCProbers(t *testing.T) {
+	svc := "foo"
+	for _, tc := range []struct {
+		name     string
+		podSpec  *corev1.PodSpec
+		expected string
+	}{
+		{
+			name: "simple gRPC liveness probe",
+			podSpec: &corev1.PodSpec{
+				Containers: []corev1.Container{
+					{
+						Name: "foo",
+						LivenessProbe: &corev1.Probe{
+							ProbeHandler: corev1.ProbeHandler{
+								GRPC: &corev1.GRPCAction{
+									Port: 1234,
+								},
+							},
+						},
+					},
+				},
+			},
+			expected: `
+{
+    "/app-health/foo/livez": {
+        "grpc": {
+            "port": 1234,
+            "service": null
+        }
+    }
+}`,
+		},
+		{
+			name: "gRPC readiness probe with service",
+			podSpec: &corev1.PodSpec{
+				Containers: []corev1.Container{
+					{
+						Name: "bar",
+						ReadinessProbe: &corev1.Probe{
+							ProbeHandler: corev1.ProbeHandler{
+								GRPC: &corev1.GRPCAction{
+									Port:    1234,
+									Service: &svc,
+								},
+							},
+						},
+					},
+				},
+			},
+			expected: `
+{
+    "/app-health/bar/readyz": {
+        "grpc": {
+            "port": 1234,
+            "service": "foo"
+        }
+    }
+}`,
+		},
+		{
+			name: "gRPC startup probe with service and timeout",
+			podSpec: &corev1.PodSpec{
+				Containers: []corev1.Container{
+					{
+						Name: "foo",
+						StartupProbe: &corev1.Probe{
+							ProbeHandler: corev1.ProbeHandler{
+								GRPC: &corev1.GRPCAction{
+									Port:    1234,
+									Service: &svc,
+								},
+							},
+							TimeoutSeconds: 10,
+						},
+					},
+				},
+			},
+			expected: `
+{
+    "/app-health/foo/startupz": {
+        "grpc": {
+            "port": 1234,
+            "service": "foo"
+        },
+        "timeoutSeconds": 10
+    }
+}`,
+		},
+	} {
+		got := DumpAppProbers(tc.podSpec, 15020)
+		test.JSONEquals(t, got, tc.expected)
+	}
+}
+
+func TestPatchRewriteProbe(t *testing.T) {
+	svc := "foo"
+	annotations := map[string]string{}
+	statusPort := intstr.FromInt(15020)
+	for _, tc := range []struct {
+		name        string
+		pod         *corev1.Pod
+		annotations map[string]string
+		expected    *corev1.Pod
+	}{
+		{
+			name: "pod with no probes",
+			pod: &corev1.Pod{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name: "foo",
+						},
+					},
+				},
+			},
+			expected: &corev1.Pod{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name: "foo",
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "pod with a gRPC liveness probe",
+			pod: &corev1.Pod{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name: "foo",
+							LivenessProbe: &corev1.Probe{
+								ProbeHandler: corev1.ProbeHandler{
+									GRPC: &corev1.GRPCAction{
+										Port: 1234,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			expected: &corev1.Pod{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name: "foo",
+							LivenessProbe: &corev1.Probe{
+								ProbeHandler: corev1.ProbeHandler{
+									HTTPGet: &corev1.HTTPGetAction{
+										Path: "/app-health/foo/livez",
+										Port: statusPort,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "pod with gRPC liveness,readiness,startup probes",
+			pod: &corev1.Pod{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name: "foo",
+							LivenessProbe: &corev1.Probe{
+								ProbeHandler: corev1.ProbeHandler{
+									GRPC: &corev1.GRPCAction{
+										Port:    1234,
+										Service: &svc,
+									},
+								},
+							},
+							ReadinessProbe: &corev1.Probe{
+								ProbeHandler: corev1.ProbeHandler{
+									GRPC: &corev1.GRPCAction{
+										Port:    1235,
+										Service: &svc,
+									},
+								},
+								TimeoutSeconds: 10,
+							},
+						},
+						{
+							Name: "bar",
+							StartupProbe: &corev1.Probe{
+								ProbeHandler: corev1.ProbeHandler{
+									GRPC: &corev1.GRPCAction{
+										Port: 1236,
+									},
+								},
+								TimeoutSeconds:      20,
+								PeriodSeconds:       10,
+								InitialDelaySeconds: 10,
+							},
+						},
+					},
+				},
+			},
+			expected: &corev1.Pod{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name: "foo",
+							LivenessProbe: &corev1.Probe{
+								ProbeHandler: corev1.ProbeHandler{
+									HTTPGet: &corev1.HTTPGetAction{
+										Path: "/app-health/foo/livez",
+										Port: statusPort,
+									},
+								},
+							},
+							ReadinessProbe: &corev1.Probe{
+								ProbeHandler: corev1.ProbeHandler{
+									HTTPGet: &corev1.HTTPGetAction{
+										Path: "/app-health/foo/readyz",
+										Port: statusPort,
+									},
+								},
+								TimeoutSeconds: 10,
+							},
+						},
+						{
+							Name: "bar",
+							StartupProbe: &corev1.Probe{
+								ProbeHandler: corev1.ProbeHandler{
+									HTTPGet: &corev1.HTTPGetAction{
+										Path: "/app-health/bar/startupz",
+										Port: statusPort,
+									},
+								},
+								TimeoutSeconds:      20,
+								PeriodSeconds:       10,
+								InitialDelaySeconds: 10,
+							},
+						},
+					},
+				},
+			},
+		},
+	} {
+		patchRewriteProbe(annotations, tc.pod, 15020)
+		if !reflect.DeepEqual(tc.pod, tc.expected) {
+			t.Errorf("[%v] failed, want %v, got %v", tc.name, tc.expected, tc.pod)
+		}
+	}
+}
diff --git a/pkg/test/echo/server/endpoint/grpc.go b/pkg/test/echo/server/endpoint/grpc.go
index 00cfa73d5c..d959924bfe 100644
--- a/pkg/test/echo/server/endpoint/grpc.go
+++ b/pkg/test/echo/server/endpoint/grpc.go
@@ -31,6 +31,8 @@ import (
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
 	xdscreds "google.golang.org/grpc/credentials/xds"
+	"google.golang.org/grpc/health"
+	grpcHealth "google.golang.org/grpc/health/grpc_health_v1"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/peer"
 	"google.golang.org/grpc/reflection"
@@ -113,6 +115,10 @@ func (s *grpcInstance) Start(onReady OnReadyFunc) error {
 	}
 	s.server = s.newServer(opts...)
 
+	// add the standard grpc health check
+	healthServer := health.NewServer()
+	grpcHealth.RegisterHealthServer(s.server, healthServer)
+
 	proto.RegisterEchoTestServiceServer(s.server, &grpcHandler{
 		Config: s.Config,
 	})
@@ -131,7 +137,10 @@ func (s *grpcInstance) Start(onReady OnReadyFunc) error {
 	}()
 
 	// Notify the WaitGroup once the port has transitioned to ready.
-	go s.awaitReady(onReady, listener)
+	go s.awaitReady(func() {
+		healthServer.SetServingStatus("", grpcHealth.HealthCheckResponse_SERVING)
+		onReady()
+	}, listener)
 
 	return nil
 }
diff --git a/pkg/test/framework/components/echo/config.go b/pkg/test/framework/components/echo/config.go
index 0ce220cf06..590f3b5e9d 100644
--- a/pkg/test/framework/components/echo/config.go
+++ b/pkg/test/framework/components/echo/config.go
@@ -106,6 +106,9 @@ type Config struct {
 	// ReadinessTCPPort if set, use this port for the TCP readiness probe (instead of using a HTTP probe).
 	ReadinessTCPPort string
 
+	// ReadinessGRPCPort if set, use this port for the GRPC readiness probe (instead of using a HTTP probe).
+	ReadinessGRPCPort string
+
 	// Subsets contains the list of Subsets config belonging to this echo
 	// service instance.
 	Subsets []SubsetConfig
diff --git a/pkg/test/framework/components/echo/kube/deployment.go b/pkg/test/framework/components/echo/kube/deployment.go
index 8eb50ec1a5..374551f30b 100644
--- a/pkg/test/framework/components/echo/kube/deployment.go
+++ b/pkg/test/framework/components/echo/kube/deployment.go
@@ -248,6 +248,9 @@ spec:
 {{- if $.ReadinessTCPPort }}
           tcpSocket:
             port: {{ $.ReadinessTCPPort }}
+{{- else if $.ReadinessGRPCPort }}
+          grpc:
+            port: {{ $.ReadinessGRPCPort }}			
 {{- else }}
           httpGet:
             path: /
@@ -693,6 +696,7 @@ func templateParams(cfg echo.Config, imgSettings *image.Settings, settings *reso
 		"Namespace":          namespace,
 		"ImagePullSecret":    imagePullSecret,
 		"ReadinessTCPPort":   cfg.ReadinessTCPPort,
+		"ReadinessGRPCPort":  cfg.ReadinessGRPCPort,
 		"VM": map[string]interface{}{
 			"Image": vmImage,
 		},
diff --git a/pkg/test/framework/features/features.yaml b/pkg/test/framework/features/features.yaml
index 51641c7735..96d5d52622 100644
--- a/pkg/test/framework/features/features.yaml
+++ b/pkg/test/framework/features/features.yaml
@@ -68,6 +68,7 @@ features:
       analysis:
         line-numbers:
       tcp-probe:
+      grpc-probe:
     helpers:
       add-to-mesh:
       remove-from-mesh:
diff --git a/tests/integration/pilot/grpc_probe_test.go b/tests/integration/pilot/grpc_probe_test.go
new file mode 100644
index 0000000000..65f7eb1e8e
--- /dev/null
+++ b/tests/integration/pilot/grpc_probe_test.go
@@ -0,0 +1,101 @@
+//go:build integ
+// +build integ
+
+//  Copyright Istio Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package pilot
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"istio.io/istio/pkg/config/protocol"
+	"istio.io/istio/pkg/test/framework"
+	"istio.io/istio/pkg/test/framework/components/echo"
+	"istio.io/istio/pkg/test/framework/components/echo/echoboot"
+	"istio.io/istio/pkg/test/framework/components/namespace"
+)
+
+func TestGRPCProbe(t *testing.T) {
+	framework.NewTest(t).
+		Features("usability.observability.grpc-probe").
+		Run(func(t framework.TestContext) {
+			ns := namespace.NewOrFail(t, t, namespace.Config{Prefix: "grpc-probe", Inject: true})
+			// apply strict mtls
+			t.ConfigKube().ApplyYAMLOrFail(t, ns.Name(), fmt.Sprintf(`
+apiVersion: security.istio.io/v1beta1
+kind: PeerAuthentication
+metadata:
+  name: grpc-probe-mtls
+  namespace: %s
+spec:
+  mtls:
+    mode: STRICT`, ns.Name()))
+
+			for _, testCase := range []struct {
+				name     string
+				rewrite  bool
+				ready    bool
+				openPort bool
+			}{
+				{name: "norewrite-unready", rewrite: false, ready: false, openPort: true},
+				{name: "rewrite-unready", rewrite: true, ready: false, openPort: false},
+				{name: "rewrite-ready", rewrite: true, ready: true, openPort: true},
+			} {
+				t.NewSubTest(testCase.name).Run(func(t framework.TestContext) {
+					runGRPCProbeDeployment(t, ns, testCase.name, testCase.rewrite, testCase.ready, testCase.openPort)
+				})
+			}
+		})
+}
+
+func runGRPCProbeDeployment(ctx framework.TestContext, ns namespace.Instance, //nolint:interfacer
+	name string, rewrite bool, wantReady bool, openPort bool) {
+	ctx.Helper()
+
+	var grpcProbe echo.Instance
+	cfg := echo.Config{
+		Namespace:         ns,
+		Service:           name,
+		ReadinessGRPCPort: "1234",
+		Subsets: []echo.SubsetConfig{
+			{
+				Annotations: echo.NewAnnotations().SetBool(echo.SidecarRewriteAppHTTPProbers, rewrite),
+			},
+		},
+	}
+
+	if openPort {
+		cfg.Ports = []echo.Port{{
+			Name:         "readiness-grpc-port",
+			Protocol:     protocol.GRPC,
+			ServicePort:  1234,
+			InstancePort: 1234,
+		}}
+	}
+
+	// Negative test, we expect the grpc readiness check fails, so set a timeout duration.
+	if !wantReady {
+		cfg.ReadinessTimeout = time.Second * 15
+	}
+	_, err := echoboot.NewBuilder(ctx).
+		With(&grpcProbe, cfg).
+		Build()
+	gotReady := err == nil
+	if gotReady != wantReady {
+		ctx.Errorf("grpcProbe app %v, got error %v, want ready = %v", name, err, wantReady)
+	}
+}