Repository: GoogleCloudPlatform/cloud-sql-proxy-operator
Pull request #228 (2023-02-28 04:50:21)

test: Demonstrate e2e tests for private IP databases and GKE clusters.
Demonstrate that the Private IP operator flag correctly allows the proxy to connect to a private IP database.

hessjcg: /gcbrun

Patch:
diff --git a/.github/renovate.json b/.github/renovate.json
index 8af344f..1e7c11f 100644
--- a/.github/renovate.json
+++ b/.github/renovate.json
@@ -1,22 +1,56 @@
 {
   "extends": [
     "config:base",
-    ":semanticCommitTypeAll(chore)"
+    ":semanticCommitTypeAll(chore)",
+    ":disableDependencyDashboard"
+  ],
+  "ignorePresets": [
+    ":semanticPrefixFixDepsChoreOthers"
   ],
-  "ignorePresets": [":semanticPrefixFixDepsChoreOthers"],
   "prConcurrentLimit": 0,
-  "rebaseStalePrs": true,
+  "rebaseWhen": "behind-base-branch",
   "dependencyDashboard": true,
-  "semanticCommits": true,
+  "semanticCommits": "enabled",
   "postUpdateOptions": [
     "gomodTidy"
   ],
-  "ignoreDeps": [
-      "golang.org/x/net"
+  "packageRules": [
+    {
+      "description": "Disable MAJOR update types",
+      "matchUpdateTypes": [
+        "major"
+      ],
+      "enabled": false,
+    },
+    {
+      "matchUpdateTypes": ["minor", "patch", "pin", "digest"],
+      "automerge": true,
+      "automergeType": "pr",
+      "automergeStrategy": "rebase"
+    }
   ],
   "force": {
     "constraints": {
       "go": "1.20"
     }
-  }
+  },
+  "regexManagers": [
+    {
+      "fileMatch": ["^Makefile$"],
+      "matchStrings": [
+        "(?<envName>.*?_VERSION)\\s*=\\s*(?<currentValue>.*?)\\s*# renovate datasource=(?<datasource>.*?) depName=(?<depName>.*?)\\n"
+      ],
+      "depNameTemplate": "{{{depName}}}",
+      "datasourceTemplate": "{{{datasource}}}",
+      "versioningTemplate": "semver"
+    },
+    {
+      "fileMatch": ["podspec_updates.go$"],
+      "matchStrings": [
+        "DefaultProxyImage = \"(?<depName>gcr.io/cloud-sql-connectors/cloud-sql-proxy):(?<currentValue>.*?)\"\\n"
+      ],
+      "depNameTemplate": "{{{depName}}}",
+      "datasourceTemplate": "docker",
+    }
+  ]
 }
diff --git a/Makefile b/Makefile
index 46f4c67..75b2042 100644
--- a/Makefile
+++ b/Makefile
@@ -210,6 +210,7 @@ deploy_operator: kustomize kubectl # Deploy controller to the K8s cluster using
 	cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG)
 	$(KUSTOMIZE) build config/default | $(KUBECTL) apply -f -
 	$(E2E_KUBECTL) rollout status deployment -n cloud-sql-proxy-operator-system cloud-sql-proxy-operator-controller-manager --timeout=90s
+	$(E2E_PRIVATE_KUBECTL) rollout status deployment -n cloud-sql-proxy-operator-system cloud-sql-proxy-operator-controller-manager --timeout=90s
 
 ##
 # Update installer
@@ -235,6 +236,7 @@ installer/install.sh: ## Build install shell script to deploy the operator
 # This is the file where Terraform will write the kubeconfig.yaml for the
 # GKE cluster.
 KUBECONFIG_E2E ?= $(PWD)/bin/e2e-kubeconfig.yaml
+PRIVATE_KUBECONFIG_E2E ?= $(PWD)/bin/e2e-private-kubeconfig.yaml
 
 # This is the file where Terraform will write the kubeconfig.yaml for the
 # GKE cluster.
@@ -242,7 +244,9 @@ ENVIRONMENT_NAME ?= $(shell whoami)
 
 # kubectl command with proper environment vars set
 E2E_KUBECTL_ENV = USE_GKE_E2E_AUTH_PLUGIN=True KUBECONFIG=$(KUBECONFIG_E2E)
+E2E_PRIVATE_KUBECTL_ENV = USE_GKE_E2E_AUTH_PLUGIN=True KUBECONFIG=$(PRIVATE_KUBECONFIG_E2E)
 E2E_KUBECTL = $(E2E_KUBECTL_ENV) $(KUBECTL)
+E2E_PRIVATE_KUBECTL = $(E2E_PRIVATE_KUBECTL_ENV) $(KUBECTL)
 
 # This is the file where Terraform will write the URL to the e2e container registry
 E2E_DOCKER_URL_FILE :=$(PWD)/bin/gcloud-docker-repo.url
@@ -287,6 +291,7 @@ e2e_cluster_job: e2e_project terraform # Build infrastructure for e2e tests in t
 	PROJECT_DIR=$(PWD) \
   		E2E_PROJECT_ID=$(E2E_PROJECT_ID) \
   		KUBECONFIG_E2E=$(KUBECONFIG_E2E) \
+  		PRIVATE_KUBECONFIG_E2E=$(PRIVATE_KUBECONFIG_E2E) \
   		E2E_DOCKER_URL_FILE=$(E2E_DOCKER_URL_FILE) \
   		ENVIRONMENT_NAME=$(ENVIRONMENT_NAME) \
   		NODEPOOL_SERVICEACCOUNT_EMAIL=$(NODEPOOL_SERVICEACCOUNT_EMAIL) \
@@ -300,6 +305,7 @@ e2e_cluster: e2e_project terraform # Build infrastructure for e2e tests
 	PROJECT_DIR=$(PWD) \
   		E2E_PROJECT_ID=$(E2E_PROJECT_ID) \
   		KUBECONFIG_E2E=$(KUBECONFIG_E2E) \
+  		PRIVATE_KUBECONFIG_E2E=$(PRIVATE_KUBECONFIG_E2E) \
   		E2E_DOCKER_URL_FILE=$(E2E_DOCKER_URL_FILE) \
   		ENVIRONMENT_NAME=$(ENVIRONMENT_NAME) \
   		TESTINFRA_JSON_FILE=$(LOCALBIN)/testinfra.json \
@@ -310,6 +316,7 @@ e2e_cluster_destroy: e2e_project terraform # Destroy the infrastructure for e2e
 	PROJECT_DIR=$(PWD) \
   		E2E_PROJECT_ID=$(E2E_PROJECT_ID) \
   		KUBECONFIG_E2E=$(KUBECONFIG_E2E) \
+  		PRIVATE_KUBECONFIG_E2E=$(PRIVATE_KUBECONFIG_E2E) \
   		E2E_DOCKER_URL_FILE=$(E2E_DOCKER_URL_FILE) \
   		ENVIRONMENT_NAME=$(ENVIRONMENT_NAME) \
   		TESTINFRA_JSON_FILE=$(LOCALBIN)/testinfra.json \
@@ -318,7 +325,11 @@ e2e_cluster_destroy: e2e_project terraform # Destroy the infrastructure for e2e
 .PHONY: e2e_cert_manager_deploy
 e2e_cert_manager_deploy: e2e_project helm # Deploy the certificate manager
 	helm repo add jetstack https://charts.jetstack.io --kubeconfig=$(KUBECONFIG_E2E)
+	helm repo add jetstack https://charts.jetstack.io --kubeconfig=$(PRIVATE_KUBECONFIG_E2E)
+
 	helm repo update --kubeconfig=$(KUBECONFIG_E2E)
+	helm repo update --kubeconfig=$(PRIVATE_KUBECONFIG_E2E)
+
 	helm get all -n cert-manager cert-manager --kubeconfig=$(KUBECONFIG_E2E)  || \
 		helm --kubeconfig=$(KUBECONFIG_E2E) install \
 			cert-manager jetstack/cert-manager \
@@ -328,21 +339,31 @@ e2e_cert_manager_deploy: e2e_project helm # Deploy the certificate manager
 			--create-namespace \
 			--set global.leaderElection.namespace=cert-manager \
 			--set installCRDs=true
+	helm get all -n cert-manager cert-manager --kubeconfig=$(PRIVATE_KUBECONFIG_E2E)  || \
+		helm --kubeconfig=$(PRIVATE_KUBECONFIG_E2E) install \
+			cert-manager jetstack/cert-manager \
+			--kubeconfig=$(PRIVATE_KUBECONFIG_E2E) \
+			--namespace cert-manager \
+			--version "$(CERT_MANAGER_VERSION)" \
+			--create-namespace \
+			--set global.leaderElection.namespace=cert-manager \
+			--set installCRDs=true
 
 
 .PHONY: e2e_install_crd
 e2e_install_crd: generate e2e_project kustomize kubectl $(E2E_WORK_DIR) # Install CRDs into the GKE cluster
 	$(KUSTOMIZE) build config/crd > $(E2E_WORK_DIR)/crd.yaml
 	$(E2E_KUBECTL) apply -f $(E2E_WORK_DIR)/crd.yaml
-
-
+	$(E2E_PRIVATE_KUBECTL) apply -f $(E2E_WORK_DIR)/crd.yaml
 
 .PHONY: e2e_deploy
 e2e_deploy: e2e_project kustomize kubectl $(E2E_WORK_DIR) # Deploy the operator to the GKE cluster
 	cd config/manager && $(KUSTOMIZE) edit set image controller=$(E2E_OPERATOR_URL)
 	$(KUSTOMIZE) build config/default > $(E2E_WORK_DIR)/operator.yaml
 	$(E2E_KUBECTL) apply -f $(E2E_WORK_DIR)/operator.yaml
+	$(E2E_PRIVATE_KUBECTL) apply -f $(E2E_WORK_DIR)/operator.yaml
 	$(E2E_KUBECTL) rollout status deployment -n cloud-sql-proxy-operator-system cloud-sql-proxy-operator-controller-manager --timeout=90s
+	$(E2E_PRIVATE_KUBECTL) rollout status deployment -n cloud-sql-proxy-operator-system cloud-sql-proxy-operator-controller-manager --timeout=90s
 
 
 # Note: `go test --count=1` is used to make sure go actually runs the tests every
@@ -360,10 +381,14 @@ e2e_cleanup_test_namespaces: e2e_project kustomize kubectl # remove e2e test nam
 	( $(E2E_KUBECTL) get ns -o=name | \
 		grep namespace/test | \
 		$(E2E_KUBECTL_ENV) xargs $(KUBECTL) delete ) || true
+	( $(E2E_PRIVATE_KUBECTL) get ns -o=name | \
+		grep namespace/test | \
+		$(E2E_PRIVATE_KUBECTL_ENV) xargs $(KUBECTL) delete ) || true
 
 .PHONY: e2e_undeploy
 e2e_undeploy: e2e_project kustomize kubectl $(E2E_WORK_DIR) # Remove the operator from the GKE cluster
 	$(E2E_KUBECTL) delete -f $(E2E_WORK_DIR)/operator.yaml
+	$(E2E_PRIVATE_KUBECTL) delete -f $(E2E_WORK_DIR)/operator.yaml
 
 ###
 # Build the operator docker image and push it to the
@@ -417,19 +442,24 @@ GOLANGCI_LINT ?= $(LOCALBIN)/golangci-lint
 CRD_REF_DOCS ?= $(LOCALBIN)/crd-ref-docs
 
 ## Tool Versions
-CONTROLLER_TOOLS_VERSION ?= latest
-KUBECTL_VERSION ?= $(shell curl -L -s https://dl.k8s.io/release/stable.txt | tr -d '\n')
-TERRAFORM_VERSION ?= 1.2.7
-KUSTOMIZE_VERSION ?= v4.5.2
-ENVTEST_VERSION ?= latest
-GOLANGCI_LINT_VERSION ?= latest
-CRD_REF_DOCS_VERSION ?= latest
+# Important note: avoid adding spaces in the macro declarations as any
+# additional whitespace will break the renovate regex rules.
+
+KUBECTL_VERSION=v0.26.2# renovate datasource=github-tags depName=kubernetes/kubectl
+TERRAFORM_VERSION=v1.3.9# renovate datasource=github-tags depName=hashicorp/terraform
+
+CONTROLLER_TOOLS_VERSION=v0.11.3# renovate datasource=go depName=sigs.k8s.io/controller-tools
+CRD_REF_DOCS_VERSION=v0.0.8# renovate datasource=go depName=github.com/elastic/crd-ref-docs
+ENVTEST_VERSION=v0.0.0-20230301194117-e2d8821b277f# renovate datasource=go depName=sigs.k8s.io/controller-runtime/tools/setup-envtest
+GOLANGCI_LINT_VERSION=v1.51.2# renovate datasource=go depName=github.com/golangci/golangci-lint/cmd/golangci-lint
+
+KUSTOMIZE_VERSION=v4.5.2# don't manage with renovate, this repo has non-standard tags
 
 GOOS?=$(shell go env GOOS | tr -d '\n')
 GOARCH?=$(shell go env GOARCH | tr -d '\n')
 
 remove_tools:
-	rm -rf $(LOCALBIN)/*
+	rm -rf $(KUSTOMIZE) $(CONTROLLER_GEN) $(KUBECTL) $(ENVTEST) $(TERRAFORM) $(GOLANGCI_LINT) $(CRD_REF_DOCS)
 
 all_tools: kustomize controller-gen envtest kubectl terraform golangci-lint crd-ref-docs
 
@@ -466,7 +496,7 @@ $(KUBECTL): $(LOCALBIN)
 terraform: $(TERRAFORM) # Download terraform
 $(TERRAFORM): $(LOCALBIN)
 	test -s $@ || \
-		( curl -L -o $@.zip https://releases.hashicorp.com/terraform/$(TERRAFORM_VERSION)/terraform_$(TERRAFORM_VERSION)_$(GOOS)_$(GOARCH).zip && \
+		( curl -L -o $@.zip https://releases.hashicorp.com/terraform/$(subst v,,$(TERRAFORM_VERSION))/terraform_$(subst v,,$(TERRAFORM_VERSION))_$(GOOS)_$(GOARCH).zip && \
 		cd $(LOCALBIN) && unzip -o $@.zip && \
 		rm -f $@.zip && \
 		chmod a+x $@ && \
diff --git a/README.md b/README.md
index 73ed69b..2ae702f 100644
--- a/README.md
+++ b/README.md
@@ -14,9 +14,11 @@ to the matching workload pods.
 
 ## Installation
 
-Check for the latest version on the [releases page][releases] and use the
+Check for the latest version on the [releases page][latest-release] and use the
 following instructions. 
 
+[latest-release]: https://github.com/GoogleCloudPlatform/cloud-sql-proxy-operator/releases/latest
+
 Confirm that kubectl can connect to your kubernetes cluster.
 
 ```shell
diff --git a/config/crd/bases/cloudsql.cloud.google.com_authproxyworkloads.yaml b/config/crd/bases/cloudsql.cloud.google.com_authproxyworkloads.yaml
index 5a25846..346f4a6 100644
--- a/config/crd/bases/cloudsql.cloud.google.com_authproxyworkloads.yaml
+++ b/config/crd/bases/cloudsql.cloud.google.com_authproxyworkloads.yaml
@@ -935,6 +935,24 @@ spec:
                           description: HTTPPort the port for Prometheus and health check server. This sets the proxy container's CLI argument `--http-port`
                           format: int32
                           type: integer
+                        prometheus:
+                          description: Prometheus Enables Prometheus HTTP endpoint /metrics on localhost This sets the proxy container's CLI argument `--prometheus`
+                          type: boolean
+                        prometheusNamespace:
+                          description: PrometheusNamespace is used the provided Prometheus namespace for metrics This sets the proxy container's CLI argument `--prometheus-namespace`
+                          type: string
+                        quotaProject:
+                          description: QuotaProject Specifies the project to use for Cloud SQL Admin API quota tracking. The IAM principal must have the "serviceusage.services.use" permission for the given project. See https://cloud.google.com/service-usage/docs/overview and https://cloud.google.com/storage/docs/requester-pays This sets the proxy container's CLI argument `--quota-project`
+                          type: string
+                        telemetryPrefix:
+                          description: TelemetryPrefix is the prefix for Cloud Monitoring metrics. This sets the proxy container's CLI argument `--telemetry-prefix`
+                          type: string
+                        telemetryProject:
+                          description: TelemetryProject enables Cloud Monitoring and Cloud Trace with the provided project ID. This sets the proxy container's CLI argument `--telemetry-project`
+                          type: string
+                        telemetrySampleRate:
+                          description: TelemetrySampleRate is the Cloud Trace sample rate. A smaller number means more traces. This sets the proxy container's CLI argument `--telemetry-sample-rate`
+                          type: integer
                       type: object
                   type: object
                 instances:
diff --git a/docs/api.md b/docs/api.md
index 771d3ea..9e3af8e 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -120,6 +120,12 @@ _Appears in:_
 
 | Field | Description |
 | --- | --- |
+| `quotaProject` _string_ | QuotaProject Specifies the project to use for Cloud SQL Admin API quota tracking. The IAM principal must have the "serviceusage.services.use" permission for the given project. See https://cloud.google.com/service-usage/docs/overview and https://cloud.google.com/storage/docs/requester-pays This sets the proxy container's CLI argument `--quota-project` |
+| `prometheus` _boolean_ | Prometheus Enables Prometheus HTTP endpoint /metrics on localhost This sets the proxy container's CLI argument `--prometheus` |
+| `prometheusNamespace` _string_ | PrometheusNamespace is used the provided Prometheus namespace for metrics This sets the proxy container's CLI argument `--prometheus-namespace` |
+| `telemetryProject` _string_ | TelemetryProject enables Cloud Monitoring and Cloud Trace with the provided project ID. This sets the proxy container's CLI argument `--telemetry-project` |
+| `telemetryPrefix` _string_ | TelemetryPrefix is the prefix for Cloud Monitoring metrics. This sets the proxy container's CLI argument `--telemetry-prefix` |
+| `telemetrySampleRate` _integer_ | TelemetrySampleRate is the Cloud Trace sample rate. A smaller number means more traces. This sets the proxy container's CLI argument `--telemetry-sample-rate` |
 | `httpPort` _integer_ | HTTPPort the port for Prometheus and health check server. This sets the proxy container's CLI argument `--http-port` |
 | `disableTraces` _boolean_ | DisableTraces disables Cloud Trace testintegration (used with telemetryProject) This sets the proxy container's CLI argument `--disable-traces` |
 | `disableMetrics` _boolean_ | DisableMetrics disables Cloud Monitoring testintegration (used with telemetryProject) This sets the proxy container's CLI argument `--disable-metrics` |
diff --git a/infra/permissions/main.tf b/infra/permissions/main.tf
index f1de397..a33e4ca 100644
--- a/infra/permissions/main.tf
+++ b/infra/permissions/main.tf
@@ -43,6 +43,7 @@ locals {
     "replicapoolupdater.googleapis.com",
     "resourceviews.googleapis.com",
     "servicemanagement.googleapis.com",
+    "servicenetworking.googleapis.com",
     "sql-component.googleapis.com",
     "sqladmin.googleapis.com",
   "storage-api.googleapis.com"])
diff --git a/infra/resources/database.tf b/infra/resources/database.tf
index 2180c4f..d9f2cd9 100644
--- a/infra/resources/database.tf
+++ b/infra/resources/database.tf
@@ -24,7 +24,7 @@ resource "random_id" "db_password" {
 
 # See versions at https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database_instance#database_version
 resource "google_sql_database_instance" "instance" {
-  name             = "inst${random_id.db_name.hex}"
+  name             = "inst${random_id.db_name.hex}${var.environment_name}"
   project          = var.project_id
   region           = var.gcloud_region
   database_version = "POSTGRES_13"
@@ -44,7 +44,7 @@ resource "google_sql_database" "db" {
 
 # See versions at https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database_instance#database_version
 resource "google_sql_database_instance" "mysql" {
-  name             = "mysql${random_id.db_name.hex}"
+  name             = "mysql${random_id.db_name.hex}${var.environment_name}"
   project          = var.project_id
   region           = var.gcloud_region
   database_version = "MYSQL_8_0"
@@ -72,7 +72,7 @@ resource "google_sql_user" "mysql_user" {
 
 # See versions at https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database_instance#database_version
 resource "google_sql_database_instance" "mssql" {
-  name             = "mssql${random_id.db_name.hex}"
+  name             = "mssql${random_id.db_name.hex}${var.environment_name}"
   project          = var.project_id
   region           = var.gcloud_region
   database_version = "SQLSERVER_2019_EXPRESS"
diff --git a/infra/resources/main.tf b/infra/resources/main.tf
index 9a9377b..93a0dd9 100644
--- a/infra/resources/main.tf
+++ b/infra/resources/main.tf
@@ -27,6 +27,11 @@ terraform {
 provider "google" {
 }
 
+provider "google-beta" {
+  region = "us-central1"
+  zone   = "us-central1-a"
+}
+
 # Enable gcloud project APIs
 locals {
   standard_labels = {
@@ -63,6 +68,15 @@ locals {
       }
       kubeconfig = var.kubeconfig_path
     }
+    private = {
+      postgres = {
+        instance     = google_sql_database_instance.private_postgres.connection_name
+        dbName       = google_sql_database.private_db.name
+        rootUser     = "postgres"
+        rootPassword = random_id.db_password.hex
+      }
+      kubeconfig = var.private_kubeconfig_path
+    }
   }
 }
 
diff --git a/infra/resources/private-database.tf b/infra/resources/private-database.tf
new file mode 100644
index 0000000..8bc0e4c
--- /dev/null
+++ b/infra/resources/private-database.tf
@@ -0,0 +1,61 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+resource "random_id" "private_db_name_suffix" {
+  byte_length = 4
+}
+
+
+# See versions at https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database_instance#database_version
+resource "google_sql_database_instance" "private_postgres" {
+  provider = google-beta
+
+  name             = "privateinst${random_id.private_db_name_suffix.hex}${var.environment_name}"
+  project          = var.project_id
+  region           = var.gcloud_region
+  database_version = "POSTGRES_13"
+  depends_on       = [google_service_networking_connection.private_vpc_connection]
+
+  settings {
+    tier        = "db-f1-micro"
+    user_labels = local.standard_labels
+    ip_configuration {
+      ipv4_enabled                                  = false
+      private_network                               = google_compute_network.private_k8s_network.id
+      enable_private_path_for_google_cloud_services = true
+    }
+  }
+  deletion_protection = "true"
+  root_password       = random_id.db_password.hex
+}
+
+resource "google_sql_database" "private_db" {
+  name     = "db"
+  instance = google_sql_database_instance.private_postgres.name
+  project  = var.project_id
+}
+
+
+output "private_db_root_password" {
+  value = random_id.db_password.hex
+}
+output "private_db_instance_name" {
+  value = google_sql_database_instance.private_postgres.name
+}
+output "private_db_database_name" {
+  value = google_sql_database.private_db.name
+}
diff --git a/infra/resources/private-network.tf b/infra/resources/private-network.tf
new file mode 100644
index 0000000..9bed5e2
--- /dev/null
+++ b/infra/resources/private-network.tf
@@ -0,0 +1,66 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+resource "google_compute_network" "private_k8s_network" {
+  provider = google-beta
+  project  = var.project_id
+
+  name                    = "test-vpc-${var.environment_name}"
+  auto_create_subnetworks = false
+}
+
+resource "google_compute_subnetwork" "private_k8s_network" {
+  provider = google-beta
+  project  = var.project_id
+  region   = var.gcloud_region
+
+  name          = "test-vpc-subnetwork-${var.environment_name}"
+  ip_cidr_range = "10.2.0.0/16"
+  network       = google_compute_network.private_k8s_network.id
+
+  secondary_ip_range {
+    range_name    = "services-range"
+    ip_cidr_range = "192.168.1.0/24"
+  }
+
+  secondary_ip_range {
+    range_name    = "pod-ranges"
+    ip_cidr_range = "192.168.64.0/22"
+  }
+
+  secondary_ip_range {
+    range_name    = "nodepool-pod-ranges"
+    ip_cidr_range = "192.168.128.0/22"
+  }
+}
+
+resource "google_compute_global_address" "private_ip_address" {
+  provider = google-beta
+  project  = var.project_id
+
+  name          = "test-vpc-private-ip-address-${var.environment_name}"
+  purpose       = "VPC_PEERING"
+  address_type  = "INTERNAL"
+  prefix_length = 16
+  network       = google_compute_network.private_k8s_network.id
+}
+
+resource "google_service_networking_connection" "private_vpc_connection" {
+  provider = google-beta
+
+  network                 = google_compute_network.private_k8s_network.id
+  service                 = "servicenetworking.googleapis.com"
+  reserved_peering_ranges = [google_compute_global_address.private_ip_address.name]
+}
diff --git a/infra/resources/private_gke_cluster.tf b/infra/resources/private_gke_cluster.tf
new file mode 100644
index 0000000..ff1c861
--- /dev/null
+++ b/infra/resources/private_gke_cluster.tf
@@ -0,0 +1,137 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# From https://github.com/hashicorp/terraform-provider-kubernetes/blob/main/kubernetes/test-infra/gke/main.tf
+
+resource "google_container_cluster" "private" {
+  project            = var.project_id
+  name               = "operator-private-${var.environment_name}"
+  location           = var.gcloud_zone
+  min_master_version = data.google_container_engine_versions.supported.latest_master_version
+  initial_node_count = 2
+
+  // Alpha features are disabled by default and can be enabled by GKE for a particular GKE control plane version.
+  // Creating an alpha cluster enables all alpha features by default.
+  // Ref: https://cloud.google.com/kubernetes-engine/docs/concepts/feature-gates
+  enable_kubernetes_alpha = var.enable_alpha
+
+  // disalbe the default nodepool and specify node pools as
+  // separate terraform resources. This way if we
+  // change the nodepool config, we don't delete the cluster too
+  remove_default_node_pool = true
+  resource_labels          = local.standard_labels
+
+  network    = google_compute_network.private_k8s_network.id
+  subnetwork = google_compute_subnetwork.private_k8s_network.id
+
+  ip_allocation_policy {
+    services_secondary_range_name = google_compute_subnetwork.private_k8s_network.secondary_ip_range.0.range_name
+    cluster_secondary_range_name  = google_compute_subnetwork.private_k8s_network.secondary_ip_range.1.range_name
+  }
+}
+
+resource "google_container_node_pool" "private_preemptible_nodes" {
+  name               = "operator-private-nodes-${var.environment_name}"
+  cluster            = google_container_cluster.private.id
+  initial_node_count = var.workers_count
+  version            = data.google_container_engine_versions.supported.latest_node_version
+  location           = var.gcloud_zone
+
+  autoscaling {
+    max_node_count = 10
+    min_node_count = 2
+  }
+
+  management {
+    auto_repair  = var.enable_alpha ? false : true
+    auto_upgrade = var.enable_alpha ? false : true
+  }
+
+  network_config {
+    enable_private_nodes = false
+    pod_range            = google_compute_subnetwork.private_k8s_network.secondary_ip_range.2.range_name
+    create_pod_range     = false
+  }
+
+  node_config {
+    preemptible     = true
+    machine_type    = "e2-standard-8"
+    resource_labels = local.standard_labels
+
+    # Google recommends custom service accounts that have cloud-platform scope and permissions granted via IAM Roles.
+    service_account = var.nodepool_serviceaccount_email
+    oauth_scopes = [
+      "https://www.googleapis.com/auth/cloud-platform",
+      "https://www.googleapis.com/auth/compute",
+      "https://www.googleapis.com/auth/devstorage.read_only",
+      "https://www.googleapis.com/auth/logging.write",
+      "https://www.googleapis.com/auth/monitoring",
+      "https://www.googleapis.com/auth/sqlservice.admin",
+    ]
+  }
+}
+
+
+locals {
+  # This is the recommended way to produce a kubeconfig file from
+  # the Google Cloud GKE terraform resource.
+  private_kubeconfig = {
+    apiVersion = "v1"
+    kind       = "Config"
+    preferences = {
+      colors = true
+    }
+    current-context = google_container_cluster.private.name
+    contexts = [
+      {
+        name = google_container_cluster.private.name
+        context = {
+          cluster   = google_container_cluster.private.name
+          user      = var.nodepool_serviceaccount_email
+          namespace = "default"
+        }
+      }
+    ]
+    clusters = [
+      {
+        name = google_container_cluster.private.name
+        cluster = {
+          server                     = "https://${google_container_cluster.private.endpoint}"
+          certificate-authority-data = google_container_cluster.private.master_auth[0].cluster_ca_certificate
+        }
+      }
+    ]
+    users = [
+      {
+        name = var.nodepool_serviceaccount_email
+        user = {
+          exec = {
+            apiVersion         = "client.authentication.k8s.io/v1beta1"
+            command            = "gke-gcloud-auth-plugin"
+            installHint        = "Install gke-gcloud-auth-plugin for use with kubectl by following https://cloud.google.com/blog/products/containers-kubernetes/kubectl-auth-changes-in-gke"
+            provideClusterInfo = true
+          }
+        }
+      }
+    ]
+  }
+
+}
+
+resource "local_file" "private_kubeconfig" {
+  content  = yamlencode(local.private_kubeconfig)
+  filename = var.private_kubeconfig_path
+}
diff --git a/infra/resources/vars.tf b/infra/resources/vars.tf
index f1a4c03..9624ef0 100644
--- a/infra/resources/vars.tf
+++ b/infra/resources/vars.tf
@@ -28,6 +28,10 @@ variable "kubeconfig_path" {
   type        = string
   description = "The path to save the kubeconfig file"
 }
+variable "private_kubeconfig_path" {
+  type        = string
+  description = "The path to save the kubeconfig file"
+}
 variable "output_json_path" {
   type        = string
   description = "The path to save test-infra.json file, input for e2e tests"
diff --git a/infra/run.sh b/infra/run.sh
index a12767c..8c1a8ae 100755
--- a/infra/run.sh
+++ b/infra/run.sh
@@ -44,6 +44,8 @@
 #   E2E_PROJECT_ID - The Google Cloud project ID to act upon.
 #   KUBECONFIG_E2E - The output filename for the kubeconfig json file
 #     for the kubernetes cluster for the e2e environment.
+#   PRIVATE_KUBECONFIG_E2E - The output filename for the kubeconfig json file
+#     for the private ip kubernetes cluster for the e2e environment.
 #   E2E_DOCKER_URL_FILE - The output filename for a text file containing the
 #     URL to the docker container registry for the e2e test environment.
 #
@@ -84,6 +86,7 @@ function apply() {
         -var "gcloud_docker_url_file=$E2E_DOCKER_URL_FILE" \
         -var "project_id=$E2E_PROJECT_ID" \
         -var "kubeconfig_path=$KUBECONFIG_E2E" \
+        -var "private_kubeconfig_path=$PRIVATE_KUBECONFIG_E2E" \
         -var "environment_name=$ENVIRONMENT_NAME" \
         -var "nodepool_serviceaccount_email=$nodepool_serviceaccount_email"
 
@@ -97,6 +100,7 @@ function destroy() {
         -var "gcloud_docker_url_file=$E2E_DOCKER_URL_FILE" \
         -var "project_id=$E2E_PROJECT_ID" \
         -var "kubeconfig_path=$KUBECONFIG_E2E" \
+        -var "private_kubeconfig_path=$PRIVATE_KUBECONFIG_E2E" \
         -var "environment_name=$ENVIRONMENT_NAME" \
         -var "nodepool_serviceaccount_email=$nodepool_serviceaccount_email"
 }
@@ -137,6 +141,7 @@ EOF
       -var "gcloud_docker_url_file=$E2E_DOCKER_URL_FILE" \
       -var "project_id=$E2E_PROJECT_ID" \
       -var "kubeconfig_path=$KUBECONFIG_E2E" \
+      -var "private_kubeconfig_path=$PRIVATE_KUBECONFIG_E2E" \
       -var "environment_name=$ENVIRONMENT_NAME" \
       -var "nodepool_serviceaccount_email=$NODEPOOL_SERVICEACCOUNT_EMAIL"
 
@@ -178,6 +183,12 @@ if [[ -z "$KUBECONFIG_E2E" ]]; then
   FAIL=1
 fi
 
+#expects PRIVATE_KUBECONFIG_E2E
+if [[ -z "$PRIVATE_KUBECONFIG_E2E" ]]; then
+  echo "expects PRIVATE_KUBECONFIG_E2E to be set the location where kubeconfig should be written."
+  FAIL=1
+fi
+
 #expects $E2E_DOCKER_URL_FILE
 if [[ -z "$E2E_DOCKER_URL_FILE" ]]; then
   echo "expects E2E_DOCKER_URL_FILE to be set the location where docker url should be written."
diff --git a/installer/cloud-sql-proxy-operator.yaml b/installer/cloud-sql-proxy-operator.yaml
index b12d277..310ba0a 100644
--- a/installer/cloud-sql-proxy-operator.yaml
+++ b/installer/cloud-sql-proxy-operator.yaml
@@ -953,6 +953,24 @@ spec:
                           description: HTTPPort the port for Prometheus and health check server. This sets the proxy container's CLI argument `--http-port`
                           format: int32
                           type: integer
+                        prometheus:
+                          description: Prometheus Enables Prometheus HTTP endpoint /metrics on localhost This sets the proxy container's CLI argument `--prometheus`
+                          type: boolean
+                        prometheusNamespace:
+                          description: PrometheusNamespace is used the provided Prometheus namespace for metrics This sets the proxy container's CLI argument `--prometheus-namespace`
+                          type: string
+                        quotaProject:
+                          description: QuotaProject Specifies the project to use for Cloud SQL Admin API quota tracking. The IAM principal must have the "serviceusage.services.use" permission for the given project. See https://cloud.google.com/service-usage/docs/overview and https://cloud.google.com/storage/docs/requester-pays This sets the proxy container's CLI argument `--quota-project`
+                          type: string
+                        telemetryPrefix:
+                          description: TelemetryPrefix is the prefix for Cloud Monitoring metrics. This sets the proxy container's CLI argument `--telemetry-prefix`
+                          type: string
+                        telemetryProject:
+                          description: TelemetryProject enables Cloud Monitoring and Cloud Trace with the provided project ID. This sets the proxy container's CLI argument `--telemetry-project`
+                          type: string
+                        telemetrySampleRate:
+                          description: TelemetrySampleRate is the Cloud Trace sample rate. A smaller number means more traces. This sets the proxy container's CLI argument `--telemetry-sample-rate`
+                          type: integer
                       type: object
                   type: object
                 instances:
diff --git a/internal/api/v1alpha1/authproxyworkload_types.go b/internal/api/v1alpha1/authproxyworkload_types.go
index eaeabda..346319c 100644
--- a/internal/api/v1alpha1/authproxyworkload_types.go
+++ b/internal/api/v1alpha1/authproxyworkload_types.go
@@ -220,6 +220,39 @@ type AdminServerSpec struct {
 
 // TelemetrySpec specifies how the proxy container will expose telemetry.
 type TelemetrySpec struct {
+	// QuotaProject Specifies the project to use for Cloud SQL Admin API quota tracking.
+	// The IAM principal must have the "serviceusage.services.use" permission
+	// for the given project. See https://cloud.google.com/service-usage/docs/overview and
+	// https://cloud.google.com/storage/docs/requester-pays
+	// This sets the proxy container's CLI argument `--quota-project`
+	//+kubebuilder:validation:Optional
+	QuotaProject *string `json:"quotaProject,omitempty"`
+
+	// Prometheus Enables Prometheus HTTP endpoint /metrics on localhost
+	// This sets the proxy container's CLI argument `--prometheus`
+	//+kubebuilder:validation:Optional
+	Prometheus *bool `json:"prometheus,omitempty"`
+
+	// PrometheusNamespace is used the provided Prometheus namespace for metrics
+	// This sets the proxy container's CLI argument `--prometheus-namespace`
+	//+kubebuilder:validation:Optional
+	PrometheusNamespace *string `json:"prometheusNamespace,omitempty"`
+
+	// TelemetryProject enables Cloud Monitoring and Cloud Trace with the provided project ID.
+	// This sets the proxy container's CLI argument `--telemetry-project`
+	//+kubebuilder:validation:Optional
+	TelemetryProject *string `json:"telemetryProject,omitempty"`
+
+	// TelemetryPrefix is the prefix for Cloud Monitoring metrics.
+	// This sets the proxy container's CLI argument `--telemetry-prefix`
+	//+kubebuilder:validation:Optional
+	TelemetryPrefix *string `json:"telemetryPrefix,omitempty"`
+
+	// TelemetrySampleRate is the Cloud Trace sample rate. A smaller number means more traces.
+	// This sets the proxy container's CLI argument `--telemetry-sample-rate`
+	//+kubebuilder:validation:Optional
+	TelemetrySampleRate *int `json:"telemetrySampleRate,omitempty"`
+
 	// HTTPPort the port for Prometheus and health check server.
 	// This sets the proxy container's CLI argument `--http-port`
 	//+kubebuilder:validation:Optional
diff --git a/internal/api/v1alpha1/zz_generated.deepcopy.go b/internal/api/v1alpha1/zz_generated.deepcopy.go
index 55666d5..6ae0c0b 100644
--- a/internal/api/v1alpha1/zz_generated.deepcopy.go
+++ b/internal/api/v1alpha1/zz_generated.deepcopy.go
@@ -247,6 +247,36 @@ func (in *InstanceSpec) DeepCopy() *InstanceSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *TelemetrySpec) DeepCopyInto(out *TelemetrySpec) {
 	*out = *in
+	if in.QuotaProject != nil {
+		in, out := &in.QuotaProject, &out.QuotaProject
+		*out = new(string)
+		**out = **in
+	}
+	if in.Prometheus != nil {
+		in, out := &in.Prometheus, &out.Prometheus
+		*out = new(bool)
+		**out = **in
+	}
+	if in.PrometheusNamespace != nil {
+		in, out := &in.PrometheusNamespace, &out.PrometheusNamespace
+		*out = new(string)
+		**out = **in
+	}
+	if in.TelemetryProject != nil {
+		in, out := &in.TelemetryProject, &out.TelemetryProject
+		*out = new(string)
+		**out = **in
+	}
+	if in.TelemetryPrefix != nil {
+		in, out := &in.TelemetryPrefix, &out.TelemetryPrefix
+		*out = new(string)
+		**out = **in
+	}
+	if in.TelemetrySampleRate != nil {
+		in, out := &in.TelemetrySampleRate, &out.TelemetrySampleRate
+		*out = new(int)
+		**out = **in
+	}
 	if in.HTTPPort != nil {
 		in, out := &in.HTTPPort, &out.HTTPPort
 		*out = new(int32)
diff --git a/internal/workload/podspec_updates.go b/internal/workload/podspec_updates.go
index 874cff2..faf3bd1 100644
--- a/internal/workload/podspec_updates.go
+++ b/internal/workload/podspec_updates.go
@@ -36,7 +36,7 @@ import (
 // package and documented here so that they appear in the godoc. These also
 // need to be documented in the CRD
 const (
-	DefaultProxyImage = "gcr.io/cloud-sql-connectors/cloud-sql-proxy:2.1.0"
+	DefaultProxyImage = "gcr.io/cloud-sql-connectors/cloud-sql-proxy:2.1.1"
 
 	// DefaultFirstPort is the first port number chose for an instance listener by the
 	// proxy.
@@ -647,12 +647,30 @@ func (s *updateState) applyTelemetrySpec(p *cloudsqlapi.AuthProxyWorkload) {
 	}
 	tel := p.Spec.AuthProxyContainer.Telemetry
 
+	if tel.TelemetrySampleRate != nil {
+		s.addProxyContainerEnvVar(p, "CSQL_PROXY_TELEMETRY_SAMPLE_RATE", fmt.Sprintf("%d", *tel.TelemetrySampleRate))
+	}
 	if tel.DisableTraces != nil && *tel.DisableTraces {
 		s.addProxyContainerEnvVar(p, "CSQL_PROXY_DISABLE_TRACES", "true")
 	}
 	if tel.DisableMetrics != nil && *tel.DisableMetrics {
 		s.addProxyContainerEnvVar(p, "CSQL_PROXY_DISABLE_METRICS", "true")
 	}
+	if tel.PrometheusNamespace != nil || (tel.Prometheus != nil && *tel.Prometheus) {
+		s.addProxyContainerEnvVar(p, "CSQL_PROXY_PROMETHEUS", "true")
+	}
+	if tel.PrometheusNamespace != nil {
+		s.addProxyContainerEnvVar(p, "CSQL_PROXY_PROMETHEUS_NAMESPACE", *tel.PrometheusNamespace)
+	}
+	if tel.TelemetryProject != nil {
+		s.addProxyContainerEnvVar(p, "CSQL_PROXY_TELEMETRY_PROJECT", *tel.TelemetryProject)
+	}
+	if tel.TelemetryPrefix != nil {
+		s.addProxyContainerEnvVar(p, "CSQL_PROXY_TELEMETRY_PREFIX", *tel.TelemetryPrefix)
+	}
+	if tel.QuotaProject != nil {
+		s.addProxyContainerEnvVar(p, "CSQL_PROXY_QUOTA_PROJECT", *tel.QuotaProject)
+	}
 
 	return
 }


Test patch:
diff --git a/internal/workload/podspec_updates_test.go b/internal/workload/podspec_updates_test.go
index ecea20b..c8c69bc 100644
--- a/internal/workload/podspec_updates_test.go
+++ b/internal/workload/podspec_updates_test.go
@@ -628,9 +628,15 @@ func TestProxyCLIArgs(t *testing.T) {
 				AuthProxyContainer: &v1alpha1.AuthProxyContainerSpec{
 					SQLAdminAPIEndpoint: "https://example.com",
 					Telemetry: &v1alpha1.TelemetrySpec{
-						HTTPPort:       ptr(int32(9092)),
-						DisableTraces:  &wantTrue,
-						DisableMetrics: &wantTrue,
+						TelemetryPrefix:     ptr("telprefix"),
+						TelemetryProject:    ptr("telproject"),
+						TelemetrySampleRate: ptr(200),
+						HTTPPort:            ptr(int32(9092)),
+						DisableTraces:       &wantTrue,
+						DisableMetrics:      &wantTrue,
+						Prometheus:          &wantTrue,
+						PrometheusNamespace: ptr("hello"),
+						QuotaProject:        ptr("qp"),
 					},
 					AdminServer: &v1alpha1.AdminServerSpec{
 						EnableAPIs: []string{"Debug", "QuitQuitQuit"},
@@ -649,6 +655,10 @@ func TestProxyCLIArgs(t *testing.T) {
 			},
 			wantWorkloadEnv: map[string]string{
 				"CSQL_PROXY_SQLADMIN_API_ENDPOINT": "https://example.com",
+				"CSQL_PROXY_TELEMETRY_SAMPLE_RATE": "200",
+				"CSQL_PROXY_PROMETHEUS_NAMESPACE":  "hello",
+				"CSQL_PROXY_TELEMETRY_PROJECT":     "telproject",
+				"CSQL_PROXY_TELEMETRY_PREFIX":      "telprefix",
 				"CSQL_PROXY_HTTP_PORT":             "9092",
 				"CSQL_PROXY_ADMIN_PORT":            "9091",
 				"CSQL_PROXY_DEBUG":                 "true",
@@ -656,6 +666,8 @@ func TestProxyCLIArgs(t *testing.T) {
 				"CSQL_PROXY_HEALTH_CHECK":          "true",
 				"CSQL_PROXY_DISABLE_TRACES":        "true",
 				"CSQL_PROXY_DISABLE_METRICS":       "true",
+				"CSQL_PROXY_PROMETHEUS":            "true",
+				"CSQL_PROXY_QUOTA_PROJECT":         "qp",
 				"CSQL_PROXY_MAX_CONNECTIONS":       "10",
 				"CSQL_PROXY_MAX_SIGTERM_DELAY":     "20",
 			},
diff --git a/tests/e2e_test.go b/tests/e2e_test.go
index 32a702e..3175499 100644
--- a/tests/e2e_test.go
+++ b/tests/e2e_test.go
@@ -15,6 +15,7 @@
 package tests
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"strings"
@@ -25,7 +26,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
-	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 func TestMain(m *testing.M) {
@@ -68,32 +68,32 @@ func TestProxyAppliedOnNewWorkload(t *testing.T) {
 
 	tests := []struct {
 		name     string
-		o        client.Object
+		o        workload.Workload
 		allOrAny string
 	}{
 		{
 			name:     "deployment",
-			o:        testhelpers.BuildDeployment(types.NamespacedName{}, "busybox"),
+			o:        &workload.DeploymentWorkload{Deployment: testhelpers.BuildDeployment(types.NamespacedName{}, "busybox")},
 			allOrAny: "all",
 		},
 		{
 			name:     "statefulset",
-			o:        testhelpers.BuildStatefulSet(types.NamespacedName{}, "busybox"),
+			o:        &workload.StatefulSetWorkload{StatefulSet: testhelpers.BuildStatefulSet(types.NamespacedName{}, "busybox")},
 			allOrAny: "all",
 		},
 		{
 			name:     "daemonset",
-			o:        testhelpers.BuildDaemonSet(types.NamespacedName{}, "busybox"),
+			o:        &workload.DaemonSetWorkload{DaemonSet: testhelpers.BuildDaemonSet(types.NamespacedName{}, "busybox")},
 			allOrAny: "all",
 		},
 		{
 			name:     "job",
-			o:        testhelpers.BuildJob(types.NamespacedName{}, "busybox"),
+			o:        &workload.JobWorkload{Job: testhelpers.BuildJob(types.NamespacedName{}, "busybox")},
 			allOrAny: "any",
 		},
 		{
 			name:     "cronjob",
-			o:        testhelpers.BuildCronJob(types.NamespacedName{}, "busybox"),
+			o:        &workload.CronJobWorkload{CronJob: testhelpers.BuildCronJob(types.NamespacedName{}, "busybox")},
 			allOrAny: "any",
 		},
 	}
@@ -103,7 +103,7 @@ func TestProxyAppliedOnNewWorkload(t *testing.T) {
 			t.Parallel()
 			ctx := testContext()
 
-			kind := test.o.GetObjectKind().GroupVersionKind().Kind
+			kind := test.o.Object().GetObjectKind().GroupVersionKind().Kind
 			tp := newPublicPostgresClient("new" + strings.ToLower(kind))
 
 			err := tp.CreateOrPatchNamespace(ctx)
@@ -140,15 +140,11 @@ func TestProxyAppliedOnNewWorkload(t *testing.T) {
 			}
 
 			t.Log("Creating ", kind)
-			test.o.SetNamespace(tp.Namespace)
-			test.o.SetName(test.name)
-			err = tp.CreateWorkload(ctx, test.o)
+			err = createWorkload(ctx, tp, test.o, test.name)
 			if err != nil {
 				t.Fatal("unable to create ", kind, err)
 			}
-			selector := &metav1.LabelSelector{
-				MatchLabels: map[string]string{"app": appLabel},
-			}
+			selector := appSelector(appLabel)
 			t.Log("Checking for container counts", kind)
 			err = tp.ExpectPodContainerCount(ctx, selector, 2, test.allOrAny)
 			if err != nil {
@@ -227,15 +223,11 @@ func TestProxyAppliedOnExistingWorkload(t *testing.T) {
 			key := types.NamespacedName{Name: pwlName, Namespace: tp.Namespace}
 
 			t.Log("Creating ", kind)
-			test.o.Object().SetNamespace(tp.Namespace)
-			test.o.Object().SetName(test.name)
-			err = tp.CreateWorkload(ctx, test.o.Object())
+			err = createWorkload(ctx, tp, test.o, test.name)
 			if err != nil {
 				t.Fatal("unable to create ", kind, err)
 			}
-			selector := &metav1.LabelSelector{
-				MatchLabels: map[string]string{"app": appLabel},
-			}
+			selector := appSelector(appLabel)
 
 			err = tp.ExpectPodContainerCount(ctx, selector, 1, test.allOrAny)
 			if err != nil {
@@ -373,15 +365,11 @@ func TestPublicDBConnections(t *testing.T) {
 			}
 
 			t.Log("Creating ", kind)
-			wl.Object().SetNamespace(tp.Namespace)
-			wl.Object().SetName(pwlName)
-			err = tp.CreateWorkload(ctx, wl.Object())
+			err = createWorkload(ctx, tp, wl, pwlName)
 			if err != nil {
 				t.Fatal("unable to create ", kind, err)
 			}
-			selector := &metav1.LabelSelector{
-				MatchLabels: map[string]string{"app": appLabel},
-			}
+			selector := appSelector(appLabel)
 			t.Log("Checking for container counts", kind)
 			err = tp.ExpectPodContainerCount(ctx, selector, 2, "all")
 			if err != nil {
@@ -454,15 +442,11 @@ func TestUpdateWorkloadOnDelete(t *testing.T) {
 
 	// Create deployment
 	t.Log("Creating ", kind)
-	o.SetNamespace(tp.Namespace)
-	o.SetName(name)
-	err = tp.CreateWorkload(ctx, o)
+	err = createWorkload(ctx, tp, wl, name)
 	if err != nil {
 		t.Fatal("unable to create ", kind, err)
 	}
-	selector := &metav1.LabelSelector{
-		MatchLabels: map[string]string{"app": appLabel},
-	}
+	selector := appSelector(appLabel)
 
 	// Check that the deployment pods are configured with the proxy: pods
 	// have 2 containers.
@@ -487,3 +471,104 @@ func TestUpdateWorkloadOnDelete(t *testing.T) {
 		t.Error(err)
 	}
 }
+
+func TestPrivateDBConnections(t *testing.T) {
+	// When running tests during development, set the SKIP_CLEANUP=true envvar so that
+	// the test namespace remains after the test ends. By default, the test
+	// namespace will be deleted when the test exits.
+	skipCleanup := loadValue("SKIP_CLEANUP", "", "false") == "true"
+	const (
+		pwlName  = "newss"
+		appLabel = "client"
+		kind     = "Deployment"
+		allOrAny = "all"
+	)
+
+	ctx := testContext()
+	tp := newPrivatePostgresClient("postgresconn")
+
+	err := tp.CreateOrPatchNamespace(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		if skipCleanup {
+			return
+		}
+
+		err = tp.DeleteNamespace(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	key := types.NamespacedName{Name: pwlName, Namespace: tp.Namespace}
+
+	s := testhelpers.BuildSecret("db-secret", tp.DBRootUsername, tp.DBRootPassword, tp.DBName)
+	s.SetNamespace(tp.Namespace)
+	err = tp.Client.Create(ctx, &s)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Log("Creating AuthProxyWorkload")
+	b := true
+	p := testhelpers.NewAuthProxyWorkload(key)
+	testhelpers.AddUnixInstance(p, tp.ConnectionString, "/var/tests/dbsocket")
+	tp.ConfigureSelector(p, appLabel, kind)
+	tp.ConfigureResources(p)
+	p.Spec.Instances[0].PrivateIP = &b
+
+	err = tp.Create(ctx, p)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Log("Waiting for AuthProxyWorkload operator to begin the reconcile loop")
+	_, err = tp.GetAuthProxyWorkloadAfterReconcile(ctx, key)
+	if err != nil {
+		t.Fatal("unable to create AuthProxyWorkload", err)
+	}
+
+	t.Log("Creating ", kind)
+	wl := &workload.DeploymentWorkload{Deployment: testhelpers.BuildDeployment(types.NamespacedName{}, appLabel)}
+	wl.Deployment.Spec.Template = testhelpers.BuildPgUnixPodSpec(
+		600, appLabel, "db-secret")
+	err = createWorkload(ctx, tp, wl, pwlName)
+	if err != nil {
+		t.Fatal("unable to create ", kind, err)
+	}
+
+	selector := appSelector(appLabel)
+	t.Log("Checking for container counts", kind)
+	err = tp.ExpectPodContainerCount(ctx, selector, 2, "all")
+	if err != nil {
+		t.Error(err)
+	}
+
+	// The pods are configured to only be ready when the real database client
+	// successfully executes a simple query on the database.
+	t.Log("Checking for ready", kind)
+	err = tp.ExpectPodReady(ctx, selector, "all")
+	if err != nil {
+		t.Error(err)
+	}
+
+	t.Log("Done, OK", kind)
+
+}
+
+// createWorkload will set name and namespace appropriately, then use the client
+// to create the workload.
+func createWorkload(ctx context.Context, tp *testhelpers.TestCaseClient, wl workload.Workload, name string) error {
+	wl.Object().SetNamespace(tp.Namespace)
+	wl.Object().SetName(name)
+	return tp.CreateWorkload(ctx, wl.Object())
+}
+
+// appSelector creates a label selector for "app={appLabel}".
+func appSelector(appLabel string) *metav1.LabelSelector {
+	return &metav1.LabelSelector{
+		MatchLabels: map[string]string{"app": appLabel},
+	}
+}
diff --git a/tests/setup_test.go b/tests/setup_test.go
index 5b86bf4..eecbbc7 100644
--- a/tests/setup_test.go
+++ b/tests/setup_test.go
@@ -47,23 +47,27 @@ var (
 	}))
 
 	// These vars hold state initialized by SetupTests.
-	c             client.Client
+	publicClient  client.Client
+	privateClient client.Client
 	infra         testInfra
 	proxyImageURL string
 	operatorURL   string
 )
 
+func newPrivatePostgresClient(ns string) *testhelpers.TestCaseClient {
+	return newTestClient(ns, infra.Private.Postgres, privateClient)
+}
 func newPublicPostgresClient(ns string) *testhelpers.TestCaseClient {
-	return newTestClient(ns, infra.Public.Postgres)
+	return newTestClient(ns, infra.Public.Postgres, publicClient)
 }
 func newPublicMySQLClient(ns string) *testhelpers.TestCaseClient {
-	return newTestClient(ns, infra.Public.MySQL)
+	return newTestClient(ns, infra.Public.MySQL, publicClient)
 }
 func newPublicMSSQLClient(ns string) *testhelpers.TestCaseClient {
-	return newTestClient(ns, infra.Public.MSSQL)
+	return newTestClient(ns, infra.Public.MSSQL, publicClient)
 }
 
-func newTestClient(ns string, db testDatabase) *testhelpers.TestCaseClient {
+func newTestClient(ns string, db testDatabase, c client.Client) *testhelpers.TestCaseClient {
 	return &testhelpers.TestCaseClient{
 		Client:           c,
 		Namespace:        testhelpers.NewNamespaceName(ns),
@@ -119,35 +123,43 @@ func setupTests() (func(), error) {
 	}
 	infra = ti
 
-	setupKubernetesClient(ctx, infra.Public)
+	publicClient, err = setupKubernetesClient(infra.Public)
+	privateClient, err = setupKubernetesClient(infra.Private)
+	waitForAndTailOperatorPods(ctx, publicClient)
+	waitForAndTailOperatorPods(ctx, privateClient)
 
 	return cancelFunc, nil
 }
 
-func setupKubernetesClient(ctx context.Context, ti testEnvironment) error {
+func setupKubernetesClient(ti testEnvironment) (client.Client, error) {
 	// Build the kubernetes client
 	config, err := clientcmd.BuildConfigFromFlags("", ti.Kubeconfig)
 	if err != nil {
-		return fmt.Errorf("unable to build kubernetes client for config %s, %v", ti.Kubeconfig, err)
+		return nil, fmt.Errorf("unable to build kubernetes client for config %s, %v", ti.Kubeconfig, err)
 	}
 	config.RateLimiter = nil
 	k8sClientSet, err = kubernetes.NewForConfig(config)
 	if err != nil {
-		return fmt.Errorf("unable to setup e2e kubernetes client  %v", err)
+		return nil, fmt.Errorf("unable to setup e2e kubernetes client  %v", err)
 	}
 	s := scheme.Scheme
 	controller.InitScheme(s)
-	c, err = client.New(config, client.Options{Scheme: s})
+
+	c, err := client.New(config, client.Options{Scheme: s})
 	if err != nil {
-		return fmt.Errorf("Unable to initialize kubernetes client %{v}", err)
+		return nil, fmt.Errorf("Unable to initialize kubernetes client %{v}", err)
 	}
 	if c == nil {
-		return fmt.Errorf("Kubernetes client was empty after initialization %v", err)
+		return nil, fmt.Errorf("Kubernetes client was empty after initialization %v", err)
 	}
+	return c, nil
+}
+
+func waitForAndTailOperatorPods(ctx context.Context, c client.Client) error {
 
 	// Check that the e2e k8s cluster is the operator that was last built from
 	// this working directory.
-	d, err := waitForCorrectOperatorPods(ctx, err)
+	d, err := waitForCorrectOperatorPods(ctx, c)
 
 	if err != nil {
 		return fmt.Errorf("unable to find manager deployment %v", err)
@@ -167,10 +179,10 @@ func setupKubernetesClient(ctx context.Context, ti testEnvironment) error {
 	return nil
 }
 
-func waitForCorrectOperatorPods(ctx context.Context, err error) (*appsv1.Deployment, error) {
+func waitForCorrectOperatorPods(ctx context.Context, c client.Client) (*appsv1.Deployment, error) {
 	deployment := &appsv1.Deployment{}
 	managerDeploymentKey := client.ObjectKey{Namespace: "cloud-sql-proxy-operator-system", Name: "cloud-sql-proxy-operator-controller-manager"}
-	err = testhelpers.RetryUntilSuccess(5, testhelpers.DefaultRetryInterval, func() error {
+	err := testhelpers.RetryUntilSuccess(5, testhelpers.DefaultRetryInterval, func() error {
 		// Fetch the deployment
 		err := c.Get(ctx, managerDeploymentKey, deployment)
 		if err != nil {
@@ -273,7 +285,8 @@ type testDatabase struct {
 }
 
 type testInfra struct {
-	Public testEnvironment `json:"public"`
+	Public  testEnvironment `json:"public"`
+	Private testEnvironment `json:"private"`
 }
 
 type testEnvironment struct {