Repository: VictoriaMetrics/operator

Issue:
Full `httpInternalListenAddr` support
In v1.111.0 for vmauth was supported flag `httpInternalListenAddr`.
It's possible to set with `extraFalgs` in resource spec, but it doesn't work correctly because operator still use the port from `spec.port` for everything including health checks, config reload url, metrics path for scrape object. 
I propose to create a new spec field `internalPort`, the setting of which will result in:
- setting flag `httpInternalListenAddr` for vmauth deployment
- creating new internal service with this port
- building URLs of service endpoints (`/metrics`, `/health`, `/reload`, etc...) with using of this port in spec of all created resources

FYI: @f41gh7 @j-sokol @jgomezselles 

Discussion:
@f41gh7:  Feature was included into [v0.56.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.56.0) release

Solution patch:
diff --git a/api/operator/v1beta1/vmauth_types.go b/api/operator/v1beta1/vmauth_types.go
index 25491e9d..36d49e82 100644
--- a/api/operator/v1beta1/vmauth_types.go
+++ b/api/operator/v1beta1/vmauth_types.go
@@ -114,6 +114,12 @@ type VMAuthSpec struct {
 	CommonDefaultableParams           `json:",inline,omitempty" yaml:",inline"`
 	CommonConfigReloaderParams        `json:",inline,omitempty" yaml:",inline"`
 	CommonApplicationDeploymentParams `json:",inline,omitempty" yaml:",inline"`
+	// InternalListenPort instructs vmauth to serve internal routes at given port
+	// available from v0.56.0 operator
+	// and v1.111.0 vmauth version
+	// related doc https://docs.victoriametrics.com/vmauth/#security
+	// +optional
+	InternalListenPort string `json:"internalListenPort,omitempty"`
 }
 
 // VMAuthUnauthorizedUserAccessSpec defines unauthorized_user section configuration for vmauth
@@ -505,6 +511,9 @@ func (cr *VMAuth) ProbeScheme() string {
 }
 
 func (cr *VMAuth) ProbePort() string {
+	if len(cr.Spec.InternalListenPort) > 0 {
+		return cr.Spec.InternalListenPort
+	}
 	return cr.Spec.Port
 }
 
diff --git a/config/crd/overlay/crd.yaml b/config/crd/overlay/crd.yaml
index 038cc276..a6c6f3fc 100644
--- a/config/crd/overlay/crd.yaml
+++ b/config/crd/overlay/crd.yaml
@@ -15308,6 +15308,13 @@ spec:
                   type: object
                   x-kubernetes-preserve-unknown-fields: true
                 type: array
+              internalListenPort:
+                description: |-
+                  InternalListenPort instructs vmauth to serve internal routes at given port
+                  available from v0.56.0 operator
+                  and v1.111.0 vmauth version
+                  related doc https://docs.victoriametrics.com/vmauth/#security
+                type: string
               license:
                 description: |-
                   License allows to configure license key to be used for enterprise features.
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index fe481cb6..3cb14c35 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -14,6 +14,7 @@ aliases:
 ## tip
 
 * [FEATURE]: [operator](https://docs.victoriametrics.com/operator): support `VM_METRICS_VERSION` and `VM_LOGS_VERSION` env variables as a source for all VM and VL related CR image versions
+* FEATURE: [vmauth](https://docs.victoriametrics.com/operator/resources/vmauth/): add new field `internalListenPort` for serving internal routes. See [this issue](https://github.com/VictoriaMetrics/operator/issues/1302) and this [docs]().
 
 ## [v0.55.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.55.0)
 
diff --git a/docs/api.md b/docs/api.md
index 102a39da..106e96b8 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -3398,6 +3398,7 @@ _Appears in:_
 | <a href="#vmauthspec-imagepullsecrets"><code id="vmauthspec-imagepullsecrets">imagePullSecrets</code></a><br/>_[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#localobjectreference-v1-core) array_ | _(Optional)_<br/>ImagePullSecrets An optional list of references to secrets in the same namespace<br />to use for pulling images from registries<br />see https://kubernetes.io/docs/concepts/containers/images/#referring-to-an-imagepullsecrets-on-a-pod |
 | <a href="#vmauthspec-ingress"><code id="vmauthspec-ingress">ingress</code></a><br/>_[EmbeddedIngress](#embeddedingress)_ | Ingress enables ingress configuration for VMAuth. |
 | <a href="#vmauthspec-initcontainers"><code id="vmauthspec-initcontainers">initContainers</code></a><br/>_[Container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#container-v1-core) array_ | _(Optional)_<br/>InitContainers allows adding initContainers to the pod definition.<br />Any errors during the execution of an initContainer will lead to a restart of the Pod.<br />More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ |
+| <a href="#vmauthspec-internallistenport"><code id="vmauthspec-internallistenport">internalListenPort</code></a><br/>_string_ | _(Optional)_<br/>InternalListenPort instructs vmauth to serve internal routes at given port<br />available from v0.56.0 operator<br />and v1.111.0 vmauth version<br />related doc https://docs.victoriametrics.com/vmauth/#security |
 | <a href="#vmauthspec-ip_filters"><code id="vmauthspec-ip_filters">ip_filters</code></a><br/>_[VMUserIPFilters](#vmuseripfilters)_ | _(Optional)_<br/>IPFilters defines per target src ip filters<br />supported only with enterprise version of [vmauth](https://docs.victoriametrics.com/vmauth/#ip-filters) |
 | <a href="#vmauthspec-license"><code id="vmauthspec-license">license</code></a><br/>_[License](#license)_ | _(Optional)_<br/>License allows to configure license key to be used for enterprise features.<br />Using license key is supported starting from VictoriaMetrics v1.94.0.<br />See [here](https://docs.victoriametrics.com/enterprise) |
 | <a href="#vmauthspec-load_balancing_policy"><code id="vmauthspec-load_balancing_policy">load_balancing_policy</code></a><br/>_string_ | _(Optional)_<br/>LoadBalancingPolicy defines load balancing policy to use for backend urls.<br />Supported policies: least_loaded, first_available.<br />See [here](https://docs.victoriametrics.com/vmauth#load-balancing) for more details (default "least_loaded") |
diff --git a/internal/controller/operator/factory/vmauth/vmauth.go b/internal/controller/operator/factory/vmauth/vmauth.go
index 60e35d25..ea5570a7 100644
--- a/internal/controller/operator/factory/vmauth/vmauth.go
+++ b/internal/controller/operator/factory/vmauth/vmauth.go
@@ -36,6 +36,7 @@ const (
 	vmAuthConfigName      = "config.yaml"
 	vmAuthConfigNameGz    = "config.yaml.gz"
 	vmAuthVolumeName      = "config"
+	internalPortName      = "internal"
 )
 
 // CreateOrUpdateVMAuth - handles VMAuth deployment reconciliation.
@@ -68,7 +69,7 @@ func CreateOrUpdateVMAuth(ctx context.Context, cr *vmv1beta1.VMAuth, rclient cli
 		return fmt.Errorf("cannot create or update ingress for vmauth: %w", err)
 	}
 	if !ptr.Deref(cr.Spec.DisableSelfServiceScrape, false) {
-		if err := reconcile.VMServiceScrapeForCRD(ctx, rclient, build.VMServiceScrapeForServiceWithSpec(svc, cr)); err != nil {
+		if err := reconcile.VMServiceScrapeForCRD(ctx, rclient, buildServiceScrape(svc, cr)); err != nil {
 			return err
 		}
 	}
@@ -153,6 +154,9 @@ func makeSpecForVMAuth(cr *vmv1beta1.VMAuth) (*corev1.PodTemplateSpec, error) {
 	}
 
 	args = append(args, fmt.Sprintf("-httpListenAddr=:%s", cr.Spec.Port))
+	if len(cr.Spec.InternalListenPort) > 0 {
+		args = append(args, fmt.Sprintf("-httpInternalListenAddr=:%s", cr.Spec.InternalListenPort))
+	}
 	if len(cr.Spec.ExtraEnvs) > 0 || len(cr.Spec.ExtraEnvsFrom) > 0 {
 		args = append(args, "-envflag.enable=true")
 	}
@@ -164,6 +168,14 @@ func makeSpecForVMAuth(cr *vmv1beta1.VMAuth) (*corev1.PodTemplateSpec, error) {
 
 	ports = append(ports, corev1.ContainerPort{Name: "http", Protocol: "TCP", ContainerPort: intstr.Parse(cr.Spec.Port).IntVal})
 
+	if len(cr.Spec.InternalListenPort) > 0 {
+		ports = append(ports, corev1.ContainerPort{
+			Name:          internalPortName,
+			Protocol:      "TCP",
+			ContainerPort: intstr.Parse(cr.Spec.InternalListenPort).IntVal,
+		})
+	}
+
 	useStrictSecurity := ptr.Deref(cr.Spec.UseStrictSecurity, false)
 	useVMConfigReloader := ptr.Deref(cr.Spec.UseVMConfigReloader, false)
 
@@ -485,14 +497,20 @@ func buildIngressConfig(cr *vmv1beta1.VMAuth) *networkingv1.Ingress {
 }
 
 func buildVMAuthConfigReloaderContainer(cr *vmv1beta1.VMAuth) corev1.Container {
+	port := cr.Spec.Port
+	if len(cr.Spec.InternalListenPort) > 0 {
+		port = cr.Spec.InternalListenPort
+	}
 	configReloaderArgs := []string{
-		fmt.Sprintf("--reload-url=%s", vmv1beta1.BuildReloadPathWithPort(cr.Spec.ExtraArgs, cr.Spec.Port)),
+		fmt.Sprintf("--reload-url=%s", vmv1beta1.BuildReloadPathWithPort(cr.Spec.ExtraArgs, port)),
 		fmt.Sprintf("--config-envsubst-file=%s", path.Join(vmAuthConfigFolder, vmAuthConfigName)),
 	}
 	useVMConfigReloader := ptr.Deref(cr.Spec.UseVMConfigReloader, false)
 	if useVMConfigReloader {
 		configReloaderArgs = append(configReloaderArgs, fmt.Sprintf("--config-secret-name=%s/%s", cr.Namespace, cr.ConfigSecretName()))
-		configReloaderArgs = vmv1beta1.MaybeEnableProxyProtocol(configReloaderArgs, cr.Spec.ExtraArgs)
+		if len(cr.Spec.InternalListenPort) == 0 {
+			configReloaderArgs = vmv1beta1.MaybeEnableProxyProtocol(configReloaderArgs, cr.Spec.ExtraArgs)
+		}
 	} else {
 		configReloaderArgs = append(configReloaderArgs, fmt.Sprintf("--config-file=%s", path.Join(vmAuthConfigMountGz, vmAuthConfigNameGz)))
 	}
@@ -605,14 +623,27 @@ func gzipConfig(buf *bytes.Buffer, conf []byte) error {
 	return nil
 }
 
+func setInternalSvcPort(cr *vmv1beta1.VMAuth) func(svc *corev1.Service) {
+	return func(svc *corev1.Service) {
+		if len(cr.Spec.InternalListenPort) > 0 {
+			p := intstr.Parse(cr.Spec.InternalListenPort)
+			svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{
+				Name:       internalPortName,
+				Port:       p.IntVal,
+				TargetPort: p,
+			})
+		}
+	}
+}
+
 // createOrUpdateVMAuthService creates service for VMAuth
 func createOrUpdateVMAuthService(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMAuth) (*corev1.Service, error) {
 	var prevService, prevAdditionalService *corev1.Service
 	if prevCR != nil {
-		prevService = build.Service(prevCR, prevCR.Spec.Port, nil)
+		prevService = build.Service(prevCR, prevCR.Spec.Port, setInternalSvcPort(prevCR))
 		prevAdditionalService = build.AdditionalServiceFromDefault(prevService, prevCR.Spec.ServiceSpec)
 	}
-	newService := build.Service(cr, cr.Spec.Port, nil)
+	newService := build.Service(cr, cr.Spec.Port, setInternalSvcPort(cr))
 	if err := cr.Spec.ServiceSpec.IsSomeAndThen(func(s *vmv1beta1.AdditionalServiceSpec) error {
 		additionalService := build.AdditionalServiceFromDefault(newService, s)
 		if additionalService.Name == newService.Name {
@@ -662,3 +693,18 @@ func deletePrevStateResources(ctx context.Context, rclient client.Client, cr, pr
 
 	return nil
 }
+
+func buildServiceScrape(svc *corev1.Service, cr *vmv1beta1.VMAuth) *vmv1beta1.VMServiceScrape {
+	b := build.VMServiceScrapeForServiceWithSpec(svc, cr)
+	if len(cr.Spec.InternalListenPort) == 0 {
+		return b
+	}
+	for idx := range b.Spec.Endpoints {
+		ep := &b.Spec.Endpoints[idx]
+		if ep.Port == "http" {
+			ep.Port = internalPortName
+			break
+		}
+	}
+	return b
+}



Test patch:
diff --git a/test/e2e/vmauth_test.go b/test/e2e/vmauth_test.go
index f3fdb047..50d85281 100644
--- a/test/e2e/vmauth_test.go
+++ b/test/e2e/vmauth_test.go
@@ -218,6 +218,56 @@ var _ = Describe("test vmauth Controller", func() {
 						},
 					},
 				),
+				Entry("by switching to internal listen port", "vm-internal-listen",
+					&v1beta1vm.VMAuth{
+						Spec: v1beta1vm.VMAuthSpec{
+							SelectAllByDefault: true,
+							CommonApplicationDeploymentParams: v1beta1vm.CommonApplicationDeploymentParams{
+								ReplicaCount: ptr.To[int32](1),
+							},
+							CommonDefaultableParams: v1beta1vm.CommonDefaultableParams{
+								UseDefaultResources: ptr.To(false),
+							},
+							CommonConfigReloaderParams: v1beta1vm.CommonConfigReloaderParams{
+								UseVMConfigReloader: ptr.To(true),
+							},
+
+							UnauthorizedAccessConfig: []v1beta1vm.UnauthorizedAccessConfigURLMap{
+								{
+									URLPrefix: []string{"http://localhost:8490"},
+									SrcPaths:  []string{"/.*"},
+								},
+							},
+						},
+					},
+					testStep{
+						modify: func(cr *v1beta1vm.VMAuth) {
+							cr.Spec.InternalListenPort = "8426"
+						},
+						verify: func(cr *v1beta1vm.VMAuth) {
+							Eventually(func() string {
+								return expectPodCount(k8sClient, 1, namespace, cr.SelectorLabels())
+							}, eventualDeploymentPodTimeout).Should(BeEmpty())
+							pod := mustGetFirstPod(k8sClient, cr.Namespace, cr.SelectorLabels())
+							Expect(pod.Spec.Containers).To(HaveLen(2))
+							ac := pod.Spec.Containers[0]
+							Expect(ac.Ports).To(HaveLen(2))
+							Expect(ac.Ports[1].ContainerPort).To(Equal(int32(8426)))
+							nsn := types.NamespacedName{
+								Namespace: cr.Namespace,
+								Name:      cr.PrefixedName(),
+							}
+							var svc corev1.Service
+							Expect(k8sClient.Get(ctx, nsn, &svc)).To(Succeed())
+							Expect(svc.Spec.Ports).To(HaveLen(2))
+							var vmss v1beta1vm.VMServiceScrape
+							Expect(k8sClient.Get(ctx, nsn, &vmss)).To(Succeed())
+							Expect(vmss.Spec.Endpoints).To(HaveLen(1))
+							ep := vmss.Spec.Endpoints[0]
+							Expect(ep.Port).To(Equal("internal"))
+						},
+					},
+				),
 				Entry("by removing podDistruptionBudget and keeping exist ingress", "vm-keep-ingress-change-pdb",
 					&v1beta1vm.VMAuth{
 						Spec: v1beta1vm.VMAuthSpec{