Repository: istio/istio (go)

Issue:
Convert citadel cert expiry timestamp metrics to citadel cert expiry time left
(This is used to request new product features, please visit <https://github.com/istio/istio/discussions> for questions on using Istio)

**Describe the feature request**

Currently istio metrics `citadel_server_cert_chain_expiry_timestamp` reports `The unix timestamp, in seconds, when Citadel cert chain will expire. A negative time indicates the cert is expired.`.
It is not possible in datadog to create alert for cert expiration is due in x days, as to get time left we need to subtract current time from metrics value. ( not supported functionality in datadog)

https://istio.io/latest/docs/reference/commands/pilot-discovery/#metrics

same case is for `citadel_server_root_cert_expiry_timestamp` metrics

**Describe alternatives you've considered**

we can add additional metrics `citadel_server_cert_chain_expiry_time_left` that report time in seconds left for expiration of certificate along with current metrics .

https://github.com/istio/istio/blob/3727b5719435c0cb4146413ac02ac16f8378cd36/security/pkg/server/ca/server.go#L166

same case is for `citadel_server_root_cert_expiry_time_left` metrics.

**Affected product area (please put an X in all that apply)**

[ ] Ambient
[ ] Docs
[ ] Dual Stack
[ ] Installation
[ ] Networking
[ ] Performance and Scalability
[*] Extensions and Telemetry
[ ] Security
[ ] Test and Release
[ ] User Experience
[ ] Developer Infrastructure

**Affected features (please put an X in all that apply)**

[ ] Multi Cluster
[ ] Virtual Machine
[ ] Multi Control Plane

**Additional context**


Pull request:
add time remaining before cert expires metrics
**Please provide a description of this PR:**

Resolves #51037

Patch:
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index b3be0ee80e..cab85934d4 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -1,6 +1,6 @@
 {
   "name": "istio build-tools",
-  "image": "gcr.io/istio-testing/build-tools:master-62b37843401cf50e072f1d7f4d565730b708d642",
+  "image": "gcr.io/istio-testing/build-tools:master-9637643344b97c5f67c2c1fc1bdd29a97d9930dc",
   "privileged": true,
   "remoteEnv": {
     "USE_GKE_GCLOUD_AUTH_PLUGIN": "True",
diff --git a/cni/pkg/cmd/root.go b/cni/pkg/cmd/root.go
index aa220f938c..4f64540b38 100644
--- a/cni/pkg/cmd/root.go
+++ b/cni/pkg/cmd/root.go
@@ -34,13 +34,14 @@ import (
 	"istio.io/istio/pkg/collateral"
 	"istio.io/istio/pkg/ctrlz"
 	"istio.io/istio/pkg/env"
-	"istio.io/istio/pkg/log"
+	istiolog "istio.io/istio/pkg/log"
 	"istio.io/istio/pkg/version"
 	iptables "istio.io/istio/tools/istio-iptables/pkg/constants"
 )
 
 var (
-	logOptions   = log.DefaultOptions()
+	logOptions   = istiolog.DefaultOptions()
+	log          = istiolog.RegisterScope(constants.CNIAgentLogScope, "CNI agent")
 	ctrlzOptions = func() *ctrlz.Options {
 		o := ctrlz.DefaultOptions()
 		o.EnablePprof = true
@@ -53,7 +54,7 @@ var rootCmd = &cobra.Command{
 	Short:        "Install and configure Istio CNI plugin on a node, detect and repair pod which is broken by race condition.",
 	SilenceUsage: true,
 	PreRunE: func(c *cobra.Command, args []string) error {
-		if err := log.Configure(logOptions); err != nil {
+		if err := istiolog.Configure(logOptions); err != nil {
 			log.Errorf("Failed to configure log %v", err)
 		}
 		return nil
@@ -75,7 +76,7 @@ var rootCmd = &cobra.Command{
 		monitoring.SetupMonitoring(cfg.InstallConfig.MonitoringPort, "/metrics", ctx.Done())
 
 		// Start UDS log server
-		udsLogger := udsLog.NewUDSLogger()
+		udsLogger := udsLog.NewUDSLogger(log.GetOutputLevel())
 		if err = udsLogger.StartUDSLogServer(cfg.InstallConfig.LogUDSAddress, ctx.Done()); err != nil {
 			log.Errorf("Failed to start up UDS Log Server: %v", err)
 			return
@@ -243,7 +244,8 @@ func constructConfig() (*config.Config, error) {
 		CNIConfName:      viper.GetString(constants.CNIConfName),
 		ChainedCNIPlugin: viper.GetBool(constants.ChainedCNIPlugin),
 
-		LogLevel:              viper.GetString(constants.LogLevel),
+		// make plugin (which runs out-of-process) inherit our current log level
+		PluginLogLevel:        istiolog.LevelToString(istiolog.FindScope(constants.CNIAgentLogScope).GetOutputLevel()),
 		KubeconfigFilename:    viper.GetString(constants.KubeconfigFilename),
 		KubeconfigMode:        viper.GetInt(constants.KubeconfigMode),
 		KubeCAFile:            viper.GetString(constants.KubeCAFile),
diff --git a/cni/pkg/config/config.go b/cni/pkg/config/config.go
index 8291dd5e8e..fd222f5764 100644
--- a/cni/pkg/config/config.go
+++ b/cni/pkg/config/config.go
@@ -35,8 +35,9 @@ type InstallConfig struct {
 	// Whether to install CNI plugin as a chained or standalone
 	ChainedCNIPlugin bool
 
-	// Logging level
-	LogLevel string
+	// Logging level for the CNI plugin
+	// Since it runs out-of-process, it has to be separately configured
+	PluginLogLevel string
 	// Name of the kubeconfig file used by the CNI plugin
 	KubeconfigFilename string
 	// The file mode to set when creating the kubeconfig file
@@ -128,7 +129,7 @@ func (c InstallConfig) String() string {
 	b.WriteString("CNIConfName: " + c.CNIConfName + "\n")
 	b.WriteString("ChainedCNIPlugin: " + fmt.Sprint(c.ChainedCNIPlugin) + "\n")
 
-	b.WriteString("LogLevel: " + c.LogLevel + "\n")
+	b.WriteString("PluginLogLevel: " + c.PluginLogLevel + "\n")
 	b.WriteString("KubeconfigFilename: " + c.KubeconfigFilename + "\n")
 	b.WriteString("KubeconfigMode: " + fmt.Sprintf("%#o", c.KubeconfigMode) + "\n")
 	b.WriteString("KubeCAFile: " + c.KubeCAFile + "\n")
diff --git a/cni/pkg/constants/constants.go b/cni/pkg/constants/constants.go
index 5a820f1de1..d2c47585a6 100644
--- a/cni/pkg/constants/constants.go
+++ b/cni/pkg/constants/constants.go
@@ -56,9 +56,10 @@ const (
 // Internal constants
 const (
 	DefaultKubeconfigMode = 0o600
-
-	CNIAddEventPath = "/cmdadd"
-	UDSLogPath      = "/log"
+	CNIAgentLogScope      = "cni-agent"
+	CNIPluginLogScope     = "cni-plugin"
+	CNIAddEventPath       = "/cmdadd"
+	UDSLogPath            = "/log"
 
 	// K8s liveness and readiness endpoints
 	LivenessEndpoint   = "/healthz"
diff --git a/cni/pkg/install/cniconfig.go b/cni/pkg/install/cniconfig.go
index a2e0a9cdda..f991c8e70b 100644
--- a/cni/pkg/install/cniconfig.go
+++ b/cni/pkg/install/cniconfig.go
@@ -33,7 +33,7 @@ import (
 
 func createCNIConfigFile(ctx context.Context, cfg *config.InstallConfig) (string, error) {
 	pluginConfig := plugin.Config{
-		LogLevel:        cfg.LogLevel,
+		PluginLogLevel:  cfg.PluginLogLevel,
 		LogUDSAddress:   cfg.LogUDSAddress,
 		CNIEventAddress: cfg.CNIEventAddress,
 		AmbientEnabled:  cfg.AmbientEnabled,
diff --git a/cni/pkg/install/install.go b/cni/pkg/install/install.go
index cd444de8e6..4630a9924f 100644
--- a/cni/pkg/install/install.go
+++ b/cni/pkg/install/install.go
@@ -22,14 +22,14 @@ import (
 	"sync/atomic"
 
 	"istio.io/istio/cni/pkg/config"
+	"istio.io/istio/cni/pkg/constants"
 	"istio.io/istio/cni/pkg/util"
 	"istio.io/istio/pkg/file"
 	"istio.io/istio/pkg/log"
 	"istio.io/istio/pkg/util/sets"
 )
 
-// TODO this should share parent scope, in practice it isn't useful to hide it within its own granular scope.
-var installLog = log.RegisterScope("install", "CNI install")
+var installLog = log.FindScope(constants.CNIAgentLogScope).WithLabels("plugin-install")
 
 type Installer struct {
 	cfg                *config.InstallConfig
diff --git a/cni/pkg/iptables/iptables.go b/cni/pkg/iptables/iptables.go
index ed6ae1e5c1..c56aad5024 100644
--- a/cni/pkg/iptables/iptables.go
+++ b/cni/pkg/iptables/iptables.go
@@ -20,6 +20,7 @@ import (
 	"net/netip"
 	"strings"
 
+	"istio.io/istio/cni/pkg/constants"
 	"istio.io/istio/cni/pkg/ipset"
 	istiolog "istio.io/istio/pkg/log"
 	"istio.io/istio/tools/istio-iptables/pkg/builder"
@@ -48,7 +49,7 @@ const (
 	ProbeIPSet                  = "istio-inpod-probes"
 )
 
-var log = istiolog.RegisterScope("iptables", "iptables helper")
+var log = istiolog.RegisterScope(constants.CNIAgentLogScope, "").WithLabels("iptables")
 
 type Config struct {
 	RestoreFormat bool `json:"RESTORE_FORMAT"`
diff --git a/cni/pkg/log/uds.go b/cni/pkg/log/uds.go
index 91d7ff74ba..fc0525f598 100644
--- a/cni/pkg/log/uds.go
+++ b/cni/pkg/log/uds.go
@@ -29,7 +29,7 @@ import (
 	"istio.io/istio/pkg/uds"
 )
 
-var pluginLog = log.RegisterScope("cni", "CNI network plugin")
+var pluginLog = log.RegisterScope(constants.CNIPluginLogScope, "CNI network plugin (forwarded logs)")
 
 type UDSLogger struct {
 	mu            sync.Mutex
@@ -42,7 +42,7 @@ type cniLog struct {
 	Msg   string    `json:"msg"`
 }
 
-func NewUDSLogger() *UDSLogger {
+func NewUDSLogger(level log.Level) *UDSLogger {
 	l := &UDSLogger{}
 	mux := http.NewServeMux()
 	mux.HandleFunc(constants.UDSLogPath, l.handleLog)
@@ -50,6 +50,7 @@ func NewUDSLogger() *UDSLogger {
 		Handler: mux,
 	}
 	l.loggingServer = loggingServer
+	pluginLog.SetOutputLevel(level)
 	return l
 }
 
diff --git a/cni/pkg/nodeagent/net.go b/cni/pkg/nodeagent/net.go
index 95672d47d5..a91010a1ed 100644
--- a/cni/pkg/nodeagent/net.go
+++ b/cni/pkg/nodeagent/net.go
@@ -24,6 +24,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 
+	"istio.io/istio/cni/pkg/constants"
 	"istio.io/istio/cni/pkg/ipset"
 	"istio.io/istio/cni/pkg/iptables"
 	"istio.io/istio/cni/pkg/util"
@@ -33,7 +34,7 @@ import (
 	dep "istio.io/istio/tools/istio-iptables/pkg/dependencies"
 )
 
-var log = istiolog.RegisterScope("ambient", "ambient controller")
+var log = istiolog.FindScope(constants.CNIAgentLogScope).WithLabels("server")
 
 // Adapts CNI to ztunnel server. decoupled from k8s for easier integration testing.
 type NetServer struct {
diff --git a/cni/pkg/plugin/plugin.go b/cni/pkg/plugin/plugin.go
index 9c6f931f65..7f763e71ec 100644
--- a/cni/pkg/plugin/plugin.go
+++ b/cni/pkg/plugin/plugin.go
@@ -65,7 +65,7 @@ type Config struct {
 	types.NetConf
 
 	// Add plugin-specific flags here
-	LogLevel        string     `json:"log_level"`
+	PluginLogLevel  string     `json:"plugin_log_level"`
 	LogUDSAddress   string     `json:"log_uds_address"`
 	CNIEventAddress string     `json:"cni_event_address"`
 	AmbientEnabled  bool       `json:"ambient_enabled"`
@@ -112,20 +112,6 @@ func parseConfig(stdin []byte) (*Config, error) {
 	return &conf, nil
 }
 
-func getLogLevel(logLevel string) log.Level {
-	switch logLevel {
-	case "debug":
-		return log.DebugLevel
-	case "warn":
-		return log.WarnLevel
-	case "error":
-		return log.ErrorLevel
-	case "info":
-		return log.InfoLevel
-	}
-	return log.InfoLevel
-}
-
 func GetLoggingOptions(udsAddress string) *log.Options {
 	loggingOptions := log.DefaultOptions()
 	loggingOptions.OutputPaths = []string{"stderr"}
@@ -316,7 +302,7 @@ func setupLogging(conf *Config) {
 			log.Error("Failed to configure istio-cni with UDS log")
 		}
 	}
-	log.FindScope("default").SetOutputLevel(getLogLevel(conf.LogLevel))
+	log.RegisterScope(constants.CNIPluginLogScope, "CNI plugin").SetOutputLevel(log.StringToLevel(conf.PluginLogLevel))
 }
 
 func pluginResponse(conf *Config) error {
diff --git a/cni/pkg/repair/repair.go b/cni/pkg/repair/repair.go
index ca88adce50..417847a5cc 100644
--- a/cni/pkg/repair/repair.go
+++ b/cni/pkg/repair/repair.go
@@ -18,11 +18,12 @@ import (
 	"context"
 
 	"istio.io/istio/cni/pkg/config"
+	"istio.io/istio/cni/pkg/constants"
 	"istio.io/istio/pkg/kube"
 	"istio.io/istio/pkg/log"
 )
 
-var repairLog = log.RegisterScope("repair", "CNI race condition repair")
+var repairLog = log.FindScope(constants.CNIAgentLogScope).WithLabels("repair")
 
 func StartRepair(ctx context.Context, cfg config.RepairConfig) {
 	if !cfg.Enabled {
diff --git a/common/.commonfiles.sha b/common/.commonfiles.sha
index cca33e0881..1d55c32622 100644
--- a/common/.commonfiles.sha
+++ b/common/.commonfiles.sha
@@ -1,1 +1,1 @@
-c042d933248a1754d5368aa59b6e2d3839797740
+d9dad780c50ae7f54b79891605c462de66986805
diff --git a/common/scripts/setup_env.sh b/common/scripts/setup_env.sh
index 5c67ce2dd6..e5e3743349 100755
--- a/common/scripts/setup_env.sh
+++ b/common/scripts/setup_env.sh
@@ -75,7 +75,7 @@ fi
 TOOLS_REGISTRY_PROVIDER=${TOOLS_REGISTRY_PROVIDER:-gcr.io}
 PROJECT_ID=${PROJECT_ID:-istio-testing}
 if [[ "${IMAGE_VERSION:-}" == "" ]]; then
-  IMAGE_VERSION=master-62b37843401cf50e072f1d7f4d565730b708d642
+  IMAGE_VERSION=master-9637643344b97c5f67c2c1fc1bdd29a97d9930dc
 fi
 if [[ "${IMAGE_NAME:-}" == "" ]]; then
   IMAGE_NAME=build-tools
diff --git a/go.mod b/go.mod
index 2a887aa11e..78927ce520 100644
--- a/go.mod
+++ b/go.mod
@@ -100,7 +100,7 @@ require (
 	gopkg.in/yaml.v3 v3.0.1
 	helm.sh/helm/v3 v3.14.4
 	istio.io/api v1.22.0-alpha.1.0.20240520153232-e68bd5f8c38c
-	istio.io/client-go v1.22.0-alpha.1.0.20240520153632-efc8e81eb96b
+	istio.io/client-go v1.22.0-alpha.1.0.20240521144430-e59a1ecd7e38
 	k8s.io/api v0.30.0
 	k8s.io/apiextensions-apiserver v0.30.0
 	k8s.io/apimachinery v0.30.0
diff --git a/go.sum b/go.sum
index 28011f1418..fc7c208692 100644
--- a/go.sum
+++ b/go.sum
@@ -1049,8 +1049,8 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 istio.io/api v1.22.0-alpha.1.0.20240520153232-e68bd5f8c38c h1:yV3aTSMEMUhGNAGJuNWVQMXJaaLKeGNEeedIze7RR3Q=
 istio.io/api v1.22.0-alpha.1.0.20240520153232-e68bd5f8c38c/go.mod h1:jkrnQ0rj3g1cOemYFAsntbAqDBOe/B5I6PSFWCfPUdc=
-istio.io/client-go v1.22.0-alpha.1.0.20240520153632-efc8e81eb96b h1:kO/hp0Ef50GmiTsEuxgaNeBiH0kT+lJ2TWqyJCsvx4Q=
-istio.io/client-go v1.22.0-alpha.1.0.20240520153632-efc8e81eb96b/go.mod h1:ATSgIWsq0dDdoYVJ1dcWQHTltDNuGU917OvqlIl69Qc=
+istio.io/client-go v1.22.0-alpha.1.0.20240521144430-e59a1ecd7e38 h1:8jaxkOmAqvUNgCgD4eZ1UoRorLEkryFnMnTaiResPr8=
+istio.io/client-go v1.22.0-alpha.1.0.20240521144430-e59a1ecd7e38/go.mod h1:ATSgIWsq0dDdoYVJ1dcWQHTltDNuGU917OvqlIl69Qc=
 k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78=
 k8s.io/api v0.18.4/go.mod h1:lOIQAKYgai1+vz9J7YcDZwC26Z0zQewYOGWdyIPUUQ4=
 k8s.io/api v0.30.0 h1:siWhRq7cNjy2iHssOB9SCGNCl2spiF1dO3dABqZ8niA=
diff --git a/istio.deps b/istio.deps
index 7afab78e59..48b63429af 100644
--- a/istio.deps
+++ b/istio.deps
@@ -4,13 +4,13 @@
     "name": "PROXY_REPO_SHA",
     "repoName": "proxy",
     "file": "",
-    "lastStableSHA": "8ecf97b08a2b245d9ea6f44f59f69e89a2b457a8"
+    "lastStableSHA": "2508c9c41ba6f611f752e5777308b0b40ef5c7fb"
   },
   {
     "_comment": "",
     "name": "ZTUNNEL_REPO_SHA",
     "repoName": "ztunnel",
     "file": "",
-    "lastStableSHA": "80ac30e0bb69c32c07d1ca5fdebfaa084ed6b11d"
+    "lastStableSHA": "deb8b0abe2165aa287869c4c1f05fed71038fe37"
   }
 ]
diff --git a/manifests/charts/istio-cni/templates/configmap-cni.yaml b/manifests/charts/istio-cni/templates/configmap-cni.yaml
index 5fa4116d89..fec0a96234 100644
--- a/manifests/charts/istio-cni/templates/configmap-cni.yaml
+++ b/manifests/charts/istio-cni/templates/configmap-cni.yaml
@@ -11,11 +11,12 @@ metadata:
     operator.istio.io/component: "Cni"
 data:
   CURRENT_AGENT_VERSION: {{ .Values.cni.tag | default .Values.global.tag | quote }}
-  LOG_LEVEL: {{ .Values.cni.logLevel | quote }}
   AMBIENT_ENABLED: {{ .Values.cni.ambient.enabled | quote }}
   AMBIENT_DNS_CAPTURE: {{ .Values.cni.ambient.dnsCapture | quote | default "false" }}
   AMBIENT_IPV6: {{ .Values.cni.ambient.ipv6 | quote | default "false" }}
+  {{- if .Values.cni.cniConfFileName }} # K8S < 1.24 doesn't like empty values
   CNI_CONF_NAME: {{ .Values.cni.cniConfFileName }} # Name of the CNI config file to create. Only override if you know the exact path your CNI requires..
+  {{- end }}
   CNI_NET_DIR: {{ .Values.cni.cniConfDir | default "/etc/cni/net.d" }}
   CHAINED_CNI_PLUGIN: {{ .Values.cni.chained | quote }}
   EXCLUDED_NAMESPACES: "{{ range $idx, $ns := .Values.cni.excludeNamespaces }}{{ if $idx }},{{ end }}{{ $ns }}{{ end }}"
diff --git a/manifests/charts/istio-cni/values.yaml b/manifests/charts/istio-cni/values.yaml
index 2c18d5993e..0e7e6e99c5 100644
--- a/manifests/charts/istio-cni/values.yaml
+++ b/manifests/charts/istio-cni/values.yaml
@@ -124,7 +124,7 @@ defaults:
 
     # change cni scope level to control logging out of istio-cni-node DaemonSet
     logging:
-      level: default:info,cni:info
+      level: info
 
     logAsJson: false
 
diff --git a/pkg/log/options.go b/pkg/log/options.go
index 790e97f57a..dc2a60fb9c 100644
--- a/pkg/log/options.go
+++ b/pkg/log/options.go
@@ -71,6 +71,14 @@ var stringToLevel = map[string]Level{
 	"none":  NoneLevel,
 }
 
+func StringToLevel(level string) Level {
+	return stringToLevel[level]
+}
+
+func LevelToString(level Level) string {
+	return levelToString[level]
+}
+
 // Options defines the set of options supported by Istio's component logging package.
 type Options struct {
 	// OutputPaths is a list of file system paths to write the log data to.
diff --git a/prow/config/calico.yaml b/prow/config/calico.yaml
index 2c94c5b1d7..408907610a 100644
--- a/prow/config/calico.yaml
+++ b/prow/config/calico.yaml
@@ -63,7 +63,7 @@ data:
       "plugins": [
         {
           "type": "calico",
-          "log_level": "info",
+          "plugin_log_level": "info",
           "log_file_path": "/var/log/calico/cni/cni.log",
           "datastore_type": "kubernetes",
           "nodename": "__KUBERNETES_NODE_NAME__",
diff --git a/releasenotes/notes/51074.yaml b/releasenotes/notes/51074.yaml
new file mode 100644
index 0000000000..d5eaf18faa
--- /dev/null
+++ b/releasenotes/notes/51074.yaml
@@ -0,0 +1,8 @@
+apiVersion: release-notes/v2
+kind: bug-fix
+area: installation
+issue:
+  - 50958
+releaseNotes:
+- |
+  **Fixed** Ensure CNI plugin inherits CNI agent log level, simplify CNI logging config
diff --git a/samples/bookinfo/src/productpage/requirements.txt b/samples/bookinfo/src/productpage/requirements.txt
index 901811d570..fd7d341e40 100644
--- a/samples/bookinfo/src/productpage/requirements.txt
+++ b/samples/bookinfo/src/productpage/requirements.txt
@@ -12,6 +12,6 @@ opentelemetry-sdk==1.22.0
 opentelemetry-semantic-conventions==0.43b0
 opentelemetry-util-http==0.43b0
 prometheus-client==0.19.0
-requests==2.31.0
+requests==2.32.0
 simplejson==3.19.2
 urllib3==2.2.0
diff --git a/security/pkg/server/ca/monitoring.go b/security/pkg/server/ca/monitoring.go
index 24db01614c..71c6f9717c 100644
--- a/security/pkg/server/ca/monitoring.go
+++ b/security/pkg/server/ca/monitoring.go
@@ -60,11 +60,21 @@ var (
 		"The unix timestamp, in seconds, when Citadel root cert will expire. "+
 			"A negative time indicates the cert is expired.",
 	)
+	rootCertExpirySeconds = monitoring.NewDerivedGauge(
+		"citadel_server_root_cert_expiry_seconds",
+		"The time remaining, in seconds, before the root certificate will expire. "+
+			"A negative value indicates the cert is expired.",
+	)
 	certChainExpiryTimestamp = monitoring.NewGauge(
 		"citadel_server_cert_chain_expiry_timestamp",
 		"The unix timestamp, in seconds, when Citadel cert chain will expire. "+
 			"A negative time indicates the cert is expired.",
 	)
+	certChainExpirySeconds = monitoring.NewDerivedGauge(
+		"citadel_server_cert_chain_expiry_seconds",
+		"The time remaining, in seconds, before the certificate chain will expire. "+
+			"A negative value indicates the cert is expired.",
+	)
 )
 
 // monitoringMetrics are counters for certificate signing related operations.
diff --git a/security/pkg/server/ca/server.go b/security/pkg/server/ca/server.go
index 492f193bac..3073ee56ec 100644
--- a/security/pkg/server/ca/server.go
+++ b/security/pkg/server/ca/server.go
@@ -155,6 +155,12 @@ func recordCertsExpiry(keyCertBundle *util.KeyCertBundle) {
 	}
 	rootCertExpiryTimestamp.Record(rootCertExpiry)
 
+	rootCertPem, err := util.ParsePemEncodedCertificate(keyCertBundle.GetRootCertPem())
+	if err != nil {
+		serverCaLog.Errorf("failed to parse the root cert: %v", err)
+	}
+	rootCertExpirySeconds.ValueFrom(func() float64 { return time.Until(rootCertPem.NotAfter).Seconds() })
+
 	if len(keyCertBundle.GetCertChainPem()) == 0 {
 		return
 	}
@@ -164,6 +170,12 @@ func recordCertsExpiry(keyCertBundle *util.KeyCertBundle) {
 		serverCaLog.Errorf("failed to extract CA cert expiry timestamp (error %v)", err)
 	}
 	certChainExpiryTimestamp.Record(certChainExpiry)
+
+	certChainPem, err := util.ParsePemEncodedCertificate(keyCertBundle.GetCertChainPem())
+	if err != nil {
+		serverCaLog.Errorf("failed to parse the cert chain: %v", err)
+	}
+	certChainExpirySeconds.ValueFrom(func() float64 { return time.Until(certChainPem.NotAfter).Seconds() })
 }
 
 // Register registers a GRPC server on the specified port.


Test patch:
diff --git a/cni/pkg/install/cniconfig_test.go b/cni/pkg/install/cniconfig_test.go
index c472dbb798..2a72923316 100644
--- a/cni/pkg/install/cniconfig_test.go
+++ b/cni/pkg/install/cniconfig_test.go
@@ -365,7 +365,7 @@ const (
   "cniVersion": "0.3.1",
   "name": "istio-cni",
   "type": "istio-cni",
-  "log_level": "__LOG_LEVEL__",
+  "plugin_log_level": "__LOG_LEVEL__",
   "kubernetes": {
       "kubeconfig": "__KUBECONFIG_FILENAME__",
       "cni_bin_dir": "/path/cni/bin"
@@ -449,14 +449,14 @@ func TestCreateCNIConfigFile(t *testing.T) {
 		cfgFile := config.InstallConfig{
 			CNIConfName:        c.specifiedConfName,
 			ChainedCNIPlugin:   c.chainedCNIPlugin,
-			LogLevel:           "debug",
+			PluginLogLevel:     "debug",
 			KubeconfigFilename: kubeconfigFilename,
 		}
 
 		cfg := config.InstallConfig{
 			CNIConfName:        c.specifiedConfName,
 			ChainedCNIPlugin:   c.chainedCNIPlugin,
-			LogLevel:           "debug",
+			PluginLogLevel:     "debug",
 			KubeconfigFilename: kubeconfigFilename,
 		}
 		test := func(cfg config.InstallConfig) func(t *testing.T) {
diff --git a/cni/pkg/install/testdata/bridge.conf.golden b/cni/pkg/install/testdata/bridge.conf.golden
index 1c380d24e2..e006de742b 100644
--- a/cni/pkg/install/testdata/bridge.conf.golden
+++ b/cni/pkg/install/testdata/bridge.conf.golden
@@ -28,9 +28,9 @@
         ],
         "kubeconfig": "/path/to/kubeconfig"
       },
-      "log_level": "debug",
       "log_uds_address": "",
       "name": "istio-cni",
+      "plugin_log_level": "debug",
       "type": "istio-cni"
     }
   ]
diff --git a/cni/pkg/install/testdata/istio-cni-prefixed.conf b/cni/pkg/install/testdata/istio-cni-prefixed.conf
index c194a5a0de..163ed94f13 100644
--- a/cni/pkg/install/testdata/istio-cni-prefixed.conf
+++ b/cni/pkg/install/testdata/istio-cni-prefixed.conf
@@ -2,7 +2,7 @@
   "cniVersion": "0.3.1",
   "name": "istio-cni",
   "type": "prefix-istio-cni",
-  "log_level": "debug",
+  "plugin_log_level": "debug",
   "kubernetes": {
       "kubeconfig": "/path/to/kubeconfig",
       "cni_bin_dir": "/path/cni/bin"
diff --git a/cni/pkg/install/testdata/istio-cni.conf b/cni/pkg/install/testdata/istio-cni.conf
index c663efe74a..ad797c20bd 100644
--- a/cni/pkg/install/testdata/istio-cni.conf
+++ b/cni/pkg/install/testdata/istio-cni.conf
@@ -4,7 +4,7 @@
   "type": "istio-cni",
   "ipam": {},
   "dns": {},
-  "log_level": "debug",
+  "plugin_log_level": "debug",
   "log_uds_address": "",
   "cni_event_address": "",
   "ambient_enabled": false,
diff --git a/cni/pkg/install/testdata/istio-cni.conf.template b/cni/pkg/install/testdata/istio-cni.conf.template
index 86a5d6fcc3..8d87e44a1e 100644
--- a/cni/pkg/install/testdata/istio-cni.conf.template
+++ b/cni/pkg/install/testdata/istio-cni.conf.template
@@ -2,7 +2,7 @@
   "cniVersion": "0.3.1",
   "name": "istio-cni",
   "type": "istio-cni",
-  "log_level": "__LOG_LEVEL__",
+  "plugin_log_level": "__LOG_LEVEL__",
   "kubernetes": {
       "kubeconfig": "__KUBECONFIG_FILENAME__",
       "cni_bin_dir": "/path/cni/bin"
diff --git a/cni/pkg/install/testdata/list-with-istio.conflist b/cni/pkg/install/testdata/list-with-istio.conflist
index 8bcea190d0..53b5e91c4e 100644
--- a/cni/pkg/install/testdata/list-with-istio.conflist
+++ b/cni/pkg/install/testdata/list-with-istio.conflist
@@ -32,8 +32,8 @@
         "cni_bin_dir": "/path/cni/bin",
         "kubeconfig": "/path/to/kubeconfig"
       },
-      "log_level": "debug",
       "name": "istio-cni",
+      "plugin_log_level": "debug",
       "type": "istio-cni"
     }
   ]
diff --git a/cni/pkg/install/testdata/list-with-istio.conflist.golden b/cni/pkg/install/testdata/list-with-istio.conflist.golden
index 7cbbbb9972..a8938f4d6c 100644
--- a/cni/pkg/install/testdata/list-with-istio.conflist.golden
+++ b/cni/pkg/install/testdata/list-with-istio.conflist.golden
@@ -38,9 +38,9 @@
         ],
         "kubeconfig": "/path/to/kubeconfig"
       },
-      "log_level": "debug",
       "log_uds_address": "",
       "name": "istio-cni",
+      "plugin_log_level": "debug",
       "type": "istio-cni"
     }
   ]
diff --git a/cni/pkg/install/testdata/list.conflist.golden b/cni/pkg/install/testdata/list.conflist.golden
index 7cbbbb9972..a8938f4d6c 100644
--- a/cni/pkg/install/testdata/list.conflist.golden
+++ b/cni/pkg/install/testdata/list.conflist.golden
@@ -38,9 +38,9 @@
         ],
         "kubeconfig": "/path/to/kubeconfig"
       },
-      "log_level": "debug",
       "log_uds_address": "",
       "name": "istio-cni",
+      "plugin_log_level": "debug",
       "type": "istio-cni"
     }
   ]
diff --git a/cni/pkg/log/uds_test.go b/cni/pkg/log/uds_test.go
index 424572f3e4..64f75dc11e 100644
--- a/cni/pkg/log/uds_test.go
+++ b/cni/pkg/log/uds_test.go
@@ -30,8 +30,7 @@ func TestUDSLog(t *testing.T) {
 	// Start UDS log server
 	udsSockDir := t.TempDir()
 	udsSock := filepath.Join(udsSockDir, "cni.sock")
-	logger := NewUDSLogger()
-	pluginLog.SetOutputLevel(log.DebugLevel) // this will be configured by global.logging.level
+	logger := NewUDSLogger(log.DebugLevel)
 	stop := make(chan struct{})
 	defer close(stop)
 	assert.NoError(t, logger.StartUDSLogServer(udsSock, stop))
diff --git a/cni/pkg/plugin/plugin_test.go b/cni/pkg/plugin/plugin_test.go
index e4793a476d..702ccfd667 100644
--- a/cni/pkg/plugin/plugin_test.go
+++ b/cni/pkg/plugin/plugin_test.go
@@ -76,7 +76,7 @@ var mockConfTmpl = `{
         "routes": []
 
     },
-    "log_level": "debug",
+    "plugin_log_level": "debug",
     "cni_event_address": "%s",
     "ambient_enabled": %t,
     "kubernetes": {
diff --git a/cni/test/install_cni.go b/cni/test/install_cni.go
index ec84baec08..3eb0f92078 100644
--- a/cni/test/install_cni.go
+++ b/cni/test/install_cni.go
@@ -271,7 +271,7 @@ func doTest(t *testing.T, chainedCNIPlugin bool, wd, preConfFile, resultFileName
 			KubeconfigFilename:    "ZZZ-istio-cni-kubeconfig",
 			CNINetDir:             "/etc/cni/net.d",
 			ChainedCNIPlugin:      chainedCNIPlugin,
-			LogLevel:              "debug",
+			PluginLogLevel:        "debug",
 			ExcludeNamespaces:     "istio-system",
 			KubeconfigMode:        constants.DefaultKubeconfigMode,
 			CNIConfName:           envPreconf,
diff --git a/cni/test/testdata/expected/10-calico.conflist-istioconfig b/cni/test/testdata/expected/10-calico.conflist-istioconfig
index 61421f2060..effec50e19 100644
--- a/cni/test/testdata/expected/10-calico.conflist-istioconfig
+++ b/cni/test/testdata/expected/10-calico.conflist-istioconfig
@@ -10,8 +10,8 @@
       "kubernetes": {
         "kubeconfig": "/etc/cni/net.d/calico-kubeconfig"
       },
-      "log_level": "info",
       "mtu": 1500,
+      "plugin_log_level": "info",
       "policy": {
         "type": "k8s"
       },
@@ -35,9 +35,9 @@
         ],
         "kubeconfig": "/etc/cni/net.d/ZZZ-istio-cni-kubeconfig"
       },
-      "log_level": "debug",
       "log_uds_address": "",
       "name": "istio-cni",
+      "plugin_log_level": "debug",
       "type": "istio-cni"
     }
   ]
diff --git a/cni/test/testdata/expected/YYY-istio-cni.conf b/cni/test/testdata/expected/YYY-istio-cni.conf
index 8c8fb54aac..d861dd2065 100644
--- a/cni/test/testdata/expected/YYY-istio-cni.conf
+++ b/cni/test/testdata/expected/YYY-istio-cni.conf
@@ -4,7 +4,7 @@
   "type": "istio-cni",
   "ipam": {},
   "dns": {},
-  "log_level": "debug",
+  "plugin_log_level": "debug",
   "log_uds_address": "",
   "cni_event_address": "/tmp/cnieventfoo",
   "ambient_enabled": false,
diff --git a/cni/test/testdata/expected/minikube_cni.conflist.expected b/cni/test/testdata/expected/minikube_cni.conflist.expected
index 168b76f54d..f84a9e64c6 100644
--- a/cni/test/testdata/expected/minikube_cni.conflist.expected
+++ b/cni/test/testdata/expected/minikube_cni.conflist.expected
@@ -32,9 +32,9 @@
         ],
         "kubeconfig": "/etc/cni/net.d/ZZZ-istio-cni-kubeconfig"
       },
-      "log_level": "debug",
       "log_uds_address": "",
       "name": "istio-cni",
+      "plugin_log_level": "debug",
       "type": "istio-cni"
     }
   ]
diff --git a/cni/test/testdata/pre/calico.conflist b/cni/test/testdata/pre/calico.conflist
index c7cbb7bd93..536679747e 100644
--- a/cni/test/testdata/pre/calico.conflist
+++ b/cni/test/testdata/pre/calico.conflist
@@ -10,8 +10,8 @@
       "kubernetes": {
         "kubeconfig": "/etc/cni/net.d/calico-kubeconfig"
       },
-      "log_level": "info",
       "mtu": 1500,
+      "plugin_log_level": "info",
       "policy": {
         "type": "k8s"
       },
diff --git a/cni/test/testdata/pre/noname_calico.conflist b/cni/test/testdata/pre/noname_calico.conflist
index 8293c9745b..7bba2a5f6e 100644
--- a/cni/test/testdata/pre/noname_calico.conflist
+++ b/cni/test/testdata/pre/noname_calico.conflist
@@ -4,7 +4,7 @@
     {
       "type": "calico",
       "etcd_endpoints": "http://10.110.0.136:6666",
-      "log_level": "info",
+      "plugin_log_level": "info",
       "mtu": 1500,
       "ipam": {
         "type": "calico-ipam"
diff --git a/cni/test/testdata/pre/noplugins_calico.conflist b/cni/test/testdata/pre/noplugins_calico.conflist
index 47f40ac97a..c6efbb1908 100644
--- a/cni/test/testdata/pre/noplugins_calico.conflist
+++ b/cni/test/testdata/pre/noplugins_calico.conflist
@@ -5,7 +5,7 @@
     {
       "type": "calico",
       "etcd_endpoints": "http://10.110.0.136:6666",
-      "log_level": "info",
+      "plugin_log_level": "info",
       "mtu": 1500,
       "ipam": {
         "type": "calico-ipam"
diff --git a/cni/test/testdata/pre/nover_calico.conflist b/cni/test/testdata/pre/nover_calico.conflist
index 0a1b2ed1a1..2f36c44bb8 100644
--- a/cni/test/testdata/pre/nover_calico.conflist
+++ b/cni/test/testdata/pre/nover_calico.conflist
@@ -4,7 +4,7 @@
     {
       "type": "calico",
       "etcd_endpoints": "http://10.110.0.136:6666",
-      "log_level": "info",
+      "plugin_log_level": "info",
       "mtu": 1500,
       "ipam": {
         "type": "calico-ipam"