Repository: giantswarm/observability-operator
Pull request #388: Create Grafana Client on every request
Author: login: QuentinBisson
Created: 2025-05-12T09:17:03Z | Merged: 2025-05-14T07:43:26Z

### What this PR does / why we need it

Closes https://github.com/giantswarm/giantswarm/issues/32664

This pull request introduces significant changes to how the Grafana API client is managed, moving from a pre-initialized client to a dynamically generated client for each request. This improves flexibility, particularly in handling secret changes and enhancing testing. Additionally, unused environment variables and configuration fields related to Grafana credentials have been removed, simplifying the codebase.

### Checklist

- [x] Update changelog in CHANGELOG.md.
- [x] Make sure the new features are scoped to supported observability-bundle versions (see `IsSupporting` booleans)
- [x] Follow deployment test procedure in the tests/manual_e2e directory and have a working branch.



Diff:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b17f1820..d83aaeda 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- Grafana API client is now generated on every requests to support grafana secret changes and allows for better mc-bootstrap testing (https://github.com/giantswarm/giantswarm/issues/32664)
 - Updated alertmanager's inhibitions, getting rid of vintage-specifics.
 
 ## [0.29.0] - 2025-05-05
diff --git a/cmd/main.go b/cmd/main.go
index ef2204e5..2483b263 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -71,8 +71,7 @@ func main() {
 		"Path to the directory where the webhook server will store its TLS certificate and key.")
 	flag.StringVar(&conf.OperatorNamespace, "operator-namespace", "",
 		"The namespace where the observability-operator is running.")
-	flag.StringVar(&grafanaURL, "grafana-url", "http://grafana.monitoring.svc.cluster.local",
-		"grafana URL")
+	flag.StringVar(&grafanaURL, "grafana-url", "http://grafana.monitoring.svc.cluster.local", "grafana URL")
 
 	// Management cluster configuration flags.
 	flag.StringVar(&conf.ManagementCluster.BaseDomain, "management-cluster-base-domain", "",
diff --git a/helm/observability-operator/templates/deployment.yaml b/helm/observability-operator/templates/deployment.yaml
index 9d039daa..82c203ca 100644
--- a/helm/observability-operator/templates/deployment.yaml
+++ b/helm/observability-operator/templates/deployment.yaml
@@ -50,26 +50,6 @@ spec:
             secretKeyRef:
               name: {{ include "resource.default.name" . }}-credentials
               key: opsgenieApiKey
-        - name: GRAFANA_ADMIN_USERNAME
-          valueFrom:
-            secretKeyRef:
-              name: grafana
-              key: admin-user
-        - name: GRAFANA_ADMIN_PASSWORD
-          valueFrom:
-            secretKeyRef:
-              name: grafana
-              key: admin-password
-        - name: GRAFANA_TLS_CERT_FILE
-          valueFrom:
-            secretKeyRef:
-              name: grafana-tls
-              key: tls.crt
-        - name: GRAFANA_TLS_KEY_FILE
-          valueFrom:
-            secretKeyRef:
-              name: grafana-tls
-              key: tls.key
         livenessProbe:
           httpGet:
             path: /healthz
diff --git a/internal/controller/dashboard_controller.go b/internal/controller/dashboard_controller.go
index 315629ca..551cc331 100644
--- a/internal/controller/dashboard_controller.go
+++ b/internal/controller/dashboard_controller.go
@@ -3,7 +3,7 @@ package controller
 import (
 	"context"
 	"encoding/json"
-	"fmt"
+	"net/url"
 
 	grafanaAPI "github.com/grafana/grafana-openapi-client-go/client"
 	"github.com/pkg/errors"
@@ -32,8 +32,9 @@ import (
 // DashboardReconciler reconciles a Dashboard object
 type DashboardReconciler struct {
 	client.Client
-	Scheme     *runtime.Scheme
-	GrafanaAPI *grafanaAPI.GrafanaHTTPAPI
+	Scheme *runtime.Scheme
+
+	grafanaURL *url.URL
 }
 
 const (
@@ -45,18 +46,14 @@ const (
 )
 
 func SetupDashboardReconciler(mgr manager.Manager, conf config.Config) error {
-	grafanaAPI, err := grafanaclient.GenerateGrafanaClient(conf.GrafanaURL, conf)
-	if err != nil {
-		return fmt.Errorf("unable to create grafana client: %w", err)
-	}
-
 	r := &DashboardReconciler{
-		Client:     mgr.GetClient(),
-		Scheme:     mgr.GetScheme(),
-		GrafanaAPI: grafanaAPI,
+		Client: mgr.GetClient(),
+		Scheme: mgr.GetScheme(),
+
+		grafanaURL: conf.GrafanaURL,
 	}
 
-	err = r.SetupWithManager(mgr)
+	err := r.SetupWithManager(mgr)
 	if err != nil {
 		return err
 	}
@@ -85,13 +82,18 @@ func (r *DashboardReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		return ctrl.Result{}, errors.WithStack(client.IgnoreNotFound(err))
 	}
 
+	grafanaAPI, err := grafanaclient.GenerateGrafanaClient(ctx, r.Client, r.grafanaURL)
+	if err != nil {
+		return ctrl.Result{}, errors.WithStack(err)
+	}
+
 	// Handle deleted grafana dashboards
 	if !dashboard.DeletionTimestamp.IsZero() {
-		return ctrl.Result{}, r.reconcileDelete(ctx, dashboard)
+		return ctrl.Result{}, r.reconcileDelete(ctx, grafanaAPI, dashboard)
 	}
 
 	// Handle non-deleted grafana dashboards
-	return r.reconcileCreate(ctx, dashboard)
+	return r.reconcileCreate(ctx, grafanaAPI, dashboard)
 }
 
 // SetupWithManager sets up the controller with the Manager.
@@ -144,7 +146,7 @@ func (r *DashboardReconciler) SetupWithManager(mgr ctrl.Manager) error {
 // reconcileCreate ensures the Grafana dashboard described in configmap is created in Grafana.
 // This function is also responsible for:
 // - Adding the finalizer to the configmap
-func (r DashboardReconciler) reconcileCreate(ctx context.Context, dashboard *v1.ConfigMap) (ctrl.Result, error) { // nolint:unparam
+func (r DashboardReconciler) reconcileCreate(ctx context.Context, grafanaAPI *grafanaAPI.GrafanaHTTPAPI, dashboard *v1.ConfigMap) (ctrl.Result, error) { // nolint:unparam
 	logger := log.FromContext(ctx)
 
 	// Add finalizer first if not set to avoid the race condition between init and delete.
@@ -166,7 +168,7 @@ func (r DashboardReconciler) reconcileCreate(ctx context.Context, dashboard *v1.
 	}
 
 	// Configure the dashboard in Grafana
-	if err := r.configureDashboard(ctx, dashboard); err != nil {
+	if err := r.configureDashboard(ctx, grafanaAPI, dashboard); err != nil {
 		return ctrl.Result{}, errors.WithStack(err)
 	}
 
@@ -204,7 +206,7 @@ func getOrgFromDashboardConfigmap(dashboard *v1.ConfigMap) (string, error) {
 	return "", errors.New("No organization label found in configmap")
 }
 
-func (r DashboardReconciler) configureDashboard(ctx context.Context, dashboardCM *v1.ConfigMap) error {
+func (r DashboardReconciler) configureDashboard(ctx context.Context, grafanaAPI *grafanaAPI.GrafanaHTTPAPI, dashboardCM *v1.ConfigMap) error {
 	logger := log.FromContext(ctx)
 
 	dashboardOrg, err := getOrgFromDashboardConfigmap(dashboardCM)
@@ -216,14 +218,14 @@ func (r DashboardReconciler) configureDashboard(ctx context.Context, dashboardCM
 	// TODO Tenant Governance: Filter the dashboards with the list of authorized tenants
 
 	// Switch context to the dashboards-defined org
-	organization, err := grafana.FindOrgByName(r.GrafanaAPI, dashboardOrg)
+	organization, err := grafana.FindOrgByName(grafanaAPI, dashboardOrg)
 	if err != nil {
 		logger.Error(err, "failed to find organization", "organization", dashboardOrg)
 		return errors.WithStack(err)
 	}
-	currentOrgID := r.GrafanaAPI.OrgID()
-	r.GrafanaAPI.WithOrgID(organization.ID)
-	defer r.GrafanaAPI.WithOrgID(currentOrgID)
+	currentOrgID := grafanaAPI.OrgID()
+	grafanaAPI.WithOrgID(organization.ID)
+	defer grafanaAPI.WithOrgID(currentOrgID)
 
 	for _, dashboardString := range dashboardCM.Data {
 		var dashboard map[string]any
@@ -243,7 +245,7 @@ func (r DashboardReconciler) configureDashboard(ctx context.Context, dashboardCM
 		cleanDashboardID(dashboard)
 
 		// Create or update dashboard
-		err = grafana.PublishDashboard(r.GrafanaAPI, dashboard)
+		err = grafana.PublishDashboard(grafanaAPI, dashboard)
 		if err != nil {
 			logger.Error(err, "Failed updating dashboard")
 			continue
@@ -256,7 +258,7 @@ func (r DashboardReconciler) configureDashboard(ctx context.Context, dashboardCM
 }
 
 // reconcileDelete deletes the grafana dashboard.
-func (r DashboardReconciler) reconcileDelete(ctx context.Context, dashboardCM *v1.ConfigMap) error {
+func (r DashboardReconciler) reconcileDelete(ctx context.Context, grafanaAPI *grafanaAPI.GrafanaHTTPAPI, dashboardCM *v1.ConfigMap) error {
 	logger := log.FromContext(ctx)
 
 	// We do not need to delete anything if there is no finalizer on the grafana dashboard
@@ -271,14 +273,14 @@ func (r DashboardReconciler) reconcileDelete(ctx context.Context, dashboardCM *v
 	}
 
 	// Switch context to the dashboards-defined org
-	organization, err := grafana.FindOrgByName(r.GrafanaAPI, dashboardOrg)
+	organization, err := grafana.FindOrgByName(grafanaAPI, dashboardOrg)
 	if err != nil {
 		logger.Error(err, "failed to find organization", "organization", dashboardOrg)
 		return errors.WithStack(err)
 	}
-	currentOrgID := r.GrafanaAPI.OrgID()
-	r.GrafanaAPI.WithOrgID(organization.ID)
-	defer r.GrafanaAPI.WithOrgID(currentOrgID)
+	currentOrgID := grafanaAPI.OrgID()
+	grafanaAPI.WithOrgID(organization.ID)
+	defer grafanaAPI.WithOrgID(currentOrgID)
 
 	for _, dashboardString := range dashboardCM.Data {
 		var dashboard map[string]interface{}
@@ -297,13 +299,13 @@ func (r DashboardReconciler) reconcileDelete(ctx context.Context, dashboardCM *v
 		// Clean the dashboard ID to avoid conflicts
 		cleanDashboardID(dashboard)
 
-		_, err = r.GrafanaAPI.Dashboards.GetDashboardByUID(dashboardUID)
+		_, err = grafanaAPI.Dashboards.GetDashboardByUID(dashboardUID)
 		if err != nil {
 			logger.Error(err, "Failed getting dashboard")
 			continue
 		}
 
-		_, err = r.GrafanaAPI.Dashboards.DeleteDashboardByUID(dashboardUID)
+		_, err = grafanaAPI.Dashboards.DeleteDashboardByUID(dashboardUID)
 		if err != nil {
 			logger.Error(err, "Failed deleting dashboard")
 			continue
diff --git a/internal/controller/grafanaorganization_controller.go b/internal/controller/grafanaorganization_controller.go
index f736905c..c73696fd 100644
--- a/internal/controller/grafanaorganization_controller.go
+++ b/internal/controller/grafanaorganization_controller.go
@@ -4,7 +4,7 @@ import (
 	"cmp"
 	"context"
 	stderrors "errors"
-	"fmt"
+	"net/url"
 	"slices"
 
 	grafanaAPI "github.com/grafana/grafana-openapi-client-go/client"
@@ -33,22 +33,17 @@ import (
 type GrafanaOrganizationReconciler struct {
 	client.Client
 	Scheme     *runtime.Scheme
-	GrafanaAPI *grafanaAPI.GrafanaHTTPAPI
+	grafanaURL *url.URL
 }
 
 func SetupGrafanaOrganizationReconciler(mgr manager.Manager, conf config.Config) error {
-	grafanaAPI, err := grafanaclient.GenerateGrafanaClient(conf.GrafanaURL, conf)
-	if err != nil {
-		return fmt.Errorf("unable to create grafana client: %w", err)
-	}
-
 	r := &GrafanaOrganizationReconciler{
 		Client:     mgr.GetClient(),
 		Scheme:     mgr.GetScheme(),
-		GrafanaAPI: grafanaAPI,
+		grafanaURL: conf.GrafanaURL,
 	}
 
-	err = r.SetupWithManager(mgr)
+	err := r.SetupWithManager(mgr)
 	if err != nil {
 		return err
 	}
@@ -77,13 +72,18 @@ func (r *GrafanaOrganizationReconciler) Reconcile(ctx context.Context, req ctrl.
 		return ctrl.Result{}, errors.WithStack(client.IgnoreNotFound(err))
 	}
 
+	grafanaAPI, err := grafanaclient.GenerateGrafanaClient(ctx, r.Client, r.grafanaURL)
+	if err != nil {
+		return ctrl.Result{}, errors.WithStack(err)
+	}
+
 	// Handle deleted grafana organizations
 	if !grafanaOrganization.DeletionTimestamp.IsZero() {
-		return ctrl.Result{}, r.reconcileDelete(ctx, grafanaOrganization)
+		return ctrl.Result{}, r.reconcileDelete(ctx, grafanaAPI, grafanaOrganization)
 	}
 
 	// Handle non-deleted grafana organizations
-	return r.reconcileCreate(ctx, grafanaOrganization)
+	return r.reconcileCreate(ctx, grafanaAPI, grafanaOrganization)
 }
 
 // SetupWithManager sets up the controller with the Manager.
@@ -141,7 +141,7 @@ func (r *GrafanaOrganizationReconciler) SetupWithManager(mgr ctrl.Manager) error
 // - Adding the finalizer to the CR
 // - Updating the CR status field
 // - Renaming the Grafana Main Org.
-func (r GrafanaOrganizationReconciler) reconcileCreate(ctx context.Context, grafanaOrganization *v1alpha1.GrafanaOrganization) (ctrl.Result, error) { // nolint:unparam
+func (r GrafanaOrganizationReconciler) reconcileCreate(ctx context.Context, grafanaAPI *grafanaAPI.GrafanaHTTPAPI, grafanaOrganization *v1alpha1.GrafanaOrganization) (ctrl.Result, error) { // nolint:unparam
 	logger := log.FromContext(ctx)
 
 	// Add finalizer first if not set to avoid the race condition between init and delete.
@@ -163,19 +163,19 @@ func (r GrafanaOrganizationReconciler) reconcileCreate(ctx context.Context, graf
 	}
 
 	// Configure the organization in Grafana
-	if err := r.configureOrganization(ctx, grafanaOrganization); err != nil {
+	if err := r.configureOrganization(ctx, grafanaAPI, grafanaOrganization); err != nil {
 		return ctrl.Result{}, errors.WithStack(err)
 	}
 
 	var errs []error
 
 	// Update the datasources in the CR's status
-	if err := r.configureDatasources(ctx, grafanaOrganization); err != nil {
+	if err := r.configureDatasources(ctx, grafanaAPI, grafanaOrganization); err != nil {
 		errs = append(errs, err)
 	}
 
 	// Configure Grafana RBAC
-	if err := r.configureGrafanaSSO(ctx); err != nil {
+	if err := r.configureGrafanaSSO(ctx, grafanaAPI); err != nil {
 		errs = append(errs, err)
 	}
 
@@ -208,11 +208,11 @@ func newOrganization(grafanaOrganization *v1alpha1.GrafanaOrganization) grafana.
 	}
 }
 
-func (r GrafanaOrganizationReconciler) configureOrganization(ctx context.Context, grafanaOrganization *v1alpha1.GrafanaOrganization) error {
+func (r GrafanaOrganizationReconciler) configureOrganization(ctx context.Context, grafanaAPI *grafanaAPI.GrafanaHTTPAPI, grafanaOrganization *v1alpha1.GrafanaOrganization) error {
 	logger := log.FromContext(ctx)
 	// Create or update organization in Grafana
 	var organization = newOrganization(grafanaOrganization)
-	err := grafana.UpsertOrganization(ctx, r.GrafanaAPI, &organization)
+	err := grafana.UpsertOrganization(ctx, grafanaAPI, &organization)
 	if err != nil {
 		logger.Error(err, "failed to upsert grafanaOrganization")
 		return errors.WithStack(err)
@@ -233,14 +233,14 @@ func (r GrafanaOrganizationReconciler) configureOrganization(ctx context.Context
 	return nil
 }
 
-func (r GrafanaOrganizationReconciler) configureDatasources(ctx context.Context, grafanaOrganization *v1alpha1.GrafanaOrganization) error {
+func (r GrafanaOrganizationReconciler) configureDatasources(ctx context.Context, grafanaAPI *grafanaAPI.GrafanaHTTPAPI, grafanaOrganization *v1alpha1.GrafanaOrganization) error {
 	logger := log.FromContext(ctx)
 
 	logger.Info("configuring data sources")
 
 	// Create or update organization in Grafana
 	var organization = newOrganization(grafanaOrganization)
-	datasources, err := grafana.ConfigureDefaultDatasources(ctx, r.GrafanaAPI, organization)
+	datasources, err := grafana.ConfigureDefaultDatasources(ctx, grafanaAPI, organization)
 	if err != nil {
 		logger.Error(err, "failed to configure the grafanaOrganization with default datasources")
 		return errors.WithStack(err)
@@ -267,7 +267,7 @@ func (r GrafanaOrganizationReconciler) configureDatasources(ctx context.Context,
 }
 
 // reconcileDelete deletes the grafana organization.
-func (r GrafanaOrganizationReconciler) reconcileDelete(ctx context.Context, grafanaOrganization *v1alpha1.GrafanaOrganization) error {
+func (r GrafanaOrganizationReconciler) reconcileDelete(ctx context.Context, grafanaAPI *grafanaAPI.GrafanaHTTPAPI, grafanaOrganization *v1alpha1.GrafanaOrganization) error {
 	logger := log.FromContext(ctx)
 
 	// We do not need to delete anything if there is no finalizer on the grafana organization
@@ -275,12 +275,10 @@ func (r GrafanaOrganizationReconciler) reconcileDelete(ctx context.Context, graf
 		return nil
 	}
 
-	// Delete organization in Grafana
-	var organization = newOrganization(grafanaOrganization)
-
 	// Delete organization in Grafana if it exists
+	var organization = newOrganization(grafanaOrganization)
 	if grafanaOrganization.Status.OrgID > 0 {
-		err := grafana.DeleteOrganization(ctx, r.GrafanaAPI, organization)
+		err := grafana.DeleteOrganization(ctx, grafanaAPI, organization)
 		if err != nil {
 			return errors.WithStack(err)
 		}
@@ -295,7 +293,7 @@ func (r GrafanaOrganizationReconciler) reconcileDelete(ctx context.Context, graf
 	var errs []error
 
 	// Configure Grafana RBAC
-	if err := r.configureGrafanaSSO(ctx); err != nil {
+	if err := r.configureGrafanaSSO(ctx, grafanaAPI); err != nil {
 		errs = append(errs, err)
 	}
 
@@ -322,7 +320,7 @@ func (r GrafanaOrganizationReconciler) reconcileDelete(ctx context.Context, graf
 }
 
 // configureGrafana ensures the RBAC configuration is set in Grafana.
-func (r *GrafanaOrganizationReconciler) configureGrafanaSSO(ctx context.Context) error {
+func (r *GrafanaOrganizationReconciler) configureGrafanaSSO(ctx context.Context, grafanaAPI *grafanaAPI.GrafanaHTTPAPI) error {
 	logger := log.FromContext(ctx)
 
 	organizationList := v1alpha1.GrafanaOrganizationList{}
@@ -337,7 +335,7 @@ func (r *GrafanaOrganizationReconciler) configureGrafanaSSO(ctx context.Context)
 	for i, organization := range organizationList.Items {
 		organizations[i] = newOrganization(&organization)
 	}
-	err = grafana.ConfigureSSOSettings(ctx, r.GrafanaAPI, organizations)
+	err = grafana.ConfigureSSOSettings(ctx, grafanaAPI, organizations)
 	if err != nil {
 		logger.Error(err, "failed to configure grafanaOrganization with SSO settings")
 		return errors.WithStack(err)
diff --git a/pkg/config/config.go b/pkg/config/config.go
index e2c19eff..7a03a2d9 100644
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@@ -25,10 +25,5 @@ type Config struct {
 }
 
 type Environment struct {
-	GrafanaAdminUsername string `env:"GRAFANA_ADMIN_USERNAME,required=true"`
-	GrafanaAdminPassword string `env:"GRAFANA_ADMIN_PASSWORD,required=true"`
-	GrafanaTLSCertFile   string `env:"GRAFANA_TLS_CERT_FILE,required=true"`
-	GrafanaTLSKeyFile    string `env:"GRAFANA_TLS_KEY_FILE,required=true"`
-
 	OpsgenieApiKey string `env:"OPSGENIE_API_KEY,required=true"`
 }
diff --git a/pkg/grafana/client/client.go b/pkg/grafana/client/client.go
index 45958bb5..636e443c 100644
--- a/pkg/grafana/client/client.go
+++ b/pkg/grafana/client/client.go
@@ -1,53 +1,99 @@
 package client
 
 import (
+	"context"
+	"crypto/tls"
 	"fmt"
 	"net/url"
 
 	grafana "github.com/grafana/grafana-openapi-client-go/client"
-
-	"github.com/giantswarm/observability-operator/pkg/config"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
 	clientConfigNumRetries = 3
+
+	// Secret names and keys for Grafana configuration
+	grafanaNamespace = "monitoring"
+
+	grafanaAdminSecretName        = "grafana"
+	grafanaAdminSecretUserKey     = "admin-user"
+	grafanaAdminSecretPasswordKey = "admin-password"
+
+	grafanaTLSSecretName    = "grafana-tls"
+	grafanaTLSSecretCertKey = "tls.crt"
+	grafanaTLSSecretKeyKey  = "tls.key"
 )
 
-func GenerateGrafanaClient(grafanaURL *url.URL, conf config.Config) (*grafana.GrafanaHTTPAPI, error) {
-	var err error
+// GenerateGrafanaClient creates a new Grafana client by fetching credentials
+// and TLS configuration from Kubernetes secrets.
+func GenerateGrafanaClient(
+	ctx context.Context,
+	k8sClient client.Client,
+	grafanaURL *url.URL,
+) (*grafana.GrafanaHTTPAPI, error) {
+	// Get Grafana admin credentials from secret
+	adminSecret := &corev1.Secret{}
+	adminSecretKey := client.ObjectKey{Namespace: grafanaNamespace, Name: grafanaAdminSecretName}
+	if err := k8sClient.Get(ctx, adminSecretKey, adminSecret); err != nil {
+		return nil, fmt.Errorf("failed to get Grafana admin secret %q in namespace %q: %w", grafanaAdminSecretName, grafanaNamespace, err)
+	}
 
-	// Generate Grafana client
-	// Get grafana admin-password and admin-user
-	adminUserCredentials := AdminCredentials{
-		Username: conf.Environment.GrafanaAdminUsername,
-		Password: conf.Environment.GrafanaAdminPassword,
+	adminUsernameBytes, ok := adminSecret.Data[grafanaAdminSecretUserKey]
+	if !ok {
+		return nil, fmt.Errorf("key %q not found in Grafana admin secret %q", grafanaAdminSecretUserKey, grafanaAdminSecretName)
+	}
+	if len(adminUsernameBytes) == 0 {
+		return nil, fmt.Errorf("GrafanaAdminUsername is empty in secret %q", grafanaAdminSecretName)
 	}
-	if adminUserCredentials.Username == "" {
-		return nil, fmt.Errorf("GrafanaAdminUsername not set: %q", conf.Environment.GrafanaAdminUsername)
 
+	adminPasswordBytes, ok := adminSecret.Data[grafanaAdminSecretPasswordKey]
+	if !ok {
+		return nil, fmt.Errorf("key %q not found in Grafana admin secret %q", grafanaAdminSecretPasswordKey, grafanaAdminSecretName)
+	}
+	if len(adminPasswordBytes) == 0 {
+		return nil, fmt.Errorf("GrafanaAdminPassword is empty in secret %q", grafanaAdminSecretName)
 	}
-	if adminUserCredentials.Password == "" {
-		return nil, fmt.Errorf("GrafanaAdminPassword not set: %q", conf.Environment.GrafanaAdminPassword)
 
+	// Get Grafana TLS configuration from secret, if it exists
+	var clientTLSConfig *tls.Config
+	tlsSecret := &corev1.Secret{}
+	tlsSecretKey := client.ObjectKey{Namespace: grafanaNamespace, Name: grafanaTLSSecretName}
+	err := k8sClient.Get(ctx, tlsSecretKey, tlsSecret)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get Grafana TLS secret %q in namespace %q: %w", grafanaTLSSecretName, grafanaNamespace, err)
+	}
+
+	// TLS Secret found, try to load cert and key
+	tlsCertBytes, certOk := tlsSecret.Data[grafanaTLSSecretCertKey]
+	tlsKeyBytes, keyOk := tlsSecret.Data[grafanaTLSSecretKeyKey]
+
+	if !certOk || len(tlsCertBytes) == 0 {
+		return nil, fmt.Errorf("key %q not found or empty in Grafana TLS secret %q", grafanaTLSSecretCertKey, grafanaTLSSecretName)
+	}
+	if !keyOk || len(tlsKeyBytes) == 0 {
+		return nil, fmt.Errorf("key %q not found or empty in Grafana TLS secret %q", grafanaTLSSecretKeyKey, grafanaTLSSecretName)
 	}
 
-	grafanaTLSConfig, err := TLSConfig{
-		Cert: conf.Environment.GrafanaTLSCertFile,
-		Key:  conf.Environment.GrafanaTLSKeyFile,
-	}.toTLSConfig()
+	// tlsConfigInput is our local struct type from ./tls.go
+	tlsConfigInput := TLSConfig{
+		Cert: string(tlsCertBytes),
+		Key:  string(tlsKeyBytes),
+	}
+	// clientTLSConfig is the *tls.Config for the Grafana client
+	clientTLSConfig, err = tlsConfigInput.toTLSConfig()
 	if err != nil {
-		return nil, fmt.Errorf("failed to build tls config: %w", err)
+		return nil, fmt.Errorf("failed to build TLS config from secret %q: %w", grafanaTLSSecretName, err)
 	}
 
 	cfg := &grafana.TransportConfig{
-		Schemes:  []string{grafanaURL.Scheme},
-		BasePath: "/api",
-		Host:     grafanaURL.Host,
-		// We use basic auth to authenticate on grafana.
-		BasicAuth: url.UserPassword(adminUserCredentials.Username, adminUserCredentials.Password),
-		// NumRetries contains the optional number of attempted retries.
+		Schemes:    []string{grafanaURL.Scheme},
+		BasePath:   "/api",
+		Host:       grafanaURL.Host,
+		BasicAuth:  url.UserPassword(string(adminUsernameBytes), string(adminPasswordBytes)),
 		NumRetries: clientConfigNumRetries,
-		TLSConfig:  grafanaTLSConfig,
+		TLSConfig:  clientTLSConfig,
 	}
 
 	return grafana.NewHTTPClientWithConfig(nil, cfg), nil


Reviews:
user: login: copilot-pull-request-reviewer[bot]
body: ## Pull Request Overview

This PR refactors the Grafana client by generating a new instance per request to support dynamic secret changes and improve testing flexibility. Key changes include:
- Moving from a pre-initialized client to dynamic generation by fetching credentials and TLS configuration from Kubernetes secrets.
- Removing deprecated environment variables and configuration fields for Grafana credentials.
- Updating controllers and helm templates to accommodate the new client instantiation logic.

### Reviewed Changes

Copilot reviewed 7 out of 7 changed files in this pull request and generated 1 comment.

<details>
<summary>Show a summary per file</summary>

| File                                              | Description                                                              |
| ------------------------------------------------- | ------------------------------------------------------------------------ |
| pkg/grafana/client/client.go                      | Implements dynamic client generation including TLS secret retrieval.     |
| pkg/config/config.go                              | Removes obsolete configuration for Grafana credentials and TLS files.    |
| internal/controller/grafanaorganization_controller.go | Updates organization reconciler to generate and use the new client.        |
| internal/controller/dashboard_controller.go       | Updates dashboard reconciler to generate and use the new client.           |
| helm/observability-operator/templates/deployment.yaml | Removes Grafana credential environment variables from the deployment.     |
| cmd/main.go                                      | Minor correction in flag formatting.                                     |
| CHANGELOG.md                                     | Adds an entry summarizing the changes made to the Grafana client usage.    |
</details>





state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-12T09:17:27Z
user: login: QuentinBisson
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-12T09:31:11Z
user: login: TheoBrigitte
body: Looks overall ok, but I don't like very much this passing around of `grafanaAPI.GrafanaHTTPAPI` this make it cumbersome to always have to add this parameter to every function. It would be best when we embed this into a struct.
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-13T10:50:27Z
user: login: QuentinBisson
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-13T11:22:52Z
user: login: TheoBrigitte
state: APPROVED
author_association: MEMBER
submitted_at: 2025-05-13T14:25:31Z

Review comments:
diff_hunk: @@ -1,53 +1,106 @@
 package client
 
 import (
+	"context"
+	"crypto/tls"
 	"fmt"
 	"net/url"
 
 	grafana "github.com/grafana/grafana-openapi-client-go/client"
-
-	"github.com/giantswarm/observability-operator/pkg/config"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
 	clientConfigNumRetries = 3
+
+	// Secret names and keys for Grafana configuration
+	grafanaNamespace = "monitoring"
+
+	grafanaAdminSecretName        = "grafana"
+	grafanaAdminSecretUserKey     = "admin-user"
+	grafanaAdminSecretPasswordKey = "admin-password"
+
+	grafanaTLSSecretName    = "grafana-tls"
+	grafanaTLSSecretCertKey = "tls.crt"
+	grafanaTLSSecretKeyKey  = "tls.key"
 )
 
-func GenerateGrafanaClient(grafanaURL *url.URL, conf config.Config) (*grafana.GrafanaHTTPAPI, error) {
-	var err error
+// GenerateGrafanaClient creates a new Grafana client by fetching credentials
+// and TLS configuration from Kubernetes secrets.
+func GenerateGrafanaClient(
+	ctx context.Context,
+	k8sClient client.Client,
+	grafanaURL *url.URL,
+) (*grafana.GrafanaHTTPAPI, error) {
+	// Get Grafana admin credentials from secret
+	adminSecret := &corev1.Secret{}
+	adminSecretKey := client.ObjectKey{Namespace: grafanaNamespace, Name: grafanaAdminSecretName}
+	if err := k8sClient.Get(ctx, adminSecretKey, adminSecret); err != nil {
+		return nil, fmt.Errorf("failed to get Grafana admin secret %q in namespace %q: %w", grafanaAdminSecretName, grafanaNamespace, err)
+	}
 
-	// Generate Grafana client
-	// Get grafana admin-password and admin-user
-	adminUserCredentials := AdminCredentials{
-		Username: conf.Environment.GrafanaAdminUsername,
-		Password: conf.Environment.GrafanaAdminPassword,
+	adminUsernameBytes, ok := adminSecret.Data[grafanaAdminSecretUserKey]
+	if !ok {
+		return nil, fmt.Errorf("key %q not found in Grafana admin secret %q", grafanaAdminSecretUserKey, grafanaAdminSecretName)
+	}
+	if len(adminUsernameBytes) == 0 {
+		return nil, fmt.Errorf("GrafanaAdminUsername is empty in secret %q", grafanaAdminSecretName)
 	}
-	if adminUserCredentials.Username == "" {
-		return nil, fmt.Errorf("GrafanaAdminUsername not set: %q", conf.Environment.GrafanaAdminUsername)
 
+	adminPasswordBytes, ok := adminSecret.Data[grafanaAdminSecretPasswordKey]
+	if !ok {
+		return nil, fmt.Errorf("key %q not found in Grafana admin secret %q", grafanaAdminSecretPasswordKey, grafanaAdminSecretName)
+	}
+	if len(adminPasswordBytes) == 0 {
+		return nil, fmt.Errorf("GrafanaAdminPassword is empty in secret %q", grafanaAdminSecretName)
 	}
-	if adminUserCredentials.Password == "" {
-		return nil, fmt.Errorf("GrafanaAdminPassword not set: %q", conf.Environment.GrafanaAdminPassword)
 
+	adminUserCredentials := AdminCredentials{
+		Username: string(adminUsernameBytes),
+		Password: string(adminPasswordBytes),
 	}
 
-	grafanaTLSConfig, err := TLSConfig{
-		Cert: conf.Environment.GrafanaTLSCertFile,
-		Key:  conf.Environment.GrafanaTLSKeyFile,
-	}.toTLSConfig()
+	// Get Grafana TLS configuration from secret, if it exists
+	var clientTLSConfig *tls.Config
+	tlsSecret := &corev1.Secret{}
+	tlsSecretKey := client.ObjectKey{Namespace: grafanaNamespace, Name: grafanaTLSSecretName}
+	err := k8sClient.Get(ctx, tlsSecretKey, tlsSecret)
 	if err != nil {
-		return nil, fmt.Errorf("failed to build tls config: %w", err)
+		return nil, fmt.Errorf("failed to get Grafana TLS secret %q in namespace %q: %w", grafanaTLSSecretName, grafanaNamespace, err)
+	}
+
+	// TLS Secret found, try to load cert and key
+	tlsCertBytes, certOk := tlsSecret.Data[grafanaTLSSecretCertKey]
+	tlsKeyBytes, keyOk := tlsSecret.Data[grafanaTLSSecretKeyKey]
+
+	if !certOk || len(tlsCertBytes) == 0 {
+		return nil, fmt.Errorf("key %q not found or empty in Grafana TLS secret %q", grafanaTLSSecretCertKey, grafanaTLSSecretName)
+	}
+	if !keyOk || len(tlsKeyBytes) == 0 {
+		return nil, fmt.Errorf("key %q not found or empty in Grafana TLS secret %q", grafanaTLSSecretKeyKey, grafanaTLSSecretName)
+	}
+
+	// tlsConfigInput is our local struct type from ./tls.go
+	tlsConfigInput := TLSConfig{
+		Cert: string(tlsCertBytes),
+		Key:  string(tlsKeyBytes),
+	}
+	// clientTLSConfig is the *tls.Config for the Grafana client
+	var buildErr error
+	clientTLSConfig, buildErr = tlsConfigInput.toTLSConfig()
+	if buildErr != nil {
+		return nil, fmt.Errorf("failed to build TLS config from secret %q: %w", grafanaTLSSecretName, buildErr)
 	}
+	fmt.Println("Successfully created Grafana client with TLS configuration.")
path: pkg/grafana/client/client.go
user: login: Copilot
body: [nitpick] Consider replacing fmt.Println with a structured logger to maintain consistency with the application's logging practices.
created_at: 2025-05-12T09:17:27Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1,53 +1,106 @@
 package client
 
 import (
+	"context"
+	"crypto/tls"
 	"fmt"
 	"net/url"
 
 	grafana "github.com/grafana/grafana-openapi-client-go/client"
-
-	"github.com/giantswarm/observability-operator/pkg/config"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
 	clientConfigNumRetries = 3
+
+	// Secret names and keys for Grafana configuration
+	grafanaNamespace = "monitoring"
+
+	grafanaAdminSecretName        = "grafana"
+	grafanaAdminSecretUserKey     = "admin-user"
+	grafanaAdminSecretPasswordKey = "admin-password"
+
+	grafanaTLSSecretName    = "grafana-tls"
+	grafanaTLSSecretCertKey = "tls.crt"
+	grafanaTLSSecretKeyKey  = "tls.key"
 )
 
-func GenerateGrafanaClient(grafanaURL *url.URL, conf config.Config) (*grafana.GrafanaHTTPAPI, error) {
-	var err error
+// GenerateGrafanaClient creates a new Grafana client by fetching credentials
+// and TLS configuration from Kubernetes secrets.
+func GenerateGrafanaClient(
+	ctx context.Context,
+	k8sClient client.Client,
+	grafanaURL *url.URL,
+) (*grafana.GrafanaHTTPAPI, error) {
+	// Get Grafana admin credentials from secret
+	adminSecret := &corev1.Secret{}
+	adminSecretKey := client.ObjectKey{Namespace: grafanaNamespace, Name: grafanaAdminSecretName}
+	if err := k8sClient.Get(ctx, adminSecretKey, adminSecret); err != nil {
+		return nil, fmt.Errorf("failed to get Grafana admin secret %q in namespace %q: %w", grafanaAdminSecretName, grafanaNamespace, err)
+	}
 
-	// Generate Grafana client
-	// Get grafana admin-password and admin-user
-	adminUserCredentials := AdminCredentials{
-		Username: conf.Environment.GrafanaAdminUsername,
-		Password: conf.Environment.GrafanaAdminPassword,
+	adminUsernameBytes, ok := adminSecret.Data[grafanaAdminSecretUserKey]
+	if !ok {
+		return nil, fmt.Errorf("key %q not found in Grafana admin secret %q", grafanaAdminSecretUserKey, grafanaAdminSecretName)
+	}
+	if len(adminUsernameBytes) == 0 {
+		return nil, fmt.Errorf("GrafanaAdminUsername is empty in secret %q", grafanaAdminSecretName)
 	}
-	if adminUserCredentials.Username == "" {
-		return nil, fmt.Errorf("GrafanaAdminUsername not set: %q", conf.Environment.GrafanaAdminUsername)
 
+	adminPasswordBytes, ok := adminSecret.Data[grafanaAdminSecretPasswordKey]
+	if !ok {
+		return nil, fmt.Errorf("key %q not found in Grafana admin secret %q", grafanaAdminSecretPasswordKey, grafanaAdminSecretName)
+	}
+	if len(adminPasswordBytes) == 0 {
+		return nil, fmt.Errorf("GrafanaAdminPassword is empty in secret %q", grafanaAdminSecretName)
 	}
-	if adminUserCredentials.Password == "" {
-		return nil, fmt.Errorf("GrafanaAdminPassword not set: %q", conf.Environment.GrafanaAdminPassword)
 
+	adminUserCredentials := AdminCredentials{
+		Username: string(adminUsernameBytes),
+		Password: string(adminPasswordBytes),
 	}
 
-	grafanaTLSConfig, err := TLSConfig{
-		Cert: conf.Environment.GrafanaTLSCertFile,
-		Key:  conf.Environment.GrafanaTLSKeyFile,
-	}.toTLSConfig()
+	// Get Grafana TLS configuration from secret, if it exists
+	var clientTLSConfig *tls.Config
+	tlsSecret := &corev1.Secret{}
+	tlsSecretKey := client.ObjectKey{Namespace: grafanaNamespace, Name: grafanaTLSSecretName}
+	err := k8sClient.Get(ctx, tlsSecretKey, tlsSecret)
 	if err != nil {
-		return nil, fmt.Errorf("failed to build tls config: %w", err)
+		return nil, fmt.Errorf("failed to get Grafana TLS secret %q in namespace %q: %w", grafanaTLSSecretName, grafanaNamespace, err)
+	}
+
+	// TLS Secret found, try to load cert and key
+	tlsCertBytes, certOk := tlsSecret.Data[grafanaTLSSecretCertKey]
+	tlsKeyBytes, keyOk := tlsSecret.Data[grafanaTLSSecretKeyKey]
+
+	if !certOk || len(tlsCertBytes) == 0 {
+		return nil, fmt.Errorf("key %q not found or empty in Grafana TLS secret %q", grafanaTLSSecretCertKey, grafanaTLSSecretName)
+	}
+	if !keyOk || len(tlsKeyBytes) == 0 {
+		return nil, fmt.Errorf("key %q not found or empty in Grafana TLS secret %q", grafanaTLSSecretKeyKey, grafanaTLSSecretName)
+	}
+
+	// tlsConfigInput is our local struct type from ./tls.go
+	tlsConfigInput := TLSConfig{
+		Cert: string(tlsCertBytes),
+		Key:  string(tlsKeyBytes),
+	}
+	// clientTLSConfig is the *tls.Config for the Grafana client
+	var buildErr error
+	clientTLSConfig, buildErr = tlsConfigInput.toTLSConfig()
+	if buildErr != nil {
+		return nil, fmt.Errorf("failed to build TLS config from secret %q: %w", grafanaTLSSecretName, buildErr)
 	}
+	fmt.Println("Successfully created Grafana client with TLS configuration.")
path: pkg/grafana/client/client.go
user: login: QuentinBisson
body: ```suggestion
```
created_at: 2025-05-12T09:31:11Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1,53 +1,105 @@
 package client
 
 import (
+	"context"
+	"crypto/tls"
 	"fmt"
 	"net/url"
 
 	grafana "github.com/grafana/grafana-openapi-client-go/client"
-
-	"github.com/giantswarm/observability-operator/pkg/config"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
 	clientConfigNumRetries = 3
+
+	// Secret names and keys for Grafana configuration
+	grafanaNamespace = "monitoring"
+
+	grafanaAdminSecretName        = "grafana"
+	grafanaAdminSecretUserKey     = "admin-user"
+	grafanaAdminSecretPasswordKey = "admin-password"
+
+	grafanaTLSSecretName    = "grafana-tls"
+	grafanaTLSSecretCertKey = "tls.crt"
+	grafanaTLSSecretKeyKey  = "tls.key"
 )
 
-func GenerateGrafanaClient(grafanaURL *url.URL, conf config.Config) (*grafana.GrafanaHTTPAPI, error) {
-	var err error
+// GenerateGrafanaClient creates a new Grafana client by fetching credentials
+// and TLS configuration from Kubernetes secrets.
+func GenerateGrafanaClient(
+	ctx context.Context,
+	k8sClient client.Client,
+	grafanaURL *url.URL,
+) (*grafana.GrafanaHTTPAPI, error) {
+	// Get Grafana admin credentials from secret
+	adminSecret := &corev1.Secret{}
+	adminSecretKey := client.ObjectKey{Namespace: grafanaNamespace, Name: grafanaAdminSecretName}
+	if err := k8sClient.Get(ctx, adminSecretKey, adminSecret); err != nil {
+		return nil, fmt.Errorf("failed to get Grafana admin secret %q in namespace %q: %w", grafanaAdminSecretName, grafanaNamespace, err)
+	}
 
-	// Generate Grafana client
-	// Get grafana admin-password and admin-user
-	adminUserCredentials := AdminCredentials{
-		Username: conf.Environment.GrafanaAdminUsername,
-		Password: conf.Environment.GrafanaAdminPassword,
+	adminUsernameBytes, ok := adminSecret.Data[grafanaAdminSecretUserKey]
+	if !ok {
+		return nil, fmt.Errorf("key %q not found in Grafana admin secret %q", grafanaAdminSecretUserKey, grafanaAdminSecretName)
+	}
+	if len(adminUsernameBytes) == 0 {
+		return nil, fmt.Errorf("GrafanaAdminUsername is empty in secret %q", grafanaAdminSecretName)
 	}
-	if adminUserCredentials.Username == "" {
-		return nil, fmt.Errorf("GrafanaAdminUsername not set: %q", conf.Environment.GrafanaAdminUsername)
 
+	adminPasswordBytes, ok := adminSecret.Data[grafanaAdminSecretPasswordKey]
+	if !ok {
+		return nil, fmt.Errorf("key %q not found in Grafana admin secret %q", grafanaAdminSecretPasswordKey, grafanaAdminSecretName)
+	}
+	if len(adminPasswordBytes) == 0 {
+		return nil, fmt.Errorf("GrafanaAdminPassword is empty in secret %q", grafanaAdminSecretName)
 	}
-	if adminUserCredentials.Password == "" {
-		return nil, fmt.Errorf("GrafanaAdminPassword not set: %q", conf.Environment.GrafanaAdminPassword)
 
+	adminUserCredentials := AdminCredentials{
+		Username: string(adminUsernameBytes),
+		Password: string(adminPasswordBytes),
 	}
 
-	grafanaTLSConfig, err := TLSConfig{
-		Cert: conf.Environment.GrafanaTLSCertFile,
-		Key:  conf.Environment.GrafanaTLSKeyFile,
-	}.toTLSConfig()
+	// Get Grafana TLS configuration from secret, if it exists
+	var clientTLSConfig *tls.Config
+	tlsSecret := &corev1.Secret{}
+	tlsSecretKey := client.ObjectKey{Namespace: grafanaNamespace, Name: grafanaTLSSecretName}
+	err := k8sClient.Get(ctx, tlsSecretKey, tlsSecret)
 	if err != nil {
-		return nil, fmt.Errorf("failed to build tls config: %w", err)
+		return nil, fmt.Errorf("failed to get Grafana TLS secret %q in namespace %q: %w", grafanaTLSSecretName, grafanaNamespace, err)
+	}
+
+	// TLS Secret found, try to load cert and key
+	tlsCertBytes, certOk := tlsSecret.Data[grafanaTLSSecretCertKey]
+	tlsKeyBytes, keyOk := tlsSecret.Data[grafanaTLSSecretKeyKey]
+
+	if !certOk || len(tlsCertBytes) == 0 {
+		return nil, fmt.Errorf("key %q not found or empty in Grafana TLS secret %q", grafanaTLSSecretCertKey, grafanaTLSSecretName)
+	}
+	if !keyOk || len(tlsKeyBytes) == 0 {
+		return nil, fmt.Errorf("key %q not found or empty in Grafana TLS secret %q", grafanaTLSSecretKeyKey, grafanaTLSSecretName)
+	}
+
+	// tlsConfigInput is our local struct type from ./tls.go
+	tlsConfigInput := TLSConfig{
+		Cert: string(tlsCertBytes),
+		Key:  string(tlsKeyBytes),
+	}
+	// clientTLSConfig is the *tls.Config for the Grafana client
+	var buildErr error
+	clientTLSConfig, buildErr = tlsConfigInput.toTLSConfig()
+	if buildErr != nil {
+		return nil, fmt.Errorf("failed to build TLS config from secret %q: %w", grafanaTLSSecretName, buildErr)
path: pkg/grafana/client/client.go
user: login: TheoBrigitte
body: Can also just use `err` here

```suggestion
	clientTLSConfig, err = tlsConfigInput.toTLSConfig()
	if err != nil {
		return nil, fmt.Errorf("failed to build TLS config from secret %q: %w", grafanaTLSSecretName, err)
```
created_at: 2025-05-13T10:40:25Z
author_association: MEMBER
diff_hunk: @@ -1,53 +1,105 @@
 package client
 
 import (
+	"context"
+	"crypto/tls"
 	"fmt"
 	"net/url"
 
 	grafana "github.com/grafana/grafana-openapi-client-go/client"
-
-	"github.com/giantswarm/observability-operator/pkg/config"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
 	clientConfigNumRetries = 3
+
+	// Secret names and keys for Grafana configuration
+	grafanaNamespace = "monitoring"
+
+	grafanaAdminSecretName        = "grafana"
+	grafanaAdminSecretUserKey     = "admin-user"
+	grafanaAdminSecretPasswordKey = "admin-password"
+
+	grafanaTLSSecretName    = "grafana-tls"
+	grafanaTLSSecretCertKey = "tls.crt"
+	grafanaTLSSecretKeyKey  = "tls.key"
 )
 
-func GenerateGrafanaClient(grafanaURL *url.URL, conf config.Config) (*grafana.GrafanaHTTPAPI, error) {
-	var err error
+// GenerateGrafanaClient creates a new Grafana client by fetching credentials
+// and TLS configuration from Kubernetes secrets.
+func GenerateGrafanaClient(
+	ctx context.Context,
+	k8sClient client.Client,
+	grafanaURL *url.URL,
+) (*grafana.GrafanaHTTPAPI, error) {
+	// Get Grafana admin credentials from secret
+	adminSecret := &corev1.Secret{}
+	adminSecretKey := client.ObjectKey{Namespace: grafanaNamespace, Name: grafanaAdminSecretName}
+	if err := k8sClient.Get(ctx, adminSecretKey, adminSecret); err != nil {
+		return nil, fmt.Errorf("failed to get Grafana admin secret %q in namespace %q: %w", grafanaAdminSecretName, grafanaNamespace, err)
+	}
 
-	// Generate Grafana client
-	// Get grafana admin-password and admin-user
-	adminUserCredentials := AdminCredentials{
-		Username: conf.Environment.GrafanaAdminUsername,
-		Password: conf.Environment.GrafanaAdminPassword,
+	adminUsernameBytes, ok := adminSecret.Data[grafanaAdminSecretUserKey]
+	if !ok {
+		return nil, fmt.Errorf("key %q not found in Grafana admin secret %q", grafanaAdminSecretUserKey, grafanaAdminSecretName)
+	}
+	if len(adminUsernameBytes) == 0 {
+		return nil, fmt.Errorf("GrafanaAdminUsername is empty in secret %q", grafanaAdminSecretName)
 	}
-	if adminUserCredentials.Username == "" {
-		return nil, fmt.Errorf("GrafanaAdminUsername not set: %q", conf.Environment.GrafanaAdminUsername)
 
+	adminPasswordBytes, ok := adminSecret.Data[grafanaAdminSecretPasswordKey]
+	if !ok {
+		return nil, fmt.Errorf("key %q not found in Grafana admin secret %q", grafanaAdminSecretPasswordKey, grafanaAdminSecretName)
+	}
+	if len(adminPasswordBytes) == 0 {
+		return nil, fmt.Errorf("GrafanaAdminPassword is empty in secret %q", grafanaAdminSecretName)
 	}
-	if adminUserCredentials.Password == "" {
-		return nil, fmt.Errorf("GrafanaAdminPassword not set: %q", conf.Environment.GrafanaAdminPassword)
 
+	adminUserCredentials := AdminCredentials{
+		Username: string(adminUsernameBytes),
+		Password: string(adminPasswordBytes),
 	}
path: pkg/grafana/client/client.go
user: login: TheoBrigitte
body: Can we rather create the needed struct right away instead of having an intermediate adminUserCredentials ?

```suggestion
	adminUserCredentials := url.UserPassword(string(adminUsernameBytes), string(adminPasswordBytes))
```
created_at: 2025-05-13T10:42:31Z
author_association: MEMBER
diff_hunk: @@ -1,53 +1,105 @@
 package client
 
 import (
+	"context"
+	"crypto/tls"
 	"fmt"
 	"net/url"
 
 	grafana "github.com/grafana/grafana-openapi-client-go/client"
-
-	"github.com/giantswarm/observability-operator/pkg/config"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
 	clientConfigNumRetries = 3
+
+	// Secret names and keys for Grafana configuration
+	grafanaNamespace = "monitoring"
+
+	grafanaAdminSecretName        = "grafana"
+	grafanaAdminSecretUserKey     = "admin-user"
+	grafanaAdminSecretPasswordKey = "admin-password"
+
+	grafanaTLSSecretName    = "grafana-tls"
+	grafanaTLSSecretCertKey = "tls.crt"
+	grafanaTLSSecretKeyKey  = "tls.key"
 )
 
-func GenerateGrafanaClient(grafanaURL *url.URL, conf config.Config) (*grafana.GrafanaHTTPAPI, error) {
-	var err error
+// GenerateGrafanaClient creates a new Grafana client by fetching credentials
+// and TLS configuration from Kubernetes secrets.
+func GenerateGrafanaClient(
+	ctx context.Context,
+	k8sClient client.Client,
+	grafanaURL *url.URL,
+) (*grafana.GrafanaHTTPAPI, error) {
+	// Get Grafana admin credentials from secret
+	adminSecret := &corev1.Secret{}
+	adminSecretKey := client.ObjectKey{Namespace: grafanaNamespace, Name: grafanaAdminSecretName}
+	if err := k8sClient.Get(ctx, adminSecretKey, adminSecret); err != nil {
+		return nil, fmt.Errorf("failed to get Grafana admin secret %q in namespace %q: %w", grafanaAdminSecretName, grafanaNamespace, err)
+	}
 
-	// Generate Grafana client
-	// Get grafana admin-password and admin-user
-	adminUserCredentials := AdminCredentials{
-		Username: conf.Environment.GrafanaAdminUsername,
-		Password: conf.Environment.GrafanaAdminPassword,
+	adminUsernameBytes, ok := adminSecret.Data[grafanaAdminSecretUserKey]
+	if !ok {
+		return nil, fmt.Errorf("key %q not found in Grafana admin secret %q", grafanaAdminSecretUserKey, grafanaAdminSecretName)
+	}
+	if len(adminUsernameBytes) == 0 {
+		return nil, fmt.Errorf("GrafanaAdminUsername is empty in secret %q", grafanaAdminSecretName)
 	}
-	if adminUserCredentials.Username == "" {
-		return nil, fmt.Errorf("GrafanaAdminUsername not set: %q", conf.Environment.GrafanaAdminUsername)
 
+	adminPasswordBytes, ok := adminSecret.Data[grafanaAdminSecretPasswordKey]
+	if !ok {
+		return nil, fmt.Errorf("key %q not found in Grafana admin secret %q", grafanaAdminSecretPasswordKey, grafanaAdminSecretName)
+	}
+	if len(adminPasswordBytes) == 0 {
+		return nil, fmt.Errorf("GrafanaAdminPassword is empty in secret %q", grafanaAdminSecretName)
 	}
-	if adminUserCredentials.Password == "" {
-		return nil, fmt.Errorf("GrafanaAdminPassword not set: %q", conf.Environment.GrafanaAdminPassword)
 
+	adminUserCredentials := AdminCredentials{
+		Username: string(adminUsernameBytes),
+		Password: string(adminPasswordBytes),
 	}
 
-	grafanaTLSConfig, err := TLSConfig{
-		Cert: conf.Environment.GrafanaTLSCertFile,
-		Key:  conf.Environment.GrafanaTLSKeyFile,
-	}.toTLSConfig()
+	// Get Grafana TLS configuration from secret, if it exists
+	var clientTLSConfig *tls.Config
+	tlsSecret := &corev1.Secret{}
+	tlsSecretKey := client.ObjectKey{Namespace: grafanaNamespace, Name: grafanaTLSSecretName}
+	err := k8sClient.Get(ctx, tlsSecretKey, tlsSecret)
 	if err != nil {
-		return nil, fmt.Errorf("failed to build tls config: %w", err)
+		return nil, fmt.Errorf("failed to get Grafana TLS secret %q in namespace %q: %w", grafanaTLSSecretName, grafanaNamespace, err)
+	}
+
+	// TLS Secret found, try to load cert and key
+	tlsCertBytes, certOk := tlsSecret.Data[grafanaTLSSecretCertKey]
+	tlsKeyBytes, keyOk := tlsSecret.Data[grafanaTLSSecretKeyKey]
+
+	if !certOk || len(tlsCertBytes) == 0 {
+		return nil, fmt.Errorf("key %q not found or empty in Grafana TLS secret %q", grafanaTLSSecretCertKey, grafanaTLSSecretName)
+	}
+	if !keyOk || len(tlsKeyBytes) == 0 {
+		return nil, fmt.Errorf("key %q not found or empty in Grafana TLS secret %q", grafanaTLSSecretKeyKey, grafanaTLSSecretName)
+	}
+
+	// tlsConfigInput is our local struct type from ./tls.go
+	tlsConfigInput := TLSConfig{
+		Cert: string(tlsCertBytes),
+		Key:  string(tlsKeyBytes),
+	}
+	// clientTLSConfig is the *tls.Config for the Grafana client
+	var buildErr error
+	clientTLSConfig, buildErr = tlsConfigInput.toTLSConfig()
+	if buildErr != nil {
+		return nil, fmt.Errorf("failed to build TLS config from secret %q: %w", grafanaTLSSecretName, buildErr)
 	}
 
 	cfg := &grafana.TransportConfig{
-		Schemes:  []string{grafanaURL.Scheme},
-		BasePath: "/api",
-		Host:     grafanaURL.Host,
-		// We use basic auth to authenticate on grafana.
-		BasicAuth: url.UserPassword(adminUserCredentials.Username, adminUserCredentials.Password),
-		// NumRetries contains the optional number of attempted retries.
+		Schemes:    []string{grafanaURL.Scheme},
+		BasePath:   "/api",
+		Host:       grafanaURL.Host,
+		BasicAuth:  url.UserPassword(adminUserCredentials.Username, adminUserCredentials.Password),
path: pkg/grafana/client/client.go
user: login: QuentinBisson
body: ```suggestion
		BasicAuth:  url.UserPassword(string(adminUsernameBytes), string(adminPasswordBytes)),
```
created_at: 2025-05-13T11:22:52Z
author_association: CONTRIBUTOR

PR comments:
user: login: QuentinBisson
created_at: 2025-05-12T10:02:15Z
body: This is running successfully on golem
author_association: CONTRIBUTOR
user: login: QuentinBisson
created_at: 2025-05-13T10:59:41Z
body: @TheoBrigitte i understand but I also don't think setting it up in the controller stuct makes sense. What do you think about adding it in the context or leaving it as is?

The other thing we could do is transform the grafana/grafana.go into a Service object but we would still need to pass the service arround 🤔 
author_association: CONTRIBUTOR
user: login: TheoBrigitte
created_at: 2025-05-13T13:22:51Z
body: > @TheoBrigitte i understand but I also don't think setting it up in the controller stuct makes sense. What do you think about adding it in the context or leaving it as is?
> 
> The other thing we could do is transform the grafana/grafana.go into a Service object but we would still need to pass the service arround 🤔

I would rather not pass the client by context as this would then lead to a lot of unnecessary complexity to get the client from context. I think we could create a service which holds the client as a field and attach the reconcile methods and their children to it, so we would then have a new service for each request, that's a bit more refactoring though.
author_association: MEMBER
user: login: QuentinBisson
created_at: 2025-05-13T13:25:56Z
body: Yes I was thinking about this. I would prefer that we do the refactoring in another PR though. What do you think?
author_association: CONTRIBUTOR
user: login: TheoBrigitte
created_at: 2025-05-13T14:17:15Z
body: > Yes I was thinking about this. I would prefer that we do the refactoring in another PR though. What do you think?

Fully agree. I would be happy to take this over in a next PR if you don't mind.
author_association: MEMBER
user: login: QuentinBisson
created_at: 2025-05-13T14:21:19Z
body: Bé my guest, i'll focus on the logging operator clean up :)
author_association: CONTRIBUTOR