Repository: grafana/metrics-drilldown
Pull request #384: refactor: generic health checks
Author: login: NWRichmond
Created: 2025-05-02T21:27:30Z | Merged: 2025-05-12T21:01:17Z

### ✨ Description

This PR refactors a health check function (currently used for the Related Logs feature) in the following ways:
- It's more generic, and not tied to Loki-type data sources
- Switches to the API endpoint from the [deprecated `id`-based endpoint](https://grafana.com/docs/grafana/latest/developers/http_api/data_source/#get-a-single-data-source-by-id) to its [`uid`-based counterpart](https://grafana.com/docs/grafana/latest/developers/http_api/data_source/#get-a-single-data-source-by-uid)
- Adds caching to prevent superfluous data source fetching & health checks


Diff:
diff --git a/src/Integrations/logs/labelsCrossReference.ts b/src/Integrations/logs/labelsCrossReference.ts
index 7d7957f78..e264fb648 100644
--- a/src/Integrations/logs/labelsCrossReference.ts
+++ b/src/Integrations/logs/labelsCrossReference.ts
@@ -3,9 +3,9 @@ import { getDataSourceSrv } from '@grafana/runtime';
 import { sceneGraph, type SceneObject } from '@grafana/scenes';
 
 import { createMetricsLogsConnector, type FoundLokiDataSource } from './base';
-import { findHealthyLokiDataSources } from '../../RelatedLogs/RelatedLogsOrchestrator';
 import { VAR_FILTERS } from '../../shared';
 import { getTrailFor } from '../../utils';
+import { getDataSourceFetcher } from '../../utils/utils.datasource';
 import { isAdHocFiltersVariable } from '../../utils/utils.variables';
 
 const knownLabelNameDiscrepancies = {
@@ -98,7 +98,7 @@ export const createLabelsCrossReferenceConnector = (scene: SceneObject) => {
       // Get current time range if available
       const timeRange = scene.state.$timeRange?.state.value;
 
-      const lokiDataSources = await findHealthyLokiDataSources();
+      const lokiDataSources = await getDataSourceFetcher().getHealthyDataSources('loki');
       const results = await Promise.all(
         lokiDataSources.map(async ({ uid, name }) => {
           const hasLabels = await hasMatchingLabels(uid, filters, timeRange);
diff --git a/src/Integrations/logs/lokiRecordingRules.test.ts b/src/Integrations/logs/lokiRecordingRules.test.ts
index 7f3a36d15..a5b0a5c7a 100644
--- a/src/Integrations/logs/lokiRecordingRules.test.ts
+++ b/src/Integrations/logs/lokiRecordingRules.test.ts
@@ -1,8 +1,9 @@
 import { of } from 'rxjs';
 
 import { type MetricsLogsConnector } from './base';
-import { lokiRecordingRulesConnector, type RecordingRuleGroup } from './lokiRecordingRules';
+import { createLokiRecordingRulesConnector, type RecordingRuleGroup } from './lokiRecordingRules';
 import { getMockPlugin } from '../../mocks/plugin';
+import { logger } from '../../tracking/logger/logger';
 
 import type { DataSourceInstanceSettings, DataSourceJsonData } from '@grafana/data';
 import type * as runtime from '@grafana/runtime';
@@ -117,17 +118,17 @@ jest.mock('@grafana/runtime', () => ({
 }));
 
 describe('LokiRecordingRulesConnector', () => {
-  let consoleSpy: jest.SpyInstance;
+  let loggerWarnSpy: jest.SpyInstance;
 
   beforeEach(() => {
     getListSpy.mockReturnValue([mockLokiDS1, mockLokiDS2]);
     fetchSpy.mockImplementation(defaultFetchImpl);
-    consoleSpy = jest.spyOn(console, 'warn').mockImplementation();
+    loggerWarnSpy = jest.spyOn(logger, 'warn').mockImplementation();
   });
 
   describe('getDataSources', () => {
     it('should find all data sources containing the metric', async () => {
-      const connector = lokiRecordingRulesConnector;
+      const connector = createLokiRecordingRulesConnector();
       const result = await connector.getDataSources('metric_a_total');
 
       expect(result).toHaveLength(2);
@@ -140,38 +141,47 @@ describe('LokiRecordingRulesConnector', () => {
     });
 
     it('should handle non-existent metrics', async () => {
-      const connector = lokiRecordingRulesConnector;
+      const connector = createLokiRecordingRulesConnector();
       const result = await connector.getDataSources('non_existent_metric');
 
       expect(result).toHaveLength(0);
     });
 
     it('should handle datasource fetch errors gracefully', async () => {
-      // Make the second datasource fail
-      fetchSpy.mockImplementation((req) => {
-        if (req.url.includes('loki1')) {
-          return of({
-            data: { data: { groups: mockRuleGroups1 } },
-            ok: true,
-            status: 200,
-            statusText: 'OK',
-            headers: new Headers(),
-            redirected: false,
-            type: 'basic',
-            url: req.url,
-            config: { url: req.url },
-          } as runtime.FetchResponse);
-        }
-        throw new Error('Failed to fetch');
-      });
-
-      const connector = lokiRecordingRulesConnector;
+      // Make the second datasource fail with an error response
+      fetchSpy.mockImplementation((req) =>
+        req.url.includes('loki2')
+          ? of({
+              data: { message: 'Failed to fetch' },
+              ok: false,
+              status: 500,
+              statusText: 'Internal Server Error',
+              headers: new Headers(),
+              redirected: false,
+              type: 'basic',
+              url: req.url,
+              config: { url: req.url },
+            } as runtime.FetchResponse)
+          : of({
+              data: { data: { groups: mockRuleGroups1 } },
+              ok: true,
+              status: 200,
+              statusText: 'OK',
+              headers: new Headers(),
+              redirected: false,
+              type: 'basic',
+              url: req.url,
+              config: { url: req.url },
+            } as runtime.FetchResponse)
+      );
+
+      const connector = createLokiRecordingRulesConnector();
       const result = await connector.getDataSources('metric_a_total');
 
       // Should still get results from the working datasource
       expect(result).toHaveLength(1);
       expect(result[0]).toEqual({ name: 'Loki Main', uid: 'loki1' });
-      expect(consoleSpy).toHaveBeenCalled();
+      expect(loggerWarnSpy).toHaveBeenCalled();
     });
   });
 
@@ -179,7 +189,7 @@ describe('LokiRecordingRulesConnector', () => {
     let connector: MetricsLogsConnector;
 
     beforeEach(async () => {
-      connector = lokiRecordingRulesConnector;
+      connector = createLokiRecordingRulesConnector();
       // Populate the rules first
       await connector.getDataSources('metric_a_total');
     });
diff --git a/src/Integrations/logs/lokiRecordingRules.ts b/src/Integrations/logs/lokiRecordingRules.ts
index a104d2154..11a1c1f0f 100644
--- a/src/Integrations/logs/lokiRecordingRules.ts
+++ b/src/Integrations/logs/lokiRecordingRules.ts
@@ -5,7 +5,8 @@ import { type SyntaxNode } from '@lezer/common';
 import { lastValueFrom } from 'rxjs';
 
 import { createMetricsLogsConnector, type FoundLokiDataSource } from './base';
-import { findHealthyLokiDataSources } from '../../RelatedLogs/RelatedLogsOrchestrator';
+import { logger } from '../../tracking/logger/logger';
+import { getDataSourceFetcher } from '../../utils/utils.datasource';
 
 export interface RecordingRuleGroup {
   name: string;
@@ -44,7 +45,7 @@ async function fetchRecordingRuleGroups(datasourceSettings: DataSourceInstanceSe
   >(getBackendSrv().fetch(recordingRules));
 
   if (!res.ok) {
-    console.warn(`Failed to fetch recording rules from Loki data source: ${datasourceSettings.name}`);
+    logger.warn(`Failed to fetch recording rules from Loki data source: ${datasourceSettings.name}`);
     return [];
   }
 
@@ -153,10 +154,10 @@ export function getLokiQueryForRelatedMetric(
  * @returns {Promise<ExtractedRecordingRules>} A promise that resolves to an object containing
  * the extracted recording rules, keyed by data source UID.
  *
- * @throws Will log an error to the console if fetching or extracting rules fails for any data source.
+ * @throws Will log a warning if fetching or extracting rules fails for any data source.
  */
 export async function fetchAndExtractLokiRecordingRules() {
-  const lokiDataSources = await findHealthyLokiDataSources();
+  const lokiDataSources = await getDataSourceFetcher().getHealthyDataSources('loki');
   const extractedRecordingRules: ExtractedRecordingRules = {};
   await Promise.all(
     lokiDataSources.map(async (dataSource) => {
@@ -165,7 +166,7 @@ export async function fetchAndExtractLokiRecordingRules() {
         const extractedRules = extractRecordingRulesFromRuleGroups(ruleGroups, dataSource);
         extractedRecordingRules[dataSource.uid] = extractedRules;
       } catch (err) {
-        console.warn(err);
+        logger.warn(err);
       }
     })
   );
@@ -173,7 +174,7 @@ export async function fetchAndExtractLokiRecordingRules() {
   return extractedRecordingRules;
 }
 
-const createLokiRecordingRulesConnector = () => {
+export const createLokiRecordingRulesConnector = () => {
   let lokiRecordingRules: ExtractedRecordingRules = {};
 
   // In this connector, conditions have been met for related logs
@@ -197,8 +198,6 @@ const createLokiRecordingRulesConnector = () => {
   });
 };
 
-export const lokiRecordingRulesConnector = createLokiRecordingRulesConnector();
-
 /**
  * Returns whether the given query is a logs query (not a metrics query)
  * A query that's at least 3 characters long and doesn't contain a MetricExpr node is considered a logs query
diff --git a/src/RelatedLogs/RelatedLogsOrchestrator.ts b/src/RelatedLogs/RelatedLogsOrchestrator.ts
index 9c0da0ca5..876882cf8 100644
--- a/src/RelatedLogs/RelatedLogsOrchestrator.ts
+++ b/src/RelatedLogs/RelatedLogsOrchestrator.ts
@@ -1,14 +1,12 @@
-import { LoadingState, type DataSourceInstanceSettings, type DataSourceJsonData } from '@grafana/data';
-import { getBackendSrv, getDataSourceSrv } from '@grafana/runtime';
+import { LoadingState } from '@grafana/data';
 import { SceneQueryRunner } from '@grafana/scenes';
 
 import { type MetricsLogsConnector } from '../Integrations/logs/base';
 import { createLabelsCrossReferenceConnector } from '../Integrations/logs/labelsCrossReference';
-import { lokiRecordingRulesConnector } from '../Integrations/logs/lokiRecordingRules';
+import { createLokiRecordingRulesConnector } from '../Integrations/logs/lokiRecordingRules';
 import { type MetricScene } from '../MetricScene';
 import pluginJson from '../plugin.json';
-
-type DataSource = DataSourceInstanceSettings<DataSourceJsonData>;
+import { getDataSourceFetcher, type DataSource } from '../utils/utils.datasource';
 
 /**
  * Manager class that handles the orchestration of related logs functionality.
@@ -17,6 +15,7 @@ type DataSource = DataSourceInstanceSettings<DataSourceJsonData>;
 export class RelatedLogsOrchestrator {
   private readonly _logsConnectors: MetricsLogsConnector[];
   private readonly _metricScene: MetricScene;
+  private readonly _dataSourceFetcher = getDataSourceFetcher();
   private readonly _changeHandlers = {
     lokiDataSources: [] as Array<(dataSources: DataSource[]) => void>,
     relatedLogsCount: [] as Array<(count: number) => void>,
@@ -31,7 +30,7 @@ export class RelatedLogsOrchestrator {
 
   constructor(metricScene: MetricScene) {
     this._metricScene = metricScene;
-    this._logsConnectors = [lokiRecordingRulesConnector, createLabelsCrossReferenceConnector(metricScene)];
+    this._logsConnectors = [createLokiRecordingRulesConnector(), createLabelsCrossReferenceConnector(metricScene)];
   }
 
   get lokiDataSources() {
@@ -92,7 +91,7 @@ export class RelatedLogsOrchestrator {
    */
   public async findAndCheckAllDatasources(): Promise<void> {
     // Get all available Loki datasources
-    const allLokiDatasources = await findHealthyLokiDataSources();
+    const allLokiDatasources = await this._dataSourceFetcher.getHealthyDataSources('loki');
 
     // Check all datasources for logs
     if (allLokiDatasources.length > 0) {
@@ -195,37 +194,3 @@ export class RelatedLogsOrchestrator {
     return this._logsConnectors.some((connector) => connector.checkConditionsMetForRelatedLogs());
   }
 }
-
-export async function findHealthyLokiDataSources() {
-  const lokiDataSources = getDataSourceSrv().getList({
-    logs: true,
-    type: 'loki',
-    filter: (ds) => ds.uid !== 'grafana',
-  });
-  const healthyLokiDataSources: DataSource[] = [];
-  const unhealthyLokiDataSources: DataSource[] = [];
-
-  await Promise.all(
-    lokiDataSources.map((ds) =>
-      getBackendSrv()
-        .get(`/api/datasources/${ds.id}/health`, undefined, undefined, {
-          showSuccessAlert: false,
-          showErrorAlert: false,
-        })
-        .then((health) =>
-          health?.status === 'OK' ? healthyLokiDataSources.push(ds) : unhealthyLokiDataSources.push(ds)
-        )
-        .catch(() => unhealthyLokiDataSources.push(ds))
-    )
-  );
-
-  if (unhealthyLokiDataSources.length) {
-    console.warn(
-      `Found ${unhealthyLokiDataSources.length} unhealthy Loki data sources: ${unhealthyLokiDataSources
-        .map((ds) => ds.name)
-        .join(', ')}`
-    );
-  }
-
-  return healthyLokiDataSources;
-}
diff --git a/src/utils/utils.datasource.ts b/src/utils/utils.datasource.ts
index 3b9f51a51..66b8a437f 100644
--- a/src/utils/utils.datasource.ts
+++ b/src/utils/utils.datasource.ts
@@ -1,4 +1,9 @@
+import { type DataSourceInstanceSettings, type DataSourceJsonData } from '@grafana/data';
 import { type PrometheusDatasource } from '@grafana/prometheus';
+import { getBackendSrv, getDataSourceSrv } from '@grafana/runtime';
+
+import { logger } from '../tracking/logger/logger';
+export type DataSource = DataSourceInstanceSettings<DataSourceJsonData>;
 
 /**
  * Helper function to determine if a datasource is a Prometheus datasource
@@ -13,3 +18,97 @@ export function isPrometheusDataSource(input: unknown): input is PrometheusDatas
     typeof input.uid === 'string'
   );
 }
+
+type DataSourceType = 'prometheus' | 'loki';
+
+/**
+ * Fetches and caches healthy data sources
+ */
+export class DataSourceFetcher {
+  private readonly pendingRequests = new Map<DataSourceType, Promise<DataSource[]>>();
+  private readonly cache = new Map<DataSourceType, DataSource[]>();
+
+  /**
+   * Retrieves healthy data sources of the specified type
+   * Results are cached indefinitely until the `DataSourceFetcher` is destroyed
+   *
+   * @param type - The type of data source to retrieve ('prometheus' or loki)
+   * @returns Array of healthy data sources
+   */
+  public async getHealthyDataSources(type: DataSourceType): Promise<DataSource[]> {
+    // Check if we have cached results
+    const cachedDataSources = this.cache.get(type);
+    if (cachedDataSources?.length) {
+      return cachedDataSources;
+    }
+
+    // If there's already a pending request for this type, wait for it
+    let pendingRequest = this.pendingRequests.get(type);
+    if (!pendingRequest) {
+      pendingRequest = this.fetchHealthyDataSources(type).finally(() => {
+        // Clean up the pending request after it completes
+        this.pendingRequests.delete(type);
+      });
+      this.pendingRequests.set(type, pendingRequest);
+    }
+
+    // Wait for the request to complete and update cache
+    const dataSources = await pendingRequest;
+    this.cache.set(type, dataSources);
+
+    return dataSources;
+  }
+
+  /**
+   * Fetches healthy data sources of the specified type
+   */
+  private async fetchHealthyDataSources(type: DataSourceType): Promise<DataSource[]> {
+    const allDataSourcesOfType = getDataSourceSrv().getList({
+      logs: true,
+      type,
+      filter: (ds) => ds.uid !== 'grafana',
+    });
+
+    const healthyDataSources: DataSource[] = [];
+    const unhealthyDataSources: DataSource[] = [];
+
+    await Promise.all(
+      allDataSourcesOfType.map(async (ds) => {
+        try {
+          const health = await getBackendSrv().get(`/api/datasources/uid/${ds.uid}/health`, undefined, undefined, {
+            showSuccessAlert: false,
+            showErrorAlert: false,
+          });
+
+          if (health?.status === 'OK') {
+            healthyDataSources.push(ds);
+          } else {
+            unhealthyDataSources.push(ds);
+          }
+        } catch {
+          unhealthyDataSources.push(ds);
+        }
+      })
+    );
+
+    if (unhealthyDataSources.length > 0) {
+      logger.warn(
+        `Found ${unhealthyDataSources.length} unhealthy ${type} data sources: ${unhealthyDataSources
+          .map((ds) => ds.name)
+          .join(', ')}`
+      );
+    }
+
+    return healthyDataSources;
+  }
+}
+
+let dataSourceFetcherSingleton: DataSourceFetcher | undefined;
+
+export function getDataSourceFetcher(): DataSourceFetcher {
+  if (!dataSourceFetcherSingleton) {
+    dataSourceFetcherSingleton = new DataSourceFetcher();
+  }
+
+  return dataSourceFetcherSingleton;
+}


Reviews:
user: login: grafakus
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-05T11:16:33Z
user: login: grafakus
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-05T11:19:29Z
user: login: NWRichmond
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-06T17:31:21Z
user: login: NWRichmond
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-06T17:34:24Z
user: login: NWRichmond
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-06T22:21:01Z
user: login: grafakus
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-07T09:54:03Z
user: login: NWRichmond
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-09T16:44:41Z
user: login: NWRichmond
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-09T20:15:38Z
user: login: grafakus
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-12T14:26:25Z
user: login: grafakus
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-12T14:31:08Z
user: login: NWRichmond
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-12T14:40:01Z
user: login: NWRichmond
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-12T14:51:10Z
user: login: grafakus
body: Nice job! As discussed in a call, we'll follow-up with https://github.com/grafana/metrics-drilldown/issues/401 after this PR is merged.
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-12T16:24:46Z

Review comments:
diff_hunk: @@ -13,3 +17,37 @@ export function isPrometheusDataSource(input: unknown): input is PrometheusDatas
     typeof input.uid === 'string'
   );
 }
+
+export async function findHealthyDataSources(type: 'prometheus' | 'loki', verbose = false) {
+  const allDataSourcesOfType = getDataSourceSrv().getList({
+    logs: true,
+    type,
+    filter: (ds) => ds.uid !== 'grafana',
+  });
+  const healthyDataSources: DataSource[] = [];
+  const unhealthyDataSources: DataSource[] = [];
+
+  await Promise.all(
+    allDataSourcesOfType.map((ds) =>
+      getBackendSrv()
+        .get(`/api/datasources/uid/${ds.uid}/health`, undefined, undefined, {
+          showSuccessAlert: false,
+          showErrorAlert: false,
+        })
+        .then((health) => (health?.status === 'OK' ? healthyDataSources.push(ds) : unhealthyDataSources.push(ds)))
+        .catch(() => unhealthyDataSources.push(ds))
+    )
+  );
+
+  if (verbose && unhealthyDataSources.length) {
+    // Why not use `logger.warn` here? While this information might be useful for observant users
+    // who open DevTools, it's not an actionable insight for Grafana Metrics Drilldown developers.
+    console.warn(
path: src/utils/utils.datasource.ts
user: login: grafakus
body: Can we use [the logger](https://github.com/grafana/metrics-drilldown/blob/main/src/tracking/logger/logger.ts#L66-L72) instead'
created_at: 2025-05-05T11:16:33Z
author_association: CONTRIBUTOR
diff_hunk: @@ -13,3 +17,37 @@ export function isPrometheusDataSource(input: unknown): input is PrometheusDatas
     typeof input.uid === 'string'
   );
 }
+
+export async function findHealthyDataSources(type: 'prometheus' | 'loki', verbose = false) {
path: src/utils/utils.datasource.ts
user: login: grafakus
body: How many times can this function called during a typical user session? My concern is that we call it more times than needed. And also, on larger instances, the `Promise.all` L30 could impact negatively the performance of the app.

If you find also my concerns valid, can we cache the results of this function? 
created_at: 2025-05-05T11:19:29Z
author_association: CONTRIBUTOR
diff_hunk: @@ -13,3 +17,37 @@ export function isPrometheusDataSource(input: unknown): input is PrometheusDatas
     typeof input.uid === 'string'
   );
 }
+
+export async function findHealthyDataSources(type: 'prometheus' | 'loki', verbose = false) {
+  const allDataSourcesOfType = getDataSourceSrv().getList({
+    logs: true,
+    type,
+    filter: (ds) => ds.uid !== 'grafana',
+  });
+  const healthyDataSources: DataSource[] = [];
+  const unhealthyDataSources: DataSource[] = [];
+
+  await Promise.all(
+    allDataSourcesOfType.map((ds) =>
+      getBackendSrv()
+        .get(`/api/datasources/uid/${ds.uid}/health`, undefined, undefined, {
+          showSuccessAlert: false,
+          showErrorAlert: false,
+        })
+        .then((health) => (health?.status === 'OK' ? healthyDataSources.push(ds) : unhealthyDataSources.push(ds)))
+        .catch(() => unhealthyDataSources.push(ds))
+    )
+  );
+
+  if (verbose && unhealthyDataSources.length) {
+    // Why not use `logger.warn` here? While this information might be useful for observant users
+    // who open DevTools, it's not an actionable insight for Grafana Metrics Drilldown developers.
+    console.warn(
path: src/utils/utils.datasource.ts
user: login: NWRichmond
body: What do you think about the contents of the comment here?
created_at: 2025-05-06T17:31:21Z
author_association: COLLABORATOR
diff_hunk: @@ -13,3 +17,37 @@ export function isPrometheusDataSource(input: unknown): input is PrometheusDatas
     typeof input.uid === 'string'
   );
 }
+
+export async function findHealthyDataSources(type: 'prometheus' | 'loki', verbose = false) {
+  const allDataSourcesOfType = getDataSourceSrv().getList({
+    logs: true,
+    type,
+    filter: (ds) => ds.uid !== 'grafana',
+  });
+  const healthyDataSources: DataSource[] = [];
+  const unhealthyDataSources: DataSource[] = [];
+
+  await Promise.all(
+    allDataSourcesOfType.map((ds) =>
+      getBackendSrv()
+        .get(`/api/datasources/uid/${ds.uid}/health`, undefined, undefined, {
+          showSuccessAlert: false,
+          showErrorAlert: false,
+        })
+        .then((health) => (health?.status === 'OK' ? healthyDataSources.push(ds) : unhealthyDataSources.push(ds)))
+        .catch(() => unhealthyDataSources.push(ds))
+    )
+  );
+
+  if (verbose && unhealthyDataSources.length) {
+    // Why not use `logger.warn` here? While this information might be useful for observant users
+    // who open DevTools, it's not an actionable insight for Grafana Metrics Drilldown developers.
+    console.warn(
path: src/utils/utils.datasource.ts
user: login: NWRichmond
body: My concern is that it's just noise in Frontend O11y. I'm not sure how it'd be valuable to us, but let me know if you see it differently.
created_at: 2025-05-06T17:34:23Z
author_association: COLLABORATOR
diff_hunk: @@ -13,3 +17,37 @@ export function isPrometheusDataSource(input: unknown): input is PrometheusDatas
     typeof input.uid === 'string'
   );
 }
+
+export async function findHealthyDataSources(type: 'prometheus' | 'loki', verbose = false) {
path: src/utils/utils.datasource.ts
user: login: NWRichmond
body: For sure! Caching implemented in https://github.com/grafana/metrics-drilldown/pull/384/commits/d62c2148c769854877194f0c60f49713b946aac7 ✅ 
created_at: 2025-05-06T22:21:01Z
author_association: COLLABORATOR
diff_hunk: @@ -13,3 +17,37 @@ export function isPrometheusDataSource(input: unknown): input is PrometheusDatas
     typeof input.uid === 'string'
   );
 }
+
+export async function findHealthyDataSources(type: 'prometheus' | 'loki', verbose = false) {
+  const allDataSourcesOfType = getDataSourceSrv().getList({
+    logs: true,
+    type,
+    filter: (ds) => ds.uid !== 'grafana',
+  });
+  const healthyDataSources: DataSource[] = [];
+  const unhealthyDataSources: DataSource[] = [];
+
+  await Promise.all(
+    allDataSourcesOfType.map((ds) =>
+      getBackendSrv()
+        .get(`/api/datasources/uid/${ds.uid}/health`, undefined, undefined, {
+          showSuccessAlert: false,
+          showErrorAlert: false,
+        })
+        .then((health) => (health?.status === 'OK' ? healthyDataSources.push(ds) : unhealthyDataSources.push(ds)))
+        .catch(() => unhealthyDataSources.push(ds))
+    )
+  );
+
+  if (verbose && unhealthyDataSources.length) {
+    // Why not use `logger.warn` here? While this information might be useful for observant users
+    // who open DevTools, it's not an actionable insight for Grafana Metrics Drilldown developers.
+    console.warn(
path: src/utils/utils.datasource.ts
user: login: grafakus
body: I'd argue that an increasing number of unhealthy DS is a valuable insight for GME developers. I don't think it'll create a lot of noise - my assumption being that unhealthy DS is not common.

From the code design perspective, the idea of having a single logger is to centralize the management of all log messages. It feels we're going backwards by using the console.

Also, if the info logged is neither actionable nor useful for GMD developers, why not displaying some information in the UI for the user and removing the `console.warn`?
created_at: 2025-05-07T09:54:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -13,3 +17,37 @@ export function isPrometheusDataSource(input: unknown): input is PrometheusDatas
     typeof input.uid === 'string'
   );
 }
+
+export async function findHealthyDataSources(type: 'prometheus' | 'loki', verbose = false) {
+  const allDataSourcesOfType = getDataSourceSrv().getList({
+    logs: true,
+    type,
+    filter: (ds) => ds.uid !== 'grafana',
+  });
+  const healthyDataSources: DataSource[] = [];
+  const unhealthyDataSources: DataSource[] = [];
+
+  await Promise.all(
+    allDataSourcesOfType.map((ds) =>
+      getBackendSrv()
+        .get(`/api/datasources/uid/${ds.uid}/health`, undefined, undefined, {
+          showSuccessAlert: false,
+          showErrorAlert: false,
+        })
+        .then((health) => (health?.status === 'OK' ? healthyDataSources.push(ds) : unhealthyDataSources.push(ds)))
+        .catch(() => unhealthyDataSources.push(ds))
+    )
+  );
+
+  if (verbose && unhealthyDataSources.length) {
+    // Why not use `logger.warn` here? While this information might be useful for observant users
+    // who open DevTools, it's not an actionable insight for Grafana Metrics Drilldown developers.
+    console.warn(
path: src/utils/utils.datasource.ts
user: login: NWRichmond
body: I'm open to `logger.warn`ing this for now, but I remain doubtful that this signal provides value to anyone but the user or the admins of the user's Grafana instance.

> why not displaying some information in the UI for the user and removing the console.warn?

I'd prefer this route, provided that we make it not too annoying. For example, I believe it'd be too annoying to display an alert banner every time unhealthy data sources are detected. A tooltip or something subtle next to the data source selector could be great. Unless the user happens to be an Admin and also happens to have control over the affected data source, it's usually not something that they should need to think about.
created_at: 2025-05-09T16:44:40Z
author_association: COLLABORATOR
diff_hunk: @@ -13,3 +17,37 @@ export function isPrometheusDataSource(input: unknown): input is PrometheusDatas
     typeof input.uid === 'string'
   );
 }
+
+export async function findHealthyDataSources(type: 'prometheus' | 'loki', verbose = false) {
+  const allDataSourcesOfType = getDataSourceSrv().getList({
+    logs: true,
+    type,
+    filter: (ds) => ds.uid !== 'grafana',
+  });
+  const healthyDataSources: DataSource[] = [];
+  const unhealthyDataSources: DataSource[] = [];
+
+  await Promise.all(
+    allDataSourcesOfType.map((ds) =>
+      getBackendSrv()
+        .get(`/api/datasources/uid/${ds.uid}/health`, undefined, undefined, {
+          showSuccessAlert: false,
+          showErrorAlert: false,
+        })
+        .then((health) => (health?.status === 'OK' ? healthyDataSources.push(ds) : unhealthyDataSources.push(ds)))
+        .catch(() => unhealthyDataSources.push(ds))
+    )
+  );
+
+  if (verbose && unhealthyDataSources.length) {
+    // Why not use `logger.warn` here? While this information might be useful for observant users
+    // who open DevTools, it's not an actionable insight for Grafana Metrics Drilldown developers.
+    console.warn(
path: src/utils/utils.datasource.ts
user: login: NWRichmond
body: Because I'd like to keep UI changes out of the scope of this PR, I shifted to using `logger` in https://github.com/grafana/metrics-drilldown/pull/384/commits/e919e98e2427607841d9dfa5f4144975d5ec6275
created_at: 2025-05-09T20:15:38Z
author_association: COLLABORATOR
diff_hunk: @@ -13,3 +18,89 @@ export function isPrometheusDataSource(input: unknown): input is PrometheusDatas
     typeof input.uid === 'string'
   );
 }
+
+type DataSourceType = 'prometheus' | 'loki';
+
+/**
+ * Fetches and caches healthy data sources
+ */
+export class DataSourceFetcher {
+  private readonly pendingRequests = new Map<DataSourceType, Promise<DataSource[]>>();
+  private readonly cache = new Map<DataSourceType, DataSource[]>();
+
+  /**
+   * Retrieves healthy data sources of the specified type
+   * Results are cached indefinitely until the `DataSourceFetcher` is destroyed
+   *
+   * @param type - The type of data source to retrieve ('prometheus' or loki)
+   * @returns Array of healthy data sources
+   */
+  public async getHealthyDataSources(type: DataSourceType): Promise<DataSource[]> {
+    // Check if we have cached results
+    const cachedDataSources = this.cache.get(type);
+    if (cachedDataSources?.length) {
+      return cachedDataSources;
+    }
+
+    // If there's already a pending request for this type, wait for it
+    let pendingRequest = this.pendingRequests.get(type);
+    if (!pendingRequest) {
+      pendingRequest = this.fetchHealthyDataSources(type).finally(() => {
+        // Clean up the pending request after it completes
+        this.pendingRequests.delete(type);
+      });
+      this.pendingRequests.set(type, pendingRequest);
+    }
+
+    // Wait for the request to complete and update cache
+    const dataSources = await pendingRequest;
+    this.cache.set(type, dataSources);
+
+    return dataSources;
+  }
+
+  /**
+   * Fetches healthy data sources of the specified type
+   */
+  private async fetchHealthyDataSources(type: DataSourceType): Promise<DataSource[]> {
+    const allDataSourcesOfType = getDataSourceSrv().getList({
+      logs: true,
+      type,
+      filter: (ds) => ds.uid !== 'grafana',
+    });
+
+    const healthyDataSources: DataSource[] = [];
+    const unhealthyDataSources: DataSource[] = [];
+
+    await Promise.all(
+      allDataSourcesOfType.map(async (ds) => {
+        try {
+          const health = await getBackendSrv().get(`/api/datasources/uid/${ds.uid}/health`, undefined, undefined, {
+            showSuccessAlert: false,
+            showErrorAlert: false,
+          });
+
+          if (health?.status === 'OK') {
+            healthyDataSources.push(ds);
+          } else {
+            unhealthyDataSources.push(ds);
+          }
+          // NOTE: This appears to be a false positive from sonarjs
+          // eslint-disable-next-line sonarjs/no-ignored-exceptions
+        } catch (error) {
+          unhealthyDataSources.push(ds);
+        }
path: src/utils/utils.datasource.ts
user: login: grafakus
body: Actually SonarJS is right. Have you tried:

```suggestion
        } catch {
          unhealthyDataSources.push(ds);
        }
```
?
created_at: 2025-05-12T14:26:25Z
author_association: CONTRIBUTOR
diff_hunk: @@ -136,6 +139,7 @@ export class DataTrail extends SceneObjectBase<DataTrailState> implements SceneO
       histogramsLoaded: state.histogramsLoaded ?? false,
       nativeHistogramMetric: state.nativeHistogramMetric ?? '',
       trailActivated: state.trailActivated ?? false,
+      datasourceFetcher: state.datasourceFetcher ?? new DataSourceFetcher(),
path: src/DataTrail.tsx
user: login: grafakus
body: Why do we instantiate `DataSourceFetcher` here and pass it around? Could we just create a singleton in `utils/datasource.ts` and import it where it's needed?
created_at: 2025-05-12T14:31:08Z
author_association: CONTRIBUTOR
diff_hunk: @@ -13,3 +18,89 @@ export function isPrometheusDataSource(input: unknown): input is PrometheusDatas
     typeof input.uid === 'string'
   );
 }
+
+type DataSourceType = 'prometheus' | 'loki';
+
+/**
+ * Fetches and caches healthy data sources
+ */
+export class DataSourceFetcher {
+  private readonly pendingRequests = new Map<DataSourceType, Promise<DataSource[]>>();
+  private readonly cache = new Map<DataSourceType, DataSource[]>();
+
+  /**
+   * Retrieves healthy data sources of the specified type
+   * Results are cached indefinitely until the `DataSourceFetcher` is destroyed
+   *
+   * @param type - The type of data source to retrieve ('prometheus' or loki)
+   * @returns Array of healthy data sources
+   */
+  public async getHealthyDataSources(type: DataSourceType): Promise<DataSource[]> {
+    // Check if we have cached results
+    const cachedDataSources = this.cache.get(type);
+    if (cachedDataSources?.length) {
+      return cachedDataSources;
+    }
+
+    // If there's already a pending request for this type, wait for it
+    let pendingRequest = this.pendingRequests.get(type);
+    if (!pendingRequest) {
+      pendingRequest = this.fetchHealthyDataSources(type).finally(() => {
+        // Clean up the pending request after it completes
+        this.pendingRequests.delete(type);
+      });
+      this.pendingRequests.set(type, pendingRequest);
+    }
+
+    // Wait for the request to complete and update cache
+    const dataSources = await pendingRequest;
+    this.cache.set(type, dataSources);
+
+    return dataSources;
+  }
+
+  /**
+   * Fetches healthy data sources of the specified type
+   */
+  private async fetchHealthyDataSources(type: DataSourceType): Promise<DataSource[]> {
+    const allDataSourcesOfType = getDataSourceSrv().getList({
+      logs: true,
+      type,
+      filter: (ds) => ds.uid !== 'grafana',
+    });
+
+    const healthyDataSources: DataSource[] = [];
+    const unhealthyDataSources: DataSource[] = [];
+
+    await Promise.all(
+      allDataSourcesOfType.map(async (ds) => {
+        try {
+          const health = await getBackendSrv().get(`/api/datasources/uid/${ds.uid}/health`, undefined, undefined, {
+            showSuccessAlert: false,
+            showErrorAlert: false,
+          });
+
+          if (health?.status === 'OK') {
+            healthyDataSources.push(ds);
+          } else {
+            unhealthyDataSources.push(ds);
+          }
+          // NOTE: This appears to be a false positive from sonarjs
+          // eslint-disable-next-line sonarjs/no-ignored-exceptions
+        } catch (error) {
+          unhealthyDataSources.push(ds);
+        }
path: src/utils/utils.datasource.ts
user: login: NWRichmond
body: it works! https://github.com/grafana/metrics-drilldown/pull/384/commits/dbd65840f2ebfbb272a43c5253ea1ef7740612dd
created_at: 2025-05-12T14:40:01Z
author_association: COLLABORATOR
diff_hunk: @@ -136,6 +139,7 @@ export class DataTrail extends SceneObjectBase<DataTrailState> implements SceneO
       histogramsLoaded: state.histogramsLoaded ?? false,
       nativeHistogramMetric: state.nativeHistogramMetric ?? '',
       trailActivated: state.trailActivated ?? false,
+      datasourceFetcher: state.datasourceFetcher ?? new DataSourceFetcher(),
path: src/DataTrail.tsx
user: login: NWRichmond
body: Good idea, implemented in https://github.com/grafana/metrics-drilldown/pull/384/commits/7af7247b9e60ddefd40c905038716bf863517cbc
created_at: 2025-05-12T14:51:10Z
author_association: COLLABORATOR

PR comments:
user: login: github-actions[bot]
created_at: 2025-05-02T21:28:18Z
body: :cry: zizmor failed with exit code 14.

<details>
<summary>Expand for full output</summary>

```
error[excessive-permissions]: overly broad permissions
  --> ./.github/workflows/playwright-published.yml:11:3
   |
11 |   id-token: 'write'
   |   ^^^^^^^^^^^^^^^^^ id-token: write is overly broad at the workflow level
   |
   = note: audit confidence → High

error[template-injection]: code injection via template expansion
  --> ./.github/workflows/playwright-published.yml:37:9
   |
37 |         - name: Package plugin
   |           ^^^^^^^^^^^^^^^^^^^^ this step
38 |           id: package
39 | /         run: |
40 | |           mkdir -p dist-artifacts
...  |
56 | |           # Clean up
57 | |           rm -rf temp
   | |_____________________^ github.event.repository.name may expand into attacker-controllable code
   |
   = note: audit confidence → High

15 findings (1 ignored, 12 suppressed): 0 unknown, 0 informational, 0 low, 0 medium, 2 high
```
</details>
<!-- comment-action/zizmor GitHub Actions static analysis/analysis -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-02T21:32:34Z
body: :cry: zizmor failed with exit code 14.

<details>
<summary>Expand for full output</summary>

```
error[excessive-permissions]: overly broad permissions
  --> ./.github/workflows/playwright-published.yml:11:3
   |
11 |   id-token: 'write'
   |   ^^^^^^^^^^^^^^^^^ id-token: write is overly broad at the workflow level
   |
   = note: audit confidence → High

error[template-injection]: code injection via template expansion
  --> ./.github/workflows/playwright-published.yml:37:9
   |
37 |         - name: Package plugin
   |           ^^^^^^^^^^^^^^^^^^^^ this step
38 |           id: package
39 | /         run: |
40 | |           mkdir -p dist-artifacts
...  |
56 | |           # Clean up
57 | |           rm -rf temp
   | |_____________________^ github.event.repository.name may expand into attacker-controllable code
   |
   = note: audit confidence → High

15 findings (1 ignored, 12 suppressed): 0 unknown, 0 informational, 0 low, 0 medium, 2 high
```
</details>
<!-- comment-action/zizmor GitHub Actions static analysis/analysis -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-02T21:54:31Z
body: :cry: zizmor failed with exit code 14.

<details>
<summary>Expand for full output</summary>

```
error[excessive-permissions]: overly broad permissions
  --> ./.github/workflows/playwright-published.yml:11:3
   |
11 |   id-token: 'write'
   |   ^^^^^^^^^^^^^^^^^ id-token: write is overly broad at the workflow level
   |
   = note: audit confidence → High

error[template-injection]: code injection via template expansion
  --> ./.github/workflows/playwright-published.yml:37:9
   |
37 |         - name: Package plugin
   |           ^^^^^^^^^^^^^^^^^^^^ this step
38 |           id: package
39 | /         run: |
40 | |           mkdir -p dist-artifacts
...  |
56 | |           # Clean up
57 | |           rm -rf temp
   | |_____________________^ github.event.repository.name may expand into attacker-controllable code
   |
   = note: audit confidence → High

15 findings (1 ignored, 12 suppressed): 0 unknown, 0 informational, 0 low, 0 medium, 2 high
```
</details>
<!-- comment-action/zizmor GitHub Actions static analysis/analysis -->
author_association: CONTRIBUTOR