Repository: kiali/kiali

Issue:
Tracing tool to help troubleshooting
### What do you want to improve?

The Tempo configuration introduced some complexity regarding the configuration of the tracing section in Kiali.
Trying to improve the situation with some documentation, this idea from @jshaughn about creating a tool that tries to autodiscover/validate the settings emerged. 

### What is the current behavior?
There is just a health check that it really doesn't help troubleshoot the problems. 

### What is the new behavior?
Create a tracing tool to correct the tracing settings for the Kiali configuration.

It can be an external tool like a hack script or, it can be integrated in Kiali. 

The idea to be integrated in Kiali is because running as an external tool could introduce some additional issues related to the permissions, location, or the Kiali settings. 

If the tool was an on-demand feature integrated in Kiali (For example, from the tracing node in the Mesh page) that based on the Kiali settings tries to perform some tests inside and outside the cluster (From the client side), and gave a report based on these tests. 

For example: 
the jaeger provider is configured but internal_url is pointing to the tempo api. Considerer change:
enternal_services.tracing.internal_url: "internal_tempo_url"
Or: 
The external link is hidden because tempo is configured but Grafana is disabled. Consider change tempo_config.url_format to jaeger.

With some basic data, Kiali could try to connect using the http client or the grpc client, analyze the urls from the configuration, and try to infer the right urls based on some parameters, like http/grpc, the provider, with authentication, ...

Hints from issue comments:
@jshaughn: Thinking more on this, maybe think about a similar, or combined effort, to ensure other external_services are correctly configured. But also maybe that if connectivity is established, that can maybe somehow also ensure decent performance.  Certainly just ensuring that users can quickly identify whether config is correct would be really valuable.  I mention perf only because that's typically the next problem.  They are connected but then experience timeouts etc.   

This all ties into stuff that @jmazzitelli and @nrfox are also doing around the improving how users can configure can configure and troubleshoot Kiali itself. 

Solution patch:
diff --git a/config/config.go b/config/config.go
index 47f7f5dd5..05834b0ee 100644
--- a/config/config.go
+++ b/config/config.go
@@ -257,6 +257,7 @@ type CustomDashboardsConfig struct {
 type GrafanaConfig struct {
 	Auth           Auth                     `yaml:"auth"`
 	Dashboards     []GrafanaDashboardConfig `yaml:"dashboards"`
+	DatasourceUID  string                   `yaml:"datasource_uid,omitempty"`
 	Enabled        bool                     `yaml:"enabled"`      // Enable or disable Grafana support in Kiali
 	ExternalURL    string                   `yaml:"external_url"` // replaces the old url
 	HealthCheckUrl string                   `yaml:"health_check_url,omitempty"`
@@ -272,11 +273,12 @@ type GrafanaDashboardConfig struct {
 }
 
 type GrafanaVariablesConfig struct {
-	App       string `yaml:"app" json:"app,omitempty"`
-	Namespace string `yaml:"namespace" json:"namespace,omitempty"`
-	Service   string `yaml:"service" json:"service,omitempty"`
-	Version   string `yaml:"version" json:"version,omitempty"`
-	Workload  string `yaml:"workload" json:"workload,omitempty"`
+	App        string `yaml:"app" json:"app,omitempty"`
+	Datasource string `yaml:"datasource" json:"datasource,omitempty"`
+	Namespace  string `yaml:"namespace" json:"namespace,omitempty"`
+	Service    string `yaml:"service" json:"service,omitempty"`
+	Version    string `yaml:"version" json:"version,omitempty"`
+	Workload   string `yaml:"workload" json:"workload,omitempty"`
 }
 
 type TempoConfig struct {
diff --git a/config/dashboards/dashboards_config.go b/config/dashboards/dashboards_config.go
index 563287a69..849184817 100644
--- a/config/dashboards/dashboards_config.go
+++ b/config/dashboards/dashboards_config.go
@@ -79,11 +79,12 @@ type MonitoringDashboardExternalLink struct {
 }
 
 type MonitoringDashboardExternalLinkVariables struct {
-	Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"`
-	App       string `json:"app,omitempty" yaml:"app,omitempty"`
-	Service   string `json:"service,omitempty" yaml:"service,omitempty"`
-	Version   string `json:"version,omitempty" yaml:"version,omitempty"`
-	Workload  string `json:"workload,omitempty" yaml:"workload,omitempty"`
+	App        string `json:"app,omitempty" yaml:"app,omitempty"`
+	Datasource string `json:"datasource,omitempty" yaml:"datasource,omitempty"`
+	Namespace  string `json:"namespace,omitempty" yaml:"namespace,omitempty"`
+	Service    string `json:"service,omitempty" yaml:"service,omitempty"`
+	Version    string `json:"version,omitempty" yaml:"version,omitempty"`
+	Workload   string `json:"workload,omitempty" yaml:"workload,omitempty"`
 }
 
 // GetMetrics provides consistent MonitoringDashboardMetric slice in a backward-compatible way, if deprecated field MetricName is used instead of Metrics in Spec.
diff --git a/frontend/src/components/Ambient/ZtunnelMetrics.tsx b/frontend/src/components/Ambient/ZtunnelMetrics.tsx
index 9d7bdce44..414032d5b 100644
--- a/frontend/src/components/Ambient/ZtunnelMetrics.tsx
+++ b/frontend/src/components/Ambient/ZtunnelMetrics.tsx
@@ -29,7 +29,7 @@ export const ZtunnelMetrics: React.FC<ZtunnelMetricsProps> = (props: ZtunnelMetr
   const expandedChart = urlParams.get('expand') ?? undefined;
   const toolbarRef = React.createRef<HTMLDivElement>();
   const [metrics, setMetrics] = React.useState<DashboardModel>();
-  const [grafanaLinks, setGrafanaLinks] = React.useState<GrafanaInfo>();
+  const [grafanaInfo, setGrafanaInfo] = React.useState<GrafanaInfo>();
   const rateParams = computePrometheusRateParams(
     props.rangeDuration.rangeDuration ? props.rangeDuration.rangeDuration : 60,
     10
@@ -63,7 +63,7 @@ export const ZtunnelMetrics: React.FC<ZtunnelMetricsProps> = (props: ZtunnelMetr
     API.getGrafanaInfo()
       .then(grafanaInfo => {
         if (grafanaInfo) {
-          setGrafanaLinks(grafanaInfo.data);
+          setGrafanaInfo(grafanaInfo.data);
         }
       })
       .catch(err => {
@@ -100,16 +100,17 @@ export const ZtunnelMetrics: React.FC<ZtunnelMetricsProps> = (props: ZtunnelMetr
 
   return (
     <div>
-      {grafanaLinks && (
+      {grafanaInfo && (
         <div ref={toolbarRef}>
           <Toolbar style={{ padding: 0, marginBottom: '1.25rem' }}>
             <ToolbarGroup>
               <ToolbarItem style={{ marginLeft: 'auto', paddingRight: '1.25rem' }}>
                 <GrafanaLinks
-                  links={grafanaLinks?.externalLinks}
+                  links={grafanaInfo.externalLinks}
                   namespace={props.namespace}
                   object="ztunnel"
                   objectType={MetricsObjectTypes.ZTUNNEL}
+                  datasourceUID={grafanaInfo.datasourceUID}
                 />
               </ToolbarItem>
             </ToolbarGroup>
diff --git a/frontend/src/components/Mesh/TraceDiagnose.tsx b/frontend/src/components/Mesh/TraceDiagnose.tsx
index efecc62db..4b9037bdc 100644
--- a/frontend/src/components/Mesh/TraceDiagnose.tsx
+++ b/frontend/src/components/Mesh/TraceDiagnose.tsx
@@ -196,7 +196,7 @@ export const TracingDiagnoseComp: React.FC<TracingDiagnoseProps> = (props: Traci
                 <>
                   <div className={configStyle}>
                     {Object.keys(item).map(key => {
-                      if (labels[key] !== undefined) {
+                      if (labels[key] !== undefined && item[key] !== null) {
                         return (
                           <span className={blockDisplay}>
                             <span className={blueDisplay}>{labels[key]}:</span> {item[key].toString()}
diff --git a/frontend/src/components/Metrics/CustomMetrics.tsx b/frontend/src/components/Metrics/CustomMetrics.tsx
index 34643a40f..3b27e289b 100644
--- a/frontend/src/components/Metrics/CustomMetrics.tsx
+++ b/frontend/src/components/Metrics/CustomMetrics.tsx
@@ -26,7 +26,7 @@ import { TimeDurationModal } from '../Time/TimeDurationModal';
 import { GrafanaLinks } from './GrafanaLinks';
 import { MetricsObjectTypes } from 'types/Metrics';
 import { SpanOverlay, JaegerLineInfo } from './SpanOverlay';
-import { DashboardModel, ExternalLink } from 'types/Dashboards';
+import { DashboardModel } from 'types/Dashboards';
 import { Overlay } from 'types/Overlay';
 import { Aggregator, DashboardQuery } from 'types/MetricsOptions';
 import { RawOrBucket } from 'types/VictoryChartInfo';
@@ -38,11 +38,12 @@ import { timeRangeSelector } from '../../store/Selectors';
 import { TimeDurationIndicator } from '../Time/TimeDurationIndicator';
 import { isParentKiosk, kioskContextMenuAction } from 'components/Kiosk/KioskActions';
 import { TraceSpansLimit } from './TraceSpansLimit';
+import { GrafanaInfo } from '../../types/GrafanaInfo';
 
 type MetricsState = {
   cluster?: string;
   dashboard?: DashboardModel;
-  grafanaLinks: ExternalLink[];
+  grafanaInfo: GrafanaInfo;
   isTimeOptionsOpen: boolean;
   labelsSettings: LabelsSettings;
   showSpans: boolean;
@@ -109,7 +110,9 @@ class CustomMetricsComponent extends React.Component<Props, MetricsState> {
     const cluster = HistoryManager.getClusterName();
     this.state = {
       cluster: cluster,
-      grafanaLinks: [],
+      grafanaInfo: {
+        externalLinks: []
+      },
       isTimeOptionsOpen: false,
       labelsSettings: settings.labelsSettings,
       showSpans: settings.showSpans,
@@ -190,7 +193,7 @@ class CustomMetricsComponent extends React.Component<Props, MetricsState> {
         this.setState({
           dashboard: response.data,
           labelsSettings: labelsSettings,
-          grafanaLinks: response.data.externalLinks
+          grafanaInfo: response.data
         });
       })
       .catch(error => {
@@ -366,10 +369,11 @@ class CustomMetricsComponent extends React.Component<Props, MetricsState> {
 
           <ToolbarGroup style={{ marginLeft: 'auto', paddingRight: '20px' }}>
             <GrafanaLinks
-              links={this.state.grafanaLinks}
+              links={this.state.grafanaInfo.externalLinks}
               namespace={this.props.namespace}
               object={this.props.workload ? this.props.workload : this.props.app}
               objectType={this.props.workload ? MetricsObjectTypes.WORKLOAD : MetricsObjectTypes.APP}
+              datasourceUID={this.state.grafanaInfo.datasourceUID}
               version={this.props.version}
             />
           </ToolbarGroup>
diff --git a/frontend/src/components/Metrics/GrafanaLinks.tsx b/frontend/src/components/Metrics/GrafanaLinks.tsx
index 701e9d386..135f8e5f5 100644
--- a/frontend/src/components/Metrics/GrafanaLinks.tsx
+++ b/frontend/src/components/Metrics/GrafanaLinks.tsx
@@ -6,6 +6,7 @@ import { MetricsObjectTypes } from 'types/Metrics';
 import { ExternalLink, ISTIO_ZTUNNEL_DASHBOARD } from 'types/Dashboards';
 
 type Props = {
+  datasourceUID?: string;
   links: ExternalLink[];
   namespace: string;
   object: string;
@@ -20,24 +21,26 @@ export class GrafanaLinks extends React.PureComponent<Props, {}> {
       if (MetricsObjectTypes.ZTUNNEL !== props.objectType || d.name !== ISTIO_ZTUNNEL_DASHBOARD) {
         const first = d.url.includes('?') ? '&' : '?';
         const nsvar = d.variables.namespace ? `&${d.variables.namespace}=${props.namespace}` : '';
+        const dsvar =
+          d.variables.datasource && props.datasourceUID ? `&${d.variables.datasource}=${props.datasourceUID}` : '';
         const vervar = d.variables.version && props.version ? `&${d.variables.version}=${props.version}` : '';
         switch (props.objectType) {
           case MetricsObjectTypes.SERVICE:
             const fullServiceName = `${props.object}.${props.namespace}.svc.cluster.local`;
             if (d.variables.service) {
-              const url = `${d.url}${first}${d.variables.service}=${fullServiceName}${nsvar}${vervar}`;
+              const url = `${d.url}${first}${d.variables.service}=${fullServiceName}${nsvar}${dsvar}${vervar}`;
               links.push([d.name, url]);
             }
             break;
           case MetricsObjectTypes.WORKLOAD:
             if (d.variables.workload) {
-              const url = `${d.url}${first}${d.variables.workload}=${props.object}${nsvar}${vervar}`;
+              const url = `${d.url}${first}${d.variables.workload}=${props.object}${nsvar}${dsvar}${vervar}`;
               links.push([d.name, url]);
             }
             break;
           case MetricsObjectTypes.APP:
             if (d.variables.app) {
-              const url = `${d.url}${first}${d.variables.app}=${props.object}${nsvar}${vervar}`;
+              const url = `${d.url}${first}${d.variables.app}=${props.object}${nsvar}${dsvar}${vervar}`;
               links.push([d.name, url]);
             }
             break;
diff --git a/frontend/src/components/Metrics/IstioMetrics.tsx b/frontend/src/components/Metrics/IstioMetrics.tsx
index b39b2a872..52207c497 100644
--- a/frontend/src/components/Metrics/IstioMetrics.tsx
+++ b/frontend/src/components/Metrics/IstioMetrics.tsx
@@ -20,9 +20,8 @@ import { location, router, URLParam } from 'app/History';
 import { MetricsObjectTypes } from 'types/Metrics';
 import { GrafanaInfo } from 'types/GrafanaInfo';
 import { MessageType } from 'types/MessageCenter';
-import { GrafanaLinks } from './GrafanaLinks';
 import { SpanOverlay, JaegerLineInfo } from './SpanOverlay';
-import { ChartModel, DashboardModel, ExternalLink } from 'types/Dashboards';
+import { ChartModel, DashboardModel } from 'types/Dashboards';
 import { Overlay } from 'types/Overlay';
 import { RawOrBucket } from 'types/VictoryChartInfo';
 import { Dashboard } from 'components/Charts/Dashboard';
@@ -33,11 +32,12 @@ import { TimeDurationIndicator } from '../Time/TimeDurationIndicator';
 import { ApiResponse } from 'types/Api';
 import { isParentKiosk, kioskContextMenuAction } from 'components/Kiosk/KioskActions';
 import { TraceSpansLimit } from './TraceSpansLimit';
+import { GrafanaLinks } from './GrafanaLinks';
 
 type MetricsState = {
   crippledFeatures?: KialiCrippledFeatures;
   dashboard?: DashboardModel;
-  grafanaLinks: ExternalLink[];
+  grafanaInfo: GrafanaInfo;
   isTimeOptionsOpen: boolean;
   labelsSettings: LabelsSettings;
   showSpans: boolean;
@@ -96,7 +96,9 @@ class IstioMetricsComponent extends React.Component<Props, MetricsState> {
     // Initialize active filters from URL
     this.state = {
       labelsSettings: settings.labelsSettings,
-      grafanaLinks: [],
+      grafanaInfo: {
+        externalLinks: []
+      },
       isTimeOptionsOpen: false,
       tabHeight: 300,
       showSpans: settings.showSpans,
@@ -231,9 +233,9 @@ class IstioMetricsComponent extends React.Component<Props, MetricsState> {
     IstioMetricsComponent.grafanaInfoPromise
       .then(grafanaInfo => {
         if (grafanaInfo) {
-          this.setState({ grafanaLinks: grafanaInfo.externalLinks });
+          this.setState({ grafanaInfo: grafanaInfo });
         } else {
-          this.setState({ grafanaLinks: [] });
+          this.setState({ grafanaInfo: { externalLinks: [] } });
         }
       })
       .catch(err => {
@@ -427,10 +429,11 @@ class IstioMetricsComponent extends React.Component<Props, MetricsState> {
 
             <ToolbarItem style={{ marginLeft: 'auto', paddingRight: '20px' }}>
               <GrafanaLinks
-                links={this.state.grafanaLinks}
+                links={this.state.grafanaInfo.externalLinks}
                 namespace={this.props.namespace}
                 object={this.props.object}
                 objectType={this.props.objectType}
+                datasourceUID={this.state.grafanaInfo.datasourceUID}
               />
             </ToolbarItem>
 
diff --git a/frontend/src/components/Metrics/__tests__/GrafanaLinks.test.ts b/frontend/src/components/Metrics/__tests__/GrafanaLinks.test.ts
index c91399014..1676b7124 100644
--- a/frontend/src/components/Metrics/__tests__/GrafanaLinks.test.ts
+++ b/frontend/src/components/Metrics/__tests__/GrafanaLinks.test.ts
@@ -8,12 +8,12 @@ describe('Grafana links', () => {
         {
           name: 'View in Grafana',
           url: 'http://grafana:3000',
-          variables: { namespace: 'var-namespace', service: 'var-service' }
+          variables: { namespace: 'var-namespace', service: 'var-service', datasource: 'var-datasource' }
         },
         {
           name: 'View in Grafana 2',
           url: 'http://grafana:3000?orgId=1',
-          variables: { namespace: 'var-namespace', service: 'var-service' }
+          variables: { namespace: 'var-namespace', service: 'var-service', datasource: 'var-datasource' }
         }
       ],
       namespace: 'my-namespace',
@@ -37,7 +37,7 @@ describe('Grafana links', () => {
         {
           name: 'View in Grafana',
           url: 'http://grafana:3000',
-          variables: { namespace: 'var-namespace', workload: 'var-workload' }
+          variables: { namespace: 'var-namespace', workload: 'var-workload', datasource: 'var-datasource' }
         },
         {
           name: 'View in Grafana 2',
@@ -47,11 +47,14 @@ describe('Grafana links', () => {
       ],
       namespace: 'my-namespace',
       object: 'my-workload',
+      datasourceUID: 'PROMETHEUSUID',
       objectType: MetricsObjectTypes.WORKLOAD
     });
     expect(links).toHaveLength(2);
     expect(links[0][0]).toEqual('View in Grafana');
-    expect(links[0][1]).toEqual('http://grafana:3000?var-workload=my-workload&var-namespace=my-namespace');
+    expect(links[0][1]).toEqual(
+      'http://grafana:3000?var-workload=my-workload&var-namespace=my-namespace&var-datasource=PROMETHEUSUID'
+    );
     expect(links[1][0]).toEqual('View in Grafana 2');
     expect(links[1][1]).toEqual('http://grafana:3000?orgId=1&var-workload=my-workload&var-namespace=my-namespace');
   });
@@ -62,7 +65,12 @@ describe('Grafana links', () => {
         {
           name: 'View in Grafana',
           url: 'http://grafana:3000',
-          variables: { namespace: 'var-namespace', app: 'var-app', version: 'var-version' }
+          variables: {
+            namespace: 'var-namespace',
+            app: 'var-app',
+            version: 'var-version',
+            datasource: 'var-datasource'
+          }
         },
         {
           name: 'View in Grafana 2',
diff --git a/frontend/src/pages/Graph/GraphElems.tsx b/frontend/src/pages/Graph/GraphElems.tsx
index 66feccd8c..7f22a5720 100644
--- a/frontend/src/pages/Graph/GraphElems.tsx
+++ b/frontend/src/pages/Graph/GraphElems.tsx
@@ -168,9 +168,13 @@ export const setNodeAttachments = (node: Node<NodeModel>, settings: GraphSetting
   if (settings.showOutOfMesh && data.isOutOfMesh) {
     attachments.push(getDecorator(node, TopologyQuadrant.lowerRight, badgeMap.get('MS')!));
   }
+  // It is OK to use the same location for mutually exclusive options.
   if (data.hasWorkloadEntry) {
     attachments.push(getDecorator(node, TopologyQuadrant.upperRight, badgeMap.get('WE')!));
   }
+  if (data.hasIngressWaypoint) {
+    attachments.push(getDecorator(node, TopologyQuadrant.upperRight, badgeMap.get('WA')!));
+  }
   if (settings.showVirtualServices) {
     if (data.hasCB) {
       attachments.push(getDecorator(node, TopologyQuadrant.upperLeft, badgeMap.get('CB')!));
diff --git a/frontend/src/types/Dashboards.ts b/frontend/src/types/Dashboards.ts
index fb06f8d5d..ac9ac97f9 100644
--- a/frontend/src/types/Dashboards.ts
+++ b/frontend/src/types/Dashboards.ts
@@ -42,6 +42,7 @@ export interface ExternalLink {
 
 export interface ExternalLinkVariables {
   app?: string;
+  datasource?: string;
   namespace?: string;
   service?: string;
   version?: string;
diff --git a/frontend/src/types/ErrorRate/__testData__/ErrorRateConfig.ts b/frontend/src/types/ErrorRate/__testData__/ErrorRateConfig.ts
index 647d6e18a..a72f00555 100644
--- a/frontend/src/types/ErrorRate/__testData__/ErrorRateConfig.ts
+++ b/frontend/src/types/ErrorRate/__testData__/ErrorRateConfig.ts
@@ -75,6 +75,7 @@ export const serverRateConfig = {
   controlPlaneClusters: [],
   gatewayAPIClasses: [],
   gatewayAPIEnabled: false,
+  grafana: {},
   logLevel: '',
   kialiFeatureFlags: {
     disabledFeatures: [],
diff --git a/frontend/src/types/GrafanaInfo.ts b/frontend/src/types/GrafanaInfo.ts
index 7205212dd..3f3ffc091 100644
--- a/frontend/src/types/GrafanaInfo.ts
+++ b/frontend/src/types/GrafanaInfo.ts
@@ -13,5 +13,6 @@ export const ISTIO_DASHBOARDS: string[] = [
 ];
 
 export interface GrafanaInfo {
+  datasourceUID?: string;
   externalLinks: ExternalLink[];
 }
diff --git a/frontend/src/types/Graph.ts b/frontend/src/types/Graph.ts
index c7f6d8735..2a5c13aa2 100644
--- a/frontend/src/types/Graph.ts
+++ b/frontend/src/types/Graph.ts
@@ -392,6 +392,7 @@ export interface GraphNodeData {
   hasCB?: boolean;
   hasFaultInjection?: boolean;
   hasHealthConfig?: HealthAnnotationType;
+  hasIngressWaypoint?: boolean;
   hasMirroring?: boolean;
   hasRequestRouting?: boolean;
   hasRequestTimeout?: boolean;
diff --git a/frontend/src/types/TracingInfo.ts b/frontend/src/types/TracingInfo.ts
index 3c38364c8..2c8f49309 100644
--- a/frontend/src/types/TracingInfo.ts
+++ b/frontend/src/types/TracingInfo.ts
@@ -132,7 +132,7 @@ export type TracingSingleResponse = {
 };
 
 export type ValidConfig = {
-  namespaceSelector: boolean;
+  namespaceSelector?: boolean;
   provider: string;
   url: string;
   useGRPC: boolean;
diff --git a/frontend/src/types/__testData__/HealthConfig.ts b/frontend/src/types/__testData__/HealthConfig.ts
index b258f1dc8..0d06c9e3b 100644
--- a/frontend/src/types/__testData__/HealthConfig.ts
+++ b/frontend/src/types/__testData__/HealthConfig.ts
@@ -9,6 +9,7 @@ export const healthConfig = {
   controlPlaneClusters: [],
   gatewayAPIClasses: [],
   gatewayAPIEnabled: false,
+  grafana: {},
   logLevel: '',
   kialiFeatureFlags: {
     disabledFeatures: [],
diff --git a/grafana/grafana.go b/grafana/grafana.go
index 40c26e0e2..c5fe568d4 100644
--- a/grafana/grafana.go
+++ b/grafana/grafana.go
@@ -139,11 +139,12 @@ func (s *Service) Info(ctx context.Context, dashboardSupplier DashboardSupplierF
 				URL:  dashboardPath,
 				Name: dashboardConfig.Name,
 				Variables: dashboards.MonitoringDashboardExternalLinkVariables{
-					App:       dashboardConfig.Variables.App,
-					Namespace: dashboardConfig.Variables.Namespace,
-					Service:   dashboardConfig.Variables.Service,
-					Version:   dashboardConfig.Variables.Version,
-					Workload:  dashboardConfig.Variables.Workload,
+					App:        dashboardConfig.Variables.App,
+					Datasource: dashboardConfig.Variables.Datasource,
+					Namespace:  dashboardConfig.Variables.Namespace,
+					Service:    dashboardConfig.Variables.Service,
+					Version:    dashboardConfig.Variables.Version,
+					Workload:   dashboardConfig.Variables.Workload,
 				},
 			}
 			links = append(links, externalLink)
@@ -151,6 +152,7 @@ func (s *Service) Info(ctx context.Context, dashboardSupplier DashboardSupplierF
 	}
 
 	grafanaInfo := models.GrafanaInfo{
+		DatasourceUID: grafanaConfig.DatasourceUID,
 		ExternalLinks: links,
 	}
 
diff --git a/graph/config/common/common.go b/graph/config/common/common.go
index 01da18af8..1be4ac72c 100644
--- a/graph/config/common/common.go
+++ b/graph/config/common/common.go
@@ -87,16 +87,17 @@ type NodeData struct {
 	Workload              string              `json:"workload,omitempty"`
 	App                   string              `json:"app,omitempty"`
 	Version               string              `json:"version,omitempty"`
-	Service               string              `json:"service,omitempty"`               // requested service for NodeTypeService
-	Aggregate             string              `json:"aggregate,omitempty"`             // set like "<aggregate>=<aggregateVal>"
-	DestServices          []graph.ServiceName `json:"destServices,omitempty"`          // requested services for [dest] node
-	Labels                map[string]string   `json:"labels,omitempty"`                // k8s labels associated with the node
-	Traffic               []ProtocolTraffic   `json:"traffic,omitempty"`               // traffic rates for all detected protocols
-	HealthData            interface{}         `json:"healthData"`                      // data to calculate health status from configurations
-	HealthDataApp         interface{}         `json:"-"`                               // for local use to generate appBox health
-	HasCB                 bool                `json:"hasCB,omitempty"`                 // true (has circuit breaker) | false
-	HasFaultInjection     bool                `json:"hasFaultInjection,omitempty"`     // true (vs has fault injection) | false
-	HasHealthConfig       HealthConfig        `json:"hasHealthConfig,omitempty"`       // set to the health config override
+	Service               string              `json:"service,omitempty"`           // requested service for NodeTypeService
+	Aggregate             string              `json:"aggregate,omitempty"`         // set like "<aggregate>=<aggregateVal>"
+	DestServices          []graph.ServiceName `json:"destServices,omitempty"`      // requested services for [dest] node
+	Labels                map[string]string   `json:"labels,omitempty"`            // k8s labels associated with the node
+	Traffic               []ProtocolTraffic   `json:"traffic,omitempty"`           // traffic rates for all detected protocols
+	HealthData            interface{}         `json:"healthData"`                  // data to calculate health status from configurations
+	HealthDataApp         interface{}         `json:"-"`                           // for local use to generate appBox health
+	HasCB                 bool                `json:"hasCB,omitempty"`             // true (has circuit breaker) | false
+	HasFaultInjection     bool                `json:"hasFaultInjection,omitempty"` // true (vs has fault injection) | false
+	HasHealthConfig       HealthConfig        `json:"hasHealthConfig,omitempty"`   // set to the health config override
+	HasIngressWaypoint    bool                `json:"hasIngressWaypoint,omitempty"`
 	HasMirroring          bool                `json:"hasMirroring,omitempty"`          // true (has mirroring) | false
 	HasRequestRouting     bool                `json:"hasRequestRouting,omitempty"`     // true (vs has request routing) | false
 	HasRequestTimeout     bool                `json:"hasRequestTimeout,omitempty"`     // true (vs has request timeout) | false
@@ -366,6 +367,11 @@ func buildConfig(trafficMap graph.TrafficMap, nodes *[]*NodeWrapper, edges *[]*E
 			nd.IsWaypoint = val.(bool)
 		}
 
+		// check if node redirects traffic to waypoint proxy
+		if val, ok := n.Metadata[graph.HasIngressWaypoint]; ok {
+			nd.HasIngressWaypoint = val.(bool)
+		}
+
 		if val, ok := n.Metadata[graph.HasMirroring]; ok {
 			nd.HasMirroring = val.(bool)
 		}
diff --git a/graph/meta.go b/graph/meta.go
index a97c3fa85..09c3cd8b9 100644
--- a/graph/meta.go
+++ b/graph/meta.go
@@ -22,6 +22,7 @@ const (
 	HasCB                 MetadataKey = "hasCB"
 	HasFaultInjection     MetadataKey = "hasFaultInjection"
 	HasHealthConfig       MetadataKey = "hasHealthConfig"
+	HasIngressWaypoint    MetadataKey = "hasIngressWaypoint"
 	HasMirroring          MetadataKey = "hasMirroring"
 	HasTCPTrafficShifting MetadataKey = "hasTCPTrafficShifting"
 	HasTrafficShifting    MetadataKey = "hasTrafficShifting"
diff --git a/graph/telemetry/istio/appender/ambient.go b/graph/telemetry/istio/appender/ambient.go
index 74d3e8fa8..905c5cbe3 100644
--- a/graph/telemetry/istio/appender/ambient.go
+++ b/graph/telemetry/istio/appender/ambient.go
@@ -69,6 +69,17 @@ func (a AmbientAppender) handleWaypoints(trafficMap graph.TrafficMap) {
 	}
 
 	for _, n := range trafficMap {
+
+		// If is a gateway and redirects traffic to a waypoint, it acts as an ingress
+		if n.Metadata[graph.IsGatewayAPI] != nil {
+			hasWaypointEdges := sliceutil.Some(n.Edges, func(edge *graph.Edge) bool {
+				return waypointNodes[edge.Dest.ID] != nil
+			})
+			if hasWaypointEdges {
+				n.Metadata[graph.HasIngressWaypoint] = true
+			}
+		}
+
 		// If not showing waypoints then delete edges going to a waypoint
 		if !a.ShowWaypoints {
 			n.Edges = sliceutil.Filter(n.Edges, func(edge *graph.Edge) bool {
diff --git a/graph/telemetry/istio/istio.go b/graph/telemetry/istio/istio.go
index 4aaba26e2..e091b7ee5 100644
--- a/graph/telemetry/istio/istio.go
+++ b/graph/telemetry/istio/istio.go
@@ -33,7 +33,6 @@ import (
 
 	"github.com/prometheus/common/model"
 
-	"github.com/kiali/kiali/config"
 	"github.com/kiali/kiali/graph"
 	"github.com/kiali/kiali/graph/telemetry"
 	"github.com/kiali/kiali/graph/telemetry/istio/appender"
@@ -41,7 +40,6 @@ import (
 	"github.com/kiali/kiali/log"
 	"github.com/kiali/kiali/observability"
 	"github.com/kiali/kiali/prometheus/internalmetrics"
-	"github.com/kiali/kiali/util/sliceutil"
 )
 
 const (
@@ -74,10 +72,7 @@ func BuildNamespacesTrafficMap(ctx context.Context, o graph.TelemetryOptions, gl
 	appenders, finalizers := appender.ParseAppenders(o)
 	trafficMap := graph.NewTrafficMap()
 
-	handleAmbient := sliceutil.Some(finalizers, func(f graph.Appender) bool {
-		return f.Name() == appender.AmbientAppenderName
-	})
-	if handleAmbient {
+	if o.Rates.Ambient != "none" {
 		globalInfo.Vendor[appender.AmbientWaypoints] = GetWaypointMap(ctx, globalInfo)
 	}
 
@@ -383,12 +378,6 @@ func populateTrafficMap(ctx context.Context, trafficMap graph.TrafficMap, vector
 			}
 			flags = string(lFlags)
 		}
-		ztunnel := false
-		if protocol == graph.TCP.Name {
-			if lApp, appOk := m["app"]; appOk {
-				ztunnel = string(lApp) == config.Ztunnel
-			}
-		}
 
 		// handle clusters
 		sourceCluster, destCluster := util.HandleClusters(lSourceCluster, sourceClusterOk, lDestCluster, destClusterOk)
@@ -433,9 +422,10 @@ func populateTrafficMap(ctx context.Context, trafficMap graph.TrafficMap, vector
 		// - dest node is already a service node
 		// - source or dest workload is an ambient waypoint
 		var inject, sourceIsWaypoint, destIsWaypoint bool
-		if ztunnel {
+		if o.Rates.Ambient != "none" {
 			sourceIsWaypoint, destIsWaypoint = hasWaypoint(setWaypointKey(wpKeySource, sourceCluster, sourceWlNs, sourceWl), setWaypointKey(wpKeyDest, destCluster, destWlNs, destWl), globalInfo)
 		}
+
 		if o.InjectServiceNodes && graph.IsOK(destSvcName) && destSvcName != graph.PassthroughCluster {
 			_, destNodeType, err := graph.Id(destCluster, destSvcNs, destSvcName, destWlNs, destWl, destApp, destVer, o.GraphType)
 			if err != nil {
@@ -582,10 +572,7 @@ func BuildNodeTrafficMap(ctx context.Context, o graph.TelemetryOptions, globalIn
 
 	appenders, finalizers := appender.ParseAppenders(o)
 
-	handleAmbient := sliceutil.Some(finalizers, func(f graph.Appender) bool {
-		return f.Name() == appender.AmbientAppenderName
-	})
-	if handleAmbient {
+	if o.Rates.Ambient != "none" {
 		globalInfo.Vendor[appender.AmbientWaypoints] = GetWaypointMap(ctx, globalInfo)
 	}
 
@@ -1135,9 +1122,11 @@ func setWaypointKey(wpKey *waypointKey, cluster, namespace, name string) *waypoi
 
 // hasWaypoint returns true if the source or dest workload is determined to be a waypoint workload.
 func hasWaypoint(wpKeySource, wpKeyDest *waypointKey, globalInfo *graph.GlobalInfo) (sourceIsWaypoint bool, destIsWaypoint bool) {
+
 	wpMap := globalInfo.Vendor[appender.AmbientWaypoints].(waypointMap)
 	sourceIsWaypoint = wpMap[*wpKeySource]
 	destIsWaypoint = wpMap[*wpKeyDest]
+
 	return sourceIsWaypoint, destIsWaypoint
 }
 
diff --git a/kiali.go b/kiali.go
index b55422843..7901ca581 100644
--- a/kiali.go
+++ b/kiali.go
@@ -227,6 +227,41 @@ func main() {
 		if !cache.WaitForCacheSync(ctx) {
 			log.Fatal("Timed out waiting for cache to sync")
 		}
+
+		// Controller-runtime lazily adds informers when they are first accessed.
+		// Normally you'd have controllers that do this but we don't have any controllers
+		// that setup watches for workloads or Istio config yet so we need to manually
+		// fetch these resources in order to ensure that the informers are created
+		// before the server starts serving requests. Otherwise, the first request
+		// that tries to access these resources will block until the informer is synced
+		// which can take a long time.
+		if _, err := layer.Workload.GetAllWorkloads(ctx, cluster, ""); err != nil {
+			log.Warningf("Unable to get workloads to sync cache for cluster %s. First request that accesses workloads may take awhile: %v", cluster, err)
+		}
+
+		if _, err := layer.IstioConfig.GetIstioConfigList(ctx, cluster, business.IstioConfigCriteria{
+			IncludeGateways:               true,
+			IncludeK8sGateways:            true,
+			IncludeK8sGRPCRoutes:          true,
+			IncludeK8sHTTPRoutes:          true,
+			IncludeK8sTCPRoutes:           true,
+			IncludeK8sTLSRoutes:           true,
+			IncludeVirtualServices:        true,
+			IncludeDestinationRules:       true,
+			IncludeSidecars:               true,
+			IncludeServiceEntries:         true,
+			IncludeWorkloadEntries:        true,
+			IncludeWorkloadGroups:         true,
+			IncludeEnvoyFilters:           true,
+			IncludeWasmPlugins:            true,
+			IncludeAuthorizationPolicies:  true,
+			IncludePeerAuthentications:    true,
+			IncludeRequestAuthentications: true,
+			IncludeTelemetry:              true,
+			IncludeK8sReferenceGrants:     true,
+		}); err != nil {
+			log.Warningf("Unable to get Istio config to sync cache for cluster %s. First request that accesses Istio config may take awhile: %v", cluster, err)
+		}
 	}
 	log.Info("All caches synced")
 
diff --git a/models/grafana_info.go b/models/grafana_info.go
index 7109e8534..31545f7b6 100644
--- a/models/grafana_info.go
+++ b/models/grafana_info.go
@@ -2,5 +2,6 @@ package models
 
 // GrafanaInfo provides information to access Grafana dashboards
 type GrafanaInfo struct {
+	DatasourceUID string         `json:"datasourceUID,omitempty"`
 	ExternalLinks []ExternalLink `json:"externalLinks"`
 }
diff --git a/tracing/discovery_client.go b/tracing/discovery_client.go
index f76a0712a..f6ea60810 100644
--- a/tracing/discovery_client.go
+++ b/tracing/discovery_client.go
@@ -408,14 +408,16 @@ func validateEndpoint(ctx context.Context, client http.Client, zl *zerolog.Logge
 			zl.Trace().Msg(msg)
 			return nil, logs, errors.New(msg)
 		}
-		vc := model.ValidConfig{Url: validEndpoint, Provider: provider, UseGRPC: false, NamespaceSelector: false}
+		ns := false
+		vc := model.ValidConfig{Url: validEndpoint, Provider: provider, UseGRPC: false}
 		for _, rd := range response.Data {
 			parts := strings.Split(rd, ".")
 			if len(parts) > 1 {
-				vc.NamespaceSelector = true
+				ns = true
 				break
 			}
 		}
+		vc.NamespaceSelector = &ns
 		msg := fmt.Sprintf("[Discovery client] Found valid Config %v", vc)
 		logs = append(logs, model.LogLine{Time: time.Now(), Test: endpoint, Result: msg})
 		zl.Trace().Msg(msg)
@@ -429,14 +431,16 @@ func validateEndpoint(ctx context.Context, client http.Client, zl *zerolog.Logge
 		zl.Trace().Msg(msg)
 		return nil, logs, errors.New(msg)
 	}
-	vc := model.ValidConfig{Url: validEndpoint, Provider: "tempo", UseGRPC: false, NamespaceSelector: false}
+	ns := false
+	vc := model.ValidConfig{Url: validEndpoint, Provider: "tempo", UseGRPC: false}
 	for _, rd := range response.Traces {
 		parts := strings.Split(rd.RootServiceName, ".")
 		if len(parts) > 1 {
-			vc.NamespaceSelector = true
+			ns = true
 			break
 		}
 	}
+	vc.NamespaceSelector = &ns
 	msg := fmt.Sprintf("[Discovery client] Found valid Config %v", vc)
 	logs = append(logs, model.LogLine{Time: time.Now(), Test: endpoint, Result: msg})
 	zl.Trace().Msg(msg)
diff --git a/tracing/jaeger/model/types.go b/tracing/jaeger/model/types.go
index 80cb99389..979d2c0e4 100644
--- a/tracing/jaeger/model/types.go
+++ b/tracing/jaeger/model/types.go
@@ -47,7 +47,7 @@ type ParsedUrl struct {
 }
 
 type ValidConfig struct {
-	NamespaceSelector bool   `json:"namespaceSelector"`
+	NamespaceSelector *bool  `json:"namespaceSelector"`
 	Provider          string `json:"provider"`
 	Url               string `json:"url"`
 	UseGRPC           bool   `json:"useGRPC"`



Test patch:
diff --git a/grafana/grafana_test.go b/grafana/grafana_test.go
index 190d73248..26882c285 100644
--- a/grafana/grafana_test.go
+++ b/grafana/grafana_test.go
@@ -63,6 +63,7 @@ func TestGetGrafanaInfoExternal(t *testing.T) {
 	assert.Equal(t, http.StatusOK, code)
 	assert.Len(t, info.ExternalLinks, 1)
 	assert.Equal(t, "http://grafana-external:3001/some_path", info.ExternalLinks[0].URL)
+	assert.Equal(t, "", info.DatasourceUID)
 }
 
 func TestGetGrafanaInfoInCluster(t *testing.T) {
@@ -70,6 +71,7 @@ func TestGetGrafanaInfoInCluster(t *testing.T) {
 	conf.ExternalServices.Grafana.ExternalURL = "http://grafana-external:3001"
 	conf.ExternalServices.Grafana.Dashboards = dashboardsConfig
 	conf.ExternalServices.Grafana.InternalURL = "http://grafana.istio-system:3001"
+	conf.ExternalServices.Grafana.DatasourceUID = "PROMETHEUS"
 
 	grafana := grafana.NewService(conf, kubetest.NewFakeK8sClient())
 
@@ -82,6 +84,7 @@ func TestGetGrafanaInfoInCluster(t *testing.T) {
 	assert.Equal(t, http.StatusOK, code)
 	assert.Len(t, info.ExternalLinks, 1)
 	assert.Equal(t, "http://grafana-external:3001/some_path", info.ExternalLinks[0].URL)
+	assert.Equal(t, "PROMETHEUS", info.DatasourceUID)
 }
 
 func TestGetGrafanaInfoGetError(t *testing.T) {
diff --git a/mesh/api/testdata/test_mesh_graph.expected b/mesh/api/testdata/test_mesh_graph.expected
index 73989b271..b3a9b5cfe 100644
--- a/mesh/api/testdata/test_mesh_graph.expected
+++ b/mesh/api/testdata/test_mesh_graph.expected
@@ -103,6 +103,7 @@
             "Username" : "xxx"
           },
           "Dashboards" : null,
+          "DatasourceUID" : "",
           "Enabled" : true,
           "ExternalURL" : "",
           "HealthCheckUrl" : "",
diff --git a/tracing/discovery_client_test.go b/tracing/discovery_client_test.go
index 3178981cb..7229c7e0a 100644
--- a/tracing/discovery_client_test.go
+++ b/tracing/discovery_client_test.go
@@ -142,8 +142,8 @@ func TestValidateSimpleTempoHTTP(t *testing.T) {
 	assert.Equal(t, "tempo", validConfigs[0].Provider)
 	assert.Equal(t, "jaeger", validConfigs[1].Provider)
 
-	assert.True(t, validConfigs[0].NamespaceSelector)
-	assert.True(t, validConfigs[1].NamespaceSelector)
+	assert.True(t, *validConfigs[0].NamespaceSelector)
+	assert.True(t, *validConfigs[1].NamespaceSelector)
 
 	assert.Equal(t, "http://tempo-host:3100", validConfigs[0].Url)
 	assert.Equal(t, "http://tempo-host:3100", validConfigs[1].Url)
@@ -268,8 +268,8 @@ func TestValidateJaegerHTTP_SuccessfulBothEndpoints(t *testing.T) {
 	assert.Equal(t, "jaeger", validConfigs[0].Provider)
 	assert.Equal(t, "jaeger", validConfigs[1].Provider)
 	assert.GreaterOrEqual(t, len(logs), 2)
-	assert.True(t, validConfigs[0].NamespaceSelector)
-	assert.True(t, validConfigs[1].NamespaceSelector)
+	assert.True(t, *validConfigs[0].NamespaceSelector)
+	assert.True(t, *validConfigs[1].NamespaceSelector)
 }
 
 func TestValidateJaegerHTTP_OnlyOneSuccess(t *testing.T) {