Grafana dashboard: Pixie K8s Cluster Overview
Description: Dashboard includes a service map of the HTTP traffic between the services in your cluster, along with the latency, error rate, and throughput per service. It also lists the nodes, namespaces and pods available in your cluster.

To learn how to monitor service performance and infrastructure health using Pixie, see the [tutorial](https://docs.px.dev/tutorials/pixie-101/service-performance/).

{
  "__inputs": [
    {
      "name": "DS_PIXIE_GRAFANA DATASOURCE PLUGIN",
      "label": "Pixie Grafana Datasource Plugin",
      "description": "",
      "type": "datasource",
      "pluginId": "pixie-pixie-datasource",
      "pluginName": "Pixie Grafana Datasource Plugin"
    }
  ],
  "__requires": [
    {
      "type": "panel",
      "id": "nodeGraph",
      "name": "Node Graph",
      "version": ""
    },
    {
      "type": "datasource",
      "id": "pixie-pixie-datasource",
      "name": "Pixie Grafana Datasource Plugin",
      "version": "0.0.9"
    },
    {
      "type": "panel",
      "id": "table",
      "name": "Table",
      "version": ""
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "target": {
          "limit": 100,
          "matchAny": false,
          "tags": [],
          "type": "dashboard"
        },
        "type": "dashboard"
      }
    ]
  },
  "editable": true,
  "graphTooltip": 0,
  "id": null,
  "iteration": 1657152736466,
  "links": [],
  "panels": [
    {
      "datasource": "${DS_PIXIE_GRAFANA DATASOURCE PLUGIN}",
      "gridPos": {
        "h": 14,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 2,
      "targets": [
        {
          "datasource": "${DS_PIXIE_GRAFANA DATASOURCE PLUGIN}",
          "queryBody": {
            "pxlScript": "# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n'''\nThis query outputs a graph of the HTTP traffic between the services in\nyour cluster. Use with Grafana's node graph panel.\n\nThis query is for use with Grafana's Pixie Datasource Plugin only,\nas it uses Grafana macros for adding Grafana dashboard context.\nThe functions in this query are pulled from the px/cluster script:\nhttps://github.com/pixie-io/pixie/tree/main/src/pxl_scripts/px/cluster\n'''\n\n# Import Pixie's module for querying data.\nimport px\n\n# Window size to use on time_ column for bucketing.\nns_per_s = 1000 * 1000 * 1000\nns_per_ms = 1000 * 1000\nwindow_ns = px.DurationNanos(10 * ns_per_s)\n\n# Flag to filter out health checks from the data.\nfilter_health_checks = True\n\n# Whether or not to include traffic from IPs that don't resolve to a known pod/service.\ninclude_ips = True\n\n# $pixieCluster - work around for grafana to update panel on variable change\n\n# Hack to get the time window for the script.\n# TODO(philkuz): Replace this with a built-in.\ndef get_time_window(start_time: int):\n    ''' Converts the start_time string into a table with a single column and single row.\n    The approach is hacky, and will round to roughly 1 second.\n    '''\n    df = px.DataFrame('process_stats', start_time=start_time)\n\n    df = df.agg(\n        time_min=('time_', px.min),\n        time_max=('time_', px.max),\n    )\n\n    df.window = px.DurationNanos(df.time_max - df.time_min)\n    df = df[['window']]\n\n    return df\n\n\ndef add_time_window_column(df, start_time):\n    tw = get_time_window(start_time)\n    df = df.merge(tw, how='inner', left_on=[], right_on=[])\n    return df\n\n\ndef process_stats_by_entity(start_time: int, entity: str):\n    ''' Gets the windowed process stats (CPU, memory, etc) per node or pod.\n    Args:\n    @start_time Starting time of the data to examine.\n    @entity: Either pod or node_name.\n    '''\n    # Window size to use on time_ column for bucketing.\n    ns_per_s = 1000 * 1000 * 1000\n    window_ns = px.DurationNanos(10 * ns_per_s)\n\n    df = px.DataFrame(table='process_stats', start_time=start_time)\n    df[entity] = df.ctx[entity]\n    df.timestamp = px.bin(df.time_, window_ns)\n    # First calculate CPU usage by process (UPID) in each k8s_object\n    # over all windows.\n    df = df.groupby([entity, 'upid', 'timestamp']).agg(\n        rss=('rss_bytes', px.mean),\n        vsize=('vsize_bytes', px.mean),\n        # The fields below are counters, so we take the min and the max to subtract them.\n        cpu_utime_ns_max=('cpu_utime_ns', px.max),\n        cpu_utime_ns_min=('cpu_utime_ns', px.min),\n        cpu_ktime_ns_max=('cpu_ktime_ns', px.max),\n        cpu_ktime_ns_min=('cpu_ktime_ns', px.min),\n        read_bytes_max=('read_bytes', px.max),\n        read_bytes_min=('read_bytes', px.min),\n        write_bytes_max=('write_bytes', px.max),\n        write_bytes_min=('write_bytes', px.min),\n        rchar_bytes_max=('rchar_bytes', px.max),\n        rchar_bytes_min=('rchar_bytes', px.min),\n        wchar_bytes_max=('wchar_bytes', px.max),\n        wchar_bytes_min=('wchar_bytes', px.min),\n    )\n    # Next calculate cpu usage and memory stats per window.\n    df.cpu_utime_ns = df.cpu_utime_ns_max - df.cpu_utime_ns_min\n    df.cpu_ktime_ns = df.cpu_ktime_ns_max - df.cpu_ktime_ns_min\n    df.read_bytes = df.read_bytes_max - df.read_bytes_min\n    df.write_bytes = df.write_bytes_max - df.write_bytes_min\n    df.rchar_bytes = df.rchar_bytes_max - df.rchar_bytes_min\n    df.wchar_bytes = df.wchar_bytes_max - df.wchar_bytes_min\n    # Sum by UPID.\n    df = df.groupby([entity, 'timestamp']).agg(\n        cpu_ktime_ns=('cpu_ktime_ns', px.sum),\n        cpu_utime_ns=('cpu_utime_ns', px.sum),\n        read_bytes=('read_bytes', px.sum),\n        write_bytes=('write_bytes', px.sum),\n        rchar_bytes=('rchar_bytes', px.sum),\n        wchar_bytes=('wchar_bytes', px.sum),\n        rss=('rss', px.sum),\n        vsize=('vsize', px.sum),\n    )\n    df.actual_disk_read_throughput = df.read_bytes / window_ns\n    df.actual_disk_write_throughput = df.write_bytes / window_ns\n    df.total_disk_read_throughput = df.rchar_bytes / window_ns\n    df.total_disk_write_throughput = df.wchar_bytes / window_ns\n    # Now take the mean value over the various timestamps.\n    df = df.groupby(entity).agg(\n        cpu_ktime_ns=('cpu_ktime_ns', px.mean),\n        cpu_utime_ns=('cpu_utime_ns', px.mean),\n        actual_disk_read_throughput=('actual_disk_read_throughput', px.mean),\n        actual_disk_write_throughput=('actual_disk_write_throughput', px.mean),\n        total_disk_read_throughput=('total_disk_read_throughput', px.mean),\n        total_disk_write_throughput=('total_disk_write_throughput', px.mean),\n        avg_rss=('rss', px.mean),\n        avg_vsize=('vsize', px.mean),\n    )\n    # Finally, calculate total (kernel + user time)  percentage used over window.\n    df.cpu_usage = px.Percent((df.cpu_ktime_ns + df.cpu_utime_ns) / window_ns)\n    return df.drop(['cpu_ktime_ns', 'cpu_utime_ns'])\n\n\ndef nodes_for_cluster(start_time: int):\n    ''' Gets a list of nodes in the current cluster since `start_time`.\n    Args:\n    @start_time Start time of the data to examine.\n    '''\n    df = px.DataFrame(table='process_stats', start_time=start_time)\n    df.node = df.ctx['node_name']\n    df.pod = df.ctx['pod_name']\n    agg = df.groupby(['node', 'pod']).agg()\n    nodes = agg.groupby('node').agg(pod_count=('pod', px.count))\n    process_stats = process_stats_by_entity(start_time, 'node')\n    output = process_stats.merge(nodes, how='inner', left_on='node', right_on='node',\n                                 suffixes=['', '_x'])\n    return output[['node', 'cpu_usage', 'pod_count']]\n\n\ndef pods_for_cluster(start_time: int):\n    ''' A list of pods in `namespace`.\n    Args:\n    @start_time: The timestamp of data to start at.\n    @namespace: The name of the namespace to filter on.\n    '''\n    df = px.DataFrame(table='process_stats', start_time=start_time)\n    df.pod = df.ctx['pod_name']\n    df.node = df.ctx['node_name']\n    df.container = df.ctx['container_name']\n    df = df.groupby(['pod', 'node', 'container']).agg()\n    df = df.groupby(['pod', 'node']).agg(container_count=('container', px.count))\n    df.start_time = px.pod_name_to_start_time(df.pod)\n    df.status = px.pod_name_to_status(df.pod)\n    process_stats = process_stats_by_entity(start_time, 'pod')\n    output = process_stats.merge(df, how='inner', left_on='pod', right_on='pod',\n                                 suffixes=['', '_x'])\n    return output[['pod', 'cpu_usage', 'total_disk_read_throughput',\n                   'total_disk_write_throughput', 'container_count',\n                   'node', 'start_time', 'status']]\n\n\ndef namespaces_for_cluster(start_time: int):\n    ''' Gets a overview of namespaces in the current cluster since `start_time`.\n    Args:\n    @start_time Start time of the data to examine.\n    '''\n    df = px.DataFrame(table='process_stats', start_time=start_time)\n    df.service = df.ctx['service_name']\n    df.pod = df.ctx['pod_name']\n    df.namespace = df.ctx['namespace']\n    agg = df.groupby(['service', 'pod', 'namespace']).agg()\n    pod_count = agg.groupby(['namespace', 'pod']).agg()\n    pod_count = pod_count.groupby('namespace').agg(pod_count=('pod', px.count))\n    svc_count = agg.groupby(['namespace', 'service']).agg()\n    svc_count = svc_count.groupby('namespace').agg(service_count=('service', px.count))\n    pod_and_svc_count = pod_count.merge(svc_count, how='inner',\n                                        left_on='namespace', right_on='namespace',\n                                        suffixes=['', '_x'])\n    process_stats = process_stats_by_entity(start_time, 'namespace')\n    output = process_stats.merge(pod_and_svc_count, how='inner', left_on='namespace',\n                                 right_on='namespace', suffixes=['', '_y'])\n    return output[['namespace', 'pod_count', 'service_count', 'avg_vsize', 'avg_rss']]\n\n\ndef services_for_cluster(start_time: int):\n    ''' Get an overview of the services in the current cluster.\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    df = px.DataFrame(table='process_stats', start_time=start_time)\n    df.service = df.ctx['service']\n    df = df[df.service != '']\n    df.pod = df.ctx['pod']\n    df = df.groupby(['service', 'pod']).agg()\n    df = df.groupby('service').agg(pod_count=('pod', px.count))\n    service_let = service_let_summary(start_time)\n    joined = df.merge(service_let, how='left', left_on='service', right_on='service',\n                      suffixes=['', '_x'])\n    return joined.drop('service_x')\n\n\ndef http_stats(start_time: int):\n    ''' Get a dataframe of HTTP events.\n    Certain traffic (like health checks) are auto removed, and some standard fields are added.\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    df = px.DataFrame(table='http_events', start_time=start_time)\n\n    # Add K8s metadata.\n    df.service = df.ctx['service']\n    df.pod = df.ctx['pod']\n\n    # Filter out non-k8s entities.\n    df = df[df.pod != '']\n\n    # Additional HTTP fields, pre-computed for convenience.\n    df.failure = df.resp_status >= 400\n\n    # Remove health checks, and anything with no remote address.\n    health_check_req = ((df.req_path == '/healthz' or df.req_path == '/readyz') or df.req_path == '/livez')\n    filter_out_conds = (health_check_req and filter_health_checks) or (df['remote_addr'] == '-')\n    df = df[not filter_out_conds]\n\n    return df\n\n\ndef http_stats_by_service(start_time: int):\n    ''' Get a data frame of HTTP stats per service. The HTTP stats are for inbound traffic,\n    and includes HTTP request count, error count and latency quantiles.\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    df = http_stats(start_time)\n\n    # Filter only to inbound service traffic (server-side).\n    # Don't include traffic initiated by this service to an external location.\n    df = df[df.trace_role == 2]\n\n    # Compute HTTP metrics.\n    df = df.groupby(['service']).agg(\n        http_req_count_in=('latency', px.count),\n        http_error_count_in=('failure', px.sum),\n        http_latency_in=('latency', px.quantiles)\n    )\n\n    return df\n\n\ndef conn_stats(start_time: int):\n    ''' Get a dataframe of connection stats.\n    For each client-server pair, the resulting data frame has the bytes sent and received.\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    df = px.DataFrame(table='conn_stats', start_time=start_time)\n\n    df.pod = df.ctx['pod']\n    df.service = df.ctx['service']\n\n    df = df[df.service != '']\n\n    # Find min/max bytes transferred over the selected time window per pod.\n    df = df.groupby(['upid', 'remote_addr', 'remote_port', 'pod', 'service', 'trace_role']).agg(\n        bytes_recv_min=('bytes_recv', px.min),\n        bytes_recv_max=('bytes_recv', px.max),\n        bytes_sent_min=('bytes_sent', px.min),\n        bytes_sent_max=('bytes_sent', px.max),\n    )\n\n    # Calculate bytes transferred over the time window\n    df.bytes_sent = df.bytes_sent_max - df.bytes_sent_min\n    df.bytes_recv = df.bytes_recv_max - df.bytes_recv_min\n    df = df.drop(['bytes_recv_min', 'bytes_recv_max', 'bytes_sent_min', 'bytes_sent_max'])\n\n    return df\n\n\ndef conn_stats_by_service(start_time: int):\n    ''' Get a dataframe of connection stats aggregated by service.\n    For each service, the resulting data frame contains rx/tx stats for server-side and client-side connections.\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    df = conn_stats(start_time)\n\n    # Group by service and trace role.\n    # Do this after computing bytes sent/received by conn_stats key ({upid, remote_addr, remote_port}).\n    # Keeping trace_role allows us to see which traffic was part of server duties vs client duties.\n    df = df.groupby(['service', 'trace_role']).agg(\n        bytes_recv=('bytes_recv', px.sum),\n        bytes_sent=('bytes_sent', px.sum),\n    )\n\n    # Get RX/TX stats for the server side connections.\n    server_df = df[df.trace_role == 2]\n    server_df.rx_server = server_df.bytes_recv\n    server_df.tx_server = server_df.bytes_sent\n    server_df = server_df[['service', 'rx_server', 'tx_server']]\n\n    # Get RX/TX stats for the client side connections.\n    client_df = df[df.trace_role == 1]\n    client_df.rx_client = client_df.bytes_recv\n    client_df.tx_client = client_df.bytes_sent\n    client_df = client_df[['service', 'rx_client', 'tx_client']]\n\n    # Create a dataframe that contains both server-side and client-side RX/TX stats.\n    df = server_df.merge(client_df,\n                         how='left',\n                         left_on='service',\n                         right_on='service',\n                         suffixes=['', '_x'])\n    df = df['service', 'rx_server', 'tx_server', 'rx_client', 'tx_client']\n\n    return df\n\n\ndef service_let_summary(start_time: int):\n    ''' Compute a summary of traffic by requesting service, for requests\n        on services in the current cluster..\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    conn_stats_df = conn_stats_by_service(start_time)\n    http_stats_df = http_stats_by_service(start_time)\n\n    # Merge conn_stats_df and http_stats_df.\n    df = conn_stats_df.merge(http_stats_df,\n                             how='left',\n                             left_on='service',\n                             right_on='service',\n                             suffixes=['', '_x'])\n\n    # Compute time window for the query and add it as a column.\n    df = add_time_window_column(df, start_time)\n\n    # Compute throughput values.\n    df.http_req_throughput_in = df.http_req_count_in / df.window\n    df.http_error_rate_in = px.Percent(\n        px.select(df.http_req_count_in != 0, df.http_error_count_in / df.http_req_count_in, 0.0))\n    df.inbound_conns = (df.rx_server + df.tx_server) / df.window\n    df.outbound_conns = (df.tx_client + df.rx_client) / df.window\n\n    return df[['service', 'http_latency_in', 'http_req_throughput_in', 'http_error_rate_in',\n               'inbound_conns', 'outbound_conns']]\n\n\ndef service_let_graph(start_time: int):\n    ''' Compute a summary of traffic by requesting service, for requests on services\n        in the current cluster. Similar to `service_let_summary` but also breaks down\n        by pod in addition to service.\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    df = http_stats(start_time)\n    df = df.groupby(['service', 'remote_addr', 'pod', 'trace_role']).agg(\n        http_req_count_in=('latency', px.count),\n        http_error_count_in=('failure', px.sum),\n        latency_quantiles=('latency', px.quantiles),\n        inbound_bytes_total=('req_body_size', px.sum),\n        outbound_bytes_total=('resp_body_size', px.sum)\n    )\n\n    # Get the traced and remote pod/service/IP information.\n    df.traced_pod = df.pod\n    df.traced_service = df.service\n    df.traced_ip = px.pod_name_to_pod_ip(df.pod)\n    df.remote_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n    df.remote_service = px.service_id_to_service_name(px.ip_to_service_id(df.remote_addr))\n    df.remote_ip = df.remote_addr\n    # If external IPs are excluded in the service graph, then we also exclude any\n    # traffic where we don't know the remote pod or remote service name.\n    df = df[include_ips or (df.remote_pod != '' or df.remote_service != '')]\n\n    # Associate it with Client/Server roles, based on the trace role.\n    df.is_server_side_tracing = df.trace_role == 2\n    df.responder_pod = px.select(df.is_server_side_tracing, df.traced_pod, df.remote_pod)\n    df.requestor_pod = px.select(df.is_server_side_tracing, df.remote_pod, df.traced_pod)\n    df.responder_service = px.select(df.is_server_side_tracing, df.traced_service, df.remote_service)\n    df.requestor_service = px.select(df.is_server_side_tracing, df.remote_service, df.traced_service)\n    df.responder_ip = px.select(df.is_server_side_tracing, df.traced_ip, df.remote_ip)\n    df.requestor_ip = px.select(df.is_server_side_tracing, df.remote_ip, df.traced_ip)\n\n    # Compute statistics about each edge of the service graph.\n    df.latency_p50 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p50')))\n    df.latency_p90 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p90')))\n    df.latency_p99 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p99')))\n    df = add_time_window_column(df, start_time)\n    df.request_throughput = df.http_req_count_in / df.window\n    df.inbound_throughput = df.inbound_bytes_total / df.window\n    df.outbound_throughput = df.outbound_bytes_total / df.window\n    df.error_rate = px.Percent(df.http_error_count_in / df.http_req_count_in)\n\n    df = df.groupby(['responder_pod', 'requestor_pod', 'responder_service',\n                     'requestor_service', 'responder_ip', 'requestor_ip']).agg(\n        latency_p50=('latency_p50', px.mean),\n        latency_p90=('latency_p90', px.mean),\n        latency_p99=('latency_p99', px.mean),\n        request_throughput=('request_throughput', px.mean),\n        error_rate=('error_rate', px.mean),\n        inbound_throughput=('inbound_throughput', px.mean),\n        outbound_throughput=('outbound_throughput', px.mean),\n        throughput_total=('http_req_count_in', px.sum)\n    )\n\n    return df\n    \ndef graphnode_sources(start_time: int):\n    df = service_let_graph(start_time)\n    # Use Pod name for source node id and title. If pod name is not available,\n    # use service name or IP address.\n    df.source_svc_ip = px.select(df.requestor_service != '', df.requestor_service, df.requestor_ip)\n    df.id = px.select(df.requestor_pod != '', df.requestor_pod, df.source_svc_ip)\n    df.title = df.id\n    df.arc__success = 0.1\n    df.arc__failure = 0.9\n    df = df.groupby(['id', 'title']).agg()\n    \n    return df\n\n\ndef graphnode_targets(start_time: int):\n    df = service_let_graph(start_time)\n    # Use Pod name for target node id and title. If pod name is not available,\n    # use service name or IP address.\n    df.target_svc_ip = px.select(df.responder_service != '', df.responder_service, df.responder_ip)\n    df.id = px.select(df.responder_pod != '', df.responder_pod, df.target_svc_ip)\n    df.title = df.id\n    df = df.groupby(['id', 'title']).agg()\n    \n    return df\n\n\ndef nodes(start_time: int):\n    node_sources = graphnode_sources(start_time)\n    node_targets = graphnode_targets(start_time)\n    df = node_sources.append(node_targets)\n    return df\n\n\ndef edges(start_time: int):\n    df = service_let_graph(start_time)\n    df.source_svc_ip = px.select(df.requestor_service != '', df.requestor_service, df.requestor_ip)\n    df.source = px.select(df.requestor_pod != '', df.requestor_pod, df.source_svc_ip)\n    df.target_svc_ip = px.select(df.responder_service != '', df.responder_service, df.responder_ip)\n    df.target = px.select(df.responder_pod != '', df.responder_pod, df.target_svc_ip)\n    df.id = df.source + '-' + df.target\n    df.mainStat = df.error_rate * 100\n    df.secondaryStat = df.latency_p90 / ns_per_ms\n    return df[['id', 'source', 'target', 'mainStat', 'secondaryStat']]\n\nstart_time = __time_from\n\nnodes_table = nodes(start_time)\nedges_table = edges(start_time)\npx.display(nodes_table, \"nodes\")\npx.display(edges_table, \"edges\")"
          },
          "queryType": "run-script",
          "refId": "A"
        }
      ],
      "title": "HTTP Service Map",
      "type": "nodeGraph"
    },
    {
      "datasource": "${DS_PIXIE_GRAFANA DATASOURCE PLUGIN}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "displayMode": "auto",
            "inspect": false
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "node"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 402
              },
              {
                "id": "links",
                "value": [
                  {
                    "title": "",
                    "url": "/d/tcpoz_jnz/px-node?orgId=1&${pixieCluster:queryparam}&var-node=${__data.fields.node}&var-groupby=node"
                  }
                ]
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "cpu_usage"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 129
              },
              {
                "id": "unit",
                "value": "percentunit"
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 9,
        "w": 9,
        "x": 0,
        "y": 14
      },
      "id": 4,
      "options": {
        "footer": {
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "7.5.1",
      "targets": [
        {
          "datasource": "${DS_PIXIE_GRAFANA DATASOURCE PLUGIN}",
          "queryBody": {
            "pxlScript": "import px \n\n# $pixieCluster - work around for grafana to update panel on variable change\n\ndef process_stats_by_entity(start_time: int, entity: str):\n    ''' Gets the windowed process stats (CPU, memory, etc) per node or pod.\n    Args:\n    @start_time Starting time of the data to examine.\n    @entity: Either pod or node_name.\n    '''\n    # Window size to use on time_ column for bucketing.\n    ns_per_s = 1000 * 1000 * 1000\n    window_ns = px.DurationNanos(10 * ns_per_s)\n\n    df = px.DataFrame(table='process_stats', start_time=start_time)\n    df[entity] = df.ctx[entity]\n    df.timestamp = px.bin(df.time_, window_ns)\n    # First calculate CPU usage by process (UPID) in each k8s_object\n    # over all windows.\n    df = df.groupby([entity, 'upid', 'timestamp']).agg(\n        rss=('rss_bytes', px.mean),\n        vsize=('vsize_bytes', px.mean),\n        # The fields below are counters, so we take the min and the max to subtract them.\n        cpu_utime_ns_max=('cpu_utime_ns', px.max),\n        cpu_utime_ns_min=('cpu_utime_ns', px.min),\n        cpu_ktime_ns_max=('cpu_ktime_ns', px.max),\n        cpu_ktime_ns_min=('cpu_ktime_ns', px.min),\n        read_bytes_max=('read_bytes', px.max),\n        read_bytes_min=('read_bytes', px.min),\n        write_bytes_max=('write_bytes', px.max),\n        write_bytes_min=('write_bytes', px.min),\n        rchar_bytes_max=('rchar_bytes', px.max),\n        rchar_bytes_min=('rchar_bytes', px.min),\n        wchar_bytes_max=('wchar_bytes', px.max),\n        wchar_bytes_min=('wchar_bytes', px.min),\n    )\n    # Next calculate cpu usage and memory stats per window.\n    df.cpu_utime_ns = df.cpu_utime_ns_max - df.cpu_utime_ns_min\n    df.cpu_ktime_ns = df.cpu_ktime_ns_max - df.cpu_ktime_ns_min\n    df.read_bytes = df.read_bytes_max - df.read_bytes_min\n    df.write_bytes = df.write_bytes_max - df.write_bytes_min\n    df.rchar_bytes = df.rchar_bytes_max - df.rchar_bytes_min\n    df.wchar_bytes = df.wchar_bytes_max - df.wchar_bytes_min\n    # Sum by UPID.\n    df = df.groupby([entity, 'timestamp']).agg(\n        cpu_ktime_ns=('cpu_ktime_ns', px.sum),\n        cpu_utime_ns=('cpu_utime_ns', px.sum),\n        read_bytes=('read_bytes', px.sum),\n        write_bytes=('write_bytes', px.sum),\n        rchar_bytes=('rchar_bytes', px.sum),\n        wchar_bytes=('wchar_bytes', px.sum),\n        rss=('rss', px.sum),\n        vsize=('vsize', px.sum),\n    )\n    df.actual_disk_read_throughput = df.read_bytes / window_ns\n    df.actual_disk_write_throughput = df.write_bytes / window_ns\n    df.total_disk_read_throughput = df.rchar_bytes / window_ns\n    df.total_disk_write_throughput = df.wchar_bytes / window_ns\n    # Now take the mean value over the various timestamps.\n    df = df.groupby(entity).agg(\n        cpu_ktime_ns=('cpu_ktime_ns', px.mean),\n        cpu_utime_ns=('cpu_utime_ns', px.mean),\n        actual_disk_read_throughput=('actual_disk_read_throughput', px.mean),\n        actual_disk_write_throughput=('actual_disk_write_throughput', px.mean),\n        total_disk_read_throughput=('total_disk_read_throughput', px.mean),\n        total_disk_write_throughput=('total_disk_write_throughput', px.mean),\n        avg_rss=('rss', px.mean),\n        avg_vsize=('vsize', px.mean),\n    )\n    # Finally, calculate total (kernel + user time)  percentage used over window.\n    df.cpu_usage = px.Percent((df.cpu_ktime_ns + df.cpu_utime_ns) / window_ns)\n    return df.drop(['cpu_ktime_ns', 'cpu_utime_ns'])\n\ndef nodes_for_cluster(start_time: int):\n    ''' Gets a list of nodes in the current cluster since `start_time`.\n    Args:\n    @start_time Start time of the data to examine.\n    '''\n    df = px.DataFrame(table='process_stats', start_time=start_time)\n    df.node = df.ctx['node_name']\n    df.pod = df.ctx['pod_name']\n    agg = df.groupby(['node', 'pod']).agg()\n    nodes = agg.groupby('node').agg(pod_count=('pod', px.count))\n    process_stats = process_stats_by_entity(start_time, 'node')\n    output = process_stats.merge(nodes, how='inner', left_on='node', right_on='node',\n                                 suffixes=['', '_x'])\n                                 \n    return output[['node', 'cpu_usage', 'pod_count']]\n\noutput = nodes_for_cluster(__time_from)\noutput = output[['node', 'cpu_usage', 'pod_count']]\npx.display(output)"
          },
          "queryType": "run-script",
          "refId": "A"
        }
      ],
      "title": "Nodes",
      "type": "table"
    },
    {
      "datasource": "${DS_PIXIE_GRAFANA DATASOURCE PLUGIN}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "displayMode": "auto",
            "inspect": false
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "namespace"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 188
              },
              {
                "id": "links",
                "value": [
                  {
                    "title": "",
                    "url": "/d/F6XZR_j7k/px-namespace?orgId=1&${pixieCluster:queryparam}&var-namespace=${__data.fields.namespace}"
                  }
                ]
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "avg_vsize"
            },
            "properties": [
              {
                "id": "unit",
                "value": "decmbytes"
              },
              {
                "id": "decimals",
                "value": 1
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "avg_rss"
            },
            "properties": [
              {
                "id": "unit",
                "value": "decmbytes"
              },
              {
                "id": "decimals",
                "value": 1
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 9,
        "w": 15,
        "x": 9,
        "y": 14
      },
      "id": 6,
      "options": {
        "footer": {
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "showHeader": true,
        "sortBy": [
          {
            "desc": true,
            "displayName": "avg_vsize_mb"
          }
        ]
      },
      "pluginVersion": "7.5.1",
      "targets": [
        {
          "datasource": "${DS_PIXIE_GRAFANA DATASOURCE PLUGIN}",
          "queryBody": {
            "pxlScript": "# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Cluster Overview\n\nThis view lists the namespaces and the nodes that are available on the current cluster.\n\n'''\nimport px\n\n\n# Flag to filter out health checks from the data.\nfilter_health_checks = True\n\n# Whether or not to include traffic from IPs that don't resolve to a known pod/service.\ninclude_ips = True\n\n# $pixieCluster - work around for grafana to update panel on variable change\n\n# Hack to get the time window for the script.\n# TODO(philkuz): Replace this with a built-in.\ndef get_time_window(start_time: int):\n    ''' Converts the start_time string into a table with a single column and single row.\n    The approach is hacky, and will round to roughly 1 second.\n    '''\n    df = px.DataFrame('process_stats', start_time=start_time)\n\n    df = df.agg(\n        time_min=('time_', px.min),\n        time_max=('time_', px.max),\n    )\n\n    df.window = px.DurationNanos(df.time_max - df.time_min)\n    df = df[['window']]\n\n    return df\n\n\ndef add_time_window_column(df, start_time):\n    tw = get_time_window(start_time)\n    df = df.merge(tw, how='inner', left_on=[], right_on=[])\n    return df\n\n\ndef process_stats_by_entity(start_time: int, entity: str):\n    ''' Gets the windowed process stats (CPU, memory, etc) per node or pod.\n    Args:\n    @start_time Starting time of the data to examine.\n    @entity: Either pod or node_name.\n    '''\n    # Window size to use on time_ column for bucketing.\n    ns_per_s = 1000 * 1000 * 1000\n    window_ns = px.DurationNanos(10 * ns_per_s)\n\n    df = px.DataFrame(table='process_stats', start_time=start_time)\n    df[entity] = df.ctx[entity]\n    df.timestamp = px.bin(df.time_, window_ns)\n    # First calculate CPU usage by process (UPID) in each k8s_object\n    # over all windows.\n    df = df.groupby([entity, 'upid', 'timestamp']).agg(\n        rss=('rss_bytes', px.mean),\n        vsize=('vsize_bytes', px.mean),\n        # The fields below are counters, so we take the min and the max to subtract them.\n        cpu_utime_ns_max=('cpu_utime_ns', px.max),\n        cpu_utime_ns_min=('cpu_utime_ns', px.min),\n        cpu_ktime_ns_max=('cpu_ktime_ns', px.max),\n        cpu_ktime_ns_min=('cpu_ktime_ns', px.min),\n        read_bytes_max=('read_bytes', px.max),\n        read_bytes_min=('read_bytes', px.min),\n        write_bytes_max=('write_bytes', px.max),\n        write_bytes_min=('write_bytes', px.min),\n        rchar_bytes_max=('rchar_bytes', px.max),\n        rchar_bytes_min=('rchar_bytes', px.min),\n        wchar_bytes_max=('wchar_bytes', px.max),\n        wchar_bytes_min=('wchar_bytes', px.min),\n    )\n    # Next calculate cpu usage and memory stats per window.\n    df.cpu_utime_ns = df.cpu_utime_ns_max - df.cpu_utime_ns_min\n    df.cpu_ktime_ns = df.cpu_ktime_ns_max - df.cpu_ktime_ns_min\n    df.read_bytes = df.read_bytes_max - df.read_bytes_min\n    df.write_bytes = df.write_bytes_max - df.write_bytes_min\n    df.rchar_bytes = df.rchar_bytes_max - df.rchar_bytes_min\n    df.wchar_bytes = df.wchar_bytes_max - df.wchar_bytes_min\n    # Sum by UPID.\n    df = df.groupby([entity, 'timestamp']).agg(\n        cpu_ktime_ns=('cpu_ktime_ns', px.sum),\n        cpu_utime_ns=('cpu_utime_ns', px.sum),\n        read_bytes=('read_bytes', px.sum),\n        write_bytes=('write_bytes', px.sum),\n        rchar_bytes=('rchar_bytes', px.sum),\n        wchar_bytes=('wchar_bytes', px.sum),\n        rss=('rss', px.sum),\n        vsize=('vsize', px.sum),\n    )\n    df.actual_disk_read_throughput = df.read_bytes / window_ns\n    df.actual_disk_write_throughput = df.write_bytes / window_ns\n    df.total_disk_read_throughput = df.rchar_bytes / window_ns\n    df.total_disk_write_throughput = df.wchar_bytes / window_ns\n    # Now take the mean value over the various timestamps.\n    df = df.groupby(entity).agg(\n        cpu_ktime_ns=('cpu_ktime_ns', px.mean),\n        cpu_utime_ns=('cpu_utime_ns', px.mean),\n        actual_disk_read_throughput=('actual_disk_read_throughput', px.mean),\n        actual_disk_write_throughput=('actual_disk_write_throughput', px.mean),\n        total_disk_read_throughput=('total_disk_read_throughput', px.mean),\n        total_disk_write_throughput=('total_disk_write_throughput', px.mean),\n        avg_rss=('rss', px.mean),\n        avg_vsize=('vsize', px.mean),\n    )\n    # Finally, calculate total (kernel + user time)  percentage used over window.\n    df.cpu_usage = px.Percent((df.cpu_ktime_ns + df.cpu_utime_ns) / window_ns)\n    return df.drop(['cpu_ktime_ns', 'cpu_utime_ns'])\n\n\ndef nodes_for_cluster(start_time: int):\n    ''' Gets a list of nodes in the current cluster since `start_time`.\n    Args:\n    @start_time Start time of the data to examine.\n    '''\n    df = px.DataFrame(table='process_stats', start_time=start_time)\n    df.node = df.ctx['node_name']\n    df.pod = df.ctx['pod_name']\n    agg = df.groupby(['node', 'pod']).agg()\n    nodes = agg.groupby('node').agg(pod_count=('pod', px.count))\n    process_stats = process_stats_by_entity(start_time, 'node')\n    output = process_stats.merge(nodes, how='inner', left_on='node', right_on='node',\n                                 suffixes=['', '_x'])\n    return output[['node', 'cpu_usage', 'pod_count']]\n\n\ndef pods_for_cluster(start_time: int):\n    ''' A list of pods in `namespace`.\n    Args:\n    @start_time: The timestamp of data to start at.\n    @namespace: The name of the namespace to filter on.\n    '''\n    df = px.DataFrame(table='process_stats', start_time=start_time)\n    df.pod = df.ctx['pod_name']\n    df.node = df.ctx['node_name']\n    df.container = df.ctx['container_name']\n    df = df.groupby(['pod', 'node', 'container']).agg()\n    df = df.groupby(['pod', 'node']).agg(container_count=('container', px.count))\n    df.start_time = px.pod_name_to_start_time(df.pod)\n    df.status = px.pod_name_to_status(df.pod)\n    process_stats = process_stats_by_entity(start_time, 'pod')\n    output = process_stats.merge(df, how='inner', left_on='pod', right_on='pod',\n                                 suffixes=['', '_x'])\n    return output[['pod', 'cpu_usage', 'total_disk_read_throughput',\n                   'total_disk_write_throughput', 'container_count',\n                   'node', 'start_time', 'status']]\n\n\ndef namespaces_for_cluster(start_time: int):\n    ''' Gets a overview of namespaces in the current cluster since `start_time`.\n    Args:\n    @start_time Start time of the data to examine.\n    '''\n    df = px.DataFrame(table='process_stats', start_time=start_time)\n    df.service = df.ctx['service_name']\n    df.pod = df.ctx['pod_name']\n    df.namespace = df.ctx['namespace']\n    agg = df.groupby(['service', 'pod', 'namespace']).agg()\n    pod_count = agg.groupby(['namespace', 'pod']).agg()\n    pod_count = pod_count.groupby('namespace').agg(pod_count=('pod', px.count))\n    svc_count = agg.groupby(['namespace', 'service']).agg()\n    svc_count = svc_count.groupby('namespace').agg(service_count=('service', px.count))\n    pod_and_svc_count = pod_count.merge(svc_count, how='inner',\n                                        left_on='namespace', right_on='namespace',\n                                        suffixes=['', '_x'])\n    process_stats = process_stats_by_entity(start_time, 'namespace')\n    output = process_stats.merge(pod_and_svc_count, how='inner', left_on='namespace',\n                                 right_on='namespace', suffixes=['', '_y'])\n    return output[['namespace', 'pod_count', 'service_count', 'avg_vsize', 'avg_rss']]\n\n\ndef services_for_cluster(start_time: int):\n    ''' Get an overview of the services in the current cluster.\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    df = px.DataFrame(table='process_stats', start_time=start_time)\n    df.service = df.ctx['service']\n    df = df[df.service != '']\n    df.pod = df.ctx['pod']\n    df = df.groupby(['service', 'pod']).agg()\n    df = df.groupby('service').agg(pod_count=('pod', px.count))\n    service_let = service_let_summary(start_time)\n    joined = df.merge(service_let, how='left', left_on='service', right_on='service',\n                      suffixes=['', '_x'])\n    return joined.drop('service_x')\n\n\ndef http_stats(start_time: int):\n    ''' Get a dataframe of HTTP events.\n    Certain traffic (like health checks) are auto removed, and some standard fields are added.\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    df = px.DataFrame(table='http_events', start_time=start_time)\n\n    # Add K8s metadata.\n    df.service = df.ctx['service']\n    df.pod = df.ctx['pod']\n\n    # Filter out non-k8s entities.\n    df = df[df.pod != '']\n\n    # Additional HTTP fields, pre-computed for convenience.\n    df.failure = df.resp_status >= 400\n\n    # Remove health checks, and anything with no remote address.\n    health_check_req = ((df.req_path == '/healthz' or df.req_path == '/readyz') or df.req_path == '/livez')\n    filter_out_conds = (health_check_req and filter_health_checks) or (df['remote_addr'] == '-')\n    df = df[not filter_out_conds]\n\n    return df\n\n\ndef http_stats_by_service(start_time: int):\n    ''' Get a data frame of HTTP stats per service. The HTTP stats are for inbound traffic,\n    and includes HTTP request count, error count and latency quantiles.\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    df = http_stats(start_time)\n\n    # Filter only to inbound service traffic (server-side).\n    # Don't include traffic initiated by this service to an external location.\n    df = df[df.trace_role == 2]\n\n    # Compute HTTP metrics.\n    df = df.groupby(['service']).agg(\n        http_req_count_in=('latency', px.count),\n        http_error_count_in=('failure', px.sum),\n        http_latency_in=('latency', px.quantiles)\n    )\n\n    return df\n\n\ndef conn_stats(start_time: int):\n    ''' Get a dataframe of connection stats.\n    For each client-server pair, the resulting data frame has the bytes sent and received.\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    df = px.DataFrame(table='conn_stats', start_time=start_time)\n\n    df.pod = df.ctx['pod']\n    df.service = df.ctx['service']\n\n    df = df[df.service != '']\n\n    # Find min/max bytes transferred over the selected time window per pod.\n    df = df.groupby(['upid', 'remote_addr', 'remote_port', 'pod', 'service', 'trace_role']).agg(\n        bytes_recv_min=('bytes_recv', px.min),\n        bytes_recv_max=('bytes_recv', px.max),\n        bytes_sent_min=('bytes_sent', px.min),\n        bytes_sent_max=('bytes_sent', px.max),\n    )\n\n    # Calculate bytes transferred over the time window\n    df.bytes_sent = df.bytes_sent_max - df.bytes_sent_min\n    df.bytes_recv = df.bytes_recv_max - df.bytes_recv_min\n    df = df.drop(['bytes_recv_min', 'bytes_recv_max', 'bytes_sent_min', 'bytes_sent_max'])\n\n    return df\n\n\ndef conn_stats_by_service(start_time: int):\n    ''' Get a dataframe of connection stats aggregated by service.\n    For each service, the resulting data frame contains rx/tx stats for server-side and client-side connections.\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    df = conn_stats(start_time)\n\n    # Group by service and trace role.\n    # Do this after computing bytes sent/received by conn_stats key ({upid, remote_addr, remote_port}).\n    # Keeping trace_role allows us to see which traffic was part of server duties vs client duties.\n    df = df.groupby(['service', 'trace_role']).agg(\n        bytes_recv=('bytes_recv', px.sum),\n        bytes_sent=('bytes_sent', px.sum),\n    )\n\n    # Get RX/TX stats for the server side connections.\n    server_df = df[df.trace_role == 2]\n    server_df.rx_server = server_df.bytes_recv\n    server_df.tx_server = server_df.bytes_sent\n    server_df = server_df[['service', 'rx_server', 'tx_server']]\n\n    # Get RX/TX stats for the client side connections.\n    client_df = df[df.trace_role == 1]\n    client_df.rx_client = client_df.bytes_recv\n    client_df.tx_client = client_df.bytes_sent\n    client_df = client_df[['service', 'rx_client', 'tx_client']]\n\n    # Create a dataframe that contains both server-side and client-side RX/TX stats.\n    df = server_df.merge(client_df,\n                         how='left',\n                         left_on='service',\n                         right_on='service',\n                         suffixes=['', '_x'])\n    df = df['service', 'rx_server', 'tx_server', 'rx_client', 'tx_client']\n\n    return df\n\n\ndef service_let_summary(start_time: int):\n    ''' Compute a summary of traffic by requesting service, for requests\n        on services in the current cluster..\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    conn_stats_df = conn_stats_by_service(start_time)\n    http_stats_df = http_stats_by_service(start_time)\n\n    # Merge conn_stats_df and http_stats_df.\n    df = conn_stats_df.merge(http_stats_df,\n                             how='left',\n                             left_on='service',\n                             right_on='service',\n                             suffixes=['', '_x'])\n\n    # Compute time window for the query and add it as a column.\n    df = add_time_window_column(df, start_time)\n\n    # Compute throughput values.\n    df.http_req_throughput_in = df.http_req_count_in / df.window\n    df.http_error_rate_in = px.Percent(\n        px.select(df.http_req_count_in != 0, df.http_error_count_in / df.http_req_count_in, 0.0))\n    df.inbound_conns = (df.rx_server + df.tx_server) / df.window\n    df.outbound_conns = (df.tx_client + df.rx_client) / df.window\n\n    return df[['service', 'http_latency_in', 'http_req_throughput_in', 'http_error_rate_in',\n               'inbound_conns', 'outbound_conns']]\n\n\ndef service_let_graph(start_time: int):\n    ''' Compute a summary of traffic by requesting service, for requests on services\n        in the current cluster. Similar to `service_let_summary` but also breaks down\n        by pod in addition to service.\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    df = http_stats(start_time)\n    df = df.groupby(['service', 'remote_addr', 'pod', 'trace_role']).agg(\n        http_req_count_in=('latency', px.count),\n        http_error_count_in=('failure', px.sum),\n        latency_quantiles=('latency', px.quantiles),\n        inbound_bytes_total=('req_body_size', px.sum),\n        outbound_bytes_total=('resp_body_size', px.sum)\n    )\n\n    # Get the traced and remote pod/service/IP information.\n    df.traced_pod = df.pod\n    df.traced_service = df.service\n    df.traced_ip = px.pod_name_to_pod_ip(df.pod)\n    df.remote_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n    df.remote_service = px.service_id_to_service_name(px.ip_to_service_id(df.remote_addr))\n    df.remote_ip = df.remote_addr\n    # If external IPs are excluded in the service graph, then we also exclude any\n    # traffic where we don't know the remote pod or remote service name.\n    df = df[include_ips or (df.remote_pod != '' or df.remote_service != '')]\n\n    # Associate it with Client/Server roles, based on the trace role.\n    df.is_server_side_tracing = df.trace_role == 2\n    df.responder_pod = px.select(df.is_server_side_tracing, df.traced_pod, df.remote_pod)\n    df.requestor_pod = px.select(df.is_server_side_tracing, df.remote_pod, df.traced_pod)\n    df.responder_service = px.select(df.is_server_side_tracing, df.traced_service, df.remote_service)\n    df.requestor_service = px.select(df.is_server_side_tracing, df.remote_service, df.traced_service)\n    df.responder_ip = px.select(df.is_server_side_tracing, df.traced_ip, df.remote_ip)\n    df.requestor_ip = px.select(df.is_server_side_tracing, df.remote_ip, df.traced_ip)\n\n    # Compute statistics about each edge of the service graph.\n    df.latency_p50 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p50')))\n    df.latency_p90 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p90')))\n    df.latency_p99 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p99')))\n    df = add_time_window_column(df, start_time)\n    df.request_throughput = df.http_req_count_in / df.window\n    df.inbound_throughput = df.inbound_bytes_total / df.window\n    df.outbound_throughput = df.outbound_bytes_total / df.window\n    df.error_rate = px.Percent(df.http_error_count_in / df.http_req_count_in)\n\n    df = df.groupby(['responder_pod', 'requestor_pod', 'responder_service',\n                     'requestor_service', 'responder_ip', 'requestor_ip']).agg(\n        latency_p50=('latency_p50', px.mean),\n        latency_p90=('latency_p90', px.mean),\n        latency_p99=('latency_p99', px.mean),\n        request_throughput=('request_throughput', px.mean),\n        error_rate=('error_rate', px.mean),\n        inbound_throughput=('inbound_throughput', px.mean),\n        outbound_throughput=('outbound_throughput', px.mean),\n        throughput_total=('http_req_count_in', px.sum)\n    )\n\n    return df\n\noutput = namespaces_for_cluster(__time_from)\noutput.avg_vsize = output.avg_vsize / px.pow(2,20)\noutput.avg_rss = output.avg_rss / px.pow(2,20)\noutput = output[['namespace', 'pod_count', 'service_count', 'avg_vsize', 'avg_rss']]\npx.display(output)"
          },
          "queryType": "run-script",
          "refId": "A"
        }
      ],
      "title": "Namespaces",
      "type": "table"
    },
    {
      "datasource": "${DS_PIXIE_GRAFANA DATASOURCE PLUGIN}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "displayMode": "auto",
            "inspect": false
          },
          "decimals": 1,
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "service"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 296
              },
              {
                "id": "links",
                "value": [
                  {
                    "title": "",
                    "url": "/d/qhHtFyCnk/px-service?orgId=1&${pixieCluster:queryparam}&var-pixieService=${__data.fields.service}"
                  }
                ]
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "http_latency_in_p99"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 271
              },
              {
                "id": "unit",
                "value": "ms"
              },
              {
                "id": "custom.displayMode",
                "value": "color-text"
              },
              {
                "id": "thresholds",
                "value": {
                  "mode": "absolute",
                  "steps": [
                    {
                      "color": "green"
                    },
                    {
                      "color": "light-orange",
                      "value": 150
                    },
                    {
                      "color": "red",
                      "value": 300
                    }
                  ]
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "http_req_throughput_in"
            },
            "properties": [
              {
                "id": "unit",
                "value": "/s"
              },
              {
                "id": "custom.width",
                "value": 299
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "http_error_rate_in"
            },
            "properties": [
              {
                "id": "unit",
                "value": "percentunit"
              },
              {
                "id": "decimals",
                "value": 1
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "inbound_conns"
            },
            "properties": [
              {
                "id": "unit",
                "value": "KBs"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "outbound_conns"
            },
            "properties": [
              {
                "id": "unit",
                "value": "KBs"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "pod_count"
            },
            "properties": [
              {
                "id": "decimals",
                "value": 0
              },
              {
                "id": "custom.width",
                "value": 264
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 9,
        "w": 24,
        "x": 0,
        "y": 23
      },
      "id": 8,
      "options": {
        "footer": {
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "showHeader": true,
        "sortBy": [
          {
            "desc": true,
            "displayName": "http_req_throughput_in"
          }
        ]
      },
      "pluginVersion": "7.5.1",
      "targets": [
        {
          "datasource": "${DS_PIXIE_GRAFANA DATASOURCE PLUGIN}",
          "queryBody": {
            "pxlScript": "# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Cluster Overview\n\nThis view lists the namespaces and the nodes that are available on the current cluster.\n\n'''\nimport px\n\n\n# Flag to filter out health checks from the data.\nfilter_health_checks = True\n\n# Whether or not to include traffic from IPs that don't resolve to a known pod/service.\ninclude_ips = True\n\n# $pixieCluster - work around for grafana to update panel on variable change\n\n# Hack to get the time window for the script.\n# TODO(philkuz): Replace this with a built-in.\ndef get_time_window(start_time: int):\n    ''' Converts the start_time string into a table with a single column and single row.\n    The approach is hacky, and will round to roughly 1 second.\n    '''\n    df = px.DataFrame('process_stats', start_time=start_time)\n\n    df = df.agg(\n        time_min=('time_', px.min),\n        time_max=('time_', px.max),\n    )\n\n    df.window = px.DurationNanos(df.time_max - df.time_min)\n    df = df[['window']]\n\n    return df\n\n\ndef add_time_window_column(df, start_time):\n    tw = get_time_window(start_time)\n    df = df.merge(tw, how='inner', left_on=[], right_on=[])\n    return df\n\n\ndef process_stats_by_entity(start_time: int, entity: str):\n    ''' Gets the windowed process stats (CPU, memory, etc) per node or pod.\n    Args:\n    @start_time Starting time of the data to examine.\n    @entity: Either pod or node_name.\n    '''\n    # Window size to use on time_ column for bucketing.\n    ns_per_s = 1000 * 1000 * 1000\n    window_ns = px.DurationNanos(10 * ns_per_s)\n\n    df = px.DataFrame(table='process_stats', start_time=start_time)\n    df[entity] = df.ctx[entity]\n    df.timestamp = px.bin(df.time_, window_ns)\n    # First calculate CPU usage by process (UPID) in each k8s_object\n    # over all windows.\n    df = df.groupby([entity, 'upid', 'timestamp']).agg(\n        rss=('rss_bytes', px.mean),\n        vsize=('vsize_bytes', px.mean),\n        # The fields below are counters, so we take the min and the max to subtract them.\n        cpu_utime_ns_max=('cpu_utime_ns', px.max),\n        cpu_utime_ns_min=('cpu_utime_ns', px.min),\n        cpu_ktime_ns_max=('cpu_ktime_ns', px.max),\n        cpu_ktime_ns_min=('cpu_ktime_ns', px.min),\n        read_bytes_max=('read_bytes', px.max),\n        read_bytes_min=('read_bytes', px.min),\n        write_bytes_max=('write_bytes', px.max),\n        write_bytes_min=('write_bytes', px.min),\n        rchar_bytes_max=('rchar_bytes', px.max),\n        rchar_bytes_min=('rchar_bytes', px.min),\n        wchar_bytes_max=('wchar_bytes', px.max),\n        wchar_bytes_min=('wchar_bytes', px.min),\n    )\n    # Next calculate cpu usage and memory stats per window.\n    df.cpu_utime_ns = df.cpu_utime_ns_max - df.cpu_utime_ns_min\n    df.cpu_ktime_ns = df.cpu_ktime_ns_max - df.cpu_ktime_ns_min\n    df.read_bytes = df.read_bytes_max - df.read_bytes_min\n    df.write_bytes = df.write_bytes_max - df.write_bytes_min\n    df.rchar_bytes = df.rchar_bytes_max - df.rchar_bytes_min\n    df.wchar_bytes = df.wchar_bytes_max - df.wchar_bytes_min\n    # Sum by UPID.\n    df = df.groupby([entity, 'timestamp']).agg(\n        cpu_ktime_ns=('cpu_ktime_ns', px.sum),\n        cpu_utime_ns=('cpu_utime_ns', px.sum),\n        read_bytes=('read_bytes', px.sum),\n        write_bytes=('write_bytes', px.sum),\n        rchar_bytes=('rchar_bytes', px.sum),\n        wchar_bytes=('wchar_bytes', px.sum),\n        rss=('rss', px.sum),\n        vsize=('vsize', px.sum),\n    )\n    df.actual_disk_read_throughput = df.read_bytes / window_ns\n    df.actual_disk_write_throughput = df.write_bytes / window_ns\n    df.total_disk_read_throughput = df.rchar_bytes / window_ns\n    df.total_disk_write_throughput = df.wchar_bytes / window_ns\n    # Now take the mean value over the various timestamps.\n    df = df.groupby(entity).agg(\n        cpu_ktime_ns=('cpu_ktime_ns', px.mean),\n        cpu_utime_ns=('cpu_utime_ns', px.mean),\n        actual_disk_read_throughput=('actual_disk_read_throughput', px.mean),\n        actual_disk_write_throughput=('actual_disk_write_throughput', px.mean),\n        total_disk_read_throughput=('total_disk_read_throughput', px.mean),\n        total_disk_write_throughput=('total_disk_write_throughput', px.mean),\n        avg_rss=('rss', px.mean),\n        avg_vsize=('vsize', px.mean),\n    )\n    # Finally, calculate total (kernel + user time)  percentage used over window.\n    df.cpu_usage = px.Percent((df.cpu_ktime_ns + df.cpu_utime_ns) / window_ns)\n    return df.drop(['cpu_ktime_ns', 'cpu_utime_ns'])\n\n\ndef nodes_for_cluster(start_time: int):\n    ''' Gets a list of nodes in the current cluster since `start_time`.\n    Args:\n    @start_time Start time of the data to examine.\n    '''\n    df = px.DataFrame(table='process_stats', start_time=start_time)\n    df.node = df.ctx['node_name']\n    df.pod = df.ctx['pod_name']\n    agg = df.groupby(['node', 'pod']).agg()\n    nodes = agg.groupby('node').agg(pod_count=('pod', px.count))\n    process_stats = process_stats_by_entity(start_time, 'node')\n    output = process_stats.merge(nodes, how='inner', left_on='node', right_on='node',\n                                 suffixes=['', '_x'])\n    return output[['node', 'cpu_usage', 'pod_count']]\n\n\ndef pods_for_cluster(start_time: int):\n    ''' A list of pods in `namespace`.\n    Args:\n    @start_time: The timestamp of data to start at.\n    @namespace: The name of the namespace to filter on.\n    '''\n    df = px.DataFrame(table='process_stats', start_time=start_time)\n    df.pod = df.ctx['pod_name']\n    df.node = df.ctx['node_name']\n    df.container = df.ctx['container_name']\n    df = df.groupby(['pod', 'node', 'container']).agg()\n    df = df.groupby(['pod', 'node']).agg(container_count=('container', px.count))\n    df.start_time = px.pod_name_to_start_time(df.pod)\n    df.status = px.pod_name_to_status(df.pod)\n    process_stats = process_stats_by_entity(start_time, 'pod')\n    output = process_stats.merge(df, how='inner', left_on='pod', right_on='pod',\n                                 suffixes=['', '_x'])\n    return output[['pod', 'cpu_usage', 'total_disk_read_throughput',\n                   'total_disk_write_throughput', 'container_count',\n                   'node', 'start_time', 'status']]\n\n\ndef namespaces_for_cluster(start_time: int):\n    ''' Gets a overview of namespaces in the current cluster since `start_time`.\n    Args:\n    @start_time Start time of the data to examine.\n    '''\n    df = px.DataFrame(table='process_stats', start_time=start_time)\n    df.service = df.ctx['service_name']\n    df.pod = df.ctx['pod_name']\n    df.namespace = df.ctx['namespace']\n    agg = df.groupby(['service', 'pod', 'namespace']).agg()\n    pod_count = agg.groupby(['namespace', 'pod']).agg()\n    pod_count = pod_count.groupby('namespace').agg(pod_count=('pod', px.count))\n    svc_count = agg.groupby(['namespace', 'service']).agg()\n    svc_count = svc_count.groupby('namespace').agg(service_count=('service', px.count))\n    pod_and_svc_count = pod_count.merge(svc_count, how='inner',\n                                        left_on='namespace', right_on='namespace',\n                                        suffixes=['', '_x'])\n    process_stats = process_stats_by_entity(start_time, 'namespace')\n    output = process_stats.merge(pod_and_svc_count, how='inner', left_on='namespace',\n                                 right_on='namespace', suffixes=['', '_y'])\n    return output[['namespace', 'pod_count', 'service_count', 'avg_vsize', 'avg_rss']]\n\n\ndef services_for_cluster(start_time: int):\n    ''' Get an overview of the services in the current cluster.\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    df = px.DataFrame(table='process_stats', start_time=start_time)\n    df.service = df.ctx['service']\n    df = df[df.service != '']\n    df.pod = df.ctx['pod']\n    df = df.groupby(['service', 'pod']).agg()\n    df = df.groupby('service').agg(pod_count=('pod', px.count))\n    service_let = service_let_summary(start_time)\n    joined = df.merge(service_let, how='left', left_on='service', right_on='service',\n                      suffixes=['', '_x'])\n    return joined.drop('service_x')\n\n\ndef http_stats(start_time: int):\n    ''' Get a dataframe of HTTP events.\n    Certain traffic (like health checks) are auto removed, and some standard fields are added.\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    df = px.DataFrame(table='http_events', start_time=start_time)\n\n    # Add K8s metadata.\n    df.service = df.ctx['service']\n    df.pod = df.ctx['pod']\n\n    # Filter out non-k8s entities.\n    df = df[df.pod != '']\n\n    # Additional HTTP fields, pre-computed for convenience.\n    df.failure = df.resp_status >= 400\n\n    # Remove health checks, and anything with no remote address.\n    health_check_req = ((df.req_path == '/healthz' or df.req_path == '/readyz') or df.req_path == '/livez')\n    filter_out_conds = (health_check_req and filter_health_checks) or (df['remote_addr'] == '-')\n    df = df[not filter_out_conds]\n\n    return df\n\n\ndef http_stats_by_service(start_time: int):\n    ''' Get a data frame of HTTP stats per service. The HTTP stats are for inbound traffic,\n    and includes HTTP request count, error count and latency quantiles.\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    df = http_stats(start_time)\n\n    # Filter only to inbound service traffic (server-side).\n    # Don't include traffic initiated by this service to an external location.\n    df = df[df.trace_role == 2]\n\n    # Compute HTTP metrics.\n    df = df.groupby(['service']).agg(\n        http_req_count_in=('latency', px.count),\n        http_error_count_in=('failure', px.sum),\n        http_latency_in=('latency', px.quantiles)\n    )\n\n    return df\n\n\ndef conn_stats(start_time: int):\n    ''' Get a dataframe of connection stats.\n    For each client-server pair, the resulting data frame has the bytes sent and received.\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    df = px.DataFrame(table='conn_stats', start_time=start_time)\n\n    df.pod = df.ctx['pod']\n    df.service = df.ctx['service']\n\n    df = df[df.service != '']\n\n    # Find min/max bytes transferred over the selected time window per pod.\n    df = df.groupby(['upid', 'remote_addr', 'remote_port', 'pod', 'service', 'trace_role']).agg(\n        bytes_recv_min=('bytes_recv', px.min),\n        bytes_recv_max=('bytes_recv', px.max),\n        bytes_sent_min=('bytes_sent', px.min),\n        bytes_sent_max=('bytes_sent', px.max),\n    )\n\n    # Calculate bytes transferred over the time window\n    df.bytes_sent = df.bytes_sent_max - df.bytes_sent_min\n    df.bytes_recv = df.bytes_recv_max - df.bytes_recv_min\n    df = df.drop(['bytes_recv_min', 'bytes_recv_max', 'bytes_sent_min', 'bytes_sent_max'])\n\n    return df\n\n\ndef conn_stats_by_service(start_time: int):\n    ''' Get a dataframe of connection stats aggregated by service.\n    For each service, the resulting data frame contains rx/tx stats for server-side and client-side connections.\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    df = conn_stats(start_time)\n\n    # Group by service and trace role.\n    # Do this after computing bytes sent/received by conn_stats key ({upid, remote_addr, remote_port}).\n    # Keeping trace_role allows us to see which traffic was part of server duties vs client duties.\n    df = df.groupby(['service', 'trace_role']).agg(\n        bytes_recv=('bytes_recv', px.sum),\n        bytes_sent=('bytes_sent', px.sum),\n    )\n\n    # Get RX/TX stats for the server side connections.\n    server_df = df[df.trace_role == 2]\n    server_df.rx_server = server_df.bytes_recv\n    server_df.tx_server = server_df.bytes_sent\n    server_df = server_df[['service', 'rx_server', 'tx_server']]\n\n    # Get RX/TX stats for the client side connections.\n    client_df = df[df.trace_role == 1]\n    client_df.rx_client = client_df.bytes_recv\n    client_df.tx_client = client_df.bytes_sent\n    client_df = client_df[['service', 'rx_client', 'tx_client']]\n\n    # Create a dataframe that contains both server-side and client-side RX/TX stats.\n    df = server_df.merge(client_df,\n                         how='left',\n                         left_on='service',\n                         right_on='service',\n                         suffixes=['', '_x'])\n    df = df['service', 'rx_server', 'tx_server', 'rx_client', 'tx_client']\n\n    return df\n\n\ndef service_let_summary(start_time: int):\n    ''' Compute a summary of traffic by requesting service, for requests\n        on services in the current cluster..\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    conn_stats_df = conn_stats_by_service(start_time)\n    http_stats_df = http_stats_by_service(start_time)\n\n    # Merge conn_stats_df and http_stats_df.\n    df = conn_stats_df.merge(http_stats_df,\n                             how='left',\n                             left_on='service',\n                             right_on='service',\n                             suffixes=['', '_x'])\n\n    # Compute time window for the query and add it as a column.\n    df = add_time_window_column(df, start_time)\n\n    # Compute throughput values.\n    df.http_req_throughput_in = df.http_req_count_in / df.window\n    df.http_error_rate_in = px.Percent(\n        px.select(df.http_req_count_in != 0, df.http_error_count_in / df.http_req_count_in, 0.0))\n    df.inbound_conns = (df.rx_server + df.tx_server) / df.window\n    df.outbound_conns = (df.tx_client + df.rx_client) / df.window\n\n    return df[['service', 'http_latency_in', 'http_req_throughput_in', 'http_error_rate_in',\n               'inbound_conns', 'outbound_conns']]\n\n\ndef service_let_graph(start_time: int):\n    ''' Compute a summary of traffic by requesting service, for requests on services\n        in the current cluster. Similar to `service_let_summary` but also breaks down\n        by pod in addition to service.\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    df = http_stats(start_time)\n    df = df.groupby(['service', 'remote_addr', 'pod', 'trace_role']).agg(\n        http_req_count_in=('latency', px.count),\n        http_error_count_in=('failure', px.sum),\n        latency_quantiles=('latency', px.quantiles),\n        inbound_bytes_total=('req_body_size', px.sum),\n        outbound_bytes_total=('resp_body_size', px.sum)\n    )\n\n    # Get the traced and remote pod/service/IP information.\n    df.traced_pod = df.pod\n    df.traced_service = df.service\n    df.traced_ip = px.pod_name_to_pod_ip(df.pod)\n    df.remote_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n    df.remote_service = px.service_id_to_service_name(px.ip_to_service_id(df.remote_addr))\n    df.remote_ip = df.remote_addr\n    # If external IPs are excluded in the service graph, then we also exclude any\n    # traffic where we don't know the remote pod or remote service name.\n    df = df[include_ips or (df.remote_pod != '' or df.remote_service != '')]\n\n    # Associate it with Client/Server roles, based on the trace role.\n    df.is_server_side_tracing = df.trace_role == 2\n    df.responder_pod = px.select(df.is_server_side_tracing, df.traced_pod, df.remote_pod)\n    df.requestor_pod = px.select(df.is_server_side_tracing, df.remote_pod, df.traced_pod)\n    df.responder_service = px.select(df.is_server_side_tracing, df.traced_service, df.remote_service)\n    df.requestor_service = px.select(df.is_server_side_tracing, df.remote_service, df.traced_service)\n    df.responder_ip = px.select(df.is_server_side_tracing, df.traced_ip, df.remote_ip)\n    df.requestor_ip = px.select(df.is_server_side_tracing, df.remote_ip, df.traced_ip)\n\n    # Compute statistics about each edge of the service graph.\n    df.latency_p50 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p50')))\n    df.latency_p90 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p90')))\n    df.latency_p99 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p99')))\n    df = add_time_window_column(df, start_time)\n    df.request_throughput = df.http_req_count_in / df.window\n    df.inbound_throughput = df.inbound_bytes_total / df.window\n    df.outbound_throughput = df.outbound_bytes_total / df.window\n    df.error_rate = px.Percent(df.http_error_count_in / df.http_req_count_in)\n\n    df = df.groupby(['responder_pod', 'requestor_pod', 'responder_service',\n                     'requestor_service', 'responder_ip', 'requestor_ip']).agg(\n        latency_p50=('latency_p50', px.mean),\n        latency_p90=('latency_p90', px.mean),\n        latency_p99=('latency_p99', px.mean),\n        request_throughput=('request_throughput', px.mean),\n        error_rate=('error_rate', px.mean),\n        inbound_throughput=('inbound_throughput', px.mean),\n        outbound_throughput=('outbound_throughput', px.mean),\n        throughput_total=('http_req_count_in', px.sum)\n    )\n\n    return df\n\noutput = services_for_cluster(__time_from)\noutput.http_latency_in_p99 = px.pluck_float64(output.http_latency_in, 'p99') / px.pow(10,6)\noutput.http_req_throughput_in = output.http_req_throughput_in * px.pow(10,9)\noutput.inbound_conns = output.inbound_conns * px.pow(10,9) / px.pow(2,10)\noutput.outbound_conns = output.outbound_conns * px.pow(10,9) / px.pow(2,10)\noutput = output[['service', 'pod_count', 'http_latency_in_p99', 'http_req_throughput_in', 'http_error_rate_in', 'inbound_conns', 'outbound_conns']]\npx.display(output)"
          },
          "queryType": "run-script",
          "refId": "A"
        }
      ],
      "title": "Services",
      "type": "table"
    },
    {
      "datasource": "${DS_PIXIE_GRAFANA DATASOURCE PLUGIN}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "displayMode": "auto",
            "inspect": false
          },
          "decimals": 1,
          "mappings": [
            {
              "options": {
                "Failed": {
                  "color": "red",
                  "index": 5
                },
                "Pending": {
                  "color": "yellow",
                  "index": 6
                },
                "Running": {
                  "color": "green",
                  "index": 0
                },
                "Succeeded": {
                  "color": "green",
                  "index": 4
                },
                "Terminated": {
                  "color": "red",
                  "index": 3
                },
                "Unknown": {
                  "color": "red",
                  "index": 7
                },
                "Waiting": {
                  "color": "yellow",
                  "index": 8
                },
                "false": {
                  "color": "red",
                  "index": 2
                },
                "true": {
                  "color": "green",
                  "index": 1
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "pod"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 378
              },
              {
                "id": "links",
                "value": [
                  {
                    "title": "",
                    "url": "/d/_t3foxCnz/px-pod?orgId=1&${pixieCluster:queryparam}&var-pixiePod=${__data.fields.pod}"
                  }
                ]
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "ready"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 66
              },
              {
                "id": "custom.displayMode",
                "value": "color-text"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "phase"
            },
            "properties": [
              {
                "id": "custom.displayMode",
                "value": "color-text"
              },
              {
                "id": "custom.width",
                "value": 73
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "cpu_usage"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 162
              },
              {
                "id": "unit",
                "value": "percentunit"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "total_disk_read_throughput"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 274
              },
              {
                "id": "unit",
                "value": "KBs"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "container_count"
            },
            "properties": [
              {
                "id": "decimals"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "total_disk_write_throughput"
            },
            "properties": [
              {
                "id": "unit",
                "value": "KBs"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "node"
            },
            "properties": [
              {
                "id": "links",
                "value": [
                  {
                    "title": "",
                    "url": "/d/tcpoz_jnz/px-node?orgId=1&${pixieCluster:queryparam}&var-node=${__data.fields.node}&var-groupby=node"
                  }
                ]
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 9,
        "w": 24,
        "x": 0,
        "y": 32
      },
      "id": 10,
      "options": {
        "footer": {
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "7.5.1",
      "targets": [
        {
          "datasource": "${DS_PIXIE_GRAFANA DATASOURCE PLUGIN}",
          "queryBody": {
            "pxlScript": "# Copyright 2018- The Pixie Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# SPDX-License-Identifier: Apache-2.0\n\n''' Cluster Overview\n\nThis view lists the namespaces and the nodes that are available on the current cluster.\n\n'''\nimport px\n\n\n# Flag to filter out health checks from the data.\nfilter_health_checks = True\n\n# Whether or not to include traffic from IPs that don't resolve to a known pod/service.\ninclude_ips = True\n\n# $pixieCluster - work around for grafana to update panel on variable change\n\n# Hack to get the time window for the script.\n# TODO(philkuz): Replace this with a built-in.\ndef get_time_window(start_time: int):\n    ''' Converts the start_time string into a table with a single column and single row.\n    The approach is hacky, and will round to roughly 1 second.\n    '''\n    df = px.DataFrame('process_stats', start_time=start_time)\n\n    df = df.agg(\n        time_min=('time_', px.min),\n        time_max=('time_', px.max),\n    )\n\n    df.window = px.DurationNanos(df.time_max - df.time_min)\n    df = df[['window']]\n\n    return df\n\n\ndef add_time_window_column(df, start_time):\n    tw = get_time_window(start_time)\n    df = df.merge(tw, how='inner', left_on=[], right_on=[])\n    return df\n\n\ndef process_stats_by_entity(start_time: int, entity: str):\n    ''' Gets the windowed process stats (CPU, memory, etc) per node or pod.\n    Args:\n    @start_time Starting time of the data to examine.\n    @entity: Either pod or node_name.\n    '''\n    # Window size to use on time_ column for bucketing.\n    ns_per_s = 1000 * 1000 * 1000\n    window_ns = px.DurationNanos(10 * ns_per_s)\n\n    df = px.DataFrame(table='process_stats', start_time=start_time)\n    df[entity] = df.ctx[entity]\n    df.timestamp = px.bin(df.time_, window_ns)\n    # First calculate CPU usage by process (UPID) in each k8s_object\n    # over all windows.\n    df = df.groupby([entity, 'upid', 'timestamp']).agg(\n        rss=('rss_bytes', px.mean),\n        vsize=('vsize_bytes', px.mean),\n        # The fields below are counters, so we take the min and the max to subtract them.\n        cpu_utime_ns_max=('cpu_utime_ns', px.max),\n        cpu_utime_ns_min=('cpu_utime_ns', px.min),\n        cpu_ktime_ns_max=('cpu_ktime_ns', px.max),\n        cpu_ktime_ns_min=('cpu_ktime_ns', px.min),\n        read_bytes_max=('read_bytes', px.max),\n        read_bytes_min=('read_bytes', px.min),\n        write_bytes_max=('write_bytes', px.max),\n        write_bytes_min=('write_bytes', px.min),\n        rchar_bytes_max=('rchar_bytes', px.max),\n        rchar_bytes_min=('rchar_bytes', px.min),\n        wchar_bytes_max=('wchar_bytes', px.max),\n        wchar_bytes_min=('wchar_bytes', px.min),\n    )\n    # Next calculate cpu usage and memory stats per window.\n    df.cpu_utime_ns = df.cpu_utime_ns_max - df.cpu_utime_ns_min\n    df.cpu_ktime_ns = df.cpu_ktime_ns_max - df.cpu_ktime_ns_min\n    df.read_bytes = df.read_bytes_max - df.read_bytes_min\n    df.write_bytes = df.write_bytes_max - df.write_bytes_min\n    df.rchar_bytes = df.rchar_bytes_max - df.rchar_bytes_min\n    df.wchar_bytes = df.wchar_bytes_max - df.wchar_bytes_min\n    # Sum by UPID.\n    df = df.groupby([entity, 'timestamp']).agg(\n        cpu_ktime_ns=('cpu_ktime_ns', px.sum),\n        cpu_utime_ns=('cpu_utime_ns', px.sum),\n        read_bytes=('read_bytes', px.sum),\n        write_bytes=('write_bytes', px.sum),\n        rchar_bytes=('rchar_bytes', px.sum),\n        wchar_bytes=('wchar_bytes', px.sum),\n        rss=('rss', px.sum),\n        vsize=('vsize', px.sum),\n    )\n    df.actual_disk_read_throughput = df.read_bytes / window_ns\n    df.actual_disk_write_throughput = df.write_bytes / window_ns\n    df.total_disk_read_throughput = df.rchar_bytes / window_ns\n    df.total_disk_write_throughput = df.wchar_bytes / window_ns\n    # Now take the mean value over the various timestamps.\n    df = df.groupby(entity).agg(\n        cpu_ktime_ns=('cpu_ktime_ns', px.mean),\n        cpu_utime_ns=('cpu_utime_ns', px.mean),\n        actual_disk_read_throughput=('actual_disk_read_throughput', px.mean),\n        actual_disk_write_throughput=('actual_disk_write_throughput', px.mean),\n        total_disk_read_throughput=('total_disk_read_throughput', px.mean),\n        total_disk_write_throughput=('total_disk_write_throughput', px.mean),\n        avg_rss=('rss', px.mean),\n        avg_vsize=('vsize', px.mean),\n    )\n    # Finally, calculate total (kernel + user time)  percentage used over window.\n    df.cpu_usage = px.Percent((df.cpu_ktime_ns + df.cpu_utime_ns) / window_ns)\n    return df.drop(['cpu_ktime_ns', 'cpu_utime_ns'])\n\n\ndef nodes_for_cluster(start_time: int):\n    ''' Gets a list of nodes in the current cluster since `start_time`.\n    Args:\n    @start_time Start time of the data to examine.\n    '''\n    df = px.DataFrame(table='process_stats', start_time=start_time)\n    df.node = df.ctx['node_name']\n    df.pod = df.ctx['pod_name']\n    agg = df.groupby(['node', 'pod']).agg()\n    nodes = agg.groupby('node').agg(pod_count=('pod', px.count))\n    process_stats = process_stats_by_entity(start_time, 'node')\n    output = process_stats.merge(nodes, how='inner', left_on='node', right_on='node',\n                                 suffixes=['', '_x'])\n    return output[['node', 'cpu_usage', 'pod_count']]\n\n\ndef pods_for_cluster(start_time: int):\n    ''' A list of pods in `namespace`.\n    Args:\n    @start_time: The timestamp of data to start at.\n    @namespace: The name of the namespace to filter on.\n    '''\n    df = px.DataFrame(table='process_stats', start_time=start_time)\n    df.pod = df.ctx['pod_name']\n    df.node = df.ctx['node_name']\n    df.container = df.ctx['container_name']\n    df = df.groupby(['pod', 'node', 'container']).agg()\n    df = df.groupby(['pod', 'node']).agg(container_count=('container', px.count))\n    df.start_time = px.pod_name_to_start_time(df.pod)\n    df.status = px.pod_name_to_status(df.pod)\n    process_stats = process_stats_by_entity(start_time, 'pod')\n    output = process_stats.merge(df, how='inner', left_on='pod', right_on='pod',\n                                 suffixes=['', '_x'])\n    return output[['pod', 'cpu_usage', 'total_disk_read_throughput',\n                   'total_disk_write_throughput', 'container_count',\n                   'node', 'start_time', 'status']]\n\n\ndef namespaces_for_cluster(start_time: int):\n    ''' Gets a overview of namespaces in the current cluster since `start_time`.\n    Args:\n    @start_time Start time of the data to examine.\n    '''\n    df = px.DataFrame(table='process_stats', start_time=start_time)\n    df.service = df.ctx['service_name']\n    df.pod = df.ctx['pod_name']\n    df.namespace = df.ctx['namespace']\n    agg = df.groupby(['service', 'pod', 'namespace']).agg()\n    pod_count = agg.groupby(['namespace', 'pod']).agg()\n    pod_count = pod_count.groupby('namespace').agg(pod_count=('pod', px.count))\n    svc_count = agg.groupby(['namespace', 'service']).agg()\n    svc_count = svc_count.groupby('namespace').agg(service_count=('service', px.count))\n    pod_and_svc_count = pod_count.merge(svc_count, how='inner',\n                                        left_on='namespace', right_on='namespace',\n                                        suffixes=['', '_x'])\n    process_stats = process_stats_by_entity(start_time, 'namespace')\n    output = process_stats.merge(pod_and_svc_count, how='inner', left_on='namespace',\n                                 right_on='namespace', suffixes=['', '_y'])\n    return output[['namespace', 'pod_count', 'service_count', 'avg_vsize', 'avg_rss']]\n\n\ndef services_for_cluster(start_time: int):\n    ''' Get an overview of the services in the current cluster.\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    df = px.DataFrame(table='process_stats', start_time=start_time)\n    df.service = df.ctx['service']\n    df = df[df.service != '']\n    df.pod = df.ctx['pod']\n    df = df.groupby(['service', 'pod']).agg()\n    df = df.groupby('service').agg(pod_count=('pod', px.count))\n    service_let = service_let_summary(start_time)\n    joined = df.merge(service_let, how='left', left_on='service', right_on='service',\n                      suffixes=['', '_x'])\n    return joined.drop('service_x')\n\n\ndef http_stats(start_time: int):\n    ''' Get a dataframe of HTTP events.\n    Certain traffic (like health checks) are auto removed, and some standard fields are added.\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    df = px.DataFrame(table='http_events', start_time=start_time)\n\n    # Add K8s metadata.\n    df.service = df.ctx['service']\n    df.pod = df.ctx['pod']\n\n    # Filter out non-k8s entities.\n    df = df[df.pod != '']\n\n    # Additional HTTP fields, pre-computed for convenience.\n    df.failure = df.resp_status >= 400\n\n    # Remove health checks, and anything with no remote address.\n    health_check_req = ((df.req_path == '/healthz' or df.req_path == '/readyz') or df.req_path == '/livez')\n    filter_out_conds = (health_check_req and filter_health_checks) or (df['remote_addr'] == '-')\n    df = df[not filter_out_conds]\n\n    return df\n\n\ndef http_stats_by_service(start_time: int):\n    ''' Get a data frame of HTTP stats per service. The HTTP stats are for inbound traffic,\n    and includes HTTP request count, error count and latency quantiles.\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    df = http_stats(start_time)\n\n    # Filter only to inbound service traffic (server-side).\n    # Don't include traffic initiated by this service to an external location.\n    df = df[df.trace_role == 2]\n\n    # Compute HTTP metrics.\n    df = df.groupby(['service']).agg(\n        http_req_count_in=('latency', px.count),\n        http_error_count_in=('failure', px.sum),\n        http_latency_in=('latency', px.quantiles)\n    )\n\n    return df\n\n\ndef conn_stats(start_time: int):\n    ''' Get a dataframe of connection stats.\n    For each client-server pair, the resulting data frame has the bytes sent and received.\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    df = px.DataFrame(table='conn_stats', start_time=start_time)\n\n    df.pod = df.ctx['pod']\n    df.service = df.ctx['service']\n\n    df = df[df.service != '']\n\n    # Find min/max bytes transferred over the selected time window per pod.\n    df = df.groupby(['upid', 'remote_addr', 'remote_port', 'pod', 'service', 'trace_role']).agg(\n        bytes_recv_min=('bytes_recv', px.min),\n        bytes_recv_max=('bytes_recv', px.max),\n        bytes_sent_min=('bytes_sent', px.min),\n        bytes_sent_max=('bytes_sent', px.max),\n    )\n\n    # Calculate bytes transferred over the time window\n    df.bytes_sent = df.bytes_sent_max - df.bytes_sent_min\n    df.bytes_recv = df.bytes_recv_max - df.bytes_recv_min\n    df = df.drop(['bytes_recv_min', 'bytes_recv_max', 'bytes_sent_min', 'bytes_sent_max'])\n\n    return df\n\n\ndef conn_stats_by_service(start_time: int):\n    ''' Get a dataframe of connection stats aggregated by service.\n    For each service, the resulting data frame contains rx/tx stats for server-side and client-side connections.\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    df = conn_stats(start_time)\n\n    # Group by service and trace role.\n    # Do this after computing bytes sent/received by conn_stats key ({upid, remote_addr, remote_port}).\n    # Keeping trace_role allows us to see which traffic was part of server duties vs client duties.\n    df = df.groupby(['service', 'trace_role']).agg(\n        bytes_recv=('bytes_recv', px.sum),\n        bytes_sent=('bytes_sent', px.sum),\n    )\n\n    # Get RX/TX stats for the server side connections.\n    server_df = df[df.trace_role == 2]\n    server_df.rx_server = server_df.bytes_recv\n    server_df.tx_server = server_df.bytes_sent\n    server_df = server_df[['service', 'rx_server', 'tx_server']]\n\n    # Get RX/TX stats for the client side connections.\n    client_df = df[df.trace_role == 1]\n    client_df.rx_client = client_df.bytes_recv\n    client_df.tx_client = client_df.bytes_sent\n    client_df = client_df[['service', 'rx_client', 'tx_client']]\n\n    # Create a dataframe that contains both server-side and client-side RX/TX stats.\n    df = server_df.merge(client_df,\n                         how='left',\n                         left_on='service',\n                         right_on='service',\n                         suffixes=['', '_x'])\n    df = df['service', 'rx_server', 'tx_server', 'rx_client', 'tx_client']\n\n    return df\n\n\ndef service_let_summary(start_time: int):\n    ''' Compute a summary of traffic by requesting service, for requests\n        on services in the current cluster..\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    conn_stats_df = conn_stats_by_service(start_time)\n    http_stats_df = http_stats_by_service(start_time)\n\n    # Merge conn_stats_df and http_stats_df.\n    df = conn_stats_df.merge(http_stats_df,\n                             how='left',\n                             left_on='service',\n                             right_on='service',\n                             suffixes=['', '_x'])\n\n    # Compute time window for the query and add it as a column.\n    df = add_time_window_column(df, start_time)\n\n    # Compute throughput values.\n    df.http_req_throughput_in = df.http_req_count_in / df.window\n    df.http_error_rate_in = px.Percent(\n        px.select(df.http_req_count_in != 0, df.http_error_count_in / df.http_req_count_in, 0.0))\n    df.inbound_conns = (df.rx_server + df.tx_server) / df.window\n    df.outbound_conns = (df.tx_client + df.rx_client) / df.window\n\n    return df[['service', 'http_latency_in', 'http_req_throughput_in', 'http_error_rate_in',\n               'inbound_conns', 'outbound_conns']]\n\n\ndef service_let_graph(start_time: int):\n    ''' Compute a summary of traffic by requesting service, for requests on services\n        in the current cluster. Similar to `service_let_summary` but also breaks down\n        by pod in addition to service.\n    Args:\n    @start_time: The timestamp of data to start at.\n    '''\n    df = http_stats(start_time)\n    df = df.groupby(['service', 'remote_addr', 'pod', 'trace_role']).agg(\n        http_req_count_in=('latency', px.count),\n        http_error_count_in=('failure', px.sum),\n        latency_quantiles=('latency', px.quantiles),\n        inbound_bytes_total=('req_body_size', px.sum),\n        outbound_bytes_total=('resp_body_size', px.sum)\n    )\n\n    # Get the traced and remote pod/service/IP information.\n    df.traced_pod = df.pod\n    df.traced_service = df.service\n    df.traced_ip = px.pod_name_to_pod_ip(df.pod)\n    df.remote_pod = px.pod_id_to_pod_name(px.ip_to_pod_id(df.remote_addr))\n    df.remote_service = px.service_id_to_service_name(px.ip_to_service_id(df.remote_addr))\n    df.remote_ip = df.remote_addr\n    # If external IPs are excluded in the service graph, then we also exclude any\n    # traffic where we don't know the remote pod or remote service name.\n    df = df[include_ips or (df.remote_pod != '' or df.remote_service != '')]\n\n    # Associate it with Client/Server roles, based on the trace role.\n    df.is_server_side_tracing = df.trace_role == 2\n    df.responder_pod = px.select(df.is_server_side_tracing, df.traced_pod, df.remote_pod)\n    df.requestor_pod = px.select(df.is_server_side_tracing, df.remote_pod, df.traced_pod)\n    df.responder_service = px.select(df.is_server_side_tracing, df.traced_service, df.remote_service)\n    df.requestor_service = px.select(df.is_server_side_tracing, df.remote_service, df.traced_service)\n    df.responder_ip = px.select(df.is_server_side_tracing, df.traced_ip, df.remote_ip)\n    df.requestor_ip = px.select(df.is_server_side_tracing, df.remote_ip, df.traced_ip)\n\n    # Compute statistics about each edge of the service graph.\n    df.latency_p50 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p50')))\n    df.latency_p90 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p90')))\n    df.latency_p99 = px.DurationNanos(px.floor(px.pluck_float64(df.latency_quantiles, 'p99')))\n    df = add_time_window_column(df, start_time)\n    df.request_throughput = df.http_req_count_in / df.window\n    df.inbound_throughput = df.inbound_bytes_total / df.window\n    df.outbound_throughput = df.outbound_bytes_total / df.window\n    df.error_rate = px.Percent(df.http_error_count_in / df.http_req_count_in)\n\n    df = df.groupby(['responder_pod', 'requestor_pod', 'responder_service',\n                     'requestor_service', 'responder_ip', 'requestor_ip']).agg(\n        latency_p50=('latency_p50', px.mean),\n        latency_p90=('latency_p90', px.mean),\n        latency_p99=('latency_p99', px.mean),\n        request_throughput=('request_throughput', px.mean),\n        error_rate=('error_rate', px.mean),\n        inbound_throughput=('inbound_throughput', px.mean),\n        outbound_throughput=('outbound_throughput', px.mean),\n        throughput_total=('http_req_count_in', px.sum)\n    )\n\n    return df\n\noutput = pods_for_cluster(__time_from)\noutput.total_disk_read_throughput =  output.total_disk_read_throughput * px.pow(10,9) / px.pow(2,10)\noutput.total_disk_write_throughput = output.total_disk_write_throughput * px.pow(10,9) / px.pow(2,10)\nphase = px.pluck(output.status, \"phase\") \nready = px.pluck(output.status, \"ready\")  \n\noutput.phase = px.pluck(output.status, \"phase\") \noutput.ready = px.pluck(output.status, \"ready\") \n\noutput = output[['pod', 'phase', 'ready', 'cpu_usage', 'total_disk_read_throughput', 'total_disk_write_throughput', 'container_count', 'node', 'start_time']]\npx.display(output)"
          },
          "queryType": "run-script",
          "refId": "A"
        }
      ],
      "title": "Pods",
      "type": "table"
    }
  ],
  "schemaVersion": 27,
  "style": "dark",
  "tags": [],
  "templating": {
    "list": [
      {
        "current": {},
        "datasource": "${DS_PIXIE_GRAFANA DATASOURCE PLUGIN}",
        "definition": "Clusters",
        "hide": 0,
        "includeAll": false,
        "multi": false,
        "name": "pixieCluster",
        "options": [],
        "query": {
          "queryType": "get-clusters"
        },
        "refresh": 2,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "type": "query"
      }
    ]
  },
  "time": {
    "from": "now-15m",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "",
  "title": "Pixie K8s Cluster Overview",
  "description": "Dashboard includes a service map of the HTTP traffic between the services in your cluster, along with the latency, error rate, and throughput per service. It also lists the nodes, namespaces and pods available in your cluster.",
  "uid": "JMSOCJj7k",
  "version": 1,
  "weekStart": "",
  "gnetId": 16982
}