Grafana dashboard: Nvidia GPU
Description: GPU dashboard for nvidia metrics 

To be used with the gpu-operator helm chart.
Some considerations related to the metrics available:
* Graph GPU utilization does not take into consideration MIG partition size :(
* Table Usage for TimeSliced MIG is not showing usage correctly.

This was tested using NODE wide definitions (no modes or individual cards tested)

Contributions welcome! Send contributions to dy090.guerra@gmail.com

## CHANGELOG:

### Revision 2
* Correct usage of Memory Metrics (instead of Bandwidth)
* Replacement of the fan speed graph for the SM metrics
* Added Profiling metrics for FP64, FP32 and FP16 together with Tensor core

The added/refactored metrics require the usage of a custom
dcgmExporter configMap that exports the following metrics
in addition to defaults:

* DCGM_FI_PROF_PIPE_FP64_ACTIVE
* DCGM_FI_PROF_PIPE_FP32_ACTIVE
* DCGM_FI_PROF_PIPE_FP16_ACTIVE
* DCGM_FI_PROF_SM_ACTIVE
* DCGM_FI_PROF_SM_OCCUPANCY
* DCGM_FI_DEV_FB_TOTAL 

__NOTE:__ consider using DCGM_FI_DEV_FB_TOTAL instead of (DCGM_FI_DEV_FB_FREE + DCGM_FI_DEV_FB_USED) in memory dashboards.

### Revision 1
First release

## Contributors:
* Diana Gaponcic
* Diogo Guerra

{
  "__inputs": [
    {
      "name": "DS_PROMETHEUS",
      "label": "Prometheus",
      "description": "",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    }
  ],
  "__elements": [],
  "__requires": [
    {
      "type": "panel",
      "id": "gauge",
      "name": "Gauge",
      "version": ""
    },
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "9.0.1"
    },
    {
      "type": "datasource",
      "id": "prometheus",
      "name": "Prometheus",
      "version": "1.0.0"
    },
    {
      "type": "panel",
      "id": "table",
      "name": "Table",
      "version": ""
    },
    {
      "type": "panel",
      "id": "timeseries",
      "name": "Time series",
      "version": ""
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": {
          "type": "datasource",
          "uid": "grafana"
        },
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "target": {
          "limit": 100,
          "matchAny": false,
          "tags": [],
          "type": "dashboard"
        },
        "type": "dashboard"
      }
    ]
  },
  "description": "GPU dashboard for nvidia metrics ",
  "editable": true,
  "fiscalYearStartMonth": 0,
  "gnetId": 18288,
  "graphTooltip": 1,
  "id": null,
  "iteration": 1681310150077,
  "links": [],
  "liveNow": false,
  "panels": [
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [
            {
              "options": {
                "match": "null",
                "result": {
                  "text": "N/A"
                }
              },
              "type": "special"
            }
          ],
          "max": 1,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 0.8
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 6,
        "w": 3,
        "x": 0,
        "y": 0
      },
      "id": 22,
      "links": [],
      "maxDataPoints": 100,
      "options": {
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true,
        "text": {}
      },
      "pluginVersion": "9.0.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "sum(avg(DCGM_FI_PROF_GR_ENGINE_ACTIVE{instance=~\"$instance\"}) by (instance, gpu)) / count(count(DCGM_FI_PROF_GR_ENGINE_ACTIVE{instance=~\"$instance\"}) by (instance, device))",
          "format": "time_series",
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "GPU Utilization",
      "type": "gauge"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [
            {
              "options": {
                "match": "null",
                "result": {
                  "text": "N/A"
                }
              },
              "type": "special"
            }
          ],
          "max": 1,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 6,
        "w": 3,
        "x": 3,
        "y": 0
      },
      "id": 23,
      "links": [],
      "maxDataPoints": 100,
      "options": {
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true,
        "text": {}
      },
      "pluginVersion": "9.0.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "sum(DCGM_FI_DEV_FB_USED{instance=~\"$instance\", gpu=~\"$gpu\"})/sum(DCGM_FI_DEV_FB_USED{instance=~\"$instance\", gpu=~\"$gpu\"}+DCGM_FI_DEV_FB_FREE{instance=~\"$instance\", gpu=~\"$gpu\"})",
          "format": "time_series",
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "GPU Memory Utilization",
      "type": "gauge"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": true,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "max": 1,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 9,
        "w": 9,
        "x": 6,
        "y": 0
      },
      "id": 10,
      "links": [],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "hidden",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "8.2.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "avg(DCGM_FI_PROF_GR_ENGINE_ACTIVE{GPU_I_PROFILE=\"\",instance=~\"$instance\", gpu=~\"$gpu\"}) BY (instance, gpu)",
          "hide": false,
          "interval": "",
          "legendFormat": "{{instance}}-full{{gpu}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "AVG(DCGM_FI_PROF_GR_ENGINE_ACTIVE{GPU_I_PROFILE!=\"\",instance=~\"$instance\", gpu=~\"$gpu\", GPU_I_PROFILE=~\"$mig\"}) BY (instance, gpu)",
          "hide": false,
          "interval": "",
          "legendFormat": "{{instance}}-mig{{gpu}}",
          "refId": "B"
        }
      ],
      "title": "GPU Utilization",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": true,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "max": 1,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 9,
        "w": 9,
        "x": 15,
        "y": 0
      },
      "id": 28,
      "links": [],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "hidden",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "8.2.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "exemplar": true,
          "expr": " AVG(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{instance=~\"$instance\", gpu=~\"$gpu\", GPU_I_PROFILE=~\"$mig\"}) BY (instance, gpu)",
          "format": "time_series",
          "hide": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{instance}}-gpu{{gpu}}-tensor",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "exemplar": true,
          "expr": " AVG(DCGM_FI_PROF_PIPE_FP64_ACTIVE{instance=~\"$instance\", gpu=~\"$gpu\", GPU_I_PROFILE=~\"$mig\"}) BY (instance, gpu)",
          "format": "time_series",
          "hide": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{instance}}-gpu{{gpu}}-fp64",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "exemplar": true,
          "expr": " AVG(DCGM_FI_PROF_PIPE_FP32_ACTIVE{instance=~\"$instance\", gpu=~\"$gpu\", GPU_I_PROFILE=~\"$mig\"}) BY (instance, gpu)",
          "format": "time_series",
          "hide": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{instance}}-gpu{{gpu}}-fp32",
          "range": true,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "exemplar": true,
          "expr": " AVG(DCGM_FI_PROF_PIPE_FP16_ACTIVE{instance=~\"$instance\", gpu=~\"$gpu\", GPU_I_PROFILE=~\"$mig\"}) BY (instance, gpu)",
          "format": "time_series",
          "hide": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{instance}}-gpu{{gpu}}-fp16",
          "range": true,
          "refId": "D"
        }
      ],
      "title": "Core Utilization",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "displayMode": "auto",
            "inspect": false,
            "minWidth": 50
          },
          "mappings": [
            {
              "options": {
                "match": "null",
                "result": {
                  "index": 0,
                  "text": "0"
                }
              },
              "type": "special"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "none"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Use [50%]"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 80
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Allocatable"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 90
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 0,
        "y": 6
      },
      "id": 25,
      "links": [],
      "maxDataPoints": 100,
      "options": {
        "footer": {
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "frameIndex": 0,
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "9.0.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "count(label_replace(DCGM_FI_PROF_GR_ENGINE_ACTIVE{instance=~\"$instance\",GPU_I_PROFILE!=\"\"}, \"GPU\",\"$1\",\"GPU_I_PROFILE\",\"(.*)\")) by (instance,GPU)",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "{{ GPU }}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "count(label_replace(DCGM_FI_PROF_GR_ENGINE_ACTIVE{instance=~\"$instance\",GPU_I_PROFILE!=\"\"} >= 0.5, \"GPU\",\"$1\",\"GPU_I_PROFILE\",\"(.*)\")) by (instance,GPU)",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "{{ GPU }}",
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "count(label_replace(DCGM_FI_PROF_GR_ENGINE_ACTIVE{instance=~\"$instance\",GPU_I_PROFILE=\"\"}, \"GPU\",\"$1\",\"modelName\",\"(.*)\")) by (GPU)",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "{{ GPU }}",
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "count(label_replace(DCGM_FI_PROF_GR_ENGINE_ACTIVE{instance=~\"$instance\",GPU_I_PROFILE=\"\"} >= 0.5, \"GPU\",\"$1\",\"modelName\",\"(.*)\")) by (GPU)",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "{{ GPU }}",
          "refId": "D"
        }
      ],
      "title": "# GPUs / MIGs",
      "transformations": [
        {
          "id": "seriesToColumns",
          "options": {
            "byField": "GPU"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Time": true,
              "Time 1": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "instance": true
            },
            "indexByName": {},
            "renameByName": {
              "Value #A": "Capacity_aux",
              "Value #B": "Use",
              "Value #C": "Capacity_aux",
              "Value #D": "Use",
              "Value #E": "USe [>=50%]"
            }
          }
        },
        {
          "id": "calculateField",
          "options": {
            "alias": "Allocatable",
            "mode": "reduceRow",
            "reduce": {
              "include": [
                "Capacity_aux"
              ],
              "reducer": "sum"
            }
          }
        },
        {
          "id": "calculateField",
          "options": {
            "alias": "Use [50%]",
            "mode": "reduceRow",
            "reduce": {
              "include": [
                "Use"
              ],
              "reducer": "sum"
            }
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Capacity": false,
              "Capacity_aux": true,
              "Use": true
            },
            "indexByName": {},
            "renameByName": {}
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": true,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "percent"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 9,
        "w": 9,
        "x": 6,
        "y": 9
      },
      "id": 11,
      "links": [],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "8.2.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "avg(DCGM_FI_DEV_FB_USED{instance=~\"$instance\", gpu=~\"$gpu\"}/(DCGM_FI_DEV_FB_USED{instance=~\"$instance\", gpu=~\"$gpu\"}+DCGM_FI_DEV_FB_FREE{instance=~\"$instance\", gpu=~\"$gpu\"})) BY (instance, gpu)",
          "format": "time_series",
          "hide": true,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{instance}}-gpu{{gpu}}",
          "refId": "A"
        }
      ],
      "title": "Memory Utilization",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": true,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "max": 1,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 9,
        "w": 9,
        "x": 15,
        "y": 9
      },
      "id": 34,
      "links": [],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "hidden",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "8.2.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "exemplar": true,
          "expr": "avg(DCGM_FI_PROF_SM_ACTIVE{instance=~\"$instance\", gpu=~\"$gpu\", GPU_I_PROFILE=~\"$mig\"}) BY (instance, gpu)",
          "hide": false,
          "interval": "",
          "legendFormat": "{{instance}}-gpu{{gpu}}-active",
          "range": true,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "exemplar": true,
          "expr": "avg(DCGM_FI_PROF_SM_OCCUPANCY{instance=~\"$instance\", gpu=~\"$gpu\", GPU_I_PROFILE=~\"$mig\"}) BY (instance, gpu)",
          "hide": false,
          "interval": "",
          "legendFormat": "{{instance}}-gpu{{gpu}}-occupancy",
          "range": true,
          "refId": "D"
        }
      ],
      "title": "GPU SM Statistics",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "displayMode": "auto",
            "inspect": false
          },
          "mappings": [
            {
              "options": {
                "match": "null",
                "result": {
                  "index": 0,
                  "text": "0"
                }
              },
              "type": "special"
            },
            {
              "options": {
                "pattern": "nvidia_com_(.*)_shared",
                "result": {
                  "index": 1,
                  "text": "$1.shared"
                }
              },
              "type": "regex"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "none"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Requests"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 75
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Allocatable"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 90
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 0,
        "y": 12
      },
      "id": 31,
      "links": [],
      "maxDataPoints": 100,
      "options": {
        "footer": {
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "frameIndex": 0,
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "9.0.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "exemplar": true,
          "expr": "sum(kube_node_status_allocatable{node=~\"$instance\",resource=~\"nvidia_com_.+_shared\"}) by (node,resource)",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "__auto",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "exemplar": true,
          "expr": "sum(kube_pod_container_resource_limits{node=~\"$instance\",resource=~\"nvidia_com_.+_shared\"}) by (node,resource)",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "__auto",
          "refId": "B"
        }
      ],
      "title": "# Timeslicing",
      "transformations": [
        {
          "id": "seriesToColumns",
          "options": {
            "byField": "resource"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Time": true,
              "Time 1": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "Value": false,
              "__name__": true,
              "container": true,
              "endpoint": true,
              "instance": true,
              "job": true,
              "namespace": true,
              "node": true,
              "pod": true,
              "service": true,
              "unit": true
            },
            "indexByName": {},
            "renameByName": {
              "Value #A": "Allocatable",
              "Value #B": "Requests",
              "Value #C": "Capacity",
              "Value #D": "Use [50%]",
              "Value #E": "USe [>=50%]"
            }
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": true,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "celsius"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 13,
        "x": 0,
        "y": 18
      },
      "id": 2,
      "links": [],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "8.2.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "AVG(DCGM_FI_DEV_GPU_TEMP{instance=~\"$instance\", gpu=~\"$gpu\"}) BY (instance, gpu)",
          "format": "time_series",
          "instant": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{instance}}-gpu{{gpu}}",
          "refId": "A"
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "op": "gt",
          "value": 90,
          "visible": true
        }
      ],
      "title": "GPU Temperature",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 40,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": true,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "watt"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 11,
        "x": 13,
        "y": 18
      },
      "id": 20,
      "links": [],
      "options": {
        "legend": {
          "calcs": [
            "mean",
            "lastNotNull"
          ],
          "displayMode": "hidden",
          "placement": "right"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "8.2.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "AVG(DCGM_FI_DEV_POWER_USAGE{instance=~\"$instance\", gpu=~\"$gpu\"}) BY (instance, gpu)",
          "format": "time_series",
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{instance}}-gpu{{gpu}}",
          "refId": "A"
        }
      ],
      "title": "Power Usage",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": true,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "Bps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 0,
        "y": 25
      },
      "id": 29,
      "links": [],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "hidden",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "8.2.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "-DCGM_FI_PROF_PCIE_TX_BYTES{instance=~\"$instance\", gpu=~\"$gpu\"}",
          "format": "time_series",
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{instance}}-gpu{{gpu}}-Tx",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "DCGM_FI_PROF_PCIE_RX_BYTES{instance=~\"$instance\", gpu=~\"$gpu\"}",
          "format": "time_series",
          "hide": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{instance}}-gpu{{gpu}}-Rx",
          "refId": "B"
        }
      ],
      "title": "PCIe Tx/Rx",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": true,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "max": 1,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 12,
        "y": 25
      },
      "id": 30,
      "links": [],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "hidden",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "8.2.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "AVG(DCGM_FI_PROF_DRAM_ACTIVE{instance=~\"$instance\", gpu=~\"$gpu\"}) BY (instance, gpu)",
          "format": "time_series",
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{instance}}-gpu{{gpu}}",
          "refId": "A"
        }
      ],
      "title": "Cycles Memory Interface",
      "type": "timeseries"
    }
  ],
  "refresh": false,
  "schemaVersion": 36,
  "style": "dark",
  "tags": [
    "gpu",
    "kubernetes-mixin"
  ],
  "templating": {
    "list": [
      {
        "allValue": ".*",
        "current": {},
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "definition": "label_values(DCGM_FI_PROF_GR_ENGINE_ACTIVE, instance)",
        "hide": 0,
        "includeAll": true,
        "label": "",
        "multi": false,
        "name": "instance",
        "options": [],
        "query": {
          "query": "label_values(DCGM_FI_PROF_GR_ENGINE_ACTIVE, instance)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 2,
        "type": "query"
      },
      {
        "allValue": ".*",
        "current": {},
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "definition": "label_values(DCGM_FI_PROF_GR_ENGINE_ACTIVE, gpu)",
        "hide": 0,
        "includeAll": true,
        "multi": false,
        "name": "gpu",
        "options": [],
        "query": {
          "query": "label_values(DCGM_FI_PROF_GR_ENGINE_ACTIVE, gpu)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "type": "query"
      },
      {
        "allValue": ".*",
        "current": {},
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "definition": "label_values(DCGM_FI_PROF_GR_ENGINE_ACTIVE, GPU_I_PROFILE)",
        "hide": 0,
        "includeAll": true,
        "multi": true,
        "name": "mig",
        "options": [],
        "query": {
          "query": "label_values(DCGM_FI_PROF_GR_ENGINE_ACTIVE, GPU_I_PROFILE)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "type": "query"
      }
    ]
  },
  "time": {
    "from": "now-7d",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ],
    "time_options": [
      "5m",
      "15m",
      "1h",
      "6h",
      "12h",
      "24h",
      "2d",
      "7d",
      "30d"
    ]
  },
  "timezone": "",
  "title": "Nvidia GPU",
  "uid": "fnvhnzGmk",
  "version": 2,
  "weekStart": ""
}