Grafana dashboard: Kubernetes Cost Report

-   Kubernetes Cost Report detailed by
    
    -   Cluster
        
        -   OnDemand InstanceType
            
        -   Spot InstanceType
            
        -   Total Daily Cluster Cost (gauge) (timeseries per instance)
            
        -   Idle Daily Cluster Cost (gauge)
            
        -   Available Daily Cluster Cost (gauge)
            
        -   Underutilization Daily Cluster Cost (gauge)
            
        -   Usage Daily Cluster Cost (gauge)
            
        -   Shared Daily Cluster Cost (gauge)
            
        -   Total Monthly Persistent Volume Cost (gauge)
            
        -   Daily Cluster Cost per Instance Type and Capacity Type (bar gauge)
            
        -   Daily Cluster Cost per Instance Type and Capacity Type (graph)
            
        -   Daily Idle Cost per Instance Type and Capacity Type (bar gauge)
            
        -   Daily Idle Cost per Instance Type and Capacity Type (graph)
            
        -   Daily Underutilization Cost per Instance Type and Capacity Type (bar gauge)
            
        -   Daily Underutilization Cost per Instance Type and Capacity Type (graph)
            
        -   Daily Shared Cost per Instance Type and Capacity Type (bar gauge)
            
        -   Daily Shared Cost per Instance Type and Capacity Type (graph)
            
    -   Namespace/Instance
        
        -   Daily Underutilization Cost per Namespace Type (bar gauge)
            
        -   Daily Requests Cost per Namespace Type (bar gauge)
            
        -   Daily Usage Cost per Namespace Type (bar gauge)
            
        -   Daily CPU Underutilization Cost Per Namespace (table)
            
        -   Daily CPU Requests Cost Per Namespace (table)
            
        -   Daily CPU Usage Cost Per Namespace (table)
            
        -   Daily Memory Underutilization Cost Per Namespace (table)
            
        -   Daily Memory Requests Cost Per Namespace (table)
            
        -   Daily Memory Usage Cost Per Namespace (table)
            
        -   Monthly PV Cost Per Namespace (bar gauge)
            
        -   Monthly PV Cost Per Namespace (graph)
            
    -   Pod Per Namespace
        
        -   Daily CPU Underutilization Costs Per Pod (table)
            
        -   Daily CPU Requests Costs Per Pod (table)
            
        -   Daily CPU Usage Costs Per Pod (table)
            
        -   Daily Memory Underutilization Costs Per Pod (table)
            
        -   Daily Memory Requests Costs Per Pod (table)
            
        -   Daily Memory Usage Costs Per Pod (table)
            
        -   CPU Requests vs Usage Per Pod (graph)
            
        -   Memory Requests vs Usage Per Pod (graph)
            
        -   Memory Usage Per Pod
            
        -   Daily CPU Underutilization Costs Per Pod (graph)
            
        -   Daily Memory Underutilization Costs Per Pod (graph)
            
        -   Daily CPU Requests Costs Per Pod (graph)
            
        -   Daily Memory Requests Costs Per Pod (graph)
            
        -   Daily CPU Usage Costs Per Pod (graph)
            
        -   Daily Memory Usage Costs Per Pod (graph)
            
        -   Monthly PV Cost Per Pod

{
  "__inputs": [
    {
      "name": "DS_PROMETHEUS",
      "label": "Prometheus",
      "description": "",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    }
  ],
  "__requires": [
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "8.3.1"
    },
    {
      "type": "panel",
      "id": "graph",
      "name": "Graph",
      "version": ""
    },
    {
      "type": "datasource",
      "id": "prometheus",
      "name": "Prometheus",
      "version": "1.0.0"
    },
    {
      "type": "panel",
      "id": "grafana-piechart-panel",
      "name": "Pie Chart",
      "version": ""
    },
    {
      "type": "panel",
      "id": "singlestat",
      "name": "Singlestat",
      "version": ""
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "target": {
          "limit": 100,
          "matchAny": false,
          "tags": [],
          "type": "dashboard"
        },
        "type": "dashboard"
      }
    ]
  },
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 0,
  "iteration": 1646930883779,
  "links": [],
  "liveNow": true,
  "panels": [
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 19,
      "panels": [],
      "title": "Cluster (Global)",
      "type": "row"
    },
    {
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 1
      },
      "id": 26,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull"
          ],
          "displayMode": "table",
          "placement": "right"
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": true,
          "expr": "sort_desc (\n  sum by (label_beta_kubernetes_io_instance_type) (kube_node_labels{label_eks_amazonaws_com_capacity_type=\"ON_DEMAND\"})\n)",
          "instant": false,
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "On Demand Instance Type",
      "transformations": [
        {
          "id": "labelsToFields",
          "options": {
            "valueLabel": "label_beta_kubernetes_io_instance_type"
          }
        }
      ],
      "type": "timeseries"
    },
    {
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 1
      },
      "id": 2,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull"
          ],
          "displayMode": "table",
          "placement": "right"
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": true,
          "expr": "sort_desc (\n  sum by (label_beta_kubernetes_io_instance_type) (kube_node_labels{label_eks_amazonaws_com_capacity_type=\"SPOT\"})\n)",
          "instant": false,
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Spot Instance Type",
      "transformations": [
        {
          "id": "labelsToFields",
          "options": {
            "valueLabel": "label_beta_kubernetes_io_instance_type"
          }
        }
      ],
      "type": "timeseries"
    },
    {
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "decimals": 2,
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "currencyUSD"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 4,
        "x": 0,
        "y": 9
      },
      "id": 43,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "text": {
          "valueSize": 100
        },
        "textMode": "auto"
      },
      "pluginVersion": "8.3.1",
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": false,
          "expr": "sum(\n  sum_over_time(\n    (\n    capacity_instance:spot_instance_cost:cost \n    OR \n    capacity_instance:on_demand_instance_cost:cost\n    )\n  [1d:1h])\n)",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "Total",
          "refId": "A"
        }
      ],
      "title": "Total Daily Cluster Cost",
      "transformations": [
        {
          "id": "seriesToColumns",
          "options": {
            "byField": "Time"
          }
        }
      ],
      "type": "stat"
    },
    {
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "orange",
                "value": 0
              }
            ]
          },
          "unit": "currencyUSD"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 8,
        "x": 4,
        "y": 9
      },
      "id": 53,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "text": {
          "valueSize": 100
        },
        "textMode": "auto"
      },
      "pluginVersion": "8.3.1",
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": false,
          "expr": "sum (\n  sum_over_time(\n    (\n      zone_capacity_instance_node_resource:kube_node_status_allocatable_idle_instance_cpu_price:spot_idle_cpu_cost\n      OR\n      capacity_instance_node_resource:kube_node_status_allocatable_idle_instance_cpu_price:on_demand_idle_cpu_cost\n      OR\n      zone_capacity_instance_node_resource:kube_node_status_allocatable_idle_instance_mem_price:spot_idle_mem_cost\n      OR\n      capacity_instance_node_resource:kube_node_status_allocatable_idle_instance_mem_price:on_demand_idle_mem_cost\n    )\n  [1d:1h])\n)",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "Total",
          "refId": "A"
        }
      ],
      "title": "Idle Daily Cluster Cost",
      "transformations": [
        {
          "id": "seriesToColumns",
          "options": {
            "byField": "Time"
          }
        }
      ],
      "type": "stat"
    },
    {
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "currencyUSD"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 4,
        "x": 12,
        "y": 9
      },
      "id": 61,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "8.3.1",
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": false,
          "expr": "sum_over_time(\n  sum (\n  \n      # See description for label_replace\n      label_replace(\n        zone_capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:spot_pod_cpu_usage_cost\n        OR\n        capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:on_demand_pod_cpu_usage_cost\n      , \"resource\", \"CPU\", \"\",\"\")\n      \n      OR \n      \n      label_replace(\n        zone_capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:spot_pod_mem_usage_cost\n        OR\n        capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:on_demand_pod_mem_usage_cost\n      , \"resource\", \"memory\", \"\",\"\")\n\n)\n[1d:1h])",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "Total",
          "refId": "A"
        }
      ],
      "title": "Usage Daily Cluster Cost",
      "transformations": [
        {
          "id": "seriesToColumns",
          "options": {
            "byField": "Time"
          }
        }
      ],
      "type": "stat"
    },
    {
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "decimals": 2,
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "currencyUSD"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 4,
        "x": 16,
        "y": 9
      },
      "id": 52,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "8.3.1",
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": false,
          "expr": "sum (\n  sum_over_time(\n    (\n          zone_capacity_instance_node_resource:kube_node_status_shared_instance_cpu_price:spot_shared_cpu_cost\n          OR\n          capacity_instance_node_resource:kube_node_status_shared_instance_cpu_price:on_demand_shared_cpu_cost\n          OR\n          zone_capacity_instance_node_resource:kube_node_status_shared_instance_mem_price:spot_shared_mem_cost\n          OR\n          capacity_instance_node_resource:kube_node_status_shared_instance_mem_price:on_demand_shared_mem_cost\n    )\n  [1d:1h])\n)",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "Total",
          "refId": "A"
        }
      ],
      "title": "Shared Daily Cluster Cost",
      "transformations": [
        {
          "id": "seriesToColumns",
          "options": {
            "byField": "Time"
          }
        }
      ],
      "type": "stat"
    },
    {
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "decgbytes"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Cost"
            },
            "properties": [
              {
                "id": "unit",
                "value": "currencyUSD"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Total Cost"
            },
            "properties": [
              {
                "id": "unit",
                "value": "currencyUSD"
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 4,
        "x": 20,
        "y": 9
      },
      "id": 24,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "8.3.1",
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": false,
          "expr": "sum (sum by (persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes/1024/1024/1024))",
          "format": "table",
          "instant": true,
          "interval": "",
          "intervalFactor": 2,
          "legendFormat": "{{persistentvolumeclaim}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${promdatasource}"
          },
          "exemplar": false,
          "expr": "sum(sum by (persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes/1024/1024/1024) * $pvcost)",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "intervalFactor": 2,
          "legendFormat": "",
          "refId": "B"
        }
      ],
      "title": "Total Monthly PV Cost ",
      "transformations": [
        {
          "id": "seriesToColumns",
          "options": {
            "byField": "Time"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Time": true,
              "Value #A": true
            },
            "indexByName": {},
            "renameByName": {
              "Value #A": "Total Size ",
              "Value #B": "Total Cost"
            }
          }
        }
      ],
      "type": "stat"
    },
    {
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "orange",
                "value": 0
              }
            ]
          },
          "unit": "currencyUSD"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 4,
        "y": 13
      },
      "id": 62,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "text": {
          "valueSize": 100
        },
        "textMode": "auto"
      },
      "pluginVersion": "8.3.1",
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": false,
          "expr": "sum (\n  sum_over_time(\n    (\n      zone_capacity_instance_node_resource:kube_node_status_allocatable_idle_instance_cpu_price:spot_idle_cpu_cost\n      OR\n      capacity_instance_node_resource:kube_node_status_allocatable_idle_instance_cpu_price:on_demand_idle_cpu_cost\n      OR\n      zone_capacity_instance_node_resource:kube_node_status_allocatable_idle_instance_mem_price:spot_idle_mem_cost\n      OR\n      capacity_instance_node_resource:kube_node_status_allocatable_idle_instance_mem_price:on_demand_idle_mem_cost\n    )\n  [1d:1h])\n)\n-\nsum_over_time(\n  sum (\n  \n      # See description for label_replace\n      label_replace(\n          ((\n              zone_capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:spot_pod_cpu_requests_cost\n              - \n              zone_capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:spot_pod_cpu_usage_cost\n          ) > 0)\n          OR\n          ((\n              capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:on_demand_pod_cpu_requests_cost\n              -\n              capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:on_demand_pod_cpu_usage_cost\n          ) > 0)\n      , \"resource\", \"CPU\", \"\",\"\")\n      \n      OR \n      \n      label_replace(\n          ((\n              zone_capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:spot_pod_mem_requests_cost\n              -\n              zone_capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:spot_pod_mem_usage_cost\n          ) > 0)\n          OR\n          ((\n              capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:on_demand_pod_mem_requests_cost\n              -\n              capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:on_demand_pod_mem_usage_cost\n          ) > 0)\n      , \"resource\", \"memory\", \"\",\"\")\n\n)\n[1d:1h])",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "Total",
          "refId": "A"
        }
      ],
      "title": "Available Daily Cluster Cost",
      "transformations": [
        {
          "id": "seriesToColumns",
          "options": {
            "byField": "Time"
          }
        }
      ],
      "type": "stat"
    },
    {
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 0
              }
            ]
          },
          "unit": "currencyUSD"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 8,
        "y": 13
      },
      "id": 60,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "text": {
          "valueSize": 100
        },
        "textMode": "auto"
      },
      "pluginVersion": "8.3.1",
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": false,
          "expr": "sum_over_time(\n  sum (\n  \n      # See description for label_replace\n      label_replace(\n          ((\n              zone_capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:spot_pod_cpu_requests_cost\n              - \n              zone_capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:spot_pod_cpu_usage_cost\n          ) > 0)\n          OR\n          ((\n              capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:on_demand_pod_cpu_requests_cost\n              -\n              capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:on_demand_pod_cpu_usage_cost\n          ) > 0)\n      , \"resource\", \"CPU\", \"\",\"\")\n      \n      OR \n      \n      label_replace(\n          ((\n              zone_capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:spot_pod_mem_requests_cost\n              -\n              zone_capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:spot_pod_mem_usage_cost\n          ) > 0)\n          OR\n          ((\n              capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:on_demand_pod_mem_requests_cost\n              -\n              capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:on_demand_pod_mem_usage_cost\n          ) > 0)\n      , \"resource\", \"memory\", \"\",\"\")\n\n)\n[1d:1h])",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "Total",
          "refId": "A"
        }
      ],
      "title": "Underutilization Daily Cluster Cost",
      "transformations": [
        {
          "id": "seriesToColumns",
          "options": {
            "byField": "Time"
          }
        }
      ],
      "type": "stat"
    },
    {
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "currencyUSD"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 0,
        "y": 17
      },
      "id": 46,
      "options": {
        "displayMode": "gradient",
        "orientation": "vertical",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showUnfilled": false
      },
      "pluginVersion": "8.3.1",
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": false,
          "expr": "sort_desc(\n  sum_over_time(\n    (    \n    capacity_instance:spot_instance_cost:cost \n    OR \n    capacity_instance:on_demand_instance_cost:cost\n    )\n  [1d:1h])\n)",
          "format": "time_series",
          "instant": true,
          "interval": "",
          "legendFormat": "{{label_beta_kubernetes_io_instance_type}}-{{label_eks_amazonaws_com_capacity_type}}",
          "refId": "A"
        }
      ],
      "title": "Daily Cluster Cost per Instance Type and Capacity Type",
      "transformations": [],
      "type": "bargauge"
    },
    {
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "currencyEUR"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 12,
        "y": 17
      },
      "id": 44,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull"
          ],
          "displayMode": "table",
          "placement": "right",
          "sortBy": "Last *",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "pluginVersion": "8.2.3",
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": true,
          "expr": "sum_over_time(\n  (\n    capacity_instance:spot_instance_cost:cost OR capacity_instance:on_demand_instance_cost:cost\n  )\n[1d:1h])",
          "hide": false,
          "instant": false,
          "interval": "",
          "legendFormat": "{{label_beta_kubernetes_io_instance_type}}-{{label_eks_amazonaws_com_capacity_type}}",
          "refId": "A"
        },
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": true,
          "expr": "sum(\n  sum_over_time(\n    (\n    capacity_instance:spot_instance_cost:cost OR capacity_instance:on_demand_instance_cost:cost\n    )\n  [1d:1h])\n)",
          "hide": false,
          "interval": "",
          "legendFormat": "Total",
          "refId": "B"
        }
      ],
      "title": "Daily Cluster Cost per Instance and Capacity Type",
      "transformations": [],
      "type": "timeseries"
    },
    {
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "currencyUSD"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 0,
        "y": 26
      },
      "id": 54,
      "options": {
        "displayMode": "gradient",
        "orientation": "vertical",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showUnfilled": false
      },
      "pluginVersion": "8.3.1",
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": false,
          "expr": "sort_desc(\n  sum by (label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type) (\n    sum_over_time(\n      (\n        zone_capacity_instance_node_resource:kube_node_status_allocatable_idle_instance_cpu_price:spot_idle_cpu_cost\n        OR\n        capacity_instance_node_resource:kube_node_status_allocatable_idle_instance_cpu_price:on_demand_idle_cpu_cost\n        OR\n        zone_capacity_instance_node_resource:kube_node_status_allocatable_idle_instance_mem_price:spot_idle_mem_cost\n        OR\n        capacity_instance_node_resource:kube_node_status_allocatable_idle_instance_mem_price:on_demand_idle_mem_cost\n      )\n    [1d:1h])\n)\n)",
          "format": "time_series",
          "instant": true,
          "interval": "",
          "legendFormat": "{{label_beta_kubernetes_io_instance_type}}-{{label_eks_amazonaws_com_capacity_type}}",
          "refId": "A"
        }
      ],
      "title": "Daily Idle Cost per Instance Type and Capacity Type",
      "transformations": [],
      "type": "bargauge"
    },
    {
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "currencyEUR"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 12,
        "y": 26
      },
      "id": 51,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull"
          ],
          "displayMode": "table",
          "placement": "right",
          "sortBy": "Last *",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "pluginVersion": "8.2.3",
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": true,
          "expr": "sum by (label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type) (\n  sum_over_time(\n    (\n      zone_capacity_instance_node_resource:kube_node_status_allocatable_idle_instance_cpu_price:spot_idle_cpu_cost\n      OR\n      capacity_instance_node_resource:kube_node_status_allocatable_idle_instance_cpu_price:on_demand_idle_cpu_cost\n      OR\n      zone_capacity_instance_node_resource:kube_node_status_allocatable_idle_instance_mem_price:spot_idle_mem_cost\n      OR\n      capacity_instance_node_resource:kube_node_status_allocatable_idle_instance_mem_price:on_demand_idle_mem_cost\n    )\n  [1d:1h])\n)",
          "hide": false,
          "instant": false,
          "interval": "",
          "legendFormat": "{{label_beta_kubernetes_io_instance_type}}-{{label_eks_amazonaws_com_capacity_type}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${promdatasource}"
          },
          "exemplar": true,
          "expr": "sum (\n  sum_over_time(\n    (\n      zone_capacity_instance_node_resource:kube_node_status_allocatable_idle_instance_cpu_price:spot_idle_cpu_cost\n      OR\n      capacity_instance_node_resource:kube_node_status_allocatable_idle_instance_cpu_price:on_demand_idle_cpu_cost\n      OR\n      zone_capacity_instance_node_resource:kube_node_status_allocatable_idle_instance_mem_price:spot_idle_mem_cost\n      OR\n      capacity_instance_node_resource:kube_node_status_allocatable_idle_instance_mem_price:on_demand_idle_mem_cost\n    )\n  [1d:1h])\n)",
          "hide": false,
          "interval": "",
          "legendFormat": "Total",
          "refId": "B"
        }
      ],
      "title": "Daily Idle Cost per Instance and Capacity Type",
      "transformations": [],
      "type": "timeseries"
    },
    {
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "currencyUSD"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 0,
        "y": 35
      },
      "id": 59,
      "options": {
        "displayMode": "gradient",
        "orientation": "vertical",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showUnfilled": false
      },
      "pluginVersion": "8.3.1",
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": false,
          "expr": "sort_desc (\n  sum_over_time(\n      sum by (label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type) (\n      \n\t      # See description for label_replace\n\t      label_replace(\n\t          ((\n\t              zone_capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:spot_pod_cpu_requests_cost\n\t              - \n\t              zone_capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:spot_pod_cpu_usage_cost\n\t          ) > 0)\n\t          OR\n\t          ((\n\t              capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:on_demand_pod_cpu_requests_cost\n\t              -\n\t              capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:on_demand_pod_cpu_usage_cost\n\t          ) > 0)\n\t      , \"resource\", \"CPU\", \"\",\"\")\n\t      \n\t      OR \n\t      \n\t      label_replace(\n\t          ((\n\t              zone_capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:spot_pod_mem_requests_cost\n\t              -\n\t              zone_capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:spot_pod_mem_usage_cost\n\t          ) > 0)\n\t          OR\n\t          ((\n\t              capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:on_demand_pod_mem_requests_cost\n\t              -\n\t              capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:on_demand_pod_mem_requests_cost\n\t          ) > 0)\n\t      , \"resource\", \"memory\", \"\",\"\")\n\n    )\n  [1d:1h])\n)",
          "format": "time_series",
          "instant": true,
          "interval": "",
          "legendFormat": "{{label_beta_kubernetes_io_instance_type}}-{{label_eks_amazonaws_com_capacity_type}}",
          "refId": "A"
        }
      ],
      "title": "Daily Underutilization Cost per Instance Type and Capacity Type",
      "transformations": [],
      "type": "bargauge"
    },
    {
      "description": "label_replace is used to ALWAYS have a different label (resource) when doing OR, so that no time series is filtered by binary operations (capacity_type for internal OR, resource for external OR)\n\nIf for example a pod has CPU requests but not memory requests, the binary sum (+) will filter the CPU sum (will not count for them)",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "currencyUSD"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 12,
        "y": 35
      },
      "id": 47,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull"
          ],
          "displayMode": "table",
          "placement": "right",
          "sortBy": "Last *",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "pluginVersion": "8.3.1",
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": true,
          "expr": "sum_over_time(\n  sum by (label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type) (\n  \n      # See description for label_replace\n      label_replace(\n          ((\n              zone_capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:spot_pod_cpu_requests_cost\n              - \n              zone_capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:spot_pod_cpu_usage_cost\n          ) > 0)\n          OR\n          ((\n              capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:on_demand_pod_cpu_requests_cost\n              -\n              capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:on_demand_pod_cpu_usage_cost\n          ) > 0)\n      , \"resource\", \"CPU\", \"\",\"\")\n      \n      OR \n      \n      label_replace(\n          ((\n              zone_capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:spot_pod_mem_requests_cost\n              -\n              zone_capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:spot_pod_mem_usage_cost\n          ) > 0)\n          OR\n          ((\n              capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:on_demand_pod_mem_requests_cost\n              -\n              capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:on_demand_pod_mem_requests_cost\n          ) > 0)\n      , \"resource\", \"memory\", \"\",\"\")\n\n)\n[1d:1h])",
          "format": "time_series",
          "instant": false,
          "interval": "",
          "legendFormat": "{{label_beta_kubernetes_io_instance_type}}-{{label_eks_amazonaws_com_capacity_type}}",
          "refId": "A"
        },
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": true,
          "expr": "sum_over_time(\n  sum (\n  \n      # See description for label_replace\n      label_replace(\n          ((\n              zone_capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:spot_pod_cpu_requests_cost\n              - \n              zone_capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:spot_pod_cpu_usage_cost\n          ) > 0)\n          OR\n          ((\n              capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:on_demand_pod_cpu_requests_cost\n              -\n              capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:on_demand_pod_cpu_usage_cost\n          ) > 0)\n      , \"resource\", \"CPU\", \"\",\"\")\n      \n      OR \n      \n      label_replace(\n          ((\n              zone_capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:spot_pod_mem_requests_cost\n              -\n              zone_capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:spot_pod_mem_usage_cost\n          ) > 0)\n          OR\n          ((\n              capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:on_demand_pod_mem_requests_cost\n              -\n              capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:on_demand_pod_mem_requests_cost\n          ) > 0)\n      , \"resource\", \"memory\", \"\",\"\")\n\n)\n[1d:1h])",
          "hide": false,
          "interval": "",
          "legendFormat": "Total",
          "refId": "B"
        }
      ],
      "title": "Daily Underutilization Cost per Instance Type and Capacity Type",
      "transformations": [],
      "type": "timeseries"
    },
    {
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "currencyUSD"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 0,
        "y": 44
      },
      "id": 55,
      "options": {
        "displayMode": "gradient",
        "orientation": "vertical",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showUnfilled": false
      },
      "pluginVersion": "8.3.1",
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": false,
          "expr": "sort_desc(\n  sum by (label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type) (\n    sum_over_time(\n      (\n            zone_capacity_instance_node_resource:kube_node_status_shared_instance_cpu_price:spot_shared_cpu_cost\n            OR\n            capacity_instance_node_resource:kube_node_status_shared_instance_cpu_price:on_demand_shared_cpu_cost\n            OR\n            zone_capacity_instance_node_resource:kube_node_status_shared_instance_mem_price:spot_shared_mem_cost\n            OR\n            capacity_instance_node_resource:kube_node_status_shared_instance_mem_price:on_demand_shared_mem_cost\n      )\n    [1d:1h])\n  )\n)",
          "format": "time_series",
          "instant": true,
          "interval": "",
          "legendFormat": "{{label_beta_kubernetes_io_instance_type}}-{{label_eks_amazonaws_com_capacity_type}}",
          "refId": "A"
        }
      ],
      "title": "Daily Shared Cost per Instance Type and Capacity Type",
      "transformations": [],
      "type": "bargauge"
    },
    {
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "currencyEUR"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 12,
        "y": 44
      },
      "id": 50,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull"
          ],
          "displayMode": "table",
          "placement": "right",
          "sortBy": "Last *",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "pluginVersion": "8.2.3",
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": true,
          "expr": "sum by (label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type) (\n  sum_over_time(\n    (\n          zone_capacity_instance_node_resource:kube_node_status_shared_instance_cpu_price:spot_shared_cpu_cost\n          OR\n          capacity_instance_node_resource:kube_node_status_shared_instance_cpu_price:on_demand_shared_cpu_cost\n          OR\n          zone_capacity_instance_node_resource:kube_node_status_shared_instance_mem_price:spot_shared_mem_cost\n          OR\n          capacity_instance_node_resource:kube_node_status_shared_instance_mem_price:on_demand_shared_mem_cost\n    )\n  [1d:1h])\n)",
          "hide": false,
          "instant": false,
          "interval": "",
          "legendFormat": "{{label_beta_kubernetes_io_instance_type}}-{{label_eks_amazonaws_com_capacity_type}}",
          "refId": "A"
        },
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": true,
          "expr": "sum (\n  sum_over_time(\n    (\n          zone_capacity_instance_node_resource:kube_node_status_shared_instance_cpu_price:spot_shared_cpu_cost\n          OR\n          capacity_instance_node_resource:kube_node_status_shared_instance_cpu_price:on_demand_shared_cpu_cost\n          OR\n          zone_capacity_instance_node_resource:kube_node_status_shared_instance_mem_price:spot_shared_mem_cost\n          OR\n          capacity_instance_node_resource:kube_node_status_shared_instance_mem_price:on_demand_shared_mem_cost\n    )\n  [1d:1h])\n)",
          "hide": false,
          "interval": "",
          "legendFormat": "Total",
          "refId": "B"
        }
      ],
      "title": "Daily Shared Cost per Instance and Capacity Type",
      "transformations": [],
      "type": "timeseries"
    },
    {
      "description": "label_replace is used to ALWAYS have a different label (resource) when doing OR, so that no time series is filtered by binary operations (capacity_type for internal OR, resource for external OR)\n\nIf for example a pod has CPU requests but not memory requests, the binary sum (+) will filter the CPU sum (will not count for them)",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "decimals": 2,
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "currencyUSD"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 9,
        "w": 24,
        "x": 0,
        "y": 53
      },
      "id": 74,
      "options": {
        "displayMode": "gradient",
        "orientation": "vertical",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showUnfilled": false
      },
      "pluginVersion": "8.3.1",
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": false,
          "expr": "sort_desc (\nsum_over_time(\n  sum by (namespace) (\n  \n      # See description for label_replace\n      label_replace(\n          ((\n              zone_capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:spot_pod_cpu_requests_cost\n              - \n              zone_capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:spot_pod_cpu_usage_cost\n          ) > 0)\n          OR\n          ((\n              capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:on_demand_pod_cpu_requests_cost\n              -\n              capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:on_demand_pod_cpu_usage_cost\n          ) > 0)\n      , \"resource\", \"CPU\", \"\",\"\")\n      \n      OR \n      \n      label_replace(\n          ((\n              zone_capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:spot_pod_mem_requests_cost\n              -\n              zone_capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:spot_pod_mem_usage_cost\n          ) > 0)\n          OR\n          ((\n              capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:on_demand_pod_mem_requests_cost\n              -\n              capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:on_demand_pod_mem_usage_cost\n          ) > 0)\n      , \"resource\", \"memory\", \"\",\"\")\n\n)\n[1d:1h])\n)",
          "format": "time_series",
          "instant": true,
          "interval": "",
          "legendFormat": "{{namespace}}",
          "refId": "A"
        }
      ],
      "title": "Daily Total Cost per Namespace",
      "transformations": [
        {
          "id": "seriesToColumns",
          "options": {
            "byField": "Time"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Time": true
            },
            "indexByName": {},
            "renameByName": {
              "Value #A": "CPU Cost Per Day"
            }
          }
        },
        {
          "id": "sortBy",
          "options": {
            "fields": {},
            "sort": [
              {
                "desc": true,
                "field": "CPU Cost Per Day"
              }
            ]
          }
        }
      ],
      "type": "bargauge"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 62
      },
      "id": 57,
      "panels": [
        {
          "description": "label_replace is used to ALWAYS have a different label (resource) when doing OR, so that no time series is filtered by binary operations (capacity_type for internal OR, resource for external OR)\n\nIf for example a pod has CPU requests but not memory requests, the binary sum (+) will filter the CPU sum (will not count for them)",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "decimals": 2,
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 9,
            "w": 24,
            "x": 0,
            "y": 2
          },
          "id": 79,
          "options": {
            "displayMode": "gradient",
            "orientation": "vertical",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "showUnfilled": false
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "sort_desc (\nsum_over_time(\n  sum by (namespace) (\n  \n      # See description for label_replace\n      label_replace(\n          # If requests is bigger than usage, use requests. Otherwise use usage.\n          (\n            (\n                zone_capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:spot_pod_cpu_requests_cost\n                > \n                zone_capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:spot_pod_cpu_usage_cost\n            )\n            OR\n            zone_capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:spot_pod_cpu_usage_cost\n          )\n          OR\n          # If requests is bigger than usage, use requests. Otherwise use usage.\n          (\n            (\n                capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:on_demand_pod_cpu_requests_cost\n                >\n                capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:on_demand_pod_cpu_usage_cost\n            )\n            OR\n            zone_capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:spot_pod_cpu_usage_cost\n          )\n      , \"resource\", \"CPU\", \"\",\"\")\n      \n      OR \n      \n      label_replace(\n          # If requests is bigger than usage, use requests. Otherwise use usage.\n          (\n            (\n                zone_capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:spot_pod_mem_requests_cost\n                >\n                zone_capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:spot_pod_mem_usage_cost\n            )\n            OR\n            zone_capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:spot_pod_mem_usage_cost\n          )\n          OR\n          # If requests is bigger than usage, use requests. Otherwise use usage.\n          (\n            (\n                capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:on_demand_pod_mem_requests_cost\n                >\n                capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:on_demand_pod_mem_usage_cost\n            ) \n            OR capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:on_demand_pod_mem_usage_cost\n          )\n      , \"resource\", \"memory\", \"\",\"\")\n\n)\n[1d:1h])\n)",
              "format": "time_series",
              "instant": true,
              "interval": "",
              "legendFormat": "{{namespace}}",
              "refId": "A"
            }
          ],
          "title": "Daily Total Cost per Namespace",
          "transformations": [
            {
              "id": "seriesToColumns",
              "options": {
                "byField": "Time"
              }
            },
            {
              "id": "organize",
              "options": {
                "excludeByName": {
                  "Time": true
                },
                "indexByName": {},
                "renameByName": {
                  "Value #A": "CPU Cost Per Day"
                }
              }
            },
            {
              "id": "sortBy",
              "options": {
                "fields": {},
                "sort": [
                  {
                    "desc": true,
                    "field": "CPU Cost Per Day"
                  }
                ]
              }
            }
          ],
          "type": "bargauge"
        },
        {
          "description": "label_replace is used to ALWAYS have a different label (resource) when doing OR, so that no time series is filtered by binary operations (capacity_type for internal OR, resource for external OR)\n\nIf for example a pod has CPU requests but not memory requests, the binary sum (+) will filter the CPU sum (will not count for them)",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "decimals": 2,
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 9,
            "w": 24,
            "x": 0,
            "y": 11
          },
          "id": 63,
          "options": {
            "displayMode": "gradient",
            "orientation": "vertical",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "showUnfilled": false
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "sort_desc (\nsum_over_time(\n  sum by (namespace) (\n  \n      # See description for label_replace\n      label_replace(\n          ((\n              zone_capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:spot_pod_cpu_requests_cost\n              - \n              zone_capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:spot_pod_cpu_usage_cost\n          ) > 0)\n          OR\n          ((\n              capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:on_demand_pod_cpu_requests_cost\n              -\n              capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:on_demand_pod_cpu_usage_cost\n          ) > 0)\n      , \"resource\", \"CPU\", \"\",\"\")\n      \n      OR \n      \n      label_replace(\n          ((\n              zone_capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:spot_pod_mem_requests_cost\n              -\n              zone_capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:spot_pod_mem_usage_cost\n          ) > 0)\n          OR\n          ((\n              capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:on_demand_pod_mem_requests_cost\n              -\n              capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:on_demand_pod_mem_usage_cost\n          ) > 0)\n      , \"resource\", \"memory\", \"\",\"\")\n\n)\n[1d:1h])\n)",
              "format": "time_series",
              "instant": true,
              "interval": "",
              "legendFormat": "{{namespace}}",
              "refId": "A"
            }
          ],
          "title": "Daily Underutilization Cost per Namespace",
          "transformations": [
            {
              "id": "seriesToColumns",
              "options": {
                "byField": "Time"
              }
            },
            {
              "id": "organize",
              "options": {
                "excludeByName": {
                  "Time": true
                },
                "indexByName": {},
                "renameByName": {
                  "Value #A": "CPU Cost Per Day"
                }
              }
            },
            {
              "id": "sortBy",
              "options": {
                "fields": {},
                "sort": [
                  {
                    "desc": true,
                    "field": "CPU Cost Per Day"
                  }
                ]
              }
            }
          ],
          "type": "bargauge"
        },
        {
          "description": "label_replace is used to ALWAYS have a different label (resource) when doing OR, so that no time series is filtered by binary operations (capacity_type for internal OR, resource for external OR)\n\nIf for example a pod has CPU requests but not memory requests, the binary sum (+) will filter the CPU sum (will not count for them)",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "decimals": 2,
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 9,
            "w": 24,
            "x": 0,
            "y": 20
          },
          "id": 58,
          "options": {
            "displayMode": "gradient",
            "orientation": "vertical",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "showUnfilled": false
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "sort_desc (\nsum_over_time(\n    sum by (namespace) (\n        # See description for label_replace\n        label_replace(\n\t\t    zone_capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:spot_pod_cpu_requests_cost\n            OR\n            capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:on_demand_pod_cpu_requests_cost\n        , \"resource\", \"CPU\", \"\",\"\")\n        \n        OR \n        \n        label_replace(\n\t\t    zone_capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:spot_pod_mem_requests_cost\n            OR\n            capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:on_demand_pod_mem_requests_cost\n        , \"resource\", \"memory\", \"\",\"\")\n\n\t)\n[1d:1h])\n)",
              "format": "time_series",
              "instant": true,
              "interval": "",
              "legendFormat": "{{namespace}}",
              "refId": "A"
            }
          ],
          "title": "Daily Requests Cost per Namespace",
          "transformations": [
            {
              "id": "seriesToColumns",
              "options": {
                "byField": "Time"
              }
            },
            {
              "id": "organize",
              "options": {
                "excludeByName": {
                  "Time": true
                },
                "indexByName": {},
                "renameByName": {
                  "Value #A": "CPU Cost Per Day"
                }
              }
            },
            {
              "id": "sortBy",
              "options": {
                "fields": {},
                "sort": [
                  {
                    "desc": true,
                    "field": "CPU Cost Per Day"
                  }
                ]
              }
            }
          ],
          "type": "bargauge"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 29
          },
          "id": 34,
          "options": {
            "displayMode": "gradient",
            "orientation": "vertical",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "showUnfilled": false
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "sort_desc(\nsum_over_time(\n    sum by (namespace) (\n        # See description for label_replace\n        label_replace(\n\t\t    zone_capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:spot_pod_cpu_usage_cost\n            OR\n            capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:on_demand_pod_cpu_usage_cost\n        , \"resource\", \"CPU\", \"\",\"\")\n        \n        OR \n        \n        label_replace(\n\t\t    zone_capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:spot_pod_mem_usage_cost\n            OR\n            capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:on_demand_pod_mem_usage_cost\n        , \"resource\", \"memory\", \"\",\"\")\n\n\t)\n[1d:1h])\n)",
              "format": "time_series",
              "instant": true,
              "interval": "",
              "legendFormat": "{{namespace}}",
              "refId": "A"
            }
          ],
          "title": "Daily Usage Cost per Namespace ",
          "transformations": [
            {
              "id": "seriesToColumns",
              "options": {
                "byField": "Time"
              }
            }
          ],
          "type": "bargauge"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "custom": {
                "align": "auto",
                "displayMode": "auto",
                "filterable": false
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 8,
            "x": 0,
            "y": 37
          },
          "id": 65,
          "options": {
            "footer": {
              "fields": "",
              "reducer": [
                "sum"
              ],
              "show": false
            },
            "showHeader": true,
            "sortBy": []
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "sort_desc(\n  sum_over_time(\n    sum by (namespace) (\n      label_replace(((\n          (sum by (namespace) (zone_capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:spot_pod_cpu_requests_cost))\n          -\n          (sum by (namespace) (zone_capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:spot_pod_cpu_usage_cost))\n      ) > 0), \"cap\", \"SPOT\", \"\",\"\")\n      OR\n      label_replace(((\n          (sum by (namespace) (capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:on_demand_pod_cpu_requests_cost))\n          -\n          (sum by (namespace) (capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:on_demand_pod_cpu_usage_cost))\n      ) > 0), \"cap\", \"ON_DEMAND\", \"\",\"\")\n  )[1d:1h])\n)",
              "format": "table",
              "instant": true,
              "interval": "",
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "title": "Daily CPU Underutilization Cost Per Namespace",
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {
                  "Time": true
                },
                "indexByName": {},
                "renameByName": {
                  "Value #A": "Daily CPU Underutilization Cost",
                  "namespace": "Namespace"
                }
              }
            }
          ],
          "type": "table"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "custom": {
                "align": "auto",
                "displayMode": "auto",
                "filterable": false
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 8,
            "x": 8,
            "y": 37
          },
          "id": 30,
          "options": {
            "footer": {
              "fields": "",
              "reducer": [
                "sum"
              ],
              "show": false
            },
            "showHeader": true,
            "sortBy": []
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "sort_desc(\n\tsum_over_time(\n\t\tsum by (namespace) (\n\n\t\t\tzone_capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:spot_pod_cpu_requests_cost\n\t\t\tOR\n\t\t\tcapacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:on_demand_pod_cpu_requests_cost\n\n\t\t)\n\t[1d:1h])\n)",
              "format": "table",
              "instant": true,
              "interval": "",
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "title": "Daily CPU Requests Cost Per Namespace",
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {
                  "Time": true
                },
                "indexByName": {},
                "renameByName": {
                  "Value #A": "Daily CPU Requests Cost ",
                  "namespace": "Namespace"
                }
              }
            }
          ],
          "type": "table"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "custom": {
                "align": "auto",
                "displayMode": "auto",
                "filterable": false
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 8,
            "x": 16,
            "y": 37
          },
          "id": 31,
          "options": {
            "footer": {
              "fields": "",
              "reducer": [
                "sum"
              ],
              "show": false
            },
            "showHeader": true,
            "sortBy": []
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${promdatasource}"
              },
              "exemplar": false,
              "expr": "sort_desc(\n\tsum_over_time(\n\t\tsum by (namespace) (\n\n\t\t\tzone_capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:spot_pod_cpu_usage_cost\n\t\t\tOR\n\t\t\tcapacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:on_demand_pod_cpu_usage_cost\n\n\t\t)\n\t[1d:1h])\n)",
              "format": "table",
              "instant": true,
              "interval": "",
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "title": "Daily CPU Usage Cost Per Namespace ",
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {
                  "Time": true
                },
                "indexByName": {},
                "renameByName": {
                  "Value #A": "Daily CPU Usage Cost",
                  "namespace": "Namespace"
                }
              }
            }
          ],
          "type": "table"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "custom": {
                "align": "auto",
                "displayMode": "auto",
                "filterable": false
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 8,
            "x": 0,
            "y": 45
          },
          "id": 64,
          "options": {
            "footer": {
              "fields": "",
              "reducer": [
                "sum"
              ],
              "show": false
            },
            "showHeader": true,
            "sortBy": []
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "sort_desc(\n  sum_over_time(\n    sum by (namespace) (\n      label_replace(((\n          (sum by (namespace) (zone_capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:spot_pod_mem_requests_cost))\n          -\n          (sum by (namespace) (zone_capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:spot_pod_mem_usage_cost))\n      ) > 0), \"cap\", \"SPOT\", \"\",\"\")\n      OR\n      label_replace(((\n          (sum by (namespace) (capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:on_demand_pod_mem_requests_cost))\n          -\n          (sum by (namespace) (capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:on_demand_pod_mem_usage_cost))\n      ) > 0), \"cap\", \"ON_DEMAND\", \"\",\"\")\n  )[1d:1h])\n)",
              "format": "table",
              "instant": true,
              "interval": "",
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "title": "Daily Memory Underutilization Cost Per Namespace",
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {
                  "Time": true
                },
                "indexByName": {},
                "renameByName": {
                  "Value #A": "Daily Memory Underutilization Cost",
                  "namespace": "Namespace"
                }
              }
            }
          ],
          "type": "table"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "custom": {
                "align": "auto",
                "displayMode": "auto",
                "filterable": false
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 8,
            "x": 8,
            "y": 45
          },
          "id": 35,
          "options": {
            "footer": {
              "fields": "",
              "reducer": [
                "sum"
              ],
              "show": false
            },
            "showHeader": true,
            "sortBy": []
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "sort_desc(\n\tsum_over_time(\n\t\tsum by (namespace) (\n\n\t\t\tzone_capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:spot_pod_mem_requests_cost\n\t\t\tOR\n\t\t\tcapacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:on_demand_pod_mem_requests_cost\n\n\t\t)\n\t[1d:1h])\n)",
              "format": "table",
              "instant": true,
              "interval": "",
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "title": "Daily Memory Requests Cost per Namespace",
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {
                  "Time": true
                },
                "indexByName": {},
                "renameByName": {
                  "Value #A": "Daily Memory Requests Cost",
                  "namespace": "Namespace"
                }
              }
            }
          ],
          "type": "table"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "custom": {
                "align": "auto",
                "displayMode": "auto",
                "filterable": false
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 8,
            "x": 16,
            "y": 45
          },
          "id": 36,
          "options": {
            "footer": {
              "fields": "",
              "reducer": [
                "sum"
              ],
              "show": false
            },
            "showHeader": true,
            "sortBy": []
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "sort_desc(\n\tsum_over_time(\n\t\tsum by (namespace) (\n\n\t\t\tzone_capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:spot_pod_mem_usage_cost\n\t\t\tOR\n\t\t\tcapacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:on_demand_pod_mem_usage_cost\n\n\t\t)\n\t[1d:1h])\n)",
              "format": "table",
              "instant": true,
              "interval": "",
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "title": "Daily Memory Usage Cost Per Namespace",
          "transformations": [
            {
              "id": "seriesToColumns",
              "options": {
                "byField": "Time"
              }
            },
            {
              "id": "organize",
              "options": {
                "excludeByName": {
                  "Time": true
                },
                "indexByName": {},
                "renameByName": {
                  "Value #A": "Daily Memory Usage Cost",
                  "namespace": "Namespace"
                }
              }
            }
          ],
          "type": "table"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "axisSoftMin": 0,
                "fillOpacity": 80,
                "gradientMode": "opacity",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineWidth": 1
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 53
          },
          "id": 38,
          "options": {
            "barWidth": 0.97,
            "groupWidth": 0.7,
            "legend": {
              "calcs": [],
              "displayMode": "table",
              "placement": "bottom"
            },
            "orientation": "auto",
            "showValue": "always",
            "stacking": "normal",
            "tooltip": {
              "mode": "single"
            },
            "xTickLabelRotation": 0
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "sum by (namespace) (sum by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes/1024/1024/1024) * $pvcost)",
              "format": "table",
              "instant": true,
              "interval": "",
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "title": "Monthly PVs Total Cost",
          "transformations": [
            {
              "id": "seriesToColumns",
              "options": {
                "byField": "Time"
              }
            },
            {
              "id": "organize",
              "options": {
                "excludeByName": {
                  "Time": true
                },
                "indexByName": {},
                "renameByName": {
                  "Value #A": "PVCs Costs Per Day"
                }
              }
            },
            {
              "id": "sortBy",
              "options": {
                "fields": {},
                "sort": [
                  {
                    "desc": true,
                    "field": "PVCs Costs Per Day"
                  }
                ]
              }
            }
          ],
          "type": "barchart"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 61
          },
          "id": 49,
          "options": {
            "legend": {
              "calcs": [
                "last"
              ],
              "displayMode": "table",
              "placement": "right",
              "sortBy": "Last",
              "sortDesc": true
            },
            "tooltip": {
              "mode": "single"
            }
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sum by (namespace) (sum by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes/1024/1024/1024) * $pvcost)",
              "format": "time_series",
              "instant": false,
              "interval": "",
              "legendFormat": "{{namespace}}",
              "refId": "A"
            }
          ],
          "title": "Monthly PVs Total Cost",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "axisSoftMin": 0,
                "fillOpacity": 80,
                "gradientMode": "opacity",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineWidth": 1
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 69
          },
          "id": 75,
          "options": {
            "barWidth": 0.97,
            "groupWidth": 0.7,
            "legend": {
              "calcs": [],
              "displayMode": "table",
              "placement": "bottom"
            },
            "orientation": "auto",
            "showValue": "always",
            "stacking": "normal",
            "tooltip": {
              "mode": "single"
            },
            "xTickLabelRotation": 0
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "(sum by (namespace) (sum by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes/1024/1024/1024))\n-\nsum by (namespace) (sum by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes/1024/1024/1024))\n) * $pvcost",
              "format": "table",
              "instant": true,
              "interval": "",
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "title": "Monthly PVs Underutilization Cost",
          "transformations": [
            {
              "id": "seriesToColumns",
              "options": {
                "byField": "Time"
              }
            },
            {
              "id": "organize",
              "options": {
                "excludeByName": {
                  "Time": true
                },
                "indexByName": {},
                "renameByName": {
                  "Value #A": "PVCs Costs Per Day"
                }
              }
            },
            {
              "id": "sortBy",
              "options": {
                "fields": {},
                "sort": [
                  {
                    "desc": true,
                    "field": "PVCs Costs Per Day"
                  }
                ]
              }
            }
          ],
          "type": "barchart"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 69
          },
          "id": 76,
          "options": {
            "legend": {
              "calcs": [
                "lastNotNull"
              ],
              "displayMode": "table",
              "placement": "right",
              "sortBy": "Last *",
              "sortDesc": true
            },
            "tooltip": {
              "mode": "single"
            }
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "(sum by (namespace) (sum by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes/1024/1024/1024))\n-\nsum by (namespace) (sum by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes/1024/1024/1024))\n) * $pvcost",
              "format": "time_series",
              "instant": false,
              "interval": "",
              "legendFormat": "{{namespace}}",
              "refId": "A"
            }
          ],
          "title": "Monthly PVs Underutilization Cost",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "axisSoftMin": 0,
                "fillOpacity": 80,
                "gradientMode": "opacity",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineWidth": 1
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 77
          },
          "id": 77,
          "options": {
            "barWidth": 0.97,
            "groupWidth": 0.7,
            "legend": {
              "calcs": [],
              "displayMode": "table",
              "placement": "bottom"
            },
            "orientation": "auto",
            "showValue": "always",
            "stacking": "normal",
            "tooltip": {
              "mode": "single"
            },
            "xTickLabelRotation": 0
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "sum by (namespace) (sum by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes/1024/1024/1024)) * $pvcost",
              "format": "table",
              "instant": true,
              "interval": "",
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "title": "Monthly PVs Usage Cost",
          "transformations": [
            {
              "id": "seriesToColumns",
              "options": {
                "byField": "Time"
              }
            },
            {
              "id": "organize",
              "options": {
                "excludeByName": {
                  "Time": true
                },
                "indexByName": {},
                "renameByName": {
                  "Value #A": "PVCs Costs Per Day"
                }
              }
            },
            {
              "id": "sortBy",
              "options": {
                "fields": {},
                "sort": [
                  {
                    "desc": true,
                    "field": "PVCs Costs Per Day"
                  }
                ]
              }
            }
          ],
          "type": "barchart"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 77
          },
          "id": 78,
          "options": {
            "legend": {
              "calcs": [
                "lastNotNull"
              ],
              "displayMode": "table",
              "placement": "right",
              "sortBy": "Last *",
              "sortDesc": true
            },
            "tooltip": {
              "mode": "single"
            }
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sum by (namespace) (sum by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes/1024/1024/1024)) * $pvcost",
              "format": "time_series",
              "instant": false,
              "interval": "",
              "legendFormat": "{{namespace}}",
              "refId": "A"
            }
          ],
          "title": "Monthly PVs Usage Cost",
          "transformations": [],
          "type": "timeseries"
        }
      ],
      "title": "Namespace",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 63
      },
      "id": 10,
      "panels": [
        {
          "description": "label_replace is used to ALWAYS have a different label (resource) when doing OR, so that no time series is filtered by binary operations (capacity_type for internal OR, resource for external OR)\n\nIf for example a pod has CPU requests but not memory requests, the binary sum (+) will filter the CPU sum (will not count for them)",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "decimals": 2,
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 9,
            "w": 24,
            "x": 0,
            "y": 3
          },
          "id": 81,
          "options": {
            "displayMode": "gradient",
            "orientation": "vertical",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "showUnfilled": false,
            "text": {}
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "sort_desc (\nsum_over_time(\n  sum by (pod) (\n  \n      # See description for label_replace\n      label_replace(\n          # If requests is bigger than usage, use requests. Otherwise use usage.\n          (\n            (\n                zone_capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:spot_pod_cpu_requests_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n                > \n                zone_capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:spot_pod_cpu_usage_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n            )\n            OR\n            zone_capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:spot_pod_cpu_usage_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n          )\n          OR\n          # If requests is bigger than usage, use requests. Otherwise use usage.\n          (\n            (\n                capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:on_demand_pod_cpu_requests_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n                >\n                capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:on_demand_pod_cpu_usage_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n            )\n            OR\n            zone_capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:spot_pod_cpu_usage_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n          )\n      , \"resource\", \"CPU\", \"\",\"\")\n      \n      OR \n      \n      label_replace(\n          # If requests is bigger than usage, use requests. Otherwise use usage.\n          (\n            (\n                zone_capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:spot_pod_mem_requests_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n                >\n                zone_capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:spot_pod_mem_usage_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n            )\n            OR\n            zone_capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:spot_pod_mem_usage_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n          )\n          OR\n          # If requests is bigger than usage, use requests. Otherwise use usage.\n          (\n            (\n                capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:on_demand_pod_mem_requests_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n                >\n                capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:on_demand_pod_mem_usage_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n            ) \n            OR capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:on_demand_pod_mem_usage_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n          )\n      , \"resource\", \"memory\", \"\",\"\")\n\n)\n[1d:1h])\n)",
              "format": "time_series",
              "instant": true,
              "interval": "",
              "legendFormat": "{{namespace}}",
              "refId": "A"
            }
          ],
          "title": "Daily Total Cost per Pod",
          "transformations": [],
          "type": "bargauge"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "custom": {
                "align": "auto",
                "displayMode": "auto",
                "filterable": false
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 8,
            "x": 0,
            "y": 12
          },
          "id": 71,
          "options": {
            "footer": {
              "fields": "",
              "reducer": [
                "sum"
              ],
              "show": false
            },
            "showHeader": true,
            "sortBy": []
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "sort_desc(\n  sum_over_time(\n\tsum by (label_beta_kubernetes_io_instance_type, label_eks_amazonaws_com_capacity_type, pod) (\n      label_replace(((\n          (zone_capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:spot_pod_cpu_requests_cost{namespace=\"$namespace\"})\n          -\n          (zone_capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:spot_pod_cpu_usage_cost{namespace=\"$namespace\"})\n      ) > 0), \"cap\", \"SPOT\", \"\",\"\")\n      OR\n      label_replace(((\n          (capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:on_demand_pod_cpu_requests_cost{namespace=\"$namespace\"})\n          -\n          (capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:on_demand_pod_cpu_usage_cost{namespace=\"$namespace\"})\n      ) > 0), \"cap\", \"ON_DEMAND\", \"\",\"\")\n  )[1d:1h])\n)",
              "format": "table",
              "instant": true,
              "interval": "",
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "title": "Daily CPU Underutilization Costs Per Pod",
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {
                  "Time": true,
                  "label_beta_kubernetes_io_instance_type": true,
                  "label_eks_amazonaws_com_capacity_type": true
                },
                "indexByName": {},
                "renameByName": {
                  "Value": "Daily CPU Underutilization Cost ",
                  "Value #A": "Daily CPU Underutilization Cost",
                  "pod": "Pod"
                }
              }
            }
          ],
          "type": "table"
        },
        {
          "datasource": "${DS_PROMETHEUS}",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "custom": {
                "align": "auto",
                "displayMode": "auto"
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 8,
            "x": 8,
            "y": 12
          },
          "id": 68,
          "options": {
            "footer": {
              "fields": "",
              "reducer": [
                "sum"
              ],
              "show": false
            },
            "showHeader": true
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "sort_desc(\n    sum_over_time(\n\tsum by (label_beta_kubernetes_io_instance_type, label_eks_amazonaws_com_capacity_type, pod) (\n\n\t\tzone_capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:spot_pod_cpu_requests_cost{namespace=\"$namespace\"}\n\t\tOR\n\t\tcapacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:on_demand_pod_cpu_requests_cost{namespace=\"$namespace\"}\n\n\t)\n[1d:1h])\n)",
              "format": "table",
              "instant": true,
              "interval": "",
              "legendFormat": "{{label_beta_kubernetes_io_instance_type}} - {{label_eks_amazonaws_com_capacity_type}} - {{pod}}",
              "refId": "A"
            }
          ],
          "title": "Daily CPU Requests Costs Per Pod",
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {
                  "Time": true,
                  "label_beta_kubernetes_io_instance_type": true,
                  "label_eks_amazonaws_com_capacity_type": true
                },
                "indexByName": {},
                "renameByName": {
                  "Value": "Daily CPU Requests Costs",
                  "Value #A": "Daily CPU Requests Costs",
                  "pod": "Pod"
                }
              }
            }
          ],
          "type": "table"
        },
        {
          "datasource": "${DS_PROMETHEUS}",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "custom": {
                "align": "auto",
                "displayMode": "auto"
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 8,
            "x": 16,
            "y": 12
          },
          "id": 66,
          "options": {
            "footer": {
              "fields": "",
              "reducer": [
                "sum"
              ],
              "show": false
            },
            "showHeader": true
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "sort_desc(\n    sum_over_time(\n\tsum by (label_beta_kubernetes_io_instance_type, label_eks_amazonaws_com_capacity_type, pod) (\n\n\t\tzone_capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:spot_pod_cpu_usage_cost{namespace=\"$namespace\"}\n\t\tOR\n\t\tcapacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:on_demand_pod_cpu_usage_cost{namespace=\"$namespace\"}\n\n\t)\n[1d:1h])\n)",
              "format": "table",
              "instant": true,
              "interval": "",
              "legendFormat": "{{label_beta_kubernetes_io_instance_type}} - {{label_eks_amazonaws_com_capacity_type}} - {{pod}}",
              "refId": "A"
            }
          ],
          "title": "Daily CPU Usage Costs Per Pod",
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {
                  "Time": true,
                  "label_beta_kubernetes_io_instance_type": true,
                  "label_eks_amazonaws_com_capacity_type": true
                },
                "indexByName": {},
                "renameByName": {
                  "Value": "Daily CPU Usage Costs",
                  "Value #A": "Daily CPU Usage Costs",
                  "pod": "Pod"
                }
              }
            }
          ],
          "type": "table"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "custom": {
                "align": "auto",
                "displayMode": "auto",
                "filterable": false
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 0,
            "y": 20
          },
          "id": 70,
          "options": {
            "footer": {
              "fields": "",
              "reducer": [
                "sum"
              ],
              "show": false
            },
            "showHeader": true,
            "sortBy": []
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "sort_desc(\n  sum_over_time(\n\tsum by (label_beta_kubernetes_io_instance_type, label_eks_amazonaws_com_capacity_type, pod) (\n      label_replace(((\n          (zone_capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:spot_pod_mem_requests_cost{namespace=\"$namespace\"})\n          -\n          (zone_capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:spot_pod_mem_usage_cost{namespace=\"$namespace\"})\n      ) > 0), \"cap\", \"SPOT\", \"\",\"\")\n      OR\n      label_replace(((\n          (capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:on_demand_pod_mem_requests_cost{namespace=\"$namespace\"})\n          -\n          (capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:on_demand_pod_mem_usage_cost{namespace=\"$namespace\"})\n      ) > 0), \"cap\", \"ON_DEMAND\", \"\",\"\")\n  )[1d:1h])\n)",
              "format": "table",
              "instant": true,
              "interval": "",
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "title": "Daily Memory Underutilization Costs Per Pod",
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {
                  "Time": true,
                  "label_beta_kubernetes_io_instance_type": true,
                  "label_eks_amazonaws_com_capacity_type": true
                },
                "indexByName": {},
                "renameByName": {
                  "Value": "Daily Memory Underutilization Cost per Pod",
                  "Value #A": "Daily Memory Underutilization Cost",
                  "pod": "Pod"
                }
              }
            }
          ],
          "type": "table"
        },
        {
          "datasource": "${DS_PROMETHEUS}",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "custom": {
                "align": "auto",
                "displayMode": "auto"
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 8,
            "y": 20
          },
          "id": 69,
          "options": {
            "footer": {
              "fields": "",
              "reducer": [
                "sum"
              ],
              "show": false
            },
            "showHeader": true
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "sort_desc(\nsum_over_time(\n\tsum by (label_beta_kubernetes_io_instance_type, label_eks_amazonaws_com_capacity_type, pod) (\n\n\t\tzone_capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:spot_pod_mem_requests_cost{namespace=\"$namespace\"}\n\t\tOR\n\t\tcapacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:on_demand_pod_mem_requests_cost{namespace=\"$namespace\"}\n\n\t)\n[1d:1h])\n)",
              "format": "table",
              "instant": true,
              "interval": "",
              "legendFormat": "{{label_beta_kubernetes_io_instance_type}} - {{label_eks_amazonaws_com_capacity_type}} - {{pod}} ",
              "refId": "A"
            }
          ],
          "title": "Daily Memory Requests Costs Per Pod",
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {
                  "Time": true,
                  "label_beta_kubernetes_io_instance_type": true,
                  "label_eks_amazonaws_com_capacity_type": true
                },
                "indexByName": {},
                "renameByName": {
                  "Value": "Daily Memory Requests Cost",
                  "Value #A": "Daily Memory Requests Costs",
                  "pod": "Pod"
                }
              }
            }
          ],
          "type": "table"
        },
        {
          "datasource": "${DS_PROMETHEUS}",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "custom": {
                "align": "auto",
                "displayMode": "auto"
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 16,
            "y": 20
          },
          "id": 67,
          "options": {
            "footer": {
              "fields": "",
              "reducer": [
                "sum"
              ],
              "show": false
            },
            "showHeader": true
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "sort_desc(\nsum_over_time(\n\tsum by (label_beta_kubernetes_io_instance_type, label_eks_amazonaws_com_capacity_type, pod) (\n\n\t\tzone_capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:spot_pod_mem_usage_cost{namespace=\"$namespace\"}\n\t\tOR\n\t\tcapacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:on_demand_pod_mem_usage_cost{namespace=\"$namespace\"}\n\n\t)\n[1d:1h])\n)",
              "format": "table",
              "instant": true,
              "interval": "",
              "legendFormat": "{{label_beta_kubernetes_io_instance_type}} - {{label_eks_amazonaws_com_capacity_type}} - {{pod}} ",
              "refId": "A"
            }
          ],
          "title": "Daily Memory Usage Costs Per Pod",
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {
                  "Time": true,
                  "label_beta_kubernetes_io_instance_type": true,
                  "label_eks_amazonaws_com_capacity_type": true
                },
                "indexByName": {},
                "renameByName": {
                  "Value": "Daily Memory Usage Cost",
                  "Value #A": "Daily Memory Usage Costs",
                  "pod": "Pod"
                }
              }
            }
          ],
          "type": "table"
        },
        {
          "datasource": "${DS_PROMETHEUS}",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              }
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 27
          },
          "id": 11,
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "single"
            }
          },
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sum by (label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type, pod) (\n    sum by (pod, node) (node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=~\"$namespace\", pod=~\"$pod\"})\n    * on (node) group_left(label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type) \n    sum by (label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type, node) (kube_node_labels{job=\"kube-state-metrics\"})\n)",
              "instant": false,
              "interval": "",
              "legendFormat": "CPU Usage - {{pod}}",
              "refId": "A"
            },
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sum by (label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type, pod) (\n    cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{namespace=~\"$namespace\", pod=~\"$pod\"}\n    * on (node) group_left(label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type) \n    sum by (label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type, node) (kube_node_labels{job=\"kube-state-metrics\"})\n)",
              "hide": false,
              "interval": "",
              "legendFormat": "CPU Requests - {{pod}}",
              "refId": "B"
            }
          ],
          "title": "CPU Requests vs Usage Per Pod",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "datasource": "${DS_PROMETHEUS}",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 27
          },
          "id": 12,
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "single"
            }
          },
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sum by (label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type, pod) (\n  sum by (pod, node) (container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"$pod\", name!=\"\"})\n  * on (node) group_left(label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type) \n  sum by (label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type, node) (kube_node_labels{job=\"kube-state-metrics\"})\n)",
              "instant": false,
              "interval": "",
              "legendFormat": "Memory Usage - {{pod}} ",
              "refId": "A"
            },
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sum by (label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type, pod) (\n    cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{namespace=~\"$namespace\", pod=~\"$pod\"}\n    * on (node) group_left(label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type) \n    sum by (label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type, node) (kube_node_labels{job=\"kube-state-metrics\"})\n)",
              "hide": false,
              "interval": "",
              "legendFormat": "Memory Requests - {{pod}} ",
              "refId": "B"
            }
          ],
          "title": "Memory Requests vs Usage Per Pod",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 35
          },
          "id": 72,
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "single"
            }
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sort_desc(\n  sum_over_time(\n\tsum by (label_beta_kubernetes_io_instance_type, label_eks_amazonaws_com_capacity_type, pod) (\n      label_replace(((\n          (zone_capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:spot_pod_cpu_requests_cost{namespace=\"$namespace\", pod=~\"$pod\"})\n          -\n          (zone_capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:spot_pod_cpu_usage_cost{namespace=\"$namespace\", pod=~\"$pod\"})\n      ) > 0), \"cap\", \"SPOT\", \"\",\"\")\n      OR\n      label_replace(((\n          (capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:on_demand_pod_cpu_requests_cost{namespace=\"$namespace\", pod=~\"$pod\"})\n          -\n          (capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:on_demand_pod_cpu_usage_cost{namespace=\"$namespace\", pod=~\"$pod\"})\n      ) > 0), \"cap\", \"ON_DEMAND\", \"\",\"\")\n  )[1d:1h])\n)",
              "interval": "",
              "legendFormat": "{{label_beta_kubernetes_io_instance_type}} - {{label_eks_amazonaws_com_capacity_type}} - {{pod}}",
              "refId": "A"
            },
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sort_desc(\n  sum_over_time(\n\tsum (\n      label_replace(((\n          (zone_capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:spot_pod_cpu_requests_cost{namespace=\"$namespace\", pod=~\"$pod\"})\n          -\n          (zone_capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:spot_pod_cpu_usage_cost{namespace=\"$namespace\", pod=~\"$pod\"})\n      ) > 0), \"cap\", \"SPOT\", \"\",\"\")\n      OR\n      label_replace(((\n          (capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:on_demand_pod_cpu_requests_cost{namespace=\"$namespace\", pod=~\"$pod\"})\n          -\n          (capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:on_demand_pod_cpu_usage_cost{namespace=\"$namespace\", pod=~\"$pod\"})\n      ) > 0), \"cap\", \"ON_DEMAND\", \"\",\"\")\n  )[1d:1h])\n)",
              "hide": false,
              "interval": "",
              "legendFormat": "Total",
              "refId": "B"
            }
          ],
          "title": "Daily CPU Underutilization Costs Per Pod",
          "type": "timeseries"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 35
          },
          "id": 73,
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "single"
            }
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sort_desc(\n  sum_over_time(\n\tsum by (label_beta_kubernetes_io_instance_type, label_eks_amazonaws_com_capacity_type, pod) (\n      label_replace(((\n          (zone_capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:spot_pod_mem_requests_cost{namespace=\"$namespace\", pod=~\"$pod\"})\n          -\n          (zone_capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:spot_pod_mem_usage_cost{namespace=\"$namespace\", pod=~\"$pod\"})\n      ) > 0), \"cap\", \"SPOT\", \"\",\"\")\n      OR\n      label_replace(((\n          (capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:on_demand_pod_mem_requests_cost{namespace=\"$namespace\", pod=~\"$pod\"})\n          -\n          (capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:on_demand_pod_mem_usage_cost{namespace=\"$namespace\", pod=~\"$pod\"})\n      ) > 0), \"cap\", \"ON_DEMAND\", \"\",\"\")\n  )[1d:1h])\n)",
              "interval": "",
              "legendFormat": "{{label_beta_kubernetes_io_instance_type}} - {{label_eks_amazonaws_com_capacity_type}} - {{pod}}",
              "refId": "A"
            },
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sort_desc(\n  sum_over_time(\n\tsum (\n      label_replace(((\n          (zone_capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:spot_pod_mem_requests_cost{namespace=\"$namespace\", pod=~\"$pod\"})\n          -\n          (zone_capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:spot_pod_mem_usage_cost{namespace=\"$namespace\", pod=~\"$pod\"})\n      ) > 0), \"cap\", \"SPOT\", \"\",\"\")\n      OR\n      label_replace(((\n          (capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:on_demand_pod_mem_requests_cost{namespace=\"$namespace\", pod=~\"$pod\"})\n          -\n          (capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:on_demand_pod_mem_usage_cost{namespace=\"$namespace\", pod=~\"$pod\"})\n      ) > 0), \"cap\", \"ON_DEMAND\", \"\",\"\")\n  )[1d:1h])\n)",
              "hide": false,
              "interval": "",
              "legendFormat": "Total",
              "refId": "B"
            }
          ],
          "title": "Daily Memory Underutilization Costs Per Pod",
          "type": "timeseries"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 43
          },
          "id": 39,
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "single"
            }
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sum_over_time(\n\tsum by (label_beta_kubernetes_io_instance_type, label_eks_amazonaws_com_capacity_type, pod) (\n\n\t\tzone_capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:spot_pod_cpu_requests_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n\t\tOR\n\t\tcapacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:on_demand_pod_cpu_requests_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n\n\t)\n[1d:1h])",
              "interval": "",
              "legendFormat": "{{label_beta_kubernetes_io_instance_type}} - {{label_eks_amazonaws_com_capacity_type}} - {{pod}}",
              "refId": "A"
            },
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sum_over_time(\n\tsum (\n\n\t\tzone_capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:spot_pod_cpu_requests_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n\t\tOR\n\t\tcapacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:on_demand_pod_cpu_requests_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n\n\t)\n[1d:1h])",
              "hide": false,
              "interval": "",
              "legendFormat": "Total",
              "refId": "B"
            }
          ],
          "title": "Daily CPU Requests Cost Per Pod",
          "type": "timeseries"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 43
          },
          "id": 40,
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "single"
            }
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sum_over_time(\n\tsum by (label_beta_kubernetes_io_instance_type, label_eks_amazonaws_com_capacity_type, pod) (\n\n\t\tzone_capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:spot_pod_mem_requests_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n\t\tOR\n\t\tcapacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:on_demand_pod_mem_requests_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n\n\t)\n[1d:1h])",
              "interval": "",
              "legendFormat": "{{label_beta_kubernetes_io_instance_type}} - {{label_eks_amazonaws_com_capacity_type}} - {{pod}}",
              "refId": "A"
            },
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sum_over_time(\n\tsum (\n\n\t\tzone_capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:spot_pod_mem_requests_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n\t\tOR\n\t\tcapacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:on_demand_pod_mem_requests_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n\n\t)\n[1d:1h])",
              "hide": false,
              "interval": "",
              "legendFormat": "Total",
              "refId": "B"
            }
          ],
          "title": "Daily Memory Requests Cost Per Pod",
          "type": "timeseries"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 51
          },
          "id": 41,
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "single"
            }
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sum_over_time(\n\tsum by (label_beta_kubernetes_io_instance_type, label_eks_amazonaws_com_capacity_type, pod) (\n\n\t\tzone_capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:spot_pod_cpu_usage_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n\t\tOR\n\t\tcapacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:on_demand_pod_cpu_usage_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n\n\t)\n[1d:1h])",
              "interval": "",
              "legendFormat": "{{label_beta_kubernetes_io_instance_type}} - {{label_eks_amazonaws_com_capacity_type}} - {{pod}}",
              "refId": "A"
            },
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sum_over_time(\n\tsum (\n\n\t\tzone_capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:spot_pod_cpu_usage_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n\t\tOR\n\t\tcapacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:on_demand_pod_cpu_usage_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n\n\t)\n[1d:1h])",
              "hide": false,
              "interval": "",
              "legendFormat": "Total",
              "refId": "B"
            }
          ],
          "title": "Daily CPU Usage Cost Per Pod",
          "type": "timeseries"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "currencyUSD"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 51
          },
          "id": 42,
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "single"
            }
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sum_over_time(\n\tsum by (label_beta_kubernetes_io_instance_type, label_eks_amazonaws_com_capacity_type, pod) (\n\n\t\tzone_capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:spot_pod_mem_usage_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n\t\tOR\n\t\tcapacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:on_demand_pod_mem_usage_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n\n\t)\n[1d:1h])",
              "interval": "",
              "legendFormat": "{{label_beta_kubernetes_io_instance_type}} - {{label_eks_amazonaws_com_capacity_type}} - {{pod}}",
              "refId": "A"
            },
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sum_over_time(\n\tsum (\n\n\t\tzone_capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:spot_pod_mem_usage_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n\t\tOR\n\t\tcapacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:on_demand_pod_mem_usage_cost{namespace=\"$namespace\", pod=~\"$pod\"}\n\n\t)\n[1d:1h])",
              "hide": false,
              "interval": "",
              "legendFormat": "Total",
              "refId": "B"
            }
          ],
          "title": "Daily Memory Usage Cost Per Pod",
          "type": "timeseries"
        },
        {
          "datasource": "${DS_PROMETHEUS}",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "custom": {
                "align": "auto",
                "displayMode": "auto"
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "decgbytes"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byRegexp",
                  "options": ".*Cost"
                },
                "properties": [
                  {
                    "id": "unit",
                    "value": "currencyUSD"
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "PVC (Persistent Volume Claim)"
                },
                "properties": [
                  {
                    "id": "custom.width",
                    "value": 554
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 59
          },
          "id": 8,
          "options": {
            "footer": {
              "fields": "",
              "reducer": [
                "sum"
              ],
              "show": false
            },
            "frameIndex": 0,
            "showHeader": true,
            "sortBy": [
              {
                "desc": true,
                "displayName": "Underutilization Cost"
              }
            ]
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "sum by (persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=~\"${namespace}\"}/1024/1024/1024)",
              "format": "table",
              "instant": true,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{persistentvolumeclaim}}",
              "refId": "A"
            },
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "sum by (persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=~\"${namespace}\"}/1024/1024/1024)",
              "format": "table",
              "hide": false,
              "instant": true,
              "interval": "",
              "legendFormat": "",
              "refId": "C"
            },
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "sum by (persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=~\"${namespace}\"}/1024/1024/1024)\n-\nsum by (persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=~\"${namespace}\"}/1024/1024/1024)",
              "format": "table",
              "hide": false,
              "instant": true,
              "interval": "",
              "legendFormat": "",
              "refId": "D"
            },
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "sum by (persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=~\"${namespace}\"}/1024/1024/1024) * $pvcost",
              "format": "table",
              "hide": false,
              "instant": true,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "",
              "refId": "B"
            },
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "sum by (persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=~\"${namespace}\"}/1024/1024/1024) * $pvcost",
              "format": "table",
              "hide": false,
              "instant": true,
              "interval": "",
              "legendFormat": "",
              "refId": "E"
            },
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "(sum by (persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=~\"${namespace}\"}/1024/1024/1024)\n-\nsum by (persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=~\"${namespace}\"}/1024/1024/1024))\n* $pvcost",
              "format": "table",
              "hide": false,
              "instant": true,
              "interval": "",
              "legendFormat": "",
              "refId": "F"
            }
          ],
          "title": "Monthly PV Cost Per Pod",
          "transformations": [
            {
              "id": "sortBy",
              "options": {
                "fields": {},
                "sort": [
                  {
                    "desc": true,
                    "field": "Value #A"
                  }
                ]
              }
            },
            {
              "id": "seriesToColumns",
              "options": {
                "byField": "persistentvolumeclaim"
              }
            },
            {
              "id": "organize",
              "options": {
                "excludeByName": {
                  "Time": true,
                  "persistentvolumeclaim 2": true
                },
                "indexByName": {
                  "Time": 1,
                  "Value #A": 2,
                  "Value #B": 3,
                  "Value #C": 4,
                  "Value #D": 6,
                  "Value #E": 5,
                  "persistentvolumeclaim": 0
                },
                "renameByName": {
                  "Value": "Cost",
                  "Value #A": "Size",
                  "Value #B": "Total Cost",
                  "Value #C": "Used",
                  "Value #D": "Free",
                  "Value #E": "Usage Cost",
                  "Value #F": "Underutilization Cost",
                  "persistentvolumeclaim": "PVC (Persistent Volume Claim)"
                }
              }
            }
          ],
          "type": "table"
        },
        {
          "datasource": "${DS_PROMETHEUS}",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "decgbytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 67
          },
          "id": 82,
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "single"
            }
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sum by (persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=~\"${namespace}\", persistentvolumeclaim=~\"$pvc\"}/1024/1024/1024)",
              "format": "time_series",
              "instant": false,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{persistentvolumeclaim}} - Capacity",
              "refId": "A"
            },
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sum by (persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=~\"${namespace}\", persistentvolumeclaim=~\"$pvc\"}/1024/1024/1024)",
              "format": "time_series",
              "hide": false,
              "instant": false,
              "interval": "",
              "legendFormat": "{{persistentvolumeclaim}} - Usage",
              "refId": "C"
            }
          ],
          "title": "PV Capacity vs Usage",
          "transformations": [],
          "type": "timeseries"
        }
      ],
      "title": "Pod Per Namespace",
      "type": "row"
    }
  ],
  "refresh": "5m",
  "schemaVersion": 33,
  "style": "dark",
  "tags": [],
  "templating": {
    "list": [
      {
        "current": {
          "selected": false,
          "text": "Prometheus",
          "value": "Prometheus"
        },
        "hide": 0,
        "includeAll": false,
        "label": "Prometheus Datasource",
        "multi": false,
        "name": "promdatasource",
        "options": [],
        "query": "prometheus",
        "refresh": 1,
        "regex": "Prometheus",
        "skipUrlSync": false,
        "type": "datasource"
      },
      {
        "hide": 2,
        "name": "region",
        "query": "eu-west-1",
        "skipUrlSync": false,
        "type": "constant"
      },
      {
        "current": {
          "selected": false,
          "text": "velero",
          "value": "velero"
        },
        "datasource": {
          "uid": "${promdatasource}"
        },
        "definition": "label_values(kube_pod_info, namespace)",
        "hide": 0,
        "includeAll": false,
        "multi": false,
        "name": "namespace",
        "options": [],
        "query": {
          "query": "label_values(kube_pod_info, namespace)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 2,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "type": "query"
      },
      {
        "current": {
          "selected": true,
          "text": [
            "All"
          ],
          "value": [
            "$__all"
          ]
        },
        "datasource": "${DS_PROMETHEUS}",
        "definition": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)",
        "hide": 0,
        "includeAll": true,
        "label": "Pod",
        "multi": true,
        "name": "pod",
        "options": [],
        "query": {
          "query": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 2,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "type": "query"
      },
      {
        "description": "Price of GiB per month",
        "hide": 2,
        "label": "PV Cost GB/month",
        "name": "pvcost",
        "query": "0.089",
        "skipUrlSync": false,
        "type": "constant"
      },
      {
        "current": {
          "selected": false,
          "text": "All",
          "value": "$__all"
        },
        "definition": "label_values(kubelet_volume_stats_capacity_bytes{namespace=\"$namespace\"}, persistentvolumeclaim)",
        "hide": 0,
        "includeAll": true,
        "label": "Persistent Volume Claim",
        "multi": true,
        "name": "pvc",
        "options": [],
        "query": {
          "query": "label_values(kubelet_volume_stats_capacity_bytes{namespace=\"$namespace\"}, persistentvolumeclaim)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "type": "query"
      }
    ]
  },
  "time": {
    "from": "now-24h",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "",
  "title": "Kubernetes Cost Report",
  "uid": "ZudkNFhnksaf",
  "version": 1,
  "weekStart": "",
  "gnetId": 15877
}