Grafana dashboard: TFA-Factomd
Description: Factomd instrumentation related to state

The TFA-factomd dashboard is designed to work with the integrated node_exporter in Factomd via prometheus.

Point Prometheus to port :8090 at the host (instead of :9100) and it will pick up the factomd-data automatically.

{
  "__inputs": [
    {
      "name": "DS_PROMETHEUS",
      "label": "Prometheus",
      "description": "",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    }
  ],
  "__requires": [
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "6.0.2"
    },
    {
      "type": "panel",
      "id": "graph",
      "name": "Graph",
      "version": "5.0.0"
    },
    {
      "type": "datasource",
      "id": "prometheus",
      "name": "Prometheus",
      "version": "5.0.0"
    },
    {
      "type": "panel",
      "id": "text",
      "name": "Text",
      "version": "5.0.0"
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "description": "Factomd instrumentation related to state",
  "editable": true,
  "gnetId": 10008,
  "graphTooltip": 1,
  "id": null,
  "iteration": 1554477843175,
  "links": [
    {
      "asDropdown": false,
      "icon": "external link",
      "includeVars": false,
      "keepTime": false,
      "tags": [],
      "targetBlank": false,
      "title": "Factomd",
      "type": "dashboards",
      "url": ""
    }
  ],
  "panels": [
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 49,
      "panels": [],
      "repeat": null,
      "title": "Title",
      "type": "row"
    },
    {
      "content": "# Factomd Instrumentation\nBe sure to set the time window in the top right to the time window you are interested in. \n",
      "gridPos": {
        "h": 3,
        "w": 24,
        "x": 0,
        "y": 1
      },
      "id": 2,
      "links": [],
      "mode": "markdown",
      "title": "",
      "type": "text"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 4
      },
      "id": 50,
      "panels": [
        {
          "content": "# Memory Analytics\n\nMemstats --> Go Tool to analyze memory usuage\n\nGC --> Garbage Collector",
          "editable": true,
          "error": false,
          "gridPos": {
            "h": 3,
            "w": 24,
            "x": 0,
            "y": 5
          },
          "height": "100px",
          "id": 16,
          "links": [],
          "mode": "markdown",
          "title": "Memory",
          "type": "text"
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
            "h": 6,
            "w": 24,
            "x": 0,
            "y": 8
          },
          "id": 15,
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": true,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "connected",
          "paceLength": 10,
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {
              "alias": "MemStats Inuse Objects",
              "yaxis": 2
            }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "go_memstats_heap_inuse_bytes{job=~\"$node\"}",
              "format": "time_series",
              "hide": true,
              "intervalFactor": 2,
              "legendFormat": "Inuse Heap Bytes",
              "refId": "A",
              "step": 2
            },
            {
              "expr": "go_memstats_heap_objects{job=~\"$node\"}",
              "format": "time_series",
              "hide": true,
              "intervalFactor": 2,
              "legendFormat": "Inuse Heap Objects",
              "refId": "B",
              "step": 2
            },
            {
              "expr": "go_memstats_frees_total{job=~\"$node\"}",
              "format": "time_series",
              "hide": true,
              "intervalFactor": 2,
              "legendFormat": "MemStats Freed",
              "refId": "C",
              "step": 20
            },
            {
              "expr": "process_resident_memory_bytes{job=~\"$node\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Resident Memory",
              "refId": "D",
              "step": 2
            },
            {
              "expr": "process_virtual_memory_bytes{job=~\"$node\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Virtual Memory",
              "refId": "E",
              "step": 2
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Memory",
          "tooltip": {
            "msResolution": false,
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "bytes",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
            "h": 5,
            "w": 24,
            "x": 0,
            "y": 14
          },
          "id": 17,
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": true,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "connected",
          "paceLength": 10,
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": true,
          "steppedLine": false,
          "targets": [
            {
              "expr": "go_memstats_heap_alloc_bytes{job=~\"$node\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Heap Allocated and Inuse",
              "refId": "D",
              "step": 2
            },
            {
              "expr": "go_memstats_heap_inuse_bytes{job=~\"$node\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Heap Inuse",
              "refId": "E",
              "step": 2
            },
            {
              "expr": "go_memstats_heap_idle_bytes{job=~\"$node\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Heap Idle Bytes",
              "refId": "G",
              "step": 2
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Heap Stats",
          "tooltip": {
            "msResolution": false,
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "decbytes",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "fill": 1,
          "gridPos": {
            "h": 5,
            "w": 24,
            "x": 0,
            "y": 19
          },
          "id": 23,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": true,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "paceLength": 10,
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "go_memstats_heap_sys_bytes{job=~\"$node\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{job}} System Reported Memory",
              "refId": "A",
              "step": 2
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "System Reported Memory",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "bytes",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
            "h": 5,
            "w": 24,
            "x": 0,
            "y": 24
          },
          "id": 18,
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": true,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "connected",
          "paceLength": 10,
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "go_memstats_gc_sys_bytes{job=~\"$node\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{job}} GC Heap Bytes",
              "refId": "A",
              "step": 2
            },
            {
              "expr": "go_memstats_buck_hash_sys_bytes{job=~\"$node\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Profiling Bucket Bytes",
              "refId": "B",
              "step": 2
            },
            {
              "expr": "factomd_database_leveldb_cacheblock{job=~\"$node\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Level DB Caching",
              "metric": "factomd_database_leveldb_cacheblock",
              "refId": "C",
              "step": 2
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Bytes used by non-factomd",
          "tooltip": {
            "msResolution": false,
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "bytes",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
            "h": 5,
            "w": 24,
            "x": 0,
            "y": 29
          },
          "id": 19,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "connected",
          "paceLength": 10,
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "go_memstats_mallocs_total{job=~\"$node\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Total Mallocs",
              "refId": "A",
              "step": 2
            },
            {
              "expr": "go_memstats_frees_total{job=~\"$node\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Total Frees",
              "refId": "B",
              "step": 2
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Malloc/Free Totals",
          "tooltip": {
            "msResolution": false,
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Database",
          "fill": 1,
          "gridPos": {
            "h": 5,
            "w": 24,
            "x": 0,
            "y": 34
          },
          "id": 28,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "paceLength": 10,
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {
              "alias": "Gets/s",
              "yaxis": 2
            },
            {
              "alias": "Puts/s",
              "yaxis": 2
            }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "factomd_database_leveldb_cacheblock{job=~\"$node\"}",
              "format": "time_series",
              "hide": true,
              "intervalFactor": 2,
              "metric": "factomd_database_leveldb_cacheblock",
              "refId": "A",
              "step": 2
            },
            {
              "expr": "factomd_database_leveldb_gets{job=~\"$node\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{job}} LevelDB Gets",
              "metric": "factomd_database_leveldb_gets",
              "refId": "B",
              "step": 2
            },
            {
              "expr": "factomd_database_leveldb_puts{job=~\"$node\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{job}} LevelDB Puts",
              "metric": "factomd_database_leveldb_puts",
              "refId": "C",
              "step": 2
            },
            {
              "expr": "rate(factomd_database_leveldb_gets{job=~\"$node\"}[1m])",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Gets/s",
              "refId": "D",
              "step": 2
            },
            {
              "expr": "rate(factomd_database_leveldb_puts{job=~\"$node\"}[1m])",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Puts/s",
              "refId": "E",
              "step": 2
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "LevelDB",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "fill": 1,
          "gridPos": {
            "h": 5,
            "w": 24,
            "x": 0,
            "y": 39
          },
          "id": 46,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "paceLength": 10,
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {
              "alias": "Gets/s",
              "yaxis": 2
            },
            {
              "alias": "Puts/s",
              "yaxis": 2
            }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "factomd_database_leveldb_cacheblock{job=~\"$node\"}",
              "format": "time_series",
              "hide": true,
              "intervalFactor": 2,
              "metric": "factomd_database_leveldb_cacheblock",
              "refId": "A",
              "step": 2
            },
            {
              "expr": "{__name__=~\"factomd_database_overlay_gets_.*\", job=\"$node\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{__name__}}",
              "metric": "factomd_database_overlay_gets_ablock",
              "refId": "B",
              "step": 2
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "LevelDB Specific",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "repeat": null,
      "title": "Memory",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 5
      },
      "id": 51,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
            "h": 7,
            "w": 24,
            "x": 0,
            "y": 6
          },
          "id": 5,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": true,
            "sortDesc": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "connected",
          "paceLength": 10,
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "go_goroutines{job=~\"$node\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Go Routines",
              "metric": "go_goroutines",
              "refId": "A",
              "step": 600
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Go Routines",
          "tooltip": {
            "msResolution": false,
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "fill": 1,
          "gridPos": {
            "h": 5,
            "w": 24,
            "x": 0,
            "y": 13
          },
          "height": "200px",
          "id": 11,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "connected",
          "paceLength": 10,
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "go_gc_duration_seconds_count{job=~\"$node\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Total Calls",
              "metric": "go_gc_duration_seconds_count",
              "refId": "A",
              "step": 600
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Garbage Collection",
          "tooltip": {
            "msResolution": false,
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
            "h": 7,
            "w": 24,
            "x": 0,
            "y": 18
          },
          "id": 12,
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": true,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "connected",
          "paceLength": 10,
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "go_gc_duration_seconds{job=~\"$node\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{job}} {{quantile}}",
              "metric": "go_gc_duration_seconds",
              "refId": "A",
              "step": 600
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Garbage Collection Duration (s)",
          "tooltip": {
            "msResolution": false,
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "s",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
            "h": 7,
            "w": 24,
            "x": 0,
            "y": 25
          },
          "id": 14,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "connected",
          "paceLength": 10,
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "go_memstats_gc_sys_bytes{job=~\"$node\"}",
              "intervalFactor": 2,
              "legendFormat": "{{job}}  GC_Bytes Used",
              "refId": "A",
              "step": 600
            },
            {
              "expr": "go_memstats_next_gc_bytes{job=~\"$node\"}",
              "intervalFactor": 2,
              "legendFormat": "{{job}}  Bytes when next GC",
              "refId": "B",
              "step": 600
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Garbage Collecter",
          "tooltip": {
            "msResolution": false,
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "bytes",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "repeat": null,
      "title": "Process",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 6
      },
      "id": 52,
      "panels": [
        {
          "content": "# Network Connections",
          "gridPos": {
            "h": 3,
            "w": 12,
            "x": 0,
            "y": 7
          },
          "height": "100px",
          "id": 13,
          "links": [],
          "mode": "markdown",
          "title": "Networking",
          "type": "text"
        },
        {
          "aliasColors": {
            "Connection Count": "#629E51",
            "Connections By Address": "#1F78C1",
            "Metrics Count": "#F4D598"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
            "h": 7,
            "w": 24,
            "x": 0,
            "y": 10
          },
          "id": 9,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "connected",
          "paceLength": 10,
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {
              "alias": "Metrics Count",
              "yaxis": 1
            }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "factomd_p2p_controller_connections_current{job=~\"$node\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Connection Count",
              "metric": "factomd_p2p_controller_connections_current",
              "refId": "A",
              "step": 40
            },
            {
              "expr": "factomd_p2p_controller_connectionsbyaddress_current{job=~\"$node\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Connections By Address",
              "metric": "factomd_p2p_controller_connectionsbyaddress_current",
              "refId": "C",
              "step": 40
            },
            {
              "expr": "factomd_p2p_controller_metrics_current{job=~\"$node\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Metrics Count",
              "metric": "factomd_p2p_controller_metrics_current",
              "refId": "B",
              "step": 40
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Current Number of Connections",
          "tooltip": {
            "msResolution": false,
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
            "h": 7,
            "w": 24,
            "x": 0,
            "y": 17
          },
          "id": 6,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "connected",
          "paceLength": 10,
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "factomd_p2p_connection_process_receives_routine_guage{job=~\"$node\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{job}}  ProcessRecieves Routines",
              "metric": "factomd_p2p_connection_process_receives_routine_guage",
              "refId": "A",
              "step": 40
            },
            {
              "expr": "factomd_p2p_connection_process_sends_routine_guage{job=~\"$node\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{job}} ProcessSends Routines",
              "metric": "factomd_p2p_connection_process_sends_routine_guage",
              "refId": "B",
              "step": 40
            },
            {
              "expr": "factomd_p2p_connection_runloop_routine_gauge{job=~\"$node\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Runloop Routines",
              "metric": "factomd_p2p_connection__runloop_routine_gauge",
              "refId": "C",
              "step": 40
            },
            {
              "expr": "factomd_p2p_connection_dialloop_routine_guage{job=~\"$node\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Dial Loop Routines",
              "metric": "factomd_p2p_connection_dialloop_routine_guage",
              "refId": "D",
              "step": 40
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Connection Routines Alive",
          "tooltip": {
            "msResolution": false,
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
            "h": 7,
            "w": 24,
            "x": 0,
            "y": 24
          },
          "id": 8,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "connected",
          "paceLength": 10,
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "factomd_p2p_connection_runloop_initialized_counter{job=~\"$node\"}",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Initalized",
              "metric": "factomd_p2p_connection_runloop_initialized_counter",
              "refId": "A",
              "step": 40
            },
            {
              "expr": "factomd_p2p_connection_runloop_online_counter{job=~\"$node\"}",
              "hide": false,
              "intervalFactor": 2,
              "legendFormat": "{{job}} Online",
              "metric": "factomd_p2p_connection_runloop_online_counter",
              "refId": "B",
              "step": 40
            },
            {
              "expr": "factomd_p2p_connection_runloop_offline_counter{job=~\"$node\"}",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Offline",
              "metric": "factomd_p2p_connection_runloop_offline_counter",
              "refId": "C",
              "step": 40
            },
            {
              "expr": "factomd_p2p_connection_runloop_shutdown_counter{job=~\"$node\"}",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Shutdown",
              "metric": "factomd_p2p_connection_runloop_shutdown_counter",
              "refId": "D",
              "step": 40
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Connection Runloops",
          "tooltip": {
            "msResolution": false,
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "repeat": null,
      "title": "Connections",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 7
      },
      "id": 53,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "fill": 1,
          "gridPos": {
            "h": 9,
            "w": 24,
            "x": 0,
            "y": 8
          },
          "id": 21,
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": true,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": true,
            "sort": "current",
            "sortDesc": false,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "paceLength": 10,
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "factomd_state_es_missing_entry_queue{job=~\"$node\"}",
              "hide": false,
              "intervalFactor": 2,
              "legendFormat": "{{job}} missing entry queue",
              "metric": "factom_es_asking_missing_entries",
              "refId": "A",
              "step": 60
            },
            {
              "expr": "factomd_state_es_missing_entries{job=~\"$node\"}",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "{{job}} known missing ",
              "metric": "factom_es_found_entries",
              "refId": "B",
              "step": 60
            },
            {
              "expr": "factomd_state_es_found_entries{job=~\"$node\"}",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Found of the missing",
              "metric": "factom_es_missing_entries",
              "refId": "C",
              "step": 60
            },
            {
              "expr": "factomd_state_es_asking_missing_entries{job=~\"$node\"}",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Asking for missing",
              "refId": "D",
              "step": 60
            },
            {
              "expr": "factomd_state_es_highest_asking_entries{job=~\"$node\"}",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Highest dbht asking for entries",
              "refId": "E",
              "step": 60
            },
            {
              "expr": "factomd_state_es_highest_missing_entries{job=~\"$node\"}",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Highest dbhight of missing",
              "refId": "F",
              "step": 60
            },
            {
              "expr": "factomd_state_es_first_missing_entries{job=~\"$node\"}",
              "intervalFactor": 2,
              "legendFormat": "{{job}} First dbhight with missing",
              "refId": "G",
              "step": 60
            },
            {
              "expr": "factomd_state_es_entry_dbheight_complete{job=~\"$node\"}",
              "intervalFactor": 2,
              "legendFormat": "{{job}} dbhight complete",
              "refId": "H",
              "step": 60
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Missing Entry Syncing",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "fill": 1,
          "gridPos": {
            "h": 9,
            "w": 24,
            "x": 0,
            "y": 17
          },
          "id": 24,
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": true,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "paceLength": 10,
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "factomd_state_es_average_requests{job=~\"$node\"}",
              "intervalFactor": 2,
              "legendFormat": "{{job}} Average Request per missing entry",
              "refId": "A",
              "step": 60
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Average Request per missing entry",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "repeat": null,
      "title": "Missing Entries",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 8
      },
      "id": 54,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "decimals": 3,
          "fill": 1,
          "gridPos": {
            "h": 7,
            "w": 24,
            "x": 0,
            "y": 9
          },
          "id": 27,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "sort": "current",
            "sortDesc": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "paceLength": 10,
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {}
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "factomd_state_highest_saved",
              "intervalFactor": 2,
              "legendFormat": " {{job}} Highest Saved",
              "metric": "factomd_state_highest_saved",
              "refId": "A",
              "step": 2
            },
            {
              "expr": "factomd_state_highest_completed",
              "hide": false,
              "intervalFactor": 2,
              "legendFormat": " {{job}} Higest Complete",
              "metric": "factomd_state_highest_completed",
              "refId": "B",
              "step": 2
            },
            {
              "expr": "factomd_state_highest_ack",
              "hide": false,
              "intervalFactor": 2,
              "legendFormat": " {{job}} Highest Ack",
              "metric": "factomd_state_highest_ack",
              "refId": "C",
              "step": 2
            },
            {
              "expr": "hfactomd_state_highest_known",
              "hide": false,
              "intervalFactor": 2,
              "legendFormat": " {{job}} Highest Known",
              "metric": "factomd_state_highest_known",
              "refId": "D",
              "step": 2
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Consensus",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "repeat": null,
      "title": "Height",
      "type": "row"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 9
      },
      "id": 55,
      "panels": [],
      "repeat": null,
      "title": "Queues",
      "type": "row"
    },
    {
      "content": "# InMsgQueue\nCurrent : Current messages in queue\n\nTotal : All messages that went through this queue",
      "gridPos": {
        "h": 3,
        "w": 6,
        "x": 0,
        "y": 10
      },
      "height": "75",
      "id": 29,
      "links": [],
      "mode": "markdown",
      "title": "",
      "type": "text"
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS}",
      "decimals": 0,
      "fill": 1,
      "gridPos": {
        "h": 7,
        "w": 24,
        "x": 0,
        "y": 13
      },
      "id": 30,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": true,
        "hideZero": false,
        "max": true,
        "min": true,
        "rightSide": true,
        "show": true,
        "sort": null,
        "sortDesc": null,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null as zero",
      "paceLength": 10,
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": true,
      "steppedLine": false,
      "targets": [
        {
          "expr": "factomd_state_queue_current_general_inmsg_vec{job=~\"$node\"}",
          "hide": false,
          "intervalFactor": 2,
          "legendFormat": "{{message}}",
          "metric": "factomd_state_queue_current_insmg_ack",
          "refId": "A",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_current_inmsg_eom{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "EOM",
          "refId": "B",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_current_inmsg_audfault{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "AudFault",
          "refId": "C",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_current_inmsg_fedfault{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "FedFault",
          "refId": "D",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_current_inmsg_fullfault{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "FullFault",
          "refId": "E",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_current_inmsg_commitchain{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "CommitChain",
          "refId": "F",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_current_inmsg_commitentry{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "CommitEntry",
          "refId": "G",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_current_inmsg_dbsig{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "DBSig",
          "refId": "H",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_current_inmsg_eomtimeout{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "EOMTimeout",
          "refId": "I",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_current_inmsg_facttx{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "FactoidTx",
          "refId": "J",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_current_inmsg_heartbeat{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "Heartbeat",
          "refId": "K",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_current_inmsg_invaliddb{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "InvalidDB",
          "refId": "L",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_current_inmsg_mmsg{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "MissingMsg",
          "refId": "M",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_current_inmsg_mmsgresp{job=~\"$node\"}",
          "hide": true,
          "interval": "",
          "intervalFactor": 2,
          "legendFormat": "MissingMsgResp",
          "refId": "N",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_current_inmsg_missingdata{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "MissingData",
          "refId": "O",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_current_inmsg_missingdataresp{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "MissingDataResp",
          "refId": "P",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_current_inmsg_revealentry{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "RevealEntry",
          "refId": "Q",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_current_inmsg_reqblock{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "ReqBlock",
          "refId": "T",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_current_inmsg_dbstatemissing{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "DBStateMissing",
          "refId": "U",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_current_inmsg_dbstate{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "DBState",
          "refId": "V",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_current_inmsg_bounce{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "Bounce",
          "refId": "W",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_current_inmsg_bounceresp{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "BounceResp",
          "refId": "X",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_current_inmsg_misc{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "Misc",
          "refId": "Y",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_current_inmsg_ack{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "Acks",
          "refId": "R",
          "step": 4
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "fill": false,
          "line": true,
          "op": "gt",
          "value": 8500
        }
      ],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Current InMsgQueue",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "label": "Current # InMsgQueue",
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        },
        {
          "format": "short",
          "label": "",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS}",
      "decimals": 4,
      "fill": 1,
      "gridPos": {
        "h": 7,
        "w": 24,
        "x": 0,
        "y": 20
      },
      "id": 31,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": true,
        "hideEmpty": false,
        "hideZero": false,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "sideWidth": 300,
        "sort": "current",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null as zero",
      "paceLength": 10,
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "factomd_state_queue_total_general_inmsg_vec{job=~\"$node\"}",
          "hide": false,
          "intervalFactor": 2,
          "legendFormat": "{{message}}",
          "metric": "factomd_state_queue_current_general_inmsg_vec",
          "refId": "A",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_inmsg_eom{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "EOM",
          "refId": "B",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_inmsg_audfault{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "AudFault",
          "refId": "C",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_inmsg_fedfault{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "FedFault",
          "refId": "D",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_inmsg_fullfault{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "FullFault",
          "refId": "E",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_inmsg_commitchain{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "CommitChain",
          "refId": "F",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_inmsg_commitentry{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "CommitEntry",
          "refId": "G",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_inmsg_dbsig{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "DBSig",
          "refId": "H",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_inmsg_eomtimeout{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "EOMTimeout",
          "refId": "I",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_inmsg_facttx{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "FactoidTx",
          "refId": "J",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_inmsg_heartbeat{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "Heartbeat",
          "refId": "K",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_inmsg_invaliddb{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "InvalidDB",
          "refId": "L",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_inmsg_mmsg{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "MissingMsg",
          "refId": "M",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_inmsg_mmsgresp{job=~\"$node\"}",
          "hide": true,
          "interval": "",
          "intervalFactor": 2,
          "legendFormat": "MissingMsgResp",
          "refId": "N",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_inmsg_missingdata{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "MissingData",
          "refId": "O",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_inmsg_missingdataresp{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "MissingDataResp",
          "refId": "P",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_inmsg_revealentry{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "RevealEntry",
          "refId": "Q",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_inmsg_reqblock{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "ReqBlock",
          "refId": "T",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_inmsg_dbstatemissing{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "DBStateMissing",
          "refId": "U",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_inmsg_dbstate{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "DBState",
          "refId": "V",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_inmsg_bounce{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "Bounce",
          "refId": "W",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_inmsg_bounceresp{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "BounceResp",
          "refId": "X",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_inmsg_misc{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "Misc",
          "refId": "Y",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_inmsg_ack{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "Acks",
          "refId": "R",
          "step": 4
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Total InMsgQueue",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "label": "Total Through InMsgQueue",
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        },
        {
          "format": "short",
          "label": "",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS}",
      "decimals": 3,
      "fill": 1,
      "gridPos": {
        "h": 7,
        "w": 24,
        "x": 0,
        "y": 27
      },
      "id": 32,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": true,
        "hideEmpty": false,
        "hideZero": false,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "sideWidth": 300,
        "sort": "current",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null as zero",
      "paceLength": 10,
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "factomd_state_queue_total_general_netoutmsg_vec{job=~\"$node\"}",
          "intervalFactor": 2,
          "legendFormat": "{{message}}",
          "metric": "factomd_state_queue_total_general_netoutmsg_vec",
          "refId": "R",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_netoutmsg_ack{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "Acks",
          "metric": "factomd_state_queue_current_insmg_ack",
          "refId": "A",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_netoutmsg_eom{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "EOM",
          "refId": "B",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_netoutmsg_audfault{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "AudFault",
          "refId": "C",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_netoutmsg_fedfault{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "FedFault",
          "refId": "D",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_netoutmsg_fullfault{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "FullFault",
          "refId": "E",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_netoutmsg_commitchain{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "CommitChain",
          "refId": "F",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_netoutmsg_commitentry{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "CommitEntry",
          "refId": "G",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_netoutmsg_dbsig{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "DBSig",
          "refId": "H",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_netoutmsg_eomtimeout{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "EOMTimeout",
          "refId": "I",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_netoutmsg_facttx{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "FactoidTx",
          "refId": "J",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_netoutmsg_heartbeat{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "Heartbeat",
          "refId": "K",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_netoutmsg_invaliddb{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "InvalidDB",
          "refId": "L",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_netoutmsg_mmsg{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "MissingMsg",
          "refId": "M",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_netoutmsg_mmsgresp{job=~\"$node\"}",
          "hide": true,
          "interval": "",
          "intervalFactor": 2,
          "legendFormat": "MissingMsgResp",
          "refId": "N",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_netoutmsg_missingdata{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "MissingData",
          "refId": "O",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_netoutmsg_missingdataresp{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "MissingDataResp",
          "refId": "P",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_netoutmsg_revealentry{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "RevealEntry",
          "refId": "Q",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_netoutmsg_reqblock{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "ReqBlock",
          "refId": "T",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_netoutmsg_dbstatemissing{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "DBStateMissing",
          "refId": "U",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_netoutmsg_dbstate{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "DBState",
          "refId": "V",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_netoutmsg_bounce{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "Bounce",
          "refId": "W",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_netoutmsg_bounceresp{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "BounceResp",
          "refId": "X",
          "step": 4
        },
        {
          "expr": "factomd_state_queue_total_netoutmsg_misc{job=~\"$node\"}",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "Misc",
          "refId": "Y",
          "step": 4
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Total NetworkOut",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "label": "Total Through NetworkOut",
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        },
        {
          "format": "short",
          "label": "",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS}",
      "fill": 1,
      "gridPos": {
        "h": 7,
        "w": 24,
        "x": 0,
        "y": 34
      },
      "id": 33,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": true,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "sort": "current",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "paceLength": 10,
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "factomd_state_msg_replay_toss_total",
          "hide": false,
          "intervalFactor": 2,
          "legendFormat": "{{job}}",
          "metric": "factomd_state_msg_replay_toss_total",
          "refId": "A",
          "step": 4
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Repeat Messages (tossed)",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        },
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS}",
      "fill": 1,
      "gridPos": {
        "h": 7,
        "w": 24,
        "x": 0,
        "y": 41
      },
      "id": 34,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": true,
        "max": true,
        "min": false,
        "rightSide": true,
        "show": true,
        "sort": null,
        "sortDesc": null,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "paceLength": 10,
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "factomd_state_broadcast_in_current{job=~\"$node\"}",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "{{job}}",
          "metric": "factomd_state_broadcast_in_current",
          "refId": "A",
          "step": 4
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "BroadCastInQueue",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        },
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS}",
      "fill": 1,
      "gridPos": {
        "h": 7,
        "w": 24,
        "x": 0,
        "y": 48
      },
      "id": 35,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": true,
        "max": true,
        "min": false,
        "rightSide": true,
        "show": true,
        "sort": "max",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "paceLength": 10,
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "sum({__name__=~\"factomd_state_queue_current_general_inmsg_vec\"}) by (job)",
          "format": "time_series",
          "hide": false,
          "intervalFactor": 2,
          "legendFormat": "{{job}}",
          "metric": "factomd_state_queue_current_general_inmsg_vec",
          "refId": "A",
          "step": 4
        },
        {
          "expr": "sum({__name__=~\"factomd_state_queue_current_general_inmsg_vec\", job=\"factom-factomd-m2prodfb\"})",
          "format": "time_series",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "m2prodfb",
          "refId": "B",
          "step": 4
        },
        {
          "expr": "sum({__name__=~\"factomd_state_queue_current_inmsg_.*\", job=\"factom-factomd-m2prodfc\"})",
          "format": "time_series",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "m2prodfc",
          "refId": "C",
          "step": 4
        },
        {
          "expr": "sum({__name__=~\"factomd_state_queue_current_inmsg_.*\", job=\"factom-factomd-m2prodfd\"})",
          "format": "time_series",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "m2prodfd",
          "refId": "D",
          "step": 4
        },
        {
          "expr": "sum({__name__=~\"factomd_state_queue_current_inmsg_.*\", job=\"factom-factomd-m2prodfe\"})",
          "format": "time_series",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "m2prodfe",
          "refId": "E",
          "step": 4
        },
        {
          "expr": "sum({__name__=~\"factomd_state_queue_current_inmsg_.*\", job=\"factom-factomd-m2prodff\"})",
          "format": "time_series",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "m2prodff",
          "refId": "F",
          "step": 4
        },
        {
          "expr": "sum({__name__=~\"factomd_state_queue_current_inmsg_.*\", job=\"factom-factomd-m2prodfg\"})",
          "format": "time_series",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "m2prodfg",
          "refId": "G",
          "step": 4
        },
        {
          "expr": "sum({__name__=~\"factomd_state_queue_current_inmsg_.*\", job=\"factom-factomd-m2prodfh\"})",
          "format": "time_series",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "m2prodfh",
          "refId": "H",
          "step": 4
        },
        {
          "expr": "sum({__name__=~\"factomd_state_queue_current_inmsg_.*\", job=\"factom-factomd-m2prodfi\"})",
          "format": "time_series",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "m2prodfi",
          "refId": "I",
          "step": 4
        },
        {
          "expr": "sum({__name__=~\"factomd_state_queue_current_inmsg_.*\", job=\"factom-factomd-m2prodaa\"})",
          "format": "time_series",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "m2prodaa",
          "refId": "J",
          "step": 4
        },
        {
          "expr": "sum({__name__=~\"factomd_state_queue_current_inmsg_.*\", job=\"factom-factomd-m2prodab\"})",
          "format": "time_series",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "m2prodab",
          "refId": "K",
          "step": 4
        },
        {
          "expr": "sum({__name__=~\"factomd_state_queue_current_inmsg_.*\", job=\"factom-factomd-m2prodac\"})",
          "format": "time_series",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "m2prodac",
          "refId": "L",
          "step": 4
        },
        {
          "expr": "sum({__name__=~\"factomd_state_queue_current_inmsg_.*\", job=\"factom-factomd-m2prodad\"})",
          "format": "time_series",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "m2prodad",
          "refId": "M",
          "step": 4
        },
        {
          "expr": "sum({__name__=~\"factomd_state_queue_current_inmsg_.*\", job=\"factom-factomd-m2prodae\"})",
          "format": "time_series",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "m2prodae",
          "refId": "N",
          "step": 4
        },
        {
          "expr": "sum({__name__=~\"factomd_state_queue_current_inmsg_.*\", job=\"factom-factomd-m2prodaf\"})",
          "format": "time_series",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "m2prodaf",
          "refId": "O",
          "step": 4
        },
        {
          "expr": "sum({__name__=~\"factomd_state_queue_current_inmsg_.*\", job=\"factom-factomd-m2prodag\"})",
          "format": "time_series",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "m2prodag",
          "refId": "P",
          "step": 4
        },
        {
          "expr": "sum({__name__=~\"factomd_state_queue_current_inmsg_.*\", job=\"factom-factomd-m2prodah\"})",
          "format": "time_series",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "m2prodah",
          "refId": "Q",
          "step": 4
        },
        {
          "expr": "sum({__name__=~\"factomd_state_queue_current_inmsg_.*\", job=\"factom-factomd-m2prodai\"})",
          "format": "time_series",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "m2prodai",
          "refId": "R",
          "step": 4
        },
        {
          "expr": "sum({__name__=~\"factomd_state_queue_current_inmsg_.*\", job=\"factom-factomd-m2prodfa\"})",
          "format": "time_series",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "m2prodfa",
          "refId": "S",
          "step": 4
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "InMsgQueue Current Per Node",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "label": "InMsgQueue Current",
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        },
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {
        "Arrival Instant": "#6D1F62",
        "Arrival Moving": "#0A437C",
        "Arrival Total": "#64B0C8",
        "Backup": "#DEDAF7",
        "Complete Instant": "#58140C",
        "Complete Moving": "#BF1B00",
        "Complete Total": "#E0752D"
      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS}",
      "fill": 1,
      "gridPos": {
        "h": 7,
        "w": 24,
        "x": 0,
        "y": 55
      },
      "id": 37,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": true,
        "max": true,
        "min": true,
        "rightSide": true,
        "show": true,
        "sort": "max",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "paceLength": 10,
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "sum(rate(factomd_state_queue_total_general_inmsg_vec[1m])) by (job)",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "{{job}}",
          "refId": "A",
          "step": 4
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "InMsgQueue Rates",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        },
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {
        "Arrival Instant": "#6D1F62",
        "Arrival Moving": "#0A437C",
        "Arrival Total": "#64B0C8",
        "Backup": "#DEDAF7",
        "Complete Instant": "#58140C",
        "Complete Moving": "#BF1B00",
        "Complete Total": "#E0752D",
        "Moving Arrival": "#0A437C",
        "Moving Complete": "#629E51"
      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS}",
      "fill": 1,
      "gridPos": {
        "h": 7,
        "w": 24,
        "x": 0,
        "y": 62
      },
      "id": 45,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": true,
        "max": true,
        "min": true,
        "rightSide": true,
        "show": true,
        "sort": null,
        "sortDesc": null,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "paceLength": 10,
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [
        {
          "alias": "Total NetOut Rate",
          "stack": false
        }
      ],
      "spaceLength": 10,
      "stack": true,
      "steppedLine": false,
      "targets": [
        {
          "expr": "sum(rate(factomd_state_queue_total_general_netoutmsg_vec[1m])) by (job)",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "Total NetOut Rate {{job}}",
          "metric": "vec",
          "refId": "H",
          "step": 4
        },
        {
          "expr": "rate(factomd_state_queue_total_general_netoutmsg_vec{job=~\"$node\"}[1 m])",
          "format": "time_series",
          "hide": true,
          "intervalFactor": 2,
          "legendFormat": "{{message}}",
          "refId": "A",
          "step": 4
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "NetOutQueue Rates",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        },
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS}",
      "fill": 1,
      "gridPos": {
        "h": 7,
        "w": 24,
        "x": 0,
        "y": 69
      },
      "id": 36,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": true,
        "max": true,
        "min": false,
        "rightSide": true,
        "show": true,
        "sort": "max",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "paceLength": 10,
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "factomd_state_txrate_instant_tps",
          "intervalFactor": 2,
          "legendFormat": "InstantTXRate {{job}} ",
          "metric": "factomd_state_txrate_instant_tps",
          "refId": "A",
          "step": 4
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Instant Transaction Rate",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS}",
      "fill": 1,
      "gridPos": {
        "h": 7,
        "w": 24,
        "x": 0,
        "y": 76
      },
      "id": 43,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": true,
        "max": true,
        "min": false,
        "rightSide": true,
        "show": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "paceLength": 10,
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "factomd_state_txrate_total_tps",
          "intervalFactor": 2,
          "legendFormat": "TotalTXRate {{job}} ",
          "metric": "factomd_state_txrate_total_tps",
          "refId": "A",
          "step": 4
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Total Transaction Rate",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 83
      },
      "id": 56,
      "panels": [
        {
          "content": "# API",
          "gridPos": {
            "h": 3,
            "w": 6,
            "x": 0,
            "y": 11
          },
          "height": "100",
          "id": 38,
          "links": [],
          "mode": "markdown",
          "title": "API",
          "type": "text"
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "fill": 1,
          "gridPos": {
            "h": 7,
            "w": 24,
            "x": 0,
            "y": 14
          },
          "id": 39,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": true,
            "sort": "current",
            "sortDesc": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "paceLength": 10,
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {
              "alias": "Total",
              "stack": false
            }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "factomd_wsapi_v2_api_general_call_ns_count{job=~\"$node\"}",
              "intervalFactor": 2,
              "legendFormat": "Total",
              "refId": "A",
              "step": 4
            },
            {
              "expr": "{__name__=~\"factomd_wsapi_v2_api_call_.*._ns_count\", job=\"$node\"}",
              "intervalFactor": 2,
              "legendFormat": "{{__name__}}",
              "refId": "B",
              "step": 4
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "API Call Counts",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": "# of Calls",
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "fill": 1,
          "gridPos": {
            "h": 7,
            "w": 24,
            "x": 0,
            "y": 21
          },
          "id": 47,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "paceLength": 10,
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {
              "alias": "Total",
              "stack": false
            }
          ],
          "spaceLength": 10,
          "stack": true,
          "steppedLine": false,
          "targets": [
            {
              "expr": "factomd_state_queue_current_general_api_vec{job=~\"$node\"}",
              "intervalFactor": 2,
              "legendFormat": "{{message}}",
              "metric": "factomd_state_queue_current_general_inmsg_vec",
              "refId": "A",
              "step": 4
            },
            {
              "expr": "sum(factomd_state_queue_current_general_api_vec{job=~\"$node\"})",
              "intervalFactor": 2,
              "legendFormat": "Total",
              "refId": "B",
              "step": 4
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Current API Queue",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "fill": 1,
          "gridPos": {
            "h": 7,
            "w": 24,
            "x": 0,
            "y": 28
          },
          "id": 48,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "paceLength": 10,
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {
              "alias": "Total",
              "stack": false
            }
          ],
          "spaceLength": 10,
          "stack": true,
          "steppedLine": false,
          "targets": [
            {
              "expr": "factomd_state_queue_total_general_api_vec{job=~\"$node\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{message}}",
              "metric": "factomd_state_queue_current_general_inmsg_vec",
              "refId": "A",
              "step": 4
            },
            {
              "expr": "sum(factomd_state_queue_total_general_api_vec{job=~\"$node\"})",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "Total",
              "refId": "B",
              "step": 4
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Total API Queue",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "fill": 1,
          "gridPos": {
            "h": 7,
            "w": 24,
            "x": 0,
            "y": 35
          },
          "id": 41,
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": true,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "paceLength": 10,
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {
              "alias": "Total Rate",
              "stack": false
            }
          ],
          "spaceLength": 10,
          "stack": true,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(rate(factomd_state_queue_total_general_api_vec{job=~\"$node\"} [1 m])) by(message)",
              "intervalFactor": 2,
              "legendFormat": "{{message}}",
              "metric": "factomd_state_queue_total_general_api_vec",
              "refId": "E",
              "step": 4
            },
            {
              "expr": "sum(rate(factomd_state_queue_total_general_api_vec{job=~\"$node\"} [1 m]))",
              "intervalFactor": 2,
              "legendFormat": "Total Rate",
              "refId": "F",
              "step": 4
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "API Calls Per Second (15s samples)",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": "# Of API/s",
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "fill": 1,
          "gridPos": {
            "h": 7,
            "w": 24,
            "x": 0,
            "y": 42
          },
          "id": 42,
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": true,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "paceLength": 10,
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "factomd_wsapi_v2_api_general_call_ns{job=~\"$node\"}",
              "intervalFactor": 2,
              "legendFormat": "{{quantile}}",
              "refId": "A",
              "step": 4
            },
            {
              "expr": "factomd_wsapi_v2_api_general_call_ns_sum/factomd_wsapi_v2_api_general_call_ns_count{job=~\"$node\"}",
              "intervalFactor": 2,
              "legendFormat": "Average",
              "refId": "B",
              "step": 4
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "API Handle Times (General)",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "ns",
              "label": "Time for a call to complete",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "repeat": null,
      "title": "API",
      "type": "row"
    }
  ],
  "refresh": "30s",
  "schemaVersion": 18,
  "style": "dark",
  "tags": [],
  "templating": {
    "list": [
      {
        "allValue": null,
        "current": {},
        "datasource": "${DS_PROMETHEUS}",
        "definition": "label_values(factomd_database_overlay_gets_eblock, job)",
        "hide": 0,
        "includeAll": true,
        "label": null,
        "multi": true,
        "name": "node",
        "options": [],
        "query": "label_values(factomd_database_overlay_gets_eblock, job)",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 5,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      }
    ]
  },
  "time": {
    "from": "now-30m",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "5s",
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ],
    "time_options": [
      "5m",
      "15m",
      "1h",
      "6h",
      "12h",
      "24h",
      "2d",
      "7d",
      "30d"
    ]
  },
  "timezone": "browser",
  "title": "TFA-Factomd",
  "uid": "K5zRPiemk",
  "version": 3
}