Grafana dashboard: NSO
Description: Metrics on Cisco NSO

Explore the performance and behavior of your Cisco NSO system through visual graphs.

This dashboard graphs the data that can be exported from a Cisco NSO system using the opentelemetry-exporter package available at https://gitlab.com/nso-developer/opentelemetry-exporter/

You can easily create your own graphs showing the details you are interested in. The opentelemetry-exporter package exports far more information that is included in this dashboard. Create your own queries to further dig into your data!

{
  "__inputs": [
    {
      "name": "DS_INFLUXDB",
      "label": "InfluxDB",
      "description": "InfluxDB datasource with NSO data",
      "type": "datasource",
      "pluginId": "influxdb",
      "pluginName": "InfluxDB"
    },
    {
      "name": "JAEGER_BASE_URL",
      "type": "constant",
      "label": "JAEGER_BASE_URL",
      "value": "http://127.0.0.1:16686/",
      "description": "Base URL to jaeger, e.g. 'http://127.0.0.1:16686/'"
    }
  ],
  "__requires": [
    {
      "id": "gauge",
      "name": "Gauge",
      "type": "panel",
      "version": ""
    },
    {
      "id": "grafana",
      "name": "Grafana",
      "type": "grafana",
      "version": "7.3.7"
    },
    {
      "id": "graph",
      "name": "Graph",
      "type": "panel",
      "version": ""
    },
    {
      "id": "influxdb",
      "name": "InfluxDB",
      "type": "datasource",
      "version": "1.0.0"
    },
    {
      "id": "singlestat",
      "name": "Singlestat",
      "type": "panel",
      "version": ""
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": false,
        "iconColor": "#B877D9",
        "limit": 100,
        "name": "Annotations & Alerts",
        "query": "operationName : \"transaction\"",
        "showIn": 0,
        "tags": [],
        "timeField": "@startTimeMillis",
        "type": "dashboard"
      },
      {
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": false,
        "iconColor": "rgba(255, 96, 96, 1)",
        "limit": 100,
        "name": "Test markers",
        "showIn": 0,
        "tags": [
          "testmarker"
        ],
        "type": "tags"
      }
    ]
  },
  "description": "Metrics on Cisco NSO",
  "editable": true,
  "gnetId": 14353,
  "graphTooltip": 0,
  "id": null,
  "iteration": 1620681183999,
  "links": [],
  "panels": [
    {
      "datasource": "${DS_INFLUXDB}",
      "fieldConfig": {
        "defaults": {
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 2,
        "x": 0,
        "y": 0
      },
      "id": 45,
      "options": {
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "/^span\\.count$/",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": false,
        "text": {}
      },
      "pluginVersion": "7.5.4",
      "targets": [
        {
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "transaction-lock",
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT count(\"duration\") FROM \"span\" WHERE (\"name\" = 'applying transaction') AND $timeFilter GROUP BY time($__interval) fill(null)",
          "rawQuery": true,
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "queue-length"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": []
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "transactions",
      "type": "gauge"
    },
    {
      "datasource": "${DS_INFLUXDB}",
      "fieldConfig": {
        "defaults": {
          "mappings": [
            {
              "from": "",
              "id": 0,
              "operator": "",
              "text": "unlocked",
              "to": "",
              "type": 1,
              "value": "0"
            },
            {
              "from": "",
              "id": 1,
              "operator": "",
              "text": "locked",
              "to": "",
              "type": 1,
              "value": "1"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "#EAB839",
                "value": 1
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 2,
        "x": 2,
        "y": 0
      },
      "id": 16,
      "options": {
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "/^held$/",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": false,
        "text": {}
      },
      "pluginVersion": "7.5.4",
      "targets": [
        {
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "limit": "1",
          "measurement": "transaction-lock",
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT \"held\" FROM \"transaction-lock\" WHERE $timeFilter",
          "rawQuery": true,
          "refId": "A",
          "resultFormat": "table",
          "select": [
            [
              {
                "params": [
                  "held"
                ],
                "type": "field"
              }
            ]
          ],
          "tags": []
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "t-lock",
      "type": "gauge"
    },
    {
      "datasource": "${DS_INFLUXDB}",
      "fieldConfig": {
        "defaults": {
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "#EAB839",
                "value": 1
              },
              {
                "color": "red",
                "value": 2
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 2,
        "x": 4,
        "y": 0
      },
      "id": 18,
      "options": {
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "/^transaction\\-lock\\.mean$/",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true,
        "text": {}
      },
      "pluginVersion": "7.5.4",
      "targets": [
        {
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "transaction-lock",
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT mean(\"queue-length\") FROM \"transaction-lock\" WHERE $timeFilter GROUP BY time($__interval) fill(null)",
          "rawQuery": true,
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "queue-length"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": []
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "queue length",
      "type": "gauge"
    },
    {
      "collapsed": false,
      "datasource": "${DS_INFLUXDB}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 4
      },
      "id": 38,
      "panels": [],
      "title": "Overview",
      "type": "row"
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_INFLUXDB}",
      "description": "Transaction throughput",
      "fieldConfig": {
        "defaults": {},
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 5
      },
      "hiddenSeries": false,
      "id": 41,
      "legend": {
        "avg": true,
        "current": false,
        "max": true,
        "min": false,
        "show": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "connected",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.5.4",
      "pointradius": 0.5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [
        {
          "$$hashKey": "object:241",
          "alias": "Transaction length",
          "bars": true,
          "lines": false,
          "yaxis": 2
        }
      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "alias": "Number of transactions",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "span",
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT count(\"duration\") FROM \"span\" WHERE (\"name\" = 'applying transaction') AND $timeFilter GROUP BY time($__interval) fill(null)",
          "rawQuery": true,
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "duration"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": [
            {
              "key": "name",
              "operator": "=",
              "value": "create"
            }
          ]
        },
        {
          "alias": "Transaction length",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "hide": false,
          "measurement": "span",
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT mean(\"duration\") FROM \"span\" WHERE (\"name\" = 'applying transaction') AND $timeFilter GROUP BY time($__interval) fill(null)",
          "rawQuery": true,
          "refId": "B",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "duration"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": [
            {
              "key": "name",
              "operator": "=",
              "value": "create"
            }
          ]
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Transactions",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:141",
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "$$hashKey": "object:142",
          "format": "ms",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "datasource": "${DS_INFLUXDB}",
      "description": "This is a top list of the longest transactions for the given time period. Use it to identify things that might be slow.\n\nA long transaction in itself might not be a bad thing since transactions can essentially be idle and not block other transactions. However, if any lock is taken, the transaction should try to finish as soon as possible.",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null,
            "displayMode": "auto",
            "filterable": false
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "#6ED0E0",
                "value": 250
              },
              {
                "color": "#EAB839",
                "value": 500
              },
              {
                "color": "red",
                "value": 1000
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "top"
            },
            "properties": [
              {
                "id": "custom.displayMode",
                "value": "gradient-gauge"
              },
              {
                "id": "unit",
                "value": "ms"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "trace_id"
            },
            "properties": [
              {
                "id": "links",
                "value": [
                  {
                    "targetBlank": true,
                    "title": "Jaeger",
                    "url": "${jaeger_base_url}trace/${__data.fields[trace_id]}"
                  }
                ]
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Time"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": null
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "tid"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 306
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 5
      },
      "id": 31,
      "links": [],
      "options": {
        "showHeader": true,
        "sortBy": [
          {
            "desc": true,
            "displayName": "top"
          }
        ]
      },
      "pluginVersion": "7.5.4",
      "targets": [
        {
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT top(\"duration\", 100), trace_id, tid FROM \"span\" WHERE \"name\" = 'transaction' AND $timeFilter",
          "rawQuery": true,
          "refId": "A",
          "resultFormat": "table",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": []
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Longest transactions",
      "type": "table"
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_INFLUXDB}",
      "description": "The shaded red background shows when the transaction-lock is held (but will tend to overestimate due to time aggregation, zoom in for a better look!). The yellow line is a moving average of the time the transaction-lock is held.\nFinally, the blue line shows the length of the queue to the transaction-lock, i.e. how many transactions are trying to acquire the transaction-lock.",
      "fieldConfig": {
        "defaults": {
          "links": []
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 13
      },
      "hiddenSeries": false,
      "id": 9,
      "legend": {
        "avg": true,
        "current": false,
        "max": true,
        "min": false,
        "show": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "connected",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.5.4",
      "pointradius": 1,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [
        {
          "$$hashKey": "object:125",
          "alias": "held",
          "color": "rgba(118, 25, 25, 0.17)",
          "fill": 4,
          "legend": false,
          "linewidth": 0,
          "steppedLine": true
        },
        {
          "$$hashKey": "object:126",
          "alias": "queue length",
          "color": "#1F60C4",
          "fill": 4,
          "lines": true,
          "steppedLine": true,
          "yaxis": 2
        },
        {
          "$$hashKey": "object:153",
          "alias": "lock",
          "color": "#FADE2A",
          "fill": 0,
          "fillGradient": 3,
          "zindex": 3
        }
      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "alias": "held",
          "groupBy": [],
          "hide": false,
          "measurement": "transaction-lock",
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT max(\"held\") FROM \"transaction-lock\" WHERE $timeFilter GROUP BY time($__interval) fill(null)",
          "rawQuery": true,
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "held"
                ],
                "type": "field"
              }
            ]
          ],
          "tags": []
        },
        {
          "alias": "queue length",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "hide": false,
          "measurement": "transaction-lock",
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT mean(\"queue-length\") FROM \"transaction-lock\" WHERE $timeFilter GROUP BY time($__interval) fill(null)",
          "rawQuery": true,
          "refId": "B",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "queue-length"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "max"
              }
            ]
          ],
          "tags": []
        },
        {
          "alias": "lock",
          "groupBy": [],
          "hide": false,
          "measurement": "transaction-lock",
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT moving_average(mean(\"held\"),10) FROM \"transaction-lock\" WHERE $timeFilter GROUP BY time($__interval) fill(null)",
          "rawQuery": true,
          "refId": "C",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "held"
                ],
                "type": "field"
              }
            ]
          ],
          "tags": []
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Transaction lock & queue length",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:96",
          "decimals": 0,
          "format": "short",
          "label": "transaction-lock held",
          "logBase": 1,
          "max": "1",
          "min": "0",
          "show": true
        },
        {
          "$$hashKey": "object:97",
          "decimals": 1,
          "format": "none",
          "label": "queue length",
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": true,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_INFLUXDB}",
      "description": "A histogram of total transaction duration. Notice this includes parts outside of the transaction-lock.\n\nX-axis shows buckets of transaction duration in seconds.\nY-axis is the number of transactions in that bucket.",
      "fieldConfig": {
        "defaults": {
          "links": []
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 13
      },
      "hiddenSeries": false,
      "id": 27,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": false,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.5.4",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "alias": "transaction duration",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "transaction",
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT max(\"transaction\")/1000000 FROM \"transaction\" WHERE $timeFilter GROUP BY time($__interval) fill(null)",
          "rawQuery": true,
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "transaction"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "max"
              }
            ]
          ],
          "tags": []
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Transaction duration",
      "tooltip": {
        "shared": false,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "min": null,
        "mode": "histogram",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:323",
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "$$hashKey": "object:324",
          "format": "ms",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "collapsed": true,
      "datasource": null,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 21
      },
      "id": 47,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_INFLUXDB}",
          "description": "Time spent synchronizing changes in a HA group.",
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 4,
          "fillGradient": 6,
          "gridPos": {
            "h": 9,
            "w": 12,
            "x": 0,
            "y": 22
          },
          "hiddenSeries": false,
          "id": 48,
          "legend": {
            "avg": true,
            "current": false,
            "max": true,
            "min": false,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "connected",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.5",
          "pointradius": 0.5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "alias": "configuration data",
              "groupBy": [
                {
                  "params": [
                    "$__interval"
                  ],
                  "type": "time"
                },
                {
                  "params": [
                    "null"
                  ],
                  "type": "fill"
                }
              ],
              "measurement": "span",
              "orderByTime": "ASC",
              "policy": "default",
              "query": "SELECT max(\"duration\") FROM \"span\" WHERE (\"name\" = 'create') AND $timeFilter GROUP BY time($__interval) fill(null)",
              "rawQuery": false,
              "refId": "B",
              "resultFormat": "time_series",
              "select": [
                [
                  {
                    "params": [
                      "duration"
                    ],
                    "type": "field"
                  },
                  {
                    "params": [],
                    "type": "max"
                  }
                ]
              ],
              "tags": [
                {
                  "key": "name",
                  "operator": "=",
                  "value": "distributing configuration data changeset to all (available) slaves"
                }
              ]
            },
            {
              "alias": "operational data",
              "groupBy": [
                {
                  "params": [
                    "$__interval"
                  ],
                  "type": "time"
                },
                {
                  "params": [
                    "null"
                  ],
                  "type": "fill"
                }
              ],
              "hide": false,
              "measurement": "span",
              "orderByTime": "ASC",
              "policy": "default",
              "query": "SELECT max(\"duration\") FROM \"span\" WHERE (\"name\" = 'create') AND $timeFilter GROUP BY time($__interval) fill(null)",
              "rawQuery": false,
              "refId": "A",
              "resultFormat": "time_series",
              "select": [
                [
                  {
                    "params": [
                      "duration"
                    ],
                    "type": "field"
                  },
                  {
                    "params": [],
                    "type": "max"
                  }
                ]
              ],
              "tags": [
                {
                  "key": "name",
                  "operator": "=",
                  "value": "distributing operational data changeset to all (available) slaves"
                }
              ]
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "HA synchronization time",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:1007",
              "format": "ms",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:1008",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "title": "System",
      "type": "row"
    },
    {
      "collapsed": true,
      "datasource": "${DS_INFLUXDB}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 22
      },
      "id": 44,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_INFLUXDB}",
          "description": "The time to grab the transaction lock plotted against the transaction lock queue length.",
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 22
          },
          "hiddenSeries": false,
          "id": 4,
          "legend": {
            "avg": true,
            "current": false,
            "max": true,
            "min": false,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {
              "alias": "transaction lock queue length",
              "color": "#1F60C4",
              "nullPointMode": "connected",
              "steppedLine": true,
              "yaxis": 2,
              "zindex": -3
            },
            {
              "alias": "grabbing transaction lock",
              "bars": true,
              "color": "#FADE2A",
              "lines": false,
              "zindex": 3
            },
            {
              "alias": "max grabbing transaction lock",
              "color": "#F2495C",
              "lines": false,
              "pointradius": 1,
              "points": true
            }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "alias": "grabbing transaction lock",
              "groupBy": [],
              "measurement": "span",
              "orderByTime": "ASC",
              "policy": "default",
              "query": "SELECT mean(\"duration\") FROM \"span\" WHERE (\"name\" = 'grabbing transaction lock') AND $timeFilter GROUP BY time($__interval) fill(null)",
              "rawQuery": true,
              "refId": "A",
              "resultFormat": "time_series",
              "select": [
                [
                  {
                    "params": [
                      "duration"
                    ],
                    "type": "field"
                  }
                ]
              ],
              "tags": [
                {
                  "key": "name",
                  "operator": "=",
                  "value": "grabbing transaction lock"
                }
              ]
            },
            {
              "alias": "max grabbing transaction lock",
              "groupBy": [
                {
                  "params": [
                    "$__interval"
                  ],
                  "type": "time"
                },
                {
                  "params": [
                    "null"
                  ],
                  "type": "fill"
                }
              ],
              "orderByTime": "ASC",
              "policy": "default",
              "query": "SELECT max(\"duration\") FROM \"span\" WHERE (\"name\" = 'grabbing transaction lock') AND $timeFilter GROUP BY time($__interval) fill(null)",
              "rawQuery": true,
              "refId": "C",
              "resultFormat": "time_series",
              "select": [
                [
                  {
                    "params": [
                      "value"
                    ],
                    "type": "field"
                  },
                  {
                    "params": [],
                    "type": "mean"
                  }
                ]
              ],
              "tags": []
            },
            {
              "alias": "transaction lock queue length",
              "groupBy": [
                {
                  "params": [
                    "$__interval"
                  ],
                  "type": "time"
                },
                {
                  "params": [
                    "null"
                  ],
                  "type": "fill"
                }
              ],
              "hide": false,
              "orderByTime": "ASC",
              "policy": "default",
              "query": "SELECT mean(\"queue-length\") FROM \"transaction-lock\" WHERE $timeFilter GROUP BY time($__interval) fill(null)",
              "rawQuery": true,
              "refId": "B",
              "resultFormat": "time_series",
              "select": [
                [
                  {
                    "params": [
                      "value"
                    ],
                    "type": "field"
                  },
                  {
                    "params": [],
                    "type": "mean"
                  }
                ]
              ],
              "tags": []
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Waiting for transaction lock",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:460",
              "format": "ms",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
            },
            {
              "$$hashKey": "object:461",
              "format": "short",
              "label": "queue length",
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "datasource": "${DS_INFLUXDB}",
          "description": "This is a top list of the transactions that had to wait the longest in queue to acquire the transaction lock.\n\nThis list is hardly actionable in the sense that you wouldn't start looking at these transactions for tuning performance. Instead, these transactions are those that have waited the longest for other transactions to finish.",
          "fieldConfig": {
            "defaults": {
              "custom": {
                "align": null,
                "displayMode": "auto",
                "filterable": false
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "#6ED0E0",
                    "value": 250
                  },
                  {
                    "color": "#EAB839",
                    "value": 500
                  },
                  {
                    "color": "red",
                    "value": 1000
                  }
                ]
              }
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "top"
                },
                "properties": [
                  {
                    "id": "custom.displayMode",
                    "value": "gradient-gauge"
                  },
                  {
                    "id": "unit",
                    "value": "ms"
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "trace_id"
                },
                "properties": [
                  {
                    "id": "links",
                    "value": [
                      {
                        "targetBlank": true,
                        "title": "Jaeger",
                        "url": "${jaeger_base_url}trace/${__data.fields[trace_id]}"
                      }
                    ]
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Time"
                },
                "properties": [
                  {
                    "id": "custom.width",
                    "value": 164
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 22
          },
          "id": 42,
          "links": [],
          "options": {
            "showHeader": true,
            "sortBy": [
              {
                "desc": true,
                "displayName": "top"
              }
            ]
          },
          "pluginVersion": "7.5.4",
          "targets": [
            {
              "groupBy": [
                {
                  "params": [
                    "$__interval"
                  ],
                  "type": "time"
                },
                {
                  "params": [
                    "null"
                  ],
                  "type": "fill"
                }
              ],
              "orderByTime": "ASC",
              "policy": "default",
              "query": "SELECT top(\"duration\", 100), trace_id, tid FROM \"span\" WHERE \"name\" = 'grabbing transaction lock' AND $timeFilter",
              "rawQuery": true,
              "refId": "A",
              "resultFormat": "table",
              "select": [
                [
                  {
                    "params": [
                      "value"
                    ],
                    "type": "field"
                  },
                  {
                    "params": [],
                    "type": "mean"
                  }
                ]
              ],
              "tags": []
            }
          ],
          "timeFrom": null,
          "timeShift": null,
          "title": "Longest wait for transaction lock",
          "type": "table"
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_INFLUXDB}",
          "description": "The transaction phases as a percentage of the total transaction time.",
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 30
          },
          "hiddenSeries": false,
          "id": 23,
          "legend": {
            "avg": true,
            "current": false,
            "max": true,
            "min": false,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "connected",
          "options": {
            "alertThreshold": true
          },
          "percentage": true,
          "pluginVersion": "7.5.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": true,
          "steppedLine": false,
          "targets": [
            {
              "alias": "validate",
              "groupBy": [
                {
                  "params": [
                    "$__interval"
                  ],
                  "type": "time"
                },
                {
                  "params": [
                    "null"
                  ],
                  "type": "fill"
                }
              ],
              "measurement": "transaction",
              "orderByTime": "ASC",
              "policy": "default",
              "query": "SELECT (max(\"validate\")/max(\"transaction\"))*100 FROM \"transaction\" WHERE $timeFilter GROUP BY time($__interval) fill(null)",
              "rawQuery": true,
              "refId": "A",
              "resultFormat": "time_series",
              "select": [
                [
                  {
                    "params": [
                      "value"
                    ],
                    "type": "field"
                  },
                  {
                    "params": [],
                    "type": "mean"
                  }
                ]
              ],
              "tags": []
            },
            {
              "alias": "write-start",
              "groupBy": [
                {
                  "params": [
                    "$__interval"
                  ],
                  "type": "time"
                },
                {
                  "params": [
                    "null"
                  ],
                  "type": "fill"
                }
              ],
              "orderByTime": "ASC",
              "policy": "default",
              "query": "SELECT (max(\"write-start\")/max(\"transaction\"))*100 FROM \"transaction\" WHERE $timeFilter GROUP BY time($__interval) fill(null)",
              "rawQuery": true,
              "refId": "B",
              "resultFormat": "time_series",
              "select": [
                [
                  {
                    "params": [
                      "value"
                    ],
                    "type": "field"
                  },
                  {
                    "params": [],
                    "type": "mean"
                  }
                ]
              ],
              "tags": []
            },
            {
              "alias": "prepare",
              "groupBy": [
                {
                  "params": [
                    "$__interval"
                  ],
                  "type": "time"
                },
                {
                  "params": [
                    "null"
                  ],
                  "type": "fill"
                }
              ],
              "orderByTime": "ASC",
              "policy": "default",
              "query": "SELECT (max(\"prepare\")/max(\"transaction\"))*100 FROM \"transaction\" WHERE $timeFilter GROUP BY time($__interval) fill(null)",
              "rawQuery": true,
              "refId": "C",
              "resultFormat": "time_series",
              "select": [
                [
                  {
                    "params": [
                      "value"
                    ],
                    "type": "field"
                  },
                  {
                    "params": [],
                    "type": "mean"
                  }
                ]
              ],
              "tags": []
            },
            {
              "alias": "commit",
              "groupBy": [
                {
                  "params": [
                    "$__interval"
                  ],
                  "type": "time"
                },
                {
                  "params": [
                    "null"
                  ],
                  "type": "fill"
                }
              ],
              "orderByTime": "ASC",
              "policy": "default",
              "query": "SELECT (max(\"commit\")/max(\"transaction\"))*100 FROM \"transaction\" WHERE $timeFilter GROUP BY time($__interval) fill(null)",
              "rawQuery": true,
              "refId": "D",
              "resultFormat": "time_series",
              "select": [
                [
                  {
                    "params": [
                      "value"
                    ],
                    "type": "field"
                  },
                  {
                    "params": [],
                    "type": "mean"
                  }
                ]
              ],
              "tags": []
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Transaction phases",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "percent",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "cards": {
            "cardPadding": null,
            "cardRound": null
          },
          "color": {
            "cardColor": "#b4ff00",
            "colorScale": "sqrt",
            "colorScheme": "interpolateOranges",
            "exponent": 0.5,
            "mode": "spectrum"
          },
          "dataFormat": "timeseries",
          "datasource": "${DS_INFLUXDB}",
          "description": "Shows the time it takes to validate the data as a percentage of the total time the transaction lock is held.\n\nA high percentage could indicate problematic XPath constraints that could be optimized.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 30
          },
          "heatmap": {},
          "hideZeroBuckets": false,
          "highlightCards": true,
          "id": 21,
          "legend": {
            "show": false
          },
          "pluginVersion": "7.0.5",
          "reverseYBuckets": false,
          "targets": [
            {
              "alias": "rollback percentage",
              "groupBy": [
                {
                  "params": [
                    "$__interval"
                  ],
                  "type": "time"
                },
                {
                  "params": [
                    "null"
                  ],
                  "type": "fill"
                }
              ],
              "hide": false,
              "measurement": "span",
              "orderByTime": "ASC",
              "policy": "default",
              "query": "SELECT ((max(\"pre validate\")+max(\"run validation over the changeset\")+max(\"run dependency-triggered validation\"))/max(\"holding transaction lock\"))*100\nFROM transaction\nWHERE $timeFilter\nGROUP BY time($__interval) fill(null)",
              "rawQuery": true,
              "refId": "A",
              "resultFormat": "time_series",
              "select": [
                [
                  {
                    "params": [
                      "duration"
                    ],
                    "type": "field"
                  },
                  {
                    "params": [],
                    "type": "max"
                  }
                ]
              ],
              "tags": [
                {
                  "key": "name",
                  "operator": "=",
                  "value": "creating rollback file"
                }
              ]
            }
          ],
          "timeFrom": null,
          "timeShift": null,
          "title": "Data validation",
          "tooltip": {
            "show": true,
            "showHistogram": false
          },
          "type": "heatmap",
          "xAxis": {
            "show": true
          },
          "xBucketNumber": null,
          "xBucketSize": null,
          "yAxis": {
            "decimals": null,
            "format": "percent",
            "logBase": 1,
            "max": null,
            "min": null,
            "show": true,
            "splitFactor": null
          },
          "yBucketBound": "auto",
          "yBucketNumber": null,
          "yBucketSize": null
        },
        {
          "cards": {
            "cardPadding": null,
            "cardRound": null
          },
          "color": {
            "cardColor": "#b4ff00",
            "colorScale": "sqrt",
            "colorScheme": "interpolateBlues",
            "exponent": 0.5,
            "mode": "spectrum"
          },
          "dataFormat": "timeseries",
          "datasource": "${DS_INFLUXDB}",
          "description": "Shows the time it takes to create the rollback file as a percentage of the total time the transaction lock is held.\n\nIt is possible to disable rollback files for NSO if not needed for the deployment and can yield a non-negligible performance improvement. For very short transactions the rollback file creation can take a unproportionately large percentage and might be somewhat misleading.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 38
          },
          "heatmap": {},
          "hideZeroBuckets": false,
          "highlightCards": true,
          "id": 28,
          "legend": {
            "show": false
          },
          "pluginVersion": "7.0.5",
          "reverseYBuckets": false,
          "targets": [
            {
              "alias": "rollback percentage",
              "groupBy": [
                {
                  "params": [
                    "$__interval"
                  ],
                  "type": "time"
                },
                {
                  "params": [
                    "null"
                  ],
                  "type": "fill"
                }
              ],
              "hide": false,
              "measurement": "span",
              "orderByTime": "ASC",
              "policy": "default",
              "query": "SELECT (max(\"creating rollback file\")/max(\"holding transaction lock\"))*100\nFROM transaction\nWHERE $timeFilter\nGROUP BY time($__interval) fill(null)",
              "rawQuery": true,
              "refId": "A",
              "resultFormat": "time_series",
              "select": [
                [
                  {
                    "params": [
                      "duration"
                    ],
                    "type": "field"
                  },
                  {
                    "params": [],
                    "type": "max"
                  }
                ]
              ],
              "tags": [
                {
                  "key": "name",
                  "operator": "=",
                  "value": "creating rollback file"
                }
              ]
            }
          ],
          "timeFrom": null,
          "timeShift": null,
          "title": "Rollback file creation",
          "tooltip": {
            "show": true,
            "showHistogram": false
          },
          "type": "heatmap",
          "xAxis": {
            "show": true
          },
          "xBucketNumber": null,
          "xBucketSize": null,
          "yAxis": {
            "decimals": null,
            "format": "percent",
            "logBase": 1,
            "max": null,
            "min": null,
            "show": true,
            "splitFactor": null
          },
          "yBucketBound": "auto",
          "yBucketNumber": null,
          "yBucketSize": null
        },
        {
          "aliasColors": {},
          "bars": true,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_INFLUXDB}",
          "description": "Time spent creating the rollback file for a transaction \n\nAlso compared ",
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 46
          },
          "hiddenSeries": false,
          "id": 12,
          "legend": {
            "avg": true,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": false,
          "linewidth": 1,
          "nullPointMode": "connected",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.4",
          "pointradius": 0.5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {
              "alias": "rollback percentage",
              "bars": false,
              "color": "#E0B400",
              "fill": 0,
              "lines": true,
              "pointradius": 1,
              "points": true
            },
            {
              "alias": "creating rollback file",
              "color": "#FF9830",
              "yaxis": 2
            },
            {
              "alias": "holding transaction lock",
              "color": "#1F60C4",
              "yaxis": 2
            }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "alias": "rollback percentage",
              "groupBy": [
                {
                  "params": [
                    "$__interval"
                  ],
                  "type": "time"
                },
                {
                  "params": [
                    "null"
                  ],
                  "type": "fill"
                }
              ],
              "hide": false,
              "measurement": "span",
              "orderByTime": "ASC",
              "policy": "default",
              "query": "SELECT (max(\"creating rollback file\")/max(\"holding transaction lock\"))*100\nFROM transaction\nWHERE $timeFilter\nGROUP BY time($__interval) fill(null)",
              "rawQuery": true,
              "refId": "A",
              "resultFormat": "time_series",
              "select": [
                [
                  {
                    "params": [
                      "duration"
                    ],
                    "type": "field"
                  },
                  {
                    "params": [],
                    "type": "max"
                  }
                ]
              ],
              "tags": [
                {
                  "key": "name",
                  "operator": "=",
                  "value": "creating rollback file"
                }
              ]
            },
            {
              "alias": "holding transaction lock",
              "groupBy": [
                {
                  "params": [
                    "$__interval"
                  ],
                  "type": "time"
                },
                {
                  "params": [
                    "null"
                  ],
                  "type": "fill"
                }
              ],
              "hide": false,
              "measurement": "span",
              "orderByTime": "ASC",
              "policy": "default",
              "query": "SELECT max(\"holding transaction lock\") FROM \"transaction\" WHERE $timeFilter GROUP BY time($__interval) fill(null)",
              "rawQuery": true,
              "refId": "C",
              "resultFormat": "time_series",
              "select": [
                [
                  {
                    "params": [
                      "duration"
                    ],
                    "type": "field"
                  },
                  {
                    "params": [],
                    "type": "mean"
                  }
                ]
              ],
              "tags": [
                {
                  "key": "name",
                  "operator": "=",
                  "value": "holding transaction lock"
                }
              ]
            },
            {
              "alias": "creating rollback file",
              "groupBy": [
                {
                  "params": [
                    "$__interval"
                  ],
                  "type": "time"
                },
                {
                  "params": [
                    "null"
                  ],
                  "type": "fill"
                }
              ],
              "hide": false,
              "measurement": "span",
              "orderByTime": "ASC",
              "policy": "default",
              "query": "SELECT max(\"creating rollback file\") FROM \"transaction\" WHERE $timeFilter GROUP BY time($__interval) fill(null)",
              "rawQuery": true,
              "refId": "B",
              "resultFormat": "time_series",
              "select": [
                [
                  {
                    "params": [
                      "duration"
                    ],
                    "type": "field"
                  },
                  {
                    "params": [],
                    "type": "mean"
                  }
                ]
              ],
              "tags": [
                {
                  "key": "name",
                  "operator": "=",
                  "value": "creating rollback file"
                }
              ]
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Rollback file creation",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "percent",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
            },
            {
              "format": "µs",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "title": "Transaction details",
      "type": "row"
    },
    {
      "collapsed": false,
      "datasource": "${DS_INFLUXDB}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 23
      },
      "id": 36,
      "panels": [],
      "title": "Services",
      "type": "row"
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_INFLUXDB}",
      "fieldConfig": {
        "defaults": {
          "links": []
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 0,
        "y": 24
      },
      "hiddenSeries": false,
      "id": 20,
      "legend": {
        "avg": true,
        "current": false,
        "max": true,
        "min": false,
        "show": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "connected",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.5.4",
      "pointradius": 0.5,
      "points": true,
      "renderer": "flot",
      "seriesOverrides": [
        {
          "$$hashKey": "object:989",
          "alias": "create () (max)",
          "color": "#C4162A",
          "lines": false,
          "points": true
        },
        {
          "$$hashKey": "object:990",
          "alias": "create()",
          "points": false
        }
      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "alias": "create()",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "span",
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT mean(\"duration\") FROM \"span\" WHERE (\"name\" = 'create') AND $timeFilter GROUP BY time($__interval) fill(null)",
          "rawQuery": true,
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "duration"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": [
            {
              "key": "name",
              "operator": "=",
              "value": "create"
            }
          ]
        },
        {
          "alias": "create () (max)",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "span",
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT max(\"duration\") FROM \"span\" WHERE (\"name\" = 'create') AND $timeFilter GROUP BY time($__interval) fill(null)",
          "rawQuery": true,
          "refId": "B",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "duration"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "max"
              }
            ]
          ],
          "tags": [
            {
              "key": "name",
              "operator": "=",
              "value": "create"
            }
          ]
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "service modifications",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:1007",
          "format": "ms",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "$$hashKey": "object:1008",
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "datasource": "${DS_INFLUXDB}",
      "fieldConfig": {
        "defaults": {
          "mappings": [
            {
              "from": "",
              "id": 0,
              "operator": "",
              "text": "-",
              "to": "",
              "type": 1,
              "value": ""
            }
          ],
          "noValue": "-",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "#6ED0E0",
                "value": 250
              },
              {
                "color": "#EAB839",
                "value": 500
              },
              {
                "color": "red",
                "value": 1000
              }
            ]
          },
          "unit": "ms"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 13,
        "w": 12,
        "x": 12,
        "y": 24
      },
      "id": 30,
      "options": {
        "displayMode": "gradient",
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
            "mean"
          ],
          "fields": "",
          "values": false
        },
        "showUnfilled": true,
        "text": {}
      },
      "pluginVersion": "7.5.4",
      "targets": [
        {
          "alias": "$tag_service_type",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "service_type"
              ],
              "type": "tag"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT mean(\"duration\") AS \"duration\" FROM \"span\" WHERE \"name\"='create' AND $timeFilter GROUP BY \"service_type\"",
          "rawQuery": true,
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": []
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Mean time in create per service type",
      "transformations": [],
      "type": "bargauge"
    },
    {
      "datasource": "${DS_INFLUXDB}",
      "description": "This is a top list of the longest service create spans for the given time period. Use it to identify your slowest services and speed them up!",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null,
            "displayMode": "auto",
            "filterable": false
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "#6ED0E0",
                "value": 250
              },
              {
                "color": "#EAB839",
                "value": 500
              },
              {
                "color": "red",
                "value": 1000
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "top"
            },
            "properties": [
              {
                "id": "custom.displayMode",
                "value": "gradient-gauge"
              },
              {
                "id": "unit",
                "value": "ms"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "trace_id"
            },
            "properties": [
              {
                "id": "links",
                "value": [
                  {
                    "targetBlank": true,
                    "title": "Jaeger",
                    "url": "${jaeger_base_url}trace/${__data.fields[trace_id]}"
                  }
                ]
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Time"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 146
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 33
      },
      "id": 25,
      "links": [],
      "options": {
        "showHeader": true,
        "sortBy": [
          {
            "desc": true,
            "displayName": "top"
          }
        ]
      },
      "pluginVersion": "7.5.4",
      "targets": [
        {
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT top(\"duration\", 100), service, trace_id FROM \"span\" WHERE \"name\" = 'create' AND $timeFilter",
          "rawQuery": true,
          "refId": "A",
          "resultFormat": "table",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": []
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Longest service create spans",
      "type": "table"
    },
    {
      "collapsed": false,
      "datasource": "${DS_INFLUXDB}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 41
      },
      "id": 40,
      "panels": [],
      "title": "Devices",
      "type": "row"
    },
    {
      "aliasColors": {},
      "bars": true,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_INFLUXDB}",
      "description": "Shows device locks held over time",
      "fieldConfig": {
        "defaults": {
          "links": []
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 42
      },
      "hiddenSeries": false,
      "id": 14,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.5.4",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "device-lock",
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT mean(\"held\") FROM \"device-lock\" WHERE $timeFilter GROUP BY \"device\", time($__interval) fill(null)",
          "rawQuery": true,
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "held"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": []
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Device locks",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:153",
          "format": "short",
          "label": "device lock held",
          "logBase": 1,
          "max": "1",
          "min": "0",
          "show": true
        },
        {
          "$$hashKey": "object:154",
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": "                tags = {\"device\": pev['device']}                 tags.update(extra_tags)                 influx_client.write_points([                     {                         \"time\": datetime.datetime.utcfromtimestamp(pev['timestamp'] / 1000000.0),                         \"measurement\": \"device-lock\",                         \"tags\": tags,                         \"fields\": {                             \"held\": 1,                             \"tid\": pev['tid']                         }                     }                 ])",
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "datasource": "${DS_INFLUXDB}",
      "description": "This is a top list of the longest get-trans-id operations for the given time period. Use it to identify things that might be slow.\n\nA long transaction in itself might not be a bad thing since transactions can essentially be idle and not block other transactions. However, if any lock is taken, the transaction should try to finish as soon as possible.",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null,
            "displayMode": "auto",
            "filterable": false
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "#6ED0E0",
                "value": 250
              },
              {
                "color": "#EAB839",
                "value": 500
              },
              {
                "color": "red",
                "value": 1000
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "top"
            },
            "properties": [
              {
                "id": "custom.displayMode",
                "value": "gradient-gauge"
              },
              {
                "id": "unit",
                "value": "ms"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "trace_id"
            },
            "properties": [
              {
                "id": "links",
                "value": [
                  {
                    "targetBlank": true,
                    "title": "Jaeger",
                    "url": "${jaeger_base_url}trace/${__data.fields[trace_id]}"
                  }
                ]
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Time"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 164
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 42
      },
      "id": 32,
      "links": [],
      "options": {
        "showHeader": true,
        "sortBy": [
          {
            "desc": true,
            "displayName": "top"
          }
        ]
      },
      "pluginVersion": "7.5.4",
      "targets": [
        {
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT top(\"duration\", 100), trace_id, tid FROM \"span\" WHERE \"name\" = 'get-trans-id' AND $timeFilter",
          "rawQuery": true,
          "refId": "A",
          "resultFormat": "table",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": []
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Longest get-trans-id",
      "type": "table"
    },
    {
      "datasource": "${DS_INFLUXDB}",
      "description": "This is a top list of the longest get-trans-id operations for the given time period. Use it to identify things that might be slow.\n\nA long transaction in itself might not be a bad thing since transactions can essentially be idle and not block other transactions. However, if any lock is taken, the transaction should try to finish as soon as possible.",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null,
            "displayMode": "auto",
            "filterable": false
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "#6ED0E0",
                "value": 250
              },
              {
                "color": "#EAB839",
                "value": 500
              },
              {
                "color": "red",
                "value": 1000
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "top"
            },
            "properties": [
              {
                "id": "custom.displayMode",
                "value": "gradient-gauge"
              },
              {
                "id": "unit",
                "value": "ms"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "trace_id"
            },
            "properties": [
              {
                "id": "links",
                "value": [
                  {
                    "targetBlank": true,
                    "title": "Jaeger",
                    "url": "${jaeger_base_url}trace/${__data.fields[trace_id]}"
                  }
                ]
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Time"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 164
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 50
      },
      "id": 33,
      "links": [],
      "options": {
        "showHeader": true,
        "sortBy": [
          {
            "desc": true,
            "displayName": "top"
          }
        ]
      },
      "pluginVersion": "7.5.4",
      "targets": [
        {
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT top(\"duration\", 100), trace_id, tid FROM \"span\" WHERE \"name\" = 'connect' AND $timeFilter",
          "rawQuery": true,
          "refId": "A",
          "resultFormat": "table",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": []
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Longest device connection",
      "type": "table"
    },
    {
      "datasource": "${DS_INFLUXDB}",
      "description": "This is a top list of the longest sync-from operations for the given time period.",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null,
            "displayMode": "auto",
            "filterable": false
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "#6ED0E0",
                "value": 250
              },
              {
                "color": "#EAB839",
                "value": 500
              },
              {
                "color": "red",
                "value": 1000
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "top"
            },
            "properties": [
              {
                "id": "custom.displayMode",
                "value": "gradient-gauge"
              },
              {
                "id": "unit",
                "value": "ms"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "trace_id"
            },
            "properties": [
              {
                "id": "links",
                "value": [
                  {
                    "targetBlank": true,
                    "title": "Jaeger",
                    "url": "${jaeger_base_url}trace/${__data.fields[trace_id]}"
                  }
                ]
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Time"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 164
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 50
      },
      "id": 34,
      "links": [],
      "options": {
        "showHeader": true,
        "sortBy": [
          {
            "desc": true,
            "displayName": "top"
          }
        ]
      },
      "pluginVersion": "7.5.4",
      "targets": [
        {
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT top(\"duration\", 100), trace_id, tid FROM \"span\" WHERE \"name\" = 'sync-from' AND $timeFilter",
          "rawQuery": true,
          "refId": "A",
          "resultFormat": "table",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": []
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Longest sync-from",
      "type": "table"
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_INFLUXDB}",
      "description": "Shows some device related operations that happen concurrently.",
      "fieldConfig": {
        "defaults": {
          "links": []
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 58
      },
      "hiddenSeries": false,
      "id": 46,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "connected",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.5.4",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "alias": "$tag_name",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "device-lock",
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT mean(\"count\") FROM \"span-count\" WHERE (\"name\" = 'connect' OR \"name\" = 'pre validate' OR \"name\" = 'show' OR \"name\" = 'sync-from' OR \"name\" = 'taking device lock' OR \"name\" = 'ned-migrate')  AND $timeFilter GROUP BY \"name\", time($__interval) fill(null)",
          "rawQuery": true,
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "held"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": []
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Concurrent device operations",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:71",
          "format": "short",
          "label": "device lock held",
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        },
        {
          "$$hashKey": "object:72",
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": "                tags = {\"device\": pev['device']}                 tags.update(extra_tags)                 influx_client.write_points([                     {                         \"time\": datetime.datetime.utcfromtimestamp(pev['timestamp'] / 1000000.0),                         \"measurement\": \"device-lock\",                         \"tags\": tags,                         \"fields\": {                             \"held\": 1,                             \"tid\": pev['tid']                         }                     }                 ])",
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    }
  ],
  "refresh": false,
  "schemaVersion": 27,
  "style": "dark",
  "tags": [],
  "templating": {
    "list": [
      {
        "current": {
          "selected": false,
          "text": "${JAEGER_BASE_URL}",
          "value": "${JAEGER_BASE_URL}"
        },
        "description": null,
        "error": null,
        "hide": 2,
        "label": null,
        "name": "jaeger_base_url",
        "query": "${JAEGER_BASE_URL}",
        "skipUrlSync": false,
        "type": "constant"
      }
    ]
  },
  "time": {
    "from": "now-5m",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ]
  },
  "timezone": "",
  "title": "NSO",
  "uid": "nso",
  "version": 10
}