{ "__inputs": [ { "name": "DS_PROMETHEUS", "label": "prometheus", "description": "Prometheus datasource", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "7.5.10" }, { "type": "panel", "id": "graph", "name": "Graph", "version": "" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, { "type": "panel", "id": "singlestat", "name": "Singlestat", "version": "" }, { "type": "panel", "id": "stat", "name": "Stat", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "description": "OpenSearch cluster monitoring with OpenSearch Prometheus Exporter and Node Exporter", "editable": true, "gnetId": 15178, "graphTooltip": 0, "id": null, "iteration": 1635775268463, "links": [], "panels": [ { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 8, "panels": [], "title": "Cluster", "type": "row" }, { "cacheTimeout": null, "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "from": "", "id": 1, "text": "green", "to": "", "type": 1, "value": "0" }, { "from": "", "id": 2, "text": "yellow", "to": "", "type": 1, "value": "1" }, { "from": "", "id": 3, "text": "red", "to": "", "type": 1, "value": "2" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "semi-dark-green", "value": 0 }, { "color": "semi-dark-yellow", "value": 1 }, { "color": "semi-dark-red", "value": 2 } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 12, "x": 0, "y": 1 }, "id": 2, "links": [], "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "7.5.10", "targets": [ { "exemplar": true, "expr": "sum(opensearch_cluster_status{cluster=~\"$cluster\"}) / count(opensearch_cluster_nodes_number{cluster=~\"$cluster\"})", "interval": "", "intervalFactor": 2, "legendFormat": "", "queryType": "randomWalk", "refId": "A" } ], "timeFrom": null, "timeShift": null, "title": "Cluster Status", "type": "stat" }, { "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "semi-dark-blue", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 12, "y": 1 }, "id": 4, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": { "valueSize": 56 }, "textMode": "auto" }, "pluginVersion": "7.5.10", "targets": [ { "exemplar": true, "expr": "sum(opensearch_cluster_nodes_number{cluster=~\"$cluster\"}) / count(opensearch_cluster_nodes_number{cluster=~\"$cluster\"})", "interval": "", "legendFormat": "", "queryType": "randomWalk", "refId": "A" } ], "timeFrom": null, "timeShift": null, "title": "Running Nodes", "type": "stat" }, { "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "semi-dark-blue", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 16, "y": 1 }, "id": 6, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": { "valueSize": 56 }, "textMode": "auto" }, "pluginVersion": "7.5.10", "targets": [ { "exemplar": true, "expr": "sum(opensearch_cluster_datanodes_number{cluster=~\"$cluster\"}) / count(opensearch_cluster_datanodes_number{cluster=~\"$cluster\"})", "interval": "", "legendFormat": "", "queryType": "randomWalk", "refId": "A" } ], "title": "Active Data Nodes", "type": "stat" }, { "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 20, "y": 1 }, "id": 10, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": { "valueSize": 56 }, "textMode": "auto" }, "pluginVersion": "7.5.10", "targets": [ { "exemplar": true, "expr": "sum(opensearch_cluster_pending_tasks_number{cluster=~\"$cluster\"}) / count(opensearch_cluster_pending_tasks_number{cluster=~\"$cluster\"})", "interval": "", "legendFormat": "", "queryType": "randomWalk", "refId": "A" } ], "title": "Pending Tasks", "type": "stat" }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 4 }, "id": 12, "panels": [], "title": "Shards", "type": "row" }, { "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "rgb(255, 255, 255)", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 0, "y": 5 }, "id": 14, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": { "valueSize": 56 }, "textMode": "auto" }, "pluginVersion": "7.5.10", "targets": [ { "exemplar": true, "expr": "sum(opensearch_cluster_shards_number{cluster=~\"$cluster\",type=\"active\"}) / count(opensearch_cluster_shards_number{cluster=~\"$cluster\",type=\"active\"})", "interval": "", "legendFormat": "", "queryType": "randomWalk", "refId": "A" } ], "title": "Active Shards", "type": "stat" }, { "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "rgb(255, 255, 255)", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 5, "x": 4, "y": 5 }, "id": 16, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": { "valueSize": 56 }, "textMode": "auto" }, "pluginVersion": "7.5.10", "targets": [ { "exemplar": true, "expr": "sum(opensearch_cluster_shards_number{cluster=~\"$cluster\",type=\"active_primary\"}) / count(opensearch_cluster_shards_number{cluster=~\"$cluster\",type=\"active_primary\"})", "interval": "", "legendFormat": "", "queryType": "randomWalk", "refId": "A" } ], "title": "Active Primary Shards", "type": "stat" }, { "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "rgb(255, 255, 255)", "value": null }, { "color": "semi-dark-orange", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 5, "x": 9, "y": 5 }, "id": 18, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": { "valueSize": 56 }, "textMode": "auto" }, "pluginVersion": "7.5.10", "targets": [ { "exemplar": true, "expr": "sum(opensearch_cluster_shards_number{cluster=~\"$cluster\",type=\"initializing\"}) / count(opensearch_cluster_shards_number{cluster=~\"$cluster\",type=\"initializing\"})", "interval": "", "legendFormat": "", "queryType": "randomWalk", "refId": "A" } ], "title": "Initializing Shards", "type": "stat" }, { "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "rgb(255, 255, 255)", "value": null }, { "color": "semi-dark-orange", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 5, "x": 14, "y": 5 }, "id": 20, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": { "valueSize": 56 }, "textMode": "auto" }, "pluginVersion": "7.5.10", "targets": [ { "exemplar": true, "expr": "sum(opensearch_cluster_shards_number{cluster=~\"$cluster\",type=\"relocating\"}) / count(opensearch_cluster_shards_number{cluster=~\"$cluster\",type=\"relocating\"})", "interval": "", "legendFormat": "", "queryType": "randomWalk", "refId": "A" } ], "title": "Relocating Shards", "type": "stat" }, { "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "rgb(255, 255, 255)", "value": null }, { "color": "semi-dark-orange", "value": 0.5 }, { "color": "semi-dark-red", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 5, "x": 19, "y": 5 }, "id": 22, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": { "valueSize": 56 }, "textMode": "auto" }, "pluginVersion": "7.5.10", "targets": [ { "exemplar": true, "expr": "sum(opensearch_cluster_shards_number{cluster=~\"$cluster\",type=\"unassigned\"}) / count(opensearch_cluster_shards_number{cluster=~\"$cluster\",type=\"unassigned\"})", "interval": "", "legendFormat": "", "queryType": "randomWalk", "refId": "A" } ], "title": "Unassigned Shards", "type": "stat" }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 8 }, "id": 26, "panels": [], "title": "Node", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "decimals": 2, "description": "Basic CPU info", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 8, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, "hiddenSeries": false, "id": 24, "interval": "", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": true, "pluginVersion": "7.5.10", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/^Busy Iowait/", "color": "#C4162A" }, { "alias": "/^Idle/", "color": "#56A64B" }, { "alias": "/^Busy System/", "color": "#E0B400" }, { "alias": "/^Busy User/", "color": "#1F60C4" }, { "alias": "/^Busy Other/", "color": "#8F3BB8" } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (instance)(irate(node_cpu_seconds_total{mode=\"system\",instance=~\".*$node.*\",job=\"$job\"}[5m])) * 100", "interval": "", "intervalFactor": 4, "legendFormat": "Busy System {{instance}}", "refId": "A" }, { "expr": "sum by (instance)(irate(node_cpu_seconds_total{mode='user',instance=~\".*$node.*\",job=\"$job\"}[5m])) * 100", "interval": "", "intervalFactor": 4, "legendFormat": "Busy User {{instance}}", "refId": "B" }, { "expr": "sum by (instance)(irate(node_cpu_seconds_total{mode='iowait',instance=~\".*$node.*\",job=\"$job\"}[5m])) * 100", "interval": "", "intervalFactor": 4, "legendFormat": "Busy Iowait {{instance}}", "refId": "C" }, { "expr": "sum by (instance)(irate(node_cpu_seconds_total{mode=~\".*irq\",instance=~\".*$node.*\",job=\"$job\"}[5m])) * 100", "interval": "", "intervalFactor": 4, "legendFormat": "Busy IRQs {{instance}}", "refId": "D" }, { "expr": "sum (irate(node_cpu_seconds_total{mode!='idle',mode!='user',mode!='system',mode!='iowait',mode!='irq',mode!='softirq',instance=~\".*$node.*\",job=\"$job\"}[5m])) * 100", "interval": "", "intervalFactor": 4, "legendFormat": "Busy Other $node", "refId": "E" }, { "expr": "sum by (mode)(irate(node_cpu_seconds_total{mode='idle',instance=~\".*$node.*\",job=\"$job\"}[5m])) * 100", "interval": "", "intervalFactor": 4, "legendFormat": "Idle $node", "refId": "F" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "CPU Basic", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1139", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:1140", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Basic network info per interface", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 }, "hiddenSeries": false, "id": 28, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.10", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "irate(node_network_receive_bytes_total{instance=~\".*$node.*\",job=\"$job\"}[5m])*8", "interval": "", "intervalFactor": 4, "legendFormat": "recv {{device}} {{instance}}", "refId": "A" }, { "expr": "irate(node_network_transmit_bytes_total{instance=~\".*$node.*\",job=\"$job\"}[5m])*8", "interval": "", "intervalFactor": 4, "legendFormat": "trans {{device}} {{instance}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Network Traffic Basic", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 17 }, "id": 30, "panels": [], "title": "Documents", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 18 }, "hiddenSeries": false, "id": 32, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.10", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(opensearch_indices_merges_total_docs_count{cluster=~\"$cluster\"})", "interval": "", "intervalFactor": 2, "legendFormat": "Documents", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Documents Indexed", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1068", "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1069", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "editable": true, "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 18 }, "hiddenSeries": false, "id": 56, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.10", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(opensearch_indices_store_size_bytes{cluster=~\"$cluster\"})", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "Index Size", "refId": "A", "step": 240 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Index Size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:672", "format": "bytes", "logBase": 1, "max": null, "min": 0, "show": true }, { "$$hashKey": "object:673", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 27 }, "hiddenSeries": false, "id": 58, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.10", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "rate(opensearch_indices_indexing_index_count{cluster=~\"$cluster\"}[1h])", "interval": "", "legendFormat": "{{ node }}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Documents Indexed Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:757", "format": "short", "label": "Documents/s", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:758", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 27 }, "hiddenSeries": false, "id": 60, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.10", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "rate(opensearch_indices_search_fetch_count{cluster=~\"$cluster\"}[1h])", "interval": "", "legendFormat": "{{ node }}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Query Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:842", "format": "short", "label": "Queris/s", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:843", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "editable": true, "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 36 }, "height": "", "hiddenSeries": false, "id": 62, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.10", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(opensearch_threadpool_tasks_number{cluster=~\"$cluster\", name!=\"management\", type=\"queue\"}) by (name)", "interval": "", "intervalFactor": 2, "legendFormat": "Name: {{ name }}", "refId": "A", "step": 240 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Queue Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2135", "format": "short", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:2136", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 44 }, "id": 34, "panels": [], "title": "System", "type": "row" }, { "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "datasource": "$datasource", "editable": true, "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, "format": "bytes", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "gridPos": { "h": 3, "w": 6, "x": 0, "y": 45 }, "height": "", "id": 36, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", "show": true }, "tableColumn": "", "targets": [ { "exemplar": true, "expr": "sum(node_memory_MemTotal_bytes{instance=~\"$node.*\"})", "interval": "1800", "intervalFactor": 2, "legendFormat": "", "refId": "A" } ], "thresholds": "", "title": "Total Memory", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "current" }, { "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "datasource": "$datasource", "editable": true, "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, "format": "bytes", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "gridPos": { "h": 3, "w": 6, "x": 6, "y": 45 }, "height": "", "id": 38, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", "show": true }, "tableColumn": "", "targets": [ { "expr": "sum(node_memory_MemFree_bytes{instance=~\"$node.*\"})", "interval": "1800", "intervalFactor": 2, "legendFormat": "", "refId": "A" } ], "thresholds": "", "title": "Total Memory Free", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "current" }, { "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "datasource": "$datasource", "editable": true, "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, "format": "bytes", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "gridPos": { "h": 3, "w": 6, "x": 12, "y": 45 }, "height": "", "id": 40, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", "show": true }, "tableColumn": "", "targets": [ { "expr": "sum(node_memory_MemAvailable_bytes{instance=~\"$node.*\"})", "interval": "1800", "intervalFactor": 2, "legendFormat": "", "refId": "A" } ], "thresholds": "", "title": "Total Memory Available", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "current" }, { "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "datasource": "$datasource", "editable": true, "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, "format": "bytes", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "gridPos": { "h": 3, "w": 6, "x": 18, "y": 45 }, "height": "", "id": 42, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", "show": true }, "tableColumn": "", "targets": [ { "expr": "sum(node_filesystem_free_bytes{instance=~\"$node.*\", mountpoint=\"/\"})", "interval": "1800", "intervalFactor": 2, "legendFormat": "", "refId": "A" } ], "thresholds": "", "title": "Total Disk Available", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "current" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "editable": true, "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 48 }, "hiddenSeries": false, "id": 44, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.10", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(opensearch_threadpool_threads_number{cluster=~\"$cluster\", name!=\"management\", type=\"active\"}) by (name)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "Name: {{ name }}", "refId": "A", "step": 240 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Thread Pools", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:168", "format": "short", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:169", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "editable": true, "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 48 }, "hiddenSeries": false, "id": 50, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.10", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "rate(opensearch_threadpool_threads_count{cluster=~\"$cluster\", name!=\"management\", type=\"rejected\"}[5m])", "interval": "", "intervalFactor": 2, "legendFormat": "{{ node }} {{ name }}", "refId": "A", "step": 240 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Thread pool rejections", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:419", "format": "short", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:420", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 56 }, "hiddenSeries": false, "id": 46, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.10", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "bucketAggs": [ { "id": "2", "settings": { "interval": "auto" }, "type": "date_histogram" } ], "dsType": "elasticsearch", "exemplar": true, "expr": "avg(irate(node_cpu_seconds_total{instance=~\"$node.*\"}[5m])) by(mode) * 100", "format": "time_series", "interval": "", "intervalFactor": 4, "legendFormat": "{{ mode }}", "metric": "elasticsearch_breakers_tripped", "metrics": [ { "id": "1", "type": "count" } ], "refId": "A", "step": 240, "timeField": "failure_tstamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Avg. CPU Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2848", "format": "short", "label": "", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:2849", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "editable": true, "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 56 }, "hiddenSeries": false, "id": 48, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.10", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "avg_over_time(opensearch_jvm_mem_heap_used_bytes{cluster=~\"$cluster\"}[15m]) / opensearch_jvm_mem_heap_max_bytes{cluster=~\"$cluster\"}", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{ node }}", "refId": "A", "step": 240 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Avg Heap in 15min", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1445", "format": "percentunit", "label": "", "logBase": 1, "max": 1, "min": 0, "show": true }, { "$$hashKey": "object:1446", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "editable": true, "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 65 }, "hiddenSeries": false, "id": 52, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.10", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(opensearch_transport_rx_packets_count{cluster=~\"$cluster\"}[5m]))", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "RX", "refId": "A", "step": 240 }, { "exemplar": true, "expr": "sum(rate(opensearch_transport_tx_packets_count{cluster=~\"$cluster\"}[5m])) * -1", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "TX", "refId": "B", "step": 240 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "RX/TX Rate 5m", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:504", "format": "short", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:505", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "editable": true, "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 65 }, "hiddenSeries": false, "id": 54, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.10", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "irate(opensearch_jvm_gc_collection_time_seconds{cluster=~\"$cluster\"}[5m])", "interval": "", "intervalFactor": 2, "legendFormat": "{{ node }} {{ gc }}", "refId": "A", "step": 240 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "GC seconds", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2952", "format": "short", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:2953", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "schemaVersion": 27, "style": "dark", "tags": [], "templating": { "list": [ { "current": { "selected": false, "text": "prometheus", "value": "prometheus" }, "description": null, "error": null, "hide": 0, "includeAll": false, "label": null, "multi": false, "name": "datasource", "options": [], "query": "prometheus", "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", "definition": "label_values(opensearch_cluster_status{}, cluster)", "description": null, "error": null, "hide": 0, "includeAll": false, "label": null, "multi": false, "name": "cluster", "options": [], "query": { "query": "label_values(opensearch_cluster_status{}, cluster)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", "definition": "label_values(node_uname_info{job=~\".*kubernetes.*|.*node_exporters.*\"}, job)", "description": null, "error": null, "hide": 0, "includeAll": false, "label": "Job", "multi": false, "name": "job", "options": [], "query": { "query": "label_values(node_uname_info{job=~\".*kubernetes.*|.*node_exporters.*\"}, job)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 5, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", "definition": "label_values(node_uname_info{job=\"$job\"}, nodename)", "description": null, "error": null, "hide": 0, "includeAll": false, "label": "Node name", "multi": true, "name": "nodename", "options": [], "query": { "query": "label_values(node_uname_info{job=\"$job\"}, nodename)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 5, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", "definition": "label_values(node_uname_info{job=\"$job\", nodename=\"$nodename\"}, instance)", "description": null, "error": null, "hide": 0, "includeAll": false, "label": "Instance", "multi": false, "name": "node", "options": [], "query": { "query": "label_values(node_uname_info{job=\"$job\", nodename=\"$nodename\"}, instance)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 5, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "OpenSearch", "uid": "X1WSwkF7k", "version": 25 }