From 3eaa52ee188241943a59571090a2759a2d1a639c Mon Sep 17 00:00:00 2001 From: Chris Raible Date: Wed, 11 Sep 2024 14:40:37 -0700 Subject: [PATCH 1/6] Added promethetus /metrics endpoint using prom-client --- .github/scripts/docker-compose.yml | 20 + .github/scripts/grafana/dashboard.yml | 15 + .../grafana/dashboards/main-dashboard.json | 1634 +++++++++++++++++ .../grafana/datasources/datasource.yml | 9 + .github/scripts/prometheus/prometheus.yml | 31 + .../core/server/services/jobs/job-service.js | 13 + ghost/core/core/server/web/api/app.js | 4 + .../web/api/middleware/prometheus-metrics.js | 19 + .../web/parent/middleware/log-request.js | 12 + .../web/parent/middleware/queue-request.js | 27 + ghost/core/package.json | 1 + yarn.lock | 22 +- 12 files changed, 1806 insertions(+), 1 deletion(-) create mode 100644 .github/scripts/grafana/dashboard.yml create mode 100644 .github/scripts/grafana/dashboards/main-dashboard.json create mode 100644 .github/scripts/grafana/datasources/datasource.yml create mode 100644 .github/scripts/prometheus/prometheus.yml create mode 100644 ghost/core/core/server/web/api/middleware/prometheus-metrics.js diff --git a/.github/scripts/docker-compose.yml b/.github/scripts/docker-compose.yml index c9ba7eda1c6..de5a42f4ab0 100644 --- a/.github/scripts/docker-compose.yml +++ b/.github/scripts/docker-compose.yml @@ -28,5 +28,25 @@ services: ports: - "6379:6379" restart: always + prometheus: + image: prom/prometheus:v2.30.3 + container_name: ghost-prometheus + ports: + - "9090:9090" + restart: always + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + grafana: + image: grafana/grafana:8.2.3 + container_name: ghost-grafana + ports: + - "3000:3000" + restart: always + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + volumes: + - ./grafana/datasources:/etc/grafana/provisioning/datasources + - ./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/main.yaml + - ./grafana/dashboards:/var/lib/grafana/dashboards volumes: mysql-data: diff --git a/.github/scripts/grafana/dashboard.yml b/.github/scripts/grafana/dashboard.yml new file mode 100644 index 00000000000..d50b2658697 --- /dev/null +++ b/.github/scripts/grafana/dashboard.yml @@ -0,0 +1,15 @@ +## This file is used to point to the folder where the dashboards are stored +## To edit or create a dashboard, add a .json file to the ./dashboards folder + +apiVersion: 1 + +providers: + - name: "Dashboard provider" + orgId: 1 + type: file + disableDeletion: false + updateIntervalSeconds: 10 + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: true \ No newline at end of file diff --git a/.github/scripts/grafana/dashboards/main-dashboard.json b/.github/scripts/grafana/dashboards/main-dashboard.json new file mode 100644 index 00000000000..a753bdc2071 --- /dev/null +++ b/.github/scripts/grafana/dashboards/main-dashboard.json @@ -0,0 +1,1634 @@ +{ + "__inputs": [], + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "7.4.3" + }, + { + "id": "graph", + "name": "Graph", + "type": "panel", + "version": "" + }, + { + "id": "prometheus", + "name": "Prometheus", + "type": "datasource", + "version": "1.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": "$datasource", + "enable": true, + "expr": "ghost_process_start_time_seconds{job=~\"$job\", instance=~\"$instance\"} * 1000", + "hide": false, + "iconColor": "#B877D9", + "name": "Ghost Start", + "showIn": 0, + "textFormat": "{{instance}}", + "titleFormat": "Ghost Start", + "useValueForTime": true + } + ] + }, + "description": "An overview of the Ghost process metrics.", + "editable": true, + "gnetId": 14058, + "graphTooltip": 0, + "id": 1, + "iteration": 1608497517213, + "links": [], + "panels": [ + { + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "title": "Overview", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "description": "The version of Node.js.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.6.2", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false, + "ymax": null, + "ymin": null + }, + "tableColumn": "version", + "targets": [ + { + "expr": "ghost_nodejs_version_info", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Node.js Version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "first" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "description": "The number of times Node.js restarted.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 3, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.6.2", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false, + "ymax": null, + "ymin": null + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(changes(ghost_process_start_time_seconds{job=~\"$job\", instance=~\"$instance\", hostname=~\"$hostname\"}[$__range]))", + "instant": false, + "legendFormat": "Node.js", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Node.js Restarts", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [], + "valueName": "current" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 4, + "panels": [], + "title": "Process", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "CPU usage.", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 4 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0-beta2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(ghost_process_cpu_user_seconds_total{job=~\"$job\", instance=~\"$instance\"}[1m]) * 100", + "interval": "", + "legendFormat": "User CPU - {{instance}}", + "refId": "A" + }, + { + "expr": "irate(ghost_process_cpu_system_seconds_total{job=~\"$job\", instance=~\"$instance\"}[1m]) * 100", + "interval": "", + "legendFormat": "System CPU - {{instance}}", + "refId": "B" + }, + { + "expr": "irate(ghost_process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[1m]) * 100", + "interval": "", + "legendFormat": "Total CPU - {{instance}}", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "CPU time spent.", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 4 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0-beta2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(ghost_process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$interval])", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Time Spent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Memory usage.", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 4 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0-beta2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "Process Memory - {{instance}}", + "refId": "A" + }, + { + "expr": "ghost_nodejs_heap_size_total_bytes{job=~\"$job\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "Heap Total - {{instance}}", + "refId": "B" + }, + { + "expr": "ghost_nodejs_heap_size_used_bytes{job=~\"$job\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "Heap Used - {{instance}}", + "refId": "C" + }, + { + "expr": "ghost_nodejs_external_memory_bytes{job=~\"$job\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "External Memory - {{instance}}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Number of active handle and active requests.", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 13 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0-beta2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "ghost_nodejs_active_handles_total{job=~\"$job\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "Active Handler - {{instance}}", + "refId": "A" + }, + { + "expr": "ghost_nodejs_active_requests_total{job=~\"$job\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "Active Request - {{instance}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Handlers and Requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Latency of the event loop.", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 13 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0-beta2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "ghost_nodejs_eventloop_lag_seconds{job=~\"$job\", instance=~\"$instance\"}", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Last - {{instance}}", + "refId": "A" + }, + { + "expr": "ghost_nodejs_eventloop_lag_p99_seconds{job=~\"$job\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "P99 - {{instance}}", + "refId": "B" + }, + { + "expr": "ghost_nodejs_eventloop_lag_p50_seconds{job=~\"$job\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "P50 - {{instance}}", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Event Loop Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 10, + "panels": [], + "title": "Garbage Collector", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Rate of garbage collection duration.", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0-beta2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(ghost_nodejs_gc_duration_seconds_sum{job=~\"$job\", instance=~\"$instance\"}[$interval])", + "interval": "", + "legendFormat": "{{kind}} - {{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GC Duration Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "s", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Duration of garbage collection.", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 23 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0-beta2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "ghost_nodejs_gc_duration_seconds_sum{job=~\"$job\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "{{kind}} - {{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GC Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "s", + "label": "", + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Usage of heap memory.", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 23 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0-beta2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "ghost_nodejs_heap_size_total_bytes{job=~\"$job\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "Total - {{instance}}", + "refId": "A" + }, + { + "expr": "ghost_nodejs_heap_size_used_bytes{job=~\"$job\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "Used - {{instance}}", + "refId": "B" + }, + { + "expr": "process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "Resident - {{instance}}", + "refId": "C" + }, + { + "expr": "ghost_nodejs_external_memory_bytes{job=~\"$job\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "External - {{instance}}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Heap Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Rate of garbage collection.", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 31 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0-beta2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(ghost_nodejs_gc_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}[$interval])", + "interval": "", + "legendFormat": "{{instance}} - {{kind}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Count of garbage collection.", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 31 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0-beta2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "ghost_nodejs_gc_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "{{kind}} - {{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GC Count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Usage of heap space.", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 31 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0-beta2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ghost_nodejs_heap_space_size_used_bytes{instance=~\"$instance\"}", + "interval": "", + "legendFormat": "{{space}} - {{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Heap Space Used", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [ + "node.js", + "nodejs" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "prometheus", + "value": "prometheus" + }, + "error": null, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "$datasource", + "definition": "label_values(ghost_nodejs_eventloop_lag_seconds, job)", + "error": null, + "hide": 0, + "includeAll": true, + "label": "Job", + "multi": true, + "name": "job", + "options": [], + "query": "label_values(ghost_nodejs_eventloop_lag_seconds, job)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "$datasource", + "definition": "label_values(ghost_nodejs_eventloop_lag_seconds, instance)", + "error": null, + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": "label_values(ghost_nodejs_eventloop_lag_seconds, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "1m", + "value": "1m" + }, + "error": null, + "hide": 0, + "label": "Interval", + "name": "interval", + "options": [ + { + "selected": true, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Ghost Dashboard", + "uid": "yX2d7k1Gk", + "version": 1 + } \ No newline at end of file diff --git a/.github/scripts/grafana/datasources/datasource.yml b/.github/scripts/grafana/datasources/datasource.yml new file mode 100644 index 00000000000..c6d21399a60 --- /dev/null +++ b/.github/scripts/grafana/datasources/datasource.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: +- name: Prometheus + type: prometheus + url: http://prometheus:9090 + isDefault: true + access: proxy + editable: true \ No newline at end of file diff --git a/.github/scripts/prometheus/prometheus.yml b/.github/scripts/prometheus/prometheus.yml new file mode 100644 index 00000000000..9361fc785b6 --- /dev/null +++ b/.github/scripts/prometheus/prometheus.yml @@ -0,0 +1,31 @@ +global: + scrape_interval: 15s # By default, scrape targets every 15 seconds. + + # Attach these labels to any time series or alerts when communicating with + # external systems (federation, remote storage, Alertmanager). + external_labels: + monitor: 'codelab-monitor' + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped from this config. + - job_name: 'prometheus' + + # Override the global default and scrape targets from this job every 5 seconds. + scrape_interval: 5s + + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'ghost' + + scrape_interval: 1s + + static_configs: + - targets: ['host.docker.internal:2368'] + + metrics_path: '/ghost/api/metrics' + +remote_write: + - url: http://grafana:3000/api/prom/push \ No newline at end of file diff --git a/ghost/core/core/server/services/jobs/job-service.js b/ghost/core/core/server/services/jobs/job-service.js index 3b8298e73c9..bdd47b70e34 100644 --- a/ghost/core/core/server/services/jobs/job-service.js +++ b/ghost/core/core/server/services/jobs/job-service.js @@ -8,6 +8,7 @@ const logging = require('@tryghost/logging'); const models = require('../../models'); const sentry = require('../../../shared/sentry'); const domainEvents = require('@tryghost/domain-events'); +const promClient = require('prom-client'); const errorHandler = (error, workerMeta) => { logging.info(`Capturing error for worker during execution of job: ${workerMeta.name}`); @@ -44,5 +45,17 @@ const initTestMode = () => { const jobManager = new JobManager({errorHandler, workerMessageHandler, JobModel: models.Job, domainEvents}); +const gauge = new promClient.Gauge({ + name: 'ghost_jobs_manager_active_workers', + help: 'Total number of active workers', + collect() { + if (Object.keys(jobManager.bree.workers).length) { + this.set(Object.keys(jobManager.bree.workers).length); + } else { + this.set(0); + } + } +}); + module.exports = jobManager; module.exports.initTestMode = initTestMode; diff --git a/ghost/core/core/server/web/api/app.js b/ghost/core/core/server/web/api/app.js index a99445d54de..91a197d83f4 100644 --- a/ghost/core/core/server/web/api/app.js +++ b/ghost/core/core/server/web/api/app.js @@ -22,6 +22,10 @@ module.exports = function setupApiApp() { apiApp.lazyUse('/content/', require('./endpoints/content/app')); apiApp.lazyUse('/admin/', require('./endpoints/admin/app')); + if (config.get('prometheus:enabled')) { + apiApp.use('/metrics', require('./middleware/prometheus-metrics')); + } + // Error handling for requests to non-existent API versions apiApp.use(errorHandler.resourceNotFound); apiApp.use(APIVersionCompatibilityService.errorHandler); diff --git a/ghost/core/core/server/web/api/middleware/prometheus-metrics.js b/ghost/core/core/server/web/api/middleware/prometheus-metrics.js new file mode 100644 index 00000000000..dc1f9e428a5 --- /dev/null +++ b/ghost/core/core/server/web/api/middleware/prometheus-metrics.js @@ -0,0 +1,19 @@ +const {collectDefaultMetrics, register} = require('prom-client'); + +const prefix = 'ghost_'; +collectDefaultMetrics({prefix}); + +/** + * @param {import('express').Request} req + * @param {import('express').Response} res + */ +async function promClientMw(req, res) { + try { + res.set('Content-Type', register.contentType); + res.end(await register.metrics()); + } catch (err) { + res.status(500).send(err.message); + } +} + +module.exports = promClientMw; \ No newline at end of file diff --git a/ghost/core/core/server/web/parent/middleware/log-request.js b/ghost/core/core/server/web/parent/middleware/log-request.js index bc1391eaf61..7f1d56dace3 100644 --- a/ghost/core/core/server/web/parent/middleware/log-request.js +++ b/ghost/core/core/server/web/parent/middleware/log-request.js @@ -1,4 +1,11 @@ const logging = require('@tryghost/logging'); +const promClient = require('prom-client'); + +const counter = new promClient.Counter({ + name: 'ghost_http_requests_total', + help: 'Total number of HTTP requests', + labelNames: ['method', 'statusCode'] +}); /** * @TODO: move this middleware to Framework monorepo? @@ -20,6 +27,11 @@ module.exports = function logRequest(req, res, next) { logging.info({req: req, res: res}); } + counter.inc({ + method: req.method, + statusCode: res.statusCode + }); + res.removeListener('finish', logResponse); res.removeListener('close', logResponse); } diff --git a/ghost/core/core/server/web/parent/middleware/queue-request.js b/ghost/core/core/server/web/parent/middleware/queue-request.js index debdf8fb2ef..426083e4035 100644 --- a/ghost/core/core/server/web/parent/middleware/queue-request.js +++ b/ghost/core/core/server/web/parent/middleware/queue-request.js @@ -2,6 +2,7 @@ const errors = require('@tryghost/errors'); const logging = require('@tryghost/logging'); const path = require('node:path'); const expressQueue = require('express-queue'); +const promClient = require('prom-client'); const debug = (message) => { logging.debug(`[queue-request] ${message}`); @@ -25,6 +26,19 @@ module.exports = function queueRequest( queuedLimit: -1 // Do not limit the number of queued requests }); + const queueGauge = new promClient.Gauge({ + name: 'ghost_request_queue_depth', + help: 'Number of HTTP requests queued', + collect() { + this.set(queue.queue.getLength()); + } + }); + + const activeRequestGauge = new promClient.Gauge({ + name: 'ghost_request_queue_in_process', + help: 'Number of HTTP requests in process' + }); + /** * Available events: * - queue - when a request is queued @@ -40,8 +54,21 @@ module.exports = function queueRequest( debug(`Request queued: ${job.data.req.path}`); }); + queue.queue.on('process', (job) => { + activeRequestGauge.inc(); + }); + queue.queue.on('complete', (job) => { debug(`Request completed: ${job.data.req.path}`); + activeRequestGauge.dec(); + }); + + queue.queue.on('reject', (job) => { + activeRequestGauge.dec(); + }); + + queue.queue.on('cancel', (job) => { + activeRequestGauge.dec(); }); /** diff --git a/ghost/core/package.json b/ghost/core/package.json index 533883344fc..9e8d3c23851 100644 --- a/ghost/core/package.json +++ b/ghost/core/package.json @@ -216,6 +216,7 @@ "node-jose": "2.2.0", "path-match": "1.2.4", "probe-image-size": "7.2.3", + "prom-client": "15.1.3", "rss": "1.2.2", "sanitize-html": "2.13.0", "semver": "7.6.3", diff --git a/yarn.lock b/yarn.lock index f295ef38ccf..a723e2374e5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4095,7 +4095,7 @@ dependencies: "@opentelemetry/api" "^1.0.0" -"@opentelemetry/api@1.9.0", "@opentelemetry/api@^1.0.0": +"@opentelemetry/api@1.9.0", "@opentelemetry/api@^1.0.0", "@opentelemetry/api@^1.4.0": version "1.9.0" resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.0.tgz#d03eba68273dc0f7509e2a3d5cba21eae10379fe" integrity sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg== @@ -10993,6 +10993,11 @@ bindings@^1.5.0: dependencies: file-uri-to-path "1.0.0" +bintrees@1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/bintrees/-/bintrees-1.0.2.tgz#49f896d6e858a4a499df85c38fb399b9aff840f8" + integrity sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw== + bl@^4.0.3, bl@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a" @@ -26371,6 +26376,14 @@ progress@^2.0.0, progress@^2.0.1: resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== +prom-client@15.1.3: + version "15.1.3" + resolved "https://registry.yarnpkg.com/prom-client/-/prom-client-15.1.3.tgz#69fa8de93a88bc9783173db5f758dc1c69fa8fc2" + integrity sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g== + dependencies: + "@opentelemetry/api" "^1.4.0" + tdigest "^0.1.1" + promise-inflight@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/promise-inflight/-/promise-inflight-1.0.1.tgz#98472870bf228132fcbdd868129bad12c3c029e3" @@ -29844,6 +29857,13 @@ tarn@^3.0.2: resolved "https://registry.yarnpkg.com/tarn/-/tarn-3.0.2.tgz#73b6140fbb881b71559c4f8bfde3d9a4b3d27693" integrity sha512-51LAVKUSZSVfI05vjPESNc5vwqqZpbXCsU+/+wxlOrUjk2SnFTt97v9ZgQrD4YmxYW1Px6w2KjaDitCfkvgxMQ== +tdigest@^0.1.1: + version "0.1.2" + resolved "https://registry.yarnpkg.com/tdigest/-/tdigest-0.1.2.tgz#96c64bac4ff10746b910b0e23b515794e12faced" + integrity sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA== + dependencies: + bintrees "1.0.2" + teex@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/teex/-/teex-1.0.1.tgz#b8fa7245ef8e8effa8078281946c85ab780a0b12" From 10eed202433544b646dd501edc62750f471838e4 Mon Sep 17 00:00:00 2001 From: Chris Raible Date: Thu, 12 Sep 2024 14:11:35 -0700 Subject: [PATCH 2/6] Added summary metric for http response time --- .../core/core/server/web/parent/middleware/log-request.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ghost/core/core/server/web/parent/middleware/log-request.js b/ghost/core/core/server/web/parent/middleware/log-request.js index 7f1d56dace3..8a278e7e794 100644 --- a/ghost/core/core/server/web/parent/middleware/log-request.js +++ b/ghost/core/core/server/web/parent/middleware/log-request.js @@ -7,6 +7,12 @@ const counter = new promClient.Counter({ labelNames: ['method', 'statusCode'] }); +const summary = new promClient.Summary({ + name: 'ghost_http_response_time', + help: 'Summary of response times for all HTTP requests', + percentiles: [0.01, 0.1, 0.9, 0.99] +}); + /** * @TODO: move this middleware to Framework monorepo? * @@ -32,6 +38,8 @@ module.exports = function logRequest(req, res, next) { statusCode: res.statusCode }); + summary.observe(Date.now() - startTime); + res.removeListener('finish', logResponse); res.removeListener('close', logResponse); } From 978ca185f861e4416adbb22e3c42d02bc59a8361 Mon Sep 17 00:00:00 2001 From: Chris Raible Date: Thu, 12 Sep 2024 14:24:49 -0700 Subject: [PATCH 3/6] Added connection pool metrics --- ghost/core/core/server/data/db/connection.js | 38 ++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/ghost/core/core/server/data/db/connection.js b/ghost/core/core/server/data/db/connection.js index 4cb555de508..8b75b2a9ced 100644 --- a/ghost/core/core/server/data/db/connection.js +++ b/ghost/core/core/server/data/db/connection.js @@ -8,6 +8,9 @@ const metrics = require('@tryghost/metrics'); const config = require('../../../shared/config'); const errors = require('@tryghost/errors'); const ConnectionPoolInstrumentation = require('./ConnectionPoolInstrumentation'); + +const promClient = require('prom-client'); + let knexInstance; // @TODO: @@ -64,6 +67,41 @@ function configure(dbConfig) { if (!knexInstance && config.get('database') && config.get('database').client) { knexInstance = knex(configure(config.get('database'))); + new promClient.Gauge({ + name: 'ghost_db_connection_pool_used', + help: 'Number of connections currently in use in the database connection pool', + collect() { + this.set(knexInstance.client.pool.numUsed()); + } + }); + new promClient.Gauge({ + name: 'ghost_db_connection_pool_free', + help: 'Number of free connections in the database connection pool', + collect() { + this.set(knexInstance.client.pool.numFree()); + } + }); + new promClient.Gauge({ + name: 'ghost_db_connection_pool_pending_creates', + help: 'Number of pending create requests in the database connection pool', + collect() { + this.set(knexInstance.client.pool.numPendingCreates()); + } + }); + new promClient.Gauge({ + name: 'ghost_db_connection_pool_pending_acquires', + help: 'Number of pending acquire requests in the database connection pool', + collect() { + this.set(knexInstance.client.pool.numPendingAcquires()); + } + }); + new promClient.Gauge({ + name: 'ghost_db_connection_pool_utilization', + help: 'Percentage of connections in use in the database connection pool', + collect() { + this.set(knexInstance.client.pool.numUsed() / (knexInstance.client.pool.numUsed() + knexInstance.client.pool.numFree())); + } + }); if (config.get('telemetry:connectionPool')) { const instrumentation = new ConnectionPoolInstrumentation({knex: knexInstance, logging, metrics, config}); instrumentation.instrument(); From 5c814dc45ef1bbc6d0251986df03a8fca244e515 Mon Sep 17 00:00:00 2001 From: Chris Raible Date: Thu, 12 Sep 2024 15:10:39 -0700 Subject: [PATCH 4/6] Added graphs to grafana dashboard for jobs manager, connection pool, and http requests --- .../grafana/dashboards/main-dashboard.json | 1040 +++++++++++++++++ ghost/core/core/server/data/db/connection.js | 7 + .../web/parent/middleware/log-request.js | 2 +- 3 files changed, 1048 insertions(+), 1 deletion(-) diff --git a/.github/scripts/grafana/dashboards/main-dashboard.json b/.github/scripts/grafana/dashboards/main-dashboard.json index a753bdc2071..915cd2e6e35 100644 --- a/.github/scripts/grafana/dashboards/main-dashboard.json +++ b/.github/scripts/grafana/dashboards/main-dashboard.json @@ -1444,6 +1444,1046 @@ "align": false, "alignLevel": null } + }, + { + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 17, + "title": "Jobs Manager", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Active workers used by the job manager.", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 39 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0-beta2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ghost_jobs_manager_active_workers", + "interval": "", + "legendFormat": "{{space}} - {{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active workers", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 19, + "title": "Connection Pool", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Connection pool utilization.", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 47 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0-beta2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ghost_db_connection_pool_utilization", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Connection pool num used.", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 47 + }, + "hiddenSeries": false, + "id": 21, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0-beta2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ghost_db_connection_pool_used", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Num Used", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Connection pool num free.", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 47 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0-beta2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ghost_db_connection_pool_free", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Num Free", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Connection pool pending acquires.", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 55 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0-beta2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ghost_db_connection_pool_pending_acquires", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pending Acquires", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Connection pool pending creates.", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 55 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0-beta2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ghost_db_connection_pool_pending_creates", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pending Creates", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Connection pool max", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 55 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0-beta2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ghost_db_connection_pool_max", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Max", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 26, + "title": "HTTP", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Request Queue Depth", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 63 + }, + "hiddenSeries": false, + "id": 27, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0-beta2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ghost_request_queue_depth", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request Queue Depth", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Request Queue In Process", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 63 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0-beta2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ghost_request_queue_in_process", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request Queue In Process", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Response Time", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 63 + }, + "hiddenSeries": false, + "id": 29, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0-beta2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ghost_http_response_time{quantile=\"0.99\"}", + "interval": "", + "legendFormat": "P99", + "refId": "A" + }, + { + "expr": "ghost_http_response_time{quantile=\"0.95\"}", + "interval": "", + "legendFormat": "P95", + "refId": "B" + }, + { + "expr": "ghost_http_response_time{quantile=\"0.5\"}", + "interval": "", + "legendFormat": "P50", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "P99 Response Time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "refresh": false, diff --git a/ghost/core/core/server/data/db/connection.js b/ghost/core/core/server/data/db/connection.js index 8b75b2a9ced..a75ddb9ddea 100644 --- a/ghost/core/core/server/data/db/connection.js +++ b/ghost/core/core/server/data/db/connection.js @@ -102,6 +102,13 @@ if (!knexInstance && config.get('database') && config.get('database').client) { this.set(knexInstance.client.pool.numUsed() / (knexInstance.client.pool.numUsed() + knexInstance.client.pool.numFree())); } }); + new promClient.Gauge({ + name: 'ghost_db_connection_pool_max', + help: 'Total number of connections in the database connection pool', + collect() { + this.set(knexInstance.client.pool.max); + } + }); if (config.get('telemetry:connectionPool')) { const instrumentation = new ConnectionPoolInstrumentation({knex: knexInstance, logging, metrics, config}); instrumentation.instrument(); diff --git a/ghost/core/core/server/web/parent/middleware/log-request.js b/ghost/core/core/server/web/parent/middleware/log-request.js index 8a278e7e794..be539eabe5c 100644 --- a/ghost/core/core/server/web/parent/middleware/log-request.js +++ b/ghost/core/core/server/web/parent/middleware/log-request.js @@ -10,7 +10,7 @@ const counter = new promClient.Counter({ const summary = new promClient.Summary({ name: 'ghost_http_response_time', help: 'Summary of response times for all HTTP requests', - percentiles: [0.01, 0.1, 0.9, 0.99] + percentiles: [0.01, 0.1, 0.5, 0.95, 0.99] }); /** From 8329b0c61802f4a153a45861ba2c8f236227c064 Mon Sep 17 00:00:00 2001 From: Chris Raible Date: Thu, 12 Sep 2024 15:43:49 -0700 Subject: [PATCH 5/6] Added db queries per second --- .../grafana/dashboards/main-dashboard.json | 113 +++++++++++++++++- ghost/core/core/server/data/db/connection.js | 9 +- 2 files changed, 120 insertions(+), 2 deletions(-) diff --git a/.github/scripts/grafana/dashboards/main-dashboard.json b/.github/scripts/grafana/dashboards/main-dashboard.json index 915cd2e6e35..d0274c8d282 100644 --- a/.github/scripts/grafana/dashboards/main-dashboard.json +++ b/.github/scripts/grafana/dashboards/main-dashboard.json @@ -2448,7 +2448,118 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "P99 Response Time", + "title": "Response Time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 71 + }, + "id": 30, + "title": "Database", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Queries per Second", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 71 + }, + "hiddenSeries": false, + "id": 31, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0-beta2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(ghost_db_queries_total[1m])", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Queries per Second", "tooltip": { "shared": true, "sort": 0, diff --git a/ghost/core/core/server/data/db/connection.js b/ghost/core/core/server/data/db/connection.js index a75ddb9ddea..51c262fd1a0 100644 --- a/ghost/core/core/server/data/db/connection.js +++ b/ghost/core/core/server/data/db/connection.js @@ -99,7 +99,7 @@ if (!knexInstance && config.get('database') && config.get('database').client) { name: 'ghost_db_connection_pool_utilization', help: 'Percentage of connections in use in the database connection pool', collect() { - this.set(knexInstance.client.pool.numUsed() / (knexInstance.client.pool.numUsed() + knexInstance.client.pool.numFree())); + this.set(knexInstance.client.pool.numUsed() / (knexInstance.client.pool.max)); } }); new promClient.Gauge({ @@ -109,6 +109,13 @@ if (!knexInstance && config.get('database') && config.get('database').client) { this.set(knexInstance.client.pool.max); } }); + const queryCounter = new promClient.Counter({ + name: 'ghost_db_queries_total', + help: 'Total number of queries executed' + }); + knexInstance.on('query', () => { + queryCounter.inc(); + }); if (config.get('telemetry:connectionPool')) { const instrumentation = new ConnectionPoolInstrumentation({knex: knexInstance, logging, metrics, config}); instrumentation.instrument(); From 6575dd16a287489c33be9ddfc7f989c9f45fdfbe Mon Sep 17 00:00:00 2001 From: Chris Raible Date: Thu, 12 Sep 2024 15:54:05 -0700 Subject: [PATCH 6/6] Added queries per second graph --- .../grafana/dashboards/main-dashboard.json | 107 +++++++++++++++++- 1 file changed, 103 insertions(+), 4 deletions(-) diff --git a/.github/scripts/grafana/dashboards/main-dashboard.json b/.github/scripts/grafana/dashboards/main-dashboard.json index d0274c8d282..d44d0a6ae4f 100644 --- a/.github/scripts/grafana/dashboards/main-dashboard.json +++ b/.github/scripts/grafana/dashboards/main-dashboard.json @@ -2485,15 +2485,114 @@ "alignLevel": null } }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Requests per Second", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 71 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0-beta2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(ghost_http_requests_total[1m])", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 71 + "y": 79 }, - "id": 30, + "id": 31, "title": "Database", "type": "row" }, @@ -2517,10 +2616,10 @@ "h": 8, "w": 8, "x": 0, - "y": 71 + "y": 79 }, "hiddenSeries": false, - "id": 31, + "id": 32, "legend": { "alignAsTable": true, "avg": true,