openclaw-grafana-lens
Version:
OpenClaw plugin that gives AI agents full Grafana access — 18 composable tools for PromQL/LogQL/TraceQL queries, dashboard creation, alerting, SRE investigation, security monitoring, data collection pipeline management via Grafana Alloy (29 recipes), and
1,105 lines (1,104 loc) • 35 kB
JSON
{
"title": "SRE Operations",
"description": "Infrastructure and operational health — queue depth, webhook delivery, session health, stuck detection, and tool loop monitoring. The Tier 3 SRE deep-dive in the OpenClaw observability hierarchy.",
"tags": [
"openclaw",
"grafana-lens",
"sre-operations"
],
"timezone": "browser",
"editable": true,
"schemaVersion": 39,
"time": {
"from": "now-6h",
"to": "now"
},
"refresh": "30s",
"links": [
{
"title": "LLM Command Center",
"tags": [
"openclaw",
"grafana-lens",
"command-center"
],
"type": "dashboards",
"asDropdown": false,
"icon": "external link"
},
{
"title": "Session Explorer",
"tags": [
"openclaw",
"grafana-lens",
"session-explorer"
],
"type": "dashboards",
"asDropdown": false,
"icon": "external link"
},
{
"title": "Tool Performance",
"tags": [
"openclaw",
"grafana-lens",
"tool-performance"
],
"type": "dashboards",
"asDropdown": false,
"icon": "external link"
},
{
"title": "GenAI Observability",
"tags": [
"openclaw",
"grafana-lens",
"genai-observability"
],
"type": "dashboards",
"asDropdown": false,
"icon": "external link"
},
{
"title": "OpenClaw Dashboards",
"tags": [
"openclaw",
"grafana-lens"
],
"type": "dashboards",
"asDropdown": true,
"icon": "external link"
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"templating": {
"list": [
{
"name": "prometheus",
"label": "Prometheus",
"type": "datasource",
"query": "prometheus",
"current": {},
"hide": 0
},
{
"name": "loki",
"label": "Loki",
"type": "datasource",
"query": "loki",
"current": {},
"hide": 0
}
]
},
"panels": [
{
"id": 1,
"title": "Health Overview",
"type": "row",
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"collapsed": false,
"panels": []
},
{
"id": 2,
"title": "Queue Depth",
"description": "Current message queue depth.",
"type": "stat",
"gridPos": {
"h": 4,
"w": 4,
"x": 0,
"y": 1
},
"datasource": {
"type": "prometheus",
"uid": "$prometheus"
},
"targets": [
{
"expr": "sum(openclaw_lens_queue_depth) or vector(0)",
"legendFormat": "depth",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 5
},
{
"color": "red",
"value": 20
}
]
},
"noValue": "0"
}
},
"options": {
"graphMode": "area",
"colorMode": "background"
}
},
{
"id": 3,
"title": "Stuck Sessions",
"description": "Number of sessions detected as stuck.",
"type": "stat",
"gridPos": {
"h": 4,
"w": 4,
"x": 4,
"y": 1
},
"datasource": {
"type": "prometheus",
"uid": "$prometheus"
},
"targets": [
{
"expr": "sum(openclaw_lens_sessions_stuck) or vector(0)",
"legendFormat": "stuck",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
},
"noValue": "0"
}
},
"options": {
"graphMode": "none",
"colorMode": "background"
}
},
{
"id": 4,
"title": "Max Stuck Age",
"description": "Maximum age (ms) of any stuck session.",
"type": "stat",
"gridPos": {
"h": 4,
"w": 4,
"x": 8,
"y": 1
},
"datasource": {
"type": "prometheus",
"uid": "$prometheus"
},
"targets": [
{
"expr": "sum(openclaw_lens_stuck_session_max_age_ms) or vector(0)",
"legendFormat": "max age",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "ms",
"decimals": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 30000
},
{
"color": "red",
"value": 120000
}
]
},
"noValue": "0ms"
}
},
"options": {
"graphMode": "none",
"colorMode": "background"
}
},
{
"id": 5,
"title": "Tool Loops",
"description": "Currently active tool loop detections (warning + critical).",
"type": "stat",
"gridPos": {
"h": 4,
"w": 4,
"x": 12,
"y": 1
},
"datasource": {
"type": "prometheus",
"uid": "$prometheus"
},
"targets": [
{
"expr": "sum(openclaw_lens_tool_loops_active) or vector(0)",
"legendFormat": "loops",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
},
"noValue": "0"
}
},
"options": {
"graphMode": "none",
"colorMode": "background"
}
},
{
"id": 6,
"title": "Webhook Errors",
"description": "Webhook error rate over the last 5 minutes.",
"type": "stat",
"gridPos": {
"h": 4,
"w": 4,
"x": 16,
"y": 1
},
"datasource": {
"type": "prometheus",
"uid": "$prometheus"
},
"targets": [
{
"expr": "sum(rate(openclaw_lens_webhook_error_total[5m])) * 60 or vector(0)",
"legendFormat": "errors/min",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 1,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 0.1
}
]
},
"noValue": "0"
}
},
"options": {
"graphMode": "area",
"colorMode": "background"
}
},
{
"id": 7,
"title": "Active Sessions",
"description": "Currently active agent sessions from heartbeat snapshot.",
"type": "stat",
"gridPos": {
"h": 4,
"w": 4,
"x": 20,
"y": 1
},
"datasource": {
"type": "prometheus",
"uid": "$prometheus"
},
"targets": [
{
"expr": "sum(openclaw_lens_sessions_active_snapshot) or vector(0)",
"legendFormat": "active",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"noValue": "0"
}
},
"options": {
"graphMode": "area"
}
},
{
"id": 21,
"title": "Sessions Today",
"description": "Total sessions completed today.",
"type": "stat",
"gridPos": {
"h": 4,
"w": 8,
"x": 0,
"y": 5
},
"datasource": {
"type": "prometheus",
"uid": "$prometheus"
},
"targets": [
{
"expr": "sum(increase(openclaw_lens_sessions_completed_total[1d])) or vector(0)",
"legendFormat": "sessions",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"noValue": "0"
}
},
"options": {
"graphMode": "area"
}
},
{
"id": 22,
"title": "Context Pressure",
"description": "Context window utilization. >60% yellow, >80% red — compaction imminent.",
"type": "gauge",
"gridPos": {
"h": 4,
"w": 8,
"x": 8,
"y": 5
},
"datasource": {
"type": "prometheus",
"uid": "$prometheus"
},
"targets": [
{
"expr": "openclaw_lens_context_tokens{type=\"used\"} / openclaw_lens_context_tokens{type=\"limit\"} * 100",
"legendFormat": "pressure %",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "percent",
"min": 0,
"max": 100,
"decimals": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 60
},
{
"color": "red",
"value": 80
}
]
},
"noValue": "0%"
}
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
]
},
"showThresholdLabels": false,
"showThresholdMarkers": true
}
},
{
"id": 23,
"title": "Messages/hr",
"description": "Current message processing rate projected to hourly.",
"type": "stat",
"gridPos": {
"h": 4,
"w": 8,
"x": 16,
"y": 5
},
"datasource": {
"type": "prometheus",
"uid": "$prometheus"
},
"targets": [
{
"expr": "sum(rate(openclaw_lens_messages_processed_total[5m])) * 3600 or vector(0)",
"legendFormat": "msg/hr",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"noValue": "0"
}
},
"options": {
"graphMode": "area"
}
},
{
"id": 8,
"title": "Queue Health",
"type": "row",
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 9
},
"collapsed": false,
"panels": []
},
{
"id": 9,
"title": "Queue Depth",
"description": "Message queue depth over time.",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 10
},
"datasource": {
"type": "prometheus",
"uid": "$prometheus"
},
"targets": [
{
"expr": "openclaw_lens_queue_depth",
"legendFormat": "depth",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"custom": {
"drawStyle": "line",
"fillOpacity": 20,
"spanNulls": true
}
}
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 10,
"title": "Enqueue vs Dequeue Rate",
"description": "Message queue enqueue and dequeue rates. Divergence indicates queue buildup.",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 10
},
"datasource": {
"type": "prometheus",
"uid": "$prometheus"
},
"targets": [
{
"expr": "sum(rate(openclaw_lens_queue_lane_enqueue_total[5m]))",
"legendFormat": "enqueue/s",
"refId": "A"
},
{
"expr": "sum(rate(openclaw_lens_queue_lane_dequeue_total[5m]))",
"legendFormat": "dequeue/s",
"refId": "B"
}
],
"fieldConfig": {
"defaults": {
"unit": "cps",
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"spanNulls": true
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "enqueue/s"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "blue",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "dequeue/s"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "green",
"mode": "fixed"
}
}
]
}
]
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 11,
"title": "Queue Wait Time (p50/p95)",
"description": "How long messages wait in the queue before processing.",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 10
},
"datasource": {
"type": "prometheus",
"uid": "$prometheus"
},
"targets": [
{
"expr": "histogram_quantile(0.50, sum(rate(openclaw_lens_queue_wait_ms_bucket[5m])) by (le))",
"legendFormat": "p50",
"refId": "A"
},
{
"expr": "histogram_quantile(0.95, sum(rate(openclaw_lens_queue_wait_ms_bucket[5m])) by (le))",
"legendFormat": "p95",
"refId": "B"
}
],
"fieldConfig": {
"defaults": {
"unit": "ms",
"custom": {
"drawStyle": "line",
"spanNulls": true
}
}
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 12,
"title": "Webhooks & Delivery",
"type": "row",
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 18
},
"collapsed": false,
"panels": []
},
{
"id": 13,
"title": "Webhook Events",
"description": "Webhook received and error rates over time.",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 19
},
"datasource": {
"type": "prometheus",
"uid": "$prometheus"
},
"targets": [
{
"expr": "sum(rate(openclaw_lens_webhook_received_total[5m]))",
"legendFormat": "received/s",
"refId": "A"
},
{
"expr": "sum(rate(openclaw_lens_webhook_error_total[5m]))",
"legendFormat": "errors/s",
"refId": "B"
}
],
"fieldConfig": {
"defaults": {
"unit": "cps",
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"spanNulls": true
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "errors/s"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "red",
"mode": "fixed"
}
}
]
}
]
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 14,
"title": "Webhook Duration (p50/p95)",
"description": "Webhook processing time percentiles.",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 19
},
"datasource": {
"type": "prometheus",
"uid": "$prometheus"
},
"targets": [
{
"expr": "histogram_quantile(0.50, sum(rate(openclaw_lens_webhook_duration_ms_bucket[5m])) by (le))",
"legendFormat": "p50",
"refId": "A"
},
{
"expr": "histogram_quantile(0.95, sum(rate(openclaw_lens_webhook_duration_ms_bucket[5m])) by (le))",
"legendFormat": "p95",
"refId": "B"
}
],
"fieldConfig": {
"defaults": {
"unit": "ms",
"custom": {
"drawStyle": "line",
"spanNulls": true
}
}
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 15,
"title": "Session Health",
"type": "row",
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 27
},
"collapsed": false,
"panels": []
},
{
"id": 16,
"title": "Session State Changes",
"description": "Session state transition events. Tracks idle→processing→idle lifecycle.",
"type": "logs",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 28
},
"datasource": {
"type": "loki",
"uid": "$loki"
},
"targets": [
{
"expr": "{service_name=\"openclaw\"} | event_name=~\"session\\\\.(start|state)\"",
"refId": "A"
}
],
"options": {
"showTime": true,
"showLabels": false,
"showCommonLabels": false,
"wrapLogMessage": true,
"prettifyLogMessage": false,
"enableLogDetails": true,
"sortOrder": "Descending",
"dedupStrategy": "none"
}
},
{
"id": 17,
"title": "Stuck Sessions & Tool Loop Alerts",
"description": "Events for stuck sessions and tool loop detections — should be empty on healthy systems.",
"type": "logs",
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 28
},
"datasource": {
"type": "loki",
"uid": "$loki"
},
"targets": [
{
"expr": "{service_name=\"openclaw\"} | event_name=~\"session\\\\.stuck|tool\\\\.loop|tool\\\\.repeated_failure\"",
"refId": "A"
}
],
"options": {
"showTime": true,
"showLabels": false,
"showCommonLabels": false,
"wrapLogMessage": true,
"prettifyLogMessage": false,
"enableLogDetails": true,
"sortOrder": "Descending",
"dedupStrategy": "none"
}
},
{
"id": 18,
"title": "GenAI SRE Signals",
"type": "row",
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 36
},
"collapsed": false,
"panels": []
},
{
"id": 19,
"title": "LLM Error Rate (gen_ai standard)",
"description": "Error rate from gen_ai standard metrics — percentage of LLM calls returning errors. Cross-validate with openclaw-specific error metrics.",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 37
},
"datasource": {
"type": "prometheus",
"uid": "$prometheus"
},
"targets": [
{
"expr": "sum(rate(gen_ai_client_operation_duration_seconds_count{error_type!=\"\"}[5m])) / sum(rate(gen_ai_client_operation_duration_seconds_count[5m])) * 100 or vector(0)",
"legendFormat": "gen_ai error %",
"refId": "A"
},
{
"expr": "sum(rate(openclaw_lens_messages_processed_total{outcome=\"error\"}[5m])) / sum(rate(openclaw_lens_messages_processed_total[5m])) * 100 or vector(0)",
"legendFormat": "message error %",
"refId": "B"
}
],
"fieldConfig": {
"defaults": {
"unit": "percent",
"decimals": 1,
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"spanNulls": true
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "red",
"value": 5
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "gen_ai error %"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "red",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "message error %"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "orange",
"mode": "fixed"
}
}
]
}
]
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"mean",
"max"
]
}
}
},
{
"id": 20,
"title": "Context Window Pressure",
"description": "Context token usage vs limit. When usage approaches the limit, sessions will trigger compaction. Watch for sustained high pressure.",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 37
},
"datasource": {
"type": "prometheus",
"uid": "$prometheus"
},
"targets": [
{
"expr": "openclaw_lens_context_tokens{type=\"used\"}",
"legendFormat": "used",
"refId": "A"
},
{
"expr": "openclaw_lens_context_tokens{type=\"limit\"}",
"legendFormat": "limit",
"refId": "B"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"spanNulls": true
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "used"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "blue",
"mode": "fixed"
}
},
{
"id": "custom.fillOpacity",
"value": 20
}
]
},
{
"matcher": {
"id": "byName",
"options": "limit"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "red",
"mode": "fixed"
}
},
{
"id": "custom.lineStyle",
"value": {
"fill": "dash",
"dash": [
10,
10
]
}
},
{
"id": "custom.fillOpacity",
"value": 0
},
{
"id": "custom.lineWidth",
"value": 2
}
]
}
]
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"mean",
"max"
]
}
}
}
]
}