489 lines
13 KiB
JSON
489 lines
13 KiB
JSON
{
|
|
"dashboard": {
|
|
"title": "HeiCode Swarm System Health",
|
|
"tags": ["swarm", "k8s", "agents"],
|
|
"timezone": "browser",
|
|
"schemaVersion": 16,
|
|
"version": 1,
|
|
"refresh": "10s",
|
|
"time": {
|
|
"from": "now-1h",
|
|
"to": "now"
|
|
},
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"title": "Active Agents",
|
|
"type": "stat",
|
|
"gridPos": {"x": 0, "y": 0, "w": 6, "h": 4},
|
|
"targets": [
|
|
{
|
|
"expr": "swarm:agents:active:total",
|
|
"legendFormat": "Active Agents"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"value": 0, "color": "green"},
|
|
{"value": 10, "color": "yellow"},
|
|
{"value": 20, "color": "red"}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 2,
|
|
"title": "Task Completion Rate",
|
|
"type": "stat",
|
|
"gridPos": {"x": 6, "y": 0, "w": 6, "h": 4},
|
|
"targets": [
|
|
{
|
|
"expr": "swarm:tasks:completion_rate:5m",
|
|
"legendFormat": "Tasks/sec"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "ops",
|
|
"decimals": 2
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 3,
|
|
"title": "Average Task Duration",
|
|
"type": "stat",
|
|
"gridPos": {"x": 12, "y": 0, "w": 6, "h": 4},
|
|
"targets": [
|
|
{
|
|
"expr": "swarm:tasks:duration:avg",
|
|
"legendFormat": "Avg Duration"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s",
|
|
"decimals": 1
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 4,
|
|
"title": "Pod Startup Time (P95)",
|
|
"type": "stat",
|
|
"gridPos": {"x": 18, "y": 0, "w": 6, "h": 4},
|
|
"targets": [
|
|
{
|
|
"expr": "swarm:pod:startup:p95",
|
|
"legendFormat": "P95 Startup"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s",
|
|
"decimals": 1,
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"value": 0, "color": "green"},
|
|
{"value": 30, "color": "yellow"},
|
|
{"value": 60, "color": "red"}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 5,
|
|
"title": "Agent Count Over Time",
|
|
"type": "graph",
|
|
"gridPos": {"x": 0, "y": 4, "w": 12, "h": 8},
|
|
"targets": [
|
|
{
|
|
"expr": "swarm:agents:count:by_status",
|
|
"legendFormat": "{{status}}"
|
|
}
|
|
],
|
|
"yaxes": [
|
|
{"format": "short", "label": "Agents"},
|
|
{"format": "short"}
|
|
],
|
|
"legend": {"show": true, "alignAsTable": true, "rightSide": false}
|
|
},
|
|
{
|
|
"id": 6,
|
|
"title": "Task Metrics",
|
|
"type": "graph",
|
|
"gridPos": {"x": 12, "y": 4, "w": 12, "h": 8},
|
|
"targets": [
|
|
{
|
|
"expr": "rate(swarm_tasks_completed_total[5m])",
|
|
"legendFormat": "Completed"
|
|
},
|
|
{
|
|
"expr": "rate(swarm_tasks_failed_total[5m])",
|
|
"legendFormat": "Failed"
|
|
}
|
|
],
|
|
"yaxes": [
|
|
{"format": "ops", "label": "Tasks/sec"},
|
|
{"format": "short"}
|
|
]
|
|
},
|
|
{
|
|
"id": 7,
|
|
"title": "Handoff Latency Distribution",
|
|
"type": "graph",
|
|
"gridPos": {"x": 0, "y": 12, "w": 12, "h": 8},
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(0.50, rate(swarm_handoff_duration_seconds_bucket[5m]))",
|
|
"legendFormat": "P50"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.95, rate(swarm_handoff_duration_seconds_bucket[5m]))",
|
|
"legendFormat": "P95"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.99, rate(swarm_handoff_duration_seconds_bucket[5m]))",
|
|
"legendFormat": "P99"
|
|
}
|
|
],
|
|
"yaxes": [
|
|
{"format": "s", "label": "Latency"},
|
|
{"format": "short"}
|
|
]
|
|
},
|
|
{
|
|
"id": 8,
|
|
"title": "Agent Creation Rate",
|
|
"type": "graph",
|
|
"gridPos": {"x": 12, "y": 12, "w": 12, "h": 8},
|
|
"targets": [
|
|
{
|
|
"expr": "swarm:agents:creation_rate:5m",
|
|
"legendFormat": "Creation Rate"
|
|
}
|
|
],
|
|
"yaxes": [
|
|
{"format": "ops", "label": "Agents/sec"},
|
|
{"format": "short"}
|
|
]
|
|
},
|
|
{
|
|
"id": 9,
|
|
"title": "System Resource Usage",
|
|
"type": "graph",
|
|
"gridPos": {"x": 0, "y": 20, "w": 12, "h": 8},
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"swarm-system\"}[5m]))",
|
|
"legendFormat": "CPU Usage"
|
|
},
|
|
{
|
|
"expr": "sum(container_memory_working_set_bytes{namespace=\"swarm-system\"}) / 1024 / 1024 / 1024",
|
|
"legendFormat": "Memory Usage (GB)"
|
|
}
|
|
],
|
|
"yaxes": [
|
|
{"format": "short", "label": "Resources"},
|
|
{"format": "short"}
|
|
]
|
|
},
|
|
{
|
|
"id": 10,
|
|
"title": "Redis Operations",
|
|
"type": "graph",
|
|
"gridPos": {"x": 12, "y": 20, "w": 12, "h": 8},
|
|
"targets": [
|
|
{
|
|
"expr": "rate(redis_commands_processed_total[5m])",
|
|
"legendFormat": "Commands/sec"
|
|
},
|
|
{
|
|
"expr": "redis_connected_clients",
|
|
"legendFormat": "Connected Clients"
|
|
}
|
|
],
|
|
"yaxes": [
|
|
{"format": "ops", "label": "Operations"},
|
|
{"format": "short"}
|
|
]
|
|
},
|
|
{
|
|
"id": 11,
|
|
"title": "Pod Status by Phase",
|
|
"type": "piechart",
|
|
"gridPos": {"x": 0, "y": 28, "w": 8, "h": 8},
|
|
"targets": [
|
|
{
|
|
"expr": "count by (phase) (kube_pod_status_phase{namespace=\"swarm-system\"})",
|
|
"legendFormat": "{{phase}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 12,
|
|
"title": "Error Rate",
|
|
"type": "graph",
|
|
"gridPos": {"x": 8, "y": 28, "w": 8, "h": 8},
|
|
"targets": [
|
|
{
|
|
"expr": "rate(swarm_errors_total[5m])",
|
|
"legendFormat": "{{error_type}}"
|
|
}
|
|
],
|
|
"yaxes": [
|
|
{"format": "ops", "label": "Errors/sec"},
|
|
{"format": "short"}
|
|
],
|
|
"alert": {
|
|
"conditions": [
|
|
{
|
|
"evaluator": {"params": [0.1], "type": "gt"},
|
|
"operator": {"type": "and"},
|
|
"query": {"params": ["A", "5m", "now"]},
|
|
"reducer": {"params": [], "type": "avg"},
|
|
"type": "query"
|
|
}
|
|
],
|
|
"executionErrorState": "alerting",
|
|
"frequency": "60s",
|
|
"handler": 1,
|
|
"name": "High Error Rate",
|
|
"noDataState": "no_data",
|
|
"notifications": []
|
|
}
|
|
},
|
|
{
|
|
"id": 13,
|
|
"title": "Network I/O",
|
|
"type": "graph",
|
|
"gridPos": {"x": 16, "y": 28, "w": 8, "h": 8},
|
|
"targets": [
|
|
{
|
|
"expr": "rate(container_network_receive_bytes_total{namespace=\"swarm-system\"}[5m])",
|
|
"legendFormat": "RX {{pod}}"
|
|
},
|
|
{
|
|
"expr": "rate(container_network_transmit_bytes_total{namespace=\"swarm-system\"}[5m])",
|
|
"legendFormat": "TX {{pod}}"
|
|
}
|
|
],
|
|
"yaxes": [
|
|
{"format": "Bps", "label": "Bytes/sec"},
|
|
{"format": "short"}
|
|
]
|
|
},
|
|
{
|
|
"id": 14,
|
|
"title": "Top Agents by Duration",
|
|
"type": "table",
|
|
"gridPos": {"x": 0, "y": 36, "w": 12, "h": 8},
|
|
"targets": [
|
|
{
|
|
"expr": "topk(10, swarm_agent_duration_seconds)",
|
|
"format": "table",
|
|
"instant": true
|
|
}
|
|
],
|
|
"transformations": [
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {"Time": true},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"agent_id": "Agent ID",
|
|
"task_id": "Task ID",
|
|
"Value": "Duration (s)"
|
|
}
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 15,
|
|
"title": "Recent Failures",
|
|
"type": "table",
|
|
"gridPos": {"x": 12, "y": 36, "w": 12, "h": 8},
|
|
"targets": [
|
|
{
|
|
"expr": "topk(10, swarm_agent_failures)",
|
|
"format": "table",
|
|
"instant": true
|
|
}
|
|
],
|
|
"transformations": [
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {"Time": true},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"agent_id": "Agent ID",
|
|
"task_id": "Task ID",
|
|
"error_type": "Error Type",
|
|
"Value": "Count"
|
|
}
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 16,
|
|
"title": "Orchestrator Health",
|
|
"type": "stat",
|
|
"gridPos": {"x": 0, "y": 44, "w": 6, "h": 4},
|
|
"targets": [
|
|
{
|
|
"expr": "up{job=\"orchestrator\"}",
|
|
"legendFormat": "Status"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"mappings": [
|
|
{"type": "value", "value": "0", "text": "DOWN"},
|
|
{"type": "value", "value": "1", "text": "UP"}
|
|
],
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"value": 0, "color": "red"},
|
|
{"value": 1, "color": "green"}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 17,
|
|
"title": "Redis Health",
|
|
"type": "stat",
|
|
"gridPos": {"x": 6, "y": 44, "w": 6, "h": 4},
|
|
"targets": [
|
|
{
|
|
"expr": "up{job=\"redis\"}",
|
|
"legendFormat": "Status"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"mappings": [
|
|
{"type": "value", "value": "0", "text": "DOWN"},
|
|
{"type": "value", "value": "1", "text": "UP"}
|
|
],
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"value": 0, "color": "red"},
|
|
{"value": 1, "color": "green"}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 18,
|
|
"title": "Kubernetes Cluster Health",
|
|
"type": "stat",
|
|
"gridPos": {"x": 12, "y": 44, "w": 6, "h": 4},
|
|
"targets": [
|
|
{
|
|
"expr": "up{job=\"kubernetes-apiservers\"}",
|
|
"legendFormat": "API Server"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"mappings": [
|
|
{"type": "value", "value": "0", "text": "DOWN"},
|
|
{"type": "value", "value": "1", "text": "UP"}
|
|
],
|
|
"color": {"mode": "thresholds"},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"value": 0, "color": "red"},
|
|
{"value": 1, "color": "green"}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 19,
|
|
"title": "Total System Cost (Estimated)",
|
|
"type": "stat",
|
|
"gridPos": {"x": 18, "y": 44, "w": 6, "h": 4},
|
|
"targets": [
|
|
{
|
|
"expr": "sum(swarm_agent_cost_usd)",
|
|
"legendFormat": "Total Cost"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "currencyUSD",
|
|
"decimals": 2
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"name": "namespace",
|
|
"type": "query",
|
|
"query": "label_values(swarm_agent_status, namespace)",
|
|
"current": {"text": "swarm-system", "value": "swarm-system"},
|
|
"hide": 0,
|
|
"includeAll": false,
|
|
"multi": false,
|
|
"options": [],
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"sort": 0
|
|
},
|
|
{
|
|
"name": "task_id",
|
|
"type": "query",
|
|
"query": "label_values(swarm_agent_status{namespace=\"$namespace\"}, task_id)",
|
|
"current": {"text": "All", "value": "$__all"},
|
|
"hide": 0,
|
|
"includeAll": true,
|
|
"multi": true,
|
|
"options": [],
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"sort": 0
|
|
}
|
|
]
|
|
},
|
|
"annotations": {
|
|
"list": [
|
|
{
|
|
"datasource": "Prometheus",
|
|
"enable": true,
|
|
"expr": "ALERTS{alertstate=\"firing\"}",
|
|
"iconColor": "red",
|
|
"name": "Alerts",
|
|
"step": "60s",
|
|
"tagKeys": "alertname",
|
|
"textFormat": "{{alertname}}",
|
|
"titleFormat": "Alert"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|