349 lines
9.1 KiB
YAML
349 lines
9.1 KiB
YAML
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: prometheus-config
|
|
namespace: swarm-system
|
|
data:
|
|
prometheus.yml: |
|
|
global:
|
|
scrape_interval: 15s
|
|
evaluation_interval: 15s
|
|
external_labels:
|
|
cluster: 'swarm-k8s'
|
|
environment: 'production'
|
|
|
|
# Alerting configuration
|
|
alerting:
|
|
alertmanagers:
|
|
- static_configs:
|
|
- targets:
|
|
- alertmanager:9093
|
|
|
|
# Scrape configurations
|
|
scrape_configs:
|
|
# Prometheus itself
|
|
- job_name: 'prometheus'
|
|
static_configs:
|
|
- targets: ['localhost:9090']
|
|
|
|
# Orchestrator metrics
|
|
- job_name: 'orchestrator'
|
|
kubernetes_sd_configs:
|
|
- role: pod
|
|
namespaces:
|
|
names:
|
|
- swarm-system
|
|
relabel_configs:
|
|
- source_labels: [__meta_kubernetes_pod_label_app]
|
|
action: keep
|
|
regex: orchestrator
|
|
- source_labels: [__meta_kubernetes_pod_name]
|
|
target_label: pod
|
|
- source_labels: [__meta_kubernetes_namespace]
|
|
target_label: namespace
|
|
- source_labels: [__address__]
|
|
target_label: __address__
|
|
regex: ([^:]+)(?::\d+)?
|
|
replacement: $1:8000
|
|
|
|
# Agent pods metrics
|
|
- job_name: 'agents'
|
|
kubernetes_sd_configs:
|
|
- role: pod
|
|
namespaces:
|
|
names:
|
|
- swarm-system
|
|
relabel_configs:
|
|
- source_labels: [__meta_kubernetes_pod_label_app]
|
|
action: keep
|
|
regex: swarm-agent
|
|
- source_labels: [__meta_kubernetes_pod_name]
|
|
target_label: pod
|
|
- source_labels: [__meta_kubernetes_pod_label_task_id]
|
|
target_label: task_id
|
|
- source_labels: [__meta_kubernetes_pod_label_agent_id]
|
|
target_label: agent_id
|
|
- source_labels: [__meta_kubernetes_namespace]
|
|
target_label: namespace
|
|
- source_labels: [__address__]
|
|
target_label: __address__
|
|
regex: ([^:]+)(?::\d+)?
|
|
replacement: $1:8080
|
|
|
|
# Redis metrics (via redis_exporter)
|
|
- job_name: 'redis'
|
|
static_configs:
|
|
- targets: ['redis-exporter:9121']
|
|
labels:
|
|
service: 'redis'
|
|
|
|
# Kubernetes API server
|
|
- job_name: 'kubernetes-apiservers'
|
|
kubernetes_sd_configs:
|
|
- role: endpoints
|
|
scheme: https
|
|
tls_config:
|
|
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
|
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
|
relabel_configs:
|
|
- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
|
|
action: keep
|
|
regex: default;kubernetes;https
|
|
|
|
# Kubernetes nodes
|
|
- job_name: 'kubernetes-nodes'
|
|
kubernetes_sd_configs:
|
|
- role: node
|
|
scheme: https
|
|
tls_config:
|
|
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
|
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
|
relabel_configs:
|
|
- action: labelmap
|
|
regex: __meta_kubernetes_node_label_(.+)
|
|
|
|
# Kubernetes pods (general)
|
|
- job_name: 'kubernetes-pods'
|
|
kubernetes_sd_configs:
|
|
- role: pod
|
|
relabel_configs:
|
|
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
|
|
action: keep
|
|
regex: true
|
|
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
|
|
action: replace
|
|
target_label: __metrics_path__
|
|
regex: (.+)
|
|
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
|
|
action: replace
|
|
regex: ([^:]+)(?::\d+)?;(\d+)
|
|
replacement: $1:$2
|
|
target_label: __address__
|
|
- action: labelmap
|
|
regex: __meta_kubernetes_pod_label_(.+)
|
|
- source_labels: [__meta_kubernetes_namespace]
|
|
target_label: kubernetes_namespace
|
|
- source_labels: [__meta_kubernetes_pod_name]
|
|
target_label: kubernetes_pod_name
|
|
|
|
# Recording rules for aggregations
|
|
rule_files:
|
|
- /etc/prometheus/rules/*.yml
|
|
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: prometheus-rules
|
|
namespace: swarm-system
|
|
data:
|
|
swarm_rules.yml: |
|
|
groups:
|
|
- name: swarm_metrics
|
|
interval: 30s
|
|
rules:
|
|
# Agent count by status
|
|
- record: swarm:agents:count:by_status
|
|
expr: count by (status) (swarm_agent_status)
|
|
|
|
# Total active agents
|
|
- record: swarm:agents:active:total
|
|
expr: count(swarm_agent_status{status="running"})
|
|
|
|
# Average task duration
|
|
- record: swarm:tasks:duration:avg
|
|
expr: avg(swarm_task_duration_seconds)
|
|
|
|
# Task completion rate (last 5m)
|
|
- record: swarm:tasks:completion_rate:5m
|
|
expr: rate(swarm_tasks_completed_total[5m])
|
|
|
|
# Agent creation rate (last 5m)
|
|
- record: swarm:agents:creation_rate:5m
|
|
expr: rate(swarm_agents_created_total[5m])
|
|
|
|
# Handoff latency p95
|
|
- record: swarm:handoff:latency:p95
|
|
expr: histogram_quantile(0.95, rate(swarm_handoff_duration_seconds_bucket[5m]))
|
|
|
|
# Pod startup time p95
|
|
- record: swarm:pod:startup:p95
|
|
expr: histogram_quantile(0.95, rate(swarm_pod_startup_seconds_bucket[5m]))
|
|
|
|
- name: swarm_alerts
|
|
interval: 30s
|
|
rules:
|
|
# Alert if too many agents are failing
|
|
- alert: HighAgentFailureRate
|
|
expr: rate(swarm_agents_failed_total[5m]) > 0.1
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "High agent failure rate detected"
|
|
description: "Agent failure rate is {{ $value }} failures/sec"
|
|
|
|
# Alert if orchestrator is down
|
|
- alert: OrchestratorDown
|
|
expr: up{job="orchestrator"} == 0
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Orchestrator is down"
|
|
description: "Orchestrator has been down for more than 1 minute"
|
|
|
|
# Alert if Redis is down
|
|
- alert: RedisDown
|
|
expr: up{job="redis"} == 0
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Redis is down"
|
|
description: "Redis has been down for more than 1 minute"
|
|
|
|
# Alert if pod startup is slow
|
|
- alert: SlowPodStartup
|
|
expr: swarm:pod:startup:p95 > 60
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Pod startup time is slow"
|
|
description: "P95 pod startup time is {{ $value }}s (target: <30s)"
|
|
|
|
# Alert if handoff latency is high
|
|
- alert: HighHandoffLatency
|
|
expr: swarm:handoff:latency:p95 > 5
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "High handoff latency detected"
|
|
description: "P95 handoff latency is {{ $value }}s"
|
|
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: prometheus
|
|
namespace: swarm-system
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app: prometheus
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: prometheus
|
|
spec:
|
|
serviceAccountName: prometheus
|
|
containers:
|
|
- name: prometheus
|
|
image: prom/prometheus:v2.45.0
|
|
args:
|
|
- '--config.file=/etc/prometheus/prometheus.yml'
|
|
- '--storage.tsdb.path=/prometheus'
|
|
- '--storage.tsdb.retention.time=30d'
|
|
- '--web.enable-lifecycle'
|
|
ports:
|
|
- containerPort: 9090
|
|
name: web
|
|
volumeMounts:
|
|
- name: config
|
|
mountPath: /etc/prometheus
|
|
- name: rules
|
|
mountPath: /etc/prometheus/rules
|
|
- name: storage
|
|
mountPath: /prometheus
|
|
resources:
|
|
requests:
|
|
memory: "2Gi"
|
|
cpu: "500m"
|
|
limits:
|
|
memory: "4Gi"
|
|
cpu: "2000m"
|
|
volumes:
|
|
- name: config
|
|
configMap:
|
|
name: prometheus-config
|
|
- name: rules
|
|
configMap:
|
|
name: prometheus-rules
|
|
- name: storage
|
|
persistentVolumeClaim:
|
|
claimName: prometheus-storage
|
|
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: prometheus
|
|
namespace: swarm-system
|
|
spec:
|
|
type: ClusterIP
|
|
ports:
|
|
- port: 9090
|
|
targetPort: 9090
|
|
name: web
|
|
selector:
|
|
app: prometheus
|
|
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: prometheus-storage
|
|
namespace: swarm-system
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
resources:
|
|
requests:
|
|
storage: 50Gi
|
|
|
|
---
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: prometheus
|
|
namespace: swarm-system
|
|
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRole
|
|
metadata:
|
|
name: prometheus
|
|
rules:
|
|
- apiGroups: [""]
|
|
resources:
|
|
- nodes
|
|
- nodes/proxy
|
|
- services
|
|
- endpoints
|
|
- pods
|
|
verbs: ["get", "list", "watch"]
|
|
- apiGroups:
|
|
- extensions
|
|
resources:
|
|
- ingresses
|
|
verbs: ["get", "list", "watch"]
|
|
- nonResourceURLs: ["/metrics"]
|
|
verbs: ["get"]
|
|
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRoleBinding
|
|
metadata:
|
|
name: prometheus
|
|
roleRef:
|
|
apiGroup: rbac.authorization.k8s.io
|
|
kind: ClusterRole
|
|
name: prometheus
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: prometheus
|
|
namespace: swarm-system
|