diff --git a/API_DOCUMENTATION.md b/API_DOCUMENTATION.md index f24f03c..4992936 100644 --- a/API_DOCUMENTATION.md +++ b/API_DOCUMENTATION.md @@ -164,7 +164,7 @@ curl http://localhost:8000/agents?template=echo_agent ### 3. 获取 Agent 状态 -获取指定 Agent 的详细状态信息。 +获取指定 Agent 的详细状态信息,包括 Pod 状态、容器健康状态、资源配额等。 **请求** @@ -178,23 +178,168 @@ GET /agents/{agent_name}/status |------|------|------| | agent_name | string | Agent 名称 | -**响应** +**响应 - 健康状态** ```json { "name": "alice-echo", "namespace": "ai-agents", "status": "Running", - "pod_ip": "10.244.2.24", - "node_name": "aks-node-123", + "health_status": "healthy", + "template": "echo_agent", "created_at": "2026-01-05T07:35:00+00:00", - "labels": { - "user-id": "alice", - "template": "echo_agent" - } + "node": "aks-node-123", + "pod_ip": "10.244.2.24", + "containers": [ + { + "name": "alice-echo", + "ready": true, + "restart_count": 0, + "state": "running", + "started_at": "2026-01-05T07:35:15+00:00" + } + ], + "resources": { + "requests": { + "cpu": "100m", + "memory": "128Mi" + }, + "limits": { + "cpu": "500m", + "memory": "512Mi" + }, + "usage": { + "cpu": "14502n", + "memory": "8704Ki", + "available": true + } + }, + "service_port": 8080, + "access_url": "http://10.244.2.24:8080", + "endpoints": { + "root": "http://10.244.2.24:8080/", + "health": "http://10.244.2.24:8080/health" + }, + "conditions": [ + { + "type": "Ready", + "status": "True", + "reason": null + }, + { + "type": "ContainersReady", + "status": "True", + "reason": null + } + ] } ``` +**响应 - 崩溃状态** + +```json +{ + "name": "my-mysql-agent", + "namespace": "ai-agents", + "status": "Waiting", + "health_status": "unhealthy", + "template": "mysql_agent", + "created_at": "2026-01-04T06:40:38+00:00", + "node": "aks-node-123", + "pod_ip": "10.244.1.53", + "containers": [ + { + "name": "my-mysql-agent", + "ready": false, + "restart_count": 599, + "state": "waiting", + "reason": "CrashLoopBackOff", + "message": "back-off 5m0s restarting failed container=my-mysql-agent pod=my-mysql-agent_ai-agents(...)" + } + ], + "resources": { + "requests": { + "cpu": "100m", + "memory": "128Mi" + }, + "limits": { + "cpu": "500m", + "memory": "512Mi" + }, + "usage": { + "cpu": null, + "memory": null, + "available": false, + "reason": "metrics-server未安装或Pod不存在" + } + }, + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "ContainersNotReady" + }, + { + "type": "ContainersReady", + "status": "False", + "reason": "ContainersNotReady" + } + ] +} +``` + +**字段说明** + +| 字段 | 类型 | 说明 | +|------|------|------| +| name | string | Agent 名称 | +| namespace | string | 命名空间 | +| status | string | Pod 状态 (Running/Pending/Waiting/Terminated/Failed) | +| **health_status** | string | **健康状态** (healthy/unhealthy/degraded) | +| template | string | 使用的模板 | +| created_at | string | 创建时间 (ISO 8601) | +| node | string | 运行的节点 | +| pod_ip | string | Pod IP 地址 | +| **containers** | array | **容器详细状态** | +| resources | object | 资源配额和使用情况 | +| service_port | int \| null | 服务端口 | +| access_url | string \| null | 访问地址 | +| endpoints | object \| null | API 端点 | +| conditions | array | Pod 条件状态 | + +**健康状态说明** + +| 状态 | 说明 | +|------|------| +| `healthy` | 所有容器运行正常且就绪 | +| `unhealthy` | 容器崩溃、终止或未就绪 | +| `degraded` | 容器重启次数过多 (>5次) | + +**容器状态字段** + +| 字段 | 类型 | 说明 | +|------|------|------| +| name | string | 容器名称 | +| ready | boolean | 是否就绪 | +| restart_count | int | 重启次数 | +| state | string | 状态 (running/waiting/terminated) | +| reason | string \| null | 状态原因 (如 CrashLoopBackOff) | +| message | string \| null | 详细消息 | +| exit_code | int \| null | 退出码 (terminated 状态) | +| started_at | string \| null | 启动时间 (running 状态) | +| finished_at | string \| null | 结束时间 (terminated 状态) | + +**Pod 状态类型** + +| 状态 | 说明 | +|------|------| +| Running | Pod 正在运行 | +| Pending | Pod 等待调度 | +| Waiting | 容器等待启动 | +| Terminated | 容器已终止 | +| Failed | Pod 失败 | +| Succeeded | Pod 成功完成 | + **状态码** - `200` - 成功 @@ -204,9 +349,20 @@ GET /agents/{agent_name}/status **示例** ```bash +# 查询健康的 Agent curl http://localhost:8000/agents/alice-echo/status + +# 查询崩溃的 Agent +curl http://localhost:8000/agents/my-mysql-agent/status ``` +**使用建议** + +1. **监控告警**:使用 `health_status` 字段而非 `status` 进行健康监控 +2. **故障排查**:检查 `containers` 数组获取容器崩溃原因和重启次数 +3. **自动化运维**:根据 `health_status` 自动触发重启或告警 +4. **日志分析**:结合 `restart_count` 和 `reason` 定位问题 + --- ### 4. 获取 Agent 资源使用情况 diff --git a/HEALTH_CHECK_FIX.md b/HEALTH_CHECK_FIX.md new file mode 100644 index 0000000..0b8643b --- /dev/null +++ b/HEALTH_CHECK_FIX.md @@ -0,0 +1,158 @@ +# 健康检查修复说明 + +## 问题描述 + +之前的健康检查实现存在一个严重问题:即使 agent 的容器已经崩溃(crashed),查询状态时仍然会显示为健康(healthy)。 + +### 根本原因 + +原实现只检查了 Pod 的 `phase`(如 Running、Pending 等),但没有检查容器的实际状态。即使容器崩溃或处于等待/终止状态,Pod 的 phase 可能仍然是 "Running"。 + +## 修复内容 + +### 1. 修改 `k8s_manager.py` 的 `get_pod_status` 方法 + +**主要改进:** +- ✅ 检查容器实际状态(running、waiting、terminated) +- ✅ 检查容器就绪状态(ready) +- ✅ 检查容器重启次数 +- ✅ 新增 `health_status` 字段,返回真实健康状态 + +**健康状态分类:** +- `healthy`: 所有容器运行正常且就绪 +- `unhealthy`: 容器崩溃、终止或未就绪 +- `degraded`: 容器重启次数过多(>5次) + +**新增字段:** +- `health_status`: 真实健康状态 +- `containers`: 容器详细信息数组,包含: + - `name`: 容器名称 + - `ready`: 是否就绪 + - `restart_count`: 重启次数 + - `state`: 当前状态(running/waiting/terminated) + - `reason`: 状态原因(如果有) + - `exit_code`: 退出码(如果已终止) + +### 2. 修改 `k8s_manager_new.py` 的 `get_deployment_status` 方法 + +对于基于 Deployment 的实现,同样增加了对底层 Pod 容器的健康检查。 + +### 3. 更新 `app.py` 的 `PodStatusResponse` 模型 + +添加了新字段以支持响应中的健康状态信息。 + +## 使用方法 + +### 查询 Agent 状态 + +```bash +curl http://localhost:8000/agents/my-mysql-agenty/status +``` + +### 示例响应(健康状态) + +```json +{ + "name": "my-mysql-agenty", + "namespace": "ai-agents", + "status": "Running", + "health_status": "healthy", + "containers": [ + { + "name": "mysql-agent", + "ready": true, + "restart_count": 0, + "state": "running", + "started_at": "2026-01-06T10:00:00Z" + } + ], + ... +} +``` + +### 示例响应(崩溃状态) + +```json +{ + "name": "my-mysql-agenty", + "namespace": "ai-agents", + "status": "Terminated", + "health_status": "unhealthy", + "containers": [ + { + "name": "mysql-agent", + "ready": false, + "restart_count": 3, + "state": "terminated", + "reason": "Error", + "exit_code": 1, + "message": "Connection refused", + "finished_at": "2026-01-06T10:30:00Z" + } + ], + ... +} +``` + +### 示例响应(等待状态) + +```json +{ + "name": "my-mysql-agenty", + "namespace": "ai-agents", + "status": "Waiting", + "health_status": "unhealthy", + "containers": [ + { + "name": "mysql-agent", + "ready": false, + "restart_count": 2, + "state": "waiting", + "reason": "CrashLoopBackOff", + "message": "Back-off restarting failed container" + } + ], + ... +} +``` + +## 测试 + +运行测试脚本验证修复: + +```bash +# 设置环境变量 +export API_URL="http://localhost:8000" +export AGENT_NAME="my-mysql-agenty" + +# 运行测试 +./test_health_check.sh +``` + +## 重启服务 + +修复后需要重启 agent-manager 服务以应用更改: + +```bash +# 如果使用 systemd +sudo systemctl restart agent-manager + +# 或者如果直接运行 +pkill -f "uvicorn.*app:app" +uvicorn app:app --host 0.0.0.0 --port 8000 --reload +``` + +## 注意事项 + +1. **向后兼容性**: + - 原有的 `status` 字段保持不变 + - 新增的 `health_status` 字段不会影响现有客户端 + +2. **建议**: + - 在监控和告警系统中使用 `health_status` 而非 `status` + - 检查 `containers` 数组获取详细的失败原因 + +3. **健康状态判断优先级**: + - 任何容器 unhealthy → 整体 unhealthy + - 任何容器 degraded(且无 unhealthy)→ 整体 degraded + - 所有容器 healthy → 整体 healthy diff --git a/app.py b/app.py index cf74908..1c3ed03 100644 --- a/app.py +++ b/app.py @@ -209,10 +209,12 @@ class PodStatusResponse(BaseModel): name: str namespace: str status: str + health_status: Optional[str] = None # 新增:健康状态 (healthy, unhealthy, degraded) template: Optional[str] = None created_at: Optional[str] = None node: Optional[str] = None pod_ip: Optional[str] = None + containers: Optional[List[Dict]] = None # 新增:容器详细信息 resources: Optional[ResourceInfo] = None service_port: Optional[int] = None access_url: Optional[str] = None diff --git a/k8s_manager.py b/k8s_manager.py index 275b5f4..091a2ba 100644 --- a/k8s_manager.py +++ b/k8s_manager.py @@ -322,14 +322,68 @@ class K8sManager: # 尝试获取实际资源使用情况(需要metrics-server) resource_usage = self._get_pod_resource_usage(pod_name) + # 获取容器实际状态 - 检查是否崩溃或异常 + container_statuses = pod.status.container_statuses or [] + actual_status = pod.status.phase # 默认使用Pod阶段 + health_status = "healthy" + container_info = [] + + for container_status in container_statuses: + container_state = {} + restart_count = container_status.restart_count + + # 检查容器状态 + if container_status.state.running: + container_state = { + "state": "running", + "started_at": container_status.state.running.started_at.isoformat() if container_status.state.running.started_at else None + } + elif container_status.state.waiting: + container_state = { + "state": "waiting", + "reason": container_status.state.waiting.reason, + "message": container_status.state.waiting.message + } + # 容器在等待状态,标记为不健康 + health_status = "unhealthy" + actual_status = "Waiting" + elif container_status.state.terminated: + container_state = { + "state": "terminated", + "reason": container_status.state.terminated.reason, + "exit_code": container_status.state.terminated.exit_code, + "message": container_status.state.terminated.message, + "finished_at": container_status.state.terminated.finished_at.isoformat() if container_status.state.terminated.finished_at else None + } + # 容器已终止,标记为不健康 + health_status = "unhealthy" + actual_status = "Terminated" + + # 检查容器是否就绪 + if not container_status.ready: + health_status = "unhealthy" + + # 如果重启次数过多,也标记为不健康 + if restart_count > 5: + health_status = "degraded" + + container_info.append({ + "name": container_status.name, + "ready": container_status.ready, + "restart_count": restart_count, + **container_state + }) + result = { "name": pod.metadata.name, "namespace": pod.metadata.namespace, - "status": pod.status.phase, + "status": actual_status, + "health_status": health_status, # 新增:真实健康状态 "template": template, "created_at": pod.metadata.creation_timestamp.isoformat() if pod.metadata.creation_timestamp else None, "node": pod.spec.node_name, "pod_ip": pod_ip, + "containers": container_info, # 新增:容器详细信息 "resources": { "requests": resource_requests, "limits": resource_limits, diff --git a/k8s_manager_new.py b/k8s_manager_new.py index 187dbe3..9edd8ac 100644 --- a/k8s_manager_new.py +++ b/k8s_manager_new.py @@ -473,12 +473,71 @@ class K8sManager: deployment_name, self.namespace ) + # 获取 Deployment 对应的 Pods 实际状态 + label_selector = f"app={deployment_name.replace('-deployment', '')}" + pods = self.core_v1.list_namespaced_pod( + self.namespace, + label_selector=label_selector + ) + + # 检查 Pod 的健康状态 + health_status = "healthy" + pod_details = [] + + for pod in pods.items: + pod_health = "healthy" + container_statuses = pod.status.container_statuses or [] + + for container_status in container_statuses: + container_info = { + "name": container_status.name, + "ready": container_status.ready, + "restart_count": container_status.restart_count + } + + # 检查容器状态 + if container_status.state.waiting: + container_info["state"] = "waiting" + container_info["reason"] = container_status.state.waiting.reason + pod_health = "unhealthy" + elif container_status.state.terminated: + container_info["state"] = "terminated" + container_info["reason"] = container_status.state.terminated.reason + container_info["exit_code"] = container_status.state.terminated.exit_code + pod_health = "unhealthy" + elif container_status.state.running: + container_info["state"] = "running" + + # 检查是否就绪 + if not container_status.ready: + pod_health = "unhealthy" + + # 检查重启次数 + if container_status.restart_count > 5: + pod_health = "degraded" + + pod_details.append({ + "name": pod.metadata.name, + "phase": pod.status.phase, + "health": pod_health, + "containers": [container_info] + }) + + # 更新整体健康状态 + if pod_health == "unhealthy": + health_status = "unhealthy" + elif pod_health == "degraded" and health_status != "unhealthy": + health_status = "degraded" + return { "name": deployment_name, "namespace": self.namespace, + "status": "Running" if deployment.status.available_replicas else "Pending", + "health_status": health_status, # 新增:真实健康状态 "replicas": deployment.status.replicas or 0, "ready_replicas": deployment.status.ready_replicas or 0, "available_replicas": deployment.status.available_replicas or 0, + "pods": pod_details, # 新增:Pod详细信息 "conditions": [ { "type": c.type, diff --git a/test_health_check.sh b/test_health_check.sh new file mode 100755 index 0000000..5b743b7 --- /dev/null +++ b/test_health_check.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +# 测试健康检查修复 +# 验证崩溃的 agent 是否能正确识别为 unhealthy + +API_URL="${API_URL:-http://localhost:8000}" +AGENT_NAME="${AGENT_NAME:-my-mysql-agenty}" + +echo "========================================" +echo "测试 Agent 健康状态检查" +echo "========================================" +echo "API URL: $API_URL" +echo "Agent Name: $AGENT_NAME" +echo "" + +# 获取 agent 状态 +echo "📊 获取 Agent 状态..." +response=$(curl -s -w "\nHTTP_STATUS:%{http_code}" "$API_URL/agents/$AGENT_NAME/status") + +http_status=$(echo "$response" | grep "HTTP_STATUS" | cut -d':' -f2) +body=$(echo "$response" | sed '/HTTP_STATUS/d') + +echo "HTTP Status: $http_status" +echo "" + +if [ "$http_status" = "200" ]; then + echo "✅ 成功获取状态" + echo "" + echo "响应内容:" + echo "$body" | python3 -m json.tool 2>/dev/null || echo "$body" + echo "" + + # 提取健康状态 + health_status=$(echo "$body" | python3 -c "import sys, json; data=json.load(sys.stdin); print(data.get('health_status', 'N/A'))" 2>/dev/null || echo "无法解析") + status=$(echo "$body" | python3 -c "import sys, json; data=json.load(sys.stdin); print(data.get('status', 'N/A'))" 2>/dev/null || echo "无法解析") + + echo "========================================" + echo "状态摘要:" + echo " Pod Status: $status" + echo " Health Status: $health_status" + echo "========================================" + echo "" + + if [ "$health_status" = "unhealthy" ]; then + echo "✅ 正确识别为不健康状态 (unhealthy)" + + # 显示容器信息 + echo "" + echo "📦 容器详情:" + echo "$body" | python3 -c " +import sys, json +data = json.load(sys.stdin) +containers = data.get('containers', []) +for c in containers: + print(f\" 容器: {c.get('name', 'N/A')}\") + print(f\" 状态: {c.get('state', 'N/A')}\") + print(f\" 就绪: {c.get('ready', 'N/A')}\") + print(f\" 重启次数: {c.get('restart_count', 0)}\") + if c.get('reason'): + print(f\" 原因: {c.get('reason')}\") + if c.get('exit_code') is not None: + print(f\" 退出码: {c.get('exit_code')}\") + print() +" 2>/dev/null || echo " 无法解析容器信息" + elif [ "$health_status" = "healthy" ]; then + echo "⚠️ 仍然显示为健康状态 (healthy) - 可能是 agent 确实在运行" + echo " 请检查容器详细信息确认" + elif [ "$health_status" = "degraded" ]; then + echo "⚠️ 显示为降级状态 (degraded) - 容器可能频繁重启" + else + echo "⚠️ 未知健康状态: $health_status" + fi + +elif [ "$http_status" = "404" ]; then + echo "❌ Agent 不存在" + echo "$body" +else + echo "❌ 获取状态失败" + echo "$body" +fi + +echo "" +echo "========================================" +echo "测试完成" +echo "========================================"