"""
Kubernetes管理模块 - 负责与Kubernetes集群交互
"""
from kubernetes import client, config
from kubernetes.client.rest import ApiException
from typing import Dict, List, Optional
import logging
import os
import re
import requests
import time
import subprocess
import tempfile
import base64

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)


def sanitize_k8s_name(name: str, max_length: int = 63) -> str:
    """将名称转换为 DNS-1035 合规格式
    
    Kubernetes Service、Deployment 等资源名称必须符合 DNS-1035 标准：
    - 只能包含小写字母、数字和连字符 '-'
    - 必须以字母开头
    - 必须以字母或数字结尾
    - 最长 63 个字符
    
    Args:
        name: 原始名称
        max_length: 最大长度（默认 63）
        
    Returns:
        合规的 K8s 资源名称
    """
    # 转小写
    name = name.lower()
    # 将下划线和空格替换为连字符
    name = name.replace("_", "-").replace(" ", "-")
    # 移除非字母、数字、连字符的字符
    name = re.sub(r'[^a-z0-9-]', '', name)
    # 合并连续的连字符
    name = re.sub(r'-+', '-', name)
    # 如果以数字开头，添加 'a' 前缀
    if name and name[0].isdigit():
        name = 'a' + name
    # 如果以连字符开头，去掉
    name = name.lstrip('-')
    # 截断到最大长度
    name = name[:max_length]
    # 去掉末尾的连字符
    name = name.rstrip('-')
    # 最终兜底：如果名称为空
    if not name:
        name = 'agent'
    return name


class K8sManager:
    """Kubernetes资源管理器"""
    
    # Azure DNS 配置（从环境变量读取）
    AZURE_TENANT_ID = os.getenv("AZURE_TENANT_ID", "263c3ff6-1be5-4141-8308-b188464fb297")
    AZURE_CLIENT_ID = os.getenv("AZURE_CLIENT_ID", "c5ba26db-f180-425f-bac3-93708d853988")
    AZURE_CLIENT_SECRET = os.getenv("AZURE_CLIENT_SECRET", "ydt8Q~DIOfKXDUOJkzfgaNVNh2d1VrbeIaYuKbpD")
    AZURE_SUBSCRIPTION_ID = os.getenv("AZURE_SUBSCRIPTION_ID", "c6c47e4c-f5f4-49f8-b26f-7728862c17d6")
    AZURE_RESOURCE_GROUP = os.getenv("AZURE_RESOURCE_GROUP", "taiji-ai-v0")
    AZURE_DNS_ZONE = os.getenv("AZURE_DNS_ZONE", "taijiagnet.com")
    
    def __init__(self, namespace: str = "ai-agents", kubeconfig_path: str = None):
        """
        初始化Kubernetes管理器
        
        Args:
            namespace: AI Agent部署的命名空间
            kubeconfig_path: kubeconfig文件路径（可选）
        """
        self.namespace = namespace
        self._load_kube_config(kubeconfig_path)
        
        self.v1 = client.CoreV1Api()
        self.apps_v1 = client.AppsV1Api()
        self.networking_v1 = client.NetworkingV1Api()
        
        # 确保命名空间存在
        self._ensure_namespace()
    
    def _load_kube_config(self, kubeconfig_path: str = None):
        """
        加载Kubernetes配置
        优先级：
        1. 集群内ServiceAccount（推荐用于生产环境）
        2. 指定的kubeconfig文件路径
        3. 默认的kubeconfig路径 (~/.kube/config)
        """
        try:
            # 方式1: 尝试加载集群内配置（当服务运行在K8s中且有ServiceAccount时）
            config.load_incluster_config()
            logger.info("✅ 使用集群内ServiceAccount配置")
        except config.ConfigException:
            try:
                if kubeconfig_path and os.path.exists(kubeconfig_path):
                    # 方式2: 使用指定的kubeconfig文件
                    config.load_kube_config(config_file=kubeconfig_path)
                    logger.info(f"✅ 使用指定的kubeconfig: {kubeconfig_path}")
                else:
                    # 方式3: 使用默认kubeconfig（开发环境）
                    config.load_kube_config()
                    logger.info("✅ 使用默认kubeconfig (~/.kube/config)")
            except Exception as e:
                logger.error(f"❌ 无法加载Kubernetes配置: {e}")
                raise Exception(f"Kubernetes配置加载失败: {e}")
    
    def _ensure_namespace(self):
        """确保AI Agent命名空间存在"""
        try:
            self.v1.read_namespace(name=self.namespace)
            logger.info(f"命名空间 {self.namespace} 已存在")
        except ApiException as e:
            if e.status == 404:
                # 创建命名空间
                namespace_manifest = client.V1Namespace(
                    metadata=client.V1ObjectMeta(name=self.namespace)
                )
                self.v1.create_namespace(body=namespace_manifest)
                logger.info(f"创建命名空间 {self.namespace}")
            else:
                raise
    
    def create_agent_namespace(self, agent_name: str, owner_id: str = None) -> str:
        """为 Agent 创建独立的命名空间
        
        Args:
            agent_name: Agent 名称
            owner_id: 所有者 ID
            
        Returns:
            创建的命名空间名称
        """
        # 生成命名空间名称（使用 agent-{sanitized_name} 格式，确保 DNS 合规）
        sanitized_name = sanitize_k8s_name(agent_name)
        namespace_name = f"agent-{sanitized_name}"[:63].rstrip('-')
        
        try:
            # 检查命名空间是否已存在
            self.v1.read_namespace(name=namespace_name)
            logger.info(f"命名空间 {namespace_name} 已存在")
            return namespace_name
        except ApiException as e:
            if e.status == 404:
                # 创建命名空间
                labels = {
                    "managed-by": "agent-manager",
                    "agent-name": agent_name
                }
                if owner_id:
                    labels["owner-id"] = owner_id
                
                namespace_manifest = client.V1Namespace(
                    metadata=client.V1ObjectMeta(
                        name=namespace_name,
                        labels=labels
                    )
                )
                self.v1.create_namespace(body=namespace_manifest)
                logger.info(f"✅ 创建命名空间 {namespace_name}")
                
                # 复制 ACR secret 到新命名空间
                self._copy_acr_secret_to_namespace(namespace_name)
                
                return namespace_name
            else:
                raise
    
    def _copy_acr_secret_to_namespace(self, target_namespace: str):
        """复制所有 ACR secrets 到目标命名空间
        
        Args:
            target_namespace: 目标命名空间
        """
        # 需要复制的 ACR secrets 列表
        acr_secrets = ["acr-secret", "openclaw-acr-secret"]
        
        for secret_name in acr_secrets:
            try:
                # 从 agent-manager 命名空间读取 secret
                source_secret = self.v1.read_namespaced_secret(
                    name=secret_name,
                    namespace="agent-manager"
                )
                
                # 创建新的 secret（去除自动生成的字段）
                new_secret = client.V1Secret(
                    metadata=client.V1ObjectMeta(
                        name=secret_name,
                        namespace=target_namespace
                    ),
                    data=source_secret.data,
                    type=source_secret.type
                )
                
                # 在目标命名空间创建 secret
                self.v1.create_namespaced_secret(
                    namespace=target_namespace,
                    body=new_secret
                )
                logger.info(f"✅ 已复制 {secret_name} 到命名空间 {target_namespace}")
            except ApiException as e:
                if e.status == 404:
                    logger.warning(f"⚠️ 源 {secret_name} 不存在，跳过复制")
                elif e.status == 409:
                    logger.info(f"{secret_name} 已存在于命名空间 {target_namespace}")
                else:
                    logger.error(f"❌ 复制 {secret_name} 失败: {e}")
    
    def create_service(self, service_name: str, namespace: str, pod_selector: Dict[str, str], 
                      service_port: int, target_port: int, service_type: str = "LoadBalancer") -> Dict:
        """为 Agent Pod 创建 Service
        
        Args:
            service_name: Service 名称
            namespace: 命名空间
            pod_selector: Pod 选择器标签
            service_port: Service 端口
            target_port: Pod 目标端口
            service_type: Service 类型，可选值: "LoadBalancer", "ClusterIP" (默认: "LoadBalancer")
            
        Returns:
            创建的 Service 信息（包含外网 IP，如果是 LoadBalancer）
        """
        try:
            service_manifest = client.V1Service(
                metadata=client.V1ObjectMeta(
                    name=service_name,
                    namespace=namespace,
                    labels={"managed-by": "agent-manager"}
                ),
                spec=client.V1ServiceSpec(
                    selector=pod_selector,
                    ports=[
                        client.V1ServicePort(
                            name="http",
                            protocol="TCP",
                            port=service_port,
                            target_port=target_port
                        )
                    ],
                    type=service_type
                )
            )
            
            response = self.v1.create_namespaced_service(
                namespace=namespace,
                body=service_manifest
            )
            
            logger.info(f"✅ {service_type} Service {service_name} 创建成功 (namespace: {namespace})")
            
            # 获取外网 IP（仅 LoadBalancer 类型）
            external_ip = None
            if service_type == "LoadBalancer" and response.status.load_balancer.ingress:
                lb_ingress = response.status.load_balancer.ingress[0]
                external_ip = lb_ingress.ip or lb_ingress.hostname
            
            return {
                "name": response.metadata.name,
                "namespace": response.metadata.namespace,
                "cluster_ip": response.spec.cluster_ip,
                "port": service_port,
                "external_ip": external_ip,
                "type": service_type
            }
        except ApiException as e:
            logger.error(f"创建 Service 失败: {e}")
            raise Exception(f"创建 Service 失败: {e.reason}")
    
    def create_self_signed_cert(self, domain: str, namespace: str, secret_name: str, days_valid: int = 365) -> Dict:
        """为指定域名生成自签名证书并创建 Kubernetes Secret
        
        Args:
            domain: 域名
            namespace: Kubernetes 命名空间
            secret_name: Secret 名称
            days_valid: 证书有效期（天）
            
        Returns:
            证书信息
        """
        try:
            logger.info(f"🔐 开始生成自签名证书: {domain} (Secret: {secret_name})")
            
            # 创建临时目录
            with tempfile.TemporaryDirectory() as temp_dir:
                key_file = os.path.join(temp_dir, "tls.key")
                cert_file = os.path.join(temp_dir, "tls.crt")
                csr_file = os.path.join(temp_dir, "tls.csr")
                
                # 1. 生成私钥
                subprocess.run(
                    ["openssl", "genrsa", "-out", key_file, "2048"],
                    check=True,
                    capture_output=True
                )
                
                # 2. 生成证书签名请求
                subprocess.run(
                    ["openssl", "req", "-new", "-key", key_file, "-out", csr_file,
                     "-subj", f"/C=CN/ST=Beijing/L=Beijing/O=OpenClaw/CN={domain}"],
                    check=True,
                    capture_output=True
                )
                
                # 3. 创建扩展配置文件
                ext_file = os.path.join(temp_dir, "ext.conf")
                with open(ext_file, "w") as f:
                    f.write(f"""[req]
distinguished_name = req_distinguished_name
req_extensions = v3_req

[v3_req]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names

[alt_names]
DNS.1 = {domain}
DNS.2 = *.{domain.split('.', 1)[-1] if '.' in domain else domain}
DNS.3 = localhost
IP.1 = 127.0.0.1
""")
                
                # 4. 生成自签名证书
                subprocess.run(
                    ["openssl", "x509", "-req", "-days", str(days_valid),
                     "-in", csr_file, "-signkey", key_file, "-out", cert_file,
                     "-extensions", "v3_req", "-extfile", ext_file],
                    check=True,
                    capture_output=True
                )
                
                # 5. 读取证书和私钥
                with open(cert_file, "rb") as f:
                    cert_data = f.read()
                with open(key_file, "rb") as f:
                    key_data = f.read()
                
                # 6. 创建 Kubernetes Secret
                secret = client.V1Secret(
                    metadata=client.V1ObjectMeta(
                        name=secret_name,
                        namespace=namespace,
                        labels={"managed-by": "agent-manager", "app": "openclaw"}
                    ),
                    type="kubernetes.io/tls",
                    data={
                        "tls.crt": base64.b64encode(cert_data).decode("utf-8"),
                        "tls.key": base64.b64encode(key_data).decode("utf-8")
                    }
                )
                
                try:
                    self.v1.create_namespaced_secret(namespace=namespace, body=secret)
                    logger.info(f"✅ 证书 Secret 创建成功: {secret_name}")
                except ApiException as e:
                    if e.status == 409:
                        logger.info(f"证书 Secret {secret_name} 已存在，尝试更新")
                        self.v1.replace_namespaced_secret(
                            name=secret_name,
                            namespace=namespace,
                            body=secret
                        )
                    else:
                        raise
                
                return {
                    "secret_name": secret_name,
                    "domain": domain,
                    "namespace": namespace,
                    "days_valid": days_valid
                }
                
        except subprocess.CalledProcessError as e:
            error_msg = e.stderr.decode() if e.stderr else str(e)
            logger.error(f"生成证书失败: {error_msg}")
            raise Exception(f"生成自签名证书失败: {error_msg}")
        except Exception as e:
            logger.error(f"创建证书 Secret 失败: {e}")
            raise Exception(f"创建证书 Secret 失败: {str(e)}")
    
    def create_ingress(self, ingress_name: str, namespace: str, service_name: str, 
                      service_port: int, host: str = None, path: str = "/", 
                      tls_secret_name: str = None) -> Dict:
        """为 Agent Service 创建 Ingress
        
        Args:
            ingress_name: Ingress 名称
            namespace: 命名空间
            service_name: 后端 Service 名称
            service_port: Service 端口
            host: 域名（可选，如果为 None 则使用默认）
            path: 路径前缀
            tls_secret_name: TLS 证书 Secret 名称（可选，如果提供则启用 HTTPS）
            
        Returns:
            创建的 Ingress 信息
        """
        try:
            # 构建路径规则
            http_ingress_path = client.V1HTTPIngressPath(
                path=path,
                path_type="Prefix",
                backend=client.V1IngressBackend(
                    service=client.V1IngressServiceBackend(
                        name=service_name,
                        port=client.V1ServiceBackendPort(number=service_port)
                    )
                )
            )
            
            # 构建规则列表
            ingress_rules = []
            
            # 如果指定了 host，创建带 host 的规则（用于 DNS 访问）
            if host:
                ingress_rule_with_host = client.V1IngressRule(
                    host=host,
                    http=client.V1HTTPIngressRuleValue(paths=[http_ingress_path])
                )
                ingress_rules.append(ingress_rule_with_host)
            
            # 注意：不再添加不带 host 的规则，避免多个 Ingress 之间的路径冲突
            # 每个 agent 应该通过自己的域名访问，而不是通过 IP + 路径
            # 如果需要通过 IP 访问，应该使用不同的路径前缀，但这会导致 nginx ingress 验证失败
            
            # 构建 Ingress 注解
            annotations = {
                "nginx.ingress.kubernetes.io/rewrite-target": "/",
            }
            
            # 如果启用了 TLS，添加相关注解
            if tls_secret_name:
                annotations["nginx.ingress.kubernetes.io/ssl-redirect"] = "true"
                annotations["nginx.ingress.kubernetes.io/proxy-read-timeout"] = "3600"
                annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = "3600"
                annotations["nginx.ingress.kubernetes.io/websocket-services"] = service_name
            else:
                annotations["nginx.ingress.kubernetes.io/ssl-redirect"] = "false"
            
            # 构建 TLS 配置
            tls_config = None
            if tls_secret_name and host:
                tls_config = [client.V1IngressTLS(
                    hosts=[host],
                    secret_name=tls_secret_name
                )]
            
            # 创建 Ingress manifest
            ingress_spec = client.V1IngressSpec(
                ingress_class_name="nginx",  # 使用 nginx ingress controller
                rules=ingress_rules
            )
            
            if tls_config:
                ingress_spec.tls = tls_config
            
            ingress_manifest = client.V1Ingress(
                metadata=client.V1ObjectMeta(
                    name=ingress_name,
                    namespace=namespace,
                    labels={"managed-by": "agent-manager"},
                    annotations=annotations
                ),
                spec=ingress_spec
            )
            
            response = self.networking_v1.create_namespaced_ingress(
                namespace=namespace,
                body=ingress_manifest
            )
            
            logger.info(f"✅ Ingress {ingress_name} 创建成功 (namespace: {namespace})")
            
            # 获取 Ingress IP/域名
            ingress_url = None
            protocol = "https" if tls_secret_name else "http"
            if response.status.load_balancer.ingress:
                lb_ingress = response.status.load_balancer.ingress[0]
                if host:
                    ingress_url = f"{protocol}://{host}{path}"
                elif lb_ingress.ip:
                    ingress_url = f"{protocol}://{lb_ingress.ip}{path}"
                elif lb_ingress.hostname:
                    ingress_url = f"{protocol}://{lb_ingress.hostname}{path}"
            
            return {
                "name": response.metadata.name,
                "namespace": response.metadata.namespace,
                "host": host,
                "path": path,
                "url": ingress_url,
                "tls_enabled": tls_secret_name is not None,
                "note": "Ingress URL 将在负载均衡器配置完成后可用"
            }
        except ApiException as e:
            logger.error(f"创建 Ingress 失败: {e}")
            raise Exception(f"创建 Ingress 失败: {e.reason}")
    
    # 模板端口映射
    TEMPLATE_PORTS = {
        "echo_agent": 8000,
        "search_agent": 8080,
        "search_agent_a2a": 8080,
        "search_agent_mcp": 8080,
        "mysql_agent": 8000,
        "postgresql_agent": 8000,
        "jina_search_agent": 8000,
        "azure_blob_agent": 8000,
        "azure_blob_agent_mcp": 8000,
        "azure_blob_agent_a2a": 8000,
        "a2a_litellm_agent": 8000,
        "coding_a2a_agent": 8000,
        "code_ai_agent": 8000,
        "facebook_agent": 8000,
        "media_downloader": 8000,
        "content_analyzer": 8000,
        "huoke": 8000,
        "microsoft_learn_agent": 8000,
        "aws_docs_mcp": 8000,
        "google_mcp": 8000,
        # OpenClaw（沙箱镜像 openclaw-sandbox 和 openclaw-sandbox-browser 会在部署时自动启用）
        "openclaw": 18789,
    }
    
    # OpenClaw 模板（沙箱镜像不作为独立模板）
    OPENCLAW_TEMPLATES = ["openclaw"]
    
    # 模板所需环境变量说明
    TEMPLATE_ENV_INFO = {
        "jina_search_agent": {
            "required": {
                "JINA_API_KEY": "Jina API密钥，从 https://jina.ai/ 获取"
            },
            "optional": {
                "SERVICE_PORT": "HTTP服务端口，默认8000",
                "SERVICE_HOST": "HTTP服务监听地址，默认0.0.0.0"
            }
        },
        "mysql_agent": {
            "required": {
                "MYSQL_HOST": "MySQL数据库主机地址",
                "MYSQL_USER": "MySQL用户名",
                "MYSQL_PASSWORD": "MySQL密码",
                "MYSQL_DATABASE": "MySQL数据库名",
                "OPENAI_API_KEY": "OpenAI API密钥"
            },
            "optional": {
                "MYSQL_PORT": "MySQL端口，默认3306"
            }
        },
        "postgresql_agent": {
            "required": {
                "POSTGRES_HOST": "PostgreSQL数据库主机地址",
                "POSTGRES_USER": "PostgreSQL用户名",
                "POSTGRES_PASSWORD": "PostgreSQL密码",
                "POSTGRES_DATABASE": "PostgreSQL数据库名",
                "OPENAI_API_KEY": "OpenAI API密钥"
            },
            "optional": {
                "POSTGRES_PORT": "PostgreSQL端口，默认5432"
            }
        },
        "azure_blob_agent": {
            "required": {
                "LITELLM_API_BASE": "LiteLLM服务地址，如 http://litellm-service:4000",
                "LITELLM_MODEL": "使用的LLM模型，如 gpt-3.5-turbo",
                "LITELLM_API_KEY": "LiteLLM API密钥"
            },
            "optional": {
                "AZURE_STORAGE_CONNECTION_STRING": "Azure Storage连接字符串（可选，也可通过 /connect API 动态传入）",
                "SERVICE_PORT": "HTTP服务端口，默认8000",
                "SERVICE_HOST": "HTTP服务监听地址，默认0.0.0.0"
            }
        },
        "azure_blob_agent_mcp": {
            "required": {
                "MODEL_PROVIDER": "模型提供商，如 openai, azure-openai",
                "MODEL_NAME": "使用的模型名称，如 gpt-4",
                "MODEL_API_KEY": "模型 API 密钥"
            },
            "optional": {
                "AZURE_STORAGE_CONNECTION_STRING": "Azure Storage连接字符串",
                "TOOLS_CONFIG": "工具配置 JSON",
                "TOOL_ENDPOINT": "外部工具端点",
                "TOOL_API_KEY": "工具 API 密钥",
                "MODEL_ENDPOINT": "模型 API 端点",
                "STORAGE_ACCOUNT_NAME": "存储账户名称",
                "USER_ID": "用户标识",
                "TENANT_ID": "租户标识",
                "NAMESPACE": "Kubernetes 命名空间"
            }
        },
        "azure_blob_agent_a2a": {
            "required": {
                "MODEL_PROVIDER": "模型提供商，如 openai, azure-openai",
                "MODEL_NAME": "使用的模型名称，如 gpt-4",
                "MODEL_API_KEY": "模型 API 密钥",
                "AGENT_ID": "Agent 唯一标识",
                "AGENT_ROLE": "Agent 角色"
            },
            "optional": {
                "AZURE_STORAGE_CONNECTION_STRING": "Azure Storage连接字符串",
                "TOOLS_CONFIG": "工具配置 JSON",
                "TOOL_ENDPOINT": "外部工具端点",
                "TOOL_API_KEY": "工具 API 密钥",
                "MODEL_ENDPOINT": "模型 API 端点",
                "STORAGE_ACCOUNT_NAME": "存储账户名称",
                "AGENT_CAPABILITIES": "Agent 能力列表 JSON",
                "USER_ID": "用户标识",
                "TENANT_ID": "租户标识",
                "NAMESPACE": "Kubernetes 命名空间"
            }
        },
        "search_agent": {
            "required": {
                "LLM_BASE_URL": "LLM 服务地址，如 https://api.openai.com/v1",
                "SERPER_API_KEY": "Serper 搜索 API 密钥，从 https://serper.dev 获取",
                "JINA_API_KEY": "Jina Reader API 密钥，从 https://jina.ai 获取"
            },
            "optional": {
                "LLM_API_KEY": "LLM API 密钥（可在搜索请求中传入）",
                "MODEL_NAME": "LLM 模型名称，默认 gpt-4o-mini",
                "MAX_ITERATIONS": "最大搜索迭代次数，默认 3",
                "MAX_RESULTS_PER_QUERY": "每次搜索最大结果数，默认 10",
                "CONTENT_MAX_LENGTH": "内容最大长度，默认 5000",
                "TIMEOUT": "超时时间（秒），默认 30",
                "SERVICE_PORT": "HTTP服务端口，默认 8080",
                "SERVICE_HOST": "HTTP服务监听地址，默认 0.0.0.0"
            }
        },
        "search_agent_a2a": {
            "required": {
                "LLM_BASE_URL": "LLM 服务地址，如 https://api.openai.com/v1",
            },
            "optional": {
                "LITELLM_API_KEY": "LiteLLM API 密钥（可在请求中传入）",
                "LLM_API_KEY": "LLM API 密钥（备选，可在请求中传入）",
                "MODEL_NAME": "LLM 模型名称（优先）",
                "LLM_MODEL": "LLM 模型名称（备选）",
                "LITELLM_MODEL": "LiteLLM 模型名称（备选）",
                "SERPER_API_KEY": "Serper 搜索 API 密钥（已内置默认值）",
                "JINA_API_KEY": "Jina Reader API 密钥（已内置默认值）",
                "SERVICE_PORT": "HTTP服务端口，默认 8080",
                "SERVICE_HOST": "HTTP服务监听地址，默认 0.0.0.0"
            },
            "description": "A2A 协议搜索 Agent，支持通过请求动态传入 API key"
        },
        "search_agent_mcp": {
            "required": {
                "LLM_BASE_URL": "LLM 服务地址，如 https://api.openai.com/v1",
            },
            "optional": {
                "LITELLM_API_KEY": "LiteLLM API 密钥（可在请求中传入）",
                "LLM_API_KEY": "LLM API 密钥（备选，可在请求中传入）",
                "MODEL_NAME": "LLM 模型名称（优先）",
                "LLM_MODEL": "LLM 模型名称（备选）",
                "LITELLM_MODEL": "LiteLLM 模型名称（备选）",
                "SERPER_API_KEY": "Serper 搜索 API 密钥（已内置默认值）",
                "JINA_API_KEY": "Jina Reader API 密钥（已内置默认值）",
                "SERVICE_PORT": "HTTP服务端口，默认 8080",
                "SERVICE_HOST": "HTTP服务监听地址，默认 0.0.0.0"
            },
            "description": "MCP 协议搜索 Agent，支持通过请求动态传入 API key"
        },
        "a2a_litellm_agent": {
            "required": {
                "LITELLM_API_BASE": "LiteLLM 服务地址",
                "LITELLM_MODEL": "LiteLLM 模型名称"
            },
            "optional": {
                "LITELLM_API_KEY": "LiteLLM API 密钥（可在请求中传入）",
                "AGENT_NAME": "Agent 名称",
                "AGENT_DESCRIPTION": "Agent 描述",
                "SERVICE_PORT": "HTTP服务端口，默认 8000",
                "SERVICE_HOST": "HTTP服务监听地址，默认 0.0.0.0"
            }
        },
        "coding_a2a_agent": {
            "required": {
                "OPENAI_BASE_URL": "LiteLLM / OpenAI 兼容网关地址，如 https://gateway.example.com/v1"
            },
            "optional": {
                "OPENAI_API_KEY": "模型 API Key",
                "LITELLM_API_KEY": "模型 API Key（兼容变量）",
                "MODEL_NAME": "模型名称",
                "LITELLM_MODEL": "模型名称（兼容变量）",
                "WORK_DIR": "默认工作区目录，默认 /workspace",
                "AGENT_ROLE_NAME": "启动时指定角色名称，例如 backend / reviewer / planner",
                "AGENT_INSTRUCTION_TEXT": "启动时注入的角色/行为说明文本，支持类似 AGENTS.md / claude.md 内容",
                "AGENT_INSTRUCTION_FILE": "启动时读取的角色说明文件路径，内容会并入系统提示词",
                "SERVICE_PORT": "HTTP服务端口，默认 8000",
                "SERVICE_HOST": "HTTP服务监听地址，默认 0.0.0.0"
            },
            "description": "Claude Code 风格编程 Agent，使用 Pydantic AI 作为核心并通过 A2A 协议对外提供服务"
        },
        "code_ai_agent": {
            "required": {
                "LLM_BASE_URL": "LLM 服务地址，如 https://api.openai.com/v1",
            },
            "optional": {
                "LLM_API_KEY": "LLM API 密钥（可在请求中传入）",
                "MODEL_NAME": "LLM 模型名称",
                "API_HOST": "API 服务监听地址，默认 0.0.0.0",
                "API_PORT": "API 服务端口，默认 8000",
                "PROJECTS_DIR": "项目存储目录，默认 /tmp/projects"
            },
            "description": "代码助手 Agent，支持代码生成、分析和执行"
        },
        "facebook_agent": {
            "required": {
                "LITELLM_GATEWAY_URL": "LiteLLM Gateway 服务地址，如 https://litellm.graystone-fb459c5d.southeastasia.azurecontainerapps.io/",
                "FACEBOOK_API_KEY": "Facebook RapidAPI 密钥"
            },
            "optional": {
                "LITELLM_API_KEY": "LiteLLM API 密钥（可在请求中传入）",
                "LITELLM_MODEL": "LiteLLM 模型名称，默认 taiji/gpt-4o-mini",
                "FACEBOOK_API_HOST": "Facebook API 主机，默认 facebook-scraper3.p.rapidapi.com",
                "FACEBOOK_MCP_URL": "Facebook MCP URL，默认 https://mcp.rapidapi.com",
                "MAX_RESULTS": "最大搜索结果数，默认 10",
                "TIMEOUT": "超时时间（秒），默认 30",
                "LOG_LEVEL": "日志级别，默认 INFO"
            },
            "description": "Facebook 搜索智能 Agent，支持 Facebook 内容搜索和 MCP 协议"
        },
        # OpenClaw（部署时自动启用沙箱：openclaw-sandbox 和 openclaw-sandbox-browser）
        "openclaw": {
            "required": {},
            "optional": {
                "LITELLM_API_KEY": "LiteLLM API 密钥",
                "LITELLM_API_BASE": "LiteLLM API 地址",
                "LITELLM_MODEL": "LiteLLM 模型名称，默认 taiji/gemini-2.5-flash",
                "TELEGRAM_BOT_TOKEN": "Telegram Bot Token",
                "FEISHU_APP_ID": "飞书 App ID",
                "FEISHU_APP_SECRET": "飞书 App Secret",
                "DISCORD_BOT_TOKEN": "Discord Bot Token",
                "SLACK_BOT_TOKEN": "Slack Bot Token",
                "SLACK_APP_TOKEN": "Slack App Token",
                "GATEWAY_AUTH_TOKEN": "Gateway 认证 Token（自动生成）",
                "OPENCLAW_GATEWAY_TOKEN": "OpenClaw Gateway Token（自动生成）"
            },
            "description": "OpenClaw 个人 AI 助手，支持多渠道消息、语音唤醒、沙箱代码执行（自动启用 openclaw-sandbox 和 openclaw-sandbox-browser）"
        }
    }
    
    def get_template_info(self, template: str) -> Dict:
        """
        获取模板信息（从数据库动态获取）
        
        Args:
            template: 模板类型
        
        Returns:
            模板信息（端口、所需环境变量等）
        """
        from template_manager import template_manager as tm
        template_data = tm.get_template(template)
        
        if template_data:
            return {
                "template": template,
                "name": template_data.get("name"),
                "display_name": template_data.get("display_name"),
                "description": template_data.get("description"),
                "port": template_data.get("port", 8000),
                "image": template_data.get("image"),
                "agent_framework": template_data.get("agent_framework"),
                "env_requirements": template_data.get("env_requirements", {}),
                # 回退到硬编码的环境变量信息（兼容旧模板）
                "env_info": self.TEMPLATE_ENV_INFO.get(template, template_data.get("env_requirements", {}))
            }
        
        # 回退到硬编码配置（兼容性）
        return {
            "template": template,
            "port": self.TEMPLATE_PORTS.get(template),
            "env_info": self.TEMPLATE_ENV_INFO.get(template, {})
        }
    
    def create_pod(self, pod_name: str, template: str, config_data: Dict) -> Dict:
        """
        创建Pod
        
        Args:
            pod_name: Pod名称
            template: 模板类型（echo_agent, chat_agent等）
            config_data: 配置信息（replicas, resources等）
        
        Returns:
            创建的Pod信息，包含访问地址
        """
        # OpenClaw 模板使用专用部署方法（自动启用沙箱：openclaw-sandbox 和 openclaw-sandbox-browser）
        if template == "openclaw":
            return self.create_openclaw_deployment(pod_name, template, config_data)
        
        try:
            # 生成Pod规格
            pod_manifest = self._generate_pod_manifest(pod_name, template, config_data)
            
            # 创建Pod
            response = self.v1.create_namespaced_pod(
                namespace=self.namespace,
                body=pod_manifest
            )
            
            logger.info(f"Pod {pod_name} 创建成功")
            
            # 获取服务端口（从数据库动态获取）
            from template_manager import template_manager as tm
            service_port = tm.get_port(template)
            
            result = {
                "name": response.metadata.name,
                "namespace": response.metadata.namespace,
                "status": response.status.phase,
                "created_at": response.metadata.creation_timestamp.isoformat() if response.metadata.creation_timestamp else None,
                "template": template
            }
            
            # 如果是HTTP服务类型的agent，添加访问信息
            if service_port:
                result["service_port"] = service_port
                result["access_info"] = {
                    "note": "Pod IP将在Pod运行后可用，请通过 /agents/{name}/status 获取",
                    "port": service_port,
                    "endpoints": {
                        "root": f"http://<pod_ip>:{service_port}/",
                        "health": f"http://<pod_ip>:{service_port}/health"
                    }
                }
            
            return result
        except ApiException as e:
            logger.error(f"创建Pod失败: {e}")
            raise Exception(f"创建Pod失败: {e.reason}")
    
    def _generate_pod_manifest(self, pod_name: str, template: str, config_data: Dict) -> client.V1Pod:
        """生成Pod配置清单"""
        
        # 默认资源配置
        replicas = config_data.get("replicas", 1)
        cpu_request = config_data.get("cpu_request", "100m")
        cpu_limit = config_data.get("cpu_limit", "500m")
        memory_request = config_data.get("memory_request", "128Mi")
        memory_limit = config_data.get("memory_limit", "512Mi")
        
        # 从数据库动态获取模板镜像（延迟导入避免循环依赖）
        from template_manager import template_manager as tm
        image = tm.get_image(template)
        
        # 构建环境变量列表
        env_vars = [
            client.V1EnvVar(name="POD_NAME", value=pod_name),
            client.V1EnvVar(name="TEMPLATE_TYPE", value=template)
        ]
        
        # NEW: 添加 Agent 框架配置
        agent_framework = config_data.get("agent_framework", "langchain")
        env_vars.append(client.V1EnvVar(name="AGENT_FRAMEWORK", value=agent_framework))
        
        # NEW: 添加工具配置
        if "tools_config" in config_data:
            import json
            env_vars.append(client.V1EnvVar(
                name="TOOLS_CONFIG", 
                value=json.dumps(config_data["tools_config"])
            ))
        
        if "tool_endpoint" in config_data:
            env_vars.append(client.V1EnvVar(name="TOOL_ENDPOINT", value=config_data["tool_endpoint"]))
        
        if "tool_api_key" in config_data:
            env_vars.append(client.V1EnvVar(name="TOOL_API_KEY", value=config_data["tool_api_key"]))
        
        # NEW: 添加模型配置
        if "model_provider" in config_data:
            env_vars.append(client.V1EnvVar(name="MODEL_PROVIDER", value=config_data["model_provider"]))
        
        if "model_name" in config_data:
            env_vars.append(client.V1EnvVar(name="MODEL_NAME", value=config_data["model_name"]))
        
        if "model_endpoint" in config_data:
            env_vars.append(client.V1EnvVar(name="MODEL_ENDPOINT", value=config_data["model_endpoint"]))
        
        if "model_api_key" in config_data:
            env_vars.append(client.V1EnvVar(name="MODEL_API_KEY", value=config_data["model_api_key"]))
        
        # NEW: 添加存储配置
        if "storage_connection_string" in config_data:
            env_vars.append(client.V1EnvVar(
                name="AZURE_STORAGE_CONNECTION_STRING", 
                value=config_data["storage_connection_string"]
            ))
        
        if "storage_account_name" in config_data:
            env_vars.append(client.V1EnvVar(name="STORAGE_ACCOUNT_NAME", value=config_data["storage_account_name"]))
        
        # NEW: 添加用户标识
        if "user_id" in config_data:
            env_vars.append(client.V1EnvVar(name="USER_ID", value=config_data["user_id"]))
        
        if "tenant_id" in config_data:
            env_vars.append(client.V1EnvVar(name="TENANT_ID", value=config_data["tenant_id"]))
        
        # NEW: 添加命名空间信息
        namespace = config_data.get("namespace", self.namespace)
        env_vars.append(client.V1EnvVar(name="NAMESPACE", value=namespace))
        
        # 为特定模板添加默认环境变量
        template_defaults = {
            "facebook_agent": {
                "LITELLM_GATEWAY_URL": "https://litellm.graystone-fb459c5d.southeastasia.azurecontainerapps.io",
                "LITELLM_MODEL": "taiji/gpt-4o-mini",
                "FACEBOOK_API_KEY": "34225c5924msh453fa7aff7a52a9p1d7adfjsn260ba0796c00"
            },
            "code_ai_agent": {
                "OPENAI_BASE_URL": "https://litellm.graystone-fb459c5d.southeastasia.azurecontainerapps.io/v1",
                "LITELLM_MODEL": "taiji/gpt-4o-mini"
            },
            "coding_a2a_agent": {
                "OPENAI_BASE_URL": "https://litellm.graystone-fb459c5d.southeastasia.azurecontainerapps.io/v1",
                "LITELLM_MODEL": "taiji/gpt-4o-mini",
                "WORK_DIR": "/workspace"
            }
        }
        
        # 获取用户自定义环境变量
        custom_env = config_data.get("env", {})
        
        # 如果模板有默认值，先添加默认值（用户自定义值会覆盖）
        if template in template_defaults:
            defaults = template_defaults[template]
            for key, value in defaults.items():
                # 只有当用户没有提供该环境变量时才使用默认值
                if key not in custom_env:
                    env_vars.append(client.V1EnvVar(name=key, value=str(value)))
                    logger.info(f"使用默认环境变量: {key}")
        
        # 添加用户自定义环境变量（会覆盖默认值）
        for key, value in custom_env.items():
            env_vars.append(client.V1EnvVar(name=key, value=str(value)))
        
        logger.info(f"Pod {pod_name} 环境变量数量: {len(env_vars)}")
        
        # 设置容器端口（如果是HTTP服务类型的agent）
        container_ports = None
        if template in ["search_agent", "search_agent_a2a", "search_agent_mcp"]:
            container_ports = [client.V1ContainerPort(container_port=8080)]
        elif template in ["jina_search_agent", "azure_blob_agent", "azure_blob_agent_mcp", "azure_blob_agent_a2a", "a2a_litellm_agent", "coding_a2a_agent", "code_ai_agent", "facebook_agent", "echo_agent", "mysql_agent", "postgresql_agent"]:
            container_ports = [client.V1ContainerPort(container_port=8000)]
        
        # 创建Pod规格
        container = client.V1Container(
            name=pod_name,
            image=image,
            resources=client.V1ResourceRequirements(
                requests={"cpu": cpu_request, "memory": memory_request},
                limits={"cpu": cpu_limit, "memory": memory_limit}
            ),
            env=env_vars,
            ports=container_ports
        )
        
        # 根据镜像来源选择合适的 imagePullSecrets
        image_pull_secrets = [client.V1LocalObjectReference(name="acr-secret")]
        if "openclawacr" in image:
            image_pull_secrets.append(client.V1LocalObjectReference(name="openclaw-acr-secret"))
        
        pod_spec = client.V1PodSpec(
            containers=[container],
            restart_policy="Always",
            image_pull_secrets=image_pull_secrets
        )
        
        # 构建标签（合并默认标签和用户自定义标签）
        labels = {
            "app": "ai-agent",
            "template": template,
            "managed-by": "agent-manager",
            "framework": agent_framework  # NEW: 添加框架标签
        }
        # 添加用户自定义标签
        if "labels" in config_data:
            labels.update(config_data["labels"])
        
        pod_manifest = client.V1Pod(
            api_version="v1",
            kind="Pod",
            metadata=client.V1ObjectMeta(
                name=pod_name,
                labels=labels
            ),
            spec=pod_spec
        )
        
        return pod_manifest
    
    def delete_pod(self, pod_name: str) -> Dict:
        """
        删除Pod
        
        Args:
            pod_name: Pod名称
        
        Returns:
            删除结果
        """
        try:
            self.v1.delete_namespaced_pod(
                name=pod_name,
                namespace=self.namespace,
                body=client.V1DeleteOptions()
            )
            logger.info(f"Pod {pod_name} 删除成功")
            return {"status": "success", "message": f"Pod {pod_name} 已删除"}
        except ApiException as e:
            if e.status == 404:
                return {"status": "not_found", "message": f"Pod {pod_name} 不存在"}
            logger.error(f"删除Pod失败: {e}")
            raise Exception(f"删除Pod失败: {e.reason}")
    
    def delete_agent_namespace(self, agent_name: str) -> Dict:
        """删除Agent的命名空间及其所有资源
        
        Args:
            agent_name: Agent 名称
            
        Returns:
            删除结果
        """
        sanitized_name = sanitize_k8s_name(agent_name)
        namespace_name = f"agent-{sanitized_name}"[:63].rstrip('-')
        # region agent log
        try:
            import json, time
            with open("/home/taiji/tools/agent-manager/.cursor/debug.log", "a") as _f:
                _f.write(json.dumps({
                    "sessionId": "debug-session",
                    "runId": "pre-fix",
                    "hypothesisId": "H1",
                    "location": "k8s_manager.py:delete_agent_namespace:entry",
                    "message": "delete_agent_namespace computed namespace",
                    "data": {
                        "agent_name": agent_name,
                        "namespace_name": namespace_name,
                        "is_manager_namespace": namespace_name == "agent-manager"
                    },
                    "timestamp": int(time.time() * 1000)
                }) + "\n")
        except Exception:
            pass
        # endregion
        
        # 保护机制：防止删除 agent-manager 命名空间
        if namespace_name == "agent-manager":
            logger.error(f"❌ 禁止删除 agent-manager 命名空间！agent_name={agent_name}, namespace_name={namespace_name}")
            raise Exception(f"禁止删除 agent-manager 命名空间。这是系统保护命名空间，不能被删除。")
        
        try:
            # 删除命名空间会自动删除其中的所有资源（Pod、Service、Ingress等）
            self.v1.delete_namespace(
                name=namespace_name,
                body=client.V1DeleteOptions()
            )
            logger.info(f"✅ 命名空间 {namespace_name} 及其所有资源删除成功")
            return {
                "status": "success", 
                "message": f"Agent {agent_name} 的命名空间 {namespace_name} 及所有相关资源已删除",
                "namespace": namespace_name
            }
        except ApiException as e:
            if e.status == 404:
                return {
                    "status": "not_found", 
                    "message": f"命名空间 {namespace_name} 不存在"
                }
            logger.error(f"删除命名空间失败: {e}")
            raise Exception(f"删除命名空间失败: {e.reason}")
    
    def _get_azure_access_token(self) -> str:
        """获取 Azure Access Token
        
        Returns:
            Azure 访问令牌
        """
        try:
            token_url = f"https://login.microsoftonline.com/{self.AZURE_TENANT_ID}/oauth2/v2.0/token"
            data = {
                "client_id": self.AZURE_CLIENT_ID,
                "client_secret": self.AZURE_CLIENT_SECRET,
                "grant_type": "client_credentials",
                "scope": "https://management.azure.com/.default"
            }
            
            response = requests.post(token_url, data=data, timeout=10)
            response.raise_for_status()
            
            token = response.json().get('access_token')
            if not token:
                raise Exception("未能获取访问令牌")
            
            return token
        except Exception as e:
            logger.error(f"获取 Azure 访问令牌失败: {e}")
            raise
    
    def create_dns_record(self, subdomain: str, ip_address: str, ttl: int = 300) -> Dict:
        """为 IP 地址创建 Azure DNS A 记录
        
        Args:
            subdomain: 子域名（不包括主域名）
            ip_address: 要指向的 IP 地址
            ttl: DNS TTL（秒），默认 300
            
        Returns:
            DNS 记录信息
        """
        try:
            # 获取访问令牌
            token = self._get_azure_access_token()
            
            # 构建 DNS API URL
            dns_url = (
                f"https://management.azure.com/subscriptions/{self.AZURE_SUBSCRIPTION_ID}/"
                f"resourceGroups/{self.AZURE_RESOURCE_GROUP}/providers/Microsoft.Network/"
                f"dnsZones/{self.AZURE_DNS_ZONE}/A/{subdomain}?api-version=2018-05-01"
            )
            
            # DNS 记录数据
            dns_data = {
                "properties": {
                    "TTL": ttl,
                    "ARecords": [{"ipv4Address": ip_address}]
                }
            }
            
            # 创建 DNS 记录
            headers = {
                "Authorization": f"Bearer {token}",
                "Content-Type": "application/json"
            }
            
            response = requests.put(dns_url, json=dns_data, headers=headers, timeout=30)
            response.raise_for_status()
            
            full_domain = f"{subdomain}.{self.AZURE_DNS_ZONE}"
            logger.info(f"✅ DNS 记录创建成功: {full_domain} -> {ip_address}")
            
            return {
                "subdomain": subdomain,
                "domain": full_domain,
                "ip_address": ip_address,
                "ttl": ttl,
                "status": "created"
            }
        except requests.exceptions.RequestException as e:
            logger.error(f"创建 DNS 记录失败: {e}")
            if hasattr(e.response, 'text'):
                logger.error(f"错误详情: {e.response.text}")
            raise Exception(f"创建 DNS 记录失败: {str(e)}")
    
    def delete_dns_record(self, subdomain: str) -> Dict:
        """删除 Azure DNS A 记录
        
        Args:
            subdomain: 要删除的子域名
            
        Returns:
            删除结果
        """
        try:
            # 获取访问令牌
            token = self._get_azure_access_token()
            
            # 构建 DNS API URL
            dns_url = (
                f"https://management.azure.com/subscriptions/{self.AZURE_SUBSCRIPTION_ID}/"
                f"resourceGroups/{self.AZURE_RESOURCE_GROUP}/providers/Microsoft.Network/"
                f"dnsZones/{self.AZURE_DNS_ZONE}/A/{subdomain}?api-version=2018-05-01"
            )
            
            headers = {"Authorization": f"Bearer {token}"}
            
            response = requests.delete(dns_url, headers=headers, timeout=30)
            
            if response.status_code == 404:
                logger.info(f"DNS 记录 {subdomain} 不存在，跳过删除")
                return {"status": "not_found", "subdomain": subdomain}
            
            response.raise_for_status()
            
            full_domain = f"{subdomain}.{self.AZURE_DNS_ZONE}"
            logger.info(f"✅ DNS 记录删除成功: {full_domain}")
            
            return {
                "subdomain": subdomain,
                "domain": full_domain,
                "status": "deleted"
            }
        except requests.exceptions.RequestException as e:
            logger.error(f"删除 DNS 记录失败: {e}")
            raise Exception(f"删除 DNS 记录失败: {str(e)}")
    
    def wait_for_loadbalancer_ip(self, service_name: str, namespace: str, 
                                  max_wait: int = 300, interval: int = 5) -> str:
        """等待 LoadBalancer 分配外网 IP
        
        Args:
            service_name: Service 名称
            namespace: 命名空间
            max_wait: 最大等待时间（秒）
            interval: 检查间隔（秒）
            
        Returns:
            分配的外网 IP 地址
        """
        logger.info(f"等待 LoadBalancer IP 分配... (最多等待 {max_wait} 秒)")
        
        elapsed = 0
        while elapsed < max_wait:
            try:
                service = self.v1.read_namespaced_service(service_name, namespace)
                if service.status.load_balancer.ingress:
                    ip = service.status.load_balancer.ingress[0].ip
                    if ip:
                        logger.info(f"✅ LoadBalancer IP 已分配: {ip}")
                        return ip
            except Exception as e:
                logger.warning(f"查询 Service 状态失败: {e}")
            
            time.sleep(interval)
            elapsed += interval
            logger.info(f"等待中... ({elapsed}/{max_wait} 秒)")
        
        raise Exception(f"LoadBalancer IP 分配超时（{max_wait} 秒）")
    
    def get_pod_status(self, pod_name: str) -> Dict:
        """
        获取Pod状态
        
        Args:
            pod_name: Pod名称
        
        Returns:
            Pod状态信息，包含访问URL和资源使用情况
        """
        try:
            pod = self.v1.read_namespaced_pod(
                name=pod_name,
                namespace=self.namespace
            )
            
            template = pod.metadata.labels.get("template", "unknown")
            pod_ip = pod.status.pod_ip
            service_port = self.TEMPLATE_PORTS.get(template)
            
            # 获取资源配额信息
            container = pod.spec.containers[0]
            resources = container.resources
            resource_requests = {
                "cpu": resources.requests.get("cpu") if resources.requests else None,
                "memory": resources.requests.get("memory") if resources.requests else None
            }
            resource_limits = {
                "cpu": resources.limits.get("cpu") if resources.limits else None,
                "memory": resources.limits.get("memory") if resources.limits else None
            }
            
            # 尝试获取实际资源使用情况（需要metrics-server）
            resource_usage = self._get_pod_resource_usage(pod_name)
            
            # 获取容器实际状态 - 检查是否崩溃或异常
            container_statuses = pod.status.container_statuses or []
            actual_status = pod.status.phase  # 默认使用Pod阶段
            health_status = "healthy"
            container_info = []
            
            for container_status in container_statuses:
                container_state = {}
                restart_count = container_status.restart_count
                
                # 检查容器状态
                if container_status.state.running:
                    container_state = {
                        "state": "running",
                        "started_at": container_status.state.running.started_at.isoformat() if container_status.state.running.started_at else None
                    }
                elif container_status.state.waiting:
                    container_state = {
                        "state": "waiting",
                        "reason": container_status.state.waiting.reason,
                        "message": container_status.state.waiting.message
                    }
                    # 容器在等待状态，标记为不健康
                    health_status = "unhealthy"
                    actual_status = "Waiting"
                elif container_status.state.terminated:
                    container_state = {
                        "state": "terminated",
                        "reason": container_status.state.terminated.reason,
                        "exit_code": container_status.state.terminated.exit_code,
                        "message": container_status.state.terminated.message,
                        "finished_at": container_status.state.terminated.finished_at.isoformat() if container_status.state.terminated.finished_at else None
                    }
                    # 容器已终止，标记为不健康
                    health_status = "unhealthy"
                    actual_status = "Terminated"
                    
                # 检查容器是否就绪
                if not container_status.ready:
                    health_status = "unhealthy"
                    
                # 如果重启次数过多，也标记为不健康
                if restart_count > 5:
                    health_status = "degraded"
                    
                container_info.append({
                    "name": container_status.name,
                    "ready": container_status.ready,
                    "restart_count": restart_count,
                    **container_state
                })
            
            result = {
                "name": pod.metadata.name,
                "namespace": pod.metadata.namespace,
                "status": actual_status,
                "health_status": health_status,  # 新增：真实健康状态
                "template": template,
                "created_at": pod.metadata.creation_timestamp.isoformat() if pod.metadata.creation_timestamp else None,
                "node": pod.spec.node_name,
                "pod_ip": pod_ip,
                "containers": container_info,  # 新增：容器详细信息
                "resources": {
                    "requests": resource_requests,
                    "limits": resource_limits,
                    "usage": resource_usage
                },
                "conditions": [
                    {
                        "type": condition.type,
                        "status": condition.status,
                        "reason": condition.reason
                    }
                    for condition in (pod.status.conditions or [])
                ]
            }
            
            # 如果是HTTP服务类型的agent且Pod已有IP，添加访问URL
            if service_port and pod_ip:
                result["service_port"] = service_port
                result["access_url"] = f"http://{pod_ip}:{service_port}"
                result["endpoints"] = {
                    "root": f"http://{pod_ip}:{service_port}/",
                    "health": f"http://{pod_ip}:{service_port}/health"
                }
            
            return result
        except ApiException as e:
            if e.status == 404:
                return {"status": "not_found", "message": f"Pod {pod_name} 不存在"}
            logger.error(f"获取Pod状态失败: {e}")
            raise Exception(f"获取Pod状态失败: {e.reason}")
    
    def _get_pod_resource_usage(self, pod_name: str) -> Dict:
        """
        获取Pod实际资源使用情况（需要metrics-server）
        
        Args:
            pod_name: Pod名称
        
        Returns:
            资源使用信息（cpu、memory）
        """
        try:
            # 使用CustomObjectsApi调用metrics API
            custom_api = client.CustomObjectsApi()
            metrics = custom_api.get_namespaced_custom_object(
                group="metrics.k8s.io",
                version="v1beta1",
                namespace=self.namespace,
                plural="pods",
                name=pod_name
            )
            
            # 解析容器资源使用情况
            containers = metrics.get("containers", [])
            if containers:
                container = containers[0]
                usage = container.get("usage", {})
                return {
                    "cpu": usage.get("cpu"),
                    "memory": usage.get("memory"),
                    "available": True
                }
            
            return {"cpu": None, "memory": None, "available": False, "reason": "无容器数据"}
        except ApiException as e:
            if e.status == 404:
                return {"cpu": None, "memory": None, "available": False, "reason": "metrics-server未安装或Pod不存在"}
            logger.warning(f"获取Pod资源使用情况失败: {e.reason}")
            return {"cpu": None, "memory": None, "available": False, "reason": f"获取失败: {e.reason}"}
        except Exception as e:
            logger.warning(f"获取Pod资源使用情况异常: {str(e)}")
            return {"cpu": None, "memory": None, "available": False, "reason": f"异常: {str(e)}"}
    
    # ==================== OpenClaw 专用部署方法 ====================
    
    def create_openclaw_deployment(self, agent_name: str, template: str, config_data: Dict) -> Dict:
        """
        创建 OpenClaw Agent 部署（包含 DinD sidecar、ConfigMap、PVC 等）
        
        Args:
            agent_name: Agent 名称
            template: 模板类型 (openclaw, openclaw-sandbox, openclaw-sandbox-browser)
            config_data: 配置信息，包含环境变量等
        
        Returns:
            部署信息
        """
        import json
        
        logger.info(f"🚀 开始创建 OpenClaw 部署: {agent_name} (模板: {template})")
        
        # 从 config_data 获取环境变量
        env_vars = config_data.get("env", {})
        
        # OpenClaw 配置
        openclaw_config = self._generate_openclaw_config(agent_name, env_vars)
        
        # 1. 创建 Secret
        secret_name = f"{agent_name}-secrets"
        self._create_openclaw_secret(secret_name, env_vars)
        
        # 2. 创建 ConfigMap
        configmap_name = f"{agent_name}-config"
        self._create_openclaw_configmap(configmap_name, openclaw_config)
        
        # 3. 创建 PVC
        pvc_name = f"{agent_name}-data"
        self._create_openclaw_pvc(pvc_name)
        
        # 4. 创建 Deployment (包含 init container + gateway + dind sidecar)
        deployment_name = agent_name
        self._create_openclaw_deployment_resource(
            deployment_name=deployment_name,
            template=template,
            secret_name=secret_name,
            configmap_name=configmap_name,
            pvc_name=pvc_name,
            config_data=config_data
        )
        
        # 5. 创建 ClusterIP Service（OpenClaw 使用 Ingress，不需要 LoadBalancer）
        service_name = f"{agent_name}-service"
        service_info = None
        try:
            import time
            time.sleep(2)  # 等待 Deployment 创建完成
            service_info = self.create_service(
                service_name=service_name,
                namespace=self.namespace,
                pod_selector={"app": agent_name},
                service_port=18789,
                target_port=18789,
                service_type="ClusterIP"  # OpenClaw 使用 ClusterIP + Ingress
            )
            logger.info(f"✅ OpenClaw Service 创建成功: {service_name}")
        except Exception as e:
            logger.warning(f"创建 Service 失败: {str(e)}")
        
        # 6. 自动生成自签名证书并创建 Ingress（仅针对 OpenClaw）
        ingress_info = None
        cert_info = None
        try:
            # 生成域名（使用 agent_name 作为子域名）
            domain = f"{agent_name}.{self.AZURE_DNS_ZONE}"
            
            # 生成自签名证书
            tls_secret_name = f"{agent_name}-tls"
            cert_info = self.create_self_signed_cert(
                domain=domain,
                namespace=self.namespace,
                secret_name=tls_secret_name,
                days_valid=365
            )
            logger.info(f"✅ 自签名证书创建成功: {tls_secret_name}")
            
            # 创建 Ingress with TLS
            ingress_name = f"{agent_name}-ingress"
            ingress_info = self.create_ingress(
                ingress_name=ingress_name,
                namespace=self.namespace,
                service_name=service_name,
                service_port=18789,
                host=domain,
                path="/",
                tls_secret_name=tls_secret_name
            )
            logger.info(f"✅ OpenClaw Ingress 创建成功: {ingress_name}")
            
            # 7. 等待 Ingress IP 分配并自动创建 DNS 记录
            try:
                import time
                logger.info("等待 Ingress 外网 IP 分配...")
                ingress_ip = None
                max_wait = 60  # 最多等待1分钟
                interval = 3
                waited = 0
                
                while waited < max_wait:
                    try:
                        ingress = self.networking_v1.read_namespaced_ingress(
                            name=ingress_name,
                            namespace=self.namespace
                        )
                        if ingress.status.load_balancer.ingress:
                            lb_ingress = ingress.status.load_balancer.ingress[0]
                            ingress_ip = lb_ingress.ip or lb_ingress.hostname
                            if ingress_ip:
                                break
                    except ApiException:
                        pass
                    
                    time.sleep(interval)
                    waited += interval
                    logger.info(f"   等待中... ({waited}/{max_wait}秒)")
                
                if ingress_ip:
                    logger.info(f"✅ Ingress 外网 IP: {ingress_ip}")
                    
                    # 自动创建 Azure DNS 记录
                    try:
                        logger.info(f"创建 DNS 记录: {domain} -> {ingress_ip}")
                        dns_info = self.create_dns_record(
                            subdomain=agent_name,
                            ip_address=ingress_ip
                        )
                        logger.info(f"✅ DNS 记录创建成功: {dns_info['domain']} -> {ingress_ip}")
                        ingress_info["dns_info"] = dns_info
                        ingress_info["ingress_ip"] = ingress_ip
                    except Exception as e:
                        logger.warning(f"DNS 记录创建失败: {str(e)}")
                        logger.info("   DNS 记录可以稍后手动创建")
                        ingress_info["ingress_ip"] = ingress_ip
                        ingress_info["dns_note"] = f"DNS 未创建: {str(e)}"
                else:
                    logger.warning("Ingress IP 分配超时，DNS 记录将在 IP 分配后手动创建")
                    ingress_info["dns_note"] = "Ingress IP 正在分配中"
                    
            except Exception as e:
                logger.warning(f"等待 Ingress IP 或创建 DNS 失败: {str(e)}")
                ingress_info["dns_note"] = f"DNS 创建失败: {str(e)}"
            
        except Exception as e:
            logger.warning(f"创建证书或 Ingress 失败: {str(e)}")
            logger.info("   可以稍后手动创建证书和 Ingress")
        
        logger.info(f"✅ OpenClaw 部署创建成功: {agent_name}")
        
        # 构建访问信息
        access_info = {
            "note": "OpenClaw 部署正在启动，请稍后通过 /agents/{name}/status 获取状态",
            "ports": {
                "http": 18789,
                "bridge": 18790
            }
        }
        
        if ingress_info:
            if ingress_info.get("url"):
                access_info["https_url"] = ingress_info["url"]
                access_info["domain"] = ingress_info.get("host")
            
            # 添加 DNS 信息
            if ingress_info.get("dns_info"):
                dns_info = ingress_info["dns_info"]
                access_info["domain"] = dns_info.get("domain")
                access_info["https_url"] = f"https://{dns_info.get('domain')}"
                access_info["ingress_ip"] = ingress_info.get("ingress_ip")
                access_info["note"] = f"访问地址: https://{dns_info.get('domain')} (自签名证书，浏览器会显示安全警告)"
            elif ingress_info.get("ingress_ip"):
                access_info["ingress_ip"] = ingress_info["ingress_ip"]
                access_info["https_url"] = f"https://{ingress_info['ingress_ip']}"
                access_info["domain"] = ingress_info.get("host")
                access_info["note"] = f"访问地址: https://{ingress_info['ingress_ip']} 或 https://{ingress_info.get('host')} (自签名证书，浏览器会显示安全警告)"
            elif ingress_info.get("url"):
                access_info["note"] = f"访问地址: {ingress_info['url']} (自签名证书，浏览器会显示安全警告)"
        
        return {
            "name": agent_name,
            "namespace": self.namespace,
            "status": "Pending",
            "template": template,
            "deployment_type": "openclaw",
            "resources": {
                "secret": secret_name,
                "configmap": configmap_name,
                "pvc": pvc_name,
                "deployment": deployment_name,
                "service": service_name if service_info else None,
                "ingress": ingress_info.get("name") if ingress_info else None,
                "tls_secret": cert_info.get("secret_name") if cert_info else None
            },
            "service_port": 18789,
            "access_info": access_info
        }
    
    def _generate_openclaw_config(self, agent_name: str, env_vars: Dict) -> str:
        """生成 OpenClaw 配置文件内容"""
        import json
        
        # 默认配置，可通过 env_vars 覆盖
        litellm_api_base = env_vars.get("LITELLM_API_BASE", 
            "https://litellm.graystone-fb459c5d.southeastasia.azurecontainerapps.io/v1")
        litellm_model = env_vars.get("LITELLM_MODEL", "taiji/gemini-2.5-flash")
        
        config = {
            "meta": {
                "lastTouchedVersion": "2026.2.3"
            },
            "models": {
                "providers": {
                    "litellm": {
                        "baseUrl": litellm_api_base,
                        "apiKey": "${LITELLM_API_KEY}",
                        "api": "openai-completions",
                        "models": [
                            {
                                "id": litellm_model,
                                "name": litellm_model.split("/")[-1],
                                "reasoning": False,
                                "input": ["text"],
                                "cost": {"input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0},
                                "contextWindow": 1000000,
                                "maxTokens": 8192
                            }
                        ]
                    }
                }
            },
            "agents": {
                "defaults": {
                    "model": {
                        "primary": f"litellm/{litellm_model}"
                    },
                    "models": {
                        f"litellm/{litellm_model}": {
                            "alias": litellm_model.split("/")[-1]
                        }
                    },
                    "workspace": "/home/node/.openclaw/workspace",
                    "compaction": {"mode": "safeguard"},
                    "maxConcurrent": 4,
                    "subagents": {"maxConcurrent": 8},
                    "sandbox": {
                        "mode": "all",
                        "workspaceAccess": "rw",
                        "scope": "agent",
                        "docker": {
                            "image": "openclawacr.azurecr.io/openclaw-sandbox:arm64",
                            "network": "bridge"
                        },
                        "browser": {
                            "enabled": True,
                            "image": "openclawacr.azurecr.io/openclaw-sandbox-browser:arm64"
                        }
                    }
                }
            },
            "messages": {
                "ackReactionScope": "group-mentions"
            },
            "commands": {
                "native": "auto",
                "nativeSkills": "auto"
            },
            "channels": {},
            "gateway": {
                "port": 18789,
                "mode": "local",
                "bind": "lan",
                "trustedProxies": ["*"],  # 信任所有代理（用于 Ingress/LoadBalancer）
                "auth": {
                    "mode": "token",
                    "token": "${GATEWAY_AUTH_TOKEN}"
                },
                "controlUi": {
                    "dangerouslyDisableDeviceAuth": True  # 禁用设备认证，避免 "pairing required" 错误
                },
                "http": {
                    "endpoints": {
                        "chatCompletions": {"enabled": True}
                    }
                }
            },
            "plugins": {
                "entries": {}
            },
            "tools": {
                "sandbox": {
                    "tools": {
                        "allow": [
                            "exec", "process", "read", "write", "edit", "browser",
                            "sessions_list", "sessions_history", "sessions_send", 
                            "sessions_spawn", "session_status"
                        ],
                        "deny": ["canvas", "nodes", "cron", "discord", "gateway"]
                    }
                }
            }
        }
        
        # 根据环境变量配置渠道
        if env_vars.get("TELEGRAM_BOT_TOKEN"):
            config["channels"]["telegram"] = {
                "enabled": True,
                "dmPolicy": "pairing",
                "botToken": "${TELEGRAM_BOT_TOKEN}",
                "groupPolicy": "allowlist",
                "streamMode": "partial"
            }
            config["plugins"]["entries"]["telegram"] = {"enabled": True}
        
        if env_vars.get("FEISHU_APP_ID") and env_vars.get("FEISHU_APP_SECRET"):
            config["channels"]["feishu"] = {
                "appId": "${FEISHU_APP_ID}",
                "appSecret": "${FEISHU_APP_SECRET}",
                "enabled": True,
                "connectionMode": "websocket",
                "dmPolicy": "open",
                "groupPolicy": "open"
            }
            config["plugins"]["entries"]["feishu"] = {"enabled": True}
            # 飞书插件需要安装
            if "installs" not in config["plugins"]:
                config["plugins"]["installs"] = {}
            config["plugins"]["installs"]["feishu"] = {
                "source": "npm",
                "spec": "@m1heng-clawd/feishu",
                "installPath": "/home/node/.openclaw/extensions/feishu",
                "version": "0.1.6"
            }
        
        if env_vars.get("DISCORD_BOT_TOKEN"):
            config["channels"]["discord"] = {
                "enabled": True,
                "botToken": "${DISCORD_BOT_TOKEN}"
            }
            config["plugins"]["entries"]["discord"] = {"enabled": True}
        
        if env_vars.get("SLACK_BOT_TOKEN"):
            config["channels"]["slack"] = {
                "enabled": True,
                "botToken": "${SLACK_BOT_TOKEN}",
                "appToken": env_vars.get("SLACK_APP_TOKEN", "${SLACK_APP_TOKEN}")
            }
            config["plugins"]["entries"]["slack"] = {"enabled": True}
        
        return json.dumps(config, indent=2)
    
    def _create_openclaw_secret(self, secret_name: str, env_vars: Dict):
        """创建 OpenClaw Secret"""
        import hashlib
        import base64
        import json
        
        # 默认值
        default_gateway_token = hashlib.sha256(f"{secret_name}-gateway".encode()).hexdigest()
        default_auth_token = hashlib.sha256(f"{secret_name}-auth".encode()).hexdigest()
        
        string_data = {
            "OPENCLAW_GATEWAY_TOKEN": env_vars.get("OPENCLAW_GATEWAY_TOKEN", default_gateway_token),
            "LITELLM_API_KEY": env_vars.get("LITELLM_API_KEY", "sk-litellm-default"),
            "GATEWAY_AUTH_TOKEN": env_vars.get("GATEWAY_AUTH_TOKEN", default_auth_token),
        }
        
        # 添加可选的渠道 Token
        optional_keys = [
            "TELEGRAM_BOT_TOKEN", "FEISHU_APP_ID", "FEISHU_APP_SECRET",
            "DISCORD_BOT_TOKEN", "SLACK_BOT_TOKEN", "SLACK_APP_TOKEN"
        ]
        for key in optional_keys:
            if env_vars.get(key):
                string_data[key] = env_vars[key]
        
        # 添加 ACR 凭据用于 DinD
        acr_server = "openclawacr.azurecr.io"
        acr_username = "openclawacr"
        acr_password = "6I2o5Fy29069FvNSTESjFo1GqkYEdBiRNhzpev1iaGYNzOOlNxUxJQQJ99CBACqBBLyEqg7NAAACAZCR2rHa"
        
        # 创建 Docker config.json 用于 DinD 容器
        docker_config = {
            "auths": {
                acr_server: {
                    "username": acr_username,
                    "password": acr_password,
                    "auth": base64.b64encode(f"{acr_username}:{acr_password}".encode()).decode()
                }
            }
        }
        string_data["docker-config.json"] = json.dumps(docker_config)
        
        try:
            secret = client.V1Secret(
                metadata=client.V1ObjectMeta(
                    name=secret_name,
                    namespace=self.namespace,
                    labels={"managed-by": "agent-manager", "app": "openclaw"}
                ),
                string_data=string_data
            )
            
            self.v1.create_namespaced_secret(namespace=self.namespace, body=secret)
            logger.info(f"✅ 创建 Secret: {secret_name}")
        except ApiException as e:
            if e.status == 409:
                logger.info(f"Secret {secret_name} 已存在，尝试更新")
                self.v1.replace_namespaced_secret(name=secret_name, namespace=self.namespace, body=secret)
            else:
                raise
    
    def _create_openclaw_configmap(self, configmap_name: str, config_content: str):
        """创建 OpenClaw ConfigMap"""
        try:
            configmap = client.V1ConfigMap(
                metadata=client.V1ObjectMeta(
                    name=configmap_name,
                    namespace=self.namespace,
                    labels={"managed-by": "agent-manager", "app": "openclaw"}
                ),
                data={"openclaw.json": config_content}
            )
            
            self.v1.create_namespaced_config_map(namespace=self.namespace, body=configmap)
            logger.info(f"✅ 创建 ConfigMap: {configmap_name}")
        except ApiException as e:
            if e.status == 409:
                logger.info(f"ConfigMap {configmap_name} 已存在，尝试更新")
                self.v1.replace_namespaced_config_map(name=configmap_name, namespace=self.namespace, body=configmap)
            else:
                raise
    
    def _create_openclaw_pvc(self, pvc_name: str, storage_size: str = "10Gi"):
        """创建 OpenClaw PVC"""
        try:
            pvc = client.V1PersistentVolumeClaim(
                metadata=client.V1ObjectMeta(
                    name=pvc_name,
                    namespace=self.namespace,
                    labels={"managed-by": "agent-manager", "app": "openclaw"}
                ),
                spec=client.V1PersistentVolumeClaimSpec(
                    access_modes=["ReadWriteOnce"],
                    storage_class_name="managed-csi",  # Azure AKS 默认存储类
                    resources=client.V1ResourceRequirements(
                        requests={"storage": storage_size}
                    )
                )
            )
            
            self.v1.create_namespaced_persistent_volume_claim(namespace=self.namespace, body=pvc)
            logger.info(f"✅ 创建 PVC: {pvc_name}")
        except ApiException as e:
            if e.status == 409:
                logger.info(f"PVC {pvc_name} 已存在，跳过创建")
            else:
                raise
    
    def _create_openclaw_deployment_resource(
        self, 
        deployment_name: str, 
        template: str,
        secret_name: str, 
        configmap_name: str, 
        pvc_name: str,
        config_data: Dict
    ):
        """创建 OpenClaw Deployment (包含 init container + gateway + dind sidecar)"""
        
        # OpenClaw 主镜像（沙箱镜像在配置中自动引用）
        gateway_image = "openclawacr.azurecr.io/openclaw:latest"
        
        # 资源配置
        gateway_resources = client.V1ResourceRequirements(
            requests={"memory": "512Mi", "cpu": "250m"},
            limits={"memory": "2Gi", "cpu": "2000m"}
        )
        
        dind_resources = client.V1ResourceRequirements(
            requests={"memory": "512Mi", "cpu": "250m"},
            limits={"memory": "4Gi", "cpu": "2000m"}
        )
        
        # Secret 环境变量引用
        secret_env_vars = [
            client.V1EnvVar(
                name="LITELLM_API_KEY",
                value_from=client.V1EnvVarSource(
                    secret_key_ref=client.V1SecretKeySelector(name=secret_name, key="LITELLM_API_KEY")
                )
            ),
            client.V1EnvVar(
                name="OPENCLAW_GATEWAY_TOKEN",
                value_from=client.V1EnvVarSource(
                    secret_key_ref=client.V1SecretKeySelector(name=secret_name, key="OPENCLAW_GATEWAY_TOKEN")
                )
            ),
            client.V1EnvVar(
                name="GATEWAY_AUTH_TOKEN",
                value_from=client.V1EnvVarSource(
                    secret_key_ref=client.V1SecretKeySelector(name=secret_name, key="GATEWAY_AUTH_TOKEN")
                )
            ),
        ]
        
        # 可选的渠道 Token
        optional_secret_keys = [
            "TELEGRAM_BOT_TOKEN", "FEISHU_APP_ID", "FEISHU_APP_SECRET",
            "DISCORD_BOT_TOKEN", "SLACK_BOT_TOKEN", "SLACK_APP_TOKEN"
        ]
        env_vars = config_data.get("env", {})
        for key in optional_secret_keys:
            if env_vars.get(key):
                secret_env_vars.append(
                    client.V1EnvVar(
                        name=key,
                        value_from=client.V1EnvVarSource(
                            secret_key_ref=client.V1SecretKeySelector(
                                name=secret_name, key=key, optional=True
                            )
                        )
                    )
                )
        
        # Init Containers 列表
        init_containers = []
        
        # Init Container 0: 创建目录（以 root 权限）
        dir_init_container = client.V1Container(
            name="dir-init",
            image="busybox:1.36",
            command=["sh", "-c", """
echo "=== Creating directories with proper permissions ==="
mkdir -p /data/extensions /data/workspace /data/sandboxes /data/agents /data/canvas /data/cron /data/sandbox /data/devices
chown -R 1000:1000 /data
chmod -R 755 /data
ls -la /data/
echo "Directories created successfully"
"""],
            volume_mounts=[
                client.V1VolumeMount(name="data", mount_path="/data"),
            ]
        )
        init_containers.append(dir_init_container)
        
        # Init Container 1: 插件安装（如果配置了飞书）
        if env_vars.get("FEISHU_APP_ID") and env_vars.get("FEISHU_APP_SECRET"):
            plugin_install_container = client.V1Container(
                name="plugin-install",
                image=gateway_image,  # 使用 OpenClaw 镜像
                command=["sh", "-c", """
echo "=== Installing Feishu plugin ==="
cd /home/node

# 安装飞书插件到持久化目录
if [ ! -d "/data/extensions/feishu/node_modules" ]; then
    echo "Installing @m1heng-clawd/feishu plugin..."
    npm pack @m1heng-clawd/feishu --pack-destination /tmp
    mkdir -p /data/extensions/feishu
    tar -xzf /tmp/m1heng-clawd-feishu-*.tgz -C /data/extensions/feishu --strip-components=1
    cd /data/extensions/feishu && npm install --production
    echo "Plugin installed successfully"
else
    echo "Plugin already installed, skipping..."
fi

ls -la /data/extensions/feishu/ || true
"""],
                volume_mounts=[
                    client.V1VolumeMount(name="data", mount_path="/data"),
                ]
            )
            init_containers.append(plugin_install_container)
            logger.info("✅ 添加飞书插件安装 init container")
        
        # Init Container 2: 配置初始化
        config_init_container = client.V1Container(
            name="config-init",
            image="busybox:1.36",
            command=["sh", "-c", """
cp /config-template/openclaw.json /config/openclaw.json
sed -i "s|\\${LITELLM_API_KEY}|$LITELLM_API_KEY|g" /config/openclaw.json
sed -i "s|\\${FEISHU_APP_ID}|$FEISHU_APP_ID|g" /config/openclaw.json
sed -i "s|\\${FEISHU_APP_SECRET}|$FEISHU_APP_SECRET|g" /config/openclaw.json
sed -i "s|\\${TELEGRAM_BOT_TOKEN}|$TELEGRAM_BOT_TOKEN|g" /config/openclaw.json
sed -i "s|\\${GATEWAY_AUTH_TOKEN}|$GATEWAY_AUTH_TOKEN|g" /config/openclaw.json
sed -i "s|\\${DISCORD_BOT_TOKEN}|$DISCORD_BOT_TOKEN|g" /config/openclaw.json
sed -i "s|\\${SLACK_BOT_TOKEN}|$SLACK_BOT_TOKEN|g" /config/openclaw.json
sed -i "s|\\${SLACK_APP_TOKEN}|$SLACK_APP_TOKEN|g" /config/openclaw.json
mkdir -p /data/workspace /data/sandboxes /data/extensions
echo "Config initialized successfully"
"""],
            env=secret_env_vars,
            volume_mounts=[
                client.V1VolumeMount(name="config-template", mount_path="/config-template"),
                client.V1VolumeMount(name="config", mount_path="/config"),
                client.V1VolumeMount(name="data", mount_path="/data"),
            ]
        )
        init_containers.append(config_init_container)
        
        # Init Container 2: 初始化 identity volume 权限
        # 由于 gateway 容器需要创建 identity 目录，使用 emptyDir volume 避免权限问题
        # 在 initContainer 中预先创建并设置正确的权限
        permission_fix_container = client.V1Container(
            name="permission-fix",
            image="busybox:1.36",
            command=["sh", "-c", """
echo "=== Initializing identity volume permissions ==="
# 创建 identity 目录并设置正确的权限（emptyDir volume）
mkdir -p /identity
chown -R 1000:1000 /identity
chmod -R 755 /identity
ls -la /identity/ 2>/dev/null || echo "Directory check completed"
echo "Identity volume initialized successfully"
"""],
            security_context=client.V1SecurityContext(
                run_as_user=0,  # 以 root 运行
                run_as_group=0
            ),
            volume_mounts=[
                client.V1VolumeMount(
                    name="identity",
                    mount_path="/identity"
                ),
            ]
        )
        init_containers.append(permission_fix_container)
        
        # Gateway Container
        gateway_container = client.V1Container(
            name="gateway",
            image=gateway_image,
            ports=[
                client.V1ContainerPort(container_port=18789, name="http"),
                client.V1ContainerPort(container_port=18790, name="bridge"),
            ],
            env=[
                client.V1EnvVar(name="HOME", value="/home/node"),
                client.V1EnvVar(name="TERM", value="xterm-256color"),
                client.V1EnvVar(name="DOCKER_HOST", value="tcp://localhost:2375"),
            ] + secret_env_vars,
            volume_mounts=[
                client.V1VolumeMount(
                    name="config", 
                    mount_path="/home/node/.openclaw/openclaw.json",
                    sub_path="openclaw.json"
                ),
                client.V1VolumeMount(
                    name="data", 
                    mount_path="/home/node/.openclaw/workspace",
                    sub_path="workspace"
                ),
                client.V1VolumeMount(
                    name="data", 
                    mount_path="/home/node/.openclaw/sandboxes",
                    sub_path="sandboxes"
                ),
                client.V1VolumeMount(
                    name="data", 
                    mount_path="/home/node/.openclaw/extensions",
                    sub_path="extensions"
                ),
                client.V1VolumeMount(
                    name="data", 
                    mount_path="/home/node/.openclaw/agents",
                    sub_path="agents"
                ),
                client.V1VolumeMount(
                    name="data", 
                    mount_path="/home/node/.openclaw/canvas",
                    sub_path="canvas"
                ),
                client.V1VolumeMount(
                    name="data", 
                    mount_path="/home/node/.openclaw/cron",
                    sub_path="cron"
                ),
                client.V1VolumeMount(
                    name="data", 
                    mount_path="/home/node/.openclaw/sandbox",
                    sub_path="sandbox"
                ),
                client.V1VolumeMount(
                    name="data", 
                    mount_path="/home/node/.openclaw/devices",
                    sub_path="devices"
                ),
                # 添加 identity volume 挂载，用于存储设备身份信息
                client.V1VolumeMount(
                    name="identity",
                    mount_path="/home/node/.openclaw/identity"
                ),
            ],
            command=["node", "dist/index.js", "gateway", "--bind", "lan", "--port", "18789"],
            resources=gateway_resources,
            readiness_probe=client.V1Probe(
                http_get=client.V1HTTPGetAction(path="/health", port=18789),
                initial_delay_seconds=10,
                period_seconds=10
            ),
            liveness_probe=client.V1Probe(
                http_get=client.V1HTTPGetAction(path="/health", port=18789),
                initial_delay_seconds=30,
                period_seconds=30
            )
        )
        
        # DinD Sidecar Container - 带有自动拉取沙箱镜像的 postStart hook
        dind_container = client.V1Container(
            name="dind",
            image="docker:24-dind",
            security_context=client.V1SecurityContext(privileged=True),
            env=[client.V1EnvVar(name="DOCKER_TLS_CERTDIR", value="")],
            ports=[client.V1ContainerPort(container_port=2375, name="docker")],
            volume_mounts=[
                client.V1VolumeMount(name="docker-storage", mount_path="/var/lib/docker"),
                client.V1VolumeMount(
                    name="data", 
                    mount_path="/home/node/.openclaw/workspace",
                    sub_path="workspace"
                ),
                client.V1VolumeMount(
                    name="data", 
                    mount_path="/home/node/.openclaw/sandboxes",
                    sub_path="sandboxes"
                ),
                # 挂载 Docker config 用于 ACR 认证
                client.V1VolumeMount(
                    name="docker-config",
                    mount_path="/root/.docker/config.json",
                    sub_path="docker-config.json"
                ),
            ],
            resources=dind_resources,
            # 自动拉取沙箱镜像的生命周期钩子
            lifecycle=client.V1Lifecycle(
                post_start=client.V1LifecycleHandler(
                    _exec=client.V1ExecAction(
                        command=[
                            "/bin/sh", "-c",
                            """
                            # 等待 Docker daemon 启动
                            for i in $(seq 1 30); do
                                docker info > /dev/null 2>&1 && break
                                sleep 2
                            done
                            # 后台拉取沙箱镜像
                            (docker pull openclawacr.azurecr.io/openclaw-sandbox:arm64 && \
                             docker pull openclawacr.azurecr.io/openclaw-sandbox-browser:arm64) &
                            """
                        ]
                    )
                )
            )
        )
        
        # Volumes
        volumes = [
            client.V1Volume(
                name="config-template",
                config_map=client.V1ConfigMapVolumeSource(name=configmap_name)
            ),
            client.V1Volume(name="config", empty_dir=client.V1EmptyDirVolumeSource()),
            client.V1Volume(
                name="data",
                persistent_volume_claim=client.V1PersistentVolumeClaimVolumeSource(claim_name=pvc_name)
            ),
            client.V1Volume(name="docker-storage", empty_dir=client.V1EmptyDirVolumeSource()),
            # Identity volume: 用于存储设备身份信息，使用 emptyDir 避免权限问题
            client.V1Volume(name="identity", empty_dir=client.V1EmptyDirVolumeSource()),
            # Docker config 从 secret 挂载，用于 DinD ACR 认证
            client.V1Volume(
                name="docker-config",
                secret=client.V1SecretVolumeSource(
                    secret_name=secret_name,
                    items=[client.V1KeyToPath(key="docker-config.json", path="docker-config.json")]
                )
            ),
        ]
        
        # Image Pull Secrets
        image_pull_secrets = [
            client.V1LocalObjectReference(name="openclaw-acr-secret"),
            client.V1LocalObjectReference(name="acr-secret")
        ]
        
        # Pod Spec
        pod_spec = client.V1PodSpec(
            init_containers=init_containers,
            containers=[gateway_container, dind_container],
            volumes=volumes,
            image_pull_secrets=image_pull_secrets,
            restart_policy="Always"
        )
        
        # Deployment
        deployment = client.V1Deployment(
            metadata=client.V1ObjectMeta(
                name=deployment_name,
                namespace=self.namespace,
                labels={
                    "app": deployment_name,
                    "template": template,
                    "managed-by": "agent-manager"
                }
            ),
            spec=client.V1DeploymentSpec(
                replicas=1,
                selector=client.V1LabelSelector(
                    match_labels={"app": deployment_name}
                ),
                template=client.V1PodTemplateSpec(
                    metadata=client.V1ObjectMeta(
                        labels={
                            "app": deployment_name,
                            "template": template,
                            "managed-by": "agent-manager"
                        }
                    ),
                    spec=pod_spec
                )
            )
        )
        
        try:
            self.apps_v1.create_namespaced_deployment(namespace=self.namespace, body=deployment)
            logger.info(f"✅ 创建 Deployment: {deployment_name}")
        except ApiException as e:
            if e.status == 409:
                logger.info(f"Deployment {deployment_name} 已存在，尝试更新")
                self.apps_v1.replace_namespaced_deployment(
                    name=deployment_name, namespace=self.namespace, body=deployment
                )
            else:
                raise
    
    def delete_openclaw_deployment(self, agent_name: str) -> Dict:
        """删除 OpenClaw 部署及相关资源"""
        results = {"deleted": [], "errors": []}
        
        resources = [
            ("deployment", f"{agent_name}"),
            ("pvc", f"{agent_name}-data"),
            ("configmap", f"{agent_name}-config"),
            ("secret", f"{agent_name}-secrets"),
        ]
        
        for resource_type, resource_name in resources:
            try:
                if resource_type == "deployment":
                    self.apps_v1.delete_namespaced_deployment(
                        name=resource_name, namespace=self.namespace
                    )
                elif resource_type == "pvc":
                    self.v1.delete_namespaced_persistent_volume_claim(
                        name=resource_name, namespace=self.namespace
                    )
                elif resource_type == "configmap":
                    self.v1.delete_namespaced_config_map(
                        name=resource_name, namespace=self.namespace
                    )
                elif resource_type == "secret":
                    self.v1.delete_namespaced_secret(
                        name=resource_name, namespace=self.namespace
                    )
                results["deleted"].append(f"{resource_type}/{resource_name}")
                logger.info(f"✅ 删除 {resource_type}: {resource_name}")
            except ApiException as e:
                if e.status == 404:
                    logger.info(f"{resource_type} {resource_name} 不存在，跳过")
                else:
                    results["errors"].append(f"{resource_type}/{resource_name}: {e.reason}")
                    logger.error(f"删除 {resource_type} {resource_name} 失败: {e}")
        
        return results
    
    def get_pod_metrics(self, pod_name: str) -> Dict:
        """
        获取Pod资源使用情况（CPU、内存）
        
        Args:
            pod_name: Pod名称
        
        Returns:
            Pod实时资源使用信息和配额信息
        """
        try:
            # 获取 Pod 配额信息
            pod = self.v1.read_namespaced_pod(
                name=pod_name,
                namespace=self.namespace
            )
            
            container = pod.spec.containers[0]
            resources = container.resources
            
            result = {
                "name": pod_name,
                "namespace": self.namespace,
                "requests": {
                    "cpu": resources.requests.get("cpu") if resources.requests else None,
                    "memory": resources.requests.get("memory") if resources.requests else None
                },
                "limits": {
                    "cpu": resources.limits.get("cpu") if resources.limits else None,
                    "memory": resources.limits.get("memory") if resources.limits else None
                }
            }
            
            # 尝试获取实时使用情况（需要 metrics-server）
            try:
                from kubernetes.client import CustomObjectsApi
                custom_api = CustomObjectsApi()
                
                # 调用 metrics.k8s.io API
                metrics = custom_api.get_namespaced_custom_object(
                    group="metrics.k8s.io",
                    version="v1beta1",
                    namespace=self.namespace,
                    plural="pods",
                    name=pod_name
                )
                
                # 提取实时使用数据
                if metrics and "containers" in metrics:
                    container_metrics = metrics["containers"][0]
                    usage = container_metrics.get("usage", {})
                    
                    result["usage"] = {
                        "cpu": usage.get("cpu"),
                        "memory": usage.get("memory")
                    }
                    result["timestamp"] = metrics.get("timestamp")
                    logger.info(f"✅ 获取到实时资源使用: CPU={usage.get('cpu')}, Memory={usage.get('memory')}")
                else:
                    result["usage"] = None
                    logger.warning(f"⚠️ Metrics 数据格式异常")
                    
            except Exception as metrics_error:
                logger.warning(f"⚠️ 无法获取实时资源使用（可能未安装 metrics-server）: {str(metrics_error)}")
                result["usage"] = None
                result["metrics_available"] = False
            
            return result
            
        except ApiException as e:
            logger.error(f"获取Pod资源信息失败: {e}")
            raise Exception(f"获取Pod资源信息失败: {e.reason}")
    
    def list_pods(self, label_selector: Optional[str] = None) -> List[Dict]:
        """
        列出所有Pod
        
        Args:
            label_selector: 标签选择器（可选）
        
        Returns:
            Pod列表
        """
        try:
            if label_selector is None:
                label_selector = "managed-by=agent-manager"
            
            pods = self.v1.list_namespaced_pod(
                namespace=self.namespace,
                label_selector=label_selector
            )
            
            return [
                {
                    "name": pod.metadata.name,
                    "status": pod.status.phase,
                    "template": pod.metadata.labels.get("template", "unknown"),
                    "created_at": pod.metadata.creation_timestamp.isoformat() if pod.metadata.creation_timestamp else None,
                    "pod_ip": pod.status.pod_ip
                }
                for pod in pods.items
            ]
        except ApiException as e:
            logger.error(f"列出Pod失败: {e}")
            raise Exception(f"列出Pod失败: {e.reason}")

    # ============================================================================
    # Sub-mode runtime helper methods
    # ============================================================================

    def create_swarm_namespace(self, swarm_id: str, role: str) -> str:
        """
        Create namespace for swarm agent.

        Args:
            swarm_id: Swarm ID
            role: Agent role

        Returns:
            Namespace name
        """
        namespace_name = sanitize_k8s_name(f"swarm-{swarm_id[:8]}-{role}")

        try:
            # Check if namespace exists
            try:
                self.v1.read_namespace(name=namespace_name)
                logger.info(f"Namespace {namespace_name} already exists")
                return namespace_name
            except ApiException as e:
                if e.status != 404:
                    raise

            # Create namespace
            namespace = client.V1Namespace(
                metadata=client.V1ObjectMeta(
                    name=namespace_name,
                    labels={
                        "managed-by": "agent-manager",
                        "swarm-id": swarm_id[:8],
                        "agent-role": role
                    }
                )
            )

            self.v1.create_namespace(body=namespace)
            logger.info(f"✅ Created namespace: {namespace_name}")
            self._copy_acr_secret_to_namespace(namespace_name)
            return namespace_name

        except ApiException as e:
            logger.error(f"Failed to create namespace {namespace_name}: {e}")
            raise Exception(f"Failed to create namespace: {e.reason}")

    def deploy_swarm_agent(
        self,
        swarm_id: str,
        agent_id: str,
        agent_config: Dict,
        namespace: str
    ) -> Dict:
        """
        Deploy swarm agent pod.

        Args:
            swarm_id: Swarm ID
            agent_id: Agent ID
            agent_config: Agent configuration
            namespace: Namespace

        Returns:
            Deployment info {pod_name, service_url, external_ip}
        """
        try:
            pod_name = sanitize_k8s_name(f"agent-{agent_id}")
            template = agent_config.get("template", "a2a_litellm_agent")
            role = agent_config.get("role", "worker")
            model = agent_config.get("model", "gpt-4")
            billing_context = agent_config.get("billing_context") or {}
            model_lower = model.lower()
            api_format = (
                billing_context.get("api_format")
                or ("anthropic_messages" if "claude" in model_lower else "openai_chat")
            )
            stream_enabled = billing_context.get("stream")
            if stream_enabled is None:
                # Default to non-streaming for runtime agents unless Manager explicitly requests it.
                # This keeps artifact extraction stable across model gateway implementations.
                stream_enabled = False
            elif isinstance(stream_enabled, str):
                stream_enabled = stream_enabled.lower() in {"1", "true", "yes", "on"}
            timeout_seconds = int(
                billing_context.get("timeout_sec")
                or billing_context.get("timeout_seconds")
                or (600 if stream_enabled else 300)
            )
            requested_max_tokens = int(billing_context.get("max_tokens") or 4096)
            role_token_cap = 1600 if role in {"backend", "frontend", "coder", "engineer", "fullstack"} else 1200
            # Clamp per-agent output size to keep sub-mode runtime tasks stable on the gateway.
            max_tokens = max(256, min(requested_max_tokens, role_token_cap))

            # Get template image
            # TODO: Load from template database
            image_map = {
                "a2a_litellm_agent": "agnettaiji.azurecr.io/ai-agents/a2a-litellm-agent:heicode-v2-runtime-20260531214518-arm64",
                "coding_a2a_agent": "agnettaiji.azurecr.io/ai-agents/coding-a2a-agent:latest",
                "code_manager_agent": "agnettaiji.azurecr.io/ai-agents/code-manager-agent:latest"
            }
            image = image_map.get(template, image_map["a2a_litellm_agent"])

            # Environment variables
            env_vars = [
                client.V1EnvVar(name="SWARM_ID", value=swarm_id),
                client.V1EnvVar(name="AGENT_ID", value=agent_id),
                client.V1EnvVar(name="AGENT_ROLE", value=role),
                client.V1EnvVar(name="MODEL_NAME", value=model),
                client.V1EnvVar(name="POD_NAME", value=pod_name),
                client.V1EnvVar(name="NAMESPACE", value=namespace),
                client.V1EnvVar(name="MODEL_API_FORMAT", value=api_format),
                client.V1EnvVar(name="LLM_API_FORMAT", value=api_format),
                client.V1EnvVar(name="LITELLM_STREAM", value="true" if stream_enabled else "false"),
                client.V1EnvVar(name="LLM_STREAM", value="true" if stream_enabled else "false"),
                client.V1EnvVar(name="LITELLM_TIMEOUT", value=str(timeout_seconds)),
                client.V1EnvVar(name="LLM_TIMEOUT", value=str(timeout_seconds)),
                client.V1EnvVar(name="LITELLM_MAX_TOKENS", value=str(max_tokens)),
                client.V1EnvVar(name="LLM_MAX_TOKENS", value=str(max_tokens)),
            ]

            gateway_url = (
                billing_context.get("model_gateway_url")
                or (
                    os.getenv("HEICODE_NEWAPI_BASE_URL")
                    if billing_context.get("provider") == "newapi"
                    else os.getenv("LITELLM_BASE_URL")
                )
                or os.getenv("HEICODE_NEWAPI_BASE_URL")
                or os.getenv("LITELLM_BASE_URL")
            )
            if gateway_url:
                env_vars.extend([
                    client.V1EnvVar(name="LITELLM_BASE_URL", value=gateway_url),
                    client.V1EnvVar(name="LLM_BASE_URL", value=gateway_url),
                    client.V1EnvVar(name="OPENAI_BASE_URL", value=gateway_url),
                ])

            # Add model API key from Runtime environment. Do not use service
            # auth tokens as model gateway credentials.
            model_api_key = (
                os.getenv("LITELLM_API_KEY")
                or os.getenv("LITELLM_USER_KEY")
                or os.getenv("MODEL_GATEWAY_API_KEY")
                or os.getenv("HEICODE_NEWAPI_USER_TOKEN")
                or os.getenv("OPENAI_API_KEY")
                or os.getenv("LLM_API_KEY")
            )
            if model_api_key:
                env_vars.extend([
                    client.V1EnvVar(name="LITELLM_API_KEY", value=model_api_key),
                    client.V1EnvVar(name="LLM_API_KEY", value=model_api_key),
                    client.V1EnvVar(name="OPENAI_API_KEY", value=model_api_key),
                ])

            # Create pod
            pod = client.V1Pod(
                metadata=client.V1ObjectMeta(
                    name=pod_name,
                    namespace=namespace,
                    labels={
                        "app": pod_name,
                        "managed-by": "agent-manager",
                        "swarm-id": swarm_id[:8],
                        "agent-id": agent_id,
                        "agent-role": role
                    }
                ),
                spec=client.V1PodSpec(
                    containers=[
                        client.V1Container(
                            name="agent",
                            image=image,
                            ports=[client.V1ContainerPort(container_port=8000)],
                            env=env_vars,
                            resources=client.V1ResourceRequirements(
                                requests={"cpu": "100m", "memory": "256Mi"},
                                limits={"cpu": "500m", "memory": "512Mi"}
                            )
                        )
                    ],
                    image_pull_secrets=[client.V1LocalObjectReference(name="acr-secret")]
                )
            )

            try:
                self.v1.create_namespaced_pod(namespace=namespace, body=pod)
                logger.info(f"✅ Created pod: {pod_name} in namespace {namespace}")
            except ApiException as e:
                if e.status != 409:
                    raise
                logger.info(f"Pod {pod_name} already exists in namespace {namespace}")

            # Create service
            service = client.V1Service(
                metadata=client.V1ObjectMeta(
                    name=pod_name,
                    namespace=namespace,
                    labels={"app": pod_name}
                ),
                spec=client.V1ServiceSpec(
                    selector={"app": pod_name},
                    ports=[client.V1ServicePort(port=8000, target_port=8000)],
                    type="ClusterIP"
                )
            )

            try:
                self.v1.create_namespaced_service(namespace=namespace, body=service)
                logger.info(f"✅ Created service: {pod_name} in namespace {namespace}")
            except ApiException as e:
                if e.status != 409:
                    raise
                logger.info(f"Service {pod_name} already exists in namespace {namespace}")

            deadline = time.time() + 120
            last_phase = "Unknown"
            while time.time() < deadline:
                current_pod = self.v1.read_namespaced_pod(name=pod_name, namespace=namespace)
                last_phase = current_pod.status.phase or "Unknown"
                container_statuses = current_pod.status.container_statuses or []
                ready = any(status.ready for status in container_statuses)
                if last_phase == "Running" and ready:
                    logger.info(f"✅ Pod {pod_name} is ready in namespace {namespace}")
                    break
                if last_phase in {"Failed", "Unknown"}:
                    raise Exception(f"Pod {pod_name} entered phase {last_phase}")
                for status in container_statuses:
                    waiting = status.state.waiting if status.state else None
                    terminated = status.state.terminated if status.state else None
                    if waiting and waiting.reason in {"CrashLoopBackOff", "ImagePullBackOff", "ErrImagePull"}:
                        raise Exception(f"Pod {pod_name} is not ready: {waiting.reason}")
                    if terminated and terminated.exit_code != 0:
                        raise Exception(f"Pod {pod_name} exited with code {terminated.exit_code}")
                time.sleep(2)
            else:
                raise Exception(f"Timed out waiting for pod {pod_name} readiness; last phase={last_phase}")

            health_deadline = time.time() + 60
            health_url = f"http://{pod_name}.{namespace}.svc.cluster.local:8000/health"
            last_health_error = None
            while time.time() < health_deadline:
                try:
                    response = requests.get(health_url, timeout=3)
                    if response.status_code == 200:
                        logger.info(f"✅ Pod {pod_name} HTTP endpoint is reachable")
                        break
                    last_health_error = f"status={response.status_code}"
                except Exception as exc:
                    last_health_error = str(exc)
                time.sleep(2)
            else:
                raise Exception(f"Timed out waiting for pod {pod_name} HTTP readiness: {last_health_error}")

            service_url = f"http://{pod_name}.{namespace}.svc.cluster.local:8000"

            return {
                "pod_name": pod_name,
                "service_url": service_url,
                "external_ip": None,
                "namespace": namespace
            }

        except ApiException as e:
            logger.error(f"Failed to deploy swarm agent: {e}")
            raise Exception(f"Failed to deploy swarm agent: {e.reason}")

    def cleanup_swarm_resources(self, swarm_id: str):
        """
        Cleanup all K8s resources for a swarm.

        Args:
            swarm_id: Swarm ID
        """
        try:
            # List all namespaces with swarm-id label
            namespaces = self.v1.list_namespace(
                label_selector=f"swarm-id={swarm_id[:8]}"
            )

            for ns in namespaces.items:
                namespace_name = ns.metadata.name
                logger.info(f"Deleting namespace: {namespace_name}")

                # Delete namespace (this will delete all resources in it)
                self.v1.delete_namespace(name=namespace_name)
                logger.info(f"✅ Deleted namespace: {namespace_name}")

            logger.info(f"✅ Cleaned up all resources for swarm {swarm_id}")

        except ApiException as e:
            logger.error(f"Failed to cleanup swarm resources: {e}")
            raise Exception(f"Failed to cleanup swarm resources: {e.reason}")

    def get_swarm_agent_logs(self, namespace: str, pod_name: str, tail_lines: int = 100) -> str:
        """
        Get logs from swarm agent pod.

        Args:
            namespace: Namespace
            pod_name: Pod name
            tail_lines: Number of lines to tail

        Returns:
            Pod logs
        """
        try:
            logs = self.v1.read_namespaced_pod_log(
                name=pod_name,
                namespace=namespace,
                tail_lines=tail_lines
            )
            return logs

        except ApiException as e:
            logger.error(f"Failed to get pod logs: {e}")
            raise Exception(f"Failed to get pod logs: {e.reason}")