elipitc
ac7d828e80
# Conflicts: # Dockerfile # k8s/agent-manager-configmap.yaml # k8s/agent-manager-deployment.yaml
2667 lines
109 KiB
Python
2667 lines
109 KiB
Python
"""
|
||
Kubernetes管理模块 - 负责与Kubernetes集群交互
|
||
"""
|
||
from kubernetes import client, config
|
||
from kubernetes.client.rest import ApiException
|
||
from typing import Dict, List, Optional
|
||
import logging
|
||
import os
|
||
import re
|
||
import requests
|
||
import time
|
||
import subprocess
|
||
import tempfile
|
||
import base64
|
||
|
||
logging.basicConfig(level=logging.INFO)
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
def sanitize_k8s_name(name: str, max_length: int = 63) -> str:
|
||
"""将名称转换为 DNS-1035 合规格式
|
||
|
||
Kubernetes Service、Deployment 等资源名称必须符合 DNS-1035 标准：
|
||
- 只能包含小写字母、数字和连字符 '-'
|
||
- 必须以字母开头
|
||
- 必须以字母或数字结尾
|
||
- 最长 63 个字符
|
||
|
||
Args:
|
||
name: 原始名称
|
||
max_length: 最大长度（默认 63）
|
||
|
||
Returns:
|
||
合规的 K8s 资源名称
|
||
"""
|
||
# 转小写
|
||
name = name.lower()
|
||
# 将下划线和空格替换为连字符
|
||
name = name.replace("_", "-").replace(" ", "-")
|
||
# 移除非字母、数字、连字符的字符
|
||
name = re.sub(r'[^a-z0-9-]', '', name)
|
||
# 合并连续的连字符
|
||
name = re.sub(r'-+', '-', name)
|
||
# 如果以数字开头，添加 'a' 前缀
|
||
if name and name[0].isdigit():
|
||
name = 'a' + name
|
||
# 如果以连字符开头，去掉
|
||
name = name.lstrip('-')
|
||
# 截断到最大长度
|
||
name = name[:max_length]
|
||
# 去掉末尾的连字符
|
||
name = name.rstrip('-')
|
||
# 最终兜底：如果名称为空
|
||
if not name:
|
||
name = 'agent'
|
||
return name
|
||
|
||
|
||
class K8sManager:
|
||
"""Kubernetes资源管理器"""
|
||
|
||
# Azure DNS 配置（从环境变量读取）
|
||
AZURE_TENANT_ID = os.getenv("AZURE_TENANT_ID", "263c3ff6-1be5-4141-8308-b188464fb297")
|
||
AZURE_CLIENT_ID = os.getenv("AZURE_CLIENT_ID", "c5ba26db-f180-425f-bac3-93708d853988")
|
||
AZURE_CLIENT_SECRET = os.getenv("AZURE_CLIENT_SECRET", "ydt8Q~DIOfKXDUOJkzfgaNVNh2d1VrbeIaYuKbpD")
|
||
AZURE_SUBSCRIPTION_ID = os.getenv("AZURE_SUBSCRIPTION_ID", "c6c47e4c-f5f4-49f8-b26f-7728862c17d6")
|
||
AZURE_RESOURCE_GROUP = os.getenv("AZURE_RESOURCE_GROUP", "taiji-ai-v0")
|
||
AZURE_DNS_ZONE = os.getenv("AZURE_DNS_ZONE", "taijiagnet.com")
|
||
|
||
def __init__(self, namespace: str = "ai-agents", kubeconfig_path: str = None):
|
||
"""
|
||
初始化Kubernetes管理器
|
||
|
||
Args:
|
||
namespace: AI Agent部署的命名空间
|
||
kubeconfig_path: kubeconfig文件路径（可选）
|
||
"""
|
||
self.namespace = namespace
|
||
self._load_kube_config(kubeconfig_path)
|
||
|
||
self.v1 = client.CoreV1Api()
|
||
self.apps_v1 = client.AppsV1Api()
|
||
self.networking_v1 = client.NetworkingV1Api()
|
||
|
||
# 确保命名空间存在
|
||
self._ensure_namespace()
|
||
|
||
def _load_kube_config(self, kubeconfig_path: str = None):
|
||
"""
|
||
加载Kubernetes配置
|
||
优先级：
|
||
1. 集群内ServiceAccount（推荐用于生产环境）
|
||
2. 指定的kubeconfig文件路径
|
||
3. 默认的kubeconfig路径 (~/.kube/config)
|
||
"""
|
||
try:
|
||
# 方式1: 尝试加载集群内配置（当服务运行在K8s中且有ServiceAccount时）
|
||
config.load_incluster_config()
|
||
logger.info("✅ 使用集群内ServiceAccount配置")
|
||
except config.ConfigException:
|
||
try:
|
||
if kubeconfig_path and os.path.exists(kubeconfig_path):
|
||
# 方式2: 使用指定的kubeconfig文件
|
||
config.load_kube_config(config_file=kubeconfig_path)
|
||
logger.info(f"✅ 使用指定的kubeconfig: {kubeconfig_path}")
|
||
else:
|
||
# 方式3: 使用默认kubeconfig（开发环境）
|
||
config.load_kube_config()
|
||
logger.info("✅ 使用默认kubeconfig (~/.kube/config)")
|
||
except Exception as e:
|
||
logger.error(f"❌ 无法加载Kubernetes配置: {e}")
|
||
raise Exception(f"Kubernetes配置加载失败: {e}")
|
||
|
||
def _ensure_namespace(self):
|
||
"""确保AI Agent命名空间存在"""
|
||
try:
|
||
self.v1.read_namespace(name=self.namespace)
|
||
logger.info(f"命名空间 {self.namespace} 已存在")
|
||
except ApiException as e:
|
||
if e.status == 404:
|
||
# 创建命名空间
|
||
namespace_manifest = client.V1Namespace(
|
||
metadata=client.V1ObjectMeta(name=self.namespace)
|
||
)
|
||
self.v1.create_namespace(body=namespace_manifest)
|
||
logger.info(f"创建命名空间 {self.namespace}")
|
||
else:
|
||
raise
|
||
|
||
def create_agent_namespace(self, agent_name: str, owner_id: str = None) -> str:
|
||
"""为 Agent 创建独立的命名空间
|
||
|
||
Args:
|
||
agent_name: Agent 名称
|
||
owner_id: 所有者 ID
|
||
|
||
Returns:
|
||
创建的命名空间名称
|
||
"""
|
||
# 生成命名空间名称（使用 agent-{sanitized_name} 格式，确保 DNS 合规）
|
||
sanitized_name = sanitize_k8s_name(agent_name)
|
||
namespace_name = f"agent-{sanitized_name}"[:63].rstrip('-')
|
||
|
||
try:
|
||
# 检查命名空间是否已存在
|
||
self.v1.read_namespace(name=namespace_name)
|
||
logger.info(f"命名空间 {namespace_name} 已存在")
|
||
return namespace_name
|
||
except ApiException as e:
|
||
if e.status == 404:
|
||
# 创建命名空间
|
||
labels = {
|
||
"managed-by": "agent-manager",
|
||
"agent-name": agent_name
|
||
}
|
||
if owner_id:
|
||
labels["owner-id"] = owner_id
|
||
|
||
namespace_manifest = client.V1Namespace(
|
||
metadata=client.V1ObjectMeta(
|
||
name=namespace_name,
|
||
labels=labels
|
||
)
|
||
)
|
||
self.v1.create_namespace(body=namespace_manifest)
|
||
logger.info(f"✅ 创建命名空间 {namespace_name}")
|
||
|
||
# 复制 ACR secret 到新命名空间
|
||
self._copy_acr_secret_to_namespace(namespace_name)
|
||
|
||
return namespace_name
|
||
else:
|
||
raise
|
||
|
||
def _copy_acr_secret_to_namespace(self, target_namespace: str):
|
||
"""复制所有 ACR secrets 到目标命名空间
|
||
|
||
Args:
|
||
target_namespace: 目标命名空间
|
||
"""
|
||
# 需要复制的 ACR secrets 列表
|
||
acr_secrets = ["acr-secret", "openclaw-acr-secret"]
|
||
|
||
for secret_name in acr_secrets:
|
||
try:
|
||
# 从 agent-manager 命名空间读取 secret
|
||
source_secret = self.v1.read_namespaced_secret(
|
||
name=secret_name,
|
||
namespace="agent-manager"
|
||
)
|
||
|
||
# 创建新的 secret（去除自动生成的字段）
|
||
new_secret = client.V1Secret(
|
||
metadata=client.V1ObjectMeta(
|
||
name=secret_name,
|
||
namespace=target_namespace
|
||
),
|
||
data=source_secret.data,
|
||
type=source_secret.type
|
||
)
|
||
|
||
# 在目标命名空间创建 secret
|
||
self.v1.create_namespaced_secret(
|
||
namespace=target_namespace,
|
||
body=new_secret
|
||
)
|
||
logger.info(f"✅ 已复制 {secret_name} 到命名空间 {target_namespace}")
|
||
except ApiException as e:
|
||
if e.status == 404:
|
||
logger.warning(f"⚠️ 源 {secret_name} 不存在，跳过复制")
|
||
elif e.status == 409:
|
||
logger.info(f"{secret_name} 已存在于命名空间 {target_namespace}")
|
||
else:
|
||
logger.error(f"❌ 复制 {secret_name} 失败: {e}")
|
||
|
||
def create_service(self, service_name: str, namespace: str, pod_selector: Dict[str, str],
|
||
service_port: int, target_port: int, service_type: str = "LoadBalancer") -> Dict:
|
||
"""为 Agent Pod 创建 Service
|
||
|
||
Args:
|
||
service_name: Service 名称
|
||
namespace: 命名空间
|
||
pod_selector: Pod 选择器标签
|
||
service_port: Service 端口
|
||
target_port: Pod 目标端口
|
||
service_type: Service 类型，可选值: "LoadBalancer", "ClusterIP" (默认: "LoadBalancer")
|
||
|
||
Returns:
|
||
创建的 Service 信息（包含外网 IP，如果是 LoadBalancer）
|
||
"""
|
||
try:
|
||
service_manifest = client.V1Service(
|
||
metadata=client.V1ObjectMeta(
|
||
name=service_name,
|
||
namespace=namespace,
|
||
labels={"managed-by": "agent-manager"}
|
||
),
|
||
spec=client.V1ServiceSpec(
|
||
selector=pod_selector,
|
||
ports=[
|
||
client.V1ServicePort(
|
||
name="http",
|
||
protocol="TCP",
|
||
port=service_port,
|
||
target_port=target_port
|
||
)
|
||
],
|
||
type=service_type
|
||
)
|
||
)
|
||
|
||
response = self.v1.create_namespaced_service(
|
||
namespace=namespace,
|
||
body=service_manifest
|
||
)
|
||
|
||
logger.info(f"✅ {service_type} Service {service_name} 创建成功 (namespace: {namespace})")
|
||
|
||
# 获取外网 IP（仅 LoadBalancer 类型）
|
||
external_ip = None
|
||
if service_type == "LoadBalancer" and response.status.load_balancer.ingress:
|
||
lb_ingress = response.status.load_balancer.ingress[0]
|
||
external_ip = lb_ingress.ip or lb_ingress.hostname
|
||
|
||
return {
|
||
"name": response.metadata.name,
|
||
"namespace": response.metadata.namespace,
|
||
"cluster_ip": response.spec.cluster_ip,
|
||
"port": service_port,
|
||
"external_ip": external_ip,
|
||
"type": service_type
|
||
}
|
||
except ApiException as e:
|
||
logger.error(f"创建 Service 失败: {e}")
|
||
raise Exception(f"创建 Service 失败: {e.reason}")
|
||
|
||
def create_self_signed_cert(self, domain: str, namespace: str, secret_name: str, days_valid: int = 365) -> Dict:
|
||
"""为指定域名生成自签名证书并创建 Kubernetes Secret
|
||
|
||
Args:
|
||
domain: 域名
|
||
namespace: Kubernetes 命名空间
|
||
secret_name: Secret 名称
|
||
days_valid: 证书有效期（天）
|
||
|
||
Returns:
|
||
证书信息
|
||
"""
|
||
try:
|
||
logger.info(f"🔐 开始生成自签名证书: {domain} (Secret: {secret_name})")
|
||
|
||
# 创建临时目录
|
||
with tempfile.TemporaryDirectory() as temp_dir:
|
||
key_file = os.path.join(temp_dir, "tls.key")
|
||
cert_file = os.path.join(temp_dir, "tls.crt")
|
||
csr_file = os.path.join(temp_dir, "tls.csr")
|
||
|
||
# 1. 生成私钥
|
||
subprocess.run(
|
||
["openssl", "genrsa", "-out", key_file, "2048"],
|
||
check=True,
|
||
capture_output=True
|
||
)
|
||
|
||
# 2. 生成证书签名请求
|
||
subprocess.run(
|
||
["openssl", "req", "-new", "-key", key_file, "-out", csr_file,
|
||
"-subj", f"/C=CN/ST=Beijing/L=Beijing/O=OpenClaw/CN={domain}"],
|
||
check=True,
|
||
capture_output=True
|
||
)
|
||
|
||
# 3. 创建扩展配置文件
|
||
ext_file = os.path.join(temp_dir, "ext.conf")
|
||
with open(ext_file, "w") as f:
|
||
f.write(f"""[req]
|
||
distinguished_name = req_distinguished_name
|
||
req_extensions = v3_req
|
||
|
||
[v3_req]
|
||
basicConstraints = CA:FALSE
|
||
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
|
||
subjectAltName = @alt_names
|
||
|
||
[alt_names]
|
||
DNS.1 = {domain}
|
||
DNS.2 = *.{domain.split('.', 1)[-1] if '.' in domain else domain}
|
||
DNS.3 = localhost
|
||
IP.1 = 127.0.0.1
|
||
""")
|
||
|
||
# 4. 生成自签名证书
|
||
subprocess.run(
|
||
["openssl", "x509", "-req", "-days", str(days_valid),
|
||
"-in", csr_file, "-signkey", key_file, "-out", cert_file,
|
||
"-extensions", "v3_req", "-extfile", ext_file],
|
||
check=True,
|
||
capture_output=True
|
||
)
|
||
|
||
# 5. 读取证书和私钥
|
||
with open(cert_file, "rb") as f:
|
||
cert_data = f.read()
|
||
with open(key_file, "rb") as f:
|
||
key_data = f.read()
|
||
|
||
# 6. 创建 Kubernetes Secret
|
||
secret = client.V1Secret(
|
||
metadata=client.V1ObjectMeta(
|
||
name=secret_name,
|
||
namespace=namespace,
|
||
labels={"managed-by": "agent-manager", "app": "openclaw"}
|
||
),
|
||
type="kubernetes.io/tls",
|
||
data={
|
||
"tls.crt": base64.b64encode(cert_data).decode("utf-8"),
|
||
"tls.key": base64.b64encode(key_data).decode("utf-8")
|
||
}
|
||
)
|
||
|
||
try:
|
||
self.v1.create_namespaced_secret(namespace=namespace, body=secret)
|
||
logger.info(f"✅ 证书 Secret 创建成功: {secret_name}")
|
||
except ApiException as e:
|
||
if e.status == 409:
|
||
logger.info(f"证书 Secret {secret_name} 已存在，尝试更新")
|
||
self.v1.replace_namespaced_secret(
|
||
name=secret_name,
|
||
namespace=namespace,
|
||
body=secret
|
||
)
|
||
else:
|
||
raise
|
||
|
||
return {
|
||
"secret_name": secret_name,
|
||
"domain": domain,
|
||
"namespace": namespace,
|
||
"days_valid": days_valid
|
||
}
|
||
|
||
except subprocess.CalledProcessError as e:
|
||
error_msg = e.stderr.decode() if e.stderr else str(e)
|
||
logger.error(f"生成证书失败: {error_msg}")
|
||
raise Exception(f"生成自签名证书失败: {error_msg}")
|
||
except Exception as e:
|
||
logger.error(f"创建证书 Secret 失败: {e}")
|
||
raise Exception(f"创建证书 Secret 失败: {str(e)}")
|
||
|
||
def create_ingress(self, ingress_name: str, namespace: str, service_name: str,
|
||
service_port: int, host: str = None, path: str = "/",
|
||
tls_secret_name: str = None) -> Dict:
|
||
"""为 Agent Service 创建 Ingress
|
||
|
||
Args:
|
||
ingress_name: Ingress 名称
|
||
namespace: 命名空间
|
||
service_name: 后端 Service 名称
|
||
service_port: Service 端口
|
||
host: 域名（可选，如果为 None 则使用默认）
|
||
path: 路径前缀
|
||
tls_secret_name: TLS 证书 Secret 名称（可选，如果提供则启用 HTTPS）
|
||
|
||
Returns:
|
||
创建的 Ingress 信息
|
||
"""
|
||
try:
|
||
# 构建路径规则
|
||
http_ingress_path = client.V1HTTPIngressPath(
|
||
path=path,
|
||
path_type="Prefix",
|
||
backend=client.V1IngressBackend(
|
||
service=client.V1IngressServiceBackend(
|
||
name=service_name,
|
||
port=client.V1ServiceBackendPort(number=service_port)
|
||
)
|
||
)
|
||
)
|
||
|
||
# 构建规则列表
|
||
ingress_rules = []
|
||
|
||
# 如果指定了 host，创建带 host 的规则（用于 DNS 访问）
|
||
if host:
|
||
ingress_rule_with_host = client.V1IngressRule(
|
||
host=host,
|
||
http=client.V1HTTPIngressRuleValue(paths=[http_ingress_path])
|
||
)
|
||
ingress_rules.append(ingress_rule_with_host)
|
||
|
||
# 注意：不再添加不带 host 的规则，避免多个 Ingress 之间的路径冲突
|
||
# 每个 agent 应该通过自己的域名访问，而不是通过 IP + 路径
|
||
# 如果需要通过 IP 访问，应该使用不同的路径前缀，但这会导致 nginx ingress 验证失败
|
||
|
||
# 构建 Ingress 注解
|
||
annotations = {
|
||
"nginx.ingress.kubernetes.io/rewrite-target": "/",
|
||
}
|
||
|
||
# 如果启用了 TLS，添加相关注解
|
||
if tls_secret_name:
|
||
annotations["nginx.ingress.kubernetes.io/ssl-redirect"] = "true"
|
||
annotations["nginx.ingress.kubernetes.io/proxy-read-timeout"] = "3600"
|
||
annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = "3600"
|
||
annotations["nginx.ingress.kubernetes.io/websocket-services"] = service_name
|
||
else:
|
||
annotations["nginx.ingress.kubernetes.io/ssl-redirect"] = "false"
|
||
|
||
# 构建 TLS 配置
|
||
tls_config = None
|
||
if tls_secret_name and host:
|
||
tls_config = [client.V1IngressTLS(
|
||
hosts=[host],
|
||
secret_name=tls_secret_name
|
||
)]
|
||
|
||
# 创建 Ingress manifest
|
||
ingress_spec = client.V1IngressSpec(
|
||
ingress_class_name="nginx", # 使用 nginx ingress controller
|
||
rules=ingress_rules
|
||
)
|
||
|
||
if tls_config:
|
||
ingress_spec.tls = tls_config
|
||
|
||
ingress_manifest = client.V1Ingress(
|
||
metadata=client.V1ObjectMeta(
|
||
name=ingress_name,
|
||
namespace=namespace,
|
||
labels={"managed-by": "agent-manager"},
|
||
annotations=annotations
|
||
),
|
||
spec=ingress_spec
|
||
)
|
||
|
||
response = self.networking_v1.create_namespaced_ingress(
|
||
namespace=namespace,
|
||
body=ingress_manifest
|
||
)
|
||
|
||
logger.info(f"✅ Ingress {ingress_name} 创建成功 (namespace: {namespace})")
|
||
|
||
# 获取 Ingress IP/域名
|
||
ingress_url = None
|
||
protocol = "https" if tls_secret_name else "http"
|
||
if response.status.load_balancer.ingress:
|
||
lb_ingress = response.status.load_balancer.ingress[0]
|
||
if host:
|
||
ingress_url = f"{protocol}://{host}{path}"
|
||
elif lb_ingress.ip:
|
||
ingress_url = f"{protocol}://{lb_ingress.ip}{path}"
|
||
elif lb_ingress.hostname:
|
||
ingress_url = f"{protocol}://{lb_ingress.hostname}{path}"
|
||
|
||
return {
|
||
"name": response.metadata.name,
|
||
"namespace": response.metadata.namespace,
|
||
"host": host,
|
||
"path": path,
|
||
"url": ingress_url,
|
||
"tls_enabled": tls_secret_name is not None,
|
||
"note": "Ingress URL 将在负载均衡器配置完成后可用"
|
||
}
|
||
except ApiException as e:
|
||
logger.error(f"创建 Ingress 失败: {e}")
|
||
raise Exception(f"创建 Ingress 失败: {e.reason}")
|
||
|
||
# 模板端口映射
|
||
TEMPLATE_PORTS = {
|
||
"echo_agent": 8000,
|
||
"search_agent": 8080,
|
||
"search_agent_a2a": 8080,
|
||
"search_agent_mcp": 8080,
|
||
"mysql_agent": 8000,
|
||
"postgresql_agent": 8000,
|
||
"jina_search_agent": 8000,
|
||
"azure_blob_agent": 8000,
|
||
"azure_blob_agent_mcp": 8000,
|
||
"azure_blob_agent_a2a": 8000,
|
||
"a2a_litellm_agent": 8000,
|
||
"code_ai_agent": 8000,
|
||
"facebook_agent": 8000,
|
||
"media_downloader": 8000,
|
||
"content_analyzer": 8000,
|
||
"huoke": 8000,
|
||
"microsoft_learn_agent": 8000,
|
||
"aws_docs_mcp": 8000,
|
||
"google_mcp": 8000,
|
||
# OpenClaw（沙箱镜像 openclaw-sandbox 和 openclaw-sandbox-browser 会在部署时自动启用）
|
||
"openclaw": 18789,
|
||
}
|
||
|
||
# OpenClaw 模板（沙箱镜像不作为独立模板）
|
||
OPENCLAW_TEMPLATES = ["openclaw"]
|
||
|
||
# 模板所需环境变量说明
|
||
TEMPLATE_ENV_INFO = {
|
||
"jina_search_agent": {
|
||
"required": {
|
||
"JINA_API_KEY": "Jina API密钥，从 https://jina.ai/ 获取"
|
||
},
|
||
"optional": {
|
||
"SERVICE_PORT": "HTTP服务端口，默认8000",
|
||
"SERVICE_HOST": "HTTP服务监听地址，默认0.0.0.0"
|
||
}
|
||
},
|
||
"mysql_agent": {
|
||
"required": {
|
||
"MYSQL_HOST": "MySQL数据库主机地址",
|
||
"MYSQL_USER": "MySQL用户名",
|
||
"MYSQL_PASSWORD": "MySQL密码",
|
||
"MYSQL_DATABASE": "MySQL数据库名",
|
||
"OPENAI_API_KEY": "OpenAI API密钥"
|
||
},
|
||
"optional": {
|
||
"MYSQL_PORT": "MySQL端口，默认3306"
|
||
}
|
||
},
|
||
"postgresql_agent": {
|
||
"required": {
|
||
"POSTGRES_HOST": "PostgreSQL数据库主机地址",
|
||
"POSTGRES_USER": "PostgreSQL用户名",
|
||
"POSTGRES_PASSWORD": "PostgreSQL密码",
|
||
"POSTGRES_DATABASE": "PostgreSQL数据库名",
|
||
"OPENAI_API_KEY": "OpenAI API密钥"
|
||
},
|
||
"optional": {
|
||
"POSTGRES_PORT": "PostgreSQL端口，默认5432"
|
||
}
|
||
},
|
||
"azure_blob_agent": {
|
||
"required": {
|
||
"LITELLM_API_BASE": "LiteLLM服务地址，如 http://litellm-service:4000",
|
||
"LITELLM_MODEL": "使用的LLM模型，如 gpt-3.5-turbo",
|
||
"LITELLM_API_KEY": "LiteLLM API密钥"
|
||
},
|
||
"optional": {
|
||
"AZURE_STORAGE_CONNECTION_STRING": "Azure Storage连接字符串（可选，也可通过 /connect API 动态传入）",
|
||
"SERVICE_PORT": "HTTP服务端口，默认8000",
|
||
"SERVICE_HOST": "HTTP服务监听地址，默认0.0.0.0"
|
||
}
|
||
},
|
||
"azure_blob_agent_mcp": {
|
||
"required": {
|
||
"MODEL_PROVIDER": "模型提供商，如 openai, azure-openai",
|
||
"MODEL_NAME": "使用的模型名称，如 gpt-4",
|
||
"MODEL_API_KEY": "模型 API 密钥"
|
||
},
|
||
"optional": {
|
||
"AZURE_STORAGE_CONNECTION_STRING": "Azure Storage连接字符串",
|
||
"TOOLS_CONFIG": "工具配置 JSON",
|
||
"TOOL_ENDPOINT": "外部工具端点",
|
||
"TOOL_API_KEY": "工具 API 密钥",
|
||
"MODEL_ENDPOINT": "模型 API 端点",
|
||
"STORAGE_ACCOUNT_NAME": "存储账户名称",
|
||
"USER_ID": "用户标识",
|
||
"TENANT_ID": "租户标识",
|
||
"NAMESPACE": "Kubernetes 命名空间"
|
||
}
|
||
},
|
||
"azure_blob_agent_a2a": {
|
||
"required": {
|
||
"MODEL_PROVIDER": "模型提供商，如 openai, azure-openai",
|
||
"MODEL_NAME": "使用的模型名称，如 gpt-4",
|
||
"MODEL_API_KEY": "模型 API 密钥",
|
||
"AGENT_ID": "Agent 唯一标识",
|
||
"AGENT_ROLE": "Agent 角色"
|
||
},
|
||
"optional": {
|
||
"AZURE_STORAGE_CONNECTION_STRING": "Azure Storage连接字符串",
|
||
"TOOLS_CONFIG": "工具配置 JSON",
|
||
"TOOL_ENDPOINT": "外部工具端点",
|
||
"TOOL_API_KEY": "工具 API 密钥",
|
||
"MODEL_ENDPOINT": "模型 API 端点",
|
||
"STORAGE_ACCOUNT_NAME": "存储账户名称",
|
||
"AGENT_CAPABILITIES": "Agent 能力列表 JSON",
|
||
"USER_ID": "用户标识",
|
||
"TENANT_ID": "租户标识",
|
||
"NAMESPACE": "Kubernetes 命名空间"
|
||
}
|
||
},
|
||
"search_agent": {
|
||
"required": {
|
||
"LLM_BASE_URL": "LLM 服务地址，如 https://api.openai.com/v1",
|
||
"SERPER_API_KEY": "Serper 搜索 API 密钥，从 https://serper.dev 获取",
|
||
"JINA_API_KEY": "Jina Reader API 密钥，从 https://jina.ai 获取"
|
||
},
|
||
"optional": {
|
||
"LLM_API_KEY": "LLM API 密钥（可在搜索请求中传入）",
|
||
"MODEL_NAME": "LLM 模型名称，默认 gpt-4o-mini",
|
||
"MAX_ITERATIONS": "最大搜索迭代次数，默认 3",
|
||
"MAX_RESULTS_PER_QUERY": "每次搜索最大结果数，默认 10",
|
||
"CONTENT_MAX_LENGTH": "内容最大长度，默认 5000",
|
||
"TIMEOUT": "超时时间（秒），默认 30",
|
||
"SERVICE_PORT": "HTTP服务端口，默认 8080",
|
||
"SERVICE_HOST": "HTTP服务监听地址，默认 0.0.0.0"
|
||
}
|
||
},
|
||
"search_agent_a2a": {
|
||
"required": {
|
||
"LLM_BASE_URL": "LLM 服务地址，如 https://api.openai.com/v1",
|
||
},
|
||
"optional": {
|
||
"LITELLM_API_KEY": "LiteLLM API 密钥（可在请求中传入）",
|
||
"LLM_API_KEY": "LLM API 密钥（备选，可在请求中传入）",
|
||
"MODEL_NAME": "LLM 模型名称（优先）",
|
||
"LLM_MODEL": "LLM 模型名称（备选）",
|
||
"LITELLM_MODEL": "LiteLLM 模型名称（备选）",
|
||
"SERPER_API_KEY": "Serper 搜索 API 密钥（已内置默认值）",
|
||
"JINA_API_KEY": "Jina Reader API 密钥（已内置默认值）",
|
||
"SERVICE_PORT": "HTTP服务端口，默认 8080",
|
||
"SERVICE_HOST": "HTTP服务监听地址，默认 0.0.0.0"
|
||
},
|
||
"description": "A2A 协议搜索 Agent，支持通过请求动态传入 API key"
|
||
},
|
||
"search_agent_mcp": {
|
||
"required": {
|
||
"LLM_BASE_URL": "LLM 服务地址，如 https://api.openai.com/v1",
|
||
},
|
||
"optional": {
|
||
"LITELLM_API_KEY": "LiteLLM API 密钥（可在请求中传入）",
|
||
"LLM_API_KEY": "LLM API 密钥（备选，可在请求中传入）",
|
||
"MODEL_NAME": "LLM 模型名称（优先）",
|
||
"LLM_MODEL": "LLM 模型名称（备选）",
|
||
"LITELLM_MODEL": "LiteLLM 模型名称（备选）",
|
||
"SERPER_API_KEY": "Serper 搜索 API 密钥（已内置默认值）",
|
||
"JINA_API_KEY": "Jina Reader API 密钥（已内置默认值）",
|
||
"SERVICE_PORT": "HTTP服务端口，默认 8080",
|
||
"SERVICE_HOST": "HTTP服务监听地址，默认 0.0.0.0"
|
||
},
|
||
"description": "MCP 协议搜索 Agent，支持通过请求动态传入 API key"
|
||
},
|
||
"a2a_litellm_agent": {
|
||
"required": {
|
||
"LITELLM_API_BASE": "LiteLLM 服务地址",
|
||
"LITELLM_MODEL": "LiteLLM 模型名称"
|
||
},
|
||
"optional": {
|
||
"LITELLM_API_KEY": "LiteLLM API 密钥（可在请求中传入）",
|
||
"AGENT_NAME": "Agent 名称",
|
||
"AGENT_DESCRIPTION": "Agent 描述",
|
||
"SERVICE_PORT": "HTTP服务端口，默认 8000",
|
||
"SERVICE_HOST": "HTTP服务监听地址，默认 0.0.0.0"
|
||
}
|
||
},
|
||
"code_ai_agent": {
|
||
"required": {
|
||
"LLM_BASE_URL": "LLM 服务地址，如 https://api.openai.com/v1",
|
||
},
|
||
"optional": {
|
||
"LLM_API_KEY": "LLM API 密钥（可在请求中传入）",
|
||
"MODEL_NAME": "LLM 模型名称",
|
||
"API_HOST": "API 服务监听地址，默认 0.0.0.0",
|
||
"API_PORT": "API 服务端口，默认 8000",
|
||
"PROJECTS_DIR": "项目存储目录，默认 /tmp/projects"
|
||
},
|
||
"description": "代码助手 Agent，支持代码生成、分析和执行"
|
||
},
|
||
"facebook_agent": {
|
||
"required": {
|
||
"LITELLM_GATEWAY_URL": "LiteLLM Gateway 服务地址，如 https://litellm.graystone-fb459c5d.southeastasia.azurecontainerapps.io/",
|
||
"FACEBOOK_API_KEY": "Facebook RapidAPI 密钥"
|
||
},
|
||
"optional": {
|
||
"LITELLM_API_KEY": "LiteLLM API 密钥（可在请求中传入）",
|
||
"LITELLM_MODEL": "LiteLLM 模型名称，默认 taiji/gpt-4o-mini",
|
||
"FACEBOOK_API_HOST": "Facebook API 主机，默认 facebook-scraper3.p.rapidapi.com",
|
||
"FACEBOOK_MCP_URL": "Facebook MCP URL，默认 https://mcp.rapidapi.com",
|
||
"MAX_RESULTS": "最大搜索结果数，默认 10",
|
||
"TIMEOUT": "超时时间（秒），默认 30",
|
||
"LOG_LEVEL": "日志级别，默认 INFO"
|
||
},
|
||
"description": "Facebook 搜索智能 Agent，支持 Facebook 内容搜索和 MCP 协议"
|
||
},
|
||
# OpenClaw（部署时自动启用沙箱：openclaw-sandbox 和 openclaw-sandbox-browser）
|
||
"openclaw": {
|
||
"required": {},
|
||
"optional": {
|
||
"LITELLM_API_KEY": "LiteLLM API 密钥",
|
||
"LITELLM_API_BASE": "LiteLLM API 地址",
|
||
"LITELLM_MODEL": "LiteLLM 模型名称，默认 taiji/gemini-2.5-flash",
|
||
"TELEGRAM_BOT_TOKEN": "Telegram Bot Token",
|
||
"FEISHU_APP_ID": "飞书 App ID",
|
||
"FEISHU_APP_SECRET": "飞书 App Secret",
|
||
"DISCORD_BOT_TOKEN": "Discord Bot Token",
|
||
"SLACK_BOT_TOKEN": "Slack Bot Token",
|
||
"SLACK_APP_TOKEN": "Slack App Token",
|
||
"GATEWAY_AUTH_TOKEN": "Gateway 认证 Token（自动生成）",
|
||
"OPENCLAW_GATEWAY_TOKEN": "OpenClaw Gateway Token（自动生成）"
|
||
},
|
||
"description": "OpenClaw 个人 AI 助手，支持多渠道消息、语音唤醒、沙箱代码执行（自动启用 openclaw-sandbox 和 openclaw-sandbox-browser）"
|
||
}
|
||
}
|
||
|
||
def get_template_info(self, template: str) -> Dict:
|
||
"""
|
||
获取模板信息（从数据库动态获取）
|
||
|
||
Args:
|
||
template: 模板类型
|
||
|
||
Returns:
|
||
模板信息（端口、所需环境变量等）
|
||
"""
|
||
from template_manager import template_manager as tm
|
||
template_data = tm.get_template(template)
|
||
|
||
if template_data:
|
||
return {
|
||
"template": template,
|
||
"name": template_data.get("name"),
|
||
"display_name": template_data.get("display_name"),
|
||
"description": template_data.get("description"),
|
||
"port": template_data.get("port", 8000),
|
||
"image": template_data.get("image"),
|
||
"agent_framework": template_data.get("agent_framework"),
|
||
"env_requirements": template_data.get("env_requirements", {}),
|
||
# 回退到硬编码的环境变量信息（兼容旧模板）
|
||
"env_info": self.TEMPLATE_ENV_INFO.get(template, template_data.get("env_requirements", {}))
|
||
}
|
||
|
||
# 回退到硬编码配置（兼容性）
|
||
return {
|
||
"template": template,
|
||
"port": self.TEMPLATE_PORTS.get(template),
|
||
"env_info": self.TEMPLATE_ENV_INFO.get(template, {})
|
||
}
|
||
|
||
def create_pod(self, pod_name: str, template: str, config_data: Dict) -> Dict:
|
||
"""
|
||
创建Pod
|
||
|
||
Args:
|
||
pod_name: Pod名称
|
||
template: 模板类型（echo_agent, chat_agent等）
|
||
config_data: 配置信息（replicas, resources等）
|
||
|
||
Returns:
|
||
创建的Pod信息，包含访问地址
|
||
"""
|
||
# OpenClaw 模板使用专用部署方法（自动启用沙箱：openclaw-sandbox 和 openclaw-sandbox-browser）
|
||
if template == "openclaw":
|
||
return self.create_openclaw_deployment(pod_name, template, config_data)
|
||
|
||
try:
|
||
# 生成Pod规格
|
||
pod_manifest = self._generate_pod_manifest(pod_name, template, config_data)
|
||
|
||
# 创建Pod
|
||
response = self.v1.create_namespaced_pod(
|
||
namespace=self.namespace,
|
||
body=pod_manifest
|
||
)
|
||
|
||
logger.info(f"Pod {pod_name} 创建成功")
|
||
|
||
# 获取服务端口（从数据库动态获取）
|
||
from template_manager import template_manager as tm
|
||
service_port = tm.get_port(template)
|
||
|
||
result = {
|
||
"name": response.metadata.name,
|
||
"namespace": response.metadata.namespace,
|
||
"status": response.status.phase,
|
||
"created_at": response.metadata.creation_timestamp.isoformat() if response.metadata.creation_timestamp else None,
|
||
"template": template
|
||
}
|
||
|
||
# 如果是HTTP服务类型的agent，添加访问信息
|
||
if service_port:
|
||
result["service_port"] = service_port
|
||
result["access_info"] = {
|
||
"note": "Pod IP将在Pod运行后可用，请通过 /agents/{name}/status 获取",
|
||
"port": service_port,
|
||
"endpoints": {
|
||
"root": f"http://<pod_ip>:{service_port}/",
|
||
"health": f"http://<pod_ip>:{service_port}/health"
|
||
}
|
||
}
|
||
|
||
return result
|
||
except ApiException as e:
|
||
logger.error(f"创建Pod失败: {e}")
|
||
raise Exception(f"创建Pod失败: {e.reason}")
|
||
|
||
def _generate_pod_manifest(self, pod_name: str, template: str, config_data: Dict) -> client.V1Pod:
|
||
"""生成Pod配置清单"""
|
||
|
||
# 默认资源配置
|
||
replicas = config_data.get("replicas", 1)
|
||
cpu_request = config_data.get("cpu_request", "100m")
|
||
cpu_limit = config_data.get("cpu_limit", "500m")
|
||
memory_request = config_data.get("memory_request", "128Mi")
|
||
memory_limit = config_data.get("memory_limit", "512Mi")
|
||
|
||
# 从数据库动态获取模板镜像（延迟导入避免循环依赖）
|
||
from template_manager import template_manager as tm
|
||
image = tm.get_image(template)
|
||
|
||
# 构建环境变量列表
|
||
env_vars = [
|
||
client.V1EnvVar(name="POD_NAME", value=pod_name),
|
||
client.V1EnvVar(name="TEMPLATE_TYPE", value=template)
|
||
]
|
||
|
||
# NEW: 添加 Agent 框架配置
|
||
agent_framework = config_data.get("agent_framework", "langchain")
|
||
env_vars.append(client.V1EnvVar(name="AGENT_FRAMEWORK", value=agent_framework))
|
||
|
||
# NEW: 添加工具配置
|
||
if "tools_config" in config_data:
|
||
import json
|
||
env_vars.append(client.V1EnvVar(
|
||
name="TOOLS_CONFIG",
|
||
value=json.dumps(config_data["tools_config"])
|
||
))
|
||
|
||
if "tool_endpoint" in config_data:
|
||
env_vars.append(client.V1EnvVar(name="TOOL_ENDPOINT", value=config_data["tool_endpoint"]))
|
||
|
||
if "tool_api_key" in config_data:
|
||
env_vars.append(client.V1EnvVar(name="TOOL_API_KEY", value=config_data["tool_api_key"]))
|
||
|
||
# NEW: 添加模型配置
|
||
if "model_provider" in config_data:
|
||
env_vars.append(client.V1EnvVar(name="MODEL_PROVIDER", value=config_data["model_provider"]))
|
||
|
||
if "model_name" in config_data:
|
||
env_vars.append(client.V1EnvVar(name="MODEL_NAME", value=config_data["model_name"]))
|
||
|
||
if "model_endpoint" in config_data:
|
||
env_vars.append(client.V1EnvVar(name="MODEL_ENDPOINT", value=config_data["model_endpoint"]))
|
||
|
||
if "model_api_key" in config_data:
|
||
env_vars.append(client.V1EnvVar(name="MODEL_API_KEY", value=config_data["model_api_key"]))
|
||
|
||
# NEW: 添加存储配置
|
||
if "storage_connection_string" in config_data:
|
||
env_vars.append(client.V1EnvVar(
|
||
name="AZURE_STORAGE_CONNECTION_STRING",
|
||
value=config_data["storage_connection_string"]
|
||
))
|
||
|
||
if "storage_account_name" in config_data:
|
||
env_vars.append(client.V1EnvVar(name="STORAGE_ACCOUNT_NAME", value=config_data["storage_account_name"]))
|
||
|
||
# NEW: 添加用户标识
|
||
if "user_id" in config_data:
|
||
env_vars.append(client.V1EnvVar(name="USER_ID", value=config_data["user_id"]))
|
||
|
||
if "tenant_id" in config_data:
|
||
env_vars.append(client.V1EnvVar(name="TENANT_ID", value=config_data["tenant_id"]))
|
||
|
||
# NEW: 添加命名空间信息
|
||
namespace = config_data.get("namespace", self.namespace)
|
||
env_vars.append(client.V1EnvVar(name="NAMESPACE", value=namespace))
|
||
|
||
# 为特定模板添加默认环境变量
|
||
template_defaults = {
|
||
"facebook_agent": {
|
||
"LITELLM_GATEWAY_URL": "https://litellm.graystone-fb459c5d.southeastasia.azurecontainerapps.io",
|
||
"LITELLM_MODEL": "taiji/gpt-4o-mini",
|
||
"FACEBOOK_API_KEY": "34225c5924msh453fa7aff7a52a9p1d7adfjsn260ba0796c00"
|
||
},
|
||
"code_ai_agent": {
|
||
"OPENAI_BASE_URL": "https://litellm.graystone-fb459c5d.southeastasia.azurecontainerapps.io/v1",
|
||
"LITELLM_MODEL": "taiji/gpt-4o-mini"
|
||
}
|
||
}
|
||
|
||
# 获取用户自定义环境变量
|
||
custom_env = config_data.get("env", {})
|
||
|
||
# 如果模板有默认值，先添加默认值（用户自定义值会覆盖）
|
||
if template in template_defaults:
|
||
defaults = template_defaults[template]
|
||
for key, value in defaults.items():
|
||
# 只有当用户没有提供该环境变量时才使用默认值
|
||
if key not in custom_env:
|
||
env_vars.append(client.V1EnvVar(name=key, value=str(value)))
|
||
logger.info(f"使用默认环境变量: {key}")
|
||
|
||
# 添加用户自定义环境变量（会覆盖默认值）
|
||
for key, value in custom_env.items():
|
||
env_vars.append(client.V1EnvVar(name=key, value=str(value)))
|
||
|
||
logger.info(f"Pod {pod_name} 环境变量数量: {len(env_vars)}")
|
||
|
||
# 设置容器端口（如果是HTTP服务类型的agent）
|
||
container_ports = None
|
||
if template in ["search_agent", "search_agent_a2a", "search_agent_mcp"]:
|
||
container_ports = [client.V1ContainerPort(container_port=8080)]
|
||
elif template in ["jina_search_agent", "azure_blob_agent", "azure_blob_agent_mcp", "azure_blob_agent_a2a", "a2a_litellm_agent", "code_ai_agent", "facebook_agent", "echo_agent", "mysql_agent", "postgresql_agent"]:
|
||
container_ports = [client.V1ContainerPort(container_port=8000)]
|
||
|
||
# 创建Pod规格
|
||
container = client.V1Container(
|
||
name=pod_name,
|
||
image=image,
|
||
resources=client.V1ResourceRequirements(
|
||
requests={"cpu": cpu_request, "memory": memory_request},
|
||
limits={"cpu": cpu_limit, "memory": memory_limit}
|
||
),
|
||
env=env_vars,
|
||
ports=container_ports
|
||
)
|
||
|
||
# 根据镜像来源选择合适的 imagePullSecrets
|
||
image_pull_secrets = [client.V1LocalObjectReference(name="acr-secret")]
|
||
if "openclawacr" in image:
|
||
image_pull_secrets.append(client.V1LocalObjectReference(name="openclaw-acr-secret"))
|
||
|
||
pod_spec = client.V1PodSpec(
|
||
containers=[container],
|
||
restart_policy="Always",
|
||
image_pull_secrets=image_pull_secrets
|
||
)
|
||
|
||
# 构建标签（合并默认标签和用户自定义标签）
|
||
labels = {
|
||
"app": "ai-agent",
|
||
"template": template,
|
||
"managed-by": "agent-manager",
|
||
"framework": agent_framework # NEW: 添加框架标签
|
||
}
|
||
# 添加用户自定义标签
|
||
if "labels" in config_data:
|
||
labels.update(config_data["labels"])
|
||
|
||
pod_manifest = client.V1Pod(
|
||
api_version="v1",
|
||
kind="Pod",
|
||
metadata=client.V1ObjectMeta(
|
||
name=pod_name,
|
||
labels=labels
|
||
),
|
||
spec=pod_spec
|
||
)
|
||
|
||
return pod_manifest
|
||
|
||
def delete_pod(self, pod_name: str) -> Dict:
|
||
"""
|
||
删除Pod
|
||
|
||
Args:
|
||
pod_name: Pod名称
|
||
|
||
Returns:
|
||
删除结果
|
||
"""
|
||
try:
|
||
self.v1.delete_namespaced_pod(
|
||
name=pod_name,
|
||
namespace=self.namespace,
|
||
body=client.V1DeleteOptions()
|
||
)
|
||
logger.info(f"Pod {pod_name} 删除成功")
|
||
return {"status": "success", "message": f"Pod {pod_name} 已删除"}
|
||
except ApiException as e:
|
||
if e.status == 404:
|
||
return {"status": "not_found", "message": f"Pod {pod_name} 不存在"}
|
||
logger.error(f"删除Pod失败: {e}")
|
||
raise Exception(f"删除Pod失败: {e.reason}")
|
||
|
||
def delete_agent_namespace(self, agent_name: str) -> Dict:
|
||
"""删除Agent的命名空间及其所有资源
|
||
|
||
Args:
|
||
agent_name: Agent 名称
|
||
|
||
Returns:
|
||
删除结果
|
||
"""
|
||
sanitized_name = sanitize_k8s_name(agent_name)
|
||
namespace_name = f"agent-{sanitized_name}"[:63].rstrip('-')
|
||
# region agent log
|
||
try:
|
||
import json, time
|
||
with open("/home/taiji/tools/agent-manager/.cursor/debug.log", "a") as _f:
|
||
_f.write(json.dumps({
|
||
"sessionId": "debug-session",
|
||
"runId": "pre-fix",
|
||
"hypothesisId": "H1",
|
||
"location": "k8s_manager.py:delete_agent_namespace:entry",
|
||
"message": "delete_agent_namespace computed namespace",
|
||
"data": {
|
||
"agent_name": agent_name,
|
||
"namespace_name": namespace_name,
|
||
"is_manager_namespace": namespace_name == "agent-manager"
|
||
},
|
||
"timestamp": int(time.time() * 1000)
|
||
}) + "\n")
|
||
except Exception:
|
||
pass
|
||
# endregion
|
||
|
||
# 保护机制：防止删除 agent-manager 命名空间
|
||
if namespace_name == "agent-manager":
|
||
logger.error(f"❌ 禁止删除 agent-manager 命名空间！agent_name={agent_name}, namespace_name={namespace_name}")
|
||
raise Exception(f"禁止删除 agent-manager 命名空间。这是系统保护命名空间，不能被删除。")
|
||
|
||
try:
|
||
# 删除命名空间会自动删除其中的所有资源（Pod、Service、Ingress等）
|
||
self.v1.delete_namespace(
|
||
name=namespace_name,
|
||
body=client.V1DeleteOptions()
|
||
)
|
||
logger.info(f"✅ 命名空间 {namespace_name} 及其所有资源删除成功")
|
||
return {
|
||
"status": "success",
|
||
"message": f"Agent {agent_name} 的命名空间 {namespace_name} 及所有相关资源已删除",
|
||
"namespace": namespace_name
|
||
}
|
||
except ApiException as e:
|
||
if e.status == 404:
|
||
return {
|
||
"status": "not_found",
|
||
"message": f"命名空间 {namespace_name} 不存在"
|
||
}
|
||
logger.error(f"删除命名空间失败: {e}")
|
||
raise Exception(f"删除命名空间失败: {e.reason}")
|
||
|
||
def _get_azure_access_token(self) -> str:
|
||
"""获取 Azure Access Token
|
||
|
||
Returns:
|
||
Azure 访问令牌
|
||
"""
|
||
try:
|
||
token_url = f"https://login.microsoftonline.com/{self.AZURE_TENANT_ID}/oauth2/v2.0/token"
|
||
data = {
|
||
"client_id": self.AZURE_CLIENT_ID,
|
||
"client_secret": self.AZURE_CLIENT_SECRET,
|
||
"grant_type": "client_credentials",
|
||
"scope": "https://management.azure.com/.default"
|
||
}
|
||
|
||
response = requests.post(token_url, data=data, timeout=10)
|
||
response.raise_for_status()
|
||
|
||
token = response.json().get('access_token')
|
||
if not token:
|
||
raise Exception("未能获取访问令牌")
|
||
|
||
return token
|
||
except Exception as e:
|
||
logger.error(f"获取 Azure 访问令牌失败: {e}")
|
||
raise
|
||
|
||
def create_dns_record(self, subdomain: str, ip_address: str, ttl: int = 300) -> Dict:
|
||
"""为 IP 地址创建 Azure DNS A 记录
|
||
|
||
Args:
|
||
subdomain: 子域名（不包括主域名）
|
||
ip_address: 要指向的 IP 地址
|
||
ttl: DNS TTL（秒），默认 300
|
||
|
||
Returns:
|
||
DNS 记录信息
|
||
"""
|
||
try:
|
||
# 获取访问令牌
|
||
token = self._get_azure_access_token()
|
||
|
||
# 构建 DNS API URL
|
||
dns_url = (
|
||
f"https://management.azure.com/subscriptions/{self.AZURE_SUBSCRIPTION_ID}/"
|
||
f"resourceGroups/{self.AZURE_RESOURCE_GROUP}/providers/Microsoft.Network/"
|
||
f"dnsZones/{self.AZURE_DNS_ZONE}/A/{subdomain}?api-version=2018-05-01"
|
||
)
|
||
|
||
# DNS 记录数据
|
||
dns_data = {
|
||
"properties": {
|
||
"TTL": ttl,
|
||
"ARecords": [{"ipv4Address": ip_address}]
|
||
}
|
||
}
|
||
|
||
# 创建 DNS 记录
|
||
headers = {
|
||
"Authorization": f"Bearer {token}",
|
||
"Content-Type": "application/json"
|
||
}
|
||
|
||
response = requests.put(dns_url, json=dns_data, headers=headers, timeout=30)
|
||
response.raise_for_status()
|
||
|
||
full_domain = f"{subdomain}.{self.AZURE_DNS_ZONE}"
|
||
logger.info(f"✅ DNS 记录创建成功: {full_domain} -> {ip_address}")
|
||
|
||
return {
|
||
"subdomain": subdomain,
|
||
"domain": full_domain,
|
||
"ip_address": ip_address,
|
||
"ttl": ttl,
|
||
"status": "created"
|
||
}
|
||
except requests.exceptions.RequestException as e:
|
||
logger.error(f"创建 DNS 记录失败: {e}")
|
||
if hasattr(e.response, 'text'):
|
||
logger.error(f"错误详情: {e.response.text}")
|
||
raise Exception(f"创建 DNS 记录失败: {str(e)}")
|
||
|
||
def delete_dns_record(self, subdomain: str) -> Dict:
|
||
"""删除 Azure DNS A 记录
|
||
|
||
Args:
|
||
subdomain: 要删除的子域名
|
||
|
||
Returns:
|
||
删除结果
|
||
"""
|
||
try:
|
||
# 获取访问令牌
|
||
token = self._get_azure_access_token()
|
||
|
||
# 构建 DNS API URL
|
||
dns_url = (
|
||
f"https://management.azure.com/subscriptions/{self.AZURE_SUBSCRIPTION_ID}/"
|
||
f"resourceGroups/{self.AZURE_RESOURCE_GROUP}/providers/Microsoft.Network/"
|
||
f"dnsZones/{self.AZURE_DNS_ZONE}/A/{subdomain}?api-version=2018-05-01"
|
||
)
|
||
|
||
headers = {"Authorization": f"Bearer {token}"}
|
||
|
||
response = requests.delete(dns_url, headers=headers, timeout=30)
|
||
|
||
if response.status_code == 404:
|
||
logger.info(f"DNS 记录 {subdomain} 不存在，跳过删除")
|
||
return {"status": "not_found", "subdomain": subdomain}
|
||
|
||
response.raise_for_status()
|
||
|
||
full_domain = f"{subdomain}.{self.AZURE_DNS_ZONE}"
|
||
logger.info(f"✅ DNS 记录删除成功: {full_domain}")
|
||
|
||
return {
|
||
"subdomain": subdomain,
|
||
"domain": full_domain,
|
||
"status": "deleted"
|
||
}
|
||
except requests.exceptions.RequestException as e:
|
||
logger.error(f"删除 DNS 记录失败: {e}")
|
||
raise Exception(f"删除 DNS 记录失败: {str(e)}")
|
||
|
||
def wait_for_loadbalancer_ip(self, service_name: str, namespace: str,
|
||
max_wait: int = 300, interval: int = 5) -> str:
|
||
"""等待 LoadBalancer 分配外网 IP
|
||
|
||
Args:
|
||
service_name: Service 名称
|
||
namespace: 命名空间
|
||
max_wait: 最大等待时间（秒）
|
||
interval: 检查间隔（秒）
|
||
|
||
Returns:
|
||
分配的外网 IP 地址
|
||
"""
|
||
logger.info(f"等待 LoadBalancer IP 分配... (最多等待 {max_wait} 秒)")
|
||
|
||
elapsed = 0
|
||
while elapsed < max_wait:
|
||
try:
|
||
service = self.v1.read_namespaced_service(service_name, namespace)
|
||
if service.status.load_balancer.ingress:
|
||
ip = service.status.load_balancer.ingress[0].ip
|
||
if ip:
|
||
logger.info(f"✅ LoadBalancer IP 已分配: {ip}")
|
||
return ip
|
||
except Exception as e:
|
||
logger.warning(f"查询 Service 状态失败: {e}")
|
||
|
||
time.sleep(interval)
|
||
elapsed += interval
|
||
logger.info(f"等待中... ({elapsed}/{max_wait} 秒)")
|
||
|
||
raise Exception(f"LoadBalancer IP 分配超时（{max_wait} 秒）")
|
||
|
||
def get_pod_status(self, pod_name: str) -> Dict:
|
||
"""
|
||
获取Pod状态
|
||
|
||
Args:
|
||
pod_name: Pod名称
|
||
|
||
Returns:
|
||
Pod状态信息，包含访问URL和资源使用情况
|
||
"""
|
||
try:
|
||
pod = self.v1.read_namespaced_pod(
|
||
name=pod_name,
|
||
namespace=self.namespace
|
||
)
|
||
|
||
template = pod.metadata.labels.get("template", "unknown")
|
||
pod_ip = pod.status.pod_ip
|
||
service_port = self.TEMPLATE_PORTS.get(template)
|
||
|
||
# 获取资源配额信息
|
||
container = pod.spec.containers[0]
|
||
resources = container.resources
|
||
resource_requests = {
|
||
"cpu": resources.requests.get("cpu") if resources.requests else None,
|
||
"memory": resources.requests.get("memory") if resources.requests else None
|
||
}
|
||
resource_limits = {
|
||
"cpu": resources.limits.get("cpu") if resources.limits else None,
|
||
"memory": resources.limits.get("memory") if resources.limits else None
|
||
}
|
||
|
||
# 尝试获取实际资源使用情况（需要metrics-server）
|
||
resource_usage = self._get_pod_resource_usage(pod_name)
|
||
|
||
# 获取容器实际状态 - 检查是否崩溃或异常
|
||
container_statuses = pod.status.container_statuses or []
|
||
actual_status = pod.status.phase # 默认使用Pod阶段
|
||
health_status = "healthy"
|
||
container_info = []
|
||
|
||
for container_status in container_statuses:
|
||
container_state = {}
|
||
restart_count = container_status.restart_count
|
||
|
||
# 检查容器状态
|
||
if container_status.state.running:
|
||
container_state = {
|
||
"state": "running",
|
||
"started_at": container_status.state.running.started_at.isoformat() if container_status.state.running.started_at else None
|
||
}
|
||
elif container_status.state.waiting:
|
||
container_state = {
|
||
"state": "waiting",
|
||
"reason": container_status.state.waiting.reason,
|
||
"message": container_status.state.waiting.message
|
||
}
|
||
# 容器在等待状态，标记为不健康
|
||
health_status = "unhealthy"
|
||
actual_status = "Waiting"
|
||
elif container_status.state.terminated:
|
||
container_state = {
|
||
"state": "terminated",
|
||
"reason": container_status.state.terminated.reason,
|
||
"exit_code": container_status.state.terminated.exit_code,
|
||
"message": container_status.state.terminated.message,
|
||
"finished_at": container_status.state.terminated.finished_at.isoformat() if container_status.state.terminated.finished_at else None
|
||
}
|
||
# 容器已终止，标记为不健康
|
||
health_status = "unhealthy"
|
||
actual_status = "Terminated"
|
||
|
||
# 检查容器是否就绪
|
||
if not container_status.ready:
|
||
health_status = "unhealthy"
|
||
|
||
# 如果重启次数过多，也标记为不健康
|
||
if restart_count > 5:
|
||
health_status = "degraded"
|
||
|
||
container_info.append({
|
||
"name": container_status.name,
|
||
"ready": container_status.ready,
|
||
"restart_count": restart_count,
|
||
**container_state
|
||
})
|
||
|
||
result = {
|
||
"name": pod.metadata.name,
|
||
"namespace": pod.metadata.namespace,
|
||
"status": actual_status,
|
||
"health_status": health_status, # 新增：真实健康状态
|
||
"template": template,
|
||
"created_at": pod.metadata.creation_timestamp.isoformat() if pod.metadata.creation_timestamp else None,
|
||
"node": pod.spec.node_name,
|
||
"pod_ip": pod_ip,
|
||
"containers": container_info, # 新增：容器详细信息
|
||
"resources": {
|
||
"requests": resource_requests,
|
||
"limits": resource_limits,
|
||
"usage": resource_usage
|
||
},
|
||
"conditions": [
|
||
{
|
||
"type": condition.type,
|
||
"status": condition.status,
|
||
"reason": condition.reason
|
||
}
|
||
for condition in (pod.status.conditions or [])
|
||
]
|
||
}
|
||
|
||
# 如果是HTTP服务类型的agent且Pod已有IP，添加访问URL
|
||
if service_port and pod_ip:
|
||
result["service_port"] = service_port
|
||
result["access_url"] = f"http://{pod_ip}:{service_port}"
|
||
result["endpoints"] = {
|
||
"root": f"http://{pod_ip}:{service_port}/",
|
||
"health": f"http://{pod_ip}:{service_port}/health"
|
||
}
|
||
|
||
return result
|
||
except ApiException as e:
|
||
if e.status == 404:
|
||
return {"status": "not_found", "message": f"Pod {pod_name} 不存在"}
|
||
logger.error(f"获取Pod状态失败: {e}")
|
||
raise Exception(f"获取Pod状态失败: {e.reason}")
|
||
|
||
def _get_pod_resource_usage(self, pod_name: str) -> Dict:
|
||
"""
|
||
获取Pod实际资源使用情况（需要metrics-server）
|
||
|
||
Args:
|
||
pod_name: Pod名称
|
||
|
||
Returns:
|
||
资源使用信息（cpu、memory）
|
||
"""
|
||
try:
|
||
# 使用CustomObjectsApi调用metrics API
|
||
custom_api = client.CustomObjectsApi()
|
||
metrics = custom_api.get_namespaced_custom_object(
|
||
group="metrics.k8s.io",
|
||
version="v1beta1",
|
||
namespace=self.namespace,
|
||
plural="pods",
|
||
name=pod_name
|
||
)
|
||
|
||
# 解析容器资源使用情况
|
||
containers = metrics.get("containers", [])
|
||
if containers:
|
||
container = containers[0]
|
||
usage = container.get("usage", {})
|
||
return {
|
||
"cpu": usage.get("cpu"),
|
||
"memory": usage.get("memory"),
|
||
"available": True
|
||
}
|
||
|
||
return {"cpu": None, "memory": None, "available": False, "reason": "无容器数据"}
|
||
except ApiException as e:
|
||
if e.status == 404:
|
||
return {"cpu": None, "memory": None, "available": False, "reason": "metrics-server未安装或Pod不存在"}
|
||
logger.warning(f"获取Pod资源使用情况失败: {e.reason}")
|
||
return {"cpu": None, "memory": None, "available": False, "reason": f"获取失败: {e.reason}"}
|
||
except Exception as e:
|
||
logger.warning(f"获取Pod资源使用情况异常: {str(e)}")
|
||
return {"cpu": None, "memory": None, "available": False, "reason": f"异常: {str(e)}"}
|
||
|
||
# ==================== OpenClaw 专用部署方法 ====================
|
||
|
||
def create_openclaw_deployment(self, agent_name: str, template: str, config_data: Dict) -> Dict:
|
||
"""
|
||
创建 OpenClaw Agent 部署（包含 DinD sidecar、ConfigMap、PVC 等）
|
||
|
||
Args:
|
||
agent_name: Agent 名称
|
||
template: 模板类型 (openclaw, openclaw-sandbox, openclaw-sandbox-browser)
|
||
config_data: 配置信息，包含环境变量等
|
||
|
||
Returns:
|
||
部署信息
|
||
"""
|
||
import json
|
||
|
||
logger.info(f"🚀 开始创建 OpenClaw 部署: {agent_name} (模板: {template})")
|
||
|
||
# 从 config_data 获取环境变量
|
||
env_vars = config_data.get("env", {})
|
||
|
||
# OpenClaw 配置
|
||
openclaw_config = self._generate_openclaw_config(agent_name, env_vars)
|
||
|
||
# 1. 创建 Secret
|
||
secret_name = f"{agent_name}-secrets"
|
||
self._create_openclaw_secret(secret_name, env_vars)
|
||
|
||
# 2. 创建 ConfigMap
|
||
configmap_name = f"{agent_name}-config"
|
||
self._create_openclaw_configmap(configmap_name, openclaw_config)
|
||
|
||
# 3. 创建 PVC
|
||
pvc_name = f"{agent_name}-data"
|
||
self._create_openclaw_pvc(pvc_name)
|
||
|
||
# 4. 创建 Deployment (包含 init container + gateway + dind sidecar)
|
||
deployment_name = agent_name
|
||
self._create_openclaw_deployment_resource(
|
||
deployment_name=deployment_name,
|
||
template=template,
|
||
secret_name=secret_name,
|
||
configmap_name=configmap_name,
|
||
pvc_name=pvc_name,
|
||
config_data=config_data
|
||
)
|
||
|
||
# 5. 创建 ClusterIP Service（OpenClaw 使用 Ingress，不需要 LoadBalancer）
|
||
service_name = f"{agent_name}-service"
|
||
service_info = None
|
||
try:
|
||
import time
|
||
time.sleep(2) # 等待 Deployment 创建完成
|
||
service_info = self.create_service(
|
||
service_name=service_name,
|
||
namespace=self.namespace,
|
||
pod_selector={"app": agent_name},
|
||
service_port=18789,
|
||
target_port=18789,
|
||
service_type="ClusterIP" # OpenClaw 使用 ClusterIP + Ingress
|
||
)
|
||
logger.info(f"✅ OpenClaw Service 创建成功: {service_name}")
|
||
except Exception as e:
|
||
logger.warning(f"创建 Service 失败: {str(e)}")
|
||
|
||
# 6. 自动生成自签名证书并创建 Ingress（仅针对 OpenClaw）
|
||
ingress_info = None
|
||
cert_info = None
|
||
try:
|
||
# 生成域名（使用 agent_name 作为子域名）
|
||
domain = f"{agent_name}.{self.AZURE_DNS_ZONE}"
|
||
|
||
# 生成自签名证书
|
||
tls_secret_name = f"{agent_name}-tls"
|
||
cert_info = self.create_self_signed_cert(
|
||
domain=domain,
|
||
namespace=self.namespace,
|
||
secret_name=tls_secret_name,
|
||
days_valid=365
|
||
)
|
||
logger.info(f"✅ 自签名证书创建成功: {tls_secret_name}")
|
||
|
||
# 创建 Ingress with TLS
|
||
ingress_name = f"{agent_name}-ingress"
|
||
ingress_info = self.create_ingress(
|
||
ingress_name=ingress_name,
|
||
namespace=self.namespace,
|
||
service_name=service_name,
|
||
service_port=18789,
|
||
host=domain,
|
||
path="/",
|
||
tls_secret_name=tls_secret_name
|
||
)
|
||
logger.info(f"✅ OpenClaw Ingress 创建成功: {ingress_name}")
|
||
|
||
# 7. 等待 Ingress IP 分配并自动创建 DNS 记录
|
||
try:
|
||
import time
|
||
logger.info("等待 Ingress 外网 IP 分配...")
|
||
ingress_ip = None
|
||
max_wait = 60 # 最多等待1分钟
|
||
interval = 3
|
||
waited = 0
|
||
|
||
while waited < max_wait:
|
||
try:
|
||
ingress = self.networking_v1.read_namespaced_ingress(
|
||
name=ingress_name,
|
||
namespace=self.namespace
|
||
)
|
||
if ingress.status.load_balancer.ingress:
|
||
lb_ingress = ingress.status.load_balancer.ingress[0]
|
||
ingress_ip = lb_ingress.ip or lb_ingress.hostname
|
||
if ingress_ip:
|
||
break
|
||
except ApiException:
|
||
pass
|
||
|
||
time.sleep(interval)
|
||
waited += interval
|
||
logger.info(f" 等待中... ({waited}/{max_wait}秒)")
|
||
|
||
if ingress_ip:
|
||
logger.info(f"✅ Ingress 外网 IP: {ingress_ip}")
|
||
|
||
# 自动创建 Azure DNS 记录
|
||
try:
|
||
logger.info(f"创建 DNS 记录: {domain} -> {ingress_ip}")
|
||
dns_info = self.create_dns_record(
|
||
subdomain=agent_name,
|
||
ip_address=ingress_ip
|
||
)
|
||
logger.info(f"✅ DNS 记录创建成功: {dns_info['domain']} -> {ingress_ip}")
|
||
ingress_info["dns_info"] = dns_info
|
||
ingress_info["ingress_ip"] = ingress_ip
|
||
except Exception as e:
|
||
logger.warning(f"DNS 记录创建失败: {str(e)}")
|
||
logger.info(" DNS 记录可以稍后手动创建")
|
||
ingress_info["ingress_ip"] = ingress_ip
|
||
ingress_info["dns_note"] = f"DNS 未创建: {str(e)}"
|
||
else:
|
||
logger.warning("Ingress IP 分配超时，DNS 记录将在 IP 分配后手动创建")
|
||
ingress_info["dns_note"] = "Ingress IP 正在分配中"
|
||
|
||
except Exception as e:
|
||
logger.warning(f"等待 Ingress IP 或创建 DNS 失败: {str(e)}")
|
||
ingress_info["dns_note"] = f"DNS 创建失败: {str(e)}"
|
||
|
||
except Exception as e:
|
||
logger.warning(f"创建证书或 Ingress 失败: {str(e)}")
|
||
logger.info(" 可以稍后手动创建证书和 Ingress")
|
||
|
||
logger.info(f"✅ OpenClaw 部署创建成功: {agent_name}")
|
||
|
||
# 构建访问信息
|
||
access_info = {
|
||
"note": "OpenClaw 部署正在启动，请稍后通过 /agents/{name}/status 获取状态",
|
||
"ports": {
|
||
"http": 18789,
|
||
"bridge": 18790
|
||
}
|
||
}
|
||
|
||
if ingress_info:
|
||
if ingress_info.get("url"):
|
||
access_info["https_url"] = ingress_info["url"]
|
||
access_info["domain"] = ingress_info.get("host")
|
||
|
||
# 添加 DNS 信息
|
||
if ingress_info.get("dns_info"):
|
||
dns_info = ingress_info["dns_info"]
|
||
access_info["domain"] = dns_info.get("domain")
|
||
access_info["https_url"] = f"https://{dns_info.get('domain')}"
|
||
access_info["ingress_ip"] = ingress_info.get("ingress_ip")
|
||
access_info["note"] = f"访问地址: https://{dns_info.get('domain')} (自签名证书，浏览器会显示安全警告)"
|
||
elif ingress_info.get("ingress_ip"):
|
||
access_info["ingress_ip"] = ingress_info["ingress_ip"]
|
||
access_info["https_url"] = f"https://{ingress_info['ingress_ip']}"
|
||
access_info["domain"] = ingress_info.get("host")
|
||
access_info["note"] = f"访问地址: https://{ingress_info['ingress_ip']} 或 https://{ingress_info.get('host')} (自签名证书，浏览器会显示安全警告)"
|
||
elif ingress_info.get("url"):
|
||
access_info["note"] = f"访问地址: {ingress_info['url']} (自签名证书，浏览器会显示安全警告)"
|
||
|
||
return {
|
||
"name": agent_name,
|
||
"namespace": self.namespace,
|
||
"status": "Pending",
|
||
"template": template,
|
||
"deployment_type": "openclaw",
|
||
"resources": {
|
||
"secret": secret_name,
|
||
"configmap": configmap_name,
|
||
"pvc": pvc_name,
|
||
"deployment": deployment_name,
|
||
"service": service_name if service_info else None,
|
||
"ingress": ingress_info.get("name") if ingress_info else None,
|
||
"tls_secret": cert_info.get("secret_name") if cert_info else None
|
||
},
|
||
"service_port": 18789,
|
||
"access_info": access_info
|
||
}
|
||
|
||
def _generate_openclaw_config(self, agent_name: str, env_vars: Dict) -> str:
|
||
"""生成 OpenClaw 配置文件内容"""
|
||
import json
|
||
|
||
# 默认配置，可通过 env_vars 覆盖
|
||
litellm_api_base = env_vars.get("LITELLM_API_BASE",
|
||
"https://litellm.graystone-fb459c5d.southeastasia.azurecontainerapps.io/v1")
|
||
litellm_model = env_vars.get("LITELLM_MODEL", "taiji/gemini-2.5-flash")
|
||
|
||
config = {
|
||
"meta": {
|
||
"lastTouchedVersion": "2026.2.3"
|
||
},
|
||
"models": {
|
||
"providers": {
|
||
"litellm": {
|
||
"baseUrl": litellm_api_base,
|
||
"apiKey": "${LITELLM_API_KEY}",
|
||
"api": "openai-completions",
|
||
"models": [
|
||
{
|
||
"id": litellm_model,
|
||
"name": litellm_model.split("/")[-1],
|
||
"reasoning": False,
|
||
"input": ["text"],
|
||
"cost": {"input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0},
|
||
"contextWindow": 1000000,
|
||
"maxTokens": 8192
|
||
}
|
||
]
|
||
}
|
||
}
|
||
},
|
||
"agents": {
|
||
"defaults": {
|
||
"model": {
|
||
"primary": f"litellm/{litellm_model}"
|
||
},
|
||
"models": {
|
||
f"litellm/{litellm_model}": {
|
||
"alias": litellm_model.split("/")[-1]
|
||
}
|
||
},
|
||
"workspace": "/home/node/.openclaw/workspace",
|
||
"compaction": {"mode": "safeguard"},
|
||
"maxConcurrent": 4,
|
||
"subagents": {"maxConcurrent": 8},
|
||
"sandbox": {
|
||
"mode": "all",
|
||
"workspaceAccess": "rw",
|
||
"scope": "agent",
|
||
"docker": {
|
||
"image": "openclawacr.azurecr.io/openclaw-sandbox:arm64",
|
||
"network": "bridge"
|
||
},
|
||
"browser": {
|
||
"enabled": True,
|
||
"image": "openclawacr.azurecr.io/openclaw-sandbox-browser:arm64"
|
||
}
|
||
}
|
||
}
|
||
},
|
||
"messages": {
|
||
"ackReactionScope": "group-mentions"
|
||
},
|
||
"commands": {
|
||
"native": "auto",
|
||
"nativeSkills": "auto"
|
||
},
|
||
"channels": {},
|
||
"gateway": {
|
||
"port": 18789,
|
||
"mode": "local",
|
||
"bind": "lan",
|
||
"trustedProxies": ["*"], # 信任所有代理（用于 Ingress/LoadBalancer）
|
||
"auth": {
|
||
"mode": "token",
|
||
"token": "${GATEWAY_AUTH_TOKEN}"
|
||
},
|
||
"controlUi": {
|
||
"dangerouslyDisableDeviceAuth": True # 禁用设备认证，避免 "pairing required" 错误
|
||
},
|
||
"http": {
|
||
"endpoints": {
|
||
"chatCompletions": {"enabled": True}
|
||
}
|
||
}
|
||
},
|
||
"plugins": {
|
||
"entries": {}
|
||
},
|
||
"tools": {
|
||
"sandbox": {
|
||
"tools": {
|
||
"allow": [
|
||
"exec", "process", "read", "write", "edit", "browser",
|
||
"sessions_list", "sessions_history", "sessions_send",
|
||
"sessions_spawn", "session_status"
|
||
],
|
||
"deny": ["canvas", "nodes", "cron", "discord", "gateway"]
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
# 根据环境变量配置渠道
|
||
if env_vars.get("TELEGRAM_BOT_TOKEN"):
|
||
config["channels"]["telegram"] = {
|
||
"enabled": True,
|
||
"dmPolicy": "pairing",
|
||
"botToken": "${TELEGRAM_BOT_TOKEN}",
|
||
"groupPolicy": "allowlist",
|
||
"streamMode": "partial"
|
||
}
|
||
config["plugins"]["entries"]["telegram"] = {"enabled": True}
|
||
|
||
if env_vars.get("FEISHU_APP_ID") and env_vars.get("FEISHU_APP_SECRET"):
|
||
config["channels"]["feishu"] = {
|
||
"appId": "${FEISHU_APP_ID}",
|
||
"appSecret": "${FEISHU_APP_SECRET}",
|
||
"enabled": True,
|
||
"connectionMode": "websocket",
|
||
"dmPolicy": "open",
|
||
"groupPolicy": "open"
|
||
}
|
||
config["plugins"]["entries"]["feishu"] = {"enabled": True}
|
||
# 飞书插件需要安装
|
||
if "installs" not in config["plugins"]:
|
||
config["plugins"]["installs"] = {}
|
||
config["plugins"]["installs"]["feishu"] = {
|
||
"source": "npm",
|
||
"spec": "@m1heng-clawd/feishu",
|
||
"installPath": "/home/node/.openclaw/extensions/feishu",
|
||
"version": "0.1.6"
|
||
}
|
||
|
||
if env_vars.get("DISCORD_BOT_TOKEN"):
|
||
config["channels"]["discord"] = {
|
||
"enabled": True,
|
||
"botToken": "${DISCORD_BOT_TOKEN}"
|
||
}
|
||
config["plugins"]["entries"]["discord"] = {"enabled": True}
|
||
|
||
if env_vars.get("SLACK_BOT_TOKEN"):
|
||
config["channels"]["slack"] = {
|
||
"enabled": True,
|
||
"botToken": "${SLACK_BOT_TOKEN}",
|
||
"appToken": env_vars.get("SLACK_APP_TOKEN", "${SLACK_APP_TOKEN}")
|
||
}
|
||
config["plugins"]["entries"]["slack"] = {"enabled": True}
|
||
|
||
return json.dumps(config, indent=2)
|
||
|
||
def _create_openclaw_secret(self, secret_name: str, env_vars: Dict):
|
||
"""创建 OpenClaw Secret"""
|
||
import hashlib
|
||
import base64
|
||
import json
|
||
|
||
# 默认值
|
||
default_gateway_token = hashlib.sha256(f"{secret_name}-gateway".encode()).hexdigest()
|
||
default_auth_token = hashlib.sha256(f"{secret_name}-auth".encode()).hexdigest()
|
||
|
||
string_data = {
|
||
"OPENCLAW_GATEWAY_TOKEN": env_vars.get("OPENCLAW_GATEWAY_TOKEN", default_gateway_token),
|
||
"LITELLM_API_KEY": env_vars.get("LITELLM_API_KEY", "sk-litellm-default"),
|
||
"GATEWAY_AUTH_TOKEN": env_vars.get("GATEWAY_AUTH_TOKEN", default_auth_token),
|
||
}
|
||
|
||
# 添加可选的渠道 Token
|
||
optional_keys = [
|
||
"TELEGRAM_BOT_TOKEN", "FEISHU_APP_ID", "FEISHU_APP_SECRET",
|
||
"DISCORD_BOT_TOKEN", "SLACK_BOT_TOKEN", "SLACK_APP_TOKEN"
|
||
]
|
||
for key in optional_keys:
|
||
if env_vars.get(key):
|
||
string_data[key] = env_vars[key]
|
||
|
||
# 添加 ACR 凭据用于 DinD
|
||
acr_server = "openclawacr.azurecr.io"
|
||
acr_username = "openclawacr"
|
||
acr_password = "6I2o5Fy29069FvNSTESjFo1GqkYEdBiRNhzpev1iaGYNzOOlNxUxJQQJ99CBACqBBLyEqg7NAAACAZCR2rHa"
|
||
|
||
# 创建 Docker config.json 用于 DinD 容器
|
||
docker_config = {
|
||
"auths": {
|
||
acr_server: {
|
||
"username": acr_username,
|
||
"password": acr_password,
|
||
"auth": base64.b64encode(f"{acr_username}:{acr_password}".encode()).decode()
|
||
}
|
||
}
|
||
}
|
||
string_data["docker-config.json"] = json.dumps(docker_config)
|
||
|
||
try:
|
||
secret = client.V1Secret(
|
||
metadata=client.V1ObjectMeta(
|
||
name=secret_name,
|
||
namespace=self.namespace,
|
||
labels={"managed-by": "agent-manager", "app": "openclaw"}
|
||
),
|
||
string_data=string_data
|
||
)
|
||
|
||
self.v1.create_namespaced_secret(namespace=self.namespace, body=secret)
|
||
logger.info(f"✅ 创建 Secret: {secret_name}")
|
||
except ApiException as e:
|
||
if e.status == 409:
|
||
logger.info(f"Secret {secret_name} 已存在，尝试更新")
|
||
self.v1.replace_namespaced_secret(name=secret_name, namespace=self.namespace, body=secret)
|
||
else:
|
||
raise
|
||
|
||
def _create_openclaw_configmap(self, configmap_name: str, config_content: str):
|
||
"""创建 OpenClaw ConfigMap"""
|
||
try:
|
||
configmap = client.V1ConfigMap(
|
||
metadata=client.V1ObjectMeta(
|
||
name=configmap_name,
|
||
namespace=self.namespace,
|
||
labels={"managed-by": "agent-manager", "app": "openclaw"}
|
||
),
|
||
data={"openclaw.json": config_content}
|
||
)
|
||
|
||
self.v1.create_namespaced_config_map(namespace=self.namespace, body=configmap)
|
||
logger.info(f"✅ 创建 ConfigMap: {configmap_name}")
|
||
except ApiException as e:
|
||
if e.status == 409:
|
||
logger.info(f"ConfigMap {configmap_name} 已存在，尝试更新")
|
||
self.v1.replace_namespaced_config_map(name=configmap_name, namespace=self.namespace, body=configmap)
|
||
else:
|
||
raise
|
||
|
||
def _create_openclaw_pvc(self, pvc_name: str, storage_size: str = "10Gi"):
|
||
"""创建 OpenClaw PVC"""
|
||
try:
|
||
pvc = client.V1PersistentVolumeClaim(
|
||
metadata=client.V1ObjectMeta(
|
||
name=pvc_name,
|
||
namespace=self.namespace,
|
||
labels={"managed-by": "agent-manager", "app": "openclaw"}
|
||
),
|
||
spec=client.V1PersistentVolumeClaimSpec(
|
||
access_modes=["ReadWriteOnce"],
|
||
storage_class_name="managed-csi", # Azure AKS 默认存储类
|
||
resources=client.V1ResourceRequirements(
|
||
requests={"storage": storage_size}
|
||
)
|
||
)
|
||
)
|
||
|
||
self.v1.create_namespaced_persistent_volume_claim(namespace=self.namespace, body=pvc)
|
||
logger.info(f"✅ 创建 PVC: {pvc_name}")
|
||
except ApiException as e:
|
||
if e.status == 409:
|
||
logger.info(f"PVC {pvc_name} 已存在，跳过创建")
|
||
else:
|
||
raise
|
||
|
||
def _create_openclaw_deployment_resource(
|
||
self,
|
||
deployment_name: str,
|
||
template: str,
|
||
secret_name: str,
|
||
configmap_name: str,
|
||
pvc_name: str,
|
||
config_data: Dict
|
||
):
|
||
"""创建 OpenClaw Deployment (包含 init container + gateway + dind sidecar)"""
|
||
|
||
# OpenClaw 主镜像（沙箱镜像在配置中自动引用）
|
||
gateway_image = "openclawacr.azurecr.io/openclaw:latest"
|
||
|
||
# 资源配置
|
||
gateway_resources = client.V1ResourceRequirements(
|
||
requests={"memory": "512Mi", "cpu": "250m"},
|
||
limits={"memory": "2Gi", "cpu": "2000m"}
|
||
)
|
||
|
||
dind_resources = client.V1ResourceRequirements(
|
||
requests={"memory": "512Mi", "cpu": "250m"},
|
||
limits={"memory": "4Gi", "cpu": "2000m"}
|
||
)
|
||
|
||
# Secret 环境变量引用
|
||
secret_env_vars = [
|
||
client.V1EnvVar(
|
||
name="LITELLM_API_KEY",
|
||
value_from=client.V1EnvVarSource(
|
||
secret_key_ref=client.V1SecretKeySelector(name=secret_name, key="LITELLM_API_KEY")
|
||
)
|
||
),
|
||
client.V1EnvVar(
|
||
name="OPENCLAW_GATEWAY_TOKEN",
|
||
value_from=client.V1EnvVarSource(
|
||
secret_key_ref=client.V1SecretKeySelector(name=secret_name, key="OPENCLAW_GATEWAY_TOKEN")
|
||
)
|
||
),
|
||
client.V1EnvVar(
|
||
name="GATEWAY_AUTH_TOKEN",
|
||
value_from=client.V1EnvVarSource(
|
||
secret_key_ref=client.V1SecretKeySelector(name=secret_name, key="GATEWAY_AUTH_TOKEN")
|
||
)
|
||
),
|
||
]
|
||
|
||
# 可选的渠道 Token
|
||
optional_secret_keys = [
|
||
"TELEGRAM_BOT_TOKEN", "FEISHU_APP_ID", "FEISHU_APP_SECRET",
|
||
"DISCORD_BOT_TOKEN", "SLACK_BOT_TOKEN", "SLACK_APP_TOKEN"
|
||
]
|
||
env_vars = config_data.get("env", {})
|
||
for key in optional_secret_keys:
|
||
if env_vars.get(key):
|
||
secret_env_vars.append(
|
||
client.V1EnvVar(
|
||
name=key,
|
||
value_from=client.V1EnvVarSource(
|
||
secret_key_ref=client.V1SecretKeySelector(
|
||
name=secret_name, key=key, optional=True
|
||
)
|
||
)
|
||
)
|
||
)
|
||
|
||
# Init Containers 列表
|
||
init_containers = []
|
||
|
||
# Init Container 0: 创建目录（以 root 权限）
|
||
dir_init_container = client.V1Container(
|
||
name="dir-init",
|
||
image="busybox:1.36",
|
||
command=["sh", "-c", """
|
||
echo "=== Creating directories with proper permissions ==="
|
||
mkdir -p /data/extensions /data/workspace /data/sandboxes /data/agents /data/canvas /data/cron /data/sandbox /data/devices
|
||
chown -R 1000:1000 /data
|
||
chmod -R 755 /data
|
||
ls -la /data/
|
||
echo "Directories created successfully"
|
||
"""],
|
||
volume_mounts=[
|
||
client.V1VolumeMount(name="data", mount_path="/data"),
|
||
]
|
||
)
|
||
init_containers.append(dir_init_container)
|
||
|
||
# Init Container 1: 插件安装（如果配置了飞书）
|
||
if env_vars.get("FEISHU_APP_ID") and env_vars.get("FEISHU_APP_SECRET"):
|
||
plugin_install_container = client.V1Container(
|
||
name="plugin-install",
|
||
image=gateway_image, # 使用 OpenClaw 镜像
|
||
command=["sh", "-c", """
|
||
echo "=== Installing Feishu plugin ==="
|
||
cd /home/node
|
||
|
||
# 安装飞书插件到持久化目录
|
||
if [ ! -d "/data/extensions/feishu/node_modules" ]; then
|
||
echo "Installing @m1heng-clawd/feishu plugin..."
|
||
npm pack @m1heng-clawd/feishu --pack-destination /tmp
|
||
mkdir -p /data/extensions/feishu
|
||
tar -xzf /tmp/m1heng-clawd-feishu-*.tgz -C /data/extensions/feishu --strip-components=1
|
||
cd /data/extensions/feishu && npm install --production
|
||
echo "Plugin installed successfully"
|
||
else
|
||
echo "Plugin already installed, skipping..."
|
||
fi
|
||
|
||
ls -la /data/extensions/feishu/ || true
|
||
"""],
|
||
volume_mounts=[
|
||
client.V1VolumeMount(name="data", mount_path="/data"),
|
||
]
|
||
)
|
||
init_containers.append(plugin_install_container)
|
||
logger.info("✅ 添加飞书插件安装 init container")
|
||
|
||
# Init Container 2: 配置初始化
|
||
config_init_container = client.V1Container(
|
||
name="config-init",
|
||
image="busybox:1.36",
|
||
command=["sh", "-c", """
|
||
cp /config-template/openclaw.json /config/openclaw.json
|
||
sed -i "s|\\${LITELLM_API_KEY}|$LITELLM_API_KEY|g" /config/openclaw.json
|
||
sed -i "s|\\${FEISHU_APP_ID}|$FEISHU_APP_ID|g" /config/openclaw.json
|
||
sed -i "s|\\${FEISHU_APP_SECRET}|$FEISHU_APP_SECRET|g" /config/openclaw.json
|
||
sed -i "s|\\${TELEGRAM_BOT_TOKEN}|$TELEGRAM_BOT_TOKEN|g" /config/openclaw.json
|
||
sed -i "s|\\${GATEWAY_AUTH_TOKEN}|$GATEWAY_AUTH_TOKEN|g" /config/openclaw.json
|
||
sed -i "s|\\${DISCORD_BOT_TOKEN}|$DISCORD_BOT_TOKEN|g" /config/openclaw.json
|
||
sed -i "s|\\${SLACK_BOT_TOKEN}|$SLACK_BOT_TOKEN|g" /config/openclaw.json
|
||
sed -i "s|\\${SLACK_APP_TOKEN}|$SLACK_APP_TOKEN|g" /config/openclaw.json
|
||
mkdir -p /data/workspace /data/sandboxes /data/extensions
|
||
echo "Config initialized successfully"
|
||
"""],
|
||
env=secret_env_vars,
|
||
volume_mounts=[
|
||
client.V1VolumeMount(name="config-template", mount_path="/config-template"),
|
||
client.V1VolumeMount(name="config", mount_path="/config"),
|
||
client.V1VolumeMount(name="data", mount_path="/data"),
|
||
]
|
||
)
|
||
init_containers.append(config_init_container)
|
||
|
||
# Init Container 2: 初始化 identity volume 权限
|
||
# 由于 gateway 容器需要创建 identity 目录，使用 emptyDir volume 避免权限问题
|
||
# 在 initContainer 中预先创建并设置正确的权限
|
||
permission_fix_container = client.V1Container(
|
||
name="permission-fix",
|
||
image="busybox:1.36",
|
||
command=["sh", "-c", """
|
||
echo "=== Initializing identity volume permissions ==="
|
||
# 创建 identity 目录并设置正确的权限（emptyDir volume）
|
||
mkdir -p /identity
|
||
chown -R 1000:1000 /identity
|
||
chmod -R 755 /identity
|
||
ls -la /identity/ 2>/dev/null || echo "Directory check completed"
|
||
echo "Identity volume initialized successfully"
|
||
"""],
|
||
security_context=client.V1SecurityContext(
|
||
run_as_user=0, # 以 root 运行
|
||
run_as_group=0
|
||
),
|
||
volume_mounts=[
|
||
client.V1VolumeMount(
|
||
name="identity",
|
||
mount_path="/identity"
|
||
),
|
||
]
|
||
)
|
||
init_containers.append(permission_fix_container)
|
||
|
||
# Gateway Container
|
||
gateway_container = client.V1Container(
|
||
name="gateway",
|
||
image=gateway_image,
|
||
ports=[
|
||
client.V1ContainerPort(container_port=18789, name="http"),
|
||
client.V1ContainerPort(container_port=18790, name="bridge"),
|
||
],
|
||
env=[
|
||
client.V1EnvVar(name="HOME", value="/home/node"),
|
||
client.V1EnvVar(name="TERM", value="xterm-256color"),
|
||
client.V1EnvVar(name="DOCKER_HOST", value="tcp://localhost:2375"),
|
||
] + secret_env_vars,
|
||
volume_mounts=[
|
||
client.V1VolumeMount(
|
||
name="config",
|
||
mount_path="/home/node/.openclaw/openclaw.json",
|
||
sub_path="openclaw.json"
|
||
),
|
||
client.V1VolumeMount(
|
||
name="data",
|
||
mount_path="/home/node/.openclaw/workspace",
|
||
sub_path="workspace"
|
||
),
|
||
client.V1VolumeMount(
|
||
name="data",
|
||
mount_path="/home/node/.openclaw/sandboxes",
|
||
sub_path="sandboxes"
|
||
),
|
||
client.V1VolumeMount(
|
||
name="data",
|
||
mount_path="/home/node/.openclaw/extensions",
|
||
sub_path="extensions"
|
||
),
|
||
client.V1VolumeMount(
|
||
name="data",
|
||
mount_path="/home/node/.openclaw/agents",
|
||
sub_path="agents"
|
||
),
|
||
client.V1VolumeMount(
|
||
name="data",
|
||
mount_path="/home/node/.openclaw/canvas",
|
||
sub_path="canvas"
|
||
),
|
||
client.V1VolumeMount(
|
||
name="data",
|
||
mount_path="/home/node/.openclaw/cron",
|
||
sub_path="cron"
|
||
),
|
||
client.V1VolumeMount(
|
||
name="data",
|
||
mount_path="/home/node/.openclaw/sandbox",
|
||
sub_path="sandbox"
|
||
),
|
||
client.V1VolumeMount(
|
||
name="data",
|
||
mount_path="/home/node/.openclaw/devices",
|
||
sub_path="devices"
|
||
),
|
||
# 添加 identity volume 挂载，用于存储设备身份信息
|
||
client.V1VolumeMount(
|
||
name="identity",
|
||
mount_path="/home/node/.openclaw/identity"
|
||
),
|
||
],
|
||
command=["node", "dist/index.js", "gateway", "--bind", "lan", "--port", "18789"],
|
||
resources=gateway_resources,
|
||
readiness_probe=client.V1Probe(
|
||
http_get=client.V1HTTPGetAction(path="/health", port=18789),
|
||
initial_delay_seconds=10,
|
||
period_seconds=10
|
||
),
|
||
liveness_probe=client.V1Probe(
|
||
http_get=client.V1HTTPGetAction(path="/health", port=18789),
|
||
initial_delay_seconds=30,
|
||
period_seconds=30
|
||
)
|
||
)
|
||
|
||
# DinD Sidecar Container - 带有自动拉取沙箱镜像的 postStart hook
|
||
dind_container = client.V1Container(
|
||
name="dind",
|
||
image="docker:24-dind",
|
||
security_context=client.V1SecurityContext(privileged=True),
|
||
env=[client.V1EnvVar(name="DOCKER_TLS_CERTDIR", value="")],
|
||
ports=[client.V1ContainerPort(container_port=2375, name="docker")],
|
||
volume_mounts=[
|
||
client.V1VolumeMount(name="docker-storage", mount_path="/var/lib/docker"),
|
||
client.V1VolumeMount(
|
||
name="data",
|
||
mount_path="/home/node/.openclaw/workspace",
|
||
sub_path="workspace"
|
||
),
|
||
client.V1VolumeMount(
|
||
name="data",
|
||
mount_path="/home/node/.openclaw/sandboxes",
|
||
sub_path="sandboxes"
|
||
),
|
||
# 挂载 Docker config 用于 ACR 认证
|
||
client.V1VolumeMount(
|
||
name="docker-config",
|
||
mount_path="/root/.docker/config.json",
|
||
sub_path="docker-config.json"
|
||
),
|
||
],
|
||
resources=dind_resources,
|
||
# 自动拉取沙箱镜像的生命周期钩子
|
||
lifecycle=client.V1Lifecycle(
|
||
post_start=client.V1LifecycleHandler(
|
||
_exec=client.V1ExecAction(
|
||
command=[
|
||
"/bin/sh", "-c",
|
||
"""
|
||
# 等待 Docker daemon 启动
|
||
for i in $(seq 1 30); do
|
||
docker info > /dev/null 2>&1 && break
|
||
sleep 2
|
||
done
|
||
# 后台拉取沙箱镜像
|
||
(docker pull openclawacr.azurecr.io/openclaw-sandbox:arm64 && \
|
||
docker pull openclawacr.azurecr.io/openclaw-sandbox-browser:arm64) &
|
||
"""
|
||
]
|
||
)
|
||
)
|
||
)
|
||
)
|
||
|
||
# Volumes
|
||
volumes = [
|
||
client.V1Volume(
|
||
name="config-template",
|
||
config_map=client.V1ConfigMapVolumeSource(name=configmap_name)
|
||
),
|
||
client.V1Volume(name="config", empty_dir=client.V1EmptyDirVolumeSource()),
|
||
client.V1Volume(
|
||
name="data",
|
||
persistent_volume_claim=client.V1PersistentVolumeClaimVolumeSource(claim_name=pvc_name)
|
||
),
|
||
client.V1Volume(name="docker-storage", empty_dir=client.V1EmptyDirVolumeSource()),
|
||
# Identity volume: 用于存储设备身份信息，使用 emptyDir 避免权限问题
|
||
client.V1Volume(name="identity", empty_dir=client.V1EmptyDirVolumeSource()),
|
||
# Docker config 从 secret 挂载，用于 DinD ACR 认证
|
||
client.V1Volume(
|
||
name="docker-config",
|
||
secret=client.V1SecretVolumeSource(
|
||
secret_name=secret_name,
|
||
items=[client.V1KeyToPath(key="docker-config.json", path="docker-config.json")]
|
||
)
|
||
),
|
||
]
|
||
|
||
# Image Pull Secrets
|
||
image_pull_secrets = [
|
||
client.V1LocalObjectReference(name="openclaw-acr-secret"),
|
||
client.V1LocalObjectReference(name="acr-secret")
|
||
]
|
||
|
||
# Pod Spec
|
||
pod_spec = client.V1PodSpec(
|
||
init_containers=init_containers,
|
||
containers=[gateway_container, dind_container],
|
||
volumes=volumes,
|
||
image_pull_secrets=image_pull_secrets,
|
||
restart_policy="Always"
|
||
)
|
||
|
||
# Deployment
|
||
deployment = client.V1Deployment(
|
||
metadata=client.V1ObjectMeta(
|
||
name=deployment_name,
|
||
namespace=self.namespace,
|
||
labels={
|
||
"app": deployment_name,
|
||
"template": template,
|
||
"managed-by": "agent-manager"
|
||
}
|
||
),
|
||
spec=client.V1DeploymentSpec(
|
||
replicas=1,
|
||
selector=client.V1LabelSelector(
|
||
match_labels={"app": deployment_name}
|
||
),
|
||
template=client.V1PodTemplateSpec(
|
||
metadata=client.V1ObjectMeta(
|
||
labels={
|
||
"app": deployment_name,
|
||
"template": template,
|
||
"managed-by": "agent-manager"
|
||
}
|
||
),
|
||
spec=pod_spec
|
||
)
|
||
)
|
||
)
|
||
|
||
try:
|
||
self.apps_v1.create_namespaced_deployment(namespace=self.namespace, body=deployment)
|
||
logger.info(f"✅ 创建 Deployment: {deployment_name}")
|
||
except ApiException as e:
|
||
if e.status == 409:
|
||
logger.info(f"Deployment {deployment_name} 已存在，尝试更新")
|
||
self.apps_v1.replace_namespaced_deployment(
|
||
name=deployment_name, namespace=self.namespace, body=deployment
|
||
)
|
||
else:
|
||
raise
|
||
|
||
def delete_openclaw_deployment(self, agent_name: str) -> Dict:
|
||
"""删除 OpenClaw 部署及相关资源"""
|
||
results = {"deleted": [], "errors": []}
|
||
|
||
resources = [
|
||
("deployment", f"{agent_name}"),
|
||
("pvc", f"{agent_name}-data"),
|
||
("configmap", f"{agent_name}-config"),
|
||
("secret", f"{agent_name}-secrets"),
|
||
]
|
||
|
||
for resource_type, resource_name in resources:
|
||
try:
|
||
if resource_type == "deployment":
|
||
self.apps_v1.delete_namespaced_deployment(
|
||
name=resource_name, namespace=self.namespace
|
||
)
|
||
elif resource_type == "pvc":
|
||
self.v1.delete_namespaced_persistent_volume_claim(
|
||
name=resource_name, namespace=self.namespace
|
||
)
|
||
elif resource_type == "configmap":
|
||
self.v1.delete_namespaced_config_map(
|
||
name=resource_name, namespace=self.namespace
|
||
)
|
||
elif resource_type == "secret":
|
||
self.v1.delete_namespaced_secret(
|
||
name=resource_name, namespace=self.namespace
|
||
)
|
||
results["deleted"].append(f"{resource_type}/{resource_name}")
|
||
logger.info(f"✅ 删除 {resource_type}: {resource_name}")
|
||
except ApiException as e:
|
||
if e.status == 404:
|
||
logger.info(f"{resource_type} {resource_name} 不存在，跳过")
|
||
else:
|
||
results["errors"].append(f"{resource_type}/{resource_name}: {e.reason}")
|
||
logger.error(f"删除 {resource_type} {resource_name} 失败: {e}")
|
||
|
||
return results
|
||
|
||
def get_pod_metrics(self, pod_name: str) -> Dict:
|
||
"""
|
||
获取Pod资源使用情况（CPU、内存）
|
||
|
||
Args:
|
||
pod_name: Pod名称
|
||
|
||
Returns:
|
||
Pod实时资源使用信息和配额信息
|
||
"""
|
||
try:
|
||
# 获取 Pod 配额信息
|
||
pod = self.v1.read_namespaced_pod(
|
||
name=pod_name,
|
||
namespace=self.namespace
|
||
)
|
||
|
||
container = pod.spec.containers[0]
|
||
resources = container.resources
|
||
|
||
result = {
|
||
"name": pod_name,
|
||
"namespace": self.namespace,
|
||
"requests": {
|
||
"cpu": resources.requests.get("cpu") if resources.requests else None,
|
||
"memory": resources.requests.get("memory") if resources.requests else None
|
||
},
|
||
"limits": {
|
||
"cpu": resources.limits.get("cpu") if resources.limits else None,
|
||
"memory": resources.limits.get("memory") if resources.limits else None
|
||
}
|
||
}
|
||
|
||
# 尝试获取实时使用情况（需要 metrics-server）
|
||
try:
|
||
from kubernetes.client import CustomObjectsApi
|
||
custom_api = CustomObjectsApi()
|
||
|
||
# 调用 metrics.k8s.io API
|
||
metrics = custom_api.get_namespaced_custom_object(
|
||
group="metrics.k8s.io",
|
||
version="v1beta1",
|
||
namespace=self.namespace,
|
||
plural="pods",
|
||
name=pod_name
|
||
)
|
||
|
||
# 提取实时使用数据
|
||
if metrics and "containers" in metrics:
|
||
container_metrics = metrics["containers"][0]
|
||
usage = container_metrics.get("usage", {})
|
||
|
||
result["usage"] = {
|
||
"cpu": usage.get("cpu"),
|
||
"memory": usage.get("memory")
|
||
}
|
||
result["timestamp"] = metrics.get("timestamp")
|
||
logger.info(f"✅ 获取到实时资源使用: CPU={usage.get('cpu')}, Memory={usage.get('memory')}")
|
||
else:
|
||
result["usage"] = None
|
||
logger.warning(f"⚠️ Metrics 数据格式异常")
|
||
|
||
except Exception as metrics_error:
|
||
logger.warning(f"⚠️ 无法获取实时资源使用（可能未安装 metrics-server）: {str(metrics_error)}")
|
||
result["usage"] = None
|
||
result["metrics_available"] = False
|
||
|
||
return result
|
||
|
||
except ApiException as e:
|
||
logger.error(f"获取Pod资源信息失败: {e}")
|
||
raise Exception(f"获取Pod资源信息失败: {e.reason}")
|
||
|
||
def list_pods(self, label_selector: Optional[str] = None) -> List[Dict]:
|
||
"""
|
||
列出所有Pod
|
||
|
||
Args:
|
||
label_selector: 标签选择器（可选）
|
||
|
||
Returns:
|
||
Pod列表
|
||
"""
|
||
try:
|
||
if label_selector is None:
|
||
label_selector = "managed-by=agent-manager"
|
||
|
||
pods = self.v1.list_namespaced_pod(
|
||
namespace=self.namespace,
|
||
label_selector=label_selector
|
||
)
|
||
|
||
return [
|
||
{
|
||
"name": pod.metadata.name,
|
||
"status": pod.status.phase,
|
||
"template": pod.metadata.labels.get("template", "unknown"),
|
||
"created_at": pod.metadata.creation_timestamp.isoformat() if pod.metadata.creation_timestamp else None,
|
||
"pod_ip": pod.status.pod_ip
|
||
}
|
||
for pod in pods.items
|
||
]
|
||
except ApiException as e:
|
||
logger.error(f"列出Pod失败: {e}")
|
||
raise Exception(f"列出Pod失败: {e.reason}")
|
||
|
||
# ============================================================================
|
||
# Sub-mode runtime helper methods
|
||
# ============================================================================
|
||
|
||
def create_swarm_namespace(self, swarm_id: str, role: str) -> str:
|
||
"""
|
||
Create namespace for swarm agent.
|
||
|
||
Args:
|
||
swarm_id: Swarm ID
|
||
role: Agent role
|
||
|
||
Returns:
|
||
Namespace name
|
||
"""
|
||
namespace_name = sanitize_k8s_name(f"swarm-{swarm_id[:8]}-{role}")
|
||
|
||
try:
|
||
# Check if namespace exists
|
||
try:
|
||
self.v1.read_namespace(name=namespace_name)
|
||
logger.info(f"Namespace {namespace_name} already exists")
|
||
return namespace_name
|
||
except ApiException as e:
|
||
if e.status != 404:
|
||
raise
|
||
|
||
# Create namespace
|
||
namespace = client.V1Namespace(
|
||
metadata=client.V1ObjectMeta(
|
||
name=namespace_name,
|
||
labels={
|
||
"managed-by": "agent-manager",
|
||
"swarm-id": swarm_id[:8],
|
||
"agent-role": role
|
||
}
|
||
)
|
||
)
|
||
|
||
self.v1.create_namespace(body=namespace)
|
||
logger.info(f"✅ Created namespace: {namespace_name}")
|
||
self._copy_acr_secret_to_namespace(namespace_name)
|
||
return namespace_name
|
||
|
||
except ApiException as e:
|
||
logger.error(f"Failed to create namespace {namespace_name}: {e}")
|
||
raise Exception(f"Failed to create namespace: {e.reason}")
|
||
|
||
def deploy_swarm_agent(
|
||
self,
|
||
swarm_id: str,
|
||
agent_id: str,
|
||
agent_config: Dict,
|
||
namespace: str
|
||
) -> Dict:
|
||
"""
|
||
Deploy swarm agent pod.
|
||
|
||
Args:
|
||
swarm_id: Swarm ID
|
||
agent_id: Agent ID
|
||
agent_config: Agent configuration
|
||
namespace: Namespace
|
||
|
||
Returns:
|
||
Deployment info {pod_name, service_url, external_ip}
|
||
"""
|
||
try:
|
||
pod_name = sanitize_k8s_name(f"agent-{agent_id}")
|
||
template = agent_config.get("template", "a2a_litellm_agent")
|
||
role = agent_config.get("role", "worker")
|
||
model = agent_config.get("model", "gpt-4")
|
||
billing_context = agent_config.get("billing_context") or {}
|
||
|
||
# Get template image
|
||
# TODO: Load from template database
|
||
image_map = {
|
||
"a2a_litellm_agent": "agnettaiji.azurecr.io/ai-agents/a2a-litellm-agent:heicode-v2-gpt54-202605300145-arm64",
|
||
"code_manager_agent": "agnettaiji.azurecr.io/ai-agents/code-manager-agent:latest"
|
||
}
|
||
image = image_map.get(template, image_map["a2a_litellm_agent"])
|
||
|
||
# Environment variables
|
||
env_vars = [
|
||
client.V1EnvVar(name="SWARM_ID", value=swarm_id),
|
||
client.V1EnvVar(name="AGENT_ID", value=agent_id),
|
||
client.V1EnvVar(name="AGENT_ROLE", value=role),
|
||
client.V1EnvVar(name="MODEL_NAME", value=model),
|
||
client.V1EnvVar(name="POD_NAME", value=pod_name),
|
||
client.V1EnvVar(name="NAMESPACE", value=namespace),
|
||
]
|
||
|
||
gateway_url = (
|
||
billing_context.get("model_gateway_url")
|
||
or (
|
||
os.getenv("HEICODE_NEWAPI_BASE_URL")
|
||
if billing_context.get("provider") == "newapi"
|
||
else os.getenv("LITELLM_BASE_URL")
|
||
)
|
||
or os.getenv("HEICODE_NEWAPI_BASE_URL")
|
||
or os.getenv("LITELLM_BASE_URL")
|
||
)
|
||
if gateway_url:
|
||
env_vars.extend([
|
||
client.V1EnvVar(name="LITELLM_BASE_URL", value=gateway_url),
|
||
client.V1EnvVar(name="LLM_BASE_URL", value=gateway_url),
|
||
client.V1EnvVar(name="OPENAI_BASE_URL", value=gateway_url),
|
||
])
|
||
|
||
# Add model API key from Runtime environment. Do not use service
|
||
# auth tokens as model gateway credentials.
|
||
model_api_key = (
|
||
os.getenv("LITELLM_API_KEY")
|
||
or os.getenv("LITELLM_USER_KEY")
|
||
or os.getenv("MODEL_GATEWAY_API_KEY")
|
||
or os.getenv("HEICODE_NEWAPI_USER_TOKEN")
|
||
or os.getenv("OPENAI_API_KEY")
|
||
or os.getenv("LLM_API_KEY")
|
||
)
|
||
if model_api_key:
|
||
env_vars.extend([
|
||
client.V1EnvVar(name="LITELLM_API_KEY", value=model_api_key),
|
||
client.V1EnvVar(name="LLM_API_KEY", value=model_api_key),
|
||
client.V1EnvVar(name="OPENAI_API_KEY", value=model_api_key),
|
||
])
|
||
|
||
# Create pod
|
||
pod = client.V1Pod(
|
||
metadata=client.V1ObjectMeta(
|
||
name=pod_name,
|
||
namespace=namespace,
|
||
labels={
|
||
"app": pod_name,
|
||
"managed-by": "agent-manager",
|
||
"swarm-id": swarm_id[:8],
|
||
"agent-id": agent_id,
|
||
"agent-role": role
|
||
}
|
||
),
|
||
spec=client.V1PodSpec(
|
||
containers=[
|
||
client.V1Container(
|
||
name="agent",
|
||
image=image,
|
||
ports=[client.V1ContainerPort(container_port=8000)],
|
||
env=env_vars,
|
||
resources=client.V1ResourceRequirements(
|
||
requests={"cpu": "100m", "memory": "256Mi"},
|
||
limits={"cpu": "500m", "memory": "512Mi"}
|
||
)
|
||
)
|
||
],
|
||
image_pull_secrets=[client.V1LocalObjectReference(name="acr-secret")]
|
||
)
|
||
)
|
||
|
||
try:
|
||
self.v1.create_namespaced_pod(namespace=namespace, body=pod)
|
||
logger.info(f"✅ Created pod: {pod_name} in namespace {namespace}")
|
||
except ApiException as e:
|
||
if e.status != 409:
|
||
raise
|
||
logger.info(f"Pod {pod_name} already exists in namespace {namespace}")
|
||
|
||
# Create service
|
||
service = client.V1Service(
|
||
metadata=client.V1ObjectMeta(
|
||
name=pod_name,
|
||
namespace=namespace,
|
||
labels={"app": pod_name}
|
||
),
|
||
spec=client.V1ServiceSpec(
|
||
selector={"app": pod_name},
|
||
ports=[client.V1ServicePort(port=8000, target_port=8000)],
|
||
type="ClusterIP"
|
||
)
|
||
)
|
||
|
||
try:
|
||
self.v1.create_namespaced_service(namespace=namespace, body=service)
|
||
logger.info(f"✅ Created service: {pod_name} in namespace {namespace}")
|
||
except ApiException as e:
|
||
if e.status != 409:
|
||
raise
|
||
logger.info(f"Service {pod_name} already exists in namespace {namespace}")
|
||
|
||
deadline = time.time() + 120
|
||
last_phase = "Unknown"
|
||
while time.time() < deadline:
|
||
current_pod = self.v1.read_namespaced_pod(name=pod_name, namespace=namespace)
|
||
last_phase = current_pod.status.phase or "Unknown"
|
||
container_statuses = current_pod.status.container_statuses or []
|
||
ready = any(status.ready for status in container_statuses)
|
||
if last_phase == "Running" and ready:
|
||
logger.info(f"✅ Pod {pod_name} is ready in namespace {namespace}")
|
||
break
|
||
if last_phase in {"Failed", "Unknown"}:
|
||
raise Exception(f"Pod {pod_name} entered phase {last_phase}")
|
||
for status in container_statuses:
|
||
waiting = status.state.waiting if status.state else None
|
||
terminated = status.state.terminated if status.state else None
|
||
if waiting and waiting.reason in {"CrashLoopBackOff", "ImagePullBackOff", "ErrImagePull"}:
|
||
raise Exception(f"Pod {pod_name} is not ready: {waiting.reason}")
|
||
if terminated and terminated.exit_code != 0:
|
||
raise Exception(f"Pod {pod_name} exited with code {terminated.exit_code}")
|
||
time.sleep(2)
|
||
else:
|
||
raise Exception(f"Timed out waiting for pod {pod_name} readiness; last phase={last_phase}")
|
||
|
||
health_deadline = time.time() + 60
|
||
health_url = f"http://{pod_name}.{namespace}.svc.cluster.local:8000/health"
|
||
last_health_error = None
|
||
while time.time() < health_deadline:
|
||
try:
|
||
response = requests.get(health_url, timeout=3)
|
||
if response.status_code == 200:
|
||
logger.info(f"✅ Pod {pod_name} HTTP endpoint is reachable")
|
||
break
|
||
last_health_error = f"status={response.status_code}"
|
||
except Exception as exc:
|
||
last_health_error = str(exc)
|
||
time.sleep(2)
|
||
else:
|
||
raise Exception(f"Timed out waiting for pod {pod_name} HTTP readiness: {last_health_error}")
|
||
|
||
service_url = f"http://{pod_name}.{namespace}.svc.cluster.local:8000"
|
||
|
||
return {
|
||
"pod_name": pod_name,
|
||
"service_url": service_url,
|
||
"external_ip": None,
|
||
"namespace": namespace
|
||
}
|
||
|
||
except ApiException as e:
|
||
logger.error(f"Failed to deploy swarm agent: {e}")
|
||
raise Exception(f"Failed to deploy swarm agent: {e.reason}")
|
||
|
||
def cleanup_swarm_resources(self, swarm_id: str):
|
||
"""
|
||
Cleanup all K8s resources for a swarm.
|
||
|
||
Args:
|
||
swarm_id: Swarm ID
|
||
"""
|
||
try:
|
||
# List all namespaces with swarm-id label
|
||
namespaces = self.v1.list_namespace(
|
||
label_selector=f"swarm-id={swarm_id[:8]}"
|
||
)
|
||
|
||
for ns in namespaces.items:
|
||
namespace_name = ns.metadata.name
|
||
logger.info(f"Deleting namespace: {namespace_name}")
|
||
|
||
# Delete namespace (this will delete all resources in it)
|
||
self.v1.delete_namespace(name=namespace_name)
|
||
logger.info(f"✅ Deleted namespace: {namespace_name}")
|
||
|
||
logger.info(f"✅ Cleaned up all resources for swarm {swarm_id}")
|
||
|
||
except ApiException as e:
|
||
logger.error(f"Failed to cleanup swarm resources: {e}")
|
||
raise Exception(f"Failed to cleanup swarm resources: {e.reason}")
|
||
|
||
def get_swarm_agent_logs(self, namespace: str, pod_name: str, tail_lines: int = 100) -> str:
|
||
"""
|
||
Get logs from swarm agent pod.
|
||
|
||
Args:
|
||
namespace: Namespace
|
||
pod_name: Pod name
|
||
tail_lines: Number of lines to tail
|
||
|
||
Returns:
|
||
Pod logs
|
||
"""
|
||
try:
|
||
logs = self.v1.read_namespaced_pod_log(
|
||
name=pod_name,
|
||
namespace=namespace,
|
||
tail_lines=tail_lines
|
||
)
|
||
return logs
|
||
|
||
except ApiException as e:
|
||
logger.error(f"Failed to get pod logs: {e}")
|
||
raise Exception(f"Failed to get pod logs: {e.reason}")
|