diff --git a/k8s/DEPLOYMENT_SUMMARY.md b/k8s/DEPLOYMENT_SUMMARY.md new file mode 100644 index 0000000..3a2c047 --- /dev/null +++ b/k8s/DEPLOYMENT_SUMMARY.md @@ -0,0 +1,129 @@ +# AKS 部署总结 + +## ✅ 部署状态 + +### 已成功部署的服务 + +1. **MCP Server** ✅ + - 状态: 运行中 + - 健康检查: 通过 + - 服务端点: `http://mcp-server:8000` (集群内) + - 端口转发: `kubectl port-forward svc/mcp-server 8002:8000 -n taiji-ai` + +2. **LiteLLM Gateway** ✅ + - 状态: 运行中 + - 健康检查: 通过(OpenRouter 账户需要充值) + - 服务端点: `http://litellm-gateway:4000` (集群内) + - 端口转发: `kubectl port-forward svc/litellm-gateway 4000:4000 -n taiji-ai` + +3. **Data Ingestion** ✅ + - 状态: 运行中 + - 服务端点: `http://data-ingestion:8000` (集群内) + - 端口转发: `kubectl port-forward svc/data-ingestion 8001:8000 -n taiji-ai` + +### 需要关注的服务 + +1. **NATS** ⚠️ + - 状态: 部分 Pod 在创建中 + - 问题: 旧 Pod 使用 amd64 镜像导致失败,新 Pod 正在创建 + - 建议: 等待新 Pod 完全启动后,删除旧的 CrashLoopBackOff Pod + +2. **Prometheus & Grafana** ⚠️ + - 状态: PVC 挂载问题 + - 建议: 检查存储类配置 + +## 📊 部署信息 + +- **AKS 集群**: taiji-ai-pda +- **资源组**: taiji-ai-pda +- **命名空间**: taiji-ai +- **ACR**: taiji.azurecr.io +- **节点架构**: ARM64 + +## 🔧 测试命令 + +### 健康检查 + +```bash +# MCP Server +kubectl port-forward svc/mcp-server 8002:8000 -n taiji-ai +curl http://localhost:8002/health + +# Data Ingestion +kubectl port-forward svc/data-ingestion 8001:8000 -n taiji-ai +curl http://localhost:8001/health + +# LiteLLM Gateway +kubectl port-forward svc/litellm-gateway 4000:4000 -n taiji-ai +curl -H "Authorization: Bearer sk-taiji-master-key" http://localhost:4000/health +``` + +### 查看日志 + +```bash +# MCP Server +kubectl logs -f deployment/mcp-server -n taiji-ai + +# Data Ingestion +kubectl logs -f deployment/data-ingestion -n taiji-ai + +# LiteLLM Gateway +kubectl logs -f deployment/litellm-gateway -n taiji-ai +``` + +### 查看服务状态 + +```bash +kubectl get pods -n taiji-ai +kubectl get services -n taiji-ai +kubectl get deployments -n taiji-ai +``` + +## ⚠️ 已知问题 + +1. **OpenRouter 账户余额不足** + - LiteLLM Gateway 健康检查显示模型端点不可用 + - 需要访问 https://openrouter.ai/settings/credits 充值 + +2. **NATS 集群配置** + - 当前配置为集群模式,但可能需要调整 + - 如果不需要集群模式,可以简化为单节点部署 + +3. **存储类** + - Prometheus 和 Grafana 的 PVC 可能需要检查存储类配置 + - 当前使用 `managed-csi`,确保 AKS 集群支持 + +## 🚀 下一步 + +1. **清理旧 Pod** + ```bash + kubectl delete pod -l app=nats -n taiji-ai --field-selector=status.phase!=Running + ``` + +2. **配置 Ingress**(可选) + - 如果需要外部访问,配置 Ingress + - 参考 `ingress.yaml` 文件 + +3. **监控配置** + - 配置 Prometheus 和 Grafana + - 设置监控仪表板 + +4. **OpenRouter 充值** + - 访问 https://openrouter.ai/settings/credits + - 充值后 LiteLLM Gateway 将正常工作 + +## 📝 部署文件位置 + +所有 Kubernetes 配置文件位于: `/home/taiji/tools/taiji-AI-PAD/k8s/` + +- `namespace.yaml` - 命名空间 +- `configmap.yaml` - 配置映射 +- `secrets.yaml.template` - Secret 模板 +- `*-deployment.yaml` - 各服务部署配置 +- `ingress.yaml` - Ingress 配置 +- `test-deployment.sh` - 测试脚本 + +## 🎉 部署完成 + +核心服务(MCP Server、LiteLLM Gateway、Data Ingestion)已成功部署并运行! + diff --git a/k8s/QUICKSTART.md b/k8s/QUICKSTART.md new file mode 100644 index 0000000..ef69666 --- /dev/null +++ b/k8s/QUICKSTART.md @@ -0,0 +1,125 @@ +# AKS 快速部署指南 + +## 🚀 5 步快速部署 + +### 步骤 1: 准备 Azure 资源 + +```bash +# 设置变量(根据实际情况修改) +export RESOURCE_GROUP="taiji-ai-rg" +export LOCATION="southeastasia" +export AKS_NAME="taiji-aks" +export ACR_NAME="taijiacr" + +# 创建资源组 +az group create --name $RESOURCE_GROUP --location $LOCATION + +# 创建 ACR +az acr create --resource-group $RESOURCE_GROUP --name $ACR_NAME --sku Basic + +# 创建 AKS 集群(最小配置) +az aks create \ + --resource-group $RESOURCE_GROUP \ + --name $AKS_NAME \ + --node-count 2 \ + --node-vm-size Standard_B2s \ + --enable-addons monitoring \ + --attach-acr $ACR_NAME \ + --generate-ssh-keys +``` + +### 步骤 2: 构建并推送镜像 + +```bash +cd k8s +./build-and-push.sh $ACR_NAME +``` + +### 步骤 3: 创建 Secrets + +```bash +# 使用交互式脚本 +./create-secrets.sh + +# 或手动创建 +kubectl create secret generic taiji-secrets \ + --from-literal=database-url="postgresql+asyncpg://user:pass@server.postgres.database.azure.com:5432/db?sslmode=require" \ + --from-literal=async-database-url="postgresql+asyncpg://user:pass@server.postgres.database.azure.com:5432/db?sslmode=require" \ + --from-literal=redis-url="rediss://:password@server.redis.cache.windows.net:6380/0?ssl_cert_reqs=required" \ + --from-literal=jwt-secret="your-secret-key" \ + --from-literal=encryption-key="your-encryption-key" \ + --from-literal=litellm-master-key="sk-taiji-master-key" \ + --from-literal=litellm-api-key="sk-taiji-master-key" \ + --from-literal=openrouter-api-key="your-openrouter-key" \ + --from-literal=rapidapi-key="your-rapidapi-key" \ + --namespace=taiji-ai +``` + +### 步骤 4: 部署到 AKS + +```bash +./deploy.sh $ACR_NAME $RESOURCE_GROUP $AKS_NAME +``` + +### 步骤 5: 验证部署 + +```bash +# 查看所有 Pod 状态 +kubectl get pods -n taiji-ai + +# 查看服务 +kubectl get services -n taiji-ai + +# 查看日志 +kubectl logs -f deployment/mcp-server -n taiji-ai +``` + +## 📝 重要提示 + +1. **数据库和 Redis**: 确保 Azure Database for PostgreSQL 和 Azure Cache for Redis 的防火墙规则允许 AKS 节点 IP 访问。 + +2. **ACR 访问**: 如果 AKS 未自动附加 ACR,需要创建 ACR 拉取密钥: + ```bash + kubectl create secret docker-registry acr-secret \ + --docker-server=${ACR_NAME}.azurecr.io \ + --docker-username= \ + --docker-password= \ + --namespace=taiji-ai + ``` + +3. **Ingress**: 默认配置使用 Azure Application Gateway。如果使用 NGINX Ingress,请修改 `ingress.yaml`。 + +4. **域名**: 部署 Ingress 前,请确保域名 DNS 已正确配置。 + +## 🔧 常用命令 + +```bash +# 查看所有资源 +kubectl get all -n taiji-ai + +# 扩展服务 +kubectl scale deployment/mcp-server --replicas=5 -n taiji-ai + +# 更新镜像 +kubectl set image deployment/mcp-server mcp-server=${ACR_NAME}.azurecr.io/mcp-server:latest -n taiji-ai + +# 进入 Pod +kubectl exec -it -n taiji-ai -- /bin/bash + +# 查看事件 +kubectl get events -n taiji-ai --sort-by='.lastTimestamp' +``` + +## 🆘 故障排查 + +如果 Pod 无法启动: + +1. 检查 Pod 状态: `kubectl describe pod -n taiji-ai` +2. 查看日志: `kubectl logs -n taiji-ai` +3. 检查 Secret: `kubectl get secret taiji-secrets -n taiji-ai -o yaml` +4. 检查 ConfigMap: `kubectl get configmap taiji-config -n taiji-ai -o yaml` + +## 📚 更多信息 + +详细文档请参考 [README.md](./README.md) + diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 0000000..6d168cd --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,328 @@ +# AKS 部署指南 + +本文档说明如何将 taiji-AI-PAD 平台部署到 Azure Kubernetes Service (AKS)。 + +## 📋 前置要求 + +1. **Azure 账户和订阅** +2. **已安装的工具**: + - Azure CLI (`az`) + - kubectl + - Docker + - envsubst (通常包含在 gettext 包中) + +3. **Azure 资源**: + - AKS 集群 + - Azure Container Registry (ACR) + - Azure Database for PostgreSQL + - Azure Cache for Redis + +## 🚀 快速开始 + +### 1. 准备 Azure 资源 + +#### 创建 AKS 集群 + +```bash +# 设置变量 +RESOURCE_GROUP="taiji-ai-rg" +LOCATION="southeastasia" +AKS_NAME="taiji-aks" +ACR_NAME="taijiacr" + +# 创建资源组 +az group create --name $RESOURCE_GROUP --location $LOCATION + +# 创建 ACR +az acr create --resource-group $RESOURCE_GROUP --name $ACR_NAME --sku Basic + +# 创建 AKS 集群 +az aks create \ + --resource-group $RESOURCE_GROUP \ + --name $AKS_NAME \ + --node-count 3 \ + --enable-addons monitoring \ + --attach-acr $ACR_NAME \ + --generate-ssh-keys +``` + +#### 配置 Azure 数据库和 Redis + +确保您已经创建了: +- Azure Database for PostgreSQL +- Azure Cache for Redis + +记录连接字符串,稍后需要配置到 Secret 中。 + +### 2. 构建和推送 Docker 镜像 + +```bash +# 给脚本添加执行权限 +chmod +x build-and-push.sh + +# 构建并推送所有镜像 +./build-and-push.sh $ACR_NAME +``` + +这将构建并推送以下镜像: +- `litellm-gateway:latest` +- `data-ingestion:latest` +- `mcp-server:latest` + +### 3. 创建 Kubernetes Secrets + +#### 方式1: 使用交互式脚本 + +```bash +chmod +x create-secrets.sh +./create-secrets.sh +``` + +#### 方式2: 使用 kubectl 命令 + +```bash +kubectl create secret generic taiji-secrets \ + --from-literal=database-url="postgresql+asyncpg://..." \ + --from-literal=redis-url="rediss://..." \ + --from-literal=jwt-secret="your-jwt-secret" \ + --from-literal=encryption-key="your-encryption-key" \ + --from-literal=litellm-master-key="sk-taiji-master-key" \ + --from-literal=openrouter-api-key="your-openrouter-key" \ + --namespace=taiji-ai +``` + +#### 方式3: 使用 Azure Key Vault (推荐生产环境) + +1. 安装 Azure Key Vault Provider: +```bash +kubectl apply -f https://raw.githubusercontent.com/Azure/secrets-store-csi-driver-provider-azure/master/deployment/secrets-store-csi-driver.yaml +``` + +2. 创建 Key Vault 并存储密钥 +3. 配置 SecretProviderClass (参考 Azure 文档) + +### 4. 部署到 AKS + +```bash +# 给脚本添加执行权限 +chmod +x deploy.sh + +# 部署所有服务 +./deploy.sh $ACR_NAME $RESOURCE_GROUP $AKS_NAME +``` + +## 📁 文件结构 + +``` +k8s/ +├── namespace.yaml # 命名空间定义 +├── configmap.yaml # 非敏感配置 +├── secrets.yaml.template # Secret 模板(参考用) +├── nats-deployment.yaml # NATS 消息队列部署 +├── model-gateway-deployment.yaml # LiteLLM 网关部署 +├── data-ingestion-deployment.yaml # 数据接入服务部署 +├── mcp-server-deployment.yaml # MCP 服务器部署 +├── monitoring-deployment.yaml # Prometheus 和 Grafana +├── ingress.yaml # Ingress 配置 +├── deploy.sh # 部署脚本 +├── build-and-push.sh # 镜像构建脚本 +├── create-secrets.sh # Secret 创建脚本 +└── README.md # 本文档 +``` + +## 🔧 配置说明 + +### 环境变量 + +所有敏感信息存储在 Kubernetes Secret 中,非敏感配置存储在 ConfigMap 中。 + +**Secret 中的配置**: +- `database-url`: PostgreSQL 连接字符串 +- `redis-url`: Redis 连接字符串 +- `jwt-secret`: JWT 签名密钥 +- `encryption-key`: 数据加密密钥 +- `litellm-master-key`: LiteLLM 主密钥 +- `openrouter-api-key`: OpenRouter API 密钥 +- `rapidapi-key`: RapidAPI 密钥(可选) +- `azure-storage-connection-string`: Azure 存储连接字符串(可选) + +**ConfigMap 中的配置**: +- NATS URL +- LiteLLM URL +- OpenRouter Base URL +- 应用环境配置 + +### 资源限制 + +每个服务的默认资源限制: + +| 服务 | CPU 请求 | CPU 限制 | 内存请求 | 内存限制 | +|------|---------|----------|----------|----------| +| NATS | 100m | 500m | 256Mi | 512Mi | +| Model Gateway | 250m | 1000m | 512Mi | 1Gi | +| Data Ingestion | 250m | 1000m | 512Mi | 1Gi | +| MCP Server | 250m | 500m | 512Mi | 1Gi | + +### 自动扩缩容 (HPA) + +所有主要服务都配置了 HorizontalPodAutoscaler: +- **最小副本数**: 2-3 +- **最大副本数**: 8-10 +- **CPU 阈值**: 70% +- **内存阈值**: 80% + +## 🌐 Ingress 配置 + +### 选项1: Azure Application Gateway (推荐) + +如果使用 Azure Application Gateway Ingress Controller,配置文件已包含相应注解。 + +### 选项2: NGINX Ingress + +如果使用 NGINX Ingress Controller,请: +1. 安装 NGINX Ingress Controller +2. 在 `ingress.yaml` 中取消注释 NGINX 配置 +3. 注释掉 AGIC 配置 + +### 域名配置 + +默认配置使用以下域名: +- `api.taiji-ai.com` - MCP Server 和 Data Ingestion API +- `gateway.taiji-ai.com` - Model Gateway (可选) + +请根据实际情况修改域名和 TLS 证书配置。 + +## 📊 监控 + +### Prometheus + +Prometheus 自动收集以下服务的指标: +- LiteLLM Gateway +- MCP Server +- Data Ingestion + +访问方式: +```bash +kubectl port-forward svc/prometheus 9090:9090 -n taiji-ai +# 然后访问 http://localhost:9090 +``` + +### Grafana + +Grafana 已配置 Prometheus 作为数据源。 + +访问方式: +```bash +kubectl port-forward svc/grafana 3000:3000 -n taiji-ai +# 然后访问 http://localhost:3000 +# 默认用户名: admin +# 密码: 在 Secret 中配置 +``` + +## 🔍 故障排查 + +### 查看 Pod 状态 + +```bash +kubectl get pods -n taiji-ai +``` + +### 查看 Pod 日志 + +```bash +# MCP Server +kubectl logs -f deployment/mcp-server -n taiji-ai + +# Model Gateway +kubectl logs -f deployment/litellm-gateway -n taiji-ai + +# Data Ingestion +kubectl logs -f deployment/data-ingestion -n taiji-ai +``` + +### 查看服务状态 + +```bash +kubectl get services -n taiji-ai +kubectl get ingress -n taiji-ai +``` + +### 检查 Secret 和 ConfigMap + +```bash +kubectl get secrets -n taiji-ai +kubectl get configmap -n taiji-ai +kubectl describe secret taiji-secrets -n taiji-ai +``` + +### 常见问题 + +1. **镜像拉取失败** + - 检查 ACR 是否正确附加到 AKS + - 验证 `acr-secret` 是否存在 + +2. **Pod 无法启动** + - 检查 Secret 是否包含所有必需的键 + - 查看 Pod 事件: `kubectl describe pod -n taiji-ai` + +3. **服务无法访问** + - 检查 Service 和 Ingress 配置 + - 验证域名 DNS 配置 + +4. **数据库连接失败** + - 检查 Azure Database for PostgreSQL 防火墙规则 + - 验证连接字符串格式 + +## 🔄 更新部署 + +### 更新镜像 + +```bash +# 1. 构建新镜像 +./build-and-push.sh $ACR_NAME + +# 2. 重启部署(触发拉取新镜像) +kubectl rollout restart deployment/mcp-server -n taiji-ai +kubectl rollout restart deployment/litellm-gateway -n taiji-ai +kubectl rollout restart deployment/data-ingestion -n taiji-ai +``` + +### 更新配置 + +```bash +# 更新 ConfigMap +kubectl apply -f configmap.yaml + +# 更新 Secret +kubectl create secret generic taiji-secrets \ + --from-literal=... \ + --namespace=taiji-ai \ + --dry-run=client -o yaml | kubectl apply -f - + +# 重启相关 Pod +kubectl rollout restart deployment/ -n taiji-ai +``` + +## 🧹 清理 + +删除所有资源: + +```bash +kubectl delete namespace taiji-ai +``` + +## 📚 参考资源 + +- [AKS 文档](https://docs.microsoft.com/azure/aks/) +- [Azure Container Registry 文档](https://docs.microsoft.com/azure/container-registry/) +- [Kubernetes 文档](https://kubernetes.io/docs/) +- [Azure Key Vault Provider](https://azure.github.io/secrets-store-csi-driver-provider-azure/) + +## 🆘 获取帮助 + +如遇问题,请检查: +1. Pod 日志 +2. 服务事件 +3. Ingress 状态 +4. Azure 资源健康状态 + diff --git a/k8s/build-and-push.sh b/k8s/build-and-push.sh new file mode 100755 index 0000000..15db473 --- /dev/null +++ b/k8s/build-and-push.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +# 构建并推送 Docker 镜像到 ACR 的脚本 +# 使用方法: ./build-and-push.sh + +set -e + +# 颜色输出 +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +# 检查参数 +if [ $# -lt 1 ]; then + echo -e "${RED}错误: 缺少参数${NC}" + echo "使用方法: $0 " + echo "示例: $0 myregistry" + exit 1 +fi + +ACR_NAME=$1 +ACR_URL="${ACR_NAME}.azurecr.io" + +echo -e "${GREEN}开始构建和推送镜像到 ACR...${NC}" +echo "ACR 名称: $ACR_NAME" +echo "ACR URL: $ACR_URL" + +# 检查 Docker 是否安装 +if ! command -v docker &> /dev/null; then + echo -e "${RED}错误: Docker 未安装${NC}" + exit 1 +fi + +# 检查 az 是否安装 +if ! command -v az &> /dev/null; then + echo -e "${RED}错误: Azure CLI 未安装${NC}" + exit 1 +fi + +# 登录到 ACR +echo -e "${YELLOW}登录到 ACR...${NC}" +az acr login --name $ACR_NAME + +# 获取项目根目录 +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" + +# 设置构建平台(ARM64) +PLATFORM="linux/arm64" + +# 检查并设置 buildx +if ! docker buildx ls | grep -q multiarch-builder; then + echo -e "${YELLOW}创建 buildx builder...${NC}" + docker buildx create --name multiarch-builder --use --bootstrap 2>/dev/null || true +fi + +# 构建和推送 Model Gateway +echo -e "${YELLOW}构建 Model Gateway 镜像 (${PLATFORM})...${NC}" +cd "$PROJECT_ROOT/services/model-gateway" +docker buildx build --platform ${PLATFORM} -t ${ACR_URL}/litellm-gateway:latest --push . || { + echo -e "${YELLOW}尝试使用标准构建...${NC}" + docker build -t ${ACR_URL}/litellm-gateway:latest . + docker push ${ACR_URL}/litellm-gateway:latest +} +echo -e "${GREEN}Model Gateway 镜像已推送${NC}" + +# 构建和推送 Data Ingestion +echo -e "${YELLOW}构建 Data Ingestion 镜像 (${PLATFORM})...${NC}" +cd "$PROJECT_ROOT/services/data-ingestion" +docker buildx build --platform ${PLATFORM} -t ${ACR_URL}/data-ingestion:latest --push . || { + echo -e "${YELLOW}尝试使用标准构建...${NC}" + docker build -t ${ACR_URL}/data-ingestion:latest . + docker push ${ACR_URL}/data-ingestion:latest +} +echo -e "${GREEN}Data Ingestion 镜像已推送${NC}" + +# 构建和推送 MCP Server +echo -e "${YELLOW}构建 MCP Server 镜像 (${PLATFORM})...${NC}" +cd "$PROJECT_ROOT/services/mcp-server" +docker buildx build --platform ${PLATFORM} -t ${ACR_URL}/mcp-server:latest --push . || { + echo -e "${YELLOW}尝试使用标准构建...${NC}" + docker build -t ${ACR_URL}/mcp-server:latest . + docker push ${ACR_URL}/mcp-server:latest +} +echo -e "${GREEN}MCP Server 镜像已推送${NC}" + +echo -e "${GREEN}所有镜像构建和推送完成!${NC}" +echo "" +echo "镜像列表:" +echo " - ${ACR_URL}/litellm-gateway:latest" +echo " - ${ACR_URL}/data-ingestion:latest" +echo " - ${ACR_URL}/mcp-server:latest" + diff --git a/k8s/configmap.yaml b/k8s/configmap.yaml new file mode 100644 index 0000000..c26dbb8 --- /dev/null +++ b/k8s/configmap.yaml @@ -0,0 +1,34 @@ +# ConfigMap - 非敏感配置 +apiVersion: v1 +kind: ConfigMap +metadata: + name: taiji-config + namespace: taiji-ai +data: + # NATS配置 + nats-url: "nats://nats:4222" + + # LiteLLM配置 + litellm-url: "http://litellm-gateway:4000" + litellm-config-path: "/app/config/litellm_simple.yaml" + + # OpenRouter配置 + openrouter-base-url: "https://openrouter.ai/api/v1" + + # RapidAPI配置 + rapidapi-host: "rapidapi.com" + rapidapi-base-url: "https://rapidapi.com" + + # 应用配置 + environment: "production" + debug: "false" + log-level: "INFO" + + # 健康检查配置 + health-check-interval: "30" + + # 资源限制 + max-agents-per-user: "100" + agent-execution-timeout: "300" + tool-execution-timeout: "60" + diff --git a/k8s/create-secrets.sh b/k8s/create-secrets.sh new file mode 100755 index 0000000..1304480 --- /dev/null +++ b/k8s/create-secrets.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +# 创建 Kubernetes Secrets 的交互式脚本 +# 使用方法: ./create-secrets.sh + +set -e + +# 颜色输出 +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +echo -e "${GREEN}创建 Kubernetes Secrets${NC}" +echo "" + +# 检查 kubectl +if ! command -v kubectl &> /dev/null; then + echo -e "${RED}错误: kubectl 未安装${NC}" + exit 1 +fi + +# 检查命名空间 +NAMESPACE="taiji-ai" +if ! kubectl get namespace $NAMESPACE &> /dev/null; then + echo -e "${YELLOW}创建命名空间 $NAMESPACE...${NC}" + kubectl create namespace $NAMESPACE +fi + +# 提示输入各个 Secret 值 +echo -e "${YELLOW}请输入以下配置信息(按 Enter 跳过可选项):${NC}" +echo "" + +read -p "数据库 URL (DATABASE_URL): " DATABASE_URL +read -p "异步数据库 URL (ASYNC_DATABASE_URL): " ASYNC_DATABASE_URL +read -p "Redis URL (REDIS_URL): " REDIS_URL +read -p "JWT Secret: " JWT_SECRET +read -p "加密密钥 (ENCRYPTION_KEY): " ENCRYPTION_KEY +read -p "LiteLLM Master Key: " LITELLM_MASTER_KEY +read -p "OpenRouter API Key: " OPENROUTER_API_KEY +read -p "RapidAPI Key (可选): " RAPIDAPI_KEY +read -p "Azure Storage Connection String (可选): " AZURE_STORAGE_CONNECTION + +# 构建 kubectl 命令 +SECRET_ARGS=() + +if [ -n "$DATABASE_URL" ]; then + SECRET_ARGS+=("--from-literal=database-url=$DATABASE_URL") +fi + +if [ -n "$ASYNC_DATABASE_URL" ]; then + SECRET_ARGS+=("--from-literal=async-database-url=$ASYNC_DATABASE_URL") +fi + +if [ -n "$REDIS_URL" ]; then + SECRET_ARGS+=("--from-literal=redis-url=$REDIS_URL") +fi + +if [ -n "$JWT_SECRET" ]; then + SECRET_ARGS+=("--from-literal=jwt-secret=$JWT_SECRET") +fi + +if [ -n "$ENCRYPTION_KEY" ]; then + SECRET_ARGS+=("--from-literal=encryption-key=$ENCRYPTION_KEY") +fi + +if [ -n "$LITELLM_MASTER_KEY" ]; then + SECRET_ARGS+=("--from-literal=litellm-master-key=$LITELLM_MASTER_KEY") + SECRET_ARGS+=("--from-literal=litellm-api-key=$LITELLM_MASTER_KEY") +fi + +if [ -n "$OPENROUTER_API_KEY" ]; then + SECRET_ARGS+=("--from-literal=openrouter-api-key=$OPENROUTER_API_KEY") +fi + +if [ -n "$RAPIDAPI_KEY" ]; then + SECRET_ARGS+=("--from-literal=rapidapi-key=$RAPIDAPI_KEY") +fi + +if [ -n "$AZURE_STORAGE_CONNECTION" ]; then + SECRET_ARGS+=("--from-literal=azure-storage-connection-string=$AZURE_STORAGE_CONNECTION") +fi + +# 删除现有 Secret(如果存在) +if kubectl get secret taiji-secrets -n $NAMESPACE &> /dev/null; then + echo -e "${YELLOW}删除现有 Secret...${NC}" + kubectl delete secret taiji-secrets -n $NAMESPACE +fi + +# 创建 Secret +if [ ${#SECRET_ARGS[@]} -gt 0 ]; then + echo -e "${YELLOW}创建 Secret...${NC}" + kubectl create secret generic taiji-secrets \ + "${SECRET_ARGS[@]}" \ + --namespace=$NAMESPACE + + echo -e "${GREEN}Secret 创建成功!${NC}" +else + echo -e "${RED}错误: 没有提供任何 Secret 值${NC}" + exit 1 +fi + +# 验证 Secret +echo "" +echo -e "${YELLOW}验证 Secret:${NC}" +kubectl get secret taiji-secrets -n $NAMESPACE + +echo "" +echo -e "${GREEN}完成!${NC}" + diff --git a/k8s/data-ingestion-deployment.yaml b/k8s/data-ingestion-deployment.yaml new file mode 100644 index 0000000..9e65cd1 --- /dev/null +++ b/k8s/data-ingestion-deployment.yaml @@ -0,0 +1,153 @@ +# Data Ingestion 服务部署配置 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: data-ingestion + namespace: taiji-ai + labels: + app: data-ingestion + component: ingestion +spec: + replicas: 2 + selector: + matchLabels: + app: data-ingestion + template: + metadata: + labels: + app: data-ingestion + component: ingestion + spec: + containers: + - name: data-ingestion + image: ${ACR_NAME}.azurecr.io/data-ingestion:latest + imagePullPolicy: Always + ports: + - containerPort: 8000 + name: http + protocol: TCP + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: taiji-secrets + key: database-url + - name: ASYNC_DATABASE_URL + valueFrom: + secretKeyRef: + name: taiji-secrets + key: async-database-url + - name: REDIS_URL + valueFrom: + secretKeyRef: + name: taiji-secrets + key: redis-url + - name: NATS_URL + valueFrom: + configMapKeyRef: + name: taiji-config + key: nats-url + - name: RAPIDAPI_KEY + valueFrom: + secretKeyRef: + name: taiji-secrets + key: rapidapi-key + - name: RAPIDAPI_HOST + valueFrom: + configMapKeyRef: + name: taiji-config + key: rapidapi-host + - name: OPENROUTER_API_KEY + valueFrom: + secretKeyRef: + name: taiji-secrets + key: openrouter-api-key + - name: OPENROUTER_BASE_URL + valueFrom: + configMapKeyRef: + name: taiji-config + key: openrouter-base-url + - name: PYTHONUNBUFFERED + value: "1" + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "1Gi" + cpu: "1000m" + volumeMounts: + - name: logs + mountPath: /app/logs + - name: cache + mountPath: /app/cache + livenessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 2 + volumes: + - name: logs + emptyDir: {} + - name: cache + emptyDir: {} + imagePullSecrets: + - name: acr-secret + +--- +apiVersion: v1 +kind: Service +metadata: + name: data-ingestion + namespace: taiji-ai + labels: + app: data-ingestion +spec: + type: ClusterIP + ports: + - port: 8000 + targetPort: 8000 + protocol: TCP + name: http + selector: + app: data-ingestion + +--- +# HorizontalPodAutoscaler +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: data-ingestion-hpa + namespace: taiji-ai +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: data-ingestion + minReplicas: 2 + maxReplicas: 8 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 + diff --git a/k8s/deploy.sh b/k8s/deploy.sh new file mode 100755 index 0000000..48ebe9b --- /dev/null +++ b/k8s/deploy.sh @@ -0,0 +1,130 @@ +#!/bin/bash + +# AKS 部署脚本 +# 使用方法: ./deploy.sh + +set -e + +# 颜色输出 +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# 检查参数 +if [ $# -lt 3 ]; then + echo -e "${RED}错误: 缺少参数${NC}" + echo "使用方法: $0 " + echo "示例: $0 myregistry myresourcegroup myakscluster" + exit 1 +fi + +ACR_NAME=$1 +AKS_RESOURCE_GROUP=$2 +AKS_CLUSTER_NAME=$3 + +echo -e "${GREEN}开始部署到 AKS...${NC}" +echo "ACR 名称: $ACR_NAME" +echo "资源组: $AKS_RESOURCE_GROUP" +echo "AKS 集群: $AKS_CLUSTER_NAME" + +# 检查 kubectl 是否安装 +if ! command -v kubectl &> /dev/null; then + echo -e "${RED}错误: kubectl 未安装${NC}" + exit 1 +fi + +# 检查 az 是否安装 +if ! command -v az &> /dev/null; then + echo -e "${RED}错误: Azure CLI 未安装${NC}" + exit 1 +fi + +# 获取 AKS 凭据 +echo -e "${YELLOW}获取 AKS 集群凭据...${NC}" +az aks get-credentials --resource-group $AKS_RESOURCE_GROUP --name $AKS_CLUSTER_NAME --overwrite-existing + +# 检查是否已连接到集群 +if ! kubectl cluster-info &> /dev/null; then + echo -e "${RED}错误: 无法连接到 AKS 集群${NC}" + exit 1 +fi + +# 创建命名空间 +echo -e "${YELLOW}创建命名空间...${NC}" +kubectl apply -f namespace.yaml + +# 创建 ACR 拉取密钥(如果需要) +echo -e "${YELLOW}配置 ACR 访问...${NC}" +# 检查 AKS 是否已附加 ACR +ACR_ATTACHED=$(az aks show -n $AKS_CLUSTER_NAME -g $AKS_RESOURCE_GROUP --query "servicePrincipalProfile" -o tsv) +if [ -z "$ACR_ATTACHED" ]; then + echo "附加 ACR 到 AKS..." + az aks update -n $AKS_CLUSTER_NAME -g $AKS_RESOURCE_GROUP --attach-acr $ACR_NAME +else + echo "ACR 已附加,跳过..." +fi + +# 创建 Secret(提示用户) +echo -e "${YELLOW}请确保已创建 Secret:${NC}" +echo " kubectl create secret generic taiji-secrets --from-literal=... --namespace=taiji-ai" +echo " 或使用 Azure Key Vault Provider" +read -p "是否已创建 Secret? (y/n) " -n 1 -r +echo +if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo -e "${RED}请先创建 Secret 后再继续${NC}" + exit 1 +fi + +# 替换配置文件中的变量 +echo -e "${YELLOW}替换配置变量...${NC}" +export ACR_NAME=$ACR_NAME + +# 应用 ConfigMap +echo -e "${YELLOW}应用 ConfigMap...${NC}" +kubectl apply -f configmap.yaml + +# 应用所有部署配置 +echo -e "${YELLOW}部署 NATS...${NC}" +envsubst < nats-deployment.yaml | kubectl apply -f - + +echo -e "${YELLOW}部署 Model Gateway...${NC}" +envsubst < model-gateway-deployment.yaml | kubectl apply -f - + +echo -e "${YELLOW}部署 Data Ingestion...${NC}" +envsubst < data-ingestion-deployment.yaml | kubectl apply -f - + +echo -e "${YELLOW}部署 MCP Server...${NC}" +envsubst < mcp-server-deployment.yaml | kubectl apply -f - + +echo -e "${YELLOW}部署监控服务...${NC}" +kubectl apply -f monitoring-deployment.yaml + +# 等待部署完成 +echo -e "${YELLOW}等待部署就绪...${NC}" +kubectl wait --for=condition=available --timeout=300s deployment/nats -n taiji-ai || true +kubectl wait --for=condition=available --timeout=300s deployment/litellm-gateway -n taiji-ai || true +kubectl wait --for=condition=available --timeout=300s deployment/data-ingestion -n taiji-ai || true +kubectl wait --for=condition=available --timeout=300s deployment/mcp-server -n taiji-ai || true + +# 应用 Ingress(可选) +read -p "是否部署 Ingress? (y/n) " -n 1 -r +echo +if [[ $REPLY =~ ^[Yy]$ ]]; then + echo -e "${YELLOW}部署 Ingress...${NC}" + kubectl apply -f ingress.yaml +fi + +# 显示部署状态 +echo -e "${GREEN}部署完成!${NC}" +echo "" +echo "查看部署状态:" +echo " kubectl get pods -n taiji-ai" +echo " kubectl get services -n taiji-ai" +echo " kubectl get ingress -n taiji-ai" +echo "" +echo "查看日志:" +echo " kubectl logs -f deployment/mcp-server -n taiji-ai" +echo " kubectl logs -f deployment/litellm-gateway -n taiji-ai" +echo " kubectl logs -f deployment/data-ingestion -n taiji-ai" + diff --git a/k8s/ingress.yaml b/k8s/ingress.yaml new file mode 100644 index 0000000..6445d2d --- /dev/null +++ b/k8s/ingress.yaml @@ -0,0 +1,147 @@ +# Ingress 配置 - 使用 Azure Application Gateway 或 NGINX Ingress +# 根据您的 AKS 配置选择相应的 Ingress Controller + +--- +# 选项1: Azure Application Gateway Ingress Controller (AGIC) +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: taiji-ingress + namespace: taiji-ai + annotations: + # Azure Application Gateway Ingress Controller 注解 + kubernetes.io/ingress.class: azure/application-gateway + appgw.ingress.kubernetes.io/ssl-redirect: "true" + appgw.ingress.kubernetes.io/connection-draining: "true" + appgw.ingress.kubernetes.io/connection-draining-timeout: "30" + appgw.ingress.kubernetes.io/backend-protocol: "http" + appgw.ingress.kubernetes.io/request-timeout: "300" + appgw.ingress.kubernetes.io/health-probe-path: "/health" + appgw.ingress.kubernetes.io/health-probe-interval: "30" + appgw.ingress.kubernetes.io/health-probe-timeout: "10" + appgw.ingress.kubernetes.io/health-probe-unhealthy-threshold: "3" + + # CORS 配置 + appgw.ingress.kubernetes.io/cors-allow-origin: "*" + appgw.ingress.kubernetes.io/cors-allow-methods: "GET,POST,PUT,DELETE,OPTIONS" + appgw.ingress.kubernetes.io/cors-allow-headers: "*" + + # 证书配置(使用 Azure Key Vault) + # cert-manager.io/cluster-issuer: "letsencrypt-prod" +spec: + tls: + - hosts: + - api.taiji-ai.com + - gateway.taiji-ai.com + secretName: taiji-tls-cert + rules: + # MCP Server API + - host: api.taiji-ai.com + http: + paths: + - path: /api + pathType: Prefix + backend: + service: + name: mcp-server + port: + number: 8000 + - path: /health + pathType: Exact + backend: + service: + name: mcp-server + port: + number: 8000 + - path: /docs + pathType: Prefix + backend: + service: + name: mcp-server + port: + number: 8000 + - path: /openapi.json + pathType: Exact + backend: + service: + name: mcp-server + port: + number: 8000 + + # Data Ingestion API + - host: api.taiji-ai.com + http: + paths: + - path: /ingestion + pathType: Prefix + backend: + service: + name: data-ingestion + port: + number: 8000 + + # Model Gateway (内部使用,可选暴露) + - host: gateway.taiji-ai.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: litellm-gateway + port: + number: 4000 + +--- +# 选项2: NGINX Ingress Controller (如果使用 NGINX 而不是 AGIC) +# 取消注释以下配置,并注释掉上面的 AGIC 配置 +# apiVersion: networking.k8s.io/v1 +# kind: Ingress +# metadata: +# name: taiji-ingress-nginx +# namespace: taiji-ai +# annotations: +# kubernetes.io/ingress.class: nginx +# cert-manager.io/cluster-issuer: letsencrypt-prod +# nginx.ingress.kubernetes.io/ssl-redirect: "true" +# nginx.ingress.kubernetes.io/proxy-body-size: "50m" +# nginx.ingress.kubernetes.io/proxy-read-timeout: "300" +# nginx.ingress.kubernetes.io/proxy-send-timeout: "300" +# nginx.ingress.kubernetes.io/cors-allow-origin: "*" +# nginx.ingress.kubernetes.io/cors-allow-methods: "GET,POST,PUT,DELETE,OPTIONS" +# nginx.ingress.kubernetes.io/cors-allow-headers: "*" +# spec: +# tls: +# - hosts: +# - api.taiji-ai.com +# - gateway.taiji-ai.com +# secretName: taiji-tls-cert +# rules: +# - host: api.taiji-ai.com +# http: +# paths: +# - path: /api +# pathType: Prefix +# backend: +# service: +# name: mcp-server +# port: +# number: 8000 +# - path: /ingestion +# pathType: Prefix +# backend: +# service: +# name: data-ingestion +# port: +# number: 8000 +# - host: gateway.taiji-ai.com +# http: +# paths: +# - path: / +# pathType: Prefix +# backend: +# service: +# name: litellm-gateway +# port: +# number: 4000 + diff --git a/k8s/mcp-server-deployment.yaml b/k8s/mcp-server-deployment.yaml new file mode 100644 index 0000000..9ef1016 --- /dev/null +++ b/k8s/mcp-server-deployment.yaml @@ -0,0 +1,183 @@ +# MCP Server 部署配置 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mcp-server + namespace: taiji-ai + labels: + app: mcp-server + component: server +spec: + replicas: 3 + selector: + matchLabels: + app: mcp-server + template: + metadata: + labels: + app: mcp-server + component: server + spec: + containers: + - name: mcp-server + image: ${ACR_NAME}.azurecr.io/mcp-server:latest + imagePullPolicy: Always + ports: + - containerPort: 8000 + name: http + protocol: TCP + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: taiji-secrets + key: database-url + - name: ASYNC_DATABASE_URL + valueFrom: + secretKeyRef: + name: taiji-secrets + key: async-database-url + - name: REDIS_URL + valueFrom: + secretKeyRef: + name: taiji-secrets + key: redis-url + - name: NATS_URL + valueFrom: + configMapKeyRef: + name: taiji-config + key: nats-url + - name: LITELLM_URL + valueFrom: + configMapKeyRef: + name: taiji-config + key: litellm-url + - name: LITELLM_API_KEY + valueFrom: + secretKeyRef: + name: taiji-secrets + key: litellm-api-key + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: taiji-secrets + key: jwt-secret + - name: ENCRYPTION_KEY + valueFrom: + secretKeyRef: + name: taiji-secrets + key: encryption-key + - name: AZURE_STORAGE_CONNECTION_STRING + valueFrom: + secretKeyRef: + name: taiji-secrets + key: azure-storage-connection-string + optional: true + - name: ENVIRONMENT + valueFrom: + configMapKeyRef: + name: taiji-config + key: environment + - name: DEBUG + valueFrom: + configMapKeyRef: + name: taiji-config + key: debug + - name: LOG_LEVEL + valueFrom: + configMapKeyRef: + name: taiji-config + key: log-level + - name: PYTHONUNBUFFERED + value: "1" + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "1Gi" + cpu: "500m" + volumeMounts: + - name: logs + mountPath: /app/logs + livenessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 2 + volumes: + - name: logs + emptyDir: {} + imagePullSecrets: + - name: acr-secret + +--- +apiVersion: v1 +kind: Service +metadata: + name: mcp-server + namespace: taiji-ai + labels: + app: mcp-server +spec: + type: ClusterIP + ports: + - port: 8000 + targetPort: 8000 + protocol: TCP + name: http + selector: + app: mcp-server + +--- +# HorizontalPodAutoscaler +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: mcp-server-hpa + namespace: taiji-ai +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: mcp-server + minReplicas: 3 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 + +--- +# PodDisruptionBudget - 确保高可用 +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: mcp-server-pdb + namespace: taiji-ai +spec: + minAvailable: 2 + selector: + matchLabels: + app: mcp-server + diff --git a/k8s/model-gateway-deployment.yaml b/k8s/model-gateway-deployment.yaml new file mode 100644 index 0000000..594d753 --- /dev/null +++ b/k8s/model-gateway-deployment.yaml @@ -0,0 +1,188 @@ +# LiteLLM Model Gateway 部署配置 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: litellm-gateway + namespace: taiji-ai + labels: + app: litellm-gateway + component: gateway +spec: + replicas: 2 + selector: + matchLabels: + app: litellm-gateway + template: + metadata: + labels: + app: litellm-gateway + component: gateway + spec: + containers: + - name: litellm-gateway + image: ${ACR_NAME}.azurecr.io/litellm-gateway:latest + imagePullPolicy: Always + ports: + - containerPort: 4000 + name: http + protocol: TCP + env: + - name: LITELLM_MASTER_KEY + valueFrom: + secretKeyRef: + name: taiji-secrets + key: litellm-master-key + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: taiji-secrets + key: database-url + - name: REDIS_URL + valueFrom: + secretKeyRef: + name: taiji-secrets + key: redis-url + - name: OPENROUTER_API_KEY + valueFrom: + secretKeyRef: + name: taiji-secrets + key: openrouter-api-key + - name: OPENROUTER_BASE_URL + valueFrom: + configMapKeyRef: + name: taiji-config + key: openrouter-base-url + - name: LITELLM_CONFIG_PATH + valueFrom: + configMapKeyRef: + name: taiji-config + key: litellm-config-path + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "1Gi" + cpu: "1000m" + volumeMounts: + - name: litellm-config + mountPath: /app/config + readOnly: true + - name: logs + mountPath: /app/logs + livenessProbe: + httpGet: + path: /health + port: 4000 + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 25 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /health + port: 4000 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 2 + volumes: + - name: litellm-config + configMap: + name: litellm-config + - name: logs + emptyDir: {} + imagePullSecrets: + - name: acr-secret + +--- +apiVersion: v1 +kind: Service +metadata: + name: litellm-gateway + namespace: taiji-ai + labels: + app: litellm-gateway +spec: + type: ClusterIP + ports: + - port: 4000 + targetPort: 4000 + protocol: TCP + name: http + selector: + app: litellm-gateway + +--- +# LiteLLM 配置文件 ConfigMap +apiVersion: v1 +kind: ConfigMap +metadata: + name: litellm-config + namespace: taiji-ai +data: + litellm_simple.yaml: | + model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: openrouter/gpt-3.5-turbo + api_key: os.environ/OPENROUTER_API_KEY + api_base: https://openrouter.ai/api/v1 + + - model_name: openrouter-gpt-3.5-turbo + litellm_params: + model: openrouter/gpt-3.5-turbo + api_key: os.environ/OPENROUTER_API_KEY + api_base: https://openrouter.ai/api/v1 + + - model_name: openrouter-gpt-4o-mini + litellm_params: + model: openrouter/gpt-4o-mini + api_key: os.environ/OPENROUTER_API_KEY + api_base: https://openrouter.ai/api/v1 + + - model_name: openrouter-claude-3.5-sonnet + litellm_params: + model: openrouter/anthropic/claude-3.5-sonnet + api_key: os.environ/OPENROUTER_API_KEY + api_base: https://openrouter.ai/api/v1 + + - model_name: openrouter-claude-3-opus + litellm_params: + model: openrouter/anthropic/claude-3-opus + api_key: os.environ/OPENROUTER_API_KEY + api_base: https://openrouter.ai/api/v1 + + general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + database_url: os.environ/DATABASE_URL + redis_url: os.environ/REDIS_URL + +--- +# HorizontalPodAutoscaler +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: litellm-gateway-hpa + namespace: taiji-ai +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: litellm-gateway + minReplicas: 2 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 + diff --git a/k8s/monitoring-deployment.yaml b/k8s/monitoring-deployment.yaml new file mode 100644 index 0000000..a6e0b9f --- /dev/null +++ b/k8s/monitoring-deployment.yaml @@ -0,0 +1,269 @@ +# Prometheus 和 Grafana 监控服务部署配置 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + namespace: taiji-ai + labels: + app: prometheus + component: monitoring +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus + template: + metadata: + labels: + app: prometheus + component: monitoring + spec: + containers: + - name: prometheus + image: prom/prometheus:latest + ports: + - containerPort: 9090 + name: http + protocol: TCP + args: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/etc/prometheus/console_libraries' + - '--web.console.templates=/etc/prometheus/consoles' + - '--web.enable-lifecycle' + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "2Gi" + cpu: "1000m" + volumeMounts: + - name: prometheus-config + mountPath: /etc/prometheus + readOnly: true + - name: prometheus-data + mountPath: /prometheus + volumes: + - name: prometheus-config + configMap: + name: prometheus-config + - name: prometheus-data + persistentVolumeClaim: + claimName: prometheus-pvc + +--- +apiVersion: v1 +kind: Service +metadata: + name: prometheus + namespace: taiji-ai + labels: + app: prometheus +spec: + type: ClusterIP + ports: + - port: 9090 + targetPort: 9090 + protocol: TCP + name: http + selector: + app: prometheus + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + namespace: taiji-ai + labels: + app: grafana + component: monitoring +spec: + replicas: 1 + selector: + matchLabels: + app: grafana + template: + metadata: + labels: + app: grafana + component: monitoring + spec: + containers: + - name: grafana + image: grafana/grafana:latest + ports: + - containerPort: 3000 + name: http + protocol: TCP + env: + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: taiji-secrets + key: grafana-admin-password + optional: true + - name: GF_SERVER_ROOT_URL + value: "http://grafana.taiji-ai.com" + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + volumeMounts: + - name: grafana-data + mountPath: /var/lib/grafana + - name: grafana-dashboards + mountPath: /etc/grafana/provisioning/dashboards + - name: grafana-datasources + mountPath: /etc/grafana/provisioning/datasources + volumes: + - name: grafana-data + persistentVolumeClaim: + claimName: grafana-pvc + - name: grafana-dashboards + configMap: + name: grafana-dashboards + - name: grafana-datasources + configMap: + name: grafana-datasources + +--- +apiVersion: v1 +kind: Service +metadata: + name: grafana + namespace: taiji-ai + labels: + app: grafana +spec: + type: ClusterIP + ports: + - port: 3000 + targetPort: 3000 + protocol: TCP + name: http + selector: + app: grafana + +--- +# Prometheus 配置 +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-config + namespace: taiji-ai +data: + prometheus.yml: | + global: + scrape_interval: 15s + evaluation_interval: 15s + + scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'litellm-gateway' + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - taiji-ai + relabel_configs: + - source_labels: [__meta_kubernetes_service_name] + action: keep + regex: litellm-gateway + + - job_name: 'mcp-server' + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - taiji-ai + relabel_configs: + - source_labels: [__meta_kubernetes_service_name] + action: keep + regex: mcp-server + + - job_name: 'data-ingestion' + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - taiji-ai + relabel_configs: + - source_labels: [__meta_kubernetes_service_name] + action: keep + regex: data-ingestion + +--- +# Grafana 数据源配置 +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-datasources + namespace: taiji-ai +data: + prometheus.yaml: | + apiVersion: 1 + datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true + +--- +# Grafana 仪表板配置 +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboards + namespace: taiji-ai +data: + dashboard-provider.yaml: | + apiVersion: 1 + providers: + - name: 'Default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /etc/grafana/provisioning/dashboards + +--- +# 持久化存储 +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: prometheus-pvc + namespace: taiji-ai +spec: + accessModes: + - ReadWriteOnce + storageClassName: managed-csi + resources: + requests: + storage: 50Gi + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: grafana-pvc + namespace: taiji-ai +spec: + accessModes: + - ReadWriteOnce + storageClassName: managed-csi + resources: + requests: + storage: 10Gi + diff --git a/k8s/namespace.yaml b/k8s/namespace.yaml new file mode 100644 index 0000000..6625655 --- /dev/null +++ b/k8s/namespace.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: taiji-ai + labels: + name: taiji-ai + environment: production + diff --git a/k8s/nats-deployment.yaml b/k8s/nats-deployment.yaml new file mode 100644 index 0000000..a9da60e --- /dev/null +++ b/k8s/nats-deployment.yaml @@ -0,0 +1,111 @@ +# NATS 消息队列部署配置 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nats + namespace: taiji-ai + labels: + app: nats + component: messaging +spec: + replicas: 3 + selector: + matchLabels: + app: nats + template: + metadata: + labels: + app: nats + component: messaging + spec: + containers: + - name: nats + image: nats:2.10-alpine + ports: + - containerPort: 4222 + name: client + protocol: TCP + - containerPort: 6222 + name: routing + protocol: TCP + - containerPort: 8222 + name: monitoring + protocol: TCP + command: + - "/nats-server" + args: + - "-js" + - "-m" + - "8222" + - "-cluster" + - "nats://0.0.0.0:6222" + - "-routes" + - "nats://nats-0.nats.taiji-ai.svc.cluster.local:6222,nats://nats-1.nats.taiji-ai.svc.cluster.local:6222,nats://nats-2.nats.taiji-ai.svc.cluster.local:6222" + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + volumeMounts: + - name: nats-data + mountPath: /data + livenessProbe: + httpGet: + path: /healthz + port: 8222 + initialDelaySeconds: 10 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /healthz + port: 8222 + initialDelaySeconds: 5 + periodSeconds: 5 + volumes: + - name: nats-data + persistentVolumeClaim: + claimName: nats-pvc + +--- +apiVersion: v1 +kind: Service +metadata: + name: nats + namespace: taiji-ai + labels: + app: nats +spec: + type: ClusterIP + ports: + - port: 4222 + targetPort: 4222 + protocol: TCP + name: client + - port: 6222 + targetPort: 6222 + protocol: TCP + name: routing + - port: 8222 + targetPort: 8222 + protocol: TCP + name: monitoring + selector: + app: nats + +--- +# NATS 持久化存储 +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nats-pvc + namespace: taiji-ai +spec: + accessModes: + - ReadWriteOnce + storageClassName: managed-csi # Azure 托管存储类 + resources: + requests: + storage: 10Gi + diff --git a/k8s/secrets.yaml.template b/k8s/secrets.yaml.template new file mode 100644 index 0000000..d13b4b8 --- /dev/null +++ b/k8s/secrets.yaml.template @@ -0,0 +1,70 @@ +# Kubernetes Secrets 模板 +# 注意:这是一个模板文件,实际部署时请使用以下方式创建 Secret: +# +# 方式1: 使用 kubectl 命令 +# kubectl create secret generic taiji-secrets \ +# --from-literal=database-url="postgresql+asyncpg://..." \ +# --from-literal=redis-url="rediss://..." \ +# --namespace=taiji-ai +# +# 方式2: 使用 Azure Key Vault (推荐) +# 安装 Azure Key Vault Provider: https://azure.github.io/secrets-store-csi-driver-provider-azure/ +# +# 方式3: 使用 sealed-secrets (推荐用于 GitOps) +# https://github.com/bitnami-labs/sealed-secrets + +apiVersion: v1 +kind: Secret +metadata: + name: taiji-secrets + namespace: taiji-ai +type: Opaque +stringData: + # 数据库连接字符串 (Azure Database for PostgreSQL) + database-url: "postgresql+asyncpg://USER:PASSWORD@SERVER.postgres.database.azure.com:5432/DATABASE?sslmode=require" + async-database-url: "postgresql+asyncpg://USER:PASSWORD@SERVER.postgres.database.azure.com:5432/DATABASE?sslmode=require" + + # Redis连接字符串 (Azure Cache for Redis) + redis-url: "rediss://:PASSWORD@SERVER.redis.cache.windows.net:6380/0?ssl_cert_reqs=required" + + # JWT密钥 + jwt-secret: "CHANGE_THIS_SECRET_KEY_IN_PRODUCTION_USE_STRONG_RANDOM_STRING" + + # 加密密钥 + encryption-key: "CHANGE_THIS_ENCRYPTION_KEY_USE_STRONG_RANDOM_STRING" + + # LiteLLM API密钥 + litellm-master-key: "sk-taiji-master-key" + litellm-api-key: "sk-taiji-master-key" + + # OpenRouter API密钥 + openrouter-api-key: "YOUR_OPENROUTER_API_KEY" + + # RapidAPI配置 + rapidapi-key: "YOUR_RAPIDAPI_KEY" + + # Azure存储连接字符串 (可选) + azure-storage-connection-string: "DefaultEndpointsProtocol=https;AccountName=ACCOUNT;AccountKey=KEY;EndpointSuffix=core.windows.net" + +--- +# ACR (Azure Container Registry) 拉取密钥 +# 创建方式: +# kubectl create secret docker-registry acr-secret \ +# --docker-server=${ACR_NAME}.azurecr.io \ +# --docker-username=${SP_APP_ID} \ +# --docker-password=${SP_PASSWORD} \ +# --namespace=taiji-ai +# +# 或者使用 AKS 与 ACR 集成(推荐): +# az aks update -n -g --attach-acr + +apiVersion: v1 +kind: Secret +metadata: + name: acr-secret + namespace: taiji-ai +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: + # 注意:实际使用时请使用上述命令创建,不要直接使用 base64 编码的配置 + diff --git a/k8s/test-deployment.sh b/k8s/test-deployment.sh new file mode 100755 index 0000000..c499061 --- /dev/null +++ b/k8s/test-deployment.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +# 测试部署的服务 +# 使用方法: ./test-deployment.sh + +set -e + +export PATH=$HOME/bin:$PATH + +NAMESPACE="taiji-ai" + +echo "==========================================" +echo "测试 AKS 部署的服务" +echo "==========================================" +echo "" + +# 1. 检查 Pod 状态 +echo "1. 检查 Pod 状态..." +kubectl get pods -n $NAMESPACE +echo "" + +# 2. 检查服务 +echo "2. 检查服务..." +kubectl get services -n $NAMESPACE +echo "" + +# 3. 测试 MCP Server 健康检查 +echo "3. 测试 MCP Server 健康检查..." +MCP_POD=$(kubectl get pods -l app=mcp-server -n $NAMESPACE -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "") +if [ -n "$MCP_POD" ]; then + echo "MCP Server Pod: $MCP_POD" + kubectl exec -n $NAMESPACE $MCP_POD -- curl -s http://localhost:8000/health || echo "健康检查失败" +else + echo "未找到 MCP Server Pod" +fi +echo "" + +# 4. 测试 Data Ingestion 健康检查 +echo "4. 测试 Data Ingestion 健康检查..." +DI_POD=$(kubectl get pods -l app=data-ingestion -n $NAMESPACE -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "") +if [ -n "$DI_POD" ]; then + echo "Data Ingestion Pod: $DI_POD" + kubectl exec -n $NAMESPACE $DI_POD -- curl -s http://localhost:8000/health || echo "健康检查失败" +else + echo "未找到 Data Ingestion Pod" +fi +echo "" + +# 5. 测试 LiteLLM Gateway 健康检查 +echo "5. 测试 LiteLLM Gateway 健康检查..." +LG_POD=$(kubectl get pods -l app=litellm-gateway -n $NAMESPACE -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "") +if [ -n "$LG_POD" ]; then + echo "LiteLLM Gateway Pod: $LG_POD" + kubectl exec -n $NAMESPACE $LG_POD -- curl -s -H "Authorization: Bearer sk-taiji-master-key" http://localhost:4000/health || echo "健康检查失败" +else + echo "未找到 LiteLLM Gateway Pod" +fi +echo "" + +# 6. 测试服务端点(通过 port-forward) +echo "6. 测试服务端点..." +echo "注意: 以下测试需要手动运行 port-forward 命令" +echo "" +echo "测试 MCP Server:" +echo " kubectl port-forward svc/mcp-server 8002:8000 -n $NAMESPACE" +echo " curl http://localhost:8002/health" +echo "" +echo "测试 Data Ingestion:" +echo " kubectl port-forward svc/data-ingestion 8001:8000 -n $NAMESPACE" +echo " curl http://localhost:8001/health" +echo "" +echo "测试 LiteLLM Gateway:" +echo " kubectl port-forward svc/litellm-gateway 4000:4000 -n $NAMESPACE" +echo " curl -H 'Authorization: Bearer sk-taiji-master-key' http://localhost:4000/health" +echo "" + +# 7. 检查日志(最近10行) +echo "7. 检查服务日志(最近10行)..." +echo "" +echo "MCP Server 日志:" +kubectl logs -l app=mcp-server -n $NAMESPACE --tail=10 2>&1 | head -10 || echo "无法获取日志" +echo "" +echo "Data Ingestion 日志:" +kubectl logs -l app=data-ingestion -n $NAMESPACE --tail=10 2>&1 | head -10 || echo "无法获取日志" +echo "" +echo "LiteLLM Gateway 日志:" +kubectl logs -l app=litellm-gateway -n $NAMESPACE --tail=10 2>&1 | head -10 || echo "无法获取日志" +echo "" + +echo "==========================================" +echo "测试完成" +echo "==========================================" +