1215 lines
44 KiB
Python
1215 lines
44 KiB
Python
"""Deployment endpoints for Heicode sub-mode runtime integration."""
|
|
from fastapi import APIRouter, Depends, Request, HTTPException, BackgroundTasks
|
|
from sqlalchemy.orm import Session
|
|
from typing import Optional
|
|
import uuid
|
|
from datetime import datetime, timedelta
|
|
import json
|
|
|
|
from database import get_db, Deployment, AgentInstance, Event, AuditLog, Swarm, SwarmAgent
|
|
from database import DeploymentStatus as DBDeploymentStatus, RiskLevel as DBRiskLevel, BillingProvider as DBBillingProvider
|
|
from database import SwarmStatus, SwarmAgentStatus
|
|
from api.agnet.models import (
|
|
CreateDeploymentRequest, CreateDeploymentResponse, AgentInstanceResponse,
|
|
ListDeploymentsResponse, GetDeploymentResponse, StopDeploymentRequest, StopDeploymentResponse,
|
|
DeploymentSummary, BudgetSummary, PaginationInfo,
|
|
GetLogsResponse, LogEntry, GetEventsResponse, EventEntry, GetMetricsResponse,
|
|
AgentMetrics, ResourceMetrics
|
|
)
|
|
from api.agnet.auth import verify_service_token, extract_headers
|
|
from api.agnet.validators import validate_no_sensitive_fields, validate_vault_references
|
|
from api.agnet.idempotency import idempotency_cache
|
|
from api.agnet.k8s_manager import k8s_manager
|
|
from api.agnet.vault_client import vault_client
|
|
from api.swarm.callback_client import CallbackDeliveryClient
|
|
from api.status_projection import RuntimeDisplayStatus, project_deployment_status, project_runtime_run_status
|
|
from config.error_codes import ErrorCode
|
|
from config.settings import settings
|
|
import logging
|
|
import hashlib
|
|
import asyncio
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
router = APIRouter()
|
|
|
|
|
|
def _project_deployment_status_for_record(db: Session, deployment: Deployment) -> str:
|
|
"""Project persisted deployment state into a Manager-facing runtime status."""
|
|
events = (
|
|
db.query(Event.event_type, Event.payload)
|
|
.filter(Event.deployment_id == deployment.deployment_id)
|
|
.order_by(Event.occurred_at.asc())
|
|
.all()
|
|
)
|
|
normalized_events = [{"event_type": event_type, "payload": payload or {}} for event_type, payload in events]
|
|
return project_deployment_status(deployment.status, phase=deployment.phase, events=normalized_events)
|
|
|
|
|
|
def _project_agent_instance_status(deployment_status: str, agent_status: str) -> str:
|
|
"""Keep agent rows aligned with the projected deployment status contract."""
|
|
if deployment_status == RuntimeDisplayStatus.ACCEPTED.value and agent_status == DBDeploymentStatus.PENDING.value:
|
|
return RuntimeDisplayStatus.ACCEPTED.value
|
|
return agent_status
|
|
|
|
|
|
def generate_deployment_id() -> str:
|
|
"""Generate unique deployment ID."""
|
|
return f"dep_{uuid.uuid4().hex[:12]}"
|
|
|
|
|
|
def generate_agent_instance_id() -> str:
|
|
"""Generate unique agent instance ID."""
|
|
return f"agi_{uuid.uuid4().hex[:12]}"
|
|
|
|
|
|
def generate_event_id() -> str:
|
|
"""Generate unique event ID."""
|
|
return f"evt_{uuid.uuid4().hex[:12]}"
|
|
|
|
|
|
def generate_audit_id() -> str:
|
|
"""Generate unique audit ID."""
|
|
return f"aud_{uuid.uuid4().hex[:12]}"
|
|
|
|
|
|
async def emit_sub_mode_lifecycle_callbacks(
|
|
deployment_id: str,
|
|
callback_config: Optional[dict],
|
|
correlation_id: Optional[str],
|
|
agile_context: Optional[dict],
|
|
agents: list[dict],
|
|
budget: Optional[dict],
|
|
billing_context: Optional[dict],
|
|
metadata: Optional[dict],
|
|
risk_level: str,
|
|
) -> None:
|
|
"""Emit a minimal ordinary sub-mode lifecycle for Manager联调."""
|
|
callback = CallbackDeliveryClient(callback_config)
|
|
if not callback.enabled:
|
|
return
|
|
|
|
metadata = metadata or {}
|
|
agile_context = agile_context or {}
|
|
callback_deployment_id = metadata.get("manager_deployment_id") or metadata.get("heicode_deployment_id") or deployment_id
|
|
agent = agents[0] if agents else {}
|
|
agent_id = agent.get("agent_instance_id") or f"agi_{agent.get('role', 'backend')}_runtime"
|
|
agent_role = agent.get("role", "backend")
|
|
|
|
async def emit(event_type: str, payload: dict, agent_instance_id: Optional[str] = None) -> None:
|
|
payload = {
|
|
**payload,
|
|
"runtime_deployment_id": deployment_id,
|
|
"manager_deployment_id": metadata.get("manager_deployment_id"),
|
|
"source": "agent-manager",
|
|
}
|
|
await callback.emit(
|
|
event_type,
|
|
callback_deployment_id,
|
|
swarm_id=deployment_id,
|
|
agent_instance_id=agent_instance_id,
|
|
correlation_id=correlation_id or metadata.get("correlation_id"),
|
|
payload=payload,
|
|
)
|
|
|
|
stage = agile_context.get("stage") or "planning"
|
|
checkpoint = agile_context.get("checkpoint") or "draft_created"
|
|
await emit("deployment.status_changed", {"status": "running", "stage": stage, "checkpoint": checkpoint})
|
|
await emit(
|
|
"phase.changed",
|
|
{
|
|
"stage": stage,
|
|
"checkpoint": checkpoint,
|
|
"title": "Runtime 已接收普通 sub 敏捷任务",
|
|
"summary": "Agent Manager 已创建 Runtime deployment 并开始执行",
|
|
"agent_role": agent_role,
|
|
"severity": "info",
|
|
"next_action": agile_context.get("next_action") or "continue",
|
|
},
|
|
agent_id,
|
|
)
|
|
await emit(
|
|
"timeline.updated",
|
|
{
|
|
"title": "Runtime accepted",
|
|
"summary": "普通 sub 敏捷任务已进入执行层",
|
|
"stage": stage,
|
|
"checkpoint": checkpoint,
|
|
"agent_role": agent_role,
|
|
"severity": "info",
|
|
"next_action": "continue",
|
|
},
|
|
agent_id,
|
|
)
|
|
|
|
await asyncio.sleep(0.1)
|
|
await emit("agent.started", {"agent_role": agent_role, "status": "running"}, agent_id)
|
|
await emit(
|
|
"phase.changed",
|
|
{
|
|
"stage": "development",
|
|
"checkpoint": "agent_running",
|
|
"title": "开发执行中",
|
|
"summary": "Runtime 正在执行普通 sub 敏捷任务",
|
|
"agent_role": agent_role,
|
|
"severity": "info",
|
|
"next_action": "submit_artifact",
|
|
},
|
|
agent_id,
|
|
)
|
|
|
|
if agile_context.get("requires_user_approval") or risk_level == "high":
|
|
await emit(
|
|
"approval.requested",
|
|
{
|
|
"approval_id": f"appr_{deployment_id}",
|
|
"operation": "runtime.high_risk_action",
|
|
"resource_id": metadata.get("manager_deployment_id") or deployment_id,
|
|
"resource_type": "deployment",
|
|
"target_role": agent_role,
|
|
"risk_level": risk_level,
|
|
"requires_credential": False,
|
|
"ttl_seconds": 900,
|
|
"reason": "普通 sub 敏捷任务需要用户审批后继续高危动作",
|
|
},
|
|
agent_id,
|
|
)
|
|
|
|
await emit(
|
|
"artifact.created",
|
|
{
|
|
"artifact_id": f"art_{deployment_id}_runtime_summary",
|
|
"artifact_type": "document",
|
|
"title": "Runtime 执行摘要",
|
|
"summary": "Agent Manager 已接收任务并产生可追踪 Runtime 摘要",
|
|
"uri": f"artifact://runtime/{deployment_id}/summary",
|
|
"stage": "development",
|
|
"checkpoint": "artifact_ready",
|
|
"metadata": {"redacted": True},
|
|
},
|
|
agent_id,
|
|
)
|
|
|
|
budget = budget or {}
|
|
billing_context = billing_context or {}
|
|
if budget.get("max_cost_usd"):
|
|
await emit(
|
|
"budget.alert",
|
|
{
|
|
"model_id": billing_context.get("default_model_id") or "unknown",
|
|
"model_tokens": 0,
|
|
"prompt_tokens": 0,
|
|
"completion_tokens": 0,
|
|
"model_cost_usd": 0,
|
|
"runtime_seconds": 0,
|
|
"cpu_core_seconds": 0,
|
|
"memory_mb_seconds": 0,
|
|
"billing_source": billing_context.get("provider") or "newapi",
|
|
"consumed_usd": 0,
|
|
"max_cost_usd": budget.get("max_cost_usd"),
|
|
"threshold_pct": budget.get("alert_threshold_pct", 80),
|
|
"severity": "info",
|
|
"budget": {
|
|
"max_tokens": budget.get("max_tokens"),
|
|
"max_cost_usd": budget.get("max_cost_usd"),
|
|
"consumed_usd": 0,
|
|
"remaining_usd": budget.get("max_cost_usd"),
|
|
},
|
|
},
|
|
agent_id,
|
|
)
|
|
|
|
|
|
def get_agent_namespace(user_id: str, binding_scope: str) -> str:
|
|
"""Generate namespace for Heicode agent deployments."""
|
|
combined = f"{user_id}:{binding_scope}"
|
|
hash_suffix = hashlib.sha256(combined.encode()).hexdigest()[:6]
|
|
namespace = f"agent-{user_id}-{hash_suffix}"[:63]
|
|
return namespace.lower().replace("_", "-")
|
|
|
|
|
|
def validate_deployment_request(request: CreateDeploymentRequest, headers: dict) -> None:
|
|
"""Validate deployment request."""
|
|
# 1. Check default_model_id in allowed_model_ids
|
|
if request.billing_context.default_model_id not in request.billing_context.allowed_model_ids:
|
|
raise HTTPException(
|
|
status_code=422,
|
|
detail={
|
|
"success": False,
|
|
"error": {
|
|
"code": ErrorCode.MODEL_NOT_ALLOWED,
|
|
"message": f"default_model_id '{request.billing_context.default_model_id}' not in allowed_model_ids"
|
|
}
|
|
}
|
|
)
|
|
|
|
# 2. High risk requires approval token
|
|
if request.risk_level == "high" and not request.approval_token:
|
|
raise HTTPException(
|
|
status_code=422,
|
|
detail={
|
|
"success": False,
|
|
"error": {
|
|
"code": ErrorCode.POLICY_REJECTED,
|
|
"message": "High-risk deployments require approval_token"
|
|
}
|
|
}
|
|
)
|
|
|
|
# 3. Validate no sensitive fields (recursive scan)
|
|
validate_no_sensitive_fields(request.model_dump())
|
|
|
|
# 4. Validate vault references
|
|
validate_vault_references(request.model_dump())
|
|
|
|
|
|
def serialize_orchestration_plan(orchestration_plan) -> str:
|
|
"""Store structured sub-mode plans without losing their shape."""
|
|
if isinstance(orchestration_plan, str):
|
|
return orchestration_plan
|
|
return json.dumps(orchestration_plan, ensure_ascii=False, sort_keys=True)
|
|
|
|
|
|
def build_plan_summary(request: CreateDeploymentRequest) -> dict:
|
|
"""Extract Heicode sub-mode metadata for events and runtime config."""
|
|
if isinstance(request.orchestration_plan, dict):
|
|
plan = request.orchestration_plan
|
|
else:
|
|
plan = {}
|
|
return {
|
|
"intent_id": plan.get("intent_id"),
|
|
"template_hint": plan.get("template_hint"),
|
|
"objective": plan.get("objective") if plan else request.orchestration_plan,
|
|
"sub_mode": request.sub_mode,
|
|
"agile_context": request.agile_context,
|
|
"user_context": plan.get("user_context", {}),
|
|
"metadata": request.metadata,
|
|
}
|
|
|
|
|
|
def build_deployment_response_from_swarm(db: Session, swarm: Swarm) -> GetDeploymentResponse:
|
|
"""Expose compatibility runtime runs through the deployment-detail shape."""
|
|
context = swarm.project_context or {}
|
|
billing_context = context.get("billing_context") or {}
|
|
budget = context.get("budget") or {}
|
|
agents = db.query(SwarmAgent).filter(SwarmAgent.swarm_id == swarm.swarm_id).all()
|
|
display_status = project_runtime_run_status(swarm.status)
|
|
return GetDeploymentResponse(
|
|
deployment_id=swarm.swarm_id,
|
|
user_id=swarm.owner_id,
|
|
binding_scope=context.get("binding_scope") or context.get("intent_id") or swarm.swarm_id,
|
|
status=display_status,
|
|
phase=swarm.phase,
|
|
orchestration_plan=json.dumps(
|
|
{
|
|
"intent_id": context.get("intent_id"),
|
|
"template_hint": context.get("template_hint"),
|
|
"objective": swarm.task_description,
|
|
"sub_mode": context.get("sub_mode", "agile"),
|
|
"agile_context": context.get("agile_context") or {},
|
|
},
|
|
ensure_ascii=False,
|
|
),
|
|
risk_level="low",
|
|
budget=BudgetSummary(
|
|
max_usd=budget.get("max_cost_usd") or 0.0,
|
|
consumed_usd=0.0,
|
|
remaining_usd=budget.get("max_cost_usd") or 0.0,
|
|
),
|
|
billing_context={
|
|
"provider": billing_context.get("provider") or "newapi",
|
|
"default_model_id": billing_context.get("default_model_id"),
|
|
"allowed_model_ids": billing_context.get("allowed_model_ids") or [],
|
|
},
|
|
agent_instances=[
|
|
AgentInstanceResponse(
|
|
agent_instance_id=agent.agent_id,
|
|
role=agent.role,
|
|
status=_project_agent_instance_status(display_status, agent.status.value),
|
|
phase=swarm.phase,
|
|
)
|
|
for agent in agents
|
|
],
|
|
resource_grants=context.get("resource_grants") or [],
|
|
created_at=swarm.created_at,
|
|
updated_at=swarm.updated_at,
|
|
)
|
|
|
|
|
|
def create_audit_log(
|
|
db: Session,
|
|
actor: str,
|
|
action: str,
|
|
resource_type: str,
|
|
resource_id: str,
|
|
result: str,
|
|
correlation_id: Optional[str] = None,
|
|
error_code: Optional[str] = None,
|
|
error_message: Optional[str] = None
|
|
):
|
|
"""Create audit log entry."""
|
|
audit_log = AuditLog(
|
|
audit_id=generate_audit_id(),
|
|
actor=actor or "system",
|
|
action=action,
|
|
resource_type=resource_type,
|
|
resource_id=resource_id,
|
|
result=result,
|
|
correlation_id=correlation_id,
|
|
error_code=error_code,
|
|
error_message=error_message,
|
|
occurred_at=datetime.utcnow()
|
|
)
|
|
db.add(audit_log)
|
|
db.commit()
|
|
|
|
|
|
@router.post("/deployments", response_model=CreateDeploymentResponse)
|
|
async def create_deployment(
|
|
request: CreateDeploymentRequest,
|
|
http_request: Request,
|
|
background_tasks: BackgroundTasks,
|
|
db: Session = Depends(get_db),
|
|
token: str = Depends(verify_service_token)
|
|
):
|
|
"""Create a new sub-agile runtime deployment."""
|
|
headers = extract_headers(http_request)
|
|
correlation_id = headers.get("correlation_id")
|
|
user_id = headers.get("user_id")
|
|
binding_scope = headers.get("binding_scope")
|
|
idempotency_key = headers.get("idempotency_key")
|
|
|
|
logger.info(f"Creating deployment - correlation_id={correlation_id}, user_id={user_id}")
|
|
|
|
try:
|
|
# Check idempotency
|
|
if idempotency_key:
|
|
cached = idempotency_cache.get(idempotency_key)
|
|
if cached:
|
|
logger.info(f"Returning cached response for idempotency_key={idempotency_key}")
|
|
return cached
|
|
|
|
# Validate request
|
|
validate_deployment_request(request, headers)
|
|
|
|
# Generate IDs
|
|
deployment_id = generate_deployment_id()
|
|
plan_summary = build_plan_summary(request)
|
|
plan_user_context = plan_summary.get("user_context") or {}
|
|
user_id = user_id or plan_user_context.get("user_id") or "default"
|
|
binding_scope = binding_scope or plan_user_context.get("binding_scope") or f"task-{plan_summary.get('intent_id') or deployment_id}"
|
|
correlation_id = correlation_id or request.metadata.get("correlation_id")
|
|
namespace = get_agent_namespace(user_id, binding_scope)
|
|
|
|
# Create deployment record
|
|
deployment = Deployment(
|
|
deployment_id=deployment_id,
|
|
user_id=user_id,
|
|
binding_scope=binding_scope,
|
|
correlation_id=correlation_id,
|
|
orchestration_plan=serialize_orchestration_plan(request.orchestration_plan),
|
|
risk_level=DBRiskLevel(request.risk_level.value),
|
|
approval_token=request.approval_token,
|
|
budget_max_usd=request.budget.max_usd,
|
|
budget_consumed_usd=0.0,
|
|
budget_alert_threshold_pct=request.budget.alert_threshold_pct,
|
|
billing_provider=DBBillingProvider(request.billing_context.provider.value),
|
|
default_model_id=request.billing_context.default_model_id,
|
|
allowed_model_ids=request.billing_context.allowed_model_ids,
|
|
secret_ref=request.billing_context.secret_ref,
|
|
resource_grants=[grant.model_dump() for grant in request.resource_grants],
|
|
status=DBDeploymentStatus.PENDING,
|
|
namespace=namespace,
|
|
created_at=datetime.utcnow()
|
|
)
|
|
db.add(deployment)
|
|
db.flush()
|
|
|
|
# Create agent instances
|
|
agent_instances = []
|
|
for agent_config in request.agents:
|
|
instance_id = generate_agent_instance_id()
|
|
instance = AgentInstance(
|
|
agent_instance_id=instance_id,
|
|
deployment_id=deployment_id,
|
|
role=agent_config.role,
|
|
image=agent_config.image,
|
|
namespace=namespace,
|
|
pod_name=f"agent-{instance_id}",
|
|
status=DBDeploymentStatus.PENDING,
|
|
created_at=datetime.utcnow()
|
|
)
|
|
db.add(instance)
|
|
agent_instances.append(instance)
|
|
|
|
# Create deployment.accepted event
|
|
event = Event(
|
|
event_id=generate_event_id(),
|
|
deployment_id=deployment_id,
|
|
event_type="deployment.accepted",
|
|
correlation_id=correlation_id,
|
|
payload={
|
|
"risk_level": request.risk_level.value,
|
|
"callback_configured": request.callback is not None,
|
|
"callback_subscribed_events": request.callback.subscribed_events if request.callback else None,
|
|
"callback_url": request.callback.url if request.callback else None,
|
|
"callback_signing_secret_ref": request.callback.signing_secret_ref if request.callback else None,
|
|
"sub_mode": request.sub_mode,
|
|
"intent_id": plan_summary.get("intent_id"),
|
|
"agile_context": request.agile_context,
|
|
},
|
|
occurred_at=datetime.utcnow()
|
|
)
|
|
db.add(event)
|
|
|
|
# Create audit log
|
|
create_audit_log(
|
|
db=db,
|
|
actor=user_id,
|
|
action="create_deployment",
|
|
resource_type="deployment",
|
|
resource_id=deployment_id,
|
|
result="success",
|
|
correlation_id=correlation_id
|
|
)
|
|
|
|
db.commit()
|
|
|
|
# Create Kubernetes resources
|
|
try:
|
|
# Create namespace
|
|
k8s_manager.create_namespace(namespace)
|
|
|
|
# Fetch secrets from Vault
|
|
model_gateway_secret = None
|
|
if request.billing_context.secret_ref and request.billing_context.secret_ref.startswith("vault:"):
|
|
model_gateway_secret = await vault_client.get_secret(
|
|
request.billing_context.secret_ref
|
|
)
|
|
|
|
# Create ConfigMap with deployment configuration
|
|
configmap_name = f"deployment-{deployment_id}"
|
|
configmap_data = {
|
|
"DEPLOYMENT_ID": deployment_id,
|
|
"BILLING_PROVIDER": request.billing_context.provider.value,
|
|
"MODEL_GATEWAY_URL": settings.HEICODE_NEWAPI_BASE_URL if request.billing_context.provider.value == "newapi" else settings.LITELLM_BASE_URL,
|
|
"DEFAULT_MODEL_ID": request.billing_context.default_model_id,
|
|
"ALLOWED_MODEL_IDS": ",".join(request.billing_context.allowed_model_ids),
|
|
"SUB_MODE": request.sub_mode or "agile",
|
|
"PLAN_SUMMARY": json.dumps(plan_summary, ensure_ascii=False),
|
|
"AGILE_CONTEXT": json.dumps(request.agile_context or {}, ensure_ascii=False),
|
|
"RESOURCE_GRANTS": json.dumps([grant.model_dump() for grant in request.resource_grants], ensure_ascii=False),
|
|
}
|
|
if request.billing_context.secret_ref:
|
|
configmap_data["MODEL_GATEWAY_SECRET_REF"] = request.billing_context.secret_ref
|
|
if request.budget.max_tokens is not None:
|
|
configmap_data["BUDGET_MAX_TOKENS"] = str(request.budget.max_tokens)
|
|
if request.budget.max_duration_sec is not None:
|
|
configmap_data["BUDGET_MAX_DURATION_SEC"] = str(request.budget.max_duration_sec)
|
|
if request.callback:
|
|
configmap_data.update({
|
|
"CALLBACK_URL": request.callback.url,
|
|
"CALLBACK_SIGNING_SECRET_REF": request.callback.signing_secret_ref,
|
|
"CALLBACK_SUBSCRIBED_EVENTS": ",".join(request.callback.subscribed_events or []),
|
|
})
|
|
k8s_manager.create_configmap(namespace, configmap_name, configmap_data)
|
|
|
|
# Update deployment with configmap name
|
|
deployment.configmap_name = configmap_name
|
|
|
|
# Create pods for each agent instance
|
|
for instance in agent_instances:
|
|
env_vars = {
|
|
"AGENT_INSTANCE_ID": instance.agent_instance_id,
|
|
"AGENT_ROLE": instance.role,
|
|
}
|
|
|
|
# Add model gateway secret if available
|
|
if model_gateway_secret:
|
|
env_vars["MODEL_GATEWAY_API_KEY"] = model_gateway_secret
|
|
|
|
# Fetch resource grant secrets
|
|
for grant in request.resource_grants:
|
|
if grant.ref and grant.ref.startswith("azkv://"):
|
|
env_var_name = f"{grant.type.upper()}_SECRET_REF"
|
|
env_vars[env_var_name] = grant.ref
|
|
elif grant.ref and grant.ref.startswith("vault:"):
|
|
secret = await vault_client.get_secret(grant.ref)
|
|
if secret:
|
|
# Use grant type as env var prefix
|
|
env_var_name = f"{grant.type.upper()}_SECRET"
|
|
env_vars[env_var_name] = secret
|
|
|
|
labels = {
|
|
"deployment_id": deployment_id,
|
|
"agent_instance_id": instance.agent_instance_id,
|
|
"role": instance.role,
|
|
}
|
|
|
|
# Get image from agent config
|
|
agent_config = next(
|
|
(a for a in request.agents if a.role == instance.role),
|
|
None
|
|
)
|
|
image = agent_config.image if agent_config else "nginx:1.27-alpine"
|
|
|
|
k8s_manager.create_pod(
|
|
namespace=namespace,
|
|
pod_name=instance.pod_name,
|
|
image=image,
|
|
env_vars=env_vars,
|
|
configmap_name=configmap_name,
|
|
labels=labels
|
|
)
|
|
|
|
db.commit()
|
|
logger.info(f"Created K8s resources for deployment {deployment_id}")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to create K8s resources: {e}")
|
|
# Don't fail the request, pods can be created later
|
|
|
|
# Build response
|
|
response = CreateDeploymentResponse(
|
|
deployment_id=deployment_id,
|
|
swarm_id=deployment_id,
|
|
status=RuntimeDisplayStatus.ACCEPTED.value,
|
|
agent_instances=[
|
|
AgentInstanceResponse(
|
|
agent_instance_id=inst.agent_instance_id,
|
|
role=inst.role,
|
|
status=RuntimeDisplayStatus.ACCEPTED.value,
|
|
phase=None
|
|
)
|
|
for inst in agent_instances
|
|
],
|
|
created_at=deployment.created_at,
|
|
estimated_ready_at=deployment.created_at + timedelta(minutes=2),
|
|
data={
|
|
"deployment_id": deployment_id,
|
|
"swarm_id": deployment_id,
|
|
"status": RuntimeDisplayStatus.ACCEPTED.value,
|
|
"estimated_ready_at": (deployment.created_at + timedelta(minutes=2)).isoformat(),
|
|
},
|
|
)
|
|
|
|
if request.callback:
|
|
background_tasks.add_task(
|
|
emit_sub_mode_lifecycle_callbacks,
|
|
deployment_id,
|
|
request.callback.model_dump(exclude_none=True),
|
|
correlation_id,
|
|
request.agile_context,
|
|
[
|
|
{
|
|
"agent_instance_id": inst.agent_instance_id,
|
|
"role": inst.role,
|
|
}
|
|
for inst in agent_instances
|
|
],
|
|
request.budget.model_dump(exclude_none=True) if request.budget else {},
|
|
request.billing_context.model_dump(exclude_none=True) if request.billing_context else {},
|
|
request.metadata,
|
|
request.risk_level.value,
|
|
)
|
|
|
|
# Cache response for idempotency
|
|
if idempotency_key:
|
|
idempotency_cache.set(idempotency_key, response.model_dump())
|
|
|
|
logger.info(f"Deployment created successfully - deployment_id={deployment_id}")
|
|
return response
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Failed to create deployment: {e}")
|
|
create_audit_log(
|
|
db=db,
|
|
actor=user_id or "unknown",
|
|
action="create_deployment",
|
|
resource_type="deployment",
|
|
resource_id="",
|
|
result="failure",
|
|
correlation_id=correlation_id,
|
|
error_code=ErrorCode.INTERNAL_ERROR,
|
|
error_message=str(e)
|
|
)
|
|
raise HTTPException(
|
|
status_code=500,
|
|
detail={
|
|
"success": False,
|
|
"error": {
|
|
"code": ErrorCode.INTERNAL_ERROR,
|
|
"message": "Failed to create deployment",
|
|
"request_id": correlation_id
|
|
}
|
|
}
|
|
)
|
|
|
|
|
|
@router.get("/deployments", response_model=ListDeploymentsResponse)
|
|
async def list_deployments(
|
|
user_id: Optional[str] = None,
|
|
binding_scope: Optional[str] = None,
|
|
status: Optional[str] = None,
|
|
limit: int = 50,
|
|
cursor: Optional[str] = None,
|
|
db: Session = Depends(get_db),
|
|
token: str = Depends(verify_service_token)
|
|
):
|
|
"""List sub-mode runtime deployments with filtering and pagination."""
|
|
query = db.query(Deployment)
|
|
|
|
# Apply filters
|
|
if user_id:
|
|
query = query.filter(Deployment.user_id == user_id)
|
|
if binding_scope:
|
|
query = query.filter(Deployment.binding_scope == binding_scope)
|
|
if status:
|
|
query = query.filter(Deployment.status == status)
|
|
|
|
# Order by created_at desc
|
|
query = query.order_by(Deployment.created_at.desc())
|
|
|
|
# Apply limit
|
|
limit = min(limit, 200) # Max 200
|
|
deployments = query.limit(limit + 1).all()
|
|
|
|
# Check if there are more results
|
|
has_more = len(deployments) > limit
|
|
if has_more:
|
|
deployments = deployments[:limit]
|
|
|
|
# Build response
|
|
deployment_summaries = []
|
|
for dep in deployments:
|
|
instance_count = db.query(AgentInstance).filter(
|
|
AgentInstance.deployment_id == dep.deployment_id
|
|
).count()
|
|
display_status = _project_deployment_status_for_record(db, dep)
|
|
|
|
deployment_summaries.append(DeploymentSummary(
|
|
deployment_id=dep.deployment_id,
|
|
status=display_status,
|
|
risk_level=dep.risk_level.value,
|
|
budget=BudgetSummary(
|
|
max_usd=dep.budget_max_usd or 0.0,
|
|
consumed_usd=dep.budget_consumed_usd or 0.0,
|
|
remaining_usd=(dep.budget_max_usd or 0.0) - (dep.budget_consumed_usd or 0.0)
|
|
),
|
|
created_at=dep.created_at,
|
|
agent_instances_count=instance_count
|
|
))
|
|
|
|
return ListDeploymentsResponse(
|
|
deployments=deployment_summaries,
|
|
pagination=PaginationInfo(
|
|
next_cursor=None, # TODO: Implement cursor pagination
|
|
has_more=has_more
|
|
)
|
|
)
|
|
|
|
|
|
@router.get("/deployments/{deployment_id}", response_model=GetDeploymentResponse)
|
|
async def get_deployment(
|
|
deployment_id: str,
|
|
db: Session = Depends(get_db),
|
|
token: str = Depends(verify_service_token)
|
|
):
|
|
"""Get sub-mode runtime deployment details."""
|
|
deployment = db.query(Deployment).filter(
|
|
Deployment.deployment_id == deployment_id
|
|
).first()
|
|
|
|
if not deployment:
|
|
swarm = db.query(Swarm).filter(Swarm.swarm_id == deployment_id).first()
|
|
if swarm:
|
|
return build_deployment_response_from_swarm(db, swarm)
|
|
raise HTTPException(
|
|
status_code=404,
|
|
detail={
|
|
"success": False,
|
|
"error": {
|
|
"code": ErrorCode.DEPLOYMENT_NOT_FOUND,
|
|
"message": f"Deployment {deployment_id} not found"
|
|
}
|
|
}
|
|
)
|
|
|
|
# Get agent instances
|
|
instances = db.query(AgentInstance).filter(
|
|
AgentInstance.deployment_id == deployment_id
|
|
).all()
|
|
display_status = _project_deployment_status_for_record(db, deployment)
|
|
|
|
return GetDeploymentResponse(
|
|
deployment_id=deployment.deployment_id,
|
|
user_id=deployment.user_id,
|
|
binding_scope=deployment.binding_scope,
|
|
status=display_status,
|
|
phase=deployment.phase,
|
|
orchestration_plan=deployment.orchestration_plan,
|
|
risk_level=deployment.risk_level.value,
|
|
budget=BudgetSummary(
|
|
max_usd=deployment.budget_max_usd or 0.0,
|
|
consumed_usd=deployment.budget_consumed_usd or 0.0,
|
|
remaining_usd=(deployment.budget_max_usd or 0.0) - (deployment.budget_consumed_usd or 0.0)
|
|
),
|
|
billing_context={
|
|
"provider": deployment.billing_provider.value,
|
|
"default_model_id": deployment.default_model_id,
|
|
"allowed_model_ids": deployment.allowed_model_ids
|
|
},
|
|
agent_instances=[
|
|
AgentInstanceResponse(
|
|
agent_instance_id=inst.agent_instance_id,
|
|
role=inst.role,
|
|
status=_project_agent_instance_status(display_status, inst.status.value),
|
|
phase=inst.phase
|
|
)
|
|
for inst in instances
|
|
],
|
|
resource_grants=deployment.resource_grants or [],
|
|
created_at=deployment.created_at,
|
|
updated_at=deployment.updated_at
|
|
)
|
|
|
|
|
|
@router.post("/deployments/{deployment_id}/stop", response_model=StopDeploymentResponse)
|
|
async def stop_deployment(
|
|
deployment_id: str,
|
|
request: StopDeploymentRequest,
|
|
http_request: Request,
|
|
db: Session = Depends(get_db),
|
|
token: str = Depends(verify_service_token)
|
|
):
|
|
"""Stop a sub-mode runtime deployment."""
|
|
headers = extract_headers(http_request)
|
|
correlation_id = headers.get("correlation_id")
|
|
user_id = headers.get("user_id")
|
|
|
|
deployment = db.query(Deployment).filter(
|
|
Deployment.deployment_id == deployment_id
|
|
).first()
|
|
|
|
if not deployment:
|
|
swarm = db.query(Swarm).filter(Swarm.swarm_id == deployment_id).first()
|
|
if swarm:
|
|
stopped_at = datetime.utcnow()
|
|
if swarm.status != SwarmStatus.STOPPED:
|
|
swarm.status = SwarmStatus.STOPPED
|
|
swarm.error_message = request.reason
|
|
swarm.updated_at = stopped_at
|
|
db.query(SwarmAgent).filter(SwarmAgent.swarm_id == deployment_id).update(
|
|
{"status": SwarmAgentStatus.FAILED if request.reason else SwarmAgentStatus.COMPLETED}
|
|
)
|
|
db.commit()
|
|
return StopDeploymentResponse(
|
|
deployment_id=deployment_id,
|
|
status=RuntimeDisplayStatus.STOPPED.value,
|
|
stopped_at=stopped_at,
|
|
)
|
|
raise HTTPException(
|
|
status_code=404,
|
|
detail={
|
|
"success": False,
|
|
"error": {
|
|
"code": ErrorCode.DEPLOYMENT_NOT_FOUND,
|
|
"message": f"Deployment {deployment_id} not found"
|
|
}
|
|
}
|
|
)
|
|
|
|
# Idempotent: if already stopped, return current status
|
|
if deployment.status == DBDeploymentStatus.STOPPED:
|
|
return StopDeploymentResponse(
|
|
deployment_id=deployment_id,
|
|
status="stopped",
|
|
stopped_at=deployment.stopped_at or deployment.updated_at
|
|
)
|
|
|
|
# Check if in terminal state
|
|
if deployment.status == DBDeploymentStatus.FAILED:
|
|
raise HTTPException(
|
|
status_code=409,
|
|
detail={
|
|
"success": False,
|
|
"error": {
|
|
"code": ErrorCode.DEPLOYMENT_CONFLICT,
|
|
"message": f"Cannot stop deployment in '{deployment.status.value}' state"
|
|
}
|
|
}
|
|
)
|
|
|
|
# High risk requires approval
|
|
if deployment.risk_level == DBRiskLevel.HIGH and not request.approval_token:
|
|
raise HTTPException(
|
|
status_code=422,
|
|
detail={
|
|
"success": False,
|
|
"error": {
|
|
"code": ErrorCode.POLICY_REJECTED,
|
|
"message": "High-risk deployment stop requires approval_token"
|
|
}
|
|
}
|
|
)
|
|
|
|
# Update deployment status
|
|
deployment.status = DBDeploymentStatus.STOPPED
|
|
deployment.stopped_at = datetime.utcnow()
|
|
deployment.updated_at = datetime.utcnow()
|
|
|
|
# Update agent instances
|
|
db.query(AgentInstance).filter(
|
|
AgentInstance.deployment_id == deployment_id
|
|
).update({"status": DBDeploymentStatus.STOPPED})
|
|
|
|
# Create event
|
|
event = Event(
|
|
event_id=generate_event_id(),
|
|
deployment_id=deployment_id,
|
|
event_type="deployment.stopped",
|
|
correlation_id=correlation_id,
|
|
payload={"reason": request.reason},
|
|
occurred_at=datetime.utcnow()
|
|
)
|
|
db.add(event)
|
|
|
|
# Create audit log
|
|
create_audit_log(
|
|
db=db,
|
|
actor=user_id or "heicode-manager",
|
|
action="stop_deployment",
|
|
resource_type="deployment",
|
|
resource_id=deployment_id,
|
|
result="success",
|
|
correlation_id=correlation_id
|
|
)
|
|
|
|
db.commit()
|
|
|
|
# Delete Kubernetes resources
|
|
try:
|
|
# Get agent instances to delete their pods
|
|
instances = db.query(AgentInstance).filter(
|
|
AgentInstance.deployment_id == deployment_id
|
|
).all()
|
|
|
|
# Delete pods
|
|
for instance in instances:
|
|
k8s_manager.delete_pod(deployment.namespace, instance.pod_name)
|
|
|
|
# Delete ConfigMap
|
|
if deployment.configmap_name:
|
|
k8s_manager.delete_configmap(deployment.namespace, deployment.configmap_name)
|
|
|
|
logger.info(f"Deleted K8s resources for deployment {deployment_id}")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to delete K8s resources: {e}")
|
|
# Don't fail the request, resources can be cleaned up later
|
|
|
|
logger.info(f"Deployment stopped - deployment_id={deployment_id}")
|
|
|
|
return StopDeploymentResponse(
|
|
deployment_id=deployment_id,
|
|
status="stopped",
|
|
stopped_at=deployment.stopped_at
|
|
)
|
|
|
|
|
|
@router.post("/deployments/{deployment_id}/approvals/{approval_id}")
|
|
async def receive_deployment_approval_decision(
|
|
deployment_id: str,
|
|
approval_id: str,
|
|
payload: dict,
|
|
http_request: Request,
|
|
db: Session = Depends(get_db),
|
|
token: str = Depends(verify_service_token),
|
|
):
|
|
"""Accept Manager approval decisions for ordinary sub-mode runtime actions."""
|
|
headers = extract_headers(http_request)
|
|
deployment = db.query(Deployment).filter(
|
|
Deployment.deployment_id == deployment_id
|
|
).first()
|
|
if not deployment:
|
|
raise HTTPException(
|
|
status_code=404,
|
|
detail={
|
|
"success": False,
|
|
"error": {
|
|
"code": ErrorCode.DEPLOYMENT_NOT_FOUND,
|
|
"message": f"Deployment {deployment_id} not found",
|
|
},
|
|
},
|
|
)
|
|
|
|
body_approval_id = payload.get("approval_id") or approval_id
|
|
if body_approval_id != approval_id:
|
|
raise HTTPException(
|
|
status_code=422,
|
|
detail={"success": False, "error": {"code": ErrorCode.INVALID_REQUEST, "message": "approval_id path/body mismatch"}},
|
|
)
|
|
|
|
decision = payload.get("decision")
|
|
if decision not in {"approved", "rejected"}:
|
|
raise HTTPException(
|
|
status_code=422,
|
|
detail={"success": False, "error": {"code": ErrorCode.INVALID_REQUEST, "message": "decision must be approved or rejected"}},
|
|
)
|
|
|
|
event = Event(
|
|
event_id=generate_event_id(),
|
|
deployment_id=deployment_id,
|
|
event_type="approval.decision",
|
|
correlation_id=headers.get("correlation_id") or deployment.correlation_id,
|
|
payload={
|
|
**payload,
|
|
"approval_id": approval_id,
|
|
"decision": decision,
|
|
"source": "heicode-manager",
|
|
},
|
|
occurred_at=datetime.utcnow(),
|
|
)
|
|
db.add(event)
|
|
create_audit_log(
|
|
db=db,
|
|
actor=headers.get("user_id") or "heicode-manager",
|
|
action=f"approval_{decision}",
|
|
resource_type="deployment",
|
|
resource_id=deployment_id,
|
|
result="success",
|
|
correlation_id=headers.get("correlation_id") or deployment.correlation_id,
|
|
)
|
|
db.commit()
|
|
|
|
return {
|
|
"success": True,
|
|
"deployment_id": deployment_id,
|
|
"approval_id": approval_id,
|
|
"decision": decision,
|
|
"status": "accepted",
|
|
}
|
|
|
|
|
|
@router.get("/deployments/{deployment_id}/logs", response_model=GetLogsResponse)
|
|
async def get_deployment_logs(
|
|
deployment_id: str,
|
|
agent_instance_id: Optional[str] = None,
|
|
since: Optional[datetime] = None,
|
|
limit: int = 100,
|
|
db: Session = Depends(get_db),
|
|
token: str = Depends(verify_service_token)
|
|
):
|
|
"""Get logs for a sub-mode runtime deployment or agent instance."""
|
|
deployment = db.query(Deployment).filter(
|
|
Deployment.deployment_id == deployment_id
|
|
).first()
|
|
|
|
if not deployment:
|
|
raise HTTPException(
|
|
status_code=404,
|
|
detail={
|
|
"success": False,
|
|
"error": {
|
|
"code": ErrorCode.DEPLOYMENT_NOT_FOUND,
|
|
"message": f"Deployment {deployment_id} not found"
|
|
}
|
|
}
|
|
)
|
|
|
|
# Get agent instances
|
|
instances_query = db.query(AgentInstance).filter(
|
|
AgentInstance.deployment_id == deployment_id
|
|
)
|
|
if agent_instance_id:
|
|
instances_query = instances_query.filter(
|
|
AgentInstance.agent_instance_id == agent_instance_id
|
|
)
|
|
instances = instances_query.all()
|
|
|
|
# Fetch actual logs from Kubernetes pods
|
|
logs = []
|
|
for instance in instances:
|
|
try:
|
|
pod_logs = k8s_manager.get_pod_logs(
|
|
deployment.namespace,
|
|
instance.pod_name,
|
|
tail_lines=limit
|
|
)
|
|
|
|
if pod_logs:
|
|
# Parse logs into entries (simple line-by-line parsing)
|
|
for line in pod_logs.strip().split('\n')[-limit:]:
|
|
if line.strip():
|
|
logs.append(LogEntry(
|
|
timestamp=datetime.utcnow(),
|
|
agent_instance_id=instance.agent_instance_id,
|
|
level="info",
|
|
message=line,
|
|
source="stdout"
|
|
))
|
|
else:
|
|
# Pod exists but no logs yet
|
|
logs.append(LogEntry(
|
|
timestamp=datetime.utcnow(),
|
|
agent_instance_id=instance.agent_instance_id,
|
|
level="info",
|
|
message=f"Pod {instance.pod_name} has no logs yet",
|
|
source="system"
|
|
))
|
|
except Exception as e:
|
|
logger.error(f"Failed to fetch logs for pod {instance.pod_name}: {e}")
|
|
logs.append(LogEntry(
|
|
timestamp=datetime.utcnow(),
|
|
agent_instance_id=instance.agent_instance_id,
|
|
level="error",
|
|
message=f"Failed to fetch logs: {str(e)}",
|
|
source="system"
|
|
))
|
|
|
|
return GetLogsResponse(
|
|
deployment_id=deployment_id,
|
|
logs=logs,
|
|
pagination=PaginationInfo(has_more=False)
|
|
)
|
|
|
|
|
|
@router.get("/deployments/{deployment_id}/events", response_model=GetEventsResponse)
|
|
async def get_deployment_events(
|
|
deployment_id: str,
|
|
event_type: Optional[str] = None,
|
|
since: Optional[datetime] = None,
|
|
limit: int = 100,
|
|
db: Session = Depends(get_db),
|
|
token: str = Depends(verify_service_token)
|
|
):
|
|
"""Get events for a sub-mode runtime deployment."""
|
|
deployment = db.query(Deployment).filter(
|
|
Deployment.deployment_id == deployment_id
|
|
).first()
|
|
|
|
if not deployment:
|
|
raise HTTPException(
|
|
status_code=404,
|
|
detail={
|
|
"success": False,
|
|
"error": {
|
|
"code": ErrorCode.DEPLOYMENT_NOT_FOUND,
|
|
"message": f"Deployment {deployment_id} not found"
|
|
}
|
|
}
|
|
)
|
|
|
|
# Query events
|
|
events_query = db.query(Event).filter(Event.deployment_id == deployment_id)
|
|
if event_type:
|
|
events_query = events_query.filter(Event.event_type == event_type)
|
|
if since:
|
|
events_query = events_query.filter(Event.occurred_at >= since)
|
|
|
|
events_query = events_query.order_by(Event.occurred_at.desc())
|
|
events = events_query.limit(limit).all()
|
|
|
|
return GetEventsResponse(
|
|
deployment_id=deployment_id,
|
|
events=[
|
|
EventEntry(
|
|
event_id=event.event_id,
|
|
event_type=event.event_type,
|
|
agent_instance_id=event.agent_instance_id,
|
|
occurred_at=event.occurred_at,
|
|
payload=event.payload or {}
|
|
)
|
|
for event in events
|
|
],
|
|
pagination=PaginationInfo(has_more=len(events) == limit)
|
|
)
|
|
|
|
|
|
@router.get("/deployments/{deployment_id}/metrics", response_model=GetMetricsResponse)
|
|
async def get_deployment_metrics(
|
|
deployment_id: str,
|
|
db: Session = Depends(get_db),
|
|
token: str = Depends(verify_service_token)
|
|
):
|
|
"""Get resource metrics for a sub-mode runtime deployment."""
|
|
deployment = db.query(Deployment).filter(
|
|
Deployment.deployment_id == deployment_id
|
|
).first()
|
|
|
|
if not deployment:
|
|
raise HTTPException(
|
|
status_code=404,
|
|
detail={
|
|
"success": False,
|
|
"error": {
|
|
"code": ErrorCode.DEPLOYMENT_NOT_FOUND,
|
|
"message": f"Deployment {deployment_id} not found"
|
|
}
|
|
}
|
|
)
|
|
|
|
# Get agent instances
|
|
instances = db.query(AgentInstance).filter(
|
|
AgentInstance.deployment_id == deployment_id
|
|
).all()
|
|
|
|
# Fetch actual metrics from Kubernetes (or use mock data)
|
|
agent_metrics = []
|
|
total_cpu = 0.0
|
|
total_memory = 0.0
|
|
total_rx = 0
|
|
total_tx = 0
|
|
|
|
for instance in instances:
|
|
try:
|
|
# Get pod status
|
|
pod_status = k8s_manager.get_pod_status(
|
|
deployment.namespace,
|
|
instance.pod_name
|
|
)
|
|
|
|
# Calculate uptime
|
|
uptime = int((datetime.utcnow() - instance.created_at).total_seconds())
|
|
|
|
# Use mock metrics for now (real metrics require metrics-server)
|
|
cpu = 0.1
|
|
memory = 128.0
|
|
rx = 1024
|
|
tx = 2048
|
|
|
|
agent_metrics.append(AgentMetrics(
|
|
agent_instance_id=instance.agent_instance_id,
|
|
role=instance.role,
|
|
status=pod_status or instance.status.value,
|
|
resources=ResourceMetrics(
|
|
cpu_usage_cores=cpu,
|
|
memory_usage_mb=memory,
|
|
network_rx_bytes=rx,
|
|
network_tx_bytes=tx
|
|
),
|
|
uptime_seconds=uptime
|
|
))
|
|
|
|
total_cpu += cpu
|
|
total_memory += memory
|
|
total_rx += rx
|
|
total_tx += tx
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to fetch metrics for pod {instance.pod_name}: {e}")
|
|
|
|
return GetMetricsResponse(
|
|
deployment_id=deployment_id,
|
|
timestamp=datetime.utcnow(),
|
|
agent_metrics=agent_metrics,
|
|
total_resources=ResourceMetrics(
|
|
cpu_usage_cores=total_cpu,
|
|
memory_usage_mb=total_memory,
|
|
network_rx_bytes=total_rx,
|
|
network_tx_bytes=total_tx
|
|
)
|
|
)
|