feat(swarm-controller): replace opensandbox-mcp with daytona-mcp sidecar
Swap the OpenSandbox cloud-sandbox integration for Daytona: - config: drop opensandbox_server_url; add daytona_api_url + daytona_api_key_secret (KV secret NAME, value never in config/ConfigMap) + daytona_mcp_image. - cloud_init: when daytona enabled, add a daytona-mcp sidecar (daytona CLI wrapped by mcp-proxy as Streamable HTTP :8090) to each tenant compose, resolve DAYTONA_API_KEY from Key Vault on-VM into .env, and register it as a swarm-scope MCP server + install for lead+workers. Remove all opensandbox code (launcher monkey-patch, register block, compose service, urlparse). - k8s/configmap: drop opensandbox, add DAYTONA_API_URL + DAYTONA_API_KEY_SECRET. - daytona-mcp/Dockerfile: sidecar image (daytona CLI + mcp-proxy@6.5.2). - tests: daytona sidecar/registration/gating coverage. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
ff4c1c14b3
commit
25d632de0c
@@ -0,0 +1,29 @@
|
|||||||
|
# daytona-mcp sidecar — Daytona CLI (stdio MCP) wrapped by mcp-proxy as
|
||||||
|
# Streamable HTTP on :8090. Added to each tenant swarm's docker-compose by
|
||||||
|
# swarm_controller.provisioning.cloud_init when daytona is enabled. Auth via
|
||||||
|
# DAYTONA_API_KEY (env, resolved on-VM from Key Vault); DAYTONA_API_URL selects
|
||||||
|
# the Daytona control plane.
|
||||||
|
#
|
||||||
|
# Build (cloud, amd64) — pushed to heicodetest ACR:
|
||||||
|
# az acr build --registry heicodetest --image daytona-mcp:heicode-test \
|
||||||
|
# --file swarm-controller/daytona-mcp/Dockerfile swarm-controller/daytona-mcp
|
||||||
|
FROM node:20-bookworm-slim
|
||||||
|
|
||||||
|
# Daytona CLI release: https://github.com/daytonaio/daytona/releases
|
||||||
|
ARG DAYTONA_VERSION=v0.190.0
|
||||||
|
ARG TARGETARCH=amd64
|
||||||
|
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends curl ca-certificates \
|
||||||
|
&& rm -rf /var/lib/apt/lists/* \
|
||||||
|
&& curl -fsSL -o /usr/local/bin/daytona \
|
||||||
|
"https://github.com/daytonaio/daytona/releases/download/${DAYTONA_VERSION}/daytona-linux-${TARGETARCH}" \
|
||||||
|
&& chmod +x /usr/local/bin/daytona \
|
||||||
|
&& npm i -g mcp-proxy@6.5.2 \
|
||||||
|
&& npm cache clean --force
|
||||||
|
|
||||||
|
EXPOSE 8090
|
||||||
|
|
||||||
|
# mcp-proxy listens on :8090 and exposes `daytona mcp start` (stdio) over
|
||||||
|
# Streamable HTTP (/mcp) and SSE (/sse).
|
||||||
|
CMD ["mcp-proxy", "--port", "8090", "--host", "0.0.0.0", "daytona", "mcp", "start"]
|
||||||
@@ -33,3 +33,8 @@ data:
|
|||||||
REAPER_INTERVAL_SECONDS: "300"
|
REAPER_INTERVAL_SECONDS: "300"
|
||||||
REAPER_ENABLED: "true"
|
REAPER_ENABLED: "true"
|
||||||
LOG_LEVEL: "INFO"
|
LOG_LEVEL: "INFO"
|
||||||
|
# Daytona cloud-sandbox MCP. Both set → a daytona-mcp sidecar is added to each
|
||||||
|
# tenant swarm and registered as a swarm-scope MCP server. The API key value is
|
||||||
|
# NOT here — DAYTONA_API_KEY_SECRET names a Key Vault secret resolved on-VM.
|
||||||
|
DAYTONA_API_URL: "https://app.daytona.io/api"
|
||||||
|
DAYTONA_API_KEY_SECRET: "swarm-daytona-key"
|
||||||
|
|||||||
@@ -91,12 +91,21 @@ class Settings(BaseSettings):
|
|||||||
|
|
||||||
log_level: str = "INFO"
|
log_level: str = "INFO"
|
||||||
|
|
||||||
# OpenSandbox integration. When set, an opensandbox-mcp sidecar is added to
|
# Daytona integration (cloud sandbox). When daytona_api_url AND
|
||||||
# the tenant compose stack and registered as a swarm-scope MCP server so all
|
# daytona_api_key_secret are both set, a daytona-mcp sidecar (daytona CLI
|
||||||
# agents can create/exec/manage sandboxes without extra setup.
|
# wrapped by mcp-proxy as Streamable HTTP) is added to the tenant compose
|
||||||
# Format: full URL, e.g. "http://52.148.119.72:80" or "https://sandbox.example.com"
|
# stack and registered as a swarm-scope MCP server, so all agents can
|
||||||
# Empty string (default) disables the feature entirely.
|
# create/exec/manage Daytona cloud sandboxes without extra setup.
|
||||||
opensandbox_server_url: str = ""
|
# The sidecar authenticates to Daytona with DAYTONA_API_KEY, resolved on-VM
|
||||||
|
# from Key Vault by NAME — the key value never lives in config/ConfigMap.
|
||||||
|
# Both empty (default) disables the feature entirely.
|
||||||
|
daytona_api_url: str = ""
|
||||||
|
# Key Vault secret NAME (not value) holding the Daytona API key, resolved on
|
||||||
|
# the tenant VM via Managed Identity against litestream_keyvault_url.
|
||||||
|
daytona_api_key_secret: str = ""
|
||||||
|
# Sidecar image (daytona CLI + mcp-proxy). Defaults to the heicodetest ACR
|
||||||
|
# build produced by this repo's daytona-mcp Dockerfile.
|
||||||
|
daytona_mcp_image: str = "heicodetest.azurecr.io/daytona-mcp:heicode-test"
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def subnet_id(self) -> str:
|
def subnet_id(self) -> str:
|
||||||
|
|||||||
@@ -17,7 +17,6 @@ from __future__ import annotations
|
|||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
import uuid
|
import uuid
|
||||||
from urllib.parse import urlparse
|
|
||||||
|
|
||||||
from ..config import Settings
|
from ..config import Settings
|
||||||
from ..models import DeploymentRecord
|
from ..models import DeploymentRecord
|
||||||
@@ -57,16 +56,11 @@ _FLUENT_BIT_IMAGE = "fluent/fluent-bit:3.1.9"
|
|||||||
_AGENT_NS = uuid.UUID("8f1d4e2a-6c3b-4f8a-9e7d-1a2b3c4d5e6f")
|
_AGENT_NS = uuid.UUID("8f1d4e2a-6c3b-4f8a-9e7d-1a2b3c4d5e6f")
|
||||||
|
|
||||||
|
|
||||||
def _opensandbox_parts(url: str) -> tuple[str, str] | None:
|
def _daytona_enabled(s: Settings) -> bool:
|
||||||
"""Parse opensandbox_server_url into (domain, protocol).
|
"""Daytona MCP sidecar is wired in only when both the API URL and the Key
|
||||||
|
Vault secret NAME holding the API key are configured. Either empty disables
|
||||||
Returns None when url is empty. Domain includes port if non-default
|
it."""
|
||||||
(e.g. "52.148.119.72:80", "api.opensandbox.io").
|
return bool(s.daytona_api_url and s.daytona_api_key_secret)
|
||||||
"""
|
|
||||||
if not url:
|
|
||||||
return None
|
|
||||||
p = urlparse(url)
|
|
||||||
return p.netloc, p.scheme or "http"
|
|
||||||
|
|
||||||
|
|
||||||
def _acr_registry(image: str) -> str | None:
|
def _acr_registry(image: str) -> str | None:
|
||||||
@@ -161,82 +155,70 @@ chmod 644 "$SWARM_DIR/fluent-bit/fluent-bit.conf" "$SWARM_DIR/fluent-bit/parsers
|
|||||||
la_config_block = ""
|
la_config_block = ""
|
||||||
la_env_lines = ""
|
la_env_lines = ""
|
||||||
|
|
||||||
osb = _opensandbox_parts(s.opensandbox_server_url)
|
# ── Daytona MCP integration (cloud sandbox) ─────────────────────────────────
|
||||||
if osb:
|
if _daytona_enabled(s):
|
||||||
# Precompute deterministic agent IDs so the registration bash block can
|
dtn_lead_id = _agent_id(record.deployment_id, "lead")
|
||||||
# install the MCP server for every agent without talking to the swarm API
|
dtn_worker_ids = [
|
||||||
# to discover them.
|
|
||||||
lead_id = _agent_id(record.deployment_id, "lead")
|
|
||||||
worker_ids = [
|
|
||||||
_agent_id(record.deployment_id, f"worker-{i}")
|
_agent_id(record.deployment_id, f"worker-{i}")
|
||||||
for i in range(1, worker_count(record) + 1)
|
for i in range(1, worker_count(record) + 1)
|
||||||
]
|
]
|
||||||
all_agent_ids = " ".join([lead_id] + worker_ids)
|
dtn_agent_ids = " ".join([dtn_lead_id] + dtn_worker_ids)
|
||||||
osb_domain, osb_protocol = osb
|
# Resolve the Daytona API key from Key Vault by NAME (value never in
|
||||||
opensandbox_register_block = f"""
|
# config/ConfigMap). Written into .env so docker-compose interpolates it
|
||||||
# ── opensandbox-mcp launcher: monkey-patch FastMCP host default to 0.0.0.0 ──
|
# into the daytona-mcp service env. Mirrors the billing/litestream creds.
|
||||||
# FastMCP.__init__ hard-codes host='127.0.0.1'; FASTMCP_HOST env is ignored
|
daytona_resolve_block = f"""
|
||||||
# because the value is passed explicitly at construction time. We monkey-patch
|
# ── Resolve Daytona API key from Key Vault ──────────────────────────────────
|
||||||
# the default before import so the server binds on all interfaces inside the
|
DAYTONA_API_KEY="$(kv_secret "{vault_host}/secrets/{s.daytona_api_key_secret}")"
|
||||||
# container and is reachable from peer containers in the compose network.
|
if [ -z "$DAYTONA_API_KEY" ]; then
|
||||||
cat > "$SWARM_DIR/opensandbox-mcp-launch.py" <<'PYEOF'
|
echo "[swarm-bootstrap] WARN: Daytona API key empty — daytona-mcp will fail to auth" >&2
|
||||||
import os, mcp.server.fastmcp as _fm
|
fi
|
||||||
_orig = _fm.FastMCP.__init__
|
"""
|
||||||
def _p(self, *a, host="0.0.0.0", **k): _orig(self, *a, host=host, **k)
|
daytona_env_lines = "DAYTONA_API_KEY=${DAYTONA_API_KEY}\n"
|
||||||
_fm.FastMCP.__init__ = _p
|
daytona_register_block = f"""
|
||||||
from opensandbox.config import ConnectionConfig
|
# ── Register daytona MCP server for all agents ──────────────────────────────
|
||||||
from opensandbox_mcp.server import create_server
|
_dtn_wait_api() {{
|
||||||
import anyio
|
|
||||||
mcp = create_server(connection_config=ConnectionConfig(
|
|
||||||
domain=os.environ["OSB_DOMAIN"], protocol=os.environ["OSB_PROTOCOL"]))
|
|
||||||
anyio.run(mcp.run_streamable_http_async)
|
|
||||||
PYEOF
|
|
||||||
chmod 644 "$SWARM_DIR/opensandbox-mcp-launch.py"
|
|
||||||
|
|
||||||
# ── Register opensandbox MCP server for all agents ──────────────────────────
|
|
||||||
_osb_wait_api() {{
|
|
||||||
for _i in $(seq 1 60); do
|
for _i in $(seq 1 60); do
|
||||||
curl -sf "http://localhost:{s.swarm_api_port}/health" >/dev/null 2>&1 && return 0
|
curl -sf "http://localhost:{s.swarm_api_port}/health" >/dev/null 2>&1 && return 0
|
||||||
sleep 2
|
sleep 2
|
||||||
done
|
done
|
||||||
return 1
|
return 1
|
||||||
}}
|
}}
|
||||||
# Port check runs inside the opensandbox-mcp container (127.0.0.1 inside =
|
# Port check runs inside the daytona-mcp container (image has bash + /dev/tcp).
|
||||||
# 0.0.0.0 listener; host-side port is not mapped).
|
_dtn_wait_mcp() {{
|
||||||
_osb_wait_mcp() {{
|
|
||||||
for _i in $(seq 1 100); do
|
for _i in $(seq 1 100); do
|
||||||
docker exec swarm-opensandbox-mcp-1 python3 -c \
|
docker exec swarm-daytona-mcp-1 bash -c "exec 3<>/dev/tcp/localhost/8090" \
|
||||||
"import socket; s=socket.socket(); s.settimeout(1); s.connect(('localhost',8000)); s.close()" \
|
|
||||||
2>/dev/null && return 0
|
2>/dev/null && return 0
|
||||||
sleep 3
|
sleep 3
|
||||||
done
|
done
|
||||||
return 1
|
return 1
|
||||||
}}
|
}}
|
||||||
|
|
||||||
echo "[swarm-bootstrap] waiting for swarm API and opensandbox-mcp …"
|
echo "[swarm-bootstrap] waiting for swarm API and daytona-mcp …"
|
||||||
if _osb_wait_api && _osb_wait_mcp; then
|
if _dtn_wait_api && _dtn_wait_mcp; then
|
||||||
_OSB_SERVER_ID=$(curl -sf -X POST "http://localhost:{s.swarm_api_port}/api/mcp-servers" \\
|
_DTN_SERVER_ID=$(curl -sf -X POST "http://localhost:{s.swarm_api_port}/api/mcp-servers" \\
|
||||||
-H "Authorization: Bearer {api_key}" \\
|
-H "Authorization: Bearer {api_key}" \\
|
||||||
-H "Content-Type: application/json" \\
|
-H "Content-Type: application/json" \\
|
||||||
-d '{{"name":"opensandbox","description":"OpenSandbox isolated code execution","transport":"http","scope":"swarm","url":"http://opensandbox-mcp:8000/mcp"}}' \\
|
-d '{{"name":"daytona","description":"Daytona cloud sandbox (create/exec/files/git/preview)","transport":"http","scope":"swarm","url":"http://daytona-mcp:8090/mcp"}}' \\
|
||||||
| jq -r '.server.id // empty')
|
| jq -r '.server.id // empty')
|
||||||
if [ -n "$_OSB_SERVER_ID" ]; then
|
if [ -n "$_DTN_SERVER_ID" ]; then
|
||||||
for _AID in {all_agent_ids}; do
|
for _AID in {dtn_agent_ids}; do
|
||||||
curl -sf -X POST "http://localhost:{s.swarm_api_port}/api/mcp-servers/$_OSB_SERVER_ID/install" \\
|
curl -sf -X POST "http://localhost:{s.swarm_api_port}/api/mcp-servers/$_DTN_SERVER_ID/install" \\
|
||||||
-H "Authorization: Bearer {api_key}" \\
|
-H "Authorization: Bearer {api_key}" \\
|
||||||
-H "Content-Type: application/json" \\
|
-H "Content-Type: application/json" \\
|
||||||
-d "{{\\"agentId\\":\\"$_AID\\"}}" >/dev/null 2>&1 || true
|
-d "{{\\"agentId\\":\\"$_AID\\"}}" >/dev/null 2>&1 || true
|
||||||
done
|
done
|
||||||
echo "[swarm-bootstrap] opensandbox MCP server registered id=$_OSB_SERVER_ID agents={all_agent_ids}"
|
echo "[swarm-bootstrap] daytona MCP server registered id=$_DTN_SERVER_ID agents={dtn_agent_ids}"
|
||||||
else
|
else
|
||||||
echo "[swarm-bootstrap] WARN: opensandbox MCP server registration returned no id — skipping install" >&2
|
echo "[swarm-bootstrap] WARN: daytona MCP server registration returned no id — skipping install" >&2
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
echo "[swarm-bootstrap] WARN: timed out waiting for swarm API or opensandbox-mcp — MCP server not registered" >&2
|
echo "[swarm-bootstrap] WARN: timed out waiting for swarm API or daytona-mcp — MCP server not registered" >&2
|
||||||
fi
|
fi
|
||||||
"""
|
"""
|
||||||
else:
|
else:
|
||||||
opensandbox_register_block = ""
|
daytona_resolve_block = ""
|
||||||
|
daytona_env_lines = ""
|
||||||
|
daytona_register_block = ""
|
||||||
|
|
||||||
acr_host = _acr_registry(s.swarm_worker_image)
|
acr_host = _acr_registry(s.swarm_worker_image)
|
||||||
if acr_host:
|
if acr_host:
|
||||||
@@ -311,7 +293,7 @@ if [ -z "$LS_ACCOUNT" ] || [ -z "$LS_KEY" ]; then
|
|||||||
echo "[swarm-bootstrap] FATAL: litestream account/key empty (account='$LS_ACCOUNT', key set=$([ -n "$LS_KEY" ] && echo yes || echo no)) — SQLite backup would be broken" >&2
|
echo "[swarm-bootstrap] FATAL: litestream account/key empty (account='$LS_ACCOUNT', key set=$([ -n "$LS_KEY" ] && echo yes || echo no)) — SQLite backup would be broken" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
{la_resolve_block}
|
{la_resolve_block}{daytona_resolve_block}
|
||||||
# ── Write .env (runtime only; never leaves the VM) ──────────────────────────
|
# ── Write .env (runtime only; never leaves the VM) ──────────────────────────
|
||||||
cat > "$SWARM_DIR/.env" <<EOF
|
cat > "$SWARM_DIR/.env" <<EOF
|
||||||
API_KEY={api_key}
|
API_KEY={api_key}
|
||||||
@@ -327,7 +309,7 @@ OPENAI_API_BASE={MODEL_GATEWAY_BASE_URL}
|
|||||||
DEPLOYMENT_ID={record.deployment_id}
|
DEPLOYMENT_ID={record.deployment_id}
|
||||||
TENANT_ID={record.tenant_id}
|
TENANT_ID={record.tenant_id}
|
||||||
VM_NAME={record.vm_name}
|
VM_NAME={record.vm_name}
|
||||||
{la_env_lines}SLACK_DISABLE=true
|
{la_env_lines}{daytona_env_lines}SLACK_DISABLE=true
|
||||||
GITHUB_DISABLE=true
|
GITHUB_DISABLE=true
|
||||||
SWARM_URL=localhost
|
SWARM_URL=localhost
|
||||||
EOF
|
EOF
|
||||||
@@ -375,7 +357,7 @@ docker compose --env-file "$SWARM_DIR/.env" up -d
|
|||||||
# ── Continuous WAL replication (foreground process under systemd) ───────────
|
# ── Continuous WAL replication (foreground process under systemd) ───────────
|
||||||
systemctl restart swarm-litestream.service || true
|
systemctl restart swarm-litestream.service || true
|
||||||
|
|
||||||
{opensandbox_register_block}
|
{daytona_register_block}
|
||||||
echo "[swarm-bootstrap] done $(date -u +%FT%TZ)"
|
echo "[swarm-bootstrap] done $(date -u +%FT%TZ)"
|
||||||
|
|
||||||
GIT_BINDINGS='{git_bindings_json}'
|
GIT_BINDINGS='{git_bindings_json}'
|
||||||
@@ -464,20 +446,18 @@ def _compose_yaml(settings: Settings, record: DeploymentRecord) -> str:
|
|||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
"""
|
"""
|
||||||
|
|
||||||
osb = _opensandbox_parts(s.opensandbox_server_url)
|
if _daytona_enabled(s):
|
||||||
if osb:
|
# daytona-mcp sidecar: daytona CLI wrapped by mcp-proxy as Streamable
|
||||||
osb_domain, osb_protocol = osb
|
# HTTP on :8090. Auth via DAYTONA_API_KEY (from .env, resolved from Key
|
||||||
|
# Vault on-VM). Reachable as http://daytona-mcp:8090/mcp by peer agents.
|
||||||
services += f"""
|
services += f"""
|
||||||
opensandbox-mcp:
|
daytona-mcp:
|
||||||
image: python:3.11-slim
|
image: {s.daytona_mcp_image}
|
||||||
pull_policy: always
|
pull_policy: always
|
||||||
stop_grace_period: 10s
|
stop_grace_period: 10s
|
||||||
environment:
|
environment:
|
||||||
- OSB_DOMAIN={osb_domain}
|
- DAYTONA_API_KEY=${{DAYTONA_API_KEY}}
|
||||||
- OSB_PROTOCOL={osb_protocol}
|
- DAYTONA_API_URL={s.daytona_api_url}
|
||||||
volumes:
|
|
||||||
- /opt/swarm/opensandbox-mcp-launch.py:/launch.py:ro
|
|
||||||
command: sh -c "pip install --quiet --no-cache-dir opensandbox-mcp && python3 /launch.py"
|
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|||||||
@@ -121,3 +121,43 @@ def test_loganalytics_and_acr_are_independent():
|
|||||||
out = render_cloud_init(s, _record(), "api-key")
|
out = render_cloud_init(s, _record(), "api-key")
|
||||||
assert "docker login heicodetest.azurecr.io" in out
|
assert "docker login heicodetest.azurecr.io" in out
|
||||||
assert "logship:" in out
|
assert "logship:" in out
|
||||||
|
|
||||||
|
|
||||||
|
# ── Daytona MCP integration ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_daytona_absent_by_default():
|
||||||
|
out = render_cloud_init(_settings(), _record(), "api-key")
|
||||||
|
assert "daytona-mcp:" not in out
|
||||||
|
assert "daytona MCP server" not in out
|
||||||
|
assert "DAYTONA_API_KEY" not in out
|
||||||
|
|
||||||
|
|
||||||
|
def test_daytona_sidecar_and_registration_when_enabled():
|
||||||
|
s = _settings(
|
||||||
|
daytona_api_url="https://app.daytona.io/api",
|
||||||
|
daytona_api_key_secret="swarm-daytona-key",
|
||||||
|
)
|
||||||
|
out = render_cloud_init(s, _record(), "api-key")
|
||||||
|
# compose sidecar
|
||||||
|
assert "daytona-mcp:" in out
|
||||||
|
assert "heicodetest.azurecr.io/daytona-mcp:heicode-test" in out
|
||||||
|
assert "DAYTONA_API_URL=https://app.daytona.io/api" in out
|
||||||
|
# key resolved from Key Vault by name, written into .env, never inlined plaintext
|
||||||
|
assert "secrets/swarm-daytona-key" in out
|
||||||
|
assert "DAYTONA_API_KEY=${DAYTONA_API_KEY}" in out
|
||||||
|
# registration + per-agent install via the swarm API
|
||||||
|
assert '"name":"daytona"' in out
|
||||||
|
assert "http://daytona-mcp:8090/mcp" in out
|
||||||
|
assert "/api/mcp-servers/$_DTN_SERVER_ID/install" in out
|
||||||
|
|
||||||
|
|
||||||
|
def test_daytona_requires_both_url_and_secret():
|
||||||
|
# only url → disabled
|
||||||
|
out = render_cloud_init(_settings(daytona_api_url="https://app.daytona.io/api"), _record(), "k")
|
||||||
|
assert "daytona-mcp:" not in out
|
||||||
|
# only secret → disabled
|
||||||
|
out = render_cloud_init(_settings(daytona_api_key_secret="swarm-daytona-key"), _record(), "k")
|
||||||
|
assert "daytona-mcp:" not in out
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user