refactor(agent): built-in per-type env mapping; fix start timeout, orphan, guards

Replace the user-facing env_map mistake with a built-in env convention keyed by
resource type + provider — users never see/edit env names; they only fill plain
resource fields. Supports git (gitea/github/gitlab), vm, database
(mysql/pg/redis/mongo, with alias normalisation), storage (azure blob / bucket).
Lenient: missing optional fields are skipped; only unsupported type or a KV read
failure errors.

Other issues found in review and fixed:
- start timeout: template-agent start now uses a longer timeout (default 60s,
  AGENT_RUNTIME_START_TIMEOUT_SECONDS) since AM provisions synchronously — 5s
  would time out. amTemplateDo takes a per-call timeout.
- orphan agent: if AM start succeeds but the Manager record fails to persist, the
  orphan is rolled back (best-effort amDeleteTemplateAgent).
- findUserTemplateAgent now guards template_id<>'' so the new endpoints can't
  touch a legacy task deployment.
- binding_ids defaults to [] (not null).
- removed ResourceBinding.EnvMap field entirely.

Tests rewritten for the built-in convention (blob metadata-only, db provider
prefixes incl pg/mg aliases, git provider-agnostic names, ownership, unsupported
type, empty); adapter round-trip + router tests still pass.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

heicode/controller/agent_template_env.go

@@ -8,27 +8,138 @@ import (
 	"github.com/heicode/manager/model"
 )
 
-// envMapEntry describes how one environment variable is sourced from a resource
-// binding when starting a template agent.
+// envField maps one environment variable a template agent receives from a bound
+// resource. Source is envMeta (non-secret, from the binding's Metadata JSON) or
+// envSecret (resolved from Key Vault at start only). Key is the field name
+// inside that JSON.
 //
-//	Source == "metadata" -> value taken from the binding's non-secret Metadata JSON
-//	Source == "secret"   -> value resolved from Key Vault (binding.SecretRef) at
-//	                        start time only; never persisted in plaintext.
+// These env names are the HM<->AM contract: the agent template reads them.
+// Users NEVER see or edit env names — they only fill plain resource fields
+// (repo url, token, host, password, ...) in the web console.
+type envField struct {
+	Env    string
+	Source string
+	Key    string
+}
+
+const (
+	envMeta   = "metadata"
+	envSecret = "secret"
+)
+
+// normalizeProvider folds provider aliases to a canonical key.
+func normalizeProvider(p string) string {
+	switch strings.ToLower(strings.TrimSpace(p)) {
+	case "postgres", "postgresql", "pg":
+		return "pg"
+	case "mysql", "mariadb":
+		return "mysql"
+	case "redis":
+		return "redis"
+	case "mongo", "mongodb", "mg":
+		return "mongo"
+	case "github":
+		return "github"
+	case "gitea":
+		return "gitea"
+	case "gitlab":
+		return "gitlab"
+	case "blob", "azureblob", "azure-blob":
+		return "blob"
+	case "bucket", "s3", "oss", "minio", "cos":
+		return "bucket"
+	default:
+		return strings.ToLower(strings.TrimSpace(p))
+	}
+}
+
+func dbEnvSpec(prefix string) []envField {
+	return []envField{
+		{prefix + "_HOST", envMeta, "host"},
+		{prefix + "_PORT", envMeta, "port"},
+		{prefix + "_DATABASE", envMeta, "database"},
+		{prefix + "_USER", envMeta, "user"},
+		{prefix + "_PASSWORD", envSecret, "password"},
+	}
+}
+
+// builtinEnvSpec returns the fixed env mapping for a binding, keyed by resource
+// type and provider. Built into HM; not user-configurable.
 //
-// EnvMap on a ResourceBinding is a JSON object: {"ENV_NAME": {"source": "...",
-// "key": "<field in metadata or secret json>"}}.
-type envMapEntry struct {
-	Source string `json:"source"`
-	Key    string `json:"key"`
+// Supported: git (gitea/github/gitlab), vm, database (mysql/pg/redis/mongo),
+// storage (azure blob / object-storage bucket).
+func builtinEnvSpec(resourceType, provider string) []envField {
+	rt := strings.ToLower(strings.TrimSpace(resourceType))
+	p := normalizeProvider(provider)
+
+	switch rt {
+	case "git":
+		// Same env names across gitea/github/gitlab; provider passed through.
+		return []envField{
+			{"GIT_PROVIDER", envMeta, "provider"},
+			{"GIT_REPO_URL", envMeta, "repo_url"},
+			{"GIT_BRANCH", envMeta, "default_branch"},
+			{"GIT_API_BASE", envMeta, "api_base"},
+			{"GIT_TOKEN", envSecret, "token"},
+		}
+	case "vm":
+		return []envField{
+			{"VM_HOST", envMeta, "host"},
+			{"VM_PORT", envMeta, "port"},
+			{"VM_USER", envMeta, "user"},
+			{"VM_PASSWORD", envSecret, "password"},
+			{"VM_SSH_KEY", envSecret, "private_key"},
+		}
+	case "database", "db":
+		switch p {
+		case "mysql":
+			return dbEnvSpec("MYSQL")
+		case "pg":
+			return dbEnvSpec("PG")
+		case "mongo":
+			return dbEnvSpec("MONGO")
+		case "redis":
+			return []envField{
+				{"REDIS_HOST", envMeta, "host"},
+				{"REDIS_PORT", envMeta, "port"},
+				{"REDIS_DB", envMeta, "db"},
+				{"REDIS_PASSWORD", envSecret, "password"},
+			}
+		default:
+			return dbEnvSpec("DB")
+		}
+	case "blob", "storage":
+		if p == "bucket" {
+			return []envField{
+				{"BUCKET_ENDPOINT", envMeta, "endpoint"},
+				{"BUCKET_REGION", envMeta, "region"},
+				{"BUCKET_NAME", envMeta, "bucket"},
+				{"BUCKET_ACCESS_KEY_ID", envSecret, "access_key_id"},
+				{"BUCKET_SECRET_ACCESS_KEY", envSecret, "secret_access_key"},
+			}
+		}
+		// Default: Azure Blob.
+		return []envField{
+			{"BLOB_ACCOUNT", envMeta, "account"},
+			{"BLOB_CONTAINER", envMeta, "container"},
+			{"BLOB_KEY", envSecret, "key"},
+		}
+	default:
+		return nil
+	}
 }
 
 // buildAgentEnvFromBindings resolves the selected resource bindings into a flat
-// environment map for a template agent's .env at start time.
+// environment map for a template agent's .env at start time, using the built-in
+// per-type env convention.
 //
 // Non-secret values come from each binding's Metadata; secret values are read
-// from Key Vault here, at start time, and returned for injection into the agent
-// — they are NOT stored back on the Manager side. Callers must not log the
-// returned map.
+// from Key Vault here, at start time, and returned for injection — they are NOT
+// persisted on the Manager side. Callers must NOT log the returned map.
+//
+// Lenient by design: a missing optional field (e.g. a vm with a password but no
+// ssh key, or a binding with no secret yet) is simply skipped, not an error.
+// Only an unsupported resource type or a Key Vault read failure errors out.
 func buildAgentEnvFromBindings(userID int, bindingIDs []int) (map[string]string, error) {
 	env := map[string]string{}
 	if len(bindingIDs) == 0 {
@@ -47,14 +158,9 @@ func buildAgentEnvFromBindings(userID int, bindingIDs []int) (map[string]string,
 			return nil, fmt.Errorf("resource binding %d not found for user", id)
 		}
 
-		entries := map[string]envMapEntry{}
-		if strings.TrimSpace(binding.EnvMap) != "" {
-			if err := common.UnmarshalJsonStr(binding.EnvMap, &entries); err != nil {
-				return nil, fmt.Errorf("binding %d env_map is not valid JSON: %w", id, err)
-			}
-		}
-		if len(entries) == 0 {
-			continue
+		spec := builtinEnvSpec(binding.ResourceType, binding.Provider)
+		if len(spec) == 0 {
+			return nil, fmt.Errorf("binding %d: unsupported resource type %q (provider %q)", id, binding.ResourceType, binding.Provider)
 		}
 
 		metadata := map[string]any{}
@@ -62,45 +168,37 @@ func buildAgentEnvFromBindings(userID int, bindingIDs []int) (map[string]string,
 			_ = common.UnmarshalJsonStr(binding.Metadata, &metadata)
 		}
 
-		// The secret JSON is fetched lazily, only if some entry needs it.
+		// Resolve the secret JSON once per binding, only if one is bound.
 		var secret map[string]any
-		secretLoaded := false
+		if strings.TrimSpace(binding.SecretRef) != "" {
+			if !storeReady {
+				client, err := newSecretStoreClientFromEnv()
+				if err != nil {
+					return nil, err
+				}
+				store = client
+				storeReady = true
+			}
+			resolved, err := store.getJSONSecret(binding.SecretRef)
+			if err != nil {
+				return nil, fmt.Errorf("binding %d secret read failed: %w", id, err)
+			}
+			secret = resolved
+		}
 
-		for envName, entry := range entries {
-			name := strings.TrimSpace(envName)
-			if name == "" {
+		for _, f := range spec {
+			src := metadata
+			if f.Source == envSecret {
+				src = secret
+			}
+			if src == nil {
 				continue
 			}
-			switch entry.Source {
-			case "metadata":
-				if value, ok := metadata[entry.Key]; ok {
-					env[name] = fmt.Sprintf("%v", value)
+			if value, ok := src[f.Key]; ok {
+				str := fmt.Sprintf("%v", value)
+				if strings.TrimSpace(str) != "" {
+					env[f.Env] = str
 				}
-			case "secret":
-				if !secretLoaded {
-					if strings.TrimSpace(binding.SecretRef) == "" {
-						return nil, fmt.Errorf("binding %d env_map needs a secret but the binding has no secret_ref", id)
-					}
-					if !storeReady {
-						client, err := newSecretStoreClientFromEnv()
-						if err != nil {
-							return nil, err
-						}
-						store = client
-						storeReady = true
-					}
-					resolved, err := store.getJSONSecret(binding.SecretRef)
-					if err != nil {
-						return nil, fmt.Errorf("binding %d secret read failed: %w", id, err)
-					}
-					secret = resolved
-					secretLoaded = true
-				}
-				if value, ok := secret[entry.Key]; ok {
-					env[name] = fmt.Sprintf("%v", value)
-				}
-			default:
-				return nil, fmt.Errorf("binding %d env_map %q: unknown source %q (want metadata|secret)", id, name, entry.Source)
 			}
 		}
 	}

heicode/controller/agent_template_handlers.go

@@ -70,6 +70,9 @@ func HeicodeDeployAgent(c *gin.Context) {
 		agentError(c, "POLICY_REJECTED", "template_id is required")
 		return
 	}
+	if req.BindingIDs == nil {
+		req.BindingIDs = []int{}
+	}
 	if model.DB == nil {
 		agentError(c, "DEPLOYMENT_PERSIST_FAILED", "database not initialised")
 		return
@@ -111,6 +114,13 @@ func HeicodeDeployAgent(c *gin.Context) {
 	}
 	if err := model.DB.Create(&row).Error; err != nil {
 		common.SysLog("HeicodeDeployAgent persist: " + err.Error())
+		// We started an agent in AM but failed to record it: roll back the
+		// orphan so it does not leak/keep running with no Manager record.
+		if strings.TrimSpace(result.RuntimeID) != "" {
+			if delErr := amDeleteTemplateAgent(c.Request.Context(), result.RuntimeID); delErr != nil {
+				common.SysLog("HeicodeDeployAgent orphan cleanup failed: " + delErr.Error())
+			}
+		}
 		agentError(c, "DEPLOYMENT_PERSIST_FAILED", "failed to persist agent")
 		return
 	}
@@ -126,7 +136,9 @@ func findUserTemplateAgent(c *gin.Context) (model.AgentDeployment, bool) {
 		agentError(c, "POLICY_REJECTED", "deployment_id and authentication required")
 		return row, false
 	}
-	if err := model.DB.Where("deployment_id = ? AND user_id = ?", deploymentID, strconv.Itoa(userID)).First(&row).Error; err != nil {
+	// template_id <> '' guards against touching a legacy task deployment of the
+	// same user through the new template-agent endpoints.
+	if err := model.DB.Where("deployment_id = ? AND user_id = ? AND template_id <> ''", deploymentID, strconv.Itoa(userID)).First(&row).Error; err != nil {
 		agentError(c, "DEPLOYMENT_CONFLICT", "agent not found")
 		return row, false
 	}

heicode/controller/agent_template_runtime.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
+	"time"
 
 	"github.com/heicode/manager/common"
 )
@@ -62,14 +63,29 @@ func agentTemplateAgentStopPath(agentRuntimeID string) string {
 	return strings.ReplaceAll(p, "{agent_id}", url.PathEscape(agentRuntimeID))
 }
 
+// agentTemplateStartTimeout is the timeout for starting a template agent —
+// longer than the default runtime timeout because AM provisions the agent
+// (container + subdomain) synchronously before responding.
+func agentTemplateStartTimeout() time.Duration {
+	sec := common.GetEnvOrDefault("AGENT_RUNTIME_START_TIMEOUT_SECONDS", 60)
+	if sec <= 0 {
+		sec = 60
+	}
+	return time.Duration(sec) * time.Second
+}
+
 // amTemplateDo performs an AM HTTP call and returns the decoded `data` object.
-// Reuses the existing runtime config (base URL / service token / timeout).
-func amTemplateDo(ctx context.Context, method, path string, body any) (map[string]any, error) {
+// Reuses the existing runtime config (base URL / service token). A non-zero
+// timeout overrides the default runtime timeout (used by the start call).
+func amTemplateDo(ctx context.Context, method, path string, body any, timeout time.Duration) (map[string]any, error) {
 	cfg := agentRuntimeClientConfigForMode(agentRuntimeModeAgent)
 	endpoint, err := agentRuntimeURL(cfg.BaseURL, path)
 	if err != nil {
 		return nil, err
 	}
+	if timeout <= 0 {
+		timeout = cfg.Timeout
+	}
 	var reader io.Reader
 	if body != nil {
 		raw, err := common.Marshal(body)
@@ -86,7 +102,7 @@ func amTemplateDo(ctx context.Context, method, path string, body any) (map[strin
 	if cfg.Token != "" {
 		req.Header.Set("Authorization", "Bearer "+cfg.Token)
 	}
-	client := &http.Client{Timeout: cfg.Timeout}
+	client := &http.Client{Timeout: timeout}
 	resp, err := client.Do(req)
 	if err != nil {
 		return nil, err
@@ -111,7 +127,7 @@ func amTemplateDo(ctx context.Context, method, path string, body any) (map[strin
 // amListTemplates lists AM's deployable templates.
 // Proposed contract: data.templates = [...]. Swap this mapping if AM differs.
 func amListTemplates(ctx context.Context) ([]agentTemplate, error) {
-	data, err := amTemplateDo(ctx, http.MethodGet, agentTemplatesPath(), nil)
+	data, err := amTemplateDo(ctx, http.MethodGet, agentTemplatesPath(), nil, 0)
 	if err != nil {
 		return nil, err
 	}
@@ -135,7 +151,7 @@ func amStartTemplateAgent(ctx context.Context, templateID, managerDeploymentID s
 		"env":                   env,
 		"callback_url":          callbackURL,
 	}
-	data, err := amTemplateDo(ctx, http.MethodPost, agentTemplateStartPath(templateID), payload)
+	data, err := amTemplateDo(ctx, http.MethodPost, agentTemplateStartPath(templateID), payload, agentTemplateStartTimeout())
 	if err != nil {
 		return amStartResult{}, err
 	}
@@ -151,7 +167,7 @@ func amStartTemplateAgent(ctx context.Context, templateID, managerDeploymentID s
 // so HM can tell whether the agent is still alive / has crashed.
 // Proposed contract: GET /api/agent/agents/{agent_id} -> data.status.
 func amGetAgentStatus(ctx context.Context, runtimeID string) (string, error) {
-	data, err := amTemplateDo(ctx, http.MethodGet, agentTemplateAgentPath(runtimeID), nil)
+	data, err := amTemplateDo(ctx, http.MethodGet, agentTemplateAgentPath(runtimeID), nil, 0)
 	if err != nil {
 		return "", err
 	}
@@ -159,11 +175,11 @@ func amGetAgentStatus(ctx context.Context, runtimeID string) (string, error) {
 }
 
 func amStopTemplateAgent(ctx context.Context, runtimeID string) error {
-	_, err := amTemplateDo(ctx, http.MethodPost, agentTemplateAgentStopPath(runtimeID), nil)
+	_, err := amTemplateDo(ctx, http.MethodPost, agentTemplateAgentStopPath(runtimeID), nil, 0)
 	return err
 }
 
 func amDeleteTemplateAgent(ctx context.Context, runtimeID string) error {
-	_, err := amTemplateDo(ctx, http.MethodDelete, agentTemplateAgentPath(runtimeID), nil)
+	_, err := amTemplateDo(ctx, http.MethodDelete, agentTemplateAgentPath(runtimeID), nil, 0)
 	return err
 }

heicode/controller/agent_template_test.go

@@ -11,59 +11,85 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestBuildAgentEnvFromBindings_MetadataOnly(t *testing.T) {
+func TestBuildAgentEnvFromBindings_BuiltinBlobMetadataOnly(t *testing.T) {
 	setupResourceControllerTestDB(t)
 
+	// No secret_ref -> secret fields (BLOB_KEY) are skipped, not an error.
 	b := model.ResourceBinding{
 		UserId:       7,
 		Name:         "blob1",
 		ResourceType: "blob",
+		Provider:     "blob",
 		Metadata:     `{"account":"myacct","container":"uploads"}`,
-		EnvMap:       `{"AZURE_BLOB_ACCOUNT":{"source":"metadata","key":"account"},"AZURE_BLOB_CONTAINER":{"source":"metadata","key":"container"}}`,
 	}
 	require.NoError(t, model.DB.Create(&b).Error)
 
 	env, err := buildAgentEnvFromBindings(7, []int{b.Id})
 	require.NoError(t, err)
-	require.Equal(t, "myacct", env["AZURE_BLOB_ACCOUNT"])
-	require.Equal(t, "uploads", env["AZURE_BLOB_CONTAINER"])
+	require.Equal(t, "myacct", env["BLOB_ACCOUNT"])
+	require.Equal(t, "uploads", env["BLOB_CONTAINER"])
+	_, hasKey := env["BLOB_KEY"]
+	require.False(t, hasKey) // no secret bound
+}
+
+func TestBuildAgentEnvFromBindings_DatabaseProviderPrefix(t *testing.T) {
+	setupResourceControllerTestDB(t)
+	cases := []struct {
+		provider string
+		wantHost string // env var that should carry the host
+	}{
+		{"mysql", "MYSQL_HOST"},
+		{"pg", "PG_HOST"},
+		{"postgresql", "PG_HOST"}, // alias normalises to pg
+		{"redis", "REDIS_HOST"},
+		{"mg", "MONGO_HOST"}, // alias normalises to mongo
+	}
+	for _, tc := range cases {
+		b := model.ResourceBinding{
+			UserId:       7,
+			Name:         "db-" + tc.provider,
+			ResourceType: "database",
+			Provider:     tc.provider,
+			Metadata:     `{"host":"db.example","port":"5432","database":"app","user":"u"}`,
+		}
+		require.NoError(t, model.DB.Create(&b).Error)
+		env, err := buildAgentEnvFromBindings(7, []int{b.Id})
+		require.NoError(t, err, tc.provider)
+		require.Equal(t, "db.example", env[tc.wantHost], "provider %s -> %s", tc.provider, tc.wantHost)
+	}
+}
+
+func TestBuildAgentEnvFromBindings_GitNamesProviderAgnostic(t *testing.T) {
+	setupResourceControllerTestDB(t)
+	for _, provider := range []string{"github", "gitea", "gitlab"} {
+		b := model.ResourceBinding{
+			UserId:       7,
+			Name:         "git-" + provider,
+			ResourceType: "git",
+			Provider:     provider,
+			Metadata:     `{"provider":"` + provider + `","repo_url":"https://x/owner/repo","default_branch":"main"}`,
+		}
+		require.NoError(t, model.DB.Create(&b).Error)
+		env, err := buildAgentEnvFromBindings(7, []int{b.Id})
+		require.NoError(t, err)
+		require.Equal(t, "https://x/owner/repo", env["GIT_REPO_URL"])
+		require.Equal(t, "main", env["GIT_BRANCH"])
+		require.Equal(t, provider, env["GIT_PROVIDER"])
+	}
 }
 
 func TestBuildAgentEnvFromBindings_OwnershipEnforced(t *testing.T) {
 	setupResourceControllerTestDB(t)
-	b := model.ResourceBinding{UserId: 7, Name: "x", ResourceType: "blob", Metadata: `{}`, EnvMap: `{}`}
+	b := model.ResourceBinding{UserId: 7, Name: "x", ResourceType: "blob", Provider: "blob", Metadata: `{}`}
 	require.NoError(t, model.DB.Create(&b).Error)
 
 	_, err := buildAgentEnvFromBindings(99, []int{b.Id}) // different user
 	require.Error(t, err)
 }
 
-func TestBuildAgentEnvFromBindings_SecretWithoutRefFailsBeforeKV(t *testing.T) {
+func TestBuildAgentEnvFromBindings_UnsupportedType(t *testing.T) {
 	setupResourceControllerTestDB(t)
-	b := model.ResourceBinding{
-		UserId:       7,
-		Name:         "vm1",
-		ResourceType: "vm",
-		Metadata:     `{"host":"1.2.3.4"}`,
-		EnvMap:       `{"VM_PASSWORD":{"source":"secret","key":"password"}}`,
-		// SecretRef intentionally empty -> must error before any Key Vault call
-	}
-	require.NoError(t, model.DB.Create(&b).Error)
-
-	_, err := buildAgentEnvFromBindings(7, []int{b.Id})
-	require.Error(t, err)
-	require.Contains(t, err.Error(), "secret_ref")
-}
-
-func TestBuildAgentEnvFromBindings_UnknownSource(t *testing.T) {
-	setupResourceControllerTestDB(t)
-	b := model.ResourceBinding{
-		UserId:       7,
-		Name:         "x",
-		ResourceType: "blob",
-		Metadata:     `{}`,
-		EnvMap:       `{"FOO":{"source":"bogus","key":"x"}}`,
-	}
+	b := model.ResourceBinding{UserId: 7, Name: "x", ResourceType: "quantum", Provider: "weird", Metadata: `{}`}
 	require.NoError(t, model.DB.Create(&b).Error)
 
 	_, err := buildAgentEnvFromBindings(7, []int{b.Id})

heicode/model/resource.go

@@ -15,13 +15,6 @@ type ResourceBinding struct {
 	ExternalId      string `json:"external_id" gorm:"type:varchar(512)"`
 	SecretRef       string `json:"secret_ref" gorm:"type:varchar(512)"`
 	Metadata        string `json:"metadata" gorm:"type:text"`
-	// EnvMap declares the environment variables this binding exposes to a
-	// template agent at start time, as a JSON object: {"ENV_NAME": {"source":
-	// "metadata|secret", "key": "<metadata field or secret json key>"}}.
-	// Non-secret values come from Metadata; secret values are resolved from the
-	// Key Vault secret (SecretRef) only at agent-start and injected into the
-	// agent's .env — never stored here in plaintext.
-	EnvMap          string `json:"env_map" gorm:"type:text"`
 	PermissionScope string `json:"permission_scope" gorm:"type:text"`
 	Constraints     string `json:"constraints" gorm:"type:text"`
 	Status          string `json:"status" gorm:"type:varchar(32);default:'active';index"`