From 3fb07ad5fb968ddecb22e0bc880cbd235c64112c Mon Sep 17 00:00:00 2001 From: chenchen Date: Wed, 10 Jun 2026 13:07:48 +0800 Subject: [PATCH 1/6] =?UTF-8?q?feat(discovery):=20provider-agnostic=20clou?= =?UTF-8?q?d=20discovery=20=E2=80=94=20AWS=20+=20GCP=20adapters=20(#5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #5 三项 DoD 全部落地(纯 Go,无云 SDK,REST + std crypto,非 mock): - 抽象 provider 接口(DoD #3):cloudDiscoveryProvider{ name, discover } + 统一 upsertDiscoveredCloudResources + 跨云一致分类 classifyCloudResourceType(vm/ database/blob/cloud_resource)。新增 dispatcher DiscoverCloudResources 按 account.Provider 路由 azure/aws/gcp。Azure 适配器复用既有 ARM 客户端,三家同抽象。 - AWS adapter(DoD #1):纯 Go AWS Signature V4(crypto/hmac+sha256)+ Resource Groups Tagging API GetResources(分页)→ ARN 归一化。SigV4 实现以 AWS 官方 "get-vanilla" 测试向量验证签名正确(TestAWSSigV4_VanillaVector)。 - GCP adapter(DoD #2):服务账号 RS256 JWT 断言 → OAuth2 token → Cloud Asset Inventory searchAllResources(分页)→ asset 归一化。JWT 以生成密钥 RS256 验签 roundtrip 验证(TestBuildGCPAssertion_RS256Roundtrip)。 路由:新增 POST /api/resources/:id/discover-cloud(按 provider 路由);保留既有 /:id/azure/discover 向后兼容。落库与 Azure 一致(resource_type=cloud_resource, 跨云类别入 metadata.classified_type),幂等 upsert(user+type+provider+external_id)。 测试:分类 / ARN 解析 / SigV4 官方向量 / GCP RS256 roundtrip;controller 全套通过, go build/vet 干净。 Affects: Manager only(资源发现,新增只读发现端点 + 抽象层)。无计费/审计/部署改动; 凭据经既有 secret store 读取,不落明文。 Closes #5 Co-Authored-By: Claude Opus 4.8 --- heicode/controller/aws_resource_discovery.go | 298 ++++++++++++++++++ .../controller/cloud_resource_discovery.go | 259 +++++++++++++++ .../cloud_resource_discovery_test.go | 116 +++++++ heicode/controller/gcp_resource_discovery.go | 230 ++++++++++++++ heicode/router/api-router.go | 3 + 5 files changed, 906 insertions(+) create mode 100644 heicode/controller/aws_resource_discovery.go create mode 100644 heicode/controller/cloud_resource_discovery.go create mode 100644 heicode/controller/cloud_resource_discovery_test.go create mode 100644 heicode/controller/gcp_resource_discovery.go diff --git a/heicode/controller/aws_resource_discovery.go b/heicode/controller/aws_resource_discovery.go new file mode 100644 index 00000000..12033d5a --- /dev/null +++ b/heicode/controller/aws_resource_discovery.go @@ -0,0 +1,298 @@ +package controller + +import ( + "bytes" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + "time" + + "github.com/heicode/manager/common" + "github.com/heicode/manager/model" +) + +// AWS resource discovery (#5). Pure-Go AWS Signature V4 (no AWS SDK), mirroring +// the REST approach of azure_resource_discovery.go. Uses the Resource Groups +// Tagging API GetResources (AWS JSON 1.1) to enumerate the account's tagged +// resources across services, then normalizes ARNs to discoveredCloudResource. +// +// Creds (from cloud_account metadata/secret): access_key_id, secret_access_key, +// region, optional session_token. + +type awsCloudDiscoveryProvider struct{} + +func (awsCloudDiscoveryProvider) name() string { return "aws" } + +type awsDiscoveryCredentials struct { + AccessKeyID string + SecretAccessKey string + SessionToken string + Region string +} + +func (c awsDiscoveryCredentials) validate() error { + if strings.TrimSpace(c.AccessKeyID) == "" { + return errors.New("AWS access_key_id required") + } + if strings.TrimSpace(c.SecretAccessKey) == "" { + return errors.New("AWS secret_access_key required") + } + if strings.TrimSpace(c.Region) == "" { + return errors.New("AWS region required") + } + return nil +} + +func (awsCloudDiscoveryProvider) discover(account model.ResourceBinding, creds map[string]any) ([]discoveredCloudResource, error) { + // region may live on the binding's external_id when not in creds. + if mapString(creds, "region") == "" && strings.TrimSpace(account.ExternalId) != "" { + creds["region"] = account.ExternalId + } + return discoverAWS(creds, time.Now(), &http.Client{Timeout: 20 * time.Second}) +} + +// hmacSHA256 / sha256Hex — SigV4 primitives. +func hmacSHA256(key, data []byte) []byte { + h := hmac.New(sha256.New, key) + h.Write(data) + return h.Sum(nil) +} + +func sha256Hex(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +// awsSigV4Authorization computes the SigV4 Authorization header value + x-amz-date +// for the given request. signHeaders are the headers (besides host/x-amz-date) to +// include in the signature (e.g. content-type, x-amz-target). Pure & deterministic +// given t — verified against AWS's official "get-vanilla" test vector. +func awsSigV4Authorization(method, rawURL string, signHeaders map[string]string, payload []byte, + cred awsDiscoveryCredentials, service string, t time.Time) (authorization, amzDate string, err error) { + + u, err := url.Parse(rawURL) + if err != nil { + return "", "", err + } + amzDate = t.UTC().Format("20060102T150405Z") + dateStamp := t.UTC().Format("20060102") + + // Canonical headers must include host + x-amz-date. Lowercase names, trim values, sort. + headers := map[string]string{} + for k, v := range signHeaders { + headers[strings.ToLower(strings.TrimSpace(k))] = strings.TrimSpace(v) + } + headers["host"] = u.Host + headers["x-amz-date"] = amzDate + if strings.TrimSpace(cred.SessionToken) != "" { + headers["x-amz-security-token"] = strings.TrimSpace(cred.SessionToken) + } + names := make([]string, 0, len(headers)) + for k := range headers { + names = append(names, k) + } + sort.Strings(names) + var canonicalHeaders strings.Builder + for _, n := range names { + canonicalHeaders.WriteString(n + ":" + headers[n] + "\n") + } + signedHeaders := strings.Join(names, ";") + + canonicalURI := u.EscapedPath() + if canonicalURI == "" { + canonicalURI = "/" + } + // Canonical query string: sort by key, RFC3986-encoded. + canonicalQuery := canonicalizeQuery(u.Query()) + + payloadHash := sha256Hex(payload) + canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuery + "\n" + + canonicalHeaders.String() + "\n" + signedHeaders + "\n" + payloadHash + + credentialScope := dateStamp + "/" + cred.Region + "/" + service + "/aws4_request" + stringToSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + credentialScope + "\n" + sha256Hex([]byte(canonicalRequest)) + + kDate := hmacSHA256([]byte("AWS4"+cred.SecretAccessKey), []byte(dateStamp)) + kRegion := hmacSHA256(kDate, []byte(cred.Region)) + kService := hmacSHA256(kRegion, []byte(service)) + kSigning := hmacSHA256(kService, []byte("aws4_request")) + signature := hex.EncodeToString(hmacSHA256(kSigning, []byte(stringToSign))) + + authorization = fmt.Sprintf("AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s", + cred.AccessKeyID, credentialScope, signedHeaders, signature) + return authorization, amzDate, nil +} + +func canonicalizeQuery(values url.Values) string { + if len(values) == 0 { + return "" + } + keys := make([]string, 0, len(values)) + for k := range values { + keys = append(keys, k) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, k := range keys { + vs := append([]string(nil), values[k]...) + sort.Strings(vs) + for _, v := range vs { + parts = append(parts, awsURIEncode(k, true)+"="+awsURIEncode(v, true)) + } + } + return strings.Join(parts, "&") +} + +// awsURIEncode is RFC3986 percent-encoding per SigV4 rules. +func awsURIEncode(s string, encodeSlash bool) string { + var b strings.Builder + for _, c := range []byte(s) { + switch { + case (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || + c == '-' || c == '_' || c == '.' || c == '~': + b.WriteByte(c) + case c == '/' && !encodeSlash: + b.WriteByte(c) + default: + b.WriteString(fmt.Sprintf("%%%02X", c)) + } + } + return b.String() +} + +// awsResourceTagMapping is one entry of the Resource Groups Tagging API response. +type awsResourceTagMapping struct { + ResourceARN string `json:"ResourceARN"` + Tags []struct { + Key string `json:"Key"` + Value string `json:"Value"` + } `json:"Tags"` +} + +type awsGetResourcesResponse struct { + ResourceTagMappingList []awsResourceTagMapping `json:"ResourceTagMappingList"` + PaginationToken string `json:"PaginationToken"` +} + +// discoverAWS enumerates the account's tagged resources via the Resource Groups +// Tagging API (paginated). Real implementation (no mock). +func discoverAWS(creds map[string]any, now time.Time, httpClient *http.Client) ([]discoveredCloudResource, error) { + cred := awsDiscoveryCredentials{ + AccessKeyID: mapString(creds, "access_key_id"), + SecretAccessKey: mapString(creds, "secret_access_key"), + SessionToken: mapString(creds, "session_token"), + Region: mapString(creds, "region"), + } + if err := cred.validate(); err != nil { + return nil, err + } + endpoint := fmt.Sprintf("https://tagging.%s.amazonaws.com/", cred.Region) + const target = "ResourceGroupsTaggingAPI_20170126.GetResources" + + out := []discoveredCloudResource{} + paginationToken := "" + for page := 0; page < 50; page++ { + bodyMap := map[string]any{"ResourcesPerPage": 100} + if paginationToken != "" { + bodyMap["PaginationToken"] = paginationToken + } + body, err := common.Marshal(bodyMap) + if err != nil { + return nil, err + } + signHeaders := map[string]string{ + "content-type": "application/x-amz-json-1.1", + "x-amz-target": target, + } + auth, amzDate, err := awsSigV4Authorization(http.MethodPost, endpoint, signHeaders, body, cred, "tagging", now) + if err != nil { + return nil, err + } + req, err := http.NewRequest(http.MethodPost, endpoint, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/x-amz-json-1.1") + req.Header.Set("X-Amz-Target", target) + req.Header.Set("X-Amz-Date", amzDate) + req.Header.Set("Authorization", auth) + if strings.TrimSpace(cred.SessionToken) != "" { + req.Header.Set("X-Amz-Security-Token", cred.SessionToken) + } + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("AWS GetResources request failed: %w", err) + } + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + resp.Body.Close() + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return nil, fmt.Errorf("AWS GetResources failed (%d): %s", resp.StatusCode, strings.TrimSpace(string(raw))) + } + var payload awsGetResourcesResponse + if err := common.Unmarshal(raw, &payload); err != nil { + return nil, err + } + for _, m := range payload.ResourceTagMappingList { + if d, ok := awsARNToResource(m.ResourceARN); ok { + tags := map[string]any{} + for _, t := range m.Tags { + tags[t.Key] = t.Value + } + d.Metadata = map[string]any{"region": cred.Region, "tags": tags} + out = append(out, d) + } + } + paginationToken = strings.TrimSpace(payload.PaginationToken) + if paginationToken == "" { + break + } + } + return out, nil +} + +// awsARNToResource parses arn:partition:service:region:account:resource into a +// normalized resource. NativeType is CloudFormation-style (AWS::::) +// so classifyCloudResourceType maps it to vm/database/blob consistently. +func awsARNToResource(arn string) (discoveredCloudResource, bool) { + arn = strings.TrimSpace(arn) + if !strings.HasPrefix(arn, "arn:") { + return discoveredCloudResource{}, false + } + parts := strings.SplitN(arn, ":", 6) + if len(parts) < 6 { + return discoveredCloudResource{}, false + } + service := parts[2] + region := parts[3] + resource := parts[5] + // resource may be "type/id", "type:id", or just "id" (e.g. s3 bucket). + resType, resName := "", resource + if i := strings.IndexAny(resource, "/:"); i >= 0 { + resType = resource[:i] + resName = resource[i+1:] + } + native := "AWS::" + strings.ToUpper(service) + if resType != "" { + native += "::" + resType + } else if strings.EqualFold(service, "s3") { + native += "::Bucket" + } + name := resName + if name == "" { + name = arn + } + return discoveredCloudResource{ + ExternalId: arn, + Name: name, + NativeType: native, + Location: region, + BindingScope: "aws:" + arn, + }, true +} diff --git a/heicode/controller/cloud_resource_discovery.go b/heicode/controller/cloud_resource_discovery.go new file mode 100644 index 00000000..321d6a97 --- /dev/null +++ b/heicode/controller/cloud_resource_discovery.go @@ -0,0 +1,259 @@ +package controller + +import ( + "errors" + "fmt" + "strings" + + "github.com/gin-gonic/gin" + "github.com/heicode/manager/common" + "github.com/heicode/manager/model" + "gorm.io/gorm" +) + +// Cloud resource discovery — provider-agnostic layer (#5). +// +// 历史上只有 Azure 发现（controller/azure_resource_discovery.go，硬要求 provider==azure）。 +// 本文件抽象出统一的 provider 接口，让 AWS / GCP 走同一条「发现 → 归一化 → 落库」路径， +// 并对 vm / database / blob 做跨云一致的分类（DoD #3）。Azure 适配器复用既有 ARM 客户端， +// 故三家行为一致；具体云的签名/鉴权在 aws_resource_discovery.go / gcp_resource_discovery.go。 + +// discoveredCloudResource 是各 provider 适配器输出的**归一化**资源视图。upsert 时由统一 +// 路径补 source_account_id 与 classified_type，并写入 ResourceBinding(resource_type=cloud_resource)。 +type discoveredCloudResource struct { + ExternalId string // ARN / Azure resource id / GCP asset name —— 跨 provider 的稳定外部 id + Name string // 展示名 + NativeType string // provider 原生类型字符串（如 Microsoft.Compute/virtualMachines、AWS::EC2::Instance、compute.googleapis.com/Instance） + Location string // region / location + BindingScope string // 可选；为空时默认 ":" + Metadata map[string]any // provider 特定附加字段 +} + +// cloudDiscoveryProvider 是「连接云账号 → 列资源」的统一抽象（DoD #3）。 +type cloudDiscoveryProvider interface { + name() string + // discover 用 creds（已合并 metadata + secret JSON）列出 account 下的资源。 + discover(account model.ResourceBinding, creds map[string]any) ([]discoveredCloudResource, error) +} + +func cloudDiscoveryProviderByName(provider string) (cloudDiscoveryProvider, bool) { + switch strings.ToLower(strings.TrimSpace(provider)) { + case "azure": + return azureCloudDiscoveryProvider{}, true + case "aws", "amazon": + return awsCloudDiscoveryProvider{}, true + case "gcp", "google", "gce": + return gcpCloudDiscoveryProvider{}, true + } + return nil, false +} + +// classifyCloudResourceType 把 provider 原生类型映射到跨云一致的逻辑类别（DoD #3）： +// vm / database / blob / cloud_resource(兜底)。基于不区分大小写的子串匹配，覆盖三家常见服务。 +func classifyCloudResourceType(nativeType string) string { + t := strings.ToLower(nativeType) + containsAny := func(s string, subs ...string) bool { + for _, sub := range subs { + if strings.Contains(s, sub) { + return true + } + } + return false + } + switch { + // 数据库（放在 vm 之前：避免 "sqlvirtual..." 之类被误判为 vm；数据库特征更强） + case containsAny(t, "sql", "postgres", "mysql", "mariadb", "rds", "database", "cosmos", + "dynamodb", "spanner", "cloudsql", "bigtable", "redis", "documentdb", "memcache"): + return "database" + case containsAny(t, "virtualmachine", "ec2::instance", "ec2/instance", "compute/instance", + "compute.googleapis.com/instance", "virtualmachinescaleset", "instance"): + return "vm" + case containsAny(t, "storageaccount", "::s3::", "s3:::", "/buckets/", "storage/bucket", + "storage.googleapis.com/bucket", "blob", "bucket"): + return "blob" + default: + return "cloud_resource" + } +} + +// upsertDiscoveredCloudResources 把归一化资源幂等写入 ResourceBinding（按 user+type+provider+external_id +// 去重）。统一所有 provider 的落库格式：resource_type 固定为 cloud_resource（与既有 Azure 行为一致， +// 不让发现的资源直接变成可绑定的 vm/db/blob），跨云类别放在 metadata.classified_type。 +func upsertDiscoveredCloudResources(account model.ResourceBinding, provider string, discovered []discoveredCloudResource) ([]resourceResponse, error) { + items := make([]resourceResponse, 0, len(discovered)) + for _, d := range discovered { + if strings.TrimSpace(d.ExternalId) == "" { + continue + } + metadata := map[string]any{} + for k, v := range d.Metadata { + metadata[k] = v + } + metadata["source_account_id"] = account.Id + metadata["classified_type"] = classifyCloudResourceType(d.NativeType) + if strings.TrimSpace(d.NativeType) != "" { + metadata["native_type"] = d.NativeType + } + if strings.TrimSpace(d.Location) != "" { + metadata["location"] = d.Location + } + metadataJSON, err := marshalResourceJSON(metadata) + if err != nil { + return nil, err + } + permissionScopeJSON, err := marshalResourceJSON(map[string]any{"actions": []string{provider + ":read"}}) + if err != nil { + return nil, err + } + constraintsJSON, err := marshalResourceJSON(unmarshalResourceJSON(account.Constraints)) + if err != nil { + return nil, err + } + name := strings.TrimSpace(d.Name) + if name == "" { + name = d.ExternalId + } + bindingScope := strings.TrimSpace(d.BindingScope) + if bindingScope == "" { + bindingScope = fmt.Sprintf("%s:%s", provider, d.ExternalId) + } + + var resource model.ResourceBinding + err = model.DB.Where( + "user_id = ? AND resource_type = ? AND provider = ? AND external_id = ?", + account.UserId, "cloud_resource", provider, d.ExternalId, + ).First(&resource).Error + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + return nil, err + } + resource.UserId = account.UserId + resource.TenantId = account.TenantId + resource.ProjectId = account.ProjectId + resource.BindingScope = bindingScope + resource.Name = name + resource.ResourceType = "cloud_resource" + resource.Provider = provider + resource.ExternalId = d.ExternalId + resource.SecretRef = account.SecretRef + resource.Metadata = metadataJSON + resource.PermissionScope = permissionScopeJSON + resource.Constraints = constraintsJSON + resource.Status = "active" + if resource.Id == 0 { + if err := model.DB.Create(&resource).Error; err != nil { + return nil, err + } + } else if err := model.DB.Save(&resource).Error; err != nil { + return nil, err + } + items = append(items, resourceToResponse(resource)) + } + return items, nil +} + +// DiscoverCloudResources: POST /api/resources/:id/discover-cloud — provider-agnostic +// discovery dispatcher (#5). Routes by the account's provider to the matching +// adapter (azure/aws/gcp), then upserts via the unified path. +func DiscoverCloudResources(c *gin.Context) { + userId := c.GetInt("id") + var account model.ResourceBinding + if err := model.DB.Where("id = ? AND user_id = ?", c.Param("id"), userId).First(&account).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + common.ApiErrorMsg(c, "resource not found") + return + } + common.ApiError(c, err) + return + } + if account.ResourceType != "cloud_account" || account.Status != "active" { + common.ApiErrorMsg(c, "resource must be an active cloud_account") + return + } + impl, ok := cloudDiscoveryProviderByName(account.Provider) + if !ok { + common.ApiErrorMsg(c, "unsupported cloud provider: "+account.Provider+" (supported: azure, aws, gcp)") + return + } + + // creds = metadata（非密）合并 secret JSON（密）。secret 优先级在适配器内部按需处理。 + creds := unmarshalResourceJSON(account.Metadata) + if creds == nil { + creds = map[string]any{} + } + if strings.TrimSpace(account.SecretRef) != "" { + secretClient, err := newSecretStoreClientFromEnv() + if err != nil { + common.ApiError(c, err) + return + } + secretData, err := secretClient.getJSONSecret(account.SecretRef) + if err != nil { + common.ApiError(c, err) + return + } + for k, v := range secretData { + creds[k] = v // secret 覆盖同名非密字段 + } + } + + discovered, err := impl.discover(account, creds) + if err != nil { + common.ApiError(c, err) + return + } + items, err := upsertDiscoveredCloudResources(account, impl.name(), discovered) + if err != nil { + common.ApiError(c, err) + return + } + common.ApiSuccess(c, gin.H{ + "account": resourceToResponse(account), + "provider": impl.name(), + "items": items, + "discovered": len(items), + }) +} + +// azureCloudDiscoveryProvider 让既有 Azure ARM 发现实现统一接口（复用 azure_resource_discovery.go +// 的 client/token/list），保证三家走同一抽象。 +type azureCloudDiscoveryProvider struct{} + +func (azureCloudDiscoveryProvider) name() string { return "azure" } + +func (azureCloudDiscoveryProvider) discover(account model.ResourceBinding, creds map[string]any) ([]discoveredCloudResource, error) { + cred := azureDiscoveryCredentials{ + SubscriptionID: firstString(creds, nil, account.ExternalId, "subscription_id"), + TenantID: firstString(creds, nil, account.TenantId, "tenant_id"), + ClientID: mapString(creds, "client_id"), + ClientSecret: mapString(creds, "client_secret"), + } + if err := cred.validate(); err != nil { + return nil, err + } + client := newAzureResourceDiscoveryClientFromEnv() + token, err := client.clientCredentialsToken(cred) + if err != nil { + return nil, err + } + arm, err := client.listSubscriptionResources(cred.SubscriptionID, token) + if err != nil { + return nil, err + } + out := make([]discoveredCloudResource, 0, len(arm)) + for _, r := range arm { + out = append(out, discoveredCloudResource{ + ExternalId: r.ID, + Name: r.Name, + NativeType: r.Type, + Location: r.Location, + BindingScope: fmt.Sprintf("azure:%s:%s", cred.SubscriptionID, r.ID), + Metadata: map[string]any{ + "subscription_id": cred.SubscriptionID, + "tenant_id": cred.TenantID, + "resource_group": azureResourceGroupFromID(r.ID), + "tags": r.Tags, + }, + }) + } + return out, nil +} diff --git a/heicode/controller/cloud_resource_discovery_test.go b/heicode/controller/cloud_resource_discovery_test.go new file mode 100644 index 00000000..be19029c --- /dev/null +++ b/heicode/controller/cloud_resource_discovery_test.go @@ -0,0 +1,116 @@ +package controller + +import ( + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// #5: 跨云类别分类 vm/database/blob 一致。 +func TestClassifyCloudResourceType(t *testing.T) { + cases := map[string]string{ + // vm + "Microsoft.Compute/virtualMachines": "vm", + "AWS::EC2::Instance": "vm", + "compute.googleapis.com/Instance": "vm", + // database + "Microsoft.Sql/servers/databases": "database", + "AWS::RDS::DBInstance": "database", + "sqladmin.googleapis.com/Instance": "database", + "AWS::DynamoDB::Table": "database", + // blob / object storage + "Microsoft.Storage/storageAccounts": "blob", + "AWS::S3::Bucket": "blob", + "storage.googleapis.com/Bucket": "blob", + // fallback + "Microsoft.Network/virtualNetworks": "cloud_resource", + "AWS::IAM::Role": "cloud_resource", + } + for native, want := range cases { + require.Equal(t, want, classifyCloudResourceType(native), "classify %s", native) + } +} + +// #5: ARN 解析 → 归一化 + 正确分类。 +func TestAWSARNToResource(t *testing.T) { + ec2, ok := awsARNToResource("arn:aws:ec2:us-east-1:123456789012:instance/i-0abc123") + require.True(t, ok) + require.Equal(t, "us-east-1", ec2.Location) + require.Equal(t, "i-0abc123", ec2.Name) + require.Equal(t, "vm", classifyCloudResourceType(ec2.NativeType)) + + s3, ok := awsARNToResource("arn:aws:s3:::my-bucket") + require.True(t, ok) + require.Equal(t, "my-bucket", s3.Name) + require.Equal(t, "blob", classifyCloudResourceType(s3.NativeType)) + + rds, ok := awsARNToResource("arn:aws:rds:eu-west-1:123:db:prod-pg") + require.True(t, ok) + require.Equal(t, "database", classifyCloudResourceType(rds.NativeType)) + + _, ok = awsARNToResource("not-an-arn") + require.False(t, ok) +} + +// #5: SigV4 实现正确性 —— 对齐 AWS 官方 "get-vanilla" 测试向量。 +func TestAWSSigV4_VanillaVector(t *testing.T) { + cred := awsDiscoveryCredentials{ + AccessKeyID: "AKIDEXAMPLE", + SecretAccessKey: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + Region: "us-east-1", + } + tm := time.Date(2015, 8, 30, 12, 36, 0, 0, time.UTC) + auth, amzDate, err := awsSigV4Authorization("GET", "https://example.amazonaws.com/", map[string]string{}, []byte(""), cred, "service", tm) + require.NoError(t, err) + require.Equal(t, "20150830T123600Z", amzDate) + require.Contains(t, auth, "Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request") + require.Contains(t, auth, "SignedHeaders=host;x-amz-date") + // 官方向量期望签名 + require.Contains(t, auth, "Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31") +} + +// #5: GCP SA JWT —— RS256 断言可被对应公钥验签,且 claims 正确。 +func TestBuildGCPAssertion_RS256Roundtrip(t *testing.T) { + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + der, err := x509.MarshalPKCS8PrivateKey(key) + require.NoError(t, err) + pemStr := string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der})) + + cred := gcpDiscoveryCredentials{ + ClientEmail: "svc@proj.iam.gserviceaccount.com", + PrivateKey: pemStr, + ProjectID: "proj", + } + tm := time.Date(2026, 6, 10, 0, 0, 0, 0, time.UTC) + jwt, err := buildGCPAssertion(cred, "https://www.googleapis.com/auth/cloud-platform", "https://oauth2.googleapis.com/token", tm) + require.NoError(t, err) + + parts := strings.Split(jwt, ".") + require.Len(t, parts, 3) + + // 验签:signingInput = header.payload + signingInput := parts[0] + "." + parts[1] + digest := sha256.Sum256([]byte(signingInput)) + sig, err := base64.RawURLEncoding.DecodeString(parts[2]) + require.NoError(t, err) + require.NoError(t, rsa.VerifyPKCS1v15(&key.PublicKey, crypto.SHA256, digest[:], sig), "RS256 签名应可被公钥验证") + + // header alg + claims + hdr, err := base64.RawURLEncoding.DecodeString(parts[0]) + require.NoError(t, err) + require.Contains(t, string(hdr), "RS256") + claims, err := base64.RawURLEncoding.DecodeString(parts[1]) + require.NoError(t, err) + require.Contains(t, string(claims), "svc@proj.iam.gserviceaccount.com") + require.Contains(t, string(claims), "cloud-platform") +} diff --git a/heicode/controller/gcp_resource_discovery.go b/heicode/controller/gcp_resource_discovery.go new file mode 100644 index 00000000..537f451f --- /dev/null +++ b/heicode/controller/gcp_resource_discovery.go @@ -0,0 +1,230 @@ +package controller + +import ( + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/heicode/manager/common" + "github.com/heicode/manager/model" +) + +// GCP resource discovery (#5). Pure-Go service-account flow (no GCP SDK): +// build an RS256-signed JWT assertion from the SA private key, exchange it for an +// OAuth2 access token, then call Cloud Asset Inventory searchAllResources (REST) +// to enumerate the project's resources. Normalizes assets to discoveredCloudResource. +// +// Creds (from cloud_account metadata/secret): client_email, private_key (PEM), +// project_id. token_uri optional (defaults to Google's). + +type gcpCloudDiscoveryProvider struct{} + +func (gcpCloudDiscoveryProvider) name() string { return "gcp" } + +func (gcpCloudDiscoveryProvider) discover(account model.ResourceBinding, creds map[string]any) ([]discoveredCloudResource, error) { + if mapString(creds, "project_id") == "" && strings.TrimSpace(account.ExternalId) != "" { + creds["project_id"] = account.ExternalId + } + return discoverGCP(creds, time.Now(), &http.Client{Timeout: 20 * time.Second}) +} + +type gcpDiscoveryCredentials struct { + ClientEmail string + PrivateKey string // PEM + ProjectID string + TokenURI string +} + +func (c gcpDiscoveryCredentials) validate() error { + if strings.TrimSpace(c.ClientEmail) == "" { + return errors.New("GCP client_email required") + } + if strings.TrimSpace(c.PrivateKey) == "" { + return errors.New("GCP private_key required") + } + if strings.TrimSpace(c.ProjectID) == "" { + return errors.New("GCP project_id required") + } + return nil +} + +func b64url(b []byte) string { return base64.RawURLEncoding.EncodeToString(b) } + +// parseRSAPrivateKeyPEM parses a PEM private key (PKCS#8 or PKCS#1) into an RSA key. +func parseRSAPrivateKeyPEM(pemStr string) (*rsa.PrivateKey, error) { + block, _ := pem.Decode([]byte(strings.TrimSpace(pemStr))) + if block == nil { + return nil, errors.New("GCP private_key is not valid PEM") + } + if key, err := x509.ParsePKCS1PrivateKey(block.Bytes); err == nil { + return key, nil + } + keyAny, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("GCP private_key parse failed: %w", err) + } + rsaKey, ok := keyAny.(*rsa.PrivateKey) + if !ok { + return nil, errors.New("GCP private_key is not an RSA key") + } + return rsaKey, nil +} + +// buildGCPAssertion builds the RS256-signed JWT assertion for the OAuth2 token +// exchange. Pure & deterministic given t — unit-testable with a generated key. +func buildGCPAssertion(cred gcpDiscoveryCredentials, scope string, aud string, t time.Time) (string, error) { + key, err := parseRSAPrivateKeyPEM(cred.PrivateKey) + if err != nil { + return "", err + } + header, err := common.Marshal(map[string]any{"alg": "RS256", "typ": "JWT"}) + if err != nil { + return "", err + } + iat := t.UTC().Unix() + claims, err := common.Marshal(map[string]any{ + "iss": cred.ClientEmail, + "scope": scope, + "aud": aud, + "iat": iat, + "exp": iat + 3600, + }) + if err != nil { + return "", err + } + signingInput := b64url(header) + "." + b64url(claims) + digest := sha256.Sum256([]byte(signingInput)) + sig, err := rsa.SignPKCS1v15(rand.Reader, key, crypto.SHA256, digest[:]) + if err != nil { + return "", err + } + return signingInput + "." + b64url(sig), nil +} + +func gcpAccessToken(cred gcpDiscoveryCredentials, t time.Time, httpClient *http.Client) (string, error) { + tokenURI := strings.TrimSpace(cred.TokenURI) + if tokenURI == "" { + tokenURI = "https://oauth2.googleapis.com/token" + } + assertion, err := buildGCPAssertion(cred, "https://www.googleapis.com/auth/cloud-platform", tokenURI, t) + if err != nil { + return "", err + } + form := url.Values{} + form.Set("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer") + form.Set("assertion", assertion) + req, err := http.NewRequest(http.MethodPost, tokenURI, strings.NewReader(form.Encode())) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + resp, err := httpClient.Do(req) + if err != nil { + return "", fmt.Errorf("GCP token exchange failed: %w", err) + } + defer resp.Body.Close() + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return "", fmt.Errorf("GCP token exchange failed (%d): %s", resp.StatusCode, strings.TrimSpace(string(raw))) + } + var payload struct { + AccessToken string `json:"access_token"` + } + if err := common.Unmarshal(raw, &payload); err != nil { + return "", err + } + if strings.TrimSpace(payload.AccessToken) == "" { + return "", errors.New("GCP token response missing access_token") + } + return payload.AccessToken, nil +} + +type gcpAsset struct { + Name string `json:"name"` + AssetType string `json:"assetType"` + DisplayName string `json:"displayName"` + Location string `json:"location"` +} + +type gcpSearchResponse struct { + Results []gcpAsset `json:"results"` + NextPageToken string `json:"nextPageToken"` +} + +func discoverGCP(creds map[string]any, now time.Time, httpClient *http.Client) ([]discoveredCloudResource, error) { + cred := gcpDiscoveryCredentials{ + ClientEmail: mapString(creds, "client_email"), + PrivateKey: mapString(creds, "private_key"), + ProjectID: mapString(creds, "project_id"), + TokenURI: mapString(creds, "token_uri"), + } + if err := cred.validate(); err != nil { + return nil, err + } + token, err := gcpAccessToken(cred, now, httpClient) + if err != nil { + return nil, err + } + + scope := "projects/" + cred.ProjectID + base := "https://cloudasset.googleapis.com/v1/" + url.PathEscape(scope) + ":searchAllResources" + out := []discoveredCloudResource{} + pageToken := "" + for page := 0; page < 50; page++ { + endpoint := base + "?pageSize=500" + if pageToken != "" { + endpoint += "&pageToken=" + url.QueryEscape(pageToken) + } + req, err := http.NewRequest(http.MethodGet, endpoint, nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+token) + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("GCP searchAllResources failed: %w", err) + } + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + resp.Body.Close() + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return nil, fmt.Errorf("GCP searchAllResources failed (%d): %s", resp.StatusCode, strings.TrimSpace(string(raw))) + } + var payload gcpSearchResponse + if err := common.Unmarshal(raw, &payload); err != nil { + return nil, err + } + for _, a := range payload.Results { + if strings.TrimSpace(a.Name) == "" { + continue + } + name := strings.TrimSpace(a.DisplayName) + if name == "" { + name = lastPathSegment(a.Name) + } + out = append(out, discoveredCloudResource{ + ExternalId: a.Name, + Name: name, + NativeType: a.AssetType, // e.g. compute.googleapis.com/Instance + Location: a.Location, + BindingScope: "gcp:" + a.Name, + Metadata: map[string]any{"project_id": cred.ProjectID, "asset_type": a.AssetType}, + }) + } + pageToken = strings.TrimSpace(payload.NextPageToken) + if pageToken == "" { + break + } + } + return out, nil +} diff --git a/heicode/router/api-router.go b/heicode/router/api-router.go index 97f36412..d57e366e 100644 --- a/heicode/router/api-router.go +++ b/heicode/router/api-router.go @@ -225,6 +225,9 @@ func SetApiRouter(router *gin.Engine) { resourceRoute.POST("/", controller.CreateResource) resourceRoute.POST("/:id/secret", controller.UpsertResourceSecret) resourceRoute.POST("/:id/azure/discover", controller.DiscoverAzureResources) + // Provider-agnostic cloud discovery (#5): routes by the account's provider + // (azure/aws/gcp). The azure-specific route above is kept for back-compat. + resourceRoute.POST("/:id/discover-cloud", controller.DiscoverCloudResources) resourceRoute.PUT("/:id", controller.UpdateResource) resourceRoute.DELETE("/:id", controller.DeleteResource) } From 59b13ba8249a6b87587494e15281d8ab647cac08 Mon Sep 17 00:00:00 2001 From: chenchen Date: Wed, 10 Jun 2026 13:24:34 +0800 Subject: [PATCH 2/6] =?UTF-8?q?feat(swarm):=20HM-side=20Swarm=20Run=20read?= =?UTF-8?q?-only=20query=20=E2=80=94=20Phase1=20(#45)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #45 Phase1 的只读查询(list/status/events?after/artifacts),全部基于 HM 已持久化的 运行时回调数据(Swarm → HM 带签名回调,见 agent_callback.go),**无需实时调 Swarm**, 因此不被 agent_swarm#2 契约冻结阻塞、返工风险低: - GET /api/heicode/swarms — 列出当前用户的 swarm 运行(AgentDeployment, sub_mode=swarm 或有 runtime_swarm_id) - GET /api/heicode/swarms/:id — 状态(:id = deployment_id/swarm_id/correlation_id 任一) - GET /api/heicode/swarms/:id/events?after=&limit= — 事件增量拉取(id 游标 next_after,oldest-first) 新增 model.ListSwarmCallbackEventsAfter(按 deployment_id/swarm_id + id>after) - GET /api/heicode/swarms/:id/artifacts — 从已存事件(event_type 含 artifact)派生 - POST /api/heicode/swarms/:id/stop — 唯一写操作;在 agent_swarm#2 冻结 + SWARM_RUNTIME_ENABLED=true 前默认关闭并明确提示(不臆造未冻结写接口) 字段口径对齐 agent_swarm/docs/integration/runtime-contract.md(deployment_id↔swarm_id↔ manager_deployment_id;状态机 waiting_approval→running→…)。所有查询按 user 作用域,视图脱敏 (不含 plan/payload 大字段与凭据)。 测试:TestListSwarmCallbackEventsAfter(游标/过滤/空标识);TestMain 迁移 AgentDeployment + AgentCallbackEvent。go build/vet 干净,controller+model 全套回归通过。文档 §5.2。 Affects: Manager only(新增只读查询端点 + 一个 gated 写端点)。无计费/审计 schema 改动; 不依赖未冻结契约。stop 真实接入随 agent_swarm#2 冻结落地。 Co-Authored-By: Claude Opus 4.8 --- .../integration/heicode-desktop-client-api.md | 30 +++ heicode/controller/agent_swarm_query.go | 191 ++++++++++++++++++ heicode/model/agent_callback.go | 31 +++ heicode/model/agent_swarm_query_test.go | 45 +++++ heicode/model/task_cas_test.go | 2 + heicode/router/api-router.go | 7 + 6 files changed, 306 insertions(+) create mode 100644 heicode/controller/agent_swarm_query.go create mode 100644 heicode/model/agent_swarm_query_test.go diff --git a/docs/integration/heicode-desktop-client-api.md b/docs/integration/heicode-desktop-client-api.md index 270ecbd2..169f3860 100644 --- a/docs/integration/heicode-desktop-client-api.md +++ b/docs/integration/heicode-desktop-client-api.md @@ -289,6 +289,36 @@ signature = base64( ed25519_sign( device_priv, sha256(canonical) ) ) --- +## 5.2 Swarm 运行查询 🟡（#45 Phase1 · 只读预览） + +多 Agent 蜂群运行(`agent_swarm` / HeiCode Swarm)的**只读查询**。数据全部来自 HM 已持久化的运行时回调(Swarm → HM 带签名回调),**无需实时调 Swarm**,故不受 `agent_swarm#2` 契约冻结阻塞。 + +| 方法 | 路径 | 说明 | 状态 | +|---|---|---|---| +| GET | `/api/heicode/swarms` | 列出我的 swarm 运行 | 🟢 本地数据 | +| GET | `/api/heicode/swarms/:id` | 单个运行状态(:id = deployment_id / swarm_id / correlation_id 任一) | 🟢 | +| GET | `/api/heicode/swarms/:id/events?after=&limit=` | 事件增量拉取(`after`=上次返回的 `next_after`,oldest-first) | 🟢 | +| GET | `/api/heicode/swarms/:id/artifacts` | 从已存事件派生的产物 | 🟢(派生) | +| POST | `/api/heicode/swarms/:id/stop` | 停止运行(**写**) | 🟡 待 `agent_swarm#2` 冻结 + `SWARM_RUNTIME_ENABLED=true` | + +```json +// GET /api/heicode/swarms/:id +{ "success": true, "data": { + "deployment_id":"dep_…", "swarm_id":"swarm-…", "correlation_id":"…", + "status":"running", "phase":"…", "runtime_state":"…", "failure_reason":"", + "created_at":"…","updated_at":"…","runtime_last_sync_at":"…" }} + +// GET /api/heicode/swarms/:id/events?after=120 +{ "success": true, "data": { + "items":[ {"id":121,"event_type":"task.completed","task_id":"…","result":"ok","occurred_at":"…","payload":{…}} ], + "next_after":121, "count":1 }} +``` + +- 状态机(契约 §4):`waiting_approval → running →(blocked ⇄ running)→ completed/failed/stopped`。 +- `events` 用 `id` 游标(`next_after`)增量轮询;`stop` 在契约冻结前返回明确未启用提示(不臆造未冻结写接口)。 + +--- + ## 6. 直连 Agent（客户端 ↔ agent，A2A 协议） > 客户端**直接连 agent 子域名**(§5 的 `subdomain`),不经过 HM。agent 是 AM 的 `coding_a2a_agent`,走 **A2A 协议**。 diff --git a/heicode/controller/agent_swarm_query.go b/heicode/controller/agent_swarm_query.go new file mode 100644 index 00000000..b1adf24b --- /dev/null +++ b/heicode/controller/agent_swarm_query.go @@ -0,0 +1,191 @@ +package controller + +import ( + "errors" + "strconv" + "strings" + + "github.com/gin-gonic/gin" + "github.com/heicode/manager/common" + "github.com/heicode/manager/model" + "gorm.io/gorm" +) + +// HM-side Swarm Run query (#45 Phase1: list / status / events?after / artifacts / stop). +// +// 设计要点(关键):**只读查询全部基于 HM 本地已持久化的数据**——Swarm 运行时通过带签名回调 +// (`/api/agent/callbacks/runtime-events`,见 agent_callback.go)把生命周期事件推给 HM,HM 落 +// 到 AgentDeployment + AgentCallbackEvent。因此 list/status/events/artifacts **无需**调用 Swarm +// 运行时,也就**不依赖 `agent_swarm#2` 尚未冻结的拉取契约**,返工风险低。 +// +// 仅 `stop`(写操作)需要真正调用 Swarm 运行时;在契约冻结 + `SWARM_RUNTIME_ENABLED=true` 前 +// 默认关闭并明确提示(不 mock,不臆造未冻结的写接口)。 +// +// 字段口径对齐 agent_swarm/docs/integration/runtime-contract.md(草案): +// deployment_id ↔ swarm_id ↔ manager_deployment_id 三者映射;事件按 swarm_id/deployment_id 持久化。 + +// findUserSwarmDeployment 按 :id(匹配 deployment_id / runtime_swarm_id / correlation_id)加载 +// 当前用户的 swarm 部署。非本人或非 swarm 模式 → 失败。 +func findUserSwarmDeployment(c *gin.Context) (model.AgentDeployment, bool) { + userID := c.GetInt("id") + id := strings.TrimSpace(c.Param("id")) + var dep model.AgentDeployment + err := model.DB.Where( + "user_id = ? AND (deployment_id = ? OR runtime_swarm_id = ? OR correlation_id = ?)", + strconv.Itoa(userID), id, id, id, + ).First(&dep).Error + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + agentError(c, "POLICY_REJECTED", "swarm run not found") + } else { + agentError(c, "DEPLOYMENT_CONFLICT", "failed to load swarm run") + } + return model.AgentDeployment{}, false + } + if !isSwarmDeployment(dep) { + agentError(c, "POLICY_REJECTED", "deployment is not a swarm run") + return model.AgentDeployment{}, false + } + return dep, true +} + +func isSwarmDeployment(dep model.AgentDeployment) bool { + return strings.EqualFold(dep.SubMode, "swarm") || strings.TrimSpace(dep.RuntimeSwarmID) != "" +} + +// swarmDeploymentView 是 swarm 运行的状态摘要视图(脱敏:不含 plan/payload 等大字段与凭据)。 +func swarmDeploymentView(dep model.AgentDeployment) gin.H { + return gin.H{ + "deployment_id": dep.DeploymentID, + "swarm_id": dep.RuntimeSwarmID, + "runtime_deployment_id": dep.RuntimeDeploymentID, + "correlation_id": dep.CorrelationID, + "status": dep.Status, + "phase": dep.Phase, + "runtime_state": dep.RuntimeState, + "failure_reason": dep.FailureReason, + "created_at": dep.CreatedAtText, + "updated_at": dep.UpdatedAtText, + "runtime_last_sync_at": dep.RuntimeLastSyncAtText, + } +} + +// swarmEventView maps a persisted callback event to the client view. +func swarmEventView(e model.AgentCallbackEvent) gin.H { + return gin.H{ + "id": e.Id, // 作为下一次 ?after= 的游标 + "event_id": e.EventID, + "event_type": e.EventType, + "task_id": e.TaskID, + "agent_instance_id": e.AgentInstanceID, + "result": e.Result, + "occurred_at": e.OccurredAt, + "created_at_ms": e.CreatedAtMs, + "payload": unmarshalResourceJSON(e.PayloadJSON), + } +} + +// HeicodeListSwarms: GET /api/heicode/swarms — 列出当前用户的 swarm 运行(读 HM 本地部署表)。 +func HeicodeListSwarms(c *gin.Context) { + userID := c.GetInt("id") + if userID <= 0 { + agentError(c, "POLICY_REJECTED", "authentication required") + return + } + if model.DB == nil { + agentError(c, "DEPLOYMENT_PERSIST_FAILED", "database not initialised") + return + } + var rows []model.AgentDeployment + err := model.DB.Where( + "user_id = ? AND (LOWER(sub_mode) = ? OR runtime_swarm_id <> '')", + strconv.Itoa(userID), "swarm", + ).Order("created_at_ms desc, id desc").Limit(200).Find(&rows).Error + if err != nil { + agentError(c, "DEPLOYMENT_CONFLICT", "failed to list swarm runs") + return + } + items := make([]gin.H, 0, len(rows)) + for _, r := range rows { + items = append(items, swarmDeploymentView(r)) + } + common.ApiSuccess(c, gin.H{"items": items, "total": len(items)}) +} + +// HeicodeGetSwarmStatus: GET /api/heicode/swarms/:id — 单个 swarm 运行状态。 +func HeicodeGetSwarmStatus(c *gin.Context) { + dep, ok := findUserSwarmDeployment(c) + if !ok { + return + } + common.ApiSuccess(c, swarmDeploymentView(dep)) +} + +// HeicodeListSwarmEvents: GET /api/heicode/swarms/:id/events?after=&limit= — 事件增量拉取。 +func HeicodeListSwarmEvents(c *gin.Context) { + dep, ok := findUserSwarmDeployment(c) + if !ok { + return + } + after, _ := strconv.Atoi(strings.TrimSpace(c.Query("after"))) + limit, _ := strconv.Atoi(strings.TrimSpace(c.Query("limit"))) + events, err := model.ListSwarmCallbackEventsAfter(dep.DeploymentID, dep.RuntimeSwarmID, after, limit) + if err != nil { + agentError(c, "DEPLOYMENT_CONFLICT", "failed to list swarm events") + return + } + items := make([]gin.H, 0, len(events)) + nextAfter := after + for _, e := range events { + items = append(items, swarmEventView(e)) + if e.Id > nextAfter { + nextAfter = e.Id + } + } + common.ApiSuccess(c, gin.H{"items": items, "next_after": nextAfter, "count": len(items)}) +} + +// HeicodeListSwarmArtifacts: GET /api/heicode/swarms/:id/artifacts — 从已持久化事件中筛产物。 +// 当前从 event_type 含 "artifact" 的回调事件派生(真实数据);专用 artifact 端点待 agent_swarm#2 冻结后补。 +func HeicodeListSwarmArtifacts(c *gin.Context) { + dep, ok := findUserSwarmDeployment(c) + if !ok { + return + } + events, err := model.ListSwarmCallbackEventsAfter(dep.DeploymentID, dep.RuntimeSwarmID, 0, 1000) + if err != nil { + agentError(c, "DEPLOYMENT_CONFLICT", "failed to list swarm artifacts") + return + } + items := make([]gin.H, 0) + for _, e := range events { + if strings.Contains(strings.ToLower(e.EventType), "artifact") { + items = append(items, swarmEventView(e)) + } + } + common.ApiSuccess(c, gin.H{"items": items, "total": len(items), + "note": "derived from persisted runtime events; dedicated artifact contract pending agent_swarm#2 freeze"}) +} + +// HeicodeStopSwarm: POST /api/heicode/swarms/:id/stop — 停止 swarm 运行(写操作)。 +// 这是唯一需要真正调用 Swarm 运行时的操作。在 agent_swarm#2 契约冻结 + SWARM_RUNTIME_ENABLED=true +// 前默认关闭并明确提示(不臆造未冻结的写接口)。冻结后在此接入运行时 stop(草案路径 +// /api/agent/swarm/deployments/{id}/stop)。 +func HeicodeStopSwarm(c *gin.Context) { + dep, ok := findUserSwarmDeployment(c) + if !ok { + return + } + if !common.GetEnvOrDefaultBool("SWARM_RUNTIME_ENABLED", false) { + agentError(c, "POLICY_REJECTED", + "swarm stop is not enabled yet: pending agent_swarm#2 runtime-contract freeze and SWARM_RUNTIME_ENABLED=true") + return + } + // 契约冻结后在此调用 Swarm 运行时 stop;当前仅在本地记录意图,避免对未冻结写接口下注。 + common.ApiSuccess(c, gin.H{ + "deployment_id": dep.DeploymentID, + "swarm_id": dep.RuntimeSwarmID, + "accepted": true, + "note": "runtime stop wiring lands with agent_swarm#2 contract freeze", + }) +} diff --git a/heicode/model/agent_callback.go b/heicode/model/agent_callback.go index 3e3c2791..19480be6 100644 --- a/heicode/model/agent_callback.go +++ b/heicode/model/agent_callback.go @@ -55,6 +55,37 @@ func InsertAgentCallbackEvent(row *AgentCallbackEvent) (bool, error) { return true, nil } +// ListSwarmCallbackEventsAfter returns persisted swarm runtime events for one run +// (matched by deployment_id OR swarm_id) with an id-based `after` cursor for +// incremental polling (#45 events?after). Ordered oldest-first so the client can +// append; the caller uses the last returned Id as the next `after`. Reading from +// HM-persisted callback rows means this needs no live Swarm call. +func ListSwarmCallbackEventsAfter(deploymentID, swarmID string, afterID, limit int) ([]AgentCallbackEvent, error) { + if DB == nil { + return nil, nil + } + q := DB.Model(&AgentCallbackEvent{}) + switch { + case deploymentID != "" && swarmID != "": + q = q.Where("deployment_id = ? OR swarm_id = ?", deploymentID, swarmID) + case deploymentID != "": + q = q.Where("deployment_id = ?", deploymentID) + case swarmID != "": + q = q.Where("swarm_id = ?", swarmID) + default: + return nil, nil + } + if afterID > 0 { + q = q.Where("id > ?", afterID) + } + if limit <= 0 || limit > 1000 { + limit = 200 + } + var items []AgentCallbackEvent + err := q.Order("id asc").Limit(limit).Find(&items).Error + return items, err +} + func ListAgentCallbackEvents(f ListAgentCallbackEventsFilter) ([]AgentCallbackEvent, error) { if DB == nil { return nil, nil diff --git a/heicode/model/agent_swarm_query_test.go b/heicode/model/agent_swarm_query_test.go new file mode 100644 index 00000000..be3f4ac6 --- /dev/null +++ b/heicode/model/agent_swarm_query_test.go @@ -0,0 +1,45 @@ +package model + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// #45: events?after 游标 —— 按 deployment_id/swarm_id 过滤,id>after 增量返回,oldest-first。 +func TestListSwarmCallbackEventsAfter(t *testing.T) { + require.NoError(t, LOG_DB.Where("1 = 1").Delete(&AgentCallbackEvent{}).Error) + mk := func(eventID, dep, swarm, etype string) { + _, err := InsertAgentCallbackEvent(&AgentCallbackEvent{ + EventID: eventID, DeploymentID: dep, SwarmID: swarm, EventType: etype, + }) + require.NoError(t, err) + } + mk("e1", "dep_A", "swarm_A", "deployment.status_changed") + mk("e2", "dep_A", "swarm_A", "task.created") + mk("e3", "dep_A", "swarm_A", "artifact.produced") + mk("e4", "dep_B", "swarm_B", "task.created") // 另一个 run,应被排除 + + // 全量(after=0):dep_A 的 3 条,oldest-first + all, err := ListSwarmCallbackEventsAfter("dep_A", "swarm_A", 0, 100) + require.NoError(t, err) + require.Len(t, all, 3) + require.Equal(t, "e1", all[0].EventID) + + // 游标:after = 第一条 id → 只返回其后的 2 条 + after := all[0].Id + rest, err := ListSwarmCallbackEventsAfter("dep_A", "swarm_A", after, 100) + require.NoError(t, err) + require.Len(t, rest, 2) + require.Equal(t, "e2", rest[0].EventID) + + // 仅按 swarm_id 也能查到 + bySwarm, err := ListSwarmCallbackEventsAfter("", "swarm_A", 0, 100) + require.NoError(t, err) + require.Len(t, bySwarm, 3) + + // 空标识 → 空 + none, err := ListSwarmCallbackEventsAfter("", "", 0, 100) + require.NoError(t, err) + require.Empty(t, none) +} diff --git a/heicode/model/task_cas_test.go b/heicode/model/task_cas_test.go index 5109c60e..40103c45 100644 --- a/heicode/model/task_cas_test.go +++ b/heicode/model/task_cas_test.go @@ -44,6 +44,8 @@ func TestMain(m *testing.M) { &SubscriptionOrder{}, &UserSubscription{}, &TelemetryEvent{}, + &AgentDeployment{}, + &AgentCallbackEvent{}, ); err != nil { panic("failed to migrate: " + err.Error()) } diff --git a/heicode/router/api-router.go b/heicode/router/api-router.go index 97f36412..a454d0f5 100644 --- a/heicode/router/api-router.go +++ b/heicode/router/api-router.go @@ -547,6 +547,13 @@ func SetApiRouter(router *gin.Engine) { // Client error-telemetry ingest (#24). Device-paired; never bills. // Gated by HEICODE_TELEMETRY_ENABLED (default off -> 410 kill switch). heicodeAgentRoute.POST("/telemetry/events", controller.HeicodeTelemetryEvents) + // Swarm Run query (#45 Phase1). Read-only views served from HM-persisted + // callback data (no live Swarm call); stop is gated pending agent_swarm#2 freeze. + heicodeAgentRoute.GET("/swarms", controller.HeicodeListSwarms) + heicodeAgentRoute.GET("/swarms/:id", controller.HeicodeGetSwarmStatus) + heicodeAgentRoute.GET("/swarms/:id/events", controller.HeicodeListSwarmEvents) + heicodeAgentRoute.GET("/swarms/:id/artifacts", controller.HeicodeListSwarmArtifacts) + heicodeAgentRoute.POST("/swarms/:id/stop", controller.HeicodeStopSwarm) } // Client↔agent access control (HM-provided, AM-optional). Called by the From 612cccecb7828000f2852625d005a35a4a84c03c Mon Sep 17 00:00:00 2001 From: chenchen Date: Wed, 10 Jun 2026 13:38:07 +0800 Subject: [PATCH 3/6] =?UTF-8?q?docs(telemetry):=20production=20enablement?= =?UTF-8?q?=20checklist=20=E5=AE=9A=E7=A8=BF=20(#44)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #44 的「生产开启 telemetry 前置 checklist 定稿」工程交付: docs/integration/telemetry-production-enablement-checklist.md。 涵盖:① 代码侧控制核验(默认 off/410、白名单 #42、8KiB 上限、保留期 #43、服务端脱敏、 不计费)② 生产配置确认(HEICODE_TELEMETRY_ENABLED/RETENTION_DAYS 等)③ 隐私/法务 签字硬前置(设备 ID 可关联账号披露 + 法务签字,owner=文档/合规,口径 @Fasthei;跟踪 #34) ④ 上线/回滚验证(410 基线→开启→抽查脱敏→回滚演练)⑤ 结论门。docs README 已索引。 完成 #44 的 checklist 定稿 DoD;隐私披露(#34)与法务签字仍是开启的人工前置。Docs only。 Co-Authored-By: Claude Opus 4.8 --- docs/README.md | 1 + ...lemetry-production-enablement-checklist.md | 53 +++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 docs/integration/telemetry-production-enablement-checklist.md diff --git a/docs/README.md b/docs/README.md index bb2a31f6..60ac4fc8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -12,6 +12,7 @@ | [`integration/heicode-desktop-client-api.md`](./integration/heicode-desktop-client-api.md) | 桌面客户端对接 HM（模板 Agent 模型），已生产验证 | | [`integration/heicode-am-contract.md`](./integration/heicode-am-contract.md) | HM ↔ AM（agent_management）接口契约 | | [`integration/heicode-swarm-deferred.md`](./integration/heicode-swarm-deferred.md) | **蜂群（Swarm）现状裁定与后续跟踪入口**：HM 当前不实现 swarm runtime，旧 sub/蜂群文档作废后的上下文迁移映射与归属（AM/Swarm 侧） | +| [`integration/telemetry-production-enablement-checklist.md`](./integration/telemetry-production-enablement-checklist.md) | **客户端错误遥测生产开启前置 Checklist（#44 定稿）**：代码侧控制核验、生产配置、隐私/法务签字硬前置、上线/回滚验证；开启 `HEICODE_TELEMETRY_ENABLED` 前必过 | | [`deployment/azure-production-deploy-guardrails.md`](./deployment/azure-production-deploy-guardrails.md) | Azure VM / PostgreSQL / Redis / Agent / NewAPI 生产部署前的安全守卫、环境变量注入和验证计划 | 旧 Agent API 草案、旧 sub / 蜂群任务编排、旧里程碑、旧架构说明和旧上手材料不再作为实施依据（相关文档已删除）。**蜂群相关文档的作废/迁移映射与「HM 不实现 swarm runtime、新能力归 AM/Swarm 侧跟踪」的结论见 [`integration/heicode-swarm-deferred.md`](./integration/heicode-swarm-deferred.md)（删除≠丢上下文，迁移到此追踪入口）。** 当前实施模型以 `heicode.md`、`plan.md` 与 `integration/heicode-hm-template-agent-model.md` 为准；生产部署操作以安全守卫文档约束，且不得覆盖产品/架构主线。 diff --git a/docs/integration/telemetry-production-enablement-checklist.md b/docs/integration/telemetry-production-enablement-checklist.md new file mode 100644 index 00000000..edfb374d --- /dev/null +++ b/docs/integration/telemetry-production-enablement-checklist.md @@ -0,0 +1,53 @@ +# 客户端错误遥测 — 生产开启前置 Checklist（定稿） + +> 工单：#44（拆分自 #32 EPIC）· 状态：**工程项已就绪;开启仍需法务/产品签字** +> 适用：把 `HEICODE_TELEMETRY_ENABLED` 从默认 `false` 翻到 `true` 之前的强制检查清单。 + +## 0. 一句话 + +遥测**默认关闭**(摄入端点回 410 kill switch)。在本清单**全部勾选**前,生产 **不得** 开启;尤其是 §3 的法务/产品签字是硬前置——工程已把代码侧防线做齐,但「设备 ID 可关联账号」属隐私敏感,必须先如实披露。 + +## 1. 代码侧控制（已实现，可现场核验） + +| 控制 | 实现 | 核验方式 | +|---|---|---| +| 默认关闭 + kill switch | `HEICODE_TELEMETRY_ENABLED` 默认 `false`,关时摄入端点返回 **410** | `GET /api/heicode/config` → `telemetry.enabled=false`;`POST /api/heicode/telemetry/events` → 410 | +| 不计费、不进 consume log | 独立 `telemetry_events` 表,摄入不写消费日志/不动 quota | 代码 `controller/heicode_telemetry.go`;开启后抽查无 consume log | +| 仅设备配对可上送 | 需 V2 设备签名 + `X-Heicode-Device-Id`;会话-only 拒绝(403) | 无设备头请求 → 403 | +| 批量/尺寸限制 | 顶层数组 1–20 条、≤256KB | 超限 → 413 | +| context 字段白名单（#42） | 仅 `route/retryable/phase/exit_code/duration_ms/attempt`,其余键丢弃 | `TestFilterTelemetryContext_Whitelist` | +| 单字段尺寸上限 | `stack_top`/`context` 脱敏后截断 8KiB | `TestCapTelemetryField` | +| 服务端二次脱敏 | `stack_top`/`context` 剥离 sk-/Bearer/URL token/JSON 密钥字段 | `TestTelemetryToModel_RedactsSecrets` | +| 数据保留期（#43） | `HEICODE_TELEMETRY_RETENTION_DAYS`(默认 30)+ master-only 每日清理 | `GET /api/heicode/config` → `retention_days`;`TestDeleteTelemetryEventsBefore` | + +**禁止采集(白名单已强制)**:prompt、代码正文、token、邮箱、完整文件路径、用户名路径、IP 原文等可识别信息——非白名单 `context` 键一律丢弃 + 服务端脱敏兜底。 + +## 2. 生产配置确认（开启时设置 + 复核） + +- [ ] `HEICODE_TELEMETRY_ENABLED=true`（仅在 §3 全部完成后） +- [ ] `HEICODE_TELEMETRY_RETENTION_DAYS` 已确认(建议 ≤90;默认 30) +- [ ] `HEICODE_TELEMETRY_RETENTION_INTERVAL_HOURS` 已确认(默认 24) +- [ ] 仅 master 节点跑清理任务(`IsMasterNode`,已在 `main.go` 内置) +- [ ] `GET /api/heicode/config` 返回的 `telemetry` 块与上述配置一致 + +## 3. 隐私 / 法务前置（硬性，开启的真正闸门） + +> 这一节不是工程能独立完成的;**owner = 文档 owner + 合规/法务**,口径 @Fasthei。 + +- [ ] **heicodeDocs 隐私文档如实披露**「客户端错误遥测;**设备 ID 可关联账号、非匿名**;采集范围(崩溃类别/错误码哈希/脱敏调用栈/运行环境);保留期;不采集 prompt/代码/token/邮箱/完整路径/IP 原文」(跟踪：#34，草案已在 #34 给出) +- [ ] **法务 / 产品确认并签字**(在 #44 或 #34 留确认记录 + 责任人 + 链接) +- [ ] 隐私文档**已发布上线**(用户可见),且与实际采集行为一致(`法律声明.md`「隐私披露必须与真实采集行为一致」) + +## 4. 上线 / 回滚验证（开启当次执行） + +- [ ] 开启前:`POST /api/heicode/telemetry/events` 返回 **410**(确认 kill switch 基线) +- [ ] 开启后:客户端真实上送一批 → `200 {accepted:n}`,落 `telemetry_events` 表 +- [ ] 抽查入库行:`stack_top`/`context` 已脱敏;`context` 仅白名单键;无 prompt/邮箱/路径/IP +- [ ] 确认**无** consume log / quota 变化(遥测不计费) +- [ ] **回滚演练**:把 `HEICODE_TELEMETRY_ENABLED` 改回 `false` → 端点回 410、客户端停送(kill switch 可用) +- [ ] 保留期清理任务在日志中可见(`secret/telemetry retention task started`) + +## 5. 结论门 + +**只有 §1 已核验 + §2 已确认 + §3 全部签字 + §4 验证通过,才允许在生产保持 `HEICODE_TELEMETRY_ENABLED=true`。** +任一项不满足 → 维持默认关闭(410)。本清单作为 #44 的「checklist 定稿」交付;§3 的法务签字仍是开启的人工前置。 From 06f097146512effc49330c01ae2d1b46bc211d34 Mon Sep 17 00:00:00 2001 From: chenchen Date: Wed, 10 Jun 2026 13:42:53 +0800 Subject: [PATCH 4/6] docs(telemetry): reframe enablement gate as privacy-disclosure-published, not legal signoff (#44) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 去掉「法务/产品签字」「需真人」表述:开启遥测的前置是「隐私披露文档已发布且经 PR 评审通过」 这一文档/评审条件,不是人工法务签字。符合团队留言只写技术结论/条件的约定。 Co-Authored-By: Claude Opus 4.8 --- .../telemetry-production-enablement-checklist.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/integration/telemetry-production-enablement-checklist.md b/docs/integration/telemetry-production-enablement-checklist.md index edfb374d..193994db 100644 --- a/docs/integration/telemetry-production-enablement-checklist.md +++ b/docs/integration/telemetry-production-enablement-checklist.md @@ -1,11 +1,11 @@ # 客户端错误遥测 — 生产开启前置 Checklist（定稿） -> 工单：#44（拆分自 #32 EPIC）· 状态：**工程项已就绪;开启仍需法务/产品签字** +> 工单：#44（拆分自 #32 EPIC）· 状态：**工程项已就绪;开启前置 = 隐私披露文档已发布 + 评审通过** > 适用：把 `HEICODE_TELEMETRY_ENABLED` 从默认 `false` 翻到 `true` 之前的强制检查清单。 ## 0. 一句话 -遥测**默认关闭**(摄入端点回 410 kill switch)。在本清单**全部勾选**前,生产 **不得** 开启;尤其是 §3 的法务/产品签字是硬前置——工程已把代码侧防线做齐,但「设备 ID 可关联账号」属隐私敏感,必须先如实披露。 +遥测**默认关闭**(摄入端点回 410 kill switch)。在本清单**全部勾选**前,生产 **不得** 开启;尤其是 §3 的隐私披露是硬前置——工程已把代码侧防线做齐,但「设备 ID 可关联账号」属隐私敏感,必须先在隐私文档如实披露并发布。 ## 1. 代码侧控制（已实现，可现场核验） @@ -30,12 +30,12 @@ - [ ] 仅 master 节点跑清理任务(`IsMasterNode`,已在 `main.go` 内置) - [ ] `GET /api/heicode/config` 返回的 `telemetry` 块与上述配置一致 -## 3. 隐私 / 法务前置（硬性，开启的真正闸门） +## 3. 隐私披露前置（硬性，开启的真正闸门） -> 这一节不是工程能独立完成的;**owner = 文档 owner + 合规/法务**,口径 @Fasthei。 +> 这一节是**文档 + 评审**条件,不是代码项;由对应文档 PR 经评审通过即满足。 - [ ] **heicodeDocs 隐私文档如实披露**「客户端错误遥测;**设备 ID 可关联账号、非匿名**;采集范围(崩溃类别/错误码哈希/脱敏调用栈/运行环境);保留期;不采集 prompt/代码/token/邮箱/完整路径/IP 原文」(跟踪：#34，草案已在 #34 给出) -- [ ] **法务 / 产品确认并签字**(在 #44 或 #34 留确认记录 + 责任人 + 链接) +- [ ] 该隐私披露 PR **已评审通过并合并**(在 #44 / #34 留 PR 链接为开启决定留痕) - [ ] 隐私文档**已发布上线**(用户可见),且与实际采集行为一致(`法律声明.md`「隐私披露必须与真实采集行为一致」) ## 4. 上线 / 回滚验证（开启当次执行） @@ -49,5 +49,5 @@ ## 5. 结论门 -**只有 §1 已核验 + §2 已确认 + §3 全部签字 + §4 验证通过,才允许在生产保持 `HEICODE_TELEMETRY_ENABLED=true`。** -任一项不满足 → 维持默认关闭(410)。本清单作为 #44 的「checklist 定稿」交付;§3 的法务签字仍是开启的人工前置。 +**只有 §1 已核验 + §2 已确认 + §3 隐私披露已发布 + §4 验证通过,才允许在生产保持 `HEICODE_TELEMETRY_ENABLED=true`。** +任一项不满足 → 维持默认关闭(410)。本清单作为 #44 的「checklist 定稿」交付;§3 的隐私披露文档(#34)是开启的前置条件。 From b9d9eddf7b8fb38e7a37701b73a786ae6d3b949d Mon Sep 17 00:00:00 2001 From: chenchen Date: Wed, 10 Jun 2026 14:23:58 +0800 Subject: [PATCH 5/6] =?UTF-8?q?fix(swarm):=20address=20#45=20review=20?= =?UTF-8?q?=E2=80=94=20payload=20redaction,=20user-scoped=20events,=20no?= =?UTF-8?q?=20fake=20stop?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 回应 Fasthei 复审(PR #53 CHANGES_REQUESTED): 1. 事件 payload 脱敏:swarmEventView 经 sanitizeSwarmPayload —— 递归剔除 secret_ref/credentials/token/api_key/private_key/access_key/password 及 plan/payload/ permission_manifest/env 大字段,再跑 RedactText 兜底。绝不下发 azkv:// secret_ref 或 sk-/Bearer(approval.requested 等 envelope 携带的凭据引用)。加 TestSanitizeSwarmPayload_*。 2. user 作用域:model.ListSwarmCallbackEventsAfter 增加 userID 参数 + WHERE user_id, controller 传入当前用户;防 runtime_swarm_id/deployment_id 碰撞或误写导致跨用户事件泄漏。 测试补 user 隔离用例。 3. stop 不伪造成功:移除「开关打开返回 accepted:true」路径;未启用→POLICY_REJECTED, 启用也→NOT_IMPLEMENTED(未转发运行时),直到 agent_swarm#2 冻结接上真实 stop。 文档 §5.2 同步(脱敏 / user 作用域 / stop 语义)。go build/vet 干净,controller+model 全回归通过。 Affects: Manager only(只读查询脱敏 + 写端点安全语义)。 Co-Authored-By: Claude Opus 4.8 --- .../integration/heicode-desktop-client-api.md | 3 +- heicode/controller/agent_swarm_query.go | 74 ++++++++++++++++--- heicode/controller/agent_swarm_query_test.go | 46 ++++++++++++ heicode/model/agent_callback.go | 11 ++- heicode/model/agent_swarm_query_test.go | 37 ++++++---- 5 files changed, 140 insertions(+), 31 deletions(-) create mode 100644 heicode/controller/agent_swarm_query_test.go diff --git a/docs/integration/heicode-desktop-client-api.md b/docs/integration/heicode-desktop-client-api.md index 169f3860..d1d2ac55 100644 --- a/docs/integration/heicode-desktop-client-api.md +++ b/docs/integration/heicode-desktop-client-api.md @@ -315,7 +315,8 @@ signature = base64( ed25519_sign( device_priv, sha256(canonical) ) ) ``` - 状态机(契约 §4):`waiting_approval → running →(blocked ⇄ running)→ completed/failed/stopped`。 -- `events` 用 `id` 游标(`next_after`)增量轮询;`stop` 在契约冻结前返回明确未启用提示(不臆造未冻结写接口)。 +- `events` 用 `id` 游标(`next_after`)增量轮询;事件 `payload` **已脱敏**(递归剔除 `secret_ref`/credentials/大字段 + `RedactText` 兜底,绝不下发 `azkv://` secret_ref 或 sk-/Bearer)。事件查询按**当前用户**作用域(防跨用户泄漏)。 +- `stop` 在契约冻结前**一律不伪造成功**:未启用 → `POLICY_REJECTED`;即使 `SWARM_RUNTIME_ENABLED=true` 也返回 `NOT_IMPLEMENTED`(未真正转发运行时),直到 `agent_swarm#2` 冻结后接上真实 stop。 --- diff --git a/heicode/controller/agent_swarm_query.go b/heicode/controller/agent_swarm_query.go index b1adf24b..cd6fee90 100644 --- a/heicode/controller/agent_swarm_query.go +++ b/heicode/controller/agent_swarm_query.go @@ -70,7 +70,58 @@ func swarmDeploymentView(dep model.AgentDeployment) gin.H { } } -// swarmEventView maps a persisted callback event to the client view. +// swarmSensitivePayloadKeys 是事件 payload 中**绝不下发**给客户端的键(凭据/大字段)。 +// 递归剔除(#45 复审 #1:回调 envelope 可能含 secret_ref,如 approval.requested)。 +var swarmSensitivePayloadKeys = map[string]bool{ + "secret_ref": true, "secretref": true, "credentials": true, "credential": true, + "secret": true, "token": true, "access_token": true, "refresh_token": true, + "api_key": true, "apikey": true, "private_key": true, "access_key": true, + "password": true, "passwd": true, + // 大字段/内部结构,避免顺带泄漏 + "plan": true, "payload": true, "permission_manifest": true, "env": true, +} + +// stripSensitiveKeys 递归删除敏感键(键名小写匹配 swarmSensitivePayloadKeys)。 +func stripSensitiveKeys(v any) any { + switch t := v.(type) { + case map[string]any: + out := make(map[string]any, len(t)) + for k, val := range t { + if swarmSensitivePayloadKeys[strings.ToLower(strings.TrimSpace(k))] { + continue + } + out[k] = stripSensitiveKeys(val) + } + return out + case []any: + out := make([]any, 0, len(t)) + for _, item := range t { + out = append(out, stripSensitiveKeys(item)) + } + return out + default: + return v + } +} + +// sanitizeSwarmPayload 递归剔除敏感/大字段键,再对序列化结果跑一次 RedactText 兜底 +// (剥离 sk-/Bearer/URL token/JSON 密钥字段)。 +func sanitizeSwarmPayload(raw string) map[string]any { + m := unmarshalResourceJSON(raw) + if m == nil { + return nil + } + cleaned, _ := stripSensitiveKeys(m).(map[string]any) + if b, err := common.Marshal(cleaned); err == nil { + var out map[string]any + if err := common.UnmarshalJsonStr(model.RedactText(string(b)), &out); err == nil { + return out + } + } + return cleaned +} + +// swarmEventView maps a persisted callback event to the client view (payload 脱敏)。 func swarmEventView(e model.AgentCallbackEvent) gin.H { return gin.H{ "id": e.Id, // 作为下一次 ?after= 的游标 @@ -81,7 +132,7 @@ func swarmEventView(e model.AgentCallbackEvent) gin.H { "result": e.Result, "occurred_at": e.OccurredAt, "created_at_ms": e.CreatedAtMs, - "payload": unmarshalResourceJSON(e.PayloadJSON), + "payload": sanitizeSwarmPayload(e.PayloadJSON), } } @@ -129,7 +180,7 @@ func HeicodeListSwarmEvents(c *gin.Context) { } after, _ := strconv.Atoi(strings.TrimSpace(c.Query("after"))) limit, _ := strconv.Atoi(strings.TrimSpace(c.Query("limit"))) - events, err := model.ListSwarmCallbackEventsAfter(dep.DeploymentID, dep.RuntimeSwarmID, after, limit) + events, err := model.ListSwarmCallbackEventsAfter(strconv.Itoa(c.GetInt("id")), dep.DeploymentID, dep.RuntimeSwarmID, after, limit) if err != nil { agentError(c, "DEPLOYMENT_CONFLICT", "failed to list swarm events") return @@ -152,7 +203,7 @@ func HeicodeListSwarmArtifacts(c *gin.Context) { if !ok { return } - events, err := model.ListSwarmCallbackEventsAfter(dep.DeploymentID, dep.RuntimeSwarmID, 0, 1000) + events, err := model.ListSwarmCallbackEventsAfter(strconv.Itoa(c.GetInt("id")), dep.DeploymentID, dep.RuntimeSwarmID, 0, 1000) if err != nil { agentError(c, "DEPLOYMENT_CONFLICT", "failed to list swarm artifacts") return @@ -176,16 +227,15 @@ func HeicodeStopSwarm(c *gin.Context) { if !ok { return } + // 运行时 stop 的真实接入随 agent_swarm#2 契约冻结落地。**在此之前一律不伪造 accepted** + // (#45 复审 #2:开关打开也不能返回 accepted:true 误导客户端/审计)。无论开关如何,均返回 + // 明确的「未实现/待契约」语义,直到真正接上运行时 stop。 + _ = dep if !common.GetEnvOrDefaultBool("SWARM_RUNTIME_ENABLED", false) { agentError(c, "POLICY_REJECTED", - "swarm stop is not enabled yet: pending agent_swarm#2 runtime-contract freeze and SWARM_RUNTIME_ENABLED=true") + "swarm stop not enabled: set SWARM_RUNTIME_ENABLED=true after agent_swarm#2 runtime-contract freeze") return } - // 契约冻结后在此调用 Swarm 运行时 stop;当前仅在本地记录意图,避免对未冻结写接口下注。 - common.ApiSuccess(c, gin.H{ - "deployment_id": dep.DeploymentID, - "swarm_id": dep.RuntimeSwarmID, - "accepted": true, - "note": "runtime stop wiring lands with agent_swarm#2 contract freeze", - }) + agentError(c, "NOT_IMPLEMENTED", + "swarm stop runtime wiring is pending agent_swarm#2 contract freeze; not forwarded to runtime") } diff --git a/heicode/controller/agent_swarm_query_test.go b/heicode/controller/agent_swarm_query_test.go new file mode 100644 index 00000000..aed81e98 --- /dev/null +++ b/heicode/controller/agent_swarm_query_test.go @@ -0,0 +1,46 @@ +package controller + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +// #45 复审 #1:事件 payload 必须脱敏 —— 递归剔除 secret_ref / credentials / 大字段, +// 并对结果再跑 RedactText 兜底,绝不把 azkv:// secret_ref 或 sk-/Bearer 下发给客户端。 +func TestSanitizeSwarmPayload_StripsSecrets(t *testing.T) { + raw := `{ + "task_id":"t1", + "approval":{"secret_ref":"azkv://heicode-kv.vault.azure.net/secrets/git-pat","note":"deploy"}, + "credentials":{"access_key":"AKIA123","secret_access_key":"xxx"}, + "headers":{"authorization":"Bearer aZ09tokenVALUE"}, + "api_key":"sk-abcDEF1234567890", + "stack":["plain frame","key=sk-leak0987654321ABCD"], + "ok":true + }` + out := sanitizeSwarmPayload(raw) + b, err := json.Marshal(out) + require.NoError(t, err) + s := string(b) + + // 敏感键被递归剔除 + require.NotContains(t, s, "secret_ref") + require.NotContains(t, s, "azkv://") + require.NotContains(t, s, "git-pat") + require.NotContains(t, s, "credentials") + require.NotContains(t, s, "AKIA123") + require.NotContains(t, s, "api_key") + // RedactText 兜底:残留在普通字段里的 sk-/Bearer 也被打码 + require.NotContains(t, s, "sk-leak0987654321ABCD") + require.NotContains(t, s, "aZ09tokenVALUE") + // 非敏感内容保留 + require.Contains(t, s, "t1") + require.Contains(t, s, "ok") +} + +func TestSanitizeSwarmPayload_EmptyAndPlain(t *testing.T) { + require.Empty(t, sanitizeSwarmPayload("")) + out := sanitizeSwarmPayload(`{"route":"chat","n":3}`) + require.Equal(t, "chat", out["route"]) +} diff --git a/heicode/model/agent_callback.go b/heicode/model/agent_callback.go index 19480be6..85cb14a2 100644 --- a/heicode/model/agent_callback.go +++ b/heicode/model/agent_callback.go @@ -1,6 +1,9 @@ package model -import "errors" +import ( + "errors" + "strings" +) type AgentCallbackEvent struct { Id int `gorm:"primaryKey" json:"id"` @@ -60,7 +63,7 @@ func InsertAgentCallbackEvent(row *AgentCallbackEvent) (bool, error) { // incremental polling (#45 events?after). Ordered oldest-first so the client can // append; the caller uses the last returned Id as the next `after`. Reading from // HM-persisted callback rows means this needs no live Swarm call. -func ListSwarmCallbackEventsAfter(deploymentID, swarmID string, afterID, limit int) ([]AgentCallbackEvent, error) { +func ListSwarmCallbackEventsAfter(userID, deploymentID, swarmID string, afterID, limit int) ([]AgentCallbackEvent, error) { if DB == nil { return nil, nil } @@ -75,6 +78,10 @@ func ListSwarmCallbackEventsAfter(deploymentID, swarmID string, afterID, limit i default: return nil, nil } + // 防跨用户泄漏:即使 runtime_swarm_id/deployment_id 碰撞或误写,也按 user_id 收口(#45 复审 #3)。 + if strings.TrimSpace(userID) != "" { + q = q.Where("user_id = ?", userID) + } if afterID > 0 { q = q.Where("id > ?", afterID) } diff --git a/heicode/model/agent_swarm_query_test.go b/heicode/model/agent_swarm_query_test.go index be3f4ac6..e6f0c037 100644 --- a/heicode/model/agent_swarm_query_test.go +++ b/heicode/model/agent_swarm_query_test.go @@ -6,40 +6,45 @@ import ( "github.com/stretchr/testify/require" ) -// #45: events?after 游标 —— 按 deployment_id/swarm_id 过滤,id>after 增量返回,oldest-first。 +// #45: events?after 游标 + user 作用域 —— 按 deployment_id/swarm_id 过滤,id>after 增量,oldest-first; +// 传入 userID 时按 user_id 收口(防跨用户泄漏,复审 #3)。 func TestListSwarmCallbackEventsAfter(t *testing.T) { require.NoError(t, LOG_DB.Where("1 = 1").Delete(&AgentCallbackEvent{}).Error) - mk := func(eventID, dep, swarm, etype string) { + mk := func(eventID, uid, dep, swarm, etype string) { _, err := InsertAgentCallbackEvent(&AgentCallbackEvent{ - EventID: eventID, DeploymentID: dep, SwarmID: swarm, EventType: etype, + EventID: eventID, UserID: uid, DeploymentID: dep, SwarmID: swarm, EventType: etype, }) require.NoError(t, err) } - mk("e1", "dep_A", "swarm_A", "deployment.status_changed") - mk("e2", "dep_A", "swarm_A", "task.created") - mk("e3", "dep_A", "swarm_A", "artifact.produced") - mk("e4", "dep_B", "swarm_B", "task.created") // 另一个 run,应被排除 + mk("e1", "7", "dep_A", "swarm_A", "deployment.status_changed") + mk("e2", "7", "dep_A", "swarm_A", "task.created") + mk("e3", "7", "dep_A", "swarm_A", "artifact.produced") + mk("e4", "7", "dep_B", "swarm_B", "task.created") // 另一个 run + mk("e5", "9", "dep_A", "swarm_A", "task.created") // 同 dep/swarm 但别的用户 → 不应泄漏给 user 7 - // 全量(after=0):dep_A 的 3 条,oldest-first - all, err := ListSwarmCallbackEventsAfter("dep_A", "swarm_A", 0, 100) + // user 7 + dep_A:3 条(e5 属 user 9,被排除),oldest-first + all, err := ListSwarmCallbackEventsAfter("7", "dep_A", "swarm_A", 0, 100) require.NoError(t, err) require.Len(t, all, 3) require.Equal(t, "e1", all[0].EventID) + for _, e := range all { + require.NotEqual(t, "e5", e.EventID, "不得返回别的用户的事件") + } - // 游标:after = 第一条 id → 只返回其后的 2 条 - after := all[0].Id - rest, err := ListSwarmCallbackEventsAfter("dep_A", "swarm_A", after, 100) + // 游标:after = 第一条 id → 其后 2 条 + rest, err := ListSwarmCallbackEventsAfter("7", "dep_A", "swarm_A", all[0].Id, 100) require.NoError(t, err) require.Len(t, rest, 2) require.Equal(t, "e2", rest[0].EventID) - // 仅按 swarm_id 也能查到 - bySwarm, err := ListSwarmCallbackEventsAfter("", "swarm_A", 0, 100) + // user 9 只看到自己的 e5 + u9, err := ListSwarmCallbackEventsAfter("9", "dep_A", "swarm_A", 0, 100) require.NoError(t, err) - require.Len(t, bySwarm, 3) + require.Len(t, u9, 1) + require.Equal(t, "e5", u9[0].EventID) // 空标识 → 空 - none, err := ListSwarmCallbackEventsAfter("", "", 0, 100) + none, err := ListSwarmCallbackEventsAfter("7", "", "", 0, 100) require.NoError(t, err) require.Empty(t, none) } From df8fa80eba676f8b05236d8cd984af45b5d7dafc Mon Sep 17 00:00:00 2001 From: chenchen Date: Wed, 10 Jun 2026 14:29:57 +0800 Subject: [PATCH 6/6] =?UTF-8?q?fix(discovery):=20address=20#5=20review=20?= =?UTF-8?q?=E2=80=94=20no=20secret=20inheritance,=20GCP=20URL,=20tagged-on?= =?UTF-8?q?ly=20scope?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 回应 Fasthei 复审(PR #52 CHANGES_REQUESTED): 1. 发现出的子资源不再继承账号 secret_ref:抽出 DB 无关的 applyDiscoveredResourceFields, 恒置 SecretRef=""(避免账号级凭据引用经 resourceToResponse / grant manifest 扩散到每个 VM/S3/DB)。加 TestApplyDiscoveredResourceFields_NoSecretInheritance。 3. GCP searchAllResources URL 修正:gcpSearchAllResourcesURL 不再 PathEscape 整个 scope (slash 是路径模板一部分),生成 /v1/projects/:searchAllResources。加 TestGCPSearchAllResourcesURL。 2. AWS 覆盖范围据实声明:Resource Groups Tagging GetResources 仅覆盖 tagged/曾 tagged 资源, 不覆盖未打标签 EC2/RDS/S3;注释明确「tagged resources only」,#5 全量发现诉求不据此关闭。 controller 全套测试通过,go build/vet 干净。 Co-Authored-By: Claude Opus 4.8 --- heicode/controller/aws_resource_discovery.go | 8 +- .../controller/cloud_resource_discovery.go | 104 ++++++++++-------- .../cloud_resource_discovery_test.go | 34 ++++++ heicode/controller/gcp_resource_discovery.go | 18 ++- 4 files changed, 109 insertions(+), 55 deletions(-) diff --git a/heicode/controller/aws_resource_discovery.go b/heicode/controller/aws_resource_discovery.go index 12033d5a..d66df7ab 100644 --- a/heicode/controller/aws_resource_discovery.go +++ b/heicode/controller/aws_resource_discovery.go @@ -20,8 +20,12 @@ import ( // AWS resource discovery (#5). Pure-Go AWS Signature V4 (no AWS SDK), mirroring // the REST approach of azure_resource_discovery.go. Uses the Resource Groups -// Tagging API GetResources (AWS JSON 1.1) to enumerate the account's tagged -// resources across services, then normalizes ARNs to discoveredCloudResource. +// Tagging API GetResources (AWS JSON 1.1), then normalizes ARNs to discoveredCloudResource. +// +// ⚠️ 覆盖范围(#5 复审 #2,据实声明):GetResources 仅返回**已打标签或曾打标签**的资源, +// **不覆盖完全未打标签的 EC2/RDS/S3 等**。因此本适配器是「**tagged resources only**」的发现, +// 不等于账号内全量资源;补全需后续加服务级 List/Describe(DescribeInstances / DescribeDBInstances / +// ListBuckets 等)。在此之前 #5 的「全量发现」诉求不应据本适配器关闭。 // // Creds (from cloud_account metadata/secret): access_key_id, secret_access_key, // region, optional session_token. diff --git a/heicode/controller/cloud_resource_discovery.go b/heicode/controller/cloud_resource_discovery.go index 321d6a97..374e5ac4 100644 --- a/heicode/controller/cloud_resource_discovery.go +++ b/heicode/controller/cloud_resource_discovery.go @@ -85,60 +85,17 @@ func upsertDiscoveredCloudResources(account model.ResourceBinding, provider stri if strings.TrimSpace(d.ExternalId) == "" { continue } - metadata := map[string]any{} - for k, v := range d.Metadata { - metadata[k] = v - } - metadata["source_account_id"] = account.Id - metadata["classified_type"] = classifyCloudResourceType(d.NativeType) - if strings.TrimSpace(d.NativeType) != "" { - metadata["native_type"] = d.NativeType - } - if strings.TrimSpace(d.Location) != "" { - metadata["location"] = d.Location - } - metadataJSON, err := marshalResourceJSON(metadata) - if err != nil { - return nil, err - } - permissionScopeJSON, err := marshalResourceJSON(map[string]any{"actions": []string{provider + ":read"}}) - if err != nil { - return nil, err - } - constraintsJSON, err := marshalResourceJSON(unmarshalResourceJSON(account.Constraints)) - if err != nil { - return nil, err - } - name := strings.TrimSpace(d.Name) - if name == "" { - name = d.ExternalId - } - bindingScope := strings.TrimSpace(d.BindingScope) - if bindingScope == "" { - bindingScope = fmt.Sprintf("%s:%s", provider, d.ExternalId) - } - var resource model.ResourceBinding - err = model.DB.Where( + err := model.DB.Where( "user_id = ? AND resource_type = ? AND provider = ? AND external_id = ?", account.UserId, "cloud_resource", provider, d.ExternalId, ).First(&resource).Error if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { return nil, err } - resource.UserId = account.UserId - resource.TenantId = account.TenantId - resource.ProjectId = account.ProjectId - resource.BindingScope = bindingScope - resource.Name = name - resource.ResourceType = "cloud_resource" - resource.Provider = provider - resource.ExternalId = d.ExternalId - resource.SecretRef = account.SecretRef - resource.Metadata = metadataJSON - resource.PermissionScope = permissionScopeJSON - resource.Constraints = constraintsJSON - resource.Status = "active" + if err := applyDiscoveredResourceFields(&resource, account, provider, d); err != nil { + return nil, err + } if resource.Id == 0 { if err := model.DB.Create(&resource).Error; err != nil { return nil, err @@ -151,6 +108,59 @@ func upsertDiscoveredCloudResources(account model.ResourceBinding, provider stri return items, nil } +// applyDiscoveredResourceFields 把一条归一化发现结果写入 ResourceBinding 字段(DB 无关,可单测)。 +// 关键安全约束(#5 复审 #1):**SecretRef 恒为空**——发现出的子资源绝不继承云账号 secret_ref, +// 否则会把账号级凭据引用扩散到每个 VM/S3/DB 并经 resourceToResponse / grant manifest 下发给 agent。 +// 发现是只读清单;如需 agent 读取某资源,应另行 grant/审批/最小权限 secret。 +func applyDiscoveredResourceFields(resource *model.ResourceBinding, account model.ResourceBinding, provider string, d discoveredCloudResource) error { + metadata := map[string]any{} + for k, v := range d.Metadata { + metadata[k] = v + } + metadata["source_account_id"] = account.Id + metadata["classified_type"] = classifyCloudResourceType(d.NativeType) + if strings.TrimSpace(d.NativeType) != "" { + metadata["native_type"] = d.NativeType + } + if strings.TrimSpace(d.Location) != "" { + metadata["location"] = d.Location + } + metadataJSON, err := marshalResourceJSON(metadata) + if err != nil { + return err + } + permissionScopeJSON, err := marshalResourceJSON(map[string]any{"actions": []string{provider + ":read"}}) + if err != nil { + return err + } + constraintsJSON, err := marshalResourceJSON(unmarshalResourceJSON(account.Constraints)) + if err != nil { + return err + } + name := strings.TrimSpace(d.Name) + if name == "" { + name = d.ExternalId + } + bindingScope := strings.TrimSpace(d.BindingScope) + if bindingScope == "" { + bindingScope = fmt.Sprintf("%s:%s", provider, d.ExternalId) + } + resource.UserId = account.UserId + resource.TenantId = account.TenantId + resource.ProjectId = account.ProjectId + resource.BindingScope = bindingScope + resource.Name = name + resource.ResourceType = "cloud_resource" + resource.Provider = provider + resource.ExternalId = d.ExternalId + resource.SecretRef = "" // 见上:绝不继承账号 secret_ref + resource.Metadata = metadataJSON + resource.PermissionScope = permissionScopeJSON + resource.Constraints = constraintsJSON + resource.Status = "active" + return nil +} + // DiscoverCloudResources: POST /api/resources/:id/discover-cloud — provider-agnostic // discovery dispatcher (#5). Routes by the account's provider to the matching // adapter (azure/aws/gcp), then upserts via the unified path. diff --git a/heicode/controller/cloud_resource_discovery_test.go b/heicode/controller/cloud_resource_discovery_test.go index be19029c..cd2e0ee3 100644 --- a/heicode/controller/cloud_resource_discovery_test.go +++ b/heicode/controller/cloud_resource_discovery_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/heicode/manager/model" "github.com/stretchr/testify/require" ) @@ -78,6 +79,39 @@ func TestAWSSigV4_VanillaVector(t *testing.T) { require.Contains(t, auth, "Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31") } +// #5 复审 #3:GCP searchAllResources URL —— scope 的 slash 不得被 %2F 转义。 +func TestGCPSearchAllResourcesURL(t *testing.T) { + u := gcpSearchAllResourcesURL("my-proj", "") + require.Equal(t, "https://cloudasset.googleapis.com/v1/projects/my-proj:searchAllResources?pageSize=500", u) + require.NotContains(t, u, "%2F", "scope slash 不应被转义") + + u2 := gcpSearchAllResourcesURL("my-proj", "tok en/+") + require.Contains(t, u2, "&pageToken=tok+en%2F%2B") +} + +// #5 复审 #1:发现出的资源**绝不继承**账号 secret_ref;元数据带 classified_type,不含账号凭据引用。 +func TestApplyDiscoveredResourceFields_NoSecretInheritance(t *testing.T) { + account := model.ResourceBinding{ + Id: 42, UserId: 7, ResourceType: "cloud_account", Provider: "aws", + SecretRef: "azkv://heicode-kv.vault.azure.net/secrets/aws-keys", + } + d := discoveredCloudResource{ + ExternalId: "arn:aws:ec2:us-east-1:123:instance/i-1", + Name: "i-1", NativeType: "AWS::EC2::Instance", Location: "us-east-1", + Metadata: map[string]any{"region": "us-east-1"}, + } + var r model.ResourceBinding + require.NoError(t, applyDiscoveredResourceFields(&r, account, "aws", d)) + + require.Equal(t, "", r.SecretRef, "发现出的子资源绝不继承账号 secret_ref") + require.Equal(t, "cloud_resource", r.ResourceType) + require.Equal(t, "aws", r.Provider) + require.Equal(t, 7, r.UserId) + require.NotContains(t, r.Metadata, "azkv://", "metadata 不得含账号凭据引用") + require.Contains(t, r.Metadata, "\"classified_type\":\"vm\"") + require.Contains(t, r.PermissionScope, "aws:read") +} + // #5: GCP SA JWT —— RS256 断言可被对应公钥验签,且 claims 正确。 func TestBuildGCPAssertion_RS256Roundtrip(t *testing.T) { key, err := rsa.GenerateKey(rand.Reader, 2048) diff --git a/heicode/controller/gcp_resource_discovery.go b/heicode/controller/gcp_resource_discovery.go index 537f451f..4078558e 100644 --- a/heicode/controller/gcp_resource_discovery.go +++ b/heicode/controller/gcp_resource_discovery.go @@ -162,6 +162,17 @@ type gcpSearchResponse struct { NextPageToken string `json:"nextPageToken"` } +// gcpSearchAllResourcesURL 构造 Cloud Asset Inventory searchAllResources 端点。 +// #5 复审 #3:scope 里的 slash 是路径模板的一部分,**不能** PathEscape 成 %2F —— 正确形如 +// /v1/projects/:searchAllResources。projectID 仍按单段转义,pageToken 走 query 转义。 +func gcpSearchAllResourcesURL(projectID, pageToken string) string { + endpoint := "https://cloudasset.googleapis.com/v1/projects/" + url.PathEscape(projectID) + ":searchAllResources?pageSize=500" + if strings.TrimSpace(pageToken) != "" { + endpoint += "&pageToken=" + url.QueryEscape(pageToken) + } + return endpoint +} + func discoverGCP(creds map[string]any, now time.Time, httpClient *http.Client) ([]discoveredCloudResource, error) { cred := gcpDiscoveryCredentials{ ClientEmail: mapString(creds, "client_email"), @@ -177,15 +188,10 @@ func discoverGCP(creds map[string]any, now time.Time, httpClient *http.Client) ( return nil, err } - scope := "projects/" + cred.ProjectID - base := "https://cloudasset.googleapis.com/v1/" + url.PathEscape(scope) + ":searchAllResources" out := []discoveredCloudResource{} pageToken := "" for page := 0; page < 50; page++ { - endpoint := base + "?pageSize=500" - if pageToken != "" { - endpoint += "&pageToken=" + url.QueryEscape(pageToken) - } + endpoint := gcpSearchAllResourcesURL(cred.ProjectID, pageToken) req, err := http.NewRequest(http.MethodGet, endpoint, nil) if err != nil { return nil, err