From 3fb07ad5fb968ddecb22e0bc880cbd235c64112c Mon Sep 17 00:00:00 2001 From: chenchen Date: Wed, 10 Jun 2026 13:07:48 +0800 Subject: [PATCH] =?UTF-8?q?feat(discovery):=20provider-agnostic=20cloud=20?= =?UTF-8?q?discovery=20=E2=80=94=20AWS=20+=20GCP=20adapters=20(#5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #5 三项 DoD 全部落地(纯 Go,无云 SDK,REST + std crypto,非 mock): - 抽象 provider 接口(DoD #3):cloudDiscoveryProvider{ name, discover } + 统一 upsertDiscoveredCloudResources + 跨云一致分类 classifyCloudResourceType(vm/ database/blob/cloud_resource)。新增 dispatcher DiscoverCloudResources 按 account.Provider 路由 azure/aws/gcp。Azure 适配器复用既有 ARM 客户端,三家同抽象。 - AWS adapter(DoD #1):纯 Go AWS Signature V4(crypto/hmac+sha256)+ Resource Groups Tagging API GetResources(分页)→ ARN 归一化。SigV4 实现以 AWS 官方 "get-vanilla" 测试向量验证签名正确(TestAWSSigV4_VanillaVector)。 - GCP adapter(DoD #2):服务账号 RS256 JWT 断言 → OAuth2 token → Cloud Asset Inventory searchAllResources(分页)→ asset 归一化。JWT 以生成密钥 RS256 验签 roundtrip 验证(TestBuildGCPAssertion_RS256Roundtrip)。 路由:新增 POST /api/resources/:id/discover-cloud(按 provider 路由);保留既有 /:id/azure/discover 向后兼容。落库与 Azure 一致(resource_type=cloud_resource, 跨云类别入 metadata.classified_type),幂等 upsert(user+type+provider+external_id)。 测试:分类 / ARN 解析 / SigV4 官方向量 / GCP RS256 roundtrip;controller 全套通过, go build/vet 干净。 Affects: Manager only(资源发现,新增只读发现端点 + 抽象层)。无计费/审计/部署改动; 凭据经既有 secret store 读取,不落明文。 Closes #5 Co-Authored-By: Claude Opus 4.8 --- heicode/controller/aws_resource_discovery.go | 298 ++++++++++++++++++ .../controller/cloud_resource_discovery.go | 259 +++++++++++++++ .../cloud_resource_discovery_test.go | 116 +++++++ heicode/controller/gcp_resource_discovery.go | 230 ++++++++++++++ heicode/router/api-router.go | 3 + 5 files changed, 906 insertions(+) create mode 100644 heicode/controller/aws_resource_discovery.go create mode 100644 heicode/controller/cloud_resource_discovery.go create mode 100644 heicode/controller/cloud_resource_discovery_test.go create mode 100644 heicode/controller/gcp_resource_discovery.go diff --git a/heicode/controller/aws_resource_discovery.go b/heicode/controller/aws_resource_discovery.go new file mode 100644 index 00000000..12033d5a --- /dev/null +++ b/heicode/controller/aws_resource_discovery.go @@ -0,0 +1,298 @@ +package controller + +import ( + "bytes" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + "time" + + "github.com/heicode/manager/common" + "github.com/heicode/manager/model" +) + +// AWS resource discovery (#5). Pure-Go AWS Signature V4 (no AWS SDK), mirroring +// the REST approach of azure_resource_discovery.go. Uses the Resource Groups +// Tagging API GetResources (AWS JSON 1.1) to enumerate the account's tagged +// resources across services, then normalizes ARNs to discoveredCloudResource. +// +// Creds (from cloud_account metadata/secret): access_key_id, secret_access_key, +// region, optional session_token. + +type awsCloudDiscoveryProvider struct{} + +func (awsCloudDiscoveryProvider) name() string { return "aws" } + +type awsDiscoveryCredentials struct { + AccessKeyID string + SecretAccessKey string + SessionToken string + Region string +} + +func (c awsDiscoveryCredentials) validate() error { + if strings.TrimSpace(c.AccessKeyID) == "" { + return errors.New("AWS access_key_id required") + } + if strings.TrimSpace(c.SecretAccessKey) == "" { + return errors.New("AWS secret_access_key required") + } + if strings.TrimSpace(c.Region) == "" { + return errors.New("AWS region required") + } + return nil +} + +func (awsCloudDiscoveryProvider) discover(account model.ResourceBinding, creds map[string]any) ([]discoveredCloudResource, error) { + // region may live on the binding's external_id when not in creds. + if mapString(creds, "region") == "" && strings.TrimSpace(account.ExternalId) != "" { + creds["region"] = account.ExternalId + } + return discoverAWS(creds, time.Now(), &http.Client{Timeout: 20 * time.Second}) +} + +// hmacSHA256 / sha256Hex — SigV4 primitives. +func hmacSHA256(key, data []byte) []byte { + h := hmac.New(sha256.New, key) + h.Write(data) + return h.Sum(nil) +} + +func sha256Hex(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +// awsSigV4Authorization computes the SigV4 Authorization header value + x-amz-date +// for the given request. signHeaders are the headers (besides host/x-amz-date) to +// include in the signature (e.g. content-type, x-amz-target). Pure & deterministic +// given t — verified against AWS's official "get-vanilla" test vector. +func awsSigV4Authorization(method, rawURL string, signHeaders map[string]string, payload []byte, + cred awsDiscoveryCredentials, service string, t time.Time) (authorization, amzDate string, err error) { + + u, err := url.Parse(rawURL) + if err != nil { + return "", "", err + } + amzDate = t.UTC().Format("20060102T150405Z") + dateStamp := t.UTC().Format("20060102") + + // Canonical headers must include host + x-amz-date. Lowercase names, trim values, sort. + headers := map[string]string{} + for k, v := range signHeaders { + headers[strings.ToLower(strings.TrimSpace(k))] = strings.TrimSpace(v) + } + headers["host"] = u.Host + headers["x-amz-date"] = amzDate + if strings.TrimSpace(cred.SessionToken) != "" { + headers["x-amz-security-token"] = strings.TrimSpace(cred.SessionToken) + } + names := make([]string, 0, len(headers)) + for k := range headers { + names = append(names, k) + } + sort.Strings(names) + var canonicalHeaders strings.Builder + for _, n := range names { + canonicalHeaders.WriteString(n + ":" + headers[n] + "\n") + } + signedHeaders := strings.Join(names, ";") + + canonicalURI := u.EscapedPath() + if canonicalURI == "" { + canonicalURI = "/" + } + // Canonical query string: sort by key, RFC3986-encoded. + canonicalQuery := canonicalizeQuery(u.Query()) + + payloadHash := sha256Hex(payload) + canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuery + "\n" + + canonicalHeaders.String() + "\n" + signedHeaders + "\n" + payloadHash + + credentialScope := dateStamp + "/" + cred.Region + "/" + service + "/aws4_request" + stringToSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + credentialScope + "\n" + sha256Hex([]byte(canonicalRequest)) + + kDate := hmacSHA256([]byte("AWS4"+cred.SecretAccessKey), []byte(dateStamp)) + kRegion := hmacSHA256(kDate, []byte(cred.Region)) + kService := hmacSHA256(kRegion, []byte(service)) + kSigning := hmacSHA256(kService, []byte("aws4_request")) + signature := hex.EncodeToString(hmacSHA256(kSigning, []byte(stringToSign))) + + authorization = fmt.Sprintf("AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s", + cred.AccessKeyID, credentialScope, signedHeaders, signature) + return authorization, amzDate, nil +} + +func canonicalizeQuery(values url.Values) string { + if len(values) == 0 { + return "" + } + keys := make([]string, 0, len(values)) + for k := range values { + keys = append(keys, k) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, k := range keys { + vs := append([]string(nil), values[k]...) + sort.Strings(vs) + for _, v := range vs { + parts = append(parts, awsURIEncode(k, true)+"="+awsURIEncode(v, true)) + } + } + return strings.Join(parts, "&") +} + +// awsURIEncode is RFC3986 percent-encoding per SigV4 rules. +func awsURIEncode(s string, encodeSlash bool) string { + var b strings.Builder + for _, c := range []byte(s) { + switch { + case (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || + c == '-' || c == '_' || c == '.' || c == '~': + b.WriteByte(c) + case c == '/' && !encodeSlash: + b.WriteByte(c) + default: + b.WriteString(fmt.Sprintf("%%%02X", c)) + } + } + return b.String() +} + +// awsResourceTagMapping is one entry of the Resource Groups Tagging API response. +type awsResourceTagMapping struct { + ResourceARN string `json:"ResourceARN"` + Tags []struct { + Key string `json:"Key"` + Value string `json:"Value"` + } `json:"Tags"` +} + +type awsGetResourcesResponse struct { + ResourceTagMappingList []awsResourceTagMapping `json:"ResourceTagMappingList"` + PaginationToken string `json:"PaginationToken"` +} + +// discoverAWS enumerates the account's tagged resources via the Resource Groups +// Tagging API (paginated). Real implementation (no mock). +func discoverAWS(creds map[string]any, now time.Time, httpClient *http.Client) ([]discoveredCloudResource, error) { + cred := awsDiscoveryCredentials{ + AccessKeyID: mapString(creds, "access_key_id"), + SecretAccessKey: mapString(creds, "secret_access_key"), + SessionToken: mapString(creds, "session_token"), + Region: mapString(creds, "region"), + } + if err := cred.validate(); err != nil { + return nil, err + } + endpoint := fmt.Sprintf("https://tagging.%s.amazonaws.com/", cred.Region) + const target = "ResourceGroupsTaggingAPI_20170126.GetResources" + + out := []discoveredCloudResource{} + paginationToken := "" + for page := 0; page < 50; page++ { + bodyMap := map[string]any{"ResourcesPerPage": 100} + if paginationToken != "" { + bodyMap["PaginationToken"] = paginationToken + } + body, err := common.Marshal(bodyMap) + if err != nil { + return nil, err + } + signHeaders := map[string]string{ + "content-type": "application/x-amz-json-1.1", + "x-amz-target": target, + } + auth, amzDate, err := awsSigV4Authorization(http.MethodPost, endpoint, signHeaders, body, cred, "tagging", now) + if err != nil { + return nil, err + } + req, err := http.NewRequest(http.MethodPost, endpoint, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/x-amz-json-1.1") + req.Header.Set("X-Amz-Target", target) + req.Header.Set("X-Amz-Date", amzDate) + req.Header.Set("Authorization", auth) + if strings.TrimSpace(cred.SessionToken) != "" { + req.Header.Set("X-Amz-Security-Token", cred.SessionToken) + } + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("AWS GetResources request failed: %w", err) + } + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + resp.Body.Close() + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return nil, fmt.Errorf("AWS GetResources failed (%d): %s", resp.StatusCode, strings.TrimSpace(string(raw))) + } + var payload awsGetResourcesResponse + if err := common.Unmarshal(raw, &payload); err != nil { + return nil, err + } + for _, m := range payload.ResourceTagMappingList { + if d, ok := awsARNToResource(m.ResourceARN); ok { + tags := map[string]any{} + for _, t := range m.Tags { + tags[t.Key] = t.Value + } + d.Metadata = map[string]any{"region": cred.Region, "tags": tags} + out = append(out, d) + } + } + paginationToken = strings.TrimSpace(payload.PaginationToken) + if paginationToken == "" { + break + } + } + return out, nil +} + +// awsARNToResource parses arn:partition:service:region:account:resource into a +// normalized resource. NativeType is CloudFormation-style (AWS::::) +// so classifyCloudResourceType maps it to vm/database/blob consistently. +func awsARNToResource(arn string) (discoveredCloudResource, bool) { + arn = strings.TrimSpace(arn) + if !strings.HasPrefix(arn, "arn:") { + return discoveredCloudResource{}, false + } + parts := strings.SplitN(arn, ":", 6) + if len(parts) < 6 { + return discoveredCloudResource{}, false + } + service := parts[2] + region := parts[3] + resource := parts[5] + // resource may be "type/id", "type:id", or just "id" (e.g. s3 bucket). + resType, resName := "", resource + if i := strings.IndexAny(resource, "/:"); i >= 0 { + resType = resource[:i] + resName = resource[i+1:] + } + native := "AWS::" + strings.ToUpper(service) + if resType != "" { + native += "::" + resType + } else if strings.EqualFold(service, "s3") { + native += "::Bucket" + } + name := resName + if name == "" { + name = arn + } + return discoveredCloudResource{ + ExternalId: arn, + Name: name, + NativeType: native, + Location: region, + BindingScope: "aws:" + arn, + }, true +} diff --git a/heicode/controller/cloud_resource_discovery.go b/heicode/controller/cloud_resource_discovery.go new file mode 100644 index 00000000..321d6a97 --- /dev/null +++ b/heicode/controller/cloud_resource_discovery.go @@ -0,0 +1,259 @@ +package controller + +import ( + "errors" + "fmt" + "strings" + + "github.com/gin-gonic/gin" + "github.com/heicode/manager/common" + "github.com/heicode/manager/model" + "gorm.io/gorm" +) + +// Cloud resource discovery — provider-agnostic layer (#5). +// +// 历史上只有 Azure 发现（controller/azure_resource_discovery.go，硬要求 provider==azure）。 +// 本文件抽象出统一的 provider 接口，让 AWS / GCP 走同一条「发现 → 归一化 → 落库」路径， +// 并对 vm / database / blob 做跨云一致的分类（DoD #3）。Azure 适配器复用既有 ARM 客户端， +// 故三家行为一致；具体云的签名/鉴权在 aws_resource_discovery.go / gcp_resource_discovery.go。 + +// discoveredCloudResource 是各 provider 适配器输出的**归一化**资源视图。upsert 时由统一 +// 路径补 source_account_id 与 classified_type，并写入 ResourceBinding(resource_type=cloud_resource)。 +type discoveredCloudResource struct { + ExternalId string // ARN / Azure resource id / GCP asset name —— 跨 provider 的稳定外部 id + Name string // 展示名 + NativeType string // provider 原生类型字符串（如 Microsoft.Compute/virtualMachines、AWS::EC2::Instance、compute.googleapis.com/Instance） + Location string // region / location + BindingScope string // 可选；为空时默认 ":" + Metadata map[string]any // provider 特定附加字段 +} + +// cloudDiscoveryProvider 是「连接云账号 → 列资源」的统一抽象（DoD #3）。 +type cloudDiscoveryProvider interface { + name() string + // discover 用 creds（已合并 metadata + secret JSON）列出 account 下的资源。 + discover(account model.ResourceBinding, creds map[string]any) ([]discoveredCloudResource, error) +} + +func cloudDiscoveryProviderByName(provider string) (cloudDiscoveryProvider, bool) { + switch strings.ToLower(strings.TrimSpace(provider)) { + case "azure": + return azureCloudDiscoveryProvider{}, true + case "aws", "amazon": + return awsCloudDiscoveryProvider{}, true + case "gcp", "google", "gce": + return gcpCloudDiscoveryProvider{}, true + } + return nil, false +} + +// classifyCloudResourceType 把 provider 原生类型映射到跨云一致的逻辑类别（DoD #3）： +// vm / database / blob / cloud_resource(兜底)。基于不区分大小写的子串匹配，覆盖三家常见服务。 +func classifyCloudResourceType(nativeType string) string { + t := strings.ToLower(nativeType) + containsAny := func(s string, subs ...string) bool { + for _, sub := range subs { + if strings.Contains(s, sub) { + return true + } + } + return false + } + switch { + // 数据库（放在 vm 之前：避免 "sqlvirtual..." 之类被误判为 vm；数据库特征更强） + case containsAny(t, "sql", "postgres", "mysql", "mariadb", "rds", "database", "cosmos", + "dynamodb", "spanner", "cloudsql", "bigtable", "redis", "documentdb", "memcache"): + return "database" + case containsAny(t, "virtualmachine", "ec2::instance", "ec2/instance", "compute/instance", + "compute.googleapis.com/instance", "virtualmachinescaleset", "instance"): + return "vm" + case containsAny(t, "storageaccount", "::s3::", "s3:::", "/buckets/", "storage/bucket", + "storage.googleapis.com/bucket", "blob", "bucket"): + return "blob" + default: + return "cloud_resource" + } +} + +// upsertDiscoveredCloudResources 把归一化资源幂等写入 ResourceBinding（按 user+type+provider+external_id +// 去重）。统一所有 provider 的落库格式：resource_type 固定为 cloud_resource（与既有 Azure 行为一致， +// 不让发现的资源直接变成可绑定的 vm/db/blob），跨云类别放在 metadata.classified_type。 +func upsertDiscoveredCloudResources(account model.ResourceBinding, provider string, discovered []discoveredCloudResource) ([]resourceResponse, error) { + items := make([]resourceResponse, 0, len(discovered)) + for _, d := range discovered { + if strings.TrimSpace(d.ExternalId) == "" { + continue + } + metadata := map[string]any{} + for k, v := range d.Metadata { + metadata[k] = v + } + metadata["source_account_id"] = account.Id + metadata["classified_type"] = classifyCloudResourceType(d.NativeType) + if strings.TrimSpace(d.NativeType) != "" { + metadata["native_type"] = d.NativeType + } + if strings.TrimSpace(d.Location) != "" { + metadata["location"] = d.Location + } + metadataJSON, err := marshalResourceJSON(metadata) + if err != nil { + return nil, err + } + permissionScopeJSON, err := marshalResourceJSON(map[string]any{"actions": []string{provider + ":read"}}) + if err != nil { + return nil, err + } + constraintsJSON, err := marshalResourceJSON(unmarshalResourceJSON(account.Constraints)) + if err != nil { + return nil, err + } + name := strings.TrimSpace(d.Name) + if name == "" { + name = d.ExternalId + } + bindingScope := strings.TrimSpace(d.BindingScope) + if bindingScope == "" { + bindingScope = fmt.Sprintf("%s:%s", provider, d.ExternalId) + } + + var resource model.ResourceBinding + err = model.DB.Where( + "user_id = ? AND resource_type = ? AND provider = ? AND external_id = ?", + account.UserId, "cloud_resource", provider, d.ExternalId, + ).First(&resource).Error + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + return nil, err + } + resource.UserId = account.UserId + resource.TenantId = account.TenantId + resource.ProjectId = account.ProjectId + resource.BindingScope = bindingScope + resource.Name = name + resource.ResourceType = "cloud_resource" + resource.Provider = provider + resource.ExternalId = d.ExternalId + resource.SecretRef = account.SecretRef + resource.Metadata = metadataJSON + resource.PermissionScope = permissionScopeJSON + resource.Constraints = constraintsJSON + resource.Status = "active" + if resource.Id == 0 { + if err := model.DB.Create(&resource).Error; err != nil { + return nil, err + } + } else if err := model.DB.Save(&resource).Error; err != nil { + return nil, err + } + items = append(items, resourceToResponse(resource)) + } + return items, nil +} + +// DiscoverCloudResources: POST /api/resources/:id/discover-cloud — provider-agnostic +// discovery dispatcher (#5). Routes by the account's provider to the matching +// adapter (azure/aws/gcp), then upserts via the unified path. +func DiscoverCloudResources(c *gin.Context) { + userId := c.GetInt("id") + var account model.ResourceBinding + if err := model.DB.Where("id = ? AND user_id = ?", c.Param("id"), userId).First(&account).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + common.ApiErrorMsg(c, "resource not found") + return + } + common.ApiError(c, err) + return + } + if account.ResourceType != "cloud_account" || account.Status != "active" { + common.ApiErrorMsg(c, "resource must be an active cloud_account") + return + } + impl, ok := cloudDiscoveryProviderByName(account.Provider) + if !ok { + common.ApiErrorMsg(c, "unsupported cloud provider: "+account.Provider+" (supported: azure, aws, gcp)") + return + } + + // creds = metadata（非密）合并 secret JSON（密）。secret 优先级在适配器内部按需处理。 + creds := unmarshalResourceJSON(account.Metadata) + if creds == nil { + creds = map[string]any{} + } + if strings.TrimSpace(account.SecretRef) != "" { + secretClient, err := newSecretStoreClientFromEnv() + if err != nil { + common.ApiError(c, err) + return + } + secretData, err := secretClient.getJSONSecret(account.SecretRef) + if err != nil { + common.ApiError(c, err) + return + } + for k, v := range secretData { + creds[k] = v // secret 覆盖同名非密字段 + } + } + + discovered, err := impl.discover(account, creds) + if err != nil { + common.ApiError(c, err) + return + } + items, err := upsertDiscoveredCloudResources(account, impl.name(), discovered) + if err != nil { + common.ApiError(c, err) + return + } + common.ApiSuccess(c, gin.H{ + "account": resourceToResponse(account), + "provider": impl.name(), + "items": items, + "discovered": len(items), + }) +} + +// azureCloudDiscoveryProvider 让既有 Azure ARM 发现实现统一接口（复用 azure_resource_discovery.go +// 的 client/token/list），保证三家走同一抽象。 +type azureCloudDiscoveryProvider struct{} + +func (azureCloudDiscoveryProvider) name() string { return "azure" } + +func (azureCloudDiscoveryProvider) discover(account model.ResourceBinding, creds map[string]any) ([]discoveredCloudResource, error) { + cred := azureDiscoveryCredentials{ + SubscriptionID: firstString(creds, nil, account.ExternalId, "subscription_id"), + TenantID: firstString(creds, nil, account.TenantId, "tenant_id"), + ClientID: mapString(creds, "client_id"), + ClientSecret: mapString(creds, "client_secret"), + } + if err := cred.validate(); err != nil { + return nil, err + } + client := newAzureResourceDiscoveryClientFromEnv() + token, err := client.clientCredentialsToken(cred) + if err != nil { + return nil, err + } + arm, err := client.listSubscriptionResources(cred.SubscriptionID, token) + if err != nil { + return nil, err + } + out := make([]discoveredCloudResource, 0, len(arm)) + for _, r := range arm { + out = append(out, discoveredCloudResource{ + ExternalId: r.ID, + Name: r.Name, + NativeType: r.Type, + Location: r.Location, + BindingScope: fmt.Sprintf("azure:%s:%s", cred.SubscriptionID, r.ID), + Metadata: map[string]any{ + "subscription_id": cred.SubscriptionID, + "tenant_id": cred.TenantID, + "resource_group": azureResourceGroupFromID(r.ID), + "tags": r.Tags, + }, + }) + } + return out, nil +} diff --git a/heicode/controller/cloud_resource_discovery_test.go b/heicode/controller/cloud_resource_discovery_test.go new file mode 100644 index 00000000..be19029c --- /dev/null +++ b/heicode/controller/cloud_resource_discovery_test.go @@ -0,0 +1,116 @@ +package controller + +import ( + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// #5: 跨云类别分类 vm/database/blob 一致。 +func TestClassifyCloudResourceType(t *testing.T) { + cases := map[string]string{ + // vm + "Microsoft.Compute/virtualMachines": "vm", + "AWS::EC2::Instance": "vm", + "compute.googleapis.com/Instance": "vm", + // database + "Microsoft.Sql/servers/databases": "database", + "AWS::RDS::DBInstance": "database", + "sqladmin.googleapis.com/Instance": "database", + "AWS::DynamoDB::Table": "database", + // blob / object storage + "Microsoft.Storage/storageAccounts": "blob", + "AWS::S3::Bucket": "blob", + "storage.googleapis.com/Bucket": "blob", + // fallback + "Microsoft.Network/virtualNetworks": "cloud_resource", + "AWS::IAM::Role": "cloud_resource", + } + for native, want := range cases { + require.Equal(t, want, classifyCloudResourceType(native), "classify %s", native) + } +} + +// #5: ARN 解析 → 归一化 + 正确分类。 +func TestAWSARNToResource(t *testing.T) { + ec2, ok := awsARNToResource("arn:aws:ec2:us-east-1:123456789012:instance/i-0abc123") + require.True(t, ok) + require.Equal(t, "us-east-1", ec2.Location) + require.Equal(t, "i-0abc123", ec2.Name) + require.Equal(t, "vm", classifyCloudResourceType(ec2.NativeType)) + + s3, ok := awsARNToResource("arn:aws:s3:::my-bucket") + require.True(t, ok) + require.Equal(t, "my-bucket", s3.Name) + require.Equal(t, "blob", classifyCloudResourceType(s3.NativeType)) + + rds, ok := awsARNToResource("arn:aws:rds:eu-west-1:123:db:prod-pg") + require.True(t, ok) + require.Equal(t, "database", classifyCloudResourceType(rds.NativeType)) + + _, ok = awsARNToResource("not-an-arn") + require.False(t, ok) +} + +// #5: SigV4 实现正确性 —— 对齐 AWS 官方 "get-vanilla" 测试向量。 +func TestAWSSigV4_VanillaVector(t *testing.T) { + cred := awsDiscoveryCredentials{ + AccessKeyID: "AKIDEXAMPLE", + SecretAccessKey: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + Region: "us-east-1", + } + tm := time.Date(2015, 8, 30, 12, 36, 0, 0, time.UTC) + auth, amzDate, err := awsSigV4Authorization("GET", "https://example.amazonaws.com/", map[string]string{}, []byte(""), cred, "service", tm) + require.NoError(t, err) + require.Equal(t, "20150830T123600Z", amzDate) + require.Contains(t, auth, "Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request") + require.Contains(t, auth, "SignedHeaders=host;x-amz-date") + // 官方向量期望签名 + require.Contains(t, auth, "Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31") +} + +// #5: GCP SA JWT —— RS256 断言可被对应公钥验签,且 claims 正确。 +func TestBuildGCPAssertion_RS256Roundtrip(t *testing.T) { + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + der, err := x509.MarshalPKCS8PrivateKey(key) + require.NoError(t, err) + pemStr := string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der})) + + cred := gcpDiscoveryCredentials{ + ClientEmail: "svc@proj.iam.gserviceaccount.com", + PrivateKey: pemStr, + ProjectID: "proj", + } + tm := time.Date(2026, 6, 10, 0, 0, 0, 0, time.UTC) + jwt, err := buildGCPAssertion(cred, "https://www.googleapis.com/auth/cloud-platform", "https://oauth2.googleapis.com/token", tm) + require.NoError(t, err) + + parts := strings.Split(jwt, ".") + require.Len(t, parts, 3) + + // 验签:signingInput = header.payload + signingInput := parts[0] + "." + parts[1] + digest := sha256.Sum256([]byte(signingInput)) + sig, err := base64.RawURLEncoding.DecodeString(parts[2]) + require.NoError(t, err) + require.NoError(t, rsa.VerifyPKCS1v15(&key.PublicKey, crypto.SHA256, digest[:], sig), "RS256 签名应可被公钥验证") + + // header alg + claims + hdr, err := base64.RawURLEncoding.DecodeString(parts[0]) + require.NoError(t, err) + require.Contains(t, string(hdr), "RS256") + claims, err := base64.RawURLEncoding.DecodeString(parts[1]) + require.NoError(t, err) + require.Contains(t, string(claims), "svc@proj.iam.gserviceaccount.com") + require.Contains(t, string(claims), "cloud-platform") +} diff --git a/heicode/controller/gcp_resource_discovery.go b/heicode/controller/gcp_resource_discovery.go new file mode 100644 index 00000000..537f451f --- /dev/null +++ b/heicode/controller/gcp_resource_discovery.go @@ -0,0 +1,230 @@ +package controller + +import ( + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/heicode/manager/common" + "github.com/heicode/manager/model" +) + +// GCP resource discovery (#5). Pure-Go service-account flow (no GCP SDK): +// build an RS256-signed JWT assertion from the SA private key, exchange it for an +// OAuth2 access token, then call Cloud Asset Inventory searchAllResources (REST) +// to enumerate the project's resources. Normalizes assets to discoveredCloudResource. +// +// Creds (from cloud_account metadata/secret): client_email, private_key (PEM), +// project_id. token_uri optional (defaults to Google's). + +type gcpCloudDiscoveryProvider struct{} + +func (gcpCloudDiscoveryProvider) name() string { return "gcp" } + +func (gcpCloudDiscoveryProvider) discover(account model.ResourceBinding, creds map[string]any) ([]discoveredCloudResource, error) { + if mapString(creds, "project_id") == "" && strings.TrimSpace(account.ExternalId) != "" { + creds["project_id"] = account.ExternalId + } + return discoverGCP(creds, time.Now(), &http.Client{Timeout: 20 * time.Second}) +} + +type gcpDiscoveryCredentials struct { + ClientEmail string + PrivateKey string // PEM + ProjectID string + TokenURI string +} + +func (c gcpDiscoveryCredentials) validate() error { + if strings.TrimSpace(c.ClientEmail) == "" { + return errors.New("GCP client_email required") + } + if strings.TrimSpace(c.PrivateKey) == "" { + return errors.New("GCP private_key required") + } + if strings.TrimSpace(c.ProjectID) == "" { + return errors.New("GCP project_id required") + } + return nil +} + +func b64url(b []byte) string { return base64.RawURLEncoding.EncodeToString(b) } + +// parseRSAPrivateKeyPEM parses a PEM private key (PKCS#8 or PKCS#1) into an RSA key. +func parseRSAPrivateKeyPEM(pemStr string) (*rsa.PrivateKey, error) { + block, _ := pem.Decode([]byte(strings.TrimSpace(pemStr))) + if block == nil { + return nil, errors.New("GCP private_key is not valid PEM") + } + if key, err := x509.ParsePKCS1PrivateKey(block.Bytes); err == nil { + return key, nil + } + keyAny, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("GCP private_key parse failed: %w", err) + } + rsaKey, ok := keyAny.(*rsa.PrivateKey) + if !ok { + return nil, errors.New("GCP private_key is not an RSA key") + } + return rsaKey, nil +} + +// buildGCPAssertion builds the RS256-signed JWT assertion for the OAuth2 token +// exchange. Pure & deterministic given t — unit-testable with a generated key. +func buildGCPAssertion(cred gcpDiscoveryCredentials, scope string, aud string, t time.Time) (string, error) { + key, err := parseRSAPrivateKeyPEM(cred.PrivateKey) + if err != nil { + return "", err + } + header, err := common.Marshal(map[string]any{"alg": "RS256", "typ": "JWT"}) + if err != nil { + return "", err + } + iat := t.UTC().Unix() + claims, err := common.Marshal(map[string]any{ + "iss": cred.ClientEmail, + "scope": scope, + "aud": aud, + "iat": iat, + "exp": iat + 3600, + }) + if err != nil { + return "", err + } + signingInput := b64url(header) + "." + b64url(claims) + digest := sha256.Sum256([]byte(signingInput)) + sig, err := rsa.SignPKCS1v15(rand.Reader, key, crypto.SHA256, digest[:]) + if err != nil { + return "", err + } + return signingInput + "." + b64url(sig), nil +} + +func gcpAccessToken(cred gcpDiscoveryCredentials, t time.Time, httpClient *http.Client) (string, error) { + tokenURI := strings.TrimSpace(cred.TokenURI) + if tokenURI == "" { + tokenURI = "https://oauth2.googleapis.com/token" + } + assertion, err := buildGCPAssertion(cred, "https://www.googleapis.com/auth/cloud-platform", tokenURI, t) + if err != nil { + return "", err + } + form := url.Values{} + form.Set("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer") + form.Set("assertion", assertion) + req, err := http.NewRequest(http.MethodPost, tokenURI, strings.NewReader(form.Encode())) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + resp, err := httpClient.Do(req) + if err != nil { + return "", fmt.Errorf("GCP token exchange failed: %w", err) + } + defer resp.Body.Close() + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return "", fmt.Errorf("GCP token exchange failed (%d): %s", resp.StatusCode, strings.TrimSpace(string(raw))) + } + var payload struct { + AccessToken string `json:"access_token"` + } + if err := common.Unmarshal(raw, &payload); err != nil { + return "", err + } + if strings.TrimSpace(payload.AccessToken) == "" { + return "", errors.New("GCP token response missing access_token") + } + return payload.AccessToken, nil +} + +type gcpAsset struct { + Name string `json:"name"` + AssetType string `json:"assetType"` + DisplayName string `json:"displayName"` + Location string `json:"location"` +} + +type gcpSearchResponse struct { + Results []gcpAsset `json:"results"` + NextPageToken string `json:"nextPageToken"` +} + +func discoverGCP(creds map[string]any, now time.Time, httpClient *http.Client) ([]discoveredCloudResource, error) { + cred := gcpDiscoveryCredentials{ + ClientEmail: mapString(creds, "client_email"), + PrivateKey: mapString(creds, "private_key"), + ProjectID: mapString(creds, "project_id"), + TokenURI: mapString(creds, "token_uri"), + } + if err := cred.validate(); err != nil { + return nil, err + } + token, err := gcpAccessToken(cred, now, httpClient) + if err != nil { + return nil, err + } + + scope := "projects/" + cred.ProjectID + base := "https://cloudasset.googleapis.com/v1/" + url.PathEscape(scope) + ":searchAllResources" + out := []discoveredCloudResource{} + pageToken := "" + for page := 0; page < 50; page++ { + endpoint := base + "?pageSize=500" + if pageToken != "" { + endpoint += "&pageToken=" + url.QueryEscape(pageToken) + } + req, err := http.NewRequest(http.MethodGet, endpoint, nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+token) + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("GCP searchAllResources failed: %w", err) + } + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + resp.Body.Close() + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return nil, fmt.Errorf("GCP searchAllResources failed (%d): %s", resp.StatusCode, strings.TrimSpace(string(raw))) + } + var payload gcpSearchResponse + if err := common.Unmarshal(raw, &payload); err != nil { + return nil, err + } + for _, a := range payload.Results { + if strings.TrimSpace(a.Name) == "" { + continue + } + name := strings.TrimSpace(a.DisplayName) + if name == "" { + name = lastPathSegment(a.Name) + } + out = append(out, discoveredCloudResource{ + ExternalId: a.Name, + Name: name, + NativeType: a.AssetType, // e.g. compute.googleapis.com/Instance + Location: a.Location, + BindingScope: "gcp:" + a.Name, + Metadata: map[string]any{"project_id": cred.ProjectID, "asset_type": a.AssetType}, + }) + } + pageToken = strings.TrimSpace(payload.NextPageToken) + if pageToken == "" { + break + } + } + return out, nil +} diff --git a/heicode/router/api-router.go b/heicode/router/api-router.go index 97f36412..d57e366e 100644 --- a/heicode/router/api-router.go +++ b/heicode/router/api-router.go @@ -225,6 +225,9 @@ func SetApiRouter(router *gin.Engine) { resourceRoute.POST("/", controller.CreateResource) resourceRoute.POST("/:id/secret", controller.UpsertResourceSecret) resourceRoute.POST("/:id/azure/discover", controller.DiscoverAzureResources) + // Provider-agnostic cloud discovery (#5): routes by the account's provider + // (azure/aws/gcp). The azure-specific route above is kept for back-compat. + resourceRoute.POST("/:id/discover-cloud", controller.DiscoverCloudResources) resourceRoute.PUT("/:id", controller.UpdateResource) resourceRoute.DELETE("/:id", controller.DeleteResource) }