diff --git a/heicode/controller/aws_resource_discovery.go b/heicode/controller/aws_resource_discovery.go index d66df7a..c99f13d 100644 --- a/heicode/controller/aws_resource_discovery.go +++ b/heicode/controller/aws_resource_discovery.go @@ -22,10 +22,11 @@ import ( // the REST approach of azure_resource_discovery.go. Uses the Resource Groups // Tagging API GetResources (AWS JSON 1.1), then normalizes ARNs to discoveredCloudResource. // -// ⚠️ 覆盖范围(#5 复审 #2,据实声明):GetResources 仅返回**已打标签或曾打标签**的资源, -// **不覆盖完全未打标签的 EC2/RDS/S3 等**。因此本适配器是「**tagged resources only**」的发现, -// 不等于账号内全量资源;补全需后续加服务级 List/Describe(DescribeInstances / DescribeDBInstances / -// ListBuckets 等)。在此之前 #5 的「全量发现」诉求不应据本适配器关闭。 +// ⚠️ 覆盖范围:GetResources 仅返回**已打标签或曾打标签**的资源,**不覆盖完全未打标签的 +// EC2/RDS/S3 等**——本函数(discoverAWSTagged)是「**tagged resources only**」的一半。 +// 全量发现由 discoverAWS(aws_resource_discovery_services.go,#62)把本结果与服务级 +// List/Describe(DescribeInstances / DescribeDBInstances / ListBuckets / ListFunctions) +// 按 ARN 合并去重补齐。 // // Creds (from cloud_account metadata/secret): access_key_id, secret_access_key, // region, optional session_token. @@ -185,18 +186,11 @@ type awsGetResourcesResponse struct { PaginationToken string `json:"PaginationToken"` } -// discoverAWS enumerates the account's tagged resources via the Resource Groups -// Tagging API (paginated). Real implementation (no mock). -func discoverAWS(creds map[string]any, now time.Time, httpClient *http.Client) ([]discoveredCloudResource, error) { - cred := awsDiscoveryCredentials{ - AccessKeyID: mapString(creds, "access_key_id"), - SecretAccessKey: mapString(creds, "secret_access_key"), - SessionToken: mapString(creds, "session_token"), - Region: mapString(creds, "region"), - } - if err := cred.validate(); err != nil { - return nil, err - } +// discoverAWSTagged enumerates the account's tagged resources via the Resource Groups +// Tagging API (paginated). Real implementation (no mock). Coverage is "tagged only" +// (see file header); discoverAWS (aws_resource_discovery_services.go) merges this with +// service-level List/Describe for full-account coverage (#62). +func discoverAWSTagged(cred awsDiscoveryCredentials, now time.Time, httpClient *http.Client) ([]discoveredCloudResource, error) { endpoint := fmt.Sprintf("https://tagging.%s.amazonaws.com/", cred.Region) const target = "ResourceGroupsTaggingAPI_20170126.GetResources" diff --git a/heicode/controller/aws_resource_discovery_services.go b/heicode/controller/aws_resource_discovery_services.go new file mode 100644 index 0000000..0d89030 --- /dev/null +++ b/heicode/controller/aws_resource_discovery_services.go @@ -0,0 +1,401 @@ +package controller + +import ( + "encoding/xml" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/heicode/manager/common" +) + +// Service-level AWS discovery (#62): augments the tagged-only GetResources pass +// (aws_resource_discovery.go) with direct List/Describe so **untagged** EC2/RDS/S3/Lambda +// resources are found too — closing the "tagged only ≠ full account" gap (#5 follow-up). +// +// 设计: +// - discoverAWS 编排:先 tagged(最全的标签信息),再各服务级 List/Describe,按 ARN 合并去重 +// (tagged 优先保留,信息更丰富)。 +// - **失败降级**:任一服务调用失败(如缺该服务读权限 / AccessDenied)只记日志并跳过,绝不让 +// 整次发现失败;仅当「tagged 报错且无任何服务返回结果」时,才把 tagged 的原始错误抛出, +// 好让账号凭据/区域问题对用户可见。 +// - 复用 awsSigV4Authorization(已对 AWS get-vanilla 向量验证)签 GET;无 AWS SDK 依赖。 +// - 区域:EC2/RDS/Lambda 用账号配置的 region;S3 ListBuckets 是全局接口(以 us-east-1 签名)。 +// 跨 region 全量扫描不在本次范围(EC2/RDS 仅扫配置 region),由 #62 备注另行跟踪。 +// - 解析逻辑(parseAWS* )为纯函数,便于用真实响应样本单测。 + +// discoverAWS enumerates the account's resources by merging the tagged-resources pass +// with service-level List/Describe (#62). Replaces the old tagged-only discoverAWS. +func discoverAWS(creds map[string]any, now time.Time, httpClient *http.Client) ([]discoveredCloudResource, error) { + cred := awsDiscoveryCredentials{ + AccessKeyID: mapString(creds, "access_key_id"), + SecretAccessKey: mapString(creds, "secret_access_key"), + SessionToken: mapString(creds, "session_token"), + Region: mapString(creds, "region"), + } + if err := cred.validate(); err != nil { + return nil, err + } + + var lists [][]discoveredCloudResource + tagged, taggedErr := discoverAWSTagged(cred, now, httpClient) + if taggedErr != nil { + common.SysLog("discoverAWS tagged: " + taggedErr.Error()) + } else { + lists = append(lists, tagged) // tagged 先入,合并时优先保留 + } + + services := []struct { + name string + fn func(awsDiscoveryCredentials, time.Time, *http.Client) ([]discoveredCloudResource, error) + }{ + {"ec2", discoverAWSEC2}, + {"rds", discoverAWSRDS}, + {"s3", discoverAWSS3}, + {"lambda", discoverAWSLambda}, + } + for _, svc := range services { + list, err := svc.fn(cred, now, httpClient) + if err != nil { + common.SysLog("discoverAWS " + svc.name + ": " + err.Error()) + continue + } + lists = append(lists, list) + } + + merged := mergeDedupeByExternalID(lists...) + // 全部为空且 tagged 当初报错 → 抛出原始错误,让凭据/区域问题可见(纯权限缺失则降级返回已得结果)。 + if len(merged) == 0 && taggedErr != nil { + return nil, taggedErr + } + return merged, nil +} + +// mergeDedupeByExternalID 按 ExternalId(ARN)合并多组发现结果,先到先得(靠前的列表优先保留), +// 保持稳定顺序。纯函数,便于单测。 +func mergeDedupeByExternalID(lists ...[]discoveredCloudResource) []discoveredCloudResource { + seen := map[string]bool{} + out := make([]discoveredCloudResource, 0) + for _, list := range lists { + for _, d := range list { + id := strings.TrimSpace(d.ExternalId) + if id == "" || seen[id] { + continue + } + seen[id] = true + out = append(out, d) + } + } + return out +} + +// awsSignedGet 发起一次 SigV4 签名的 GET(空 body),返回响应体。非 2xx 视为错误。 +func awsSignedGet(cred awsDiscoveryCredentials, rawURL, service string, now time.Time, httpClient *http.Client) ([]byte, error) { + auth, amzDate, err := awsSigV4Authorization(http.MethodGet, rawURL, map[string]string{}, nil, cred, service, now) + if err != nil { + return nil, err + } + req, err := http.NewRequest(http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("X-Amz-Date", amzDate) + req.Header.Set("Authorization", auth) + if strings.TrimSpace(cred.SessionToken) != "" { + req.Header.Set("X-Amz-Security-Token", cred.SessionToken) + } + resp, err := httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 16<<20)) + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return body, fmt.Errorf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + return body, nil +} + +// ---- EC2 DescribeInstances (Query 协议, XML) ---- + +type ec2DescribeInstancesResponse struct { + XMLName xml.Name `xml:"DescribeInstancesResponse"` + Reservations []struct { + OwnerID string `xml:"ownerId"` + Instances []struct { + InstanceID string `xml:"instanceId"` + InstanceType string `xml:"instanceType"` + State struct { + Name string `xml:"name"` + } `xml:"instanceState"` + Placement struct { + AvailabilityZone string `xml:"availabilityZone"` + } `xml:"placement"` + Tags []struct { + Key string `xml:"key"` + Value string `xml:"value"` + } `xml:"tagSet>item"` + } `xml:"instancesSet>item"` + } `xml:"reservationSet>item"` + NextToken string `xml:"nextToken"` +} + +// parseEC2Instances 解析 DescribeInstances XML → 归一化资源 + nextToken。纯函数。 +func parseEC2Instances(body []byte, region string) ([]discoveredCloudResource, string, error) { + var r ec2DescribeInstancesResponse + if err := xml.Unmarshal(body, &r); err != nil { + return nil, "", err + } + out := []discoveredCloudResource{} + for _, res := range r.Reservations { + for _, inst := range res.Instances { + if strings.TrimSpace(inst.InstanceID) == "" { + continue + } + arn := fmt.Sprintf("arn:aws:ec2:%s:%s:instance/%s", region, res.OwnerID, inst.InstanceID) + tags := map[string]any{} + name := inst.InstanceID + for _, t := range inst.Tags { + tags[t.Key] = t.Value + if strings.EqualFold(t.Key, "Name") && strings.TrimSpace(t.Value) != "" { + name = t.Value + } + } + out = append(out, discoveredCloudResource{ + ExternalId: arn, + Name: name, + NativeType: "AWS::EC2::Instance", + Location: region, + BindingScope: "aws:" + arn, + Metadata: map[string]any{ + "region": region, + "instance_type": inst.InstanceType, + "state": inst.State.Name, + "tags": tags, + }, + }) + } + } + return out, strings.TrimSpace(r.NextToken), nil +} + +func discoverAWSEC2(cred awsDiscoveryCredentials, now time.Time, httpClient *http.Client) ([]discoveredCloudResource, error) { + out := []discoveredCloudResource{} + nextToken := "" + for page := 0; page < 50; page++ { + q := url.Values{} + q.Set("Action", "DescribeInstances") + q.Set("Version", "2016-11-15") + q.Set("MaxResults", "100") + if nextToken != "" { + q.Set("NextToken", nextToken) + } + rawURL := fmt.Sprintf("https://ec2.%s.amazonaws.com/?%s", cred.Region, q.Encode()) + body, err := awsSignedGet(cred, rawURL, "ec2", now, httpClient) + if err != nil { + return nil, err + } + page1, token, err := parseEC2Instances(body, cred.Region) + if err != nil { + return nil, err + } + out = append(out, page1...) + nextToken = token + if nextToken == "" { + break + } + } + return out, nil +} + +// ---- RDS DescribeDBInstances (Query 协议, XML) ---- + +type rdsDescribeDBInstancesResponse struct { + XMLName xml.Name `xml:"DescribeDBInstancesResponse"` + Result struct { + Marker string `xml:"Marker"` + DBInstances []struct { + Arn string `xml:"DBInstanceArn"` + Identifier string `xml:"DBInstanceIdentifier"` + Engine string `xml:"Engine"` + Status string `xml:"DBInstanceStatus"` + AvailabilityZone string `xml:"AvailabilityZone"` + } `xml:"DBInstances>DBInstance"` + } `xml:"DescribeDBInstancesResult"` +} + +// parseRDSInstances 解析 DescribeDBInstances XML → 归一化资源 + marker。纯函数。 +func parseRDSInstances(body []byte, region string) ([]discoveredCloudResource, string, error) { + var r rdsDescribeDBInstancesResponse + if err := xml.Unmarshal(body, &r); err != nil { + return nil, "", err + } + out := []discoveredCloudResource{} + for _, db := range r.Result.DBInstances { + arn := strings.TrimSpace(db.Arn) + if arn == "" { + continue // 无 ARN 不入(稳定外部 id 缺失) + } + name := strings.TrimSpace(db.Identifier) + if name == "" { + name = arn + } + out = append(out, discoveredCloudResource{ + ExternalId: arn, + Name: name, + NativeType: "AWS::RDS::DBInstance", + Location: region, + BindingScope: "aws:" + arn, + Metadata: map[string]any{ + "region": region, + "engine": db.Engine, + "status": db.Status, + "az": db.AvailabilityZone, + }, + }) + } + return out, strings.TrimSpace(r.Result.Marker), nil +} + +func discoverAWSRDS(cred awsDiscoveryCredentials, now time.Time, httpClient *http.Client) ([]discoveredCloudResource, error) { + out := []discoveredCloudResource{} + marker := "" + for page := 0; page < 50; page++ { + q := url.Values{} + q.Set("Action", "DescribeDBInstances") + q.Set("Version", "2014-10-31") + q.Set("MaxRecords", "100") + if marker != "" { + q.Set("Marker", marker) + } + rawURL := fmt.Sprintf("https://rds.%s.amazonaws.com/?%s", cred.Region, q.Encode()) + body, err := awsSignedGet(cred, rawURL, "rds", now, httpClient) + if err != nil { + return nil, err + } + page1, next, err := parseRDSInstances(body, cred.Region) + if err != nil { + return nil, err + } + out = append(out, page1...) + marker = next + if marker == "" { + break + } + } + return out, nil +} + +// ---- S3 ListBuckets (REST-XML, 全局) ---- + +type s3ListAllMyBucketsResult struct { + XMLName xml.Name `xml:"ListAllMyBucketsResult"` + Buckets []struct { + Name string `xml:"Name"` + } `xml:"Buckets>Bucket"` +} + +// parseS3Buckets 解析 ListBuckets XML → 归一化资源。纯函数。S3 bucket ARN 无 region/account 段。 +func parseS3Buckets(body []byte) ([]discoveredCloudResource, error) { + var r s3ListAllMyBucketsResult + if err := xml.Unmarshal(body, &r); err != nil { + return nil, err + } + out := []discoveredCloudResource{} + for _, b := range r.Buckets { + name := strings.TrimSpace(b.Name) + if name == "" { + continue + } + arn := "arn:aws:s3:::" + name + out = append(out, discoveredCloudResource{ + ExternalId: arn, + Name: name, + NativeType: "AWS::S3::Bucket", + Location: "", // 全局列举不含 region;按需 GetBucketLocation(未做,避免逐桶请求) + BindingScope: "aws:" + arn, + Metadata: map[string]any{"bucket": name}, + }) + } + return out, nil +} + +func discoverAWSS3(cred awsDiscoveryCredentials, now time.Time, httpClient *http.Client) ([]discoveredCloudResource, error) { + // ListBuckets 是全局接口,固定以 us-east-1 / service=s3 签名。 + s3Cred := cred + s3Cred.Region = "us-east-1" + body, err := awsSignedGet(s3Cred, "https://s3.amazonaws.com/", "s3", now, httpClient) + if err != nil { + return nil, err + } + return parseS3Buckets(body) +} + +// ---- Lambda ListFunctions (REST-JSON) ---- + +type lambdaListFunctionsResponse struct { + Functions []struct { + FunctionArn string `json:"FunctionArn"` + FunctionName string `json:"FunctionName"` + Runtime string `json:"Runtime"` + } `json:"Functions"` + NextMarker string `json:"NextMarker"` +} + +// parseLambdaFunctions 解析 ListFunctions JSON → 归一化资源 + nextMarker。纯函数。 +func parseLambdaFunctions(body []byte, region string) ([]discoveredCloudResource, string, error) { + var r lambdaListFunctionsResponse + if err := common.Unmarshal(body, &r); err != nil { + return nil, "", err + } + out := []discoveredCloudResource{} + for _, fn := range r.Functions { + arn := strings.TrimSpace(fn.FunctionArn) + if arn == "" { + continue + } + name := strings.TrimSpace(fn.FunctionName) + if name == "" { + name = arn + } + out = append(out, discoveredCloudResource{ + ExternalId: arn, + Name: name, + NativeType: "AWS::Lambda::Function", + Location: region, + BindingScope: "aws:" + arn, + Metadata: map[string]any{"region": region, "runtime": fn.Runtime}, + }) + } + return out, strings.TrimSpace(r.NextMarker), nil +} + +func discoverAWSLambda(cred awsDiscoveryCredentials, now time.Time, httpClient *http.Client) ([]discoveredCloudResource, error) { + out := []discoveredCloudResource{} + marker := "" + for page := 0; page < 50; page++ { + q := url.Values{} + q.Set("MaxItems", "50") + if marker != "" { + q.Set("Marker", marker) + } + rawURL := fmt.Sprintf("https://lambda.%s.amazonaws.com/2015-03-31/functions/?%s", cred.Region, q.Encode()) + body, err := awsSignedGet(cred, rawURL, "lambda", now, httpClient) + if err != nil { + return nil, err + } + page1, next, err := parseLambdaFunctions(body, cred.Region) + if err != nil { + return nil, err + } + out = append(out, page1...) + marker = next + if marker == "" { + break + } + } + return out, nil +} diff --git a/heicode/controller/cloud_resource_discovery_test.go b/heicode/controller/cloud_resource_discovery_test.go index cd2e0ee..5b33a51 100644 --- a/heicode/controller/cloud_resource_discovery_test.go +++ b/heicode/controller/cloud_resource_discovery_test.go @@ -20,21 +20,21 @@ import ( func TestClassifyCloudResourceType(t *testing.T) { cases := map[string]string{ // vm - "Microsoft.Compute/virtualMachines": "vm", - "AWS::EC2::Instance": "vm", - "compute.googleapis.com/Instance": "vm", + "Microsoft.Compute/virtualMachines": "vm", + "AWS::EC2::Instance": "vm", + "compute.googleapis.com/Instance": "vm", // database - "Microsoft.Sql/servers/databases": "database", - "AWS::RDS::DBInstance": "database", - "sqladmin.googleapis.com/Instance": "database", - "AWS::DynamoDB::Table": "database", + "Microsoft.Sql/servers/databases": "database", + "AWS::RDS::DBInstance": "database", + "sqladmin.googleapis.com/Instance": "database", + "AWS::DynamoDB::Table": "database", // blob / object storage - "Microsoft.Storage/storageAccounts": "blob", - "AWS::S3::Bucket": "blob", - "storage.googleapis.com/Bucket": "blob", + "Microsoft.Storage/storageAccounts": "blob", + "AWS::S3::Bucket": "blob", + "storage.googleapis.com/Bucket": "blob", // fallback - "Microsoft.Network/virtualNetworks": "cloud_resource", - "AWS::IAM::Role": "cloud_resource", + "Microsoft.Network/virtualNetworks": "cloud_resource", + "AWS::IAM::Role": "cloud_resource", } for native, want := range cases { require.Equal(t, want, classifyCloudResourceType(native), "classify %s", native) @@ -148,3 +148,121 @@ func TestBuildGCPAssertion_RS256Roundtrip(t *testing.T) { require.Contains(t, string(claims), "svc@proj.iam.gserviceaccount.com") require.Contains(t, string(claims), "cloud-platform") } + +// #62: EC2 DescribeInstances XML 解析 —— 含 Name 标签 / ownerId / nextToken;ARN 由 region+owner+id 组装。 +func TestParseEC2Instances(t *testing.T) { + body := []byte(` + + + + 123456789012 + + + i-0abc123 + t3.micro + running + us-east-1a + + Nameweb-1 + envprod + + + + i-0def456 + t3.small + + + + + NEXT== +`) + out, next, err := parseEC2Instances(body, "us-east-1") + require.NoError(t, err) + require.Equal(t, "NEXT==", next) + require.Len(t, out, 2) + require.Equal(t, "arn:aws:ec2:us-east-1:123456789012:instance/i-0abc123", out[0].ExternalId) + require.Equal(t, "web-1", out[0].Name) // Name 标签优先 + require.Equal(t, "AWS::EC2::Instance", out[0].NativeType) + require.Equal(t, "vm", classifyCloudResourceType(out[0].NativeType)) + require.Equal(t, "i-0def456", out[1].Name) // 无 Name 标签回退 instanceId +} + +// #62: RDS DescribeDBInstances XML 解析 —— ARN 直接取 DBInstanceArn;无 ARN 跳过;Marker 续页。 +func TestParseRDSInstances(t *testing.T) { + body := []byte(` + + m2 + + + arn:aws:rds:eu-west-1:123456789012:db:prod-pg + prod-pg + postgres + available + + + no-arn + mysql + + + +`) + out, marker, err := parseRDSInstances(body, "eu-west-1") + require.NoError(t, err) + require.Equal(t, "m2", marker) + require.Len(t, out, 1) // 无 ARN 的被跳过 + require.Equal(t, "arn:aws:rds:eu-west-1:123456789012:db:prod-pg", out[0].ExternalId) + require.Equal(t, "prod-pg", out[0].Name) + require.Equal(t, "database", classifyCloudResourceType(out[0].NativeType)) +} + +// #62: S3 ListBuckets XML 解析 —— ARN 为 arn:aws:s3:::,全局无 region。 +func TestParseS3Buckets(t *testing.T) { + body := []byte(` + x + + my-bucket2024-01-01T00:00:00Z + logs-bucket + +`) + out, err := parseS3Buckets(body) + require.NoError(t, err) + require.Len(t, out, 2) + require.Equal(t, "arn:aws:s3:::my-bucket", out[0].ExternalId) + require.Equal(t, "my-bucket", out[0].Name) + require.Equal(t, "blob", classifyCloudResourceType(out[0].NativeType)) + require.Empty(t, out[0].Location) +} + +// #62: Lambda ListFunctions JSON 解析 —— ARN 取 FunctionArn;NextMarker 续页。 +func TestParseLambdaFunctions(t *testing.T) { + body := []byte(`{"Functions":[ + {"FunctionArn":"arn:aws:lambda:us-east-1:123456789012:function:fn-a","FunctionName":"fn-a","Runtime":"go1.x"}, + {"FunctionArn":"arn:aws:lambda:us-east-1:123456789012:function:fn-b","FunctionName":"fn-b","Runtime":"python3.12"} + ],"NextMarker":"mk"}`) + out, next, err := parseLambdaFunctions(body, "us-east-1") + require.NoError(t, err) + require.Equal(t, "mk", next) + require.Len(t, out, 2) + require.Equal(t, "arn:aws:lambda:us-east-1:123456789012:function:fn-a", out[0].ExternalId) + require.Equal(t, "fn-a", out[0].Name) + require.Equal(t, "AWS::Lambda::Function", out[0].NativeType) +} + +// #62: 合并去重 —— 先到先得(tagged 优先保留),空 ExternalId 丢弃,顺序稳定。 +func TestMergeDedupeByExternalID(t *testing.T) { + tagged := []discoveredCloudResource{ + {ExternalId: "arn:a", Name: "tagged-a", Metadata: map[string]any{"tags": map[string]any{"k": "v"}}}, + {ExternalId: "arn:b", Name: "tagged-b"}, + } + ec2 := []discoveredCloudResource{ + {ExternalId: "arn:a", Name: "ec2-a"}, // 与 tagged 重复 → 保留 tagged + {ExternalId: "arn:c", Name: "ec2-c"}, // 未打标签的新资源 + {ExternalId: "", Name: "empty"}, // 空 id 丢弃 + } + merged := mergeDedupeByExternalID(tagged, ec2) + require.Len(t, merged, 3) + require.Equal(t, "tagged-a", merged[0].Name) // tagged 优先 + require.NotNil(t, merged[0].Metadata["tags"]) // 保留更丰富的 tagged 元数据 + require.Equal(t, "tagged-b", merged[1].Name) + require.Equal(t, "ec2-c", merged[2].Name) // 补齐未打标签资源 +}