Derive GCP sub types from asset type (#1145)

resources/fetching/fetchers/gcp/assets_fetcher.go

@@ -19,6 +19,10 @@ package fetchers
 
 import (
 	"context"
+	"fmt"
+	"strings"
+
+	"github.com/huandu/xstrings"
 
 	"cloud.google.com/go/asset/apiv1/assetpb"
 	"github.com/elastic/elastic-agent-libs/logp"
@@ -40,19 +44,23 @@ type GcpAsset struct {
 	Asset *assetpb.Asset `json:"asset,omitempty"`
 }
 
-var GcpAssetTypes = map[string]map[string][]string{
+// map of types to asset types.
+// sub-type is derived from asset type by using the first and last segments of the asset type name
+// example: gcp-cloudkms-crypto-key
+var GcpAssetTypes = map[string][]string{
 	fetching.KeyManagement: {
-		"gcp-kms": {"cloudkms.googleapis.com/CryptoKey"},
+		"cloudkms.googleapis.com/CryptoKey",
 	},
 	fetching.CloudIdentity: {
-		"gcp-iam": {"iam.googleapis.com/ServiceAccount"},
+		"iam.googleapis.com/ServiceAccount",
+		"iam.googleapis.com/ServiceAccountKey",
 	},
 	fetching.CloudDatabase: {
-		"gcp-bq-dataset": {"bigquery.googleapis.com/Dataset"},
-		"gcp-bq-table":   {"bigquery.googleapis.com/Table"},
+		"bigquery.googleapis.com/Dataset",
+		"bigquery.googleapis.com/Table",
 	},
 	fetching.CloudStorage: {
-		"gcp-gcs": {"storage.googleapis.com/Bucket"},
+		"storage.googleapis.com/Bucket",
 	},
 }
 
@@ -67,23 +75,21 @@ func NewGcpAssetsFetcher(_ context.Context, log *logp.Logger, ch chan fetching.R
 func (f *GcpAssetsFetcher) Fetch(ctx context.Context, cMetadata fetching.CycleMetadata) error {
 	f.log.Info("Starting GcpAssetsFetcher.Fetch")
 
-	for typeName, subtypes := range GcpAssetTypes {
-		for subTypeName, assetTypes := range subtypes {
-			assets, err := f.provider.ListAllAssetTypesByName(assetTypes)
-			if err != nil {
-				f.log.Errorf("Failed to list assets for type %s: %s", typeName, err)
-				continue
-			}
+	for typeName, assetTypes := range GcpAssetTypes {
+		assets, err := f.provider.ListAllAssetTypesByName(assetTypes)
+		if err != nil {
+			f.log.Errorf("Failed to list assets for type %s: %s", typeName, err)
+			continue
+		}
 
-			for _, asset := range assets {
-				f.resourceCh <- fetching.ResourceInfo{
-					CycleMetadata: cMetadata,
-					Resource: &GcpAsset{
-						Type:    typeName,
-						SubType: subTypeName,
-						Asset:   asset,
-					},
-				}
+		for _, asset := range assets {
+			f.resourceCh <- fetching.ResourceInfo{
+				CycleMetadata: cMetadata,
+				Resource: &GcpAsset{
+					Type:    typeName,
+					SubType: getGcpSubType(asset.AssetType),
+					Asset:   asset,
+				},
 			}
 		}
 	}
@@ -116,3 +122,13 @@ func (r *GcpAsset) GetMetadata() (fetching.ResourceMetadata, error) {
 }
 
 func (r *GcpAsset) GetElasticCommonData() any { return nil }
+
+func getGcpSubType(assetType string) string {
+	dotIndex := strings.Index(assetType, ".")
+	slashIndex := strings.Index(assetType, "/")
+
+	prefix := assetType[:dotIndex]
+	suffix := assetType[slashIndex+1:]
+
+	return strings.ToLower(fmt.Sprintf("gcp-%s-%s", prefix, xstrings.ToKebabCase(suffix)))
+}

resources/fetching/fetchers/gcp/assets_fetcher_test.go

@@ -65,23 +65,15 @@ func (s *GcpAssetsFetcherTestSuite) TestFetcher_Fetch() {
 		return true
 	})).Return(
 		[]*assetpb.Asset{
-			{Name: "a"}, // 1 asset for each subtype
+			{Name: "a", AssetType: "iam.googleapis.com/ServiceAccount"},
 		}, nil,
 	)
 
 	err := GcpAssetsFetcher.Fetch(ctx, fetching.CycleMetadata{})
 	s.NoError(err)
-
 	results := testhelper.CollectResources(s.resourceCh)
-	s.Equal(getSubtypesCount(), len(results))
-}
 
-func getSubtypesCount() int {
-	var count int
-	for _, subtypes := range GcpAssetTypes {
-		for range subtypes {
-			count++
-		}
-	}
-	return count
+	// ListAllAssetTypesByName mocked to return a single asset
+	// Will be called N times, where N is the number of types in GcpAssetTypes
+	s.Equal(len(GcpAssetTypes), len(results))
 }