[Asset Inventory][Azure] Add storage assets

internal/inventory/ASSETS.md

@@ -98,9 +98,9 @@ Infrastructure: 24% (18/73)
 
 ## AZURE Resources
 
-**Progress: 23% (12/51)**
+**Progress: 29% (15/51)**
 Identity: 0% (0/8)
-Infrastructure: 27% (12/43)
+Infrastructure: 34% (15/43)
 
 <details> <summary>Full table</summary>
 
@@ -114,8 +114,8 @@ Infrastructure: 27% (12/43)
 | Identity | Directory | Group | Azure AD Group | No ❌ |
 | Identity | Directory | User | Azure AD User | No ❌ |
 | Identity | Service Identity | Service Principal | Azure AD Service Principal | No ❌ |
-| Infrastructure | Application Integration | Message Queue | Azure Storage Queue | No ❌ |
-| Infrastructure | Application Integration | Message Queue | Azure Storage Queue Service | No ❌ |
+| Infrastructure | Application Integration | Message Queue | Azure Storage Queue | Yes ✅ |
+| Infrastructure | Application Integration | Message Queue | Azure Storage Queue Service | Yes ✅ |
 | Infrastructure | Application | Web Application | Azure App Service | Yes ✅ |
 | Infrastructure | Compute | Virtual Machine | Azure Virtual Machine | Yes ✅ |
 | Infrastructure | Container | Registry | Azure Container Registry | Yes ✅ |
@@ -154,7 +154,7 @@ Infrastructure: 27% (12/43)
 | Infrastructure | Storage | File Storage | Azure Storage File Service | No ❌ |
 | Infrastructure | Storage | File Storage | Azure Storage File Share | No ❌ |
 | Infrastructure | Storage | Object Storage | Azure Storage Blob Container | No ❌ |
-| Infrastructure | Storage | Object Storage | Azure Storage Blob Service | No ❌ |
+| Infrastructure | Storage | Object Storage | Azure Storage Blob Service | Yes ✅ |
 | Infrastructure | Storage | Snapshot | Azure Snapshot | Yes ✅ |
 | Infrastructure | Storage | Storage | Azure Storage Account | Yes ✅ |
 

internal/inventory/asset.go

@@ -29,20 +29,21 @@ const (
 type AssetSubCategory string
 
 const (
-	SubCategoryApplication      AssetSubCategory = "application"
-	SubCategoryAuthorization    AssetSubCategory = "authorization"
-	SubCategoryCompute          AssetSubCategory = "compute"
-	SubCategoryContainer        AssetSubCategory = "container"
-	SubCategoryDatabase         AssetSubCategory = "database"
-	SubCategoryDigitalIdentity  AssetSubCategory = "digital-identity"
-	SubCategoryIntegration      AssetSubCategory = "integration"
-	SubCategoryManagement       AssetSubCategory = "management"
-	SubCategoryMessaging        AssetSubCategory = "messaging"
-	SubCategoryNetwork          AssetSubCategory = "network"
-	SubCategoryStorage          AssetSubCategory = "storage"
-	SubCategoryServiceIdentity  AssetSubCategory = "service-identity"
-	SubCategoryAccessManagement AssetSubCategory = "access-management"
-	SubCategoryServerless       AssetSubCategory = "serverless"
+	SubCategoryAccessManagement       AssetSubCategory = "access-management"
+	SubCategoryApplication            AssetSubCategory = "application"
+	SubCategoryApplicationIntegration AssetSubCategory = "application-integration"
+	SubCategoryAuthorization          AssetSubCategory = "authorization"
+	SubCategoryCompute                AssetSubCategory = "compute"
+	SubCategoryContainer              AssetSubCategory = "container"
+	SubCategoryDatabase               AssetSubCategory = "database"
+	SubCategoryDigitalIdentity        AssetSubCategory = "digital-identity"
+	SubCategoryIntegration            AssetSubCategory = "integration"
+	SubCategoryManagement             AssetSubCategory = "management"
+	SubCategoryMessaging              AssetSubCategory = "messaging"
+	SubCategoryNetwork                AssetSubCategory = "network"
+	SubCategoryServerless             AssetSubCategory = "serverless"
+	SubCategoryServiceIdentity        AssetSubCategory = "service-identity"
+	SubCategoryStorage                AssetSubCategory = "storage"
 )
 
 // AssetType is used to build the document index. Use only numbers, letters and dashes (-)
@@ -57,6 +58,7 @@ const (
 	TypeGateway             AssetType = "gateway"
 	TypeInterface           AssetType = "interface"
 	TypeLoadBalancer        AssetType = "load-balancer"
+	TypeMessageQueue        AssetType = "message-queue"
 	TypeNoSQLDatabase       AssetType = "nosql-database"
 	TypeNotificationService AssetType = "notification-service"
 	TypeObjectStorage       AssetType = "object-storage"
@@ -68,15 +70,15 @@ const (
 	TypeRole                AssetType = "role"
 	TypeScalability         AssetType = "scalability"
 	TypeServerless          AssetType = "serverless"
+	TypeServiceAccount      AssetType = "service-account"
+	TypeServiceAccountKey   AssetType = "service-account-key"
 	TypeSnapshot            AssetType = "snapshot"
 	TypeStorage             AssetType = "storage"
 	TypeSubnet              AssetType = "subnet"
 	TypeUser                AssetType = "user"
 	TypeVirtualMachine      AssetType = "virtual-machine"
 	TypeVirtualNetwork      AssetType = "virtual-network"
 	TypeWebApplication      AssetType = "web-application"
-	TypeServiceAccount      AssetType = "service-account"
-	TypeServiceAccountKey   AssetType = "service-account-key"
 	TypeResourceHierarchy   AssetType = "resource-hierarchy"
 	TypeOrchestration       AssetType = "orchestration"
 	TypeFunction            AssetType = "function"
@@ -99,6 +101,9 @@ const (
 	SubTypeAzureSQLServer           AssetSubType = "azure-sql-server"
 	SubTypeAzureSnapshot            AssetSubType = "azure-snapshot"
 	SubTypeAzureStorageAccount      AssetSubType = "azure-storage-account"
+	SubTypeAzureStorageBlobService  AssetSubType = "azure-storage-blob-service"
+	SubTypeAzureStorageQueue        AssetSubType = "azure-storage-queue"
+	SubTypeAzureStorageQueueService AssetSubType = "azure-storage-queue-service"
 	SubTypeAzureSubscription        AssetSubType = "azure-subscription"
 	SubTypeAzureTenant              AssetSubType = "azure-tenant"
 	SubTypeAzureVirtualMachine      AssetSubType = "azure-virtual-machine"
@@ -193,6 +198,9 @@ var (
 	AssetClassificationAzureSQLServer           = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryDatabase, Type: TypeRelationalDatabase, SubType: SubTypeAzureSQLServer}
 	AssetClassificationAzureSnapshot            = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryStorage, Type: TypeSnapshot, SubType: SubTypeAzureSnapshot}
 	AssetClassificationAzureStorageAccount      = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryStorage, Type: TypeStorage, SubType: SubTypeAzureStorageAccount}
+	AssetClassificationAzureStorageBlobService  = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryStorage, Type: TypeObjectStorage, SubType: SubTypeAzureStorageBlobService}
+	AssetClassificationAzureStorageQueue        = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryApplicationIntegration, Type: TypeMessageQueue, SubType: SubTypeAzureStorageQueue}
+	AssetClassificationAzureStorageQueueService = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryApplicationIntegration, Type: TypeMessageQueue, SubType: SubTypeAzureStorageQueueService}
 	AssetClassificationAzureSubscription        = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryManagement, Type: TypeCloudAccount, SubType: SubTypeAzureSubscription}
 	AssetClassificationAzureTenant              = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryManagement, Type: TypeCloudAccount, SubType: SubTypeAzureTenant}
 	AssetClassificationAzureVirtualMachine      = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryCompute, Type: TypeVirtualMachine, SubType: SubTypeAzureVirtualMachine}

internal/inventory/azurefetcher/azurefetchers.go

@@ -29,5 +29,6 @@ func New(logger *logp.Logger, provider azurelib.ProviderAPI, _ *azure_auth.Azure
 	return []inventory.AssetFetcher{
 		newAccountFetcher(logger, provider),
 		newResourceGraphFetcher(logger, provider),
+		newStorageFetcher(logger, provider),
 	}
 }

internal/inventory/azurefetcher/fetcher_account.go

@@ -28,18 +28,18 @@ import (
 
 type accountFetcher struct {
 	logger   *logp.Logger
-	provider subscriptionProvider
+	provider accountProvider
 }
 
 type (
-	subscriptionProviderFunc func(context.Context) ([]azurelib.AzureAsset, error)
-	subscriptionProvider     interface {
+	accountProviderFunc func(context.Context) ([]azurelib.AzureAsset, error)
+	accountProvider     interface {
 		ListTenants(ctx context.Context) ([]azurelib.AzureAsset, error)
 		ListSubscriptions(ctx context.Context) ([]azurelib.AzureAsset, error)
 	}
 )
 
-func newAccountFetcher(logger *logp.Logger, provider subscriptionProvider) inventory.AssetFetcher {
+func newAccountFetcher(logger *logp.Logger, provider accountProvider) inventory.AssetFetcher {
 	return &accountFetcher{
 		logger:   logger,
 		provider: provider,
@@ -49,7 +49,7 @@ func newAccountFetcher(logger *logp.Logger, provider subscriptionProvider) inven
 func (f *accountFetcher) Fetch(ctx context.Context, assetChan chan<- inventory.AssetEvent) {
 	resourcesToFetch := []struct {
 		name           string
-		function       subscriptionProviderFunc
+		function       accountProviderFunc
 		classification inventory.AssetClassification
 	}{
 		{"Tenants", f.provider.ListTenants, inventory.AssetClassificationAzureTenant},
@@ -60,7 +60,7 @@ func (f *accountFetcher) Fetch(ctx context.Context, assetChan chan<- inventory.A
 	}
 }
 
-func (f *accountFetcher) fetch(ctx context.Context, resourceName string, function subscriptionProviderFunc, classification inventory.AssetClassification, assetChan chan<- inventory.AssetEvent) {
+func (f *accountFetcher) fetch(ctx context.Context, resourceName string, function accountProviderFunc, classification inventory.AssetClassification, assetChan chan<- inventory.AssetEvent) {
 	f.logger.Infof("Fetching %s", resourceName)
 	defer f.logger.Infof("Fetching %s - Finished", resourceName)
 

internal/inventory/azurefetcher/fetcher_account_test.go

@@ -57,7 +57,7 @@ func TestAccountFetcher_Fetch_Tenants(t *testing.T) {
 
 	// setup
 	logger := logp.NewLogger("azurefetcher_test")
-	provider := newMockSubscriptionProvider(t)
+	provider := newMockAccountProvider(t)
 	provider.EXPECT().ListTenants(mock.Anything).Return(azureAssets, nil)
 	provider.EXPECT().ListSubscriptions(mock.Anything).Return(nil, nil)
 	fetcher := newAccountFetcher(logger, provider)
@@ -94,7 +94,7 @@ func TestAccountFetcher_Fetch_Subscriptions(t *testing.T) {
 
 	// setup
 	logger := logp.NewLogger("azurefetcher_test")
-	provider := newMockSubscriptionProvider(t)
+	provider := newMockAccountProvider(t)
 	provider.EXPECT().ListTenants(mock.Anything).Return(nil, nil)
 	provider.EXPECT().ListSubscriptions(mock.Anything).Return(azureAssets, nil)
 	fetcher := newAccountFetcher(logger, provider)

internal/inventory/azurefetcher/fetcher_storage.go

@@ -0,0 +1,121 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package azurefetcher
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/elastic/elastic-agent-libs/logp"
+
+	"github.com/elastic/cloudbeat/internal/inventory"
+	azurelib "github.com/elastic/cloudbeat/internal/resources/providers/azurelib/inventory"
+)
+
+type storageFetcher struct {
+	logger   *logp.Logger
+	provider storageProvider
+}
+
+type (
+	storageProviderFunc func(context.Context, []azurelib.AzureAsset) ([]azurelib.AzureAsset, error)
+	storageProvider     interface {
+		ListSubscriptions(ctx context.Context) ([]azurelib.AzureAsset, error)
+		ListStorageAccountBlobServices(ctx context.Context, storageAccounts []azurelib.AzureAsset) ([]azurelib.AzureAsset, error)
+		ListStorageAccountQueues(ctx context.Context, storageAccounts []azurelib.AzureAsset) ([]azurelib.AzureAsset, error)
+		ListStorageAccountQueueServices(ctx context.Context, storageAccounts []azurelib.AzureAsset) ([]azurelib.AzureAsset, error)
+		ListStorageAccounts(ctx context.Context, storageAccountsSubscriptionsIds []string) ([]azurelib.AzureAsset, error)
+	}
+)
+
+func newStorageFetcher(logger *logp.Logger, provider storageProvider) inventory.AssetFetcher {
+	return &storageFetcher{
+		logger:   logger,
+		provider: provider,
+	}
+}
+
+func (f *storageFetcher) Fetch(ctx context.Context, assetChan chan<- inventory.AssetEvent) {
+	resourcesToFetch := []struct {
+		name           string
+		function       storageProviderFunc
+		classification inventory.AssetClassification
+	}{
+		{"Storage Blob Services", f.provider.ListStorageAccountBlobServices, inventory.AssetClassificationAzureStorageBlobService},
+		{"Storage Queue Services", f.provider.ListStorageAccountQueueServices, inventory.AssetClassificationAzureStorageQueueService},
+		{"Storage Queues", f.provider.ListStorageAccountQueues, inventory.AssetClassificationAzureStorageQueue},
+	}
+
+	storageAccounts, err := f.listStorageAccounts(ctx)
+	if err != nil {
+		f.logger.Errorf("Could not fetch anything: %v", err)
+		return
+	}
+
+	for _, r := range resourcesToFetch {
+		f.fetch(ctx, storageAccounts, r.name, r.function, r.classification, assetChan)
+	}
+}
+
+func (f *storageFetcher) listStorageAccounts(ctx context.Context) ([]azurelib.AzureAsset, error) {
+	subscriptions, err := f.provider.ListSubscriptions(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("error listing subscriptions: %v", err)
+	}
+
+	subscriptionIds := make([]string, len(subscriptions))
+	for _, subscription := range subscriptions {
+		subscriptionIds = append(subscriptionIds, subscription.Name)
+	}
+
+	storageAccounts, err := f.provider.ListStorageAccounts(ctx, subscriptionIds)
+	if err != nil {
+		return nil, fmt.Errorf("error listing storage accounts: %v", err)
+	}
+
+	return storageAccounts, nil
+}
+
+func (f *storageFetcher) fetch(ctx context.Context, storageAccounts []azurelib.AzureAsset, resourceName string, function storageProviderFunc, classification inventory.AssetClassification, assetChan chan<- inventory.AssetEvent) {
+	f.logger.Infof("Fetching %s", resourceName)
+	defer f.logger.Infof("Fetching %s - Finished", resourceName)
+
+	azureAssets, err := function(ctx, storageAccounts)
+	if err != nil {
+		f.logger.Errorf("Could not fetch %s: %v", resourceName, err)
+		return
+	}
+
+	for _, item := range azureAssets {
+		assetChan <- inventory.NewAssetEvent(
+			classification,
+			[]string{item.Id},
+			item.DisplayName,
+			inventory.WithRawAsset(item),
+			inventory.WithCloud(inventory.AssetCloud{
+				Provider: inventory.AzureCloudProvider,
+				Account: inventory.AssetCloudAccount{
+					Id: item.TenantId,
+				},
+				Service: &inventory.AssetCloudService{
+					Name: "Azure",
+				},
+			}),
+		)
+	}
+}