Skip to content

Commit

Permalink
[AWS Orgs] Implement running fetchers for each account (#1086)
Browse files Browse the repository at this point in the history
  • Loading branch information
orestisfl authored Jul 17, 2023
1 parent c444c93 commit ea05488
Show file tree
Hide file tree
Showing 25 changed files with 1,619 additions and 163 deletions.
1 change: 1 addition & 0 deletions .golangci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ linters:
enable:
- contextcheck
- exhaustruct
- exportloopref
- gocyclo
- reassign
- misspell
Expand Down
6 changes: 0 additions & 6 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ import (
"github.com/elastic/beats/v7/libbeat/processors"
"github.com/elastic/beats/v7/x-pack/libbeat/common/aws"
"github.com/elastic/elastic-agent-libs/config"
"github.com/elastic/elastic-agent-libs/logp"

"github.com/elastic/cloudbeat/launcher"
)
Expand Down Expand Up @@ -100,11 +99,6 @@ func New(cfg *config.C) (*Config, error) {
case "":
case SingleAccount:
case OrganizationAccount:
logp.NewLogger("config").Errorf(
"aws.account_type '%s' not implemented yet",
c.CloudConfig.Aws.AccountType,
)
c.CloudConfig.Aws.AccountType = SingleAccount
default:
return nil, launcher.NewUnhealthyError(fmt.Sprintf(
"aws.account_type '%s' is not supported",
Expand Down
4 changes: 2 additions & 2 deletions config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ config:
v1:
benchmark: cis_eks
aws:
account_type: single_account
account_type: organization_account
credentials:
access_key_id: key
secret_access_key: secret
Expand All @@ -84,7 +84,7 @@ config:
ProfileName: "credential_profile_name",
RoleArn: "role_arn",
},
AccountType: "single_account",
AccountType: "organization_account",
},
},
3,
Expand Down
5 changes: 3 additions & 2 deletions dataprovider/providers/aws/data_provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (

"github.com/elastic/cloudbeat/dataprovider/types"
"github.com/elastic/cloudbeat/resources/fetching"
"github.com/elastic/cloudbeat/resources/utils/strings"
"github.com/elastic/cloudbeat/version"
)

Expand Down Expand Up @@ -59,12 +60,12 @@ func (a DataProvider) FetchData(_ string, id string) (types.Data, error) {
}

func (a DataProvider) EnrichEvent(event *beat.Event, resMetadata fetching.ResourceMetadata) error {
_, err := event.Fields.Put(cloudAccountIdField, a.accountId)
_, err := event.Fields.Put(cloudAccountIdField, strings.FirstNonEmpty(resMetadata.AwsAccountId, a.accountId))
if err != nil {
return err
}

_, err = event.Fields.Put(cloudAccountNameField, a.accountName)
_, err = event.Fields.Put(cloudAccountNameField, strings.FirstNonEmpty(resMetadata.AwsAccountAlias, a.accountName))
if err != nil {
return err
}
Expand Down
107 changes: 80 additions & 27 deletions dataprovider/providers/aws/data_provider_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"github.com/elastic/beats/v7/libbeat/beat"
"github.com/elastic/elastic-agent-libs/mapstr"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"

"github.com/elastic/cloudbeat/dataprovider/types"
Expand Down Expand Up @@ -92,35 +93,87 @@ func (s *AwsDataProviderTestSuite) TestAwsDataProvider_FetchData() {
}
}

func TestAWSDataProvider_EnrichEvent(t *testing.T) {
options := []Option{
WithLogger(testhelper.NewLogger(t)),
WithAccount(awslib.Identity{
Account: accountId,
Alias: accountName,
}),
func TestDataProvider_EnrichEvent(t *testing.T) {
tests := []struct {
name string
resMetadata fetching.ResourceMetadata
identity awslib.Identity
expectedFields map[string]string
}{
{
name: "no replacement",
resMetadata: fetching.ResourceMetadata{
Region: someRegion,
},
identity: awslib.Identity{
Account: accountId,
Alias: accountName,
},
expectedFields: map[string]string{
cloudAccountIdField: accountId,
cloudAccountNameField: accountName,
cloudProviderField: "aws",
cloudRegionField: someRegion,
},
},
{
name: "replace alias",
resMetadata: fetching.ResourceMetadata{
Region: someRegion,
AwsAccountId: "",
AwsAccountAlias: "some alias",
},
identity: awslib.Identity{
Account: accountId,
Alias: accountName,
},
expectedFields: map[string]string{
cloudAccountIdField: accountId,
cloudAccountNameField: "some alias",
cloudProviderField: "aws",
cloudRegionField: someRegion,
},
},
{
name: "replace both",
resMetadata: fetching.ResourceMetadata{
Region: someRegion,
AwsAccountId: "12345654321",
AwsAccountAlias: "some alias",
},
identity: awslib.Identity{
Account: accountId,
Alias: accountName,
},
expectedFields: map[string]string{
cloudAccountIdField: "12345654321",
cloudAccountNameField: "some alias",
cloudProviderField: "aws",
cloudRegionField: someRegion,
},
},
}

k := New(options...)
e := &beat.Event{
Fields: mapstr.M{},
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
p := New(WithLogger(testhelper.NewLogger(t)), WithAccount(tt.identity))
e := &beat.Event{
Fields: mapstr.M{},
}

err := p.EnrichEvent(e, tt.resMetadata)
require.NoError(t, err)

for key, expectedValue := range tt.expectedFields {
assertField(t, e.Fields, key, expectedValue)
}
})
}
err := k.EnrichEvent(e, fetching.ResourceMetadata{Region: someRegion})
assert.NoError(t, err)

accountID, err := e.Fields.GetValue(cloudAccountIdField)
assert.NoError(t, err)
assert.Equal(t, "accountId", accountID)

accountName, err := e.Fields.GetValue(cloudAccountNameField)
assert.NoError(t, err)
assert.Equal(t, "accountName", accountName)
}

cloud, err := e.Fields.GetValue(cloudProviderField)
assert.NoError(t, err)
assert.Equal(t, "aws", cloud)
func assertField(t *testing.T, fields mapstr.M, key string, expectedValue string) {
t.Helper()

region, err := e.Fields.GetValue(cloudRegionField)
assert.NoError(t, err)
assert.Equal(t, someRegion, region)
got, err := fields.GetValue(key)
require.NoError(t, err)
assert.Equal(t, expectedValue, got)
}
131 changes: 131 additions & 0 deletions flavors/benchmark/aws_org.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package benchmark

import (
"context"
"fmt"

awssdk "github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
"github.com/aws/aws-sdk-go-v2/service/sts"
"github.com/elastic/beats/v7/x-pack/libbeat/common/aws"
"github.com/elastic/elastic-agent-libs/logp"

"github.com/elastic/cloudbeat/config"
"github.com/elastic/cloudbeat/dataprovider"
aws_dataprovider "github.com/elastic/cloudbeat/dataprovider/providers/aws"
"github.com/elastic/cloudbeat/resources/fetching"
"github.com/elastic/cloudbeat/resources/fetching/factory"
"github.com/elastic/cloudbeat/resources/fetching/registry"
"github.com/elastic/cloudbeat/resources/providers/awslib"
)

type AWSOrg struct{}

func (A *AWSOrg) Initialize(
ctx context.Context,
log *logp.Logger,
cfg *config.Config,
ch chan fetching.ResourceInfo,
dependencies *Dependencies,
) (registry.Registry, dataprovider.CommonDataProvider, error) {
// TODO: make this mock-able
awsConfig, err := aws.InitializeAWSConfig(cfg.CloudConfig.Aws.Cred)
if err != nil {
return nil, nil, fmt.Errorf("failed to initialize AWS credentials: %w", err)
}

awsIdentity, err := dependencies.AWSIdentity(ctx, awsConfig)
if err != nil {
return nil, nil, fmt.Errorf("failed to get AWS identity: %w", err)
}

accounts, err := getAwsAccounts(ctx, awsConfig, dependencies, awsIdentity)
if err != nil {
return nil, nil, fmt.Errorf("failed to get AWS accounts: %w", err)
}

return registry.NewRegistry(
log,
factory.NewCisAwsOrganizationFactory(ctx, log, ch, accounts),
), aws_dataprovider.New(
aws_dataprovider.WithLogger(log),
aws_dataprovider.WithAccount(*awsIdentity),
), nil
}

func getAwsAccounts(
ctx context.Context,
initialCfg awssdk.Config,
dependencies *Dependencies,
rootIdentity *awslib.Identity,
) ([]factory.AwsAccount, error) {
const (
rootRole = "cloudbeat-root"
memberRole = "cloudbeat-securityaudit"
)

rootCfg := assumeRole(
sts.NewFromConfig(initialCfg),
initialCfg,
fmtIAMRole(rootIdentity.Account, rootRole),
)
stsClient := sts.NewFromConfig(rootCfg)

accountIdentities, err := dependencies.AWSAccounts(ctx, rootCfg)
if err != nil {
return nil, err
}

accounts := []factory.AwsAccount{
{
Identity: *rootIdentity,
Config: rootCfg,
},
}
for _, identity := range accountIdentities {
if identity.Account == rootIdentity.Account {
continue
}

memberCfg := assumeRole(
stsClient,
rootCfg,
fmtIAMRole(identity.Account, memberRole),
)

accounts = append(accounts, factory.AwsAccount{
Identity: identity,
Config: memberCfg,
})
}
return accounts, nil
}

func assumeRole(client *sts.Client, cfg awssdk.Config, arn string) awssdk.Config {
cfg.Credentials = awssdk.NewCredentialsCache(stscreds.NewAssumeRoleProvider(client, arn))
return cfg
}

func fmtIAMRole(account string, role string) string {
return fmt.Sprintf("arn:aws:iam::%s:role/%s", account, role)
}

func (A *AWSOrg) Run(context.Context) error { return nil }
func (A *AWSOrg) Stop() {}
Loading

0 comments on commit ea05488

Please sign in to comment.