Skip to content

Commit

Permalink
[Asset Inventory POC] Fetch S3 buckets (#2045)
Browse files Browse the repository at this point in the history 
* Fetch S3 buckets

* Enrich cloud field

* Convert bucket policy to an indexable format
  • Loading branch information
@romulets
romulets authored Mar 21, 2024
1 parent 963e244 commit b1fd4ae
Show file tree
Hide file tree
Showing 13 changed files with 727 additions and 98 deletions.
94 changes: 72 additions & 22 deletions internal/inventory/asset.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,43 +35,45 @@ type assetSubCategory string

const (
SubCategoryCompute assetSubCategory = "compute"
SubCategoryStorage assetSubCategory = "storage"
)

// assetType is used to build the document index. Use only numbers, letters and dashes (-)
type assetType string

const (
TypeVirtualMachine assetType = "virtual-machine"
TypeObjectStorage assetType = "object-storage"
)

// assetSubType is used to build the document index. Use only numbers, letters and dashes (-)
type assetSubType string

const (
SubTypeEC2 assetSubType = "ec2"
SubTypeS3 assetSubType = "s3"
)

type assetCloudProvider string

const (
AwsCloudProvider assetCloudProvider = "aws"
AwsCloudProvider = "aws"
)

// AssetEvent holds the whole asset
type AssetEvent struct {
Asset Asset
Network *AssetNetwork
Cloud *AssetCloud
Host *AssetHost
IAM *AssetIAM
Asset Asset
Network *AssetNetwork
Cloud *AssetCloud
Host *AssetHost
IAM *AssetIAM
ResourcePolicies []AssetResourcePolicy
}

// AssetClassification holds the taxonomy of an asset
type AssetClassification struct {
Category assetCategory `json:"category"`
SubCategory assetSubCategory `json:"subCategory"`
SubCategory assetSubCategory `json:"sub_category"`
Type assetType `json:"type"`
SubStype assetSubType `json:"subStype"`
SubType assetSubType `json:"sub_type"`
}

// Asset contains the identifiers of the asset
Expand All @@ -86,35 +88,77 @@ type Asset struct {

// AssetNetwork contains network information
type AssetNetwork struct {
NetworkId *string `json:"networkId"`
SubnetId *string `json:"subnetId"`
Ipv6Address *string `json:"ipv6Address"`
PublicIpAddress *string `json:"publicIpAddress"`
PrivateIpAddress *string `json:"privateIpAddress"`
PublicDnsName *string `json:"publicDnsName"`
PrivateDnsName *string `json:"privateDnsName"`
NetworkId *string `json:"network_id"`
SubnetId *string `json:"subnet_id"`
Ipv6Address *string `json:"ipv6_address"`
PublicIpAddress *string `json:"public_ip_address"`
PrivateIpAddress *string `json:"private_ip_address"`
PublicDnsName *string `json:"public_dns_name"`
PrivateDnsName *string `json:"private_dns_name"`
}

// AssetCloud contains information about the cloud provider
type AssetCloud struct {
Provider assetCloudProvider `json:"provider"`
Region string `json:"region"`
AvailabilityZone *string `json:"availability_zone,omitempty"`
Provider string `json:"provider,omitempty"`
Region string `json:"region,omitempty"`
Account AssetCloudAccount `json:"account"`
Instance *AssetCloudInstance `json:"instance,omitempty"`
Machine *AssetCloudMachine `json:"machine,omitempty"`
Project *AssetCloudProject `json:"project,omitempty"`
Service *AssetCloudService `json:"service,omitempty"`
}

type AssetCloudAccount struct {
Id string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
}

type AssetCloudInstance struct {
Id string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
}

type AssetCloudMachine struct {
MachineType string `json:"machine_type,omitempty"`
}

type AssetCloudProject struct {
Id string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
}

type AssetCloudService struct {
Name string `json:"name,omitempty"`
}

// AssetHost contains information of the asset in case it is a host
type AssetHost struct {
Architecture string `json:"architecture"`
ImageId *string `json:"imageId"`
InstanceType string `json:"instanceType"`
InstanceType string `json:"instance_type"`
Platform string `json:"platform"`
PlatformDetails *string `json:"platformDetails"`
PlatformDetails *string `json:"platform_details"`
}

type AssetIAM struct {
Id *string `json:"id"`
Arn *string `json:"arn"`
}

// AssetResourcePolicy maps security policies applied directly on resources
type AssetResourcePolicy struct {
Version *string `json:"version,omitempty"`
Id *string `json:"id,omitempty"`
Effect string `json:"effect,omitempty"`
Principal map[string]any `json:"principal,omitempty"`
Action []string `json:"action,omitempty"`
NotAction []string `json:"notAction,omitempty"`
Resource []string `json:"resource,omitempty"`
NoResource []string `json:"noResource,omitempty"`
Condition map[string]any `json:"condition,omitempty"`
}

// AssetEnricher functional builder function
type AssetEnricher func(asset *AssetEvent)

Expand Down Expand Up @@ -171,13 +215,19 @@ func WithIAM(iam AssetIAM) AssetEnricher {
}
}

func WithResourcePolicies(policies ...AssetResourcePolicy) AssetEnricher {
return func(a *AssetEvent) {
a.ResourcePolicies = policies
}
}

func EmptyEnricher() AssetEnricher {
return func(_ *AssetEvent) {}
}

func generateUniqueId(c AssetClassification, resourceId string) string {
hasher := sha256.New()
toBeHashed := fmt.Sprintf("%s-%s-%s-%s-%s", resourceId, c.Category, c.SubCategory, c.Type, c.SubStype)
toBeHashed := fmt.Sprintf("%s-%s-%s-%s-%s", resourceId, c.Category, c.SubCategory, c.Type, c.SubType)
hasher.Write([]byte(toBeHashed)) //nolint:revive
hash := hasher.Sum(nil)
encoded := base64.StdEncoding.EncodeToString(hash)
Expand Down
76 changes: 53 additions & 23 deletions internal/inventory/aws/fetcher_ec2.go → ...nal/inventory/aws/fetcher_ec2_instance.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,31 +30,35 @@ import (
"github.com/elastic/cloudbeat/internal/resources/utils/pointers"
)

type Ec2Fetcher struct {
logger *logp.Logger
provider instancesProvider
type Ec2InstanceFetcher struct {
logger *logp.Logger
provider ec2InstancesProvider
AccountId string
AccountName string
}

type instancesProvider interface {
type ec2InstancesProvider interface {
DescribeInstances(ctx context.Context) ([]*ec2.Ec2Instance, error)
}

var ec2Classification = inventory.AssetClassification{
var ec2InstanceClassification = inventory.AssetClassification{
Category: inventory.CategoryInfrastructure,
SubCategory: inventory.SubCategoryCompute,
Type: inventory.TypeVirtualMachine,
SubStype: inventory.SubTypeEC2,
SubType: inventory.SubTypeEC2,
}

func newEc2Fetcher(logger *logp.Logger, identity *cloud.Identity, cfg aws.Config) inventory.AssetFetcher {
provider := ec2.NewEC2Provider(logger, identity.Account, cfg, &awslib.MultiRegionClientFactory[ec2.Client]{})
return &Ec2Fetcher{
logger: logger,
provider: provider,
return &Ec2InstanceFetcher{
logger: logger,
provider: provider,
AccountId: identity.Account,
AccountName: identity.AccountAlias,
}
}

func (e *Ec2Fetcher) Fetch(ctx context.Context, assetChannel chan<- inventory.AssetEvent) {
func (e *Ec2InstanceFetcher) Fetch(ctx context.Context, assetChannel chan<- inventory.AssetEvent) {
instances, err := e.provider.DescribeInstances(ctx)
if err != nil {
e.logger.Errorf("Could not list ec2 instances: %v", err)
Expand All @@ -74,25 +78,31 @@ func (e *Ec2Fetcher) Fetch(ctx context.Context, assetChannel chan<- inventory.As
})
}

tags := make(map[string]string, len(instance.Tags))
for _, t := range instance.Tags {
if t.Key == nil {
continue
}

tags[*t.Key] = pointers.Deref(t.Value)
}

assetChannel <- inventory.NewAssetEvent(
ec2Classification,
ec2InstanceClassification,
instance.GetResourceArn(),
instance.GetResourceName(),

inventory.WithRawAsset(instance),
inventory.WithTags(tags),
inventory.WithTags(e.getTags(instance)),
inventory.WithCloud(inventory.AssetCloud{
Provider: inventory.AwsCloudProvider,
Region: instance.Region,
Provider: inventory.AwsCloudProvider,
Region: instance.Region,
AvailabilityZone: e.getAvailabilityZone(instance),
Account: inventory.AssetCloudAccount{
Id: e.AccountId,
Name: e.AccountName,
},
Instance: &inventory.AssetCloudInstance{
Id: pointers.Deref(instance.InstanceId),
Name: instance.GetResourceName(),
},
Machine: &inventory.AssetCloudMachine{
MachineType: string(instance.InstanceType),
},
Service: &inventory.AssetCloudService{
Name: "AWS EC2",
},
}),
inventory.WithHost(inventory.AssetHost{
Architecture: string(instance.Architecture),
Expand All @@ -114,3 +124,23 @@ func (e *Ec2Fetcher) Fetch(ctx context.Context, assetChannel chan<- inventory.As
)
}
}

func (e *Ec2InstanceFetcher) getTags(instance *ec2.Ec2Instance) map[string]string {
tags := make(map[string]string, len(instance.Tags))
for _, t := range instance.Tags {
if t.Key == nil {
continue
}

tags[*t.Key] = pointers.Deref(t.Value)
}
return tags
}

func (e *Ec2InstanceFetcher) getAvailabilityZone(instance *ec2.Ec2Instance) *string {
if instance.Placement == nil {
return nil
}

return instance.Placement.AvailabilityZone
}
52 changes: 43 additions & 9 deletions internal/inventory/aws/fetcher_ec2_test.go → ...nventory/aws/fetcher_ec2_instance_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import (
"github.com/elastic/cloudbeat/internal/resources/utils/pointers"
)

func TestFetch(t *testing.T) {
func TestEC2InstanceFetcher_Fetch(t *testing.T) {
instance1 := &ec2beat.Ec2Instance{
Instance: types.Instance{
IamInstanceProfile: &types.IamInstanceProfile{
Expand Down Expand Up @@ -62,6 +62,9 @@ func TestFetch(t *testing.T) {
PrivateIpAddress: pointers.Ref("private-ip-addre"),
PublicDnsName: pointers.Ref("public-dns"),
PrivateDnsName: pointers.Ref("private-dns"),
Placement: &types.Placement{
AvailabilityZone: pointers.Ref("1a"),
},
},
Region: "us-east",
}
Expand All @@ -75,14 +78,29 @@ func TestFetch(t *testing.T) {

expected := []inventory.AssetEvent{
inventory.NewAssetEvent(
ec2Classification,
ec2InstanceClassification,
"arn:aws:ec2:us-east::ec2/234567890",
"test-server",
inventory.WithRawAsset(instance1),
inventory.WithTags(map[string]string{"Name": "test-server", "key": "value"}),
inventory.WithCloud(inventory.AssetCloud{
Provider: inventory.AwsCloudProvider,
Region: "us-east",
Provider: inventory.AwsCloudProvider,
Region: "us-east",
AvailabilityZone: pointers.Ref("1a"),
Account: inventory.AssetCloudAccount{
Id: "123",
Name: "alias",
},
Instance: &inventory.AssetCloudInstance{
Id: "234567890",
Name: "test-server",
},
Machine: &inventory.AssetCloudMachine{
MachineType: "instance-type",
},
Service: &inventory.AssetCloudService{
Name: "AWS EC2",
},
}),
inventory.WithHost(inventory.AssetHost{
Architecture: string(types.ArchitectureValuesX8664),
Expand All @@ -107,27 +125,43 @@ func TestFetch(t *testing.T) {
),

inventory.NewAssetEvent(
ec2Classification,
ec2InstanceClassification,
"",
"",
inventory.WithRawAsset(instance2),
inventory.WithTags(map[string]string{}),
inventory.WithCloud(inventory.AssetCloud{
Provider: inventory.AwsCloudProvider,
Region: "us-east",
Account: inventory.AssetCloudAccount{
Id: "123",
Name: "alias",
},
Instance: &inventory.AssetCloudInstance{
Id: "",
Name: "",
},
Machine: &inventory.AssetCloudMachine{
MachineType: "",
},
Service: &inventory.AssetCloudService{
Name: "AWS EC2",
},
}),
inventory.WithHost(inventory.AssetHost{}),
inventory.WithNetwork(inventory.AssetNetwork{}),
),
}

logger := logp.NewLogger("test_fetcher_ec2")
provider := newMockInstancesProvider(t)
provider := newMockEc2InstancesProvider(t)
provider.EXPECT().DescribeInstances(mock.Anything).Return(in, nil)

fetcher := Ec2Fetcher{
logger: logger,
provider: provider,
fetcher := Ec2InstanceFetcher{
logger: logger,
provider: provider,
AccountId: "123",
AccountName: "alias",
}

ch := make(chan inventory.AssetEvent)
Expand Down
Loading

0 comments on commit b1fd4ae

Please sign in to comment.