Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Cloud data provider #1144

Merged
merged 25 commits into from
Jul 25, 2023
Merged

Cloud data provider #1144

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
e97e4f5
add util
uri-weisman Jul 10, 2023
17c044f
gcp benchmark setup
orouz Jul 16, 2023
d0fdd55
add gcp assets inventory sdk
orouz Jul 16, 2023
9856702
add empty gcp data provider
orouz Jul 16, 2023
404dd97
add gcp factory, fetcher and provider
orouz Jul 16, 2023
e8b7a1f
move common funcs to utils
orouz Jul 16, 2023
3ae8690
Merge branch 'main' into gcp_data_provider
uri-weisman Jul 16, 2023
4a1ff37
Merge remote-tracking branch 'or/gcp_assets' into gcp_data_provider
uri-weisman Jul 16, 2023
239d526
with recursion
uri-weisman Jul 17, 2023
6c1c231
with recursion
uri-weisman Jul 17, 2023
91db6f4
Merge branch 'main' into gcp_data_provider
uri-weisman Jul 18, 2023
3915978
add tests
uri-weisman Jul 18, 2023
b25a0b8
prepare for review
uri-weisman Jul 18, 2023
951e315
remove org fields
uri-weisman Jul 19, 2023
6382618
prepare for review
uri-weisman Jul 19, 2023
86aec2e
remove gcp dp
uri-weisman Jul 19, 2023
a720267
cr changes
uri-weisman Jul 25, 2023
8a42ced
Merge remote-tracking branch 'upstream/main' into gcp_data_provider
uri-weisman Jul 25, 2023
118a67d
cr changes 2
uri-weisman Jul 25, 2023
cdd8601
Merge branch 'main' into gcp_data_provider
uri-weisman Jul 25, 2023
57733ba
cr changes 3
uri-weisman Jul 25, 2023
abc7de3
cr changes 3
uri-weisman Jul 25, 2023
859c48a
Merge branch 'gcp_data_provider' of https://github.com/uri-weisman/cl…
uri-weisman Jul 25, 2023
957b2c0
update mocks
uri-weisman Jul 25, 2023
9afd1ea
cr changes 4
uri-weisman Jul 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 0 additions & 83 deletions dataprovider/providers/aws/mock_option.go
View file
Open in desktop

This file was deleted.

50 changes: 40 additions & 10 deletions dataprovider/providers/aws/data_provider.go → ...provider/providers/cloud/data_provider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.

package aws
package cloud

import (
"github.com/elastic/beats/v7/libbeat/beat"
Expand All @@ -32,16 +32,28 @@ const (
cloudAccountNameField = "cloud.account.name"
cloudProviderField = "cloud.provider"
cloudRegionField = "cloud.region"
cloudProviderValue = "aws"
cloudProjectIdField = "cloud.project.id"
cloudProjectNameField = "cloud.project.name"
)

type Identity struct {
Provider string
Account string
AccountAlias string
ProjectId string
ProjectName string
}

type DataProvider struct {
log *logp.Logger
accountId string
accountName string
log *logp.Logger
accountId string
accountName string
providerName string
projectName string
projectId string
}

func New(options ...Option) DataProvider {
func NewDataProvider(options ...Option) DataProvider {
adp := DataProvider{}
for _, opt := range options {
opt(&adp)
Expand All @@ -60,25 +72,43 @@ func (a DataProvider) FetchData(_ string, id string) (types.Data, error) {
}

func (a DataProvider) EnrichEvent(event *beat.Event, resMetadata fetching.ResourceMetadata) error {
_, err := event.Fields.Put(cloudAccountIdField, strings.FirstNonEmpty(resMetadata.AwsAccountId, a.accountId))
err := insertIfNotEmpty(cloudAccountIdField, strings.FirstNonEmpty(resMetadata.AwsAccountId, a.accountId), event)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Blocker: make sure this doesn't break aws orgs account fields (LGTM but let's double check)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@orestisfl how this can be validated?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right now, manually since we don't have E2E tests yet.

However, the unit tests cover the case already so maybe it's overkill.

if err != nil {
return err
}

err = insertIfNotEmpty(cloudAccountNameField, strings.FirstNonEmpty(resMetadata.AwsAccountAlias, a.accountName), event)
if err != nil {
return err
}

_, err = event.Fields.Put(cloudAccountNameField, strings.FirstNonEmpty(resMetadata.AwsAccountAlias, a.accountName))
err = insertIfNotEmpty(cloudProviderField, a.providerName, event)
if err != nil {
return err
}

_, err = event.Fields.Put(cloudProviderField, cloudProviderValue)
err = insertIfNotEmpty(cloudRegionField, resMetadata.Region, event)
if err != nil {
return err
}

_, err = event.Fields.Put(cloudRegionField, resMetadata.Region)
err = insertIfNotEmpty(cloudProjectIdField, a.projectId, event)
if err != nil {
return err
}

err = insertIfNotEmpty(cloudProjectNameField, a.projectName, event)
if err != nil {
return err
}

return nil
}

func insertIfNotEmpty(field string, value string, event *beat.Event) error {
if value != "" {
_, err := event.Fields.Put(field, value)
return err
}
return nil
}
81 changes: 52 additions & 29 deletions ...vider/providers/aws/data_provider_test.go → ...der/providers/cloud/data_provider_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.

package aws
package cloud

import (
"testing"
Expand All @@ -28,32 +28,35 @@ import (

"github.com/elastic/cloudbeat/dataprovider/types"
"github.com/elastic/cloudbeat/resources/fetching"
"github.com/elastic/cloudbeat/resources/providers/awslib"
"github.com/elastic/cloudbeat/resources/utils/testhelper"
"github.com/elastic/cloudbeat/version"
)

var (
accountName = "accountName"
accountId = "accountId"
someRegion = "eu-west-1"
accountName = "accountName"
accountId = "accountId"
awsProvider = "aws"
gcpProvider = "gcp"
gcpProjectName = "projectName"
gcpProjectId = "projectId"
someRegion = "eu-west-1"
)

type AwsDataProviderTestSuite struct {
type CloudDataProviderTestSuite struct {
suite.Suite
}

func TestAwsDataProviderTestSuite(t *testing.T) {
s := new(AwsDataProviderTestSuite)
func TestCloudDataProviderTestSuite(t *testing.T) {
s := new(CloudDataProviderTestSuite)

suite.Run(t, s)
}

func (s *AwsDataProviderTestSuite) SetupTest() {}
func (s *CloudDataProviderTestSuite) SetupTest() {}

func (s *AwsDataProviderTestSuite) TearDownTest() {}
func (s *CloudDataProviderTestSuite) TearDownTest() {}

func (s *AwsDataProviderTestSuite) TestAwsDataProvider_FetchData() {
func (s *CloudDataProviderTestSuite) TestAwsDataProvider_FetchData() {
tests := []struct {
name string
options []Option
Expand All @@ -66,9 +69,9 @@ func (s *AwsDataProviderTestSuite) TestAwsDataProvider_FetchData() {
name: "get data",
options: []Option{
WithLogger(testhelper.NewLogger(s.T())),
WithAccount(awslib.Identity{
Account: accountId,
Alias: accountName,
WithAccount(Identity{
Account: accountId,
AccountAlias: accountName,
}),
},
expected: types.Data{
Expand All @@ -82,7 +85,7 @@ func (s *AwsDataProviderTestSuite) TestAwsDataProvider_FetchData() {
}

for _, test := range tests {
p := New(test.options...)
p := NewDataProvider(test.options...)
result, err := p.FetchData(test.resource, test.id)
if test.expectError {
s.Error(err)
Expand All @@ -97,22 +100,23 @@ func TestDataProvider_EnrichEvent(t *testing.T) {
tests := []struct {
name string
resMetadata fetching.ResourceMetadata
identity awslib.Identity
identity Identity
expectedFields map[string]string
}{
{
name: "no replacement",
resMetadata: fetching.ResourceMetadata{
Region: someRegion,
},
identity: awslib.Identity{
Account: accountId,
Alias: accountName,
identity: Identity{
Account: accountId,
AccountAlias: accountName,
Provider: awsProvider,
},
expectedFields: map[string]string{
cloudAccountIdField: accountId,
cloudAccountNameField: accountName,
cloudProviderField: "aws",
cloudProviderField: awsProvider,
cloudRegionField: someRegion,
},
},
Expand All @@ -123,14 +127,15 @@ func TestDataProvider_EnrichEvent(t *testing.T) {
AwsAccountId: "",
AwsAccountAlias: "some alias",
},
identity: awslib.Identity{
Account: accountId,
Alias: accountName,
identity: Identity{
Account: accountId,
AccountAlias: accountName,
Provider: awsProvider,
},
expectedFields: map[string]string{
cloudAccountIdField: accountId,
cloudAccountNameField: "some alias",
cloudProviderField: "aws",
cloudProviderField: awsProvider,
cloudRegionField: someRegion,
},
},
Expand All @@ -141,21 +146,39 @@ func TestDataProvider_EnrichEvent(t *testing.T) {
AwsAccountId: "12345654321",
AwsAccountAlias: "some alias",
},
identity: awslib.Identity{
Account: accountId,
Alias: accountName,
identity: Identity{
Account: accountId,
AccountAlias: accountName,
Provider: awsProvider,
},
expectedFields: map[string]string{
cloudAccountIdField: "12345654321",
cloudAccountNameField: "some alias",
cloudProviderField: "aws",
cloudProviderField: awsProvider,
cloudRegionField: someRegion,
},
},
{
name: "enrich a gcp event",
resMetadata: fetching.ResourceMetadata{
Region: someRegion,
},
identity: Identity{
Provider: gcpProvider,
ProjectId: gcpProjectId,
ProjectName: gcpProjectName,
},
expectedFields: map[string]string{
cloudProviderField: gcpProvider,
cloudRegionField: someRegion,
cloudProjectIdField: gcpProjectId,
cloudProjectNameField: gcpProjectName,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
p := New(WithLogger(testhelper.NewLogger(t)), WithAccount(tt.identity))
p := NewDataProvider(WithLogger(testhelper.NewLogger(t)), WithAccount(tt.identity))
e := &beat.Event{
Fields: mapstr.M{},
}
Expand Down
2 changes: 1 addition & 1 deletion dataprovider/providers/gcp/mock_option.go → dataprovider/providers/cloud/mock_option.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 6 additions & 5 deletions dataprovider/providers/aws/option.go → dataprovider/providers/cloud/option.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,10 @@
// specific language governing permissions and limitations
// under the License.

package aws
package cloud

import (
"github.com/elastic/elastic-agent-libs/logp"

"github.com/elastic/cloudbeat/resources/providers/awslib"
)

type Option func(*DataProvider)
Expand All @@ -31,9 +29,12 @@ func WithLogger(log *logp.Logger) Option {
}
}

func WithAccount(identity awslib.Identity) Option {
func WithAccount(identity Identity) Option {
return func(dp *DataProvider) {
dp.accountId = identity.Account
dp.accountName = identity.Alias
dp.accountName = identity.AccountAlias
dp.providerName = identity.Provider
dp.projectId = identity.ProjectId
dp.projectName = identity.ProjectName
}
}
Loading
Loading