Cloud data provider

[uri-weisman]

Summary of your changes

1. Extend the existing aws data provider to serve all cloud benchmarks - AWS, GCP, and AZURE in the future.
2. Create a new identity provider for GCP.
3. Bump csp policies version.

Screenshot/Data

GCP

AWS

Related Issues

• [CIS GCP] Create a data provider #1120
• https://www.elastic.co/guide/en/ecs/master/ecs-cloud.html

Checklist

• I have added tests that prove my fix is effective or that my feature works
• I have added the necessary README/documentation (if appropriate)

[mergify]

This pull request does not have a backport label. Could you fix it @uri-weisman? 🙏
To fixup this pull request, you need to add the backport labels for the needed
branches, such as:

• backport-v./d./d./d is the label to automatically backport to the 8./d branch. /d is the digit
  NOTE: backport-skip has been added to this pull request.

[github-actions]

📊 Allure Report - 💔 Some tests failed or were broken.

Result	Count
🟥 Failed	4
🟩 Passed	29
⬜ Skipped	1

[orestisfl]

Blocker: make sure this doesn't break aws orgs account fields (LGTM but let's double check)

[olegsu]

@orestisfl how this can be validated?

[orestisfl]

Right now, manually since we don't have E2E tests yet.

However, the unit tests cover the case already so maybe it's overkill.

[orestisfl]

Returning nil from NewProvider can introduce segfault risks. GetIdentity is only called once in the lifetime of the app so we can move all initialization code in the body of the function like we do for other Dependencies in the Benchmark

Suggested change

	type ProviderGetter interface {
	GetIdentity(ctx context.Context, cfg config.GcpConfig) (*cloud.Identity, error)
	}
	type Provider struct {
	service ResourceManager
	logger *logp.Logger
	}
	// CloudResourceManagerService is a wrapper around the GCP resource manager service to make it easier to mock
	type CloudResourceManagerService struct {
	service *cloudresourcemanager.Service
	}
	type ResourceManager interface {
	projectsGet(context.Context, string) (*cloudresourcemanager.Project, error)
	}
	func NewProvider(ctx context.Context, cfg *config.Config, logger *logp.Logger) *Provider {
	gcpClientOpt, err := auth.GetGcpClientConfig(cfg, logger)
	if err != nil {
	logger.Errorf("failed to get GCP client config: %v", err)
	return nil
	}
	gcpClientOpt = append(gcpClientOpt, option.WithScopes(cloudresourcemanager.CloudPlatformReadOnlyScope))
	crmService, err := cloudresourcemanager.NewService(ctx, gcpClientOpt...)
	if err != nil {
	logger.Errorf("failed to create GCP resource manager service: %v", err)
	return nil
	}
	return &Provider{
	service: &CloudResourceManagerService{service: crmService},
	logger: logger,
	}
	}
	// GetIdentity returns GCP identity information
	func (p *Provider) GetIdentity(ctx context.Context, cfg config.GcpConfig) (*cloud.Identity, error) {
	proj, err := p.service.projectsGet(ctx, "projects/"+cfg.ProjectId)
	if err != nil {
	return nil, err
	}
	return &cloud.Identity{
	Provider: provider,
	ProjectId: proj.ProjectId,
	ProjectName: proj.DisplayName,
	}, nil
	}
	func (p *CloudResourceManagerService) projectsGet(ctx context.Context, id string) (*cloudresourcemanager.Project, error) {
	project, err := p.service.Projects.Get(id).Context(ctx).Do()
	if err != nil {
	return nil, fmt.Errorf("unable to get project with id '%v': %v", id, err)
	}
	return project, nil
	}
	type ProviderGetter interface {
	GetIdentity(ctx context.Context, cfg config.GcpConfig) (*cloud.Identity, error)
	}
	type Provider struct {
	}
	// CloudResourceManagerService is a wrapper around the GCP resource manager service to make it easier to mock
	type CloudResourceManagerService struct {
	service *cloudresourcemanager.Service
	}
	type ResourceManager interface {
	projectsGet(context.Context, string) (*cloudresourcemanager.Project, error)
	}
	// GetIdentity returns GCP identity information
	func (p *Provider) GetIdentity(ctx context.Context, logger *logp.Logger, cfg config.GcpConfig) (*cloud.Identity, error) {
	gcpClientOpt, err := auth.GetGcpClientConfig(cfg, logger)
	if err != nil {
	return nil, err
	}
	gcpClientOpt = append(gcpClientOpt, option.WithScopes(cloudresourcemanager.CloudPlatformReadOnlyScope))
	crmService, err := cloudresourcemanager.NewService(ctx, gcpClientOpt...)
	if err != nil {
	logger.Errorf("failed to create GCP resource manager service: %v", err)
	return nil, err
	}
	proj, err := CloudResourceManagerService{service: crmService}.projectsGet(ctx, "projects/"+cfg.ProjectId)
	if err != nil {
	return nil, err
	}
	return &cloud.Identity{
	Provider: provider,
	ProjectId: proj.ProjectId,
	ProjectName: proj.DisplayName,
	}, nil
	}
	func (p CloudResourceManagerService) projectsGet(ctx context.Context, id string) (*cloudresourcemanager.Project, error) {
	project, err := p.service.Projects.Get(id).Context(ctx).Do()
	if err != nil {
	return nil, fmt.Errorf("unable to get project with id '%v': %v", id, err)
	}
	return project, nil
	}

which also requires changing GetGcpClientConfig:

func GetGcpClientConfig(gcpCred config.GcpConfig, log *logp.Logger) ([]option.ClientOption, error) {

[uri-weisman]

@orestisfl - I get your point but in your implementation, I won't be able to mock the cloudresourcemanager service.

[olegsu]

can we return also an error in that case?

[orestisfl]

@uri-weisman

How about:

// GetIdentity returns GCP identity information
func (p *Provider) GetIdentity(ctx context.Context, cfg config.GcpConfig, logger *logp.Logger) (*cloud.Identity, error) {
	if err := p.initialize(ctx, cfg, logger); err != nil {
		return nil, err
	}

	proj, err := p.service.projectsGet(ctx, "projects/"+cfg.ProjectId)
	if err != nil {
		return nil, err
	}

	return &cloud.Identity{
		Provider:    provider,
		ProjectId:   proj.ProjectId,
		ProjectName: proj.DisplayName,
	}, nil
}

func (p *Provider) initialize(ctx context.Context, cfg config.GcpConfig, logger *logp.Logger) error {
	if p.service != nil {
		return nil
	}

	gcpClientOpt, err := auth.GetGcpClientConfig(cfg, logger)
	if err != nil {
		return err
	}
	gcpClientOpt = append(gcpClientOpt, option.WithScopes(cloudresourcemanager.CloudPlatformReadOnlyScope))
	crmService, err := cloudresourcemanager.NewService(ctx, gcpClientOpt...)
	if err != nil {
		return err
	}

	p.service = &CloudResourceManagerService{service: crmService}
	return nil
}

[mergify]

This pull request is now in conflicts. Could you fix it? 🙏
To fixup this pull request, you can check out it locally. See documentation: https://help.github.com/articles/checking-out-pull-requests-locally/

git fetch upstream
git checkout -b gcp_data_provider upstream/gcp_data_provider
git merge upstream/main
git push upstream gcp_data_provider

[orestisfl]

Suggested change

	return fmt.Errorf("the file %q cannot be read", filePath)
	return fmt.Errorf("file %q cannot be read", filePath)

[orestisfl]

Suggested change

	return fmt.Errorf("the file %q does not contain valid JSON", filePath)
	return fmt.Errorf("file %q does not contain valid JSON", filePath)