Skip to content

Commit

Permalink
Merge branch 'main' into updatecli_main_updatecli-beats-main
Browse files Browse the repository at this point in the history
  • Loading branch information
@oren-zohar
oren-zohar authored Mar 25, 2024
2 parents f8304c0 + c1d9c4c commit 0cebe1c
Show file tree
Hide file tree
Showing 35 changed files with 2,130 additions and 281 deletions.
6 changes: 4 additions & 2 deletions deploy/test-environments/delete_env.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,9 @@ else
GCP_FILTER="name:'$ENV_PREFIX*'"
fi

ALL_GCP_DEPLOYMENTS=$(gcloud deployment-manager deployments list --filter="$GCP_FILTER" --format="value(name)")
while IFS= read -r line; do
ALL_GCP_DEPLOYMENTS+=("$line")
done < <(gcloud deployment-manager deployments list --filter="$GCP_FILTER" --format="value(name)")

# Divide environments into those to be deleted and those to be skipped
TO_DELETE_ENVS=()
Expand Down Expand Up @@ -163,7 +165,7 @@ printf "%s\n" "${FAILED_STACKS[@]}"
# Delete GCP deployments
PROJECT_NAME=$(gcloud config get-value core/project)
PROJECT_NUMBER=$(gcloud projects list --filter="${PROJECT_NAME}" --format="value(PROJECT_NUMBER)")
./delete_gcp_env.sh "$PROJECT_NAME" "$PROJECT_NUMBER" "$ALL_GCP_DEPLOYMENTS"
./delete_gcp_env.sh "$PROJECT_NAME" "$PROJECT_NUMBER" "${ALL_GCP_DEPLOYMENTS[@]}"

# Delete Azure groups
FAILED_AZURE_GROUPS=()
Expand Down
30 changes: 16 additions & 14 deletions deploy/test-environments/delete_gcp_env.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,15 @@ PROJECT_NUMBER=$2
shift 2
GCP_DEPLOYMENTS=("$@")

echo "Project Name: $PROJECT_NAME"
echo "Project Number: $PROJECT_NUMBER"
echo "GCP Deployments: ${GCP_DEPLOYMENTS[*]}"
# Add the needed roles to delete the templates to the project using the deployment manager
gcloud projects add-iam-policy-binding "${PROJECT_NAME}" --member=serviceAccount:"${PROJECT_NUMBER}"@cloudservices.gserviceaccount.com --role=roles/iam.roleAdmin --no-user-output-enabled
gcloud projects add-iam-policy-binding "${PROJECT_NAME}" --member=serviceAccount:"${PROJECT_NUMBER}"@cloudservices.gserviceaccount.com --role=roles/resourcemanager.projectIamAdmin --no-user-output-enabled

for DEPLOYMENT in "${GCP_DEPLOYMENTS[@]}"; do
# Add the needed roles to delete the templates to the project using the deployment manager
gcloud projects add-iam-policy-binding "${PROJECT_NAME}" --member=serviceAccount:"${PROJECT_NUMBER}"@cloudservices.gserviceaccount.com --role=roles/iam.roleAdmin --no-user-output-enabled
gcloud projects add-iam-policy-binding "${PROJECT_NAME}" --member=serviceAccount:"${PROJECT_NUMBER}"@cloudservices.gserviceaccount.com --role=roles/resourcemanager.projectIamAdmin --no-user-output-enabled
DELETED_DEPLOYMENTS=()
FAILED_DEPLOYMENTS=()

for DEPLOYMENT in "${GCP_DEPLOYMENTS[@]}"; do
echo "Deleting GCP deployment: $DEPLOYMENT"
if gcloud deployment-manager deployments delete "$DEPLOYMENT" -q; then
echo "Successfully deleted GCP deployment: $DEPLOYMENT"
DELETED_DEPLOYMENTS+=("$DEPLOYMENT")
Expand All @@ -30,18 +30,20 @@ for DEPLOYMENT in "${GCP_DEPLOYMENTS[@]}"; do
FAILED_DEPLOYMENTS+=("$DEPLOYMENT")
fi

# Remove the roles required to deploy the DM templates
gcloud projects remove-iam-policy-binding "${PROJECT_NAME}" --member=serviceAccount:"${PROJECT_NUMBER}"@cloudservices.gserviceaccount.com --role=roles/iam.roleAdmin --no-user-output-enabled
gcloud projects remove-iam-policy-binding "${PROJECT_NAME}" --member=serviceAccount:"${PROJECT_NUMBER}"@cloudservices.gserviceaccount.com --role=roles/resourcemanager.projectIamAdmin --no-user-output-enabled

done

# Print summary of gcp deployments deletions
# Remove the roles required to deploy the DM templates
gcloud projects remove-iam-policy-binding "${PROJECT_NAME}" --member=serviceAccount:"${PROJECT_NUMBER}"@cloudservices.gserviceaccount.com --role=roles/iam.roleAdmin --no-user-output-enabled
gcloud projects remove-iam-policy-binding "${PROJECT_NAME}" --member=serviceAccount:"${PROJECT_NUMBER}"@cloudservices.gserviceaccount.com --role=roles/resourcemanager.projectIamAdmin --no-user-output-enabled

echo "Successfully deleted GCP deployments (${#DELETED_DEPLOYMENTS[@]}):"
printf "%s\n" "${DELETED_DEPLOYMENTS[@]}"
# Print summary of gcp deployments deletions
if [ ${#DELETED_DEPLOYMENTS[@]} -gt 0 ]; then
printf "%s\n" "${DELETED_DEPLOYMENTS[@]}"
fi

echo "Failed to delete GCP deployments (${#FAILED_DEPLOYMENTS[@]}):"
if [ ${#FAILED_DEPLOYMENTS[@]} -gt 0 ]; then
echo "Failed to delete GCP deployments (${#FAILED_DEPLOYMENTS[@]}):"
printf "%s\n" "${FAILED_DEPLOYMENTS[@]}"
exit 1
fi
12 changes: 12 additions & 0 deletions dev-docs/Cloud-Env-Upgrade.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,18 @@ The [`Test Upgrade Environment`](https://github.com/elastic/cloudbeat/actions/wo
It also facilitates the upgrade of the environment to a new version of the ELK stack and all installed agents, while also performing checks for findings retrieval. For example, if the target ELK version is 8.12.0 and the base version was not selected, the workflow will automatically calculate the previously released version (e.g., 8.11.3), install that version, and then proceed to upgrade to the specified target version (8.12.0). Essentially, this workflow is designed to test the upgrade feature on upcoming versions that are currently in development or will be release candidates (BC).


## Overview of the Upgrade Process

The upgrade process comprises the following main steps:

1. Install the released version, including all integrations (CSPM/KSPM), and deploy their agents.
2. Upgrade the ELK stack version.
3. Upgrade CSPM/KSPM integration versions:
- If the integration has a `preview` version, the workflow will execute a script to update the integration to the latest `preview` version.
- If the latest version is released (no `preview` suffix), the integration upgrade will be automatically performed after the stack upgrade.
4. Upgrade KSPM agents by reapplying Kubernetes manifests with the latest image versions.
5. Upgrade Linux-type agents (CSPM/CNVM) by using the Fleet upgrade API.

## How to Run the Workflow

Follow these steps to run the workflow:
Expand Down
82 changes: 41 additions & 41 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ module github.com/elastic/cloudbeat
go 1.21

require (
cloud.google.com/go/asset v1.18.0
cloud.google.com/go/iam v1.1.6
cloud.google.com/go/asset v1.18.1
cloud.google.com/go/iam v1.1.7
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0
Expand All @@ -20,30 +20,30 @@ require (
github.com/aquasecurity/go-dep-parser v0.0.0-20231120074854-8322cc2242bf
github.com/aquasecurity/trivy v0.48.3
github.com/aquasecurity/trivy-db v0.0.0-20240220070059-88dc6466aa40
github.com/aws/aws-sdk-go v1.50.35
github.com/aws/aws-sdk-go-v2 v1.25.3
github.com/aws/aws-sdk-go-v2/config v1.27.7
github.com/aws/aws-sdk-go-v2/credentials v1.17.7
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.3
github.com/aws/aws-sdk-go-v2/service/accessanalyzer v1.28.3
github.com/aws/aws-sdk-go-v2/service/autoscaling v1.40.3
github.com/aws/aws-sdk-go-v2/service/cloudformation v1.47.1
github.com/aws/aws-sdk-go-v2/service/cloudtrail v1.39.0
github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.36.2
github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.34.3
github.com/aws/aws-sdk-go-v2/service/configservice v1.46.2
github.com/aws/aws-sdk-go-v2/service/ec2 v1.150.0
github.com/aws/aws-sdk-go-v2/service/ecr v1.27.2
github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.24.2
github.com/aws/aws-sdk-go-v2/service/iam v1.31.2
github.com/aws/aws-sdk-go-v2/service/kms v1.29.2
github.com/aws/aws-sdk-go-v2/service/organizations v1.27.1
github.com/aws/aws-sdk-go-v2/service/rds v1.75.1
github.com/aws/aws-sdk-go-v2/service/s3 v1.51.4
github.com/aws/aws-sdk-go-v2/service/s3control v1.44.2
github.com/aws/aws-sdk-go-v2/service/securityhub v1.46.2
github.com/aws/aws-sdk-go-v2/service/sns v1.29.2
github.com/aws/aws-sdk-go-v2/service/sts v1.28.4
github.com/aws/aws-sdk-go v1.51.6
github.com/aws/aws-sdk-go-v2 v1.26.0
github.com/aws/aws-sdk-go-v2/config v1.27.9
github.com/aws/aws-sdk-go-v2/credentials v1.17.9
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0
github.com/aws/aws-sdk-go-v2/service/accessanalyzer v1.29.0
github.com/aws/aws-sdk-go-v2/service/autoscaling v1.40.4
github.com/aws/aws-sdk-go-v2/service/cloudformation v1.48.0
github.com/aws/aws-sdk-go-v2/service/cloudtrail v1.39.1
github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.36.3
github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.35.0
github.com/aws/aws-sdk-go-v2/service/configservice v1.46.3
github.com/aws/aws-sdk-go-v2/service/ec2 v1.152.0
github.com/aws/aws-sdk-go-v2/service/ecr v1.27.3
github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.24.3
github.com/aws/aws-sdk-go-v2/service/iam v1.31.3
github.com/aws/aws-sdk-go-v2/service/kms v1.30.0
github.com/aws/aws-sdk-go-v2/service/organizations v1.27.2
github.com/aws/aws-sdk-go-v2/service/rds v1.76.0
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0
github.com/aws/aws-sdk-go-v2/service/s3control v1.44.3
github.com/aws/aws-sdk-go-v2/service/securityhub v1.47.0
github.com/aws/aws-sdk-go-v2/service/sns v1.29.3
github.com/aws/aws-sdk-go-v2/service/sts v1.28.5
github.com/aws/smithy-go v1.20.1
github.com/dgraph-io/ristretto v0.1.1
github.com/djherbis/times v1.6.0
Expand Down Expand Up @@ -120,14 +120,14 @@ require (
)

require (
cloud.google.com/go v0.112.0 // indirect
cloud.google.com/go v0.112.1 // indirect
cloud.google.com/go/accesscontextmanager v1.8.5 // indirect
cloud.google.com/go/compute v1.24.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/longrunning v0.5.5 // indirect
cloud.google.com/go/orgpolicy v1.12.1 // indirect
cloud.google.com/go/osconfig v1.12.5 // indirect
cloud.google.com/go/storage v1.36.0 // indirect
cloud.google.com/go/storage v1.38.0 // indirect
code.cloudfoundry.org/go-diodes v0.0.0-20190809170250-f77fb823c7ee // indirect
code.cloudfoundry.org/go-loggregator v7.4.0+incompatible // indirect
code.cloudfoundry.org/gofileutils v0.0.0-20170111115228-4d0c80011a0f // indirect
Expand Down Expand Up @@ -181,18 +181,18 @@ require (
github.com/armon/go-radix v1.0.0 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4 // indirect
github.com/aws/aws-sdk-go-v2/service/ebs v1.21.7 // indirect
github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.21.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.20.2 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
github.com/bmatcuk/doublestar/v4 v4.6.0 // indirect
Expand Down Expand Up @@ -225,7 +225,7 @@ require (
github.com/dnephin/pflag v1.0.7 // indirect
github.com/docker/cli v24.0.5+incompatible // indirect
github.com/docker/distribution v2.8.2+incompatible // indirect
github.com/docker/docker v24.0.7+incompatible // indirect
github.com/docker/docker v24.0.9+incompatible // indirect
github.com/docker/docker-credential-helpers v0.7.0 // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
Expand Down Expand Up @@ -283,7 +283,7 @@ require (
github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
github.com/golang/glog v1.2.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/gomodule/redigo v1.8.3 // indirect
github.com/google/btree v1.1.2 // indirect
Expand Down Expand Up @@ -479,9 +479,9 @@ require (
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto v0.0.0-20240228224816-df926f6c8641 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240304161311-37d4d3c04a78 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240304161311-37d4d3c04a78 // indirect
google.golang.org/grpc v1.62.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 // indirect
google.golang.org/grpc v1.62.1 // indirect
google.golang.org/protobuf v1.33.0
gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
Expand Down
Loading

0 comments on commit 0cebe1c

Please sign in to comment.