Merge branch '8.x' into mergify/bp/8.x/pr-2574

.github/workflows/test-environment.yml

@@ -118,7 +118,6 @@ env:
   WORKING_DIR: deploy/test-environments
   INTEGRATIONS_SETUP_DIR: tests/integrations_setup
   AWS_DEFAULT_TAGS: "Key=division,Value=engineering Key=org,Value=security Key=team,Value=cloud-security-posture Key=project,Value=test-environments"
-  GCP_DEFAULT_TAGS: "division=engineering,org=security,team=cloud-security-posture,project=test-environments,owner=${{ github.actor }}"
   GCP_ZONE: "us-central1-a"
   AZURE_DEFAULT_TAGS: "division=engineering org=security team=cloud-security-posture project=test-environments owner=${{ github.actor }}"
   TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }}
@@ -409,7 +408,14 @@ jobs:
       - name: Deploy CSPM GCP agent
         id: cspm-gcp-agent
         working-directory: deploy/deployment-manager
+        env:
+          ACTOR: ${{ github.actor }}
         run: |
+          # GCP labeling rules:
+          # Only hyphens (-), underscores (_), lowercase characters, and numbers are allowed. International characters are allowed.
+          # Convert github.actor to lowercase, replace disallowed characters
+          GCP_LABEL=$(echo "$ACTOR" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9_-]/_/g')
+          GCP_DEFAULT_TAGS="division=engineering,org=security,team=cloud-security-posture,project=test-environments,owner=$GCP_LABEL"
           . ./set_env.sh && ./deploy.sh && gcloud compute instances update "${DEPLOYMENT_NAME}" --update-labels "${GCP_DEFAULT_TAGS}" --zone="${GCP_ZONE}"
 
       - name: Install CSPM Azure integration

.github/workflows/updatecli.yml

@@ -54,7 +54,7 @@ jobs:
       - name: Init Hermit
         run: ./bin/hermit env -r >> $GITHUB_ENV
       - name: Install Updatecli in the runner
-        uses: updatecli/updatecli-action@6b8881a17fc8038e884ec94ff72a49e8e8a4069f # v0.76.1
+        uses: updatecli/updatecli-action@92a13b95c2cd9f1c6742c965509203c6a5635ed7 # v0.76.1
       - name: Run Updatecli in Apply mode
         run: updatecli apply --config .ci/updatecli/updatecli.d/update-${{ matrix.pipeline-name }}.yml --values .ci/updatecli/values.yml
         env:
@@ -84,7 +84,7 @@ jobs:
 
           echo "GIT_BRANCH=$branchName" >> $GITHUB_ENV
       - name: Install Updatecli in the runner
-        uses: updatecli/updatecli-action@6b8881a17fc8038e884ec94ff72a49e8e8a4069f # v0.76.1
+        uses: updatecli/updatecli-action@92a13b95c2cd9f1c6742c965509203c6a5635ed7 # v0.76.1
       - name: Run Updatecli in Apply mode
         run: updatecli apply --config .ci/updatecli/updatecli.d/update-${{ matrix.pipeline-name }}.yml --values .ci/updatecli/values.yml
         env:

.github/workflows/weekly-serverless.yml

@@ -30,7 +30,7 @@ jobs:
       deployment_name: ${{ needs.naming.outputs.deployment_name }}
       # For now, the region is not used because it's overridden in the tf, but it's here for future compatibility.
       ess-region: "aws-us-east-1"
-      elk-stack-version: 8.12.0
+      elk-stack-version: 8.15.3
       serverless_mode: true
       run-sanity-tests: true
       expiration_days: 0

.go-version

@@ -1 +1 @@
-1.23.1
+1.23.2

.mergify.yml

@@ -313,6 +313,19 @@ pull_request_rules:
         labels:
           - "backport"
         title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
+  - name: backport patches to 8.16 branch
+    conditions:
+      - merged
+      - label=backport-v8.16.0
+    actions:
+      backport:
+        assignees:
+          - "{{ author }}"
+        branches:
+          - "8.16"
+        labels:
+          - "backport"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
   - name: backport patches to 8.x branch
     conditions:
       - merged

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ exclude: (bin/(activate-)?hermit)|(scripts/make/gimme/.*)
 repos:
   ## General
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: check-merge-conflict
       - id: check-added-large-files
@@ -42,7 +42,7 @@ repos:
         files: ^(.ci/scripts|.github/actions|tests)/
 
   - repo: https://github.com/pycqa/pylint
-    rev: v3.3.0
+    rev: v3.3.1
     hooks:
       - id: pylint
         args: [
@@ -77,7 +77,7 @@ repos:
         files: deploy/cloudformation/.*yml
 
   - repo: https://github.com/aws-cloudformation/cfn-lint
-    rev: v1.14.2
+    rev: v1.16.0
     hooks:
       - id: cfn-python-lint
         files: deploy/cloudformation/.*.yml
@@ -170,14 +170,15 @@ repos:
         files: (\.rego)$
 
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.41.0
+    rev: v0.42.0
     hooks:
       - id: markdownlint
         args: [ '--disable',
                 MD013,  # Line length can be ignored for now
                 MD033,  # Allow inline HTML
                 MD046,  # Allow code blocks to be fenced with backticks
                 MD041,  # Allow multiple top level headers
+                MD058,  # Allow tables without blank lines around
                 '--' ]
         # TODO: Apply makrdownlint in other files
         files: security-policies/.*\.(md|markdown)$

bin/bq

@@ -1 +1 @@
-.gcloud-493.0.0.pkg
+.gcloud-495.0.0.pkg

bin/docker-credential-gcloud

@@ -1 +1 @@
-.gcloud-493.0.0.pkg
+.gcloud-495.0.0.pkg

bin/gcloud

@@ -1 +1 @@
-.gcloud-493.0.0.pkg
+.gcloud-495.0.0.pkg

bin/gh

@@ -1 +1 @@
-.gh-2.57.0.pkg
+.gh-2.58.0.pkg

bin/git-credential-gcloud.sh

@@ -1 +1 @@
-.gcloud-493.0.0.pkg
+.gcloud-495.0.0.pkg

bin/go

@@ -1 +1 @@
-.go-1.23.1.pkg
+.go-1.23.2.pkg

bin/gofmt

@@ -1 +1 @@
-.go-1.23.1.pkg
+.go-1.23.2.pkg

bin/gsutil

@@ -1 +1 @@
-.gcloud-493.0.0.pkg
+.gcloud-495.0.0.pkg

bin/just

@@ -1 +1 @@
-.just-1.35.0.pkg
+.just-1.36.0.pkg

bin/opa

@@ -1 +1 @@
-.opa-0.68.0.pkg
+.opa-0.69.0.pkg

bin/pre-commit

@@ -1 +1 @@
-.pre-commit-3.8.0.pkg
+.pre-commit-4.0.0.pkg

deploy/azure/ARM-for-organization-account.dev.json

@@ -27,7 +27,7 @@
             "metadata": {
                 "description": "The version of elastic-agent to install"
             },
-            "defaultValue": "8.16.0"
+            "defaultValue": "8.17.0"
         },
         "FleetUrl": {
             "type": "string",

deploy/azure/ARM-for-organization-account.json

@@ -27,7 +27,7 @@
             "metadata": {
                 "description": "The version of elastic-agent to install"
             },
-            "defaultValue": "8.16.0"
+            "defaultValue": "8.17.0"
         },
         "FleetUrl": {
             "type": "string",

deploy/azure/ARM-for-single-account.dev.json

@@ -14,7 +14,7 @@
             "metadata": {
                 "description": "The version of elastic-agent to install"
             },
-            "defaultValue": "8.16.0"
+            "defaultValue": "8.17.0"
         },
         "FleetUrl": {
             "type": "string",

deploy/azure/ARM-for-single-account.json

@@ -14,7 +14,7 @@
             "metadata": {
                 "description": "The version of elastic-agent to install"
             },
-            "defaultValue": "8.16.0"
+            "defaultValue": "8.17.0"
         },
         "FleetUrl": {
             "type": "string",

deploy/azure/install_agent_az_cli.sh

@@ -33,11 +33,17 @@
 # Exit immediately if a command exits with a non-zero status, print each command before executing it, and fail pipelines if any command fails.
 set -euo pipefail
 
+# Extract major and minor versions
+MAJOR_VERSION=$(echo "${STACK_VERSION}" | cut -d'.' -f1)
 MINOR_VERSION=$(echo "${STACK_VERSION}" | cut -d'.' -f2)
-# Check if minor version is less than 12, ie. 8.11 and below
-if ((MINOR_VERSION < 12)); then
-    echo "Versions 8.11 and below are not supported. Please use versions 8.12+"
-    exit 0
+
+# Check for unsupported versions
+if [[ "$MAJOR_VERSION" -eq 8 && "$MINOR_VERSION" -lt 12 ]]; then
+    echo "Versions 8.11 and below are not supported. Please use versions 8.12+."
+    exit 1
+elif [[ "$MAJOR_VERSION" -lt 8 ]]; then
+    echo "Unsupported version: ${STACK_VERSION}. Please use versions 8.12+ or 9.x.x and above."
+    exit 1
 fi
 
 # Create a resource group with the name DEPLOYMENT_NAME

docs/version.asciidoc

@@ -1,6 +1,6 @@
 // doc-branch can be: master, 8.1, 8.2, etc.
 :doc-branch: master
-:go-version: 1.23.1
+:go-version: 1.23.2
 :python: 3.7
 :docker: 1.12
 :docker-compose: 1.11

go.mod

@@ -79,7 +79,6 @@ require (
 	go.uber.org/goleak v1.3.0
 	go.uber.org/zap v1.27.0
 	go.uber.org/zap/exp v0.1.1-0.20240530135403-273f9ea8c887
-	golang.org/x/exp v0.0.0-20240909161429-701f63a606c0
 	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616
 	golang.org/x/oauth2 v0.23.0
 	google.golang.org/api v0.199.0
@@ -162,6 +161,7 @@ require (
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.29.0 // indirect
+	golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect
 	golang.org/x/tools v0.25.0 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect

internal/flavors/benchmark/azure.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math"
 	"time"
 
 	"github.com/elastic/elastic-agent-libs/logp"
@@ -50,7 +51,7 @@ func (a *Azure) NewBenchmark(ctx context.Context, log *logp.Logger, cfg *config.
 
 	return builder.New(
 		builder.WithBenchmarkDataProvider(bdp),
-		builder.WithManagerTimeout(20*time.Minute),
+		builder.WithManagerTimeout(calculateFetcherTimeout(cfg.Period)),
 	).Build(ctx, log, cfg, resourceCh, reg)
 }
 
@@ -95,3 +96,16 @@ func (a *Azure) checkDependencies() error {
 	}
 	return nil
 }
+
+// calculateFetcherTimeout calculates the timeout for each fetcher based on period as ~70% of the period duration. If less than 3 hours, it returns 3 hours.
+func calculateFetcherTimeout(period time.Duration) time.Duration {
+	roundedHours := math.Round(period.Hours() * 0.7)
+	to := time.Duration(roundedHours) * time.Hour
+
+	const min = 3 * time.Hour
+	if to < min {
+		return min
+	}
+
+	return to
+}

internal/flavors/benchmark/azure_test.go

@@ -20,8 +20,10 @@ package benchmark
 import (
 	"errors"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/cloudbeat/internal/config"
 	"github.com/elastic/cloudbeat/internal/resources/providers/azurelib"
@@ -119,3 +121,46 @@ func mockAzureInventoryInitializerService(err error) azurelib.ProviderInitialize
 	}
 	return initializer
 }
+
+func TestCalculateFetcherTimeout(t *testing.T) {
+	tests := map[string]struct {
+		inputPeriod time.Duration
+		expected    time.Duration
+	}{
+		"48h": {
+			inputPeriod: 48 * time.Hour,
+			expected:    34 * time.Hour,
+		},
+		"24h": {
+			inputPeriod: 24 * time.Hour,
+			expected:    17 * time.Hour,
+		},
+		"3h": {
+			inputPeriod: 3 * time.Hour,
+			expected:    3 * time.Hour,
+		},
+		"30m": {
+			inputPeriod: 30 * time.Minute,
+			expected:    3 * time.Hour,
+		},
+		"0": {
+			inputPeriod: 0,
+			expected:    3 * time.Hour,
+		},
+		"-30m": {
+			inputPeriod: -30 * time.Minute,
+			expected:    3 * time.Hour,
+		},
+		"-3h": {
+			inputPeriod: -3 * time.Hour,
+			expected:    3 * time.Hour,
+		},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			got := calculateFetcherTimeout(tc.inputPeriod)
+			require.Equal(t, tc.expected, got)
+		})
+	}
+}

internal/resources/fetching/fetchers/azure/assets_fetcher.go

@@ -21,9 +21,10 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"maps"
+	"slices"
 
 	"github.com/elastic/elastic-agent-libs/logp"
-	"golang.org/x/exp/maps"
 
 	"github.com/elastic/cloudbeat/internal/resources/fetching"
 	"github.com/elastic/cloudbeat/internal/resources/fetching/cycle"
@@ -98,7 +99,7 @@ func (f *AzureAssetsFetcher) Fetch(ctx context.Context, cycleMetadata cycle.Meta
 	var assets []inventory.AzureAsset
 	for _, assetGroup := range AzureAssetGroups {
 		// Fetching all types even if non-existent in asset group for simplicity
-		r, err := f.provider.ListAllAssetTypesByName(ctx, assetGroup, maps.Keys(AzureAssetTypeToTypePair))
+		r, err := f.provider.ListAllAssetTypesByName(ctx, assetGroup, slices.Collect(maps.Keys(AzureAssetTypeToTypePair)))
 		if err != nil {
 			f.log.Errorf("AzureAssetsFetcher.Fetch failed to fetch asset group %s: %s", assetGroup, err.Error())
 			errAgg = errors.Join(errAgg, err)

internal/resources/fetching/fetchers/azure/assets_fetcher_test.go

@@ -20,12 +20,12 @@ package fetchers
 import (
 	"context"
 	"errors"
+	"maps"
 	"testing"
 
 	"github.com/samber/lo"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/suite"
-	"golang.org/x/exp/maps"
 
 	"github.com/elastic/cloudbeat/internal/resources/fetching"
 	"github.com/elastic/cloudbeat/internal/resources/fetching/cycle"
@@ -61,7 +61,7 @@ func (s *AzureAssetsFetcherTestSuite) TestFetcher_Fetch() {
 	var flatMockAssets []inventory.AzureAsset
 	for _, assetGroup := range AzureAssetGroups {
 		var mockAssets []inventory.AzureAsset
-		for _, assetType := range maps.Keys(AzureAssetTypeToTypePair) {
+		for assetType := range maps.Keys(AzureAssetTypeToTypePair) {
 			mockAssets = append(mockAssets,
 				inventory.AzureAsset{
 					Id:             "id",