From a1294f0d6c62c6a0547aa14d623a61ae27e04a19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Sobo=C5=84?= Date: Thu, 24 Oct 2024 14:36:24 +0200 Subject: [PATCH] [Asset Inventory][AWS & Azure] Support organization account deployment type (#2591) --- .github/workflows/ci-pull_request.yml | 2 +- .github/workflows/publish-cloudformation.yml | 1 + deploy/asset-inventory-arm/.gitignore | 1 + .../ARM-for-organization-account.dev.json | 444 +++++++++++++++++ .../ARM-for-organization-account.json | 450 ++++++++++++++++++ .../ARM-for-single-account.dev.json | 368 ++++++++++++++ .../ARM-for-single-account.json | 359 ++++++++++++++ deploy/asset-inventory-arm/README.md | 59 +++ .../generate_dev_template.py | 305 ++++++++++++ .../asset-inventory-arm/install-agent-dev.sh | 16 + deploy/asset-inventory-arm/install-agent.sh | 24 + .../install_agent_az_cli.sh | 67 +++ .../asset-inventory-cloudformation/.gitignore | 4 + .../asset-inventory-cloudformation/README.md | 34 ++ .../asset-inventory-cloudformation/config.go | 126 +++++ .../ec2-types.yml | 156 ++++++ .../elastic-agent-ec2-organization.yml | 256 ++++++++++ .../elastic-agent-ec2.yml | 151 ++++++ .../asset-inventory-cloudformation/gomain.go | 209 ++++++++ internal/flavors/assetinventory/strategy.go | 24 +- .../flavors/assetinventory/strategy_aws.go | 110 +++++ justfile | 3 + scripts/bump_integration.sh | 3 + scripts/publish_cft.sh | 2 + 24 files changed, 3155 insertions(+), 19 deletions(-) create mode 100644 deploy/asset-inventory-arm/.gitignore create mode 100644 deploy/asset-inventory-arm/ARM-for-organization-account.dev.json create mode 100644 deploy/asset-inventory-arm/ARM-for-organization-account.json create mode 100644 deploy/asset-inventory-arm/ARM-for-single-account.dev.json create mode 100644 deploy/asset-inventory-arm/ARM-for-single-account.json create mode 100644 deploy/asset-inventory-arm/README.md create mode 100755 deploy/asset-inventory-arm/generate_dev_template.py create mode 100644 deploy/asset-inventory-arm/install-agent-dev.sh create mode 100644 deploy/asset-inventory-arm/install-agent.sh create mode 100755 deploy/asset-inventory-arm/install_agent_az_cli.sh create mode 100644 deploy/asset-inventory-cloudformation/.gitignore create mode 100644 deploy/asset-inventory-cloudformation/README.md create mode 100644 deploy/asset-inventory-cloudformation/config.go create mode 100644 deploy/asset-inventory-cloudformation/ec2-types.yml create mode 100644 deploy/asset-inventory-cloudformation/elastic-agent-ec2-organization.yml create mode 100644 deploy/asset-inventory-cloudformation/elastic-agent-ec2.yml create mode 100644 deploy/asset-inventory-cloudformation/gomain.go create mode 100644 internal/flavors/assetinventory/strategy_aws.go diff --git a/.github/workflows/ci-pull_request.yml b/.github/workflows/ci-pull_request.yml index 12813116d7..8013b15664 100644 --- a/.github/workflows/ci-pull_request.yml +++ b/.github/workflows/ci-pull_request.yml @@ -94,7 +94,7 @@ jobs: run: | go install gotest.tools/gotestsum GOOS=linux TEST_DIRECTORY=./... gotestsum --format pkgname -- -race -coverpkg=./... -coverprofile=cover.out.tmp - cat cover.out.tmp | grep -v "mock_.*.go" > cover.out # remove mock files from coverage report + cat cover.out.tmp | grep -v "mock_.*.go" | grep -v "elastic/cloudbeat/deploy" > cover.out # remove mock files and deploy dir - name: Upload coverage artifact uses: actions/upload-artifact@v4 diff --git a/.github/workflows/publish-cloudformation.yml b/.github/workflows/publish-cloudformation.yml index 3eb8e01f14..b104e19ac3 100644 --- a/.github/workflows/publish-cloudformation.yml +++ b/.github/workflows/publish-cloudformation.yml @@ -6,6 +6,7 @@ on: - main - "[0-9]+.[0-9]+" paths: + - deploy/asset-inventory-cloudformation/*.yml - deploy/cloudformation/*.yml - scripts/publish_cft.sh - .github/workflows/publish-cloudformation.yml diff --git a/deploy/asset-inventory-arm/.gitignore b/deploy/asset-inventory-arm/.gitignore new file mode 100644 index 0000000000..d78a7a5f4f --- /dev/null +++ b/deploy/asset-inventory-arm/.gitignore @@ -0,0 +1 @@ +dev-flags.conf diff --git a/deploy/asset-inventory-arm/ARM-for-organization-account.dev.json b/deploy/asset-inventory-arm/ARM-for-organization-account.dev.json new file mode 100644 index 0000000000..1621e11b7b --- /dev/null +++ b/deploy/asset-inventory-arm/ARM-for-organization-account.dev.json @@ -0,0 +1,444 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-08-01/managementGroupDeploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "ResourceGroupName": { + "type": "string", + "defaultValue": "[concat('cloudbeat-resource-group-', dateTimeToEpoch(utcNow('u')))]", + "metadata": { + "description": "The resource group name where the virtual machine with the Elastic Agent is running on" + } + }, + "SubscriptionId": { + "type": "string", + "metadata": { + "description": "The id of the subscription where the virtual machine with the Elastic Agent is running on" + } + }, + "ElasticArtifactServer": { + "type": "string", + "defaultValue": "https://artifacts.elastic.co/downloads/beats/elastic-agent", + "metadata": { + "description": "The URL of the artifact server" + } + }, + "ElasticAgentVersion": { + "type": "string", + "metadata": { + "description": "The version of elastic-agent to install" + }, + "defaultValue": "9.0.0" + }, + "FleetUrl": { + "type": "string", + "metadata": { + "description": "The fleet URL of elastic-agent" + } + }, + "EnrollmentToken": { + "type": "string", + "metadata": { + "description": "The enrollment token of elastic-agent" + } + }, + "DeploymentLocation": { + "type": "string", + "defaultValue": "[deployment().location]", + "metadata": { + "description": "Deployment location" + } + }, + "PublicKeyDevOnly": { + "type": "string", + "metadata": { + "description": "The public key of the SSH key pair" + } + } + }, + "variables": { + "resourceGroupDeployment": "[concat('resource-group-deployment-', parameters('DeploymentLocation'))]", + "roleAssignmentDeployment": "[concat('role-assignment-deployment-', parameters('DeploymentLocation'))]", + "roleGUID": "[guid(parameters('SubscriptionId'), parameters('ResourceGroupName'))]" + }, + "resources": [ + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "[variables('resourceGroupDeployment')]", + "location": "[parameters('DeploymentLocation')]", + "subscriptionId": "[parameters('SubscriptionId')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "ResourceGroupName": { + "value": "[parameters('ResourceGroupName')]" + }, + "DeploymentLocation": { + "value": "[parameters('DeploymentLocation')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "ResourceGroupName": { + "type": "string" + }, + "DeploymentLocation": { + "type": "string" + } + }, + "resources": [ + { + "type": "Microsoft.Resources/resourceGroups", + "apiVersion": "2022-09-01", + "name": "[parameters('ResourceGroupName')]", + "location": "[parameters('DeploymentLocation')]" + } + ] + } + } + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "[variables('roleAssignmentDeployment')]", + "location": "[parameters('DeploymentLocation')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "AdditionalRoleGUID": { + "value": "[variables('roleGUID')]" + }, + "ManagementGroupID": { + "value": "[managementGroup().id]" + }, + "ResourceGroupName": { + "value": "[parameters('ResourceGroupName')]" + }, + "SubscriptionId": { + "value": "[parameters('SubscriptionId')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-08-01/managementGroupDeploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "AdditionalRoleGUID": { + "type": "string" + }, + "ManagementGroupID": { + "type": "string" + }, + "ResourceGroupName": { + "type": "string" + }, + "SubscriptionId": { + "type": "string" + } + }, + "resources": [ + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(parameters('ManagementGroupID'), parameters('SubscriptionId'), parameters('ResourceGroupName'), deployment().name, 'securityaudit')]", + "properties": { + "roleDefinitionId": "/providers/Microsoft.Authorization/roleDefinitions/acdd72a7-3385-48ef-bd42-f606fba81ae7", + "principalId": "[reference(resourceId(parameters('SubscriptionId'), parameters('ResourceGroupName'), 'Microsoft.Compute/virtualMachines', 'cloudbeatVM'), '2023-09-01', 'Full').identity.principalId]", + "principalType": "ServicePrincipal" + } + }, + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(parameters('SubscriptionId'), parameters('ResourceGroupName'), deployment().name, 'additional-role')]", + "properties": { + "roleDefinitionId": "[concat('/providers/Microsoft.Authorization/roleDefinitions/', parameters('AdditionalRoleGUID'))]", + "principalId": "[reference(resourceId(parameters('SubscriptionId'), parameters('ResourceGroupName'), 'Microsoft.Compute/virtualMachines', 'cloudbeatVM'), '2023-09-01', 'Full').identity.principalId]", + "principalType": "ServicePrincipal" + } + } + ] + } + }, + "dependsOn": [ + "[variables('resourceGroupDeployment')]", + "cloudbeat-vm-deployment" + ] + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "cloudbeat-vm-deployment", + "resourceGroup": "[parameters('ResourceGroupName')]", + "subscriptionId": "[parameters('SubscriptionId')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "parameters": { + "PublicKeyDevOnly": { + "value": "[parameters('PublicKeyDevOnly')]" + } + }, + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "PublicKeyDevOnly": { + "type": "string" + } + }, + "resources": [ + { + "type": "Microsoft.Compute/virtualMachines", + "apiVersion": "2023-09-01", + "name": "cloudbeatVM", + "location": "[parameters('ResourceGroupLocation')]", + "dependsOn": [ + "cloudbeatNic" + ], + "properties": { + "hardwareProfile": { + "vmSize": "[parameters('VMSize')]" + }, + "storageProfile": { + "osDisk": { + "createOption": "FromImage", + "managedDisk": { + "storageAccountType": "Standard_LRS" + } + }, + "imageReference": { + "publisher": "canonical", + "offer": "0001-com-ubuntu-server-jammy", + "sku": "22_04-lts-gen2", + "version": "latest" + } + }, + "osProfile": { + "computerName": "cloudbeatVM", + "adminUsername": "[parameters('AdminUsername')]", + "linuxConfiguration": { + "disablePasswordAuthentication": true, + "ssh": { + "publicKeys": [ + { + "path": "/home/cloudbeat/.ssh/authorized_keys", + "keyData": "[parameters('PublicKeyDevOnly')]" + } + ] + } + } + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": "[resourceId('Microsoft.Network/networkInterfaces', 'cloudbeatNic')]" + } + ] + } + }, + "identity": { + "type": "SystemAssigned" + } + }, + { + "type": "Microsoft.Network/virtualNetworks", + "apiVersion": "2023-06-01", + "name": "cloudbeatVNet", + "location": "[parameters('ResourceGroupLocation')]", + "properties": { + "addressSpace": { + "addressPrefixes": [ + "10.0.0.0/16" + ] + }, + "subnets": [ + { + "name": "cloudbeatSubnet", + "properties": { + "addressPrefix": "10.0.0.0/24", + "networkSecurityGroup": { + "id": "[resourceId('Microsoft.Network/networkSecurityGroups', 'cloudbeatNSGDevOnly')]" + } + } + } + ] + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/networkSecurityGroups', 'cloudbeatNSGDevOnly')]" + ] + }, + { + "type": "Microsoft.Network/networkInterfaces", + "apiVersion": "2023-06-01", + "name": "cloudbeatNic", + "location": "[parameters('ResourceGroupLocation')]", + "dependsOn": [ + "cloudbeatVNet", + "[resourceId('Microsoft.Network/publicIPAddresses', 'cloudbeatPublicIPDevOnly')]" + ], + "properties": { + "ipConfigurations": [ + { + "name": "ipconfig1", + "properties": { + "subnet": { + "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', 'cloudbeatVNet', 'cloudbeatSubnet')]" + }, + "publicIpAddress": { + "id": "[resourceId('Microsoft.Network/publicIPAddresses', 'cloudbeatPublicIPDevOnly')]" + } + } + } + ] + } + }, + { + "type": "Microsoft.Authorization/roleDefinitions", + "apiVersion": "2022-04-01", + "name": "[parameters('AdditionalRoleGUID')]", + "properties": { + "assignableScopes": [ + "[parameters('ManagementGroupID')]", + "[concat('/subscriptions/', parameters('SubscriptionId'))]", + "[concat('/subscriptions/', parameters('SubscriptionId'), '/resourcegroups/', parameters('ResourceGroupName'))]" + ], + "description": "Additional read permissions for cloudbeatVM", + "permissions": [ + { + "actions": [ + "Microsoft.Web/sites/*/read", + "Microsoft.Web/sites/config/Read", + "Microsoft.Web/sites/config/list/Action" + ] + } + ], + "roleName": "[concat('cloudbeatVM additional permissions ', parameters('ResourceGroupName'))]", + "type": "CustomRole" + } + }, + { + "type": "Microsoft.Network/publicIPAddresses", + "name": "cloudbeatPublicIpDevOnly", + "apiVersion": "2020-05-01", + "location": "[resourceGroup().location]", + "properties": { + "publicIPAllocationMethod": "Dynamic" + } + }, + { + "type": "Microsoft.Network/networkSecurityGroups", + "name": "cloudbeatNSGDevOnly", + "apiVersion": "2021-04-01", + "location": "[resourceGroup().location]", + "properties": { + "securityRules": [ + { + "name": "AllowSshAll", + "properties": { + "access": "Allow", + "destinationAddressPrefix": "*", + "destinationPortRange": "22", + "direction": "Inbound", + "priority": 100, + "protocol": "Tcp", + "sourceAddressPrefix": "*", + "sourcePortRange": "*" + } + } + ] + } + } + ] + } + }, + "dependsOn": [ + "[variables('resourceGroupDeployment')]" + ] + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "elastic-agent-deployment", + "resourceGroup": "[parameters('ResourceGroupName')]", + "subscriptionId": "[parameters('SubscriptionId')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "ElasticArtifactServer": { + "value": "[parameters('ElasticArtifactServer')]" + }, + "FleetUrl": { + "value": "[parameters('FleetUrl')]" + }, + "EnrollmentToken": { + "value": "[parameters('EnrollmentToken')]" + }, + "ElasticAgentVersion": { + "value": "[parameters('ElasticAgentVersion')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "ElasticArtifactServer": { + "type": "string" + }, + "FleetUrl": { + "type": "string" + }, + "EnrollmentToken": { + "type": "string" + }, + "ElasticAgentVersion": { + "type": "string" + }, + "ResourceGroupLocation": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Resource Group Location" + } + } + }, + "resources": [ + { + "type": "Microsoft.Compute/virtualMachines/extensions", + "apiVersion": "2023-09-01", + "name": "cloudbeatVM/customScriptExtension", + "location": "[parameters('ResourceGroupLocation')]", + "properties": { + "publisher": "Microsoft.Azure.Extensions", + "type": "CustomScript", + "typeHandlerVersion": "2.1", + "settings": { + "fileUris": [ + "https://raw.githubusercontent.com/elastic/cloudbeat/main/deploy/azure/install-agent-dev.sh" + ], + "commandToExecute": "[concat('bash install-agent-dev.sh ', parameters('ElasticAgentVersion'), ' ', parameters('ElasticArtifactServer'), ' ', parameters('FleetUrl'), ' ', parameters('EnrollmentToken'))]" + } + } + } + ] + } + }, + "dependsOn": [ + "cloudbeat-vm-deployment", + "[variables('roleAssignmentDeployment')]" + ] + } + ] +} diff --git a/deploy/asset-inventory-arm/ARM-for-organization-account.json b/deploy/asset-inventory-arm/ARM-for-organization-account.json new file mode 100644 index 0000000000..08942e3b32 --- /dev/null +++ b/deploy/asset-inventory-arm/ARM-for-organization-account.json @@ -0,0 +1,450 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-08-01/managementGroupDeploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "ResourceGroupName": { + "type": "string", + "defaultValue": "[concat('cloudbeat-resource-group-', dateTimeToEpoch(utcNow('u')))]", + "metadata": { + "description": "The resource group name where the virtual machine with the Elastic Agent is running on" + } + }, + "SubscriptionId": { + "type": "string", + "metadata": { + "description": "The id of the subscription where the virtual machine with the Elastic Agent is running on" + } + }, + "ElasticArtifactServer": { + "type": "string", + "defaultValue": "https://artifacts.elastic.co/downloads/beats/elastic-agent", + "metadata": { + "description": "The URL of the artifact server" + } + }, + "ElasticAgentVersion": { + "type": "string", + "metadata": { + "description": "The version of elastic-agent to install" + }, + "defaultValue": "9.0.0" + }, + "FleetUrl": { + "type": "string", + "metadata": { + "description": "The fleet URL of elastic-agent" + } + }, + "EnrollmentToken": { + "type": "string", + "metadata": { + "description": "The enrollment token of elastic-agent" + } + }, + "DeploymentLocation": { + "type": "string", + "defaultValue": "[deployment().location]", + "metadata": { + "description": "Deployment location" + } + } + }, + "variables": { + "resourceGroupDeployment": "[concat('resource-group-deployment-', parameters('DeploymentLocation'))]", + "roleAssignmentDeployment": "[concat('role-assignment-deployment-', parameters('DeploymentLocation'))]", + "roleGUID": "[guid(parameters('SubscriptionId'), parameters('ResourceGroupName'))]" + }, + "resources": [ + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "[variables('resourceGroupDeployment')]", + "location": "[parameters('DeploymentLocation')]", + "subscriptionId": "[parameters('SubscriptionId')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "ResourceGroupName": { + "value": "[parameters('ResourceGroupName')]" + }, + "DeploymentLocation": { + "value": "[parameters('DeploymentLocation')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "ResourceGroupName": { + "type": "string" + }, + "DeploymentLocation": { + "type": "string" + } + }, + "resources": [ + { + "type": "Microsoft.Resources/resourceGroups", + "apiVersion": "2022-09-01", + "name": "[parameters('ResourceGroupName')]", + "location": "[parameters('DeploymentLocation')]" + } + ] + } + } + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "[variables('roleAssignmentDeployment')]", + "location": "[parameters('DeploymentLocation')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "AdditionalRoleGUID": { + "value": "[variables('roleGUID')]" + }, + "ManagementGroupID": { + "value": "[managementGroup().id]" + }, + "ResourceGroupName": { + "value": "[parameters('ResourceGroupName')]" + }, + "SubscriptionId": { + "value": "[parameters('SubscriptionId')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-08-01/managementGroupDeploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "AdditionalRoleGUID": { + "type": "string" + }, + "ManagementGroupID": { + "type": "string" + }, + "ResourceGroupName": { + "type": "string" + }, + "SubscriptionId": { + "type": "string" + } + }, + "resources": [ + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(parameters('ManagementGroupID'), parameters('SubscriptionId'), parameters('ResourceGroupName'), deployment().name, 'securityaudit')]", + "properties": { + "roleDefinitionId": "/providers/Microsoft.Authorization/roleDefinitions/acdd72a7-3385-48ef-bd42-f606fba81ae7", + "principalId": "[reference(resourceId(parameters('SubscriptionId'), parameters('ResourceGroupName'), 'Microsoft.Compute/virtualMachines', 'cloudbeatVM'), '2023-09-01', 'Full').identity.principalId]", + "principalType": "ServicePrincipal" + } + }, + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(parameters('SubscriptionId'), parameters('ResourceGroupName'), deployment().name, 'additional-role')]", + "properties": { + "roleDefinitionId": "[concat('/providers/Microsoft.Authorization/roleDefinitions/', parameters('AdditionalRoleGUID'))]", + "principalId": "[reference(resourceId(parameters('SubscriptionId'), parameters('ResourceGroupName'), 'Microsoft.Compute/virtualMachines', 'cloudbeatVM'), '2023-09-01', 'Full').identity.principalId]", + "principalType": "ServicePrincipal" + } + } + ] + } + }, + "dependsOn": [ + "[variables('resourceGroupDeployment')]", + "cloudbeat-vm-deployment" + ] + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "cloudbeat-vm-deployment", + "resourceGroup": "[parameters('ResourceGroupName')]", + "subscriptionId": "[parameters('SubscriptionId')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "parameters": { + "AdditionalRoleGUID": { + "value": "[variables('roleGUID')]" + }, + "ManagementGroupID": { + "value": "[managementGroup().id]" + }, + "ResourceGroupName": { + "value": "[parameters('ResourceGroupName')]" + }, + "SubscriptionId": { + "value": "[parameters('SubscriptionId')]" + } + }, + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "AdditionalRoleGUID": { + "type": "string" + }, + "ManagementGroupID": { + "type": "string" + }, + "ResourceGroupName": { + "type": "string" + }, + "SubscriptionId": { + "type": "string" + }, + "AdminUsername": { + "type": "string", + "defaultValue": "cloudbeat", + "metadata": { + "description": "Admin username for the OS profile (Don't change)" + } + }, + "ResourceGroupLocation": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Resource Group Location" + } + }, + "VMSize": { + "type": "string", + "defaultValue": "Standard_DS2_v2", + "metadata": { + "description": "Size of the VM to be deployed" + } + } + }, + "resources": [ + { + "type": "Microsoft.Compute/virtualMachines", + "apiVersion": "2023-09-01", + "name": "cloudbeatVM", + "location": "[parameters('ResourceGroupLocation')]", + "dependsOn": [ + "cloudbeatNic", + "cloudbeatGenerateKeypair" + ], + "properties": { + "hardwareProfile": { + "vmSize": "[parameters('VMSize')]" + }, + "storageProfile": { + "osDisk": { + "createOption": "FromImage", + "managedDisk": { + "storageAccountType": "Standard_LRS" + } + }, + "imageReference": { + "publisher": "canonical", + "offer": "0001-com-ubuntu-server-jammy", + "sku": "22_04-lts-gen2", + "version": "latest" + } + }, + "osProfile": { + "computerName": "cloudbeatVM", + "adminUsername": "[parameters('AdminUsername')]", + "linuxConfiguration": { + "disablePasswordAuthentication": true, + "ssh": { + "publicKeys": [ + { + "path": "/home/cloudbeat/.ssh/authorized_keys", + "keyData": "[reference('cloudbeatGenerateKeypair').outputs.public_key]" + } + ] + } + } + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": "[resourceId('Microsoft.Network/networkInterfaces', 'cloudbeatNic')]" + } + ] + } + }, + "identity": { + "type": "SystemAssigned" + } + }, + { + "type": "Microsoft.Network/virtualNetworks", + "apiVersion": "2023-06-01", + "name": "cloudbeatVNet", + "location": "[parameters('ResourceGroupLocation')]", + "properties": { + "addressSpace": { + "addressPrefixes": [ + "10.0.0.0/16" + ] + }, + "subnets": [ + { + "name": "cloudbeatSubnet", + "properties": { + "addressPrefix": "10.0.0.0/24" + } + } + ] + } + }, + { + "type": "Microsoft.Network/networkInterfaces", + "apiVersion": "2023-06-01", + "name": "cloudbeatNic", + "location": "[parameters('ResourceGroupLocation')]", + "dependsOn": [ + "cloudbeatVNet" + ], + "properties": { + "ipConfigurations": [ + { + "name": "ipconfig1", + "properties": { + "subnet": { + "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', 'cloudbeatVNet', 'cloudbeatSubnet')]" + } + } + } + ] + } + }, + { + "type": "Microsoft.Resources/deploymentScripts", + "apiVersion": "2023-08-01", + "name": "cloudbeatGenerateKeypair", + "location": "[parameters('ResourceGroupLocation')]", + "kind": "AzureCLI", + "properties": { + "azCliVersion": "2.51.0", + "cleanupPreference": "Always", + "retentionInterval": "P1D", + "scriptContent": "#/bin/bash -e\nyes | ssh-keygen -f sshkey -N ''\necho \"{\\\"public_key\\\":\\\"$(cat sshkey.pub)\\\"}\" > $AZ_SCRIPTS_OUTPUT_PATH", + "timeout": "PT30M" + } + }, + { + "type": "Microsoft.Authorization/roleDefinitions", + "apiVersion": "2022-04-01", + "name": "[parameters('AdditionalRoleGUID')]", + "properties": { + "assignableScopes": [ + "[parameters('ManagementGroupID')]", + "[concat('/subscriptions/', parameters('SubscriptionId'))]", + "[concat('/subscriptions/', parameters('SubscriptionId'), '/resourcegroups/', parameters('ResourceGroupName'))]" + ], + "description": "Additional read permissions for cloudbeatVM", + "permissions": [ + { + "actions": [ + "Microsoft.Web/sites/*/read", + "Microsoft.Web/sites/config/Read", + "Microsoft.Web/sites/config/list/Action" + ] + } + ], + "roleName": "[concat('cloudbeatVM additional permissions ', parameters('ResourceGroupName'))]", + "type": "CustomRole" + } + } + ] + } + }, + "dependsOn": [ + "[variables('resourceGroupDeployment')]" + ] + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "elastic-agent-deployment", + "resourceGroup": "[parameters('ResourceGroupName')]", + "subscriptionId": "[parameters('SubscriptionId')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "ElasticArtifactServer": { + "value": "[parameters('ElasticArtifactServer')]" + }, + "FleetUrl": { + "value": "[parameters('FleetUrl')]" + }, + "EnrollmentToken": { + "value": "[parameters('EnrollmentToken')]" + }, + "ElasticAgentVersion": { + "value": "[parameters('ElasticAgentVersion')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "ElasticArtifactServer": { + "type": "string" + }, + "FleetUrl": { + "type": "string" + }, + "EnrollmentToken": { + "type": "string" + }, + "ElasticAgentVersion": { + "type": "string" + }, + "ResourceGroupLocation": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Resource Group Location" + } + } + }, + "resources": [ + { + "type": "Microsoft.Compute/virtualMachines/extensions", + "apiVersion": "2023-09-01", + "name": "cloudbeatVM/customScriptExtension", + "location": "[parameters('ResourceGroupLocation')]", + "properties": { + "publisher": "Microsoft.Azure.Extensions", + "type": "CustomScript", + "typeHandlerVersion": "2.1", + "settings": { + "fileUris": [ + "https://raw.githubusercontent.com/elastic/cloudbeat/main/deploy/azure/install-agent.sh" + ], + "commandToExecute": "[concat('bash install-agent.sh ', parameters('ElasticAgentVersion'), ' ', parameters('ElasticArtifactServer'), ' ', parameters('FleetUrl'), ' ', parameters('EnrollmentToken'))]" + } + } + } + ] + } + }, + "dependsOn": [ + "cloudbeat-vm-deployment", + "[variables('roleAssignmentDeployment')]" + ] + } + ] +} diff --git a/deploy/asset-inventory-arm/ARM-for-single-account.dev.json b/deploy/asset-inventory-arm/ARM-for-single-account.dev.json new file mode 100644 index 0000000000..a8e98b16fe --- /dev/null +++ b/deploy/asset-inventory-arm/ARM-for-single-account.dev.json @@ -0,0 +1,368 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "ElasticArtifactServer": { + "type": "string", + "defaultValue": "https://artifacts.elastic.co/downloads/beats/elastic-agent", + "metadata": { + "description": "The URL of the artifact server" + } + }, + "ElasticAgentVersion": { + "type": "string", + "metadata": { + "description": "The version of elastic-agent to install" + }, + "defaultValue": "9.0.0" + }, + "FleetUrl": { + "type": "string", + "metadata": { + "description": "The fleet URL of elastic-agent" + } + }, + "EnrollmentToken": { + "type": "string", + "metadata": { + "description": "The enrollment token of elastic-agent" + } + }, + "ResourceGroupLocation": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Resource Group location" + } + }, + "PublicKeyDevOnly": { + "type": "string", + "metadata": { + "description": "The public key of the SSH key pair" + } + } + }, + "variables": { + "roleAssignmentDeployment": "[concat('role-assignment-deployment-', resourceGroup().name)]", + "roleGUID": "[guid(subscription().subscriptionId)]" + }, + "resources": [ + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "[variables('roleAssignmentDeployment')]", + "subscriptionId": "[subscription().subscriptionId]", + "location": "[parameters('ResourceGroupLocation')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "ResourceGroupName": { + "value": "[resourceGroup().name]" + }, + "AdditionalRoleGUID": { + "value": "[variables('roleGUID')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "ResourceGroupName": { + "type": "string" + }, + "AdditionalRoleGUID": { + "type": "string" + } + }, + "resources": [ + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(subscription().id, parameters('ResourceGroupName'), deployment().name, 'securityaudit')]", + "properties": { + "roleDefinitionId": "/providers/Microsoft.Authorization/roleDefinitions/acdd72a7-3385-48ef-bd42-f606fba81ae7", + "principalId": "[reference(resourceId(subscription().subscriptionId, parameters('ResourceGroupName'), 'Microsoft.Compute/virtualMachines', 'cloudbeatVM'), '2023-09-01', 'Full').identity.principalId]", + "principalType": "ServicePrincipal" + } + }, + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(subscription().id, parameters('ResourceGroupName'), deployment().name, 'additional-role')]", + "properties": { + "roleDefinitionId": "[concat('/providers/Microsoft.Authorization/roleDefinitions/', parameters('AdditionalRoleGUID'))]", + "principalId": "[reference(resourceId(subscription().subscriptionId, parameters('ResourceGroupName'), 'Microsoft.Compute/virtualMachines', 'cloudbeatVM'), '2023-09-01', 'Full').identity.principalId]", + "principalType": "ServicePrincipal" + } + } + ] + } + }, + "dependsOn": [ + "cloudbeat-vm-deployment" + ] + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "cloudbeat-vm-deployment", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "parameters": { + "PublicKeyDevOnly": { + "value": "[parameters('PublicKeyDevOnly')]" + } + }, + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "PublicKeyDevOnly": { + "type": "string" + } + }, + "resources": [ + { + "type": "Microsoft.Compute/virtualMachines", + "apiVersion": "2023-09-01", + "name": "cloudbeatVM", + "location": "[parameters('ResourceGroupLocation')]", + "dependsOn": [ + "cloudbeatNic" + ], + "properties": { + "hardwareProfile": { + "vmSize": "[parameters('VMSize')]" + }, + "storageProfile": { + "osDisk": { + "createOption": "FromImage", + "managedDisk": { + "storageAccountType": "Standard_LRS" + } + }, + "imageReference": { + "publisher": "canonical", + "offer": "0001-com-ubuntu-server-jammy", + "sku": "22_04-lts-gen2", + "version": "latest" + } + }, + "osProfile": { + "computerName": "cloudbeatVM", + "adminUsername": "[parameters('AdminUsername')]", + "linuxConfiguration": { + "disablePasswordAuthentication": true, + "ssh": { + "publicKeys": [ + { + "path": "/home/cloudbeat/.ssh/authorized_keys", + "keyData": "[parameters('PublicKeyDevOnly')]" + } + ] + } + } + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": "[resourceId('Microsoft.Network/networkInterfaces', 'cloudbeatNic')]" + } + ] + } + }, + "identity": { + "type": "SystemAssigned" + } + }, + { + "type": "Microsoft.Network/virtualNetworks", + "apiVersion": "2023-06-01", + "name": "cloudbeatVNet", + "location": "[parameters('ResourceGroupLocation')]", + "properties": { + "addressSpace": { + "addressPrefixes": [ + "10.0.0.0/16" + ] + }, + "subnets": [ + { + "name": "cloudbeatSubnet", + "properties": { + "addressPrefix": "10.0.0.0/24", + "networkSecurityGroup": { + "id": "[resourceId('Microsoft.Network/networkSecurityGroups', 'cloudbeatNSGDevOnly')]" + } + } + } + ] + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/networkSecurityGroups', 'cloudbeatNSGDevOnly')]" + ] + }, + { + "type": "Microsoft.Network/networkInterfaces", + "apiVersion": "2023-06-01", + "name": "cloudbeatNic", + "location": "[parameters('ResourceGroupLocation')]", + "dependsOn": [ + "cloudbeatVNet", + "[resourceId('Microsoft.Network/publicIPAddresses', 'cloudbeatPublicIPDevOnly')]" + ], + "properties": { + "ipConfigurations": [ + { + "name": "ipconfig1", + "properties": { + "subnet": { + "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', 'cloudbeatVNet', 'cloudbeatSubnet')]" + }, + "publicIpAddress": { + "id": "[resourceId('Microsoft.Network/publicIPAddresses', 'cloudbeatPublicIPDevOnly')]" + } + } + } + ] + } + }, + { + "type": "Microsoft.Authorization/roleDefinitions", + "apiVersion": "2022-04-01", + "name": "[parameters('AdditionalRoleGUID')]", + "properties": { + "assignableScopes": [ + "[concat('/subscriptions/', subscription().subscriptionId)]", + "[concat('/subscriptions/', subscription().subscriptionId, '/resourcegroups/', parameters('ResourceGroupName'))]" + ], + "description": "Additional read permissions for cloudbeatVM", + "permissions": [ + { + "actions": [ + "Microsoft.Web/sites/*/read", + "Microsoft.Web/sites/config/Read", + "Microsoft.Web/sites/config/list/Action" + ] + } + ], + "roleName": "cloudbeatVM additional permissions", + "type": "CustomRole" + } + }, + { + "type": "Microsoft.Network/publicIPAddresses", + "name": "cloudbeatPublicIpDevOnly", + "apiVersion": "2020-05-01", + "location": "[resourceGroup().location]", + "properties": { + "publicIPAllocationMethod": "Dynamic" + } + }, + { + "type": "Microsoft.Network/networkSecurityGroups", + "name": "cloudbeatNSGDevOnly", + "apiVersion": "2021-04-01", + "location": "[resourceGroup().location]", + "properties": { + "securityRules": [ + { + "name": "AllowSshAll", + "properties": { + "access": "Allow", + "destinationAddressPrefix": "*", + "destinationPortRange": "22", + "direction": "Inbound", + "priority": 100, + "protocol": "Tcp", + "sourceAddressPrefix": "*", + "sourcePortRange": "*" + } + } + ] + } + } + ] + } + } + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "elastic-agent-deployment", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "ElasticArtifactServer": { + "value": "[parameters('ElasticArtifactServer')]" + }, + "FleetUrl": { + "value": "[parameters('FleetUrl')]" + }, + "EnrollmentToken": { + "value": "[parameters('EnrollmentToken')]" + }, + "ElasticAgentVersion": { + "value": "[parameters('ElasticAgentVersion')]" + }, + "ResourceGroupLocation": { + "value": "[parameters('ResourceGroupLocation')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "ElasticArtifactServer": { + "type": "string" + }, + "FleetUrl": { + "type": "string" + }, + "EnrollmentToken": { + "type": "string" + }, + "ElasticAgentVersion": { + "type": "string" + }, + "ResourceGroupLocation": { + "type": "string" + } + }, + "resources": [ + { + "type": "Microsoft.Compute/virtualMachines/extensions", + "apiVersion": "2023-09-01", + "name": "cloudbeatVM/customScriptExtension", + "location": "[parameters('ResourceGroupLocation')]", + "properties": { + "publisher": "Microsoft.Azure.Extensions", + "type": "CustomScript", + "typeHandlerVersion": "2.1", + "settings": { + "fileUris": [ + "https://raw.githubusercontent.com/elastic/cloudbeat/main/deploy/azure/install-agent-dev.sh" + ], + "commandToExecute": "[concat('bash install-agent-dev.sh ', parameters('ElasticAgentVersion'), ' ', parameters('ElasticArtifactServer'), ' ', parameters('FleetUrl'), ' ', parameters('EnrollmentToken'))]" + } + } + } + ] + } + }, + "dependsOn": [ + "cloudbeat-vm-deployment", + "[variables('roleAssignmentDeployment')]" + ] + } + ] +} diff --git a/deploy/asset-inventory-arm/ARM-for-single-account.json b/deploy/asset-inventory-arm/ARM-for-single-account.json new file mode 100644 index 0000000000..68564ce5c9 --- /dev/null +++ b/deploy/asset-inventory-arm/ARM-for-single-account.json @@ -0,0 +1,359 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "ElasticArtifactServer": { + "type": "string", + "defaultValue": "https://artifacts.elastic.co/downloads/beats/elastic-agent", + "metadata": { + "description": "The URL of the artifact server" + } + }, + "ElasticAgentVersion": { + "type": "string", + "metadata": { + "description": "The version of elastic-agent to install" + }, + "defaultValue": "9.0.0" + }, + "FleetUrl": { + "type": "string", + "metadata": { + "description": "The fleet URL of elastic-agent" + } + }, + "EnrollmentToken": { + "type": "string", + "metadata": { + "description": "The enrollment token of elastic-agent" + } + }, + "ResourceGroupLocation": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Resource Group location" + } + } + }, + "variables": { + "roleAssignmentDeployment": "[concat('role-assignment-deployment-', resourceGroup().name)]", + "roleGUID": "[guid(subscription().subscriptionId)]" + }, + "resources": [ + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "[variables('roleAssignmentDeployment')]", + "subscriptionId": "[subscription().subscriptionId]", + "location": "[parameters('ResourceGroupLocation')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "ResourceGroupName": { + "value": "[resourceGroup().name]" + }, + "AdditionalRoleGUID": { + "value": "[variables('roleGUID')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "ResourceGroupName": { + "type": "string" + }, + "AdditionalRoleGUID": { + "type": "string" + } + }, + "resources": [ + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(subscription().id, parameters('ResourceGroupName'), deployment().name, 'securityaudit')]", + "properties": { + "roleDefinitionId": "/providers/Microsoft.Authorization/roleDefinitions/acdd72a7-3385-48ef-bd42-f606fba81ae7", + "principalId": "[reference(resourceId(subscription().subscriptionId, parameters('ResourceGroupName'), 'Microsoft.Compute/virtualMachines', 'cloudbeatVM'), '2023-09-01', 'Full').identity.principalId]", + "principalType": "ServicePrincipal" + } + }, + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(subscription().id, parameters('ResourceGroupName'), deployment().name, 'additional-role')]", + "properties": { + "roleDefinitionId": "[concat('/providers/Microsoft.Authorization/roleDefinitions/', parameters('AdditionalRoleGUID'))]", + "principalId": "[reference(resourceId(subscription().subscriptionId, parameters('ResourceGroupName'), 'Microsoft.Compute/virtualMachines', 'cloudbeatVM'), '2023-09-01', 'Full').identity.principalId]", + "principalType": "ServicePrincipal" + } + } + ] + } + }, + "dependsOn": [ + "cloudbeat-vm-deployment" + ] + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "cloudbeat-vm-deployment", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "parameters": { + "ResourceGroupName": { + "value": "[resourceGroup().name]" + }, + "AdditionalRoleGUID": { + "value": "[variables('roleGUID')]" + } + }, + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "ResourceGroupName": { + "type": "string" + }, + "AdditionalRoleGUID": { + "type": "string" + }, + "VMSize": { + "type": "string", + "defaultValue": "Standard_DS2_v2", + "metadata": { + "description": "VM Size" + } + }, + "ResourceGroupLocation": { + "type": "string", + "defaultValue": "[resourceGroup().location]" + }, + "AdminUsername": { + "type": "string", + "defaultValue": "cloudbeat", + "metadata": { + "description": "Admin username for the OS profile (Don't change)" + } + } + }, + "resources": [ + { + "type": "Microsoft.Compute/virtualMachines", + "apiVersion": "2023-09-01", + "name": "cloudbeatVM", + "location": "[parameters('ResourceGroupLocation')]", + "dependsOn": [ + "cloudbeatNic", + "cloudbeatGenerateKeypair" + ], + "properties": { + "hardwareProfile": { + "vmSize": "[parameters('VMSize')]" + }, + "storageProfile": { + "osDisk": { + "createOption": "FromImage", + "managedDisk": { + "storageAccountType": "Standard_LRS" + } + }, + "imageReference": { + "publisher": "canonical", + "offer": "0001-com-ubuntu-server-jammy", + "sku": "22_04-lts-gen2", + "version": "latest" + } + }, + "osProfile": { + "computerName": "cloudbeatVM", + "adminUsername": "[parameters('AdminUsername')]", + "linuxConfiguration": { + "disablePasswordAuthentication": true, + "ssh": { + "publicKeys": [ + { + "path": "/home/cloudbeat/.ssh/authorized_keys", + "keyData": "[reference('cloudbeatGenerateKeypair').outputs.public_key]" + } + ] + } + } + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": "[resourceId('Microsoft.Network/networkInterfaces', 'cloudbeatNic')]" + } + ] + } + }, + "identity": { + "type": "SystemAssigned" + } + }, + { + "type": "Microsoft.Network/virtualNetworks", + "apiVersion": "2023-06-01", + "name": "cloudbeatVNet", + "location": "[parameters('ResourceGroupLocation')]", + "properties": { + "addressSpace": { + "addressPrefixes": [ + "10.0.0.0/16" + ] + }, + "subnets": [ + { + "name": "cloudbeatSubnet", + "properties": { + "addressPrefix": "10.0.0.0/24" + } + } + ] + } + }, + { + "type": "Microsoft.Network/networkInterfaces", + "apiVersion": "2023-06-01", + "name": "cloudbeatNic", + "location": "[parameters('ResourceGroupLocation')]", + "dependsOn": [ + "cloudbeatVNet" + ], + "properties": { + "ipConfigurations": [ + { + "name": "ipconfig1", + "properties": { + "subnet": { + "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', 'cloudbeatVNet', 'cloudbeatSubnet')]" + } + } + } + ] + } + }, + { + "type": "Microsoft.Resources/deploymentScripts", + "apiVersion": "2023-08-01", + "name": "cloudbeatGenerateKeypair", + "location": "[parameters('ResourceGroupLocation')]", + "kind": "AzureCLI", + "properties": { + "azCliVersion": "2.51.0", + "cleanupPreference": "Always", + "retentionInterval": "P1D", + "scriptContent": "#/bin/bash -e\nyes | ssh-keygen -f sshkey -N ''\necho \"{\\\"public_key\\\":\\\"$(cat sshkey.pub)\\\"}\" > $AZ_SCRIPTS_OUTPUT_PATH", + "timeout": "PT30M" + } + }, + { + "type": "Microsoft.Authorization/roleDefinitions", + "apiVersion": "2022-04-01", + "name": "[parameters('AdditionalRoleGUID')]", + "properties": { + "assignableScopes": [ + "[concat('/subscriptions/', subscription().subscriptionId)]", + "[concat('/subscriptions/', subscription().subscriptionId, '/resourcegroups/', parameters('ResourceGroupName'))]" + ], + "description": "Additional read permissions for cloudbeatVM", + "permissions": [ + { + "actions": [ + "Microsoft.Web/sites/*/read", + "Microsoft.Web/sites/config/Read", + "Microsoft.Web/sites/config/list/Action" + ] + } + ], + "roleName": "cloudbeatVM additional permissions", + "type": "CustomRole" + } + } + ] + } + } + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "elastic-agent-deployment", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "ElasticArtifactServer": { + "value": "[parameters('ElasticArtifactServer')]" + }, + "FleetUrl": { + "value": "[parameters('FleetUrl')]" + }, + "EnrollmentToken": { + "value": "[parameters('EnrollmentToken')]" + }, + "ElasticAgentVersion": { + "value": "[parameters('ElasticAgentVersion')]" + }, + "ResourceGroupLocation": { + "value": "[parameters('ResourceGroupLocation')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "ElasticArtifactServer": { + "type": "string" + }, + "FleetUrl": { + "type": "string" + }, + "EnrollmentToken": { + "type": "string" + }, + "ElasticAgentVersion": { + "type": "string" + }, + "ResourceGroupLocation": { + "type": "string" + } + }, + "resources": [ + { + "type": "Microsoft.Compute/virtualMachines/extensions", + "apiVersion": "2023-09-01", + "name": "cloudbeatVM/customScriptExtension", + "location": "[parameters('ResourceGroupLocation')]", + "properties": { + "publisher": "Microsoft.Azure.Extensions", + "type": "CustomScript", + "typeHandlerVersion": "2.1", + "settings": { + "fileUris": [ + "https://raw.githubusercontent.com/elastic/cloudbeat/main/deploy/azure/install-agent.sh" + ], + "commandToExecute": "[concat('bash install-agent.sh ', parameters('ElasticAgentVersion'), ' ', parameters('ElasticArtifactServer'), ' ', parameters('FleetUrl'), ' ', parameters('EnrollmentToken'))]" + } + } + } + ] + } + }, + "dependsOn": [ + "cloudbeat-vm-deployment", + "[variables('roleAssignmentDeployment')]" + ] + } + ] +} diff --git a/deploy/asset-inventory-arm/README.md b/deploy/asset-inventory-arm/README.md new file mode 100644 index 0000000000..93d4d73d5a --- /dev/null +++ b/deploy/asset-inventory-arm/README.md @@ -0,0 +1,59 @@ +## ARM deployment for developers + +The [`generate_dev_template.py`](./generate_dev_template.py) script generates an ARM template for deploying the Elastic +Agent with SSH access enabled to the VM. This script works both for the single subscription and management group +templates. + +Usage: + +```text +usage: generate_dev_template.py [-h] + [--template-type {single-account,organization-account}] + [--output-file OUTPUT_FILE] [--deploy] + [--resource-group RESOURCE_GROUP] + [--public-ssh-key PUBLIC_SSH_KEY] + [--artifact-server ARTIFACT_SERVER] + [--elastic-agent-version ELASTIC_AGENT_VERSION] + [--fleet-url FLEET_URL] + [--enrollment-token ENROLLMENT_TOKEN] + +Deploy Azure resources for a single account + +options: + -h, --help show this help message and exit + --template-type {single-account,organization-account} + The type of template to use + --output-file OUTPUT_FILE + The output file to write the modified template to + --deploy Perform deployment + --resource-group RESOURCE_GROUP + The resource group to deploy to + --public-ssh-key PUBLIC_SSH_KEY + SSH public key to use for the VMs + --artifact-server ARTIFACT_SERVER + The URL of the artifact server + --elastic-agent-version ELASTIC_AGENT_VERSION + The version of elastic-agent to install + --fleet-url FLEET_URL + The fleet URL of elastic-agent + --enrollment-token ENROLLMENT_TOKEN + The enrollment token of elastic-agent +``` + +Arguments are also read from the `dev-flags.conf` file in the same directory as the script. Write the arguments in the +file as you would pass them to the script. Notice that you need to properly quote arguments. Example: + +```text +--artifact-server https://snapshots.elastic.co/8.12.0-t9e0i58r/downloads/beats/elastic-agent +--elastic-agent-version 8.12.0-SNAPSHOT +--fleet-url +--enrollment-token +--public-ssh-key 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC3e38/Q26WUsyUVb4D7N1McL9QbrcamMfZw23+txivvP13QXzIEyvMjsqUpX0kqjg+C4OD7osfZ+wlVI3QFkomjDjjPMx/FYGUGk5ZKvKh9vXyxN2brYZq8C24lWQSpZbmvNF4+FueFx1eo6wMllLzmzzQ60LpeBhNhRiDPiLQKBotDn1mD6zymnhSANpS/+rWX5HVguSQgtEZP4vvxpKVxEM8hnT8V0PvWFfuNQpTf7zVpZtFvGTLoosvvGbQ27wiufHdF8vv9mF5cXhy02N4IaREcJEMu5wmQaD7zUcJ67aN4v7FTwkA6D3sppb7cJolUJJiOWh4kt7K03BEBYIM9g88lhHDFxwpUvMNWhwp/RHnu8/Ic3HL623W5EDcXxsjH1gsIpXtNuSaUP6G+c2k1zvmST7Oom6EXLT47hv9MXWcS7zY1YZtqVlboZiBRH5MfqwRPFHl6r04yqq1vithW/LeBweH8/q4iWaVYABda0Zmq8qFKKu/5VZStqbOt5wa0bIZrMn+dU6NUHlP6gOuM1yb7kbR2Y/x7AnHvNZ8YtcXDmoMjX93/7A+4Dr3qZd0FKtVoYqUspg0jOGH/Kj3sswp7oM98yJz5F/3/7VwSdzO/DzSGr9Of9BLCQHfcS6qJUZjsErPDqc0T7v7c+Dsz73t5zYq8uYovtUt6m3Anw== user@hostname' +--deploy +``` + +Executing the deployment with `--deploy` requires the `az` CLI to be installed and logged in to the correct +subscription. + +The script is included the pre-commit pipelines so new dev templates will be generated each time a change is made to the +source templates. diff --git a/deploy/asset-inventory-arm/generate_dev_template.py b/deploy/asset-inventory-arm/generate_dev_template.py new file mode 100755 index 0000000000..7c6428daf5 --- /dev/null +++ b/deploy/asset-inventory-arm/generate_dev_template.py @@ -0,0 +1,305 @@ +#!/usr/bin/env python +# pylint: disable=duplicate-code +""" +Generate and deploy development templates for Azure deployment. + +Enables SSH access to the VMs and installs the elastic-agent with the given version and enrollment token. +""" +import argparse +import json +import os +import pathlib +import shlex +import subprocess +import sys +import time + + +def main(): + """ + Parse arguments and run the script. + """ + args = parse_args(load_file_args() + sys.argv[1:]) + + with open(args.template_file) as f: + template = json.load(f) + + modify_template(template) + with open(args.output_file, "w") as f: + print(json.dumps(template, indent=4), file=f) # Pretty-print the template in a JSON file. + + if args.deploy: + if args.template_type == "organization-account": + deploy_to_management_group(args) + else: + deploy_to_subscription(args) + + +def load_file_args(): + """ + Load extra command-line arguments from a file. + """ + config_file = pathlib.Path(__file__).parent / "dev-flags.conf" + if not config_file.exists(): + return [] + with open(config_file) as f: + return shlex.split(f.read().strip()) + + +def parse_args(argv): + """ + Parse command-line arguments. + :param argv: The arguments + :return: Parsed argparse namespace + """ + will_call_az_cli = "--deploy" in argv + + parser = argparse.ArgumentParser(description="Deploy Azure resources for a single account") + parser.add_argument( + "--template-type", + help="The type of template to use", + default="single-account", + choices=["single-account", "organization-account"], + ) + parser.add_argument( + "--output-file", + help="The output file to write the modified template to", + default=None, # Replace later + ) + parser.add_argument("--deploy", help="Perform deployment", action="store_true") + parser.add_argument( + "--resource-group", + help="The resource group to deploy to", + default=f"{os.environ.get('USER', 'unknown')}-cloudbeat-dev-{int(time.time())}", + ) + parser.add_argument("--location", help="The location to deploy to", default=os.environ.get("LOCATION", "centralus")) + parser.add_argument("--subscription-id", help="The subscription ID to deploy to (defaults to current)") + parser.add_argument("--management-group-id", help="The management group ID to deploy to") + + parser.add_argument("--public-ssh-key", help="SSH public key to use for the VMs", required=will_call_az_cli) + parser.add_argument("--artifact-server", help="The URL of the artifact server", required=will_call_az_cli) + parser.add_argument( + "--elastic-agent-version", + help="The version of elastic-agent to install", + default=os.environ.get("ELK_VERSION", ""), + ) + parser.add_argument("--fleet-url", help="The fleet URL of elastic-agent", required=will_call_az_cli) + parser.add_argument("--enrollment-token", help="The enrollment token of elastic-agent", required=will_call_az_cli) + args = parser.parse_args(argv) + + if args.deploy != will_call_az_cli: + parser.error("Assertion failed: --deploy detected but parser returned different result") + + args.template_file = pathlib.Path(__file__).parent / f"ARM-for-{args.template_type}.json" + if args.output_file is None: + args.output_file = str(args.template_file).replace(".json", ".dev.json") + if args.template_type == "single-account" and args.management_group_id is not None: + parser.error("Cannot specify management group for single-account template") + elif args.deploy and args.template_type == "organization-account" and args.management_group_id is None: + parser.error("Must specify management group for organization-account template") + + return args + + +def modify_template(template): + """ + Modify the template in-place. + :param template: Parsed dictionary of the template + """ + template["parameters"]["PublicKeyDevOnly"] = { + "type": "string", + "metadata": {"description": "The public key of the SSH key pair"}, + } + + # Shallow copy of all resources and resources of deployments + all_resources = template["resources"][:] + for resource in template["resources"]: + if resource["type"] == "Microsoft.Resources/deployments": + all_resources += resource["properties"]["template"]["resources"] + for resource in all_resources: + modify_resource(resource) + + +def modify_resource(resource): + """ + Modify a single resource in-place. + :param resource: Parsed dictionary of the resource + """ + # Delete generated key pair from all dependencies + depends_on = [d for d in resource.get("dependsOn", []) if not d.startswith("cloudbeatGenerateKeypair")] + + if resource["name"] == "cloudbeatVM": + # Use user-provided public key + resource["properties"]["osProfile"]["linuxConfiguration"]["ssh"]["publicKeys"] = [ + { + "path": "/home/cloudbeat/.ssh/authorized_keys", + "keyData": "[parameters('PublicKeyDevOnly')]", + }, + ] + elif resource["name"] == "cloudbeatVNet": + # Add network security group to virtual network + nsg_resource_id = "[resourceId('Microsoft.Network/networkSecurityGroups', 'cloudbeatNSGDevOnly')]" + resource["properties"]["subnets"][0]["properties"]["networkSecurityGroup"] = {"id": nsg_resource_id} + depends_on += [nsg_resource_id] + elif resource["name"] == "cloudbeatNic": + # Add public IP to network interface + public_ip_resource_id = "[resourceId('Microsoft.Network/publicIPAddresses', 'cloudbeatPublicIPDevOnly')]" + resource["properties"]["ipConfigurations"][0]["properties"]["publicIpAddress"] = {"id": public_ip_resource_id} + depends_on += [public_ip_resource_id] + elif resource["name"] == "cloudbeatVM/customScriptExtension": + # Modify agent installation to *not* disable SSH + resource["properties"]["settings"] = { + "fileUris": ["https://raw.githubusercontent.com/elastic/cloudbeat/main/deploy/azure/install-agent-dev.sh"], + "commandToExecute": ( + "[concat('" + "bash install-agent-dev.sh ', " + "parameters('ElasticAgentVersion'), ' ', " + "parameters('ElasticArtifactServer'), ' ', " + "parameters('FleetUrl'), ' ', " + "parameters('EnrollmentToken'))]" + ), + } + elif resource["name"] == "cloudbeat-vm-deployment": + resource["properties"]["parameters"] = {"PublicKeyDevOnly": {"value": "[parameters('PublicKeyDevOnly')]"}} + resource["properties"]["template"]["parameters"] = {"PublicKeyDevOnly": {"type": "string"}} + modify_vm_deployment_template_resources_array(resource["properties"]["template"]) + + if depends_on: + resource["dependsOn"] = depends_on + + +def modify_vm_deployment_template_resources_array(template): + """ + Modify the resources array of the cloudbeat VM deployment template in-place. + :param template: Parsed dictionary of the template + """ + template["resources"] = [ + resource + for resource in template["resources"] + # Delete generated key pair since we provide our own + if resource["name"] != "cloudbeatGenerateKeypair" + ] + [ + { + "type": "Microsoft.Network/publicIPAddresses", + "name": "cloudbeatPublicIpDevOnly", + "apiVersion": "2020-05-01", + "location": "[resourceGroup().location]", + "properties": {"publicIPAllocationMethod": "Dynamic"}, + }, + { + "type": "Microsoft.Network/networkSecurityGroups", + "name": "cloudbeatNSGDevOnly", + "apiVersion": "2021-04-01", + "location": "[resourceGroup().location]", + "properties": { + "securityRules": [ + { + "name": "AllowSshAll", + "properties": { + "access": "Allow", + "destinationAddressPrefix": "*", + "destinationPortRange": "22", + "direction": "Inbound", + "priority": 100, + "protocol": "Tcp", + "sourceAddressPrefix": "*", + "sourcePortRange": "*", + }, + }, + ], + }, + }, + ] + + +def deploy_to_subscription(args): + """ + Deploy the template to a subscription. + :param args: The parsed arguments + """ + parameters = parameters_from_args(args) + subscription_args = ["--subscription", args.subscription_id] if args.subscription_id else [] + subprocess.check_call( + [ + "az", + "group", + "create", + "--name", + args.resource_group, + "--location", + args.location, + ] + + subscription_args, + ) + subprocess.check_call( + [ + "az", + "deployment", + "group", + "create", + "--resource-group", + args.resource_group, + "--template-file", + args.output_file, + "--parameters", + json.dumps(parameters), + ] + + subscription_args, + ) + + +def deploy_to_management_group(args): + """ + Deploy the template to a management group. + :param args: The parsed arguments + """ + parameters = parameters_from_args(args) + parameters["parameters"]["ResourceGroupName"] = {"value": args.resource_group} + if args.subscription_id is None: + args.subscription_id = ( + subprocess.check_output(["az", "account", "show", "--query", "id", "-o", "tsv"]) + .decode( + "utf-8", + ) + .strip() + ) + parameters["parameters"]["SubscriptionId"] = {"value": args.subscription_id} + subprocess.check_call( + [ + "az", + "deployment", + "mg", + "create", + "--location", + args.location, + "--template-file", + args.output_file, + "--parameters", + json.dumps(parameters), + "--management-group-id", + args.management_group_id, + ], + ) + + +def parameters_from_args(args): + """ + Generate the deployment parameters file from the parsed arguments. + :param args: The parsed arguments + :return: + """ + return { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "ElasticArtifactServer": {"value": args.artifact_server}, + "ElasticAgentVersion": {"value": args.elastic_agent_version}, + "FleetUrl": {"value": args.fleet_url}, + "EnrollmentToken": {"value": args.enrollment_token}, + "PublicKeyDevOnly": {"value": args.public_ssh_key}, + }, + } + + +if __name__ == "__main__": + main() diff --git a/deploy/asset-inventory-arm/install-agent-dev.sh b/deploy/asset-inventory-arm/install-agent-dev.sh new file mode 100644 index 0000000000..fa700ac332 --- /dev/null +++ b/deploy/asset-inventory-arm/install-agent-dev.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -euxo pipefail + +usage="$0 " +ElasticAgentVersion=${1:?$usage} +ElasticArtifactServer=${2:?$usage} +FleetUrl=${3:?$usage} +EnrollmentToken=${4:?$usage} + +ElasticAgentArtifact="elastic-agent-$ElasticAgentVersion-linux-x86_64" +curl -L -O "${ElasticArtifactServer}/$ElasticAgentArtifact.tar.gz" +tar xzf "${ElasticAgentArtifact}.tar.gz" +cd "${ElasticAgentArtifact}" +sudo ./elastic-agent install --non-interactive --url="${FleetUrl}" --enrollment-token="${EnrollmentToken}" + +wait diff --git a/deploy/asset-inventory-arm/install-agent.sh b/deploy/asset-inventory-arm/install-agent.sh new file mode 100644 index 0000000000..a895da2077 --- /dev/null +++ b/deploy/asset-inventory-arm/install-agent.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -euxo pipefail + +# Delete password, stop and disable ssh +sudo passwd --delete cloudbeat & +( + sudo systemctl disable --now ssh || true + sudo systemctl mask ssh.service || true + sudo killall sshd || true +) & + +usage="$0 " +ElasticAgentVersion=${1:?$usage} +ElasticArtifactServer=${2:?$usage} +FleetUrl=${3:?$usage} +EnrollmentToken=${4:?$usage} + +ElasticAgentArtifact="elastic-agent-$ElasticAgentVersion-linux-x86_64" +curl -L -O "${ElasticArtifactServer}/$ElasticAgentArtifact.tar.gz" +tar xzf "${ElasticAgentArtifact}.tar.gz" +cd "${ElasticAgentArtifact}" +sudo ./elastic-agent install --non-interactive --url="${FleetUrl}" --enrollment-token="${EnrollmentToken}" + +wait diff --git a/deploy/asset-inventory-arm/install_agent_az_cli.sh b/deploy/asset-inventory-arm/install_agent_az_cli.sh new file mode 100755 index 0000000000..406741d4b3 --- /dev/null +++ b/deploy/asset-inventory-arm/install_agent_az_cli.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +# This script is used to deploy Azure resources and apply tags to them using Azure CLI. +# Requiered environment variables: +# - AZURE_TAGS: The default tags to apply to the resources. For example, "key1=value1 key2=value2". +# - DEPLOYMENT_NAME: The name of the deployment. +# - STACK_VERSION: The version of the stack. +# Pre-requisites: +# arm_parameters.json: The parameters file for the ARM template should be present in the same directory as this script. +# Example: +# { +# "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", +# "contentVersion": "1.0.0.0", +# "parameters": { +# "EnrollmentToken": { +# "value": "your-enrollment-token" +# }, +# "FleetUrl": { +# "value": "https://your-fleet-url" +# }, +# "ElasticArtifactServer": { +# "value": "https://artifacts.elastic.co/downloads/beats/elastic-agent" +# }, +# "ElasticAgentVersion": { +# "value": "8.14.1" +# } +# } +# } +# Usage: +# cd ./deploy/azure +# STACK_VERSION="8.14.1" DEPLOYMENT_NAME="elastic-agent-cspm" AZURE_TAGS="key1=value1 key2=value2" ./install_agent_az_cli.sh + +# Exit immediately if a command exits with a non-zero status, print each command before executing it, and fail pipelines if any command fails. +set -euo pipefail + +MINOR_VERSION=$(echo "${STACK_VERSION}" | cut -d'.' -f2) +# Check if minor version is less than 12, ie. 8.11 and below +if ((MINOR_VERSION < 12)); then + echo "Versions 8.11 and below are not supported. Please use versions 8.12+" + exit 0 +fi + +# Create a resource group with the name DEPLOYMENT_NAME +az group create --location EastUS --name "${DEPLOYMENT_NAME}" + +# Create a group deployment using the ARM-for-single-account.json +az deployment group create --resource-group "${DEPLOYMENT_NAME}" --template-file ARM-for-single-account.json --parameters @arm_parameters.json + +# Get the resource group ID of the DEPLOYMENT_NAME and store it in group +group=$(az group show -n "${DEPLOYMENT_NAME}" --query id --output tsv) + +# --tags requires the tags to be passed as separate arguments +# shellcheck disable=SC2086 +az tag update --resource-id "$group" --operation Merge --tags ${AZURE_TAGS} +# Get all resources within the resource group and store their IDs in an array +resources=() +while IFS= read -r resource_id; do + resources+=("$resource_id") +done < <(az resource list --resource-group "${DEPLOYMENT_NAME}" --query "[].id" --output tsv) + +# Loop over each resource ID in resources +for resource in "${resources[@]}"; do + # Merge the deploy_tags with the existing tags of the resource + # --tags requires the tags to be passed as separate arguments + # shellcheck disable=SC2086 + az tag update --resource-id "$resource" --operation Merge --tags ${AZURE_TAGS} +done diff --git a/deploy/asset-inventory-cloudformation/.gitignore b/deploy/asset-inventory-cloudformation/.gitignore new file mode 100644 index 0000000000..c3b2c2453a --- /dev/null +++ b/deploy/asset-inventory-cloudformation/.gitignore @@ -0,0 +1,4 @@ +elastic-agent-ec2-dev-*.yml +*generated.yml +config.env +config.json diff --git a/deploy/asset-inventory-cloudformation/README.md b/deploy/asset-inventory-cloudformation/README.md new file mode 100644 index 0000000000..76aef2d16d --- /dev/null +++ b/deploy/asset-inventory-cloudformation/README.md @@ -0,0 +1,34 @@ +## Elastic Agent EC2 CloudFormation template + +### What it does +This CloudFormation template creates a role for elastic-agent and attaches it to a newly created EC2 instance. +The EC2 instance has elastic-agent preinstalled in it using the fleet URL and enrollment token. + +### How to test it +*Prerequisites:* +1. You have an elastic stack deployed in the cloud that includes Kibana, elasticsearch and fleet-server (check https://github.com/elastic/cloudbeat/blob/main/dev-docs/ELK-Deployment.md to deploy your own stack) +2. You have AWS CLI installed on your laptop and configured to work with our dev account `elastic-security-cloud-security-dev` (in particular, `~/.aws/config` and `~/.aws/credentials` should be set, check https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html for more information) + +*Steps:* +1. Install the Vulnerability Management integration on a new agent policy, you might have to check the "Display beta integrations" checkbox. +2. After you installed the integration you can install a new elastic-agent, you should keep the fleet URL and the enrollment token. +3. On cloudbeat repo, create a `deploy/cloudformation/config.env` file of the form: +``` +STACK_NAME="" # john-qa-bc2-8-9-0-May28 +FLEET_URL="" +ENROLLMENT_TOKEN="" +ELASTIC_ARTIFACT_SERVER="https://artifacts.elastic.co/downloads/beats/elastic-agent" # Replace artifact URL with a pre-release version (BC or snapshot) +ELASTIC_AGENT_VERSION="" # e.g: 8.8.0 | 8.8.0-SNAPSHOT + +DEV.ALLOW_SSH=false # Set to true to allow SSH connections to the deployed instance +DEV.KEY_NAME="" # When SSH is allowed, you must provide the key name that will be used to ssh into the EC2 +``` +4. Run `just deploy-cloudformation` to create a CloudFormation stack with an elastic-agent that will automatically enroll to your fleet. + +*Debugging:* +1. CloudFormation stack creation may take a few minutes, to see the progress, find your stack on https://console.aws.amazon.com/cloudformation/ and check the "Event" tab. +2. If the stack was created successfully but elastic-agent didn't enroll to your fleet, try to ssh into the EC2 by running `ssh -i ~/.ssh/ ubuntu@` and then get the initialization logs by `cat /var/log/cloud-init-output.log`. +3. If ssh is not enabled, you can get the system logs from the EC2 instance +![right click on instance -> monitor and troubleshoot -> get sytem logs](https://github.com/orestisfl/cloudbeat/assets/5778622/3f0158ed-ba46-4fe5-b7e9-7a800b3de020) +![system logs window](https://github.com/orestisfl/cloudbeat/assets/5778622/af8105d1-7d6c-47d0-b386-eea124816b53) +You might need to wait a bit until the logs become available but terminating the instance doesn't immediately delete them diff --git a/deploy/asset-inventory-cloudformation/config.go b/deploy/asset-inventory-cloudformation/config.go new file mode 100644 index 0000000000..af54ff63d9 --- /dev/null +++ b/deploy/asset-inventory-cloudformation/config.go @@ -0,0 +1,126 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Config is put into a different package to prevent cyclic imports in case +// it is needed in several locations + +package main + +import ( + "fmt" + "reflect" + "strings" + + "github.com/spf13/viper" +) + +type config struct { + StackName string `mapstructure:"STACK_NAME"` + FleetURL string `mapstructure:"FLEET_URL"` + EnrollmentToken string `mapstructure:"ENROLLMENT_TOKEN"` + ElasticArtifactServer *string `mapstructure:"ELASTIC_ARTIFACT_SERVER"` + ElasticAgentVersion string `mapstructure:"ELASTIC_AGENT_VERSION"` + Dev *devConfig `mapstructure:"DEV"` +} + +type devConfig struct { + AllowSSH bool `mapstructure:"ALLOW_SSH"` + KeyName string `mapstructure:"KEY_NAME"` +} + +func parseConfig() (*config, error) { + // Read in configuration from on of the files: config.json, config.yml or config.env + viper.AddConfigPath("./") + err := viper.ReadInConfig() + if err != nil { + return nil, fmt.Errorf("failed to read configuration: %v", err) + } + + var cfg config + err = bindEnvs(cfg) + if err != nil { + return nil, fmt.Errorf("failed to bind environment variables: %v", err) + } + + err = viper.Unmarshal(&cfg) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal configuration file: %v", err) + } + + err = validateConfig(&cfg) + if err != nil { + return nil, err + } + + return &cfg, nil +} + +func bindEnvs(iface any, parts ...string) error { + ifv := reflect.ValueOf(iface) + ift := reflect.TypeOf(iface) + for i := 0; i < ift.NumField(); i++ { + v := ifv.Field(i) + t := ift.Field(i) + tv, ok := t.Tag.Lookup("mapstructure") + if !ok { + continue + } + var err error + switch v.Kind() { + case reflect.Struct: + err = bindEnvs(v.Interface(), append(parts, tv)...) + default: + err = viper.BindEnv(strings.Join(append(parts, tv), ".")) + } + if err != nil { + return err + } + } + return nil +} + +func validateConfig(cfg *config) error { + if cfg.StackName == "" { + return fmt.Errorf("missing required flag: STACK_NAME") + } + + if cfg.FleetURL == "" { + return fmt.Errorf("missing required flag: FLEET_URL") + } + + if cfg.EnrollmentToken == "" { + return fmt.Errorf("missing required flag: ENROLLMENT_TOKEN") + } + + if cfg.ElasticAgentVersion == "" { + return fmt.Errorf("missing required flag: ELASTIC_AGENT_VERSION") + } + + if cfg.Dev != nil { + return validateDevConfig(cfg.Dev) + } + + return nil +} + +func validateDevConfig(cfg *devConfig) error { + if cfg.AllowSSH && cfg.KeyName == "" { + return fmt.Errorf("missing required flag for SSH enablement mode: DEV.KEY_NAME") + } + + return nil +} diff --git a/deploy/asset-inventory-cloudformation/ec2-types.yml b/deploy/asset-inventory-cloudformation/ec2-types.yml new file mode 100644 index 0000000000..3e94fc7126 --- /dev/null +++ b/deploy/asset-inventory-cloudformation/ec2-types.yml @@ -0,0 +1,156 @@ +# The purpose of this template is to create a set of EC2 instances with different instance types and AMIs. +# Then scan these instances with vulnerability scanner in order to verify that the scanner works well. +# Currently this process is being done manually, but eventually we would like to automate it. +# This template is separate to the elastic-agent-ec2.yml template and is not used in the production deployment process. +# All AMIs in this template are only available in eu-west-1 region. +AWSTemplateFormatVersion: "2010-09-09" + +Parameters: + Encrypted: + Description: Whether to encrypt the EC2 instance root volume + Type: String + AllowedValues: + - "true" + - "false" + Default: "false" + +Resources: + arm64AmazonLinux2: + Type: AWS::EC2::Instance + Properties: + Tags: + - Key: Name + Value: !Join + - '-' + - - vuln-mgmt-types-arm64AmazonLinux2 + - !Select + - 2 + - !Split + - / + - !Ref AWS::StackId + InstanceType: t4g.nano + ImageId: ami-0a0ae3c8519bff7f0 + BlockDeviceMappings: + - DeviceName: /dev/xvda + Ebs: + VolumeSize: "10" + Encrypted: !Ref Encrypted + + arm64RHLinux9: + Type: AWS::EC2::Instance + Properties: + Tags: + - Key: Name + Value: !Join + - '-' + - - vuln-mgmt-types-arm64RHLinux9 + - !Select + - 2 + - !Split + - / + - !Ref AWS::StackId + InstanceType: t4g.small + ImageId: ami-062e673cc4273dad8 + BlockDeviceMappings: + - DeviceName: /dev/sda1 + Ebs: + VolumeSize: "10" + Encrypted: !Ref Encrypted + + x86SuseLinux15: + Type: AWS::EC2::Instance + Properties: + Tags: + - Key: Name + Value: !Join + - '-' + - - vuln-mgmt-types-x86SuseLinux15 + - !Select + - 2 + - !Split + - / + - !Ref AWS::StackId + InstanceType: t2.nano + ImageId: ami-09ee771fad415a6d7 + BlockDeviceMappings: + - DeviceName: /dev/sda1 + Ebs: + VolumeSize: "10" + Encrypted: !Ref Encrypted + + x86UbuntuLinux2204: + Type: AWS::EC2::Instance + Properties: + Tags: + - Key: Name + Value: !Join + - '-' + - - vuln-mgmt-types-x86UbuntuLinux2204 + - !Select + - 2 + - !Split + - / + - !Ref AWS::StackId + InstanceType: t2.nano + ImageId: ami-00aa9d3df94c6c354 + BlockDeviceMappings: + - DeviceName: /dev/sda1 + Ebs: + VolumeSize: "10" + Encrypted: !Ref Encrypted + + x86DebianLinux11: + Type: AWS::EC2::Instance + Properties: + Tags: + - Key: Name + Value: !Join + - '-' + - - vuln-mgmt-types-x86DebianLinux11 + - !Select + - 2 + - !Split + - / + - !Ref AWS::StackId + InstanceType: t2.nano + ImageId: ami-089f338f3a2e69431 + BlockDeviceMappings: + - DeviceName: /dev/xvda + Ebs: + VolumeSize: "10" + Encrypted: !Ref Encrypted + + x86AmazonLinux2023: + Type: AWS::EC2::Instance + Properties: + Tags: + - Key: Name + Value: !Join + - '-' + - - vuln-mgmt-types-x86AmazonLinux2023 + - !Select + - 2 + - !Split + - / + - !Ref AWS::StackId + InstanceType: t2.nano + ImageId: ami-04b1c88a6bbd48f8e + BlockDeviceMappings: + - DeviceName: /dev/xvda + Ebs: + VolumeSize: "10" + Encrypted: !Ref Encrypted + + SecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupName: !Join + - '-' + - - vuln-mgmt-types + - !Select + - 2 + - !Split + - / + - !Ref AWS::StackId + GroupDescription: Block incoming traffic + SecurityGroupIngress: [] diff --git a/deploy/asset-inventory-cloudformation/elastic-agent-ec2-organization.yml b/deploy/asset-inventory-cloudformation/elastic-agent-ec2-organization.yml new file mode 100644 index 0000000000..fa88fc0c7e --- /dev/null +++ b/deploy/asset-inventory-cloudformation/elastic-agent-ec2-organization.yml @@ -0,0 +1,256 @@ +AWSTemplateFormatVersion: "2010-09-09" + +Description: Creates IAM roles needed for multi-account access + +Parameters: + OrganizationalUnitIds: + Description: | + Comma-separated list of organizational units to deploy the IAM roles to. + Specify the unique IDs of the organizational units where the roles should be deployed. + Example: ou-abc123,ou-def456,ou-ghi789 + Type: CommaDelimitedList + AllowedPattern: ^(ou-[0-9a-z]{4,32}-[a-z0-9]{8,32}|r-[0-9a-z]{4,32})$ + + ScanManagementAccount: + Description: | + When set to "Yes", the Management Account resources will be scanned, + regardless of selected Organizational Unit IDs. Likewise, when set to + "No", the Management Account resources will not be scanned, even if + the Management Account belongs to a selected Organizational Unit. + Type: String + AllowedValues: + - "Yes" + - "No" + Default: "Yes" + ConstraintDescription: Must specify "Yes" or "No" + + LatestAmiId: + Type: AWS::SSM::Parameter::Value + Default: /aws/service/ami-amazon-linux-latest/al2023-ami-minimal-kernel-default-arm64 + + InstanceType: + Description: The type of EC2 instance to create + Type: String + AllowedValues: + - m6g.xlarge + - m6g.2xlarge + - m6g.4xlarge + Default: m6g.xlarge + + EnrollmentToken: + Description: The enrollment token of elastic-agent + Type: String + + FleetUrl: + Description: The fleet URL of elastic-agent + Type: String + + ElasticArtifactServer: + Description: The URL of the artifact server + Type: String + Default: https://artifacts.elastic.co/downloads/beats/elastic-agent + + ElasticAgentVersion: + Description: The version of elastic-agent to install + Type: String + +Conditions: + ScanManagementAccountEnabled: !Equals + - !Ref ScanManagementAccount + - "Yes" + +Resources: + + # Security Group for EC2 instance + ElasticAgentSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupName: !Join + - '-' + - - elastic-agent-security-group + - !Select + - 2 + - !Split + - / + - !Ref AWS::StackId + GroupDescription: Block incoming traffic + SecurityGroupIngress: [] + + # IAM Role for EC2 instance and for assuming member account roles + CloudbeatRootRole: + Type: AWS::IAM::Role + Properties: + RoleName: cloudbeat-asset-inventory-root + Description: Role that cloudbeat uses to assume roles in other accounts + Tags: + - Key: cloudbeat_scan_management_account + Value: !Ref ScanManagementAccount + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + AWS: !Ref AWS::AccountId + Action: + - sts:AssumeRole + - Effect: Allow + Principal: + Service: + - ec2.amazonaws.com + Action: + - sts:AssumeRole + Path: / + Policies: + - PolicyName: cloudbeat-asset-inventory-root-permissions + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - iam:GetRole + - iam:ListAccountAliases + - iam:ListGroup + - iam:ListRoles + - iam:ListUsers + Resource: '*' + - Effect: Allow + Action: + - organizations:List* + - organizations:Describe* + Resource: '*' + - Effect: Allow + Action: + - sts:AssumeRole + Resource: '*' + + # Instance profile to attach to EC2 instance + ElasticAgentInstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + InstanceProfileName: !Join + - '-' + - - elastic-agent-instance-profile + - !Select + - 2 + - !Split + - / + - !Ref AWS::StackId + Path: / + Roles: + - !Ref CloudbeatRootRole + + # EC2 Instance to run elastic-agent + ElasticAgentEc2Instance: + CreationPolicy: + ResourceSignal: + Timeout: PT10M + Type: AWS::EC2::Instance + Properties: + Tags: + - Key: Name + Value: !Join + - '-' + - - elastic-agent-instance + - !Select + - 2 + - !Split + - / + - !Ref AWS::StackId + - Key: Task + Value: Organization Cloud Security Posture Management Scanner + ImageId: !Ref LatestAmiId + InstanceType: !Ref InstanceType + IamInstanceProfile: !Ref ElasticAgentInstanceProfile + SecurityGroupIds: + - !Ref ElasticAgentSecurityGroup + UserData: !Base64 + Fn::Sub: | + #!/bin/bash -x + signal() { + sudo yum install -y aws-cfn-bootstrap || { sudo apt-get update && sudo apt-get -y install python3-pip && sudo pip3 install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-latest.tar.gz; } + cfn-signal --exit-code "$1" --stack ${AWS::StackName} --resource ElasticAgentEc2Instance --region ${AWS::Region} + } + ElasticAgentArtifact=elastic-agent-${ElasticAgentVersion}-linux-arm64 + curl -L -O ${ElasticArtifactServer}/$ElasticAgentArtifact.tar.gz + tar xzvf $ElasticAgentArtifact.tar.gz + cd $ElasticAgentArtifact + sudo ./elastic-agent install --non-interactive --url=${FleetUrl} --enrollment-token=${EnrollmentToken} --tag=cft_version:cloudformation-asset-inventory-organization-account-CFT_VERSION.yml --tag=cft_arn:${AWS::StackId} + signal $? + BlockDeviceMappings: + - DeviceName: /dev/xvda + Ebs: + VolumeSize: 32 + + CloudbeatRoleStackSet: + Type: AWS::CloudFormation::StackSet + Properties: + StackSetName: cloudbeat-role-asset-inventory-stackset + Description: StackSet for deploying the cloudbeat-asset-inventory-securityaudit IAM role to member accounts in the specified organizational units. + AutoDeployment: + Enabled: true + RetainStacksOnAccountRemoval: false + Capabilities: + - CAPABILITY_NAMED_IAM + ManagedExecution: + Active: true + Parameters: + - ParameterKey: RootRoleArn + ParameterValue: !GetAtt CloudbeatRootRole.Arn + PermissionModel: SERVICE_MANAGED + StackInstancesGroup: + - DeploymentTargets: + OrganizationalUnitIds: !Ref OrganizationalUnitIds + Regions: + - !Ref AWS::Region + TemplateBody: | + AWSTemplateFormatVersion: '2010-09-09' + Description: Creates IAM roles needed for multi-account access + Parameters: + RootRoleArn: + Type: String + Resources: + CloudbeatMemberRole: + Type: 'AWS::IAM::Role' + Properties: + RoleName: cloudbeat-asset-inventory-securityaudit + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + AWS: !Ref RootRoleArn + Action: + - sts:AssumeRole + Path: / + ManagedPolicyArns: + - arn:aws:iam::aws:policy/SecurityAudit + + CloudbeatManagementAccountAuditRole: + Type: AWS::IAM::Role + Properties: + RoleName: cloudbeat-asset-inventory-securityaudit + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + AWS: !GetAtt CloudbeatRootRole.Arn + Action: + - sts:AssumeRole + Path: / + ManagedPolicyArns: + - arn:aws:iam::aws:policy/SecurityAudit + Condition: ScanManagementAccountEnabled + +Outputs: + CloudbeatRootRoleArn: + Description: The cloudbeat IAM role in the management account + Value: !GetAtt CloudbeatRootRole.Arn + + Ec2InstanceId: + Description: The EC2 instance ID + Value: !Ref ElasticAgentEc2Instance + + Ec2InstancePublicIp: + Description: The EC2 instance public IP + Value: !GetAtt ElasticAgentEc2Instance.PublicIp diff --git a/deploy/asset-inventory-cloudformation/elastic-agent-ec2.yml b/deploy/asset-inventory-cloudformation/elastic-agent-ec2.yml new file mode 100644 index 0000000000..c4f6d0dd62 --- /dev/null +++ b/deploy/asset-inventory-cloudformation/elastic-agent-ec2.yml @@ -0,0 +1,151 @@ +AWSTemplateFormatVersion: "2010-09-09" + +Description: Creates an EC2 instance with permissions to run elastic-agent + +Parameters: + LatestAmiId: + Type: AWS::SSM::Parameter::Value + Default: /aws/service/ami-amazon-linux-latest/al2023-ami-minimal-kernel-default-arm64 + + InstanceType: + Description: The type of EC2 instance to create + Type: String + AllowedValues: + - m6g.xlarge + - m6g.2xlarge + - m6g.4xlarge + Default: m6g.xlarge + + EnrollmentToken: + Description: The enrollment token of elastic-agent + Type: String + + FleetUrl: + Description: The fleet URL of elastic-agent + Type: String + + ElasticArtifactServer: + Description: The URL of the artifact server + Type: String + Default: https://artifacts.elastic.co/downloads/beats/elastic-agent + + ElasticAgentVersion: + Description: The version of elastic-agent to install + Type: String + +Resources: + + # Security Group for EC2 instance + ElasticAgentSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupName: !Join + - '-' + - - elastic-agent-security-group + - !Select + - 2 + - !Split + - / + - !Ref AWS::StackId + GroupDescription: Block incoming traffic + SecurityGroupIngress: [] + + # IAM Role for EC2 instance + ElasticAgentRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: + - ec2.amazonaws.com + Action: + - sts:AssumeRole + Path: / + ManagedPolicyArns: + - arn:aws:iam::aws:policy/SecurityAudit + + # IAM Role to assume for Management Account + CloudbeatRootRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + AWS: !GetAtt ElasticAgentRole.Arn + Action: + - sts:AssumeRole + Path: / + ManagedPolicyArns: + - arn:aws:iam::aws:policy/SecurityAudit + + # Instance profile to attach to EC2 instance + ElasticAgentInstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + InstanceProfileName: !Join + - '-' + - - elastic-agent-instance-profile + - !Select + - 2 + - !Split + - / + - !Ref AWS::StackId + Path: / + Roles: + - !Ref ElasticAgentRole + + # EC2 Instance to run elastic-agent + ElasticAgentEc2Instance: + CreationPolicy: + ResourceSignal: + Timeout: PT10M + Type: AWS::EC2::Instance + Properties: + Tags: + - Key: Name + Value: !Join + - '-' + - - elastic-agent-instance + - !Select + - 2 + - !Split + - / + - !Ref AWS::StackId + - Key: Task + Value: Cloud Security Posture Management Scanner + ImageId: !Ref LatestAmiId + InstanceType: !Ref InstanceType + IamInstanceProfile: !Ref ElasticAgentInstanceProfile + SecurityGroupIds: + - !Ref ElasticAgentSecurityGroup + UserData: !Base64 + Fn::Sub: | + #!/bin/bash -x + signal() { + sudo yum install -y aws-cfn-bootstrap || { sudo apt-get update && sudo apt-get -y install python3-pip && sudo pip3 install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-latest.tar.gz; } + cfn-signal --exit-code "$1" --stack ${AWS::StackName} --resource ElasticAgentEc2Instance --region ${AWS::Region} + } + ElasticAgentArtifact=elastic-agent-${ElasticAgentVersion}-linux-arm64 + curl -L -O ${ElasticArtifactServer}/$ElasticAgentArtifact.tar.gz + tar xzvf $ElasticAgentArtifact.tar.gz + cd $ElasticAgentArtifact + sudo ./elastic-agent install --non-interactive --url=${FleetUrl} --enrollment-token=${EnrollmentToken} --tag=cft_version:cloudformation-asset-inventory-single-account-CFT_VERSION.yml --tag=cft_arn:${AWS::StackId} + signal $? + BlockDeviceMappings: + - DeviceName: /dev/xvda + Ebs: + VolumeSize: 32 + +Outputs: + Ec2InstanceId: + Description: The EC2 instance ID + Value: !Ref ElasticAgentEc2Instance + + Ec2InstancePublicIp: + Description: The EC2 instance public IP + Value: !GetAtt ElasticAgentEc2Instance.PublicIp diff --git a/deploy/asset-inventory-cloudformation/gomain.go b/deploy/asset-inventory-cloudformation/gomain.go new file mode 100644 index 0000000000..3ebb25d465 --- /dev/null +++ b/deploy/asset-inventory-cloudformation/gomain.go @@ -0,0 +1,209 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package main + +import ( + "context" + "fmt" + "log" + "os" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + awsConfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/cloudformation" + "github.com/aws/aws-sdk-go-v2/service/cloudformation/types" + "github.com/mikefarah/yq/v4/pkg/yqlib" +) + +const ( + DEV = "DEV_TEMPLATE" + PROD = "PROD_TEMPLATE" +) + +func main() { + cfg, err := parseConfig() + if err != nil { + log.Fatal(err) + } + + err = createFromConfig(cfg) + if err != nil { + log.Fatal(err) + } +} + +func createFromConfig(cfg *config) error { + params := map[string]string{} + + params["FleetUrl"] = cfg.FleetURL + params["EnrollmentToken"] = cfg.EnrollmentToken + params["ElasticAgentVersion"] = cfg.ElasticAgentVersion + + if cfg.ElasticArtifactServer != nil { + params["ElasticArtifactServer"] = *cfg.ElasticArtifactServer + } + + templateSourcePath := "elastic-agent-ec2.yml" + templateTargetPath := getTemplateTargetPath(templateSourcePath) + if err := generateProdTemplate(templateSourcePath, templateTargetPath); err != nil { + return fmt.Errorf("failed to generate prod template: %w", err) + } + + if cfg.Dev != nil && cfg.Dev.AllowSSH { + params["KeyName"] = cfg.Dev.KeyName + + err := generateDevTemplate(templateTargetPath, templateTargetPath) + if err != nil { + return fmt.Errorf("failed to generate dev template: %w", err) + } + } + + err := createStack(cfg.StackName, templateTargetPath, params) + if err != nil { + return fmt.Errorf("failed to create CloudFormation stack: %w", err) + } + + return nil +} + +func generateDevTemplate(prodTemplatePath string, devTemplatePath string) error { + const yqExpression = ` +.Parameters.KeyName = { + "Description": "SSH Keypair to login to the instance", + "Type": "AWS::EC2::KeyPair::KeyName" +} | +.Resources.ElasticAgentEc2Instance.Properties.KeyName = { "Ref": "KeyName" } | +.Resources.ElasticAgentSecurityGroup.Properties.GroupDescription = "Allow SSH from anywhere" | +.Resources.ElasticAgentSecurityGroup.Properties.SecurityGroupIngress += { + "CidrIp": "0.0.0.0/0", + "FromPort": 22, + "IpProtocol": "tcp", + "ToPort": 22 +} +` + return generateTemplate(prodTemplatePath, devTemplatePath, yqExpression) +} + +func generateProdTemplate(prodTemplatePath string, devTemplatePath string) error { + const yqExpression = ` +.Resources.ElasticAgentEc2Instance.Properties.Tags += { + "Key": "division", + "Value": "engineering" +} | +.Resources.ElasticAgentEc2Instance.Properties.Tags += { + "Key": "org", + "Value": "security" +} | +.Resources.ElasticAgentEc2Instance.Properties.Tags += { + "Key": "team", + "Value": "cloud-security" +} | +.Resources.ElasticAgentEc2Instance.Properties.Tags += { + "Key": "project", + "Value": "cloudformation" +} +.Resources.ElasticAgentEc2Instance.Properties.Tags += { + "Key": "deployment-type", + "Value": "asset-inventory" +} +` + return generateTemplate(prodTemplatePath, devTemplatePath, yqExpression) +} + +func generateTemplate(sourcePath string, targetPath string, yqExpression string) (err error) { + inputBytes, err := os.ReadFile(sourcePath) + if err != nil { + return err + } + + preferences := yqlib.NewDefaultYamlPreferences() + preferences.Indent = 2 + preferences.ColorsEnabled = false + + generatedTemplateString, err := yqlib.NewStringEvaluator().Evaluate( + yqExpression, + string(inputBytes), + yqlib.NewYamlEncoder(preferences), + yqlib.NewYamlDecoder(yqlib.NewDefaultYamlPreferences()), + ) + if err != nil { + return err + } + + f, err := os.Create(targetPath) + if err != nil { + return err + } + defer func(f *os.File) { + closeErr := f.Close() + if closeErr != nil && err == nil { + err = fmt.Errorf("failed to close file: %w", closeErr) + } + }(f) + + _, err = f.WriteString(generatedTemplateString) + if err != nil { + return fmt.Errorf("failed to write template: %w", err) + } + + return +} + +func createStack(stackName string, templatePath string, params map[string]string) error { + ctx := context.Background() + + cfg, err := awsConfig.LoadDefaultConfig(ctx) + if err != nil { + return fmt.Errorf("failed to load AWS SDK config: %v", err) + } + + svc := cloudformation.NewFromConfig(cfg) + cfParams := make([]types.Parameter, 0, len(params)) + for key, value := range params { + p := types.Parameter{ + ParameterKey: aws.String(key), + ParameterValue: aws.String(value), + } + cfParams = append(cfParams, p) + } + + bodyBytes, err := os.ReadFile(templatePath) + if err != nil { + return fmt.Errorf("failed to open template file: %v", err) + } + + createStackInput := &cloudformation.CreateStackInput{ + StackName: &stackName, + TemplateBody: aws.String(string(bodyBytes)), + Parameters: cfParams, + Capabilities: []types.Capability{types.CapabilityCapabilityNamedIam}, + } + + stackOutput, err := svc.CreateStack(ctx, createStackInput) + if err != nil { + return fmt.Errorf("failed to call AWS CloudFormation CreateStack: %v", err) + } + + log.Printf("Created stack %s", *stackOutput.StackId) + return nil +} + +func getTemplateTargetPath(source string) string { + return strings.Replace(source, ".yml", "-generated.yml", 1) +} diff --git a/internal/flavors/assetinventory/strategy.go b/internal/flavors/assetinventory/strategy.go index b51e75c68b..d793a8bb5d 100644 --- a/internal/flavors/assetinventory/strategy.go +++ b/internal/flavors/assetinventory/strategy.go @@ -28,10 +28,8 @@ import ( "github.com/elastic/cloudbeat/internal/config" "github.com/elastic/cloudbeat/internal/inventory" - "github.com/elastic/cloudbeat/internal/inventory/awsfetcher" "github.com/elastic/cloudbeat/internal/inventory/azurefetcher" "github.com/elastic/cloudbeat/internal/inventory/gcpfetcher" - "github.com/elastic/cloudbeat/internal/resources/providers/awslib" "github.com/elastic/cloudbeat/internal/resources/providers/azurelib" azure_auth "github.com/elastic/cloudbeat/internal/resources/providers/azurelib/auth" gcp_auth "github.com/elastic/cloudbeat/internal/resources/providers/gcplib/auth" @@ -53,7 +51,12 @@ func (s *strategy) NewAssetInventory(ctx context.Context, client beat.Client) (i switch s.cfg.AssetInventoryProvider { case config.ProviderAWS: - fetchers, err = s.initAwsFetchers(ctx) + switch s.cfg.CloudConfig.Aws.AccountType { + case config.SingleAccount, config.OrganizationAccount: + fetchers, err = s.initAwsFetchers(ctx) + default: + err = fmt.Errorf("unsupported account_type: %q", s.cfg.CloudConfig.Aws.AccountType) + } case config.ProviderAzure: fetchers, err = s.initAzureFetchers(ctx) case config.ProviderGCP: @@ -72,21 +75,6 @@ func (s *strategy) NewAssetInventory(ctx context.Context, client beat.Client) (i return inventory.NewAssetInventory(s.logger, fetchers, client, now), nil } -func (s *strategy) initAwsFetchers(ctx context.Context) ([]inventory.AssetFetcher, error) { - awsConfig, err := awslib.InitializeAWSConfig(s.cfg.CloudConfig.Aws.Cred) - if err != nil { - return nil, err - } - - idProvider := awslib.IdentityProvider{Logger: s.logger} - awsIdentity, err := idProvider.GetIdentity(ctx, *awsConfig) - if err != nil { - return nil, err - } - - return awsfetcher.New(s.logger, awsIdentity, *awsConfig), nil -} - func (s *strategy) initAzureFetchers(_ context.Context) ([]inventory.AssetFetcher, error) { cfgProvider := &azure_auth.ConfigProvider{AuthProvider: &azure_auth.AzureAuthProvider{}} azureConfig, err := cfgProvider.GetAzureClientConfig(s.cfg.CloudConfig.Azure) diff --git a/internal/flavors/assetinventory/strategy_aws.go b/internal/flavors/assetinventory/strategy_aws.go new file mode 100644 index 0000000000..a06add2825 --- /dev/null +++ b/internal/flavors/assetinventory/strategy_aws.go @@ -0,0 +1,110 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package assetinventory + +import ( + "context" + "fmt" + "strings" + + awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/elastic/elastic-agent-libs/logp" + + "github.com/elastic/cloudbeat/internal/config" + "github.com/elastic/cloudbeat/internal/inventory" + "github.com/elastic/cloudbeat/internal/inventory/awsfetcher" + "github.com/elastic/cloudbeat/internal/resources/providers/awslib" + "github.com/elastic/cloudbeat/internal/resources/utils/pointers" +) + +const ( + rootRole = "cloudbeat-asset-inventory-root" + memberRole = "cloudbeat-asset-inventory-securityaudit" +) + +func (s *strategy) initAwsFetchers(ctx context.Context) ([]inventory.AssetFetcher, error) { + awsConfig, err := awslib.InitializeAWSConfig(s.cfg.CloudConfig.Aws.Cred) + if err != nil { + return nil, err + } + + idProvider := awslib.IdentityProvider{Logger: s.logger} + awsIdentity, err := idProvider.GetIdentity(ctx, *awsConfig) + if err != nil { + return nil, err + } + + // Early exit if we're scanning the entire account. + if s.cfg.CloudConfig.Aws.AccountType == config.SingleAccount { + return awsfetcher.New(s.logger, awsIdentity, *awsConfig), nil + } + + // Assume audit roles per selected account and generate fetchers for them + rootRoleConfig := assumeRole( + sts.NewFromConfig(*awsConfig), + *awsConfig, + fmtIAMRole(awsIdentity.Account, rootRole), + ) + accountProvider := &awslib.AccountProvider{} + accountIdentities, err := accountProvider.ListAccounts(ctx, s.logger, rootRoleConfig) + if err != nil { + return nil, err + } + var fetchers []inventory.AssetFetcher + stsClient := sts.NewFromConfig(rootRoleConfig) + for _, identity := range accountIdentities { + assumedRoleConfig := assumeRole( + stsClient, + rootRoleConfig, + fmtIAMRole(identity.Account, memberRole), + ) + if ok := tryListingBuckets(ctx, s.logger, assumedRoleConfig); !ok { + // role does not exist, skip identity/account + s.logger.Infof("Skipping identity on purpose %+v", identity) + continue + } + accountFetchers := awsfetcher.New(s.logger, &identity, assumedRoleConfig) + fetchers = append(fetchers, accountFetchers...) + } + + return fetchers, nil +} + +func tryListingBuckets(ctx context.Context, log *logp.Logger, roleConfig awssdk.Config) bool { + s3Client := s3.NewFromConfig(roleConfig) + _, err := s3Client.ListBuckets(ctx, &s3.ListBucketsInput{MaxBuckets: pointers.Ref(int32(1))}) + if err == nil { + return true + } + if !strings.Contains(err.Error(), "not authorized to perform: sts:AssumeRole") { + log.Errorf("Expected a 403 autorization error, but got: %v", err) + } + return false +} + +func assumeRole(client stscreds.AssumeRoleAPIClient, cfg awssdk.Config, arn string) awssdk.Config { + cfg.Credentials = awssdk.NewCredentialsCache(stscreds.NewAssumeRoleProvider(client, arn)) + return cfg +} + +func fmtIAMRole(account string, role string) string { + return fmt.Sprintf("arn:aws:iam::%s:role/%s", account, role) +} diff --git a/justfile b/justfile index 4fbb5bf1cb..79a1abd163 100644 --- a/justfile +++ b/justfile @@ -125,6 +125,9 @@ deploy-arm: deploy-cloudformation: cd deploy/cloudformation && go run . +deploy-asset-inventory-cloudformation: + cd deploy/asset-inventory-cloudformation && go run . + deploy-dm: .deploy/deployment-manager/deploy.sh diff --git a/scripts/bump_integration.sh b/scripts/bump_integration.sh index acdd8a4df9..c029b69f7b 100755 --- a/scripts/bump_integration.sh +++ b/scripts/bump_integration.sh @@ -57,6 +57,9 @@ update_manifest_version_vars() { echo "Update cloudshell_git_branch in manifest.yml" sed -i'' -E "s/cloudbeat%2F[0-9]+\.[0-9]+/cloudbeat%2F$MAJOR_MINOR_CLOUDBEAT/g" "$manifest_path" + # aws asset inventory + sed -i'' -E "s/cloudformation-asset-inventory-ACCOUNT_TYPE-[0-9]+\.[0-9]+\.[0-9]+/cloudformation-asset-inventory-ACCOUNT_TYPE-$CURRENT_CLOUDBEAT_VERSION/g" "$manifest_path" + git add "$manifest_path" if git diff --cached --quiet; then echo "No changes to commit in $manifest_path" diff --git a/scripts/publish_cft.sh b/scripts/publish_cft.sh index 7921c57f90..ba7cd7bcb7 100755 --- a/scripts/publish_cft.sh +++ b/scripts/publish_cft.sh @@ -20,3 +20,5 @@ upload_file deploy/cloudformation/elastic-agent-ec2-cspm.yml "cloudformation-csp upload_file deploy/cloudformation/elastic-agent-ec2-cspm-organization.yml "cloudformation-cspm-organization-account" "$version" upload_file deploy/cloudformation/elastic-agent-direct-access-key-cspm.yml "cloudformation-cspm-direct-access-key-single-account" "$version" upload_file deploy/cloudformation/elastic-agent-direct-access-key-cspm-organization.yml "cloudformation-cspm-direct-access-key-organization-account" "$version" +upload_file deploy/asset-inventory-cloudformation/elastic-agent-ec2.yml "cloudformation-asset-inventory-single-account" "$version" +upload_file deploy/asset-inventory-cloudformation/elastic-agent-ec2-organization.yml "cloudformation-asset-inventory-organization-account" "$version"