From 2e8f2b70b80ec67df64f3f241b0d413a68a10fff Mon Sep 17 00:00:00 2001 From: AkhtarAmir Date: Tue, 16 Jul 2024 17:44:24 +0500 Subject: [PATCH 1/6] H-plugin synapse workspace diagnostic logs enabled --- exports.js | 1 + helpers/azure/api.js | 5 + .../synapse/workspaceDiagnosticLogsEnabled.js | 63 +++++++++ .../workspaceDiagnosticLogsEnabled.spec.js | 127 ++++++++++++++++++ 4 files changed, 196 insertions(+) create mode 100644 plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js create mode 100644 plugins/azure/synapse/workspaceDiagnosticLogsEnabled.spec.js diff --git a/exports.js b/exports.js index dcbf2a3dbb..45afeaddb9 100644 --- a/exports.js +++ b/exports.js @@ -1217,6 +1217,7 @@ module.exports = { 'workspaceManagedIdentity' : require(__dirname + '/plugins/azure/synapse/workspaceManagedIdentity.js'), 'synapseWorkspaceAdAuthEnabled' : require(__dirname + '/plugins/azure/synapse/synapseWorkspaceAdAuthEnabled.js'), 'synapseWorkspacPrivateEndpoint': require(__dirname + '/plugins/azure/synapse/synapseWorkspacPrivateEndpoint.js'), + 'workspaceDiagnosticLogsEnabled': require(__dirname + '/plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js'), 'apiInstanceManagedIdentity' : require(__dirname + '/plugins/azure/apiManagement/apiInstanceManagedIdentity.js'), 'apiInstanceHasTags' : require(__dirname + '/plugins/azure/apiManagement/apiInstanceHasTags.js'), diff --git a/helpers/azure/api.js b/helpers/azure/api.js index 000b92f186..a194c7bbc2 100644 --- a/helpers/azure/api.js +++ b/helpers/azure/api.js @@ -1277,6 +1277,11 @@ var tertiarycalls = { properties: ['id'], url: 'https://management.azure.com/{id}/providers/microsoft.insights/diagnosticSettings?api-version=2021-05-01-preview' }, + listByWorkspaces: { + reliesOnPath: 'synapse.listWorkspaces', + properties: ['id'], + url: 'https://management.azure.com/{id}/providers/microsoft.insights/diagnosticSettings?api-version=2021-05-01-preview' + } }, backupShortTermRetentionPolicies: { listByDatabase: { diff --git a/plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js b/plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js new file mode 100644 index 0000000000..b6c2db4e46 --- /dev/null +++ b/plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js @@ -0,0 +1,63 @@ +var async = require('async'); +var helpers = require('../../../helpers/azure'); + +module.exports = { + title: 'Synapse Workspace Diagnostic Logging Enabled', + category: 'AI & ML', + domain: 'Machine Learning', + severity: 'Medium', + description: 'Ensures diagnostic logging is enabled for Synapse workspace.', + more_info: 'Enabling diagnostic logging for Azure Synapse workspace enhances performance monitoring, troubleshooting, and security optimization. This feature captures detailed logs of workspace activities, allowing you to gain insights, identify issues, and maintain the integrity and efficiency of data operations.', + recommended_action: 'Enable diagnostic logging for all Synapse workspaces.', + link: 'https://learn.microsoft.com/en-gb/azure/azure-monitor/essentials/diagnostic-settings', + apis: ['synapse:listWorkspaces', 'diagnosticSettings:listByWorkspaces'], + realtime_triggers: ['microsoftsynapse:workspaces:write','microsoftsynapse:workspaces:delete','microsoftinsights:diagnosticSettings:delete'], + + run: function(cache, settings, callback) { + const results = []; + const source = {}; + const locations = helpers.locations(settings.govcloud); + + async.each(locations.synapse, function(location, rcb) { + const workspaces = helpers.addSource(cache, source, + ['synapse', 'listWorkspaces', location]); + + if (!workspaces) return rcb(); + + + if (workspaces.err || !workspaces.data) { + helpers.addResult(results, 3, 'Unable to query Synapse workspaces: ' + helpers.addError(workspaces), location); + return rcb(); + } + + if (!workspaces.data.length) { + helpers.addResult(results, 0, 'No existing Synapse workspaces found', location); + return rcb(); + } + + for (let workspace of workspaces.data) { + var diagnosticSettings = helpers.addSource(cache, source, + ['diagnosticSettings', 'listByWorkspaces', location, workspace.id]); + + if (!diagnosticSettings || diagnosticSettings.err || !diagnosticSettings.data) { + helpers.addResult(results, 3, `Unable to query for Synapse workspace diagnostic settings: ${helpers.addError(diagnosticSettings)}`, + location, workspace.id); + continue; + } + + var found = diagnosticSettings.data.find(ds => ds.logs && ds.logs.length); + + if (found) { + helpers.addResult(results, 0, 'Synapse workspace has diagnostic logs enabled', location, workspace.id); + } else { + helpers.addResult(results, 2, 'Synapse workspace does not have diagnostic logs enabled', location, workspace.id); + } + } + + rcb(); + }, function() { + // Global checking goes here + callback(null, results, source); + }); + } +}; \ No newline at end of file diff --git a/plugins/azure/synapse/workspaceDiagnosticLogsEnabled.spec.js b/plugins/azure/synapse/workspaceDiagnosticLogsEnabled.spec.js new file mode 100644 index 0000000000..73d35d1877 --- /dev/null +++ b/plugins/azure/synapse/workspaceDiagnosticLogsEnabled.spec.js @@ -0,0 +1,127 @@ +var expect = require('chai').expect; +var workspaceDiagnosticLogsEnabled = require('./workspaceDiagnosticLogsEnabled'); + +const workspaces = [ + { + type: "Microsoft.Synapse/workspaces", + id: "/subscriptions/123/resourceGroups/rsgrp/providers/Microsoft.Synapse/workspaces/test", + location: "eastus", + name: "test", + } +]; + + +const diagnosticSettings = [ + { + id: "/subscriptions/123/resourceGroups/rsgrp/providers/Microsoft.Synapse/workspaces/test", + type: 'Microsoft.Insights/diagnosticSettings', + name: 'test', + location: 'eastus', + kind: null, + tags: null, + eventHubName: null, + metrics: [], + logs: [ + { + "category": null, + "categoryGroup": "allLogs", + "enabled": true, + "retentionPolicy": { + "enabled": false, + "days": 0 + } + }, + { + "category": null, + "categoryGroup": "audit", + "enabled": false, + "retentionPolicy": { + "enabled": false, + "days": 0 + } + } + ], + logAnalyticsDestinationType: null + } +]; + +const createCache = (workspaces, ds) => { + const id = workspaces && workspaces.length ? workspaces[0].id : null; + return { + synapse: { + listWorkspaces: { + 'eastus': { + data: workspaces + } + } + }, + diagnosticSettings: { + listByWorkspaces: { + 'eastus': { + [id]: { + data: ds + } + } + } + + }, + }; +}; + +describe('workspaceDiagnosticLogsEnabled', function() { + describe('run', function() { + it('should give a passing result if no Synapse workspaces are found', function (done) { + const cache = createCache([], null); + workspaceDiagnosticLogsEnabled.run(cache, {}, (err, results) => { + expect(results.length).to.equal(1); + expect(results[0].status).to.equal(0); + expect(results[0].message).to.include('No existing Synapse workspaces found'); + expect(results[0].region).to.equal('eastus'); + done(); + }); + }); + + it('should give unknown result if unable to query for Synapse workspaces', function (done) { + const cache = createCache(null, ['error']); + workspaceDiagnosticLogsEnabled.run(cache, {}, (err, results) => { + expect(results.length).to.equal(1); + expect(results[0].status).to.equal(3); + expect(results[0].message).to.include('Unable to query Synapse workspaces: Unable to obtain data'); + expect(results[0].region).to.equal('eastus'); + done(); + }); + }); + it('should give unknown result if unable to query for diagnostic settings', function(done) { + const cache = createCache([workspaces[0]], null); + workspaceDiagnosticLogsEnabled.run(cache, {}, (err, results) => { + expect(results.length).to.equal(1); + expect(results[0].status).to.equal(3); + expect(results[0].message).to.include('Unable to query for Synapse workspace diagnostic settings'); + expect(results[0].region).to.equal('eastus'); + done(); + }); + }); + + it('should give passing result if diagnostic logs enabled', function(done) { + const cache = createCache([workspaces[0]], [diagnosticSettings[0]]); + workspaceDiagnosticLogsEnabled.run(cache, {}, (err, results) => { + expect(results.length).to.equal(1); + expect(results[0].status).to.equal(0); + expect(results[0].message).to.include('Synapse workspace has diagnostic logs enabled'); + expect(results[0].region).to.equal('eastus'); + done(); + }); + }); + + it('should give failing result if diagnostic logs not enabled', function(done) { + const cache = createCache([workspaces[0]], [[]]); + workspaceDiagnosticLogsEnabled.run(cache, {}, (err, results) => { + expect(results.length).to.equal(1); + expect(results[0].status).to.equal(2); + expect(results[0].message).to.include('Synapse workspace does not have diagnostic logs enabled'); + expect(results[0].region).to.equal('eastus'); + done(); + }); + }); + }); +}); From 695bcada9b95f62004a030b6a91ea9bc2d4e5d4c Mon Sep 17 00:00:00 2001 From: AkhtarAmir Date: Tue, 16 Jul 2024 22:15:19 +0500 Subject: [PATCH 2/6] update file --- plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js b/plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js index b6c2db4e46..2bb4fe8840 100644 --- a/plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js +++ b/plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js @@ -11,7 +11,7 @@ module.exports = { recommended_action: 'Enable diagnostic logging for all Synapse workspaces.', link: 'https://learn.microsoft.com/en-gb/azure/azure-monitor/essentials/diagnostic-settings', apis: ['synapse:listWorkspaces', 'diagnosticSettings:listByWorkspaces'], - realtime_triggers: ['microsoftsynapse:workspaces:write','microsoftsynapse:workspaces:delete','microsoftinsights:diagnosticSettings:delete'], + realtime_triggers: ['microsoftsynapse:workspaces:write','microsoftsynapse:workspaces:delete','microsoftinsights:diagnosticSettings:delete','microsoftinsights:diagnosticSettings:write'], run: function(cache, settings, callback) { const results = []; From 624728f6f46d55b343c6e9ea75e0360f12a2596d Mon Sep 17 00:00:00 2001 From: AkhtarAmir Date: Mon, 26 Aug 2024 14:37:30 +0500 Subject: [PATCH 3/6] update files --- plugins/aws/eks/eksKubernetesVersion.spec.js | 2 +- plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/aws/eks/eksKubernetesVersion.spec.js b/plugins/aws/eks/eksKubernetesVersion.spec.js index b53206f8d2..0997f85358 100644 --- a/plugins/aws/eks/eksKubernetesVersion.spec.js +++ b/plugins/aws/eks/eksKubernetesVersion.spec.js @@ -82,7 +82,7 @@ describe('eksKubernetesVersion', function () { "cluster": { "name": "mycluster", "arn": "arn:aws:eks:us-east-1:012345678911:cluster/mycluster", - "version": "1.27", + "version": "1.29", } } ); diff --git a/plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js b/plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js index 2bb4fe8840..34748a2305 100644 --- a/plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js +++ b/plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js @@ -6,8 +6,8 @@ module.exports = { category: 'AI & ML', domain: 'Machine Learning', severity: 'Medium', - description: 'Ensures diagnostic logging is enabled for Synapse workspace.', - more_info: 'Enabling diagnostic logging for Azure Synapse workspace enhances performance monitoring, troubleshooting, and security optimization. This feature captures detailed logs of workspace activities, allowing you to gain insights, identify issues, and maintain the integrity and efficiency of data operations.', + description: 'Ensures that diagnostic logging is enabled for Synapse workspace.', + more_info: 'Enabling diagnostic logs in Azure Synapse workspace is important for monitoring, troubleshooting, and optimizing performance. These logs provide detailed insights into resource usage, query execution, and potential issues, allowing administrators to identify bottlenecks, track errors, and improve the overall efficiency and reliability of the workspace.', recommended_action: 'Enable diagnostic logging for all Synapse workspaces.', link: 'https://learn.microsoft.com/en-gb/azure/azure-monitor/essentials/diagnostic-settings', apis: ['synapse:listWorkspaces', 'diagnosticSettings:listByWorkspaces'], @@ -24,7 +24,6 @@ module.exports = { if (!workspaces) return rcb(); - if (workspaces.err || !workspaces.data) { helpers.addResult(results, 3, 'Unable to query Synapse workspaces: ' + helpers.addError(workspaces), location); return rcb(); @@ -36,6 +35,7 @@ module.exports = { } for (let workspace of workspaces.data) { + var diagnosticSettings = helpers.addSource(cache, source, ['diagnosticSettings', 'listByWorkspaces', location, workspace.id]); From 8ec13352fb01033a2299c0559b8d8bd66489c878 Mon Sep 17 00:00:00 2001 From: AkhtarAmir Date: Wed, 18 Sep 2024 12:59:58 +0500 Subject: [PATCH 4/6] updated the link --- plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js b/plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js index 34748a2305..df0f737c10 100644 --- a/plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js +++ b/plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js @@ -9,7 +9,7 @@ module.exports = { description: 'Ensures that diagnostic logging is enabled for Synapse workspace.', more_info: 'Enabling diagnostic logs in Azure Synapse workspace is important for monitoring, troubleshooting, and optimizing performance. These logs provide detailed insights into resource usage, query execution, and potential issues, allowing administrators to identify bottlenecks, track errors, and improve the overall efficiency and reliability of the workspace.', recommended_action: 'Enable diagnostic logging for all Synapse workspaces.', - link: 'https://learn.microsoft.com/en-gb/azure/azure-monitor/essentials/diagnostic-settings', + link: 'https://learn.microsoft.com/en-us/azure/synapse-analytics/monitor-synapse-analytics', apis: ['synapse:listWorkspaces', 'diagnosticSettings:listByWorkspaces'], realtime_triggers: ['microsoftsynapse:workspaces:write','microsoftsynapse:workspaces:delete','microsoftinsights:diagnosticSettings:delete','microsoftinsights:diagnosticSettings:write'], From bd578d330a7a6861cfdc81a24dba634bf9f50c17 Mon Sep 17 00:00:00 2001 From: AkhtarAmir Date: Wed, 18 Sep 2024 13:01:31 +0500 Subject: [PATCH 5/6] fixed --- plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js b/plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js index df0f737c10..a9beb8a458 100644 --- a/plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js +++ b/plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js @@ -35,6 +35,7 @@ module.exports = { } for (let workspace of workspaces.data) { + if (!workspace.id) continue; var diagnosticSettings = helpers.addSource(cache, source, ['diagnosticSettings', 'listByWorkspaces', location, workspace.id]); From d15fc1a9e0bea70ecdb5e8c1c8576977ea593953 Mon Sep 17 00:00:00 2001 From: alphadev4 <113519745+alphadev4@users.noreply.github.com> Date: Wed, 18 Sep 2024 16:27:31 +0500 Subject: [PATCH 6/6] Update exports.js --- exports.js | 1 - 1 file changed, 1 deletion(-) diff --git a/exports.js b/exports.js index 2af7ee0fe7..725e0d0871 100644 --- a/exports.js +++ b/exports.js @@ -1225,7 +1225,6 @@ module.exports = { 'workspaceDiagnosticLogsEnabled': require(__dirname + '/plugins/azure/synapse/workspaceDiagnosticLogsEnabled.js'), 'workspaceDoubleEncryption' : require(__dirname + '/plugins/azure/synapse/workspaceDoubleEncryption.js'), - 'apiInstanceManagedIdentity' : require(__dirname + '/plugins/azure/apiManagement/apiInstanceManagedIdentity.js'), 'apiInstanceHasTags' : require(__dirname + '/plugins/azure/apiManagement/apiInstanceHasTags.js'),