From 91b713b2178c706d4805051ce6bd66fe7cf55abb Mon Sep 17 00:00:00 2001 From: si458 Date: Wed, 24 Jul 2024 21:20:57 +0100 Subject: [PATCH] add s3 autobackup support Signed-off-by: si458 --- db.js | 93 ++++++++++++++++++++++++++++++++++ meshcentral-config-schema.json | 48 ++++++++++++++++++ meshcentral.js | 2 + sample-config-advanced.json | 10 ++++ 4 files changed, 153 insertions(+) diff --git a/db.js b/db.js index 294a947677..10bcf59285 100644 --- a/db.js +++ b/db.js @@ -3575,6 +3575,99 @@ module.exports.CreateDB = function (parent, func) { }); }); } + + // S3 Backup + if ((typeof parent.config.settings.autobackup == 'object') && (typeof parent.config.settings.autobackup.s3 == 'object')) { + var s3folderName = 'MeshCentral-Backups'; + if (typeof parent.config.settings.autobackup.s3.foldername == 'string') { s3folderName = parent.config.settings.autobackup.s3.foldername; } + // Construct the config object + var accessKey = parent.config.settings.autobackup.s3.accesskey, + secretKey = parent.config.settings.autobackup.s3.secretkey, + endpoint = parent.config.settings.autobackup.s3.endpoint ? parent.config.settings.autobackup.s3.endpoint : 's3.amazonaws.com', + port = parent.config.settings.autobackup.s3.port ? parent.config.settings.autobackup.s3.port : 443, + useSsl = parent.config.settings.autobackup.s3.ssl ? parent.config.settings.autobackup.s3.ssl : true, + bucketName = parent.config.settings.autobackup.s3.bucketname, + pathPrefix = s3folderName, + threshold = parent.config.settings.autobackup.s3.maxfiles ? parent.config.settings.autobackup.s3.maxfiles : 0, + fileToUpload = filename; + // Create a MinIO client + const Minio = require('minio'); + var minioClient = new Minio.Client({ + endPoint: endpoint, + port: port, + useSSL: useSsl, + accessKey: accessKey, + secretKey: secretKey + }); + // List objects in the specified bucket and path prefix + var listObjectsPromise = new Promise(function(resolve, reject) { + var items = []; + var stream = minioClient.listObjects(bucketName, pathPrefix, true); + stream.on('data', function(item) { + if (!item.name.endsWith('/')) { // Exclude directories + items.push(item); + } + }); + stream.on('end', function() { + resolve(items); + }); + stream.on('error', function(err) { + reject(err); + }); + }); + listObjectsPromise.then(function(objects) { + // Count the number of files + var fileCount = objects.length; + // Return if no files to carry on uploading + if (fileCount === 0) { return Promise.resolve(); } + // Sort the files by LastModified date (oldest first) + objects.sort(function(a, b) { return new Date(a.lastModified) - new Date(b.lastModified); }); + // Check if the threshold is zero and return if + if (threshold === 0) { return Promise.resolve(); } + // Check if the number of files exceeds the threshold (maxfiles) is 0 + if (fileCount >= threshold) { + // Calculate how many files need to be deleted to make space for the new file + var filesToDelete = fileCount - threshold + 1; // +1 to make space for the new file + if (func) { func('Deleting ' + filesToDelete + ' older ' + (filesToDelete == 1 ? 'file' : 'files') + ' from S3 ...'); } + // Create an array of promises for deleting files + var deletePromises = objects.slice(0, filesToDelete).map(function(fileToDelete) { + return new Promise(function(resolve, reject) { + minioClient.removeObject(bucketName, fileToDelete.name, function(err) { + if (err) { + reject(err); + } else { + if (func) { func('Deleted file: ' + fileToDelete.name + ' from S3'); } + resolve(); + } + }); + }); + }); + // Wait for all deletions to complete + return Promise.all(deletePromises); + } else { + return Promise.resolve(); // No deletion needed + } + }).then(function() { + // Determine the upload path by combining the pathPrefix with the filename + var fileName = require('path').basename(fileToUpload); + var uploadPath = require('path').join(pathPrefix, fileName); + // Upload a new file + var uploadPromise = new Promise(function(resolve, reject) { + if (func) { func('Uploading file ' + uploadPath + ' to S3'); } + minioClient.fPutObject(bucketName, uploadPath, fileToUpload, function(err, etag) { + if (err) { + reject(err); + } else { + if (func) { func('Uploaded file: ' + uploadPath + ' to S3'); } + resolve(etag); + } + }); + }); + return uploadPromise; + }).catch(function(error) { + if (func) { func('Error managing files in S3: ' + error); } + }); + } } // Transfer NeDB data into the current database diff --git a/meshcentral-config-schema.json b/meshcentral-config-schema.json index a57c9fd68a..7599f3d10d 100644 --- a/meshcentral-config-schema.json +++ b/meshcentral-config-schema.json @@ -897,6 +897,54 @@ "description": "The maximum number of files to keep in the WebDAV folder, older files will be removed if needed." } } + }, + "s3": { + "type": "object", + "description": "Enabled automated upload of the server backups to an S3 server.", + "required": [ + "accessKey", + "secretKey", + "bucketName" + ], + "properties": { + "endpoint": { + "type": "string", + "default": "s3.amazonaws.com", + "description": "S3 Endpoint address e.g. myS3.myserver.com" + }, + "accessKey": { + "type": "string", + "description": "S3 accessKey" + }, + "secretKey": { + "type": "string", + "description": "S3 secretKey" + }, + "port": { + "type": "integer", + "default": 443, + "description": "S3 Endpoint port number, Default is 443" + }, + "ssl": { + "type": "boolean", + "default": true, + "description": "If \"true\", the S3 Endpoint will use \"https\", else it will use \"http\", Default is true" + }, + "bucketName": { + "type": "string", + "description": "S3 Bucket Name" + }, + "folderName": { + "type": "string", + "default": "MeshCentral-Backups", + "description": "The name of the folder to create in the WebDAV account." + }, + "maxFiles": { + "type": "integer", + "default": 0, + "description": "The maximum number of files to keep in the S3 folder, older files will be removed if needed. Default is 0 which is keep everything" + } + } } } }, diff --git a/meshcentral.js b/meshcentral.js index 0ed62bdcfb..a184e3426b 100644 --- a/meshcentral.js +++ b/meshcentral.js @@ -4101,6 +4101,8 @@ function mainStart() { if (typeof config.settings.autobackup.webdav == 'object') { if ((typeof config.settings.autobackup.webdav.url != 'string') || (typeof config.settings.autobackup.webdav.username != 'string') || (typeof config.settings.autobackup.webdav.password != 'string')) { addServerWarning("Missing WebDAV parameters.", 2, null, !args.launch); } else { modules.push('webdav@4.11.3'); } } + // Enable S3 Support + if (typeof config.settings.autobackup.s3 == 'object') { modules.push('minio@8.0.1'); } } // Setup common password blocking diff --git a/sample-config-advanced.json b/sample-config-advanced.json index 19ea1415ca..771de45939 100644 --- a/sample-config-advanced.json +++ b/sample-config-advanced.json @@ -132,6 +132,16 @@ "password": "pass", "folderName": "MeshCentral-Backups", "maxFiles": 10 + }, + "_s3": { + "accessKey": "MYLONGACCESSKEY", + "secretKey": "MYLONGSECRETKEY", + "endpoint": "myS3.myserver.com", + "port": 9000, + "ssl": false, + "bucketName": "test", + "folderName": "MeshCentral-Backups", + "maxfiles": 10 } }, "_redirects": {