From e9bf7643273ae41488549d3b513616c36eaca346 Mon Sep 17 00:00:00 2001 From: Bruno Friedmann Date: Tue, 21 Nov 2023 09:19:22 +0100 Subject: [PATCH 1/4] plugin: fd mariabackup add support for mariadb 11+ - cmake introduce consistent cmake variable MARIADB_BACKUP_BINARY instead of MARIABACKUP_BINARY - cmake replace all call of mysql cmd to their mariadb equivalent Mariadb >11 start echoing deprecated warning. + mysql -> mariadb + mysqld -> mariadbd + mariabackup -> mariadb-backup - split testrunner to setup,cleanup,default - fileset replace hardcoded mysql by @MARIADB_CLIENT_BINARY@ - add function check_plugin_mariadb_version() this is needed to pick the right stream filename depending of the version. (xbstream or mbstream) - refactor start_backup_job() function - add function get_lsn_by_command() - add function parse_innodb_status(res) which can be used by cmd and mysqlclient - re-add self.restorecommand init in parse_plugin_definition so restore directory is new for each file - reword xbstream to mbstream expect for the protocol name - improve logging placement statements - use black for formating - fix my.cnf usage mariadb package hard require python3-mysqlclient module on SUSE. The code using this module is then used during testing; to be able to connect to the right instance we need to provide a default config file. Signed-off-by: Bruno Friedmann --- core/cmake/BareosFindPrograms.cmake | 16 +- .../mariabackup/bareos-fd-mariabackup.py | 218 +++++++++++------- systemtests/environment.in | 2 +- .../py3plug-fd-mariabackup/CMakeLists.txt | 14 +- .../fileset/MariabackupTest.conf.in | 5 +- .../tests/py3plug-fd-mariabackup/testrunner | 124 ---------- .../py3plug-fd-mariabackup/testrunner-default | 159 +++++++++++++ 7 files changed, 317 insertions(+), 221 deletions(-) delete mode 100755 systemtests/tests/py3plug-fd-mariabackup/testrunner create mode 100755 systemtests/tests/py3plug-fd-mariabackup/testrunner-default diff --git a/core/cmake/BareosFindPrograms.cmake b/core/cmake/BareosFindPrograms.cmake index af6a822ef57..da70bb24eb5 100644 --- a/core/cmake/BareosFindPrograms.cmake +++ b/core/cmake/BareosFindPrograms.cmake @@ -1,6 +1,6 @@ # BAREOS® - Backup Archiving REcovery Open Sourced # -# Copyright (C) 2017-2021 Bareos GmbH & Co. KG +# Copyright (C) 2017-2023 Bareos GmbH & Co. KG # # This program is Free Software; you can redistribute it and/or # modify it under the terms of version three of the GNU Affero General Public @@ -117,26 +117,26 @@ message(STATUS "XTRABACKUP_BINARY: ${XTRABACKUP_BINARY}") message(STATUS "MYSQL_DAEMON_BINARY:${MYSQL_DAEMON_BINARY}") message(STATUS "MYSQL_CLIENT_BINARY:${MYSQL_CLIENT_BINARY}") -# For mariadb: MARIABACKUP_BINARY MARIADB_DAEMON_BINARY MARIADB_CLIENT_BINARY +# For mariadb: MARIADB_BACKUP_BINARY MARIADB_DAEMON_BINARY MARIADB_CLIENT_BINARY # MARIADB_MYSQL_INSTALL_DB_SCRIPT -if(NOT DEFINED MARIABACKUP_BINARY) - find_program(MARIABACKUP_BINARY mariabackup) +if(NOT DEFINED MARIADB_BACKUP_BINARY) + find_program(MARIADB_BACKUP_BINARY mariadb-backup) endif() find_program_and_verify_version_string( - MARIADB_DAEMON_BINARY mysqld MariaDB "/usr/libexec" "" + MARIADB_DAEMON_BINARY mariadbd MariaDB "/usr/libexec" "" ) find_program_and_verify_version_string( - MARIADB_CLIENT_BINARY mysql MariaDB "" "" + MARIADB_CLIENT_BINARY mariadb MariaDB "" "" ) if(NOT DEFINED MARIADB_MYSQL_INSTALL_DB_SCRIPT) find_program(MARIADB_MYSQL_INSTALL_DB_SCRIPT mysql_install_db) endif() -message(STATUS "MARIABACKUP_BINARY: ${MARIABACKUP_BINARY}") -message(STATUS "MARIADB_DAEMON_BINARY:${MARIADB_DAEMON_BINARY}") +message(STATUS "MARIADB_BACKUP_BINARY: ${MARIADB_BACKUP_BINARY}") +message(STATUS "MARIADB_DAEMON_BINARY: ${MARIADB_DAEMON_BINARY}") message(STATUS "MARIADB_CLIENT_BINARY: ${MARIADB_CLIENT_BINARY}") message( STATUS "MARIADB_MYSQL_INSTALL_DB_SCRIPT: ${MARIADB_MYSQL_INSTALL_DB_SCRIPT}" diff --git a/core/src/plugins/filed/python/mariabackup/bareos-fd-mariabackup.py b/core/src/plugins/filed/python/mariabackup/bareos-fd-mariabackup.py index bea71003709..0470f0d82d2 100644 --- a/core/src/plugins/filed/python/mariabackup/bareos-fd-mariabackup.py +++ b/core/src/plugins/filed/python/mariabackup/bareos-fd-mariabackup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright (C) 2015-2023 Bareos GmbH & Co. KG # @@ -19,29 +19,28 @@ # # Author: Philipp Storz # -# Uses mariabackup for backup and restore of mariadb databases - -# This module contains the wrapper functions called by the Bareos-FD, the functions call the corresponding -# methods from your plugin class -from BareosFdWrapper import * - -from bareosfd import * +""" +bareos-fd-mariabckup is a plugin to backup and restore MariaDB with mariadb-backup and mbstream +""" import os -from subprocess import * -from BareosFdPluginBaseclass import * -import BareosFdWrapper -import datetime +import json import time +import datetime import tempfile import shutil -import json +from subprocess import * +import bareosfd +from bareosfd import * + +from BareosFdPluginBaseclass import BareosFdPluginBaseclass +from BareosFdWrapper import * # noqa @BareosPlugin class BareosFdMariabackup(BareosFdPluginBaseclass): """ Plugin for backing up all innodb databases found in a specific mariadb server - using the native mariabackup tool. + using the native mariadb-backup tool. """ def __init__(self, plugindef): @@ -53,27 +52,36 @@ def __init__(self, plugindef): self.tempdir = tempfile.mkdtemp() # self.logdir = GetValue(bVarWorkingDir) self.logdir = "/var/log/bareos/" - self.log = "bareos-plugin-mariabackup.log" + self.log = "bareos-plugin-mariadb-backup.log" self.rop_data = {} self.max_to_lsn = 0 self.err_fd = None + self.dumpbinary = "mariadb-backup" + self.restorecommand = None + self.mysqlcmd = None + self.mycnf = "" + self.strictIncremental = False + self.dumpoptions = "" + # xtrabackup_checkpoints for <11 and mariadb_backup_checkpoints for >=11 + # version needs to be retrieved with + # mariadb --batch --skip-column-names --execute "select version();" + self.checkpoints_filename = None def parse_plugin_definition(self, plugindef): """ - We have default options that should work out of the box in the most use cases - that the mysql/mariadb is on the same host and can be accessed without user/password information, + We have default options that should work out of the box in the most use cases + when the mysql/mariadb is on the same host and can be accessed without user/password + information, e.g. with a valid my.cnf for user root. """ BareosFdPluginBaseclass.parse_plugin_definition(self, plugindef) if "dumpbinary" in self.options: self.dumpbinary = self.options["dumpbinary"] - else: - self.dumpbinary = "mariabackup" - if "restorecommand" not in self.options: - self.restorecommand = "mbstream -x -C " - else: + # The reset to default is needed for each file. + self.restorecommand = "mbstream -x -C " + if "restorecommand" in self.options: self.restorecommand = self.options["restorecommand"] # Default is not to write an extra logfile @@ -86,13 +94,15 @@ def parse_plugin_definition(self, plugindef): else: self.log = os.path.join(self.logdir, self.options["log"]) - # By default, standard mysql-config files will be used, set - # this option to use extra files + # By default, standard mysql-config files will be used, + # set this option to use extra files self.connect_options = {"read_default_group": "client"} if "mycnf" in self.options: self.connect_options["read_default_file"] = self.options["mycnf"] - self.mycnf = "--defaults-extra-file=%s " % self.options["mycnf"] - else: + # self.mycnf = "--defaults-extra-file=%s " % self.options["mycnf"] + # defaults-extra-file can't be added after default-file which is + # set in the fileset, mycnf will be mainly used by mysqlclient + # python module. Can be improved. self.mycnf = "" # If true, incremental jobs will only be performed, if LSN has increased @@ -102,8 +112,6 @@ def parse_plugin_definition(self, plugindef): and self.options["strictIncremental"] == "true" ): self.strictIncremental = True - else: - self.strictIncremental = False self.dumpoptions = self.mycnf @@ -118,11 +126,11 @@ def parse_plugin_definition(self, plugindef): if "extradumpoptions" in self.options: self.dumpoptions += " " + self.options["extradumpoptions"] - # We need to call mysql to get the current Log Sequece Number (LSN) + # We need to call mariadb to get the current Log Sequece Number (LSN) if "mysqlcmd" in self.options: self.mysqlcmd = self.options["mysqlcmd"] else: - self.mysqlcmd = "mysql %s -r" % self.mycnf + self.mysqlcmd = "mariadb %s -r" % self.mycnf return bRC_OK @@ -137,10 +145,81 @@ def check_plugin_options(self, mandatory_options=None): else: return bRC_OK + def check_plugin_mariadb_version(self): + get_version_command = self.mysqlcmd.split() + get_version_command.extend( + ["--batch", "--skip-column-names", "--execute", "select version();"] + ) + DebugMessage(100, f'mariadb version check: "{get_version_command}"\n') + try: + ret = check_output(get_version_command).decode().split(".")[0] + mariadb_major_version = int(ret) + if mariadb_major_version is not None and mariadb_major_version != 0: + if mariadb_major_version < 11: + self.checkpoints_filename = "xtrabackup_checkpoints" + elif mariadb_major_version >= 11: + self.checkpoints_filename = "mariadb_backup_checkpoints" + DebugMessage( + 100, + ( + f"Detected mariadb version {mariadb_major_version} " + f": {self.checkpoints_filename}\n" + ), + ) + return bRC_OK + except CalledProcessError as run_err: + DebugMessage( + 100, + f"No mariadb version detected: {run_err}\n", + ) + return bRC_Error + except ValueError as val_err: + DebugMessage( + 100, + f"mariadb version detected seems wrong: {val_err}\n", + ) + return bRC_Error + + def get_lsn_by_command(self): + get_lsn_cmd = self.mysqlcmd.split() + get_lsn_cmd.extend( + [ + "--batch", + "--skip-column-names", + "--execute", + "SHOW ENGINE INNODB STATUS;", + ] + ) + DebugMessage(100, f'get_lsn_by_command: "{get_lsn_cmd}"\n') + try: + ret = check_output(get_lsn_cmd).decode() + return ret + except CalledProcessError as run_err: + DebugMessage( + 100, + f"Error while looking for LSN: {run_err}\n", + ) + return bRC_Error + + def parse_innodb_status(self, res): + last_lsn = None + try: + for line in res.split("\n"): + if line.startswith("Log sequence number"): + last_lsn = int(line.split(" ")[-1]) + except ValueError as val_err: + DebugMessage( + 100, + f"Returned LSN seems wrong: {val_err}\n", + ) + return bRC_Error + return last_lsn + def create_file(self, restorepkt): """ On restore we create a subdirectory for the first base backup and each incremental backup. - Because mariabackup expects an empty directory, we create a tree starting with jobId/ of restore job + Because mariadb-backup expects an empty directory, we create a tree starting with jobId/ + of restore job """ FNAME = restorepkt.ofname DebugMessage(100, "create file with %s called\n" % FNAME) @@ -150,7 +229,7 @@ def create_file(self, restorepkt): if origJobId in self.rop_data: rop_from_lsn = int(self.rop_data[origJobId]["from_lsn"]) rop_to_lsn = int(self.rop_data[origJobId]["to_lsn"]) - self.writeDir += "/%020d_" % rop_from_lsn + self.writeDir += "%020d_" % rop_from_lsn self.writeDir += "%020d_" % rop_to_lsn self.writeDir += "%010d" % origJobId else: @@ -167,17 +246,17 @@ def create_file(self, restorepkt): "Directory %s does not exist, creating it now\n" % self.writeDir, ) os.makedirs(self.writeDir) - # Mariabackup requires empty directory + # mbstream requires empty directory if os.listdir(self.writeDir): JobMessage( M_FATAL, - "Restore with xbstream needs empty directory: %s\n" % self.writeDir, + "Restore with mbstream needs empty directory: %s\n" % self.writeDir, ) return bRC_Error self.restorecommand += self.writeDir DebugMessage( 100, - 'Restore using xbstream to extract files with "%s"\n' % self.restorecommand, + 'Restore using mbstream to extract files with "%s"\n' % self.restorecommand, ) restorepkt.create_status = CF_EXTRACT return bRC_OK @@ -187,6 +266,13 @@ def start_backup_job(self): We will check, if database has changed since last backup in the incremental case """ + check_version_bRC = self.check_plugin_mariadb_version() + if check_version_bRC != bRC_OK: + JobMessage( + M_FATAL, + "Unable to determine mariadb version\n", + ) + return check_version_bRC check_option_bRC = self.check_plugin_options() if check_option_bRC != bRC_OK: return check_option_bRC @@ -211,6 +297,8 @@ def start_backup_job(self): 100, "Import of module MySQLdb failed. Using command pipe instead\n", ) + # TODO Fix as this doesn't use the mysqldefault passed file and always try to connect + # to default instance. # contributed by https://github.com/kjetilho if hasMySQLdbModule: try: @@ -224,11 +312,8 @@ def start_backup_job(self): "Could not fetch SHOW ENGINE INNODB STATUS, unprivileged user?", ) return bRC_Error - info = result[0][2] + innodb_status = result[0][2] conn.close() - for line in info.split("\n"): - if line.startswith("Log sequence number"): - last_lsn = int(line.split(" ")[-1]) except Exception as e: JobMessage( M_FATAL, @@ -237,32 +322,9 @@ def start_backup_job(self): return bRC_Error # use old method as fallback, if module MySQLdb not available else: - get_lsn_command = ( - "echo 'SHOW ENGINE INNODB STATUS' | %s | grep 'Log sequence number' | awk '{ print $4 }'" - % self.mysqlcmd - ) - last_lsn_proc = Popen( - get_lsn_command, shell=True, stdout=PIPE, stderr=PIPE - ) - last_lsn_proc.wait() - returnCode = last_lsn_proc.poll() - (mysqlStdOut, mysqlStdErr) = last_lsn_proc.communicate() - if returnCode != 0 or mysqlStdErr: - JobMessage( - M_FATAL, - 'Could not get LSN with command "%s", Error: %s' - % (get_lsn_command, mysqlStdErr), - ) - return bRC_Error - else: - try: - last_lsn = int(mysqlStdOut) - except: - JobMessage( - M_FATAL, - 'Error reading LSN: "%s" not an integer' % mysqlStdOut, - ) - return bRC_Error + innodb_status = self.get_lsn_by_command() + + last_lsn = self.parse_innodb_status(innodb_status) JobMessage(M_INFO, "Backup until LSN: %d\n" % last_lsn) if ( self.max_to_lsn > 0 @@ -275,7 +337,6 @@ def start_backup_job(self): % (last_lsn, self.max_to_lsn), ) self.files_to_backup = ["lsn_only"] - return bRC_OK return bRC_OK def start_backup_file(self, savepkt): @@ -294,7 +355,7 @@ def start_backup_file(self, savepkt): if self.file_to_backup == "stream": # This is the database backup as xbstream - savepkt.fname = "/_mariabackup/xbstream.%010d" % self.jobId + savepkt.fname = "/_mariadb-backup/xbstream.%010d" % self.jobId savepkt.type = FT_REG if self.max_to_lsn > 0: self.dumpoptions += " --incremental-lsn=%d" % self.max_to_lsn @@ -305,11 +366,11 @@ def start_backup_file(self, savepkt): # Read checkpoints and create restore object checkpoints = {} # improve: Error handling - with open("%s/xtrabackup_checkpoints" % self.tempdir) as lsnfile: + with open(f"{self.tempdir}/{self.checkpoints_filename}") as lsnfile: for line in lsnfile: key, value = line.partition("=")[::2] checkpoints[key.strip()] = value.strip() - savepkt.fname = "/_mariabackup/xtrabackup_checkpoints" + savepkt.fname = f"/_mariadb-backup/{self.checkpoints_filename}" savepkt.type = FT_RESTORE_FIRST savepkt.object_name = savepkt.fname savepkt.object = bytearray(json.dumps(checkpoints), encoding="utf8") @@ -319,7 +380,7 @@ def start_backup_file(self, savepkt): elif self.file_to_backup == "lsn_only": # We have nothing to backup incremental, so we just have to pass # the restore object from previous job - savepkt.fname = "/_mariabackup/xtrabackup_checkpoints" + savepkt.fname = f"/_mariadb-backup/{self.checkpoints_filename}" savepkt.type = FT_RESTORE_FIRST savepkt.object_name = savepkt.fname savepkt.object = bytearray(self.row_rop_raw) @@ -341,7 +402,7 @@ def start_backup_file(self, savepkt): def plugin_io(self, IOP): """ Called for io operations. We read from pipe into buffers or on restore - send to xbstream + send to mbstream """ DebugMessage(200, "plugin_io called with " + str(IOP.func) + "\n") @@ -405,7 +466,7 @@ def plugin_io(self, IOP): M_ERROR, "Dump / restore command not finished properly\n", ) - bRC_Error + return bRC_Error return bRC_OK else: DebugMessage( @@ -429,8 +490,9 @@ def end_backup_file(self): """ Check if dump was successful. """ - # Usually the mariabackup process should have terminated here, but on some servers - # it has not always. + # Usually the mariadb-backup process should have terminated here, + # but on some servers it has not always. + DebugMessage(100, "end_backup_file() entry point in Python called\n") if self.file_to_backup == "stream": returnCode = self.subprocess_returnCode if returnCode is None: @@ -442,8 +504,7 @@ def end_backup_file(self): else: DebugMessage( 100, - "end_backup_file() entry point in Python called. Returncode: %d\n" - % self.stream.returncode, + "end_backup_file() Returncode: %d\n" % self.stream.returncode, ) if returnCode != 0: msg = [ @@ -453,8 +514,7 @@ def end_backup_file(self): if self.log: msg += ['log file: "%s"' % self.log] JobMessage(M_FATAL, ", ".join(msg) + "\n") - if returnCode != 0: - return bRC_Error + return bRC_Error if self.log: self.err_fd.write( @@ -486,7 +546,7 @@ def end_restore_file(self): % self.stream.returncode, ) if returnCode != 0: - msg = ["Restore command returned non-zero value: %d" % return_code] + msg = ["Restore command returned non-zero value: %d" % returnCode] if self.log: msg += ['log file: "%s"' % self.log] JobMessage(M_ERROR, ", ".join(msg) + "\n") diff --git a/systemtests/environment.in b/systemtests/environment.in index 36b8fd634ad..211fb6b5d0c 100644 --- a/systemtests/environment.in +++ b/systemtests/environment.in @@ -150,7 +150,7 @@ XTRABACKUP_BINARY=@XTRABACKUP_BINARY@ MYSQL_DAEMON_BINARY=@MYSQL_DAEMON_BINARY@ MYSQL_CLIENT_BINARY=@MYSQL_CLIENT_BINARY@ -MARIABACKUP_BINARY=@MARIABACKUP_BINARY@ +MARIADB_BACKUP_BINARY=@MARIADB_BACKUP_BINARY@ MARIADB_DAEMON_BINARY=@MARIADB_DAEMON_BINARY@ MARIADB_CLIENT_BINARY=@MARIADB_CLIENT_BINARY@ MARIADB_MYSQL_INSTALL_DB_SCRIPT=@MARIADB_MYSQL_INSTALL_DB_SCRIPT@ diff --git a/systemtests/tests/py3plug-fd-mariabackup/CMakeLists.txt b/systemtests/tests/py3plug-fd-mariabackup/CMakeLists.txt index 4f6c9c48a5e..6b3f13a47c6 100644 --- a/systemtests/tests/py3plug-fd-mariabackup/CMakeLists.txt +++ b/systemtests/tests/py3plug-fd-mariabackup/CMakeLists.txt @@ -19,13 +19,13 @@ # This test requires the following variables to be set: # -# -D MARIABACKUP_BINARY=/usr/bin/mariabackup \ -D -# MARIADB_DAEMON_BINARY=/usr/libexec/mysqld \ -D -# MARIADB_CLIENT_BINARY=/usr/bin/mysql \ -D +# -D MARIADB_BACKUP_BINARY=/usr/bin/mariadb-backup \ -D +# MARIADB_DAEMON_BINARY=/usr/bin/mariadbd \ -D +# MARIADB_CLIENT_BINARY=/usr/bin/mariadb \ -D # MARIADB_MYSQL_INSTALL_DB_SCRIPT=/usr/bin/mysql_install_db if(TARGET python3-fd - AND MARIABACKUP_BINARY + AND MARIADB_BACKUP_BINARY AND MARIADB_DAEMON_BINARY AND MARIADB_CLIENT_BINARY AND MARIADB_MYSQL_INSTALL_DB_SCRIPT @@ -38,11 +38,11 @@ else() DISABLED COMMENT " - no python3-fd or MARIABACKUP_BINARY,MARIADB_DAEMON_BINARY,MARIADB_CLIENT_BINARY or MARIADB_MYSQL_INSTALL_DB_SCRIPT not set. + no python3-fd or MARIADB_BACKUP_BINARY,MARIADB_DAEMON_BINARY,MARIADB_CLIENT_BINARY or MARIADB_MYSQL_INSTALL_DB_SCRIPT not set. example: - cmake -D MARIABACKUP_BINARY=/usr/bin/mariabackup \ + cmake -D MARIADB_BACKUP_BINARY=/usr/bin/mariadb-backup \ -D MARIADB_DAEMON_BINARY=/usr/libexec/mysqld \ - -D MARIADB_CLIENT_BINARY=/usr/bin/mysql \ + -D MARIADB_CLIENT_BINARY=/usr/bin/mariadb \ -D MARIADB_MYSQL_INSTALL_DB_SCRIPT=/usr/bin/mysql_install_db" ) endif() diff --git a/systemtests/tests/py3plug-fd-mariabackup/etc/bareos/bareos-dir.d/fileset/MariabackupTest.conf.in b/systemtests/tests/py3plug-fd-mariabackup/etc/bareos/bareos-dir.d/fileset/MariabackupTest.conf.in index 2e2e6d6c121..c6142817326 100644 --- a/systemtests/tests/py3plug-fd-mariabackup/etc/bareos/bareos-dir.d/fileset/MariabackupTest.conf.in +++ b/systemtests/tests/py3plug-fd-mariabackup/etc/bareos/bareos-dir.d/fileset/MariabackupTest.conf.in @@ -8,8 +8,9 @@ FileSet { Plugin = "@python_module_name@:" "module_path=@python_plugin_module_src_test_dir@:" "module_name=bareos-fd-mariabackup:" - "dumpbinary=@MARIABACKUP_BINARY@ --defaults-file=mysqldefaults:" + "mycnf=my.cnf:" + "dumpbinary=@MARIADB_BACKUP_BINARY@ --defaults-file=mysqldefaults:" "extradumpoptions=--user=@USER@@extradumpoptions@:" - "mysqlcmd=mysql --defaults-file=mysqldefaults --user=@USER@ --raw" + "mysqlcmd=@MARIADB_CLIENT_BINARY@ --defaults-file=mysqldefaults --user=@USER@ --raw" } } diff --git a/systemtests/tests/py3plug-fd-mariabackup/testrunner b/systemtests/tests/py3plug-fd-mariabackup/testrunner deleted file mode 100755 index 4e6fc0d3cb7..00000000000 --- a/systemtests/tests/py3plug-fd-mariabackup/testrunner +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/bash -set -e -set -o pipefail -set -u -# -# This systemtest tests the mariadb plugin functionality -# of the Bareos FD by using the supplied module -# BareosFdPluginMariabackup.py -# -TestName="$(basename "$(pwd)")" -export TestName - -#shellcheck source=../environment.in -. ./environment -#shellcheck source=../scripts/functions -. "${rscripts}"/functions - -skip_if_root - -JobName=backup-bareos-fd -MARIABACKUP="${MARIABACKUP_BINARY} --defaults-file=mysqldefaults" -mariabackup_test_db="${db_name}_mariabackup" - -# override MYSQL_*_BINARY variables to be able to use mysql.sh -MYSQL_DAEMON_BINARY="${MARIADB_DAEMON_BINARY}" -MYSQL_CLIENT_BINARY="${MARIADB_CLIENT_BINARY}" - -#shellcheck source=../scripts/mysql.sh -. "${rscripts}"/mysql.sh -"${rscripts}"/cleanup -"${rscripts}"/setup - - -start_test - -mysql_init -mysql_server_start - -echo "create database ${mariabackup_test_db}" | $MYSQL_CLIENT -echo "CREATE TABLE test ( id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, data VARCHAR(100), created TIMESTAMP DEFAULT NOW()) " | $MYSQL_CLIENT "${mariabackup_test_db}" -echo "insert into test (data) VALUES ('test entry 1') " | $MYSQL_CLIENT "${mariabackup_test_db}" - -cat <"$tmp/bconcmds" -@$out $tmp/log1.out -run job=$JobName yes -wait JobName=$JobName -status dir - -run job=$JobName level=Incremental yes -wait JobName=$JobName -status dir - - -END_OF_DATA - -# insert data and run incremental -echo "insert into test (data) VALUES ('test entry 2') " | $MYSQL_CLIENT "${mariabackup_test_db}" - -run_bareos "$@" - -cat <"$tmp/bconcmds" -@# run incremental again without any new data -run job=$JobName yes -wait JobName=$JobName -status dir -wait -messages -quit -END_OF_DATA2 -run_bareos "$@" - -cat <"$tmp/bconcmds" -restore client=bareos-fd fileset=MariaBackupTest yes restorejob=RestoreFile select all done -@$out $tmp/log2.out -wait -END_OF_DATA3 - -run_bareos "$@" - -mysql_server_stop - - -# Check if mariabackup has extracted some files at least -# TODO: verify that mariabackup --prepare works and eventually do complete datbase restore -ls -lR "$tmp/bareos-restores/_mariabackup/" -if [ -z "$(ls -A "$tmp"/bareos-restores/_mariabackup/)" ]; then - echo "No restore data found" - estat=1 -fi - - -# create new empty data dir -rm -Rf mysql/data/* -mkdir -p mysql/data/ - -TARGETDIR=$(find ./tmp/bareos-restores/_mariabackup/4/ -type d -name '*001') -INCDIR1=$(find ./tmp/bareos-restores/_mariabackup/4/ -type d -name '*002') -INCDIR2=$(find ./tmp/bareos-restores/_mariabackup/4/ -type d -name '*002') - -${MARIABACKUP} --prepare --target-dir=${TARGETDIR} -# after prepare a file is left in data dir, this seems to be a bug -rm -Rf mysql/data/* -${MARIABACKUP} --copy-back --target-dir=${TARGETDIR} --incremental-dir=${INCDIR1} --incremental-dir=${INCDIR2} - - -mysql_server_start - -if ! echo "SELECT * from test " | $MYSQL_CLIENT "${mariabackup_test_db}" | grep "test entry 1"; then - echo "test entry 1 not found" - estat=2 -fi - -if ! echo "SELECT * from test " | $MYSQL_CLIENT "${mariabackup_test_db}" | grep "test entry 2"; then - echo "test entry 2 not found" - estat=3 -fi - -check_for_zombie_jobs storage=File - -mysql_server_stop - -check_two_logs - -end_test diff --git a/systemtests/tests/py3plug-fd-mariabackup/testrunner-default b/systemtests/tests/py3plug-fd-mariabackup/testrunner-default new file mode 100755 index 00000000000..13652878392 --- /dev/null +++ b/systemtests/tests/py3plug-fd-mariabackup/testrunner-default @@ -0,0 +1,159 @@ +#!/bin/bash +set -e +set -o pipefail +set -u +# +# This systemtest tests the mariadb plugin functionality +# of the Bareos FD by using the supplied module +# BareosFdPluginMariabackup.py +# +TestName="$(basename "$(pwd)")" +export TestName + +#shellcheck source=../environment.in +. ./environment +#shellcheck source=../scripts/functions +. "${rscripts}"/functions + +skip_if_root + +JobName=backup-bareos-fd +MARIABACKUP="${MARIADB_BACKUP_BINARY} --defaults-file=mysqldefaults" +mariabackup_test_db="${db_name}_mariabackup" + +# override MYSQL_*_BINARY variables to be able to use mysql.sh +MYSQL_DAEMON_BINARY="${MARIADB_DAEMON_BINARY}" +MYSQL_CLIENT_BINARY="${MARIADB_CLIENT_BINARY}" + +#shellcheck source=../scripts/mysql.sh +. "${rscripts}"/mysql.sh + +start_test + +# TODO test job failed when Accurate is On + +mysql_init +mysql_server_start + +echo "create database ${mariabackup_test_db}" | ${MYSQL_CLIENT} +echo "CREATE TABLE test ( id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, data VARCHAR(100), created TIMESTAMP DEFAULT NOW()) " | ${MYSQL_CLIENT} "${mariabackup_test_db}" +echo "insert into test (data) VALUES ('test entry 1') " | ${MYSQL_CLIENT} "${mariabackup_test_db}" + +cat <"${tmp}/bconcmds" +@$out ${tmp}/log1.out +setdebug level=150 trace=1 timestamp=1 client=bareos-fd +run job=${JobName} yes +wait JobName=${JobName} +status dir +wait +messages +quit +END_OF_DATA1 +echo "Run first Full job" +run_bconsole +expect_grep "Backup OK" "${tmp}/log1.out" "Backup OK for Full not found in job log" +if [ ${estat} -ne 0 ]; then + exit ${estat} +fi + +# insert data and run incremental +${MYSQL_CLIENT} "${mariabackup_test_db}" <<< "insert into test (data) VALUES ('test entry 2')" + +cat <"${tmp}/bconcmds" +@$out ${tmp}/log2.out +@# run incremental with new data +run job=$JobName level=Incremental yes +wait JobName=$JobName +status dir +wait +messages +quit +END_OF_DATA2 +echo "Run first incremental job" +run_bconsole + +expect_grep "Backup OK" "${tmp}/log2.out" "Backup OK for incremental not found in job log" +if [ ${estat} -ne 0 ]; then + exit ${estat} +fi + +cat <"$tmp/bconcmds" +@$out ${tmp}/log3.out +@# run incremental again without any new data +run job=${JobName} yes +wait JobName=${JobName} +status dir +wait +messages +quit +END_OF_DATA3 +echo "Run second empty incremental job" +run_bconsole + +expect_grep "Backup OK" "${tmp}/log3.out" "Backup OK for empty incremental not found in job log" +if [ ${estat} -ne 0 ]; then + exit ${estat} +fi + + +cat <"$tmp/bconcmds" +@$out ${tmp}/log4.out +restore client=bareos-fd fileset=MariaBackupTest yes restorejob=RestoreFile select all done +wait +status dir +wait +messages +quit +END_OF_DATA4 +echo "Run restore job" +run_bconsole + +expect_grep "Restore OK" "${tmp}/log4.out" "Restore OK not found in job log" +if [ ${estat} -ne 0 ]; then + exit ${estat} +fi + +# Check if mariabackup has extracted some files at least +# TODO: verify that mariabackup --prepare works and eventually do complete database restore +ls -lR "${tmp}/bareos-restores/_mariadb-backup/" +if [ -z "$(ls -A "${tmp}"/bareos-restores/_mariadb-backup/)" ]; then + echo "No restore data found" + estat=1 +fi + +mysql_server_stop + +# create new empty data dir +rm -Rf mysql/data/* +mkdir -p mysql/data/ + +TARGETDIR=$(find ./tmp/bareos-restores/_mariadb-backup/4/ -type d -name '*001') +INCDIR1=$(find ./tmp/bareos-restores/_mariadb-backup/4/ -type d -name '*002') +INCDIR2=$(find ./tmp/bareos-restores/_mariadb-backup/4/ -type d -name '*003') + +# First, prepare the base backup: +${MARIABACKUP} --prepare --target-dir="${TARGETDIR}" +# Then, apply the incremental changes to the base full backup: +${MARIABACKUP} --prepare --target-dir="${TARGETDIR}" --incremental-dir="${INCDIR1}" +# after prepare a file is left in data dir, this seems to be a bug +rm -Rvf mysql/data/* +# Finally transfer the restore +${MARIABACKUP} --copy-back --target-dir="${TARGETDIR}" + +mysql_server_start + +if ! echo "SELECT * from test " | ${MYSQL_CLIENT} "${mariabackup_test_db}" | grep "test entry 1"; then + echo "test entry 1 not found" + estat=2 +fi + +if ! echo "SELECT * from test " | ${MYSQL_CLIENT} "${mariabackup_test_db}" | grep "test entry 2"; then + echo "test entry 2 not found" + estat=3 +fi + +check_for_zombie_jobs storage=File + +mysql_server_stop + +end_test From 8b66bbe1d4d14468738d28dfe1ca90b1821426ab Mon Sep 17 00:00:00 2001 From: "Bruno Friedmann @Bareos" <89836284+bruno-at-bareos@users.noreply.github.com> Date: Mon, 15 Jul 2024 15:12:21 +0200 Subject: [PATCH 2/4] improve logging and error handling Co-authored-by: sduehr --- .../python/mariabackup/bareos-fd-mariabackup.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/core/src/plugins/filed/python/mariabackup/bareos-fd-mariabackup.py b/core/src/plugins/filed/python/mariabackup/bareos-fd-mariabackup.py index 0470f0d82d2..b5479ae32c9 100644 --- a/core/src/plugins/filed/python/mariabackup/bareos-fd-mariabackup.py +++ b/core/src/plugins/filed/python/mariabackup/bareos-fd-mariabackup.py @@ -195,8 +195,8 @@ def get_lsn_by_command(self): ret = check_output(get_lsn_cmd).decode() return ret except CalledProcessError as run_err: - DebugMessage( - 100, + JobMessage( + M_FATAL, f"Error while looking for LSN: {run_err}\n", ) return bRC_Error @@ -208,11 +208,17 @@ def parse_innodb_status(self, res): if line.startswith("Log sequence number"): last_lsn = int(line.split(" ")[-1]) except ValueError as val_err: - DebugMessage( - 100, + JobMessage( + M_FATAL, f"Returned LSN seems wrong: {val_err}\n", ) return bRC_Error + if last_lsn is None: + JobMessage( + M_FATAL, + f"Could not get LSN from {res}\n", + ) + return bRC_Error return last_lsn def create_file(self, restorepkt): @@ -325,6 +331,8 @@ def start_backup_job(self): innodb_status = self.get_lsn_by_command() last_lsn = self.parse_innodb_status(innodb_status) + if last_lsn == bRC_Error: + return bRC_Error JobMessage(M_INFO, "Backup until LSN: %d\n" % last_lsn) if ( self.max_to_lsn > 0 From b2aa55d2e3eb5974763f56aa1d4d42cad8867b16 Mon Sep 17 00:00:00 2001 From: Bruno Friedmann Date: Mon, 15 Jul 2024 16:48:29 +0200 Subject: [PATCH 3/4] fix copyright year --- .../plugins/filed/python/mariabackup/bareos-fd-mariabackup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/plugins/filed/python/mariabackup/bareos-fd-mariabackup.py b/core/src/plugins/filed/python/mariabackup/bareos-fd-mariabackup.py index b5479ae32c9..5030d1b6c29 100644 --- a/core/src/plugins/filed/python/mariabackup/bareos-fd-mariabackup.py +++ b/core/src/plugins/filed/python/mariabackup/bareos-fd-mariabackup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (C) 2015-2023 Bareos GmbH & Co. KG +# Copyright (C) 2015-2024 Bareos GmbH & Co. KG # # This program is Free Software; you can redistribute it and/or # modify it under the terms of version three of the GNU Affero General Public From c6c1084c2813c2001cdc2bc3a143132e9762d698 Mon Sep 17 00:00:00 2001 From: Bareos Bot Date: Mon, 15 Jul 2024 15:39:04 +0000 Subject: [PATCH 4/4] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07e241aac5e..e12dd37abc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -114,6 +114,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - dir: fix crash on purge with job without client [PR #1857] - fix runtime status [PR #1872] - Fix multiple ACL handling bugs [PR #1875] +- fix #1775 plugin: fd mariabackup add support mariadb 11+ [PR #1835] [PR #1538]: https://github.com/bareos/bareos/pull/1538 [PR #1581]: https://github.com/bareos/bareos/pull/1581 @@ -201,6 +202,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [PR #1810]: https://github.com/bareos/bareos/pull/1810 [PR #1821]: https://github.com/bareos/bareos/pull/1821 [PR #1829]: https://github.com/bareos/bareos/pull/1829 +[PR #1835]: https://github.com/bareos/bareos/pull/1835 [PR #1840]: https://github.com/bareos/bareos/pull/1840 [PR #1846]: https://github.com/bareos/bareos/pull/1846 [PR #1847]: https://github.com/bareos/bareos/pull/1847