diff --git a/.github/workflows/ci-units-types.yml b/.github/workflows/ci-units-types.yml index bb70be5aa93..00c2e6d846e 100644 --- a/.github/workflows/ci-units-types.yml +++ b/.github/workflows/ci-units-types.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 diff --git a/kiwi/builder/install.py b/kiwi/builder/install.py index 23d424fd4c1..42215542205 100644 --- a/kiwi/builder/install.py +++ b/kiwi/builder/install.py @@ -280,9 +280,9 @@ def create_install_pxe_archive(self) -> None: source_filename=self.diskname, keep_source_on_compress=True ) - compress.xz(self.xz_options) + xz_archive = compress.xz(self.xz_options) Command.run( - ['mv', compress.compressed_filename, pxe_image_filename] + ['mv', xz_archive, pxe_image_filename] ) # the system image transfer is checked against a checksum diff --git a/kiwi/command.py b/kiwi/command.py index 057ea610350..95e68a3f7dc 100644 --- a/kiwi/command.py +++ b/kiwi/command.py @@ -15,11 +15,17 @@ # You should have received a copy of the GNU General Public License # along with kiwi. If not, see # -from typing import NamedTuple -import select -import os +from typing import IO, Callable, List, MutableMapping, NamedTuple, Optional, overload import logging +import os +import select import subprocess +import sys + +if sys.version_info >= (3, 8): + from typing import Literal # pragma: no cover +else: # pragma: no cover + from typing_extensions import Literal # pragma: no cover # project from kiwi.utils.codec import Codec @@ -31,23 +37,19 @@ log = logging.getLogger('kiwi') -command_type = NamedTuple( - 'command_type', [ - ('output', str), - ('error', str), - ('returncode', int) - ] -) -command_call_type = NamedTuple( - 'command_call_type', [ - ('output', str), - ('output_available', bool), - ('error', str), - ('error_available', bool), - ('process', subprocess.Popen) - ] -) +class CommandT(NamedTuple): + output: str + error: str + returncode: int + + +class CommandCallT(NamedTuple): + output: IO[bytes] + output_available: Callable[[], bool] + error: IO[bytes] + error_available: Callable[[], bool] + process: subprocess.Popen class Command: @@ -58,16 +60,39 @@ class Command: commands in blocking and non blocking mode. Control of stdout and stderr is given to the caller """ + + @overload + @staticmethod + def run( + command: List[str], custom_env: Optional[MutableMapping[str, str]] = None, + raise_on_error: bool = True, stderr_to_stdout: bool = False, + raise_on_command_not_found: Literal[False] = False + ) -> CommandT: + ... # pragma: no cover + + @overload + @staticmethod + def run( + command: List[str], custom_env: Optional[MutableMapping[str, str]] = None, + raise_on_error: bool = True, stderr_to_stdout: bool = False, + raise_on_command_not_found: bool = True + ) -> Optional[CommandT]: + ... # pragma: no cover + @staticmethod def run( - command, custom_env=None, raise_on_error=True, stderr_to_stdout=False - ): + command: List[str], custom_env: Optional[MutableMapping[str, str]] = None, + raise_on_error: bool = True, stderr_to_stdout: bool = False, + raise_on_command_not_found: bool = True + ) -> Optional[CommandT]: """ Execute a program and block the caller. The return value - is a hash containing the stdout, stderr and return code - information. Unless raise_on_error is set to false an - exception is thrown if the command exits with an error - code not equal to zero + is a CommandT namedtuple containing the stdout, stderr + and return code information. Unless raise_on_error is + set to `False` an exception is thrown if the command + exits with an error code not equal to zero. If + raise_on_command_not_found is `False` and the command is + not found, then `None` is returned. Example: @@ -76,7 +101,7 @@ def run( result = Command.run(['ls', '-l']) :param list command: command and arguments - :param list custom_env: custom os.environ + :param dict custom_env: custom os.environ :param bool raise_on_error: control error behaviour :param bool stderr_to_stdout: redirects stderr to stdout @@ -85,40 +110,42 @@ def run( .. code:: python - command(output='string', error='string', returncode=int) + CommandT(output='string', error='string', returncode=int) - :rtype: namedtuple + :rtype: CommandT """ from .path import Path log.debug('EXEC: [%s]', ' '.join(command)) - environment = os.environ - if custom_env: - environment = custom_env - if not Path.which( - command[0], custom_env=environment, access_mode=os.X_OK - ): + environment = custom_env or os.environ + cmd_abspath: Optional[str] + if command[0].startswith("/"): + cmd_abspath = command[0] + if not os.path.exists(cmd_abspath): + cmd_abspath = None + else: + cmd_abspath = Path.which( + command[0], custom_env=environment, access_mode=os.X_OK + ) + + if not cmd_abspath: message = 'Command "%s" not found in the environment' % command[0] - if not raise_on_error: - log.debug('EXEC: %s', message) - return command_type( - output=None, - error=None, - returncode=-1 - ) - else: + if raise_on_command_not_found: raise KiwiCommandNotFound(message) + log.debug('EXEC: %s', message) + return None stderr = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE try: process = subprocess.Popen( - command, + [cmd_abspath] + command[1:], stdout=subprocess.PIPE, stderr=stderr, env=environment ) - except Exception as e: + except (OSError, subprocess.SubprocessError) as e: raise KiwiCommandError( '%s: %s: %s' % (command[0], type(e).__name__, format(e)) - ) + ) from e + output, error = process.communicate() if process.returncode != 0 and raise_on_error: if not error: @@ -135,14 +162,17 @@ def run( command[0], Codec.decode(error), Codec.decode(output) ) ) - return command_type( + return CommandT( output=Codec.decode(output), error=Codec.decode(error), returncode=process.returncode ) @staticmethod - def call(command, custom_env=None): + def call( + command: List[str], + custom_env: Optional[MutableMapping[str, str]] = None + ) -> CommandCallT: """ Execute a program and return an io file handle pair back. stdout and stderr are both on different channels. The caller @@ -174,9 +204,7 @@ def call(command, custom_env=None): """ from .path import Path log.debug('EXEC: [%s]', ' '.join(command)) - environment = os.environ - if custom_env: - environment = custom_env + environment = custom_env or os.environ if not Path.which( command[0], custom_env=environment, access_mode=os.X_OK ): @@ -193,31 +221,32 @@ def call(command, custom_env=None): except Exception as e: raise KiwiCommandError( '%s: %s' % (type(e).__name__, format(e)) - ) + ) from e + + # guaranteed to be true as stdout & stderr equal subprocess.PIPE + assert process.stdout and process.stderr - def output_available(): + def output_available() -> Callable[[], bool]: def _select(): - descriptor_lists = select.select( + readable, _, exceptional = select.select( [process.stdout], [], [process.stdout], 1e-4 ) - readable = descriptor_lists[0] - exceptional = descriptor_lists[2] if readable and not exceptional: return True + return False return _select - def error_available(): + def error_available() -> Callable[[], bool]: def _select(): - descriptor_lists = select.select( + readable, _, exceptional = select.select( [process.stderr], [], [process.stderr], 1e-4 ) - readable = descriptor_lists[0] - exceptional = descriptor_lists[2] if readable and not exceptional: return True + return False return _select - return command_call_type( + return CommandCallT( output=process.stdout, output_available=output_available(), error=process.stderr, diff --git a/kiwi/command_process.py b/kiwi/command_process.py index 45e6a876951..42d0680918c 100644 --- a/kiwi/command_process.py +++ b/kiwi/command_process.py @@ -17,6 +17,7 @@ # import logging from collections import namedtuple +from kiwi.command import CommandCallT # project from kiwi.utils.codec import Codec @@ -37,7 +38,7 @@ class CommandProcess: :param subprocess command: instance of subprocess :param string log_topic: topic string for logging """ - def __init__(self, command, log_topic='system'): + def __init__(self, command: CommandCallT, log_topic='system') -> None: self.command = CommandIterator(command) self.log_topic = log_topic self.items_processed = 0 @@ -154,7 +155,7 @@ class CommandIterator: :param subprocess command: instance of subprocess """ - def __init__(self, command): + def __init__(self, command: CommandCallT) -> None: self.command = command self.command_error_output = bytes(b'') self.command_output_line = bytes(b'') @@ -196,7 +197,7 @@ def get_error_output(self): """ return Codec.decode(self.command_error_output) - def get_error_code(self): + def get_error_code(self) -> int: """ Provide return value from processed command @@ -206,7 +207,7 @@ def get_error_code(self): """ return self.command.process.returncode - def get_pid(self): + def get_pid(self) -> int: """ Provide process ID of command while running @@ -216,7 +217,7 @@ def get_pid(self): """ return self.command.process.pid - def kill(self): + def kill(self) -> None: """ Send kill signal SIGTERM to command process """ diff --git a/kiwi/mount_manager.py b/kiwi/mount_manager.py index fc6d0314d86..15377cf9537 100644 --- a/kiwi/mount_manager.py +++ b/kiwi/mount_manager.py @@ -216,7 +216,4 @@ def is_mounted(self) -> bool: command=['mountpoint', '-q', self.mountpoint], raise_on_error=False ) - if mountpoint_call.returncode == 0: - return True - else: - return False + return mountpoint_call.returncode == 0 diff --git a/kiwi/package_manager/apt.py b/kiwi/package_manager/apt.py index 535e511769a..a87abf3fea5 100644 --- a/kiwi/package_manager/apt.py +++ b/kiwi/package_manager/apt.py @@ -24,7 +24,7 @@ ) # project -from kiwi.command import command_call_type +from kiwi.command import CommandCallT from kiwi.command import Command from kiwi.path import Path from kiwi.package_manager.base import PackageManagerBase @@ -128,7 +128,7 @@ def setup_repository_modules( def process_install_requests_bootstrap( self, root_bind: RootBind = None, bootstrap_package: str = None - ) -> command_call_type: + ) -> CommandCallT: """ Process package install requests for bootstrap phase (no chroot) Either debootstrap or a prebuilt bootstrap package can be used @@ -155,7 +155,7 @@ def process_install_requests_bootstrap( def _process_install_requests_bootstrap_prebuild_root( self, bootstrap_package: str - ) -> command_call_type: + ) -> CommandCallT: """ Process bootstrap phase (no chroot) using a prebuilt bootstrap package. The package has to provide a tarball below the @@ -207,7 +207,7 @@ def _process_install_requests_bootstrap_prebuild_root( def _process_install_requests_bootstrap_debootstrap( self, root_bind: RootBind = None - ) -> command_call_type: + ) -> CommandCallT: """ Process package install requests for bootstrap phase (no chroot) The debootstrap program is used to bootstrap a new system @@ -316,7 +316,7 @@ def post_process_install_requests_bootstrap( if root_bind: root_bind.mount_kernel_file_systems(delta_root) - def process_install_requests(self) -> command_call_type: + def process_install_requests(self) -> CommandCallT: """ Process package install requests for image phase (chroot) @@ -344,7 +344,7 @@ def process_install_requests(self) -> command_call_type: apt_get_command, self.command_env ) - def process_delete_requests(self, force: bool = False) -> command_call_type: + def process_delete_requests(self, force: bool = False) -> CommandCallT: """ Process package delete requests (chroot) @@ -452,7 +452,7 @@ def post_process_delete_requests( ] ) - def update(self) -> command_call_type: + def update(self) -> CommandCallT: """ Process package update requests (chroot) diff --git a/kiwi/package_manager/base.py b/kiwi/package_manager/base.py index a7a01d1e110..4567000d228 100644 --- a/kiwi/package_manager/base.py +++ b/kiwi/package_manager/base.py @@ -20,7 +20,7 @@ ) from kiwi.api_helper import decommissioned -from kiwi.command import command_call_type +from kiwi.command import CommandCallT from kiwi.system.root_bind import RootBind from kiwi.repository.base import RepositoryBase @@ -117,7 +117,7 @@ def setup_repository_modules( def process_install_requests_bootstrap( self, root_bind: RootBind = None, bootstrap_package: str = None - ) -> command_call_type: + ) -> CommandCallT: """ Process package install requests for bootstrap phase (no chroot) @@ -125,7 +125,7 @@ def process_install_requests_bootstrap( """ raise NotImplementedError - def process_install_requests(self) -> command_call_type: + def process_install_requests(self) -> CommandCallT: """ Process package install requests for image phase (chroot) @@ -133,7 +133,7 @@ def process_install_requests(self) -> command_call_type: """ raise NotImplementedError - def process_delete_requests(self, force: bool = False) -> command_call_type: + def process_delete_requests(self, force: bool = False) -> CommandCallT: """ Process package delete requests (chroot) @@ -143,7 +143,7 @@ def process_delete_requests(self, force: bool = False) -> command_call_type: """ raise NotImplementedError - def update(self) -> command_call_type: + def update(self) -> CommandCallT: """ Process package update requests (chroot) @@ -243,7 +243,7 @@ def has_failed(returncode: int) -> bool: :rtype: boolean """ - return True if returncode != 0 else False + return returncode != 0 def get_error_details(self) -> str: """ diff --git a/kiwi/package_manager/dnf.py b/kiwi/package_manager/dnf.py index 248b032afa9..8e0b7631582 100644 --- a/kiwi/package_manager/dnf.py +++ b/kiwi/package_manager/dnf.py @@ -20,7 +20,7 @@ ) # project -from kiwi.command import command_call_type +from kiwi.command import CommandCallT from kiwi.package_manager.base import PackageManagerBase from kiwi.system.root_bind import RootBind from kiwi.api_helper import decommissioned @@ -60,22 +60,22 @@ def setup_repository_modules( @decommissioned def process_install_requests_bootstrap( self, root_bind: RootBind = None, bootstrap_package: str = None - ) -> command_call_type: + ) -> CommandCallT: pass # pragma: no cover @no_type_check @decommissioned - def process_install_requests(self) -> command_call_type: + def process_install_requests(self) -> CommandCallT: pass # pragma: no cover @no_type_check @decommissioned - def process_delete_requests(self, force: bool = False) -> command_call_type: + def process_delete_requests(self, force: bool = False) -> CommandCallT: pass # pragma: no cover @no_type_check @decommissioned - def update(self) -> command_call_type: + def update(self) -> CommandCallT: pass # pragma: no cover @decommissioned diff --git a/kiwi/package_manager/dnf4.py b/kiwi/package_manager/dnf4.py index 10395fe43e3..18599d41fe4 100644 --- a/kiwi/package_manager/dnf4.py +++ b/kiwi/package_manager/dnf4.py @@ -21,7 +21,7 @@ ) # project -from kiwi.command import command_call_type +from kiwi.command import CommandCallT from kiwi.command import Command from kiwi.utils.rpm_database import RpmDataBase from kiwi.utils.rpm import Rpm @@ -136,7 +136,7 @@ def setup_repository_modules( def process_install_requests_bootstrap( self, root_bind: RootBind = None, bootstrap_package: str = None - ) -> command_call_type: + ) -> CommandCallT: """ Process package install requests for bootstrap phase (no chroot) @@ -165,7 +165,7 @@ def process_install_requests_bootstrap( dnf_command, self.command_env ) - def process_install_requests(self) -> command_call_type: + def process_install_requests(self) -> CommandCallT: """ Process package install requests for image phase (chroot) @@ -196,7 +196,7 @@ def process_install_requests(self) -> command_call_type: dnf_command, self.command_env ) - def process_delete_requests(self, force: bool = False) -> command_call_type: + def process_delete_requests(self, force: bool = False) -> CommandCallT: """ Process package delete requests (chroot) @@ -245,7 +245,7 @@ def process_delete_requests(self, force: bool = False) -> command_call_type: dnf_command, self.command_env ) - def update(self) -> command_call_type: + def update(self) -> CommandCallT: """ Process package update requests (chroot) diff --git a/kiwi/package_manager/dnf5.py b/kiwi/package_manager/dnf5.py index 56ce89ddc2d..3be05d1e83c 100644 --- a/kiwi/package_manager/dnf5.py +++ b/kiwi/package_manager/dnf5.py @@ -21,7 +21,7 @@ ) # project -from kiwi.command import command_call_type +from kiwi.command import CommandCallT from kiwi.command import Command from kiwi.utils.rpm_database import RpmDataBase from kiwi.utils.rpm import Rpm @@ -136,7 +136,7 @@ def setup_repository_modules( def process_install_requests_bootstrap( self, root_bind: RootBind = None, bootstrap_package: str = None - ) -> command_call_type: + ) -> CommandCallT: """ Process package install requests for bootstrap phase (no chroot) @@ -165,7 +165,7 @@ def process_install_requests_bootstrap( dnf_command, self.command_env ) - def process_install_requests(self) -> command_call_type: + def process_install_requests(self) -> CommandCallT: """ Process package install requests for image phase (chroot) @@ -196,7 +196,7 @@ def process_install_requests(self) -> command_call_type: dnf_command, self.command_env ) - def process_delete_requests(self, force: bool = False) -> command_call_type: + def process_delete_requests(self, force: bool = False) -> CommandCallT: """ Process package delete requests (chroot) @@ -245,7 +245,7 @@ def process_delete_requests(self, force: bool = False) -> command_call_type: dnf_command, self.command_env ) - def update(self) -> command_call_type: + def update(self) -> CommandCallT: """ Process package update requests (chroot) diff --git a/kiwi/package_manager/microdnf.py b/kiwi/package_manager/microdnf.py index b783fe6348b..2636d56a329 100644 --- a/kiwi/package_manager/microdnf.py +++ b/kiwi/package_manager/microdnf.py @@ -23,7 +23,7 @@ # project -from kiwi.command import command_call_type +from kiwi.command import CommandCallT from kiwi.command import Command from kiwi.utils.rpm_database import RpmDataBase from kiwi.utils.rpm import Rpm @@ -155,7 +155,7 @@ def setup_repository_modules( def process_install_requests_bootstrap( self, root_bind: RootBind = None, bootstrap_package: str = None - ) -> command_call_type: + ) -> CommandCallT: """ Process package install requests for bootstrap phase (no chroot) @@ -187,7 +187,7 @@ def process_install_requests_bootstrap( microdnf_command, self.command_env ) - def process_install_requests(self) -> command_call_type: + def process_install_requests(self) -> CommandCallT: """ Process package install requests for image phase (chroot) @@ -218,7 +218,7 @@ def process_install_requests(self) -> command_call_type: microdnf_command, self.command_env ) - def process_delete_requests(self, force: bool = False) -> command_call_type: + def process_delete_requests(self, force: bool = False) -> CommandCallT: """ Process package delete requests (chroot) @@ -267,7 +267,7 @@ def process_delete_requests(self, force: bool = False) -> command_call_type: dnf_command, self.command_env ) - def update(self) -> command_call_type: + def update(self) -> CommandCallT: """ Process package update requests (chroot) diff --git a/kiwi/package_manager/pacman.py b/kiwi/package_manager/pacman.py index 95c68675f17..be863796387 100644 --- a/kiwi/package_manager/pacman.py +++ b/kiwi/package_manager/pacman.py @@ -22,7 +22,7 @@ ) # project -from kiwi.command import command_call_type +from kiwi.command import CommandCallT from kiwi.command import Command from kiwi.package_manager.base import PackageManagerBase from kiwi.system.root_bind import RootBind @@ -110,7 +110,7 @@ def setup_repository_modules( def process_install_requests_bootstrap( self, root_bind: RootBind = None, bootstrap_package: str = None - ) -> command_call_type: + ) -> CommandCallT: """ Process package install requests for bootstrap phase (no chroot) @@ -139,7 +139,7 @@ def process_install_requests_bootstrap( pacman_command, self.command_env ) - def process_install_requests(self) -> command_call_type: + def process_install_requests(self) -> CommandCallT: """ Process package install requests for image phase (chroot) @@ -158,7 +158,7 @@ def process_install_requests(self) -> command_call_type: pacman_command, self.command_env ) - def process_delete_requests(self, force: bool = False) -> command_call_type: + def process_delete_requests(self, force: bool = False) -> CommandCallT: """ Process package delete requests (chroot) @@ -195,7 +195,7 @@ def process_delete_requests(self, force: bool = False) -> command_call_type: self.command_env ) - def update(self) -> command_call_type: + def update(self) -> CommandCallT: """ Process package update requests (chroot) diff --git a/kiwi/package_manager/zypper.py b/kiwi/package_manager/zypper.py index 865c282903a..2f71ec7b66c 100644 --- a/kiwi/package_manager/zypper.py +++ b/kiwi/package_manager/zypper.py @@ -23,7 +23,7 @@ # project -from kiwi.command import command_call_type +from kiwi.command import CommandCallT from kiwi.command import Command from kiwi.package_manager.base import PackageManagerBase from kiwi.system.root_bind import RootBind @@ -114,7 +114,7 @@ def setup_repository_modules( def process_install_requests_bootstrap( self, root_bind: RootBind = None, bootstrap_package: str = None - ) -> command_call_type: + ) -> CommandCallT: """ Process package install requests for bootstrap phase (no chroot) @@ -134,7 +134,7 @@ def process_install_requests_bootstrap( command, self.command_env ) - def process_install_requests(self) -> command_call_type: + def process_install_requests(self) -> CommandCallT: """ Process package install requests for image phase (chroot) @@ -165,7 +165,7 @@ def process_install_requests(self) -> command_call_type: self.chroot_command_env ) - def process_delete_requests(self, force: bool = False) -> command_call_type: + def process_delete_requests(self, force: bool = False) -> CommandCallT: """ Process package delete requests (chroot) @@ -211,7 +211,7 @@ def process_delete_requests(self, force: bool = False) -> command_call_type: zypper_command, self.chroot_command_env ) - def update(self) -> command_call_type: + def update(self) -> CommandCallT: """ Process package update requests (chroot) diff --git a/kiwi/partitioner/msdos.py b/kiwi/partitioner/msdos.py index fb105e6d749..55ab202769d 100644 --- a/kiwi/partitioner/msdos.py +++ b/kiwi/partitioner/msdos.py @@ -91,7 +91,8 @@ def set_flag(self, partition_id: int, flag_name: str) -> None: raise KiwiPartitionerMsDosFlagError( 'Unknown partition flag %s' % flag_name ) - if self.flag_map[flag_name]: + flag_val = self.flag_map[flag_name] + if flag_val: if flag_name == 'f.active': Command.run( [ @@ -100,10 +101,11 @@ def set_flag(self, partition_id: int, flag_name: str) -> None: ] ) else: + assert isinstance(flag_val, str), f"flag {flag_name} must be a string but got a {type(flag_val)}" Command.run( [ 'sfdisk', '-c', self.disk_device, - format(partition_id), self.flag_map[flag_name] + format(partition_id), flag_val ] ) else: diff --git a/kiwi/path.py b/kiwi/path.py index 35fe9dd99a9..787426c0464 100644 --- a/kiwi/path.py +++ b/kiwi/path.py @@ -18,7 +18,7 @@ import os import logging import collections -from typing import Dict, List, Optional +from typing import Dict, List, MutableMapping, Optional # project from kiwi.command import Command @@ -224,7 +224,7 @@ def rebase_to_root(root: str, elements: List[str]) -> List[str]: @staticmethod def which( filename: str, alternative_lookup_paths: Optional[List[str]] = None, - custom_env: Optional[Dict[str, str]] = None, + custom_env: Optional[MutableMapping[str, str]] = None, access_mode: Optional[int] = None, root_dir: Optional[str] = None ) -> Optional[str]: diff --git a/kiwi/storage/raid_device.py b/kiwi/storage/raid_device.py index 58394079241..21f35aa1a10 100644 --- a/kiwi/storage/raid_device.py +++ b/kiwi/storage/raid_device.py @@ -108,6 +108,8 @@ def create_raid_config(self, filename: str) -> None: :param string filename: config file name """ + if not self.raid_device: + raise KiwiRaidSetupError("No raid device defined, cannot create raid config!") mdadm_call = Command.run( ['mdadm', '-Db', self.raid_device] ) diff --git a/kiwi/system/setup.py b/kiwi/system/setup.py index e9f780f88a5..706f4bdc9b4 100644 --- a/kiwi/system/setup.py +++ b/kiwi/system/setup.py @@ -23,7 +23,7 @@ from collections import OrderedDict from collections import namedtuple from typing import ( - Any, List + Any, List, Optional ) # project @@ -1154,7 +1154,7 @@ def _call_script( self.setup_selinux_file_contexts() def _call_script_no_chroot( - self, name, option_list, working_directory + self, name: str, option_list: List[str], working_directory: Optional[str] ): if not working_directory: working_directory = self.root_dir diff --git a/kiwi/tasks/result_bundle.py b/kiwi/tasks/result_bundle.py index 7aa7cb157ef..ca1eec43246 100644 --- a/kiwi/tasks/result_bundle.py +++ b/kiwi/tasks/result_bundle.py @@ -196,8 +196,7 @@ def process(self): if result_file.compress: log.info('--> XZ compressing') compress = Compress(bundle_file) - compress.xz(self.runtime_config.get_xz_options()) - bundle_file = compress.compressed_filename + bundle_file = compress.xz(self.runtime_config.get_xz_options()) if self.command_args['--zsync-source'] and result_file.shasum: # Files with a checksum are considered to be image files diff --git a/kiwi/utils/command_capabilities.py b/kiwi/utils/command_capabilities.py index e009aefbc70..d9df2054f2c 100644 --- a/kiwi/utils/command_capabilities.py +++ b/kiwi/utils/command_capabilities.py @@ -78,7 +78,7 @@ def has_option_in_help( def check_version( call, version_waterline, version_flags=None, root=None, raise_on_error=True - ): + ) -> bool: """ Checks if the given command version is equal or higher than the given version tuple. diff --git a/test/unit/builder/install_test.py b/test/unit/builder/install_test.py index 1d66a8b937a..a4351de4099 100644 --- a/test/unit/builder/install_test.py +++ b/test/unit/builder/install_test.py @@ -342,6 +342,8 @@ def test_create_install_pxe_archive( mock_md5.return_value = checksum compress = mock.Mock() + src = 'target_dir/result-image.x86_64-1.2.3.raw' + compress.xz.return_value = src mock_compress.return_value = compress m_open = mock_open() @@ -349,15 +351,11 @@ def test_create_install_pxe_archive( self.install_image.create_install_pxe_archive() mock_compress.assert_called_once_with( - keep_source_on_compress=True, - source_filename='target_dir/result-image.x86_64-1.2.3.raw' + keep_source_on_compress=True, source_filename=src ) compress.xz.assert_called_once_with(None) assert mock_command.call_args_list[0] == call( - [ - 'mv', compress.compressed_filename, - 'tmpdir/result-image.x86_64-1.2.3.xz' - ] + ['mv', src, 'tmpdir/result-image.x86_64-1.2.3.xz'] ) mock_md5.assert_called_once_with( 'target_dir/result-image.x86_64-1.2.3.raw' diff --git a/test/unit/command_test.py b/test/unit/command_test.py index 5b485c164d7..8e83899e6cb 100644 --- a/test/unit/command_test.py +++ b/test/unit/command_test.py @@ -4,6 +4,8 @@ import unittest.mock as mock import os +import pytest + from kiwi.command import Command from kiwi.exceptions import ( @@ -30,7 +32,7 @@ def test_run_raises_error(self, mock_popen, mock_which): @patch('subprocess.Popen') def test_run_failure(self, mock_popen, mock_which): mock_which.return_value = 'command' - mock_popen.side_effect = KiwiCommandError('Run failure') + mock_popen.side_effect = OSError('Run failure') with raises(KiwiCommandError): Command.run(['command', 'args']) @@ -55,10 +57,33 @@ def test_run_does_not_raise_error(self, mock_popen, mock_which): @patch('kiwi.path.Path.which') def test_run_does_not_raise_error_if_command_not_found(self, mock_which): mock_which.return_value = None - result = Command.run(['command', 'args'], os.environ, False) - assert result.error is None - assert result.output is None - assert result.returncode == -1 + result = Command.run(['command', 'args'], os.environ, raise_on_command_not_found=False) + assert result is None + + @patch('kiwi.path.Path.which') + @patch('os.path.exists') + @patch('subprocess.Popen') + def test_run_does_not_call_which_for_abspaths(self, mock_popen, mock_exists, mock_which): + mock_exists.return_value = True + proc = mock.MagicMock() + proc.communicate.return_value = (str.encode("stdout"), str.encode("stderr")) + proc.returncode = 0 + mock_popen.return_value = proc + + result = Command.run(['/bin/command', 'args']) + mock_which.assert_not_called() + assert result is not None + + @patch('kiwi.path.Path.which') + @patch('os.path.exists') + def test_run_fails_for_non_existent_abspath(self, mock_exists, mock_which): + mock_exists.return_value = False + + with pytest.raises(KiwiCommandNotFound) as cmd_not_found_err: + Command.run(['/bin/command', 'args']) + + mock_which.assert_not_called() + assert '"/bin/command" not found' in str(cmd_not_found_err.value) @patch('os.access') @patch('os.path.exists') @@ -90,17 +115,18 @@ def test_call_command_does_not_exist(self): with raises(KiwiCommandNotFound): Command.call(['does-not-exist'], os.environ) + @pytest.mark.parametrize("available", (True, False)) @patch('kiwi.path.Path.which') @patch('subprocess.Popen') @patch('select.select') - def test_call(self, mock_select, mock_popen, mock_which): + def test_call(self, mock_select, mock_popen, mock_which, available: bool): mock_which.return_value = 'command' - mock_select.return_value = [True, False, False] + mock_select.return_value = [True, False, not available] mock_process = mock.Mock() mock_popen.return_value = mock_process call = Command.call(['command', 'args']) - assert call.output_available() - assert call.error_available() + assert call.output_available() == available + assert call.error_available() == available assert call.output == mock_process.stdout assert call.error == mock_process.stderr assert call.process == mock_process diff --git a/test/unit/path_test.py b/test/unit/path_test.py index dbd04bddb3b..eebdef002bd 100644 --- a/test/unit/path_test.py +++ b/test/unit/path_test.py @@ -108,7 +108,7 @@ def test_which(self, mock_exists, mock_env, mock_access): assert Path.which('some-file') is None mock_env.return_value = None mock_exists.return_value = True - assert Path.which('some-file', ['alternative']) == \ + assert Path.which('some-file', alternative_lookup_paths=['alternative']) == \ 'alternative/some-file' mock_access.return_value = False mock_env.return_value = '/usr/local/bin:/usr/bin:/bin' @@ -134,7 +134,6 @@ def test_which_not_found_log( mock_exists.return_value = False with self._caplog.at_level(logging.DEBUG): assert Path.which('file') is None - print(self._caplog.text) assert ( '"file": in paths "{0}" exists: "False" mode match: ' 'not checked' diff --git a/test/unit/shell_test.py b/test/unit/shell_test.py index 983afe0fe51..8b6204d4426 100644 --- a/test/unit/shell_test.py +++ b/test/unit/shell_test.py @@ -11,7 +11,7 @@ def test_quote(self): @patch('kiwi.path.Path.which') def test_quote_key_value_file(self, mock_which): - mock_which.return_value = 'cp' + mock_which.side_effect = ['cp', 'bash'] assert Shell.quote_key_value_file('../data/key_value') == [ "foo='bar'", "bar='xxx'", diff --git a/test/unit/storage/raid_device_test.py b/test/unit/storage/raid_device_test.py index af7f2fea257..70828dc11ee 100644 --- a/test/unit/storage/raid_device_test.py +++ b/test/unit/storage/raid_device_test.py @@ -80,6 +80,16 @@ def test_create_raid_config(self, mock_command): m_open.return_value.write.assert_called_once_with('data') self.raid.raid_device = None + @patch('kiwi.storage.raid_device.Command.run') + def test_create_raid_config_without_raid_device(self, mock_command): + self.raid.raid_device = None + + with raises(KiwiRaidSetupError) as raid_err_ctx: + self.raid.create_raid_config('mdadm.conf') + + assert "No raid device" in str(raid_err_ctx.value) + mock_command.assert_not_called() + def test_is_loop(self): assert self.raid.is_loop() is True diff --git a/test/unit/system/profile_test.py b/test/unit/system/profile_test.py index b7f9a0d16e9..a8736b18f58 100644 --- a/test/unit/system/profile_test.py +++ b/test/unit/system/profile_test.py @@ -27,7 +27,7 @@ def setup_method(self, cls): @patch('kiwi.path.Path.which') def test_create_live(self, mock_which): - mock_which.return_value = 'cp' + mock_which.side_effect = ['cp', 'bash'] self.live_profile.create(self.profile_file) os.remove(self.profile_file) assert self.live_profile.dot_profile == { @@ -76,7 +76,7 @@ def test_create_live(self, mock_which): @patch('kiwi.path.Path.which') def test_create_oem(self, mock_which): - mock_which.return_value = 'cp' + mock_which.side_effect = ['cp', 'bash'] self.profile.create(self.profile_file) os.remove(self.profile_file) assert self.profile.dot_profile == { @@ -159,7 +159,7 @@ def test_create_oem(self, mock_which): @patch('kiwi.path.Path.which') def test_create_displayname_is_image_name(self, mock_which): - mock_which.return_value = 'cp' + mock_which.side_effect = ['cp', 'bash'] description = XMLDescription('../data/example_pxe_config.xml') profile = Profile( XMLState(description.load()) @@ -171,7 +171,7 @@ def test_create_displayname_is_image_name(self, mock_which): @patch('kiwi.path.Path.which') def test_create_cpio(self, mock_which): - mock_which.return_value = 'cp' + mock_which.side_effect = ['cp', 'bash'] description = XMLDescription('../data/example_dot_profile_config.xml') profile = Profile( XMLState(description.load(), None, 'cpio') diff --git a/test/unit/tasks/result_bundle_test.py b/test/unit/tasks/result_bundle_test.py index 7bb2f6caba0..3174134dc5b 100644 --- a/test/unit/tasks/result_bundle_test.py +++ b/test/unit/tasks/result_bundle_test.py @@ -102,7 +102,7 @@ def test_process_result_bundle( checksum = Mock() compress = Mock() mock_path_which.return_value = 'zsyncmake' - compress.compressed_filename = 'compressed_filename' + compress.xz.return_value = 'compressed_filename' mock_compress.return_value = compress mock_checksum.return_value = checksum mock_exists.return_value = False @@ -139,7 +139,7 @@ def test_process_result_bundle( os.sep.join([self.abs_bundle_dir, 'test-image-1.2.3-Build_42']) ) mock_checksum.assert_called_once_with( - compress.compressed_filename + 'compressed_filename' ) checksum.sha256.assert_called_once_with() m_open.return_value.write.assert_called_once_with( @@ -168,7 +168,7 @@ def test_process_result_bundle_as_rpm( checksum = Mock() compress = Mock() mock_path_which.return_value = 'zsyncmake' - compress.compressed_filename = 'compressed_filename' + compress.xz.return_value = 'compressed_filename' mock_compress.return_value = compress mock_checksum.return_value = checksum mock_exists.return_value = False @@ -303,7 +303,7 @@ def test_process_result_bundle_zsyncmake_missing( checksum = Mock() compress = Mock() mock_path_which.return_value = None - compress.compressed_filename = 'compressed_filename' + compress.xz.return_value = 'compressed_filename' mock_compress.return_value = compress mock_checksum.return_value = checksum mock_exists.return_value = False diff --git a/test/unit/utils/compress_test.py b/test/unit/utils/compress_test.py index 199e87c52eb..a5f2dabc3ce 100644 --- a/test/unit/utils/compress_test.py +++ b/test/unit/utils/compress_test.py @@ -95,9 +95,11 @@ def test_uncompress_unknown_format(self, mock_format): @patch('kiwi.path.Path.which') def test_get_format(self, mock_which): - mock_which.return_value = 'ziptool' + mock_which.side_effect = ['xz'] xz = Compress('../data/xz_data.xz') assert xz.get_format() == 'xz' + + mock_which.side_effect = ['xz', 'gzip'] gzip = Compress('../data/gz_data.gz') assert gzip.get_format() == 'gzip' diff --git a/tox.ini b/tox.ini index 3fbd704b7cd..55865535eee 100644 --- a/tox.ini +++ b/tox.ini @@ -21,12 +21,14 @@ envlist = unit_py3_11, unit_py3_10, unit_py3_9, + unit_py3_8, + unit_py3_7, packagedoc [testenv] description = - {unit_py3_9,unit_py3_10,unit_py3_11,unit_py3_12}: Unit Test run with basepython set to {basepython} + {unit_py3_7,unit_py3_8,unit_py3_9,unit_py3_10,unit_py3_11,unit_py3_12}: Unit Test run with basepython set to {basepython} devel: Test KIWI allowlist_externals = bash @@ -44,6 +46,8 @@ basepython = unit_py3_11: python3.11 unit_py3_10: python3.10 unit_py3_9: python3.9 + unit_py3_8: python3.8 + unit_py3_7: python3.7 release: python3.10 check: python3 devel: python3 @@ -55,42 +59,80 @@ deps = -r.virtualenv.dev-requirements.txt -# Unit Test run with basepython set to 3.9 +# Test run with basepython set to 3.7 +# Only static type checking is performed for 3.7 +# because the unit test code base requires python >= 3.8 +[testenv:unit_py3_7] +setenv = + PYTHONPATH={toxinidir}/test +changedir=test/unit +commands = + {[testenv:mypy]commands} + +# Test run with basepython set to 3.8 +[testenv:unit_py3_8] +setenv = + PYTHONPATH={toxinidir}/test +changedir=test/unit +commands = + {[testenv:mypy]commands} + {[testenv:unit]commands} + +# Test run with basepython set to 3.9 [testenv:unit_py3_9] setenv = PYTHONPATH={toxinidir}/test changedir=test/unit commands = + {[testenv:mypy]commands} {[testenv:unit]commands} -# Unit Test run with basepython set to 3.10 +# Test run with basepython set to 3.10 [testenv:unit_py3_10] setenv = PYTHONPATH={toxinidir}/test changedir=test/unit commands = + {[testenv:mypy]commands} {[testenv:unit]commands} -# Unit Test run with basepython set to 3.11 +# Test run with basepython set to 3.11 [testenv:unit_py3_11] setenv = PYTHONPATH={toxinidir}/test changedir=test/unit commands = + {[testenv:mypy]commands} {[testenv:unit]commands} -# Unit Test run with basepython set to 3.12 +# Test run with basepython set to 3.12 [testenv:unit_py3_12] setenv = PYTHONPATH={toxinidir}/test changedir=test/unit commands = + {[testenv:mypy]commands} {[testenv:unit]commands} +[testenv:mypy] +description = Static Type Checking Base +skip_install = True +usedevelop = True +setenv = + PYTHONUNBUFFERED=yes + WITH_COVERAGE=yes +passenv = + * +deps = {[testenv]deps} +changedir=test/unit +commands = + bash -c 'cd ../../ && mypy kiwi' + + [testenv:unit] description = Unit Test Base skip_install = True