diff --git a/CHANGELOG.md b/CHANGELOG.md index fe7fdcc28b..823e000beb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ - Update gitpod/workspace-base Docker digest to 0f38224 ([#3048](https://github.com/nf-core/tools/pull/3048)) - update output_dir for api docs to new website structure ([#3051](https://github.com/nf-core/tools/pull/3051)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.5.1 ([#3052](https://github.com/nf-core/tools/pull/3052)) +- Add `--limit-output` argument for modules/subworkflow update ([#3047](https://github.com/nf-core/tools/pull/3047)) - update api docs to new structure ([#3054](https://github.com/nf-core/tools/pull/3054)) - Update to pytest v8 and move it to dev dependencies ([#3058](https://github.com/nf-core/tools/pull/3058)) - handle new jsonschema error type ([#3061](https://github.com/nf-core/tools/pull/3061)) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 4fa640436d..f33c63e87c 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -928,6 +928,13 @@ def command_modules_install(ctx, tool, dir, prompt, force, sha): default=False, help="Prompt for the version of the module", ) +@click.option( + "--limit-output", + "limit_output", + is_flag=True, + default=False, + help="Limit output to only the difference in main.nf", +) @click.option("-s", "--sha", type=str, metavar="", help="Install module at commit SHA") @click.option( "-a", @@ -970,11 +977,12 @@ def command_modules_update( preview, save_diff, update_deps, + limit_output, ): """ Update DSL2 modules within a pipeline. """ - modules_update(ctx, tool, directory, force, prompt, sha, install_all, preview, save_diff, update_deps) + modules_update(ctx, tool, directory, force, prompt, sha, install_all, preview, save_diff, update_deps, limit_output) # nf-core modules patch @@ -1538,6 +1546,14 @@ def command_subworkflows_remove(ctx, dir, subworkflow): metavar="", help="Install subworkflow at commit SHA", ) +@click.option( + "-l", + "--limit-output", + "limit_output", + is_flag=True, + default=False, + help="Limit ouput to only the difference in main.nf", +) @click.option( "-a", "--all", @@ -1579,11 +1595,14 @@ def command_subworkflows_update( preview, save_diff, update_deps, + limit_output, ): """ Update DSL2 subworkflow within a pipeline. """ - subworkflows_update(ctx, subworkflow, dir, force, prompt, sha, install_all, preview, save_diff, update_deps) + subworkflows_update( + ctx, subworkflow, dir, force, prompt, sha, install_all, preview, save_diff, update_deps, limit_output + ) ## DEPRECATED commands since v3.0.0 diff --git a/nf_core/commands_modules.py b/nf_core/commands_modules.py index 5f7191436c..3d96d332b0 100644 --- a/nf_core/commands_modules.py +++ b/nf_core/commands_modules.py @@ -86,6 +86,7 @@ def modules_update( preview, save_diff, update_deps, + limit_output, ): """ Update DSL2 modules within a pipeline. @@ -107,6 +108,7 @@ def modules_update( ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], + limit_output, ) exit_status = module_install.update(tool) if not exit_status and install_all: diff --git a/nf_core/commands_subworkflows.py b/nf_core/commands_subworkflows.py index cc1a544eca..a3abce3f85 100644 --- a/nf_core/commands_subworkflows.py +++ b/nf_core/commands_subworkflows.py @@ -229,6 +229,7 @@ def subworkflows_update( preview, save_diff, update_deps, + limit_output, ): """ Update DSL2 subworkflow within a pipeline. @@ -250,6 +251,7 @@ def subworkflows_update( ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], + limit_output, ) exit_status = subworkflow_install.update(subworkflow) if not exit_status and install_all: diff --git a/nf_core/components/update.py b/nf_core/components/update.py index a54c47232e..1e31e56271 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -38,6 +38,7 @@ def __init__( remote_url=None, branch=None, no_pull=False, + limit_output=False, ): super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull) self.force = force @@ -46,6 +47,7 @@ def __init__( self.update_all = update_all self.show_diff = show_diff self.save_diff_fn = save_diff_fn + self.limit_output = limit_output self.update_deps = update_deps self.component = None self.update_config = None @@ -75,6 +77,8 @@ def _parameter_checks(self): if not self.has_valid_directory(): raise UserWarning("The command was not run in a valid pipeline directory.") + if self.limit_output and not (self.save_diff_fn or self.show_diff): + raise UserWarning("The '--limit-output' flag can only be used with '--preview' or '--save-diff'.") def update(self, component=None, silent=False, updated=None, check_diff_exist=True) -> bool: """Updates a specified module/subworkflow or all modules/subworkflows in a pipeline. @@ -124,7 +128,6 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr components_info = ( self.get_all_components_info() if self.update_all else [self.get_single_component_info(component)] ) - # Save the current state of the modules.json old_modules_json = self.modules_json.get_modules_json() @@ -231,6 +234,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr version, dsp_from_dir=component_dir, dsp_to_dir=component_dir, + limit_output=self.limit_output, ) updated.append(component) except UserWarning as e: @@ -271,8 +275,8 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr version, dsp_from_dir=component_dir, dsp_to_dir=component_dir, + limit_output=self.limit_output, ) - # Ask the user if they want to install the component dry_run = not questionary.confirm( f"Update {self.component_type[:-1]} '{component}'?", @@ -389,6 +393,8 @@ def get_single_component_info(self, component): sha = self.sha config_entry = None + if self.update_config is None: + raise UserWarning("Could not find '.nf-core.yml' file in pipeline directory") if any( [ entry.count("/") == 1 @@ -829,6 +835,7 @@ def try_apply_patch( for_git=False, dsp_from_dir=component_relpath, dsp_to_dir=component_relpath, + limit_output=self.limit_output, ) # Move the patched files to the install dir @@ -875,7 +882,13 @@ def get_components_to_update(self, component): return modules_to_update, subworkflows_to_update - def update_linked_components(self, modules_to_update, subworkflows_to_update, updated=None, check_diff_exist=True): + def update_linked_components( + self, + modules_to_update, + subworkflows_to_update, + updated=None, + check_diff_exist=True, + ): """ Update modules and subworkflows linked to the component being updated. """ @@ -883,7 +896,12 @@ def update_linked_components(self, modules_to_update, subworkflows_to_update, up if s_update in updated: continue original_component_type, original_update_all = self._change_component_type("subworkflows") - self.update(s_update, silent=True, updated=updated, check_diff_exist=check_diff_exist) + self.update( + s_update, + silent=True, + updated=updated, + check_diff_exist=check_diff_exist, + ) self._reset_component_type(original_component_type, original_update_all) for m_update in modules_to_update: @@ -891,7 +909,12 @@ def update_linked_components(self, modules_to_update, subworkflows_to_update, up continue original_component_type, original_update_all = self._change_component_type("modules") try: - self.update(m_update, silent=True, updated=updated, check_diff_exist=check_diff_exist) + self.update( + m_update, + silent=True, + updated=updated, + check_diff_exist=check_diff_exist, + ) except LookupError as e: # If the module to be updated is not available, check if there has been a name change if "not found in list of available" in str(e): diff --git a/nf_core/modules/modules_differ.py b/nf_core/modules/modules_differ.py index dc2b163dd4..e79554f2b6 100644 --- a/nf_core/modules/modules_differ.py +++ b/nf_core/modules/modules_differ.py @@ -133,6 +133,7 @@ def write_diff_file( for_git=True, dsp_from_dir=None, dsp_to_dir=None, + limit_output=False, ): """ Writes the diffs of a module to the diff file. @@ -154,6 +155,7 @@ def write_diff_file( adds a/ and b/ prefixes to the file paths dsp_from_dir (str | Path): The 'from' directory displayed in the diff dsp_to_dir (str | Path): The 'to' directory displayed in the diff + limit_output (bool): If true, don't write the diff for files other than main.nf """ if dsp_from_dir is None: dsp_from_dir = from_dir @@ -174,9 +176,22 @@ def write_diff_file( else: fh.write(f"Changes in module '{Path(repo_path, module)}'\n") - for _, (diff_status, diff) in diffs.items(): - if diff_status != ModulesDiffer.DiffEnum.UNCHANGED: + for file, (diff_status, diff) in diffs.items(): + if diff_status == ModulesDiffer.DiffEnum.UNCHANGED: + # The files are identical + fh.write(f"'{Path(dsp_from_dir, file)}' is unchanged\n") + elif diff_status == ModulesDiffer.DiffEnum.CREATED: + # The file was created between the commits + fh.write(f"'{Path(dsp_from_dir, file)}' was created\n") + elif diff_status == ModulesDiffer.DiffEnum.REMOVED: + # The file was removed between the commits + fh.write(f"'{Path(dsp_from_dir, file)}' was removed\n") + elif limit_output and not file.suffix == ".nf": + # Skip printing the diff for files other than main.nf + fh.write(f"Changes in '{Path(module, file)}' but not shown\n") + else: # The file has changed write the diff lines to the file + fh.write(f"Changes in '{Path(module, file)}':\n") for line in diff: fh.write(line) fh.write("\n") @@ -219,7 +234,15 @@ def append_modules_json_diff(diff_path, old_modules_json, new_modules_json, modu @staticmethod def print_diff( - module, repo_path, from_dir, to_dir, current_version=None, new_version=None, dsp_from_dir=None, dsp_to_dir=None + module, + repo_path, + from_dir, + to_dir, + current_version=None, + new_version=None, + dsp_from_dir=None, + dsp_to_dir=None, + limit_output=False, ): """ Prints the diffs between two module versions to the terminal @@ -234,6 +257,7 @@ def print_diff( new_version (str): The version of the module the diff is computed against dsp_from_dir (str | Path): The 'from' directory displayed in the diff dsp_to_dir (str | Path): The 'to' directory displayed in the diff + limit_output (bool): If true, don't print the diff for files other than main.nf """ if dsp_from_dir is None: dsp_from_dir = from_dir @@ -261,6 +285,9 @@ def print_diff( elif diff_status == ModulesDiffer.DiffEnum.REMOVED: # The file was removed between the commits log.info(f"'{Path(dsp_from_dir, file)}' was removed") + elif limit_output and not file.suffix == ".nf": + # Skip printing the diff for files other than main.nf + log.info(f"Changes in '{Path(module, file)}' but not shown") else: # The file has changed log.info(f"Changes in '{Path(module, file)}':") diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index b0a4fa661f..42c633ed23 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -862,7 +862,7 @@ def module_present(self, module_name, repo_url, install_dir): install_dir, {} ) - def get_modules_json(self): + def get_modules_json(self) -> dict: """ Returns a copy of the loaded modules.json @@ -871,7 +871,8 @@ def get_modules_json(self): """ if self.modules_json is None: self.load() - return copy.deepcopy(self.modules_json) + + return copy.deepcopy(self.modules_json) # type: ignore def get_component_version(self, component_type, component_name, repo_url, install_dir): """ diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 9d53bf2017..f6cf5235a4 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -15,6 +15,7 @@ def __init__( remote_url=None, branch=None, no_pull=False, + limit_output=False, ): super().__init__( pipeline_dir, @@ -29,4 +30,5 @@ def __init__( remote_url, branch, no_pull, + limit_output, ) diff --git a/nf_core/subworkflows/update.py b/nf_core/subworkflows/update.py index 3cd4ad59fd..9b6bf16928 100644 --- a/nf_core/subworkflows/update.py +++ b/nf_core/subworkflows/update.py @@ -15,6 +15,7 @@ def __init__( remote_url=None, branch=None, no_pull=False, + limit_output=False, ): super().__init__( pipeline_dir, @@ -29,4 +30,5 @@ def __init__( remote_url, branch, no_pull, + limit_output, ) diff --git a/tests/modules/update.py b/tests/modules/update.py index 81eb85716e..e02b058fbb 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -1,5 +1,4 @@ -import filecmp -import os +import logging import shutil import tempfile from pathlib import Path @@ -24,6 +23,7 @@ GITLAB_URL, OLD_TRIMGALORE_BRANCH, OLD_TRIMGALORE_SHA, + cmp_component, ) @@ -33,13 +33,13 @@ def test_install_and_update(self): update_obj = ModuleUpdate(self.pipeline_dir, show_diff=False) # Copy the module files and check that they are unaffected by the update - tmpdir = tempfile.mkdtemp() - trimgalore_tmpdir = os.path.join(tmpdir, "trimgalore") - trimgalore_path = os.path.join(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "trimgalore") + tmpdir = Path(tempfile.TemporaryDirectory().name) + trimgalore_tmpdir = tmpdir / "trimgalore" + trimgalore_path = Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "trimgalore") shutil.copytree(trimgalore_path, trimgalore_tmpdir) assert update_obj.update("trimgalore") is True - assert cmp_module(trimgalore_tmpdir, trimgalore_path) is True + assert cmp_component(trimgalore_tmpdir, trimgalore_path) is True def test_install_at_hash_and_update(self): @@ -50,13 +50,13 @@ def test_install_at_hash_and_update(self): ) # Copy the module files and check that they are affected by the update - tmpdir = tempfile.mkdtemp() - trimgalore_tmpdir = os.path.join(tmpdir, "trimgalore") - trimgalore_path = os.path.join(self.pipeline_dir, "modules", GITLAB_REPO, "trimgalore") + tmpdir = Path(tempfile.TemporaryDirectory().name) + trimgalore_tmpdir = tmpdir / "trimgalore" + trimgalore_path = Path(self.pipeline_dir, "modules", GITLAB_REPO, "trimgalore") shutil.copytree(trimgalore_path, trimgalore_tmpdir) assert update_obj.update("trimgalore") is True - assert cmp_module(trimgalore_tmpdir, trimgalore_path) is False + assert cmp_component(trimgalore_tmpdir, trimgalore_path) is False # Check that the modules.json is correctly updated mod_json_obj = ModulesJson(self.pipeline_dir) @@ -67,10 +67,36 @@ def test_install_at_hash_and_update(self): assert correct_git_sha == current_git_sha +# Mock questionary answer: do not update module, only show diffs +@mock.patch.object(questionary.Question, "unsafe_ask", return_value=True) +def test_install_at_hash_and_update_limit_output(self, mock_prompt): + """Installs an old version of a module in the pipeline and updates it with limited output reporting""" + self.caplog.set_level(logging.INFO) + assert self.mods_install_old.install("trimgalore") + + update_obj = ModuleUpdate( + self.pipeline_dir, + show_diff=True, + update_deps=True, + remote_url=GITLAB_URL, + branch=OLD_TRIMGALORE_BRANCH, + limit_output=True, + ) + assert update_obj.update("trimgalore") + + # Check changes not shown for non-.nf files + assert "Changes in 'trimgalore/meta.yml' but not shown" in self.caplog.text + # Check changes shown for .nf files + assert "Changes in 'trimgalore/main.nf'" in self.caplog.text + for line in self.caplog.text.split("\n"): + if line.startswith("---"): + assert line.endswith("main.nf") + + def test_install_at_hash_and_update_and_save_diff_to_file(self): """Installs an old version of a module in the pipeline and updates it""" self.mods_install_old.install("trimgalore") - patch_path = os.path.join(self.pipeline_dir, "trimgalore.patch") + patch_path = Path(self.pipeline_dir, "trimgalore.patch") update_obj = ModuleUpdate( self.pipeline_dir, save_diff_fn=patch_path, @@ -80,17 +106,47 @@ def test_install_at_hash_and_update_and_save_diff_to_file(self): ) # Copy the module files and check that they are affected by the update - tmpdir = tempfile.mkdtemp() - trimgalore_tmpdir = os.path.join(tmpdir, "trimgalore") - trimgalore_path = os.path.join(self.pipeline_dir, "modules", GITLAB_REPO, "trimgalore") + tmpdir = Path(tempfile.TemporaryDirectory().name) + trimgalore_tmpdir = tmpdir / "trimgalore" + trimgalore_path = Path(self.pipeline_dir, "modules", GITLAB_REPO, "trimgalore") shutil.copytree(trimgalore_path, trimgalore_tmpdir) assert update_obj.update("trimgalore") is True - assert cmp_module(trimgalore_tmpdir, trimgalore_path) is True + assert cmp_component(trimgalore_tmpdir, trimgalore_path) is True # TODO: Apply the patch to the module +def test_install_at_hash_and_update_and_save_diff_to_file_limit_output(self): + """Installs an old version of a module in the pipeline and updates it""" + # Install old version of trimgalore + self.mods_install_old.install("trimgalore") + patch_path = Path(self.pipeline_dir, "trimgalore.patch") + # Update saving the differences to a patch file and with `limit_output` + update_obj = ModuleUpdate( + self.pipeline_dir, + save_diff_fn=patch_path, + remote_url=GITLAB_URL, + branch=OLD_TRIMGALORE_BRANCH, + limit_output=True, + ) + assert update_obj.update("trimgalore") + + # Check that the patch file was created + assert patch_path.exists(), f"Patch file was not created at {patch_path}" + + # Read the contents of the patch file + with open(patch_path) as fh: + patch_content = fh.read() + # Check changes not shown for non-.nf files + assert "Changes in 'trimgalore/meta.yml' but not shown" in patch_content + # Check changes only shown for main.nf + assert "Changes in 'trimgalore/main.nf'" in patch_content + for line in patch_content: + if line.startswith("---"): + assert line.endswith("main.nf") + + def test_update_all(self): """Updates all modules present in the pipeline""" update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False) @@ -116,7 +172,7 @@ def test_update_with_config_fixed_version(self): update_config = {GITLAB_URL: {GITLAB_REPO: {"trimgalore": OLD_TRIMGALORE_SHA}}} config_fn, tools_config = nf_core.utils.load_tools_config(self.pipeline_dir) tools_config["update"] = update_config - with open(os.path.join(self.pipeline_dir, config_fn), "w") as f: + with open(Path(self.pipeline_dir, config_fn), "w") as f: yaml.dump(tools_config, f) # Update all modules in the pipeline @@ -141,7 +197,7 @@ def test_update_with_config_dont_update(self): update_config = {GITLAB_URL: {GITLAB_REPO: {"trimgalore": False}}} config_fn, tools_config = nf_core.utils.load_tools_config(self.pipeline_dir) tools_config["update"] = update_config - with open(os.path.join(self.pipeline_dir, config_fn), "w") as f: + with open(Path(self.pipeline_dir, config_fn), "w") as f: yaml.dump(tools_config, f) # Update all modules in the pipeline @@ -170,7 +226,7 @@ def test_update_with_config_fix_all(self): update_config = {GITLAB_URL: OLD_TRIMGALORE_SHA} config_fn, tools_config = nf_core.utils.load_tools_config(self.pipeline_dir) tools_config["update"] = update_config - with open(os.path.join(self.pipeline_dir, config_fn), "w") as f: + with open(Path(self.pipeline_dir, config_fn), "w") as f: yaml.dump(tools_config, f) # Update all modules in the pipeline @@ -194,7 +250,7 @@ def test_update_with_config_no_updates(self): update_config = {GITLAB_URL: False} config_fn, tools_config = nf_core.utils.load_tools_config(self.pipeline_dir) tools_config["update"] = update_config - with open(os.path.join(self.pipeline_dir, config_fn), "w") as f: + with open(Path(self.pipeline_dir, config_fn), "w") as f: yaml.dump(tools_config, f) # Update all modules in the pipeline @@ -298,10 +354,8 @@ def test_update_different_branch_mix_modules_branch_test(self): @mock.patch.object(questionary.Question, "unsafe_ask", return_value=False) def test_update_only_show_differences(self, mock_prompt): """Try updating all modules showing differences. - Don't update some of them. + Only show diffs, don't actually save any updated files. Check that the sha in modules.json is not changed.""" - modules_json = ModulesJson(self.pipeline_dir) - update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=True) # Update modules to a fixed old SHA update_old = ModuleUpdate( @@ -309,21 +363,21 @@ def test_update_only_show_differences(self, mock_prompt): ) update_old.update() - tmpdir = tempfile.mkdtemp() - shutil.rmtree(tmpdir) + tmpdir = Path(tempfile.TemporaryDirectory().name) shutil.copytree(Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME), tmpdir) - assert update_obj.update() is True + update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=True) + assert ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=True).update() - mod_json = modules_json.get_modules_json() + mod_json = ModulesJson(self.pipeline_dir).get_modules_json() # Loop through all modules and check that they are NOT updated (according to the modules.json file) # A module that can be updated but shouldn't is fastqc # Module multiqc is already up to date so don't check mod = "fastqc" - correct_git_sha = list(update_obj.modules_repo.get_component_git_log(mod, "modules", depth=1))[0]["git_sha"] + non_updated_git_sha = list(update_obj.modules_repo.get_component_git_log(mod, "modules", depth=1))[0]["git_sha"] current_git_sha = mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME][mod]["git_sha"] - assert correct_git_sha != current_git_sha - assert cmp_module(Path(tmpdir, mod), Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, mod)) is True + assert non_updated_git_sha != current_git_sha + assert cmp_component(Path(tmpdir, mod), Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, mod)) is True # Mock questionary answer: do not update module, only show diffs @@ -339,7 +393,7 @@ def test_update_only_show_differences_when_patch(self, mock_prompt): update_old = ModuleUpdate( self.pipeline_dir, update_all=True, show_diff=False, sha="5e34754d42cd2d5d248ca8673c0a53cdf5624905" ) - update_old.update() + assert update_old.update() # Modify fastqc module, it will have a patch which will be applied during update # We modify fastqc because it's one of the modules that can be updated and there's another one before it (custom/dumpsoftwareversions) @@ -357,7 +411,7 @@ def test_update_only_show_differences_when_patch(self, mock_prompt): patch_obj = ModulePatch(self.pipeline_dir) patch_obj.patch("fastqc") # Check that a patch file with the correct name has been created - assert "fastqc.diff" in set(os.listdir(module_path)) + assert "fastqc.diff" in [f.name for f in module_path.glob("*.diff")] # Update all modules assert update_obj.update() is True @@ -372,12 +426,6 @@ def test_update_only_show_differences_when_patch(self, mock_prompt): assert correct_git_sha != current_git_sha -def cmp_module(dir1, dir2): - """Compare two versions of the same module""" - files = ["main.nf", "meta.yml"] - return all(filecmp.cmp(os.path.join(dir1, f), os.path.join(dir2, f), shallow=False) for f in files) - - def test_update_module_with_extra_config_file(self): """Try updating a module with a config file""" # Install the module diff --git a/tests/subworkflows/update.py b/tests/subworkflows/update.py index 9ddc9bec0c..42ed716b1c 100644 --- a/tests/subworkflows/update.py +++ b/tests/subworkflows/update.py @@ -1,8 +1,10 @@ -import filecmp +import logging import shutil import tempfile from pathlib import Path +from unittest import mock +import questionary import yaml import nf_core.utils @@ -11,7 +13,7 @@ from nf_core.modules.update import ModuleUpdate from nf_core.subworkflows.update import SubworkflowUpdate -from ..utils import OLD_SUBWORKFLOWS_SHA +from ..utils import OLD_SUBWORKFLOWS_SHA, cmp_component def test_install_and_update(self): @@ -20,8 +22,7 @@ def test_install_and_update(self): update_obj = SubworkflowUpdate(self.pipeline_dir, show_diff=False) # Copy the sw files and check that they are unaffected by the update - tmpdir = tempfile.mkdtemp() - shutil.rmtree(tmpdir) + tmpdir = Path(tempfile.TemporaryDirectory().name) sw_path = Path(self.pipeline_dir, "subworkflows", NF_CORE_MODULES_NAME, "bam_stats_samtools") shutil.copytree(sw_path, tmpdir) @@ -36,8 +37,8 @@ def test_install_at_hash_and_update(self): old_mod_json = ModulesJson(self.pipeline_dir).get_modules_json() # Copy the sw files and check that they are affected by the update - tmpdir = tempfile.mkdtemp() - shutil.rmtree(tmpdir) + tmpdir = Path(tempfile.TemporaryDirectory().name) + sw_path = Path(self.pipeline_dir, "subworkflows", NF_CORE_MODULES_NAME, "fastq_align_bowtie2") shutil.copytree(sw_path, tmpdir) @@ -57,6 +58,29 @@ def test_install_at_hash_and_update(self): ) +# Mock questionary answer: update components +@mock.patch.object(questionary.Question, "unsafe_ask", return_value=True) +def test_install_at_hash_and_update_limit_output(self, mock_prompt): + """Installs an old version of a subworkflow in the pipeline and updates it with limit_output=True""" + self.caplog.set_level(logging.INFO) + assert self.subworkflow_install_old.install("fastq_align_bowtie2") + + update_obj = SubworkflowUpdate(self.pipeline_dir, show_diff=True, update_deps=True, limit_output=True) + + assert update_obj.update("fastq_align_bowtie2") + + # Check changes not shown for non-.nf files + assert "Changes in 'fastq_align_bowtie2/meta.yml' but not shown" in self.caplog.text + assert "Changes in 'bam_sort_stats_samtools/meta.yml' but not shown" in self.caplog.text + assert "Changes in 'bam_stats_samtools/meta.yml' but not shown" in self.caplog.text + assert "Changes in 'samtools/flagstat/meta.yml' but not shown" in self.caplog.text + # Check changes only shown for main.nf files + assert "Changes in 'fastq_align_bowtie2/main.nf'" in self.caplog.text + for line in self.caplog.text.split("\n"): + if line.startswith("---"): + assert line.endswith("main.nf") + + def test_install_at_hash_and_update_and_save_diff_to_file(self): """Installs an old version of a sw in the pipeline and updates it. Save differences to a file.""" assert self.subworkflow_install_old.install("fastq_align_bowtie2") @@ -64,8 +88,8 @@ def test_install_at_hash_and_update_and_save_diff_to_file(self): update_obj = SubworkflowUpdate(self.pipeline_dir, save_diff_fn=patch_path, update_deps=True) # Copy the sw files and check that they are affected by the update - tmpdir = tempfile.mkdtemp() - shutil.rmtree(tmpdir) + tmpdir = Path(tempfile.TemporaryDirectory().name) + sw_path = Path(self.pipeline_dir, "subworkflows", NF_CORE_MODULES_NAME, "fastq_align_bowtie2") shutil.copytree(sw_path, tmpdir) @@ -79,6 +103,33 @@ def test_install_at_hash_and_update_and_save_diff_to_file(self): ) +def test_install_at_hash_and_update_and_save_diff_limit_output(self): + """Installs an old version of a sw in the pipeline and updates it. Save differences to a file.""" + # Install old version of fastq_align_bowtie2 + self.subworkflow_install_old.install("fastq_align_bowtie2") + patch_path = Path(self.pipeline_dir, "fastq_align_bowtie2.patch") + # Update saving the differences to a patch file and with `limit_output` + update_obj = SubworkflowUpdate(self.pipeline_dir, save_diff_fn=patch_path, update_deps=True, limit_output=True) + assert update_obj.update("fastq_align_bowtie2") + + # Check that the patch file was created + assert patch_path.exists(), f"Patch file was not created at {patch_path}" + + # Read the contents of the patch file + with open(patch_path) as fh: + content = fh.read() + # Check changes not shown for non-.nf files + assert "Changes in 'fastq_align_bowtie2/meta.yml' but not shown" in content + assert "Changes in 'bam_sort_stats_samtools/meta.yml' but not shown" in content + assert "Changes in 'bam_stats_samtools/meta.yml' but not shown" in content + assert "Changes in 'samtools/flagstat/meta.yml' but not shown" in content + # Check changes only shown for main.nf files + assert "Changes in 'fastq_align_bowtie2/main.nf'" in content + for line in content: + if line.startswith("---"): + assert line.endswith("main.nf") + + def test_update_all(self): """Updates all subworkflows present in the pipeline""" # Install subworkflows fastq_align_bowtie2, bam_sort_stats_samtools, bam_stats_samtools @@ -223,8 +274,7 @@ def test_update_all_linked_components_from_subworkflow(self): old_mod_json = ModulesJson(self.pipeline_dir).get_modules_json() # Copy the sw files and check that they are affected by the update - tmpdir = tempfile.mkdtemp() - shutil.rmtree(tmpdir) + tmpdir = Path(tempfile.TemporaryDirectory().name) subworkflows_path = Path(self.pipeline_dir, "subworkflows", NF_CORE_MODULES_NAME) modules_path = Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME) shutil.copytree(subworkflows_path, Path(tmpdir, "subworkflows")) @@ -270,8 +320,7 @@ def test_update_all_subworkflows_from_module(self): old_mod_json = ModulesJson(self.pipeline_dir).get_modules_json() # Copy the sw files and check that they are affected by the update - tmpdir = tempfile.mkdtemp() - shutil.rmtree(tmpdir) + tmpdir = Path(tempfile.TemporaryDirectory().name) sw_path = Path(self.pipeline_dir, "subworkflows", NF_CORE_MODULES_NAME, "fastq_align_bowtie2") shutil.copytree(sw_path, Path(tmpdir, "fastq_align_bowtie2")) @@ -325,9 +374,3 @@ def test_update_change_of_included_modules(self): assert "ensemblvep" not in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME] assert "ensemblvep/vep" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME] assert Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "ensemblvep/vep").is_dir() - - -def cmp_component(dir1, dir2): - """Compare two versions of the same component""" - files = ["main.nf", "meta.yml"] - return all(filecmp.cmp(Path(dir1, f), Path(dir2, f), shallow=False) for f in files) diff --git a/tests/test_modules.py b/tests/test_modules.py index 107b245663..6e601ce7ad 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -6,6 +6,7 @@ import unittest from pathlib import Path +import pytest import requests_cache import responses import yaml @@ -156,6 +157,10 @@ def test_modulesrepo_class(self): assert modrepo.repo_path == "nf-core" assert modrepo.branch == "master" + @pytest.fixture(autouse=True) + def _use_caplog(self, caplog): + self.caplog = caplog + ############################################ # Test of the individual modules commands. # ############################################ @@ -269,6 +274,8 @@ def test_modulesrepo_class(self): test_install_and_update, test_install_at_hash_and_update, test_install_at_hash_and_update_and_save_diff_to_file, + test_install_at_hash_and_update_and_save_diff_to_file_limit_output, + test_install_at_hash_and_update_limit_output, test_update_all, test_update_different_branch_mix_modules_branch_test, test_update_different_branch_mixed_modules_main, diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 786ba53836..17bc678cad 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -6,6 +6,8 @@ import unittest from pathlib import Path +import pytest + import nf_core.modules import nf_core.pipelines.create.create import nf_core.subworkflows @@ -113,6 +115,10 @@ def tearDown(self): if os.path.exists(self.tmp_dir): shutil.rmtree(self.tmp_dir) + @pytest.fixture(autouse=True) + def _use_caplog(self, caplog): + self.caplog = caplog + ################################################ # Test of the individual subworkflow commands. # ################################################ @@ -175,7 +181,9 @@ def tearDown(self): from .subworkflows.update import ( # type: ignore[misc] test_install_and_update, test_install_at_hash_and_update, + test_install_at_hash_and_update_and_save_diff_limit_output, test_install_at_hash_and_update_and_save_diff_to_file, + test_install_at_hash_and_update_limit_output, test_update_all, test_update_all_linked_components_from_subworkflow, test_update_all_subworkflows_from_module, diff --git a/tests/utils.py b/tests/utils.py index 9a0fd0896f..151655b8f3 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -2,9 +2,10 @@ Helper functions for tests """ +import filecmp import functools -import os import tempfile +from pathlib import Path from typing import Any, Callable, Tuple import responses @@ -93,18 +94,24 @@ def mock_biocontainers_api_calls(rsps: responses.RequestsMock, module: str, vers rsps.get(biocontainers_api_url, json=biocontainers_mock, status=200) -def create_tmp_pipeline() -> Tuple[str, str, str, str]: +def create_tmp_pipeline() -> Tuple[Path, Path, str, Path]: """Create a new Pipeline for testing""" - tmp_dir = tempfile.mkdtemp() - root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - template_dir = os.path.join(root_repo_dir, "nf_core", "pipeline-template") + tmp_dir = Path(tempfile.mkdtemp()) + root_repo_dir = Path(__file__).resolve().parent.parent + template_dir = root_repo_dir / "nf_core" / "pipeline-template" pipeline_name = "mypipeline" - pipeline_dir = os.path.join(tmp_dir, pipeline_name) + pipeline_dir = tmp_dir / pipeline_name nf_core.pipelines.create.create.PipelineCreate( - pipeline_name, "it is mine", "me", no_git=True, outdir=pipeline_dir + pipeline_name, "it is mine", "me", no_git=True, outdir=str(pipeline_dir) ).init_pipeline() # return values to instance variables for later use in test methods return tmp_dir, template_dir, pipeline_name, pipeline_dir + + +def cmp_component(dir1: Path, dir2: Path) -> bool: + """Compare two versions of the same component""" + files = ["main.nf", "meta.yml"] + return all(filecmp.cmp(dir1 / f, dir2 / f, shallow=False) for f in files)