Merge pull request #3047 from Joon-Klaps/add-flag-update-modules

add flagg limit_output for modules update
nf-core · Jul 16, 2024 · 97fd791 · 97fd791
2 parents cb3bb67 + 6325769
commit 97fd791
Show file tree

Hide file tree

Showing 14 changed files with 266 additions and 74 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -46,6 +46,7 @@
 - Update gitpod/workspace-base Docker digest to 0f38224 ([#3048](https://github.com/nf-core/tools/pull/3048))
 - update output_dir for api docs to new website structure ([#3051](https://github.com/nf-core/tools/pull/3051))
 - Update pre-commit hook astral-sh/ruff-pre-commit to v0.5.1 ([#3052](https://github.com/nf-core/tools/pull/3052))
+- Add `--limit-output` argument for modules/subworkflow update ([#3047](https://github.com/nf-core/tools/pull/3047))
 - update api docs to new structure ([#3054](https://github.com/nf-core/tools/pull/3054))
 - Update to pytest v8 and move it to dev dependencies ([#3058](https://github.com/nf-core/tools/pull/3058))
 - handle new jsonschema error type ([#3061](https://github.com/nf-core/tools/pull/3061))

diff --git a/nf_core/__main__.py b/nf_core/__main__.py
@@ -928,6 +928,13 @@ def command_modules_install(ctx, tool, dir, prompt, force, sha):
     default=False,
     help="Prompt for the version of the module",
 )
+@click.option(
+    "--limit-output",
+    "limit_output",
+    is_flag=True,
+    default=False,
+    help="Limit output to only the difference in main.nf",
+)
 @click.option("-s", "--sha", type=str, metavar="<commit sha>", help="Install module at commit SHA")
 @click.option(
     "-a",
@@ -970,11 +977,12 @@ def command_modules_update(
     preview,
     save_diff,
     update_deps,
+    limit_output,
 ):
     """
     Update DSL2 modules within a pipeline.
     """
-    modules_update(ctx, tool, directory, force, prompt, sha, install_all, preview, save_diff, update_deps)
+    modules_update(ctx, tool, directory, force, prompt, sha, install_all, preview, save_diff, update_deps, limit_output)
 
 
 # nf-core modules patch
@@ -1538,6 +1546,14 @@ def command_subworkflows_remove(ctx, dir, subworkflow):
     metavar="<commit sha>",
     help="Install subworkflow at commit SHA",
 )
+@click.option(
+    "-l",
+    "--limit-output",
+    "limit_output",
+    is_flag=True,
+    default=False,
+    help="Limit ouput to only the difference in main.nf",
+)
 @click.option(
     "-a",
     "--all",
@@ -1579,11 +1595,14 @@ def command_subworkflows_update(
     preview,
     save_diff,
     update_deps,
+    limit_output,
 ):
     """
     Update DSL2 subworkflow within a pipeline.
     """
-    subworkflows_update(ctx, subworkflow, dir, force, prompt, sha, install_all, preview, save_diff, update_deps)
+    subworkflows_update(
+        ctx, subworkflow, dir, force, prompt, sha, install_all, preview, save_diff, update_deps, limit_output
+    )
 
 
 ## DEPRECATED commands since v3.0.0

diff --git a/nf_core/commands_modules.py b/nf_core/commands_modules.py
@@ -86,6 +86,7 @@ def modules_update(
     preview,
     save_diff,
     update_deps,
+    limit_output,
 ):
     """
     Update DSL2 modules within a pipeline.
@@ -107,6 +108,7 @@ def modules_update(
             ctx.obj["modules_repo_url"],
             ctx.obj["modules_repo_branch"],
             ctx.obj["modules_repo_no_pull"],
+            limit_output,
         )
         exit_status = module_install.update(tool)
         if not exit_status and install_all:

diff --git a/nf_core/commands_subworkflows.py b/nf_core/commands_subworkflows.py
@@ -229,6 +229,7 @@ def subworkflows_update(
     preview,
     save_diff,
     update_deps,
+    limit_output,
 ):
     """
     Update DSL2 subworkflow within a pipeline.
@@ -250,6 +251,7 @@ def subworkflows_update(
             ctx.obj["modules_repo_url"],
             ctx.obj["modules_repo_branch"],
             ctx.obj["modules_repo_no_pull"],
+            limit_output,
         )
         exit_status = subworkflow_install.update(subworkflow)
         if not exit_status and install_all:

diff --git a/nf_core/components/update.py b/nf_core/components/update.py
@@ -38,6 +38,7 @@ def __init__(
         remote_url=None,
         branch=None,
         no_pull=False,
+        limit_output=False,
     ):
         super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull)
         self.force = force
@@ -46,6 +47,7 @@ def __init__(
         self.update_all = update_all
         self.show_diff = show_diff
         self.save_diff_fn = save_diff_fn
+        self.limit_output = limit_output
         self.update_deps = update_deps
         self.component = None
         self.update_config = None
@@ -75,6 +77,8 @@ def _parameter_checks(self):
 
         if not self.has_valid_directory():
             raise UserWarning("The command was not run in a valid pipeline directory.")
+        if self.limit_output and not (self.save_diff_fn or self.show_diff):
+            raise UserWarning("The '--limit-output' flag can only be used with '--preview' or '--save-diff'.")
 
     def update(self, component=None, silent=False, updated=None, check_diff_exist=True) -> bool:
         """Updates a specified module/subworkflow or all modules/subworkflows in a pipeline.
@@ -124,7 +128,6 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr
         components_info = (
             self.get_all_components_info() if self.update_all else [self.get_single_component_info(component)]
         )
-
         # Save the current state of the modules.json
         old_modules_json = self.modules_json.get_modules_json()
 
@@ -231,6 +234,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr
                             version,
                             dsp_from_dir=component_dir,
                             dsp_to_dir=component_dir,
+                            limit_output=self.limit_output,
                         )
                         updated.append(component)
                     except UserWarning as e:
@@ -271,8 +275,8 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr
                         version,
                         dsp_from_dir=component_dir,
                         dsp_to_dir=component_dir,
+                        limit_output=self.limit_output,
                     )
-
                     # Ask the user if they want to install the component
                     dry_run = not questionary.confirm(
                         f"Update {self.component_type[:-1]} '{component}'?",
@@ -389,6 +393,8 @@ def get_single_component_info(self, component):
 
         sha = self.sha
         config_entry = None
+        if self.update_config is None:
+            raise UserWarning("Could not find '.nf-core.yml' file in pipeline directory")
         if any(
             [
                 entry.count("/") == 1
@@ -829,6 +835,7 @@ def try_apply_patch(
             for_git=False,
             dsp_from_dir=component_relpath,
             dsp_to_dir=component_relpath,
+            limit_output=self.limit_output,
         )
 
         # Move the patched files to the install dir
@@ -875,23 +882,39 @@ def get_components_to_update(self, component):
 
         return modules_to_update, subworkflows_to_update
 
-    def update_linked_components(self, modules_to_update, subworkflows_to_update, updated=None, check_diff_exist=True):
+    def update_linked_components(
+        self,
+        modules_to_update,
+        subworkflows_to_update,
+        updated=None,
+        check_diff_exist=True,
+    ):
         """
         Update modules and subworkflows linked to the component being updated.
         """
         for s_update in subworkflows_to_update:
             if s_update in updated:
                 continue
             original_component_type, original_update_all = self._change_component_type("subworkflows")
-            self.update(s_update, silent=True, updated=updated, check_diff_exist=check_diff_exist)
+            self.update(
+                s_update,
+                silent=True,
+                updated=updated,
+                check_diff_exist=check_diff_exist,
+            )
             self._reset_component_type(original_component_type, original_update_all)
 
         for m_update in modules_to_update:
             if m_update in updated:
                 continue
             original_component_type, original_update_all = self._change_component_type("modules")
             try:
-                self.update(m_update, silent=True, updated=updated, check_diff_exist=check_diff_exist)
+                self.update(
+                    m_update,
+                    silent=True,
+                    updated=updated,
+                    check_diff_exist=check_diff_exist,
+                )
             except LookupError as e:
                 # If the module to be updated is not available, check if there has been a name change
                 if "not found in list of available" in str(e):

diff --git a/nf_core/modules/modules_differ.py b/nf_core/modules/modules_differ.py
@@ -133,6 +133,7 @@ def write_diff_file(
         for_git=True,
         dsp_from_dir=None,
         dsp_to_dir=None,
+        limit_output=False,
     ):
         """
         Writes the diffs of a module to the diff file.
@@ -154,6 +155,7 @@ def write_diff_file(
                             adds a/ and b/ prefixes to the file paths
             dsp_from_dir (str | Path): The 'from' directory displayed in the diff
             dsp_to_dir (str | Path): The 'to' directory displayed in the diff
+            limit_output (bool): If true, don't write the diff for files other than main.nf
         """
         if dsp_from_dir is None:
             dsp_from_dir = from_dir
@@ -174,9 +176,22 @@ def write_diff_file(
             else:
                 fh.write(f"Changes in module '{Path(repo_path, module)}'\n")
 
-            for _, (diff_status, diff) in diffs.items():
-                if diff_status != ModulesDiffer.DiffEnum.UNCHANGED:
+            for file, (diff_status, diff) in diffs.items():
+                if diff_status == ModulesDiffer.DiffEnum.UNCHANGED:
+                    # The files are identical
+                    fh.write(f"'{Path(dsp_from_dir, file)}' is unchanged\n")
+                elif diff_status == ModulesDiffer.DiffEnum.CREATED:
+                    # The file was created between the commits
+                    fh.write(f"'{Path(dsp_from_dir, file)}' was created\n")
+                elif diff_status == ModulesDiffer.DiffEnum.REMOVED:
+                    # The file was removed between the commits
+                    fh.write(f"'{Path(dsp_from_dir, file)}' was removed\n")
+                elif limit_output and not file.suffix == ".nf":
+                    # Skip printing the diff for files other than main.nf
+                    fh.write(f"Changes in '{Path(module, file)}' but not shown\n")
+                else:
                     # The file has changed write the diff lines to the file
+                    fh.write(f"Changes in '{Path(module, file)}':\n")
                     for line in diff:
                         fh.write(line)
                     fh.write("\n")
@@ -219,7 +234,15 @@ def append_modules_json_diff(diff_path, old_modules_json, new_modules_json, modu
 
     @staticmethod
     def print_diff(
-        module, repo_path, from_dir, to_dir, current_version=None, new_version=None, dsp_from_dir=None, dsp_to_dir=None
+        module,
+        repo_path,
+        from_dir,
+        to_dir,
+        current_version=None,
+        new_version=None,
+        dsp_from_dir=None,
+        dsp_to_dir=None,
+        limit_output=False,
     ):
         """
         Prints the diffs between two module versions to the terminal
@@ -234,6 +257,7 @@ def print_diff(
             new_version (str): The version of the module the diff is computed against
             dsp_from_dir (str | Path): The 'from' directory displayed in the diff
             dsp_to_dir (str | Path): The 'to' directory displayed in the diff
+            limit_output (bool): If true, don't print the diff for files other than main.nf
         """
         if dsp_from_dir is None:
             dsp_from_dir = from_dir
@@ -261,6 +285,9 @@ def print_diff(
             elif diff_status == ModulesDiffer.DiffEnum.REMOVED:
                 # The file was removed between the commits
                 log.info(f"'{Path(dsp_from_dir, file)}' was removed")
+            elif limit_output and not file.suffix == ".nf":
+                # Skip printing the diff for files other than main.nf
+                log.info(f"Changes in '{Path(module, file)}' but not shown")
             else:
                 # The file has changed
                 log.info(f"Changes in '{Path(module, file)}':")

diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py
@@ -862,7 +862,7 @@ def module_present(self, module_name, repo_url, install_dir):
             install_dir, {}
         )
 
-    def get_modules_json(self):
+    def get_modules_json(self) -> dict:
         """
         Returns a copy of the loaded modules.json
 
@@ -871,7 +871,8 @@ def get_modules_json(self):
         """
         if self.modules_json is None:
             self.load()
-        return copy.deepcopy(self.modules_json)
+
+        return copy.deepcopy(self.modules_json)  # type: ignore
 
     def get_component_version(self, component_type, component_name, repo_url, install_dir):
         """

diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py
@@ -15,6 +15,7 @@ def __init__(
         remote_url=None,
         branch=None,
         no_pull=False,
+        limit_output=False,
     ):
         super().__init__(
             pipeline_dir,
@@ -29,4 +30,5 @@ def __init__(
             remote_url,
             branch,
             no_pull,
+            limit_output,
         )
diff --git a/nf_core/subworkflows/update.py b/nf_core/subworkflows/update.py
@@ -15,6 +15,7 @@ def __init__(
         remote_url=None,
         branch=None,
         no_pull=False,
+        limit_output=False,
     ):
         super().__init__(
             pipeline_dir,
@@ -29,4 +30,5 @@ def __init__(
             remote_url,
             branch,
             no_pull,
+            limit_output,
         )