allow mixed str and dict in lint config

CHANGELOG.md

@@ -12,6 +12,8 @@
 
 ### Linting
 
+- allow mixed str and dict in lint config ([#3228](https://github.com/nf-core/tools/pull/3228))
+
 ### Modules
 
 ### Subworkflows

nf_core/components/lint/__init__.py

@@ -22,7 +22,7 @@
 from nf_core.components.nfcore_component import NFCoreComponent
 from nf_core.modules.modules_json import ModulesJson
 from nf_core.pipelines.lint_utils import console
-from nf_core.utils import LintConfigType
+from nf_core.utils import NFCoreYamlLintConfig
 from nf_core.utils import plural_s as _s
 
 log = logging.getLogger(__name__)
@@ -80,7 +80,7 @@ def __init__(
         self.failed: List[LintResult] = []
         self.all_local_components: List[NFCoreComponent] = []
 
-        self.lint_config: Optional[LintConfigType] = None
+        self.lint_config: Optional[NFCoreYamlLintConfig] = None
         self.modules_json: Optional[ModulesJson] = None
 
         if self.component_type == "modules":

nf_core/pipelines/create/create.py

@@ -8,7 +8,7 @@
 import re
 import shutil
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple, Union, cast
+from typing import Dict, List, Optional, Tuple, Union
 
 import git
 import git.config
@@ -21,7 +21,7 @@
 from nf_core.pipelines.create.utils import CreateConfig, features_yml_path, load_features_yaml
 from nf_core.pipelines.create_logo import create_logo
 from nf_core.pipelines.lint_utils import run_prettier_on_file
-from nf_core.utils import LintConfigType, NFCoreTemplateConfig
+from nf_core.utils import NFCoreTemplateConfig, NFCoreYamlLintConfig
 
 log = logging.getLogger(__name__)
 
@@ -395,7 +395,7 @@ def fix_linting(self):
         # Add the lint content to the preexisting nf-core config
         config_fn, nf_core_yml = nf_core.utils.load_tools_config(self.outdir)
         if config_fn is not None and nf_core_yml is not None:
-            nf_core_yml.lint = cast(LintConfigType, lint_config)
+            nf_core_yml.lint = NFCoreYamlLintConfig(**lint_config)
             with open(self.outdir / config_fn, "w") as fh:
                 yaml.dump(nf_core_yml.model_dump(), fh, default_flow_style=False, sort_keys=False)
 

nf_core/pipelines/lint/__init__.py

@@ -27,8 +27,8 @@
 from nf_core import __version__
 from nf_core.components.lint import ComponentLint
 from nf_core.pipelines.lint_utils import console
+from nf_core.utils import NFCoreYamlConfig, NFCoreYamlLintConfig, strip_ansi_codes
 from nf_core.utils import plural_s as _s
-from nf_core.utils import strip_ansi_codes
 
 from .actions_awsfulltest import actions_awsfulltest
 from .actions_awstest import actions_awstest
@@ -112,7 +112,7 @@ def __init__(
         # Initialise the parent object
         super().__init__(wf_path)
 
-        self.lint_config = {}
+        self.lint_config: Optional[NFCoreYamlLintConfig] = None
         self.release_mode = release_mode
         self.fail_ignored = fail_ignored
         self.fail_warned = fail_warned
@@ -173,12 +173,11 @@ def _load_lint_config(self) -> bool:
         Add parsed config to the `self.lint_config` class attribute.
         """
         _, tools_config = nf_core.utils.load_tools_config(self.wf_path)
-        self.lint_config = getattr(tools_config, "lint", {}) or {}
+        self.lint_config = getattr(tools_config, "lint", None) or None
         is_correct = True
-
         # Check if we have any keys that don't match lint test names
         if self.lint_config is not None:
-            for k in self.lint_config:
+            for k, v in self.lint_config:
                 if k != "nfcore_components" and k not in self.lint_tests:
                     # nfcore_components is an exception to allow custom pipelines without nf-core components
                     log.warning(f"Found unrecognised test name '{k}' in pipeline lint config")
@@ -594,7 +593,7 @@ def run_linting(
     lint_obj._load_lint_config()
     lint_obj.load_pipeline_config()
 
-    if "nfcore_components" in lint_obj.lint_config and not lint_obj.lint_config["nfcore_components"]:
+    if lint_obj.lint_config and lint_obj.lint_config["nfcore_components"] is False:
         module_lint_obj = None
         subworkflow_lint_obj = None
     else:
@@ -679,5 +678,4 @@ def run_linting(
     if len(lint_obj.failed) > 0:
         if release_mode:
             log.info("Reminder: Lint tests were run in --release mode.")
-
     return lint_obj, module_lint_obj, subworkflow_lint_obj

nf_core/pipelines/lint/multiqc_config.py

@@ -31,6 +31,15 @@ def multiqc_config(self) -> Dict[str, List[str]]:
             lint:
                 multiqc_config: False
 
+        To disable this test only for specific sections, you can specify a list of section names.
+        For example:
+
+        .. code-block:: yaml
+        lint:
+                multiqc_config:
+                    - report_section_order
+                    - report_comment
+
     """
 
     passed: List[str] = []

nf_core/pipelines/lint/nfcore_yml.py

@@ -1,7 +1,8 @@
-import re
 from pathlib import Path
 from typing import Dict, List
 
+from ruamel.yaml import YAML
+
 from nf_core import __version__
 
 REPOSITORY_TYPES = ["pipeline", "modules"]
@@ -26,21 +27,23 @@ def nfcore_yml(self) -> Dict[str, List[str]]:
     failed: List[str] = []
     ignored: List[str] = []
 
+    yaml = YAML()
+
     # Remove field that should be ignored according to the linting config
     ignore_configs = self.lint_config.get(".nf-core", []) if self.lint_config is not None else []
-    try:
-        with open(Path(self.wf_path, ".nf-core.yml")) as fh:
-            content = fh.read()
-    except FileNotFoundError:
-        with open(Path(self.wf_path, ".nf-core.yaml")) as fh:
-            content = fh.read()
+    for ext in (".yml", ".yaml"):
+        try:
+            nf_core_yml = yaml.load(Path(self.wf_path) / f".nf-core{ext}")
+            break
+        except FileNotFoundError:
+            continue
+    else:
+        raise FileNotFoundError("No `.nf-core.yml` file found.")
 
     if "repository_type" not in ignore_configs:
         # Check that the repository type is set in the .nf-core.yml
-        repo_type_re = r"repository_type: (.+)"
-        match = re.search(repo_type_re, content)
-        if match:
-            repo_type = match.group(1)
+        if "repository_type" in nf_core_yml:
+            repo_type = nf_core_yml["repository_type"]
             if repo_type not in REPOSITORY_TYPES:
                 failed.append(
                     f"Repository type in `.nf-core.yml` is not valid. "
@@ -55,10 +58,8 @@ def nfcore_yml(self) -> Dict[str, List[str]]:
 
     if "nf_core_version" not in ignore_configs:
         # Check that the nf-core version is set in the .nf-core.yml
-        nf_core_version_re = r"nf_core_version: (.+)"
-        match = re.search(nf_core_version_re, content)
-        if match:
-            nf_core_version = match.group(1).strip('"')
+        if "nf_core_version" in nf_core_yml:
+            nf_core_version = nf_core_yml["nf_core_version"]
             if nf_core_version != __version__ and "dev" not in nf_core_version:
                 warned.append(
                     f"nf-core version in `.nf-core.yml` is not set to the latest version. "

nf_core/pipelines/lint/template_strings.py

@@ -39,8 +39,8 @@ def template_strings(self):
     ignored = []
     # Files that should be ignored according to the linting config
     ignore_files = self.lint_config.get("template_strings", []) if self.lint_config is not None else []
-    files = self.list_files()
 
+    files = self.list_files()
     # Loop through files, searching for string
     num_matches = 0
     for fn in files:

nf_core/utils.py

@@ -1088,7 +1088,64 @@ def get(self, item: str, default: Any = None) -> Any:
         return getattr(self, item, default)
 
 
-LintConfigType = Optional[Dict[str, Union[List[str], List[Dict[str, List[str]]], bool]]]
+class NFCoreYamlLintConfig(BaseModel):
+    """
+    schema for linting config in `.nf-core.yml` should cover:
+
+    .. code-block:: yaml
+        files_unchanged:
+            - .github/workflows/branch.yml
+        modules_config: False
+        modules_config:
+                - fastqc
+        # merge_markers: False
+        merge_markers:
+                - docs/my_pdf.pdf
+        nextflow_config: False
+        nextflow_config:
+            - manifest.name
+            - config_defaults:
+                - params.annotation_db
+                - params.multiqc_comment_headers
+                - params.custom_table_headers
+        # multiqc_config: False
+        multiqc_config:
+            - report_section_order
+            - report_comment
+        files_exist:
+            - .github/CONTRIBUTING.md
+            - CITATIONS.md
+        template_strings: False
+        template_strings:
+                - docs/my_pdf.pdf
+        nfcore_components: False
+    """
+
+    files_unchanged: Union[bool, List[str]] = []
+    """ List of files that should not be changed """
+    modules_config: Optional[Union[bool, List[str]]] = []
+    """ List of modules that should not be changed """
+    merge_markers: Optional[Union[bool, List[str]]] = []
+    """ List of files that should not contain merge markers """
+    nextflow_config: Optional[Union[bool, List[Union[str, Dict[str, List[str]]]]]] = []
+    """ List of Nextflow config files that should not be changed """
+    multiqc_config: Union[bool, List[str]] = []
+    """ List of MultiQC config options that be changed """
+    files_exist: Union[bool, List[str]] = []
+    """ List of files that can not exist """
+    template_strings: Optional[Union[bool, List[str]]] = []
+    """ List of files that can contain template strings """
+    nfcore_components: Optional[bool] = None
+    """ Include all required files to use nf-core modules and subworkflows """
+
+    def __getitem__(self, item: str) -> Any:
+        return getattr(self, item)
+
+    def get(self, item: str, default: Any = None) -> Any:
+        return getattr(self, item, default)
+
+    def __setitem__(self, item: str, value: Any) -> None:
+        setattr(self, item, value)
 
 
 class NFCoreYamlConfig(BaseModel):
@@ -1100,7 +1157,7 @@ class NFCoreYamlConfig(BaseModel):
     """ Version of nf-core/tools used to create/update the pipeline"""
     org_path: Optional[str] = None
     """ Path to the organisation's modules repository (used for modules repo_type only) """
-    lint: Optional[LintConfigType] = None
+    lint: Optional[NFCoreYamlLintConfig] = None
     """ Pipeline linting configuration, see https://nf-co.re/docs/nf-core-tools/pipelines/lint#linting-config for examples and documentation """
     template: Optional[NFCoreTemplateConfig] = None
     """ Pipeline template configuration """
@@ -1115,6 +1172,9 @@ def __getitem__(self, item: str) -> Any:
     def get(self, item: str, default: Any = None) -> Any:
         return getattr(self, item, default)
 
+    def __setitem__(self, item: str, value: Any) -> None:
+        setattr(self, item, value)
+
 
 def load_tools_config(directory: Union[str, Path] = ".") -> Tuple[Optional[Path], Optional[NFCoreYamlConfig]]:
     """
@@ -1153,7 +1213,7 @@ def load_tools_config(directory: Union[str, Path] = ".") -> Tuple[Optional[Path]
     except ValidationError as e:
         error_message = f"Config file '{config_fn}' is invalid"
         for error in e.errors():
-            error_message += f"\n{error['loc'][0]}: {error['msg']}"
+            error_message += f"\n{error['loc'][0]}: {error['msg']}\ninput: {error['input']}"
         raise AssertionError(error_message)
 
     wf_config = fetch_wf_config(Path(directory))