diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 1ce9902a3..af057b67f 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -170,7 +170,7 @@ def lint( self.lint_modules(local_modules, registry=registry, local=True, fix_version=fix_version) # Lint nf-core modules - if len(remote_modules) > 0: + if not local and len(remote_modules) > 0: self.lint_modules(remote_modules, registry=registry, local=False, fix_version=fix_version) if print_results: @@ -248,9 +248,8 @@ def lint_module( elif test_name in ["meta_yml", "environment_yml"]: # Allow files to be missing for local getattr(self, test_name)(mod, allow_missing=True) - """ - self.main_nf(mod, fix_version, self.registry, progress_bar) - """ + else: + getattr(self, test_name)(mod) self.passed += [LintResult(mod, *m) for m in mod.passed] warned = [LintResult(mod, *m) for m in (mod.warned + mod.failed)] diff --git a/nf_core/subworkflows/lint/__init__.py b/nf_core/subworkflows/lint/__init__.py index cedae62f1..ee46f3f3b 100644 --- a/nf_core/subworkflows/lint/__init__.py +++ b/nf_core/subworkflows/lint/__init__.py @@ -152,7 +152,7 @@ def lint( self.lint_subworkflows(local_subworkflows, registry=registry, local=True) # Lint nf-core subworkflows - if len(remote_subworkflows) > 0: + if not local and len(remote_subworkflows) > 0: self.lint_subworkflows(remote_subworkflows, registry=registry, local=False) if print_results: @@ -208,6 +208,7 @@ def lint_subworkflow(self, swf, progress_bar, registry, local=False): # Only check the main script in case of a local subworkflow if local: self.main_nf(swf) + self.meta_yml(swf, allow_missing=True) self.passed += [LintResult(swf, *s) for s in swf.passed] warned = [LintResult(swf, *m) for m in (swf.warned + swf.failed)] if not self.fail_warned: diff --git a/nf_core/subworkflows/lint/meta_yml.py b/nf_core/subworkflows/lint/meta_yml.py index be282bc45..262c80f94 100644 --- a/nf_core/subworkflows/lint/meta_yml.py +++ b/nf_core/subworkflows/lint/meta_yml.py @@ -6,11 +6,12 @@ import yaml import nf_core.components.components_utils +from nf_core.components.lint import LintExceptionError log = logging.getLogger(__name__) -def meta_yml(subworkflow_lint_object, subworkflow): +def meta_yml(subworkflow_lint_object, subworkflow, allow_missing: bool = False): """ Lint a ``meta.yml`` file @@ -28,6 +29,18 @@ def meta_yml(subworkflow_lint_object, subworkflow): """ # Read the meta.yml file + if subworkflow.meta_yml is None: + if allow_missing: + subworkflow.warned.append( + ( + "meta_yml_exists", + "Subworkflow `meta.yml` does not exist", + Path(subworkflow.component_dir, "meta.yml"), + ) + ) + return + raise LintExceptionError("Subworkflow does not have a `meta.yml` file") + try: with open(subworkflow.meta_yml) as fh: meta_yaml = yaml.safe_load(fh) @@ -49,7 +62,7 @@ def meta_yml(subworkflow_lint_object, subworkflow): if len(e.path) > 0: hint = f"\nCheck the entry for `{e.path[0]}`." if e.message.startswith("None is not of type 'object'") and len(e.path) > 2: - hint = f"\nCheck that the child entries of {e.path[0]+'.'+e.path[2]} are indented correctly." + hint = f"\nCheck that the child entries of {e.path[0]}.{e.path[2]} are indented correctly." subworkflow.failed.append( ( "meta_yml_valid", @@ -96,10 +109,9 @@ def meta_yml(subworkflow_lint_object, subworkflow): ) # confirm that all included components in ``main.nf`` are specified in ``meta.yml`` - included_components = nf_core.components.components_utils.get_components_to_install(subworkflow.component_dir) - included_components = ( - included_components[0] + included_components[1] - ) # join included modules and included subworkflows in a single list + included_components_ = nf_core.components.components_utils.get_components_to_install(subworkflow.component_dir) + included_components = included_components_[0] + included_components_[1] + # join included modules and included subworkflows in a single list if "components" in meta_yaml: meta_components = [x for x in meta_yaml["components"]] for component in set(included_components): diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 5242c28eb..d216f9bed 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -9,12 +9,13 @@ import yaml +from nf_core.components.lint import LintExceptionError from nf_core.components.nfcore_component import NFCoreComponent log = logging.getLogger(__name__) -def subworkflow_tests(_, subworkflow: NFCoreComponent): +def subworkflow_tests(_, subworkflow: NFCoreComponent, allow_missing: bool = False): """ Lint the tests of a subworkflow in ``nf-core/modules`` @@ -23,8 +24,30 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): Additionally, checks that all included components in test ``main.nf`` are specified in ``test.yml`` """ - if subworkflow.nftest_testdir is None or subworkflow.nftest_main_nf is None: - raise ValueError() + if subworkflow.nftest_testdir is None: + if allow_missing: + subworkflow.warned.append( + ( + "test_dir_exists", + "nf-test directory is missing", + Path(subworkflow.component_dir, "tests"), + ) + ) + return + raise LintExceptionError("Module does not have a `tests` dir") + + if subworkflow.nftest_main_nf is None: + if allow_missing: + subworkflow.warned.append( + ( + "test_main_nf_exists", + "test `main.nf.test` does not exist", + Path(subworkflow.component_dir, "tests", "main.nf.test"), + ) + ) + return + raise LintExceptionError("Subworkflow does not have a `tests` dir") + repo_dir = subworkflow.component_dir.parts[ : subworkflow.component_dir.parts.index(subworkflow.component_name.split("/")[0]) ][-1] diff --git a/tests/modules/test_lint.py b/tests/modules/test_lint.py index 537280798..37f992a7c 100644 --- a/tests/modules/test_lint.py +++ b/tests/modules/test_lint.py @@ -1,4 +1,5 @@ import json +import shutil from pathlib import Path from typing import Union @@ -158,7 +159,7 @@ ] -class TestModulesCreate(TestModules): +class TestModulesLint(TestModules): def _setup_patch(self, pipeline_dir: Union[str, Path], modify_module: bool): install_obj = nf_core.modules.install.ModuleInstall( pipeline_dir, @@ -760,6 +761,46 @@ def test_modules_empty_file_in_stub_snapshot(self): with open(snap_file, "w") as fh: fh.write(content) + def test_modules_lint_local(self): + assert self.mods_install.install("trimgalore") + installed = Path(self.pipeline_dir, "modules", "nf-core", "trimgalore") + local = Path(self.pipeline_dir, "modules", "local", "trimgalore") + shutil.move(installed, local) + module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) + module_lint.lint(print_results=False, local=True, all_modules=True) + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + + def test_modules_lint_local_missing_files(self): + assert self.mods_install.install("trimgalore") + installed = Path(self.pipeline_dir, "modules", "nf-core", "trimgalore") + local = Path(self.pipeline_dir, "modules", "local", "trimgalore") + shutil.move(installed, local) + Path(self.pipeline_dir, "modules", "local", "trimgalore", "environment.yml").unlink() + Path(self.pipeline_dir, "modules", "local", "trimgalore", "meta.yml").unlink() + module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) + module_lint.lint(print_results=False, local=True, all_modules=True) + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + warnings = [x.message for x in module_lint.warned] + assert "Module's `environment.yml` does not exist" in warnings + assert "Module `meta.yml` does not exist" in warnings + + def test_modules_lint_local_old_format(self): + assert self.mods_install.install("trimgalore") + installed = Path(self.pipeline_dir, "modules", "nf-core", "trimgalore", "main.nf") + Path(self.pipeline_dir, "modules", "local").mkdir() + local = Path(self.pipeline_dir, "modules", "local", "trimgalore.nf") + shutil.copy(installed, local) + self.mods_remove.remove("trimgalore", force=True) + module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) + module_lint.lint(print_results=False, local=True, all_modules=True) + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + # A skeleton object with the passed/warned/failed list attrs # Use this in place of a ModuleLint object to test behaviour of diff --git a/tests/subworkflows/test_lint.py b/tests/subworkflows/test_lint.py index d94b55b3d..49c671c08 100644 --- a/tests/subworkflows/test_lint.py +++ b/tests/subworkflows/test_lint.py @@ -31,7 +31,6 @@ def test_subworkflows_lint_new_subworkflow(self): subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.nfcore_modules) subworkflow_lint.lint(print_results=True, all_subworkflows=True) assert len(subworkflow_lint.failed) == 0 - assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0 @@ -397,3 +396,41 @@ def test_subworkflows_empty_file_in_stub_snapshot(self): # reset the file with open(snap_file, "w") as fh: fh.write(content) + + def test_subworkflows_lint_local(self): + assert self.subworkflow_install.install("fastq_align_bowtie2") + installed = Path(self.pipeline_dir, "subworkflows", "nf-core", "fastq_align_bowtie2") + local = Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2") + shutil.move(installed, local) + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) + subworkflow_lint.lint(print_results=False, local=True, all_subworkflows=True) + assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + + def test_subworkflows_lint_local_missing_files(self): + assert self.subworkflow_install.install("fastq_align_bowtie2") + installed = Path(self.pipeline_dir, "subworkflows", "nf-core", "fastq_align_bowtie2") + local = Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2") + shutil.move(installed, local) + Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2", "meta.yml").unlink() + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) + subworkflow_lint.lint(print_results=False, local=True, all_subworkflows=True) + assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + warnings = [x.message for x in subworkflow_lint.warned] + assert "Subworkflow `meta.yml` does not exist" in warnings + + def test_subworkflows_lint_local_old_format(self): + assert self.subworkflow_install.install("fastq_align_bowtie2") + installed = Path(self.pipeline_dir, "subworkflows", "nf-core", "fastq_align_bowtie2", "main.nf") + Path(self.pipeline_dir, "subworkflows", "local").mkdir(exist_ok=True) + local = Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2.nf") + shutil.copy(installed, local) + self.subworkflow_remove.remove("fastq_align_bowtie2", force=True) + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) + subworkflow_lint.lint(print_results=False, local=True, all_subworkflows=True) + assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0