From 5c82fec3f82aed6c1c27cc4801769bfb1d36b014 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 27 Aug 2024 11:06:25 +0200 Subject: [PATCH] allow linting a pipeline without nf-core components --- nf_core/commands_pipelines.py | 5 +- nf_core/pipeline-template/main.nf | 6 +- .../pipeline-template/nextflow_schema.json | 2 + .../pipeline-template/workflows/pipeline.nf | 2 +- nf_core/pipelines/create/templatefeatures.yml | 41 ++++--- nf_core/pipelines/lint/__init__.py | 105 ++++++++++-------- 6 files changed, 92 insertions(+), 69 deletions(-) diff --git a/nf_core/commands_pipelines.py b/nf_core/commands_pipelines.py index 4b6fa75f3e..23affb1d27 100644 --- a/nf_core/commands_pipelines.py +++ b/nf_core/commands_pipelines.py @@ -144,9 +144,12 @@ def pipelines_lint( ctx.obj["hide_progress"], ) swf_failed = 0 + module_failed = 0 if subworkflow_lint_obj is not None: swf_failed = len(subworkflow_lint_obj.failed) - if len(lint_obj.failed) + len(module_lint_obj.failed) + swf_failed > 0: + if module_lint_obj is not None: + module_failed = len(module_lint_obj.failed) + if len(lint_obj.failed) + module_failed + swf_failed > 0: sys.exit(1) except AssertionError as e: log.critical(e) diff --git a/nf_core/pipeline-template/main.nf b/nf_core/pipeline-template/main.nf index 100e82417f..e1e5fc7626 100644 --- a/nf_core/pipeline-template/main.nf +++ b/nf_core/pipeline-template/main.nf @@ -19,7 +19,6 @@ include { {{ short_name|upper }} } from './workflows/{{ short_name }}' {%- if modules %} -{%- if modules %} include { PIPELINE_INITIALISATION } from './subworkflows/local/utils_nfcore_{{ short_name }}_pipeline' include { PIPELINE_COMPLETION } from './subworkflows/local/utils_nfcore_{{ short_name }}_pipeline' {%- if igenomes %} @@ -58,10 +57,10 @@ workflow {{ prefix_nodash|upper }}_{{ short_name|upper }} { {{ short_name|upper }} ( samplesheet ) -{%- if multiqc or modules %} +{%- if multiqc %}{%- if modules %} emit: multiqc_report = {{ short_name|upper }}.out.multiqc_report // channel: /path/to/multiqc_report.html -{%- endif %} +{%- endif %}{%- endif %} } /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -74,7 +73,6 @@ workflow { main: {%- if modules %} - {%- modules %} // // SUBWORKFLOW: Run initialisation tasks // diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 758689e89e..7a94e6098d 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -57,6 +57,7 @@ "fa_icon": "fas fa-book", "help_text": "If using a reference genome configured in the pipeline using iGenomes, use this parameter to give the ID for the reference. This is then used to build the full paths for all required reference genome files e.g. `--genome GRCh38`. \n\nSee the [nf-core website docs](https://nf-co.re/usage/reference_genomes) for more details." }, + {%- if modules %} "fasta": { "type": "string", "format": "file-path", @@ -67,6 +68,7 @@ "help_text": "This parameter is *mandatory* if `--genome` is not specified. If you don't have a BWA index available this will be generated for you automatically. Combine with `--save_reference` to save BWA index for future runs.", "fa_icon": "far fa-file-code" }, + {%- endif %} "igenomes_ignore": { "type": "boolean", "description": "Do not load the iGenomes reference config.", diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index dcb6be61ec..ebec9a3397 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -24,9 +24,9 @@ workflow {{ short_name|upper }} { take: ch_samplesheet // channel: samplesheet read in from --input + {%- if modules %} main: - {%- if modules %} ch_versions = Channel.empty() {% if multiqc %}ch_multiqc_files = Channel.empty(){% endif %} diff --git a/nf_core/pipelines/create/templatefeatures.yml b/nf_core/pipelines/create/templatefeatures.yml index 8b92b6e55f..307c418c67 100644 --- a/nf_core/pipelines/create/templatefeatures.yml +++ b/nf_core/pipelines/create/templatefeatures.yml @@ -238,6 +238,33 @@ fastqc: The pipeline will include the FastQC module. nfcore_pipelines: True custom_pipelines: True +modules: + skippable_paths: + - "conf/base.config" + - "conf/modules.config" + - "conf/modules.json" + - "modules" + - "subworkflows" + short_description: "Use nf-core components" + description: "Include all required files to use nf-core modules and subworkflows" + help_text: | + It is *recommended* to use this feature if you want to use modules and subworkflows in your pipeline. + This will add all required files to use nf-core components or any compatible components from private repos by using `nf-core modules` and `nf-core subworkflows` commands. + linting: + nfcore_components: False + modules_json: False + base_config: False + modules_config: False + files_exist: + - "conf/base.config" + - "conf/modules.config" + - "conf/modules.json" + nextflow_config: + - "process.cpus" + - "process.memory" + - "process.time" + nfcore_pipelines: False + custom_pipelines: True changelog: skippable_paths: - "CHANGELOG.md" @@ -285,20 +312,6 @@ email: - ".prettierignore" nfcore_pipelines: False custom_pipelines: True -modules: - skippable_paths: - - "conf/base.config" - - "conf/modules.config" - - "conf/modules.json" - - "modules" - - "subworkflows" - short_description: "Use nf-core components" - description: "Include all required files to use nf-core modules and subworkflows" - help_text: | - It is *recommended* to use this feature if you want to use modules and subworkflows in your pipeline. - This will add all required files to use nf-core components or any compatible components from private repos by using `nf-core modules` and `nf-core subworkflows` commands. - nfcore_pipelines: False - custom_pipelines: True adaptivecard: skippable_paths: - "assets/adaptivecard.json" diff --git a/nf_core/pipelines/lint/__init__.py b/nf_core/pipelines/lint/__init__.py index ed833d3219..3835aac452 100644 --- a/nf_core/pipelines/lint/__init__.py +++ b/nf_core/pipelines/lint/__init__.py @@ -149,6 +149,7 @@ def _get_all_lint_tests(release_mode): "base_config", "modules_config", "nfcore_yml", + "nfcore_components", ] + (["version_consistency"] if release_mode else []) def _load(self) -> bool: @@ -543,7 +544,7 @@ def run_linting( md_fn=None, json_fn=None, hide_progress: bool = False, -) -> Tuple[PipelineLint, ComponentLint, Union[ComponentLint, None]]: +) -> Tuple[PipelineLint, Union[ComponentLint, None], Union[ComponentLint, None]]: """Runs all nf-core linting checks on a given Nextflow pipeline project in either `release` mode or `normal` mode (default). Returns an object of type :class:`PipelineLint` after finished. @@ -588,41 +589,45 @@ def run_linting( lint_obj._load_lint_config() lint_obj.load_pipeline_config() - # Create the modules lint object - module_lint_obj = nf_core.modules.lint.ModuleLint(pipeline_dir, hide_progress=hide_progress) - # Create the subworkflows lint object - try: - subworkflow_lint_obj = nf_core.subworkflows.lint.SubworkflowLint(pipeline_dir, hide_progress=hide_progress) - except LookupError: - subworkflow_lint_obj = None - - # Verify that the pipeline is correctly configured and has a modules.json file - module_lint_obj.has_valid_directory() - module_lint_obj.has_modules_file() - # Run only the tests we want - if key: - # Select only the module lint tests - module_lint_tests = list( - set(key).intersection(set(nf_core.modules.lint.ModuleLint.get_all_module_lint_tests(is_pipeline=True))) - ) - # Select only the subworkflow lint tests - subworkflow_lint_tests = list( - set(key).intersection( - set(nf_core.subworkflows.lint.SubworkflowLint.get_all_subworkflow_lint_tests(is_pipeline=True)) + if "nfcore_components" in lint_obj.lint_config and not lint_obj.lint_config: + # Create the modules lint object + module_lint_obj = nf_core.modules.lint.ModuleLint(pipeline_dir, hide_progress=hide_progress) + # Create the subworkflows lint object + try: + subworkflow_lint_obj = nf_core.subworkflows.lint.SubworkflowLint(pipeline_dir, hide_progress=hide_progress) + except LookupError: + subworkflow_lint_obj = None + + # Verify that the pipeline is correctly configured and has a modules.json file + module_lint_obj.has_valid_directory() + module_lint_obj.has_modules_file() + # Run only the tests we want + if key: + # Select only the module lint tests + module_lint_tests = list( + set(key).intersection(set(nf_core.modules.lint.ModuleLint.get_all_module_lint_tests(is_pipeline=True))) ) - ) + # Select only the subworkflow lint tests + subworkflow_lint_tests = list( + set(key).intersection( + set(nf_core.subworkflows.lint.SubworkflowLint.get_all_subworkflow_lint_tests(is_pipeline=True)) + ) + ) + else: + # If no key is supplied, run the default modules tests + module_lint_tests = list(("module_changes", "module_version")) + subworkflow_lint_tests = list(("subworkflow_changes", "subworkflow_version")) + module_lint_obj.filter_tests_by_key(module_lint_tests) + if subworkflow_lint_obj is not None: + subworkflow_lint_obj.filter_tests_by_key(subworkflow_lint_tests) + + # Set up files for component linting test + module_lint_obj.set_up_pipeline_files() + if subworkflow_lint_obj is not None: + subworkflow_lint_obj.set_up_pipeline_files() else: - # If no key is supplied, run the default modules tests - module_lint_tests = list(("module_changes", "module_version")) - subworkflow_lint_tests = list(("subworkflow_changes", "subworkflow_version")) - module_lint_obj.filter_tests_by_key(module_lint_tests) - if subworkflow_lint_obj is not None: - subworkflow_lint_obj.filter_tests_by_key(subworkflow_lint_tests) - - # Set up files for component linting test - module_lint_obj.set_up_pipeline_files() - if subworkflow_lint_obj is not None: - subworkflow_lint_obj.set_up_pipeline_files() + module_lint_obj = None + subworkflow_lint_obj = None # Run the pipeline linting tests try: @@ -632,24 +637,26 @@ def run_linting( log.info("Stopping tests...") return lint_obj, module_lint_obj, subworkflow_lint_obj - # Run the module lint tests - if len(module_lint_obj.all_local_components) > 0: - module_lint_obj.lint_modules(module_lint_obj.all_local_components, local=True) - if len(module_lint_obj.all_remote_components) > 0: - module_lint_obj.lint_modules(module_lint_obj.all_remote_components, local=False) - # Run the subworkflows lint tests - if subworkflow_lint_obj is not None: - if len(subworkflow_lint_obj.all_local_components) > 0: - subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_local_components, local=True) - if len(subworkflow_lint_obj.all_remote_components) > 0: - subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_remote_components, local=False) + if module_lint_obj is not None and subworkflow_lint_obj is not None: + # Run the module lint tests + if len(module_lint_obj.all_local_components) > 0: + module_lint_obj.lint_modules(module_lint_obj.all_local_components, local=True) + if len(module_lint_obj.all_remote_components) > 0: + module_lint_obj.lint_modules(module_lint_obj.all_remote_components, local=False) + # Run the subworkflows lint tests + if subworkflow_lint_obj is not None: + if len(subworkflow_lint_obj.all_local_components) > 0: + subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_local_components, local=True) + if len(subworkflow_lint_obj.all_remote_components) > 0: + subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_remote_components, local=False) # Print the results lint_obj._print_results(show_passed) - module_lint_obj._print_results(show_passed, sort_by=sort_by) - if subworkflow_lint_obj is not None: - subworkflow_lint_obj._print_results(show_passed, sort_by=sort_by) - nf_core.pipelines.lint_utils.print_joint_summary(lint_obj, module_lint_obj, subworkflow_lint_obj) + if module_lint_obj is not None and subworkflow_lint_obj is not None: + module_lint_obj._print_results(show_passed, sort_by=sort_by) + if subworkflow_lint_obj is not None: + subworkflow_lint_obj._print_results(show_passed, sort_by=sort_by) + nf_core.pipelines.lint_utils.print_joint_summary(lint_obj, module_lint_obj, subworkflow_lint_obj) nf_core.pipelines.lint_utils.print_fixes(lint_obj) # Save results to Markdown file