From ee2241a4e763239c3d45f343f7d5e930c8f1561f Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 25 Jun 2024 14:52:57 +0200 Subject: [PATCH 01/96] Replace check_max with resourceLimits --- CHANGELOG.md | 1 + nf_core/pipeline-template/conf/base.config | 40 +++++++++++++--------- nf_core/pipeline-template/nextflow.config | 33 ------------------ 3 files changed, 24 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 008c755848..c65b999a5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ - Don't cache pip in `linting.yml` ([#2961](https://github.com/nf-core/tools/pull/2961)) - Lint pipelines with the nf-core template version and post comment if it is outdated ([#2978](https://github.com/nf-core/tools/pull/2978)) +- Replaces the old custom `check_max()` function with the Nextflow native `resourceLimits` directive ([]()) ### General diff --git a/nf_core/pipeline-template/conf/base.config b/nf_core/pipeline-template/conf/base.config index 9c62bf0634..72a912fe47 100644 --- a/nf_core/pipeline-template/conf/base.config +++ b/nf_core/pipeline-template/conf/base.config @@ -11,9 +11,15 @@ process { // TODO nf-core: Check the defaults for all processes - cpus = { check_max( 1 * task.attempt, 'cpus' ) } - memory = { check_max( 6.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } + cpus = { 1 * task.attempt } + memory = { 6.GB * task.attempt } + time = { 4.h * task.attempt } + + resourceLimits = [ + cpus: params.max_cpus, + memory: params.max_memory, + time: params.max_time + ] errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' } maxRetries = 1 @@ -27,30 +33,30 @@ process { // TODO nf-core: Customise requirements for specific processes. // See https://www.nextflow.io/docs/latest/config.html#config-process-selectors withLabel:process_single { - cpus = { check_max( 1 , 'cpus' ) } - memory = { check_max( 6.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } + cpus = { 1 } + memory = { 6.GB * task.attempt } + time = { 4.h * task.attempt } } withLabel:process_low { - cpus = { check_max( 2 * task.attempt, 'cpus' ) } - memory = { check_max( 12.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } + cpus = { 2 * task.attempt } + memory = { 12.GB * task.attempt } + time = { 4.h * task.attempt } } withLabel:process_medium { - cpus = { check_max( 6 * task.attempt, 'cpus' ) } - memory = { check_max( 36.GB * task.attempt, 'memory' ) } - time = { check_max( 8.h * task.attempt, 'time' ) } + cpus = { 6 * task.attempt } + memory = { 36.GB * task.attempt } + time = { 8.h * task.attempt } } withLabel:process_high { - cpus = { check_max( 12 * task.attempt, 'cpus' ) } - memory = { check_max( 72.GB * task.attempt, 'memory' ) } - time = { check_max( 16.h * task.attempt, 'time' ) } + cpus = { 12 * task.attempt } + memory = { 72.GB * task.attempt } + time = { 16.h * task.attempt } } withLabel:process_long { - time = { check_max( 20.h * task.attempt, 'time' ) } + time = { 20.h * task.attempt } } withLabel:process_high_memory { - memory = { check_max( 200.GB * task.attempt, 'memory' ) } + memory = { 200.GB * task.attempt } } withLabel:error_ignore { errorStrategy = 'ignore' diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 2e6a56b001..0316c2e0c5 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -265,36 +265,3 @@ manifest { // Load modules.config for DSL2 module specific options includeConfig 'conf/modules.config' - -// Function to ensure that resource requirements don't go beyond -// a maximum limit -def check_max(obj, type) { - if (type == 'memory') { - try { - if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1) - return params.max_memory as nextflow.util.MemoryUnit - else - return obj - } catch (all) { - println " ### ERROR ### Max memory '${params.max_memory}' is not valid! Using default value: $obj" - return obj - } - } else if (type == 'time') { - try { - if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1) - return params.max_time as nextflow.util.Duration - else - return obj - } catch (all) { - println " ### ERROR ### Max time '${params.max_time}' is not valid! Using default value: $obj" - return obj - } - } else if (type == 'cpus') { - try { - return Math.min( obj, params.max_cpus as int ) - } catch (all) { - println " ### ERROR ### Max cpus '${params.max_cpus}' is not valid! Using default value: $obj" - return obj - } - } -} From ce49baddf0a39478d1d5483d5cfeac56c32e1e77 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 25 Jun 2024 14:55:43 +0200 Subject: [PATCH 02/96] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c65b999a5a..60808610d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,7 +42,7 @@ - Don't cache pip in `linting.yml` ([#2961](https://github.com/nf-core/tools/pull/2961)) - Lint pipelines with the nf-core template version and post comment if it is outdated ([#2978](https://github.com/nf-core/tools/pull/2978)) -- Replaces the old custom `check_max()` function with the Nextflow native `resourceLimits` directive ([]()) +- Replaces the old custom `check_max()` function with the Nextflow native `resourceLimits` directive ([#3037](https://github.com/nf-core/tools/pull/3037)) ### General From c8abbef61080458001fe8eb590a26a05c627cea1 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 25 Jun 2024 15:13:28 +0200 Subject: [PATCH 03/96] Code alignment, and try to get test to run on GHA Actions runner --- .github/workflows/create-test-wf.yml | 2 +- nf_core/pipeline-template/conf/base.config | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index a95a477459..cb2aad2675 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -75,7 +75,7 @@ jobs: pwd # echo content of current directory ls -la - nextflow run nf-core-testpipeline -profile test,self_hosted_runner --outdir ./results + nextflow run nf-core-testpipeline -profile test,self_hosted_runner --outdir ./results --max_cpus 4 - name: Upload log file artifact if: ${{ always() }} diff --git a/nf_core/pipeline-template/conf/base.config b/nf_core/pipeline-template/conf/base.config index 72a912fe47..fb3a8456bd 100644 --- a/nf_core/pipeline-template/conf/base.config +++ b/nf_core/pipeline-template/conf/base.config @@ -16,9 +16,9 @@ process { time = { 4.h * task.attempt } resourceLimits = [ - cpus: params.max_cpus, + cpus: params.max_cpus, memory: params.max_memory, - time: params.max_time + time: params.max_time ] errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' } From 044fc1d6750f4cf9d3ccf87bcb6ed9f3e61760b8 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Mon, 1 Jul 2024 14:30:55 +0200 Subject: [PATCH 04/96] Remove default basic resource limits in template (should be set by users) --- nf_core/pipeline-template/conf/base.config | 6 ------ 1 file changed, 6 deletions(-) diff --git a/nf_core/pipeline-template/conf/base.config b/nf_core/pipeline-template/conf/base.config index fb3a8456bd..fa292339e3 100644 --- a/nf_core/pipeline-template/conf/base.config +++ b/nf_core/pipeline-template/conf/base.config @@ -15,12 +15,6 @@ process { memory = { 6.GB * task.attempt } time = { 4.h * task.attempt } - resourceLimits = [ - cpus: params.max_cpus, - memory: params.max_memory, - time: params.max_time - ] - errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' } maxRetries = 1 maxErrors = '-1' From 528a207ca8e2e5169140876b5d7cdbf5c5845d5f Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Mon, 1 Jul 2024 14:31:31 +0200 Subject: [PATCH 05/96] Add a basic set of resource limits in test configs, matching GithubActions runner limits --- nf_core/pipeline-template/conf/test.config | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nf_core/pipeline-template/conf/test.config b/nf_core/pipeline-template/conf/test.config index 827e21b7b7..2b9715efb6 100644 --- a/nf_core/pipeline-template/conf/test.config +++ b/nf_core/pipeline-template/conf/test.config @@ -10,6 +10,14 @@ ---------------------------------------------------------------------------------------- */ +process { + resourceLimits = [ + cpus: 4, + memory: '16.GB', + time: '1.h' + ] +} + params { config_profile_name = 'Test profile' config_profile_description = 'Minimal test dataset to check pipeline function' From 936802fbfc553dde07a2acb9df134c995f5ab2d3 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Mon, 1 Jul 2024 14:37:13 +0200 Subject: [PATCH 06/96] Bump minimum NXF version to allow resourceLimits --- nf_core/pipeline-template/nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 0316c2e0c5..d575c6186f 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -258,7 +258,7 @@ manifest { homePage = 'https://github.com/{{ name }}' description = """{{ description }}""" mainScript = 'main.nf' - nextflowVersion = '!>=23.04.0' + nextflowVersion = '!>=24.04.2' version = '{{ version }}' doi = '' } From 06a1dbe41203c284b92d34912d14f12d77aebb72 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Mon, 1 Jul 2024 14:43:23 +0200 Subject: [PATCH 07/96] Add linting check for now deprecated params --- nf_core/pipelines/lint/nextflow_config.py | 89 ++++++++++++++++++----- 1 file changed, 69 insertions(+), 20 deletions(-) diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index f62100a70a..f1c4c536c5 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -87,6 +87,9 @@ def nextflow_config(self): * ``params.nf_required_version``: The old method for specifying the minimum Nextflow version. Replaced by ``manifest.nextflowVersion`` * ``params.container``: The old method for specifying the dockerhub container address. Replaced by ``process.container`` * ``igenomesIgnore``: Changed to ``igenomes_ignore`` + * ``params.max_cpus``: Old method of specifying the maximum number of CPUs a process can request. Replaced by native Nextflow `resourceLimits`directive. + * ``params.max_memory``: Old method of specifying the maximum number of memory can request. Replaced by native Nextflow `resourceLimits`directive. + * ``params.max_time``: Old method of specifying the maximum number of CPUs can request. Replaced by native Nextflow `resourceLimits`directive. .. tip:: The ``snake_case`` convention should now be used when defining pipeline parameters @@ -170,6 +173,9 @@ def nextflow_config(self): "params.igenomesIgnore", "params.name", "params.enable_conda", + "params.max_cpus", + "params.max_memory", + "params.max_time", ] # Remove field that should be ignored according to the linting config @@ -200,9 +206,13 @@ def nextflow_config(self): ignored.append(f"Config variable ignored: {self._wrap_quotes(cf)}") break if cf not in self.nf_config.keys(): - passed.append(f"Config variable (correctly) not found: {self._wrap_quotes(cf)}") + passed.append( + f"Config variable (correctly) not found: {self._wrap_quotes(cf)}" + ) else: - failed.append(f"Config variable (incorrectly) found: {self._wrap_quotes(cf)}") + failed.append( + f"Config variable (incorrectly) found: {self._wrap_quotes(cf)}" + ) # Check and warn if the process configuration is done with deprecated syntax process_with_deprecated_syntax = list( @@ -222,9 +232,13 @@ def nextflow_config(self): if k in ignore_configs: continue if self.nf_config.get(k) == "true": - passed.append(f"Config ``{k}`` had correct value: ``{self.nf_config.get(k)}``") + passed.append( + f"Config ``{k}`` had correct value: ``{self.nf_config.get(k)}``" + ) else: - failed.append(f"Config ``{k}`` did not have correct value: ``{self.nf_config.get(k)}``") + failed.append( + f"Config ``{k}`` did not have correct value: ``{self.nf_config.get(k)}``" + ) if "manifest.name" not in ignore_configs: # Check that the pipeline name starts with nf-core @@ -233,7 +247,9 @@ def nextflow_config(self): if not manifest_name.startswith("nf-core/"): raise AssertionError() except (AssertionError, IndexError): - failed.append(f"Config ``manifest.name`` did not begin with ``nf-core/``:\n {manifest_name}") + failed.append( + f"Config ``manifest.name`` did not begin with ``nf-core/``:\n {manifest_name}" + ) else: passed.append("Config ``manifest.name`` began with ``nf-core/``") @@ -249,7 +265,9 @@ def nextflow_config(self): ) else: - passed.append("Config variable ``manifest.homePage`` began with https://github.com/nf-core/") + passed.append( + "Config variable ``manifest.homePage`` began with https://github.com/nf-core/" + ) # Check that the DAG filename ends in ``.svg`` if "dag.file" in self.nf_config: @@ -257,12 +275,21 @@ def nextflow_config(self): if self.nf_config["dag.file"].strip("'\"").endswith(default_dag_format): passed.append(f"Config ``dag.file`` ended with ``{default_dag_format}``") else: - failed.append(f"Config ``dag.file`` did not end with ``{default_dag_format}``") + failed.append( + f"Config ``dag.file`` did not end with ``{default_dag_format}``" + ) # Check that the minimum nextflowVersion is set properly if "manifest.nextflowVersion" in self.nf_config: - if self.nf_config.get("manifest.nextflowVersion", "").strip("\"'").lstrip("!").startswith(">="): - passed.append("Config variable ``manifest.nextflowVersion`` started with >= or !>=") + if ( + self.nf_config.get("manifest.nextflowVersion", "") + .strip("\"'") + .lstrip("!") + .startswith(">=") + ): + passed.append( + "Config variable ``manifest.nextflowVersion`` started with >= or !>=" + ) else: failed.append( "Config ``manifest.nextflowVersion`` did not start with ``>=`` or ``!>=`` : " @@ -272,7 +299,9 @@ def nextflow_config(self): # Check that the pipeline version contains ``dev`` if not self.release_mode and "manifest.version" in self.nf_config: if self.nf_config["manifest.version"].strip(" '\"").endswith("dev"): - passed.append(f"Config ``manifest.version`` ends in ``dev``: ``{self.nf_config['manifest.version']}``") + passed.append( + f"Config ``manifest.version`` ends in ``dev``: ``{self.nf_config['manifest.version']}``" + ) else: warned.append( f"Config ``manifest.version`` should end in ``dev``: ``{self.nf_config['manifest.version']}``" @@ -291,18 +320,32 @@ def nextflow_config(self): if "custom_config" not in ignore_configs: # Check if custom profile params are set correctly - if self.nf_config.get("params.custom_config_version", "").strip("'") == "master": + if ( + self.nf_config.get("params.custom_config_version", "").strip("'") + == "master" + ): passed.append("Config `params.custom_config_version` is set to `master`") else: - failed.append("Config `params.custom_config_version` is not set to `master`") + failed.append( + "Config `params.custom_config_version` is not set to `master`" + ) - custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/{}".format( - self.nf_config.get("params.custom_config_version", "").strip("'") + custom_config_base = ( + "https://raw.githubusercontent.com/nf-core/configs/{}".format( + self.nf_config.get("params.custom_config_version", "").strip("'") + ) ) - if self.nf_config.get("params.custom_config_base", "").strip("'") == custom_config_base: - passed.append(f"Config `params.custom_config_base` is set to `{custom_config_base}`") + if ( + self.nf_config.get("params.custom_config_base", "").strip("'") + == custom_config_base + ): + passed.append( + f"Config `params.custom_config_base` is set to `{custom_config_base}`" + ) else: - failed.append(f"Config `params.custom_config_base` is not set to `{custom_config_base}`") + failed.append( + f"Config `params.custom_config_base` is not set to `{custom_config_base}`" + ) # Check that lines for loading custom profiles exist lines = [ @@ -344,7 +387,9 @@ def nextflow_config(self): match = re.search(r"\bprofiles\s*{", cleaned_content) if not match: - failed.append("nextflow.config does not contain `profiles` scope, but `test` profile is required") + failed.append( + "nextflow.config does not contain `profiles` scope, but `test` profile is required" + ) else: # Extract profiles scope content and check for test profile start = match.end() @@ -360,7 +405,9 @@ def nextflow_config(self): if re.search(r"\btest\s*{", profiles_content): passed.append("nextflow.config contains configuration profile `test`") else: - failed.append("nextflow.config does not contain configuration profile `test`") + failed.append( + "nextflow.config does not contain configuration profile `test`" + ) # Check that the default values in nextflow.config match the default values defined in the nextflow_schema.json ignore_defaults = [] @@ -404,7 +451,9 @@ def nextflow_config(self): schema_default = str(schema.schema_defaults[param_name]) config_default = str(self.nf_config[param]) if config_default is not None and config_default == schema_default: - passed.append(f"Config default value correct: {param}= {schema_default}") + passed.append( + f"Config default value correct: {param}= {schema_default}" + ) else: failed.append( f"Config default value incorrect: `{param}` is set as {self._wrap_quotes(schema_default)} in `nextflow_schema.json` but is {self._wrap_quotes(self.nf_config[param])} in `nextflow.config`." From 47abfc2a2dbf9f8ae42a70bc5b833e1836fdb121 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Mon, 1 Jul 2024 14:44:55 +0200 Subject: [PATCH 08/96] Add linting check for check_ params and continue removing from everywhere (not finished) --- nf_core/pipeline-template/conf/test.config | 5 --- nf_core/pipeline-template/nextflow.config | 6 ---- .../pipeline-template/nextflow_schema.json | 35 ------------------- 3 files changed, 46 deletions(-) diff --git a/nf_core/pipeline-template/conf/test.config b/nf_core/pipeline-template/conf/test.config index 2b9715efb6..0f1d974101 100644 --- a/nf_core/pipeline-template/conf/test.config +++ b/nf_core/pipeline-template/conf/test.config @@ -22,11 +22,6 @@ params { config_profile_name = 'Test profile' config_profile_description = 'Minimal test dataset to check pipeline function' - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' - // Input data // TODO nf-core: Specify the paths to your test data on nf-core/test-datasets // TODO nf-core: Give any required params for the test so that command line flags are not needed diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index d575c6186f..7d9040104c 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -50,12 +50,6 @@ params { config_profile_url = null {%- endif %} - // Max resource options - // Defaults only, expecting to be overwritten - max_memory = '128.GB' - max_cpus = 16 - max_time = '240.h' - // Schema validation default options validationFailUnrecognisedParams = false validationLenientMode = false diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 18bad71b76..07a90f6259 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -124,41 +124,6 @@ } } }, - "max_job_request_options": { - "title": "Max job request options", - "type": "object", - "fa_icon": "fab fa-acquisitions-incorporated", - "description": "Set the top limit for requested resources for any single job.", - "help_text": "If you are running on a smaller system, a pipeline step requesting more resources than are available may cause the Nextflow to stop the run with an error. These options allow you to cap the maximum resources requested by any single job so that the pipeline will run on your system.\n\nNote that you can not _increase_ the resources requested by any job using these options. For that you will need your own configuration file. See [the nf-core website](https://nf-co.re/usage/configuration) for details.", - "properties": { - "max_cpus": { - "type": "integer", - "description": "Maximum number of CPUs that can be requested for any single job.", - "default": 16, - "fa_icon": "fas fa-microchip", - "hidden": true, - "help_text": "Use to set an upper-limit for the CPU requirement for each process. Should be an integer e.g. `--max_cpus 1`" - }, - "max_memory": { - "type": "string", - "description": "Maximum amount of memory that can be requested for any single job.", - "default": "128.GB", - "fa_icon": "fas fa-memory", - "pattern": "^\\d+(\\.\\d+)?\\.?\\s*(K|M|G|T)?B$", - "hidden": true, - "help_text": "Use to set an upper-limit for the memory requirement for each process. Should be a string in the format integer-unit e.g. `--max_memory '8.GB'`" - }, - "max_time": { - "type": "string", - "description": "Maximum amount of time that can be requested for any single job.", - "default": "240.h", - "fa_icon": "far fa-clock", - "pattern": "^(\\d+\\.?\\s*(s|m|h|d|day)\\s*)+$", - "hidden": true, - "help_text": "Use to set an upper-limit for the time requirement for each process. Should be a string in the format integer-unit e.g. `--max_time '2.h'`" - } - } - }, "generic_options": { "title": "Generic options", "type": "object", From 17adacadecde4d0fad857fad25ee6e0ef44b4cc2 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 23 Feb 2024 09:42:47 +0100 Subject: [PATCH 09/96] 10 minute hack to try to parse proper input structure from module processes, for meta.yml --- nf_core/components/nfcore_component.py | 30 ++++++++++++++++++-------- nf_core/modules/lint/meta_yml.py | 2 ++ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 5d0baf63dd..e8f6f0b7a3 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -173,15 +173,27 @@ def get_inputs_from_main_nf(self): log.debug(f"Could not find any inputs in {self.main_nf}") return inputs input_data = data.split("input:")[1].split("output:")[0] - regex = r"(val|path)\s*(\(([^)]+)\)|\s*([^)\s,]+))" - matches = re.finditer(regex, input_data, re.MULTILINE) - for _, match in enumerate(matches, start=1): - if match.group(3): - input_val = match.group(3).split(",")[0] # handle `files, stageAs: "inputs/*"` cases - inputs.append(input_val) - elif match.group(4): - input_val = match.group(4).split(",")[0] # handle `files, stageAs: "inputs/*"` cases - inputs.append(input_val) + for line in input_data.split("\n"): + theseinputs = [] + regex = r"(val|path)\s*(\(([^)]+)\)|\s*([^)\s,]+))" + matches = re.finditer(regex, line) + for _, match in enumerate(matches, start=1): + input_type = None + input_val = None + if match.group(1): + input_type = match.group(1) + if match.group(3): + input_val = match.group(3).split(",")[0] # handle `files, stageAs: "inputs/*"` cases + elif match.group(4): + input_val = match.group(4).split(",")[0] # handle `files, stageAs: "inputs/*"` cases + if input_type and input_val: + theseinputs.append({ + input_val: { + "type": input_type + } + }) + if len(theseinputs) > 0: + inputs.append(theseinputs) log.debug(f"Found {len(inputs)} inputs in {self.main_nf}") self.inputs = inputs diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index 481d50b3ef..6718ed5c21 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -40,6 +40,8 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None """ module.get_inputs_from_main_nf() + print(yaml.dump({"input": module.inputs})) + exit() module.get_outputs_from_main_nf() # Check if we have a patch file, get original file in that case meta_yaml = None From 970b4a0e09249476eaf2153ff57a0aad061d6750 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 4 Jun 2024 11:51:25 +0200 Subject: [PATCH 10/96] get correct format of inputs and outputs --- nf_core/components/nfcore_component.py | 36 +++++++++++++++++--------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index e8f6f0b7a3..9a266ab56e 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -174,7 +174,7 @@ def get_inputs_from_main_nf(self): return inputs input_data = data.split("input:")[1].split("output:")[0] for line in input_data.split("\n"): - theseinputs = [] + channel_elements = [] regex = r"(val|path)\s*(\(([^)]+)\)|\s*([^)\s,]+))" matches = re.finditer(regex, line) for _, match in enumerate(matches, start=1): @@ -187,13 +187,9 @@ def get_inputs_from_main_nf(self): elif match.group(4): input_val = match.group(4).split(",")[0] # handle `files, stageAs: "inputs/*"` cases if input_type and input_val: - theseinputs.append({ - input_val: { - "type": input_type - } - }) - if len(theseinputs) > 0: - inputs.append(theseinputs) + channel_elements.append({input_val: {"type": input_type}}) + if len(channel_elements) > 0: + inputs.append(channel_elements) log.debug(f"Found {len(inputs)} inputs in {self.main_nf}") self.inputs = inputs @@ -206,9 +202,25 @@ def get_outputs_from_main_nf(self): log.debug(f"Could not find any outputs in {self.main_nf}") return outputs output_data = data.split("output:")[1].split("when:")[0] - regex = r"emit:\s*([^)\s,]+)" - matches = re.finditer(regex, output_data, re.MULTILINE) - for _, match in enumerate(matches, start=1): - outputs.append(match.group(1)) + regex_emit = r"emit:\s*([^)\s,]+)" + regex_elements = r"(val|path|env|stdout)\s*(\(([^)]+)\)|\s*([^)\s,]+))" + for line in output_data.split("\n"): + match_emit = re.search(regex_emit, line) + matches_elements = re.finditer(regex_elements, line) + if not match_emit: + continue + output_channel = {match_emit.group(1): []} + for _, match_element in enumerate(matches_elements, start=1): + output_type = None + output_val = None + if match_element.group(1): + output_type = match_element.group(1) + if match_element.group(3): + output_val = match_element.group(3).split(",")[0] # handle `files, stageAs: "inputs/*"` cases + elif match_element.group(4): + output_val = match_element.group(4).split(",")[0] # handle `files, stageAs: "inputs/*"` cases + if output_type and output_val: + output_channel[match_emit.group(1)].append({output_val: {"type": output_type}}) + outputs.append(output_channel) log.debug(f"Found {len(outputs)} outputs in {self.main_nf}") self.outputs = outputs From 744b0fd4ae76e684b19affdb8a79f09ce3815daa Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 5 Jun 2024 14:12:23 +0200 Subject: [PATCH 11/96] update module template meta.yml --- nf_core/module-template/meta.yml | 48 +++++++++++++++++--------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/nf_core/module-template/meta.yml b/nf_core/module-template/meta.yml index 9d3f3c1c12..2dd8e74efa 100644 --- a/nf_core/module-template/meta.yml +++ b/nf_core/module-template/meta.yml @@ -26,42 +26,44 @@ tools: {% endif -%} input: #{% if has_meta %} Only when we have meta - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. `[ id:'sample1', single_end:false ]` + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` {% endif %} {% if not_empty_template -%} ## TODO nf-core: Delete / customise this example input {%- endif %} - - {{ 'bam:' if not_empty_template else "input:" }} - type: file - description: {{ 'Sorted BAM/CRAM/SAM file' if not_empty_template else "" }} - pattern: {{ '"*.{bam,cram,sam}"' if not_empty_template else "" }} + - {{ 'bam:' if not_empty_template else "input:" }} + type: file + description: {{ 'Sorted BAM/CRAM/SAM file' if not_empty_template else "" }} + pattern: {{ '"*.{bam,cram,sam}"' if not_empty_template else "" }} {% if not_empty_template -%} ## TODO nf-core: Add a description of all of the variables used as output {% endif -%} output: + - versions: + - "versions.yml": + type: file + description: File containing software versions + pattern: "versions.yml" + - {{ 'bam:' if not_empty_template else "output:" }} #{% if has_meta -%} Only when we have meta - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. `[ id:'sample1', single_end:false ]` + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` {% endif %} - - versions: - type: file - description: File containing software versions - pattern: "versions.yml" {% if not_empty_template -%} - ## TODO nf-core: Delete / customise this example output + ## TODO nf-core: Delete / customise this example output {%- endif %} - - {{ 'bam:' if not_empty_template else "output:" }} - type: file - description: {{ 'Sorted BAM/CRAM/SAM file' if not_empty_template else "" }} - pattern: {{ '"*.{bam,cram,sam}"' if not_empty_template else "" }} + - {{ '"*.bam":' if not_empty_template else '"*":' }} + type: file + description: {{ 'Sorted BAM/CRAM/SAM file' if not_empty_template else "" }} + pattern: {{ '"*.{bam,cram,sam}"' if not_empty_template else "" }} authors: - "{{ author }}" From c9f6f8fc016a5e0cf8dd4b060876075bb2b655b7 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 5 Jun 2024 15:27:22 +0200 Subject: [PATCH 12/96] add linting for correct inputs and outputs in meta.yml --- nf_core/components/nfcore_component.py | 5 +- nf_core/modules/lint/meta_yml.py | 177 ++++++++++++++----------- 2 files changed, 101 insertions(+), 81 deletions(-) diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 9a266ab56e..6ff1f6c171 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -216,10 +216,11 @@ def get_outputs_from_main_nf(self): if match_element.group(1): output_type = match_element.group(1) if match_element.group(3): - output_val = match_element.group(3).split(",")[0] # handle `files, stageAs: "inputs/*"` cases + output_val = match_element.group(3) elif match_element.group(4): - output_val = match_element.group(4).split(",")[0] # handle `files, stageAs: "inputs/*"` cases + output_val = match_element.group(4) if output_type and output_val: + output_val = output_val.strip("'").strip('"') # remove quotes output_channel[match_emit.group(1)].append({output_val: {"type": output_type}}) outputs.append(output_channel) log.debug(f"Found {len(outputs)} outputs in {self.main_nf}") diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index 6718ed5c21..d5615059a5 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -40,8 +40,6 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None """ module.get_inputs_from_main_nf() - print(yaml.dump({"input": module.inputs})) - exit() module.get_outputs_from_main_nf() # Check if we have a patch file, get original file in that case meta_yaml = None @@ -93,93 +91,114 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None ) ) - # Confirm that all input and output channels are specified + # Confirm that all input and output channels are correctly specified if valid_meta_yml: + # Check that inputs are specified in meta.yml + if len(module.inputs) > 0 and "input" not in meta_yaml: + module.failed.append( + ( + "meta_input", + "Inputs not specified in module `meta.yml`", + module.meta_yml, + ) + ) + elif len(module.inputs) > 0: + module.passed.append( + ( + "meta_input", + "Inputs specified in module `meta.yml`", + module.meta_yml, + ) + ) + # Check that all inputs are correctly specified if "input" in meta_yaml: - meta_input = [list(x.keys())[0] for x in meta_yaml["input"]] - for input in module.inputs: - if input in meta_input: - module.passed.append(("meta_input_main_only", f"`{input}` specified", module.meta_yml)) - else: - module.warned.append( - ( - "meta_input_main_only", - f"`{input}` is present as an input in the `main.nf`, but missing in `meta.yml`", - module.meta_yml, - ) + # Obtain list of correct inputs and elements of each input channel + correct_inputs = [] + for input_channel in module.inputs: + channel_elements = [] + for element in input_channel: + channel_elements.append(list(element.keys())[0]) + correct_inputs.append(channel_elements) + # Obtain list of inputs specified in meta.yml + meta_inputs = [] + for input_channel in meta_yaml["input"]: + if isinstance(input_channel, list): # Correct format + channel_elements = [] + for element in input_channel: + channel_elements.append(list(element.keys())[0]) + meta_inputs.append(channel_elements) + elif isinstance(input_channel, dict): # Old format + meta_inputs.append(list(input_channel.keys())[0]) + + if correct_inputs == meta_inputs: + module.passed.append( + ( + "correct_meta_inputs", + "Correct inputs specified in module `meta.yml`", + module.meta_yml, ) - # check if there are any inputs in meta.yml that are not in main.nf - for input in meta_input: - if input in module.inputs: - module.passed.append( - ( - "meta_input_meta_only", - f"`{input}` is present as an input in `meta.yml` and `main.nf`", - module.meta_yml, - ) - ) - else: - module.warned.append( - ( - "meta_input_meta_only", - f"`{input}` is present as an input in `meta.yml` but not in `main.nf`", - module.meta_yml, - ) + ) + else: + module.failed.append( + ( + "correct_meta_inputs", + f"Incorrect inputs specified in module `meta.yml`. Inputs should contain: {correct_inputs}\nRun `nf-core modules lint --update-meta-yml` to update the `meta.yml` file.", + module.meta_yml, ) + ) - if "output" in meta_yaml and meta_yaml["output"] is not None: - meta_output = [list(x.keys())[0] for x in meta_yaml["output"]] - for output in module.outputs: - if output in meta_output: - module.passed.append(("meta_output_main_only", f"`{output}` specified", module.meta_yml)) - else: - module.warned.append( - ( - "meta_output_main_only", - f"`{output}` is present as an output in the `main.nf`, but missing in `meta.yml`", - module.meta_yml, - ) - ) - # check if there are any outputs in meta.yml that are not in main.nf - for output in meta_output: - if output in module.outputs: - module.passed.append( - ( - "meta_output_meta_only", - f"`{output}` is present as an output in `meta.yml` and `main.nf`", - module.meta_yml, - ) - ) - elif output == "meta": - module.passed.append( - ( - "meta_output_meta_only", - f"`{output}` is skipped for `meta.yml` outputs", - module.meta_yml, - ) - ) - else: - module.warned.append( - ( - "meta_output_meta_only", - f"`{output}` is present as an output in `meta.yml` but not in `main.nf`", - module.meta_yml, - ) - ) - # confirm that the name matches the process name in main.nf - if meta_yaml["name"].upper() == module.process_name: - module.passed.append( + # Check that outputs are specified in meta.yml + if len(module.outputs) > 0 and "output" not in meta_yaml: + module.failed.append( ( - "meta_name", - "Correct name specified in `meta.yml`.", + "meta_output", + "Outputs not specified in module `meta.yml`", module.meta_yml, ) ) - else: - module.failed.append( + elif len(module.outputs) > 0: + module.passed.append( ( - "meta_name", - f"Conflicting `process` name between meta.yml (`{meta_yaml['name']}`) and main.nf (`{module.process_name}`)", + "meta_output", + "Outputs specified in module `meta.yml`", module.meta_yml, ) ) + # Check that all outputs are correctly specified + if "output" in meta_yaml: + # Obtain dictionary of correct outputs and elements of each output channel + correct_outputs = {} + for output_channel in module.outputs: + channel_name = list(output_channel.keys())[0] + channel_elements = [] + for element in output_channel[channel_name]: + channel_elements.append(list(element.keys())[0]) + correct_outputs[channel_name] = channel_elements + # Obtain dictionary of outputs specified in meta.yml + meta_outputs = {} + for output_channel in meta_yaml["output"]: + channel_name = list(output_channel.keys())[0] + if isinstance(output_channel[channel_name], list): # Correct format + channel_elements = [] + for element in output_channel[channel_name]: + channel_elements.append(list(element.keys())[0]) + meta_outputs[channel_name] = channel_elements + elif isinstance(output_channel[channel_name], dict): # Old format + meta_outputs[channel_name] = [] + + if correct_outputs == meta_outputs: + module.passed.append( + ( + "correct_meta_outputs", + "Correct outputs specified in module `meta.yml`", + module.meta_yml, + ) + ) + else: + module.failed.append( + ( + "correct_meta_outputs", + f"Incorrect outputs specified in module `meta.yml`. Outputs should contain: {correct_outputs}\nRun `nf-core modules lint --update-meta-yml` to update the `meta.yml` file.", + module.meta_yml, + ) + ) From d79f487bdaacddedc1781c263574996e1c10760d Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 17 Jun 2024 17:26:37 +0200 Subject: [PATCH 13/96] add update-meta-yml option --- nf_core/__main__.py | 5 +- nf_core/commands_modules.py | 3 +- nf_core/components/lint/__init__.py | 2 + nf_core/components/nfcore_component.py | 4 +- nf_core/modules/lint/__init__.py | 105 +++++++++++++++- nf_core/modules/lint/meta_yml.py | 167 ++++++++++++++++--------- 6 files changed, 225 insertions(+), 61 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 4fa640436d..584947afd0 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1196,7 +1196,10 @@ def command_modules_test(ctx, tool, dir, no_prompts, update, once, profile): is_flag=True, help="Fix the module version if a newer version is available", ) -def command_modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, passed, sort_by, fix_version): +@click.option( + "--update-meta-yml", is_flag=True, help="Update the meta.yml file with the correct format of input and outputs" +) +def command_modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, passed, sort_by, fix_version, update_meta_yml): """ Lint one or more modules in a directory. """ diff --git a/nf_core/commands_modules.py b/nf_core/commands_modules.py index 5f7191436c..85471184ca 100644 --- a/nf_core/commands_modules.py +++ b/nf_core/commands_modules.py @@ -244,7 +244,7 @@ def modules_test(ctx, tool, dir, no_prompts, update, once, profile): sys.exit(1) -def modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, passed, sort_by, fix_version): +def modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, passed, sort_by, fix_version, update_meta_yml): """ Lint one or more modules in a directory. @@ -277,6 +277,7 @@ def modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, passed, show_passed=passed, sort_by=sort_by, fix_version=fix_version, + update_meta_yml=update_meta_yml, ) if len(module_lint.failed) > 0: sys.exit(1) diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index 499d31e71e..b4194e768e 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -56,6 +56,7 @@ def __init__( component_type, dir, fail_warned=False, + update_meta_yml=False, remote_url=None, branch=None, no_pull=False, @@ -72,6 +73,7 @@ def __init__( ) self.fail_warned = fail_warned + self.update_meta_yml = update_meta_yml self.passed = [] self.warned = [] self.failed = [] diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 6ff1f6c171..dcd76b0e8f 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -187,7 +187,7 @@ def get_inputs_from_main_nf(self): elif match.group(4): input_val = match.group(4).split(",")[0] # handle `files, stageAs: "inputs/*"` cases if input_type and input_val: - channel_elements.append({input_val: {"type": input_type}}) + channel_elements.append({input_val: {"qualifier": input_type}}) if len(channel_elements) > 0: inputs.append(channel_elements) log.debug(f"Found {len(inputs)} inputs in {self.main_nf}") @@ -221,7 +221,7 @@ def get_outputs_from_main_nf(self): output_val = match_element.group(4) if output_type and output_val: output_val = output_val.strip("'").strip('"') # remove quotes - output_channel[match_emit.group(1)].append({output_val: {"type": output_type}}) + output_channel[match_emit.group(1)].append({output_val: {"qualifier": output_type}}) outputs.append(output_channel) log.debug(f"Found {len(outputs)} outputs in {self.main_nf}") self.outputs = outputs diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index b780144ef7..f6c2069bda 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -11,6 +11,7 @@ import questionary import rich +import yaml import nf_core.modules.modules_utils import nf_core.utils @@ -29,7 +30,12 @@ class ModuleLint(ComponentLint): # Import lint functions from .environment_yml import environment_yml # type: ignore[misc] from .main_nf import main_nf # type: ignore[misc] - from .meta_yml import meta_yml # type: ignore[misc] + from .meta_yml import ( # type: ignore[misc] + meta_yml, + obtain_correct_and_specified_inputs, + obtain_correct_and_specified_outputs, + read_meta_yml, + ) from .module_changes import module_changes # type: ignore[misc] from .module_deprecations import module_deprecations # type: ignore[misc] from .module_patch import module_patch # type: ignore[misc] @@ -41,6 +47,7 @@ def __init__( self, dir, fail_warned=False, + update_meta_yml=False, remote_url=None, branch=None, no_pull=False, @@ -51,6 +58,7 @@ def __init__( component_type="modules", dir=dir, fail_warned=fail_warned, + update_meta_yml=update_meta_yml, remote_url=remote_url, branch=branch, no_pull=no_pull, @@ -213,6 +221,12 @@ def lint_module(self, mod, progress_bar, registry, local=False, fix_version=Fals # Otherwise run all the lint tests else: + mod.get_inputs_from_main_nf() + mod.get_outputs_from_main_nf() + # Update meta.yml file if requested + if self.update_meta_yml: + self.update_meta_yml_file(mod) + if self.repo_type == "pipeline" and self.modules_json: # Set correct sha version = self.modules_json.get_module_version(mod.component_name, mod.repo_url, mod.org) @@ -232,3 +246,92 @@ def lint_module(self, mod, progress_bar, registry, local=False, fix_version=Fals self.failed += warned self.failed += [LintResult(mod, *m) for m in mod.failed] + + def update_meta_yml_file(self, mod): + """ + Update the meta.yml file with the correct inputs and outputs + """ + meta_yml = self.read_meta_yml(mod) + corrected_meta_yml = meta_yml.copy() + + # Obtain inputs and outputs from main.nf and meta.yml + # Used to compare only the structure of channels and elements + # Do not compare features to allow for custom features in meta.yml (i.e. pattern) + if "input" in meta_yml: + correct_inputs, meta_inputs = self.obtain_correct_and_specified_inputs(mod, meta_yml) + if "output" in meta_yml: + correct_outputs, meta_outputs = self.obtain_correct_and_specified_outputs(mod, meta_yml) + + if correct_inputs != meta_inputs: + log.debug( + f"Correct inputs: '{correct_inputs}' differ from current inputs: '{meta_inputs}' in '{mod.meta_yml}'" + ) + corrected_meta_yml["input"] = mod.inputs.copy() # list of lists (channels) of dicts (elements) + for i, channel in enumerate(corrected_meta_yml["input"]): + for j, element in enumerate(channel): + element_name = list(element.keys())[0] + for k, meta_element in enumerate(meta_yml["input"]): + try: + # Handle old format of meta.yml: list of dicts (channels) + if element_name in meta_element.keys(): + # Copy current features of that input element form meta.yml + for feature in meta_element[element_name].keys(): + if feature not in element[element_name].keys(): + corrected_meta_yml["input"][i][j][element_name][feature] = meta_element[ + element_name + ][feature] + break + except AttributeError: + # Handle new format of meta.yml: list of lists (channels) of elements (dicts) + for x, meta_ch_element in enumerate(meta_element): + if element_name in meta_ch_element.keys(): + # Copy current features of that input element form meta.yml + for feature in meta_element[x][element_name].keys(): + if feature not in element[element_name].keys(): + corrected_meta_yml["input"][i][j][element_name][feature] = meta_element[x][ + element_name + ][feature] + break + + if correct_outputs != meta_outputs: + log.debug( + f"Correct outputs: '{correct_outputs}' differ from current outputs: '{meta_outputs}' in '{mod.meta_yml}'" + ) + corrected_meta_yml["output"] = mod.outputs.copy() # list of dicts (channels) with list of dicts (elements) + for i, channel in enumerate(corrected_meta_yml["output"]): + ch_name = list(channel.keys())[0] + for j, element in enumerate(channel[ch_name]): + element_name = list(element.keys())[0] + for k, meta_element in enumerate(meta_yml["output"]): + if element_name in meta_element.keys(): + # Copy current features of that output element form meta.yml + for feature in meta_element[element_name].keys(): + if feature not in element[element_name].keys(): + corrected_meta_yml["output"][i][ch_name][j][element_name][feature] = meta_element[ + element_name + ][feature] + break + elif ch_name in meta_element.keys(): + # When the previous output element was using the name of the channel + # Copy current features of that output element form meta.yml + try: + # Handle old format of meta.yml + for feature in meta_element[ch_name].keys(): + if feature not in element[element_name].keys(): + corrected_meta_yml["output"][i][ch_name][j][element_name][feature] = ( + meta_element[ch_name][feature] + ) + except AttributeError: + # Handle new format of meta.yml + for x, meta_ch_element in enumerate(meta_element[ch_name]): + for meta_ch_element_name in meta_ch_element.keys(): + for feature in meta_ch_element[meta_ch_element_name].keys(): + if feature not in element[element_name].keys(): + corrected_meta_yml["output"][i][ch_name][j][element_name][feature] = ( + meta_ch_element[meta_ch_element_name][feature] + ) + break + + with open(mod.meta_yml, "w") as fh: + log.info(f"Updating {mod.meta_yml}") + yaml.dump(corrected_meta_yml, fh, sort_keys=False, Dumper=nf_core.utils.custom_yaml_dumper()) diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index d5615059a5..9c074ba731 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -1,5 +1,7 @@ import json +import logging from pathlib import Path +from typing import Union import yaml from jsonschema import exceptions, validators @@ -8,6 +10,8 @@ from nf_core.components.nfcore_component import NFCoreComponent from nf_core.modules.modules_differ import ModulesDiffer +log = logging.getLogger(__name__) + def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None: """ @@ -39,28 +43,13 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None """ - module.get_inputs_from_main_nf() - module.get_outputs_from_main_nf() # Check if we have a patch file, get original file in that case - meta_yaml = None - if module.is_patched: - lines = ModulesDiffer.try_apply_patch( - module.component_name, - module_lint_object.modules_repo.repo_path, - module.patch_path, - Path(module.component_dir).relative_to(module.base_dir), - reverse=True, - ).get("meta.yml") - if lines is not None: - meta_yaml = yaml.safe_load("".join(lines)) + meta_yaml = read_meta_yml(module_lint_object, module) if meta_yaml is None: - try: - with open(module.meta_yml) as fh: - meta_yaml = yaml.safe_load(fh) - module.passed.append(("meta_yml_exists", "Module `meta.yml` exists", module.meta_yml)) - except FileNotFoundError: - module.failed.append(("meta_yml_exists", "Module `meta.yml` does not exist", module.meta_yml)) - return + module.failed.append(("meta_yml_exists", "Module `meta.yml` does not exist", module.meta_yml)) + return + else: + module.passed.append(("meta_yml_exists", "Module `meta.yml` exists", module.meta_yml)) # Confirm that the meta.yml file is valid according to the JSON schema valid_meta_yml = False @@ -110,25 +99,11 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None module.meta_yml, ) ) + else: + log.debug(f"No inputs specified in module `main.nf`: {module.component_name}") # Check that all inputs are correctly specified if "input" in meta_yaml: - # Obtain list of correct inputs and elements of each input channel - correct_inputs = [] - for input_channel in module.inputs: - channel_elements = [] - for element in input_channel: - channel_elements.append(list(element.keys())[0]) - correct_inputs.append(channel_elements) - # Obtain list of inputs specified in meta.yml - meta_inputs = [] - for input_channel in meta_yaml["input"]: - if isinstance(input_channel, list): # Correct format - channel_elements = [] - for element in input_channel: - channel_elements.append(list(element.keys())[0]) - meta_inputs.append(channel_elements) - elif isinstance(input_channel, dict): # Old format - meta_inputs.append(list(input_channel.keys())[0]) + correct_inputs, meta_inputs = obtain_correct_and_specified_inputs(module_lint_object, module, meta_yaml) if correct_inputs == meta_inputs: module.passed.append( @@ -166,25 +141,7 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None ) # Check that all outputs are correctly specified if "output" in meta_yaml: - # Obtain dictionary of correct outputs and elements of each output channel - correct_outputs = {} - for output_channel in module.outputs: - channel_name = list(output_channel.keys())[0] - channel_elements = [] - for element in output_channel[channel_name]: - channel_elements.append(list(element.keys())[0]) - correct_outputs[channel_name] = channel_elements - # Obtain dictionary of outputs specified in meta.yml - meta_outputs = {} - for output_channel in meta_yaml["output"]: - channel_name = list(output_channel.keys())[0] - if isinstance(output_channel[channel_name], list): # Correct format - channel_elements = [] - for element in output_channel[channel_name]: - channel_elements.append(list(element.keys())[0]) - meta_outputs[channel_name] = channel_elements - elif isinstance(output_channel[channel_name], dict): # Old format - meta_outputs[channel_name] = [] + correct_outputs, meta_outputs = obtain_correct_and_specified_outputs(module_lint_object, module, meta_yaml) if correct_outputs == meta_outputs: module.passed.append( @@ -202,3 +159,101 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None module.meta_yml, ) ) + + +def read_meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> Union[dict, None]: + """ + Read a `meta.yml` file and return it as a dictionary + + Args: + module_lint_object (ComponentLint): The lint object for the module + module (NFCoreComponent): The module to read + + Returns: + dict: The `meta.yml` file as a dictionary + """ + meta_yaml = None + # Check if we have a patch file, get original file in that case + if module.is_patched: + lines = ModulesDiffer.try_apply_patch( + module.component_name, + module_lint_object.modules_repo.repo_path, + module.patch_path, + Path(module.component_dir).relative_to(module.base_dir), + reverse=True, + ).get("meta.yml") + if lines is not None: + meta_yaml = yaml.safe_load("".join(lines)) + if meta_yaml is None: + try: + with open(module.meta_yml) as fh: + meta_yaml = yaml.safe_load(fh) + except FileNotFoundError: + return None + return meta_yaml + + +def obtain_correct_and_specified_inputs(_, module, meta_yaml): + """ + Obtain the list of correct inputs and the elements of each input channel. + + Args: + module (object): The module object. + meta_yaml (dict): The meta.yml dictionary. + + Returns: + tuple: A tuple containing two lists. The first list contains the correct inputs, + and the second list contains the inputs specified in meta.yml. + """ + correct_inputs = [] + for input_channel in module.inputs: + channel_elements = [] + for element in input_channel: + channel_elements.append(list(element.keys())[0]) + correct_inputs.append(channel_elements) + + meta_inputs = [] + for input_channel in meta_yaml["input"]: + if isinstance(input_channel, list): # Correct format + channel_elements = [] + for element in input_channel: + channel_elements.append(list(element.keys())[0]) + meta_inputs.append(channel_elements) + elif isinstance(input_channel, dict): # Old format + meta_inputs.append(list(input_channel.keys())[0]) + + return correct_inputs, meta_inputs + + +def obtain_correct_and_specified_outputs(_, module, meta_yaml): + """ + Obtain the dictionary of correct outputs and elements of each output channel. + + Args: + module (object): The module object. + meta_yaml (dict): The meta.yml dictionary. + + Returns: + correct_outputs (dict): A dictionary containing the correct outputs and their elements. + meta_outputs (dict): A dictionary containing the outputs specified in meta.yml. + """ + correct_outputs = {} + for output_channel in module.outputs: + channel_name = list(output_channel.keys())[0] + channel_elements = [] + for element in output_channel[channel_name]: + channel_elements.append(list(element.keys())[0]) + correct_outputs[channel_name] = channel_elements + + meta_outputs = {} + for output_channel in meta_yaml["output"]: + channel_name = list(output_channel.keys())[0] + if isinstance(output_channel[channel_name], list): # Correct format + channel_elements = [] + for element in output_channel[channel_name]: + channel_elements.append(list(element.keys())[0]) + meta_outputs[channel_name] = channel_elements + elif isinstance(output_channel[channel_name], dict): # Old format + meta_outputs[channel_name] = [] + + return correct_outputs, meta_outputs From 070d792821c2b26e8cc560b0acdc69c21cfe535d Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 21 Jun 2024 11:54:34 +0200 Subject: [PATCH 14/96] add test for nf-core modules lint --update-meta-yml --- tests/__snapshots__/test_create_app.ambr | 264 ++++++++++++----------- tests/modules/lint.py | 9 + tests/test_modules.py | 1 + 3 files changed, 144 insertions(+), 130 deletions(-) diff --git a/tests/__snapshots__/test_create_app.ambr b/tests/__snapshots__/test_create_app.ambr index 2ad0772587..ea853c75a3 100644 --- a/tests/__snapshots__/test_create_app.ambr +++ b/tests/__snapshots__/test_create_app.ambr @@ -1405,258 +1405,262 @@ font-weight: 700; } - .terminal-3175764146-matrix { + .terminal-1242773313-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3175764146-title { + .terminal-1242773313-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3175764146-r1 { fill: #c5c8c6 } - .terminal-3175764146-r2 { fill: #e3e3e3 } - .terminal-3175764146-r3 { fill: #989898 } - .terminal-3175764146-r4 { fill: #e1e1e1 } - .terminal-3175764146-r5 { fill: #4ebf71;font-weight: bold } - .terminal-3175764146-r6 { fill: #a5a5a5;font-style: italic; } - .terminal-3175764146-r7 { fill: #454a50 } - .terminal-3175764146-r8 { fill: #e2e3e3;font-weight: bold } - .terminal-3175764146-r9 { fill: #1e1e1e } - .terminal-3175764146-r10 { fill: #008139 } - .terminal-3175764146-r11 { fill: #000000 } - .terminal-3175764146-r12 { fill: #e2e2e2 } - .terminal-3175764146-r13 { fill: #18954b } - .terminal-3175764146-r14 { fill: #e2e2e2;font-weight: bold } - .terminal-3175764146-r15 { fill: #969696;font-weight: bold } - .terminal-3175764146-r16 { fill: #808080 } - .terminal-3175764146-r17 { fill: #7ae998 } - .terminal-3175764146-r18 { fill: #507bb3 } - .terminal-3175764146-r19 { fill: #0a180e;font-weight: bold } - .terminal-3175764146-r20 { fill: #dde6ed;font-weight: bold } - .terminal-3175764146-r21 { fill: #001541 } - .terminal-3175764146-r22 { fill: #fea62b;font-weight: bold } - .terminal-3175764146-r23 { fill: #a7a9ab } - .terminal-3175764146-r24 { fill: #e2e3e3 } + .terminal-1242773313-r1 { fill: #c5c8c6 } + .terminal-1242773313-r2 { fill: #e3e3e3 } + .terminal-1242773313-r3 { fill: #989898 } + .terminal-1242773313-r4 { fill: #e1e1e1 } + .terminal-1242773313-r5 { fill: #4ebf71;font-weight: bold } + .terminal-1242773313-r6 { fill: #18954b } + .terminal-1242773313-r7 { fill: #e2e2e2 } + .terminal-1242773313-r8 { fill: #e2e2e2;font-style: italic; } + .terminal-1242773313-r9 { fill: #e2e2e2;font-style: italic;;text-decoration: underline; } + .terminal-1242773313-r10 { fill: #a5a5a5;font-style: italic; } + .terminal-1242773313-r11 { fill: #1e1e1e } + .terminal-1242773313-r12 { fill: #008139 } + .terminal-1242773313-r13 { fill: #454a50 } + .terminal-1242773313-r14 { fill: #787878 } + .terminal-1242773313-r15 { fill: #e2e3e3;font-weight: bold } + .terminal-1242773313-r16 { fill: #000000 } + .terminal-1242773313-r17 { fill: #b93c5b } + .terminal-1242773313-r18 { fill: #e2e2e2;font-weight: bold } + .terminal-1242773313-r19 { fill: #969696;font-weight: bold } + .terminal-1242773313-r20 { fill: #808080 } + .terminal-1242773313-r21 { fill: #7ae998 } + .terminal-1242773313-r22 { fill: #507bb3 } + .terminal-1242773313-r23 { fill: #0a180e;font-weight: bold } + .terminal-1242773313-r24 { fill: #dde6ed;font-weight: bold } + .terminal-1242773313-r25 { fill: #001541 } + .terminal-1242773313-r26 { fill: #fea62b;font-weight: bold } + .terminal-1242773313-r27 { fill: #a7a9ab } + .terminal-1242773313-r28 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core create + nf-core create - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Create GitHub repository - -   Now that we have created a new pipeline locally, we can create a new GitHub repository and push    -   the code to it. - - - - - Your GitHub usernameYour GitHub personal access token▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - for login. Show  - ▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - ▊GitHub username▎▊••••••••••••                  ▎ - ▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ - - - - The name of the organisation where the The name of the new GitHub repository - GitHub repo will be cretaed - ▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ - ▊nf-core                               ▎▊mypipeline                            ▎ - ▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ - - - ▌⚠️ You can't create a repository directly in the nf-core organisation. - ▌Please create the pipeline repo to an organisation where you have access or use your user  - ▌account. A core-team member will be able to transfer the repo to nf-core once the development - ▌has started. - - ▌💡 Your GitHub user account will be used by default if nf-core is given as the org name. - - - ▊▔▔▔▔▔▔▔▔▎Private - ▊▎Select to make the new GitHub repo private. - ▊▁▁▁▁▁▁▁▁▎ - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Create GitHub repo  Finish without creating a repo  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - - - - - - - - - -  d Toggle dark mode  q Quit  + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + + Create GitHub repository + + Now that we have created a new pipeline locally, we can create a new GitHub repository and push  + the code to it. + + ▌💡 Found GitHub username in local GitHub CLI config + + + + Your GitHub usernameYour GitHub personal access token + ▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎for login.▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊GitHub username▎▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎Show + ▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎▊GitHub token▎▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + ▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ + + + The name of the organisation where the The name of the new GitHub repository + GitHub repo will be cretaed▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ + ▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎▊mypipeline▎ + ▊nf-core▎▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ + ▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ + + + ▌⚠️ You can't create a repository directly in the nf-core organisation. + ▌Please create the pipeline repo to an organisation where you have access or use your user  + ▌account. A core-team member will be able to transfer the repo to nf-core once the development + ▌has started. + + ▌💡 Your GitHub user account will be used by default if nf-core is given as the org name. + + + ▊▔▔▔▔▔▔▔▔▎Private + ▊▎Select to make the new GitHub repo private. + ▊▁▁▁▁▁▁▁▁▎ + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + BackCreate GitHub repoFinish without creating a repo + ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + + + + + + + + + +  d Toggle dark mode q Quit diff --git a/tests/modules/lint.py b/tests/modules/lint.py index e1a4e27ff8..7798c0a948 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -59,6 +59,15 @@ def test_modules_lint_new_modules(self): assert len(module_lint.warned) >= 0 +def test_modules_lint_update_meta_yml(self): + """update the meta.yml of a module""" + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules, update_meta_yml=True) + module_lint.lint(print_results=False, all_modules="fastqc") + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + + def test_modules_lint_no_gitlab(self): """Test linting a pipeline with no modules installed""" self.mods_remove.remove("fastqc", force=True) diff --git a/tests/test_modules.py b/tests/test_modules.py index 107b245663..69778dcf0b 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -214,6 +214,7 @@ def test_modulesrepo_class(self): test_modules_lint_snapshot_file_missing_fail, test_modules_lint_snapshot_file_not_needed, test_modules_lint_trimgalore, + test_modules_lint_update_meta_yml, test_modules_meta_yml_incorrect_licence_field, test_modules_meta_yml_incorrect_name, test_modules_meta_yml_input_mismatch, From 2b705572a1e8a07a40ac7ea7cb7274d7654a21d1 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 4 Jul 2024 11:48:11 +0200 Subject: [PATCH 15/96] remove qualifier --- nf_core/components/nfcore_component.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index dcd76b0e8f..3e5a68648d 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -178,16 +178,13 @@ def get_inputs_from_main_nf(self): regex = r"(val|path)\s*(\(([^)]+)\)|\s*([^)\s,]+))" matches = re.finditer(regex, line) for _, match in enumerate(matches, start=1): - input_type = None input_val = None - if match.group(1): - input_type = match.group(1) if match.group(3): input_val = match.group(3).split(",")[0] # handle `files, stageAs: "inputs/*"` cases elif match.group(4): input_val = match.group(4).split(",")[0] # handle `files, stageAs: "inputs/*"` cases - if input_type and input_val: - channel_elements.append({input_val: {"qualifier": input_type}}) + if input_val: + channel_elements.append({input_val: {}}) if len(channel_elements) > 0: inputs.append(channel_elements) log.debug(f"Found {len(inputs)} inputs in {self.main_nf}") @@ -211,17 +208,14 @@ def get_outputs_from_main_nf(self): continue output_channel = {match_emit.group(1): []} for _, match_element in enumerate(matches_elements, start=1): - output_type = None output_val = None - if match_element.group(1): - output_type = match_element.group(1) if match_element.group(3): output_val = match_element.group(3) elif match_element.group(4): output_val = match_element.group(4) - if output_type and output_val: + if output_val: output_val = output_val.strip("'").strip('"') # remove quotes - output_channel[match_emit.group(1)].append({output_val: {"qualifier": output_type}}) + output_channel[match_emit.group(1)].append({output_val: {}}) outputs.append(output_channel) log.debug(f"Found {len(outputs)} outputs in {self.main_nf}") self.outputs = outputs From 0f49a88ec2354361c37a7bf3cb43bf6ce980c0ea Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 4 Jul 2024 12:18:12 +0200 Subject: [PATCH 16/96] use param --fix to update the meta.yml --- nf_core/__main__.py | 6 ++---- nf_core/commands_modules.py | 4 ++-- nf_core/components/lint/__init__.py | 4 ++-- nf_core/modules/lint/__init__.py | 6 +++--- nf_core/modules/lint/meta_yml.py | 4 ++-- tests/modules/lint.py | 2 +- 6 files changed, 12 insertions(+), 14 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 584947afd0..9ba23a8b08 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1196,10 +1196,8 @@ def command_modules_test(ctx, tool, dir, no_prompts, update, once, profile): is_flag=True, help="Fix the module version if a newer version is available", ) -@click.option( - "--update-meta-yml", is_flag=True, help="Update the meta.yml file with the correct format of input and outputs" -) -def command_modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, passed, sort_by, fix_version, update_meta_yml): +@click.option("--fix", is_flag=True, help="Fix all linting tests if possible.") +def command_modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, passed, sort_by, fix_version, fix): """ Lint one or more modules in a directory. """ diff --git a/nf_core/commands_modules.py b/nf_core/commands_modules.py index 85471184ca..ca80e87bd1 100644 --- a/nf_core/commands_modules.py +++ b/nf_core/commands_modules.py @@ -244,7 +244,7 @@ def modules_test(ctx, tool, dir, no_prompts, update, once, profile): sys.exit(1) -def modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, passed, sort_by, fix_version, update_meta_yml): +def modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, passed, sort_by, fix_version, fix): """ Lint one or more modules in a directory. @@ -261,6 +261,7 @@ def modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, passed, module_lint = ModuleLint( dir, fail_warned=fail_warned, + fix=fix, registry=ctx.params["registry"], remote_url=ctx.obj["modules_repo_url"], branch=ctx.obj["modules_repo_branch"], @@ -277,7 +278,6 @@ def modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, passed, show_passed=passed, sort_by=sort_by, fix_version=fix_version, - update_meta_yml=update_meta_yml, ) if len(module_lint.failed) > 0: sys.exit(1) diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index b4194e768e..0e3aba23f7 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -56,7 +56,7 @@ def __init__( component_type, dir, fail_warned=False, - update_meta_yml=False, + fix=False, remote_url=None, branch=None, no_pull=False, @@ -73,7 +73,7 @@ def __init__( ) self.fail_warned = fail_warned - self.update_meta_yml = update_meta_yml + self.fix = fix self.passed = [] self.warned = [] self.failed = [] diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index f6c2069bda..58a4c47159 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -47,7 +47,7 @@ def __init__( self, dir, fail_warned=False, - update_meta_yml=False, + fix=False, remote_url=None, branch=None, no_pull=False, @@ -58,7 +58,7 @@ def __init__( component_type="modules", dir=dir, fail_warned=fail_warned, - update_meta_yml=update_meta_yml, + fix=fix, remote_url=remote_url, branch=branch, no_pull=no_pull, @@ -224,7 +224,7 @@ def lint_module(self, mod, progress_bar, registry, local=False, fix_version=Fals mod.get_inputs_from_main_nf() mod.get_outputs_from_main_nf() # Update meta.yml file if requested - if self.update_meta_yml: + if self.fix: self.update_meta_yml_file(mod) if self.repo_type == "pipeline" and self.modules_json: diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index 9c074ba731..c3b5f3f283 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -117,7 +117,7 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None module.failed.append( ( "correct_meta_inputs", - f"Incorrect inputs specified in module `meta.yml`. Inputs should contain: {correct_inputs}\nRun `nf-core modules lint --update-meta-yml` to update the `meta.yml` file.", + f"Incorrect inputs specified in module `meta.yml`. Inputs should contain: {correct_inputs}\nRun `nf-core modules lint --fix` to update the `meta.yml` file.", module.meta_yml, ) ) @@ -155,7 +155,7 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None module.failed.append( ( "correct_meta_outputs", - f"Incorrect outputs specified in module `meta.yml`. Outputs should contain: {correct_outputs}\nRun `nf-core modules lint --update-meta-yml` to update the `meta.yml` file.", + f"Incorrect outputs specified in module `meta.yml`. Outputs should contain: {correct_outputs}\nRun `nf-core modules lint --fix` to update the `meta.yml` file.", module.meta_yml, ) ) diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 7798c0a948..9b6c6a78c3 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -61,7 +61,7 @@ def test_modules_lint_new_modules(self): def test_modules_lint_update_meta_yml(self): """update the meta.yml of a module""" - module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules, update_meta_yml=True) + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules, fix=True) module_lint.lint(print_results=False, all_modules="fastqc") assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) > 0 From ef7364dfbc18e8d63271c0be9732d8ac01c45efa Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 4 Jul 2024 12:27:24 +0200 Subject: [PATCH 17/96] create app snapshot from dev branch --- tests/__snapshots__/test_create_app.ambr | 264 +++++++++++------------ 1 file changed, 130 insertions(+), 134 deletions(-) diff --git a/tests/__snapshots__/test_create_app.ambr b/tests/__snapshots__/test_create_app.ambr index ea853c75a3..2ad0772587 100644 --- a/tests/__snapshots__/test_create_app.ambr +++ b/tests/__snapshots__/test_create_app.ambr @@ -1405,262 +1405,258 @@ font-weight: 700; } - .terminal-1242773313-matrix { + .terminal-3175764146-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1242773313-title { + .terminal-3175764146-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1242773313-r1 { fill: #c5c8c6 } - .terminal-1242773313-r2 { fill: #e3e3e3 } - .terminal-1242773313-r3 { fill: #989898 } - .terminal-1242773313-r4 { fill: #e1e1e1 } - .terminal-1242773313-r5 { fill: #4ebf71;font-weight: bold } - .terminal-1242773313-r6 { fill: #18954b } - .terminal-1242773313-r7 { fill: #e2e2e2 } - .terminal-1242773313-r8 { fill: #e2e2e2;font-style: italic; } - .terminal-1242773313-r9 { fill: #e2e2e2;font-style: italic;;text-decoration: underline; } - .terminal-1242773313-r10 { fill: #a5a5a5;font-style: italic; } - .terminal-1242773313-r11 { fill: #1e1e1e } - .terminal-1242773313-r12 { fill: #008139 } - .terminal-1242773313-r13 { fill: #454a50 } - .terminal-1242773313-r14 { fill: #787878 } - .terminal-1242773313-r15 { fill: #e2e3e3;font-weight: bold } - .terminal-1242773313-r16 { fill: #000000 } - .terminal-1242773313-r17 { fill: #b93c5b } - .terminal-1242773313-r18 { fill: #e2e2e2;font-weight: bold } - .terminal-1242773313-r19 { fill: #969696;font-weight: bold } - .terminal-1242773313-r20 { fill: #808080 } - .terminal-1242773313-r21 { fill: #7ae998 } - .terminal-1242773313-r22 { fill: #507bb3 } - .terminal-1242773313-r23 { fill: #0a180e;font-weight: bold } - .terminal-1242773313-r24 { fill: #dde6ed;font-weight: bold } - .terminal-1242773313-r25 { fill: #001541 } - .terminal-1242773313-r26 { fill: #fea62b;font-weight: bold } - .terminal-1242773313-r27 { fill: #a7a9ab } - .terminal-1242773313-r28 { fill: #e2e3e3 } + .terminal-3175764146-r1 { fill: #c5c8c6 } + .terminal-3175764146-r2 { fill: #e3e3e3 } + .terminal-3175764146-r3 { fill: #989898 } + .terminal-3175764146-r4 { fill: #e1e1e1 } + .terminal-3175764146-r5 { fill: #4ebf71;font-weight: bold } + .terminal-3175764146-r6 { fill: #a5a5a5;font-style: italic; } + .terminal-3175764146-r7 { fill: #454a50 } + .terminal-3175764146-r8 { fill: #e2e3e3;font-weight: bold } + .terminal-3175764146-r9 { fill: #1e1e1e } + .terminal-3175764146-r10 { fill: #008139 } + .terminal-3175764146-r11 { fill: #000000 } + .terminal-3175764146-r12 { fill: #e2e2e2 } + .terminal-3175764146-r13 { fill: #18954b } + .terminal-3175764146-r14 { fill: #e2e2e2;font-weight: bold } + .terminal-3175764146-r15 { fill: #969696;font-weight: bold } + .terminal-3175764146-r16 { fill: #808080 } + .terminal-3175764146-r17 { fill: #7ae998 } + .terminal-3175764146-r18 { fill: #507bb3 } + .terminal-3175764146-r19 { fill: #0a180e;font-weight: bold } + .terminal-3175764146-r20 { fill: #dde6ed;font-weight: bold } + .terminal-3175764146-r21 { fill: #001541 } + .terminal-3175764146-r22 { fill: #fea62b;font-weight: bold } + .terminal-3175764146-r23 { fill: #a7a9ab } + .terminal-3175764146-r24 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core create + nf-core create - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Create GitHub repository - - Now that we have created a new pipeline locally, we can create a new GitHub repository and push  - the code to it. - - ▌💡 Found GitHub username in local GitHub CLI config - - - - Your GitHub usernameYour GitHub personal access token - ▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎for login.▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊GitHub username▎▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎Show - ▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎▊GitHub token▎▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - ▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ - - - The name of the organisation where the The name of the new GitHub repository - GitHub repo will be cretaed▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ - ▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎▊mypipeline▎ - ▊nf-core▎▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ - ▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ - - - ▌⚠️ You can't create a repository directly in the nf-core organisation. - ▌Please create the pipeline repo to an organisation where you have access or use your user  - ▌account. A core-team member will be able to transfer the repo to nf-core once the development - ▌has started. - - ▌💡 Your GitHub user account will be used by default if nf-core is given as the org name. - - - ▊▔▔▔▔▔▔▔▔▎Private - ▊▎Select to make the new GitHub repo private. - ▊▁▁▁▁▁▁▁▁▎ - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - BackCreate GitHub repoFinish without creating a repo - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - - - - - - - - - -  d Toggle dark mode q Quit + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + + Create GitHub repository + +   Now that we have created a new pipeline locally, we can create a new GitHub repository and push    +   the code to it. + + + + + Your GitHub usernameYour GitHub personal access token▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + for login. Show  + ▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + ▊GitHub username▎▊••••••••••••                  ▎ + ▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ + + + + The name of the organisation where the The name of the new GitHub repository + GitHub repo will be cretaed + ▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ + ▊nf-core                               ▎▊mypipeline                            ▎ + ▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ + + + ▌⚠️ You can't create a repository directly in the nf-core organisation. + ▌Please create the pipeline repo to an organisation where you have access or use your user  + ▌account. A core-team member will be able to transfer the repo to nf-core once the development + ▌has started. + + ▌💡 Your GitHub user account will be used by default if nf-core is given as the org name. + + + ▊▔▔▔▔▔▔▔▔▎Private + ▊▎Select to make the new GitHub repo private. + ▊▁▁▁▁▁▁▁▁▎ + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +  Back  Create GitHub repo  Finish without creating a repo  + ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + + + + + + + + + +  d Toggle dark mode  q Quit  From c7ff082a46f0693ad5e603fc18a9197e7fcc3e90 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 5 Jul 2024 12:20:20 +0200 Subject: [PATCH 18/96] add fix argument --- nf_core/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 9ba23a8b08..1dd92d6b8f 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1201,7 +1201,7 @@ def command_modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, """ Lint one or more modules in a directory. """ - modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, passed, sort_by, fix_version) + modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, passed, sort_by, fix_version, fix) # nf-core modules info From 7c4fcc119648ba01977535c07fcfd1044b6e6514 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 21 Jun 2024 11:07:36 +0200 Subject: [PATCH 19/96] add tool identifier from bio.tools and intput and output ontologies --- nf_core/components/components_utils.py | 27 ++++++++++++++++++++++++++ nf_core/components/create.py | 4 ++++ nf_core/module-template/meta.yml | 19 +++++++++++++++++- nf_core/modules/lint/__init__.py | 16 +++++++++++++-- nf_core/modules/lint/meta_yml.py | 8 +++++--- requirements.txt | 1 + 6 files changed, 69 insertions(+), 6 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 01650a643d..965b28e16a 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -4,6 +4,7 @@ from typing import List, Optional, Tuple import questionary +import requests import rich.prompt import nf_core.utils @@ -152,3 +153,29 @@ def get_components_to_install(subworkflow_dir: str) -> Tuple[List[str], List[str elif link.startswith("../"): subworkflows.append(name.lower()) return modules, subworkflows + + +def get_biotools_id(tool_name) -> str: + """ + Try to find a bio.tools ID for 'tool' + """ + url = f"https://bio.tools/api/t/?q={tool_name}&format=json" + try: + # Send a GET request to the API + response = requests.get(url) + response.raise_for_status() # Raise an error for bad status codes + # Parse the JSON response + data = response.json() + + # Iterate through the tools in the response to find the tool name + for tool in data["list"]: + if tool["name"].lower() == tool_name: + return tool["biotoolsCURIE"] + + # If the tool name was not found in the response + log.warning(f"Could not find a bio.tools ID for '{tool_name}'") + return "" + + except requests.exceptions.RequestException as e: + log.warning(f"Could not find a bio.tools ID for '{tool_name}': {e}") + return "" diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 5d6c411bdc..426b91c98a 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -20,6 +20,7 @@ import nf_core import nf_core.utils from nf_core.components.components_command import ComponentCommand +from nf_core.components.components_utils import get_biotools_id from nf_core.pipelines.lint_utils import run_prettier_on_file log = logging.getLogger(__name__) @@ -60,6 +61,7 @@ def __init__( self.file_paths: Dict[str, Path] = {} self.not_empty_template = not empty_template self.migrate_pytest = migrate_pytest + self.tool_identifier = "" def create(self): """ @@ -148,6 +150,8 @@ def create(self): if self.component_type == "modules": # Try to find a bioconda package for 'component' self._get_bioconda_tool() + # Try to find a biotools entry for 'component' + self.tool_identifier = get_biotools_id(self.component) # Prompt for GitHub username self._get_username() diff --git a/nf_core/module-template/meta.yml b/nf_core/module-template/meta.yml index 2dd8e74efa..c7c16dcb38 100644 --- a/nf_core/module-template/meta.yml +++ b/nf_core/module-template/meta.yml @@ -20,6 +20,7 @@ tools: tool_dev_url: "{{ tool_dev_url }}" doi: "" licence: {{ tool_licence }} + identifier: {{ tool_identifier }} {% if not_empty_template -%} ## TODO nf-core: Add a description of all of the variables used as input @@ -39,6 +40,14 @@ input: type: file description: {{ 'Sorted BAM/CRAM/SAM file' if not_empty_template else "" }} pattern: {{ '"*.{bam,cram,sam}"' if not_empty_template else "" }} + ontologies: + {% if not_empty_template -%} + - edam: "http://edamontology.org/format_25722" + - edam: "http://edamontology.org/format_2573" + - edam: "http://edamontology.org/format_3462" + {% else %} + - edam: "" + {%- endif %} {% if not_empty_template -%} ## TODO nf-core: Add a description of all of the variables used as output @@ -56,7 +65,7 @@ output: description: | Groovy Map containing sample information e.g. `[ id:'sample1', single_end:false ]` - {% endif %} + {%- endif %} {% if not_empty_template -%} ## TODO nf-core: Delete / customise this example output {%- endif %} @@ -64,6 +73,14 @@ output: type: file description: {{ 'Sorted BAM/CRAM/SAM file' if not_empty_template else "" }} pattern: {{ '"*.{bam,cram,sam}"' if not_empty_template else "" }} + ontologies: + {% if not_empty_template -%} + - edam: "http://edamontology.org/format_25722" + - edam: "http://edamontology.org/format_2573" + - edam: "http://edamontology.org/format_3462" + {% else -%} + - edam: "" + {%- endif %} authors: - "{{ author }}" diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 58a4c47159..44e00e333a 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -11,10 +11,11 @@ import questionary import rich -import yaml +import ruamel.yaml import nf_core.modules.modules_utils import nf_core.utils +from nf_core.components.components_utils import get_biotools_id from nf_core.components.lint import ComponentLint, LintExceptionError, LintResult from nf_core.pipelines.lint_utils import console @@ -253,6 +254,9 @@ def update_meta_yml_file(self, mod): """ meta_yml = self.read_meta_yml(mod) corrected_meta_yml = meta_yml.copy() + yaml = ruamel.yaml.YAML() + yaml.preserve_quotes = True + yaml.indent(mapping=2, sequence=2, offset=2) # Obtain inputs and outputs from main.nf and meta.yml # Used to compare only the structure of channels and elements @@ -332,6 +336,14 @@ def update_meta_yml_file(self, mod): ) break + # Add bio.tools identifier + for i, tool in enumerate(corrected_meta_yml["tools"]): + tool_name = list(tool.keys())[0] + if "identifier" not in tool[tool_name]: + corrected_meta_yml["tools"][i][tool_name]["identifier"] = get_biotools_id( + mod.component_name if "/" not in mod.component_name else mod.component_name.split("/")[0] + ) + with open(mod.meta_yml, "w") as fh: log.info(f"Updating {mod.meta_yml}") - yaml.dump(corrected_meta_yml, fh, sort_keys=False, Dumper=nf_core.utils.custom_yaml_dumper()) + yaml.dump(corrected_meta_yml, fh) diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index c3b5f3f283..5bba74acfb 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import Union -import yaml +import ruamel.yaml from jsonschema import exceptions, validators from nf_core.components.lint import ComponentLint @@ -173,6 +173,8 @@ def read_meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> dict: The `meta.yml` file as a dictionary """ meta_yaml = None + yaml = ruamel.yaml.YAML() + yaml.preserve_quotes = True # Check if we have a patch file, get original file in that case if module.is_patched: lines = ModulesDiffer.try_apply_patch( @@ -183,11 +185,11 @@ def read_meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> reverse=True, ).get("meta.yml") if lines is not None: - meta_yaml = yaml.safe_load("".join(lines)) + meta_yaml = yaml.load("".join(lines)) if meta_yaml is None: try: with open(module.meta_yml) as fh: - meta_yaml = yaml.safe_load(fh) + meta_yaml = yaml.load(fh) except FileNotFoundError: return None return meta_yaml diff --git a/requirements.txt b/requirements.txt index ccfc1bc9c1..a85f4e15c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,3 +24,4 @@ tabulate textual==0.71.0 trogon pdiff +ruamel From e757a214ed677daa542853d9f83aee92e76782b0 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 25 Jun 2024 11:28:23 +0200 Subject: [PATCH 20/96] fix indentation and run prettier on meta.yml files --- nf_core/modules/lint/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 44e00e333a..9314ba07b2 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -17,7 +17,7 @@ import nf_core.utils from nf_core.components.components_utils import get_biotools_id from nf_core.components.lint import ComponentLint, LintExceptionError, LintResult -from nf_core.pipelines.lint_utils import console +from nf_core.pipelines.lint_utils import console, run_prettier_on_file log = logging.getLogger(__name__) @@ -256,7 +256,7 @@ def update_meta_yml_file(self, mod): corrected_meta_yml = meta_yml.copy() yaml = ruamel.yaml.YAML() yaml.preserve_quotes = True - yaml.indent(mapping=2, sequence=2, offset=2) + yaml.indent(mapping=2, sequence=2, offset=0) # Obtain inputs and outputs from main.nf and meta.yml # Used to compare only the structure of channels and elements @@ -347,3 +347,4 @@ def update_meta_yml_file(self, mod): with open(mod.meta_yml, "w") as fh: log.info(f"Updating {mod.meta_yml}") yaml.dump(corrected_meta_yml, fh) + run_prettier_on_file(fh.name) From 7979ced0852c0f66e357bf876b02c2cea8f4e301 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 5 Jul 2024 12:14:56 +0200 Subject: [PATCH 21/96] install correct ruamel library --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a85f4e15c5..61e181cac5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,4 +24,4 @@ tabulate textual==0.71.0 trogon pdiff -ruamel +ruamel.yaml From 297735e06df52450a6a133dade10ad45a3f1799e Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 24 Jul 2024 10:46:34 +0200 Subject: [PATCH 22/96] add --fix option to fix the meta.yml of subworkflows --- nf_core/__main__.py | 5 +- nf_core/commands_subworkflows.py | 3 +- nf_core/components/nfcore_component.py | 142 +++++++++++++++---------- nf_core/subworkflows/lint/__init__.py | 55 +++++++++- nf_core/subworkflows/lint/meta_yml.py | 6 ++ tests/subworkflows/lint.py | 9 ++ tests/test_subworkflows.py | 1 + 7 files changed, 162 insertions(+), 59 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 1dd92d6b8f..dd800ad810 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1431,11 +1431,12 @@ def command_subworkflows_list_local(ctx, keywords, json, dir): # pylint: disabl help="Sort lint output by subworkflow or test name.", show_default=True, ) -def command_subworkflows_lint(ctx, subworkflow, dir, registry, key, all, fail_warned, local, passed, sort_by): +@click.option("--fix", is_flag=True, help="Fix all linting tests if possible.") +def command_subworkflows_lint(ctx, subworkflow, dir, registry, key, all, fail_warned, local, passed, sort_by, fix): """ Lint one or more subworkflows in a directory. """ - subworkflows_lint(ctx, subworkflow, dir, registry, key, all, fail_warned, local, passed, sort_by) + subworkflows_lint(ctx, subworkflow, dir, registry, key, all, fail_warned, local, passed, sort_by, fix) # nf-core subworkflows info diff --git a/nf_core/commands_subworkflows.py b/nf_core/commands_subworkflows.py index cc1a544eca..70a0c94edc 100644 --- a/nf_core/commands_subworkflows.py +++ b/nf_core/commands_subworkflows.py @@ -102,7 +102,7 @@ def subworkflows_list_local(ctx, keywords, json, dir): # pylint: disable=redefi sys.exit(1) -def subworkflows_lint(ctx, subworkflow, dir, registry, key, all, fail_warned, local, passed, sort_by): +def subworkflows_lint(ctx, subworkflow, dir, registry, key, all, fail_warned, local, passed, sort_by, fix): """ Lint one or more subworkflows in a directory. @@ -119,6 +119,7 @@ def subworkflows_lint(ctx, subworkflow, dir, registry, key, all, fail_warned, lo subworkflow_lint = SubworkflowLint( dir, fail_warned=fail_warned, + fix=fix, registry=ctx.params["registry"], remote_url=ctx.obj["modules_repo_url"], branch=ctx.obj["modules_repo_branch"], diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 3e5a68648d..db817db905 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -41,6 +41,7 @@ def __init__( remote_component (bool): Whether the module is to be treated as a nf-core or local component """ + self.component_type = component_type self.component_name = component_name self.repo_url = repo_url self.component_dir = component_dir @@ -155,67 +156,98 @@ def _get_included_components_in_chained_tests(self, main_nf_test: Union[Path, st included_components.append(component) return included_components - def get_inputs_from_main_nf(self): + def get_inputs_from_main_nf(self) -> None: """Collect all inputs from the main.nf file.""" inputs = [] with open(self.main_nf) as f: data = f.read() - # get input values from main.nf after "input:", which can be formatted as tuple val(foo) path(bar) or val foo or val bar or path bar or path foo - # regex matches: - # val(foo) - # path(bar) - # val foo - # val bar - # path bar - # path foo - # don't match anything inside comments or after "output:" - if "input:" not in data: - log.debug(f"Could not find any inputs in {self.main_nf}") - return inputs - input_data = data.split("input:")[1].split("output:")[0] - for line in input_data.split("\n"): - channel_elements = [] - regex = r"(val|path)\s*(\(([^)]+)\)|\s*([^)\s,]+))" - matches = re.finditer(regex, line) - for _, match in enumerate(matches, start=1): - input_val = None - if match.group(3): - input_val = match.group(3).split(",")[0] # handle `files, stageAs: "inputs/*"` cases - elif match.group(4): - input_val = match.group(4).split(",")[0] # handle `files, stageAs: "inputs/*"` cases - if input_val: - channel_elements.append({input_val: {}}) - if len(channel_elements) > 0: - inputs.append(channel_elements) - log.debug(f"Found {len(inputs)} inputs in {self.main_nf}") - self.inputs = inputs + if self.component_type == "modules": + # get input values from main.nf after "input:", which can be formatted as tuple val(foo) path(bar) or val foo or val bar or path bar or path foo + # regex matches: + # val(foo) + # path(bar) + # val foo + # val bar + # path bar + # path foo + # don't match anything inside comments or after "output:" + if "input:" not in data: + log.debug(f"Could not find any inputs in {self.main_nf}") + return inputs + input_data = data.split("input:")[1].split("output:")[0] + for line in input_data.split("\n"): + channel_elements = [] + regex = r"(val|path)\s*(\(([^)]+)\)|\s*([^)\s,]+))" + matches = re.finditer(regex, line) + for _, match in enumerate(matches, start=1): + input_val = None + if match.group(3): + input_val = match.group(3).split(",")[0] # handle `files, stageAs: "inputs/*"` cases + elif match.group(4): + input_val = match.group(4).split(",")[0] # handle `files, stageAs: "inputs/*"` cases + if input_val: + channel_elements.append({input_val: {}}) + if len(channel_elements) > 0: + inputs.append(channel_elements) + log.debug(f"Found {len(inputs)} inputs in {self.main_nf}") + self.inputs = inputs + elif self.component_type == "subworkflows": + # get input values from main.nf after "take:" + if "take:" not in data: + log.debug(f"Could not find any inputs in {self.main_nf}") + return inputs + # get all lines between "take" and "main" or "emit" + input_data = data.split("take:")[1].split("main:")[0].split("emit:")[0] + for line in input_data.split("\n"): + try: + inputs.append(line.split()[0]) + except IndexError: + # Empty lines + pass + log.debug(f"Found {len(inputs)} inputs in {self.main_nf}") + self.inputs = inputs def get_outputs_from_main_nf(self): outputs = [] with open(self.main_nf) as f: data = f.read() - # get output values from main.nf after "output:". the names are always after "emit:" - if "output:" not in data: - log.debug(f"Could not find any outputs in {self.main_nf}") - return outputs - output_data = data.split("output:")[1].split("when:")[0] - regex_emit = r"emit:\s*([^)\s,]+)" - regex_elements = r"(val|path|env|stdout)\s*(\(([^)]+)\)|\s*([^)\s,]+))" - for line in output_data.split("\n"): - match_emit = re.search(regex_emit, line) - matches_elements = re.finditer(regex_elements, line) - if not match_emit: - continue - output_channel = {match_emit.group(1): []} - for _, match_element in enumerate(matches_elements, start=1): - output_val = None - if match_element.group(3): - output_val = match_element.group(3) - elif match_element.group(4): - output_val = match_element.group(4) - if output_val: - output_val = output_val.strip("'").strip('"') # remove quotes - output_channel[match_emit.group(1)].append({output_val: {}}) - outputs.append(output_channel) - log.debug(f"Found {len(outputs)} outputs in {self.main_nf}") - self.outputs = outputs + if self.component_type == "modules": + # get output values from main.nf after "output:". the names are always after "emit:" + if "output:" not in data: + log.debug(f"Could not find any outputs in {self.main_nf}") + return outputs + output_data = data.split("output:")[1].split("when:")[0] + regex_emit = r"emit:\s*([^)\s,]+)" + regex_elements = r"(val|path|env|stdout)\s*(\(([^)]+)\)|\s*([^)\s,]+))" + for line in output_data.split("\n"): + match_emit = re.search(regex_emit, line) + matches_elements = re.finditer(regex_elements, line) + if not match_emit: + continue + output_channel = {match_emit.group(1): []} + for _, match_element in enumerate(matches_elements, start=1): + output_val = None + if match_element.group(3): + output_val = match_element.group(3) + elif match_element.group(4): + output_val = match_element.group(4) + if output_val: + output_val = output_val.strip("'").strip('"') # remove quotes + output_channel[match_emit.group(1)].append({output_val: {}}) + outputs.append(output_channel) + log.debug(f"Found {len(outputs)} outputs in {self.main_nf}") + self.outputs = outputs + elif self.component_type == "subworkflows": + # get output values from main.nf after "emit:". Can be named outputs or not. + if "emit:" not in data: + log.debug(f"Could not find any outputs in {self.main_nf}") + return outputs + output_data = data.split("emit:")[1].split("}")[0] + for line in output_data.split("\n"): + try: + outputs.append(line.split("=")[0].split()[0]) + except IndexError: + # Empty lines + pass + log.debug(f"Found {len(outputs)} outputs in {self.main_nf}") + self.outputs = outputs diff --git a/nf_core/subworkflows/lint/__init__.py b/nf_core/subworkflows/lint/__init__.py index a3cacf2952..e0911e5bd9 100644 --- a/nf_core/subworkflows/lint/__init__.py +++ b/nf_core/subworkflows/lint/__init__.py @@ -11,11 +11,12 @@ import questionary import rich +import ruamel.yaml import nf_core.modules.modules_utils import nf_core.utils from nf_core.components.lint import ComponentLint, LintExceptionError, LintResult -from nf_core.pipelines.lint_utils import console +from nf_core.pipelines.lint_utils import console, run_prettier_on_file log = logging.getLogger(__name__) @@ -38,6 +39,7 @@ def __init__( self, dir, fail_warned=False, + fix=False, remote_url=None, branch=None, no_pull=False, @@ -48,6 +50,7 @@ def __init__( component_type="subworkflows", dir=dir, fail_warned=fail_warned, + fix=fix, remote_url=remote_url, branch=branch, no_pull=no_pull, @@ -207,6 +210,10 @@ def lint_subworkflow(self, swf, progress_bar, registry, local=False): # Otherwise run all the lint tests else: + # Update meta.yml file if requested + if self.fix: + self.update_meta_yml_file(swf) + if self.repo_type == "pipeline" and self.modules_json: # Set correct sha version = self.modules_json.get_subworkflow_version(swf.component_name, swf.repo_url, swf.org) @@ -223,3 +230,49 @@ def lint_subworkflow(self, swf, progress_bar, registry, local=False): self.failed += warned self.failed += [LintResult(swf, *s) for s in swf.failed] + + + def update_meta_yml_file(self, swf): + """ + Update the meta.yml file with the correct inputs and outputs + """ + yaml = ruamel.yaml.YAML() + yaml.preserve_quotes = True + yaml.indent(mapping=2, sequence=2, offset=0) + + # Read meta.yml + with open(swf.meta_yml) as fh: + meta_yaml = yaml.load(fh) + meta_yaml_corrected = meta_yaml.copy() + # Obtain inputs and outputs from main.nf + swf.get_inputs_from_main_nf() + swf.get_outputs_from_main_nf() + + # Compare inputs and add them if missing + if "input" in meta_yaml: + # Delete inputs from meta.yml which are not present in main.nf + meta_yaml_corrected["input"] = [input for input in meta_yaml["input"] if list(input.keys())[0] in swf.inputs] + # Obtain inputs from main.nf missing in meta.yml + inputs_correct = [list(input.keys())[0] for input in meta_yaml_corrected["input"] if list(input.keys())[0] in swf.inputs] + inputs_missing = [input for input in swf.inputs if input not in inputs_correct] + # Add missing inputs to meta.yml + for missing_input in inputs_missing: + meta_yaml_corrected["input"].append({missing_input: {"description": ""}}) + + if "output" in meta_yaml: + # Delete outputs from meta.yml which are not present in main.nf + meta_yaml_corrected["output"] = [output for output in meta_yaml["output"] if list(output.keys())[0] in swf.outputs] + # Obtain output from main.nf missing in meta.yml + outputs_correct = [list(output.keys())[0] for output in meta_yaml_corrected["output"] if list(output.keys())[0] in swf.outputs] + outputs_missing = [output for output in swf.outputs if output not in outputs_correct] + # Add missing inputs to meta.yml + for missing_output in outputs_missing: + meta_yaml_corrected["output"].append({missing_output: {"description": ""}}) + + # Write corrected meta.yml to file + with open(swf.meta_yml, "w") as fh: + log.info(f"Updating {swf.meta_yml}") + yaml.dump(meta_yaml_corrected, fh) + run_prettier_on_file(fh.name) + + diff --git a/nf_core/subworkflows/lint/meta_yml.py b/nf_core/subworkflows/lint/meta_yml.py index 24e75eddbf..633061e247 100644 --- a/nf_core/subworkflows/lint/meta_yml.py +++ b/nf_core/subworkflows/lint/meta_yml.py @@ -1,4 +1,5 @@ import json +import logging from pathlib import Path import jsonschema.validators @@ -6,6 +7,7 @@ import nf_core.components.components_utils +log = logging.getLogger(__name__) def meta_yml(subworkflow_lint_object, subworkflow): """ @@ -65,6 +67,8 @@ def meta_yml(subworkflow_lint_object, subworkflow): subworkflow.passed.append(("meta_input", f"`{input}` specified", subworkflow.meta_yml)) else: subworkflow.failed.append(("meta_input", f"`{input}` missing in `meta.yml`", subworkflow.meta_yml)) + else: + log.debug(f"No inputs specified in subworkflow `main.nf`: {subworkflow.component_name}") if "output" in meta_yaml: meta_output = [list(x.keys())[0] for x in meta_yaml["output"]] @@ -75,6 +79,8 @@ def meta_yml(subworkflow_lint_object, subworkflow): subworkflow.failed.append( ("meta_output", f"`{output}` missing in `meta.yml`", subworkflow.meta_yml) ) + else: + log.debug(f"No outputs specified in subworkflow `main.nf`: {subworkflow.component_name}") # confirm that the name matches the process name in main.nf if meta_yaml["name"].upper() == subworkflow.workflow_name: diff --git a/tests/subworkflows/lint.py b/tests/subworkflows/lint.py index 540f421ad2..35e2c6b519 100644 --- a/tests/subworkflows/lint.py +++ b/tests/subworkflows/lint.py @@ -38,6 +38,15 @@ def test_subworkflows_lint_new_subworkflow(self): assert len(subworkflow_lint.warned) >= 0 +def test_subworkflows_lint_update_meta_yml(self): + """update the meta.yml of a subworkflow""" + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules, fix=True) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + + def test_subworkflows_lint_no_gitlab(self): """Test linting a pipeline with no subworkflows installed""" with pytest.raises(LookupError): diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 786ba53836..cb8bcf7a56 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -159,6 +159,7 @@ def tearDown(self): test_subworkflows_lint_snapshot_file, test_subworkflows_lint_snapshot_file_missing_fail, test_subworkflows_lint_snapshot_file_not_needed, + test_subworkflows_lint_update_meta_yml, ) from .subworkflows.list import ( # type: ignore[misc] test_subworkflows_install_and_list_subworkflows, From a79a4f0e0c4ad3546c39d22ef46ce74e9bc25de8 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 24 Jul 2024 12:12:17 +0200 Subject: [PATCH 23/96] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16db104fc6..e25e8f8948 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ ### Components - The `modules_nfcore` tag in the `main.nf.test` file of modules/subworkflows now displays the organization name in custom modules repositories ([#3005](https://github.com/nf-core/tools/pull/3005)) +- Add option `--fix` to update the `meta.yml` file of subworkflows ([#3077](https://github.com/nf-core/tools/pull/3077)) ### General From 2d2be3877fe04dbc4ddb80cf525be0db13083740 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 24 Jul 2024 16:03:33 +0200 Subject: [PATCH 24/96] fix typing and linting --- nf_core/components/nfcore_component.py | 11 +++++------ nf_core/subworkflows/lint/__init__.py | 21 ++++++++++++++------- nf_core/subworkflows/lint/meta_yml.py | 1 + 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index db817db905..39a4ba51f4 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -158,7 +158,7 @@ def _get_included_components_in_chained_tests(self, main_nf_test: Union[Path, st def get_inputs_from_main_nf(self) -> None: """Collect all inputs from the main.nf file.""" - inputs = [] + inputs: list[list | str] = [] with open(self.main_nf) as f: data = f.read() if self.component_type == "modules": @@ -173,10 +173,10 @@ def get_inputs_from_main_nf(self) -> None: # don't match anything inside comments or after "output:" if "input:" not in data: log.debug(f"Could not find any inputs in {self.main_nf}") - return inputs + return input_data = data.split("input:")[1].split("output:")[0] for line in input_data.split("\n"): - channel_elements = [] + channel_elements: list[dict] = [] regex = r"(val|path)\s*(\(([^)]+)\)|\s*([^)\s,]+))" matches = re.finditer(regex, line) for _, match in enumerate(matches, start=1): @@ -195,15 +195,14 @@ def get_inputs_from_main_nf(self) -> None: # get input values from main.nf after "take:" if "take:" not in data: log.debug(f"Could not find any inputs in {self.main_nf}") - return inputs + return # get all lines between "take" and "main" or "emit" input_data = data.split("take:")[1].split("main:")[0].split("emit:")[0] for line in input_data.split("\n"): try: inputs.append(line.split()[0]) except IndexError: - # Empty lines - pass + pass # Empty lines log.debug(f"Found {len(inputs)} inputs in {self.main_nf}") self.inputs = inputs diff --git a/nf_core/subworkflows/lint/__init__.py b/nf_core/subworkflows/lint/__init__.py index e0911e5bd9..22025d2a07 100644 --- a/nf_core/subworkflows/lint/__init__.py +++ b/nf_core/subworkflows/lint/__init__.py @@ -231,7 +231,6 @@ def lint_subworkflow(self, swf, progress_bar, registry, local=False): self.failed += [LintResult(swf, *s) for s in swf.failed] - def update_meta_yml_file(self, swf): """ Update the meta.yml file with the correct inputs and outputs @@ -251,9 +250,13 @@ def update_meta_yml_file(self, swf): # Compare inputs and add them if missing if "input" in meta_yaml: # Delete inputs from meta.yml which are not present in main.nf - meta_yaml_corrected["input"] = [input for input in meta_yaml["input"] if list(input.keys())[0] in swf.inputs] + meta_yaml_corrected["input"] = [ + input for input in meta_yaml["input"] if list(input.keys())[0] in swf.inputs + ] # Obtain inputs from main.nf missing in meta.yml - inputs_correct = [list(input.keys())[0] for input in meta_yaml_corrected["input"] if list(input.keys())[0] in swf.inputs] + inputs_correct = [ + list(input.keys())[0] for input in meta_yaml_corrected["input"] if list(input.keys())[0] in swf.inputs + ] inputs_missing = [input for input in swf.inputs if input not in inputs_correct] # Add missing inputs to meta.yml for missing_input in inputs_missing: @@ -261,9 +264,15 @@ def update_meta_yml_file(self, swf): if "output" in meta_yaml: # Delete outputs from meta.yml which are not present in main.nf - meta_yaml_corrected["output"] = [output for output in meta_yaml["output"] if list(output.keys())[0] in swf.outputs] + meta_yaml_corrected["output"] = [ + output for output in meta_yaml["output"] if list(output.keys())[0] in swf.outputs + ] # Obtain output from main.nf missing in meta.yml - outputs_correct = [list(output.keys())[0] for output in meta_yaml_corrected["output"] if list(output.keys())[0] in swf.outputs] + outputs_correct = [ + list(output.keys())[0] + for output in meta_yaml_corrected["output"] + if list(output.keys())[0] in swf.outputs + ] outputs_missing = [output for output in swf.outputs if output not in outputs_correct] # Add missing inputs to meta.yml for missing_output in outputs_missing: @@ -274,5 +283,3 @@ def update_meta_yml_file(self, swf): log.info(f"Updating {swf.meta_yml}") yaml.dump(meta_yaml_corrected, fh) run_prettier_on_file(fh.name) - - diff --git a/nf_core/subworkflows/lint/meta_yml.py b/nf_core/subworkflows/lint/meta_yml.py index 633061e247..be282bc453 100644 --- a/nf_core/subworkflows/lint/meta_yml.py +++ b/nf_core/subworkflows/lint/meta_yml.py @@ -9,6 +9,7 @@ log = logging.getLogger(__name__) + def meta_yml(subworkflow_lint_object, subworkflow): """ Lint a ``meta.yml`` file From d295defdd9e7a4b8d3d1da1ae026437ab274ee54 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 5 Aug 2024 11:20:39 +0200 Subject: [PATCH 25/96] add tests to ensure all files are part of a template customisation group --- tests/pipelines/test_create.py | 39 ++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/pipelines/test_create.py b/tests/pipelines/test_create.py index 13fd3b24cd..0d93afb117 100644 --- a/tests/pipelines/test_create.py +++ b/tests/pipelines/test_create.py @@ -7,6 +7,7 @@ import git import jinja2 import yaml +import itertools import nf_core.pipelines.create.create from nf_core.pipelines.create.utils import load_features_yaml @@ -15,6 +16,7 @@ PIPELINE_TEMPLATE_YML = TEST_DATA_DIR / "pipeline_create_template.yml" PIPELINE_TEMPLATE_YML_SKIP = TEST_DATA_DIR / "pipeline_create_template_skip.yml" +PIPELINE_TEMPLATE = Path(nf_core.__file__).parent / "pipeline-template" class NfcoreCreateTest(unittest.TestCase): @@ -134,3 +136,40 @@ def test_pipeline_creation_with_yml_skip(self, tmp_path): assert not (pipeline.outdir / ".github").exists() assert not (pipeline.outdir / "conf" / "igenomes.config").exists() assert not (pipeline.outdir / ".editorconfig").exists() + + def test_template_customisation_all_files_grouping(self): + """Test that all pipeline template files are included in a pipeline customisation group.""" + create_obj = nf_core.pipelines.create.create.PipelineCreate( + template_config=PIPELINE_TEMPLATE_YML_SKIP, + default_branch=self.default_branch, + ) + all_skippable_paths = itertools.chain(*[sp for sp in create_obj.skippable_paths.values()]) + for _, _, files in PIPELINE_TEMPLATE.walk(): + for file in files: + str_path = str(Path(file).relative_to(PIPELINE_TEMPLATE)) + assert str_path in all_skippable_paths, f"Template file `{str_path}` not present in a group for pipeline customisation `PipelineCreate.skippable_paths`." + + def test_template_customisation_all_template_areas(self): + """Check that all groups in `skippable_paths` are template areas.""" + create_obj = nf_core.pipelines.create.create.PipelineCreate( + template_config=PIPELINE_TEMPLATE_YML_SKIP, + default_branch=self.default_branch, + ) + for area in create_obj.skippable_paths.keys(): + if area != "is_nfcore": + assert area in create_obj.template_areas.keys(), f"Customisation template group `{area}` not present in `PipelineCreate.template_areas`." + + def test_template_customisation_all_features_tested(self): + "Check that all customisation groups are tested on CI." + create_obj = nf_core.pipelines.create.create.PipelineCreate( + template_config=PIPELINE_TEMPLATE_YML_SKIP, + default_branch=self.default_branch, + ) + with open(PIPELINE_TEMPLATE_YML_SKIP) as fh: + skip_yaml = yaml.safe_load(fh) + with open(Path(nf_core.__file__).parent.parent / ".github" / "workflows" / "create-test-lint-wf-template.yml") as fh: + ci_workflow = yaml.safe_load(fh) + for area in create_obj.skippable_paths.keys(): + assert area in skip_yaml["skip_features"], f"Customisation template group `{area}` not tested in `tests/data/pipeline_create_template_skip.yml`." + if area != "github": + assert f"template_skip_{area}.yml" in ci_workflow["jobs"]["RunTestWorkflow"]["strategy"]["matrix"]["TEMPLATE"], f"Customisation template group `{area}` not tested in `create-test-lint-wf-template.yml` github workflow." From f700e61932f0ff3bb0b40e0570145ad389a86cab Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 5 Aug 2024 11:28:46 +0200 Subject: [PATCH 26/96] fix linting --- tests/pipelines/test_create.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/pipelines/test_create.py b/tests/pipelines/test_create.py index 0d93afb117..cb8151849d 100644 --- a/tests/pipelines/test_create.py +++ b/tests/pipelines/test_create.py @@ -1,5 +1,6 @@ """Some tests covering the pipeline creation sub command.""" +import itertools import os import unittest from pathlib import Path @@ -7,7 +8,6 @@ import git import jinja2 import yaml -import itertools import nf_core.pipelines.create.create from nf_core.pipelines.create.utils import load_features_yaml @@ -147,7 +147,9 @@ def test_template_customisation_all_files_grouping(self): for _, _, files in PIPELINE_TEMPLATE.walk(): for file in files: str_path = str(Path(file).relative_to(PIPELINE_TEMPLATE)) - assert str_path in all_skippable_paths, f"Template file `{str_path}` not present in a group for pipeline customisation `PipelineCreate.skippable_paths`." + assert ( + str_path in all_skippable_paths + ), f"Template file `{str_path}` not present in a group for pipeline customisation `PipelineCreate.skippable_paths`." def test_template_customisation_all_template_areas(self): """Check that all groups in `skippable_paths` are template areas.""" @@ -157,7 +159,9 @@ def test_template_customisation_all_template_areas(self): ) for area in create_obj.skippable_paths.keys(): if area != "is_nfcore": - assert area in create_obj.template_areas.keys(), f"Customisation template group `{area}` not present in `PipelineCreate.template_areas`." + assert ( + area in create_obj.template_areas.keys() + ), f"Customisation template group `{area}` not present in `PipelineCreate.template_areas`." def test_template_customisation_all_features_tested(self): "Check that all customisation groups are tested on CI." @@ -167,9 +171,16 @@ def test_template_customisation_all_features_tested(self): ) with open(PIPELINE_TEMPLATE_YML_SKIP) as fh: skip_yaml = yaml.safe_load(fh) - with open(Path(nf_core.__file__).parent.parent / ".github" / "workflows" / "create-test-lint-wf-template.yml") as fh: + with open( + Path(nf_core.__file__).parent.parent / ".github" / "workflows" / "create-test-lint-wf-template.yml" + ) as fh: ci_workflow = yaml.safe_load(fh) for area in create_obj.skippable_paths.keys(): - assert area in skip_yaml["skip_features"], f"Customisation template group `{area}` not tested in `tests/data/pipeline_create_template_skip.yml`." + assert ( + area in skip_yaml["skip_features"] + ), f"Customisation template group `{area}` not tested in `tests/data/pipeline_create_template_skip.yml`." if area != "github": - assert f"template_skip_{area}.yml" in ci_workflow["jobs"]["RunTestWorkflow"]["strategy"]["matrix"]["TEMPLATE"], f"Customisation template group `{area}` not tested in `create-test-lint-wf-template.yml` github workflow." + assert ( + f"template_skip_{area}.yml" + in ci_workflow["jobs"]["RunTestWorkflow"]["strategy"]["matrix"]["TEMPLATE"] + ), f"Customisation template group `{area}` not tested in `create-test-lint-wf-template.yml` github workflow." From b8812e8abc220d5022c81198d12f66b84b2fe265 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 5 Aug 2024 11:29:30 +0200 Subject: [PATCH 27/96] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84a3018fed..88d6355693 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ - add option to exclude documentation from pipeline template ([#3130](https://github.com/nf-core/tools/pull/3130)) - add option to exclude test configs from pipeline template ([#3133](https://github.com/nf-core/tools/pull/3133)) - add option to exclude tower.yml from pipeline template ([#3134](https://github.com/nf-core/tools/pull/3134)) +- Add tests to ensure all files are part of a template customisation group and all groups are tested ([#3099](https://github.com/nf-core/tools/pull/3099)) ### Linting From e6507ccd2c2e11a23ac5fcbacb7c5389deb7fbd8 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 28 Aug 2024 09:43:32 +0200 Subject: [PATCH 28/96] rename templatefeatures.yml to template_features.yml --- .github/workflows/create-test-lint-wf-template.yml | 2 +- MANIFEST.in | 2 +- .../create/{templatefeatures.yml => template_features.yml} | 0 nf_core/pipelines/create/utils.py | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename nf_core/pipelines/create/{templatefeatures.yml => template_features.yml} (100%) diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 85ce2230ab..1fb521b4bb 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -44,7 +44,7 @@ jobs: - name: Create Matrix id: create_matrix run: | - echo "matrix=$(yq 'keys | filter(. != "github") | filter(. != "is_nfcore") | filter(. != "test_config") | tojson(0)' nf_core/pipelines/create/templatefeatures.yml)" >> $GITHUB_OUTPUT + echo "matrix=$(yq 'keys | filter(. != "github") | filter(. != "is_nfcore") | filter(. != "test_config") | tojson(0)' nf_core/pipelines/create/template_features.yml)" >> $GITHUB_OUTPUT RunTestWorkflow: runs-on: ${{ matrix.runner }} diff --git a/MANIFEST.in b/MANIFEST.in index 2bec403804..ce2e08c090 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -10,4 +10,4 @@ include nf_core/assets/logo/nf-core-repo-logo-base-darkbg.png include nf_core/assets/logo/placeholder_logo.svg include nf_core/assets/logo/MavenPro-Bold.ttf include nf_core/pipelines/create/create.tcss -include nf_core/pipelines/create/templatefeatures.yml +include nf_core/pipelines/create/template_features.yml diff --git a/nf_core/pipelines/create/templatefeatures.yml b/nf_core/pipelines/create/template_features.yml similarity index 100% rename from nf_core/pipelines/create/templatefeatures.yml rename to nf_core/pipelines/create/template_features.yml diff --git a/nf_core/pipelines/create/utils.py b/nf_core/pipelines/create/utils.py index 0b72c2bcf5..9b331c2a3b 100644 --- a/nf_core/pipelines/create/utils.py +++ b/nf_core/pipelines/create/utils.py @@ -37,7 +37,7 @@ def init_context(value: Dict[str, Any]) -> Iterator[None]: NFCORE_PIPELINE_GLOBAL: bool = True # YAML file describing template features -features_yml_path = Path(nf_core.__file__).parent / "pipelines" / "create" / "templatefeatures.yml" +features_yml_path = Path(nf_core.__file__).parent / "pipelines" / "create" / "template_features.yml" class CreateConfig(NFCoreTemplateConfig): From fcfdbaf08031d94504e1ee7f14bea7b967101591 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 28 Aug 2024 10:12:34 +0200 Subject: [PATCH 29/96] update test to use template_features.yml and add missing files to a feature group --- .../pipelines/create/template_features.yml | 6 +- tests/pipelines/test_create.py | 78 ++++++++----------- 2 files changed, 38 insertions(+), 46 deletions(-) diff --git a/nf_core/pipelines/create/template_features.yml b/nf_core/pipelines/create/template_features.yml index d72e2cff77..a8300700ae 100644 --- a/nf_core/pipelines/create/template_features.yml +++ b/nf_core/pipelines/create/template_features.yml @@ -1,7 +1,8 @@ github: skippable_paths: - - ".github/" + - ".github" - ".gitignore" + - ".gitattributes" short_description: "Skip the creation of a local Git repository." description: "" help_text: "" @@ -282,6 +283,9 @@ changelog: nf_schema: skippable_paths: - "subworkflows/nf-core/utils_nfschema_plugin" + - "nextflow_schema.json" + - "assets/schema_input.json" + - "assets/samplesheet.csv" short_description: "Use nf-schema" description: "Use the nf-schema Nextflow plugin for this pipeline." help_text: | diff --git a/tests/pipelines/test_create.py b/tests/pipelines/test_create.py index cb8151849d..cccd9643f2 100644 --- a/tests/pipelines/test_create.py +++ b/tests/pipelines/test_create.py @@ -1,6 +1,5 @@ """Some tests covering the pipeline creation sub command.""" -import itertools import os import unittest from pathlib import Path @@ -139,48 +138,37 @@ def test_pipeline_creation_with_yml_skip(self, tmp_path): def test_template_customisation_all_files_grouping(self): """Test that all pipeline template files are included in a pipeline customisation group.""" - create_obj = nf_core.pipelines.create.create.PipelineCreate( - template_config=PIPELINE_TEMPLATE_YML_SKIP, - default_branch=self.default_branch, - ) - all_skippable_paths = itertools.chain(*[sp for sp in create_obj.skippable_paths.values()]) - for _, _, files in PIPELINE_TEMPLATE.walk(): + template_features_yml = load_features_yaml() + base_required_files = [ + ".nf-core.yml", + "README.md", + "nextflow.config", + "CITATIONS.md", + "main.nf", + "workflows/pipeline.nf", + ] + all_skipped_files = [] + for feature in template_features_yml.keys(): + if template_features_yml[feature]["skippable_paths"]: + all_skipped_files.extend(template_features_yml[feature]["skippable_paths"]) + + for root, _, files in PIPELINE_TEMPLATE.walk(): for file in files: - str_path = str(Path(file).relative_to(PIPELINE_TEMPLATE)) - assert ( - str_path in all_skippable_paths - ), f"Template file `{str_path}` not present in a group for pipeline customisation `PipelineCreate.skippable_paths`." - - def test_template_customisation_all_template_areas(self): - """Check that all groups in `skippable_paths` are template areas.""" - create_obj = nf_core.pipelines.create.create.PipelineCreate( - template_config=PIPELINE_TEMPLATE_YML_SKIP, - default_branch=self.default_branch, - ) - for area in create_obj.skippable_paths.keys(): - if area != "is_nfcore": - assert ( - area in create_obj.template_areas.keys() - ), f"Customisation template group `{area}` not present in `PipelineCreate.template_areas`." - - def test_template_customisation_all_features_tested(self): - "Check that all customisation groups are tested on CI." - create_obj = nf_core.pipelines.create.create.PipelineCreate( - template_config=PIPELINE_TEMPLATE_YML_SKIP, - default_branch=self.default_branch, - ) - with open(PIPELINE_TEMPLATE_YML_SKIP) as fh: - skip_yaml = yaml.safe_load(fh) - with open( - Path(nf_core.__file__).parent.parent / ".github" / "workflows" / "create-test-lint-wf-template.yml" - ) as fh: - ci_workflow = yaml.safe_load(fh) - for area in create_obj.skippable_paths.keys(): - assert ( - area in skip_yaml["skip_features"] - ), f"Customisation template group `{area}` not tested in `tests/data/pipeline_create_template_skip.yml`." - if area != "github": - assert ( - f"template_skip_{area}.yml" - in ci_workflow["jobs"]["RunTestWorkflow"]["strategy"]["matrix"]["TEMPLATE"] - ), f"Customisation template group `{area}` not tested in `create-test-lint-wf-template.yml` github workflow." + str_path = str((root / file).relative_to(PIPELINE_TEMPLATE)) + if str_path not in base_required_files: + try: + assert ( + str_path in all_skipped_files + ), f"Template file `{str_path}` not present in a group for pipeline customisation in `template_features.yml`." + except AssertionError: + if "/" in str_path: + # Check if the parent directory is in the skipped files + upper_dir_present = False + for i in range(1, len(str_path.split("/"))): + upper_dir = "/".join(str_path.split("/")[:i]) + if upper_dir in all_skipped_files: + upper_dir_present = True + break + assert upper_dir_present, f"Template file `{str_path}` not present in a group for pipeline customisation in `template_features.yml`." + else: + raise From ea52f9bf241c0f7302d385b74123624b65bbe11d Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 28 Aug 2024 10:27:39 +0200 Subject: [PATCH 30/96] fix pipeline linting when skipping nf_schema --- nf_core/pipelines/create/template_features.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nf_core/pipelines/create/template_features.yml b/nf_core/pipelines/create/template_features.yml index a8300700ae..b3b124f622 100644 --- a/nf_core/pipelines/create/template_features.yml +++ b/nf_core/pipelines/create/template_features.yml @@ -292,6 +292,13 @@ nf_schema: [nf-schema](https://nextflow-io.github.io/nf-schema/latest/) is used to validate input parameters based on a JSON schema. It also provides helper functionality to create help messages, get a summary of changed parameters and validate and convert a samplesheet to a channel. + linting: + files_exist: + - "nextflow_schema.json" + schema_params: False + schema_lint: False + schema_description: False + nextflow_config: False nfcore_pipelines: True custom_pipelines: True license: From fc7c1e810542aebf976fa10378c78637f9fa2788 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 28 Aug 2024 10:34:46 +0200 Subject: [PATCH 31/96] switch to os.walk to support <3.10 versions of Python --- tests/pipelines/test_create.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pipelines/test_create.py b/tests/pipelines/test_create.py index cccd9643f2..d72b346903 100644 --- a/tests/pipelines/test_create.py +++ b/tests/pipelines/test_create.py @@ -152,9 +152,9 @@ def test_template_customisation_all_files_grouping(self): if template_features_yml[feature]["skippable_paths"]: all_skipped_files.extend(template_features_yml[feature]["skippable_paths"]) - for root, _, files in PIPELINE_TEMPLATE.walk(): + for root, _, files in os.walk(PIPELINE_TEMPLATE): for file in files: - str_path = str((root / file).relative_to(PIPELINE_TEMPLATE)) + str_path = str((Path(root) / file).relative_to(PIPELINE_TEMPLATE)) if str_path not in base_required_files: try: assert ( From 283284c5ab705c032b6e75850d44fe8a61f7530c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 29 Aug 2024 12:31:47 +0200 Subject: [PATCH 32/96] some more minimal template fixes --- nf_core/pipeline-template/README.md | 6 ++++++ nf_core/pipeline-template/nextflow.config | 9 +++++---- nf_core/pipeline-template/nextflow_schema.json | 12 +++++++----- nf_core/pipelines/create/template_features.yml | 2 +- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 1fa210c0e0..f4cbce19c3 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -7,6 +7,12 @@ +{%- else %} + +

+ {{ name }} +

+ {% endif -%} {% if github_badges -%} [![GitHub Actions CI Status](https://github.com/{{ name }}/actions/workflows/ci.yml/badge.svg)](https://github.com/{{ name }}/actions/workflows/ci.yml) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 7776bc03f1..8201916b3f 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -31,14 +31,14 @@ params { // Boilerplate options outdir = null - publish_dir_mode = 'copy' + {% if modules %}publish_dir_mode = 'copy'{% endif %} {%- if email %} email = null email_on_fail = null plaintext_email = false {%- endif %} - monochrome_logs = false - hook_url = null + {% if modules %}monochrome_logs = false{% endif %} + {% if slackreport or adaptivecard %}hook_url = null{% endif %} {% if nf_schema %}help = false help_full = false show_hidden = false{% endif %} @@ -62,9 +62,10 @@ params { max_cpus = 16 max_time = '240.h' + {%- if nf_schema %} // Schema validation default options validate_params = true - + {% endif %} } {% if modules %} // Load base.config by default for all pipelines diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index c84f652d07..446da25aed 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -184,6 +184,7 @@ "fa_icon": "fas fa-question-circle", "hidden": true }, + {%- if modules %} "publish_dir_mode": { "type": "string", "default": "copy", @@ -192,7 +193,7 @@ "fa_icon": "fas fa-copy", "enum": ["symlink", "rellink", "link", "copy", "copyNoFollow", "move"], "hidden": true - },{% if email %} + },{% endif %}{% if email %} "email_on_fail": { "type": "string", "description": "Email address for completion summary, only when pipeline fails.", @@ -215,21 +216,22 @@ "default": "25.MB", "fa_icon": "fas fa-file-upload", "hidden": true - }, - {% endif %} + },{% endif %} + {%- if modules %} "monochrome_logs": { "type": "boolean", "description": "Do not use coloured log outputs.", "fa_icon": "fas fa-palette", "hidden": true - }, + },{% endif %} + {%- if slackreport or adaptivecard %} "hook_url": { "type": "string", "description": "Incoming hook URL for messaging service", "fa_icon": "fas fa-people-group", "help_text": "Incoming hook URL for messaging service. Currently, MS Teams and Slack are supported.", "hidden": true - }, + },{% endif %} {%- if multiqc %} "multiqc_config": { "type": "string", diff --git a/nf_core/pipelines/create/template_features.yml b/nf_core/pipelines/create/template_features.yml index b3b124f622..744c054075 100644 --- a/nf_core/pipelines/create/template_features.yml +++ b/nf_core/pipelines/create/template_features.yml @@ -27,7 +27,7 @@ github: readme: - "nextflow_badge" nfcore_pipelines: False - custom_pipelines: False + custom_pipelines: True ci: skippable_paths: - ".github/workflows/" From 43b35748a4d4078d080ffbe4f494cbb6f2aed1ff Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 29 Aug 2024 13:49:20 +0200 Subject: [PATCH 33/96] update textual snapshots --- .../__snapshots__/test_create_app.ambr | 511 +++++++++--------- 1 file changed, 256 insertions(+), 255 deletions(-) diff --git a/tests/pipelines/__snapshots__/test_create_app.ambr b/tests/pipelines/__snapshots__/test_create_app.ambr index 8a0f4dd68d..9a7e52288f 100644 --- a/tests/pipelines/__snapshots__/test_create_app.ambr +++ b/tests/pipelines/__snapshots__/test_create_app.ambr @@ -851,257 +851,257 @@ font-weight: 700; } - .terminal-2784035137-matrix { + .terminal-1873624262-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2784035137-title { + .terminal-1873624262-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2784035137-r1 { fill: #c5c8c6 } - .terminal-2784035137-r2 { fill: #e3e3e3 } - .terminal-2784035137-r3 { fill: #989898 } - .terminal-2784035137-r4 { fill: #e1e1e1 } - .terminal-2784035137-r5 { fill: #4ebf71;font-weight: bold } - .terminal-2784035137-r6 { fill: #1e1e1e } - .terminal-2784035137-r7 { fill: #507bb3 } - .terminal-2784035137-r8 { fill: #e2e2e2 } - .terminal-2784035137-r9 { fill: #808080 } - .terminal-2784035137-r10 { fill: #dde6ed;font-weight: bold } - .terminal-2784035137-r11 { fill: #001541 } - .terminal-2784035137-r12 { fill: #0178d4 } - .terminal-2784035137-r13 { fill: #454a50 } - .terminal-2784035137-r14 { fill: #e2e3e3;font-weight: bold } - .terminal-2784035137-r15 { fill: #000000 } - .terminal-2784035137-r16 { fill: #14191f } - .terminal-2784035137-r17 { fill: #e4e4e4 } - .terminal-2784035137-r18 { fill: #7ae998 } - .terminal-2784035137-r19 { fill: #0a180e;font-weight: bold } - .terminal-2784035137-r20 { fill: #008139 } - .terminal-2784035137-r21 { fill: #fea62b;font-weight: bold } - .terminal-2784035137-r22 { fill: #a7a9ab } - .terminal-2784035137-r23 { fill: #e2e3e3 } + .terminal-1873624262-r1 { fill: #c5c8c6 } + .terminal-1873624262-r2 { fill: #e3e3e3 } + .terminal-1873624262-r3 { fill: #989898 } + .terminal-1873624262-r4 { fill: #e1e1e1 } + .terminal-1873624262-r5 { fill: #4ebf71;font-weight: bold } + .terminal-1873624262-r6 { fill: #1e1e1e } + .terminal-1873624262-r7 { fill: #507bb3 } + .terminal-1873624262-r8 { fill: #e2e2e2 } + .terminal-1873624262-r9 { fill: #dde6ed;font-weight: bold } + .terminal-1873624262-r10 { fill: #001541 } + .terminal-1873624262-r11 { fill: #808080 } + .terminal-1873624262-r12 { fill: #14191f } + .terminal-1873624262-r13 { fill: #0178d4 } + .terminal-1873624262-r14 { fill: #454a50 } + .terminal-1873624262-r15 { fill: #e2e3e3;font-weight: bold } + .terminal-1873624262-r16 { fill: #000000 } + .terminal-1873624262-r17 { fill: #e4e4e4 } + .terminal-1873624262-r18 { fill: #7ae998 } + .terminal-1873624262-r19 { fill: #0a180e;font-weight: bold } + .terminal-1873624262-r20 { fill: #008139 } + .terminal-1873624262-r21 { fill: #fea62b;font-weight: bold } + .terminal-1873624262-r22 { fill: #a7a9ab } + .terminal-1873624262-r23 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core create + nf-core create - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Template features - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github CI testsThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - actions for Continuous - Integration (CI)  - testing - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use reference genomesThe pipeline will be  Hide help  - ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - copy of the most ▁▁ - common reference  - genome files from  - iGenomes - - - Nf-core pipelines are configured to use a copy of the most common reference  - genome files. - - By selecting this option, your pipeline will include a configuration file  - specifying the paths to these files. - - The required code to use these files will also be included in the template.  - When the pipeline user provides an appropriate genome key, the pipeline will - automatically download the required reference files. - ▅▅ - For more information about reference genomes in nf-core pipelines, see the  - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github badgesThe README.md file of  Show help  - ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - include GitHub badges - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add configuration The pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - profiles containing  - custom parameters  - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + + Template features + + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Skip the creation of a                                       Show help  + ▊▁▁▁▁▁▁▁▁▎        local Git repository.▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github CI testsThe pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + actions for Continuous + Integration (CI)  + testing + ▄▄ + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Use reference genomesThe pipeline will be  Hide help  + ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + copy of the most  + common reference  + genome files from  + iGenomes + + + Nf-core pipelines are configured to use a copy of the most common reference  + genome files. + + By selecting this option, your pipeline will include a configuration file  + specifying the paths to these files. + + The required code to use these files will also be included in the template.  + When the pipeline user provides an appropriate genome key, the pipeline will + automatically download the required reference files. + ▅▅ + For more information about reference genomes in nf-core pipelines, see the  + + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github badgesThe README.md file of  Show help  + ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + include GitHub badges + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +  Back  Continue  + ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + +  d Toggle dark mode  q Quit  @@ -2233,254 +2233,255 @@ font-weight: 700; } - .terminal-1508200152-matrix { + .terminal-2724681558-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1508200152-title { + .terminal-2724681558-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1508200152-r1 { fill: #c5c8c6 } - .terminal-1508200152-r2 { fill: #e3e3e3 } - .terminal-1508200152-r3 { fill: #989898 } - .terminal-1508200152-r4 { fill: #e1e1e1 } - .terminal-1508200152-r5 { fill: #4ebf71;font-weight: bold } - .terminal-1508200152-r6 { fill: #1e1e1e } - .terminal-1508200152-r7 { fill: #507bb3 } - .terminal-1508200152-r8 { fill: #e2e2e2 } - .terminal-1508200152-r9 { fill: #808080 } - .terminal-1508200152-r10 { fill: #dde6ed;font-weight: bold } - .terminal-1508200152-r11 { fill: #001541 } - .terminal-1508200152-r12 { fill: #454a50 } - .terminal-1508200152-r13 { fill: #7ae998 } - .terminal-1508200152-r14 { fill: #e2e3e3;font-weight: bold } - .terminal-1508200152-r15 { fill: #0a180e;font-weight: bold } - .terminal-1508200152-r16 { fill: #000000 } - .terminal-1508200152-r17 { fill: #008139 } - .terminal-1508200152-r18 { fill: #fea62b;font-weight: bold } - .terminal-1508200152-r19 { fill: #a7a9ab } - .terminal-1508200152-r20 { fill: #e2e3e3 } + .terminal-2724681558-r1 { fill: #c5c8c6 } + .terminal-2724681558-r2 { fill: #e3e3e3 } + .terminal-2724681558-r3 { fill: #989898 } + .terminal-2724681558-r4 { fill: #e1e1e1 } + .terminal-2724681558-r5 { fill: #4ebf71;font-weight: bold } + .terminal-2724681558-r6 { fill: #1e1e1e } + .terminal-2724681558-r7 { fill: #507bb3 } + .terminal-2724681558-r8 { fill: #e2e2e2 } + .terminal-2724681558-r9 { fill: #dde6ed;font-weight: bold } + .terminal-2724681558-r10 { fill: #001541 } + .terminal-2724681558-r11 { fill: #808080 } + .terminal-2724681558-r12 { fill: #14191f } + .terminal-2724681558-r13 { fill: #454a50 } + .terminal-2724681558-r14 { fill: #7ae998 } + .terminal-2724681558-r15 { fill: #e2e3e3;font-weight: bold } + .terminal-2724681558-r16 { fill: #0a180e;font-weight: bold } + .terminal-2724681558-r17 { fill: #000000 } + .terminal-2724681558-r18 { fill: #008139 } + .terminal-2724681558-r19 { fill: #fea62b;font-weight: bold } + .terminal-2724681558-r20 { fill: #a7a9ab } + .terminal-2724681558-r21 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core create + nf-core create - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Template features - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github CI testsThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - actions for Continuous - Integration (CI)  - testing - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use reference genomesThe pipeline will be  Show help  - ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - copy of the most  - common reference  - genome files from  - iGenomes - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github badgesThe README.md file of  Show help  - ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - include GitHub badges - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add configuration The pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - profiles containing  - custom parameters  - requried to run  - nf-core pipelines at  - different institutions - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use code lintersThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include code linters ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - and CI tests to lint  - your code: pre-commit, - editor-config and  - prettier. - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Include citationsInclude pipeline tools Show help  - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + + Template features + + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Skip the creation of a                                       Show help  + ▊▁▁▁▁▁▁▁▁▎        local Git repository.▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github CI testsThe pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + actions for Continuous + Integration (CI)  + testing + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▃▃ + ▊▎        Use reference genomesThe pipeline will be  Show help  + ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + copy of the most  + common reference  + genome files from  + iGenomes + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github badgesThe README.md file of  Show help  + ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + include GitHub badges + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add configuration The pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + profiles containing  + custom parameters  + requried to run  + nf-core pipelines at  + different institutions + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Use code lintersThe pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎include code linters ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + and CI tests to lint  + your code: pre-commit, + editor-config and  + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +  Back  Continue  + ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + +  d Toggle dark mode  q Quit  From a0682c3fa7c9bd2b2b3f16dd06097efa79943b36 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 29 Aug 2024 13:58:30 +0200 Subject: [PATCH 34/96] don't remove .gitignore when unselecting github --- nf_core/pipelines/create/template_features.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/nf_core/pipelines/create/template_features.yml b/nf_core/pipelines/create/template_features.yml index 744c054075..b59a2e51a6 100644 --- a/nf_core/pipelines/create/template_features.yml +++ b/nf_core/pipelines/create/template_features.yml @@ -1,11 +1,18 @@ github: skippable_paths: - ".github" - - ".gitignore" - ".gitattributes" - short_description: "Skip the creation of a local Git repository." - description: "" - help_text: "" + short_description: "Use a GitHub repository." + description: "Create a GitHub repository for the pipeline." + help_text: | + This will create a GitHub repository for the pipeline. + + The repository will include: + - Continuous Integration (CI) tests + - Issues and pull requests templates + + The initialisation of a git repository is required to use the nf-core/tools. + This means that even if you unselect this option, your pipeline will still contain a `.git` directory and `.gitignore` file. linting: files_exist: - ".github/ISSUE_TEMPLATE/bug_report.yml" @@ -13,7 +20,6 @@ github: - ".github/PULL_REQUEST_TEMPLATE.md" - ".github/CONTRIBUTING.md" - ".github/.dockstore.yml" - - ".gitignore" files_unchanged: - ".github/ISSUE_TEMPLATE/bug_report.yml" - ".github/ISSUE_TEMPLATE/config.yml" From 825d7b68b928267c4cfcff6989907dc389165165 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 29 Aug 2024 14:07:59 +0200 Subject: [PATCH 35/96] update textual snapshots --- .../__snapshots__/test_create_app.ambr | 512 +++++++++--------- 1 file changed, 256 insertions(+), 256 deletions(-) diff --git a/tests/pipelines/__snapshots__/test_create_app.ambr b/tests/pipelines/__snapshots__/test_create_app.ambr index 9a7e52288f..6e3009e18e 100644 --- a/tests/pipelines/__snapshots__/test_create_app.ambr +++ b/tests/pipelines/__snapshots__/test_create_app.ambr @@ -851,257 +851,257 @@ font-weight: 700; } - .terminal-1873624262-matrix { + .terminal-1208643741-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1873624262-title { + .terminal-1208643741-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1873624262-r1 { fill: #c5c8c6 } - .terminal-1873624262-r2 { fill: #e3e3e3 } - .terminal-1873624262-r3 { fill: #989898 } - .terminal-1873624262-r4 { fill: #e1e1e1 } - .terminal-1873624262-r5 { fill: #4ebf71;font-weight: bold } - .terminal-1873624262-r6 { fill: #1e1e1e } - .terminal-1873624262-r7 { fill: #507bb3 } - .terminal-1873624262-r8 { fill: #e2e2e2 } - .terminal-1873624262-r9 { fill: #dde6ed;font-weight: bold } - .terminal-1873624262-r10 { fill: #001541 } - .terminal-1873624262-r11 { fill: #808080 } - .terminal-1873624262-r12 { fill: #14191f } - .terminal-1873624262-r13 { fill: #0178d4 } - .terminal-1873624262-r14 { fill: #454a50 } - .terminal-1873624262-r15 { fill: #e2e3e3;font-weight: bold } - .terminal-1873624262-r16 { fill: #000000 } - .terminal-1873624262-r17 { fill: #e4e4e4 } - .terminal-1873624262-r18 { fill: #7ae998 } - .terminal-1873624262-r19 { fill: #0a180e;font-weight: bold } - .terminal-1873624262-r20 { fill: #008139 } - .terminal-1873624262-r21 { fill: #fea62b;font-weight: bold } - .terminal-1873624262-r22 { fill: #a7a9ab } - .terminal-1873624262-r23 { fill: #e2e3e3 } + .terminal-1208643741-r1 { fill: #c5c8c6 } + .terminal-1208643741-r2 { fill: #e3e3e3 } + .terminal-1208643741-r3 { fill: #989898 } + .terminal-1208643741-r4 { fill: #e1e1e1 } + .terminal-1208643741-r5 { fill: #4ebf71;font-weight: bold } + .terminal-1208643741-r6 { fill: #1e1e1e } + .terminal-1208643741-r7 { fill: #507bb3 } + .terminal-1208643741-r8 { fill: #e2e2e2 } + .terminal-1208643741-r9 { fill: #808080 } + .terminal-1208643741-r10 { fill: #dde6ed;font-weight: bold } + .terminal-1208643741-r11 { fill: #001541 } + .terminal-1208643741-r12 { fill: #14191f } + .terminal-1208643741-r13 { fill: #0178d4 } + .terminal-1208643741-r14 { fill: #454a50 } + .terminal-1208643741-r15 { fill: #e2e3e3;font-weight: bold } + .terminal-1208643741-r16 { fill: #000000 } + .terminal-1208643741-r17 { fill: #e4e4e4 } + .terminal-1208643741-r18 { fill: #7ae998 } + .terminal-1208643741-r19 { fill: #0a180e;font-weight: bold } + .terminal-1208643741-r20 { fill: #008139 } + .terminal-1208643741-r21 { fill: #fea62b;font-weight: bold } + .terminal-1208643741-r22 { fill: #a7a9ab } + .terminal-1208643741-r23 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core create + nf-core create - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Template features - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Skip the creation of a                                       Show help  - ▊▁▁▁▁▁▁▁▁▎        local Git repository.▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github CI testsThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - actions for Continuous - Integration (CI)  - testing - ▄▄ - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use reference genomesThe pipeline will be  Hide help  - ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - copy of the most  - common reference  - genome files from  - iGenomes - - - Nf-core pipelines are configured to use a copy of the most common reference  - genome files. - - By selecting this option, your pipeline will include a configuration file  - specifying the paths to these files. - - The required code to use these files will also be included in the template.  - When the pipeline user provides an appropriate genome key, the pipeline will - automatically download the required reference files. - ▅▅ - For more information about reference genomes in nf-core pipelines, see the  - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github badgesThe README.md file of  Show help  - ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - include GitHub badges - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + + Template features + + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Use a GitHub Create a GitHub  Show help  + ▊▁▁▁▁▁▁▁▁▎        repository.repository for the ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + pipeline. + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github CI testsThe pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + actions for Continuous + Integration (CI)  + testing▄▄ + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Use reference genomesThe pipeline will be  Hide help  + ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + copy of the most  + common reference  + genome files from  + iGenomes + + + Nf-core pipelines are configured to use a copy of the most common reference  + genome files. + + By selecting this option, your pipeline will include a configuration file  + specifying the paths to these files. + + The required code to use these files will also be included in the template.  + When the pipeline user provides an appropriate genome key, the pipeline will + automatically download the required reference files. + ▅▅ + For more information about reference genomes in nf-core pipelines, see the  + + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github badgesThe README.md file of  Show help  + ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + include GitHub badges + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +  Back  Continue  + ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + +  d Toggle dark mode  q Quit  @@ -2233,255 +2233,255 @@ font-weight: 700; } - .terminal-2724681558-matrix { + .terminal-3747616910-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2724681558-title { + .terminal-3747616910-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2724681558-r1 { fill: #c5c8c6 } - .terminal-2724681558-r2 { fill: #e3e3e3 } - .terminal-2724681558-r3 { fill: #989898 } - .terminal-2724681558-r4 { fill: #e1e1e1 } - .terminal-2724681558-r5 { fill: #4ebf71;font-weight: bold } - .terminal-2724681558-r6 { fill: #1e1e1e } - .terminal-2724681558-r7 { fill: #507bb3 } - .terminal-2724681558-r8 { fill: #e2e2e2 } - .terminal-2724681558-r9 { fill: #dde6ed;font-weight: bold } - .terminal-2724681558-r10 { fill: #001541 } - .terminal-2724681558-r11 { fill: #808080 } - .terminal-2724681558-r12 { fill: #14191f } - .terminal-2724681558-r13 { fill: #454a50 } - .terminal-2724681558-r14 { fill: #7ae998 } - .terminal-2724681558-r15 { fill: #e2e3e3;font-weight: bold } - .terminal-2724681558-r16 { fill: #0a180e;font-weight: bold } - .terminal-2724681558-r17 { fill: #000000 } - .terminal-2724681558-r18 { fill: #008139 } - .terminal-2724681558-r19 { fill: #fea62b;font-weight: bold } - .terminal-2724681558-r20 { fill: #a7a9ab } - .terminal-2724681558-r21 { fill: #e2e3e3 } + .terminal-3747616910-r1 { fill: #c5c8c6 } + .terminal-3747616910-r2 { fill: #e3e3e3 } + .terminal-3747616910-r3 { fill: #989898 } + .terminal-3747616910-r4 { fill: #e1e1e1 } + .terminal-3747616910-r5 { fill: #4ebf71;font-weight: bold } + .terminal-3747616910-r6 { fill: #1e1e1e } + .terminal-3747616910-r7 { fill: #507bb3 } + .terminal-3747616910-r8 { fill: #e2e2e2 } + .terminal-3747616910-r9 { fill: #808080 } + .terminal-3747616910-r10 { fill: #dde6ed;font-weight: bold } + .terminal-3747616910-r11 { fill: #001541 } + .terminal-3747616910-r12 { fill: #14191f } + .terminal-3747616910-r13 { fill: #454a50 } + .terminal-3747616910-r14 { fill: #7ae998 } + .terminal-3747616910-r15 { fill: #e2e3e3;font-weight: bold } + .terminal-3747616910-r16 { fill: #0a180e;font-weight: bold } + .terminal-3747616910-r17 { fill: #000000 } + .terminal-3747616910-r18 { fill: #008139 } + .terminal-3747616910-r19 { fill: #fea62b;font-weight: bold } + .terminal-3747616910-r20 { fill: #a7a9ab } + .terminal-3747616910-r21 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core create + nf-core create - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Template features - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Skip the creation of a                                       Show help  - ▊▁▁▁▁▁▁▁▁▎        local Git repository.▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github CI testsThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - actions for Continuous - Integration (CI)  - testing - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▃▃ - ▊▎        Use reference genomesThe pipeline will be  Show help  - ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - copy of the most  - common reference  - genome files from  - iGenomes - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github badgesThe README.md file of  Show help  - ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - include GitHub badges - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add configuration The pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - profiles containing  - custom parameters  - requried to run  - nf-core pipelines at  - different institutions - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use code lintersThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include code linters ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - and CI tests to lint  - your code: pre-commit, - editor-config and  - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + + Template features + + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Use a GitHub Create a GitHub  Show help  + ▊▁▁▁▁▁▁▁▁▎        repository.repository for the ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + pipeline. + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github CI testsThe pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + actions for Continuous + Integration (CI)  + testing + ▃▃ + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Use reference genomesThe pipeline will be  Show help  + ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + copy of the most  + common reference  + genome files from  + iGenomes + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github badgesThe README.md file of  Show help  + ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + include GitHub badges + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add configuration The pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + profiles containing  + custom parameters  + requried to run  + nf-core pipelines at  + different institutions + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Use code lintersThe pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎include code linters ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + and CI tests to lint  + your code: pre-commit, + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +  Back  Continue  + ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + +  d Toggle dark mode  q Quit  From 5b9eff5dfb1b579a334bd8b2903eb8bba25a46ce Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 29 Aug 2024 14:14:29 +0200 Subject: [PATCH 36/96] add .gitignore to base required files --- tests/pipelines/test_create.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pipelines/test_create.py b/tests/pipelines/test_create.py index d72b346903..f83cc274fc 100644 --- a/tests/pipelines/test_create.py +++ b/tests/pipelines/test_create.py @@ -140,6 +140,7 @@ def test_template_customisation_all_files_grouping(self): """Test that all pipeline template files are included in a pipeline customisation group.""" template_features_yml = load_features_yaml() base_required_files = [ + ".gitignore", ".nf-core.yml", "README.md", "nextflow.config", From 21e03ec5fd8effad4b48c8a269c2a248596f4086 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 30 Aug 2024 12:42:44 +0200 Subject: [PATCH 37/96] run nf-core lint --release on PRs to master --- nf_core/pipeline-template/.github/workflows/linting.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index ecd5e32f6c..00e8fd302a 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -58,6 +58,15 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_PR_COMMIT: ${{ github.event.pull_request.head.sha }} run: nf-core -l lint_log.txt pipelines lint --dir ${GITHUB_WORKSPACE} --markdown lint_results.md + if: ${{ github.base_ref == 'dev' }} + + - name: Run nf-core pipelines lint --release + env: + GITHUB_COMMENTS_URL: ${{ github.event.pull_request.comments_url }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_PR_COMMIT: ${{ github.event.pull_request.head.sha }} + run: nf-core -l lint_log.txt pipelines lint --release --dir ${GITHUB_WORKSPACE} --markdown lint_results.md + if: ${{ github.base_ref == 'master' }} - name: Save PR number if: ${{ always() }} From 424743eedb2cb6ed0e906e7682fd3e4107aa256f Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 30 Aug 2024 12:45:54 +0200 Subject: [PATCH 38/96] udpate changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e198caa50..eb8b762b1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ - add option to exclude documentation from pipeline template ([#3130](https://github.com/nf-core/tools/pull/3130)) - add option to exclude test configs from pipeline template ([#3133](https://github.com/nf-core/tools/pull/3133)) - add option to exclude tower.yml from pipeline template ([#3134](https://github.com/nf-core/tools/pull/3134)) +- run nf-core lint --release` on PRs to master`([#3148](https://github.com/nf-core/tools/pull/3148)) ### Linting From 1be881e4d0047d1d8d037afdd09c2353ec016213 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 1 Sep 2024 01:36:52 +0000 Subject: [PATCH 39/96] Update pre-commit hook pre-commit/mirrors-mypy to v1.11.2 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4f08d8419d..14a1a597f5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ repos: alias: ec - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.11.1" + rev: "v1.11.2" hooks: - id: mypy additional_dependencies: From d67733cb05348d4430cf78d3fad90afc59dbe0ef Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 1 Sep 2024 03:58:35 +0000 Subject: [PATCH 40/96] Update dependency pytest-textual-snapshot to v1 --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index aa43ee3fe3..04c6372d72 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -16,7 +16,7 @@ types-requests types-setuptools typing_extensions >=4.0.0 pytest-asyncio -pytest-textual-snapshot==0.4.0 +pytest-textual-snapshot==1.0.0 pytest-workflow>=2.0.0 pytest>=8.0.0 ruff From 94d28d177c688c7071a0fc11b27414eae922a486 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 1 Sep 2024 03:58:38 +0000 Subject: [PATCH 41/96] Update pre-commit hook editorconfig-checker/editorconfig-checker.python to v3 --- .pre-commit-config.yaml | 2 +- nf_core/pipeline-template/.pre-commit-config.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4f08d8419d..e1a9e59961 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: - prettier@3.3.3 - repo: https://github.com/editorconfig-checker/editorconfig-checker.python - rev: "2.7.3" + rev: "3.0.3" hooks: - id: editorconfig-checker alias: ec diff --git a/nf_core/pipeline-template/.pre-commit-config.yaml b/nf_core/pipeline-template/.pre-commit-config.yaml index 4dc0f1dcd7..9e9f0e1c4e 100644 --- a/nf_core/pipeline-template/.pre-commit-config.yaml +++ b/nf_core/pipeline-template/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: - prettier@3.2.5 - repo: https://github.com/editorconfig-checker/editorconfig-checker.python - rev: "2.7.3" + rev: "3.0.3" hooks: - id: editorconfig-checker alias: ec From fef0829832068384635a7ea36b956cbf663938aa Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sun, 1 Sep 2024 08:34:03 +0200 Subject: [PATCH 42/96] markdown header --- nf_core/pipeline-template/README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index f4cbce19c3..beb45ed511 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -9,9 +9,7 @@ {%- else %} -

- {{ name }} -

+# {{ name }} {% endif -%} {% if github_badges -%} From 6482287913cfb0776fe1ca00b14e5589c3f1dcfb Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 3 Sep 2024 16:52:11 +0200 Subject: [PATCH 43/96] fix issue where tool and subtool have the same name --- nf_core/components/nfcore_component.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 5f29bea4eb..433dff438f 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -64,12 +64,19 @@ def __init__( self.process_name = "" self.environment_yml: Optional[Path] = Path(self.component_dir, "environment.yml") + component_list = self.component_name.split("/") + name_index = ( len(self.component_dir.parts) - 1 - - self.component_dir.parts[::-1].index(self.component_name.split("/")[0]) + - self.component_dir.parts[::-1].index(component_list[0]) ) + if len(component_list) != 1 and component_list[0] == component_list[1]: + # Handle cases where the subtool has the same name as the tool + name_index -= 1 + repo_dir = self.component_dir.parts[:name_index][-1] + self.org = repo_dir self.nftest_testdir = Path(self.component_dir, "tests") self.nftest_main_nf = Path(self.nftest_testdir, "main.nf.test") From 05c238e6d0f9dbbf784c9e6bd0ba00dc711e998c Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 3 Sep 2024 17:46:09 +0200 Subject: [PATCH 44/96] pre commit --- nf_core/components/nfcore_component.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 433dff438f..0f3cdcdfbd 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -66,17 +66,13 @@ def __init__( component_list = self.component_name.split("/") - name_index = ( - len(self.component_dir.parts) - - 1 - - self.component_dir.parts[::-1].index(component_list[0]) - ) + name_index = len(self.component_dir.parts) - 1 - self.component_dir.parts[::-1].index(component_list[0]) if len(component_list) != 1 and component_list[0] == component_list[1]: # Handle cases where the subtool has the same name as the tool name_index -= 1 repo_dir = self.component_dir.parts[:name_index][-1] - + self.org = repo_dir self.nftest_testdir = Path(self.component_dir, "tests") self.nftest_main_nf = Path(self.nftest_testdir, "main.nf.test") From 736d5e8ce5764555020074cb03d11267fe4eef54 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 4 Sep 2024 11:51:11 +0200 Subject: [PATCH 45/96] Removes all remaining references to max_* params --- .github/workflows/create-test-wf.yml | 2 +- CHANGELOG.md | 2 +- nf_core/pipelines/lint/nextflow_config.py | 16 ++- tests/pipelines/lint/test_nextflow_config.py | 87 ++------------ tests/pipelines/test_launch.py | 118 +++++++++++++++---- 5 files changed, 116 insertions(+), 109 deletions(-) diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 7d69cb7b7b..56c6c822a9 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -75,7 +75,7 @@ jobs: pwd # echo content of current directory ls -la - nextflow run nf-core-testpipeline -profile test,self_hosted_runner --outdir ./results --max_cpus 4 + nextflow run nf-core-testpipeline -profile test,self_hosted_runner --outdir ./results - name: Upload log file artifact if: ${{ always() }} diff --git a/CHANGELOG.md b/CHANGELOG.md index cd3e558153..c5e22af180 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ - add option to exclude test configs from pipeline template ([#3133](https://github.com/nf-core/tools/pull/3133)) - add option to exclude tower.yml from pipeline template ([#3134](https://github.com/nf-core/tools/pull/3134)) - Add tests to ensure all files are part of a template customisation group and all groups are tested ([#3099](https://github.com/nf-core/tools/pull/3099)) +- Replaces the old custom `check_max()` function with the Nextflow native `resourceLimits` directive ([#3037](https://github.com/nf-core/tools/pull/3037)) ### Linting @@ -98,7 +99,6 @@ - Don't cache pip in `linting.yml` ([#2961](https://github.com/nf-core/tools/pull/2961)) - Lint pipelines with the nf-core template version and post comment if it is outdated ([#2978](https://github.com/nf-core/tools/pull/2978)) -- Replaces the old custom `check_max()` function with the Nextflow native `resourceLimits` directive ([#3037](https://github.com/nf-core/tools/pull/3037)) ### General diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index f694a1ab2c..4179517880 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -80,7 +80,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: * ``params.nf_required_version``: The old method for specifying the minimum Nextflow version. Replaced by ``manifest.nextflowVersion`` * ``params.container``: The old method for specifying the dockerhub container address. Replaced by ``process.container`` * ``igenomesIgnore``: Changed to ``igenomes_ignore`` - * ``params.max_cpus``: Old method of specifying the maximum number of CPUs a process can request. Replaced by native Nextflow `resourceLimits`directive. + * ``params.max_cpus``: Old method of specifying the maximum number of CPUs a process can request. Replaced by native Nextflow `resourceLimits`directive in config files. * ``params.max_memory``: Old method of specifying the maximum number of memory can request. Replaced by native Nextflow `resourceLimits`directive. * ``params.max_time``: Old method of specifying the maximum number of CPUs can request. Replaced by native Nextflow `resourceLimits`directive. @@ -149,7 +149,13 @@ def nextflow_config(self) -> Dict[str, List[str]]: ["params.input"], ] # Throw a warning if these are missing - config_warn = [["manifest.mainScript"], ["timeline.file"], ["trace.file"], ["report.file"], ["dag.file"]] + config_warn = [ + ["manifest.mainScript"], + ["timeline.file"], + ["trace.file"], + ["report.file"], + ["dag.file"], + ] # Old depreciated vars - fail if present config_fail_ifdefined = [ "params.nf_required_version", @@ -207,7 +213,11 @@ def nextflow_config(self) -> Dict[str, List[str]]: ) # Remove field that should be ignored according to the linting config - ignore_configs = self.lint_config.get("nextflow_config", []) if self.lint_config is not None else [] + ignore_configs = ( + self.lint_config.get("nextflow_config", []) + if self.lint_config is not None + else [] + ) for cfs in config_fail: for cf in cfs: diff --git a/tests/pipelines/lint/test_nextflow_config.py b/tests/pipelines/lint/test_nextflow_config.py index 3cc9355452..88b5e4503e 100644 --- a/tests/pipelines/lint/test_nextflow_config.py +++ b/tests/pipelines/lint/test_nextflow_config.py @@ -30,8 +30,9 @@ def test_default_values_match(self): result = lint_obj.nextflow_config() assert len(result["failed"]) == 0 assert len(result["warned"]) == 0 - assert "Config default value correct: params.max_cpus" in str(result["passed"]) - assert "Config default value correct: params.validate_params" in str(result["passed"]) + assert "Config default value correct: params.validate_params" in str( + result["passed"] + ) def test_nextflow_config_bad_name_fail(self): """Tests that config variable existence test fails with bad pipeline name""" @@ -69,80 +70,6 @@ def test_nextflow_config_missing_test_profile_failed(self): assert len(result["failed"]) > 0 assert len(result["warned"]) == 0 - def test_default_values_fail(self): - """Test linting fails if the default values in nextflow.config do not match the ones defined in the nextflow_schema.json.""" - # Change the default value of max_cpus in nextflow.config - nf_conf_file = Path(self.new_pipeline) / "nextflow.config" - with open(nf_conf_file) as f: - content = f.read() - fail_content = re.sub(r"\bmax_cpus\s*=\s*16\b", "max_cpus = 0", content) - with open(nf_conf_file, "w") as f: - f.write(fail_content) - # Change the default value of max_memory in nextflow_schema.json - nf_schema_file = Path(self.new_pipeline) / "nextflow_schema.json" - with open(nf_schema_file) as f: - content = f.read() - fail_content = re.sub(r'"default": "128.GB"', '"default": "18.GB"', content) - with open(nf_schema_file, "w") as f: - f.write(fail_content) - lint_obj = nf_core.pipelines.lint.PipelineLint(self.new_pipeline) - lint_obj.load_pipeline_config() - result = lint_obj.nextflow_config() - assert len(result["failed"]) == 2 - assert ( - "Config default value incorrect: `params.max_cpus` is set as `16` in `nextflow_schema.json` but is `0` in `nextflow.config`." - in result["failed"] - ) - assert ( - "Config default value incorrect: `params.max_memory` is set as `18.GB` in `nextflow_schema.json` but is `128.GB` in `nextflow.config`." - in result["failed"] - ) - - def test_catch_params_assignment_in_main_nf(self): - """Test linting fails if main.nf contains an assignment to a parameter from nextflow_schema.json.""" - # Add parameter assignment in main.nf - main_nf_file = Path(self.new_pipeline) / "main.nf" - with open(main_nf_file, "a") as f: - f.write("params.max_time = 42") - lint_obj = nf_core.pipelines.lint.PipelineLint(self.new_pipeline) - lint_obj.load_pipeline_config() - result = lint_obj.nextflow_config() - assert len(result["failed"]) == 1 - assert ( - result["failed"][0] - == "Config default value incorrect: `params.max_time` is set as `240.h` in `nextflow_schema.json` but is `null` in `nextflow.config`." - ) - - def test_allow_params_reference_in_main_nf(self): - """Test linting allows for references like `params.aligner == 'bwa'` in main.nf. The test will detect if the bug mentioned in GitHub-issue #2833 reemerges.""" - # Add parameter reference in main.nf - main_nf_file = Path(self.new_pipeline) / "main.nf" - with open(main_nf_file, "a") as f: - f.write("params.max_time == 42") - lint_obj = nf_core.pipelines.lint.PipelineLint(self.new_pipeline) - lint_obj.load_pipeline_config() - result = lint_obj.nextflow_config() - assert len(result["failed"]) == 0 - - def test_default_values_ignored(self): - """Test ignoring linting of default values.""" - # Add max_cpus to the ignore list - nf_core_yml_path = Path(self.new_pipeline) / ".nf-core.yml" - nf_core_yml = NFCoreYamlConfig( - repository_type="pipeline", lint={"nextflow_config": [{"config_defaults": ["params.max_cpus"]}]} - ) - with open(nf_core_yml_path, "w") as f: - yaml.dump(nf_core_yml.model_dump(), f) - - lint_obj = nf_core.pipelines.lint.PipelineLint(self.new_pipeline) - lint_obj.load_pipeline_config() - lint_obj._load_lint_config() - result = lint_obj.nextflow_config() - assert len(result["failed"]) == 0 - assert len(result["ignored"]) == 1 - assert "Config default value correct: params.max_cpu" not in str(result["passed"]) - assert "Config default ignored: params.max_cpus" in str(result["ignored"]) - def test_default_values_float(self): """Test comparing two float values.""" # Add a float value `dummy=0.0001` to the nextflow.config below `validate_params` @@ -150,7 +77,9 @@ def test_default_values_float(self): with open(nf_conf_file) as f: content = f.read() fail_content = re.sub( - r"validate_params\s*=\s*true", "params.validate_params = true\ndummy = 0.000000001", content + r"validate_params\s*=\s*true", + "params.validate_params = true\ndummy = 0.000000001", + content, ) with open(nf_conf_file, "w") as f: f.write(fail_content) @@ -180,7 +109,9 @@ def test_default_values_float_fail(self): with open(nf_conf_file) as f: content = f.read() fail_content = re.sub( - r"validate_params\s*=\s*true", "params.validate_params = true\ndummy = 0.000000001", content + r"validate_params\s*=\s*true", + "params.validate_params = true\ndummy = 0.000000001", + content, ) with open(nf_conf_file, "w") as f: f.write(fail_content) diff --git a/tests/pipelines/test_launch.py b/tests/pipelines/test_launch.py index d63e7b6dc5..5598fc695b 100644 --- a/tests/pipelines/test_launch.py +++ b/tests/pipelines/test_launch.py @@ -17,9 +17,13 @@ class TestLaunch(TestPipelines): def setUp(self) -> None: super().setUp() self.nf_params_fn = Path(self.pipeline_dir, "nf-params.json") - self.launcher = nf_core.pipelines.launch.Launch(self.pipeline_dir, params_out=self.nf_params_fn) + self.launcher = nf_core.pipelines.launch.Launch( + self.pipeline_dir, params_out=self.nf_params_fn + ) - @mock.patch.object(nf_core.pipelines.launch.Launch, "prompt_web_gui", side_effect=[True]) + @mock.patch.object( + nf_core.pipelines.launch.Launch, "prompt_web_gui", side_effect=[True] + ) @mock.patch.object(nf_core.pipelines.launch.Launch, "launch_web_gui") def test_launch_pipeline(self, mock_webbrowser, mock_lauch_web_gui): """Test the main launch function""" @@ -34,10 +38,14 @@ def test_launch_file_exists(self, mock_confirm): # Try and to launch, return with error assert self.launcher.launch_pipeline() is False - @mock.patch.object(nf_core.pipelines.launch.Launch, "prompt_web_gui", side_effect=[True]) + @mock.patch.object( + nf_core.pipelines.launch.Launch, "prompt_web_gui", side_effect=[True] + ) @mock.patch.object(nf_core.pipelines.launch.Launch, "launch_web_gui") @mock.patch.object(nf_core.pipelines.launch.Confirm, "ask", side_effect=[False]) - def test_launch_file_exists_overwrite(self, mock_webbrowser, mock_lauch_web_gui, mock_confirm): + def test_launch_file_exists_overwrite( + self, mock_webbrowser, mock_lauch_web_gui, mock_confirm + ): """Test that we detect an existing params file and we overwrite it""" # Make an empty params file to be overwritten open(self.nf_params_fn, "a").close() @@ -47,7 +55,14 @@ def test_launch_file_exists_overwrite(self, mock_webbrowser, mock_lauch_web_gui, def test_get_pipeline_schema(self): """Test loading the params schema from a pipeline""" self.launcher.get_pipeline_schema() - assert len(self.launcher.schema_obj.schema["$defs"]["input_output_options"]["properties"]) > 2 + assert ( + len( + self.launcher.schema_obj.schema["$defs"]["input_output_options"][ + "properties" + ] + ) + > 2 + ) @with_temporary_folder def test_make_pipeline_schema(self, tmp_path): @@ -58,10 +73,21 @@ def test_make_pipeline_schema(self, tmp_path): ) create_obj.init_pipeline() Path(test_pipeline_dir, "nextflow_schema.json").unlink() - self.launcher = nf_core.pipelines.launch.Launch(test_pipeline_dir, params_out=self.nf_params_fn) + self.launcher = nf_core.pipelines.launch.Launch( + test_pipeline_dir, params_out=self.nf_params_fn + ) self.launcher.get_pipeline_schema() - assert len(self.launcher.schema_obj.schema["$defs"]["input_output_options"]["properties"]) >= 2 - assert self.launcher.schema_obj.schema["$defs"]["input_output_options"]["properties"]["outdir"] == { + assert ( + len( + self.launcher.schema_obj.schema["$defs"]["input_output_options"][ + "properties" + ] + ) + >= 2 + ) + assert self.launcher.schema_obj.schema["$defs"]["input_output_options"][ + "properties" + ]["outdir"] == { "type": "string", "format": "directory-path", "description": "The output directory where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure.", @@ -91,8 +117,13 @@ def test_nf_merge_schema(self): self.launcher.get_pipeline_schema() self.launcher.set_schema_inputs() self.launcher.merge_nxf_flag_schema() - assert self.launcher.schema_obj.schema["allOf"][0] == {"$ref": "#/$defs/coreNextflow"} - assert "-resume" in self.launcher.schema_obj.schema["$defs"]["coreNextflow"]["properties"] + assert self.launcher.schema_obj.schema["allOf"][0] == { + "$ref": "#/$defs/coreNextflow" + } + assert ( + "-resume" + in self.launcher.schema_obj.schema["$defs"]["coreNextflow"]["properties"] + ) def test_ob_to_questionary_string(self): """Check converting a python dict to a pyenquirer format - simple strings""" @@ -101,14 +132,21 @@ def test_ob_to_questionary_string(self): "default": "data/*{1,2}.fastq.gz", } result = self.launcher.single_param_to_questionary("input", sc_obj) - assert result == {"type": "input", "name": "input", "message": "", "default": "data/*{1,2}.fastq.gz"} + assert result == { + "type": "input", + "name": "input", + "message": "", + "default": "data/*{1,2}.fastq.gz", + } @mock.patch("questionary.unsafe_prompt", side_effect=[{"use_web_gui": "Web based"}]) def test_prompt_web_gui_true(self, mock_prompt): """Check the prompt to launch the web schema or use the cli""" assert self.launcher.prompt_web_gui() is True - @mock.patch("questionary.unsafe_prompt", side_effect=[{"use_web_gui": "Command line"}]) + @mock.patch( + "questionary.unsafe_prompt", side_effect=[{"use_web_gui": "Command line"}] + ) def test_prompt_web_gui_false(self, mock_prompt): """Check the prompt to launch the web schema or use the cli""" assert self.launcher.prompt_web_gui() is False @@ -123,17 +161,23 @@ def test_launch_web_gui_missing_keys(self, mock_poll_nfcore_web_api): assert exc_info.value.args[0].startswith("Web launch response not recognised:") @mock.patch( - "nf_core.utils.poll_nfcore_web_api", side_effect=[{"api_url": "foo", "web_url": "bar", "status": "recieved"}] + "nf_core.utils.poll_nfcore_web_api", + side_effect=[{"api_url": "foo", "web_url": "bar", "status": "recieved"}], ) @mock.patch("webbrowser.open") @mock.patch("nf_core.utils.wait_cli_function") - def test_launch_web_gui(self, mock_poll_nfcore_web_api, mock_webbrowser, mock_wait_cli_function): + def test_launch_web_gui( + self, mock_poll_nfcore_web_api, mock_webbrowser, mock_wait_cli_function + ): """Check the code that opens the web browser""" self.launcher.get_pipeline_schema() self.launcher.merge_nxf_flag_schema() assert self.launcher.launch_web_gui() is None - @mock.patch("nf_core.utils.poll_nfcore_web_api", side_effect=[{"status": "error", "message": "foo"}]) + @mock.patch( + "nf_core.utils.poll_nfcore_web_api", + side_effect=[{"status": "error", "message": "foo"}], + ) def test_get_web_launch_response_error(self, mock_poll_nfcore_web_api): """Test polling the website for a launch response - status error""" with pytest.raises(AssertionError) as exc_info: @@ -145,14 +189,22 @@ def test_get_web_launch_response_unexpected(self, mock_poll_nfcore_web_api): """Test polling the website for a launch response - status error""" with pytest.raises(AssertionError) as exc_info: self.launcher.get_web_launch_response() - assert exc_info.value.args[0].startswith("Web launch GUI returned unexpected status (foo): ") + assert exc_info.value.args[0].startswith( + "Web launch GUI returned unexpected status (foo): " + ) - @mock.patch("nf_core.utils.poll_nfcore_web_api", side_effect=[{"status": "waiting_for_user"}]) + @mock.patch( + "nf_core.utils.poll_nfcore_web_api", + side_effect=[{"status": "waiting_for_user"}], + ) def test_get_web_launch_response_waiting(self, mock_poll_nfcore_web_api): """Test polling the website for a launch response - status waiting_for_user""" assert self.launcher.get_web_launch_response() is False - @mock.patch("nf_core.utils.poll_nfcore_web_api", side_effect=[{"status": "launch_params_complete"}]) + @mock.patch( + "nf_core.utils.poll_nfcore_web_api", + side_effect=[{"status": "launch_params_complete"}], + ) def test_get_web_launch_response_missing_keys(self, mock_poll_nfcore_web_api): """Test polling the website for a launch response - complete, but missing keys""" with pytest.raises(AssertionError) as exc_info: @@ -175,7 +227,9 @@ def test_get_web_launch_response_missing_keys(self, mock_poll_nfcore_web_api): ], ) @mock.patch.object(nf_core.pipelines.launch.Launch, "sanitise_web_response") - def test_get_web_launch_response_valid(self, mock_poll_nfcore_web_api, mock_sanitise): + def test_get_web_launch_response_valid( + self, mock_poll_nfcore_web_api, mock_sanitise + ): """Test polling the website for a launch response - complete, valid response""" self.launcher.get_pipeline_schema() assert self.launcher.get_web_launch_response() is True @@ -185,11 +239,9 @@ def test_sanitise_web_response(self): self.launcher.get_pipeline_schema() self.launcher.nxf_flags["-name"] = "" self.launcher.schema_obj.input_params["igenomes_ignore"] = "true" - self.launcher.schema_obj.input_params["max_cpus"] = "12" self.launcher.sanitise_web_response() assert "-name" not in self.launcher.nxf_flags assert self.launcher.schema_obj.input_params["igenomes_ignore"] is True - assert self.launcher.schema_obj.input_params["max_cpus"] == 12 def test_ob_to_questionary_bool(self): """Check converting a python dict to a pyenquirer format - booleans""" @@ -262,7 +314,10 @@ def test_ob_to_questionary_enum(self): def test_ob_to_questionary_pattern(self): """Check converting a python dict to a questionary format - with pattern""" - sc_obj = {"type": "string", "pattern": "^([a-zA-Z0-9_\\-\\.]+)@([a-zA-Z0-9_\\-\\.]+)\\.([a-zA-Z]{2,5})$"} + sc_obj = { + "type": "string", + "pattern": "^([a-zA-Z0-9_\\-\\.]+)@([a-zA-Z0-9_\\-\\.]+)\\.([a-zA-Z]{2,5})$", + } result = self.launcher.single_param_to_questionary("email", sc_obj) assert result["type"] == "input" assert result["validate"]("test@email.com") is True @@ -295,7 +350,10 @@ def test_build_command_nf(self): self.launcher.nxf_flags["-name"] = "Test_Workflow" self.launcher.nxf_flags["-resume"] = True self.launcher.build_command() - assert self.launcher.nextflow_cmd == f'nextflow run {self.pipeline_dir} -name "Test_Workflow" -resume' + assert ( + self.launcher.nextflow_cmd + == f'nextflow run {self.pipeline_dir} -name "Test_Workflow" -resume' + ) def test_build_command_params(self): """Test the functionality to build a nextflow command - params supplied""" @@ -303,13 +361,18 @@ def test_build_command_params(self): self.launcher.schema_obj.input_params.update({"input": "custom_input"}) self.launcher.build_command() # Check command - assert self.launcher.nextflow_cmd == f'nextflow run {self.pipeline_dir} -params-file "{self.nf_params_fn}"' + assert ( + self.launcher.nextflow_cmd + == f'nextflow run {self.pipeline_dir} -params-file "{self.nf_params_fn}"' + ) # Check saved parameters file with open(self.nf_params_fn) as fh: try: saved_json = json.load(fh) except json.JSONDecodeError as e: - raise UserWarning(f"Unable to load JSON file '{self.nf_params_fn}' due to error {e}") + raise UserWarning( + f"Unable to load JSON file '{self.nf_params_fn}' due to error {e}" + ) assert saved_json == {"input": "custom_input"} @@ -319,4 +382,7 @@ def test_build_command_params_cl(self): self.launcher.get_pipeline_schema() self.launcher.schema_obj.input_params.update({"input": "custom_input"}) self.launcher.build_command() - assert self.launcher.nextflow_cmd == f'nextflow run {self.pipeline_dir} --input "custom_input"' + assert ( + self.launcher.nextflow_cmd + == f'nextflow run {self.pipeline_dir} --input "custom_input"' + ) From 556336034ffc1107a2c212fc2cd326a57953e150 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 4 Sep 2024 09:57:17 +0000 Subject: [PATCH 46/96] [automated] Fix code linting --- nf_core/pipelines/lint/nextflow_config.py | 89 +++++--------------- tests/pipelines/lint/test_nextflow_config.py | 7 +- tests/pipelines/test_launch.py | 86 ++++--------------- 3 files changed, 40 insertions(+), 142 deletions(-) diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index 4179517880..14e91cc609 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -213,11 +213,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: ) # Remove field that should be ignored according to the linting config - ignore_configs = ( - self.lint_config.get("nextflow_config", []) - if self.lint_config is not None - else [] - ) + ignore_configs = self.lint_config.get("nextflow_config", []) if self.lint_config is not None else [] for cfs in config_fail: for cf in cfs: @@ -244,13 +240,9 @@ def nextflow_config(self) -> Dict[str, List[str]]: ignored.append(f"Config variable ignored: {self._wrap_quotes(cf)}") break if cf not in self.nf_config.keys(): - passed.append( - f"Config variable (correctly) not found: {self._wrap_quotes(cf)}" - ) + passed.append(f"Config variable (correctly) not found: {self._wrap_quotes(cf)}") else: - failed.append( - f"Config variable (incorrectly) found: {self._wrap_quotes(cf)}" - ) + failed.append(f"Config variable (incorrectly) found: {self._wrap_quotes(cf)}") # Check and warn if the process configuration is done with deprecated syntax @@ -271,13 +263,9 @@ def nextflow_config(self) -> Dict[str, List[str]]: if k in ignore_configs: continue if self.nf_config.get(k) == "true": - passed.append( - f"Config ``{k}`` had correct value: ``{self.nf_config.get(k)}``" - ) + passed.append(f"Config ``{k}`` had correct value: ``{self.nf_config.get(k)}``") else: - failed.append( - f"Config ``{k}`` did not have correct value: ``{self.nf_config.get(k)}``" - ) + failed.append(f"Config ``{k}`` did not have correct value: ``{self.nf_config.get(k)}``") if "manifest.name" not in ignore_configs: # Check that the pipeline name starts with nf-core @@ -286,9 +274,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: if not manifest_name.startswith("nf-core/"): raise AssertionError() except (AssertionError, IndexError): - failed.append( - f"Config ``manifest.name`` did not begin with ``nf-core/``:\n {manifest_name}" - ) + failed.append(f"Config ``manifest.name`` did not begin with ``nf-core/``:\n {manifest_name}") else: passed.append("Config ``manifest.name`` began with ``nf-core/``") @@ -304,9 +290,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: ) else: - passed.append( - "Config variable ``manifest.homePage`` began with https://github.com/nf-core/" - ) + passed.append("Config variable ``manifest.homePage`` began with https://github.com/nf-core/") # Check that the DAG filename ends in ``.svg`` if "dag.file" in self.nf_config: @@ -314,21 +298,12 @@ def nextflow_config(self) -> Dict[str, List[str]]: if self.nf_config["dag.file"].strip("'\"").endswith(default_dag_format): passed.append(f"Config ``dag.file`` ended with ``{default_dag_format}``") else: - failed.append( - f"Config ``dag.file`` did not end with ``{default_dag_format}``" - ) + failed.append(f"Config ``dag.file`` did not end with ``{default_dag_format}``") # Check that the minimum nextflowVersion is set properly if "manifest.nextflowVersion" in self.nf_config: - if ( - self.nf_config.get("manifest.nextflowVersion", "") - .strip("\"'") - .lstrip("!") - .startswith(">=") - ): - passed.append( - "Config variable ``manifest.nextflowVersion`` started with >= or !>=" - ) + if self.nf_config.get("manifest.nextflowVersion", "").strip("\"'").lstrip("!").startswith(">="): + passed.append("Config variable ``manifest.nextflowVersion`` started with >= or !>=") else: failed.append( "Config ``manifest.nextflowVersion`` did not start with ``>=`` or ``!>=`` : " @@ -338,9 +313,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: # Check that the pipeline version contains ``dev`` if not self.release_mode and "manifest.version" in self.nf_config: if self.nf_config["manifest.version"].strip(" '\"").endswith("dev"): - passed.append( - f"Config ``manifest.version`` ends in ``dev``: ``{self.nf_config['manifest.version']}``" - ) + passed.append(f"Config ``manifest.version`` ends in ``dev``: ``{self.nf_config['manifest.version']}``") else: warned.append( f"Config ``manifest.version`` should end in ``dev``: ``{self.nf_config['manifest.version']}``" @@ -359,32 +332,18 @@ def nextflow_config(self) -> Dict[str, List[str]]: if "custom_config" not in ignore_configs: # Check if custom profile params are set correctly - if ( - self.nf_config.get("params.custom_config_version", "").strip("'") - == "master" - ): + if self.nf_config.get("params.custom_config_version", "").strip("'") == "master": passed.append("Config `params.custom_config_version` is set to `master`") else: - failed.append( - "Config `params.custom_config_version` is not set to `master`" - ) + failed.append("Config `params.custom_config_version` is not set to `master`") - custom_config_base = ( - "https://raw.githubusercontent.com/nf-core/configs/{}".format( - self.nf_config.get("params.custom_config_version", "").strip("'") - ) + custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/{}".format( + self.nf_config.get("params.custom_config_version", "").strip("'") ) - if ( - self.nf_config.get("params.custom_config_base", "").strip("'") - == custom_config_base - ): - passed.append( - f"Config `params.custom_config_base` is set to `{custom_config_base}`" - ) + if self.nf_config.get("params.custom_config_base", "").strip("'") == custom_config_base: + passed.append(f"Config `params.custom_config_base` is set to `{custom_config_base}`") else: - failed.append( - f"Config `params.custom_config_base` is not set to `{custom_config_base}`" - ) + failed.append(f"Config `params.custom_config_base` is not set to `{custom_config_base}`") # Check that lines for loading custom profiles exist lines = [ @@ -426,9 +385,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: match = re.search(r"\bprofiles\s*{", cleaned_content) if not match: - failed.append( - "nextflow.config does not contain `profiles` scope, but `test` profile is required" - ) + failed.append("nextflow.config does not contain `profiles` scope, but `test` profile is required") else: # Extract profiles scope content and check for test profile start = match.end() @@ -444,9 +401,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: if re.search(r"\btest\s*{", profiles_content): passed.append("nextflow.config contains configuration profile `test`") else: - failed.append( - "nextflow.config does not contain configuration profile `test`" - ) + failed.append("nextflow.config does not contain configuration profile `test`") # Check that the default values in nextflow.config match the default values defined in the nextflow_schema.json ignore_defaults = [] @@ -490,9 +445,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: schema_default = str(schema.schema_defaults[param_name]) config_default = str(self.nf_config[param]) if config_default is not None and config_default == schema_default: - passed.append( - f"Config default value correct: {param}= {schema_default}" - ) + passed.append(f"Config default value correct: {param}= {schema_default}") else: failed.append( f"Config default value incorrect: `{param}` is set as {self._wrap_quotes(schema_default)} in `nextflow_schema.json` but is {self._wrap_quotes(self.nf_config[param])} in `nextflow.config`." diff --git a/tests/pipelines/lint/test_nextflow_config.py b/tests/pipelines/lint/test_nextflow_config.py index 88b5e4503e..4c3893f4c4 100644 --- a/tests/pipelines/lint/test_nextflow_config.py +++ b/tests/pipelines/lint/test_nextflow_config.py @@ -2,11 +2,8 @@ import re from pathlib import Path -import yaml - import nf_core.pipelines.create.create import nf_core.pipelines.lint -from nf_core.utils import NFCoreYamlConfig from ..test_lint import TestLint @@ -30,9 +27,7 @@ def test_default_values_match(self): result = lint_obj.nextflow_config() assert len(result["failed"]) == 0 assert len(result["warned"]) == 0 - assert "Config default value correct: params.validate_params" in str( - result["passed"] - ) + assert "Config default value correct: params.validate_params" in str(result["passed"]) def test_nextflow_config_bad_name_fail(self): """Tests that config variable existence test fails with bad pipeline name""" diff --git a/tests/pipelines/test_launch.py b/tests/pipelines/test_launch.py index 5598fc695b..5e230528a7 100644 --- a/tests/pipelines/test_launch.py +++ b/tests/pipelines/test_launch.py @@ -17,13 +17,9 @@ class TestLaunch(TestPipelines): def setUp(self) -> None: super().setUp() self.nf_params_fn = Path(self.pipeline_dir, "nf-params.json") - self.launcher = nf_core.pipelines.launch.Launch( - self.pipeline_dir, params_out=self.nf_params_fn - ) + self.launcher = nf_core.pipelines.launch.Launch(self.pipeline_dir, params_out=self.nf_params_fn) - @mock.patch.object( - nf_core.pipelines.launch.Launch, "prompt_web_gui", side_effect=[True] - ) + @mock.patch.object(nf_core.pipelines.launch.Launch, "prompt_web_gui", side_effect=[True]) @mock.patch.object(nf_core.pipelines.launch.Launch, "launch_web_gui") def test_launch_pipeline(self, mock_webbrowser, mock_lauch_web_gui): """Test the main launch function""" @@ -38,14 +34,10 @@ def test_launch_file_exists(self, mock_confirm): # Try and to launch, return with error assert self.launcher.launch_pipeline() is False - @mock.patch.object( - nf_core.pipelines.launch.Launch, "prompt_web_gui", side_effect=[True] - ) + @mock.patch.object(nf_core.pipelines.launch.Launch, "prompt_web_gui", side_effect=[True]) @mock.patch.object(nf_core.pipelines.launch.Launch, "launch_web_gui") @mock.patch.object(nf_core.pipelines.launch.Confirm, "ask", side_effect=[False]) - def test_launch_file_exists_overwrite( - self, mock_webbrowser, mock_lauch_web_gui, mock_confirm - ): + def test_launch_file_exists_overwrite(self, mock_webbrowser, mock_lauch_web_gui, mock_confirm): """Test that we detect an existing params file and we overwrite it""" # Make an empty params file to be overwritten open(self.nf_params_fn, "a").close() @@ -55,14 +47,7 @@ def test_launch_file_exists_overwrite( def test_get_pipeline_schema(self): """Test loading the params schema from a pipeline""" self.launcher.get_pipeline_schema() - assert ( - len( - self.launcher.schema_obj.schema["$defs"]["input_output_options"][ - "properties" - ] - ) - > 2 - ) + assert len(self.launcher.schema_obj.schema["$defs"]["input_output_options"]["properties"]) > 2 @with_temporary_folder def test_make_pipeline_schema(self, tmp_path): @@ -73,21 +58,10 @@ def test_make_pipeline_schema(self, tmp_path): ) create_obj.init_pipeline() Path(test_pipeline_dir, "nextflow_schema.json").unlink() - self.launcher = nf_core.pipelines.launch.Launch( - test_pipeline_dir, params_out=self.nf_params_fn - ) + self.launcher = nf_core.pipelines.launch.Launch(test_pipeline_dir, params_out=self.nf_params_fn) self.launcher.get_pipeline_schema() - assert ( - len( - self.launcher.schema_obj.schema["$defs"]["input_output_options"][ - "properties" - ] - ) - >= 2 - ) - assert self.launcher.schema_obj.schema["$defs"]["input_output_options"][ - "properties" - ]["outdir"] == { + assert len(self.launcher.schema_obj.schema["$defs"]["input_output_options"]["properties"]) >= 2 + assert self.launcher.schema_obj.schema["$defs"]["input_output_options"]["properties"]["outdir"] == { "type": "string", "format": "directory-path", "description": "The output directory where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure.", @@ -117,13 +91,8 @@ def test_nf_merge_schema(self): self.launcher.get_pipeline_schema() self.launcher.set_schema_inputs() self.launcher.merge_nxf_flag_schema() - assert self.launcher.schema_obj.schema["allOf"][0] == { - "$ref": "#/$defs/coreNextflow" - } - assert ( - "-resume" - in self.launcher.schema_obj.schema["$defs"]["coreNextflow"]["properties"] - ) + assert self.launcher.schema_obj.schema["allOf"][0] == {"$ref": "#/$defs/coreNextflow"} + assert "-resume" in self.launcher.schema_obj.schema["$defs"]["coreNextflow"]["properties"] def test_ob_to_questionary_string(self): """Check converting a python dict to a pyenquirer format - simple strings""" @@ -144,9 +113,7 @@ def test_prompt_web_gui_true(self, mock_prompt): """Check the prompt to launch the web schema or use the cli""" assert self.launcher.prompt_web_gui() is True - @mock.patch( - "questionary.unsafe_prompt", side_effect=[{"use_web_gui": "Command line"}] - ) + @mock.patch("questionary.unsafe_prompt", side_effect=[{"use_web_gui": "Command line"}]) def test_prompt_web_gui_false(self, mock_prompt): """Check the prompt to launch the web schema or use the cli""" assert self.launcher.prompt_web_gui() is False @@ -166,9 +133,7 @@ def test_launch_web_gui_missing_keys(self, mock_poll_nfcore_web_api): ) @mock.patch("webbrowser.open") @mock.patch("nf_core.utils.wait_cli_function") - def test_launch_web_gui( - self, mock_poll_nfcore_web_api, mock_webbrowser, mock_wait_cli_function - ): + def test_launch_web_gui(self, mock_poll_nfcore_web_api, mock_webbrowser, mock_wait_cli_function): """Check the code that opens the web browser""" self.launcher.get_pipeline_schema() self.launcher.merge_nxf_flag_schema() @@ -189,9 +154,7 @@ def test_get_web_launch_response_unexpected(self, mock_poll_nfcore_web_api): """Test polling the website for a launch response - status error""" with pytest.raises(AssertionError) as exc_info: self.launcher.get_web_launch_response() - assert exc_info.value.args[0].startswith( - "Web launch GUI returned unexpected status (foo): " - ) + assert exc_info.value.args[0].startswith("Web launch GUI returned unexpected status (foo): ") @mock.patch( "nf_core.utils.poll_nfcore_web_api", @@ -227,9 +190,7 @@ def test_get_web_launch_response_missing_keys(self, mock_poll_nfcore_web_api): ], ) @mock.patch.object(nf_core.pipelines.launch.Launch, "sanitise_web_response") - def test_get_web_launch_response_valid( - self, mock_poll_nfcore_web_api, mock_sanitise - ): + def test_get_web_launch_response_valid(self, mock_poll_nfcore_web_api, mock_sanitise): """Test polling the website for a launch response - complete, valid response""" self.launcher.get_pipeline_schema() assert self.launcher.get_web_launch_response() is True @@ -350,10 +311,7 @@ def test_build_command_nf(self): self.launcher.nxf_flags["-name"] = "Test_Workflow" self.launcher.nxf_flags["-resume"] = True self.launcher.build_command() - assert ( - self.launcher.nextflow_cmd - == f'nextflow run {self.pipeline_dir} -name "Test_Workflow" -resume' - ) + assert self.launcher.nextflow_cmd == f'nextflow run {self.pipeline_dir} -name "Test_Workflow" -resume' def test_build_command_params(self): """Test the functionality to build a nextflow command - params supplied""" @@ -361,18 +319,13 @@ def test_build_command_params(self): self.launcher.schema_obj.input_params.update({"input": "custom_input"}) self.launcher.build_command() # Check command - assert ( - self.launcher.nextflow_cmd - == f'nextflow run {self.pipeline_dir} -params-file "{self.nf_params_fn}"' - ) + assert self.launcher.nextflow_cmd == f'nextflow run {self.pipeline_dir} -params-file "{self.nf_params_fn}"' # Check saved parameters file with open(self.nf_params_fn) as fh: try: saved_json = json.load(fh) except json.JSONDecodeError as e: - raise UserWarning( - f"Unable to load JSON file '{self.nf_params_fn}' due to error {e}" - ) + raise UserWarning(f"Unable to load JSON file '{self.nf_params_fn}' due to error {e}") assert saved_json == {"input": "custom_input"} @@ -382,7 +335,4 @@ def test_build_command_params_cl(self): self.launcher.get_pipeline_schema() self.launcher.schema_obj.input_params.update({"input": "custom_input"}) self.launcher.build_command() - assert ( - self.launcher.nextflow_cmd - == f'nextflow run {self.pipeline_dir} --input "custom_input"' - ) + assert self.launcher.nextflow_cmd == f'nextflow run {self.pipeline_dir} --input "custom_input"' From 6633769fa4cf5c6f9a84745a9424fb07443b5a90 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 4 Sep 2024 12:06:55 +0200 Subject: [PATCH 47/96] Fix linting fail --- nf_core/pipeline-template/nextflow_schema.json | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index f75354d4e1..818adf9619 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -135,6 +135,7 @@ } } }, + {%- endif %} "generic_options": { "title": "Generic options", "type": "object", From d819232314df5ef888ea84ed366253f0f3ddf3ae Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 4 Sep 2024 12:10:36 +0200 Subject: [PATCH 48/96] Fix schema error --- nf_core/pipeline-template/nextflow_schema.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 818adf9619..4136a0b490 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -243,9 +243,6 @@ {% if nf_core_configs %}{ "$ref": "#/$defs/institutional_config_options" },{% endif %} - { - "$ref": "#/$defs/max_job_request_options" - }, { "$ref": "#/$defs/generic_options" } From b6e19d8a750c2fdcb63851d81a2337852f43e84f Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 5 Sep 2024 09:36:45 +0200 Subject: [PATCH 49/96] add a tabix tabix test --- tests/modules/test_lint.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/modules/test_lint.py b/tests/modules/test_lint.py index cc7e0565e0..144d163b27 100644 --- a/tests/modules/test_lint.py +++ b/tests/modules/test_lint.py @@ -186,6 +186,15 @@ def test_modules_lint_trimgalore(self): assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 + def test_modules_lint_tabix_tabix(self): + """Test linting the tabix/tabix module""" + self.mods_install.install("tabix/tabix") + module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) + module_lint.lint(print_results=False, module="tabix/tabix") + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + def test_modules_lint_empty(self): """Test linting a pipeline with no modules installed""" self.mods_remove.remove("fastqc", force=True) From dad359cc43a17e587eb97213350f46dffa5c1021 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Thu, 5 Sep 2024 09:08:49 +0000 Subject: [PATCH 50/96] Add --update-all flag to remove defaults channels dependents --- nf_core/gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index 849729966b..c462c6a47e 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -45,7 +45,7 @@ USER gitpod RUN conda config --add channels bioconda && \ conda config --add channels conda-forge && \ conda config --set channel_priority strict && \ - conda install --quiet --yes --name base \ + conda install --quiet --yes --update-all --name base \ nextflow \ nf-test \ prettier \ From 1c6b5776187d4410ffe584cb3fd748335cc760d6 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Thu, 5 Sep 2024 09:20:23 +0000 Subject: [PATCH 51/96] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8858c89bb3..80e94dfaa3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,6 +83,7 @@ - Add bot action to update textual snapshots and write bot documentation ([#3102](https://github.com/nf-core/tools/pull/3102)) - Update gitpod setup ([#3136](https://github.com/nf-core/tools/pull/3136)) - fix syncing a pipeline from current directory ([#3143](https://github.com/nf-core/tools/pull/3143)) +- Patch gitpod conda setup to not use defaults channel ([#3159](https://github.com/nf-core/tools/pull/3159)) ## Version updates From 4f930c19b8c5086f713a8040507aca1e30674432 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 6 Sep 2024 14:55:38 +0200 Subject: [PATCH 52/96] Don't test conda `environment.yml` `name` attribute (which should no longer be there) --- CHANGELOG.md | 1 + nf_core/modules/lint/environment_yml.py | 39 ------------------------- 2 files changed, 1 insertion(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 80e94dfaa3..ad84fb3978 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,7 @@ - Linting will now check the use of the right validation plugin include statements in the workflow scripts ([#3116])(https://github.com/nf-core/tools/pull/3116) - Full linting for correct use of nf-schema and nf-validation ([#3116](https://github.com/nf-core/tools/pull/3116)) - Handle cases where the directory path contains the name of the component ([#3147](https://github.com/nf-core/tools/pull/3147)) +- Don't test conda `environment.yml` `name` attribute (which should no longer be there) ([#3161](https://github.com/nf-core/tools/pull/3161)) ### Pipeline create command diff --git a/nf_core/modules/lint/environment_yml.py b/nf_core/modules/lint/environment_yml.py index 341b9cd730..4488b0befa 100644 --- a/nf_core/modules/lint/environment_yml.py +++ b/nf_core/modules/lint/environment_yml.py @@ -90,42 +90,3 @@ def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent) env_yml["dependencies"].sort() with open(Path(module.component_dir, "environment.yml"), "w") as fh: yaml.dump(env_yml, fh, Dumper=custom_yaml_dumper()) - - # Check that the name in the environment.yml file matches the name in the meta.yml file - with open(Path(module.component_dir, "meta.yml")) as fh: - meta_yml = yaml.safe_load(fh) - - if env_yml["name"] == meta_yml["name"]: - module.passed.append( - ( - "environment_yml_name", - "The module's `environment.yml` name matches module name", - module.environment_yml, - ) - ) - else: - module.failed.append( - ( - "environment_yml_name", - f"Conflicting process name between environment.yml (`{env_yml['name']}`) and meta.yml (`{module.component_name}`)", - module.environment_yml, - ) - ) - - # Check that the name is lowercase - if env_yml["name"] == env_yml["name"].lower(): - module.passed.append( - ( - "environment_yml_name_lowercase", - "The module's `environment.yml` name is lowercase", - module.environment_yml, - ) - ) - else: - module.failed.append( - ( - "environment_yml_name_lowercase", - "The module's `environment.yml` name is not lowercase", - module.environment_yml, - ) - ) From 0593114791f5a96304f27ad4070e4ea47c3936a2 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 6 Sep 2024 15:50:50 +0200 Subject: [PATCH 53/96] Remove name from conda enviroment.yml in module template --- nf_core/module-template/environment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/module-template/environment.yml b/nf_core/module-template/environment.yml index f234f85422..a8a40a8e03 100644 --- a/nf_core/module-template/environment.yml +++ b/nf_core/module-template/environment.yml @@ -1,6 +1,5 @@ --- # yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json -name: "{{ component_name_underscore }}" channels: - conda-forge - bioconda From ac88c16e187eba7ea607b904c35c521189970222 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 6 Sep 2024 16:20:49 +0200 Subject: [PATCH 54/96] Remove tests involving environment.yml with 'name' --- tests/modules/test_lint.py | 86 -------------------------------------- 1 file changed, 86 deletions(-) diff --git a/tests/modules/test_lint.py b/tests/modules/test_lint.py index 144d163b27..51c814b88c 100644 --- a/tests/modules/test_lint.py +++ b/tests/modules/test_lint.py @@ -487,54 +487,6 @@ def test_modules_environment_yml_file_not_array(self): assert len(module_lint.warned) >= 0 assert module_lint.failed[0].lint_test == "environment_yml_valid" - def test_modules_environment_yml_file_name_mismatch(self): - """Test linting a module with a different name in the environment.yml file""" - with open( - Path( - self.nfcore_modules, - "modules", - "nf-core", - "bpipe", - "test", - "environment.yml", - ) - ) as fh: - yaml_content = yaml.safe_load(fh) - yaml_content["name"] = "bpipe-test" - with open( - Path( - self.nfcore_modules, - "modules", - "nf-core", - "bpipe", - "test", - "environment.yml", - ), - "w", - ) as fh: - fh.write(yaml.dump(yaml_content)) - module_lint = nf_core.modules.lint.ModuleLint(directory=self.nfcore_modules) - module_lint.lint(print_results=False, module="bpipe/test") - # reset changes - yaml_content["name"] = "bpipe_test" - with open( - Path( - self.nfcore_modules, - "modules", - "nf-core", - "bpipe", - "test", - "environment.yml", - ), - "w", - ) as fh: - fh.write(yaml.dump(yaml_content)) - - assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" - assert len(module_lint.passed) > 0 - assert len(module_lint.warned) >= 0 - assert module_lint.failed[0].lint_test == "environment_yml_name" - def test_modules_meta_yml_incorrect_licence_field(self): """Test linting a module with an incorrect Licence field in meta.yml""" with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "meta.yml")) as fh: @@ -604,36 +556,11 @@ def test_modules_meta_yml_incorrect_name(self): with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "meta.yml")) as fh: meta_yml = yaml.safe_load(fh) meta_yml["name"] = "bpipe/test" - # need to make the same change to the environment.yml file - with open( - Path( - self.nfcore_modules, - "modules", - "nf-core", - "bpipe", - "test", - "environment.yml", - ) - ) as fh: - environment_yml = yaml.safe_load(fh) - environment_yml["name"] = "bpipe/test" with open( Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "meta.yml"), "w", ) as fh: fh.write(yaml.dump(meta_yml)) - with open( - Path( - self.nfcore_modules, - "modules", - "nf-core", - "bpipe", - "test", - "environment.yml", - ), - "w", - ) as fh: - fh.write(yaml.dump(environment_yml)) module_lint = nf_core.modules.lint.ModuleLint(directory=self.nfcore_modules) module_lint.lint(print_results=False, module="bpipe/test") @@ -644,19 +571,6 @@ def test_modules_meta_yml_incorrect_name(self): "w", ) as fh: fh.write(yaml.dump(meta_yml)) - environment_yml["name"] = "bpipe_test" - with open( - Path( - self.nfcore_modules, - "modules", - "nf-core", - "bpipe", - "test", - "environment.yml", - ), - "w", - ) as fh: - fh.write(yaml.dump(environment_yml)) assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) >= 0 From 6b3e8489190084b8fdd4014fae527eb70cd4d80e Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 6 Sep 2024 17:07:05 +0200 Subject: [PATCH 55/96] Don't look for dev, look for not master Co-authored-by: Matthieu Muffato --- nf_core/pipeline-template/.github/workflows/linting.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 00e8fd302a..826c66763f 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -58,7 +58,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_PR_COMMIT: ${{ github.event.pull_request.head.sha }} run: nf-core -l lint_log.txt pipelines lint --dir ${GITHUB_WORKSPACE} --markdown lint_results.md - if: ${{ github.base_ref == 'dev' }} + if: ${{ github.base_ref != 'master' }} - name: Run nf-core pipelines lint --release env: From 3e020cf2c5d08d0e4c53cc1e64e26cd989b8a745 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 6 Sep 2024 17:07:52 +0200 Subject: [PATCH 56/96] Move if statements to top of YAML blocks --- nf_core/pipeline-template/.github/workflows/linting.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 826c66763f..dbba830ec1 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -53,20 +53,20 @@ jobs: pip install nf-core==${{ steps.read_yml.outputs['nf_core_version'] }} - name: Run nf-core pipelines lint + if: ${{ github.base_ref != 'master' }} env: GITHUB_COMMENTS_URL: ${{ github.event.pull_request.comments_url }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_PR_COMMIT: ${{ github.event.pull_request.head.sha }} run: nf-core -l lint_log.txt pipelines lint --dir ${GITHUB_WORKSPACE} --markdown lint_results.md - if: ${{ github.base_ref != 'master' }} - name: Run nf-core pipelines lint --release + if: ${{ github.base_ref == 'master' }} env: GITHUB_COMMENTS_URL: ${{ github.event.pull_request.comments_url }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_PR_COMMIT: ${{ github.event.pull_request.head.sha }} run: nf-core -l lint_log.txt pipelines lint --release --dir ${GITHUB_WORKSPACE} --markdown lint_results.md - if: ${{ github.base_ref == 'master' }} - name: Save PR number if: ${{ always() }} From 9765f18a4e2f6dd208f62abb276829a5213aca4b Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 16 Sep 2024 11:14:47 +0200 Subject: [PATCH 57/96] Remove if/else block to include igenomes.config and add igenomes_ignored.config --- CHANGELOG.md | 1 + nf_core/pipeline-template/conf/igenomes_ignored.config | 9 +++++++++ nf_core/pipeline-template/nextflow.config | 6 +----- nf_core/pipelines/create/template_features.yml | 1 + 4 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 nf_core/pipeline-template/conf/igenomes_ignored.config diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bcc937852..4713b562ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ - add option to exclude tower.yml from pipeline template ([#3134](https://github.com/nf-core/tools/pull/3134)) - run nf-core lint `--release` on PRs to master ([#3148](https://github.com/nf-core/tools/pull/3148)) - Add tests to ensure all files are part of a template customisation group and all groups are tested ([#3099](https://github.com/nf-core/tools/pull/3099)) +- Remove if/else block to include `igenomes.config` ([#3168](https://github.com/nf-core/tools/pull/3168)) ### Linting diff --git a/nf_core/pipeline-template/conf/igenomes_ignored.config b/nf_core/pipeline-template/conf/igenomes_ignored.config new file mode 100644 index 0000000000..b4034d8243 --- /dev/null +++ b/nf_core/pipeline-template/conf/igenomes_ignored.config @@ -0,0 +1,9 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Nextflow config file for iGenomes paths +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Empty genomes dictionary to use when igenomes is ignored. +---------------------------------------------------------------------------------------- +*/ + +params.genomes = [:] diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 8201916b3f..529dccb2ad 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -216,11 +216,7 @@ charliecloud.registry = 'quay.io' {% if igenomes -%} // Load igenomes.config if required -if (!params.igenomes_ignore) { - includeConfig 'conf/igenomes.config' -} else { - params.genomes = [:] -} +includeConfig !params.igenomes_ignore ? 'conf/igenomes.config' : 'conf/igenomes_ignored.config' {% endif -%} // Export these variables to prevent local Python/R libraries from conflicting with those in the container diff --git a/nf_core/pipelines/create/template_features.yml b/nf_core/pipelines/create/template_features.yml index b59a2e51a6..c166429431 100644 --- a/nf_core/pipelines/create/template_features.yml +++ b/nf_core/pipelines/create/template_features.yml @@ -60,6 +60,7 @@ ci: igenomes: skippable_paths: - "conf/igenomes.config" + - "conf/igenomes_ignored.config" short_description: "Use reference genomes" description: "The pipeline will be configured to use a copy of the most common reference genome files from iGenomes" help_text: | From 559903976a9dd5d81ed58df81f13bd8523de3618 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 16 Sep 2024 16:49:39 +0200 Subject: [PATCH 58/96] add tests back --- tests/pipelines/lint/test_nextflow_config.py | 78 ++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/tests/pipelines/lint/test_nextflow_config.py b/tests/pipelines/lint/test_nextflow_config.py index 4c3893f4c4..d2296618b0 100644 --- a/tests/pipelines/lint/test_nextflow_config.py +++ b/tests/pipelines/lint/test_nextflow_config.py @@ -2,8 +2,11 @@ import re from pathlib import Path +import yaml + import nf_core.pipelines.create.create import nf_core.pipelines.lint +from nf_core.utils import NFCoreYamlConfig from ..test_lint import TestLint @@ -65,6 +68,81 @@ def test_nextflow_config_missing_test_profile_failed(self): assert len(result["failed"]) > 0 assert len(result["warned"]) == 0 + def test_default_values_fail(self): + """Test linting fails if the default values in nextflow.config do not match the ones defined in the nextflow_schema.json.""" + # Change the default value of max_multiqc_email_size in nextflow.config + nf_conf_file = Path(self.new_pipeline) / "nextflow.config" + with open(nf_conf_file) as f: + content = f.read() + fail_content = re.sub(r"\bmax_multiqc_email_size\s*=\s*'25.MB'", "max_multiqc_email_size = '0'", content) + with open(nf_conf_file, "w") as f: + f.write(fail_content) + # Change the default value of custom_config_version in nextflow_schema.json + nf_schema_file = Path(self.new_pipeline) / "nextflow_schema.json" + with open(nf_schema_file) as f: + content = f.read() + fail_content = re.sub(r'"default": "master"', '"default": "main"', content) + with open(nf_schema_file, "w") as f: + f.write(fail_content) + lint_obj = nf_core.pipelines.lint.PipelineLint(self.new_pipeline) + lint_obj.load_pipeline_config() + result = lint_obj.nextflow_config() + assert len(result["failed"]) == 2 + assert ( + "Config default value incorrect: `params.max_multiqc_email_size` is set as `25.MB` in `nextflow_schema.json` but is `0` in `nextflow.config`." + in result["failed"] + ) + assert ( + "Config default value incorrect: `params.custom_config_version` is set as `main` in `nextflow_schema.json` but is `master` in `nextflow.config`." + in result["failed"] + ) + + def test_catch_params_assignment_in_main_nf(self): + """Test linting fails if main.nf contains an assignment to a parameter from nextflow_schema.json.""" + # Add parameter assignment in main.nf + main_nf_file = Path(self.new_pipeline) / "main.nf" + with open(main_nf_file, "a") as f: + f.write("params.custom_config_base = 'test'") + lint_obj = nf_core.pipelines.lint.PipelineLint(self.new_pipeline) + lint_obj.load_pipeline_config() + result = lint_obj.nextflow_config() + assert len(result["failed"]) == 1 + assert ( + result["failed"][0] + == "Config default value incorrect: `params.custom_config_base` is set as `https://raw.githubusercontent.com/nf-core/configs/master` in `nextflow_schema.json` but is `null` in `nextflow.config`." + ) + + def test_allow_params_reference_in_main_nf(self): + """Test linting allows for references like `params.aligner == 'bwa'` in main.nf. The test will detect if the bug mentioned in GitHub-issue #2833 reemerges.""" + # Add parameter reference in main.nf + main_nf_file = Path(self.new_pipeline) / "main.nf" + with open(main_nf_file, "a") as f: + f.write("params.custom_config_version == 'main'") + lint_obj = nf_core.pipelines.lint.PipelineLint(self.new_pipeline) + lint_obj.load_pipeline_config() + result = lint_obj.nextflow_config() + assert len(result["failed"]) == 0 + + def test_default_values_ignored(self): + """Test ignoring linting of default values.""" + # Add custom_config_version to the ignore list + nf_core_yml_path = Path(self.new_pipeline) / ".nf-core.yml" + nf_core_yml = NFCoreYamlConfig( + repository_type="pipeline", + lint={"nextflow_config": [{"config_defaults": ["params.custom_config_version"]}]}, + ) + with open(nf_core_yml_path, "w") as f: + yaml.dump(nf_core_yml.model_dump(), f) + + lint_obj = nf_core.pipelines.lint.PipelineLint(self.new_pipeline) + lint_obj.load_pipeline_config() + lint_obj._load_lint_config() + result = lint_obj.nextflow_config() + assert len(result["failed"]) == 0 + assert len(result["ignored"]) == 1 + assert "Config default value correct: params.custom_config_version" not in str(result["passed"]) + assert "Config default ignored: params.custom_config_version" in str(result["ignored"]) + def test_default_values_float(self): """Test comparing two float values.""" # Add a float value `dummy=0.0001` to the nextflow.config below `validate_params` From c568ce9e207d6c2bd9b43954912754661004147f Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 16 Sep 2024 17:06:04 +0200 Subject: [PATCH 59/96] more pytest fixing --- tests/pipelines/lint/test_nextflow_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pipelines/lint/test_nextflow_config.py b/tests/pipelines/lint/test_nextflow_config.py index d2296618b0..a655fb8ace 100644 --- a/tests/pipelines/lint/test_nextflow_config.py +++ b/tests/pipelines/lint/test_nextflow_config.py @@ -106,9 +106,9 @@ def test_catch_params_assignment_in_main_nf(self): lint_obj = nf_core.pipelines.lint.PipelineLint(self.new_pipeline) lint_obj.load_pipeline_config() result = lint_obj.nextflow_config() - assert len(result["failed"]) == 1 + assert len(result["failed"]) == 2 assert ( - result["failed"][0] + result["failed"][1] == "Config default value incorrect: `params.custom_config_base` is set as `https://raw.githubusercontent.com/nf-core/configs/master` in `nextflow_schema.json` but is `null` in `nextflow.config`." ) From c2488b5218975975cc6ad4a17f5e76ef84c7c8dd Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 18 Sep 2024 10:33:42 +0200 Subject: [PATCH 60/96] Strip hyphens from hashtags for mastodon compatibiluty --- .../.github/workflows/release-announcements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/release-announcements.yml b/nf_core/pipeline-template/.github/workflows/release-announcements.yml index 8fee061fdd..035ed63bba 100644 --- a/nf_core/pipeline-template/.github/workflows/release-announcements.yml +++ b/nf_core/pipeline-template/.github/workflows/release-announcements.yml @@ -12,7 +12,7 @@ jobs: - name: get topics and convert to hashtags id: get_topics run: | - echo "topics=$(curl -s https://nf-co.re/pipelines.json | jq -r '.remote_workflows[] | select(.full_name == "${{ github.repository }}") | .topics[]' | awk '{print "#"$0}' | tr '\n' ' ')" >> $GITHUB_OUTPUT + echo "topics=$(curl -s https://nf-co.re/pipelines.json | jq -r '.remote_workflows[] | select(.full_name == "${{ github.repository }}") | .topics[]' | awk '{print "#"$0}' | tr '\n' ' ')" | sed 's/-//g' >> $GITHUB_OUTPUT - uses: rzr/fediverse-action@master with: From 53448ce341b6280f0a4967f100c1bd0d9f649965 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 18 Sep 2024 10:38:16 +0200 Subject: [PATCH 61/96] fixes for linting modules without an input --- nf_core/components/nfcore_component.py | 1 + nf_core/modules/lint/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index b6e3520ee5..84c9a651ed 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -184,6 +184,7 @@ def get_inputs_from_main_nf(self) -> None: # don't match anything inside comments or after "output:" if "input:" not in data: log.debug(f"Could not find any inputs in {self.main_nf}") + return input_data = data.split("input:")[1].split("output:")[0] for line in input_data.split("\n"): channel_elements: list[dict[str, dict[str, str]]] = [] diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 43a5fe71e9..49012cff40 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -288,7 +288,7 @@ def update_meta_yml_file(self, mod): if "output" in meta_yml: correct_outputs, meta_outputs = self.obtain_correct_and_specified_outputs(mod, meta_yml) - if correct_inputs != meta_inputs: + if "input" in meta_yml and correct_inputs != meta_inputs: log.debug( f"Correct inputs: '{correct_inputs}' differ from current inputs: '{meta_inputs}' in '{mod.meta_yml}'" ) @@ -319,7 +319,7 @@ def update_meta_yml_file(self, mod): ][feature] break - if correct_outputs != meta_outputs: + if "output" in meta_yml and correct_outputs != meta_outputs: log.debug( f"Correct outputs: '{correct_outputs}' differ from current outputs: '{meta_outputs}' in '{mod.meta_yml}'" ) From bebc6bc1cdb065654a0d33e25c42e7f6e9a2ccca Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 18 Sep 2024 13:45:44 +0200 Subject: [PATCH 62/96] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bcc937852..f2d494ab13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ - add option to exclude tower.yml from pipeline template ([#3134](https://github.com/nf-core/tools/pull/3134)) - run nf-core lint `--release` on PRs to master ([#3148](https://github.com/nf-core/tools/pull/3148)) - Add tests to ensure all files are part of a template customisation group and all groups are tested ([#3099](https://github.com/nf-core/tools/pull/3099)) +- Fixed release announcement hashtags for Mastodon ([#3099](https://github.com/nf-core/tools/pull/3176)) ### Linting From 494a03ff12a45d235d2a8c62ca86d52e5de064dc Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 19 Sep 2024 12:15:19 +0200 Subject: [PATCH 63/96] fix broken lint test by adding module dependency version --- tests/modules/test_lint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modules/test_lint.py b/tests/modules/test_lint.py index 51c814b88c..07e12924c6 100644 --- a/tests/modules/test_lint.py +++ b/tests/modules/test_lint.py @@ -432,7 +432,7 @@ def test_modules_environment_yml_file_sorted_incorrectly(self): ) as fh: yaml_content = yaml.safe_load(fh) # Add a new dependency to the environment.yml file and reverse the order - yaml_content["dependencies"].append("z") + yaml_content["dependencies"].append("z=0.0.0") yaml_content["dependencies"].reverse() yaml_content = yaml.dump(yaml_content) with open( From 17dc0bc34cfd22782221532508a883d65a2c6590 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 19 Sep 2024 12:39:02 +0200 Subject: [PATCH 64/96] change order of profiles used for CI --- .github/workflows/create-test-wf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 56c6c822a9..e3918bffdf 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -75,7 +75,7 @@ jobs: pwd # echo content of current directory ls -la - nextflow run nf-core-testpipeline -profile test,self_hosted_runner --outdir ./results + nextflow run nf-core-testpipeline -profile self_hosted_runner,test --outdir ./results - name: Upload log file artifact if: ${{ always() }} From c494c268e0873f82e340a47aabcf24e36d59a29a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 19 Sep 2024 12:55:36 +0200 Subject: [PATCH 65/96] update nextflow version on CI --- .github/workflows/create-lint-wf.yml | 2 +- .github/workflows/create-test-wf.yml | 2 +- nf_core/pipeline-template/.github/workflows/ci.yml | 2 +- nf_core/pipeline-template/README.md | 2 +- nf_core/pipelines/lint/readme.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index f07b31e9de..1a3e283d00 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -38,7 +38,7 @@ jobs: strategy: matrix: NXF_VER: - - "23.10.0" + - "24.04.2" - "latest-everything" steps: - name: go to subdirectory and change nextflow workdir diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index e3918bffdf..09c5b01c37 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -39,7 +39,7 @@ jobs: strategy: matrix: NXF_VER: - - "23.10.0" + - "24.04.2" - "latest-everything" steps: - name: go to working directory diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 63fa99cec8..06371cc974 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: strategy: matrix: NXF_VER: - - "23.10.0" + - "24.04.2" - "latest-everything" steps: - name: Check out pipeline code diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index beb45ed511..4331454288 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -20,7 +20,7 @@ [![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) [![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com) -[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.10.0-23aa62.svg)](https://www.nextflow.io/) +[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A524.04.2-23aa62.svg)](https://www.nextflow.io/) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) diff --git a/nf_core/pipelines/lint/readme.py b/nf_core/pipelines/lint/readme.py index 1c09104258..bdfad5200f 100644 --- a/nf_core/pipelines/lint/readme.py +++ b/nf_core/pipelines/lint/readme.py @@ -36,7 +36,7 @@ def readme(self): if "nextflow_badge" not in ignore_configs: # Check that there is a readme badge showing the minimum required version of Nextflow - # [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.10.0-23aa62.svg)](https://www.nextflow.io/) + # [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A524.04.2-23aa62.svg)](https://www.nextflow.io/) # and that it has the correct version nf_badge_re = r"\[!\[Nextflow\]\(https://img\.shields\.io/badge/nextflow%20DSL2-!?(?:%E2%89%A5|%3E%3D)([\d\.]+)-23aa62\.svg\)\]\(https://www\.nextflow\.io/\)" match = re.search(nf_badge_re, content) From 2c0d2f6a325892a407500df8022c534167b4a7e0 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 19 Sep 2024 13:31:14 +0200 Subject: [PATCH 66/96] configs are included after profiles --- nf_core/pipeline-template/nextflow.config | 31 ++++++++++++----------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 8c1d33244f..d129d1a492 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -76,21 +76,6 @@ process { maxErrors = '-1' } {% endif %} -{% if nf_core_configs -%} -// Load nf-core custom profiles from different Institutions -try { - includeConfig "${params.custom_config_base}/nfcore_custom.config" -} catch (Exception e) { - System.err.println("WARNING: Could not load nf-core/config profiles: ${params.custom_config_base}/nfcore_custom.config") -} - -// Load {{ name }} custom profiles from different institutions. -try { - includeConfig "${params.custom_config_base}/pipeline/{{ short_name }}.config" -} catch (Exception e) { - System.err.println("WARNING: Could not load nf-core/config/{{ short_name }} profiles: ${params.custom_config_base}/pipeline/{{ short_name }}.config") -} -{% endif -%} profiles { debug { @@ -199,6 +184,22 @@ profiles { {%- endif %} } +{% if nf_core_configs -%} +// Load nf-core custom profiles from different Institutions +try { + includeConfig "${params.custom_config_base}/nfcore_custom.config" +} catch (Exception e) { + System.err.println("WARNING: Could not load nf-core/config profiles: ${params.custom_config_base}/nfcore_custom.config") +} + +// Load {{ name }} custom profiles from different institutions. +try { + includeConfig "${params.custom_config_base}/pipeline/{{ short_name }}.config" +} catch (Exception e) { + System.err.println("WARNING: Could not load nf-core/config/{{ short_name }} profiles: ${params.custom_config_base}/pipeline/{{ short_name }}.config") +} +{% endif -%} + // Set default registry for Apptainer, Docker, Podman, Charliecloud and Singularity independent of -profile // Will not be used unless Apptainer / Docker / Podman / Charliecloud / Singularity are enabled // Set to your registry if you have a mirror of containers From d2959edebb53efb1479785d3258d2577546cf67d Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 19 Sep 2024 13:44:58 +0200 Subject: [PATCH 67/96] reduce memory in test profile --- nf_core/pipeline-template/conf/test.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/conf/test.config b/nf_core/pipeline-template/conf/test.config index 0f1d974101..bea6f670d0 100644 --- a/nf_core/pipeline-template/conf/test.config +++ b/nf_core/pipeline-template/conf/test.config @@ -13,7 +13,7 @@ process { resourceLimits = [ cpus: 4, - memory: '16.GB', + memory: '15.GB', time: '1.h' ] } From 535477086cd8f90fc06ef6d98cfff3f0e0dbf4bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 19 Sep 2024 14:00:02 +0200 Subject: [PATCH 68/96] Apply suggestions from code review Co-authored-by: Phil Ewels --- nf_core/modules/lint/meta_yml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index 80b40b0221..4f7ffd0739 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -130,7 +130,7 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None module.failed.append( ( "correct_meta_inputs", - f"Incorrect inputs specified in module `meta.yml`. Inputs should contain: {correct_inputs}\nRun `nf-core modules lint --fix` to update the `meta.yml` file.", + f"Module `meta.yml` does not match `main.nf`. Inputs should contain: {correct_inputs}\nRun `nf-core modules lint --fix` to update the `meta.yml` file.", module.meta_yml, ) ) @@ -168,7 +168,7 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None module.failed.append( ( "correct_meta_outputs", - f"Incorrect outputs specified in module `meta.yml`. Outputs should contain: {correct_outputs}\nRun `nf-core modules lint --fix` to update the `meta.yml` file.", + f"Module `meta.yml` does not match `main.nf`. Outputs should contain: {correct_outputs}\nRun `nf-core modules lint --fix` to update the `meta.yml` file.", module.meta_yml, ) ) From 1e18f146899a99e3f80c17570322fdec92c07ff7 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Fri, 20 Sep 2024 11:07:14 +0200 Subject: [PATCH 69/96] bump nf-schema to v2.1.1 in the template --- nf_core/pipeline-template/nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index d129d1a492..16f57c94c3 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -274,7 +274,7 @@ manifest { {% if nf_schema -%} // Nextflow plugins plugins { - id 'nf-schema@2.1.0' // Validation of pipeline parameters and creation of an input channel from a sample sheet + id 'nf-schema@2.1.1' // Validation of pipeline parameters and creation of an input channel from a sample sheet } validation { From e2c47405fba6c66609a982da928db6fa4d9a4b96 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Fri, 20 Sep 2024 13:08:30 +0200 Subject: [PATCH 70/96] remove help params from ignore params --- nf_core/pipeline-template/nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 16f57c94c3..3e301f0b0b 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -278,7 +278,7 @@ plugins { } validation { - defaultIgnoreParams = ["genomes", "helpFull", "showHidden", "help-full", "show-hidden"] // The last 4 parameters are here because of a bug in nf-schema. This will be fixed in a later version + defaultIgnoreParams = ["genomes"] help { enabled = true command = "nextflow run $manifest.name -profile --input samplesheet.csv --outdir " From ebfbd9884f9c6dddfbf18feeb5945ce659c7c43d Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 20 Sep 2024 18:16:40 +0200 Subject: [PATCH 71/96] fix pytests by mocking biotools, adding a meta to the test module and running prettier to meta.yml file --- tests/test_modules.py | 15 +++++++++++---- tests/utils.py | 9 +++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/tests/test_modules.py b/tests/test_modules.py index 0e16497176..d0692236e8 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -7,7 +7,7 @@ import pytest import requests_cache import responses -import yaml +import ruamel.yaml import nf_core.modules import nf_core.modules.create @@ -16,6 +16,7 @@ import nf_core.modules.remove import nf_core.pipelines.create.create from nf_core import __version__ +from nf_core.pipelines.lint_utils import run_prettier_on_file from nf_core.utils import NFCoreYamlConfig from .utils import ( @@ -28,11 +29,15 @@ create_tmp_pipeline, mock_anaconda_api_calls, mock_biocontainers_api_calls, + mock_biotools_api_calls, ) def create_modules_repo_dummy(tmp_dir): """Create a dummy copy of the nf-core/modules repo""" + yaml = ruamel.yaml.YAML() + yaml.preserve_quotes = True + yaml.indent(mapping=2, sequence=2, offset=0) root_dir = Path(tmp_dir, "modules") Path(root_dir, "modules", "nf-core").mkdir(parents=True) @@ -42,13 +47,14 @@ def create_modules_repo_dummy(tmp_dir): nf_core_yml = NFCoreYamlConfig(nf_core_version=__version__, repository_type="modules", org_path="nf-core") with open(Path(root_dir, ".nf-core.yml"), "w") as fh: yaml.dump(nf_core_yml.model_dump(), fh) - # mock biocontainers and anaconda response + # mock biocontainers and anaconda response and biotools response with responses.RequestsMock() as rsps: mock_anaconda_api_calls(rsps, "bpipe", "0.9.13--hdfd78af_0") mock_biocontainers_api_calls(rsps, "bpipe", "0.9.13--hdfd78af_0") + mock_biotools_api_calls(rsps, "bpipe") # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules module_create = nf_core.modules.create.ModuleCreate( - root_dir, "bpipe/test", "@author", "process_single", False, False + root_dir, "bpipe/test", "@author", "process_single", True, False ) with requests_cache.disabled(): assert module_create.create() @@ -57,10 +63,11 @@ def create_modules_repo_dummy(tmp_dir): meta_yml_path = Path(root_dir, "modules", "nf-core", "bpipe", "test", "meta.yml") with open(str(meta_yml_path)) as fh: - meta_yml = yaml.safe_load(fh) + meta_yml = yaml.load(fh) del meta_yml["tools"][0]["bpipe"]["doi"] with open(str(meta_yml_path), "w") as fh: yaml.dump(meta_yml, fh) + run_prettier_on_file(fh.name) # Add dummy content to main.nf.test.snap test_snap_path = Path(root_dir, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap") diff --git a/tests/utils.py b/tests/utils.py index 1d5a8a115d..6f4b73cccc 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -98,6 +98,15 @@ def mock_biocontainers_api_calls(rsps: responses.RequestsMock, module: str, vers rsps.get(biocontainers_api_url, json=biocontainers_mock, status=200) +def mock_biotools_api_calls(rsps: responses.RequestsMock, module: str) -> None: + """Mock biotools api calls for module""" + biotools_api_url = f"https://bio.tools/api/t/?q={module}&format=json" + biotools_mock = { + "list": [{"name": "Bpipe", "biotoolsCURIE": "biotools:bpipe"}], + } + rsps.get(biotools_api_url, json=biotools_mock, status=200) + + def create_tmp_pipeline(no_git: bool = False) -> Tuple[Path, Path, str, Path]: """Create a new Pipeline for testing""" From 0447d1a28a802922bd3e9fa9f184ed9f3eb0b8da Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 23 Sep 2024 12:52:22 +0200 Subject: [PATCH 72/96] update template modules --- nf_core/pipeline-template/modules.json | 4 +- .../modules/nf-core/fastqc/main.nf | 5 +- .../modules/nf-core/fastqc/meta.yml | 57 ++++++++------ .../modules/nf-core/multiqc/environment.yml | 3 +- .../modules/nf-core/multiqc/main.nf | 14 +++- .../modules/nf-core/multiqc/meta.yml | 78 ++++++++++++------- 6 files changed, 100 insertions(+), 61 deletions(-) diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 367202155a..9bc344e7d1 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -8,12 +8,12 @@ {%- if fastqc %} "fastqc": { "branch": "master", - "git_sha": "285a50500f9e02578d90b3ce6382ea3c30216acd", + "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1", "installed_by": ["modules"] }{% endif %}{%- if multiqc %}{% if fastqc %},{% endif %} "multiqc": { "branch": "master", - "git_sha": "b7ebe95761cd389603f9cc0e0dc384c0f663815a", + "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1", "installed_by": ["modules"] } {%- endif %} diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf index d79f1c862d..d8989f4812 100644 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf @@ -26,7 +26,10 @@ process FASTQC { def rename_to = old_new_pairs*.join(' ').join(' ') def renamed_files = old_new_pairs.collect{ old_name, new_name -> new_name }.join(' ') - def memory_in_mb = MemoryUnit.of("${task.memory}").toUnit('MB') + // The total amount of allocated RAM by FastQC is equal to the number of threads defined (--threads) time the amount of RAM defined (--memory) + // https://github.com/s-andrews/FastQC/blob/1faeea0412093224d7f6a07f777fad60a5650795/fastqc#L211-L222 + // Dividing the task.memory by task.cpu allows to stick to requested amount of RAM in the label + def memory_in_mb = MemoryUnit.of("${task.memory}").toUnit('MB') / task.cpus // FastQC memory value allowed range (100 - 10000) def fastqc_memory = memory_in_mb > 10000 ? 10000 : (memory_in_mb < 100 ? 100 : memory_in_mb) diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/meta.yml b/nf_core/pipeline-template/modules/nf-core/fastqc/meta.yml index ee5507e06b..4827da7af2 100644 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/meta.yml +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/meta.yml @@ -16,35 +16,44 @@ tools: homepage: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/ documentation: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/Help/ licence: ["GPL-2.0-only"] + identifier: biotools:fastqc input: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] - - reads: - type: file - description: | - List of input FastQ files of size 1 and 2 for single-end and paired-end data, - respectively. + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - reads: + type: file + description: | + List of input FastQ files of size 1 and 2 for single-end and paired-end data, + respectively. output: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] - html: - type: file - description: FastQC report - pattern: "*_{fastqc.html}" + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - "*.html": + type: file + description: FastQC report + pattern: "*_{fastqc.html}" - zip: - type: file - description: FastQC report archive - pattern: "*_{fastqc.zip}" + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - "*.zip": + type: file + description: FastQC report archive + pattern: "*_{fastqc.zip}" - versions: - type: file - description: File containing software versions - pattern: "versions.yml" + - versions.yml: + type: file + description: File containing software versions + pattern: "versions.yml" authors: - "@drpatelh" - "@grst" diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml b/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml index 329ddb4870..f1cd99b079 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml @@ -1,6 +1,5 @@ -name: multiqc channels: - conda-forge - bioconda dependencies: - - bioconda::multiqc=1.21 + - bioconda::multiqc=1.24.1 diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf index 47ac352f94..b9ccebdbbc 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf @@ -3,14 +3,16 @@ process MULTIQC { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.21--pyhdfd78af_0' : - 'biocontainers/multiqc:1.21--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.25--pyhdfd78af_0' : + 'biocontainers/multiqc:1.25--pyhdfd78af_0' }" input: path multiqc_files, stageAs: "?/*" path(multiqc_config) path(extra_multiqc_config) path(multiqc_logo) + path(replace_names) + path(sample_names) output: path "*multiqc_report.html", emit: report @@ -23,16 +25,22 @@ process MULTIQC { script: def args = task.ext.args ?: '' + def prefix = task.ext.prefix ? "--filename ${task.ext.prefix}.html" : '' def config = multiqc_config ? "--config $multiqc_config" : '' def extra_config = extra_multiqc_config ? "--config $extra_multiqc_config" : '' - def logo = multiqc_logo ? /--cl-config 'custom_logo: "${multiqc_logo}"'/ : '' + def logo = multiqc_logo ? "--cl-config 'custom_logo: \"${multiqc_logo}\"'" : '' + def replace = replace_names ? "--replace-names ${replace_names}" : '' + def samples = sample_names ? "--sample-names ${sample_names}" : '' """ multiqc \\ --force \\ $args \\ $config \\ + $prefix \\ $extra_config \\ $logo \\ + $replace \\ + $samples \\ . cat <<-END_VERSIONS > versions.yml diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/meta.yml b/nf_core/pipeline-template/modules/nf-core/multiqc/meta.yml index 45a9bc35e1..b16c187923 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/meta.yml +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/meta.yml @@ -1,5 +1,6 @@ name: multiqc -description: Aggregate results from bioinformatics analyses across many samples into a single report +description: Aggregate results from bioinformatics analyses across many samples into + a single report keywords: - QC - bioinformatics tools @@ -12,40 +13,59 @@ tools: homepage: https://multiqc.info/ documentation: https://multiqc.info/docs/ licence: ["GPL-3.0-or-later"] + identifier: biotools:multiqc input: - - multiqc_files: - type: file - description: | - List of reports / files recognised by MultiQC, for example the html and zip output of FastQC - - multiqc_config: - type: file - description: Optional config yml for MultiQC - pattern: "*.{yml,yaml}" - - extra_multiqc_config: - type: file - description: Second optional config yml for MultiQC. Will override common sections in multiqc_config. - pattern: "*.{yml,yaml}" - - multiqc_logo: - type: file - description: Optional logo file for MultiQC - pattern: "*.{png}" + - - multiqc_files: + type: file + description: | + List of reports / files recognised by MultiQC, for example the html and zip output of FastQC + - - multiqc_config: + type: file + description: Optional config yml for MultiQC + pattern: "*.{yml,yaml}" + - - extra_multiqc_config: + type: file + description: Second optional config yml for MultiQC. Will override common sections + in multiqc_config. + pattern: "*.{yml,yaml}" + - - multiqc_logo: + type: file + description: Optional logo file for MultiQC + pattern: "*.{png}" + - - replace_names: + type: file + description: | + Optional two-column sample renaming file. First column a set of + patterns, second column a set of corresponding replacements. Passed via + MultiQC's `--replace-names` option. + pattern: "*.{tsv}" + - - sample_names: + type: file + description: | + Optional TSV file with headers, passed to the MultiQC --sample_names + argument. + pattern: "*.{tsv}" output: - report: - type: file - description: MultiQC report file - pattern: "multiqc_report.html" + - "*multiqc_report.html": + type: file + description: MultiQC report file + pattern: "multiqc_report.html" - data: - type: directory - description: MultiQC data dir - pattern: "multiqc_data" + - "*_data": + type: directory + description: MultiQC data dir + pattern: "multiqc_data" - plots: - type: file - description: Plots created by MultiQC - pattern: "*_data" + - "*_plots": + type: file + description: Plots created by MultiQC + pattern: "*_data" - versions: - type: file - description: File containing software versions - pattern: "versions.yml" + - versions.yml: + type: file + description: File containing software versions + pattern: "versions.yml" authors: - "@abhi18av" - "@bunop" From 200ae85466eef6f544227bf9556be686c2161946 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 23 Sep 2024 12:53:06 +0200 Subject: [PATCH 73/96] add testing the module name back --- nf_core/modules/lint/meta_yml.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index 4f7ffd0739..4ad728d10b 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -95,6 +95,23 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None # Confirm that all input and output channels are correctly specified if valid_meta_yml: + # confirm that the name matches the process name in main.nf + if meta_yaml["name"].upper() == module.process_name: + module.passed.append( + ( + "meta_name", + "Correct name specified in `meta.yml`.", + module.meta_yml, + ) + ) + else: + module.failed.append( + ( + "meta_name", + f"Conflicting `process` name between meta.yml (`{meta_yaml['name']}`) and main.nf (`{module.process_name}`)", + module.meta_yml, + ) + ) # Check that inputs are specified in meta.yml if len(module.inputs) > 0 and "input" not in meta_yaml: module.failed.append( From 0e3a35b64aa814911e9eb61ee7c23cee51413cbc Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 23 Sep 2024 12:53:42 +0200 Subject: [PATCH 74/96] fix info command for the new yaml structure --- nf_core/components/info.py | 53 ++++++++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 14 deletions(-) diff --git a/nf_core/components/info.py b/nf_core/components/info.py index 98f8be5272..e4a58d1618 100644 --- a/nf_core/components/info.py +++ b/nf_core/components/info.py @@ -280,14 +280,26 @@ def generate_component_info_help(self): inputs_table = Table(expand=True, show_lines=True, box=box.MINIMAL_HEAVY_HEAD, padding=0) inputs_table.add_column(":inbox_tray: Inputs") inputs_table.add_column("Description") - inputs_table.add_column("Pattern", justify="right", style="green") + if self.component_type == "modules": + inputs_table.add_column("Pattern", justify="right", style="green") + elif self.component_type == "subworkflows": + inputs_table.add_column("Structure", justify="right", style="green") for input in self.meta["input"]: - for key, info in input.items(): - inputs_table.add_row( - f"[orange1 on black] {key} [/][dim i] ({info['type']})", - Markdown(info["description"] if info["description"] else ""), - info.get("pattern", ""), - ) + if self.component_type == "modules": + for element in input: + for key, info in element.items(): + inputs_table.add_row( + f"[orange1 on black] {key} [/][dim i] ({info['type']})", + Markdown(info["description"] if info["description"] else ""), + info.get("pattern", ""), + ) + elif self.component_type == "subworkflows": + for key, info in input.items(): + inputs_table.add_row( + f"[orange1 on black] {key} [/][dim i]", + Markdown(info["description"] if info["description"] else ""), + info.get("structure", ""), + ) renderables.append(inputs_table) @@ -296,14 +308,27 @@ def generate_component_info_help(self): outputs_table = Table(expand=True, show_lines=True, box=box.MINIMAL_HEAVY_HEAD, padding=0) outputs_table.add_column(":outbox_tray: Outputs") outputs_table.add_column("Description") - outputs_table.add_column("Pattern", justify="right", style="green") + if self.component_type == "modules": + inputs_table.add_column("Pattern", justify="right", style="green") + elif self.component_type == "subworkflows": + inputs_table.add_column("Structure", justify="right", style="green") for output in self.meta["output"]: - for key, info in output.items(): - outputs_table.add_row( - f"[orange1 on black] {key} [/][dim i] ({info['type']})", - Markdown(info["description"] if info["description"] else ""), - info.get("pattern", ""), - ) + if self.component_type == "modules": + for ch_name, elements in output.items(): + for element in elements: + for key, info in element.items(): + outputs_table.add_row( + f"[orange1 on black] {key} [/][dim i] ({info['type']})", + Markdown(info["description"] if info["description"] else ""), + info.get("pattern", ""), + ) + elif self.component_type == "subworkflows": + for key, info in output.items(): + outputs_table.add_row( + f"[orange1 on black] {key} [/][dim i]", + Markdown(info["description"] if info["description"] else ""), + info.get("structure", ""), + ) renderables.append(outputs_table) From 23c2612c561dd4489c17df1f0f4b3c95d091e8e5 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 23 Sep 2024 12:54:32 +0200 Subject: [PATCH 75/96] fix modules pytest --- nf_core/module-template/meta.yml | 10 +++---- tests/modules/test_lint.py | 45 ++++++++------------------------ tests/modules/test_patch.py | 4 +-- 3 files changed, 18 insertions(+), 41 deletions(-) diff --git a/nf_core/module-template/meta.yml b/nf_core/module-template/meta.yml index c7c16dcb38..d9d1cc8ae8 100644 --- a/nf_core/module-template/meta.yml +++ b/nf_core/module-template/meta.yml @@ -53,11 +53,6 @@ input: ## TODO nf-core: Add a description of all of the variables used as output {% endif -%} output: - - versions: - - "versions.yml": - type: file - description: File containing software versions - pattern: "versions.yml" - {{ 'bam:' if not_empty_template else "output:" }} #{% if has_meta -%} Only when we have meta - meta: @@ -81,6 +76,11 @@ output: {% else -%} - edam: "" {%- endif %} + - versions: + - "versions.yml": + type: file + description: File containing software versions + pattern: "versions.yml" authors: - "{{ author }}" diff --git a/tests/modules/test_lint.py b/tests/modules/test_lint.py index f4adedfcb5..5372807987 100644 --- a/tests/modules/test_lint.py +++ b/tests/modules/test_lint.py @@ -210,14 +210,6 @@ def test_modules_lint_new_modules(self): assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 - def test_modules_lint_update_meta_yml(self): - """update the meta.yml of a module""" - module_lint = nf_core.modules.ModuleLint(directory=self.nfcore_modules, fix=True) - module_lint.lint(print_results=False, module="fastqc") - assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" - assert len(module_lint.passed) > 0 - assert len(module_lint.warned) >= 0 - def test_modules_lint_no_gitlab(self): """Test linting a pipeline with no modules installed""" self.mods_remove.remove("fastqc", force=True) @@ -282,7 +274,7 @@ def test_modules_lint_patched_modules(self): all_modules=True, ) - assert len(module_lint.failed) == 1 + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 @@ -313,6 +305,14 @@ def test_modules_lint_check_url(self): len(mocked_ModuleLint.failed) == failed ), f"{test}: Expected {failed} FAIL, got {len(mocked_ModuleLint.failed)}." + def test_modules_lint_update_meta_yml(self): + """update the meta.yml of a module""" + module_lint = nf_core.modules.lint.ModuleLint(directory=self.nfcore_modules, fix=True) + module_lint.lint(print_results=False, module="bpipe/test") + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + def test_modules_lint_snapshot_file(self): """Test linting a module with a snapshot file""" module_lint = nf_core.modules.lint.ModuleLint(directory=self.nfcore_modules) @@ -521,25 +521,6 @@ def test_modules_meta_yml_incorrect_licence_field(self): assert len(module_lint.warned) >= 0 assert module_lint.failed[0].lint_test == "meta_yml_valid" - def test_modules_meta_yml_input_mismatch(self): - """Test linting a module with an extra entry in input fields in meta.yml compared to module.input""" - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "main.nf")) as fh: - main_nf = fh.read() - main_nf_new = main_nf.replace("path bam", "path bai") - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "main.nf"), "w") as fh: - fh.write(main_nf_new) - module_lint = nf_core.modules.lint.ModuleLint(directory=self.nfcore_modules) - module_lint.lint(print_results=False, module="bpipe/test") - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "main.nf"), "w") as fh: - fh.write(main_nf) - assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" - assert len(module_lint.passed) >= 0 - assert len(module_lint.warned) == 2, f"Linting warning with {[x.__dict__ for x in module_lint.warned]}" - lint_tests = [x.lint_test for x in module_lint.warned] - # check that it is there twice: - assert lint_tests.count("meta_input_meta_only") == 1 - assert lint_tests.count("meta_input_main_only") == 1 - def test_modules_meta_yml_output_mismatch(self): """Test linting a module with an extra entry in output fields in meta.yml compared to module.output""" with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "main.nf")) as fh: @@ -551,13 +532,9 @@ def test_modules_meta_yml_output_mismatch(self): module_lint.lint(print_results=False, module="bpipe/test") with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "main.nf"), "w") as fh: fh.write(main_nf) - assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) >= 0 - assert len(module_lint.warned) == 2 - lint_tests = [x.lint_test for x in module_lint.warned] - # check that it is there twice: - assert lint_tests.count("meta_output_meta_only") == 1 - assert lint_tests.count("meta_output_main_only") == 1 + assert "Module `meta.yml` does not match `main.nf`" in module_lint.failed[0].message def test_modules_meta_yml_incorrect_name(self): """Test linting a module with an incorrect name in meta.yml""" diff --git a/tests/modules/test_patch.py b/tests/modules/test_patch.py index c3eb94d374..1cdb400ef1 100644 --- a/tests/modules/test_patch.py +++ b/tests/modules/test_patch.py @@ -21,8 +21,8 @@ testing if the update commands works correctly with patch files """ -ORG_SHA = "002623ccc88a3b0cb302c7d8f13792a95354d9f2" -CORRECT_SHA = "1dff30bfca2d98eb7ac7b09269a15e822451d99f" +ORG_SHA = "63e780200600e340365b669f9c673b670764c569" +CORRECT_SHA = "63e780200600e340365b669f9c673b670764c569" SUCCEED_SHA = "ba15c20c032c549d77c5773659f19c2927daf48e" FAIL_SHA = "67b642d4471c4005220a342cad3818d5ba2b5a73" BISMARK_ALIGN = "bismark/align" From b41e64c2e664db3c3464bf19dd75d99ec67228cb Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 23 Sep 2024 13:41:48 +0200 Subject: [PATCH 76/96] fix subworkflows pytest --- tests/subworkflows/test_lint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/subworkflows/test_lint.py b/tests/subworkflows/test_lint.py index 1598531d2d..d94b55b3d3 100644 --- a/tests/subworkflows/test_lint.py +++ b/tests/subworkflows/test_lint.py @@ -65,7 +65,7 @@ def test_subworkflows_lint_multiple_remotes(self): def test_subworkflows_lint_update_meta_yml(self): """update the meta.yml of a subworkflow""" - subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules, fix=True) + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.nfcore_modules, fix=True) subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" assert len(subworkflow_lint.passed) > 0 From b249b135ed5810fbdc29f6013e297cc2820f9234 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 23 Sep 2024 14:02:41 +0200 Subject: [PATCH 77/96] update pipeline for multiqc update --- nf_core/pipeline-template/workflows/pipeline.nf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 5b7e0ff143..f878bb31a2 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -95,7 +95,9 @@ workflow {{ short_name|upper }} { ch_multiqc_files.collect(), ch_multiqc_config.toList(), ch_multiqc_custom_config.toList(), - ch_multiqc_logo.toList() + ch_multiqc_logo.toList(), + [], + [] ) {% endif %} emit: From a2ffa4427e7feb877c2fe1fe4e0f7ebf82463893 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 23 Sep 2024 15:10:02 +0200 Subject: [PATCH 78/96] fix more pytests --- .../modules/nf-core/fastqc/tests/main.nf.test | 225 ++++++++--- .../nf-core/fastqc/tests/main.nf.test.snap | 370 ++++++++++++++++-- .../nf-core/multiqc/tests/main.nf.test | 8 + .../nf-core/multiqc/tests/main.nf.test.snap | 20 +- tests/modules/test_patch.py | 26 +- 5 files changed, 529 insertions(+), 120 deletions(-) diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test index 70edae4d99..e9d79a074e 100644 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test @@ -23,17 +23,14 @@ nextflow_process { then { assertAll ( - { assert process.success }, - - // NOTE The report contains the date inside it, which means that the md5sum is stable per day, but not longer than that. So you can't md5sum it. - // looks like this:
Mon 2 Oct 2023
test.gz
- // https://github.com/nf-core/modules/pull/3903#issuecomment-1743620039 - - { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, - { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, - { assert path(process.out.html[0][1]).text.contains("File typeConventional base calls") }, - - { assert snapshot(process.out.versions).match("fastqc_versions_single") } + { assert process.success }, + // NOTE The report contains the date inside it, which means that the md5sum is stable per day, but not longer than that. So you can't md5sum it. + // looks like this:
Mon 2 Oct 2023
test.gz
+ // https://github.com/nf-core/modules/pull/3903#issuecomment-1743620039 + { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, + { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, + { assert path(process.out.html[0][1]).text.contains("File typeConventional base calls") }, + { assert snapshot(process.out.versions).match() } ) } } @@ -54,16 +51,14 @@ nextflow_process { then { assertAll ( - { assert process.success }, - - { assert process.out.html[0][1][0] ==~ ".*/test_1_fastqc.html" }, - { assert process.out.html[0][1][1] ==~ ".*/test_2_fastqc.html" }, - { assert process.out.zip[0][1][0] ==~ ".*/test_1_fastqc.zip" }, - { assert process.out.zip[0][1][1] ==~ ".*/test_2_fastqc.zip" }, - { assert path(process.out.html[0][1][0]).text.contains("File typeConventional base calls") }, - { assert path(process.out.html[0][1][1]).text.contains("File typeConventional base calls") }, - - { assert snapshot(process.out.versions).match("fastqc_versions_paired") } + { assert process.success }, + { assert process.out.html[0][1][0] ==~ ".*/test_1_fastqc.html" }, + { assert process.out.html[0][1][1] ==~ ".*/test_2_fastqc.html" }, + { assert process.out.zip[0][1][0] ==~ ".*/test_1_fastqc.zip" }, + { assert process.out.zip[0][1][1] ==~ ".*/test_2_fastqc.zip" }, + { assert path(process.out.html[0][1][0]).text.contains("File typeConventional base calls") }, + { assert path(process.out.html[0][1][1]).text.contains("File typeConventional base calls") }, + { assert snapshot(process.out.versions).match() } ) } } @@ -83,13 +78,11 @@ nextflow_process { then { assertAll ( - { assert process.success }, - - { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, - { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, - { assert path(process.out.html[0][1]).text.contains("File typeConventional base calls") }, - - { assert snapshot(process.out.versions).match("fastqc_versions_interleaved") } + { assert process.success }, + { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, + { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, + { assert path(process.out.html[0][1]).text.contains("File typeConventional base calls") }, + { assert snapshot(process.out.versions).match() } ) } } @@ -109,13 +102,11 @@ nextflow_process { then { assertAll ( - { assert process.success }, - - { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, - { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, - { assert path(process.out.html[0][1]).text.contains("File typeConventional base calls") }, - - { assert snapshot(process.out.versions).match("fastqc_versions_bam") } + { assert process.success }, + { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, + { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, + { assert path(process.out.html[0][1]).text.contains("File typeConventional base calls") }, + { assert snapshot(process.out.versions).match() } ) } } @@ -138,22 +129,20 @@ nextflow_process { then { assertAll ( - { assert process.success }, - - { assert process.out.html[0][1][0] ==~ ".*/test_1_fastqc.html" }, - { assert process.out.html[0][1][1] ==~ ".*/test_2_fastqc.html" }, - { assert process.out.html[0][1][2] ==~ ".*/test_3_fastqc.html" }, - { assert process.out.html[0][1][3] ==~ ".*/test_4_fastqc.html" }, - { assert process.out.zip[0][1][0] ==~ ".*/test_1_fastqc.zip" }, - { assert process.out.zip[0][1][1] ==~ ".*/test_2_fastqc.zip" }, - { assert process.out.zip[0][1][2] ==~ ".*/test_3_fastqc.zip" }, - { assert process.out.zip[0][1][3] ==~ ".*/test_4_fastqc.zip" }, - { assert path(process.out.html[0][1][0]).text.contains("File typeConventional base calls") }, - { assert path(process.out.html[0][1][1]).text.contains("File typeConventional base calls") }, - { assert path(process.out.html[0][1][2]).text.contains("File typeConventional base calls") }, - { assert path(process.out.html[0][1][3]).text.contains("File typeConventional base calls") }, - - { assert snapshot(process.out.versions).match("fastqc_versions_multiple") } + { assert process.success }, + { assert process.out.html[0][1][0] ==~ ".*/test_1_fastqc.html" }, + { assert process.out.html[0][1][1] ==~ ".*/test_2_fastqc.html" }, + { assert process.out.html[0][1][2] ==~ ".*/test_3_fastqc.html" }, + { assert process.out.html[0][1][3] ==~ ".*/test_4_fastqc.html" }, + { assert process.out.zip[0][1][0] ==~ ".*/test_1_fastqc.zip" }, + { assert process.out.zip[0][1][1] ==~ ".*/test_2_fastqc.zip" }, + { assert process.out.zip[0][1][2] ==~ ".*/test_3_fastqc.zip" }, + { assert process.out.zip[0][1][3] ==~ ".*/test_4_fastqc.zip" }, + { assert path(process.out.html[0][1][0]).text.contains("File typeConventional base calls") }, + { assert path(process.out.html[0][1][1]).text.contains("File typeConventional base calls") }, + { assert path(process.out.html[0][1][2]).text.contains("File typeConventional base calls") }, + { assert path(process.out.html[0][1][3]).text.contains("File typeConventional base calls") }, + { assert snapshot(process.out.versions).match() } ) } } @@ -173,21 +162,18 @@ nextflow_process { then { assertAll ( - { assert process.success }, - - { assert process.out.html[0][1] ==~ ".*/mysample_fastqc.html" }, - { assert process.out.zip[0][1] ==~ ".*/mysample_fastqc.zip" }, - { assert path(process.out.html[0][1]).text.contains("File typeConventional base calls") }, - - { assert snapshot(process.out.versions).match("fastqc_versions_custom_prefix") } + { assert process.success }, + { assert process.out.html[0][1] ==~ ".*/mysample_fastqc.html" }, + { assert process.out.zip[0][1] ==~ ".*/mysample_fastqc.zip" }, + { assert path(process.out.html[0][1]).text.contains("File typeConventional base calls") }, + { assert snapshot(process.out.versions).match() } ) } } test("sarscov2 single-end [fastq] - stub") { - options "-stub" - + options "-stub" when { process { """ @@ -201,12 +187,123 @@ nextflow_process { then { assertAll ( - { assert process.success }, - { assert snapshot(process.out.html.collect { file(it[1]).getName() } + - process.out.zip.collect { file(it[1]).getName() } + - process.out.versions ).match("fastqc_stub") } + { assert process.success }, + { assert snapshot(process.out).match() } ) } } + test("sarscov2 paired-end [fastq] - stub") { + + options "-stub" + when { + process { + """ + input[0] = Channel.of([ + [id: 'test', single_end: false], // meta map + [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) ] + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } + + test("sarscov2 interleaved [fastq] - stub") { + + options "-stub" + when { + process { + """ + input[0] = Channel.of([ + [id: 'test', single_end: false], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_interleaved.fastq.gz', checkIfExists: true) + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } + + test("sarscov2 paired-end [bam] - stub") { + + options "-stub" + when { + process { + """ + input[0] = Channel.of([ + [id: 'test', single_end: false], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true) + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } + + test("sarscov2 multiple [fastq] - stub") { + + options "-stub" + when { + process { + """ + input[0] = Channel.of([ + [id: 'test', single_end: false], // meta map + [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test2_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test2_2.fastq.gz', checkIfExists: true) ] + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } + + test("sarscov2 custom_prefix - stub") { + + options "-stub" + when { + process { + """ + input[0] = Channel.of([ + [ id:'mysample', single_end:true ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } } diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test.snap b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test.snap index 86f7c31154..d5db3092fb 100644 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test.snap +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test.snap @@ -1,88 +1,392 @@ { - "fastqc_versions_interleaved": { + "sarscov2 custom_prefix": { "content": [ [ "versions.yml:md5,e1cc25ca8af856014824abd842e93978" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "24.04.3" }, - "timestamp": "2024-01-31T17:40:07.293713" + "timestamp": "2024-07-22T11:02:16.374038" }, - "fastqc_stub": { + "sarscov2 single-end [fastq] - stub": { "content": [ - [ - "test.html", - "test.zip", - "versions.yml:md5,e1cc25ca8af856014824abd842e93978" - ] + { + "0": [ + [ + { + "id": "test", + "single_end": true + }, + "test.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test", + "single_end": true + }, + "test.zip:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ], + "html": [ + [ + { + "id": "test", + "single_end": true + }, + "test.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ], + "zip": [ + [ + { + "id": "test", + "single_end": true + }, + "test.zip:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.3" + }, + "timestamp": "2024-07-22T11:02:24.993809" + }, + "sarscov2 custom_prefix - stub": { + "content": [ + { + "0": [ + [ + { + "id": "mysample", + "single_end": true + }, + "mysample.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "mysample", + "single_end": true + }, + "mysample.zip:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ], + "html": [ + [ + { + "id": "mysample", + "single_end": true + }, + "mysample.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ], + "zip": [ + [ + { + "id": "mysample", + "single_end": true + }, + "mysample.zip:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "24.04.3" }, - "timestamp": "2024-01-31T17:31:01.425198" + "timestamp": "2024-07-22T11:03:10.93942" }, - "fastqc_versions_multiple": { + "sarscov2 interleaved [fastq]": { "content": [ [ "versions.yml:md5,e1cc25ca8af856014824abd842e93978" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "24.04.3" }, - "timestamp": "2024-01-31T17:40:55.797907" + "timestamp": "2024-07-22T11:01:42.355718" }, - "fastqc_versions_bam": { + "sarscov2 paired-end [bam]": { "content": [ [ "versions.yml:md5,e1cc25ca8af856014824abd842e93978" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "24.04.3" }, - "timestamp": "2024-01-31T17:40:26.795862" + "timestamp": "2024-07-22T11:01:53.276274" }, - "fastqc_versions_single": { + "sarscov2 multiple [fastq]": { "content": [ [ "versions.yml:md5,e1cc25ca8af856014824abd842e93978" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "24.04.3" }, - "timestamp": "2024-01-31T17:39:27.043675" + "timestamp": "2024-07-22T11:02:05.527626" }, - "fastqc_versions_paired": { + "sarscov2 paired-end [fastq]": { "content": [ [ "versions.yml:md5,e1cc25ca8af856014824abd842e93978" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "24.04.3" + }, + "timestamp": "2024-07-22T11:01:31.188871" + }, + "sarscov2 paired-end [fastq] - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": false + }, + "test.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test", + "single_end": false + }, + "test.zip:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ], + "html": [ + [ + { + "id": "test", + "single_end": false + }, + "test.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ], + "zip": [ + [ + { + "id": "test", + "single_end": false + }, + "test.zip:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.3" + }, + "timestamp": "2024-07-22T11:02:34.273566" + }, + "sarscov2 multiple [fastq] - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": false + }, + "test.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test", + "single_end": false + }, + "test.zip:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ], + "html": [ + [ + { + "id": "test", + "single_end": false + }, + "test.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ], + "zip": [ + [ + { + "id": "test", + "single_end": false + }, + "test.zip:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.3" }, - "timestamp": "2024-01-31T17:39:47.584191" + "timestamp": "2024-07-22T11:03:02.304411" }, - "fastqc_versions_custom_prefix": { + "sarscov2 single-end [fastq]": { "content": [ [ "versions.yml:md5,e1cc25ca8af856014824abd842e93978" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "24.04.3" + }, + "timestamp": "2024-07-22T11:01:19.095607" + }, + "sarscov2 interleaved [fastq] - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": false + }, + "test.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test", + "single_end": false + }, + "test.zip:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ], + "html": [ + [ + { + "id": "test", + "single_end": false + }, + "test.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ], + "zip": [ + [ + { + "id": "test", + "single_end": false + }, + "test.zip:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.3" + }, + "timestamp": "2024-07-22T11:02:44.640184" + }, + "sarscov2 paired-end [bam] - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": false + }, + "test.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test", + "single_end": false + }, + "test.zip:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ], + "html": [ + [ + { + "id": "test", + "single_end": false + }, + "test.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ], + "zip": [ + [ + { + "id": "test", + "single_end": false + }, + "test.zip:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.3" }, - "timestamp": "2024-01-31T17:41:14.576531" + "timestamp": "2024-07-22T11:02:53.550742" } } \ No newline at end of file diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test b/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test index f1c4242ef2..33316a7ddb 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test @@ -8,6 +8,8 @@ nextflow_process { tag "modules_nfcore" tag "multiqc" + config "./nextflow.config" + test("sarscov2 single-end [fastqc]") { when { @@ -17,6 +19,8 @@ nextflow_process { input[1] = [] input[2] = [] input[3] = [] + input[4] = [] + input[5] = [] """ } } @@ -41,6 +45,8 @@ nextflow_process { input[1] = Channel.of(file("https://github.com/nf-core/tools/raw/dev/nf_core/pipeline-template/assets/multiqc_config.yml", checkIfExists: true)) input[2] = [] input[3] = [] + input[4] = [] + input[5] = [] """ } } @@ -66,6 +72,8 @@ nextflow_process { input[1] = [] input[2] = [] input[3] = [] + input[4] = [] + input[5] = [] """ } } diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap b/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap index bfebd80298..b779e46924 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap @@ -2,14 +2,14 @@ "multiqc_versions_single": { "content": [ [ - "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" + "versions.yml:md5,8c8724363a5efe0c6f43ab34faa57efd" ] ], "meta": { "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nextflow": "24.04.2" }, - "timestamp": "2024-02-29T08:48:55.657331" + "timestamp": "2024-07-10T12:41:34.562023" }, "multiqc_stub": { "content": [ @@ -17,25 +17,25 @@ "multiqc_report.html", "multiqc_data", "multiqc_plots", - "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" + "versions.yml:md5,8c8724363a5efe0c6f43ab34faa57efd" ] ], "meta": { "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nextflow": "24.04.2" }, - "timestamp": "2024-02-29T08:49:49.071937" + "timestamp": "2024-07-10T11:27:11.933869532" }, "multiqc_versions_config": { "content": [ [ - "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" + "versions.yml:md5,8c8724363a5efe0c6f43ab34faa57efd" ] ], "meta": { "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nextflow": "24.04.2" }, - "timestamp": "2024-02-29T08:49:25.457567" + "timestamp": "2024-07-10T11:26:56.709849369" } -} \ No newline at end of file +} diff --git a/tests/modules/test_patch.py b/tests/modules/test_patch.py index 1cdb400ef1..2f60cd4a20 100644 --- a/tests/modules/test_patch.py +++ b/tests/modules/test_patch.py @@ -21,10 +21,10 @@ testing if the update commands works correctly with patch files """ -ORG_SHA = "63e780200600e340365b669f9c673b670764c569" +ORG_SHA = "3dc7c14d29af40f1a0871a675364e437559d97a8" CORRECT_SHA = "63e780200600e340365b669f9c673b670764c569" -SUCCEED_SHA = "ba15c20c032c549d77c5773659f19c2927daf48e" -FAIL_SHA = "67b642d4471c4005220a342cad3818d5ba2b5a73" +SUCCEED_SHA = "0d0515c3f11266e1314e129bec3e308f804c8dc7" +FAIL_SHA = "cb64a5c1ef85619b89ab99dec2e9097fe84e1dc8" BISMARK_ALIGN = "bismark/align" REPO_NAME = "nf-core-test" PATCH_BRANCH = "patch-tester" @@ -76,7 +76,7 @@ def test_create_patch_no_change(self): module_path = Path(self.pipeline_dir, "modules", REPO_NAME, BISMARK_ALIGN) # Check that no patch file has been added to the directory - assert set(os.listdir(module_path)) == {"main.nf", "meta.yml"} + assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", "environment.yml"} # Check the 'modules.json' contains no patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) @@ -94,7 +94,7 @@ def test_create_patch_change(self): patch_fn = f"{'-'.join(BISMARK_ALIGN.split('/'))}.diff" # Check that a patch file with the correct name has been created - assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", patch_fn} + assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", "environment.yml", patch_fn} # Check the 'modules.json' contains a patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) @@ -127,7 +127,7 @@ def test_create_patch_try_apply_successful(self): patch_fn = f"{'-'.join(BISMARK_ALIGN.split('/'))}.diff" # Check that a patch file with the correct name has been created - assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", patch_fn} + assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", "environment.yml", patch_fn} # Check the 'modules.json' contains a patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) @@ -153,7 +153,7 @@ def test_create_patch_try_apply_successful(self): update_obj.move_files_from_tmp_dir(BISMARK_ALIGN, install_dir, REPO_NAME, SUCCEED_SHA) # Check that a patch file with the correct name has been created - assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", patch_fn} + assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", "environment.yml", patch_fn} # Check the 'modules.json' contains a patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) @@ -195,7 +195,7 @@ def test_create_patch_try_apply_failed(self): patch_fn = f"{'-'.join(BISMARK_ALIGN.split('/'))}.diff" # Check that a patch file with the correct name has been created - assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", patch_fn} + assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", "environment.yml", patch_fn} # Check the 'modules.json' contains a patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) @@ -234,7 +234,7 @@ def test_create_patch_update_success(self): patch_fn = f"{'-'.join(BISMARK_ALIGN.split('/'))}.diff" # Check that a patch file with the correct name has been created - assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", patch_fn} + assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", "environment.yml", patch_fn} # Check the 'modules.json' contains a patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) @@ -254,7 +254,7 @@ def test_create_patch_update_success(self): assert update_obj.update(BISMARK_ALIGN) # Check that a patch file with the correct name has been created - assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", patch_fn} + assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", "environment.yml", patch_fn} # Check the 'modules.json' contains a patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) @@ -295,7 +295,7 @@ def test_create_patch_update_fail(self): patch_fn = f"{'-'.join(BISMARK_ALIGN.split('/'))}.diff" # Check that a patch file with the correct name has been created - assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", patch_fn} + assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", "environment.yml", patch_fn} # Check the 'modules.json' contains a patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) @@ -349,7 +349,7 @@ def test_remove_patch(self): # Check that a patch file with the correct name has been created patch_fn = f"{'-'.join(BISMARK_ALIGN.split('/'))}.diff" - assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", patch_fn} + assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", "environment.yml", patch_fn} # Check the 'modules.json' contains a patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) @@ -361,7 +361,7 @@ def test_remove_patch(self): mock_questionary.unsafe_ask.return_value = True patch_obj.remove(BISMARK_ALIGN) # Check that the diff file has been removed - assert set(os.listdir(module_path)) == {"main.nf", "meta.yml"} + assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", "environment.yml"} # Check that the 'modules.json' entry has been removed modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) From 34ead3efa80d57652237878f64b7870c1a4d382e Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 13 Sep 2024 16:46:16 +0200 Subject: [PATCH 79/96] remove try-catch blocks from nextflow.config --- CHANGELOG.md | 1 + nf_core/pipeline-template/nextflow.config | 13 +++---------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68399a9c0b..a1cab78972 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ - Add tests to ensure all files are part of a template customisation group and all groups are tested ([#3099](https://github.com/nf-core/tools/pull/3099)) - Replaces the old custom `check_max()` function with the Nextflow native `resourceLimits` directive ([#3037](https://github.com/nf-core/tools/pull/3037)) - Fixed release announcement hashtags for Mastodon ([#3099](https://github.com/nf-core/tools/pull/3176)) +- Remove try/catch blocks from `nextflow.config` ([#3167](https://github.com/nf-core/tools/pull/3167)) ### Linting diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 3e301f0b0b..28b7d7963e 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -186,18 +186,11 @@ profiles { {% if nf_core_configs -%} // Load nf-core custom profiles from different Institutions -try { - includeConfig "${params.custom_config_base}/nfcore_custom.config" -} catch (Exception e) { - System.err.println("WARNING: Could not load nf-core/config profiles: ${params.custom_config_base}/nfcore_custom.config") -} +includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? "${params.custom_config_base}/nfcore_custom.config" : "/dev/null" // Load {{ name }} custom profiles from different institutions. -try { - includeConfig "${params.custom_config_base}/pipeline/{{ short_name }}.config" -} catch (Exception e) { - System.err.println("WARNING: Could not load nf-core/config/{{ short_name }} profiles: ${params.custom_config_base}/pipeline/{{ short_name }}.config") -} +// TODO nf-core: Optionally, you can add a pipeline-specific nf-core config at https://github.com/nf-core/configs +includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? "${params.custom_config_base}/pipeline/{{ short_name }}.config" : "/dev/null" {% endif -%} // Set default registry for Apptainer, Docker, Podman, Charliecloud and Singularity independent of -profile From 930933b9ee077160907b61a2d5a7fcd3b776ba0f Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 16 Sep 2024 11:52:34 +0200 Subject: [PATCH 80/96] change the name of tmp pipeline to be consistent with all tests --- tests/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index 1d5a8a115d..e04a4b038e 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -104,7 +104,7 @@ def create_tmp_pipeline(no_git: bool = False) -> Tuple[Path, Path, str, Path]: tmp_dir = Path(tempfile.TemporaryDirectory().name) root_repo_dir = Path(__file__).resolve().parent.parent template_dir = root_repo_dir / "nf_core" / "pipeline-template" - pipeline_name = "mypipeline" + pipeline_name = "testpipeline" pipeline_dir = tmp_dir / pipeline_name pipeline_dir.mkdir(parents=True) @@ -114,7 +114,7 @@ def create_tmp_pipeline(no_git: bool = False) -> Tuple[Path, Path, str, Path]: org_path="nf-core", lint=None, template=NFCoreTemplateConfig( - name="mypipeline", + name="testpipeline", author="me", description="it is mine", org="nf-core", From e6fdba8886cfdf5bbcb7cef2c6b319da7825a3a9 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 16 Sep 2024 12:19:46 +0200 Subject: [PATCH 81/96] update nextflow_config linting to include the new includeconfig line --- nf_core/pipelines/lint/nextflow_config.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index 14e91cc609..dd45621bc6 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -346,7 +346,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: failed.append(f"Config `params.custom_config_base` is not set to `{custom_config_base}`") # Check that lines for loading custom profiles exist - lines = [ + old_lines = [ r"// Load nf-core custom profiles from different Institutions", r"try {", r'includeConfig "${params.custom_config_base}/nfcore_custom.config"', @@ -354,11 +354,19 @@ def nextflow_config(self) -> Dict[str, List[str]]: r'System.err.println("WARNING: Could not load nf-core/config profiles: ${params.custom_config_base}/nfcore_custom.config")', r"}", ] + lines = [ + r"// Load nf-core custom profiles from different Institutions", + r'''includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? "${params.custom_config_base}/nfcore_custom.config" : "/dev/null"''', + ] path = Path(self.wf_path, "nextflow.config") i = 0 with open(path) as f: for line in f: - if lines[i] in line: + if old_lines[i] in line: + i += 1 + if i == len(old_lines): + break + elif lines[i] in line: i += 1 if i == len(lines): break @@ -366,6 +374,12 @@ def nextflow_config(self) -> Dict[str, List[str]]: i = 0 if i == len(lines): passed.append("Lines for loading custom profiles found") + elif i == len(old_lines): + failed.append( + "Old lines for loading custom profiles found. File should contain: ```groovy\n{}".format( + "\n".join(lines) + ) + ) else: lines[2] = f"\t{lines[2]}" lines[4] = f"\t{lines[4]}" From fc6d875203059109f3d75b8a28f11f0ec0d0f945 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 25 Sep 2024 11:35:48 +0200 Subject: [PATCH 82/96] add linting for igenomes_ignored.config --- nf_core/pipelines/lint/files_exist.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipelines/lint/files_exist.py b/nf_core/pipelines/lint/files_exist.py index bd25ff33d0..9dd307d8b5 100644 --- a/nf_core/pipelines/lint/files_exist.py +++ b/nf_core/pipelines/lint/files_exist.py @@ -167,6 +167,7 @@ def files_exist(self) -> Dict[str, List[str]]: [Path("assets", "multiqc_config.yml")], [Path("conf", "base.config")], [Path("conf", "igenomes.config")], + [Path("conf", "igenomes_ignored.config")], [Path(".github", "workflows", "awstest.yml")], [Path(".github", "workflows", "awsfulltest.yml")], [Path("modules.json")], From 854ddf9ce85ebc72711e1e7474b07c05925e38ca Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 25 Sep 2024 16:00:19 +0200 Subject: [PATCH 83/96] remove missing check_max from nextflow.config --- nf_core/pipeline-template/nextflow.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index d129d1a492..c51fe52d3b 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -67,9 +67,9 @@ includeConfig 'conf/base.config' {%- else %} process { // TODO nf-core: Check the defaults for all processes - cpus = { check_max( 1 * task.attempt, 'cpus' ) } - memory = { check_max( 6.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } + cpus = { 1 * task.attempt } + memory = { 6.GB * task.attempt } + time = { 4.h * task.attempt } errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' } maxRetries = 1 From 2c64df3680c37518fde6480772398cd1aef32445 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Fri, 27 Sep 2024 08:33:57 +0000 Subject: [PATCH 84/96] regenerate textual snapshots which are now stored in separate files --- .../__snapshots__/test_create_app.ambr | 3321 ----------------- .../test_basic_details_custom.svg | 271 ++ .../test_basic_details_nfcore.svg | 274 ++ .../test_create_app/test_choose_type.svg | 269 ++ .../test_customisation_help.svg | 275 ++ .../test_create_app/test_final_details.svg | 269 ++ .../test_create_app/test_github_details.svg | 276 ++ .../test_github_exit_message.svg | 272 ++ .../test_create_app/test_github_question.svg | 265 ++ .../test_create_app/test_type_custom.svg | 273 ++ .../test_create_app/test_type_nfcore.svg | 272 ++ .../test_type_nfcore_validation.svg | 273 ++ .../test_create_app/test_welcome.svg | 271 ++ 13 files changed, 3260 insertions(+), 3321 deletions(-) delete mode 100644 tests/pipelines/__snapshots__/test_create_app.ambr create mode 100644 tests/pipelines/__snapshots__/test_create_app/test_basic_details_custom.svg create mode 100644 tests/pipelines/__snapshots__/test_create_app/test_basic_details_nfcore.svg create mode 100644 tests/pipelines/__snapshots__/test_create_app/test_choose_type.svg create mode 100644 tests/pipelines/__snapshots__/test_create_app/test_customisation_help.svg create mode 100644 tests/pipelines/__snapshots__/test_create_app/test_final_details.svg create mode 100644 tests/pipelines/__snapshots__/test_create_app/test_github_details.svg create mode 100644 tests/pipelines/__snapshots__/test_create_app/test_github_exit_message.svg create mode 100644 tests/pipelines/__snapshots__/test_create_app/test_github_question.svg create mode 100644 tests/pipelines/__snapshots__/test_create_app/test_type_custom.svg create mode 100644 tests/pipelines/__snapshots__/test_create_app/test_type_nfcore.svg create mode 100644 tests/pipelines/__snapshots__/test_create_app/test_type_nfcore_validation.svg create mode 100644 tests/pipelines/__snapshots__/test_create_app/test_welcome.svg diff --git a/tests/pipelines/__snapshots__/test_create_app.ambr b/tests/pipelines/__snapshots__/test_create_app.ambr deleted file mode 100644 index 6e3009e18e..0000000000 --- a/tests/pipelines/__snapshots__/test_create_app.ambr +++ /dev/null @@ -1,3321 +0,0 @@ -# serializer version: 1 -# name: test_basic_details_custom - ''' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - nf-core create - - - - - - - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Basic details - - - - - GitHub organisationWorkflow name - - ▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ - ▊nf-core▎▊Pipeline Name▎ - ▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ - - - - A short description of your pipeline. - - ▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ - ▊Description▎ - ▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ - - - - Name of the main author / authors - - ▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ - ▊Author(s)▎ - ▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Next  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - - - - - - - - - - - - - - - - -  d Toggle dark mode  q Quit  - - - - - ''' -# --- -# name: test_basic_details_nfcore - ''' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - nf-core create - - - - - - - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Basic details - - - - - GitHub organisationWorkflow name - - ▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ - ▊nf-core                                   ▎▊Pipeline Name▎ - ▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ - - - - A short description of your pipeline. - - ▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ - ▊Description▎ - ▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ - - - - Name of the main author / authors - - ▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ - ▊Author(s)▎ - ▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Next  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - - - - - - - - - - - - - - - - -  d Toggle dark mode  q Quit  - - - - - ''' -# --- -# name: test_choose_type - ''' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - nf-core create - - - - - - - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Choose pipeline type - - - - - Choose "nf-core" if:Choose "Custom" if: - - ● You want your pipeline to be part of the ● Your pipeline will never be part of  - nf-core communitynf-core - ● You think that there's an outside chance ● You want full control over all features  - that it ever could be part of nf-corethat are included from the template  - (including those that are mandatory for  - nf-core). - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  nf-core  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Custom  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - - - - - What's the difference? - -   Choosing "nf-core" effectively pre-selects the following template features: - - ● GitHub Actions continuous-integration configuration files: - ▪ Pipeline test runs: Small-scale (GitHub) and large-scale (AWS) - ▪ Code formatting checks with Prettier - ▪ Auto-fix linting functionality using @nf-core-bot - ▪ Marking old issues as stale - ● Inclusion of shared nf-core configuration profiles - - - - - - - - - - - - - - -  d Toggle dark mode  q Quit  - - - - - ''' -# --- -# name: test_customisation_help - ''' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - nf-core create - - - - - - - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Template features - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use a GitHub Create a GitHub  Show help  - ▊▁▁▁▁▁▁▁▁▎        repository.repository for the ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - pipeline. - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github CI testsThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - actions for Continuous - Integration (CI)  - testing▄▄ - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use reference genomesThe pipeline will be  Hide help  - ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - copy of the most  - common reference  - genome files from  - iGenomes - - - Nf-core pipelines are configured to use a copy of the most common reference  - genome files. - - By selecting this option, your pipeline will include a configuration file  - specifying the paths to these files. - - The required code to use these files will also be included in the template.  - When the pipeline user provides an appropriate genome key, the pipeline will - automatically download the required reference files. - ▅▅ - For more information about reference genomes in nf-core pipelines, see the  - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github badgesThe README.md file of  Show help  - ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - include GitHub badges - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  - - - - - ''' -# --- -# name: test_final_details - ''' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - nf-core create - - - - - - - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Final details - - - - - First version of the pipelinePath to the output directory where the  - pipeline will be created - ▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ - ▊1.0.0dev▎▊.                                         ▎ - ▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Finish  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -  d Toggle dark mode  q Quit  - - - - - ''' -# --- -# name: test_github_details - ''' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - nf-core create - - - - - - - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Create GitHub repository - -   Now that we have created a new pipeline locally, we can create a new GitHub repository and push    -   the code to it. - - - - - Your GitHub usernameYour GitHub personal access token▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - for login. Show  - ▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - ▊GitHub username▎▊••••••••••••                  ▎ - ▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ - - - - The name of the organisation where the The name of the new GitHub repository - GitHub repo will be cretaed - ▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ - ▊nf-core                               ▎▊mypipeline                            ▎ - ▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ - - - ▌⚠️ You can't create a repository directly in the nf-core organisation. - ▌Please create the pipeline repo to an organisation where you have access or use your user  - ▌account. A core-team member will be able to transfer the repo to nf-core once the development - ▌has started. - - ▌💡 Your GitHub user account will be used by default if nf-core is given as the org name. - - - ▊▔▔▔▔▔▔▔▔▎Private - ▊▎Select to make the new GitHub repo private. - ▊▁▁▁▁▁▁▁▁▎ - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Create GitHub repo  Finish without creating a repo  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - - - - - - - - - -  d Toggle dark mode  q Quit  - - - - - ''' -# --- -# name: test_github_exit_message - ''' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - nf-core create - - - - - - - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - HowTo create a GitHub repository - - - -                                           ,--./,-. -           ___     __   __   __   ___     /,-._.--~\  -     |\ | |__  __ /  ` /  \ |__) |__         }  { -     | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                           `._,._,' - -   If you would like to create the GitHub repository later, you can do it manually by following  -   these steps: - -  1. Create a new GitHub repository -  2. Add the remote to your local repository: - - - cd <pipeline_directory> - git remote add origin git@github.com:<username>/<repo_name>.git - - -  3. Push the code to the remote: - - - git push --all origin - - - ▌💡 Note the --all flag: this is needed to push all branches to the remote. - - - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Close  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - - - - - - - - - - - - -  d Toggle dark mode  q Quit  - - - - - ''' -# --- -# name: test_github_question - ''' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - nf-core create - - - - - - - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Create GitHub repository - - -   After creating the pipeline template locally, we can create a GitHub repository and push the  -   code to it. - -   Do you want to create a GitHub repository? - - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Create GitHub repo  Finish without creating a repo  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -  d Toggle dark mode  q Quit  - - - - - ''' -# --- -# name: test_type_custom - ''' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - nf-core create - - - - - - - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Template features - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use a GitHub Create a GitHub  Show help  - ▊▁▁▁▁▁▁▁▁▎        repository.repository for the ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - pipeline. - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github CI testsThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - actions for Continuous - Integration (CI)  - testing - ▃▃ - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use reference genomesThe pipeline will be  Show help  - ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - copy of the most  - common reference  - genome files from  - iGenomes - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github badgesThe README.md file of  Show help  - ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - include GitHub badges - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add configuration The pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - profiles containing  - custom parameters  - requried to run  - nf-core pipelines at  - different institutions - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use code lintersThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include code linters ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - and CI tests to lint  - your code: pre-commit, - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  - - - - - ''' -# --- -# name: test_type_nfcore - ''' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - nf-core create - - - - - - - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Template features - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use reference genomesThe pipeline will be  Show help  - ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - copy of the most common - reference genome files  - from iGenomes - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use multiqcThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include the MultiQC ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - module which generates  - an HTML report for  - quality control. - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use fastqcThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include the FastQC ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - module which performs  - quality control  - analysis of input FASTQ - files. - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use nf-schemaUse the nf-schema  Show help  - ▊▁▁▁▁▁▁▁▁▎Nextflow plugin for ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - this pipeline. - - - - - - - - - - - - - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  - - - - - ''' -# --- -# name: test_type_nfcore_validation - ''' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - nf-core create - - - - - - - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Basic details - - - - - GitHub organisationWorkflow name - - ▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ - ▊nf-core                                   ▎▊Pipeline Name▎ - ▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ - Value error, Must be lowercase without  - punctuation. - - - - A short description of your pipeline. - - ▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ - ▊Description▎ - ▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ - Value error, Cannot be left empty. - - - - Name of the main author / authors - - ▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ - ▊Author(s)▎ - ▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ - Value error, Cannot be left empty. - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Next  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - - - - - - - - - - - - -  d Toggle dark mode  q Quit  - - - - - ''' -# --- -# name: test_welcome - ''' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - nf-core create - - - - - - - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - -                                           ,--./,-. -           ___     __   __   __   ___     /,-._.--~\  -     |\ | |__  __ /  ` /  \ |__) |__         }  { -     | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                           `._,._,' - - - - Welcome to the nf-core pipeline creation wizard - -   This app will help you create a new Nextflow pipeline from the nf-core/tools pipeline template. - -   The template helps anyone benefit from nf-core best practices, and is a requirement for nf-core    -   pipelines. - - ▌💡 If you want to add a pipeline to nf-core, please join on Slack and discuss your plans with - ▌the community as early as possible; ideally before you start on your pipeline! See the  - ▌nf-core guidelines and the #new-pipelines Slack channel for more information. - - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Let's go!  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - - - - - - - - - - - - - - - - - - - - - - - - -  d Toggle dark mode  q Quit  - - - - - ''' -# --- diff --git a/tests/pipelines/__snapshots__/test_create_app/test_basic_details_custom.svg b/tests/pipelines/__snapshots__/test_create_app/test_basic_details_custom.svg new file mode 100644 index 0000000000..5c4a15831c --- /dev/null +++ b/tests/pipelines/__snapshots__/test_create_app/test_basic_details_custom.svg @@ -0,0 +1,271 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + nf-core create + + + + + + + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + +Basic details + + + + +GitHub organisationWorkflow name + +▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ +▊nf-core▎▊Pipeline Name▎ +▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ + + + +A short description of your pipeline. + +▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ +▊Description▎ +▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ + + + +Name of the main author / authors + +▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ +▊Author(s)▎ +▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ + +▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + Back  Next  +▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + + + + + + + + + + + + + + + + + d Toggle dark mode  q Quit  + + + diff --git a/tests/pipelines/__snapshots__/test_create_app/test_basic_details_nfcore.svg b/tests/pipelines/__snapshots__/test_create_app/test_basic_details_nfcore.svg new file mode 100644 index 0000000000..4445dc68a6 --- /dev/null +++ b/tests/pipelines/__snapshots__/test_create_app/test_basic_details_nfcore.svg @@ -0,0 +1,274 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + nf-core create + + + + + + + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + +Basic details + + + + +GitHub organisationWorkflow name + +▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ +▊nf-core                                   ▎▊Pipeline Name▎ +▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ + + + +A short description of your pipeline. + +▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ +▊Description▎ +▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ + + + +Name of the main author / authors + +▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ +▊Author(s)▎ +▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ + +▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + Back  Next  +▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + + + + + + + + + + + + + + + + + d Toggle dark mode  q Quit  + + + diff --git a/tests/pipelines/__snapshots__/test_create_app/test_choose_type.svg b/tests/pipelines/__snapshots__/test_create_app/test_choose_type.svg new file mode 100644 index 0000000000..f32d5271ea --- /dev/null +++ b/tests/pipelines/__snapshots__/test_create_app/test_choose_type.svg @@ -0,0 +1,269 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + nf-core create + + + + + + + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + +Choose pipeline type + + + + +Choose "nf-core" if:Choose "Custom" if: + +● You want your pipeline to be part of the ● Your pipeline will never be part of  +nf-core communitynf-core +● You think that there's an outside chance ● You want full control over all features  +that it ever could be part of nf-corethat are included from the template  +(including those that are mandatory for  +nf-core). +▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + nf-core  +▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + Custom  +▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + + + + +What's the difference? + +  Choosing "nf-core" effectively pre-selects the following template features: + +● GitHub Actions continuous-integration configuration files: +▪ Pipeline test runs: Small-scale (GitHub) and large-scale (AWS) +▪ Code formatting checks with Prettier +▪ Auto-fix linting functionality using @nf-core-bot +▪ Marking old issues as stale +● Inclusion of shared nf-core configuration profiles + + + + + + + + + + + + + + + d Toggle dark mode  q Quit  + + + diff --git a/tests/pipelines/__snapshots__/test_create_app/test_customisation_help.svg b/tests/pipelines/__snapshots__/test_create_app/test_customisation_help.svg new file mode 100644 index 0000000000..8969694cbc --- /dev/null +++ b/tests/pipelines/__snapshots__/test_create_app/test_customisation_help.svg @@ -0,0 +1,275 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + nf-core create + + + + + + + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + +Template features + + +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎        Use a GitHub Create a GitHub  Show help  +▊▁▁▁▁▁▁▁▁▎        repository.repository for the ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +pipeline. + +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎        Add Github CI testsThe pipeline will  Show help  +▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +actions for Continuous +Integration (CI)  +testing▄▄ + +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎        Use reference genomesThe pipeline will be  Hide help  +▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +copy of the most  +common reference  +genome files from  +iGenomes + + +Nf-core pipelines are configured to use a copy of the most common reference  +genome files. + +By selecting this option, your pipeline will include a configuration file  +specifying the paths to these files. + +The required code to use these files will also be included in the template.  +When the pipeline user provides an appropriate genome key, the pipeline will +automatically download the required reference files. +▅▅ +For more information about reference genomes in nf-core pipelines, see the  + + +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎        Add Github badgesThe README.md file of  Show help  +▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +include GitHub badges + +▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + Back  Continue  +▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + + d Toggle dark mode  q Quit  + + + diff --git a/tests/pipelines/__snapshots__/test_create_app/test_final_details.svg b/tests/pipelines/__snapshots__/test_create_app/test_final_details.svg new file mode 100644 index 0000000000..ddd0ff57ec --- /dev/null +++ b/tests/pipelines/__snapshots__/test_create_app/test_final_details.svg @@ -0,0 +1,269 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + nf-core create + + + + + + + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + +Final details + + + + +First version of the pipelinePath to the output directory where the  +pipeline will be created +▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ +▊1.0.0dev▎▊.                                         ▎ +▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ + +▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + Back  Finish  +▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + d Toggle dark mode  q Quit  + + + diff --git a/tests/pipelines/__snapshots__/test_create_app/test_github_details.svg b/tests/pipelines/__snapshots__/test_create_app/test_github_details.svg new file mode 100644 index 0000000000..3013b9961e --- /dev/null +++ b/tests/pipelines/__snapshots__/test_create_app/test_github_details.svg @@ -0,0 +1,276 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + nf-core create + + + + + + + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + +Create GitHub repository + +  Now that we have created a new pipeline locally, we can create a new GitHub repository and push    +  the code to it. + + + + +Your GitHub usernameYour GitHub personal access token▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +for login. Show  +▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +▊GitHub username▎▊••••••••••••                  ▎ +▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ + + + +The name of the organisation where the The name of the new GitHub repository +GitHub repo will be cretaed +▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ +▊nf-core                               ▎▊mypipeline                            ▎ +▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ + + +▌⚠️ You can't create a repository directly in the nf-core organisation. +▌Please create the pipeline repo to an organisation where you have access or use your user  +▌account. A core-team member will be able to transfer the repo to nf-core once the development +▌has started. + +▌💡 Your GitHub user account will be used by default if nf-core is given as the org name. + + +▊▔▔▔▔▔▔▔▔▎Private +▊▎Select to make the new GitHub repo private. +▊▁▁▁▁▁▁▁▁▎ +▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + Back  Create GitHub repo  Finish without creating a repo  +▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + + + + + + + + + + d Toggle dark mode  q Quit  + + + diff --git a/tests/pipelines/__snapshots__/test_create_app/test_github_exit_message.svg b/tests/pipelines/__snapshots__/test_create_app/test_github_exit_message.svg new file mode 100644 index 0000000000..3612a062c7 --- /dev/null +++ b/tests/pipelines/__snapshots__/test_create_app/test_github_exit_message.svg @@ -0,0 +1,272 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + nf-core create + + + + + + + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + +HowTo create a GitHub repository + + + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\  +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +  If you would like to create the GitHub repository later, you can do it manually by following  +  these steps: + + 1. Create a new GitHub repository + 2. Add the remote to your local repository: + + +cd <pipeline_directory> +git remote add origin git@github.com:<username>/<repo_name>.git + + + 3. Push the code to the remote: + + +git push --all origin + + +▌💡 Note the --all flag: this is needed to push all branches to the remote. + + + +▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + Close  +▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + + + + + + + + + + + + + d Toggle dark mode  q Quit  + + + diff --git a/tests/pipelines/__snapshots__/test_create_app/test_github_question.svg b/tests/pipelines/__snapshots__/test_create_app/test_github_question.svg new file mode 100644 index 0000000000..a0ca3d70d1 --- /dev/null +++ b/tests/pipelines/__snapshots__/test_create_app/test_github_question.svg @@ -0,0 +1,265 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + nf-core create + + + + + + + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + +Create GitHub repository + + +  After creating the pipeline template locally, we can create a GitHub repository and push the  +  code to it. + +  Do you want to create a GitHub repository? + + +▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + Create GitHub repo  Finish without creating a repo  +▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + d Toggle dark mode  q Quit  + + + diff --git a/tests/pipelines/__snapshots__/test_create_app/test_type_custom.svg b/tests/pipelines/__snapshots__/test_create_app/test_type_custom.svg new file mode 100644 index 0000000000..7ff071efa0 --- /dev/null +++ b/tests/pipelines/__snapshots__/test_create_app/test_type_custom.svg @@ -0,0 +1,273 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + nf-core create + + + + + + + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + +Template features + + +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎        Use a GitHub Create a GitHub  Show help  +▊▁▁▁▁▁▁▁▁▎        repository.repository for the ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +pipeline. + +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎        Add Github CI testsThe pipeline will  Show help  +▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +actions for Continuous +Integration (CI)  +testing +▃▃ +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎        Use reference genomesThe pipeline will be  Show help  +▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +copy of the most  +common reference  +genome files from  +iGenomes + +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎        Add Github badgesThe README.md file of  Show help  +▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +include GitHub badges + +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎        Add configuration The pipeline will  Show help  +▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +profiles containing  +custom parameters  +requried to run  +nf-core pipelines at  +different institutions + +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎        Use code lintersThe pipeline will  Show help  +▊▁▁▁▁▁▁▁▁▎include code linters ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +and CI tests to lint  +your code: pre-commit, +▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + Back  Continue  +▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + + d Toggle dark mode  q Quit  + + + diff --git a/tests/pipelines/__snapshots__/test_create_app/test_type_nfcore.svg b/tests/pipelines/__snapshots__/test_create_app/test_type_nfcore.svg new file mode 100644 index 0000000000..48b9b91e9d --- /dev/null +++ b/tests/pipelines/__snapshots__/test_create_app/test_type_nfcore.svg @@ -0,0 +1,272 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + nf-core create + + + + + + + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + +Template features + + +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎        Use reference genomesThe pipeline will be  Show help  +▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +copy of the most common +reference genome files  +from iGenomes + +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎        Use multiqcThe pipeline will  Show help  +▊▁▁▁▁▁▁▁▁▎include the MultiQC ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +module which generates  +an HTML report for  +quality control. + +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎        Use fastqcThe pipeline will  Show help  +▊▁▁▁▁▁▁▁▁▎include the FastQC ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +module which performs  +quality control  +analysis of input FASTQ +files. + +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎        Use nf-schemaUse the nf-schema  Show help  +▊▁▁▁▁▁▁▁▁▎Nextflow plugin for ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +this pipeline. + + + + + + + + + + + + + +▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + Back  Continue  +▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + + d Toggle dark mode  q Quit  + + + diff --git a/tests/pipelines/__snapshots__/test_create_app/test_type_nfcore_validation.svg b/tests/pipelines/__snapshots__/test_create_app/test_type_nfcore_validation.svg new file mode 100644 index 0000000000..7e55b2b0f2 --- /dev/null +++ b/tests/pipelines/__snapshots__/test_create_app/test_type_nfcore_validation.svg @@ -0,0 +1,273 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + nf-core create + + + + + + + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + +Basic details + + + + +GitHub organisationWorkflow name + +▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ +▊nf-core                                   ▎▊Pipeline Name▎ +▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ +Value error, Must be lowercase without  +punctuation. + + + +A short description of your pipeline. + +▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ +▊Description▎ +▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ +Value error, Cannot be left empty. + + + +Name of the main author / authors + +▊▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▎ +▊Author(s)▎ +▊▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▎ +Value error, Cannot be left empty. + +▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + Back  Next  +▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + + + + + + + + + + + + + d Toggle dark mode  q Quit  + + + diff --git a/tests/pipelines/__snapshots__/test_create_app/test_welcome.svg b/tests/pipelines/__snapshots__/test_create_app/test_welcome.svg new file mode 100644 index 0000000000..2670307c22 --- /dev/null +++ b/tests/pipelines/__snapshots__/test_create_app/test_welcome.svg @@ -0,0 +1,271 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + nf-core create + + + + + + + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\  +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + + + +Welcome to the nf-core pipeline creation wizard + +  This app will help you create a new Nextflow pipeline from the nf-core/tools pipeline template. + +  The template helps anyone benefit from nf-core best practices, and is a requirement for nf-core    +  pipelines. + +▌💡 If you want to add a pipeline to nf-core, please join on Slack and discuss your plans with +▌the community as early as possible; ideally before you start on your pipeline! See the  +▌nf-core guidelines and the #new-pipelines Slack channel for more information. + + +▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + Let's go!  +▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + + + + + + + + + + + + + + + + + + + + + + + + + d Toggle dark mode  q Quit  + + + From 8d5f898f3114d1b029963af7ed60590b56288175 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 27 Sep 2024 11:51:12 +0200 Subject: [PATCH 85/96] remove code duplication and format components info command --- nf_core/components/info.py | 43 +++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/nf_core/components/info.py b/nf_core/components/info.py index e4a58d1618..f3e5bf617c 100644 --- a/nf_core/components/info.py +++ b/nf_core/components/info.py @@ -229,6 +229,25 @@ def get_remote_yaml(self) -> Optional[dict]: self.remote_location = self.modules_repo.remote_url return yaml.safe_load(file_contents) + def generate_params_table(self, type) -> Table: + "Generate a rich table for inputs and outputs" + table = Table(expand=True, show_lines=True, box=box.MINIMAL_HEAVY_HEAD, padding=0) + table.add_column(f":inbox_tray: {type}") + table.add_column("Description") + if self.component_type == "modules": + table.add_column("Pattern", justify="right", style="green") + elif self.component_type == "subworkflows": + table.add_column("Structure", justify="right", style="green") + return table + + def get_channel_structure(self, structure: dict) -> str: + "Get the structure of a channel" + structure_str = "" + for key, info in structure.items(): + pattern = f" - {info['pattern']}" if info.get("pattern") else "" + structure_str += f"{key} ({info['type']}{pattern})" + return structure_str + def generate_component_info_help(self): """Take the parsed meta.yml and generate rich help. @@ -277,14 +296,9 @@ def generate_component_info_help(self): # Inputs if self.meta.get("input"): - inputs_table = Table(expand=True, show_lines=True, box=box.MINIMAL_HEAVY_HEAD, padding=0) - inputs_table.add_column(":inbox_tray: Inputs") - inputs_table.add_column("Description") - if self.component_type == "modules": - inputs_table.add_column("Pattern", justify="right", style="green") - elif self.component_type == "subworkflows": - inputs_table.add_column("Structure", justify="right", style="green") - for input in self.meta["input"]: + inputs_table = self.generate_params_table("Inputs") + for i, input in enumerate(self.meta["input"]): + inputs_table.add_row(f"[italic]input[{i}][/]", "", "") if self.component_type == "modules": for element in input: for key, info in element.items(): @@ -298,23 +312,18 @@ def generate_component_info_help(self): inputs_table.add_row( f"[orange1 on black] {key} [/][dim i]", Markdown(info["description"] if info["description"] else ""), - info.get("structure", ""), + self.get_channel_structure(info["structure"]) if info.get("structure") else "", ) renderables.append(inputs_table) # Outputs if self.meta.get("output"): - outputs_table = Table(expand=True, show_lines=True, box=box.MINIMAL_HEAVY_HEAD, padding=0) - outputs_table.add_column(":outbox_tray: Outputs") - outputs_table.add_column("Description") - if self.component_type == "modules": - inputs_table.add_column("Pattern", justify="right", style="green") - elif self.component_type == "subworkflows": - inputs_table.add_column("Structure", justify="right", style="green") + outputs_table = self.generate_params_table("Outputs") for output in self.meta["output"]: if self.component_type == "modules": for ch_name, elements in output.items(): + outputs_table.add_row(f"{ch_name}", "", "") for element in elements: for key, info in element.items(): outputs_table.add_row( @@ -327,7 +336,7 @@ def generate_component_info_help(self): outputs_table.add_row( f"[orange1 on black] {key} [/][dim i]", Markdown(info["description"] if info["description"] else ""), - info.get("structure", ""), + self.get_channel_structure(info["structure"]) if info.get("structure") else "", ) renderables.append(outputs_table) From efc11ba691b08b06df1e1631e9416de36a503a02 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 27 Sep 2024 16:35:00 +0200 Subject: [PATCH 86/96] add conf/igenomes_ignored.config to linting --- nf_core/pipelines/create/template_features.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipelines/create/template_features.yml b/nf_core/pipelines/create/template_features.yml index c166429431..6f476cdfbe 100644 --- a/nf_core/pipelines/create/template_features.yml +++ b/nf_core/pipelines/create/template_features.yml @@ -77,6 +77,7 @@ igenomes: linting: files_exist: - "conf/igenomes.config" + - "conf/igenomes_ignored.config" nfcore_pipelines: True custom_pipelines: True github_badges: From 27093939c2dec3d770a97ea17222ffbe7c2b6934 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 13 Sep 2024 16:46:16 +0200 Subject: [PATCH 87/96] remove try-catch blocks from nextflow.config --- CHANGELOG.md | 1 + nf_core/pipeline-template/nextflow.config | 13 +++---------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a11aee515..b1c5463a9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ - Remove if/else block to include `igenomes.config` ([#3168](https://github.com/nf-core/tools/pull/3168)) - Replaces the old custom `check_max()` function with the Nextflow native `resourceLimits` directive ([#3037](https://github.com/nf-core/tools/pull/3037)) - Fixed release announcement hashtags for Mastodon ([#3099](https://github.com/nf-core/tools/pull/3176)) +- Remove try/catch blocks from `nextflow.config` ([#3167](https://github.com/nf-core/tools/pull/3167)) ### Linting diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 39c3521363..dc2aea0b72 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -186,18 +186,11 @@ profiles { {% if nf_core_configs -%} // Load nf-core custom profiles from different Institutions -try { - includeConfig "${params.custom_config_base}/nfcore_custom.config" -} catch (Exception e) { - System.err.println("WARNING: Could not load nf-core/config profiles: ${params.custom_config_base}/nfcore_custom.config") -} +includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? "${params.custom_config_base}/nfcore_custom.config" : "/dev/null" // Load {{ name }} custom profiles from different institutions. -try { - includeConfig "${params.custom_config_base}/pipeline/{{ short_name }}.config" -} catch (Exception e) { - System.err.println("WARNING: Could not load nf-core/config/{{ short_name }} profiles: ${params.custom_config_base}/pipeline/{{ short_name }}.config") -} +// TODO nf-core: Optionally, you can add a pipeline-specific nf-core config at https://github.com/nf-core/configs +includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? "${params.custom_config_base}/pipeline/{{ short_name }}.config" : "/dev/null" {% endif -%} // Set default registry for Apptainer, Docker, Podman, Charliecloud and Singularity independent of -profile From b1df038c263bd8893d7971e0cfe30dac746f86d1 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 16 Sep 2024 11:52:34 +0200 Subject: [PATCH 88/96] change the name of tmp pipeline to be consistent with all tests --- tests/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index 6f4b73cccc..022b91227f 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -113,7 +113,7 @@ def create_tmp_pipeline(no_git: bool = False) -> Tuple[Path, Path, str, Path]: tmp_dir = Path(tempfile.TemporaryDirectory().name) root_repo_dir = Path(__file__).resolve().parent.parent template_dir = root_repo_dir / "nf_core" / "pipeline-template" - pipeline_name = "mypipeline" + pipeline_name = "testpipeline" pipeline_dir = tmp_dir / pipeline_name pipeline_dir.mkdir(parents=True) @@ -123,7 +123,7 @@ def create_tmp_pipeline(no_git: bool = False) -> Tuple[Path, Path, str, Path]: org_path="nf-core", lint=None, template=NFCoreTemplateConfig( - name="mypipeline", + name="testpipeline", author="me", description="it is mine", org="nf-core", From 7e738eab0ed726ea21d007822a30e396a62c8ded Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 16 Sep 2024 12:19:46 +0200 Subject: [PATCH 89/96] update nextflow_config linting to include the new includeconfig line --- nf_core/pipelines/lint/nextflow_config.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index 14e91cc609..dd45621bc6 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -346,7 +346,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: failed.append(f"Config `params.custom_config_base` is not set to `{custom_config_base}`") # Check that lines for loading custom profiles exist - lines = [ + old_lines = [ r"// Load nf-core custom profiles from different Institutions", r"try {", r'includeConfig "${params.custom_config_base}/nfcore_custom.config"', @@ -354,11 +354,19 @@ def nextflow_config(self) -> Dict[str, List[str]]: r'System.err.println("WARNING: Could not load nf-core/config profiles: ${params.custom_config_base}/nfcore_custom.config")', r"}", ] + lines = [ + r"// Load nf-core custom profiles from different Institutions", + r'''includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? "${params.custom_config_base}/nfcore_custom.config" : "/dev/null"''', + ] path = Path(self.wf_path, "nextflow.config") i = 0 with open(path) as f: for line in f: - if lines[i] in line: + if old_lines[i] in line: + i += 1 + if i == len(old_lines): + break + elif lines[i] in line: i += 1 if i == len(lines): break @@ -366,6 +374,12 @@ def nextflow_config(self) -> Dict[str, List[str]]: i = 0 if i == len(lines): passed.append("Lines for loading custom profiles found") + elif i == len(old_lines): + failed.append( + "Old lines for loading custom profiles found. File should contain: ```groovy\n{}".format( + "\n".join(lines) + ) + ) else: lines[2] = f"\t{lines[2]}" lines[4] = f"\t{lines[4]}" From e46779b6b8709af658c43ed5533603db306231e1 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 30 Sep 2024 11:04:28 +0200 Subject: [PATCH 90/96] add test on pipeline release to check that custom configs are included --- nf_core/pipeline-template/nextflow.config | 2 +- nf_core/pipelines/lint/__init__.py | 4 ++- nf_core/pipelines/lint/included_configs.py | 37 ++++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 nf_core/pipelines/lint/included_configs.py diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index dc2aea0b72..4c816a2a2c 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -190,7 +190,7 @@ includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? "${pa // Load {{ name }} custom profiles from different institutions. // TODO nf-core: Optionally, you can add a pipeline-specific nf-core config at https://github.com/nf-core/configs -includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? "${params.custom_config_base}/pipeline/{{ short_name }}.config" : "/dev/null" +// includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? "${params.custom_config_base}/pipeline/{{ short_name }}.config" : "/dev/null" {% endif -%} // Set default registry for Apptainer, Docker, Podman, Charliecloud and Singularity independent of -profile diff --git a/nf_core/pipelines/lint/__init__.py b/nf_core/pipelines/lint/__init__.py index 6d27351b62..8cc7c37cb2 100644 --- a/nf_core/pipelines/lint/__init__.py +++ b/nf_core/pipelines/lint/__init__.py @@ -37,6 +37,7 @@ from .configs import base_config, modules_config from .files_exist import files_exist from .files_unchanged import files_unchanged +from .included_configs import included_configs from .merge_markers import merge_markers from .modules_json import modules_json from .modules_structure import modules_structure @@ -101,6 +102,7 @@ class PipelineLint(nf_core.utils.Pipeline): system_exit = system_exit template_strings = template_strings version_consistency = version_consistency + included_configs = included_configs def __init__( self, wf_path, release_mode=False, fix=(), key=None, fail_ignored=False, fail_warned=False, hide_progress=False @@ -152,7 +154,7 @@ def _get_all_lint_tests(release_mode): "base_config", "modules_config", "nfcore_yml", - ] + (["version_consistency"] if release_mode else []) + ] + (["version_consistency", "included_configs"] if release_mode else []) def _load(self) -> bool: """Load information about the pipeline into the PipelineLint object""" diff --git a/nf_core/pipelines/lint/included_configs.py b/nf_core/pipelines/lint/included_configs.py new file mode 100644 index 0000000000..9d4eb1f990 --- /dev/null +++ b/nf_core/pipelines/lint/included_configs.py @@ -0,0 +1,37 @@ +from pathlib import Path + + +def included_configs(self): + """Check that the pipeline nextflow.config includes the pipeline custom configs. + + If the include line is uncommented, the test passes. + If the include line is commented, the test fails. + If the include line is missing, the test warns. + + Can be skipped by adding the following to the .nf-core.yml file: + lint: + included_configs: False + """ + passed = [] + failed = [] + warned = [] + + config_file = Path(self.wf_path / "nextflow.config") + + with open(config_file) as fh: + config = fh.read() + print(self.pipeline_name) + if ( + f"// includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? \"${{params.custom_config_base}}/pipeline/{self.pipeline_name}.config\"" + in config + ): + failed.append("Pipeline config does not include custom configs. Please uncomment the includeConfig line.") + elif ( + "includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? \"${{params.custom_config_base}}/pipeline/{self.pipeline_name}.config\"" + in config + ): + passed.append("Pipeline config includes custom configs.") + else: + warned.append("Pipeline config does not include custom configs. Please add the includeConfig line.") + + return {"passed": passed, "failed": failed, "warned": warned} From 70b231e9ee4eb1361f5bfb0cc47aec41bc73f849 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 30 Sep 2024 11:43:28 +0200 Subject: [PATCH 91/96] fix tests by uncommenting includeConfig statements --- .github/actions/create-lint-wf/action.yml | 6 ++++++ .github/workflows/create-lint-wf.yml | 5 +++++ .github/workflows/create-test-lint-wf-template.yml | 5 +++++ nf_core/pipelines/create/template_features.yml | 1 + nf_core/pipelines/lint/included_configs.py | 1 - 5 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml index 8760901db1..7cc6f85d4f 100644 --- a/.github/actions/create-lint-wf/action.yml +++ b/.github/actions/create-lint-wf/action.yml @@ -53,6 +53,12 @@ runs: run: find nf-core-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \; working-directory: create-lint-wf + # Uncomment includeConfig statement + - name: uncomment include config + shell: bash + run: sed -i 's/\/\/ includeConfig/includeConfig/' nf-core-testpipeline/nextflow.config + working-directory: create-lint-wf + # Replace zenodo.XXXXXX to pass readme linting - name: replace zenodo.XXXXXX shell: bash diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 1a3e283d00..190cd01170 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -78,6 +78,11 @@ jobs: run: find nf-core-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \; working-directory: create-lint-wf + # Uncomment includeConfig statement + - name: uncomment include config + run: sed -i 's/\/\/ includeConfig/includeConfig/' nf-core-testpipeline/nextflow.config + working-directory: create-lint-wf + # Run the other nf-core commands - name: nf-core pipelines list run: nf-core --log-file log.txt pipelines list diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 1fb521b4bb..10aeccd297 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -137,6 +137,11 @@ jobs: run: find my-prefix-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \; working-directory: create-test-lint-wf + # Uncomment includeConfig statement + - name: uncomment include config + run: sed -i 's/\/\/ includeConfig/includeConfig/' nf-core-testpipeline/nextflow.config + working-directory: create-lint-wf + # Replace zenodo.XXXXXX to pass readme linting - name: replace zenodo.XXXXXX run: find my-prefix-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \; diff --git a/nf_core/pipelines/create/template_features.yml b/nf_core/pipelines/create/template_features.yml index 6f476cdfbe..55bb3b0b3e 100644 --- a/nf_core/pipelines/create/template_features.yml +++ b/nf_core/pipelines/create/template_features.yml @@ -120,6 +120,7 @@ nf_core_configs: - "custom_config" - "params.custom_config_version" - "params.custom_config_base" + included_configs: False nfcore_pipelines: False custom_pipelines: True is_nfcore: diff --git a/nf_core/pipelines/lint/included_configs.py b/nf_core/pipelines/lint/included_configs.py index 9d4eb1f990..6cfeb3f8a5 100644 --- a/nf_core/pipelines/lint/included_configs.py +++ b/nf_core/pipelines/lint/included_configs.py @@ -20,7 +20,6 @@ def included_configs(self): with open(config_file) as fh: config = fh.read() - print(self.pipeline_name) if ( f"// includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? \"${{params.custom_config_base}}/pipeline/{self.pipeline_name}.config\"" in config From 3a0a8ca33cc876bfb6299ded1b09f1b9cd4e4dcd Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 30 Sep 2024 11:56:51 +0200 Subject: [PATCH 92/96] fix sed commands --- .github/actions/create-lint-wf/action.yml | 2 +- .github/workflows/create-lint-wf.yml | 2 +- .github/workflows/create-test-lint-wf-template.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml index 7cc6f85d4f..ecd0eef873 100644 --- a/.github/actions/create-lint-wf/action.yml +++ b/.github/actions/create-lint-wf/action.yml @@ -56,7 +56,7 @@ runs: # Uncomment includeConfig statement - name: uncomment include config shell: bash - run: sed -i 's/\/\/ includeConfig/includeConfig/' nf-core-testpipeline/nextflow.config + run: find nf-core-testpipeline -type f -exec sed -i 's/\/\/ includeConfig/includeConfig/' {} \; working-directory: create-lint-wf # Replace zenodo.XXXXXX to pass readme linting diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 190cd01170..e0b4c67cfc 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -80,7 +80,7 @@ jobs: # Uncomment includeConfig statement - name: uncomment include config - run: sed -i 's/\/\/ includeConfig/includeConfig/' nf-core-testpipeline/nextflow.config + run: find nf-core-testpipeline -type f -exec sed -i 's/\/\/ includeConfig/includeConfig/' {} \; working-directory: create-lint-wf # Run the other nf-core commands diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 10aeccd297..86d6b3f6b7 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -139,7 +139,7 @@ jobs: # Uncomment includeConfig statement - name: uncomment include config - run: sed -i 's/\/\/ includeConfig/includeConfig/' nf-core-testpipeline/nextflow.config + run: find nf-core-testpipeline -type f -exec sed -i 's/\/\/ includeConfig/includeConfig/' {} \; working-directory: create-lint-wf # Replace zenodo.XXXXXX to pass readme linting From c779bf4197ae545527cc4251ce27aeb3ee011c3b Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 30 Sep 2024 12:10:19 +0200 Subject: [PATCH 93/96] formatted string --- nf_core/pipelines/lint/included_configs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipelines/lint/included_configs.py b/nf_core/pipelines/lint/included_configs.py index 6cfeb3f8a5..75c4594f41 100644 --- a/nf_core/pipelines/lint/included_configs.py +++ b/nf_core/pipelines/lint/included_configs.py @@ -26,7 +26,7 @@ def included_configs(self): ): failed.append("Pipeline config does not include custom configs. Please uncomment the includeConfig line.") elif ( - "includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? \"${{params.custom_config_base}}/pipeline/{self.pipeline_name}.config\"" + f"includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? \"${{params.custom_config_base}}/pipeline/{self.pipeline_name}.config\"" in config ): passed.append("Pipeline config includes custom configs.") From d625d923933294b9da93cc63de0b4beccdd35752 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 30 Sep 2024 12:16:03 +0200 Subject: [PATCH 94/96] test output directory typo --- .github/workflows/create-test-lint-wf-template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 86d6b3f6b7..debbf8e691 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -140,7 +140,7 @@ jobs: # Uncomment includeConfig statement - name: uncomment include config run: find nf-core-testpipeline -type f -exec sed -i 's/\/\/ includeConfig/includeConfig/' {} \; - working-directory: create-lint-wf + working-directory: create-test-lint-wf # Replace zenodo.XXXXXX to pass readme linting - name: replace zenodo.XXXXXX From 19d21bb91ab2c765406224417db417189ece6af2 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 30 Sep 2024 12:19:38 +0200 Subject: [PATCH 95/96] fix more output directory errors --- .github/workflows/create-test-lint-wf-template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index debbf8e691..d8df2f6905 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -139,7 +139,7 @@ jobs: # Uncomment includeConfig statement - name: uncomment include config - run: find nf-core-testpipeline -type f -exec sed -i 's/\/\/ includeConfig/includeConfig/' {} \; + run: find my-prefix-testpipeline -type f -exec sed -i 's/\/\/ includeConfig/includeConfig/' {} \; working-directory: create-test-lint-wf # Replace zenodo.XXXXXX to pass readme linting From 5f367d18ffa4274ba8eb3084cf94abe6c10e19c2 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 30 Sep 2024 12:32:51 +0200 Subject: [PATCH 96/96] generate API docs --- docs/api/_src/pipeline_lint_tests/included_configs.md | 5 +++++ docs/api/_src/pipeline_lint_tests/index.md | 2 ++ 2 files changed, 7 insertions(+) create mode 100644 docs/api/_src/pipeline_lint_tests/included_configs.md diff --git a/docs/api/_src/pipeline_lint_tests/included_configs.md b/docs/api/_src/pipeline_lint_tests/included_configs.md new file mode 100644 index 0000000000..f68f7da25e --- /dev/null +++ b/docs/api/_src/pipeline_lint_tests/included_configs.md @@ -0,0 +1,5 @@ +# included_configs + + ```{eval-rst} + .. automethod:: nf_core.pipelines.lint.PipelineLint.included_configs + ``` diff --git a/docs/api/_src/pipeline_lint_tests/index.md b/docs/api/_src/pipeline_lint_tests/index.md index 3575c08db4..4dd93442d2 100644 --- a/docs/api/_src/pipeline_lint_tests/index.md +++ b/docs/api/_src/pipeline_lint_tests/index.md @@ -7,6 +7,7 @@ - [base_config](./base_config/) - [files_exist](./files_exist/) - [files_unchanged](./files_unchanged/) + - [included_configs](./included_configs/) - [merge_markers](./merge_markers/) - [modules_config](./modules_config/) - [modules_json](./modules_json/) @@ -16,6 +17,7 @@ - [nfcore_yml](./nfcore_yml/) - [pipeline_name_conventions](./pipeline_name_conventions/) - [pipeline_todos](./pipeline_todos/) + - [plugin_includes](./plugin_includes/) - [readme](./readme/) - [schema_description](./schema_description/) - [schema_lint](./schema_lint/)