diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index bd661ce07c..70125a10f3 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -40,18 +40,20 @@ jobs: - name: 🏗 Set up yq uses: frenck/action-setup-yq@v1 - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Create Matrix id: create_matrix - run: echo "matrix=$(yq 'keys' ${{ github.workspace }}/nf_core/pipelines/create/templatefeatures.yml | awk '{print $2}' | paste -sd "," - | awk '{print "[" $0 "]"}')" >> $GITHUB_OUTPUT + run: | + echo "matrix=$(yq 'keys | tojson(0)' nf_core/pipelines/create/templatefeatures.yml)" >> $GITHUB_OUTPUT RunTestWorkflow: runs-on: ${{ matrix.runner }} + needs: prepare-matrix env: NXF_ANSI_LOG: false strategy: matrix: - TEMPLATE: ${{ needs.prepare-matrix.outputs.all_features }} + TEMPLATE: ${{ fromJson(needs.prepare-matrix.outputs.all_features) }} runner: # use the runner given by the input if it is dispatched manually, run on github if it is a rerun or on self-hosted by default - ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} @@ -66,6 +68,8 @@ jobs: exclude: - TEMPLATE: github - TEMPLATE: is_nfcore + - TEMPLATE: nf_core_configs + profile: "self_hosted_runner" fail-fast: false steps: @@ -94,15 +98,16 @@ jobs: version: latest-everything # Create template files - - name: Create template skip all (except github) + - name: Create template skip ${{ matrix.TEMPLATE }} run: | mkdir create-test-lint-wf export NXF_WORK=$(pwd) - printf "org: my-prefix\nskip: ${{ needs.prepare-matrix.outputs.all_features }}" > create-test-lint-wf/template_skip_all.yml - - - name: Create template skip {{ matrix.TEMPLATE }} - run: | - printf "org: my-prefix\nskip: {{ matrix.TEMPLATE }}" > create-test-lint-wf/template_skip_{{ matrix.TEMPLATE }}.yml + if [ ${{ matrix.TEMPLATE }} == "all" ] + then + printf "org: my-prefix\nskip_features: ${{ needs.prepare-matrix.outputs.all_features }}" > create-test-lint-wf/template_skip_all.yml + else + printf "org: my-prefix\nskip_features: [${{ matrix.TEMPLATE }}]" > create-test-lint-wf/template_skip_${{ matrix.TEMPLATE }}.yml + fi # Create a pipeline from the template - name: create a pipeline from the template ${{ matrix.TEMPLATE }} diff --git a/.prettierignore b/.prettierignore index 9387ee9509..cbe7274a4a 100644 --- a/.prettierignore +++ b/.prettierignore @@ -7,6 +7,7 @@ nf_core/module-template/meta.yml nf_core/pipeline-template/nextflow_schema.json nf_core/pipeline-template/modules.json nf_core/pipeline-template/tower.yml +nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml tests/data/pipeline_create_template_skip.yml # don't run on things handled by ruff *.py diff --git a/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml b/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml index 063690f29f..412f5bd3b3 100644 --- a/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml @@ -2,6 +2,7 @@ name: Bug report description: Report something that is broken or incorrect labels: bug body: +{%- if is_nfcore %} - type: markdown attributes: value: | @@ -9,6 +10,7 @@ body: - [nf-core website: troubleshooting](https://nf-co.re/usage/troubleshooting) - [{{ name }} pipeline documentation](https://nf-co.re/{{ short_name }}/usage) +{%- endif %} - type: textarea id: description diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 3a7f544805..0469d6cfec 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -41,11 +41,11 @@ params { version = false pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/' + {%- if nf_core_configs %} // Config options config_profile_name = null config_profile_description = null - {%- if nf_core_configs %} custom_config_version = 'master' custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" config_profile_contact = null diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 1c52e2298d..4a1a22c3ee 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -77,6 +77,7 @@ } }, {%- endif %} + {%- if nf_core_configs %} "institutional_config_options": { "title": "Institutional config options", "type": "object", @@ -125,6 +126,7 @@ } } }, + {%- endif %} "max_job_request_options": { "title": "Max job request options", "type": "object", @@ -289,9 +291,9 @@ {% if igenomes %}{ "$ref": "#/definitions/reference_genome_options" },{% endif %} - { + {% if nf_core_configs %}{ "$ref": "#/definitions/institutional_config_options" - }, + },{% endif %} { "$ref": "#/definitions/max_job_request_options" }, diff --git a/nf_core/pipelines/create/create.py b/nf_core/pipelines/create/create.py index 200507d8a0..05b04a5422 100644 --- a/nf_core/pipelines/create/create.py +++ b/nf_core/pipelines/create/create.py @@ -218,6 +218,7 @@ def obtain_jinja_params_dict( # Add is_nfcore as an area to skip for non-nf-core pipelines, to skip all nf-core files if not self.config.is_nfcore: skip_areas.append("is_nfcore") + jinja_params["is_nfcore"] = False # Set the last parameters based on the ones provided jinja_params["short_name"] = ( @@ -349,17 +350,9 @@ def render_template(self) -> None: template_stat = os.stat(template_fn_path) os.chmod(output_path, template_stat.st_mode) - # Remove all unused parameters in the nextflow schema - if not self.jinja_params["igenomes"] or not self.jinja_params["nf_core_configs"]: - self.update_nextflow_schema() if self.config.is_nfcore: # Make a logo and save it, if it is a nf-core pipeline self.make_pipeline_logo() - else: - if self.jinja_params["github"]: - # Remove field mentioning nf-core docs - # in the github bug report template - self.remove_nf_core_in_bug_report_template() # Update the .nf-core.yml with linting configurations self.fix_linting() @@ -374,38 +367,8 @@ def render_template(self) -> None: log.debug(f"Dumping pipeline template yml to pipeline config file '{config_fn.name}'") run_prettier_on_file(self.outdir / config_fn) - def update_nextflow_schema(self): - """ - Removes unused parameters from the nextflow schema. - """ - schema_path = self.outdir / "nextflow_schema.json" - - schema = nf_core.pipelines.schema.PipelineSchema() - schema.schema_filename = schema_path - schema.no_prompts = True - schema.load_schema() - schema.get_wf_params() - schema.remove_schema_notfound_configs() - schema.save_schema(suppress_logging=True) - run_prettier_on_file(schema_path) - - def remove_nf_core_in_bug_report_template(self): - """ - Remove the field mentioning nf-core documentation - in the github bug report template - """ - bug_report_path = self.outdir / ".github" / "ISSUE_TEMPLATE" / "bug_report.yml" - - with open(bug_report_path) as fh: - contents = yaml.load(fh, Loader=yaml.FullLoader) - - # Remove the first item in the body, which is the information about the docs - contents["body"].pop(0) - - with open(bug_report_path, "w") as fh: - yaml.dump(contents, fh, default_flow_style=False, sort_keys=False) - - run_prettier_on_file(bug_report_path) + # Run prettier on files + run_prettier_on_file(self.outdir) def fix_linting(self): """ @@ -414,17 +377,18 @@ def fix_linting(self): """ # Create a lint config lint_config = {} - for area in self.skip_areas: + for area in (self.config.skip_features or []) + self.skip_areas: try: for lint_test in self.template_features_yml[area]["linting"]: - if not lint_config[lint_test]: - pass - if self.template_features_yml[area]["linting"][lint_test]: - lint_config.setdefault(lint_test, []).extend( - self.template_features_yml[area]["linting"][lint_test] - ) - else: - lint_config[lint_test] = False + try: + if self.template_features_yml[area]["linting"][lint_test]: + lint_config.setdefault(lint_test, []).extend( + self.template_features_yml[area]["linting"][lint_test] + ) + else: + lint_config[lint_test] = False + except AttributeError: + pass # When linting is False except KeyError: pass # Areas without linting diff --git a/nf_core/pipelines/create/templatefeatures.yml b/nf_core/pipelines/create/templatefeatures.yml index 48ce200551..b97bf347ab 100644 --- a/nf_core/pipelines/create/templatefeatures.yml +++ b/nf_core/pipelines/create/templatefeatures.yml @@ -109,6 +109,8 @@ nf_core_configs: - "process.memory" - "process.time" - "custom_config" + - "params.custom_config_version" + - "params.custom_config_base" nfcore_pipelines: False custom_pipelines: True is_nfcore: @@ -160,6 +162,11 @@ code_linters: - pre-commit (https://pre-commit.com/): used to run all code-linters on every PR and on ever commit if you run `pre-commit install` to install it in your local repository. - editor-config (https://github.com/editorconfig-checker/editorconfig-checker): checks rules such as indentation or trailing spaces. - prettier (https://github.com/prettier/prettier): enforces a consistent style (indentation, quoting, line length, etc). + linting: + files_exist: + - ".editorconfig" + - ".prettierignore" + - ".prettierrc.yml" nfcore_pipelines: False custom_pipelines: True citations: