Merge pull request #3113 from mirpedrol/template-customisation-yaml-file

GHA using the template features yaml file was not working as expected [skip changelog]

.github/workflows/create-test-lint-wf-template.yml

@@ -40,18 +40,20 @@ jobs:
       - name: 🏗 Set up yq
         uses: frenck/action-setup-yq@v1
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
       - name: Create Matrix
         id: create_matrix
-        run: echo "matrix=$(yq 'keys' ${{ github.workspace }}/nf_core/pipelines/create/templatefeatures.yml | awk '{print $2}' | paste -sd "," - | awk '{print "[" $0 "]"}')" >> $GITHUB_OUTPUT
+        run: |
+          echo "matrix=$(yq 'keys | tojson(0)' nf_core/pipelines/create/templatefeatures.yml)" >> $GITHUB_OUTPUT
 
   RunTestWorkflow:
     runs-on: ${{ matrix.runner }}
+    needs: prepare-matrix
     env:
       NXF_ANSI_LOG: false
     strategy:
       matrix:
-        TEMPLATE: ${{ needs.prepare-matrix.outputs.all_features }}
+        TEMPLATE: ${{ fromJson(needs.prepare-matrix.outputs.all_features) }}
         runner:
           # use the runner given by the input if it is dispatched manually, run on github if it is a rerun or on self-hosted by default
           - ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }}
@@ -66,6 +68,8 @@ jobs:
         exclude:
           - TEMPLATE: github
           - TEMPLATE: is_nfcore
+          - TEMPLATE: nf_core_configs
+            profile: "self_hosted_runner"
       fail-fast: false
 
     steps:
@@ -94,15 +98,16 @@ jobs:
           version: latest-everything
 
       # Create template files
-      - name: Create template skip all (except github)
+      - name: Create template skip ${{ matrix.TEMPLATE }}
         run: |
           mkdir create-test-lint-wf
           export NXF_WORK=$(pwd)
-          printf "org: my-prefix\nskip: ${{ needs.prepare-matrix.outputs.all_features }}" > create-test-lint-wf/template_skip_all.yml
-
-      - name: Create template skip {{ matrix.TEMPLATE }}
-        run: |
-          printf "org: my-prefix\nskip: {{ matrix.TEMPLATE }}" > create-test-lint-wf/template_skip_{{ matrix.TEMPLATE }}.yml
+          if [ ${{ matrix.TEMPLATE }} == "all" ]
+          then
+            printf "org: my-prefix\nskip_features: ${{ needs.prepare-matrix.outputs.all_features }}" > create-test-lint-wf/template_skip_all.yml
+          else
+            printf "org: my-prefix\nskip_features: [${{ matrix.TEMPLATE }}]" > create-test-lint-wf/template_skip_${{ matrix.TEMPLATE }}.yml
+          fi
 
       # Create a pipeline from the template
       - name: create a pipeline from the template ${{ matrix.TEMPLATE }}

.prettierignore

@@ -7,6 +7,7 @@ nf_core/module-template/meta.yml
 nf_core/pipeline-template/nextflow_schema.json
 nf_core/pipeline-template/modules.json
 nf_core/pipeline-template/tower.yml
+nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml
 tests/data/pipeline_create_template_skip.yml
 # don't run on things handled by ruff
 *.py

nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -2,13 +2,15 @@ name: Bug report
 description: Report something that is broken or incorrect
 labels: bug
 body:
+{%- if is_nfcore %}
   - type: markdown
     attributes:
       value: |
         Before you post this issue, please check the documentation:
 
         - [nf-core website: troubleshooting](https://nf-co.re/usage/troubleshooting)
         - [{{ name }} pipeline documentation](https://nf-co.re/{{ short_name }}/usage)
+{%- endif %}
 
   - type: textarea
     id: description

nf_core/pipeline-template/nextflow.config

@@ -41,11 +41,11 @@ params {
     version                      = false
     pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/'
 
+    {%- if nf_core_configs %}
     // Config options
     config_profile_name        = null
     config_profile_description = null
 
-    {%- if nf_core_configs %}
     custom_config_version      = 'master'
     custom_config_base         = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}"
     config_profile_contact     = null

nf_core/pipeline-template/nextflow_schema.json

@@ -77,6 +77,7 @@
             }
         },
         {%- endif %}
+        {%- if nf_core_configs %}
         "institutional_config_options": {
             "title": "Institutional config options",
             "type": "object",
@@ -125,6 +126,7 @@
                 }
             }
         },
+        {%- endif %}
         "max_job_request_options": {
             "title": "Max job request options",
             "type": "object",
@@ -289,9 +291,9 @@
         {% if igenomes %}{
             "$ref": "#/definitions/reference_genome_options"
         },{% endif %}
-        {
+        {% if nf_core_configs %}{
             "$ref": "#/definitions/institutional_config_options"
-        },
+        },{% endif %}
         {
             "$ref": "#/definitions/max_job_request_options"
         },

nf_core/pipelines/create/create.py

@@ -218,6 +218,7 @@ def obtain_jinja_params_dict(
         # Add is_nfcore as an area to skip for non-nf-core pipelines, to skip all nf-core files
         if not self.config.is_nfcore:
             skip_areas.append("is_nfcore")
+            jinja_params["is_nfcore"] = False
 
         # Set the last parameters based on the ones provided
         jinja_params["short_name"] = (
@@ -349,17 +350,9 @@ def render_template(self) -> None:
                 template_stat = os.stat(template_fn_path)
                 os.chmod(output_path, template_stat.st_mode)
 
-        # Remove all unused parameters in the nextflow schema
-        if not self.jinja_params["igenomes"] or not self.jinja_params["nf_core_configs"]:
-            self.update_nextflow_schema()
         if self.config.is_nfcore:
             # Make a logo and save it, if it is a nf-core pipeline
             self.make_pipeline_logo()
-        else:
-            if self.jinja_params["github"]:
-                # Remove field mentioning nf-core docs
-                # in the github bug report template
-                self.remove_nf_core_in_bug_report_template()
 
         # Update the .nf-core.yml with linting configurations
         self.fix_linting()
@@ -374,38 +367,8 @@ def render_template(self) -> None:
                     log.debug(f"Dumping pipeline template yml to pipeline config file '{config_fn.name}'")
                     run_prettier_on_file(self.outdir / config_fn)
 
-    def update_nextflow_schema(self):
-        """
-        Removes unused parameters from the nextflow schema.
-        """
-        schema_path = self.outdir / "nextflow_schema.json"
-
-        schema = nf_core.pipelines.schema.PipelineSchema()
-        schema.schema_filename = schema_path
-        schema.no_prompts = True
-        schema.load_schema()
-        schema.get_wf_params()
-        schema.remove_schema_notfound_configs()
-        schema.save_schema(suppress_logging=True)
-        run_prettier_on_file(schema_path)
-
-    def remove_nf_core_in_bug_report_template(self):
-        """
-        Remove the field mentioning nf-core documentation
-        in the github bug report template
-        """
-        bug_report_path = self.outdir / ".github" / "ISSUE_TEMPLATE" / "bug_report.yml"
-
-        with open(bug_report_path) as fh:
-            contents = yaml.load(fh, Loader=yaml.FullLoader)
-
-        # Remove the first item in the body, which is the information about the docs
-        contents["body"].pop(0)
-
-        with open(bug_report_path, "w") as fh:
-            yaml.dump(contents, fh, default_flow_style=False, sort_keys=False)
-
-        run_prettier_on_file(bug_report_path)
+        # Run prettier on files
+        run_prettier_on_file(self.outdir)
 
     def fix_linting(self):
         """
@@ -414,17 +377,18 @@ def fix_linting(self):
         """
         # Create a lint config
         lint_config = {}
-        for area in self.skip_areas:
+        for area in (self.config.skip_features or []) + self.skip_areas:
             try:
                 for lint_test in self.template_features_yml[area]["linting"]:
-                    if not lint_config[lint_test]:
-                        pass
-                    if self.template_features_yml[area]["linting"][lint_test]:
-                        lint_config.setdefault(lint_test, []).extend(
-                            self.template_features_yml[area]["linting"][lint_test]
-                        )
-                    else:
-                        lint_config[lint_test] = False
+                    try:
+                        if self.template_features_yml[area]["linting"][lint_test]:
+                            lint_config.setdefault(lint_test, []).extend(
+                                self.template_features_yml[area]["linting"][lint_test]
+                            )
+                        else:
+                            lint_config[lint_test] = False
+                    except AttributeError:
+                        pass  # When linting is False
             except KeyError:
                 pass  # Areas without linting
 

nf_core/pipelines/create/templatefeatures.yml

@@ -109,6 +109,8 @@ nf_core_configs:
       - "process.memory"
       - "process.time"
       - "custom_config"
+      - "params.custom_config_version"
+      - "params.custom_config_base"
   nfcore_pipelines: False
   custom_pipelines: True
 is_nfcore:
@@ -160,6 +162,11 @@ code_linters:
     - pre-commit (https://pre-commit.com/): used to run all code-linters on every PR and on ever commit if you run `pre-commit install` to install it in your local repository.
     - editor-config (https://github.com/editorconfig-checker/editorconfig-checker): checks rules such as indentation or trailing spaces.
     - prettier (https://github.com/prettier/prettier): enforces a consistent style (indentation, quoting, line length, etc).
+  linting:
+    files_exist:
+      - ".editorconfig"
+      - ".prettierignore"
+      - ".prettierrc.yml"
   nfcore_pipelines: False
   custom_pipelines: True
 citations: