From c616e8af41f275223fc53dd128ea5b6bdf8324b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 8 Feb 2021 16:02:26 +0100 Subject: [PATCH 01/41] prepare new client package without config repo --- prepare-client-packages.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/prepare-client-packages.yml b/prepare-client-packages.yml index bd6df32a..d9ca67bb 100644 --- a/prepare-client-packages.yml +++ b/prepare-client-packages.yml @@ -5,7 +5,7 @@ vars: package_vendor: EESSI package_maintainer: EESSI - package_description: CVMFS config repository package for EESSI. + package_description: CVMFS configuration package for EESSI. package_source_dir: "{{ lookup('env', 'GITHUB_WORKSPACE') | default('/tmp', True) }}/package" tasks: - name: Create directory structure for the CVMFS config @@ -18,9 +18,9 @@ - "{{ package_source_dir }}/etc" - "{{ package_source_dir }}/etc/cvmfs" - "{{ package_source_dir }}/etc/cvmfs/keys" - - "{{ package_source_dir }}/etc/cvmfs/keys/{{ eessi_cvmfs_config_repo.domain }}" - - "{{ package_source_dir }}/etc/cvmfs/config.d" + - "{{ package_source_dir }}/etc/cvmfs/keys/{{ eessi_cvmfs_server_urls.domain }}" - "{{ package_source_dir }}/etc/cvmfs/default.d" + - "{{ package_source_dir }}/etc/cvmfs/domain.d" - name: Make EESSI CVMFS public key file copy: @@ -28,18 +28,18 @@ dest: "{{ package_source_dir }}{{ eessi_cvmfs_config_repo.key.path }}" mode: 0644 - - name: Make EESSI CVMFS configuration file + - name: Make EESSI CVMFS domain configuration file copy: content: | CVMFS_SERVER_URL="{{ eessi_cvmfs_config_repo.urls|join(';') }}" - CVMFS_PUBLIC_KEY="{{ eessi_cvmfs_config_repo.key.path }}" - dest: "{{ package_source_dir }}/etc/cvmfs/config.d/{{ eessi_cvmfs_config_repo.repository.repository }}.conf" + CVMFS_KEYS_DIR="{{ eessi_cvmfs_repositories.key_dir }}" + dest: "{{ package_source_dir }}/etc/cvmfs/domain.d/{{ eessi_cvmfs_server_urls.domain }}.conf" mode: 0644 - - name: Make EESSI CVMFS default configuration file - copy: - content: | - CVMFS_CONFIG_REPOSITORY="{{ eessi_cvmfs_config_repo.repository.repository }}" - CVMFS_DEFAULT_DOMAIN="{{ eessi_cvmfs_config_repo.domain }}" - dest: "{{ package_source_dir }}/etc/cvmfs/default.d/80-eessi-cvmfs.conf" - mode: 0644 +# - name: Make EESSI CVMFS default configuration file +# copy: +# content: | +# CVMFS_CONFIG_REPOSITORY="{{ eessi_cvmfs_config_repo.repository.repository }}" +# CVMFS_DEFAULT_DOMAIN="{{ eessi_cvmfs_config_repo.domain }}" +# dest: "{{ package_source_dir }}/etc/cvmfs/default.d/80-eessi-cvmfs.conf" +# mode: 0644 From 40c5caefe3aebb86ffe640b0b2bd41901f052246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 8 Feb 2021 16:05:15 +0100 Subject: [PATCH 02/41] enable ci on this branch --- .github/workflows/build-test-release-client-packages.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-test-release-client-packages.yml b/.github/workflows/build-test-release-client-packages.yml index 946e8c16..4a538682 100644 --- a/.github/workflows/build-test-release-client-packages.yml +++ b/.github/workflows/build-test-release-client-packages.yml @@ -4,13 +4,12 @@ on: push: branches: - master - - test_client_packages_ci + - remove_config_repo tags: - 'v*' pull_request: branches: - master - - test_client_packages_ci jobs: From 6df5d7098269b3f6dc4cf2354d62611ebd185e6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 8 Feb 2021 16:10:07 +0100 Subject: [PATCH 03/41] enable geoapi --- prepare-client-packages.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/prepare-client-packages.yml b/prepare-client-packages.yml index d9ca67bb..b299643b 100644 --- a/prepare-client-packages.yml +++ b/prepare-client-packages.yml @@ -33,6 +33,7 @@ content: | CVMFS_SERVER_URL="{{ eessi_cvmfs_config_repo.urls|join(';') }}" CVMFS_KEYS_DIR="{{ eessi_cvmfs_repositories.key_dir }}" + CVMFS_USE_GEOAPI="yes" dest: "{{ package_source_dir }}/etc/cvmfs/domain.d/{{ eessi_cvmfs_server_urls.domain }}.conf" mode: 0644 From 353f1bfbe442b3cd7c67027b3c21109393ad2e84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 8 Feb 2021 16:12:35 +0100 Subject: [PATCH 04/41] use config repo domain for now --- prepare-client-packages.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prepare-client-packages.yml b/prepare-client-packages.yml index b299643b..aa4b90dd 100644 --- a/prepare-client-packages.yml +++ b/prepare-client-packages.yml @@ -18,7 +18,7 @@ - "{{ package_source_dir }}/etc" - "{{ package_source_dir }}/etc/cvmfs" - "{{ package_source_dir }}/etc/cvmfs/keys" - - "{{ package_source_dir }}/etc/cvmfs/keys/{{ eessi_cvmfs_server_urls.domain }}" + - "{{ package_source_dir }}/etc/cvmfs/keys/{{ eessi_cvmfs_config_repo.domain }}" - "{{ package_source_dir }}/etc/cvmfs/default.d" - "{{ package_source_dir }}/etc/cvmfs/domain.d" @@ -34,7 +34,7 @@ CVMFS_SERVER_URL="{{ eessi_cvmfs_config_repo.urls|join(';') }}" CVMFS_KEYS_DIR="{{ eessi_cvmfs_repositories.key_dir }}" CVMFS_USE_GEOAPI="yes" - dest: "{{ package_source_dir }}/etc/cvmfs/domain.d/{{ eessi_cvmfs_server_urls.domain }}.conf" + dest: "{{ package_source_dir }}/etc/cvmfs/domain.d/{{ eessi_cvmfs_config_repo.domain }}.conf" mode: 0644 # - name: Make EESSI CVMFS default configuration file From e745d593432cd2483ef22dd68bd479d113b7164b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 10 Feb 2021 16:58:57 +0100 Subject: [PATCH 05/41] use existing config in group vars --- prepare-client-packages.yml | 40 ++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/prepare-client-packages.yml b/prepare-client-packages.yml index aa4b90dd..485f379b 100644 --- a/prepare-client-packages.yml +++ b/prepare-client-packages.yml @@ -8,6 +8,15 @@ package_description: CVMFS configuration package for EESSI. package_source_dir: "{{ lookup('env', 'GITHUB_WORKSPACE') | default('/tmp', True) }}/package" tasks: + - name: Check the number of defined domains (should be one!) + fail: + msg: "Cannot build a client package when eessi_cvmfs_server_urls contains different domains" + when: eessi_cvmfs_server_urls | selectattr('domain') | list | unique | count > 1 + + - name: Determine domain name + set_fact: + eessi_domain: "{{ eessi_cvmfs_server_urls[0].domain }}" + - name: Create directory structure for the CVMFS config file: path: "{{ item }}" @@ -18,29 +27,28 @@ - "{{ package_source_dir }}/etc" - "{{ package_source_dir }}/etc/cvmfs" - "{{ package_source_dir }}/etc/cvmfs/keys" - - "{{ package_source_dir }}/etc/cvmfs/keys/{{ eessi_cvmfs_config_repo.domain }}" - - "{{ package_source_dir }}/etc/cvmfs/default.d" + - "{{ package_source_dir }}/etc/cvmfs/keys/{{ eessi_domain }}" - "{{ package_source_dir }}/etc/cvmfs/domain.d" - name: Make EESSI CVMFS public key file copy: - content: "{{ eessi_cvmfs_config_repo.key.key }}" - dest: "{{ package_source_dir }}{{ eessi_cvmfs_config_repo.key.path }}" + content: "{{ item.key }}" + dest: "{{ package_source_dir }}{{ item.path }}" mode: 0644 + with_items: "{{ eessi_cvmfs_keys }}" - name: Make EESSI CVMFS domain configuration file copy: content: | - CVMFS_SERVER_URL="{{ eessi_cvmfs_config_repo.urls|join(';') }}" - CVMFS_KEYS_DIR="{{ eessi_cvmfs_repositories.key_dir }}" - CVMFS_USE_GEOAPI="yes" - dest: "{{ package_source_dir }}/etc/cvmfs/domain.d/{{ eessi_cvmfs_config_repo.domain }}.conf" + CVMFS_SERVER_URL="{{ item.urls|join(';') }}" + CVMFS_KEYS_DIR="/etc/cvmfs/keys/{{ item.domain }}" + CVMFS_USE_GEOAPI="{{ item.use_geoapi | ternary('yes', 'no') }}" + + if [ "$CVMFS_HTTP_PROXY" = "" ]; then + # For individual clients (laptops, clusters < 5 nodes), the default is to + # use a site proxy where possible and otherwise a direct stratum 1 connection + CVMFS_HTTP_PROXY="auto;DIRECT" + fi + dest: "{{ package_source_dir }}/etc/cvmfs/domain.d/{{ item.domain }}.conf" mode: 0644 - -# - name: Make EESSI CVMFS default configuration file -# copy: -# content: | -# CVMFS_CONFIG_REPOSITORY="{{ eessi_cvmfs_config_repo.repository.repository }}" -# CVMFS_DEFAULT_DOMAIN="{{ eessi_cvmfs_config_repo.domain }}" -# dest: "{{ package_source_dir }}/etc/cvmfs/default.d/80-eessi-cvmfs.conf" -# mode: 0644 + with_items: "{{ eessi_cvmfs_server_urls }}" From 2628f25274b66615dcac94f03027471caeda148b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 10 Feb 2021 17:00:02 +0100 Subject: [PATCH 06/41] add public keys, remove new part --- inventory/group_vars/all.yml | 37 +++++++++++------------------------- 1 file changed, 11 insertions(+), 26 deletions(-) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index b8c74e72..3ad2cc9e 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -2,16 +2,20 @@ # vars file for eessi project # Automatically configure EESSI CVMFS repos. -eessi_cvmfs_repos_enabled: config-repo +eessi_cvmfs_repos_enabled: true # Email address for the project, which will be put in the contact file on the config repo. eessi_email: eessi@list.rug.nl -# Defaults for eessi-hpc.org config repo, syntax for each key is the same as that of cvmfs_ -eessi_cvmfs_config_repo: - domain: eessi-hpc.org - key: - path: /etc/cvmfs/keys/eessi-hpc.org/cvmfs-config.eessi-hpc.org.pub +# +# Defaults for eessi-hpc.org repos. +# + +# Public keys for the repositories, which you can find on the Stratum 0 at: +# /etc/cvmfs/keys/*.pub +# Note: you first have to run the stratum0.yml playbook once to generate the repositories and keys. +eessi_cvmfs_keys: + - path: /etc/cvmfs/keys/eessi-hpc.org/cvmfs-config.eessi-hpc.org.pub key: | -----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqoTBc/Eksr9ECEOoCzHA @@ -22,26 +26,7 @@ eessi_cvmfs_config_repo: +hCt494UsFs3tchmTebpK+x3pVRXybQ66Qm1mQiaz+2VbUJdQEov4RCC5PcOmOOk BwIDAQAB -----END PUBLIC KEY----- - urls: - - "http://cvmfs-s1-rug.eessi-hpc.org/cvmfs/@fqrn@" - - "http://cvmfs-s1-bgo.eessi-hpc.org/cvmfs/@fqrn@" - repository: - repository: cvmfs-config.eessi-hpc.org - stratum0: cvmfs-s0.eessi-hpc.org - owner: "{{ cvmfs_repo_owner | default('root') }}" - key_dir: /etc/cvmfs/keys/eessi-hpc.org - server_options: [] - client_options: [] - -# -# Defaults for eessi-hpc.org repos. -# - -# Public keys for the repositories, which you can find on the Stratum 0 at: -# /etc/cvmfs/keys/*.pub -# Note: you first have to run the stratum0.yml playbook once to generate the repositories and keys. -eessi_cvmfs_keys: - - path: /etc/cvmfs/keys/eessi-hpc.org/pilot.eessi-hpc.org.pub + - path: /etc/cvmfs/keys/eessi-hpc.org/cvmfs-config.eessi-hpc.org.pub key: | -----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAommtaoqQ36Rb/cFnqUWg From 381bec02b8d0f0f0df08dfd7d8543e97a3c43c84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 10 Feb 2021 17:42:03 +0100 Subject: [PATCH 07/41] remove whitespaces --- prepare-client-packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prepare-client-packages.yml b/prepare-client-packages.yml index 485f379b..1f584eef 100644 --- a/prepare-client-packages.yml +++ b/prepare-client-packages.yml @@ -43,7 +43,7 @@ CVMFS_SERVER_URL="{{ item.urls|join(';') }}" CVMFS_KEYS_DIR="/etc/cvmfs/keys/{{ item.domain }}" CVMFS_USE_GEOAPI="{{ item.use_geoapi | ternary('yes', 'no') }}" - + if [ "$CVMFS_HTTP_PROXY" = "" ]; then # For individual clients (laptops, clusters < 5 nodes), the default is to # use a site proxy where possible and otherwise a direct stratum 1 connection From 207c20194aa3c5d5756a2b767cb949d1f27f93f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 10 Feb 2021 17:44:47 +0100 Subject: [PATCH 08/41] change name of task --- prepare-client-packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prepare-client-packages.yml b/prepare-client-packages.yml index 1f584eef..cfe3e4f4 100644 --- a/prepare-client-packages.yml +++ b/prepare-client-packages.yml @@ -30,7 +30,7 @@ - "{{ package_source_dir }}/etc/cvmfs/keys/{{ eessi_domain }}" - "{{ package_source_dir }}/etc/cvmfs/domain.d" - - name: Make EESSI CVMFS public key file + - name: Make EESSI CVMFS public key files copy: content: "{{ item.key }}" dest: "{{ package_source_dir }}{{ item.path }}" From aa2b10b3da7be037bc15043afe830c386a3de406 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 10 Feb 2021 17:44:55 +0100 Subject: [PATCH 09/41] correct name of pilot key --- inventory/group_vars/all.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 3ad2cc9e..d20f52f0 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -26,7 +26,7 @@ eessi_cvmfs_keys: +hCt494UsFs3tchmTebpK+x3pVRXybQ66Qm1mQiaz+2VbUJdQEov4RCC5PcOmOOk BwIDAQAB -----END PUBLIC KEY----- - - path: /etc/cvmfs/keys/eessi-hpc.org/cvmfs-config.eessi-hpc.org.pub + - path: /etc/cvmfs/keys/eessi-hpc.org/pilot.eessi-hpc.org.pub key: | -----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAommtaoqQ36Rb/cFnqUWg From 8c3ffa6bd951bb75794a6272ef8c8742c8537640 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 10 Feb 2021 17:58:30 +0100 Subject: [PATCH 10/41] Let CI test access to the pilot repo instead of the cvmfs-config repo --- .../workflows/build-test-release-client-packages.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-test-release-client-packages.yml b/.github/workflows/build-test-release-client-packages.yml index 4a538682..17e5646f 100644 --- a/.github/workflows/build-test-release-client-packages.yml +++ b/.github/workflows/build-test-release-client-packages.yml @@ -162,7 +162,7 @@ jobs: run: sudo cvmfs_config setup - name: Test repository access - run: ls /cvmfs/cvmfs-config.eessi-hpc.org/etc/cvmfs + run: ls /cvmfs/pilot.eessi-hpc.org/etc/cvmfs test-rpm-package: needs: build-linux-packages @@ -198,10 +198,10 @@ jobs: - name: Mount the repositories #run: cvmfs_config setup - run: mkdir -p /cvmfs/cvmfs-config.eessi-hpc.org && mount -t cvmfs cvmfs-config.eessi-hpc.org /cvmfs/cvmfs-config.eessi-hpc.org + run: mkdir -p /cvmfs/pilot.eessi-hpc.org && mount -t cvmfs pilot.eessi-hpc.org /cvmfs/pilot.eessi-hpc.org - name: Test repository access - run: ls /cvmfs/cvmfs-config.eessi-hpc.org/etc/cvmfs + run: ls /cvmfs/pilot.eessi-hpc.org/etc/cvmfs test-macos-package: needs: build-macos-package @@ -232,10 +232,10 @@ jobs: run: echo "CVMFS_HTTP_PROXY=DIRECT" | sudo tee /etc/cvmfs/default.local - name: Mount the repositories - run: sudo mkdir -p /Users/Shared/cvmfs/cvmfs-config.eessi-hpc.org && sudo mount -t cvmfs cvmfs-config.eessi-hpc.org /Users/Shared/cvmfs/cvmfs-config.eessi-hpc.org + run: sudo mkdir -p /Users/Shared/cvmfs/pilot.eessi-hpc.org && sudo mount -t cvmfs pilot.eessi-hpc.org /Users/Shared/cvmfs/pilot.eessi-hpc.org - name: Test repository access - run: ls /Users/Shared/cvmfs/cvmfs-config.eessi-hpc.org/etc/cvmfs + run: ls /Users/Shared/cvmfs/pilot.eessi-hpc.org/etc/cvmfs release: needs: [build-linux-packages, build-macos-package, test-deb-package, test-rpm-package, test-macos-package] From 8f4ef56835f6ead166cbb02fd0fe3c6d43d19667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 10 Feb 2021 18:04:34 +0100 Subject: [PATCH 11/41] build tar package --- .../build-test-release-client-packages.yml | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.github/workflows/build-test-release-client-packages.yml b/.github/workflows/build-test-release-client-packages.yml index 17e5646f..78ccdac7 100644 --- a/.github/workflows/build-test-release-client-packages.yml +++ b/.github/workflows/build-test-release-client-packages.yml @@ -57,14 +57,23 @@ jobs: fpm_args: "etc" fpm_opts: "--debug -n cvmfs-config-eessi -v ${{ steps.get_version.outputs.version }} -t deb -a all -s dir -C ./package --description 'CVMFS config repository package for EESSI.'" + - name: Build tar package + id: build-tar + uses: bpicode/github-action-fpm@master + with: + fpm_args: "etc" + fpm_opts: "--debug -n cvmfs-config-eessi -v ${{ steps.get_version.outputs.version }} -t tar -a all -s dir -C ./package --description 'CVMFS config repository package for EESSI.'" + - name: Find filenames of generated packages id: find_filenames shell: bash run: | rpmfile="$(ls -1 cvmfs-config-eessi*.rpm)" debfile="$(ls -1 cvmfs-config-eessi*.deb)" + tarfile="$(ls -1 cvmfs-config-eessi*.tar)" echo ::set-output name=rpmfile::${rpmfile} echo ::set-output name=debfile::${debfile} + echo ::set-output name=tarfile::${tarfile} - name: Upload Deb package as artifact uses: actions/upload-artifact@v2 @@ -78,6 +87,12 @@ jobs: name: RPM package path: ${{ steps.find_filenames.outputs.rpmfile }} + - name: Upload tar package as artifact + uses: actions/upload-artifact@v2 + with: + name: Tar package + path: ${{ steps.find_filenames.outputs.tarfile }} + build-macos-package: runs-on: macos-latest steps: @@ -252,6 +267,11 @@ jobs: with: name: RPM package + - name: Download tar package + uses: actions/download-artifact@v2 + with: + name: Tar package + - name: Download macOS package uses: actions/download-artifact@v2 with: @@ -264,9 +284,11 @@ jobs: rpmfile="$(ls -1 cvmfs-config-eessi*.rpm)" debfile="$(ls -1 cvmfs-config-eessi*.deb)" pkgfile="$(ls -1 cvmfs-config-eessi*.pkg)" + tarfile="$(ls -1 cvmfs-config-eessi*.tar)" echo ::set-output name=rpmfile::${rpmfile} echo ::set-output name=debfile::${debfile} echo ::set-output name=pkgfile::${pkgfile} + echo ::set-output name=tarfile::${tarfile} - name: Create Release id: create_release @@ -308,3 +330,13 @@ jobs: asset_path: ${{ steps.find_filenames.outputs.pkgfile }} asset_name: ${{ steps.find_filenames.outputs.pkgfile }} asset_content_type: application/x-newton-compatible-pkg + + - name: Upload Tar as release asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ${{ steps.find_filenames.outputs.tarfile }} + asset_name: ${{ steps.find_filenames.outputs.tarfile }} + asset_content_type: application/x-tar From c5723673b932f6ecd548f200cea6fc035659d73e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 10 Feb 2021 18:16:44 +0100 Subject: [PATCH 12/41] fix the ls test, add test for tarball --- .../build-test-release-client-packages.yml | 43 +++++++++++++++++-- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-test-release-client-packages.yml b/.github/workflows/build-test-release-client-packages.yml index 78ccdac7..2b6c6bc3 100644 --- a/.github/workflows/build-test-release-client-packages.yml +++ b/.github/workflows/build-test-release-client-packages.yml @@ -177,7 +177,7 @@ jobs: run: sudo cvmfs_config setup - name: Test repository access - run: ls /cvmfs/pilot.eessi-hpc.org/etc/cvmfs + run: ls /cvmfs/pilot.eessi-hpc.org/ test-rpm-package: needs: build-linux-packages @@ -216,7 +216,44 @@ jobs: run: mkdir -p /cvmfs/pilot.eessi-hpc.org && mount -t cvmfs pilot.eessi-hpc.org /cvmfs/pilot.eessi-hpc.org - name: Test repository access - run: ls /cvmfs/pilot.eessi-hpc.org/etc/cvmfs + run: ls /cvmfs/pilot.eessi-hpc.org/ + + test-tar-package: + needs: build-linux-packages + runs-on: ubuntu-latest + steps: + - name: Install CVMFS dependencies + run: sudo apt-get install lsb-release + + - name: Download and install CVMFS client + run: wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest_all.deb && sudo dpkg -i cvmfs-release-latest_all.deb + + - name: Install CVMFS client + run: sudo apt-get update && sudo apt-get install cvmfs + + - name: Download cvmfs-config-eessi package + uses: actions/download-artifact@v2 + with: + name: Tar package + + - name: Find filename of package + id: find_filename + shell: bash + run: | + tarfile="$(ls -1 cvmfs-config-eessi*.tar)" + echo ::set-output name=tarfile::${tarfile} + + - name: Install package + run: sudo tar -C / -xf ./${{ steps.find_filename.outputs.tarfile }} + + - name: Add local CVMFS configuration (/etc/cvmfs/default.local) + run: echo "CVMFS_HTTP_PROXY=DIRECT" | sudo tee /etc/cvmfs/default.local + + - name: Mount the repositories + run: sudo cvmfs_config setup + + - name: Test repository access + run: ls /cvmfs/pilot.eessi-hpc.org/ test-macos-package: needs: build-macos-package @@ -253,7 +290,7 @@ jobs: run: ls /Users/Shared/cvmfs/pilot.eessi-hpc.org/etc/cvmfs release: - needs: [build-linux-packages, build-macos-package, test-deb-package, test-rpm-package, test-macos-package] + needs: [build-linux-packages, build-macos-package, test-deb-package, test-rpm-package, test-macos-package, test-tar-package] if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest steps: From e6c345cba6fe4b44b1af5fba406d4bf54c33a935 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 10 Feb 2021 18:29:39 +0100 Subject: [PATCH 13/41] fix macos test --- .github/workflows/build-test-release-client-packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test-release-client-packages.yml b/.github/workflows/build-test-release-client-packages.yml index 2b6c6bc3..8397e704 100644 --- a/.github/workflows/build-test-release-client-packages.yml +++ b/.github/workflows/build-test-release-client-packages.yml @@ -287,7 +287,7 @@ jobs: run: sudo mkdir -p /Users/Shared/cvmfs/pilot.eessi-hpc.org && sudo mount -t cvmfs pilot.eessi-hpc.org /Users/Shared/cvmfs/pilot.eessi-hpc.org - name: Test repository access - run: ls /Users/Shared/cvmfs/pilot.eessi-hpc.org/etc/cvmfs + run: ls /Users/Shared/cvmfs/pilot.eessi-hpc.org/ release: needs: [build-linux-packages, build-macos-package, test-deb-package, test-rpm-package, test-macos-package, test-tar-package] From efe57f0d0b5ad30620d23b4b2bd901c2c4373620 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 10 Feb 2021 18:53:04 +0100 Subject: [PATCH 14/41] remove current test branch for building packages --- .github/workflows/build-test-release-client-packages.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-test-release-client-packages.yml b/.github/workflows/build-test-release-client-packages.yml index 8397e704..f17ed181 100644 --- a/.github/workflows/build-test-release-client-packages.yml +++ b/.github/workflows/build-test-release-client-packages.yml @@ -4,7 +4,6 @@ on: push: branches: - master - - remove_config_repo tags: - 'v*' pull_request: From 5d82d366bf9357318429301d571caa33d7c4d53a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 11 Feb 2021 09:44:53 +0100 Subject: [PATCH 15/41] comment the eessi config repo --- inventory/group_vars/all.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index d20f52f0..e7a3482d 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -61,7 +61,7 @@ eessi_cvmfs_repositories: # Override all the Galaxy defaults by our EESSI defaults. # This is required, beucase the galaxy_* variables are used inside the Ansible tasks. -galaxy_cvmfs_config_repo: "{{ eessi_cvmfs_config_repo }}" +#galaxy_cvmfs_config_repo: "{{ eessi_cvmfs_config_repo }}" galaxy_cvmfs_keys: "{{ eessi_cvmfs_keys }}" galaxy_cvmfs_server_urls: "{{ eessi_cvmfs_server_urls }}" galaxy_cvmfs_repositories: "{{ eessi_cvmfs_repositories }}" From f767fc8e0a2f67f93714d9147117e2d99c35ab3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 11 Feb 2021 09:52:19 +0100 Subject: [PATCH 16/41] comment config repo for stratum 0/1 --- inventory/group_vars/cvmfsstratum0servers | 2 +- inventory/group_vars/cvmfsstratum1servers | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/inventory/group_vars/cvmfsstratum0servers b/inventory/group_vars/cvmfsstratum0servers index ec952dfc..df6c11c8 100644 --- a/inventory/group_vars/cvmfsstratum0servers +++ b/inventory/group_vars/cvmfsstratum0servers @@ -1,3 +1,3 @@ --- -cvmfs_repositories: "{{ eessi_cvmfs_repositories + [eessi_cvmfs_config_repo.repository] }}" +#cvmfs_repositories: "{{ eessi_cvmfs_repositories + [eessi_cvmfs_config_repo.repository] }}" diff --git a/inventory/group_vars/cvmfsstratum1servers b/inventory/group_vars/cvmfsstratum1servers index 1478980a..9e777265 100644 --- a/inventory/group_vars/cvmfsstratum1servers +++ b/inventory/group_vars/cvmfsstratum1servers @@ -1,5 +1,5 @@ --- -cvmfs_repositories: "{{ eessi_cvmfs_repositories + [eessi_cvmfs_config_repo.repository] }}" -cvmfs_keys: "{{ eessi_cvmfs_keys + [eessi_cvmfs_config_repo.key] }}" +#cvmfs_repositories: "{{ eessi_cvmfs_repositories + [eessi_cvmfs_config_repo.repository] }}" +#cvmfs_keys: "{{ eessi_cvmfs_keys + [eessi_cvmfs_config_repo.key] }}" cvmfs_squid_conf_src: "{{ local_stratum1_cvmfs_squid_conf_src | default('eessi_stratum1_squid.conf.j2') }}" From 2e446afb1a4e6cd84372fc856ea893d4e2f846e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 11 Feb 2021 09:57:29 +0100 Subject: [PATCH 17/41] still include the old cvmfs-config repo for the CI --- inventory/group_vars/all.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index e7a3482d..8bcfa697 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -4,6 +4,11 @@ # Automatically configure EESSI CVMFS repos. eessi_cvmfs_repos_enabled: true +# Keep this for legacy reasons (and for the Stratum 1 CI) +eessi_cvmfs_config_repo: + repository: + repository: cvmfs-config.eessi-hpc.org + # Email address for the project, which will be put in the contact file on the config repo. eessi_email: eessi@list.rug.nl From 7fde248d6b93e9f56bc60b895b7fdc4f40b7f1f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 11 Feb 2021 10:55:03 +0100 Subject: [PATCH 18/41] still include the old cvmfs-config repo for the CI --- inventory/group_vars/all.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 8bcfa697..c3544fea 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -5,9 +5,31 @@ eessi_cvmfs_repos_enabled: true # Keep this for legacy reasons (and for the Stratum 1 CI) +# Rename this to ci.eessi-hpc.org later on? eessi_cvmfs_config_repo: + domain: eessi-hpc.org + key: + path: /etc/cvmfs/keys/eessi-hpc.org/cvmfs-config.eessi-hpc.org.pub + key: | + -----BEGIN PUBLIC KEY----- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqoTBc/Eksr9ECEOoCzHA + F20UjePR7mJyWB13eTBXnjq7PLKKilfk2llLKetPIRMEmbvp97NiQ9ALe/tiGDzT + 1Wg82Mf9oO4DNQYrTfE/MP5inIrMtSv1WnYJYV4Bgm24s8IhHXytIf1D4qt0x1v5 + 0yRui0dwaZrErWKRKufbJ6ehmEDHuVSEAujrORyig/jYFKVyTaCsXnmwFDJmX2Uw + RGCaf0M9ZLYs9t/SWA0s08rle5kH82hgBsIvUr7PLT7hbU407VK6+Y02tuxbI0JR + +hCt494UsFs3tchmTebpK+x3pVRXybQ66Qm1mQiaz+2VbUJdQEov4RCC5PcOmOOk + BwIDAQAB + -----END PUBLIC KEY----- + urls: + - "http://cvmfs-s1-rug.eessi-hpc.org/cvmfs/@fqrn@" + - "http://cvmfs-s1-bgo.eessi-hpc.org/cvmfs/@fqrn@" repository: repository: cvmfs-config.eessi-hpc.org + stratum0: cvmfs-s0.eessi-hpc.org + owner: "{{ cvmfs_repo_owner | default('root') }}" + key_dir: /etc/cvmfs/keys/eessi-hpc.org + server_options: [] + client_options: [] # Email address for the project, which will be put in the contact file on the config repo. eessi_email: eessi@list.rug.nl From 1e6156e9ed844179aec06094f6b6e135418b8b26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 11 Feb 2021 11:14:12 +0100 Subject: [PATCH 19/41] dont run the cvmfs-config playbook in CI --- .github/workflows/test-playbooks.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-playbooks.yml b/.github/workflows/test-playbooks.yml index ba01c59f..d1c62094 100644 --- a/.github/workflows/test-playbooks.yml +++ b/.github/workflows/test-playbooks.yml @@ -34,8 +34,8 @@ jobs: run: docker run -d --workdir /github/workspace --rm -e INPUT_PLAYBOOK -e HOME -e GITHUB_JOB -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_REPOSITORY_OWNER -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RETENTION_DAYS -e GITHUB_ACTOR -e GITHUB_WORKFLOW -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GITHUB_EVENT_NAME -e GITHUB_SERVER_URL -e GITHUB_API_URL -e GITHUB_GRAPHQL_URL -e GITHUB_WORKSPACE -e GITHUB_ACTION -e GITHUB_EVENT_PATH -e GITHUB_PATH -e GITHUB_ENV -e RUNNER_OS -e RUNNER_TOOL_CACHE -e RUNNER_TEMP -e RUNNER_WORKSPACE -e ACTIONS_RUNTIME_URL -e ACTIONS_RUNTIME_TOKEN -e ACTIONS_CACHE_URL -e GITHUB_ACTIONS=true -e CI=true -v "/var/run/docker.sock":"/var/run/docker.sock" -v $HOME:"/github/home" -v "$HOME/work/_temp/_github_workflow":"/github/workflow" -v "/home/runner/work/_temp/_runner_file_commands":"/github/file_commands" -v "$GITHUB_WORKSPACE":"/github/workspace" -v /sys/fs/cgroup:/sys/fs/cgroup --privileged --device /dev/fuse --mount type=bind,source=${GITHUB_WORKSPACE}/srv,target=/srv --mount type=bind,source=${GITHUB_WORKSPACE}/srv,target=/var/spool/cvmfs --name ${{ matrix.component }}-${{ matrix.os }} docker.pkg.github.com/$(echo $GITHUB_REPOSITORY | tr '[A-Z]' '[a-z]')/${{ matrix.os }} - name: Execute the playbook run: docker exec ${{ matrix.component }}-${{ matrix.os }} /test-playbook.sh ${{ matrix.component }}.yml - - name: Execute additional playbook for Stratum 0 - run: docker exec ${{ matrix.component }}-${{ matrix.os }} /test-playbook.sh ${{ matrix.component }}-deploy-cvmfs-config.yml - if: ${{ matrix.component == 'stratum0' }} +# - name: Execute additional playbook for Stratum 0 +# run: docker exec ${{ matrix.component }}-${{ matrix.os }} /test-playbook.sh ${{ matrix.component }}-deploy-cvmfs-config.yml +# if: ${{ matrix.component == 'stratum0' }} - name: Stop the container run: docker kill ${{ matrix.component }}-${{ matrix.os }} From 35f77397dad9f81177c3fd56a6c4dd48094ff9d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 11 Feb 2021 15:55:11 +0100 Subject: [PATCH 20/41] replace cvmfs-config by ci repo --- .github/workflows/test-playbook.sh | 2 +- inventory/group_vars/all.yml | 37 ++++++++---------------------- 2 files changed, 11 insertions(+), 28 deletions(-) diff --git a/.github/workflows/test-playbook.sh b/.github/workflows/test-playbook.sh index 3e599330..e5fe2f62 100755 --- a/.github/workflows/test-playbook.sh +++ b/.github/workflows/test-playbook.sh @@ -18,7 +18,7 @@ export CVMFS_GEO_DB_FILE=NONE # Only test the cvmfs-config repo on the Stratum 1, as the other ones may be very large. if [ $playbook == "stratum1.yml" ] then - echo 'cvmfs_repositories: "[{{ eessi_cvmfs_config_repo.repository }}]"' >> inventory/local_site_specific_vars.yml + echo 'cvmfs_repositories: "{{ eessi_cvmfs_repositories | selectattr('use_for_ci', 'defined') | selectattr('use_for_ci', 'equalto', true) | map(attribute='repository') | list }}"' >> inventory/local_site_specific_vars.yml fi # Install the Ansible dependencies. diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index c3544fea..34c81981 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -4,33 +4,6 @@ # Automatically configure EESSI CVMFS repos. eessi_cvmfs_repos_enabled: true -# Keep this for legacy reasons (and for the Stratum 1 CI) -# Rename this to ci.eessi-hpc.org later on? -eessi_cvmfs_config_repo: - domain: eessi-hpc.org - key: - path: /etc/cvmfs/keys/eessi-hpc.org/cvmfs-config.eessi-hpc.org.pub - key: | - -----BEGIN PUBLIC KEY----- - MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqoTBc/Eksr9ECEOoCzHA - F20UjePR7mJyWB13eTBXnjq7PLKKilfk2llLKetPIRMEmbvp97NiQ9ALe/tiGDzT - 1Wg82Mf9oO4DNQYrTfE/MP5inIrMtSv1WnYJYV4Bgm24s8IhHXytIf1D4qt0x1v5 - 0yRui0dwaZrErWKRKufbJ6ehmEDHuVSEAujrORyig/jYFKVyTaCsXnmwFDJmX2Uw - RGCaf0M9ZLYs9t/SWA0s08rle5kH82hgBsIvUr7PLT7hbU407VK6+Y02tuxbI0JR - +hCt494UsFs3tchmTebpK+x3pVRXybQ66Qm1mQiaz+2VbUJdQEov4RCC5PcOmOOk - BwIDAQAB - -----END PUBLIC KEY----- - urls: - - "http://cvmfs-s1-rug.eessi-hpc.org/cvmfs/@fqrn@" - - "http://cvmfs-s1-bgo.eessi-hpc.org/cvmfs/@fqrn@" - repository: - repository: cvmfs-config.eessi-hpc.org - stratum0: cvmfs-s0.eessi-hpc.org - owner: "{{ cvmfs_repo_owner | default('root') }}" - key_dir: /etc/cvmfs/keys/eessi-hpc.org - server_options: [] - client_options: [] - # Email address for the project, which will be put in the contact file on the config repo. eessi_email: eessi@list.rug.nl @@ -82,9 +55,19 @@ eessi_cvmfs_repositories: server_options: - CVMFS_AUTO_GC=false - CVMFS_GARBAGE_COLLECTION=true + client_options: + - CVMFS_NFILES=4096 + use_for_ci: no + - repository: ci.eessi-hpc.org + stratum0: cvmfs-s0.eessi-hpc.org + owner: "{{ cvmfs_repo_owner | default('root') }}" + key_dir: /etc/cvmfs/keys/eessi-hpc.org + server_options: - CVMFS_AUTO_GC=false + - CVMFS_GARBAGE_COLLECTION=true client_options: - CVMFS_NFILES=4096 + use_for_ci: yes # Override all the Galaxy defaults by our EESSI defaults. # This is required, beucase the galaxy_* variables are used inside the Ansible tasks. From 458b7ef091c09b0c84937777ffb62ef1b32b4a1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 11 Feb 2021 17:02:55 +0100 Subject: [PATCH 21/41] escape quotes --- .github/workflows/test-playbook.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-playbook.sh b/.github/workflows/test-playbook.sh index e5fe2f62..d1789e5f 100755 --- a/.github/workflows/test-playbook.sh +++ b/.github/workflows/test-playbook.sh @@ -18,7 +18,7 @@ export CVMFS_GEO_DB_FILE=NONE # Only test the cvmfs-config repo on the Stratum 1, as the other ones may be very large. if [ $playbook == "stratum1.yml" ] then - echo 'cvmfs_repositories: "{{ eessi_cvmfs_repositories | selectattr('use_for_ci', 'defined') | selectattr('use_for_ci', 'equalto', true) | map(attribute='repository') | list }}"' >> inventory/local_site_specific_vars.yml + echo 'cvmfs_repositories: "{{ eessi_cvmfs_repositories | selectattr(\"use_for_ci\", \"defined\") | selectattr(\"use_for_ci\", \"equalto\", true) | map(attribute=\"repository\") | list }}"' >> inventory/local_site_specific_vars.yml fi # Install the Ansible dependencies. From 809d9d50137650c7290dc2da30eed8e664127d8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 11 Feb 2021 17:15:57 +0100 Subject: [PATCH 22/41] use heredoc instead of echo --- .github/workflows/test-playbook.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-playbook.sh b/.github/workflows/test-playbook.sh index d1789e5f..b4c45b38 100755 --- a/.github/workflows/test-playbook.sh +++ b/.github/workflows/test-playbook.sh @@ -18,7 +18,9 @@ export CVMFS_GEO_DB_FILE=NONE # Only test the cvmfs-config repo on the Stratum 1, as the other ones may be very large. if [ $playbook == "stratum1.yml" ] then - echo 'cvmfs_repositories: "{{ eessi_cvmfs_repositories | selectattr(\"use_for_ci\", \"defined\") | selectattr(\"use_for_ci\", \"equalto\", true) | map(attribute=\"repository\") | list }}"' >> inventory/local_site_specific_vars.yml + cat >> inventory/local_site_specific_vars.yml <<-EOF + cvmfs_repositories: "{{ eessi_cvmfs_repositories | selectattr('use_for_ci', 'defined') | selectattr('use_for_ci', 'equalto', true) | map(attribute='repository') | list }}" + EOF fi # Install the Ansible dependencies. From f4934f35cc3a0f5a2b9d271b321efbbb0f96e583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 11 Feb 2021 17:30:02 +0100 Subject: [PATCH 23/41] use eq instead of equalto --- .github/workflows/test-playbook.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-playbook.sh b/.github/workflows/test-playbook.sh index b4c45b38..2bd5ee0c 100755 --- a/.github/workflows/test-playbook.sh +++ b/.github/workflows/test-playbook.sh @@ -19,7 +19,7 @@ export CVMFS_GEO_DB_FILE=NONE if [ $playbook == "stratum1.yml" ] then cat >> inventory/local_site_specific_vars.yml <<-EOF - cvmfs_repositories: "{{ eessi_cvmfs_repositories | selectattr('use_for_ci', 'defined') | selectattr('use_for_ci', 'equalto', true) | map(attribute='repository') | list }}" + cvmfs_repositories: "{{ eessi_cvmfs_repositories | selectattr('use_for_ci', 'defined') | selectattr('use_for_ci', 'eq', true) | map(attribute='repository') | list }}" EOF fi From f89ebb4c6ca0baf77ad14c4132d9835f76c6d239 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 11 Feb 2021 17:42:41 +0100 Subject: [PATCH 24/41] use sameas instead of eq --- .github/workflows/test-playbook.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-playbook.sh b/.github/workflows/test-playbook.sh index 2bd5ee0c..9501a2be 100755 --- a/.github/workflows/test-playbook.sh +++ b/.github/workflows/test-playbook.sh @@ -19,7 +19,7 @@ export CVMFS_GEO_DB_FILE=NONE if [ $playbook == "stratum1.yml" ] then cat >> inventory/local_site_specific_vars.yml <<-EOF - cvmfs_repositories: "{{ eessi_cvmfs_repositories | selectattr('use_for_ci', 'defined') | selectattr('use_for_ci', 'eq', true) | map(attribute='repository') | list }}" + cvmfs_repositories: "{{ eessi_cvmfs_repositories | selectattr('use_for_ci', 'defined') | selectattr('use_for_ci', 'sameas', true) | map(attribute='repository') | list }}" EOF fi From 02f93ac993919143252954c6371bab91b64b9e2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 11 Feb 2021 17:48:03 +0100 Subject: [PATCH 25/41] dont select only the repo name --- .github/workflows/test-playbook.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-playbook.sh b/.github/workflows/test-playbook.sh index 9501a2be..ab8d0bcb 100755 --- a/.github/workflows/test-playbook.sh +++ b/.github/workflows/test-playbook.sh @@ -19,7 +19,7 @@ export CVMFS_GEO_DB_FILE=NONE if [ $playbook == "stratum1.yml" ] then cat >> inventory/local_site_specific_vars.yml <<-EOF - cvmfs_repositories: "{{ eessi_cvmfs_repositories | selectattr('use_for_ci', 'defined') | selectattr('use_for_ci', 'sameas', true) | map(attribute='repository') | list }}" + cvmfs_repositories: "{{ eessi_cvmfs_repositories | selectattr('use_for_ci', 'defined') | selectattr('use_for_ci', 'sameas', true) | list }}" EOF fi From 787d4385b57e7e713bda503c6f6a8dc9b46756aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 11 Feb 2021 18:09:37 +0100 Subject: [PATCH 26/41] add pubkey for ci repo --- inventory/group_vars/all.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 34c81981..02262b94 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -37,6 +37,17 @@ eessi_cvmfs_keys: HJN0HHOVdRIwNjjPkV2jIqhJoI85no89v8/V45SFPVo+A7N0Cj4QdQrQTezpnlby FQIDAQAB -----END PUBLIC KEY----- + - path: /etc/cvmfs/keys/eessi-hpc.org/ci.eessi-hpc.org.pub + key: | + -----BEGIN PUBLIC KEY----- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAommtaoqQ36Rb/cFnqUWg + 8FoYCOjJWVBMQ4sVZ87oFRWehOUbcfy/ah85D9055HQws7i/hLNGJJAa7Es3E2o8 + v8LtSlRdfR1A7b7LA6p3213HHz6dNU6I1riwtE2OP7/w4Q3+Agj1iRtW535GwzGo + 4ZbewL8T1kbyaEm4HyefoitHXJthoPyTlYAhvAegIil1VJPhJZW+q/NWxnsd9QGJ + uAtbSICQg6mBJI50KiCBYmKHOAjH2W8qEnLWc7JoPoNvAHuMx1ya6YerBM5T9Esm + HJN0HHOVdRIwNjjPkV2jIqhJoI85no89v8/V45SFPVo+A7N0Cj4QdQrQTezpnlby + FQIDAQAB + -----END PUBLIC KEY----- # URLs for all the Stratum 1 servers. eessi_cvmfs_server_urls: From 29defa4e924cc5b0a18f85cce4ab4d4cd8f0297c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 12 Feb 2021 09:34:41 +0100 Subject: [PATCH 27/41] add note about PR https://github.com/galaxyproject/ansible-cvmfs/pull/32 --- inventory/group_vars/all.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 02262b94..a1299ec9 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -37,6 +37,8 @@ eessi_cvmfs_keys: HJN0HHOVdRIwNjjPkV2jIqhJoI85no89v8/V45SFPVo+A7N0Cj4QdQrQTezpnlby FQIDAQAB -----END PUBLIC KEY----- + # The following (duplicate) key can be removed after this PR has been merged: + # https://github.com/galaxyproject/ansible-cvmfs/pull/32 - path: /etc/cvmfs/keys/eessi-hpc.org/ci.eessi-hpc.org.pub key: | -----BEGIN PUBLIC KEY----- From 3f0d2082698fc2f8be339815e6e1eb79a67aa2b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 12 Feb 2021 14:08:30 +0100 Subject: [PATCH 28/41] don't use config repo for clients --- inventory/group_vars/cvmfsclients | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/group_vars/cvmfsclients b/inventory/group_vars/cvmfsclients index ff211fcb..2c06a768 100644 --- a/inventory/group_vars/cvmfsclients +++ b/inventory/group_vars/cvmfsclients @@ -5,4 +5,4 @@ cvmfs_http_proxies: "{{ local_cvmfs_http_proxies | default(['DIRECT']) }}" # Use the CVMFS configuration repository for the clients. -eessi_cvmfs_repos_enabled: config-repo +eessi_cvmfs_repos_enabled: true From 5d606af7f55ee6a21578b0b7a8893d1154f1292e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 12 Feb 2021 14:16:07 +0100 Subject: [PATCH 29/41] configure clients statically (no config repo) --- inventory/group_vars/all.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index a1299ec9..62a25a8d 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -11,6 +11,9 @@ eessi_email: eessi@list.rug.nl # Defaults for eessi-hpc.org repos. # +# We do not use a config repo anymore. +eessi_cvmfs_config_repo: {} + # Public keys for the repositories, which you can find on the Stratum 0 at: # /etc/cvmfs/keys/*.pub # Note: you first have to run the stratum0.yml playbook once to generate the repositories and keys. @@ -68,8 +71,8 @@ eessi_cvmfs_repositories: server_options: - CVMFS_AUTO_GC=false - CVMFS_GARBAGE_COLLECTION=true - client_options: - - CVMFS_NFILES=4096 + client_options: [] +# - CVMFS_NFILES=4096 use_for_ci: no - repository: ci.eessi-hpc.org stratum0: cvmfs-s0.eessi-hpc.org @@ -78,13 +81,13 @@ eessi_cvmfs_repositories: server_options: - CVMFS_AUTO_GC=false - CVMFS_GARBAGE_COLLECTION=true - client_options: - - CVMFS_NFILES=4096 + client_options: [] +# - CVMFS_NFILES=4096 use_for_ci: yes # Override all the Galaxy defaults by our EESSI defaults. # This is required, beucase the galaxy_* variables are used inside the Ansible tasks. -#galaxy_cvmfs_config_repo: "{{ eessi_cvmfs_config_repo }}" +galaxy_cvmfs_config_repo: "{{ eessi_cvmfs_config_repo }}" galaxy_cvmfs_keys: "{{ eessi_cvmfs_keys }}" galaxy_cvmfs_server_urls: "{{ eessi_cvmfs_server_urls }}" galaxy_cvmfs_repositories: "{{ eessi_cvmfs_repositories }}" From c008c1bfdafa2ae0efeb347980da74a92a2ffd45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 12 Feb 2021 14:33:00 +0100 Subject: [PATCH 30/41] remove config repo lines --- inventory/group_vars/cvmfsclients | 3 --- inventory/group_vars/cvmfsstratum0servers | 1 - inventory/group_vars/cvmfsstratum1servers | 2 -- 3 files changed, 6 deletions(-) diff --git a/inventory/group_vars/cvmfsclients b/inventory/group_vars/cvmfsclients index 2c06a768..795c9a47 100644 --- a/inventory/group_vars/cvmfsclients +++ b/inventory/group_vars/cvmfsclients @@ -3,6 +3,3 @@ # List of proxies to be used for the clients. # Override this setting in your local_site_specific_vars.yml file. cvmfs_http_proxies: "{{ local_cvmfs_http_proxies | default(['DIRECT']) }}" - -# Use the CVMFS configuration repository for the clients. -eessi_cvmfs_repos_enabled: true diff --git a/inventory/group_vars/cvmfsstratum0servers b/inventory/group_vars/cvmfsstratum0servers index df6c11c8..cd21505a 100644 --- a/inventory/group_vars/cvmfsstratum0servers +++ b/inventory/group_vars/cvmfsstratum0servers @@ -1,3 +1,2 @@ --- -#cvmfs_repositories: "{{ eessi_cvmfs_repositories + [eessi_cvmfs_config_repo.repository] }}" diff --git a/inventory/group_vars/cvmfsstratum1servers b/inventory/group_vars/cvmfsstratum1servers index 9e777265..51f05182 100644 --- a/inventory/group_vars/cvmfsstratum1servers +++ b/inventory/group_vars/cvmfsstratum1servers @@ -1,5 +1,3 @@ --- -#cvmfs_repositories: "{{ eessi_cvmfs_repositories + [eessi_cvmfs_config_repo.repository] }}" -#cvmfs_keys: "{{ eessi_cvmfs_keys + [eessi_cvmfs_config_repo.key] }}" cvmfs_squid_conf_src: "{{ local_stratum1_cvmfs_squid_conf_src | default('eessi_stratum1_squid.conf.j2') }}" From f0a03833df25bfd354078a8682271726463077a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 12 Feb 2021 15:14:55 +0100 Subject: [PATCH 31/41] improve domain config, add comments --- prepare-client-packages.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/prepare-client-packages.yml b/prepare-client-packages.yml index cfe3e4f4..27294b62 100644 --- a/prepare-client-packages.yml +++ b/prepare-client-packages.yml @@ -40,15 +40,24 @@ - name: Make EESSI CVMFS domain configuration file copy: content: | + # Stratum 1 servers for the eessi-hpc.org domain CVMFS_SERVER_URL="{{ item.urls|join(';') }}" + + # Public keys for the eessi-hpc.org domain CVMFS_KEYS_DIR="/etc/cvmfs/keys/{{ item.domain }}" + + # Ènable or disable the Geo API, based on the configuration file CVMFS_USE_GEOAPI="{{ item.use_geoapi | ternary('yes', 'no') }}" - if [ "$CVMFS_HTTP_PROXY" = "" ]; then - # For individual clients (laptops, clusters < 5 nodes), the default is to - # use a site proxy where possible and otherwise a direct stratum 1 connection + # The client profile is supposed to be set in /etc/cvmfs/default.local + if [ "$CVMFS_CLIENT_PROFILE" = "single" ]; then + if [ "$CVMFS_HTTP_PROXY" = "" ]; then + # For individual clients (laptops, clusters < 5 nodes), use a site proxy + # where possible and otherwise a direct stratum 1 connection CVMFS_HTTP_PROXY="auto;DIRECT" + fi fi + dest: "{{ package_source_dir }}/etc/cvmfs/domain.d/{{ item.domain }}.conf" mode: 0644 with_items: "{{ eessi_cvmfs_server_urls }}" From 3c9ebf91b191e142c3c5049a9d7bfaad58461379 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 12 Feb 2021 15:47:32 +0100 Subject: [PATCH 32/41] use CVMFS_CLIENT_PROFILE=single instead of setting a proxy --- .github/workflows/build-test-release-client-packages.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-test-release-client-packages.yml b/.github/workflows/build-test-release-client-packages.yml index f17ed181..4eac4b7f 100644 --- a/.github/workflows/build-test-release-client-packages.yml +++ b/.github/workflows/build-test-release-client-packages.yml @@ -170,7 +170,7 @@ jobs: run: sudo dpkg -i ./${{ steps.find_filename.outputs.debfile }} - name: Add local CVMFS configuration (/etc/cvmfs/default.local) - run: echo "CVMFS_HTTP_PROXY=DIRECT" | sudo tee /etc/cvmfs/default.local + run: echo "CVMFS_CLIENT_PROFILE=single" | sudo tee /etc/cvmfs/default.local - name: Mount the repositories run: sudo cvmfs_config setup @@ -208,7 +208,7 @@ jobs: run: rpm -i ./${{ steps.find_filename.outputs.rpmfile }} - name: Add local CVMFS configuration (/etc/cvmfs/default.local) - run: echo "CVMFS_HTTP_PROXY=DIRECT" | tee /etc/cvmfs/default.local + run: echo "CVMFS_CLIENT_PROFILE=single" | tee /etc/cvmfs/default.local - name: Mount the repositories #run: cvmfs_config setup @@ -246,7 +246,7 @@ jobs: run: sudo tar -C / -xf ./${{ steps.find_filename.outputs.tarfile }} - name: Add local CVMFS configuration (/etc/cvmfs/default.local) - run: echo "CVMFS_HTTP_PROXY=DIRECT" | sudo tee /etc/cvmfs/default.local + run: echo "CVMFS_CLIENT_PROFILE=single" | sudo tee /etc/cvmfs/default.local - name: Mount the repositories run: sudo cvmfs_config setup @@ -280,7 +280,7 @@ jobs: run: sudo installer -target / -pkg ./${{ steps.find_filename.outputs.pkgfile }} - name: Add local CVMFS configuration (/etc/cvmfs/default.local) - run: echo "CVMFS_HTTP_PROXY=DIRECT" | sudo tee /etc/cvmfs/default.local + run: echo "CVMFS_CLIENT_PROFILE=single" | sudo tee /etc/cvmfs/default.local - name: Mount the repositories run: sudo mkdir -p /Users/Shared/cvmfs/pilot.eessi-hpc.org && sudo mount -t cvmfs pilot.eessi-hpc.org /Users/Shared/cvmfs/pilot.eessi-hpc.org From 646f2c4a5f6c690f52a248a9cb105c3dea68c2a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 12 Feb 2021 15:49:49 +0100 Subject: [PATCH 33/41] update README --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1e56a64d..d52dd810 100644 --- a/README.md +++ b/README.md @@ -166,9 +166,13 @@ Next, you need to make a file `/etc/cvmfs/default.local` manually; this file is contains, for instance, the URL to your local proxy and the size of the local cache. As an example, you can put the following in this file, which corresponds to not using a proxy and setting the local quota limit to 40000MB: ``` -CVMFS_HTTP_PROXY=DIRECT +CVMFS_CLIENT_PROFILE=single CVMFS_QUOTA_LIMIT=40000 ``` +If you do want to use your own proxy, replace the first line by: +``` +CVMFS_HTTP_PROXY=: +``` For more details about configuring your client, see https://cvmfs.readthedocs.io/en/stable/cpt-configure.html. Finally, run `cvmfs_config setup` to set up CVMFS. From 0a7c5932299e7cafc09fdb5bf4ab40c026898b2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 12 Feb 2021 16:19:14 +0100 Subject: [PATCH 34/41] fix typo --- prepare-client-packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prepare-client-packages.yml b/prepare-client-packages.yml index 27294b62..5005411e 100644 --- a/prepare-client-packages.yml +++ b/prepare-client-packages.yml @@ -46,7 +46,7 @@ # Public keys for the eessi-hpc.org domain CVMFS_KEYS_DIR="/etc/cvmfs/keys/{{ item.domain }}" - # Ènable or disable the Geo API, based on the configuration file + # Enable or disable the Geo API, based on the configuration file CVMFS_USE_GEOAPI="{{ item.use_geoapi | ternary('yes', 'no') }}" # The client profile is supposed to be set in /etc/cvmfs/default.local From 432ed69717a7d116b1d9197a3cebf1ff9499c559 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 15 Feb 2021 10:03:25 +0100 Subject: [PATCH 35/41] remove auto keyword --- prepare-client-packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prepare-client-packages.yml b/prepare-client-packages.yml index 5005411e..39e545f5 100644 --- a/prepare-client-packages.yml +++ b/prepare-client-packages.yml @@ -54,7 +54,7 @@ if [ "$CVMFS_HTTP_PROXY" = "" ]; then # For individual clients (laptops, clusters < 5 nodes), use a site proxy # where possible and otherwise a direct stratum 1 connection - CVMFS_HTTP_PROXY="auto;DIRECT" + CVMFS_HTTP_PROXY="DIRECT" fi fi From 8567142dc965162861437e45967edbdddb411f39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 15 Feb 2021 10:29:51 +0100 Subject: [PATCH 36/41] remove config repo from package description --- .github/workflows/build-test-release-client-packages.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-test-release-client-packages.yml b/.github/workflows/build-test-release-client-packages.yml index 4eac4b7f..12e9885e 100644 --- a/.github/workflows/build-test-release-client-packages.yml +++ b/.github/workflows/build-test-release-client-packages.yml @@ -47,21 +47,21 @@ jobs: uses: bpicode/github-action-fpm@master with: fpm_args: "etc" - fpm_opts: "--debug -n cvmfs-config-eessi -v ${{ steps.get_version.outputs.version }} -t rpm -a all -s dir -C ./package --description 'CVMFS config repository package for EESSI.'" + fpm_opts: "--debug -n cvmfs-config-eessi -v ${{ steps.get_version.outputs.version }} -t rpm -a all -s dir -C ./package --description 'CVMFS configuration package for EESSI.'" - name: Build Deb package id: build-deb uses: bpicode/github-action-fpm@master with: fpm_args: "etc" - fpm_opts: "--debug -n cvmfs-config-eessi -v ${{ steps.get_version.outputs.version }} -t deb -a all -s dir -C ./package --description 'CVMFS config repository package for EESSI.'" + fpm_opts: "--debug -n cvmfs-config-eessi -v ${{ steps.get_version.outputs.version }} -t deb -a all -s dir -C ./package --description 'CVMFS configuration package for EESSI.'" - name: Build tar package id: build-tar uses: bpicode/github-action-fpm@master with: fpm_args: "etc" - fpm_opts: "--debug -n cvmfs-config-eessi -v ${{ steps.get_version.outputs.version }} -t tar -a all -s dir -C ./package --description 'CVMFS config repository package for EESSI.'" + fpm_opts: "--debug -n cvmfs-config-eessi -v ${{ steps.get_version.outputs.version }} -t tar -a all -s dir -C ./package --description 'CVMFS configuration package for EESSI.'" - name: Find filenames of generated packages id: find_filenames @@ -126,7 +126,7 @@ jobs: run: gem install --no-document fpm - name: Build package - run: "fpm --debug -n cvmfs-config-eessi -v ${{ steps.get_version.outputs.version }} -t osxpkg -a all -s dir -C ./package --description 'CVMFS config repository package for EESSI.' etc" + run: "fpm --debug -n cvmfs-config-eessi -v ${{ steps.get_version.outputs.version }} -t osxpkg -a all -s dir -C ./package --description 'CVMFS configuration package for EESSI.' etc" - name: Find filename of generated package id: find_filename From bfd6143bc967312ca97ba5b06fccd6c7d76632f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 15 Feb 2021 10:47:52 +0100 Subject: [PATCH 37/41] remove config repo from Stratum 1 CI --- .github/workflows/test-playbook.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-playbook.sh b/.github/workflows/test-playbook.sh index ab8d0bcb..d6db6487 100755 --- a/.github/workflows/test-playbook.sh +++ b/.github/workflows/test-playbook.sh @@ -15,7 +15,7 @@ echo ' - 127.0.0.1' >> inventory/local_site_specific_vars.yml # Don't use the GEO API for the Stratum 1, since we do not have a key here. export CVMFS_GEO_DB_FILE=NONE -# Only test the cvmfs-config repo on the Stratum 1, as the other ones may be very large. +# Only test CI suitable repos on the Stratum 1, as the other ones may be very large. if [ $playbook == "stratum1.yml" ] then cat >> inventory/local_site_specific_vars.yml <<-EOF From efd868a6f8c9fb6caa0a2f5f3ca0911d4dacfb1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 15 Feb 2021 10:48:50 +0100 Subject: [PATCH 38/41] remove config repo from comment --- inventory/group_vars/all.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 62a25a8d..ef3dca03 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -4,7 +4,7 @@ # Automatically configure EESSI CVMFS repos. eessi_cvmfs_repos_enabled: true -# Email address for the project, which will be put in the contact file on the config repo. +# Email address for the project. eessi_email: eessi@list.rug.nl # From 2e2f22cf588a7d74a3cfa341e24d62efc757b0e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 15 Feb 2021 10:49:14 +0100 Subject: [PATCH 39/41] remove stratum0-deploy-cvmfs-config.yml --- stratum0-deploy-cvmfs-config.yml | 57 -------------------------------- 1 file changed, 57 deletions(-) delete mode 100644 stratum0-deploy-cvmfs-config.yml diff --git a/stratum0-deploy-cvmfs-config.yml b/stratum0-deploy-cvmfs-config.yml deleted file mode 100644 index 309c5b07..00000000 --- a/stratum0-deploy-cvmfs-config.yml +++ /dev/null @@ -1,57 +0,0 @@ -# Start a transaction on the Stratum 0 and put all the files for the config repo in place. -# Note: first deploy your Stratum 0 and make sure that group_vars/all.yml contains the right configuration, -# including all the keys of the repositories that have to be configured. ---- -- name: CVMFS Stratum 0 deploy config repository - hosts: cvmfsstratum0servers - tasks: - - name: Start the CVMFS transaction, create directories, and copy files - block: - - name: start CVMFS transaction - command: cvmfs_server transaction "{{ eessi_cvmfs_config_repo.repository.repository }}" - - - name: create directories - file: - mode: "755" - path: "{{ item }}" - state: directory - with_list: - - "/cvmfs/{{ eessi_cvmfs_config_repo.repository.repository }}/etc/cvmfs/keys/{{ eessi_cvmfs_config_repo.domain }}" - - "/cvmfs/{{ eessi_cvmfs_config_repo.repository.repository }}/etc/cvmfs/contact" - - "/cvmfs/{{ eessi_cvmfs_config_repo.repository.repository }}/etc/cvmfs/domain.d" - - - name: deploy public keys - copy: - content: "{{ item.key }}" - dest: "/cvmfs/{{ eessi_cvmfs_config_repo.repository.repository }}{{ item.path }}" - mode: "644" - with_items: "{{ eessi_cvmfs_keys }}" - - - name: create contacts file - copy: - content: "{{ eessi_email }}\n" - dest: "/cvmfs/{{ eessi_cvmfs_config_repo.repository.repository }}/etc/cvmfs/contact/{{ eessi_cvmfs_config_repo.domain }}" - mode: "644" - - - name: create domain configuration file - copy: - content: | - CVMFS_SERVER_URL="{{ eessi_cvmfs_config_repo.urls | join(';') }}" - CVMFS_KEYS_DIR="/cvmfs/{{ eessi_cvmfs_config_repo.repository.repository }}/etc/cvmfs/keys/{{ eessi_cvmfs_config_repo.domain }}" - CVMFS_USE_GEOAPI="{{ eessi_cvmfs_server_urls[0].use_geoapi | default(false) | ternary('yes', 'no') }}" - dest: "/cvmfs/{{ eessi_cvmfs_config_repo.repository.repository }}/etc/cvmfs/domain.d/{{ eessi_cvmfs_config_repo.domain }}.conf" - mode: "644" - - - name: create default CVMFS configuration file - copy: - content: | - # For reference see https://sft.its.cern.ch/jira/browse/CVM-878 - CVMFS_LOW_SPEED_LIMIT=10000 - dest: "/cvmfs/{{ eessi_cvmfs_config_repo.repository.repository }}/etc/cvmfs/default.conf" - mode: "644" - - - name: publish CVMFS transaction - command: cvmfs_server publish "{{ eessi_cvmfs_config_repo.repository.repository }}" - rescue: - - name: cancel the transaction - command: cvmfs_server abort -f "{{ eessi_cvmfs_config_repo.repository.repository }}" From e9f65690b10d5b422d6a1ff94de65835f44243da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 2 Mar 2021 20:08:31 +0100 Subject: [PATCH 40/41] remove commented lines --- .github/workflows/test-playbooks.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/test-playbooks.yml b/.github/workflows/test-playbooks.yml index d1c62094..52fb3751 100644 --- a/.github/workflows/test-playbooks.yml +++ b/.github/workflows/test-playbooks.yml @@ -34,8 +34,5 @@ jobs: run: docker run -d --workdir /github/workspace --rm -e INPUT_PLAYBOOK -e HOME -e GITHUB_JOB -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_REPOSITORY_OWNER -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RETENTION_DAYS -e GITHUB_ACTOR -e GITHUB_WORKFLOW -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GITHUB_EVENT_NAME -e GITHUB_SERVER_URL -e GITHUB_API_URL -e GITHUB_GRAPHQL_URL -e GITHUB_WORKSPACE -e GITHUB_ACTION -e GITHUB_EVENT_PATH -e GITHUB_PATH -e GITHUB_ENV -e RUNNER_OS -e RUNNER_TOOL_CACHE -e RUNNER_TEMP -e RUNNER_WORKSPACE -e ACTIONS_RUNTIME_URL -e ACTIONS_RUNTIME_TOKEN -e ACTIONS_CACHE_URL -e GITHUB_ACTIONS=true -e CI=true -v "/var/run/docker.sock":"/var/run/docker.sock" -v $HOME:"/github/home" -v "$HOME/work/_temp/_github_workflow":"/github/workflow" -v "/home/runner/work/_temp/_runner_file_commands":"/github/file_commands" -v "$GITHUB_WORKSPACE":"/github/workspace" -v /sys/fs/cgroup:/sys/fs/cgroup --privileged --device /dev/fuse --mount type=bind,source=${GITHUB_WORKSPACE}/srv,target=/srv --mount type=bind,source=${GITHUB_WORKSPACE}/srv,target=/var/spool/cvmfs --name ${{ matrix.component }}-${{ matrix.os }} docker.pkg.github.com/$(echo $GITHUB_REPOSITORY | tr '[A-Z]' '[a-z]')/${{ matrix.os }} - name: Execute the playbook run: docker exec ${{ matrix.component }}-${{ matrix.os }} /test-playbook.sh ${{ matrix.component }}.yml -# - name: Execute additional playbook for Stratum 0 -# run: docker exec ${{ matrix.component }}-${{ matrix.os }} /test-playbook.sh ${{ matrix.component }}-deploy-cvmfs-config.yml -# if: ${{ matrix.component == 'stratum0' }} - name: Stop the container run: docker kill ${{ matrix.component }}-${{ matrix.os }} From ebe44f104fa99f1e9bdda1ac17b51f7a8c67ed8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 2 Mar 2021 20:14:44 +0100 Subject: [PATCH 41/41] remove commented CVMFS_NFILES --- inventory/group_vars/all.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index ab22654b..f9e3e541 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -73,7 +73,6 @@ eessi_cvmfs_repositories: - CVMFS_AUTO_TAG_TIMESPAN="30 days ago" - CVMFS_GARBAGE_COLLECTION=true client_options: [] -# - CVMFS_NFILES=4096 use_for_ci: no - repository: ci.eessi-hpc.org stratum0: cvmfs-s0.eessi-hpc.org @@ -83,7 +82,6 @@ eessi_cvmfs_repositories: - CVMFS_AUTO_GC=false - CVMFS_GARBAGE_COLLECTION=true client_options: [] -# - CVMFS_NFILES=4096 use_for_ci: yes # Override all the Galaxy defaults by our EESSI defaults.