Skip to content

Commit

Permalink
chore: parallelize PR checks and run only if relevant files are modif…
Browse files Browse the repository at this point in the history
…ied (#566)

Signed-off-by: Lenin Mehedy <lenin.mehedy@swirldslabs.com>
Signed-off-by: Nathan Klick <nathan@swirldslabs.com>
Co-authored-by: Nathan Klick <nathan@swirldslabs.com>
  • Loading branch information
leninmehedy and nathanklick authored Nov 28, 2023
1 parent f2e2449 commit d9c04f2
Show file tree
Hide file tree
Showing 9 changed files with 336 additions and 84 deletions.
17 changes: 10 additions & 7 deletions .github/workflows/flow-pull-request-checks.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -44,20 +44,23 @@ jobs:
enable-spotless-check: true

unit-tests:
name: Unit Tests
name: FST Java Unit Tests
uses: ./.github/workflows/zxc-compile-code.yaml
with:
custom-job-label: Standard
enable-unit-tests: true

nodejs-tests:
name: NodeJS CLI Tests
uses: ./.github/workflows/zxc-compile-code.yaml
chart-tests:
name: FS Helm Chart Tests
uses: ./.github/workflows/zxc-helm-chart-tests.yaml
with:
custom-job-label: Standard

fsnetman-tests:
name: FS Network Manager Tests
uses: ./.github/workflows/zxc-fsnetman-tests.yaml
with:
custom-job-label: Standard
enable-unit-tests: false
enable-nodejs-tests: true
node-version: 18

codecov:
name: CodeCov
Expand Down
49 changes: 5 additions & 44 deletions .github/workflows/zxc-compile-code.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,6 @@ on:
type: boolean
required: false
default: false
enable-nodejs-tests:
description: "NodeJS Testing Enabled"
type: boolean
required: false
default: false
enable-spotless-check:
description: "Spotless Check Enabled"
type: boolean
Expand Down Expand Up @@ -114,7 +109,7 @@ jobs:

- name: Setup Kind
uses: helm/kind-action@dda0770415bac9fc20092cacbc54aa298604d140 # v1.8.0
if: ${{ (inputs.enable-unit-tests || inputs.enable-nodejs-tests) && !cancelled() && !failure() }}
if: ${{ inputs.enable-unit-tests && !cancelled() && !failure() }}
with:
config: dev/dev-cluster.yaml
version: v0.19.0
Expand All @@ -123,13 +118,13 @@ jobs:

- name: Setup Helm
uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5
if: ${{ (inputs.enable-unit-tests || inputs.enable-nodejs-tests) && !cancelled() && !failure() }}
if: ${{ inputs.enable-unit-tests && !cancelled() && !failure() }}
with:
version: "v3.12.3" # helm version

# Technically, this step is not required for the unit tests to run, but it is useful for debugging setup issues.
- name: Kubernetes Cluster Info
if: ${{ (inputs.enable-unit-tests || inputs.enable-nodejs-tests) && !cancelled() && !failure() }}
if: ${{ inputs.enable-unit-tests && !cancelled() && !failure() }}
run: |
kubectl config get-contexts
kubectl get crd
Expand All @@ -140,7 +135,7 @@ jobs:
- name: Authenticate to Google Cloud
id: google-auth
uses: google-github-actions/auth@35b0e87d162680511bf346c299f71c9c5c379033 # v1.1.1
if: ${{ (inputs.enable-unit-tests || inputs.enable-nodejs-tests) && !cancelled() && !failure() }}
if: ${{ inputs.enable-unit-tests && !cancelled() && !failure() }}
with:
token_format: 'access_token'
workload_identity_provider: "projects/101730247931/locations/global/workloadIdentityPools/hedera-registry-pool/providers/hedera-registry-gh-actions"
Expand All @@ -152,40 +147,6 @@ jobs:
if: ${{ inputs.enable-unit-tests && !cancelled() }}
uses: google-github-actions/setup-gcloud@e30db14379863a8c79331b04a9969f4c1e225e0b # v1.1.1

- name: NodeJS CLI Unit Tests
id: nodejs-test
working-directory: fullstack-network-manager
if: ${{ inputs.enable-nodejs-tests && !cancelled() && !failure() }}
run: |
npm i
npm test
- name: NodeJS CLI E2E Tests
id: nodejs-test-e2e
working-directory: fullstack-network-manager
if: ${{ inputs.enable-nodejs-tests && !cancelled() && !failure() }}
run: |
export DEV_MODE=true
npm i
npm link
fsnetman init -d ../charts
fsnetman cluster create
fsnetman cluster setup
fsnetman chart install
npm run test-e2e
# This step tests the Helm chart direct mode of operation which uses the ubi8-init-java17 image.
- name: Helm Chart Test (Direct Install)
working-directory: dev
if: ${{ inputs.enable-unit-tests && !cancelled() && !failure() }}
run: make ci-test SCRIPT_NAME=direct-install.sh

# This step tests the Helm chart NMT mode of operation which uses the ubi8-init-dind image.
- name: Helm Chart Test (NMT Install)
working-directory: dev
if: ${{ inputs.enable-unit-tests && !cancelled() && !failure() }}
run: make ci-test SCRIPT_NAME=nmt-install.sh

- name: Compile
id: gradle-build
uses: gradle/gradle-build-action@842c587ad8aa4c68eeba24c396e15af4c2e9f30a # v2.9.0
Expand Down Expand Up @@ -237,7 +198,7 @@ jobs:

# This step is to cache the helm charts before we start unit tests
- name: Setup Helm Charts
if: ${{ (inputs.enable-unit-tests || inputs.enable-nodejs-tests) && !cancelled() && !failure() }}
if: ${{ inputs.enable-unit-tests && !cancelled() && !failure() }}
run: |
helm repo add haproxytech https://haproxytech.github.io/helm-charts
helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
Expand Down
135 changes: 135 additions & 0 deletions .github/workflows/zxc-fsnetman-tests.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
##
# Copyright (C) 2023 Hedera Hashgraph, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##

name: "ZXC: FS Network Manager Tests"
# The purpose of this reusable workflow is to compile the code and run the unit tests on every PR and commit.
# This reusable component is called by the following workflows:
# - .github/workflows/flow-pull-request-checks.yaml

on:
workflow_call:
inputs:
custom-job-label:
description: "Custom Job Label:"
type: string
required: false
default: "fsnetman Test"
node-version:
description: "NodeJS Version:"
type: string
required: false
default: "18"

defaults:
run:
shell: bash

permissions:
id-token: write
contents: read
actions: read
pull-requests: write
checks: write
statuses: write

jobs:
test:
name: ${{ inputs.custom-job-label || 'fsnetman Test' }}
runs-on: [self-hosted, Linux, medium, ephemeral]
steps:
- name: Get related changed files
id: changed-files
uses: tj-actions/changed-files@25ef3926d147cd02fc7e931c1ef50772bbb0d25d #v40.1.1
with:
files: |
fullstack-network-manager/**
- name: List all related changed files
id: check-changed-files
run: |
echo "Modified files"
echo "-------------------------------------------------------------------"
for file in ${{ steps.changed-files.outputs.all_changed_files }}; do
echo " - ${file} was changed"
done
echo ""
if [ "${{ steps.changed-files.outputs.any_changed }}" ]; then
echo "run-tests=true" >> "${GITHUB_OUTPUT}"
echo "Executing fsnetman tests...."
else
echo ">> No relevant files are changed. No need to run fsnetman tests"
fi
- name: Checkout Code
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
if: ${{ steps.check-changed-files.outputs.run-tests && !cancelled() && !failure() }}
with:
# the fetch depth defaults to only the commit that triggered the workflow unless the spotless check was enabled
fetch-depth: ${{ inputs.enable-spotless-check && '0' || '' }}

- name: Setup Node
uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
if: ${{ steps.check-changed-files.outputs.run-tests && !cancelled() && !failure() }}
with:
node-version: ${{ inputs.node-version }}

- name: Setup Kind
uses: helm/kind-action@dda0770415bac9fc20092cacbc54aa298604d140 # v1.8.0
if: ${{ steps.check-changed-files.outputs.run-tests && !cancelled() && !failure() }}
with:
config: dev/dev-cluster.yaml
version: v0.19.0
verbosity: 3
wait: 120s

- name: Setup Helm
uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5
if: ${{ steps.check-changed-files.outputs.run-tests && !cancelled() && !failure() }}
with:
version: "v3.12.3" # helm version

# Technically, this step is not required for the unit tests to run, but it is useful for debugging setup issues.
- name: Kubernetes Cluster Info
if: ${{ steps.check-changed-files.outputs.run-tests && !cancelled() && !failure() }}
run: |
kubectl config get-contexts
kubectl get crd
kubectl get node --show-labels
- name: NodeJS CLI Unit Tests
id: nodejs-test
working-directory: fullstack-network-manager
if: ${{ steps.check-changed-files.outputs.run-tests && !cancelled() && !failure() }}
run: |
which npm
which bash
npm -v
npm i
npm test
- name: NodeJS CLI E2E Tests
id: nodejs-test-e2e
working-directory: fullstack-network-manager
if: ${{ steps.check-changed-files.outputs.run-tests && !cancelled() && !failure() }}
run: |
export DEV_MODE=true
npm i
npm link
fsnetman init -d ../charts
fsnetman cluster create
fsnetman cluster setup
fsnetman chart install
npm run test-e2e
127 changes: 127 additions & 0 deletions .github/workflows/zxc-helm-chart-tests.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
##
# Copyright (C) 2023 Hedera Hashgraph, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##

name: "ZXC: FS Helm Chart Tests"
# The purpose of this reusable workflow is to compile the code and run the unit tests on every PR and commit.
# This reusable component is called by the following workflows:
# - .github/workflows/flow-pull-request-checks.yaml

on:
workflow_call:
inputs:
custom-job-label:
description: "Custom Job Label:"
type: string
required: false
default: "Helm Chart Test"

defaults:
run:
shell: bash

permissions:
id-token: write
contents: read
actions: read
pull-requests: write
checks: write
statuses: write

jobs:
test:
name: ${{ inputs.custom-job-label || 'Helm Chart Test' }} (${{ matrix.scriptName }})
runs-on: [self-hosted, Linux, medium, ephemeral]
strategy:
fail-fast: false
matrix:
# direct-install.sh uses ubi8-init-java17 image
# nmt-install.sh uses ubi8-init-dind image
scriptName: [ direct-install.sh, nmt-install.sh ]
steps:
- name: Get changed files related to charts
id: changed-files
uses: tj-actions/changed-files@25ef3926d147cd02fc7e931c1ef50772bbb0d25d #v40.1.1
with:
files: |
chart:
- charts/**
scripts:
- dev/**
- name: List all changed files related to charts
id: check-changed-files
run: |
echo "Modified charts files"
echo "-------------------------------------------------------------------"
for file in ${{ steps.changed-files.outputs.chart_all_changed_files }}; do
echo " - ${file} was changed"
done
echo ""
echo "Modified script files"
echo "-------------------------------------------------------------------"
for file in ${{ steps.changed-files.outputs.script_all_changed_files }}; do
echo " - ${file} was changed"
done
echo ""
if [ "${{ steps.changed-files.outputs.scripts_any_changed }}" ] || [ "${{ steps.changed-files.outputs.scripts_any_changed }}" ]; then
echo "run-tests=true" >> "${GITHUB_OUTPUT}"
echo "Executing helm chart tests...."
else
echo ">> No relevant files are changed. No need to run helm chart tests"
fi
- name: Initialize Job Variables
id: vars
if: ${{ steps.check-changed-files.outputs.run-tests && !cancelled() && !failure() }}
run: |
echo "script_name=${{ matrix.scriptName }}" >> "${GITHUB_OUTPUT}"
- name: Checkout Code
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
if: ${{ steps.check-changed-files.outputs.run-tests && !cancelled() && !failure() }}
with:
# the fetch depth defaults to only the commit that triggered the workflow unless the spotless check was enabled
fetch-depth: ${{ inputs.enable-spotless-check && '0' || '' }}

- name: Setup Kind
uses: helm/kind-action@dda0770415bac9fc20092cacbc54aa298604d140 # v1.8.0
if: ${{ steps.check-changed-files.outputs.run-tests && !cancelled() && !failure() }}
with:
config: dev/dev-cluster.yaml
version: v0.19.0
verbosity: 3
wait: 120s

- name: Setup Helm
uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5
if: ${{ steps.check-changed-files.outputs.run-tests && !cancelled() && !failure() }}
with:
version: "v3.12.3" # helm version

# Technically, this step is not required for the unit tests to run, but it is useful for debugging setup issues.
- name: Kubernetes Cluster Info
if: ${{ steps.check-changed-files.outputs.run-tests && !cancelled() && !failure() }}
run: |
kubectl config get-contexts
kubectl get crd
kubectl get node --show-labels
- name: Helm Chart Test
working-directory: dev
if: ${{ steps.check-changed-files.outputs.run-tests && !cancelled() && !failure() }}
run: |
echo "Running test with SCRIPT_NAME=${{ steps.vars.outputs.script_name}}"
make ci-test SCRIPT_NAME="${{ steps.vars.outputs.script_name}}"
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -515,3 +515,4 @@ dev/temp/.env
charts/deleteme.yaml
node_modules
fullstack-network-manager/coverage
dev/resources/nmt/version.properties
Loading

0 comments on commit d9c04f2

Please sign in to comment.