Skip to content

Stream curl downloads directly to GCP (#56) #97

Stream curl downloads directly to GCP (#56)

Stream curl downloads directly to GCP (#56) #97

Workflow file for this run

name: Deploy resources and config
on:
push:
branches:
- main
permissions:
id-token: write
contents: read
env:
PROJECT: "cpg-common"
REFERENCES_PREFIX: "gs://cpg-common-main/references"
CONFIG_DESTINATION: "gs://cpg-config/templates/references/references.toml"
WORKLOAD_ID_PROVIDER: "projects/1051897107465/locations/global/workloadIdentityPools/github-pool/providers/github-provider"
SERVICE_ACCOUNT: "gh-references-deployer@cpg-common.iam.gserviceaccount.com"
jobs:
make_matrix:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.set_matrix.outputs.matrix }}
steps:
- name: "Checkout repo"
uses: actions/checkout@v4
- name: "Checkout revision before change"
uses: actions/checkout@v4
with:
ref: ${{ github.event.before }}
path: "before"
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- id: "google-cloud-auth"
name: "Authenticate to Google Cloud"
uses: "google-github-actions/auth@v2"
with:
project_id: ${{ env.PROJECT }}
workload_identity_provider: ${{ env.WORKLOAD_ID_PROVIDER }}
service_account: ${{ env.SERVICE_ACCOUNT }}
- id: "google-cloud-sdk-setup"
name: "Set up Cloud SDK"
uses: google-github-actions/setup-gcloud@v2
- name: "Set matrix"
id: set_matrix
run: |
pip install google-cloud-storage
cp references.py .github/workflows/references.py
cp before/references.py .github/workflows/references_before.py;
result=$(python .github/workflows/prep_matrix.py --references-prefix ${{ env.REFERENCES_PREFIX }})
echo "matrix=$result" >> $GITHUB_OUTPUT
transfer:
runs-on: ubuntu-latest
needs:
- make_matrix
if: ${{ needs.make_matrix.outputs.matrix != '{}' && needs.make_matrix.outputs.matrix != '' }}
strategy:
matrix: ${{ fromJson(needs.make_matrix.outputs.matrix) }}
env:
CLOUDSDK_CORE_DISABLE_PROMPTS: 1
steps:
- name: "checkout repo"
uses: actions/checkout@v4
- id: "google-cloud-auth"
name: "Authenticate to Google Cloud"
uses: google-github-actions/auth@v2
with:
project_id: ${{ env.PROJECT }}
workload_identity_provider: ${{ env.WORKLOAD_ID_PROVIDER }}
service_account: ${{ env.SERVICE_ACCOUNT }}
- id: "google-cloud-sdk-setup"
name: "Set up Cloud SDK"
uses: google-github-actions/setup-gcloud@v2
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: "transfer"
run: |
cp references.py .github/workflows/references.py
python .github/workflows/transfer.py \
--gcp-project ${{env.PROJECT}} \
--references-prefix ${{ env.REFERENCES_PREFIX }} \
${{ matrix.name }}
deploy_config:
runs-on: ubuntu-latest
needs:
- transfer
if: ${{ ! failure() }}
env:
CLOUDSDK_CORE_DISABLE_PROMPTS: 1
steps:
- name: "checkout repo"
uses: actions/checkout@v4
- id: "google-cloud-auth"
name: "Authenticate to Google Cloud"
uses: "google-github-actions/auth@v2"
with:
project_id: ${{ env.PROJECT }}
workload_identity_provider: ${{ env.WORKLOAD_ID_PROVIDER }}
service_account: ${{ env.SERVICE_ACCOUNT }}
- id: "google-cloud-sdk-setup"
name: "Set up Cloud SDK"
uses: google-github-actions/setup-gcloud@v2
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: "prepare config"
run: |
cp references.py .github/workflows/references.py
pip install toml
python .github/workflows/prep_config.py --references-prefix ${{ env.REFERENCES_PREFIX }} > config.toml
- name: "deploy config toml"
run: |
gcloud storage cp config.toml ${{ env.CONFIG_DESTINATION }}