metadata-ingestion.yml

name: metadata ingestion
on:
  push:
    branches:
      - master
    paths:
      - ".github/workflows/metadata-ingestion.yml"
      - "metadata-ingestion/**"
      - "metadata-ingestion-modules/**"
      - "metadata-models/**"
  pull_request:
    branches:
      - "**"
    paths:
      - ".github/workflows/metadata-ingestion.yml"
      - "metadata-ingestion/**"
      - "metadata-ingestion-modules/**"
      - "metadata-models/**"
  release:
    types: [published]

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  metadata-ingestion:
    runs-on: ubuntu-latest
    timeout-minutes: 40
    env:
      SPARK_VERSION: 3.3.2
      DATAHUB_TELEMETRY_ENABLED: false
      # TODO: Enable this once the test is fixed.
      # DATAHUB_LOOKML_GIT_TEST_SSH_KEY: ${{ secrets.DATAHUB_LOOKML_GIT_TEST_SSH_KEY }}
    strategy:
      matrix:
        python-version: ["3.8", "3.10"]
        command:
          [
            "testQuick",
            "testIntegrationBatch0",
            "testIntegrationBatch1",
            "testIntegrationBatch2",
          ]
        include:
          - python-version: "3.8"
          - python-version: "3.10"
      fail-fast: false
    steps:
      - name: Free up disk space
        run: |
          sudo apt-get remove 'dotnet-*' azure-cli || true
          sudo rm -rf /usr/local/lib/android/ || true
          sudo docker image prune -a -f || true
      - name: Set up JDK 17
        uses: actions/setup-java@v4
        with:
          distribution: "zulu"
          java-version: 17
      - uses: gradle/actions/setup-gradle@v3
      - uses: acryldata/sane-checkout-action@v3
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cache/uv
          key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
      - name: Install dependencies
        run: ./metadata-ingestion/scripts/install_deps.sh
      - name: Install package
        run: ./gradlew :metadata-ingestion:installPackageOnly
      - name: Run lint alongwith testQuick
        if: ${{  matrix.command == 'testQuick' }}
        run: ./gradlew :metadata-ingestion:lint
      - name: Run metadata-ingestion tests
        run: ./gradlew :metadata-ingestion:${{ matrix.command }}
      - name: Debug info
        if: always()
        run: |
          source metadata-ingestion/venv/bin/activate && uv pip freeze
          set -x
          df -hl
          docker image ls
          docker system df
      - uses: actions/upload-artifact@v3
        with:
          name: Test Results (metadata ingestion ${{ matrix.python-version }})
          path: |
            **/build/reports/tests/test/**
            **/build/test-results/test/**
            **/junit.*.xml
            !**/binary/**
      - name: Upload coverage to Codecov
        if: ${{ always() && matrix.python-version == '3.10' }}
        uses: codecov/codecov-action@v3
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          directory: .
          fail_ci_if_error: false
          flags: pytest-${{ matrix.command }}
          name: pytest-${{ matrix.command }}
          verbose: true

  event-file:
    runs-on: ubuntu-latest
    steps:
      - name: Upload
        uses: actions/upload-artifact@v3
        with:
          name: Event File
          path: ${{ github.event_path }}